[ { "path": ".gitattributes", "content": "" }, { "path": ".github/workflows/codeql.yml", "content": "# For most projects, this workflow file will not need changing; you simply need\n# to commit it to your repository.\n#\n# You may wish to alter this file to override the set of languages analyzed,\n# or to provide custom queries or build logic.\nname: \"CodeQL\"\n\non:\n push:\n branches: [master]\n pull_request:\n # The branches below must be a subset of the branches above\n branches: [master]\n\njobs:\n analyze:\n name: Analyze\n runs-on: ubuntu-latest\n\n strategy:\n fail-fast: false\n matrix:\n # Override automatic language detection by changing the below list\n # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']\n language: ['go']\n # Learn more...\n # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection\n\n steps:\n - name: Checkout repository\n uses: actions/checkout@v4\n with:\n # We must fetch at least the immediate parents so that if this is\n # a pull request then we can checkout the head.\n fetch-depth: 2\n\n # Initializes the CodeQL tools for scanning.\n - name: Initialize CodeQL\n uses: github/codeql-action/init@v3\n with:\n languages: ${{ matrix.language }}\n # If you wish to specify custom queries, you can do so here or in a config file.\n # By default, queries listed here will override any specified in a config file. \n # Prefix the list here with \"+\" to use these queries and those in the config file.\n # queries: ./path/to/local/query, your-org/your-repo/queries@main\n\n # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).\n # If this step fails, then you should remove it and run the build manually (see below)\n - name: Autobuild\n uses: github/codeql-action/autobuild@v3\n\n # ℹī¸ Command-line programs to run using the OS shell.\n # 📚 https://git.io/JvXDl\n\n # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines\n # and modify them (or add more) to build your code if your project\n # uses a compiled language\n\n #- run: |\n # make bootstrap\n # make release\n\n - name: Perform CodeQL Analysis\n uses: github/codeql-action/analyze@v3\n\n\n" }, { "path": ".github/workflows/docker.yml", "content": "name: Docker images\n\non:\n push:\n branches: [ master ]\n tags: [ 'v*' ]\n pull_request:\n branches: [ master ]\n\njobs:\n docker:\n name: Build image\n runs-on: ubuntu-latest\n steps:\n - name: Check out code\n uses: actions/checkout@v4\n - name: Docker meta\n id: meta\n uses: docker/metadata-action@v3\n with:\n images: ghcr.io/${{ github.repository }}\n # create latest tag for branch events\n flavor: |\n latest=${{ github.event_name == 'push' && github.ref_type == 'branch' }}\n tags: |\n type=ref,event=branch\n type=ref,event=pr\n type=semver,pattern={{version}}\n type=semver,pattern={{major}}.{{minor}}\n type=semver,pattern={{major}}.{{minor}}.{{patch}}\n - name: Login to DockerHub\n if: github.event_name != 'pull_request'\n uses: docker/login-action@v1\n with:\n registry: ghcr.io\n username: ${{ github.actor }}\n password: ${{ secrets.GITHUB_TOKEN }}\n - name: Build and push\n id: docker_build\n uses: docker/build-push-action@v2\n with:\n # push for non-pr events\n push: ${{ github.event_name != 'pull_request' }}\n context: .\n tags: ${{ steps.meta.outputs.tags }}\n labels: ${{ steps.meta.outputs.labels }}\n\n" }, { "path": ".github/workflows/lint.yml", "content": "name: Lint\non:\n pull_request:\n\njobs:\n golangci:\n name: lint\n runs-on: ubuntu-22.04\n steps:\n - uses: actions/checkout@v4\n \n - uses: actions/setup-go@v4\n with:\n go-version-file: go.mod\n\n - name: Run linter\n uses: golangci/golangci-lint-action@v7\n with:\n version: v2.0.2\n\n" }, { "path": ".github/workflows/release.yml", "content": "name: Upload Packages to new release\n\non:\n release:\n types:\n - published\n\njobs:\n build:\n name: Build\n runs-on: ubuntu-latest\n env:\n BINARY: ${{ github.event.repository.name }}\n CGO_ENABLED: 0\n\n outputs:\n matrix: ${{ steps.build.outputs.matrix }}\n steps:\n - name: Set up Go\n uses: actions/setup-go@v5\n with:\n go-version: ^1\n\n - uses: actions/checkout@v4\n name: Checkout\n\n - name: Test\n run: make test\n env:\n CGO_ENABLED: 1\n\n - name: Build packages\n id: build\n run: |\n go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.40.0\n make nfpm-deb nfpm-rpm\n make sum-files\n ARTIFACTS=\n # Upload all deb and rpm packages\n for package in *deb *rpm; do ARTIFACTS=${ARTIFACTS}\\\"$package\\\",\\ ; done\n echo ::set-output name=matrix::{\\\"file\\\": [${ARTIFACTS} \\\"sha256sum\\\", \\\"md5sum\\\"]}\n\n - name: Check version\n id: check_version\n run: |\n ./out/${BINARY}-linux-amd64 -version\n [ v$(./out/${BINARY}-linux-amd64 -version) = ${{ github.event.release.tag_name }} ]\n\n - name: Artifact\n id: artifact\n uses: actions/upload-artifact@v4\n with:\n name: packages\n retention-days: 1\n path: |\n *.deb\n *.rpm\n sha256sum\n md5sum\n\n - name: Push packages to the stable repo\n run: make packagecloud-stable\n env:\n PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}\n\n upload:\n needs: build\n runs-on: ubuntu-latest\n strategy:\n matrix: ${{fromJson(needs.build.outputs.matrix)}}\n steps:\n - name: Download artifact\n uses: actions/download-artifact@v4.1.7\n with:\n name: packages\n - name: Upload ${{ matrix.file }}\n id: upload\n uses: actions/upload-release-asset@v1\n env:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n with:\n upload_url: ${{ github.event.release.upload_url }}\n asset_path: ${{ matrix.file }}\n asset_name: ${{ matrix.file }}\n asset_content_type: application/octet-stream\n" }, { "path": ".github/workflows/tests-sd.yml", "content": "name: Tests register in SD\n\non:\n push:\n branches: [ master ]\n pull_request:\n branches: [ master ]\n\njobs:\n\n tests:\n env:\n CGO_ENABLED: 0\n name: Test register in SD\n runs-on: ubuntu-latest\n strategy:\n matrix:\n go:\n - ^1\n steps:\n\n - name: Set up Go\n uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Check out code\n uses: actions/checkout@v4\n\n - name: Start consul\n run: |\n ./tests/consul.sh 1.15.2 > /tmp/consul.log &\n sleep 30\n shell: bash\n\n - name: Test\n run: go test ./sd/nginx -tags=test_sd -v\n" }, { "path": ".github/workflows/tests.yml", "content": "name: Tests\n\non:\n push:\n branches: [ master ]\n pull_request:\n branches: [ master ]\n\njobs:\n\n tests:\n env:\n CGO_ENABLED: 0\n name: Test code\n runs-on: ubuntu-latest\n strategy:\n matrix:\n go:\n - ^1.20\n - ^1.21\n - ^1\n steps:\n\n - name: Set up Go\n uses: actions/setup-go@v5\n with:\n go-version: ${{ matrix.go }}\n\n - name: Check out code\n uses: actions/checkout@v4\n\n - name: Checkout to the latest tag\n run: |\n # Fetch all tags\n git fetch --depth=1 --tags\n # Get the latest tag\n VERS=$(git tag -l | sort -Vr | head -n1)\n # Fetch everything to the latest tag\n git fetch --shallow-since=$(git log $VERS -1 --format=%at)\n if: ${{ github.event_name == 'push' }} # only when built from master\n\n - name: Build project\n run: make\n\n - name: Validate default configs\n run: |\n ./graphite-clickhouse -config-print-default > /tmp/graphite-clickhouse.conf\n ./graphite-clickhouse -config /tmp/graphite-clickhouse.conf -check-config\n\n - name: Check documentation consistency\n run: |\n make config\n git diff --exit-code\n\n - name: Test\n run: make test\n env:\n CGO_ENABLED: 1\n\n - name: Test (with GMT-5)\n run: |\n go clean -testcache\n TZ=Etc/GMT-5 make test\n env:\n CGO_ENABLED: 1\n\n - name: Test (with GMT+5)\n run: |\n go clean -testcache\n TZ=Etc/GMT+5 make test\n env:\n CGO_ENABLED: 1\n\n - name: Integration tests\n run: |\n make e2e-test\n ./e2e-test -config tests -abort -rmi\n\n # TODO (msaf1980): find a way to set TZ in carbon-clickhouse docker (or run locally)\n # run with clickhouse.date-format = \"both\"\n # - name: Integration tests (with Etc/GMT-5)\n # run: |\n # make e2e-test\n # sudo timedatectl set-timezone Etc/GMT-5\n # ./e2e-test -config issues/daytime\n\n # - name: Integration tests (with Etc/GMT+5)\n # run: |\n # make e2e-test\n # sudo timedatectl set-timezone Etc/GMT+5\n # ./e2e-test -config issues/daytime\n\n - name: Check packaging\n run: |\n go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.40.0\n make DEVEL=1 nfpm-deb nfpm-rpm\n make sum-files\n\n - name: Artifact\n id: artifact\n uses: actions/upload-artifact@v4\n with:\n name: packages-${{ matrix.go }}\n path: |\n *.deb\n *.rpm\n sha256sum\n md5sum\n\n - name: Push packages to the autobuilds repo\n if: ${{ github.event_name == 'push' && matrix.go == '^1' }} # only when built from master with latest go\n run: make DEVEL=1 packagecloud-autobuilds\n env:\n PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}\n" }, { "path": ".gitignore", "content": "# Compiled Object files, Static and Dynamic libs (Shared Objects)\n*.o\n*.a\n*.so\ngraphite-clickhouse\n.vscode\n/out/\n\n# Folders\n_obj\n_test\n\n# Architecture specific extensions/prefixes\n*.[568vq]\n[568vq].out\n\n*.cgo1.go\n*.cgo2.c\n_cgo_defun.c\n_cgo_gotypes.go\n_cgo_export.*\n\n_testmain.go\n\n*.exe\n*.test\n*.prof\n" }, { "path": ".golangci.yml", "content": "version: \"2\"\nlinters:\n default: none\n enable:\n - asasalint\n - asciicheck\n - bidichk\n - bodyclose\n# - contextcheck\n - decorder\n# - dogsled\n - durationcheck\n# - errcheck\n# - errorlint\n# - fatcontext\n - ginkgolinter\n - gocheckcompilerdirectives\n - gochecksumtype\n# - goconst\n# - gocyclo\n# - godot\n - goheader\n - govet\n - grouper\n# - ineffassign\n - loggercheck\n# - makezero\n# - misspell\n# - mnd\n# - nilerr\n# - noctx\n# - nosprintfhostport\n - prealloc\n# - predeclared\n - promlinter\n - protogetter\n - reassign\n# - revive\n - rowserrcheck\n - sloglint\n - spancheck\n - sqlclosecheck\n# - staticcheck\n# - testifylint\n - tparallel\n - unconvert\n# - unparam\n# - unused\n - usestdlibvars\n# - wastedassign\n - whitespace\n - wsl\n settings:\n gocyclo:\n min-complexity: 15\n govet:\n settings:\n printf:\n funcs:\n - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof\n - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf\n - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf\n - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf\n unparam:\n check-exported: false\n exclusions:\n generated: lax\n presets:\n - comments\n - common-false-positives\n - legacy\n - std-error-handling\n rules:\n - linters:\n - errcheck\n - contextcheck\n - goconst\n - mnd\n path: _test\\.go\n - linters:\n - godot\n path: notifier/registrator.go\n paths:\n - third_party$\n - builtin$\n - examples$\nformatters:\n enable:\n - gofmt\n# - gofumpt\n - goimports\n exclusions:\n generated: lax\n paths:\n - third_party$\n - builtin$\n - examples$\n" }, { "path": "Dockerfile", "content": "FROM golang:alpine as builder\n\nWORKDIR /go/src/github.com/lomik/graphite-clickhouse\nCOPY . .\n\nENV GOPATH=/go\n\nRUN apk add git --no-cache\n\nRUN go build -ldflags '-extldflags \"-static\"' github.com/lomik/graphite-clickhouse\n\nFROM alpine:latest\n\nRUN apk --no-cache add ca-certificates\nWORKDIR /\n\nCOPY --from=builder /go/src/github.com/lomik/graphite-clickhouse/graphite-clickhouse /usr/bin/graphite-clickhouse\n\nCMD [\"graphite-clickhouse\"]\n\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2016 Roman Lomonosov\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "Makefile", "content": "NAME:=graphite-clickhouse\nDESCRIPTION:=\"Graphite cluster backend with ClickHouse support\"\nMODULE:=github.com/lomik/graphite-clickhouse\n\nGO ?= go\nexport GO111MODULE := on\nTEMPDIR:=$(shell mktemp -d)\n\nDEVEL ?= 0\nifeq ($(DEVEL), 0)\nVERSION:=$(shell sh -c 'grep \"const Version\" $(NAME).go | cut -d\\\" -f2')\nelse\nVERSION:=$(shell sh -c 'git describe --always --tags | sed -e \"s/^v//i\"')\nendif\n\nSRCS:=$(shell find . -name '*.go')\n\nall: $(NAME)\n\n.PHONY: clean\nclean:\n\trm -f $(NAME) $(NAME)-client\n\trm -rf out\n\trm -f *deb *rpm\n\trm -f sha256sum md5sum\n\n$(NAME): $(SRCS)\n\t$(GO) build -ldflags '-X main.BuildVersion=$(VERSION)' $(MODULE) \n\ndebug: $(SRCS)\n\t$(GO) build -ldflags '-X main.BuildVersion=$(VERSION)' -gcflags=all='-N -l' $(MODULE)\n\ndeploy/doc/graphite-clickhouse.conf: $(NAME)\n\t./$(NAME) -config-print-default > $@\n\ndoc/config.md: deploy/doc/graphite-clickhouse.conf deploy/doc/config.md\n\t@echo 'Generating $@...'\n\t@printf '[//]: # (This file is built out of deploy/doc/config.md, please do not edit it manually) \\n' > $@\n\t@printf '[//]: # (To rebuild it run `make config`)\\n\\n' >> $@\n\t@cat deploy/doc/config.md >> $@\n\t@printf '\\n```toml\\n' >> $@\n\t@cat deploy/doc/graphite-clickhouse.conf >> $@\n\t@printf '```\\n' >> $@\n\nconfig: doc/config.md\n\n# run after prometheus upgrade\nprometheus/ui:\n\tvendor_prometheus_ui.sh\n\ntest:\n\t$(GO) test -race ./...\n\ne2e-test: $(NAME)\n\t$(GO) build $(MODULE)/cmd/e2e-test\n\nclient: $(NAME)\n\t$(GO) build $(MODULE)/cmd/graphite-clickhouse-client\n\ngox-build: out/$(NAME)-linux-amd64 out/$(NAME)-linux-arm64 out/root/etc/$(NAME)/$(NAME).conf\n\nARCH = amd64 arm64\nout/$(NAME)-linux-%: out $(SRCS)\n\tGOOS=linux GOARCH=$* $(GO) build -ldflags '-X main.BuildVersion=$(VERSION)' -o $@ $(MODULE)\n\nout:\n\tmkdir -p out\n\nout/root/etc/$(NAME)/$(NAME).conf: $(NAME)\n\tmkdir -p \"$(shell dirname $@)\"\n\t./$(NAME) -config-print-default > $@\n\nnfpm-deb: gox-build\n\t$(MAKE) nfpm-build-deb ARCH=amd64\n\t$(MAKE) nfpm-build-deb ARCH=arm64\nnfpm-rpm: gox-build\n\t$(MAKE) nfpm-build-rpm ARCH=amd64\n\t$(MAKE) nfpm-build-rpm ARCH=arm64\n\nnfpm-build-%: nfpm.yaml\n\tNAME=$(NAME) DESCRIPTION=$(DESCRIPTION) ARCH=$(ARCH) VERSION_STRING=$(VERSION) nfpm package --packager $*\n\n.ONESHELL:\nRPM_VERSION:=$(subst -,_,$(VERSION))\npackagecloud-push-rpm: $(wildcard $(NAME)-$(RPM_VERSION)-1.*.rpm)\n\tfor pkg in $^; do\n\t\tpackage_cloud push $(REPO)/el/7 $${pkg} || true\n\t\tpackage_cloud push $(REPO)/el/8 $${pkg} || true\n\t\tpackage_cloud push $(REPO)/el/9 $${pkg} || true\n\tdone\n\n.ONESHELL:\npackagecloud-push-deb: $(wildcard $(NAME)_$(VERSION)_*.deb)\n\tfor pkg in $^; do\n\t\tpackage_cloud push $(REPO)/ubuntu/xenial $${pkg} || true\n\t\tpackage_cloud push $(REPO)/ubuntu/bionic $${pkg} || true\n\t\tpackage_cloud push $(REPO)/ubuntu/focal $${pkg} || true\n\t\tpackage_cloud push $(REPO)/debian/stretch $${pkg} || true\n\t\tpackage_cloud push $(REPO)/debian/buster $${pkg} || true\n\t\tpackage_cloud push $(REPO)/debian/bullseye $${pkg} || true\n\tdone\n\npackagecloud-push:\n\t@$(MAKE) packagecloud-push-rpm\n\t@$(MAKE) packagecloud-push-deb\n\npackagecloud-autobuilds:\n\t$(MAKE) packagecloud-push REPO=go-graphite/autobuilds\n\npackagecloud-stable:\n\t$(MAKE) packagecloud-push REPO=go-graphite/stable\n\nsum-files: | sha256sum md5sum\n\nmd5sum:\n\tmd5sum $(wildcard $(NAME)_$(VERSION)*.deb) $(wildcard $(NAME)-$(VERSION)*.rpm) > md5sum\n\nsha256sum:\n\tsha256sum $(wildcard $(NAME)_$(VERSION)*.deb) $(wildcard $(NAME)-$(VERSION)*.rpm) > sha256sum\n\n.PHONY: lint\nlint:\n\tgolangci-lint run\n" }, { "path": "README.md", "content": "[![deb](https://img.shields.io/badge/deb-packagecloud.io-844fec.svg)](https://packagecloud.io/go-graphite/stable)\n[![rpm](https://img.shields.io/badge/rpm-packagecloud.io-844fec.svg)](https://packagecloud.io/go-graphite/stable)\n\n# graphite-clickhouse\nGraphite cluster backend with ClickHouse support\n\n## Work scheme\n![stack.png](doc/stack.png?v3)\n\nGray components are optional or alternative\n\n## TL;DR\n[Preconfigured docker-compose](https://github.com/lomik/graphite-clickhouse-tldr)\n\n### Docker\nDocker images are available on [packages](https://github.com/lomik/graphite-clickhouse/pkgs/container/graphite-clickhouse) page.\n\n## Compatibility\n- [x] [graphite-web 1.1.0](https://github.com/graphite-project/graphite-web)\n- [x] [graphite-web 0.9.15](https://github.com/graphite-project/graphite-web/tree/0.9.15)\n- [x] [graphite-web 1.0.0](https://github.com/graphite-project/graphite-web)\n- [x] [carbonapi 0.14.1+](https://github.com/go-graphite/carbonapi)\n- [x] [carbonzipper](https://github.com/go-graphite/carbonzipper) (DEPRECATED, is part of carbonapi currently)\n\n## Build\nRequired golang 1.18+\n```sh\n# build binary\ngit clone https://github.com/lomik/graphite-clickhouse.git\ncd graphite-clickhouse\nmake\n```\n\n## Installation\n1. Setup [Yandex ClickHouse](https://github.com/yandex/ClickHouse) and [carbon-clickhouse](https://github.com/lomik/carbon-clickhouse)\n2. Setup and configure `graphite-clickhouse`\n3. Add graphite-clickhouse `host:port` to graphite-web [CLUSTER_SERVERS](http://graphite.readthedocs.io/en/latest/config-local-settings.html#cluster-configuration)\n\n## Configuration\nSee [configuration documentation](./doc/config.md).\n\n### Special headers processing\n\nSome HTTP headers are processed specially by the service\n\n#### Request headers\n\n*Grafana headers*: `X-Dashboard-Id`, `X-Grafana-Org-Id`, and `X-Panel-Id` are logged and passed further to the ClickHouse.\n\n*Debug headers* (see [debugging.md](./doc/debugging.md) for details):\n\n- `X-Gch-Debug-External-Data` - when this header is set to anything and every of `directory`, `directory-perm`, and `external-data-perm` parameters in `[debug]` is set and valid, service will save the dump of external data tables in the directory for debug output.\n- `X-Gch-Debug-Output` - header to enable special processing for `format=carbonapi_v3_pb` and `format=json` render output.\n- `X-Gch-Debug-Protobuf` - header enables the original marshallers for `protobuf` and `carbonapi_v3_pb` to check the binary data integrity.\n\n#### Response headers\n\n- `X-Gch-Request-Id` - the current request ID.\n- `X-Cached-Find` - Flag for find cache hit.\n\n## Run on same host with old graphite-web 0.9.x\nBy default graphite-web won't connect to CLUSTER_SERVER on localhost. Cheat:\n```python\nclass ForceLocal(str):\n def split(self, *args, **kwargs):\n return [\"8.8.8.8\", \"8080\"]\n\nCLUSTER_SERVERS = [ForceLocal(\"127.0.0.1:9090\")]\n```\n" }, { "path": "autocomplete/autocomplete.go", "content": "package autocomplete\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/go-graphite/carbonapi/pkg/parser\"\n\t\"github.com/msaf1980/go-stringutils\"\n\t\"go.uber.org/zap\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/finder\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/helper/date\"\n\t\"github.com/lomik/graphite-clickhouse/helper/utils\"\n\t\"github.com/lomik/graphite-clickhouse/logs\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/where\"\n)\n\n// override in unit tests for stable results\nvar timeNow = time.Now\n\ntype Handler struct {\n\tconfig *config.Config\n\tisValues bool\n}\n\nfunc NewTags(config *config.Config) *Handler {\n\th := &Handler{\n\t\tconfig: config,\n\t}\n\n\treturn h\n}\n\nfunc NewValues(config *config.Config) *Handler {\n\th := &Handler{\n\t\tconfig: config,\n\t\tisValues: true,\n\t}\n\n\treturn h\n}\n\nfunc dateString(autocompleteDays int, tm time.Time) (string, string) {\n\tfromDate := date.FromTimeToDaysFormat(tm.AddDate(0, 0, -autocompleteDays))\n\tuntilDate := date.UntilTimeToDaysFormat(tm)\n\n\treturn fromDate, untilDate\n}\n\nfunc (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {\n\t// Don't process, if the tagged table is not set\n\tif h.config.ClickHouse.TaggedTable == \"\" {\n\t\tw.Write([]byte{'[', ']'})\n\t\treturn\n\t}\n\n\tif h.isValues {\n\t\th.ServeValues(w, r)\n\t} else {\n\t\th.ServeTags(w, r)\n\t}\n}\n\nfunc getTagCountQuerier(config *config.Config, opts clickhouse.Options) *finder.TagCountQuerier {\n\tvar tcq *finder.TagCountQuerier = nil\n\tif config.ClickHouse.TagsCountTable != \"\" {\n\t\ttcq = finder.NewTagCountQuerier(\n\t\t\tconfig.ClickHouse.URL,\n\t\t\tconfig.ClickHouse.TagsCountTable,\n\t\t\topts,\n\t\t\tconfig.FeatureFlags.UseCarbonBehavior,\n\t\t\tconfig.FeatureFlags.DontMatchMissingTags,\n\t\t\tconfig.ClickHouse.TaggedUseDaily,\n\t\t)\n\t}\n\n\treturn tcq\n}\n\nfunc (h *Handler) requestExpr(r *http.Request, tcq *finder.TagCountQuerier, from, until int64) (*where.Where, *where.Where, map[string]bool, error) {\n\tformExpr := r.Form[\"expr\"]\n\texpr := make([]string, 0, len(formExpr))\n\n\tfor i := 0; i < len(formExpr); i++ {\n\t\tif formExpr[i] != \"\" {\n\t\t\texpr = append(expr, formExpr[i])\n\t\t}\n\t}\n\n\tusedTags := make(map[string]bool)\n\n\twr := where.New()\n\tpw := where.New()\n\n\tif len(expr) == 0 {\n\t\treturn wr, pw, usedTags, nil\n\t}\n\n\tterms, err := finder.ParseTaggedConditions(expr, h.config, true)\n\tif err != nil {\n\t\treturn wr, pw, usedTags, err\n\t}\n\n\tif tcq != nil {\n\t\ttagValuesCosts, err := tcq.GetCostsFromCountTable(r.Context(), terms, from, until)\n\t\tif err != nil {\n\t\t\treturn wr, pw, usedTags, err\n\t\t}\n\n\t\tif tagValuesCosts != nil {\n\t\t\tfinder.SetCosts(terms, tagValuesCosts)\n\t\t} else if len(h.config.ClickHouse.TaggedCosts) != 0 {\n\t\t\tfinder.SetCosts(terms, h.config.ClickHouse.TaggedCosts)\n\t\t}\n\t}\n\n\tfinder.SortTaggedTermsByCost(terms)\n\n\twr, pw, err = finder.TaggedWhere(terms, h.config.FeatureFlags.UseCarbonBehavior, h.config.FeatureFlags.DontMatchMissingTags)\n\tif err != nil {\n\t\treturn wr, pw, usedTags, err\n\t}\n\n\tfor i := 0; i < len(expr); i++ {\n\t\ta := strings.Split(expr[i], \"=\")\n\t\tusedTags[a[0]] = true\n\t}\n\n\treturn wr, pw, usedTags, nil\n}\n\nfunc taggedKey(typ string, truncateSec int32, fromDate, untilDate string, tag string, exprs []string, tagPrefix string, limit int) (string, string) {\n\tts := utils.TimestampTruncate(timeNow().Unix(), time.Duration(truncateSec)*time.Second)\n\n\tvar sb stringutils.Builder\n\n\tsb.Grow(128)\n\tsb.WriteString(typ)\n\tsb.WriteString(fromDate)\n\tsb.WriteByte(';')\n\tsb.WriteString(untilDate)\n\tsb.WriteString(\";limit=\")\n\tsb.WriteInt(int64(limit), 10)\n\ttagStart := sb.Len()\n\n\tif tagPrefix != \"\" {\n\t\tsb.WriteString(\";tagPrefix=\")\n\t\tsb.WriteString(tagPrefix)\n\t}\n\n\tif tag != \"\" {\n\t\tsb.WriteString(\";tag=\")\n\t\tsb.WriteString(tag)\n\t}\n\n\tfor _, expr := range exprs {\n\t\tsb.WriteString(\";expr='\")\n\t\tsb.WriteString(strings.Replace(expr, \" = \", \"=\", 1))\n\t\tsb.WriteByte('\\'')\n\t}\n\n\texprEnd := sb.Len()\n\tsb.WriteString(\";ts=\")\n\tsb.WriteString(strconv.FormatInt(ts, 10))\n\n\ts := sb.String()\n\n\treturn s, s[tagStart:exprEnd]\n}\n\nfunc taggedValuesKey(typ string, truncateSec int32, fromDate, untilDate string, tag string, exprs []string, valuePrefix string, limit int) (string, string) {\n\tts := utils.TimestampTruncate(timeNow().Unix(), time.Duration(truncateSec)*time.Second)\n\n\tvar sb stringutils.Builder\n\n\tsb.Grow(128)\n\tsb.WriteString(typ)\n\tsb.WriteString(fromDate)\n\tsb.WriteByte(';')\n\tsb.WriteString(untilDate)\n\tsb.WriteString(\";limit=\")\n\tsb.WriteInt(int64(limit), 10)\n\ttagStart := sb.Len()\n\n\tif valuePrefix != \"\" {\n\t\tsb.WriteString(\";valuePrefix=\")\n\t\tsb.WriteString(valuePrefix)\n\t}\n\n\tif tag != \"\" {\n\t\tsb.WriteString(\";tag=\")\n\t\tsb.WriteString(tag)\n\t}\n\n\tfor _, expr := range exprs {\n\t\tsb.WriteString(\";expr='\")\n\t\tsb.WriteString(strings.Replace(expr, \" = \", \"=\", 1))\n\t\tsb.WriteByte('\\'')\n\t}\n\n\texprEnd := sb.Len()\n\tsb.WriteString(\";ts=\")\n\tsb.WriteString(strconv.FormatInt(ts, 10))\n\n\ts := sb.String()\n\n\treturn s, s[tagStart:exprEnd]\n}\n\n// func taggedTagsQuery(exprs []string, tagPrefix string, limit int) []string {\n// \tquery := make([]string, 0, 3+len(exprs))\n// \tif tagPrefix != \"\" {\n// \t\tquery = append(query, \"tagPrefix=\"+tagPrefix)\n// \t}\n// \tfor _, expr := range exprs {\n// \t\tquery = append(query, \"expr='\"+expr+\"'\")\n// \t}\n// \tquery = append(query, \"limit=\"+strconv.Itoa(limit))\n// \treturn query\n// }\n\nfunc (h *Handler) ServeTags(w http.ResponseWriter, r *http.Request) {\n\tstart := timeNow()\n\tstatus := http.StatusOK\n\taccessLogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named(\"http\")\n\tlogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named(\"autocomplete\")\n\tr = r.WithContext(scope.WithLogger(r.Context(), logger))\n\n\tvar (\n\t\terr error\n\t\tchReadRows int64\n\t\tchReadBytes int64\n\t\tmetricsCount int64\n\t\treadBytes int64\n\t\tqueueFail bool\n\t\tqueueDuration time.Duration\n\t\tfindCache bool\n\t\topts clickhouse.Options\n\t)\n\n\tusername := r.Header.Get(\"X-Forwarded-User\")\n\tlimiter := h.config.GetUserTagsLimiter(username)\n\n\tdefer func() {\n\t\tif rec := recover(); rec != nil {\n\t\t\tstatus = http.StatusInternalServerError\n\n\t\t\tlogger.Error(\"panic during eval:\",\n\t\t\t\tzap.String(\"requestID\", scope.String(r.Context(), \"requestID\")),\n\t\t\t\tzap.Any(\"reason\", rec),\n\t\t\t\tzap.Stack(\"stack\"),\n\t\t\t)\n\n\t\t\tanswer := fmt.Sprintf(\"%v\\nStack trace: %v\", rec, zap.Stack(\"\").String)\n\t\t\thttp.Error(w, answer, status)\n\t\t}\n\n\t\td := time.Since(start)\n\t\tdMS := d.Milliseconds()\n\t\tlogs.AccessLog(accessLogger, h.config, r, status, d, queueDuration, findCache, queueFail)\n\t\tlimiter.SendDuration(queueDuration.Milliseconds())\n\t\tmetrics.SendFindMetrics(metrics.TagsRequestMetric, status, dMS, 0, h.config.Metrics.ExtendedStat, metricsCount)\n\n\t\tif !findCache && chReadRows > 0 && chReadBytes > 0 {\n\t\t\terrored := status != http.StatusOK && status != http.StatusNotFound\n\t\t\tmetrics.SendQueryRead(metrics.AutocompleteQMetric, 0, 0, dMS, metricsCount, readBytes, chReadRows, chReadBytes, errored)\n\t\t}\n\t}()\n\n\tr.ParseMultipartForm(1024 * 1024)\n\ttagPrefix := r.FormValue(\"tagPrefix\")\n\tlimitStr := r.FormValue(\"limit\")\n\tlimit := 10000\n\n\tvar body []byte\n\n\tif limitStr != \"\" {\n\t\tlimit, err = strconv.Atoi(limitStr)\n\t\tif err == finder.ErrCostlySeriesByTag {\n\t\t\tstatus = http.StatusForbidden\n\t\t\thttp.Error(w, err.Error(), status)\n\n\t\t\treturn\n\t\t} else if err != nil {\n\t\t\tstatus = http.StatusBadRequest\n\t\t\thttp.Error(w, err.Error(), status)\n\n\t\t\treturn\n\t\t}\n\t}\n\n\tfromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, start)\n\n\tvar key string\n\n\texprs := r.Form[\"expr\"]\n\t// params := taggedTagsQuery(exprs, tagPrefix, limit)\n\n\tuseCache := h.config.Common.FindCache != nil && h.config.Common.FindCacheConfig.FindTimeoutSec > 0 && !parser.TruthyBool(r.FormValue(\"noCache\"))\n\tif useCache {\n\t\tkey, _ = taggedKey(\"tags;\", h.config.Common.FindCacheConfig.FindTimeoutSec, fromDate, untilDate, \"\", exprs, tagPrefix, limit)\n\n\t\tbody, err = h.config.Common.FindCache.Get(key)\n\t\tif err == nil {\n\t\t\tif metrics.FinderCacheMetrics != nil {\n\t\t\t\tmetrics.FinderCacheMetrics.CacheHits.Add(1)\n\t\t\t}\n\n\t\t\tfindCache = true\n\n\t\t\tw.Header().Set(\"X-Cached-Find\", strconv.Itoa(int(h.config.Common.FindCacheConfig.FindTimeoutSec)))\n\t\t}\n\t}\n\n\topts = clickhouse.Options{\n\t\tTLSConfig: h.config.ClickHouse.TLSConfig,\n\t\tTimeout: h.config.ClickHouse.IndexTimeout,\n\t\tConnectTimeout: h.config.ClickHouse.ConnectTimeout,\n\t\tCheckRequestProgress: h.config.FeatureFlags.LogQueryProgress,\n\t\tProgressSendingInterval: h.config.ClickHouse.ProgressSendingInterval,\n\t}\n\n\twr, pw, usedTags, err := h.requestExpr(\n\t\tr,\n\t\tgetTagCountQuerier(h.config, opts),\n\t\tstart.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Unix(),\n\t\tstart.Unix(),\n\t)\n\n\tif err != nil {\n\t\tstatus = http.StatusBadRequest\n\t\thttp.Error(w, err.Error(), status)\n\n\t\treturn\n\t}\n\n\tif !findCache {\n\t\tvar valueSQL string\n\n\t\tif len(usedTags) == 0 {\n\t\t\tvalueSQL = \"splitByChar('=', Tag1)[1] AS value\"\n\n\t\t\tif tagPrefix != \"\" {\n\t\t\t\twr.And(where.HasPrefix(\"Tag1\", tagPrefix))\n\t\t\t}\n\t\t} else {\n\t\t\tvalueSQL = \"splitByChar('=', arrayJoin(Tags))[1] AS value\"\n\n\t\t\tif tagPrefix != \"\" {\n\t\t\t\twr.And(where.HasPrefix(\"arrayJoin(Tags)\", tagPrefix))\n\t\t\t}\n\t\t}\n\n\t\tqueryLimit := limit + len(usedTags)\n\n\t\twr.Andf(\"Date >= '%s' AND Date <= '%s'\", fromDate, untilDate)\n\n\t\tsql := fmt.Sprintf(\"SELECT %s FROM %s %s %s GROUP BY value ORDER BY value LIMIT %d\",\n\t\t\tvalueSQL,\n\t\t\th.config.ClickHouse.TaggedTable,\n\t\t\tpw.PreWhereSQL(),\n\t\t\twr.SQL(),\n\t\t\tqueryLimit,\n\t\t)\n\n\t\tvar (\n\t\t\tentered bool\n\t\t\tctx context.Context\n\t\t\tcancel context.CancelFunc\n\t\t)\n\n\t\tif limiter.Enabled() {\n\t\t\tctx, cancel = context.WithTimeout(context.Background(), h.config.ClickHouse.IndexTimeout)\n\t\t\tdefer cancel()\n\n\t\t\terr = limiter.Enter(ctx, \"tags\")\n\t\t\tqueueDuration = time.Since(start)\n\n\t\t\tif err != nil {\n\t\t\t\tstatus = http.StatusServiceUnavailable\n\t\t\t\tqueueFail = true\n\n\t\t\t\tlogger.Error(err.Error())\n\t\t\t\thttp.Error(w, err.Error(), status)\n\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tqueueDuration = time.Since(start)\n\t\t\tentered = true\n\n\t\t\tdefer func() {\n\t\t\t\tif entered {\n\t\t\t\t\tlimiter.Leave(ctx, \"tags\")\n\n\t\t\t\t\tentered = false\n\t\t\t\t}\n\t\t\t}()\n\t\t}\n\n\t\tbody, chReadRows, chReadBytes, err = clickhouse.Query(\n\t\t\tscope.WithTable(r.Context(), h.config.ClickHouse.TaggedTable),\n\t\t\th.config.ClickHouse.URL,\n\t\t\tsql,\n\t\t\topts,\n\t\t\tnil,\n\t\t)\n\n\t\tif entered {\n\t\t\t// release early as possible\n\t\t\tlimiter.Leave(ctx, \"tags\")\n\n\t\t\tentered = false\n\t\t}\n\n\t\tif err != nil {\n\t\t\tstatus, _ = clickhouse.HandleError(w, err)\n\t\t\treturn\n\t\t}\n\n\t\treadBytes = int64(len(body))\n\n\t\tif useCache {\n\t\t\tif metrics.FinderCacheMetrics != nil {\n\t\t\t\tmetrics.FinderCacheMetrics.CacheMisses.Add(1)\n\t\t\t}\n\n\t\t\th.config.Common.FindCache.Set(key, body, h.config.Common.FindCacheConfig.FindTimeoutSec)\n\t\t}\n\t}\n\n\trows := strings.Split(stringutils.UnsafeString(body), \"\\n\")\n\ttags := make([]string, 0, uint64(len(rows))+1) // +1 - reserve for \"name\" tag\n\n\thasName := false\n\n\tfor i := 0; i < len(rows); i++ {\n\t\tif rows[i] == \"\" {\n\t\t\tcontinue\n\t\t}\n\n\t\tif rows[i] == \"__name__\" {\n\t\t\trows[i] = \"name\"\n\t\t}\n\n\t\tif usedTags[rows[i]] {\n\t\t\tcontinue\n\t\t}\n\n\t\ttags = append(tags, rows[i])\n\n\t\tif rows[i] == \"name\" {\n\t\t\thasName = true\n\t\t}\n\t}\n\n\tif !hasName && !usedTags[\"name\"] && (tagPrefix == \"\" || strings.HasPrefix(\"name\", tagPrefix)) {\n\t\ttags = append(tags, \"name\")\n\t}\n\n\tsort.Strings(tags)\n\n\tif len(tags) > limit {\n\t\ttags = tags[:limit]\n\t}\n\n\tif useCache {\n\t\tif findCache {\n\t\t\tlogger.Info(\"finder\", zap.String(\"get_cache\", key),\n\t\t\t\tzap.Int(\"metrics\", len(rows)), zap.Bool(\"find_cached\", true),\n\t\t\t\tzap.Int32(\"ttl\", h.config.Common.FindCacheConfig.FindTimeoutSec))\n\t\t} else {\n\t\t\tlogger.Info(\"finder\", zap.String(\"set_cache\", key),\n\t\t\t\tzap.Int(\"metrics\", len(rows)), zap.Bool(\"find_cached\", false),\n\t\t\t\tzap.Int32(\"ttl\", h.config.Common.FindCacheConfig.FindTimeoutSec))\n\t\t}\n\t}\n\n\tb, err := json.Marshal(tags)\n\tif err != nil {\n\t\tstatus = http.StatusInternalServerError\n\t\thttp.Error(w, err.Error(), status)\n\n\t\treturn\n\t}\n\n\tmetricsCount = int64(len(tags))\n\n\tw.Write(b)\n}\n\n// func taggedValuesQuery(tag string, exprs []string, valuePrefix string, limit int) []string {\n// \tquery := make([]string, 0, 3+len(exprs))\n// \tif tag != \"\" {\n// \t\tquery = append(query, \"tag=\"+tag)\n// \t}\n// \tif valuePrefix != \"\" {\n// \t\tquery = append(query, \"valuePrefix=\"+valuePrefix)\n// \t}\n// \tfor _, expr := range exprs {\n// \t\tquery = append(query, \"expr='\"+expr+\"'\")\n// \t}\n// \tquery = append(query, \"limit=\"+strconv.Itoa(limit))\n// \treturn query\n// }\n\nfunc (h *Handler) ServeValues(w http.ResponseWriter, r *http.Request) {\n\tstart := timeNow()\n\tstatus := http.StatusOK\n\taccessLogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named(\"http\")\n\tlogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named(\"autocomplete\")\n\tr = r.WithContext(scope.WithLogger(r.Context(), logger))\n\n\tvar (\n\t\terr error\n\t\tbody []byte\n\t\tchReadRows int64\n\t\tchReadBytes int64\n\t\tmetricsCount int64\n\t\tqueueFail bool\n\t\tqueueDuration time.Duration\n\t\tfindCache bool\n\t\topts clickhouse.Options\n\t)\n\n\tusername := r.Header.Get(\"X-Forwarded-User\")\n\tlimiter := h.config.GetUserTagsLimiter(username)\n\n\tdefer func() {\n\t\tif rec := recover(); rec != nil {\n\t\t\tstatus = http.StatusInternalServerError\n\n\t\t\tlogger.Error(\"panic during eval:\",\n\t\t\t\tzap.String(\"requestID\", scope.String(r.Context(), \"requestID\")),\n\t\t\t\tzap.Any(\"reason\", rec),\n\t\t\t\tzap.Stack(\"stack\"),\n\t\t\t)\n\n\t\t\tanswer := fmt.Sprintf(\"%v\\nStack trace: %v\", rec, zap.Stack(\"\").String)\n\t\t\thttp.Error(w, answer, status)\n\t\t}\n\n\t\td := time.Since(start)\n\t\tdMS := d.Milliseconds()\n\t\tlogs.AccessLog(accessLogger, h.config, r, status, d, queueDuration, findCache, queueFail)\n\t\tlimiter.SendDuration(queueDuration.Milliseconds())\n\t\tmetrics.SendFindMetrics(metrics.TagsRequestMetric, status, dMS, 0, h.config.Metrics.ExtendedStat, metricsCount)\n\n\t\tif !findCache && chReadRows > 0 && chReadBytes > 0 {\n\t\t\terrored := status != http.StatusOK && status != http.StatusNotFound\n\t\t\tmetrics.SendQueryRead(metrics.AutocompleteQMetric, 0, 0, dMS, metricsCount, int64(len(body)), chReadRows, chReadBytes, errored)\n\t\t}\n\t}()\n\n\tr.ParseMultipartForm(1024 * 1024)\n\n\ttag := r.FormValue(\"tag\")\n\tif tag == \"name\" {\n\t\ttag = \"__name__\"\n\t}\n\n\tvaluePrefix := r.FormValue(\"valuePrefix\")\n\tlimitStr := r.FormValue(\"limit\")\n\tlimit := 10000\n\n\tif limitStr != \"\" {\n\t\tlimit, err = strconv.Atoi(limitStr)\n\t\tif err != nil {\n\t\t\tstatus = http.StatusBadRequest\n\t\t\thttp.Error(w, err.Error(), status)\n\n\t\t\treturn\n\t\t}\n\t}\n\n\tfromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, start)\n\n\tvar key string\n\n\texprs := r.Form[\"expr\"]\n\t// params := taggedValuesQuery(tag, exprs, valuePrefix, limit)\n\n\t// taggedKey(tag, , \"valuePrefix=\"+valuePrefix, limit)\n\tuseCache := h.config.Common.FindCache != nil && h.config.Common.FindCacheConfig.FindTimeoutSec > 0 && !parser.TruthyBool(r.FormValue(\"noCache\"))\n\tif useCache {\n\t\t// logger = logger.With(zap.String(\"use_cache\", \"true\"))\n\t\tkey, _ = taggedValuesKey(\"values;\", h.config.Common.FindCacheConfig.FindTimeoutSec, fromDate, untilDate, tag, exprs, valuePrefix, limit)\n\n\t\tbody, err = h.config.Common.FindCache.Get(key)\n\t\tif err == nil {\n\t\t\tif metrics.FinderCacheMetrics != nil {\n\t\t\t\tmetrics.FinderCacheMetrics.CacheHits.Add(1)\n\t\t\t}\n\n\t\t\tfindCache = true\n\n\t\t\tw.Header().Set(\"X-Cached-Find\", strconv.Itoa(int(h.config.Common.FindCacheConfig.FindTimeoutSec)))\n\t\t}\n\t}\n\n\topts = clickhouse.Options{\n\t\tTLSConfig: h.config.ClickHouse.TLSConfig,\n\t\tTimeout: h.config.ClickHouse.IndexTimeout,\n\t\tConnectTimeout: h.config.ClickHouse.ConnectTimeout,\n\t\tCheckRequestProgress: h.config.FeatureFlags.LogQueryProgress,\n\t\tProgressSendingInterval: h.config.ClickHouse.ProgressSendingInterval,\n\t}\n\n\tif !findCache {\n\t\twr, pw, usedTags, err := h.requestExpr(\n\t\t\tr,\n\t\t\tgetTagCountQuerier(h.config, opts),\n\t\t\tstart.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Unix(),\n\t\t\tstart.Unix(),\n\t\t)\n\n\t\tif err == finder.ErrCostlySeriesByTag {\n\t\t\tstatus = http.StatusForbidden\n\t\t\thttp.Error(w, err.Error(), status)\n\n\t\t\treturn\n\t\t} else if err != nil {\n\t\t\tstatus = http.StatusBadRequest\n\t\t\thttp.Error(w, err.Error(), status)\n\n\t\t\treturn\n\t\t}\n\n\t\tvar valueSQL string\n\t\tif len(usedTags) == 0 {\n\t\t\tvalueSQL = fmt.Sprintf(\"substr(Tag1, %d) AS value\", len(tag)+2)\n\t\t\twr.And(where.HasPrefix(\"Tag1\", tag+\"=\"+valuePrefix))\n\t\t} else {\n\t\t\tprefixSelector := where.HasPrefix(\"x\", tag+\"=\"+valuePrefix)\n\t\t\tvalueSQL = fmt.Sprintf(\"substr(arrayFilter(x -> %s, Tags)[1], %d) AS value\", prefixSelector, len(tag)+2)\n\t\t\twr.And(\"arrayExists(x -> \" + prefixSelector + \", Tags)\")\n\t\t}\n\n\t\twr.Andf(\"Date >= '%s' AND Date <= '%s'\", fromDate, untilDate)\n\n\t\tsql := fmt.Sprintf(\"SELECT %s FROM %s %s %s GROUP BY value ORDER BY value LIMIT %d\",\n\t\t\tvalueSQL,\n\t\t\th.config.ClickHouse.TaggedTable,\n\t\t\tpw.PreWhereSQL(),\n\t\t\twr.SQL(),\n\t\t\tlimit,\n\t\t)\n\n\t\tvar (\n\t\t\tentered bool\n\t\t\tctx context.Context\n\t\t\tcancel context.CancelFunc\n\t\t)\n\n\t\tif limiter.Enabled() {\n\t\t\tctx, cancel = context.WithTimeout(context.Background(), h.config.ClickHouse.IndexTimeout)\n\t\t\tdefer cancel()\n\n\t\t\terr = limiter.Enter(ctx, \"tags\")\n\t\t\tqueueDuration = time.Since(start)\n\n\t\t\tif err != nil {\n\t\t\t\tstatus = http.StatusServiceUnavailable\n\t\t\t\tqueueFail = true\n\n\t\t\t\tlogger.Error(err.Error())\n\t\t\t\thttp.Error(w, err.Error(), status)\n\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tqueueDuration = time.Since(start)\n\t\t\tentered = true\n\n\t\t\tdefer func() {\n\t\t\t\tif entered {\n\t\t\t\t\tlimiter.Leave(ctx, \"tags\")\n\n\t\t\t\t\tentered = false\n\t\t\t\t}\n\t\t\t}()\n\t\t}\n\n\t\tbody, chReadRows, chReadBytes, err = clickhouse.Query(\n\t\t\tscope.WithTable(r.Context(), h.config.ClickHouse.TaggedTable),\n\t\t\th.config.ClickHouse.URL,\n\t\t\tsql,\n\t\t\topts,\n\t\t\tnil,\n\t\t)\n\n\t\tif entered {\n\t\t\t// release early as possible\n\t\t\tlimiter.Leave(ctx, \"tags\")\n\n\t\t\tentered = false\n\t\t}\n\n\t\tif err != nil {\n\t\t\tstatus, _ = clickhouse.HandleError(w, err)\n\t\t\treturn\n\t\t}\n\n\t\tif useCache {\n\t\t\tif metrics.FinderCacheMetrics != nil {\n\t\t\t\tmetrics.FinderCacheMetrics.CacheMisses.Add(1)\n\t\t\t}\n\n\t\t\th.config.Common.FindCache.Set(key, body, h.config.Common.FindCacheConfig.FindTimeoutSec)\n\t\t}\n\t}\n\n\tvar rows []string\n\tif len(body) > 0 {\n\t\trows = strings.Split(stringutils.UnsafeString(body), \"\\n\")\n\t\tif len(rows) > 0 && rows[len(rows)-1] == \"\" {\n\t\t\trows = rows[:len(rows)-1]\n\t\t}\n\n\t\tmetricsCount = int64(len(rows))\n\t}\n\n\tif useCache {\n\t\tif findCache {\n\t\t\tlogger.Info(\"finder\", zap.String(\"get_cache\", key),\n\t\t\t\tzap.Int(\"metrics\", len(rows)), zap.Bool(\"find_cached\", true),\n\t\t\t\tzap.Int32(\"ttl\", h.config.Common.FindCacheConfig.FindTimeoutSec))\n\t\t} else {\n\t\t\tlogger.Info(\"finder\", zap.String(\"set_cache\", key),\n\t\t\t\tzap.Int(\"metrics\", len(rows)), zap.Bool(\"find_cached\", false),\n\t\t\t\tzap.Int32(\"ttl\", h.config.Common.FindCacheConfig.FindTimeoutSec))\n\t\t}\n\t}\n\n\tb, err := json.Marshal(rows)\n\tif err != nil {\n\t\tstatus = http.StatusInternalServerError\n\t\thttp.Error(w, err.Error(), status)\n\n\t\treturn\n\t}\n\n\tw.Write(b)\n}\n" }, { "path": "autocomplete/autocomplete_test.go", "content": "package autocomplete\n\nimport (\n\t\"io\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/date\"\n\tchtest \"github.com/lomik/graphite-clickhouse/helper/tests/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc NewRequest(method, url string, body io.Reader) *http.Request {\n\tr, _ := http.NewRequest(method, url, body)\n\n\treturn r\n}\n\ntype testStruct struct {\n\trequest *http.Request\n\twantCode int\n\twant string\n\twantContent string\n}\n\nfunc testResponce(t *testing.T, step int, h *Handler, tt *testStruct, wantCachedFind string) {\n\tw := httptest.NewRecorder()\n\n\th.ServeHTTP(w, tt.request)\n\n\ts := w.Body.String()\n\n\tassert.Equalf(t, tt.wantCode, w.Code, \"code mismatch step %d\\n,%s\", step, s)\n\n\tif w.Code == http.StatusOK {\n\t\tif tt.wantContent != \"\" {\n\t\t\tcontentType := w.Header().Get(\"Content-Type\")\n\t\t\tassert.Equalf(t, tt.wantContent, contentType, \"content type mismatch, step %d\", step)\n\t\t}\n\n\t\tcachedFindHeader := w.Header().Get(\"X-Cached-Find\")\n\t\tassert.Equalf(t, cachedFindHeader, wantCachedFind, \"cached find '%s' mismatch, want be %v, step %d\", cachedFindHeader, wantCachedFind, step)\n\n\t\tassert.Equalf(t, tt.want, s, \"Step %d\", step)\n\t}\n}\n\nfunc TestHandler_ServeTags(t *testing.T) {\n\ttimeNow = func() time.Time {\n\t\treturn time.Unix(1669714247, 0)\n\t}\n\n\tmetrics.DisableMetrics()\n\n\tsrv := chtest.NewTestServer()\n\tdefer srv.Close()\n\n\tcfg, _ := config.DefaultConfig()\n\tcfg.ClickHouse.URL = srv.URL\n\tcfg.ClickHouse.TaggedTable = \"graphite_tagged\"\n\n\th := NewTags(cfg)\n\n\tnow := timeNow()\n\tfromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)\n\n\t// Test 1: Get all tags without filters\n\tsrv.AddResponce(\n\t\t\"SELECT splitByChar('=', Tag1)[1] AS value FROM graphite_tagged WHERE \"+\n\t\t\t\"Date >= '\"+fromDate+\"' AND Date <= '\"+untilDate+\"' GROUP BY value ORDER BY value LIMIT 10000\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"__name__\\nenvironment\\nproject\\nhost\\n\"),\n\t\t})\n\n\t// Test 2: Get tags with prefix filter\n\tsrv.AddResponce(\n\t\t\"SELECT splitByChar('=', Tag1)[1] AS value FROM graphite_tagged WHERE \"+\n\t\t\t\"(Tag1 LIKE 'pr%') AND (Date >= '\"+fromDate+\"' AND Date <= '\"+untilDate+\"') GROUP BY value ORDER BY value LIMIT 10000\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"project\\n\"),\n\t\t})\n\n\t// Test 3: Get tags with expr filters\n\tsrv.AddResponce(\n\t\t\"SELECT splitByChar('=', arrayJoin(Tags))[1] AS value FROM graphite_tagged WHERE \"+\n\t\t\t\"(Tag1='environment=production') AND (Date >= '\"+fromDate+\"' AND Date <= '\"+untilDate+\"') GROUP BY value ORDER BY value LIMIT 10001\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"__name__\\nhost\\nproject\\n\"),\n\t\t})\n\n\t// Test 4: Get tags with multiple expr filters\n\tsrv.AddResponce(\n\t\t\"SELECT splitByChar('=', arrayJoin(Tags))[1] AS value FROM graphite_tagged WHERE \"+\n\t\t\t\"((Tag1='environment=production') AND (has(Tags, 'project=web'))) AND (Date >= '\"+fromDate+\"' AND Date <= '\"+untilDate+\"') GROUP BY value ORDER BY value LIMIT 10002\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"__name__\\nhost\\n\"),\n\t\t})\n\n\t// Test 5: Get tags with prefix and expr filters\n\tsrv.AddResponce(\n\t\t\"SELECT splitByChar('=', arrayJoin(Tags))[1] AS value FROM graphite_tagged WHERE \"+\n\t\t\t\"((Tag1='environment=production') AND (arrayJoin(Tags) LIKE 'h%')) AND (Date >= '\"+fromDate+\"' AND Date <= '\"+untilDate+\"') GROUP BY value ORDER BY value LIMIT 10001\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"host\\n\"),\n\t\t})\n\n\ttests := []testStruct{\n\t\t{\n\t\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/tags\", nil),\n\t\t\twantCode: http.StatusOK,\n\t\t\twant: `[\"environment\",\"host\",\"name\",\"project\"]`,\n\t\t},\n\t\t{\n\t\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/tags?tagPrefix=pr\", nil),\n\t\t\twantCode: http.StatusOK,\n\t\t\twant: `[\"project\"]`,\n\t\t},\n\t\t{\n\t\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/tags?expr=environment%3Dproduction\", nil),\n\t\t\twantCode: http.StatusOK,\n\t\t\twant: `[\"host\",\"name\",\"project\"]`,\n\t\t},\n\t\t{\n\t\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/tags?expr=environment%3Dproduction&expr=project%3Dweb\", nil),\n\t\t\twantCode: http.StatusOK,\n\t\t\twant: `[\"host\",\"name\"]`,\n\t\t},\n\t\t{\n\t\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/tags?expr=environment%3Dproduction&tagPrefix=h\", nil),\n\t\t\twantCode: http.StatusOK,\n\t\t\twant: `[\"host\"]`,\n\t\t},\n\t}\n\n\tfor i, tt := range tests {\n\t\tt.Run(\"Test#\"+strconv.Itoa(i), func(t *testing.T) {\n\t\t\ttestResponce(t, i, h, &tt, \"\")\n\t\t})\n\t}\n}\n\nfunc TestHandler_ServeTagsWithCache(t *testing.T) {\n\ttimeNow = func() time.Time {\n\t\treturn time.Unix(1669714247, 0)\n\t}\n\n\tmetrics.DisableMetrics()\n\n\tsrv := chtest.NewTestServer()\n\tdefer srv.Close()\n\n\tcfg, _ := config.DefaultConfig()\n\tcfg.ClickHouse.URL = srv.URL\n\tcfg.ClickHouse.TaggedTable = \"graphite_tagged\"\n\n\t// Enable cache\n\tcfg.Common.FindCacheConfig = config.CacheConfig{\n\t\tType: \"mem\",\n\t\tSize: 8192,\n\t\tFindTimeoutSec: 1,\n\t}\n\n\tvar err error\n\n\tcfg.Common.FindCache, err = config.CreateCache(\"autocomplete\", &cfg.Common.FindCacheConfig)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create find cache: %v\", err)\n\t}\n\n\th := NewTags(cfg)\n\n\tnow := timeNow()\n\tfromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)\n\n\tsrv.AddResponce(\n\t\t\"SELECT splitByChar('=', Tag1)[1] AS value FROM graphite_tagged WHERE \"+\n\t\t\t\"Date >= '\"+fromDate+\"' AND Date <= '\"+untilDate+\"' GROUP BY value ORDER BY value LIMIT 10000\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"__name__\\nenvironment\\nproject\\nhost\\n\"),\n\t\t})\n\n\ttest := testStruct{\n\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/tags\", nil),\n\t\twantCode: http.StatusOK,\n\t\twant: `[\"environment\",\"host\",\"name\",\"project\"]`,\n\t}\n\n\t// First request - should hit the database\n\ttestResponce(t, 0, h, &test, \"\")\n\tassert.Equal(t, uint64(1), srv.Queries())\n\n\t// Second request - should hit the cache\n\ttestResponce(t, 1, h, &test, \"1\")\n\tassert.Equal(t, uint64(1), srv.Queries()) // No new queries\n\n\t// Wait for cache expiration\n\ttime.Sleep(time.Second * 2)\n\n\t// Third request - should hit the database again\n\ttestResponce(t, 2, h, &test, \"\")\n\tassert.Equal(t, uint64(2), srv.Queries())\n}\n\nfunc TestHandler_ServeValues(t *testing.T) {\n\ttimeNow = func() time.Time {\n\t\treturn time.Unix(1669714247, 0)\n\t}\n\n\tmetrics.DisableMetrics()\n\n\tsrv := chtest.NewTestServer()\n\tdefer srv.Close()\n\n\tcfg, _ := config.DefaultConfig()\n\tcfg.ClickHouse.URL = srv.URL\n\tcfg.ClickHouse.TaggedTable = \"graphite_tagged\"\n\n\th := NewValues(cfg)\n\n\tnow := timeNow()\n\tuntil := strconv.FormatInt(now.Unix(), 10)\n\tfrom := strconv.FormatInt(now.Add(-time.Minute).Unix(), 10)\n\tfromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)\n\n\tsrv.AddResponce(\n\t\t\"SELECT substr(arrayFilter(x -> x LIKE 'host=%', Tags)[1], 6) AS value FROM graphite_tagged WHERE (((Tag1='environment=production') AND (has(Tags, 'project=web'))) AND (arrayExists(x -> x LIKE 'host=%', Tags))) AND \"+\n\t\t\t\"(Date >= '\"+fromDate+\"' AND Date <= '\"+untilDate+\"') GROUP BY value ORDER BY value LIMIT 10000\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"host1\\nhost2\\ndc-host2\\ndc-host3\\n\"),\n\t\t})\n\n\ttests := []testStruct{\n\t\t{\n\t\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/values?\"+\n\t\t\t\t\"expr=environment%3Dproduction\"+\"&\"+\"expr=project%3Dweb\"+\"&\"+\"tag=host\"+\n\t\t\t\t\"&limit=10000&from=\"+from+\"&until=\"+until, nil),\n\t\t\twantCode: http.StatusOK,\n\t\t\twant: \"[\\\"host1\\\",\\\"host2\\\",\\\"dc-host2\\\",\\\"dc-host3\\\"]\",\n\t\t\twantContent: \"text/plain; charset=utf-8\",\n\t\t},\n\t}\n\n\tvar queries uint64\n\n\tfor i, tt := range tests {\n\t\tt.Run(tt.request.URL.RawQuery+\"#\"+strconv.Itoa(i), func(t *testing.T) {\n\t\t\tfor i := 0; i < 2; i++ {\n\t\t\t\ttestResponce(t, i, h, &tt, \"\")\n\t\t\t}\n\n\t\t\tassert.Equal(t, uint64(2), srv.Queries()-queries)\n\t\t\tqueries = srv.Queries()\n\t\t})\n\t}\n}\n\nfunc TestHandler_ServeValuesWithValuePrefix(t *testing.T) {\n\ttimeNow = func() time.Time {\n\t\treturn time.Unix(1669714247, 0)\n\t}\n\n\tmetrics.DisableMetrics()\n\n\tsrv := chtest.NewTestServer()\n\tdefer srv.Close()\n\n\tcfg, _ := config.DefaultConfig()\n\tcfg.ClickHouse.URL = srv.URL\n\tcfg.ClickHouse.TaggedTable = \"graphite_tagged\"\n\n\th := NewValues(cfg)\n\n\tnow := timeNow()\n\tfromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)\n\n\t// Test with valuePrefix\n\tsrv.AddResponce(\n\t\t\"SELECT substr(Tag1, 6) AS value FROM graphite_tagged WHERE \"+\n\t\t\t\"(Tag1 LIKE 'host=dc-%') AND (Date >= '\"+fromDate+\"' AND Date <= '\"+untilDate+\"') GROUP BY value ORDER BY value LIMIT 100\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"dc-host1\\ndc-host2\\ndc-host3\\n\"),\n\t\t})\n\n\ttest := testStruct{\n\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/values?tag=host&valuePrefix=dc-&limit=100\", nil),\n\t\twantCode: http.StatusOK,\n\t\twant: \"[\\\"dc-host1\\\",\\\"dc-host2\\\",\\\"dc-host3\\\"]\",\n\t}\n\n\ttestResponce(t, 0, h, &test, \"\")\n}\n\nfunc TestHandler_ServeValuesNameTag(t *testing.T) {\n\ttimeNow = func() time.Time {\n\t\treturn time.Unix(1669714247, 0)\n\t}\n\n\tmetrics.DisableMetrics()\n\n\tsrv := chtest.NewTestServer()\n\tdefer srv.Close()\n\n\tcfg, _ := config.DefaultConfig()\n\tcfg.ClickHouse.URL = srv.URL\n\tcfg.ClickHouse.TaggedTable = \"graphite_tagged\"\n\n\th := NewValues(cfg)\n\n\tnow := timeNow()\n\tfromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)\n\n\t// Test with name tag (which should be converted to __name__)\n\tsrv.AddResponce(\n\t\t`SELECT substr(Tag1, 10) AS value FROM graphite_tagged WHERE `+\n\t\t\t`(Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=metric.%') AND (Date >= '`+fromDate+`' AND Date <= '`+untilDate+`') GROUP BY value ORDER BY value LIMIT 10000`,\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"metric.cpu.usage\\nmetric.memory.used\\nmetric.disk.io\\n\"),\n\t\t})\n\n\ttest := testStruct{\n\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/values?tag=name&valuePrefix=metric.\", nil),\n\t\twantCode: http.StatusOK,\n\t\twant: \"[\\\"metric.cpu.usage\\\",\\\"metric.memory.used\\\",\\\"metric.disk.io\\\"]\",\n\t}\n\n\ttestResponce(t, 0, h, &test, \"\")\n}\n\nfunc TestHandler_ServeValuesWithCache(t *testing.T) {\n\ttimeNow = func() time.Time {\n\t\treturn time.Unix(1669714247, 0)\n\t}\n\n\tmetrics.DisableMetrics()\n\n\tsrv := chtest.NewTestServer()\n\tdefer srv.Close()\n\n\tcfg, _ := config.DefaultConfig()\n\tcfg.ClickHouse.URL = srv.URL\n\tcfg.ClickHouse.TaggedTable = \"graphite_tagged\"\n\n\t// Enable cache\n\tcfg.Common.FindCacheConfig = config.CacheConfig{\n\t\tType: \"mem\",\n\t\tSize: 8192,\n\t\tFindTimeoutSec: 1,\n\t}\n\n\tvar err error\n\n\tcfg.Common.FindCache, err = config.CreateCache(\"autocomplete\", &cfg.Common.FindCacheConfig)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create find cache: %v\", err)\n\t}\n\n\th := NewValues(cfg)\n\n\tnow := timeNow()\n\tfromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)\n\n\tsrv.AddResponce(\n\t\t\"SELECT substr(Tag1, 6) AS value FROM graphite_tagged WHERE \"+\n\t\t\t\"(Tag1 LIKE 'host=%') AND (Date >= '\"+fromDate+\"' AND Date <= '\"+untilDate+\"') GROUP BY value ORDER BY value LIMIT 10000\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"host1\\nhost2\\nhost3\\n\"),\n\t\t})\n\n\ttest := testStruct{\n\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/values?tag=host\", nil),\n\t\twantCode: http.StatusOK,\n\t\twant: \"[\\\"host1\\\",\\\"host2\\\",\\\"host3\\\"]\",\n\t}\n\n\t// First request - should hit the database\n\ttestResponce(t, 0, h, &test, \"\")\n\tassert.Equal(t, uint64(1), srv.Queries())\n\n\t// Second request - should hit the cache\n\ttestResponce(t, 1, h, &test, \"1\")\n\tassert.Equal(t, uint64(1), srv.Queries()) // No new queries\n\n\t// Wait for cache expiration\n\ttime.Sleep(time.Second * 2)\n\n\t// Third request - should hit the database again\n\ttestResponce(t, 2, h, &test, \"\")\n\tassert.Equal(t, uint64(2), srv.Queries())\n}\n\nfunc TestHandler_ServeValuesWithCacheAndExpr(t *testing.T) {\n\ttimeNow = func() time.Time {\n\t\treturn time.Unix(1669714247, 0)\n\t}\n\n\tmetrics.DisableMetrics()\n\n\tsrv := chtest.NewTestServer()\n\tdefer srv.Close()\n\n\tcfg, _ := config.DefaultConfig()\n\tcfg.ClickHouse.URL = srv.URL\n\tcfg.ClickHouse.TaggedTable = \"graphite_tagged\"\n\n\t// Enable cache\n\tcfg.Common.FindCacheConfig = config.CacheConfig{\n\t\tType: \"mem\",\n\t\tSize: 8192,\n\t\tFindTimeoutSec: 1,\n\t}\n\n\tvar err error\n\n\tcfg.Common.FindCache, err = config.CreateCache(\"autocomplete\", &cfg.Common.FindCacheConfig)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create find cache: %v\", err)\n\t}\n\n\th := NewValues(cfg)\n\n\tnow := timeNow()\n\tfromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)\n\n\tsrv.AddResponce(\n\t\t\"SELECT substr(arrayFilter(x -> x LIKE 'host=%', Tags)[1], 6) AS value FROM graphite_tagged WHERE \"+\n\t\t\t\"((Tag1='environment=production') AND (arrayExists(x -> x LIKE 'host=%', Tags))) AND (Date >= '\"+fromDate+\"' AND Date <= '\"+untilDate+\"') GROUP BY value ORDER BY value LIMIT 10000\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"prod-host1\\nprod-host2\\n\"),\n\t\t})\n\n\ttest := testStruct{\n\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/values?tag=host&expr=environment%3Dproduction\", nil),\n\t\twantCode: http.StatusOK,\n\t\twant: \"[\\\"prod-host1\\\",\\\"prod-host2\\\"]\",\n\t}\n\n\t// First request - should hit the database\n\ttestResponce(t, 0, h, &test, \"\")\n\tassert.Equal(t, uint64(1), srv.Queries())\n\n\t// Second request - should hit the cache\n\ttestResponce(t, 1, h, &test, \"1\")\n\tassert.Equal(t, uint64(1), srv.Queries()) // No new queries\n\n\t// Test with different valuePrefix - should not hit cache\n\ttest2 := testStruct{\n\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/values?tag=host&expr=environment%3Dproduction&valuePrefix=prod-host1\", nil),\n\t\twantCode: http.StatusOK,\n\t\twant: \"[\\\"prod-host1\\\"]\",\n\t}\n\n\tsrv.AddResponce(\n\t\t\"SELECT substr(arrayFilter(x -> x LIKE 'host=prod-host1%', Tags)[1], 6) AS value FROM graphite_tagged WHERE \"+\n\t\t\t\"((Tag1='environment=production') AND (arrayExists(x -> x LIKE 'host=prod-host1%', Tags))) AND (Date >= '\"+fromDate+\"' AND Date <= '\"+untilDate+\"') GROUP BY value ORDER BY value LIMIT 10000\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"prod-host1\\n\"),\n\t\t})\n\n\t// Should hit the database because valuePrefix is different\n\ttestResponce(t, 2, h, &test2, \"\")\n\tassert.Equal(t, uint64(2), srv.Queries())\n}\n\nfunc TestHandler_ServeValuesWithInvalidLimit(t *testing.T) {\n\ttimeNow = func() time.Time {\n\t\treturn time.Unix(1669714247, 0)\n\t}\n\n\tmetrics.DisableMetrics()\n\n\tsrv := chtest.NewTestServer()\n\tdefer srv.Close()\n\n\tcfg, _ := config.DefaultConfig()\n\tcfg.ClickHouse.URL = srv.URL\n\tcfg.ClickHouse.TaggedTable = \"graphite_tagged\"\n\n\th := NewValues(cfg)\n\n\ttest := testStruct{\n\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/values?tag=host&limit=invalid\", nil),\n\t\twantCode: http.StatusBadRequest,\n\t\twant: \"\", // Error response\n\t}\n\n\ttestResponce(t, 0, h, &test, \"\")\n}\n\nfunc TestHandler_ServeValuesWithMultipleExpr(t *testing.T) {\n\ttimeNow = func() time.Time {\n\t\treturn time.Unix(1669714247, 0)\n\t}\n\n\tmetrics.DisableMetrics()\n\n\tsrv := chtest.NewTestServer()\n\tdefer srv.Close()\n\n\tcfg, _ := config.DefaultConfig()\n\tcfg.ClickHouse.URL = srv.URL\n\tcfg.ClickHouse.TaggedTable = \"graphite_tagged\"\n\n\th := NewValues(cfg)\n\n\tnow := timeNow()\n\tfromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)\n\n\t// Test with multiple expressions and valuePrefix\n\tsrv.AddResponce(\n\t\t\"SELECT substr(arrayFilter(x -> x LIKE 'host=dc-%', Tags)[1], 6) AS value FROM graphite_tagged WHERE \"+\n\t\t\t\"(((Tag1='environment=production') AND (has(Tags, 'project=web'))) AND (arrayExists(x -> x LIKE 'host=dc-%', Tags))) AND \"+\n\t\t\t\"(Date >= '\"+fromDate+\"' AND Date <= '\"+untilDate+\"') GROUP BY value ORDER BY value LIMIT 10000\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"dc-host1\\ndc-host2\\n\"),\n\t\t})\n\n\ttest := testStruct{\n\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/values?tag=host&expr=environment%3Dproduction&expr=project%3Dweb&valuePrefix=dc-\", nil),\n\t\twantCode: http.StatusOK,\n\t\twant: \"[\\\"dc-host1\\\",\\\"dc-host2\\\"]\",\n\t}\n\n\ttestResponce(t, 0, h, &test, \"\")\n}\n\nfunc TestHandler_ServeValuesNoCache(t *testing.T) {\n\ttimeNow = func() time.Time {\n\t\treturn time.Unix(1669714247, 0)\n\t}\n\n\tmetrics.DisableMetrics()\n\n\tsrv := chtest.NewTestServer()\n\tdefer srv.Close()\n\n\tcfg, _ := config.DefaultConfig()\n\tcfg.ClickHouse.URL = srv.URL\n\tcfg.ClickHouse.TaggedTable = \"graphite_tagged\"\n\n\t// Enable cache\n\tcfg.Common.FindCacheConfig = config.CacheConfig{\n\t\tType: \"mem\",\n\t\tSize: 8192,\n\t\tFindTimeoutSec: 60,\n\t}\n\n\tvar err error\n\n\tcfg.Common.FindCache, err = config.CreateCache(\"autocomplete\", &cfg.Common.FindCacheConfig)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create find cache: %v\", err)\n\t}\n\n\th := NewValues(cfg)\n\n\tnow := timeNow()\n\tfromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)\n\n\tsrv.AddResponce(\n\t\t\"SELECT substr(Tag1, 6) AS value FROM graphite_tagged WHERE \"+\n\t\t\t\"(Tag1 LIKE 'host=%') AND (Date >= '\"+fromDate+\"' AND Date <= '\"+untilDate+\"') GROUP BY value ORDER BY value LIMIT 10000\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"host1\\nhost2\\n\"),\n\t\t})\n\n\ttest := testStruct{\n\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/values?tag=host&noCache=true\", nil),\n\t\twantCode: http.StatusOK,\n\t\twant: \"[\\\"host1\\\",\\\"host2\\\"]\",\n\t}\n\n\t// First request with noCache=true - should always hit the database\n\ttestResponce(t, 0, h, &test, \"\")\n\tassert.Equal(t, uint64(1), srv.Queries())\n\n\t// Second request with noCache=true - should hit the database again\n\ttestResponce(t, 1, h, &test, \"\")\n\tassert.Equal(t, uint64(2), srv.Queries()) // Should increase\n}\n\nfunc TestHandler_ServeValuesEmptyResult(t *testing.T) {\n\ttimeNow = func() time.Time {\n\t\treturn time.Unix(1669714247, 0)\n\t}\n\n\tmetrics.DisableMetrics()\n\n\tsrv := chtest.NewTestServer()\n\tdefer srv.Close()\n\n\tcfg, _ := config.DefaultConfig()\n\tcfg.ClickHouse.URL = srv.URL\n\tcfg.ClickHouse.TaggedTable = \"graphite_tagged\"\n\n\th := NewValues(cfg)\n\n\tnow := timeNow()\n\tfromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)\n\n\t// Test empty result\n\tsrv.AddResponce(\n\t\t\"SELECT substr(Tag1, 13) AS value FROM graphite_tagged WHERE \"+\n\t\t\t\"(Tag1 LIKE 'nonexistent=%') AND (Date >= '\"+fromDate+\"' AND Date <= '\"+untilDate+\"') GROUP BY value ORDER BY value LIMIT 10000\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"\"), // Empty response\n\t\t})\n\n\ttest := testStruct{\n\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/values?tag=nonexistent\", nil),\n\t\twantCode: http.StatusOK,\n\t\twant: \"null\", // Empty array\n\t}\n\n\ttestResponce(t, 0, h, &test, \"\")\n}\n\nfunc TestHandler_ServeTagsWithCostOptimization(t *testing.T) {\n\ttimeNow = func() time.Time {\n\t\treturn time.Unix(1669714247, 0)\n\t}\n\n\tmetrics.DisableMetrics()\n\n\tsrv := chtest.NewTestServer()\n\tdefer srv.Close()\n\n\tcfg, _ := config.DefaultConfig()\n\tcfg.ClickHouse.URL = srv.URL\n\tcfg.ClickHouse.TaggedTable = \"graphite_tagged\"\n\tcfg.ClickHouse.TagsCountTable = \"tag1_count_per_day\"\n\n\th := NewTags(cfg)\n\n\tnow := timeNow()\n\tfromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)\n\tfrom := now.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Unix()\n\tuntil := now.Unix()\n\n\t// Test case: Tags query with multiple expressions should use cost optimization\n\t// First response: tags count query to get costs\n\tsrv.AddResponce(\n\t\t\"SELECT Tag1, sum(Count) as cnt FROM tag1_count_per_day WHERE \"+\n\t\t\t\"((Tag1='environment=production') OR (Tag1='project=web')) AND \"+\n\t\t\t\"(Date >= '\"+date.FromTimestampToDaysFormat(from)+\"' AND Date <= '\"+date.UntilTimestampToDaysFormat(until)+\"') GROUP BY Tag1 FORMAT TabSeparatedRaw\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"environment=production\\t10000\\nproject=web\\t500\\n\"),\n\t\t})\n\n\t// Second response: main tags query (should be ordered based on costs)\n\tsrv.AddResponce(\n\t\t\"SELECT splitByChar('=', arrayJoin(Tags))[1] AS value FROM graphite_tagged WHERE \"+\n\t\t\t\"((Tag1='project=web') AND (has(Tags, 'environment=production'))) AND \"+ // Lower cost term first\n\t\t\t\"(Date >= '\"+fromDate+\"' AND Date <= '\"+untilDate+\"') GROUP BY value ORDER BY value LIMIT 10002\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"__name__\\nhost\\nregion\\n\"),\n\t\t})\n\n\ttest := testStruct{\n\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/tags?expr=environment%3Dproduction&expr=project%3Dweb\", nil),\n\t\twantCode: http.StatusOK,\n\t\twant: `[\"host\",\"name\",\"region\"]`,\n\t}\n\n\ttestResponce(t, 0, h, &test, \"\")\n\tassert.Equal(t, uint64(2), srv.Queries()) // Should have 2 queries: cost query + main query\n}\n\nfunc TestHandler_ServeValuesWithCostOptimization(t *testing.T) {\n\ttimeNow = func() time.Time {\n\t\treturn time.Unix(1669714247, 0)\n\t}\n\n\tmetrics.DisableMetrics()\n\n\tsrv := chtest.NewTestServer()\n\tdefer srv.Close()\n\n\tcfg, _ := config.DefaultConfig()\n\tcfg.ClickHouse.URL = srv.URL\n\tcfg.ClickHouse.TaggedTable = \"graphite_tagged\"\n\tcfg.ClickHouse.TagsCountTable = \"tag1_count_per_day\"\n\n\th := NewValues(cfg)\n\n\tnow := timeNow()\n\tfromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)\n\tfrom := now.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Unix()\n\tuntil := now.Unix()\n\n\t// First response: tags count query for cost optimization\n\tsrv.AddResponce(\n\t\t\"SELECT Tag1, sum(Count) as cnt FROM tag1_count_per_day WHERE \"+\n\t\t\t\"((Tag1='environment=production') OR (Tag1='datacenter=us-east')) AND \"+\n\t\t\t\"(Date >= '\"+date.FromTimestampToDaysFormat(from)+\"' AND Date <= '\"+date.UntilTimestampToDaysFormat(until)+\"') GROUP BY Tag1 FORMAT TabSeparatedRaw\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"environment=production\\t5000\\ndatacenter=us-east\\t100\\n\"),\n\t\t})\n\n\t// Second response: values query (should use optimized order)\n\tsrv.AddResponce(\n\t\t\"SELECT substr(arrayFilter(x -> x LIKE 'host=%', Tags)[1], 6) AS value FROM graphite_tagged WHERE \"+\n\t\t\t\"(((Tag1='datacenter=us-east') AND (has(Tags, 'environment=production'))) AND \"+ // Lower cost first\n\t\t\t\"(arrayExists(x -> x LIKE 'host=%', Tags))) AND \"+\n\t\t\t\"(Date >= '\"+fromDate+\"' AND Date <= '\"+untilDate+\"') GROUP BY value ORDER BY value LIMIT 10000\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"host1\\nhost2\\nhost3\\n\"),\n\t\t})\n\n\ttest := testStruct{\n\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/values?tag=host&expr=environment%3Dproduction&expr=datacenter%3Dus-east\", nil),\n\t\twantCode: http.StatusOK,\n\t\twant: \"[\\\"host1\\\",\\\"host2\\\",\\\"host3\\\"]\",\n\t}\n\n\ttestResponce(t, 0, h, &test, \"\")\n\tassert.Equal(t, uint64(2), srv.Queries())\n}\n\nfunc TestHandler_ServeTagsWithWildcardExpressions(t *testing.T) {\n\ttimeNow = func() time.Time {\n\t\treturn time.Unix(1669714247, 0)\n\t}\n\n\tmetrics.DisableMetrics()\n\n\tsrv := chtest.NewTestServer()\n\tdefer srv.Close()\n\n\tcfg, _ := config.DefaultConfig()\n\tcfg.ClickHouse.URL = srv.URL\n\tcfg.ClickHouse.TaggedTable = \"graphite_tagged\"\n\tcfg.ClickHouse.TagsCountTable = \"tag1_count_per_day\"\n\n\th := NewTags(cfg)\n\n\tnow := timeNow()\n\tfromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)\n\n\t// Test with wildcard expressions (should not query tags count table)\n\tsrv.AddResponce(\n\t\t\"SELECT splitByChar('=', arrayJoin(Tags))[1] AS value FROM graphite_tagged WHERE \"+\n\t\t\t\"(Tag1 LIKE 'environment=prod%') AND (Date >= '\"+fromDate+\"' AND Date <= '\"+untilDate+\"') GROUP BY value ORDER BY value LIMIT 10001\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"__name__\\nhost\\nproject\\n\"),\n\t\t})\n\n\ttest := testStruct{\n\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/tags?expr=environment%3Dprod*\", nil),\n\t\twantCode: http.StatusOK,\n\t\twant: `[\"host\",\"name\",\"project\"]`,\n\t}\n\n\ttestResponce(t, 0, h, &test, \"\")\n\tassert.Equal(t, uint64(1), srv.Queries()) // Only 1 query since wildcards skip cost optimization\n}\n\nfunc TestHandler_ServeValuesWithNoEqualityTerms(t *testing.T) {\n\ttimeNow = func() time.Time {\n\t\treturn time.Unix(1669714247, 0)\n\t}\n\n\tmetrics.DisableMetrics()\n\n\tsrv := chtest.NewTestServer()\n\tdefer srv.Close()\n\n\tcfg, _ := config.DefaultConfig()\n\tcfg.ClickHouse.URL = srv.URL\n\tcfg.ClickHouse.TaggedTable = \"graphite_tagged\"\n\tcfg.ClickHouse.TagsCountTable = \"tag1_count_per_day\"\n\n\th := NewValues(cfg)\n\n\tnow := timeNow()\n\tfromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)\n\n\t// Test with != operator (should not use tags count table)\n\tsrv.AddResponce(\n\t\t\"SELECT substr(arrayFilter(x -> x LIKE 'host=%', Tags)[1], 6) AS value FROM graphite_tagged WHERE \"+\n\t\t\t\"((NOT arrayExists((x) -> x='environment=development', Tags)) AND (arrayExists(x -> x LIKE 'host=%', Tags))) AND \"+\n\t\t\t\"(Date >= '\"+fromDate+\"' AND Date <= '\"+untilDate+\"') GROUP BY value ORDER BY value LIMIT 10000\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"host1\\nhost2\\n\"),\n\t\t})\n\n\ttest := testStruct{\n\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/values?tag=host&expr=environment%21%3Ddevelopment\", nil),\n\t\twantCode: http.StatusOK,\n\t\twant: \"[\\\"host1\\\",\\\"host2\\\"]\",\n\t}\n\n\ttestResponce(t, 0, h, &test, \"\")\n\tassert.Equal(t, uint64(1), srv.Queries()) // Only 1 query since no equality terms\n}\n\nfunc TestHandler_ServeTagsWithHighCostTags(t *testing.T) {\n\ttimeNow = func() time.Time {\n\t\treturn time.Unix(1669714247, 0)\n\t}\n\n\tmetrics.DisableMetrics()\n\n\tsrv := chtest.NewTestServer()\n\tdefer srv.Close()\n\n\tcfg, _ := config.DefaultConfig()\n\tcfg.ClickHouse.URL = srv.URL\n\tcfg.ClickHouse.TaggedTable = \"graphite_tagged\"\n\tcfg.ClickHouse.TagsCountTable = \"tag1_count_per_day\"\n\n\th := NewTags(cfg)\n\n\tnow := timeNow()\n\tfromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)\n\tfrom := now.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Unix()\n\tuntil := now.Unix()\n\n\t// Test with high cardinality tags - tags should be reordered based on cost\n\tsrv.AddResponce(\n\t\t\"SELECT Tag1, sum(Count) as cnt FROM tag1_count_per_day WHERE \"+\n\t\t\t\"(((Tag1='__name__=high.cost.metric') OR (Tag1='environment=production')) OR (Tag1='dc=west')) AND \"+\n\t\t\t\"(Date >= '\"+date.FromTimestampToDaysFormat(from)+\"' AND Date <= '\"+date.UntilTimestampToDaysFormat(until)+\"') GROUP BY Tag1 FORMAT TabSeparatedRaw\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"__name__=high.cost.metric\\t1000000\\nenvironment=production\\t10000\\ndc=west\\t50\\n\"),\n\t\t})\n\n\t// Query should use lowest cost tag first (dc=west)\n\tsrv.AddResponce(\n\t\t\"SELECT splitByChar('=', arrayJoin(Tags))[1] AS value FROM graphite_tagged WHERE \"+\n\t\t\t\"(((Tag1='dc=west') AND (has(Tags, 'environment=production'))) AND (has(Tags, '__name__=high.cost.metric'))) AND \"+\n\t\t\t\"(Date >= '\"+fromDate+\"' AND Date <= '\"+untilDate+\"') GROUP BY value ORDER BY value LIMIT 10003\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"host\\nproject\\nregion\\n\"),\n\t\t})\n\n\ttest := testStruct{\n\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/tags?expr=name%3Dhigh.cost.metric&expr=environment%3Dproduction&expr=dc%3Dwest\", nil),\n\t\twantCode: http.StatusOK,\n\t\twant: `[\"host\",\"project\",\"region\"]`,\n\t}\n\n\ttestResponce(t, 0, h, &test, \"\")\n\tassert.Equal(t, uint64(2), srv.Queries())\n}\n\nfunc TestHandler_ServeValuesWithMixedOperators(t *testing.T) {\n\ttimeNow = func() time.Time {\n\t\treturn time.Unix(1669714247, 0)\n\t}\n\n\tmetrics.DisableMetrics()\n\n\tsrv := chtest.NewTestServer()\n\tdefer srv.Close()\n\n\tcfg, _ := config.DefaultConfig()\n\tcfg.ClickHouse.URL = srv.URL\n\tcfg.ClickHouse.TaggedTable = \"graphite_tagged\"\n\tcfg.ClickHouse.TagsCountTable = \"tag1_count_per_day\"\n\n\th := NewValues(cfg)\n\n\tnow := timeNow()\n\tfromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)\n\tfrom := now.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Unix()\n\tuntil := now.Unix()\n\n\t// Test with mixed operators (only equality operators should be in cost query)\n\tsrv.AddResponce(\n\t\t\"SELECT Tag1, sum(Count) as cnt FROM tag1_count_per_day WHERE \"+\n\t\t\t\"((Tag1='environment=production') OR (Tag1='project=api')) AND \"+\n\t\t\t\"(Date >= '\"+date.FromTimestampToDaysFormat(from)+\"' AND Date <= '\"+date.UntilTimestampToDaysFormat(until)+\"') GROUP BY Tag1 FORMAT TabSeparatedRaw\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"environment=production\\t8000\\nproject=api\\t200\\n\"),\n\t\t})\n\n\t// Main query should include != operator but order by cost\n\tsrv.AddResponce(\n\t\t\"SELECT substr(arrayFilter(x -> x LIKE 'host=%', Tags)[1], 6) AS value FROM graphite_tagged WHERE \"+\n\t\t\t\"((((Tag1='project=api') AND (has(Tags, 'environment=production'))) AND (NOT arrayExists((x) -> x='dc=east', Tags))) AND \"+\n\t\t\t\"(arrayExists(x -> x LIKE 'host=%', Tags))) AND \"+\n\t\t\t\"(Date >= '\"+fromDate+\"' AND Date <= '\"+untilDate+\"') GROUP BY value ORDER BY value LIMIT 10000\",\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"host1\\nhost2\\n\"),\n\t\t})\n\n\ttest := testStruct{\n\t\trequest: NewRequest(\"GET\", srv.URL+\"/tags/autoComplete/values?tag=host&expr=environment%3Dproduction&expr=project%3Dapi&expr=dc%21%3Deast\", nil),\n\t\twantCode: http.StatusOK,\n\t\twant: \"[\\\"host1\\\",\\\"host2\\\"]\",\n\t}\n\n\ttestResponce(t, 0, h, &test, \"\")\n\tassert.Equal(t, uint64(2), srv.Queries()) // Cost query only for equality terms\n}\n" }, { "path": "cache/cache.go", "content": "package cache\n\nimport (\n\t\"crypto/sha256\"\n\t\"encoding/hex\"\n\t\"errors\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/bradfitz/gomemcache/memcache\"\n\n\t\"github.com/msaf1980/go-expirecache\"\n)\n\nvar (\n\tErrTimeout = errors.New(\"cache: timeout\")\n\tErrNotFound = errors.New(\"cache: not found\")\n)\n\ntype BytesCache interface {\n\tGet(k string) ([]byte, error)\n\tSet(k string, v []byte, expire int32)\n}\n\nfunc NewExpireCache(maxsize uint64) BytesCache {\n\tec := expirecache.New[string, []byte](maxsize)\n\tgo ec.ApproximateCleaner(10 * time.Second)\n\n\treturn &ExpireCache{ec: ec}\n}\n\ntype ExpireCache struct {\n\tec *expirecache.Cache[string, []byte]\n}\n\nfunc (ec ExpireCache) Get(k string) ([]byte, error) {\n\tv, ok := ec.ec.Get(k)\n\n\tif !ok {\n\t\treturn nil, ErrNotFound\n\t}\n\n\treturn v, nil\n}\n\nfunc (ec ExpireCache) Set(k string, v []byte, expire int32) {\n\tec.ec.Set(k, v, uint64(len(v)), expire)\n}\n\nfunc NewMemcached(prefix string, servers ...string) BytesCache {\n\treturn &MemcachedCache{prefix: prefix, client: memcache.New(servers...)}\n}\n\ntype MemcachedCache struct {\n\tprefix string\n\tclient *memcache.Client\n\ttimeouts uint64\n}\n\nfunc (m *MemcachedCache) Get(k string) ([]byte, error) {\n\tkey := sha256.Sum256([]byte(k))\n\thk := hex.EncodeToString(key[:])\n\tdone := make(chan bool, 1)\n\n\tvar err error\n\n\tvar item *memcache.Item\n\n\tgo func() {\n\t\titem, err = m.client.Get(m.prefix + hk)\n\t\tdone <- true\n\t}()\n\n\ttimeout := time.After(50 * time.Millisecond)\n\n\tselect {\n\tcase <-timeout:\n\t\tatomic.AddUint64(&m.timeouts, 1)\n\t\treturn nil, ErrTimeout\n\tcase <-done:\n\t}\n\n\tif err != nil {\n\t\t// translate to internal cache miss error\n\t\tif errors.Is(err, memcache.ErrCacheMiss) {\n\t\t\terr = ErrNotFound\n\t\t}\n\n\t\treturn nil, err\n\t}\n\n\treturn item.Value, nil\n}\n\nfunc (m *MemcachedCache) Set(k string, v []byte, expire int32) {\n\tkey := sha256.Sum256([]byte(k))\n\thk := hex.EncodeToString(key[:])\n\n\tgo func() {\n\t\t_ = m.client.Set(&memcache.Item{Key: m.prefix + hk, Value: v, Expiration: expire})\n\t}()\n}\n\nfunc (m *MemcachedCache) Timeouts() uint64 {\n\treturn atomic.LoadUint64(&m.timeouts)\n}\n" }, { "path": "capabilities/handler.go", "content": "package capabilities\n\nimport (\n\t\"encoding/json\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\t\"time\"\n\n\tv3pb \"github.com/go-graphite/protocol/carbonapi_v3_pb\"\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/logs\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n)\n\ntype Handler struct {\n\tconfig *config.Config\n}\n\nfunc NewHandler(config *config.Config) *Handler {\n\treturn &Handler{\n\t\tconfig: config,\n\t}\n}\n\nfunc (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {\n\taccessLogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named(\"http\")\n\tlogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named(\"capabilities\")\n\n\tr = r.WithContext(scope.WithLogger(r.Context(), logger))\n\n\tr.ParseMultipartForm(1024 * 1024)\n\n\tformat := r.FormValue(\"format\")\n\n\taccepts := r.Header[\"Accept\"]\n\tfor _, accept := range accepts {\n\t\tif accept == \"application/x-carbonapi-v3-pb\" {\n\t\t\tformat = \"carbonapi_v3_pb\"\n\t\t\tbreak\n\t\t}\n\t}\n\n\tstatus := http.StatusOK\n\tstart := time.Now()\n\n\tdefer func() {\n\t\td := time.Since(start)\n\t\tlogs.AccessLog(accessLogger, h.config, r, status, d, time.Duration(0), false, false)\n\t}()\n\n\tif format == \"carbonapi_v3_pb\" || format == \"json\" {\n\t\tbody, err := io.ReadAll(r.Body)\n\t\tif err != nil {\n\t\t\tstatus = http.StatusBadRequest\n\t\t\thttp.Error(w, \"Bad request (malformed body)\", status)\n\t\t}\n\n\t\tvar pv3Request v3pb.CapabilityRequest\n\n\t\terr = pv3Request.Unmarshal(body)\n\t\tif err != nil {\n\t\t\tstatus = http.StatusBadRequest\n\t\t\thttp.Error(w, \"Bad request (malformed body)\", status)\n\t\t}\n\n\t\thostname, err := os.Hostname()\n\t\tif err != nil {\n\t\t\thostname = \"(unknown)\"\n\t\t}\n\n\t\tpvResponse := v3pb.CapabilityResponse{\n\t\t\tSupportedProtocols: []string{\"carbonapi_v3_pb\", \"carbonapi_v2_pb\", \"graphite-web-pickle\"},\n\t\t\tName: hostname,\n\t\t\tHighPrecisionTimestamps: false,\n\t\t\tSupportFilteringFunctions: false,\n\t\t\tLikeSplittedRequests: false,\n\t\t\tSupportStreaming: false,\n\t\t}\n\n\t\tvar data []byte\n\n\t\tcontentType := \"\"\n\n\t\tswitch format {\n\t\tcase \"json\":\n\t\t\tcontentType = \"application/json\"\n\n\t\t\tdata, err = json.Marshal(pvResponse)\n\t\t\tif err != nil {\n\t\t\t\tstatus = http.StatusInternalServerError\n\t\t\t\thttp.Error(w, err.Error(), status)\n\n\t\t\t\treturn\n\t\t\t}\n\t\tcase \"carbonapi_v3_pb\":\n\t\t\tcontentType = \"application/x-carbonapi-v3-pb\"\n\n\t\t\tdata, err = pvResponse.Marshal()\n\t\t\tif err != nil {\n\t\t\t\tstatus = http.StatusBadRequest\n\t\t\t\thttp.Error(w, \"Bad request (unsupported format)\", status)\n\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\n\t\tw.Header().Set(\"Content-Type\", contentType)\n\t\tw.Write(data)\n\t} else {\n\t\tstatus = http.StatusBadRequest\n\t\thttp.Error(w, \"Bad request (unsupported format)\", status)\n\t}\n}\n" }, { "path": "cmd/e2e-test/carbon-clickhouse.go", "content": "package main\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"text/template\"\n)\n\nvar CchContainerName = \"carbon-clickhouse-gch-test\"\n\ntype CarbonClickhouse struct {\n\tVersion string `toml:\"version\"`\n\n\tDockerImage string `toml:\"image\"`\n\n\tTemplate string `toml:\"template\"` // carbon-clickhouse config template\n\n\tTZ string `toml:\"tz\"` // override timezone\n\n\taddress string `toml:\"-\"`\n\tcontainer string `toml:\"-\"`\n\tstoreDir string `toml:\"-\"`\n}\n\nfunc (c *CarbonClickhouse) Start(testDir, clickhouseURL string) (string, error) {\n\tif len(c.Version) == 0 {\n\t\tc.Version = \"latest\"\n\t}\n\n\tif len(c.DockerImage) == 0 {\n\t\tc.DockerImage = \"ghcr.io/go-graphite/carbon-clickhouse\"\n\t}\n\n\tvar err error\n\n\tc.address, err = getFreeTCPPort(\"\")\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tc.container = CchContainerName\n\n\tc.storeDir, err = os.MkdirTemp(\"\", \"carbon-clickhouse\")\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tc.address, err = getFreeTCPPort(\"\")\n\tif err != nil {\n\t\tc.Cleanup()\n\t\treturn \"\", err\n\t}\n\n\tname := filepath.Base(c.Template)\n\ttpl := path.Join(testDir, c.Template)\n\n\ttmpl, err := template.New(name).ParseFiles(tpl)\n\tif err != nil {\n\t\tc.Cleanup()\n\t\treturn \"\", err\n\t}\n\n\tparam := struct {\n\t\tCLICKHOUSE_URL string\n\t\tCCH_ADDR string\n\t}{\n\t\tCLICKHOUSE_URL: clickhouseURL,\n\t\tCCH_ADDR: c.address,\n\t}\n\n\tconfigFile := path.Join(c.storeDir, \"carbon-clickhouse.conf\")\n\n\tf, err := os.OpenFile(configFile, os.O_WRONLY|os.O_CREATE, 0644)\n\tif err != nil {\n\t\tc.Cleanup()\n\t\treturn \"\", err\n\t}\n\n\terr = tmpl.ExecuteTemplate(f, name, param)\n\tif err != nil {\n\t\tc.Cleanup()\n\t\treturn \"\", err\n\t}\n\n\ttz := os.Getenv(\"TZ\")\n\n\tcchStart := []string{\"run\", \"-d\",\n\t\t\"--name\", c.container,\n\t\t\"-p\", c.address + \":2003\",\n\t\t\"-v\", c.storeDir + \":/etc/carbon-clickhouse\",\n\t\t// TZ, need to be same as graphite-clickhouse for prevent bugs, ike issue #184\n\t\t\"-v\", \"/etc/timezone:/etc/timezone:ro\",\n\t\t\"-v\", \"/etc/localtime:/etc/localtime:ro\",\n\t\t\"-e\", \"TZ=\" + tz,\n\t\t\"--network\", DockerNetwork,\n\t}\n\n\tcchStart = append(cchStart, c.DockerImage+\":\"+c.Version)\n\n\tcmd := exec.Command(DockerBinary, cchStart...)\n\n\tout, err := cmd.CombinedOutput()\n\tif err == nil {\n\t\tdateLocal, _ := exec.Command(\"date\").Output()\n\t\tdateLocalStr := strings.TrimRight(string(dateLocal), \"\\n\")\n\t\t_, dateOnCCH := containerExec(c.container, []string{\"date\"})\n\t\tfmt.Printf(\"date local %s, on carbon-clickhouse %s\\n\", dateLocalStr, dateOnCCH)\n\t}\n\n\treturn string(out), err\n}\n\nfunc (c *CarbonClickhouse) Stop(delete bool) (string, error) {\n\tif len(c.container) == 0 {\n\t\treturn \"\", nil\n\t}\n\n\tchStop := []string{\"stop\", c.container}\n\n\tcmd := exec.Command(DockerBinary, chStop...)\n\tout, err := cmd.CombinedOutput()\n\n\tif err == nil && delete {\n\t\treturn c.Delete()\n\t}\n\n\treturn string(out), err\n}\n\nfunc (c *CarbonClickhouse) Delete() (string, error) {\n\tif len(c.container) == 0 {\n\t\treturn \"\", nil\n\t}\n\n\tchDel := []string{\"rm\", c.container}\n\n\tcmd := exec.Command(DockerBinary, chDel...)\n\tout, err := cmd.CombinedOutput()\n\n\tif err == nil {\n\t\tc.container = \"\"\n\t}\n\n\tc.Cleanup()\n\n\treturn string(out), err\n}\n\nfunc (c *CarbonClickhouse) Cleanup() {\n\tif c.storeDir != \"\" {\n\t\tos.RemoveAll(c.storeDir)\n\t\tc.storeDir = \"\"\n\t}\n}\n\nfunc (c *CarbonClickhouse) Address() string {\n\treturn c.address\n}\n\nfunc (c *CarbonClickhouse) Container() string {\n\treturn c.container\n}\n" }, { "path": "cmd/e2e-test/checks.go", "content": "package main\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"os\"\n\t\"reflect\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/go-graphite/protocol/carbonapi_v3_pb\"\n\t\"github.com/lomik/graphite-clickhouse/helper/client\"\n\t\"github.com/lomik/graphite-clickhouse/helper/datetime\"\n\t\"github.com/lomik/graphite-clickhouse/helper/tests/compare\"\n)\n\nfunc isFindCached(header http.Header) (string, bool) {\n\tif header == nil {\n\t\treturn \"\", false\n\t}\n\n\tv, exist := header[\"X-Cached-Find\"]\n\tif len(v) == 0 {\n\t\treturn \"\", false\n\t}\n\n\treturn v[0], exist\n}\n\nfunc requestId(header http.Header) string {\n\tif header == nil {\n\t\treturn \"\"\n\t}\n\n\tv, exist := header[\"X-Gch-Request-Id\"]\n\tif exist && len(v) > 0 {\n\t\treturn v[0]\n\t}\n\n\treturn \"\"\n}\n\nfunc compareFindMatch(errors *[]string, name, url string, actual, expected []client.FindMatch, findCached bool, cacheTTL int, header http.Header) {\n\tvar cacheTTLStr string\n\tif findCached {\n\t\tcacheTTLStr = strconv.Itoa(cacheTTL)\n\t}\n\n\tid := requestId(header)\n\n\tif header != nil {\n\t\tv, actualFindCached := isFindCached(header)\n\t\tif actualFindCached != findCached || cacheTTLStr != v {\n\t\t\t*errors = append(*errors, fmt.Sprintf(\"TRY[%s] %s %s: X-Cached-Find want '%s', got '%s'\", name, id, url, cacheTTLStr, v))\n\t\t}\n\t}\n\n\tmaxLen := compare.Max(len(expected), len(actual))\n\tfor i := 0; i < maxLen; i++ {\n\t\tif i > len(actual)-1 {\n\t\t\t*errors = append(*errors, fmt.Sprintf(\"- TRY[%s] %s %s [%d] = %+v\", name, id, url, i, expected[i]))\n\t\t} else if i > len(expected)-1 {\n\t\t\t*errors = append(*errors, fmt.Sprintf(\"+ TRY[%s] %s %s [%d] = %+v\", name, id, url, i, actual[i]))\n\t\t} else if expected[i] != actual[i] {\n\t\t\t*errors = append(*errors, fmt.Sprintf(\"- TRY[%s] %s %s [%d] = %+v\", name, id, url, i, expected[i]))\n\t\t\t*errors = append(*errors, fmt.Sprintf(\"+ TRY[%s] %s %s [%d] = %+v\", name, id, url, i, actual[i]))\n\t\t}\n\t}\n}\n\nfunc verifyMetricsFind(ch *Clickhouse, gch *GraphiteClickhouse, check *MetricsFindCheck) []string {\n\tvar errors []string\n\n\thttpClient := http.Client{\n\t\tTimeout: check.Timeout,\n\t}\n\taddress := gch.URL()\n\n\tfor _, format := range check.Formats {\n\t\tname := \"\"\n\n\t\tif url, result, respHeader, err := client.MetricsFind(&httpClient, address, format, check.Query, check.from, check.until); err == nil {\n\t\t\tid := requestId(respHeader)\n\t\t\tif check.ErrorRegexp != \"\" {\n\t\t\t\terrors = append(errors, fmt.Sprintf(\"TRY[%s] %s %s: want error with '%s'\", \"\", id, url, check.ErrorRegexp))\n\t\t\t}\n\n\t\t\tcompareFindMatch(&errors, name, url, result, check.Result, check.InCache, check.CacheTTL, respHeader)\n\n\t\t\tif len(result) == 0 && len(check.Result) > 0 {\n\t\t\t\tgch.Grep(id)\n\n\t\t\t\tif len(check.DumpIfEmpty) > 0 {\n\t\t\t\t\tfor _, q := range check.DumpIfEmpty {\n\t\t\t\t\t\tif out, err := ch.Query(q); err == nil {\n\t\t\t\t\t\t\tfmt.Fprintf(os.Stderr, \"%s\\n%s\", q, out)\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tfmt.Fprintf(os.Stderr, \"%s: %s\\n\", err.Error(), q)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif check.CacheTTL > 0 && check.ErrorRegexp == \"\" {\n\t\t\t\t// second query must be find-cached\n\t\t\t\tname = \"cache\"\n\t\t\t\tif url, result, respHeader, err = client.MetricsFind(&httpClient, address, format, check.Query, check.from, check.until); err == nil {\n\t\t\t\t\tcompareFindMatch(&errors, name, url, result, check.Result, true, check.CacheTTL, respHeader)\n\t\t\t\t} else {\n\t\t\t\t\terrStr := strings.TrimRight(err.Error(), \"\\n\")\n\t\t\t\t\terrors = append(errors, fmt.Sprintf(\"TRY[%s] %s %s: %s\", name, requestId(respHeader), url, errStr))\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\terrStr := strings.TrimRight(err.Error(), \"\\n\")\n\t\t\tif check.errorRegexp == nil || !check.errorRegexp.MatchString(errStr) {\n\t\t\t\terrors = append(errors, fmt.Sprintf(\"TRY[%s] %s %s: want error with '%s', got '%s'\", \"\", requestId(respHeader), url, check.ErrorRegexp, errStr))\n\t\t\t} else {\n\t\t\t\tfmt.Printf(\"EXPECTED ERROR, SUCCESS %s : %s\\n\", url, errStr)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn errors\n}\n\nfunc compareTags(errors *[]string, name, url string, actual, expected []string, findCached bool, cacheTTL int, header http.Header) {\n\tvar cacheTTLStr string\n\tif findCached {\n\t\tcacheTTLStr = strconv.Itoa(cacheTTL)\n\t}\n\n\tid := requestId(header)\n\n\tif header != nil {\n\t\tv, actualFindCached := isFindCached(header)\n\t\tif actualFindCached != findCached || cacheTTLStr != v {\n\t\t\t*errors = append(*errors, fmt.Sprintf(\"TRY[%s] %s %s: X-Cached-Find want '%s', got '%s'\", name, id, url, cacheTTLStr, v))\n\t\t}\n\t}\n\n\tmaxLen := compare.Max(len(expected), len(actual))\n\tfor i := 0; i < maxLen; i++ {\n\t\tif i > len(actual)-1 {\n\t\t\t*errors = append(*errors, fmt.Sprintf(\"- TRY[%s] %s %s [%d] = %+v\", name, id, url, i, expected[i]))\n\t\t} else if i > len(expected)-1 {\n\t\t\t*errors = append(*errors, fmt.Sprintf(\"+ TRY[%s] %s %s [%d] = %+v\", name, id, url, i, actual[i]))\n\t\t} else if expected[i] != actual[i] {\n\t\t\t*errors = append(*errors, fmt.Sprintf(\"- TRY[%s] %s %s [%d] = %+v\", name, id, url, i, expected[i]))\n\t\t\t*errors = append(*errors, fmt.Sprintf(\"+ TRY[%s] %s %s [%d] = %+v\", name, id, url, i, actual[i]))\n\t\t}\n\t}\n}\n\nfunc verifyTags(ch *Clickhouse, gch *GraphiteClickhouse, check *TagsCheck) []string {\n\tvar errors []string\n\n\thttpClient := http.Client{\n\t\tTimeout: check.Timeout,\n\t}\n\taddress := gch.URL()\n\n\tfor _, format := range check.Formats {\n\t\tvar (\n\t\t\tresult []string\n\t\t\terr error\n\t\t\turl string\n\t\t\trespHeader http.Header\n\t\t)\n\n\t\tname := \"\"\n\n\t\tif check.Names {\n\t\t\turl, result, respHeader, err = client.TagsNames(&httpClient, address, format, check.Query, check.Limits, check.from, check.until)\n\t\t} else {\n\t\t\turl, result, respHeader, err = client.TagsValues(&httpClient, address, format, check.Query, check.Limits, check.from, check.until)\n\t\t}\n\n\t\tif err == nil {\n\t\t\tid := requestId(respHeader)\n\t\t\tif check.ErrorRegexp != \"\" {\n\t\t\t\terrors = append(errors, fmt.Sprintf(\"TRY[%s] %s %s: want error with '%s'\", \"\", id, url, check.ErrorRegexp))\n\t\t\t}\n\n\t\t\tcompareTags(&errors, name, url, result, check.Result, check.InCache, check.CacheTTL, respHeader)\n\n\t\t\tif len(result) == 0 && len(check.Result) > 0 {\n\t\t\t\tgch.Grep(id)\n\n\t\t\t\tif len(check.DumpIfEmpty) > 0 {\n\t\t\t\t\tfor _, q := range check.DumpIfEmpty {\n\t\t\t\t\t\tif out, err := ch.Query(q); err == nil {\n\t\t\t\t\t\t\tfmt.Fprintf(os.Stderr, \"%s\\n%s\", q, out)\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tfmt.Fprintf(os.Stderr, \"%s: %s\\n\", err.Error(), q)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif check.CacheTTL > 0 && check.ErrorRegexp == \"\" {\n\t\t\t\t// second query must be find-cached\n\t\t\t\tname = \"cache\"\n\n\t\t\t\tif check.Names {\n\t\t\t\t\turl, result, respHeader, err = client.TagsNames(&httpClient, address, format, check.Query, check.Limits, check.from, check.until)\n\t\t\t\t} else {\n\t\t\t\t\turl, result, respHeader, err = client.TagsValues(&httpClient, address, format, check.Query, check.Limits, check.from, check.until)\n\t\t\t\t}\n\n\t\t\t\tif err == nil {\n\t\t\t\t\tcompareTags(&errors, name, url, result, check.Result, true, check.CacheTTL, respHeader)\n\t\t\t\t} else {\n\t\t\t\t\terrStr := strings.TrimRight(err.Error(), \"\\n\")\n\t\t\t\t\terrors = append(errors, fmt.Sprintf(\"TRY[%s] %s %s: %s\", name, requestId(respHeader), url, errStr))\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\terrStr := strings.TrimRight(err.Error(), \"\\n\")\n\t\t\tif check.errorRegexp == nil || !check.errorRegexp.MatchString(errStr) {\n\t\t\t\terrors = append(errors, fmt.Sprintf(\"TRY[%s] %s %s: want error with '%s', got '%s'\", \"\", requestId(respHeader), url, check.ErrorRegexp, errStr))\n\t\t\t} else {\n\t\t\t\tfmt.Printf(\"EXPECTED ERROR, SUCCESS %s : %s\\n\", url, errStr)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn errors\n}\n\nfunc compareRender(errors *[]string, name, url string, actual, expected []client.Metric, findCached bool, header http.Header, cacheTTL int) {\n\tvar cacheTTLStr string\n\tif findCached {\n\t\tcacheTTLStr = strconv.Itoa(cacheTTL)\n\t}\n\n\tsort.Slice(actual, func(i, j int) bool {\n\t\treturn actual[i].Name < actual[j].Name\n\t})\n\n\tid := requestId(header)\n\n\tif header != nil {\n\t\tv, actualFindCached := isFindCached(header)\n\t\tif actualFindCached != findCached || cacheTTLStr != v {\n\t\t\t*errors = append(*errors, fmt.Sprintf(\"TRY[%s] %s %s: X-Cached-Find want '%s', got '%s'\", name, id, url, cacheTTLStr, v))\n\t\t}\n\t}\n\n\tmaxLen := compare.Max(len(expected), len(actual))\n\tfor i := 0; i < maxLen; i++ {\n\t\tif i > len(actual)-1 {\n\t\t\t*errors = append(*errors, fmt.Sprintf(\"- TRY[%s] %s %s [%d] = %+v\", name, id, url, i, expected[i]))\n\t\t} else if i > len(expected)-1 {\n\t\t\t*errors = append(*errors, fmt.Sprintf(\"+ TRY[%s] %s %s [%d] = %+v\", name, id, url, i, actual[i]))\n\t\t} else if actual[i].Name != expected[i].Name {\n\t\t\t*errors = append(*errors, fmt.Sprintf(\"- TRY[%s] %s %s [%d] = %+v\", name, id, url, i, expected[i]))\n\t\t\t*errors = append(*errors, fmt.Sprintf(\"+ TRY[%s] %s %s [%d] = %+v\", name, id, url, i, actual[i]))\n\t\t} else {\n\t\t\tif actual[i].PathExpression != expected[i].PathExpression {\n\t\t\t\t*errors = append(*errors, fmt.Sprintf(\"TRY[%s] %s %s '%s': mismatch [%d].PathExpression, got '%s', want '%s'\", name, id, url, actual[i].Name, i, actual[i].PathExpression, expected[i].PathExpression))\n\t\t\t}\n\n\t\t\tif actual[i].ConsolidationFunc != expected[i].ConsolidationFunc {\n\t\t\t\t*errors = append(*errors, fmt.Sprintf(\"TRY[%s] %s %s '%s': mismatch [%d].ConsolidationFunc, got '%s', want '%s'\", name, id, url, actual[i].Name, i, actual[i].ConsolidationFunc, expected[i].ConsolidationFunc))\n\t\t\t}\n\n\t\t\tif actual[i].ConsolidationFunc != expected[i].ConsolidationFunc {\n\t\t\t\t*errors = append(*errors, fmt.Sprintf(\"TRY[%s] %s %s '%s': mismatch [%d].ConsolidationFunc, got '%s', want '%s'\", name, id, url, actual[i].Name, i, actual[i].ConsolidationFunc, expected[i].ConsolidationFunc))\n\t\t\t}\n\n\t\t\tif actual[i].StartTime != expected[i].StartTime {\n\t\t\t\t*errors = append(*errors, fmt.Sprintf(\"TRY[%s] %s %s '%s': mismatch [%d].StartTime, got %d, want %d\", name, id, url, actual[i].Name, i, actual[i].StartTime, expected[i].StartTime))\n\t\t\t}\n\n\t\t\tif actual[i].StopTime != expected[i].StopTime {\n\t\t\t\t*errors = append(*errors, fmt.Sprintf(\"TRY[%s] %s %s '%s': mismatch [%d].StopTime, got %d, want %d\", name, id, url, actual[i].Name, i, actual[i].StopTime, expected[i].StopTime))\n\t\t\t}\n\n\t\t\tif actual[i].StepTime != expected[i].StepTime {\n\t\t\t\t*errors = append(*errors, fmt.Sprintf(\"TRY[%s] %s %s '%s': mismatch [%d].StepTime, got %d, want %d\", name, id, url, actual[i].Name, i, actual[i].StepTime, expected[i].StepTime))\n\t\t\t}\n\n\t\t\tif actual[i].RequestStartTime != expected[i].RequestStartTime {\n\t\t\t\t*errors = append(*errors, fmt.Sprintf(\"TRY[%s] %s %s '%s': mismatch [%d].RequestStartTime, got %d, want %d\", name, id, url, actual[i].Name, i, actual[i].RequestStartTime, expected[i].RequestStartTime))\n\t\t\t}\n\n\t\t\tif actual[i].RequestStopTime != expected[i].RequestStopTime {\n\t\t\t\t*errors = append(*errors, fmt.Sprintf(\"TRY[%s] %s %s '%s': mismatch [%d].RequestStopTime, got %d, want %d\", name, id, url, actual[i].Name, i, actual[i].RequestStopTime, expected[i].RequestStopTime))\n\t\t\t}\n\n\t\t\tif actual[i].HighPrecisionTimestamps != expected[i].HighPrecisionTimestamps {\n\t\t\t\t*errors = append(*errors, fmt.Sprintf(\"TRY[%s] %s %s '%s': mismatch [%d].HighPrecisionTimestamps, got %v, want %v\", name, id, url, actual[i].Name, i, actual[i].HighPrecisionTimestamps, expected[i].HighPrecisionTimestamps))\n\t\t\t}\n\n\t\t\tif !reflect.DeepEqual(actual[i].AppliedFunctions, expected[i].AppliedFunctions) {\n\t\t\t\t*errors = append(*errors, fmt.Sprintf(\"TRY[%s] %s %s '%s': mismatch [%d].AppliedFunctions, got '%s', want '%s'\", name, id, url, actual[i].Name, i, actual[i].AppliedFunctions, expected[i].AppliedFunctions))\n\t\t\t}\n\n\t\t\tif !compare.NearlyEqual(float64(actual[i].XFilesFactor), float64(expected[i].XFilesFactor)) {\n\t\t\t\t*errors = append(*errors, fmt.Sprintf(\"TRY[%s] %s %s '%s': mismatch [%d].XFilesFactor, got %g, want %g\", name, id, url, actual[i].Name, i, actual[i].XFilesFactor, expected[i].XFilesFactor))\n\t\t\t}\n\n\t\t\tif !compare.NearlyEqualSlice(actual[i].Values, expected[i].Values) {\n\t\t\t\t*errors = append(*errors, fmt.Sprintf(\"TRY[%s] %s %s '%s': mismatch [%d].Values, got %g, want %g\", name, id, url, actual[i].Name, i, actual[i].Values, expected[i].Values))\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc parseFilteringFunctions(strFilteringFuncs []string) ([]*carbonapi_v3_pb.FilteringFunction, error) {\n\tres := make([]*carbonapi_v3_pb.FilteringFunction, 0, len(strFilteringFuncs))\n\n\tfor _, strFF := range strFilteringFuncs {\n\t\tstrFFSplit := strings.Split(strFF, \"(\")\n\t\tif len(strFFSplit) != 2 {\n\t\t\treturn nil, fmt.Errorf(\"could not parse filtering function: %s\", strFF)\n\t\t}\n\n\t\tname := strFFSplit[0]\n\n\t\targs := strings.Split(strFFSplit[1], \",\")\n\t\tfor i := range args {\n\t\t\targs[i] = strings.TrimSpace(args[i])\n\t\t\targs[i] = strings.Trim(args[i], \")'\")\n\t\t}\n\n\t\tres = append(res, &carbonapi_v3_pb.FilteringFunction{Name: name, Arguments: args})\n\t}\n\n\treturn res, nil\n}\n\nfunc verifyRender(ch *Clickhouse, gch *GraphiteClickhouse, check *RenderCheck, defaultPreision time.Duration) []string {\n\tvar errors []string\n\n\thttpClient := http.Client{\n\t\tTimeout: check.Timeout,\n\t}\n\taddress := gch.URL()\n\tfrom := datetime.TimestampTruncate(check.from, defaultPreision)\n\tuntil := datetime.TimestampTruncate(check.until, defaultPreision)\n\n\tfor _, format := range check.Formats {\n\t\tvar filteringFunctions []*carbonapi_v3_pb.FilteringFunction\n\n\t\tif format == client.FormatPb_v3 {\n\t\t\tvar err error\n\n\t\t\tfilteringFunctions, err = parseFilteringFunctions(check.FilteringFunctions)\n\t\t\tif err != nil {\n\t\t\t\terrors = append(errors, err.Error())\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tif url, result, respHeader, err := client.Render(&httpClient, address, format, check.Targets, filteringFunctions, check.MaxDataPoints, from, until); err == nil {\n\t\t\tid := requestId(respHeader)\n\t\t\tname := \"\"\n\n\t\t\tif check.ErrorRegexp != \"\" {\n\t\t\t\terrors = append(errors, fmt.Sprintf(\"TRY[%s] %s %s: want error with '%s'\", \"\", id, url, check.ErrorRegexp))\n\t\t\t}\n\n\t\t\tcompareRender(&errors, name, url, result, check.result, check.InCache, respHeader, check.CacheTTL)\n\n\t\t\tif len(result) == 0 && len(check.result) > 0 {\n\t\t\t\tgch.Grep(id)\n\n\t\t\t\tif len(check.DumpIfEmpty) > 0 {\n\t\t\t\t\tfor _, q := range check.DumpIfEmpty {\n\t\t\t\t\t\tif out, err := ch.Query(q); err == nil {\n\t\t\t\t\t\t\tfmt.Fprintf(os.Stderr, \"%s\\n%s\", q, out)\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tfmt.Fprintf(os.Stderr, \"%s: %s\\n\", err.Error(), q)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif check.CacheTTL > 0 && check.ErrorRegexp == \"\" {\n\t\t\t\t// second query must be find-cached\n\t\t\t\tname = \"cache\"\n\t\t\t\tif url, result, respHeader, err = client.Render(&httpClient, address, format, check.Targets, filteringFunctions, check.MaxDataPoints, from, until); err == nil {\n\t\t\t\t\tcompareRender(&errors, name, url, result, check.result, true, respHeader, check.CacheTTL)\n\t\t\t\t} else {\n\t\t\t\t\terrStr := strings.TrimRight(err.Error(), \"\\n\")\n\t\t\t\t\terrors = append(errors, fmt.Sprintf(\"TRY[%s] %s %s: %s\", name, requestId(respHeader), url, errStr))\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\terrStr := strings.TrimRight(err.Error(), \"\\n\")\n\t\t\tif check.errorRegexp == nil || !check.errorRegexp.MatchString(errStr) {\n\t\t\t\terrors = append(errors, fmt.Sprintf(\"TRY[%s] %s %s: want error with '%s', got '%s'\", \"\", requestId(respHeader), url, check.ErrorRegexp, errStr))\n\t\t\t} else {\n\t\t\t\tfmt.Printf(\"EXPECTED ERROR, SUCCESS %s : %s\\n\", url, errStr)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn errors\n}\n\nfunc debug(test *TestSchema, ch *Clickhouse, gch *GraphiteClickhouse) {\n\tfor {\n\t\tcmd := gch.Cmd()\n\t\tfmt.Println(cmd)\n\t\tfmt.Printf(\"graphite-clickhouse URL: %s , clickhouse URL: %s , proxy URL: %s (delay %v)\\n\",\n\t\t\tgch.URL(), ch.URL(), test.Proxy.URL(), test.Proxy.GetDelay())\n\t\tfmt.Printf(\"graphite-clickhouse log: %s , clickhouse container: %s\\n\",\n\t\t\tgch.storeDir+\"/graphite-clickhouse.log\", ch.container)\n\t\tfmt.Println(\"Some queries was failed, press y for continue after debug test, k for kill graphite-clickhouse:\")\n\n\t\tin := bufio.NewScanner(os.Stdin)\n\t\tin.Scan()\n\n\t\ts := in.Text()\n\t\tif s == \"y\" || s == \"Y\" {\n\t\t\tbreak\n\t\t} else if s == \"k\" || s == \"K\" {\n\t\t\tgch.Stop(false)\n\t\t}\n\t}\n}\n" }, { "path": "cmd/e2e-test/clickhouse.go", "content": "package main\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/msaf1980/go-stringutils\"\n)\n\nvar (\n\tClickhouseContainerName = \"clickhouse-server-gch-test\"\n\tClickhouseOldImage = \"yandex/clickhouse-server\"\n\tClickhouseDefaultImage = \"clickhouse/clickhouse-server\"\n)\n\ntype Clickhouse struct {\n\tVersion string `toml:\"version\"`\n\tDir string `toml:\"dir\"`\n\tTLSEnabled bool `toml:\"tls\"`\n\n\tDockerImage string `toml:\"image\"`\n\n\tTZ string `toml:\"tz\"` // override timezone\n\n\thttpAddress string `toml:\"-\"`\n\thttpsAddress string `toml:\"-\"`\n\turl string `toml:\"-\"`\n\ttlsurl string `toml:\"-\"`\n\tcontainer string `toml:\"-\"`\n}\n\nfunc (c *Clickhouse) CheckConfig(rootDir string) error {\n\tif c.Version == \"\" {\n\t\tc.Version = \"latest\"\n\t}\n\n\tif len(c.Dir) == 0 {\n\t\treturn ErrNoSetDir\n\t}\n\n\tif !strings.HasPrefix(c.Dir, \"/\") {\n\t\tc.Dir = rootDir + \"/\" + c.Dir\n\t}\n\n\tif c.DockerImage == \"\" {\n\t\tif c.Version == \"latest\" {\n\t\t\tc.DockerImage = ClickhouseDefaultImage\n\t\t} else {\n\t\t\tsplitV := strings.Split(c.Version, \".\")\n\n\t\t\tmajorV, err := strconv.Atoi(splitV[0])\n\t\t\tif err != nil {\n\t\t\t\tc.DockerImage = ClickhouseDefaultImage\n\t\t\t} else if majorV >= 21 {\n\t\t\t\tc.DockerImage = ClickhouseDefaultImage\n\t\t\t} else {\n\t\t\t\tc.DockerImage = ClickhouseOldImage\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (c *Clickhouse) Key() string {\n\treturn c.DockerImage + \":\" + c.Version + \" \" + c.Dir + \" TZ \" + c.TZ\n}\n\nfunc (c *Clickhouse) Start() (string, error) {\n\tvar err error\n\n\tc.httpAddress, err = getFreeTCPPort(\"\")\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tport := strings.Split(c.httpAddress, \":\")[1]\n\tc.url = \"http://\" + c.httpAddress\n\n\tc.container = ClickhouseContainerName\n\n\t// tz, _ := localTZLocationName()\n\n\tchStart := []string{\"run\", \"-d\",\n\t\t\"--name\", c.container,\n\t\t\"--ulimit\", \"nofile=262144:262144\",\n\t\t\"-p\", port + \":8123\",\n\t\t// \"-e\", \"TZ=\" + tz, // workaround for TZ=\":/etc/localtime\"\n\t\t\"-v\", c.Dir + \"/config.xml:/etc/clickhouse-server/config.xml\",\n\t\t\"-v\", c.Dir + \"/users.xml:/etc/clickhouse-server/users.xml\",\n\t\t\"-v\", c.Dir + \"/rollup.xml:/etc/clickhouse-server/config.d/rollup.xml\",\n\t\t\"-v\", c.Dir + \"/init.sql:/docker-entrypoint-initdb.d/init.sql\",\n\t\t\"--network\", DockerNetwork,\n\t}\n\n\tif c.TLSEnabled {\n\t\tc.httpsAddress, err = getFreeTCPPort(\"\")\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\n\t\tport = strings.Split(c.httpsAddress, \":\")[1]\n\t\tc.tlsurl = \"https://\" + c.httpsAddress\n\t\tchStart = append(chStart,\n\t\t\t\"-v\", c.Dir+\"/server.crt:/etc/clickhouse-server/server.crt\",\n\t\t\t\"-v\", c.Dir+\"/server.key:/etc/clickhouse-server/server.key\",\n\t\t\t\"-v\", c.Dir+\"/rootCA.crt:/etc/clickhouse-server/rootCA.crt\",\n\t\t\t\"-p\", port+\":8443\",\n\t\t)\n\t}\n\n\tif c.TZ != \"\" {\n\t\tchStart = append(chStart, \"-e\", \"TZ=\"+c.TZ)\n\t}\n\n\tchStart = append(chStart, c.DockerImage+\":\"+c.Version)\n\n\tcmd := exec.Command(DockerBinary, chStart...)\n\tout, err := cmd.CombinedOutput()\n\n\treturn string(out), err\n}\n\nfunc (c *Clickhouse) Stop(delete bool) (string, error) {\n\tif len(c.container) == 0 {\n\t\treturn \"\", nil\n\t}\n\n\tchStop := []string{\"stop\", c.container}\n\n\tcmd := exec.Command(DockerBinary, chStop...)\n\tout, err := cmd.CombinedOutput()\n\n\tif err == nil && delete {\n\t\treturn c.Delete()\n\t}\n\n\treturn string(out), err\n}\n\nfunc (c *Clickhouse) Delete() (string, error) {\n\tif len(c.container) == 0 {\n\t\treturn \"\", nil\n\t}\n\n\tchDel := []string{\"rm\", c.container}\n\n\tcmd := exec.Command(DockerBinary, chDel...)\n\tout, err := cmd.CombinedOutput()\n\n\tif err == nil {\n\t\tc.container = \"\"\n\t}\n\n\treturn string(out), err\n}\n\nfunc (c *Clickhouse) URL() string {\n\treturn c.url\n}\n\nfunc (c *Clickhouse) TLSURL() string {\n\treturn c.tlsurl\n}\n\nfunc (c *Clickhouse) Container() string {\n\treturn c.container\n}\n\nfunc (c *Clickhouse) Exec(sql string) (bool, string) {\n\treturn containerExec(c.container, []string{\"sh\", \"-c\", \"clickhouse-client -q '\" + sql + \"'\"})\n}\n\nfunc (c *Clickhouse) Query(sql string) (string, error) {\n\treader := strings.NewReader(sql)\n\n\trequest, err := http.NewRequest(http.MethodPost, c.URL(), reader)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tresp, err := http.DefaultClient.Do(request)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tdefer resp.Body.Close()\n\n\tmsg, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tif resp.StatusCode != http.StatusOK {\n\t\treturn \"\", errors.New(resp.Status + \": \" + string(bytes.TrimRight(msg, \"\\n\")))\n\t}\n\n\treturn string(msg), nil\n}\n\nfunc (c *Clickhouse) Alive() bool {\n\tif len(c.container) == 0 {\n\t\treturn false\n\t}\n\n\treq, err := http.DefaultClient.Get(c.URL())\n\tif err != nil {\n\t\treturn false\n\t}\n\n\tdefer req.Body.Close()\n\n\treturn req.StatusCode == http.StatusOK\n}\n\nfunc (c *Clickhouse) CopyLog(destDir string, tail uint64) error {\n\tif len(c.container) == 0 {\n\t\treturn nil\n\t}\n\n\tdest := destDir + \"/clickhouse-server.log\"\n\n\tchArgs := []string{\"cp\", c.container + \":/var/log/clickhouse-server/clickhouse-server.log\", dest}\n\n\tcmd := exec.Command(DockerBinary, chArgs...)\n\n\tout, err := cmd.CombinedOutput()\n\tif err != nil {\n\t\treturn errors.New(err.Error() + \": \" + string(bytes.TrimRight(out, \"\\n\")))\n\t}\n\n\tif tail > 0 {\n\t\tout, _ := exec.Command(\"tail\", \"-\"+strconv.FormatUint(tail, 10), dest).Output()\n\t\tfmt.Fprintf(os.Stderr, \"CLICKHOUSE-SERVER.LOG %s\", stringutils.UnsafeString(out))\n\t}\n\n\treturn nil\n}\n\nfunc (c *Clickhouse) CopyErrLog(destDir string, tail uint64) error {\n\tif len(c.container) == 0 {\n\t\treturn nil\n\t}\n\n\tdest := destDir + \"/clickhouse-server.err.log\"\n\n\tchArgs := []string{\"cp\", c.container + \":/var/log/clickhouse-server/clickhouse-server.err.log\", dest}\n\n\tcmd := exec.Command(DockerBinary, chArgs...)\n\n\tout, err := cmd.CombinedOutput()\n\tif err != nil {\n\t\treturn errors.New(err.Error() + \": \" + string(bytes.TrimRight(out, \"\\n\")))\n\t}\n\n\tif tail > 0 {\n\t\tout, _ := exec.Command(\"tail\", \"-\"+strconv.FormatUint(tail, 10), dest).Output()\n\t\tfmt.Fprintf(os.Stderr, \"CLICKHOUSE-SERVER.ERR %s\", stringutils.UnsafeString(out))\n\t}\n\n\treturn nil\n}\n" }, { "path": "cmd/e2e-test/container.go", "content": "package main\n\nimport (\n\t\"os/exec\"\n\t\"strings\"\n)\n\nvar (\n\tDockerBinary string\n\tDockerNetwork string = \"graphite-ch-test\"\n)\n\nfunc imageDelete(image, version string) (bool, string) {\n\tif len(DockerBinary) == 0 {\n\t\tpanic(\"docker not set\")\n\t}\n\n\tchArgs := []string{\"rmi\", image + \":\" + version}\n\n\tcmd := exec.Command(DockerBinary, chArgs...)\n\tout, err := cmd.CombinedOutput()\n\ts := strings.Trim(string(out), \"\\n\")\n\n\tif err == nil {\n\t\treturn true, s\n\t}\n\n\treturn false, err.Error() + \": \" + s\n}\n\nfunc containerExist(name string) (bool, string) {\n\tif len(DockerBinary) == 0 {\n\t\tpanic(\"docker not set\")\n\t}\n\n\tchInspect := []string{\"inspect\", \"--format\", \"'{{.Name}}'\", name}\n\n\tcmd := exec.Command(DockerBinary, chInspect...)\n\tout, err := cmd.CombinedOutput()\n\ts := strings.Trim(string(out), \"\\n\")\n\n\tif err == nil {\n\t\treturn true, s\n\t}\n\n\treturn false, err.Error() + \": \" + s\n}\n\nfunc containerRemove(name string) (bool, string) {\n\tif len(DockerBinary) == 0 {\n\t\tpanic(\"docker not set\")\n\t}\n\n\tchInspect := []string{\"rm\", \"-f\", name}\n\n\tcmd := exec.Command(DockerBinary, chInspect...)\n\tout, err := cmd.CombinedOutput()\n\ts := strings.Trim(string(out), \"\\n\")\n\n\tif err == nil {\n\t\treturn true, s\n\t}\n\n\treturn false, err.Error() + \": \" + s\n}\n\nfunc containerExec(name string, args []string) (bool, string) {\n\tif len(DockerBinary) == 0 {\n\t\tpanic(\"docker not set\")\n\t}\n\n\tdCmd := []string{\"exec\", name}\n\tdCmd = append(dCmd, args...)\n\n\tcmd := exec.Command(DockerBinary, dCmd...)\n\tout, err := cmd.CombinedOutput()\n\ts := strings.Trim(string(out), \"\\n\")\n\n\tif err == nil {\n\t\treturn true, s\n\t}\n\n\treturn false, err.Error() + \": \" + s\n}\n" }, { "path": "cmd/e2e-test/e2etesting.go", "content": "package main\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"path\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"go.uber.org/zap\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/client\"\n\t\"github.com/lomik/graphite-clickhouse/helper/datetime\"\n\n\t\"github.com/pelletier/go-toml\"\n)\n\nvar (\n\tpreSQL = []string{\n\t\t\"TRUNCATE TABLE IF EXISTS graphite_reverse\",\n\t\t\"TRUNCATE TABLE IF EXISTS graphite\",\n\t\t\"TRUNCATE TABLE IF EXISTS graphite_index\",\n\t\t\"TRUNCATE TABLE IF EXISTS graphite_tags\",\n\t}\n)\n\ntype Point struct {\n\tValue float64 `toml:\"value\"`\n\tTime string `toml:\"time\"`\n\tDelay time.Duration `toml:\"delay\"`\n\n\ttime int64 `toml:\"-\"`\n}\n\ntype InputMetric struct {\n\tName string `toml:\"name\"`\n\tPoints []Point `toml:\"points\"`\n\tRound time.Duration `toml:\"round\"`\n}\n\ntype Metric struct {\n\tName string `toml:\"name\"`\n\tPathExpression string `toml:\"path\"`\n\tConsolidationFunc string `toml:\"consolidation\"`\n\tStartTime string `toml:\"start\"`\n\tStopTime string `toml:\"stop\"`\n\tStepTime int64 `toml:\"step\"`\n\tXFilesFactor float32 `toml:\"xfiles\"`\n\tHighPrecisionTimestamps bool `toml:\"high_precision\"`\n\tValues []float64 `toml:\"values\"`\n\tAppliedFunctions []string `toml:\"applied_functions\"`\n\tRequestStartTime string `toml:\"req_start\"`\n\tRequestStopTime string `toml:\"req_stop\"`\n}\n\ntype RenderCheck struct {\n\tName string `toml:\"name\"`\n\tFormats []client.FormatType `toml:\"formats\"`\n\tFrom string `toml:\"from\"`\n\tUntil string `toml:\"until\"`\n\tTargets []string `toml:\"targets\"`\n\tMaxDataPoints int64 `toml:\"max_data_points\"`\n\tFilteringFunctions []string `toml:\"filtering_functions\"`\n\tTimeout time.Duration `toml:\"timeout\"`\n\tDumpIfEmpty []string `toml:\"dump_if_empty\"`\n\n\tOptimize []string `toml:\"optimize\"` // optimize tables before run tests\n\n\tInCache bool `toml:\"in_cache\"` // already in cache\n\tCacheTTL int `toml:\"cache_ttl\"`\n\n\tProxyDelay time.Duration `toml:\"proxy_delay\"`\n\tProxyBreakWithCode int `toml:\"proxy_break_with_code\"`\n\n\tResult []Metric `toml:\"result\"`\n\tErrorRegexp string `toml:\"error_regexp\"`\n\n\tfrom int64 `toml:\"-\"`\n\tuntil int64 `toml:\"-\"`\n\terrorRegexp *regexp.Regexp `toml:\"-\"`\n\tresult []client.Metric `toml:\"-\"`\n}\n\ntype MetricsFindCheck struct {\n\tName string `toml:\"name\"`\n\tFormats []client.FormatType `toml:\"formats\"`\n\tFrom string `toml:\"from\"`\n\tUntil string `toml:\"until\"`\n\tQuery string `toml:\"query\"`\n\tTimeout time.Duration `toml:\"timeout\"`\n\n\tDumpIfEmpty []string `toml:\"dump_if_empty\"`\n\n\tInCache bool `toml:\"in_cache\"` // already in cache\n\tCacheTTL int `toml:\"cache_ttl\"`\n\n\tProxyDelay time.Duration `toml:\"proxy_delay\"`\n\tProxyBreakWithCode int `toml:\"proxy_break_with_code\"`\n\n\tResult []client.FindMatch `toml:\"result\"`\n\tErrorRegexp string `toml:\"error_regexp\"`\n\n\tfrom int64 `toml:\"-\"`\n\tuntil int64 `toml:\"-\"`\n\terrorRegexp *regexp.Regexp `toml:\"-\"`\n}\n\ntype TagsCheck struct {\n\tName string `toml:\"name\"`\n\tNames bool `toml:\"names\"` // TagNames or TagValues\n\tFormats []client.FormatType `toml:\"formats\"`\n\tFrom string `toml:\"from\"`\n\tUntil string `toml:\"until\"`\n\tQuery string `toml:\"query\"`\n\tLimits uint64 `toml:\"limits\"`\n\tTimeout time.Duration `toml:\"timeout\"`\n\n\tDumpIfEmpty []string `toml:\"dump_if_empty\"`\n\n\tInCache bool `toml:\"in_cache\"` // already in cache\n\tCacheTTL int `toml:\"cache_ttl\"`\n\n\tProxyDelay time.Duration `toml:\"proxy_delay\"`\n\tProxyBreakWithCode int `toml:\"proxy_break_with_code\"`\n\n\tResult []string `toml:\"result\"`\n\tErrorRegexp string `toml:\"error_regexp\"`\n\n\tfrom int64 `toml:\"-\"`\n\tuntil int64 `toml:\"-\"`\n\terrorRegexp *regexp.Regexp `toml:\"-\"`\n}\n\ntype TestSchema struct {\n\tInput []InputMetric `toml:\"input\"` // carbon-clickhouse input\n\tClickhouse []Clickhouse `toml:\"clickhouse\"`\n\tProxy HttpReverseProxy `toml:\"clickhouse_proxy\"`\n\tCch CarbonClickhouse `toml:\"carbon_clickhouse\"`\n\tGch []GraphiteClickhouse `toml:\"graphite_clickhouse\"`\n\n\tFindChecks []*MetricsFindCheck `toml:\"find_checks\"`\n\tTagsChecks []*TagsCheck `toml:\"tags_checks\"`\n\tRenderChecks []*RenderCheck `toml:\"render_checks\"`\n\n\tPrecision time.Duration `toml:\"precision\"`\n\n\tdir string `toml:\"-\"`\n\tname string `toml:\"-\"` // test alias (from config name)\n\tchVersions map[string]bool `toml:\"-\"`\n\t// input map[string][]Point `toml:\"-\"`\n}\n\nfunc (schema *TestSchema) HasTLSSettings() bool {\n\treturn strings.Contains(schema.dir, \"tls\")\n}\n\nfunc getFreeTCPPort(name string) (string, error) {\n\tif len(name) == 0 {\n\t\tname = \"127.0.0.1:0\"\n\t} else if !strings.Contains(name, \":\") {\n\t\tname = name + \":0\"\n\t}\n\n\taddr, err := net.ResolveTCPAddr(\"tcp\", name)\n\tif err != nil {\n\t\treturn name, err\n\t}\n\n\tl, err := net.ListenTCP(\"tcp\", addr)\n\tif err != nil {\n\t\treturn name, err\n\t}\n\n\tdefer l.Close()\n\n\treturn l.Addr().String(), nil\n}\n\nfunc sendPlain(network, address string, metrics []InputMetric) error {\n\tif conn, err := net.DialTimeout(network, address, time.Second); err != nil {\n\t\treturn err\n\t} else {\n\t\tbw := bufio.NewWriter(conn)\n\n\t\tfor _, m := range metrics {\n\t\t\tconn.SetDeadline(time.Now().Add(time.Second))\n\n\t\t\tfor _, point := range m.Points {\n\t\t\t\tif _, err = fmt.Fprintf(bw, \"%s %f %d\\n\", m.Name, point.Value, point.time); err != nil {\n\t\t\t\t\tconn.Close()\n\t\t\t\t\treturn err\n\t\t\t\t}\n\n\t\t\t\tif point.Delay > 0 {\n\t\t\t\t\tif err = bw.Flush(); err != nil {\n\t\t\t\t\t\tconn.Close()\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\n\t\t\t\t\ttime.Sleep(point.Delay)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif err = bw.Flush(); err != nil {\n\t\t\tconn.Close()\n\t\t\treturn err\n\t\t}\n\n\t\treturn conn.Close()\n\t}\n}\n\nfunc verifyGraphiteClickhouse(test *TestSchema, gch *GraphiteClickhouse, clickhouse *Clickhouse, testDir, clickhouseDir string, verbose, breakOnError bool, logger *zap.Logger) (testSuccess bool, verifyCount, verifyFailed int) {\n\ttestSuccess = true\n\n\terr := gch.Start(testDir, clickhouse.URL(), test.Proxy.URL(), clickhouse.TLSURL())\n\tif err != nil {\n\t\tlogger.Error(\"starting graphite-clickhouse\",\n\t\t\tzap.String(\"config\", test.name),\n\t\t\tzap.String(\"clickhouse version\", clickhouse.Version),\n\t\t\tzap.String(\"clickhouse config\", clickhouseDir),\n\t\t\tzap.String(\"graphite-clickhouse config\", gch.ConfigTpl),\n\t\t\tzap.Error(err),\n\t\t)\n\n\t\ttestSuccess = false\n\n\t\treturn\n\t}\n\n\tfor i := 100; i < 1000; i += 200 {\n\t\ttime.Sleep(time.Duration(i) * time.Millisecond)\n\n\t\tif gch.Alive() {\n\t\t\tbreak\n\t\t}\n\t}\n\n\t// start tests\n\tfor n, check := range test.FindChecks {\n\t\tverifyCount++\n\n\t\ttest.Proxy.SetDelay(check.ProxyDelay)\n\t\ttest.Proxy.SetBreakStatusCode(check.ProxyBreakWithCode)\n\n\t\tif len(check.Formats) == 0 {\n\t\t\tcheck.Formats = []client.FormatType{client.FormatPb_v3}\n\t\t}\n\n\t\tif errs := verifyMetricsFind(clickhouse, gch, check); len(errs) > 0 {\n\t\t\tverifyFailed++\n\n\t\t\tfor _, e := range errs {\n\t\t\t\tfmt.Fprintln(os.Stderr, e)\n\t\t\t}\n\n\t\t\tlogger.Error(\"verify metrics find\",\n\t\t\t\tzap.String(\"config\", test.name),\n\t\t\t\tzap.String(\"clickhouse version\", clickhouse.Version),\n\t\t\t\tzap.String(\"clickhouse config\", clickhouseDir),\n\t\t\t\tzap.String(\"graphite-clickhouse config\", gch.ConfigTpl),\n\t\t\t\tzap.String(\"query\", check.Query),\n\t\t\t\tzap.String(\"from_raw\", check.From),\n\t\t\t\tzap.String(\"until_raw\", check.Until),\n\t\t\t\tzap.Int64(\"from\", check.from),\n\t\t\t\tzap.Int64(\"until\", check.until),\n\t\t\t\tzap.String(\"name\", check.Name+\"[\"+strconv.Itoa(n)+\"]\"),\n\t\t\t)\n\n\t\t\tif breakOnError {\n\t\t\t\tdebug(test, clickhouse, gch)\n\t\t\t}\n\t\t} else if verbose {\n\t\t\tlogger.Info(\"verify metrics find\",\n\t\t\t\tzap.String(\"config\", test.name),\n\t\t\t\tzap.String(\"clickhouse version\", clickhouse.Version),\n\t\t\t\tzap.String(\"clickhouse config\", clickhouseDir),\n\t\t\t\tzap.String(\"graphite-clickhouse config\", gch.ConfigTpl),\n\t\t\t\tzap.String(\"query\", check.Query),\n\t\t\t\tzap.String(\"from_raw\", check.From),\n\t\t\t\tzap.String(\"until_raw\", check.Until),\n\t\t\t\tzap.Int64(\"from\", check.from),\n\t\t\t\tzap.Int64(\"until\", check.until),\n\t\t\t\tzap.String(\"name\", check.Name+\"[\"+strconv.Itoa(n)+\"]\"),\n\t\t\t)\n\t\t}\n\t}\n\n\tfor n, check := range test.TagsChecks {\n\t\tverifyCount++\n\n\t\ttest.Proxy.SetDelay(check.ProxyDelay)\n\t\ttest.Proxy.SetBreakStatusCode(check.ProxyBreakWithCode)\n\n\t\tif len(check.Formats) == 0 {\n\t\t\tcheck.Formats = []client.FormatType{client.FormatJSON}\n\t\t}\n\n\t\tif errs := verifyTags(clickhouse, gch, check); len(errs) > 0 {\n\t\t\tverifyFailed++\n\n\t\t\tfor _, e := range errs {\n\t\t\t\tfmt.Fprintln(os.Stderr, e)\n\t\t\t}\n\n\t\t\tlogger.Error(\"verify tags\",\n\t\t\t\tzap.String(\"config\", test.name),\n\t\t\t\tzap.String(\"clickhouse version\", clickhouse.Version),\n\t\t\t\tzap.String(\"clickhouse config\", clickhouseDir),\n\t\t\t\tzap.String(\"graphite-clickhouse config\", gch.ConfigTpl),\n\t\t\t\tzap.Bool(\"name\", check.Names),\n\t\t\t\tzap.String(\"query\", check.Query),\n\t\t\t\tzap.String(\"from_raw\", check.From),\n\t\t\t\tzap.String(\"until_raw\", check.Until),\n\t\t\t\tzap.Int64(\"from\", check.from),\n\t\t\t\tzap.Int64(\"until\", check.until),\n\t\t\t\tzap.String(\"name\", check.Name+\"[\"+strconv.Itoa(n)+\"]\"),\n\t\t\t)\n\n\t\t\tif breakOnError {\n\t\t\t\tdebug(test, clickhouse, gch)\n\t\t\t}\n\t\t} else if verbose {\n\t\t\tlogger.Info(\"verify tags\",\n\t\t\t\tzap.String(\"config\", test.name),\n\t\t\t\tzap.String(\"clickhouse version\", clickhouse.Version),\n\t\t\t\tzap.String(\"clickhouse config\", clickhouseDir),\n\t\t\t\tzap.String(\"graphite-clickhouse config\", gch.ConfigTpl),\n\t\t\t\tzap.Bool(\"name\", check.Names),\n\t\t\t\tzap.String(\"query\", check.Query),\n\t\t\t\tzap.String(\"from_raw\", check.From),\n\t\t\t\tzap.String(\"until_raw\", check.Until),\n\t\t\t\tzap.Int64(\"from\", check.from),\n\t\t\t\tzap.Int64(\"until\", check.until),\n\t\t\t\tzap.String(\"name\", check.Name+\"[\"+strconv.Itoa(n)+\"]\"),\n\t\t\t)\n\t\t}\n\t}\n\n\tfor n, check := range test.RenderChecks {\n\t\tverifyCount++\n\n\t\ttest.Proxy.SetDelay(check.ProxyDelay)\n\t\ttest.Proxy.SetBreakStatusCode(check.ProxyBreakWithCode)\n\n\t\tif len(check.Formats) == 0 {\n\t\t\tcheck.Formats = []client.FormatType{client.FormatPb_v3}\n\t\t}\n\n\t\tif len(check.Optimize) > 0 {\n\t\t\tfor _, table := range check.Optimize {\n\t\t\t\tif success, out := clickhouse.Exec(\"OPTIMIZE TABLE \" + table + \" FINAL\"); !success {\n\t\t\t\t\tlogger.Error(\"optimize table\",\n\t\t\t\t\t\tzap.String(\"config\", test.name),\n\t\t\t\t\t\tzap.String(\"clickhouse version\", clickhouse.Version),\n\t\t\t\t\t\tzap.String(\"clickhouse config\", clickhouseDir),\n\t\t\t\t\t\tzap.String(\"graphite-clickhouse config\", gch.ConfigTpl),\n\t\t\t\t\t\tzap.Strings(\"targets\", check.Targets),\n\t\t\t\t\t\tzap.Strings(\"filtering_functions\", check.FilteringFunctions),\n\t\t\t\t\t\tzap.String(\"from_raw\", check.From),\n\t\t\t\t\t\tzap.String(\"until_raw\", check.Until),\n\t\t\t\t\t\tzap.Int64(\"from\", check.from),\n\t\t\t\t\t\tzap.Int64(\"until\", check.until),\n\t\t\t\t\t\tzap.String(\"name\", check.Name+\"[\"+strconv.Itoa(n)+\"]\"),\n\t\t\t\t\t\tzap.String(\"table\", table),\n\t\t\t\t\t\tzap.String(\"out\", out),\n\t\t\t\t\t)\n\t\t\t\t\ttime.Sleep(5 * time.Second)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif errs := verifyRender(clickhouse, gch, check, test.Precision); len(errs) > 0 {\n\t\t\tverifyFailed++\n\n\t\t\tfor _, e := range errs {\n\t\t\t\tfmt.Fprintln(os.Stderr, e)\n\t\t\t}\n\n\t\t\tlogger.Error(\"verify render\",\n\t\t\t\tzap.String(\"config\", test.name),\n\t\t\t\tzap.String(\"clickhouse version\", clickhouse.Version),\n\t\t\t\tzap.String(\"clickhouse config\", clickhouseDir),\n\t\t\t\tzap.String(\"graphite-clickhouse config\", gch.ConfigTpl),\n\t\t\t\tzap.Strings(\"targets\", check.Targets),\n\t\t\t\tzap.Strings(\"filtering_functions\", check.FilteringFunctions),\n\t\t\t\tzap.String(\"from_raw\", check.From),\n\t\t\t\tzap.String(\"until_raw\", check.Until),\n\t\t\t\tzap.Int64(\"from\", check.from),\n\t\t\t\tzap.Int64(\"until\", check.until),\n\t\t\t\tzap.String(\"name\", check.Name+\"[\"+strconv.Itoa(n)+\"]\"),\n\t\t\t)\n\n\t\t\tif breakOnError {\n\t\t\t\tdebug(test, clickhouse, gch)\n\t\t\t}\n\t\t} else if verbose {\n\t\t\tlogger.Info(\"verify render\",\n\t\t\t\tzap.String(\"config\", test.name),\n\t\t\t\tzap.String(\"clickhouse version\", clickhouse.Version),\n\t\t\t\tzap.String(\"clickhouse config\", clickhouseDir),\n\t\t\t\tzap.String(\"graphite-clickhouse config\", gch.ConfigTpl),\n\t\t\t\tzap.Strings(\"targets\", check.Targets),\n\t\t\t\tzap.Strings(\"filtering_functions\", check.FilteringFunctions),\n\t\t\t\tzap.String(\"from_raw\", check.From),\n\t\t\t\tzap.String(\"until_raw\", check.Until),\n\t\t\t\tzap.Int64(\"from\", check.from),\n\t\t\t\tzap.Int64(\"until\", check.until),\n\t\t\t\tzap.String(\"name\", check.Name+\"[\"+strconv.Itoa(n)+\"]\"),\n\t\t\t)\n\t\t}\n\t}\n\n\tif verifyFailed > 0 {\n\t\ttestSuccess = false\n\n\t\tlogger.Error(\"verify\",\n\t\t\tzap.String(\"config\", test.name),\n\t\t\tzap.String(\"clickhouse version\", clickhouse.Version),\n\t\t\tzap.String(\"clickhouse config\", clickhouseDir),\n\t\t\tzap.String(\"graphite-clickhouse config\", gch.ConfigTpl),\n\t\t\tzap.Int64(\"count\", int64(verifyCount)),\n\t\t\tzap.Int64(\"failed\", int64(verifyFailed)),\n\t\t)\n\t}\n\n\terr = gch.Stop(true)\n\tif err != nil {\n\t\tlogger.Error(\"stoping graphite-clickhouse\",\n\t\t\tzap.String(\"config\", test.name),\n\t\t\tzap.String(\"gch\", gch.ConfigTpl),\n\t\t\tzap.String(\"clickhouse version\", clickhouse.Version),\n\t\t\tzap.String(\"clickhouse config\", clickhouseDir),\n\t\t\tzap.Error(err),\n\t\t)\n\n\t\ttestSuccess = false\n\t}\n\n\treturn\n}\n\nfunc testGraphiteClickhouse(test *TestSchema, clickhouse *Clickhouse, testDir, rootDir string, verbose, breakOnError bool, logger *zap.Logger) (testSuccess bool, verifyCount, verifyFailed int) {\n\ttestSuccess = true\n\n\tfor _, sql := range preSQL {\n\t\tif success, out := clickhouse.Exec(sql); !success {\n\t\t\tlogger.Error(\"pre-execute\",\n\t\t\t\tzap.String(\"config\", test.name),\n\t\t\t\tzap.Any(\"clickhouse version\", clickhouse.Version),\n\t\t\t\tzap.String(\"clickhouse config\", clickhouse.Dir),\n\t\t\t\tzap.String(\"sql\", sql),\n\t\t\t\tzap.String(\"out\", out),\n\t\t\t)\n\n\t\t\treturn\n\t\t}\n\t}\n\n\tif err := test.Proxy.Start(clickhouse.URL()); err != nil {\n\t\tlogger.Error(\"starting clickhouse proxy\",\n\t\t\tzap.String(\"config\", test.name),\n\t\t\tzap.Any(\"clickhouse version\", clickhouse.Version),\n\t\t\tzap.String(\"clickhouse config\", clickhouse.Dir),\n\t\t\tzap.Error(err),\n\t\t)\n\n\t\treturn\n\t}\n\n\tout, err := test.Cch.Start(testDir, \"http://\"+clickhouse.Container()+\":8123\")\n\tif err != nil {\n\t\tlogger.Error(\"starting carbon-clickhouse\",\n\t\t\tzap.String(\"config\", test.name),\n\t\t\tzap.String(\"clickhouse version\", clickhouse.Version),\n\t\t\tzap.String(\"clickhouse config\", clickhouse.Dir),\n\t\t\tzap.Error(err),\n\t\t\tzap.String(\"out\", out),\n\t\t)\n\n\t\ttestSuccess = false\n\t}\n\n\tif testSuccess {\n\t\tlogger.Info(\"starting e2e test\",\n\t\t\tzap.String(\"config\", test.name),\n\t\t\tzap.String(\"clickhouse version\", clickhouse.Version),\n\t\t\tzap.String(\"clickhouse config\", clickhouse.Dir),\n\t\t)\n\t\ttime.Sleep(200 * time.Millisecond)\n\t\t// Populate test data\n\t\terr = sendPlain(\"tcp\", test.Cch.address, test.Input)\n\t\tif err != nil {\n\t\t\tlogger.Error(\"send plain to carbon-clickhouse\",\n\t\t\t\tzap.String(\"config\", test.name),\n\t\t\t\tzap.String(\"clickhouse version\", clickhouse.Version),\n\t\t\t\tzap.String(\"clickhouse config\", clickhouse.Dir),\n\t\t\t\tzap.Error(err),\n\t\t\t)\n\n\t\t\ttestSuccess = false\n\t\t}\n\n\t\tif testSuccess {\n\t\t\ttime.Sleep(2 * time.Second)\n\t\t}\n\n\t\tif testSuccess {\n\t\t\tfor _, gch := range test.Gch {\n\t\t\t\tstepSuccess, vCount, vFailed := verifyGraphiteClickhouse(test, &gch, clickhouse, testDir, clickhouse.Dir, verbose, breakOnError, logger)\n\t\t\t\tverifyCount += vCount\n\t\t\t\tverifyFailed += vFailed\n\n\t\t\t\tif !stepSuccess {\n\t\t\t\t\ttestSuccess = false\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tout, err = test.Cch.Stop(true)\n\tif err != nil {\n\t\tlogger.Error(\"stoping carbon-clickhouse\",\n\t\t\tzap.String(\"config\", test.name),\n\t\t\tzap.String(\"clickhouse version\", clickhouse.Version),\n\t\t\tzap.String(\"clickhouse config\", clickhouse.Dir),\n\t\t\tzap.Error(err),\n\t\t\tzap.String(\"out\", out),\n\t\t)\n\n\t\ttestSuccess = false\n\t}\n\n\ttest.Proxy.Stop()\n\n\tif testSuccess {\n\t\tlogger.Info(\"end e2e test\",\n\t\t\tzap.String(\"config\", test.name),\n\t\t\tzap.String(\"status\", \"success\"),\n\t\t\tzap.String(\"clickhouse version\", clickhouse.Version),\n\t\t\tzap.String(\"clickhouse config\", clickhouse.Dir),\n\t\t)\n\t} else {\n\t\tlogger.Error(\"end e2e test\",\n\t\t\tzap.String(\"config\", test.name),\n\t\t\tzap.String(\"status\", \"failed\"),\n\t\t\tzap.String(\"clickhouse version\", clickhouse.Version),\n\t\t\tzap.String(\"clickhouse config\", clickhouse.Dir),\n\t\t)\n\t}\n\n\treturn\n}\n\nfunc runTest(cfg *MainConfig, clickhouse *Clickhouse, rootDir string, now time.Time, verbose, breakOnError bool, logger *zap.Logger) (failed, total, verifyCount, verifyFailed int) {\n\tvar isRunning bool\n\n\ttotal++\n\n\tif exist, out := containerExist(CchContainerName); exist {\n\t\tlogger.Error(\"carbon-clickhouse already exist\",\n\t\t\tzap.String(\"container\", CchContainerName),\n\t\t\tzap.String(\"out\", out),\n\t\t)\n\n\t\tisRunning = true\n\t}\n\n\tif isRunning {\n\t\tfailed++\n\t\treturn\n\t}\n\n\tsuccess, vCount, vFailed := testGraphiteClickhouse(cfg.Test, clickhouse, cfg.Test.dir, rootDir, verbose, breakOnError, logger)\n\tif !success {\n\t\tfailed++\n\t}\n\n\tverifyCount += vCount\n\tverifyFailed += vFailed\n\n\treturn\n}\n\nfunc clickhouseStart(clickhouse *Clickhouse, logger *zap.Logger) bool {\n\tout, err := clickhouse.Start()\n\tif err != nil {\n\t\tlogger.Error(\"starting clickhouse\",\n\t\t\tzap.Any(\"clickhouse version\", clickhouse.Version),\n\t\t\tzap.String(\"clickhouse config\", clickhouse.Dir),\n\t\t\tzap.Error(err),\n\t\t\tzap.String(\"out\", out),\n\t\t)\n\t\tclickhouse.Stop(true)\n\n\t\treturn false\n\t}\n\n\treturn true\n}\n\nfunc clickhouseStop(clickhouse *Clickhouse, logger *zap.Logger) (result bool) {\n\tresult = true\n\n\tif !clickhouse.Alive() {\n\t\tclickhouse.CopyLog(os.TempDir(), 10)\n\n\t\tresult = false\n\t}\n\n\tout, err := clickhouse.Stop(true)\n\tif err != nil {\n\t\tlogger.Error(\"stoping clickhouse\",\n\t\t\tzap.String(\"clickhouse version\", clickhouse.Version),\n\t\t\tzap.String(\"clickhouse config\", clickhouse.Dir),\n\t\t\tzap.Error(err),\n\t\t\tzap.String(\"out\", out),\n\t\t)\n\n\t\tresult = false\n\t}\n\n\treturn result\n}\n\nfunc initTest(cfg *MainConfig, rootDir string, now time.Time, verbose, breakOnError bool, logger *zap.Logger) bool {\n\ttz, err := datetime.Timezone(\"\")\n\tif err != nil {\n\t\tfmt.Printf(\"can't get timezone: %s\\n\", err.Error())\n\t\tos.Exit(1)\n\t}\n\n\t// prepare\n\tfor n, m := range cfg.Test.Input {\n\t\tfor i := range m.Points {\n\t\t\tm.Points[i].time = datetime.DateParamToEpoch(m.Points[i].Time, tz, now, cfg.Test.Precision)\n\t\t\tif m.Points[i].time == 0 {\n\t\t\t\terr = ErrTimestampInvalid\n\t\t\t}\n\n\t\t\tif err != nil {\n\t\t\t\tlogger.Error(\"failed to read config\",\n\t\t\t\t\tzap.String(\"config\", cfg.Test.name),\n\t\t\t\t\tzap.Error(err),\n\t\t\t\t\tzap.String(\"input\", m.Name),\n\t\t\t\t\tzap.Int(\"metric\", n),\n\t\t\t\t\tzap.Int(\"point\", i),\n\t\t\t\t\tzap.String(\"time\", m.Points[i].Time),\n\t\t\t\t)\n\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t}\n\n\tfor n, find := range cfg.Test.FindChecks {\n\t\tif find.Timeout == 0 {\n\t\t\tfind.Timeout = 10 * time.Second\n\t\t}\n\n\t\tfind.from = datetime.DateParamToEpoch(find.From, tz, now, cfg.Test.Precision)\n\t\tif find.from == 0 && find.From != \"\" {\n\t\t\terr = ErrTimestampInvalid\n\t\t}\n\n\t\tif err != nil {\n\t\t\tlogger.Error(\"failed to read config\",\n\t\t\t\tzap.String(\"config\", cfg.Test.name),\n\t\t\t\tzap.Error(err),\n\t\t\t\tzap.String(\"query\", find.Query),\n\t\t\t\tzap.String(\"from\", find.From),\n\t\t\t\tzap.Int(\"step\", n),\n\t\t\t)\n\n\t\t\treturn false\n\t\t}\n\n\t\tfind.until = datetime.DateParamToEpoch(find.Until, tz, now, cfg.Test.Precision)\n\t\tif find.until == 0 && find.Until != \"\" {\n\t\t\terr = ErrTimestampInvalid\n\t\t}\n\n\t\tif err != nil {\n\t\t\tlogger.Error(\"failed to read config\",\n\t\t\t\tzap.String(\"config\", cfg.Test.name),\n\t\t\t\tzap.Error(err),\n\t\t\t\tzap.String(\"query\", find.Query),\n\t\t\t\tzap.String(\"until\", find.Until),\n\t\t\t\tzap.Int(\"step\", n),\n\t\t\t)\n\n\t\t\treturn false\n\t\t}\n\n\t\tif find.ErrorRegexp != \"\" {\n\t\t\tfind.errorRegexp = regexp.MustCompile(find.ErrorRegexp)\n\t\t}\n\t}\n\n\tfor n, tags := range cfg.Test.TagsChecks {\n\t\tif tags.Timeout == 0 {\n\t\t\ttags.Timeout = 10 * time.Second\n\t\t}\n\n\t\ttags.from = datetime.DateParamToEpoch(tags.From, tz, now, cfg.Test.Precision)\n\t\tif tags.from == 0 && tags.From != \"\" {\n\t\t\terr = ErrTimestampInvalid\n\t\t}\n\n\t\tif err != nil {\n\t\t\tlogger.Error(\"failed to read config\",\n\t\t\t\tzap.String(\"config\", cfg.Test.name),\n\t\t\t\tzap.Error(err),\n\t\t\t\tzap.String(\"query\", tags.Query),\n\t\t\t\tzap.String(\"from\", tags.From),\n\t\t\t\tzap.Int(\"find\", n),\n\t\t\t)\n\n\t\t\treturn false\n\t\t}\n\n\t\ttags.until = datetime.DateParamToEpoch(tags.Until, tz, now, cfg.Test.Precision)\n\t\tif tags.until == 0 && tags.Until != \"\" {\n\t\t\terr = ErrTimestampInvalid\n\t\t}\n\n\t\tif err != nil {\n\t\t\tlogger.Error(\"failed to read config\",\n\t\t\t\tzap.String(\"config\", cfg.Test.name),\n\t\t\t\tzap.Error(err),\n\t\t\t\tzap.String(\"query\", tags.Query),\n\t\t\t\tzap.String(\"until\", tags.Until),\n\t\t\t\tzap.Int(\"tags\", n),\n\t\t\t\tzap.Bool(\"names\", tags.Names),\n\t\t\t)\n\n\t\t\treturn false\n\t\t}\n\n\t\tif tags.ErrorRegexp != \"\" {\n\t\t\ttags.errorRegexp = regexp.MustCompile(tags.ErrorRegexp)\n\t\t}\n\t}\n\n\tfor n, r := range cfg.Test.RenderChecks {\n\t\tif r.Timeout == 0 {\n\t\t\tr.Timeout = 10 * time.Second\n\t\t}\n\n\t\tr.from = datetime.DateParamToEpoch(r.From, tz, now, cfg.Test.Precision)\n\t\tif r.from == 0 && r.From != \"\" {\n\t\t\terr = ErrTimestampInvalid\n\t\t}\n\n\t\tif err != nil {\n\t\t\tlogger.Error(\"failed to read config\",\n\t\t\t\tzap.String(\"config\", cfg.Test.name),\n\t\t\t\tzap.Error(err),\n\t\t\t\tzap.Strings(\"targets\", r.Targets),\n\t\t\t\tzap.String(\"from\", r.From),\n\t\t\t\tzap.Int(\"render\", n),\n\t\t\t)\n\n\t\t\treturn false\n\t\t}\n\n\t\tr.until = datetime.DateParamToEpoch(r.Until, tz, now, cfg.Test.Precision)\n\t\tif r.until == 0 && r.Until != \"\" {\n\t\t\terr = ErrTimestampInvalid\n\t\t}\n\n\t\tif err != nil {\n\t\t\tlogger.Error(\"failed to read config\",\n\t\t\t\tzap.String(\"config\", cfg.Test.name),\n\t\t\t\tzap.Error(err),\n\t\t\t\tzap.Strings(\"targets\", r.Targets),\n\t\t\t\tzap.String(\"until\", r.Until),\n\t\t\t\tzap.Int(\"render\", n),\n\t\t\t)\n\n\t\t\treturn false\n\t\t}\n\n\t\tif r.ErrorRegexp != \"\" {\n\t\t\tr.errorRegexp = regexp.MustCompile(r.ErrorRegexp)\n\t\t}\n\n\t\tsort.Slice(r.Result, func(i, j int) bool {\n\t\t\treturn r.Result[i].Name < r.Result[j].Name\n\t\t})\n\n\t\tr.result = make([]client.Metric, len(r.Result))\n\t\tfor i, result := range r.Result {\n\t\t\tr.result[i].StartTime = datetime.DateParamToEpoch(result.StartTime, tz, now, cfg.Test.Precision)\n\t\t\tif r.result[i].StartTime == 0 && result.StartTime != \"\" {\n\t\t\t\terr = ErrTimestampInvalid\n\t\t\t}\n\n\t\t\tif err != nil {\n\t\t\t\tlogger.Error(\"failed to read config\",\n\t\t\t\t\tzap.String(\"config\", cfg.Test.name),\n\t\t\t\t\tzap.Error(err),\n\t\t\t\t\tzap.Strings(\"targets\", r.Targets),\n\t\t\t\t\tzap.Int(\"render\", n),\n\t\t\t\t\tzap.String(\"metric\", result.Name),\n\t\t\t\t\tzap.String(\"start\", result.StartTime),\n\t\t\t\t)\n\n\t\t\t\treturn false\n\t\t\t}\n\n\t\t\tr.result[i].StopTime = datetime.DateParamToEpoch(result.StopTime, tz, now, cfg.Test.Precision)\n\t\t\tif r.result[i].StopTime == 0 && result.StopTime != \"\" {\n\t\t\t\terr = ErrTimestampInvalid\n\t\t\t}\n\n\t\t\tif err != nil {\n\t\t\t\tlogger.Error(\"failed to read config\",\n\t\t\t\t\tzap.String(\"config\", cfg.Test.name),\n\t\t\t\t\tzap.Error(err),\n\t\t\t\t\tzap.Strings(\"targets\", r.Targets),\n\t\t\t\t\tzap.Int(\"render\", n),\n\t\t\t\t\tzap.String(\"metric\", result.Name),\n\t\t\t\t\tzap.String(\"stop\", result.StopTime),\n\t\t\t\t)\n\n\t\t\t\treturn false\n\t\t\t}\n\n\t\t\tr.result[i].RequestStartTime = datetime.DateParamToEpoch(result.RequestStartTime, tz, now, cfg.Test.Precision)\n\t\t\tif r.result[i].RequestStartTime == 0 && result.RequestStartTime != \"\" {\n\t\t\t\terr = ErrTimestampInvalid\n\t\t\t}\n\n\t\t\tif err != nil {\n\t\t\t\tlogger.Error(\"failed to read config\",\n\t\t\t\t\tzap.String(\"config\", cfg.Test.name),\n\t\t\t\t\tzap.Error(err),\n\t\t\t\t\tzap.Strings(\"targets\", r.Targets),\n\t\t\t\t\tzap.Int(\"render\", n),\n\t\t\t\t\tzap.String(\"metric\", result.Name),\n\t\t\t\t\tzap.String(\"req_start\", result.RequestStartTime),\n\t\t\t\t)\n\n\t\t\t\treturn false\n\t\t\t}\n\n\t\t\tr.result[i].RequestStopTime = datetime.DateParamToEpoch(result.RequestStopTime, tz, now, cfg.Test.Precision)\n\t\t\tif r.result[i].RequestStopTime == 0 && result.RequestStopTime != \"\" {\n\t\t\t\terr = ErrTimestampInvalid\n\t\t\t}\n\n\t\t\tif err != nil {\n\t\t\t\tlogger.Error(\"failed to read config\",\n\t\t\t\t\tzap.String(\"config\", cfg.Test.name),\n\t\t\t\t\tzap.Error(err),\n\t\t\t\t\tzap.Strings(\"targets\", r.Targets),\n\t\t\t\t\tzap.Int(\"render\", n),\n\t\t\t\t\tzap.String(\"metric\", result.Name),\n\t\t\t\t\tzap.String(\"req_stop\", result.RequestStopTime),\n\t\t\t\t)\n\n\t\t\t\treturn false\n\t\t\t}\n\n\t\t\tr.result[i].StepTime = result.StepTime\n\t\t\tr.result[i].Name = result.Name\n\t\t\tr.result[i].PathExpression = result.PathExpression\n\t\t\tr.result[i].ConsolidationFunc = result.ConsolidationFunc\n\t\t\tr.result[i].XFilesFactor = result.XFilesFactor\n\t\t\tr.result[i].HighPrecisionTimestamps = result.HighPrecisionTimestamps\n\t\t\tr.result[i].AppliedFunctions = result.AppliedFunctions\n\t\t\tr.result[i].Values = result.Values\n\t\t}\n\t}\n\n\treturn true\n}\n\nfunc loadConfig(config string, rootDir string) (*MainConfig, error) {\n\td, err := os.ReadFile(config)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tconfShort := strings.ReplaceAll(config, rootDir+\"/\", \"\")\n\n\tvar cfg = &MainConfig{}\n\tif err := toml.Unmarshal(d, cfg); err != nil {\n\t\treturn nil, err\n\t}\n\n\tcfg.Test.name = confShort\n\tcfg.Test.dir = path.Dir(config)\n\n\tif cfg.Test == nil {\n\t\treturn nil, ErrNoTest\n\t}\n\n\tcfg.Test.chVersions = make(map[string]bool)\n\tfor i := range cfg.Test.Clickhouse {\n\t\tif err := cfg.Test.Clickhouse[i].CheckConfig(rootDir); err == nil {\n\t\t\tcfg.Test.chVersions[cfg.Test.Clickhouse[i].Key()] = true\n\t\t} else {\n\t\t\treturn nil, fmt.Errorf(\"[%d] %s\", i, err.Error())\n\t\t}\n\t}\n\n\treturn cfg, nil\n}\n" }, { "path": "cmd/e2e-test/errors.go", "content": "package main\n\nimport \"errors\"\n\nvar (\n\tErrTimestampInvalid = errors.New(\"invalid timestamp\")\n\tErrNoTest = errors.New(\"no test section\")\n\tErrNoSetDir = errors.New(\"dir not set\")\n)\n" }, { "path": "cmd/e2e-test/graphite-clickhouse.go", "content": "package main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"syscall\"\n\t\"text/template\"\n\n\t\"github.com/msaf1980/go-stringutils\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/client\"\n)\n\ntype GraphiteClickhouse struct {\n\tBinary string `toml:\"binary\"`\n\tConfigTpl string `toml:\"template\"`\n\tTestDir string `toml:\"-\"`\n\n\tTZ string `toml:\"tz\"` // override timezone\n\n\tstoreDir string `toml:\"-\"`\n\tconfigFile string `toml:\"-\"`\n\taddress string `toml:\"-\"`\n\tcmd *exec.Cmd `toml:\"-\"`\n}\n\nfunc (c *GraphiteClickhouse) Start(testDir, chURL, chProxyURL, chTLSURL string) error {\n\tif c.cmd != nil {\n\t\treturn errors.New(\"carbon-clickhouse already started\")\n\t}\n\n\tif len(c.Binary) == 0 {\n\t\tc.Binary = \"./graphite-clickhouse\"\n\t}\n\n\tif len(c.ConfigTpl) == 0 {\n\t\treturn errors.New(\"graphite-clickhouse config template not set\")\n\t}\n\n\tvar err error\n\n\tc.storeDir, err = os.MkdirTemp(\"\", \"graphite-clickhouse\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tc.address, err = getFreeTCPPort(\"\")\n\tif err != nil {\n\t\tc.Cleanup()\n\t\treturn err\n\t}\n\n\tc.TestDir, err = filepath.Abs(testDir)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tname := filepath.Base(c.ConfigTpl)\n\n\ttmpl, err := template.New(name).ParseFiles(path.Join(testDir, c.ConfigTpl))\n\tif err != nil {\n\t\tc.Cleanup()\n\t\treturn err\n\t}\n\n\tparam := struct {\n\t\tCLICKHOUSE_URL string\n\t\tCLICKHOUSE_TLS_URL string\n\t\tPROXY_URL string\n\t\tGCH_ADDR string\n\t\tGCH_DIR string\n\t\tTEST_DIR string\n\t}{\n\t\tCLICKHOUSE_URL: chURL,\n\t\tCLICKHOUSE_TLS_URL: chTLSURL,\n\t\tPROXY_URL: chProxyURL,\n\t\tGCH_ADDR: c.address,\n\t\tGCH_DIR: c.storeDir,\n\t\tTEST_DIR: c.TestDir,\n\t}\n\n\tc.configFile = path.Join(c.storeDir, \"graphite-clickhouse.conf\")\n\tf, err := os.OpenFile(c.configFile, os.O_WRONLY|os.O_CREATE, 0644)\n\n\tif err != nil {\n\t\tc.Cleanup()\n\t\treturn err\n\t}\n\n\terr = tmpl.ExecuteTemplate(f, name, param)\n\tif err != nil {\n\t\tc.Cleanup()\n\t\treturn err\n\t}\n\n\tc.cmd = exec.Command(c.Binary, \"-config\", c.configFile)\n\tc.cmd.Stdout = os.Stdout\n\tc.cmd.Stderr = os.Stderr\n\n\tif c.TZ != \"\" {\n\t\tc.cmd.Env = append(c.cmd.Env, \"TZ=\"+c.TZ)\n\t}\n\n\terr = c.cmd.Start()\n\tif err != nil {\n\t\tc.Cleanup()\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc (c *GraphiteClickhouse) Alive() bool {\n\tif c.cmd == nil {\n\t\treturn false\n\t}\n\n\t_, _, _, err := client.MetricsFind(http.DefaultClient, \"http://\"+c.address+\"/alive\", client.FormatDefault, \"NonExistentTarget\", 0, 0)\n\n\treturn err == nil\n}\n\nfunc (c *GraphiteClickhouse) Stop(cleanup bool) error {\n\tif cleanup {\n\t\tdefer c.Cleanup()\n\t}\n\n\tif c.cmd == nil {\n\t\treturn nil\n\t}\n\n\tvar err error\n\tif err = c.cmd.Process.Kill(); err == nil {\n\t\tif err = c.cmd.Wait(); err != nil {\n\t\t\tif exitErr, ok := err.(*exec.ExitError); ok {\n\t\t\t\tif status, ok := exitErr.Sys().(syscall.WaitStatus); ok {\n\t\t\t\t\tec := status.ExitStatus()\n\t\t\t\t\tif ec == 0 || ec == -1 {\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn err\n}\n\nfunc (c *GraphiteClickhouse) Cleanup() {\n\tif len(c.storeDir) > 0 {\n\t\tos.RemoveAll(c.storeDir)\n\t\tc.storeDir = \"\"\n\t\tc.cmd = nil\n\t}\n}\n\nfunc (c *GraphiteClickhouse) URL() string {\n\treturn \"http://\" + c.address\n}\n\nfunc (c *GraphiteClickhouse) Cmd() string {\n\treturn strings.Join(c.cmd.Args, \" \")\n}\n\nfunc (c *GraphiteClickhouse) Grep(s string) {\n\tout, _ := exec.Command(\"grep\", \"-F\", s, c.storeDir+\"/graphite-clickhouse.log\").Output()\n\tfmt.Fprintf(os.Stderr, \"GREP %s\", stringutils.UnsafeString(out))\n}\n" }, { "path": "cmd/e2e-test/main.go", "content": "package main\n\nimport (\n\t\"flag\"\n\t\"log\"\n\t\"os\"\n\t\"path\"\n\t\"runtime\"\n\t\"time\"\n\n\t\"go.uber.org/zap\"\n)\n\ntype MainConfig struct {\n\tTest *TestSchema `toml:\"test\"`\n}\n\nfunc IsDir(filename string) (bool, error) {\n\tinfo, err := os.Stat(filename)\n\tif os.IsNotExist(err) {\n\t\treturn false, nil\n\t} else if err != nil {\n\t\treturn false, err\n\t}\n\n\treturn info.IsDir(), nil\n}\n\nfunc expandDir(dirname string, paths *[]string) error {\n\tfiles, err := os.ReadDir(dirname)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor _, file := range files {\n\t\tif file.IsDir() {\n\t\t\tif err = expandDir(path.Join(dirname, file.Name()), paths); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t} else {\n\t\t\text := path.Ext(file.Name())\n\t\t\tif ext == \".toml\" {\n\t\t\t\t*paths = append(*paths, path.Join(dirname, file.Name()))\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc expandFilename(filename string, paths *[]string) error {\n\tif len(filename) == 0 {\n\t\treturn nil\n\t}\n\n\tisDir, err := IsDir(filename)\n\tif err == nil {\n\t\tif isDir {\n\t\t\tif err = expandDir(filename, paths); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t} else {\n\t\t\t*paths = append(*paths, filename)\n\t\t}\n\t}\n\n\treturn err\n}\n\nfunc main() {\n\t_, filename, _, _ := runtime.Caller(0)\n\trootDir := path.Dir(path.Dir(path.Dir(filename))) // carbon-clickhouse repositiry root dir\n\n\tconfig := flag.String(\"config\", \"\", \"toml configuration file or dir where toml files is searched (recursieve)\")\n\tverbose := flag.Bool(\"verbose\", false, \"verbose\")\n\tbreakOnError := flag.Bool(\"break\", false, \"break and wait user response if request failed\")\n\tabortOnError := flag.Bool(\"abort\", false, \"abort tests if test failed\")\n\tcleanup := flag.Bool(\"cleanup\", false, \"delete containers if exists before start\")\n\trmi := flag.Bool(\"rmi\", false, \"delete images after test end (for low space usage))\")\n\tflag.Parse()\n\n\tlogger, err := zap.NewProduction()\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tDockerBinary = os.Getenv(\"DOCKER_E2E\")\n\tif DockerBinary == \"\" {\n\t\tDockerBinary = \"docker\"\n\t}\n\n\tif *cleanup {\n\t\tif exist, _ := containerExist(CchContainerName); exist {\n\t\t\tif ok, out := containerRemove(CchContainerName); !ok {\n\t\t\t\tlogger.Fatal(\"failed to cleanup\",\n\t\t\t\t\tzap.String(\"container\", CchContainerName),\n\t\t\t\t\tzap.String(\"error\", out),\n\t\t\t\t)\n\t\t\t}\n\t\t}\n\n\t\tif exist, _ := containerExist(ClickhouseContainerName); exist {\n\t\t\tif ok, out := containerRemove(ClickhouseContainerName); !ok {\n\t\t\t\tlogger.Fatal(\"failed to cleanup\",\n\t\t\t\t\tzap.String(\"container\", ClickhouseContainerName),\n\t\t\t\t\tzap.String(\"error\", out),\n\t\t\t\t)\n\t\t\t}\n\t\t}\n\n\t\tif *config == \"\" {\n\t\t\treturn\n\t\t}\n\t}\n\n\tvar allConfigs []string\n\n\terr = expandFilename(*config, &allConfigs)\n\tif err != nil {\n\t\tlogger.Fatal(\n\t\t\t\"config\",\n\t\t\tzap.Error(err),\n\t\t)\n\t}\n\n\tif len(allConfigs) == 0 {\n\t\tlogger.Fatal(\"config should be non-null\")\n\t}\n\n\tchVersions := make(map[string]Clickhouse)\n\tconfigs := make([]*MainConfig, 0, len(allConfigs))\n\n\tfor _, config := range allConfigs {\n\t\tcfg, err := loadConfig(config, rootDir)\n\t\tif err == nil {\n\t\t\tconfigs = append(configs, cfg)\n\n\t\t\tfor _, ch := range cfg.Test.Clickhouse {\n\t\t\t\tchVersions[ch.Key()] = ch\n\t\t\t}\n\n\t\t\tnow := time.Now()\n\t\t\tif !initTest(cfg, rootDir, now, *verbose, *breakOnError, logger) {\n\t\t\t\tos.Exit(1)\n\t\t\t}\n\t\t} else {\n\t\t\tlogger.Error(\"failed to read config\",\n\t\t\t\tzap.String(\"config\", config),\n\t\t\t\tzap.Error(err),\n\t\t\t\tzap.Any(\"decode\", cfg),\n\t\t\t)\n\t\t}\n\t}\n\n\tfailed := 0\n\ttotal := 0\n\tverifyCount := 0\n\tverifyFailed := 0\n\n\t_, err = cmdExec(DockerBinary, \"network\", \"inspect\", DockerNetwork)\n\tif err != nil {\n\t\tout, err := cmdExec(DockerBinary, \"network\", \"create\", DockerNetwork)\n\t\tif err != nil {\n\t\t\tlogger.Error(\"failed to create network\",\n\t\t\t\tzap.Error(err),\n\t\t\t\tzap.String(\"out\", out),\n\t\t\t)\n\t\t\tos.Exit(1)\n\t\t}\n\t}\n\n\tfor chVersion := range chVersions {\n\t\tch := chVersions[chVersion]\n\n\t\tif exist, out := containerExist(ClickhouseContainerName); exist {\n\t\t\tlogger.Error(\"clickhouse already exist\",\n\t\t\t\tzap.String(\"container\", ClickhouseContainerName),\n\t\t\t\tzap.String(\"out\", out),\n\t\t\t)\n\t\t\tos.Exit(1)\n\t\t}\n\n\t\tlogger.Info(\"clickhouse\",\n\t\t\tzap.Any(\"clickhouse image\", ch.DockerImage),\n\t\t\tzap.Any(\"clickhouse version\", ch.Version),\n\t\t\tzap.String(\"clickhouse config\", ch.Dir),\n\t\t\tzap.String(\"tz\", ch.TZ),\n\t\t)\n\n\t\tif clickhouseStart(&ch, logger) {\n\t\t\ttime.Sleep(100 * time.Millisecond)\n\n\t\t\tfor i := 200; i < 3000; i += 200 {\n\t\t\t\tif ch.Alive() {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\n\t\t\t\ttime.Sleep(time.Duration(i) * time.Millisecond)\n\t\t\t}\n\n\t\t\tif !ch.Alive() {\n\t\t\t\tlogger.Error(\"starting clickhouse\",\n\t\t\t\t\tzap.Any(\"clickhouse version\", ch.Version),\n\t\t\t\t\tzap.String(\"clickhouse config\", ch.Dir),\n\t\t\t\t\tzap.String(\"error\", \"clickhouse is down\"),\n\t\t\t\t)\n\n\t\t\t\tfailed++\n\t\t\t\ttotal++\n\t\t\t\tverifyCount++\n\t\t\t\tverifyFailed++\n\t\t\t} else {\n\t\t\t\tfor _, config := range configs {\n\t\t\t\t\tif config.Test.chVersions[chVersion] {\n\t\t\t\t\t\tnow := time.Now()\n\t\t\t\t\t\tif initTest(config, rootDir, now, *verbose, *breakOnError, logger) {\n\t\t\t\t\t\t\ttestFailed, testTotal, vCount, vFailed := runTest(config, &ch, rootDir, now, *verbose, *breakOnError, logger)\n\t\t\t\t\t\t\tfailed += testFailed\n\t\t\t\t\t\t\ttotal += testTotal\n\t\t\t\t\t\t\tverifyCount += vCount\n\t\t\t\t\t\t\tverifyFailed += vFailed\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tfailed++\n\t\t\t\t\t\t\ttotal++\n\t\t\t\t\t\t\tverifyCount++\n\t\t\t\t\t\t\tverifyFailed++\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif !clickhouseStop(&ch, logger) {\n\t\t\t\tfailed++\n\t\t\t\tverifyFailed++\n\t\t\t}\n\t\t} else {\n\t\t\tfailed++\n\t\t\ttotal++\n\t\t\tverifyCount++\n\t\t\tverifyFailed++\n\t\t}\n\n\t\tif *rmi {\n\t\t\tif success, out := imageDelete(ch.DockerImage, ch.Version); !success {\n\t\t\t\tlogger.Error(\"docker remove image\",\n\t\t\t\t\tzap.Any(\"clickhouse version\", ch.Version),\n\t\t\t\t\tzap.String(\"clickhouse config\", ch.Dir),\n\t\t\t\t\tzap.String(\"out\", out),\n\t\t\t\t)\n\t\t\t}\n\t\t}\n\n\t\tif *abortOnError && failed > 0 {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tif failed > 0 {\n\t\tlogger.Error(\"tests ended\",\n\t\t\tzap.String(\"status\", \"failed\"),\n\t\t\tzap.Int(\"test_count\", total),\n\t\t\tzap.Int(\"test_failed\", failed),\n\t\t\tzap.Int(\"checks\", verifyCount),\n\t\t\tzap.Int(\"failed\", verifyFailed),\n\t\t\tzap.Int(\"configs\", len(allConfigs)),\n\t\t)\n\t\tos.Exit(1)\n\t} else {\n\t\tlogger.Info(\"tests ended\",\n\t\t\tzap.String(\"status\", \"success\"),\n\t\t\tzap.Int(\"test_count\", total),\n\t\t\tzap.Int(\"test_failed\", failed),\n\t\t\tzap.Int(\"checks\", verifyCount),\n\t\t\tzap.Int(\"failed\", verifyFailed),\n\t\t\tzap.Int(\"configs\", len(allConfigs)),\n\t\t)\n\t}\n}\n" }, { "path": "cmd/e2e-test/rproxy.go", "content": "package main\n\nimport (\n\t\"errors\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"net/http/httputil\"\n\t\"net/url\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/pkg/dry\"\n)\n\ntype AtomicDuration struct {\n\tval int64\n}\n\nfunc (d *AtomicDuration) Store(duration time.Duration) {\n\tatomic.StoreInt64(&d.val, duration.Nanoseconds())\n}\n\nfunc (d *AtomicDuration) Load() time.Duration {\n\treturn time.Duration(atomic.LoadInt64(&d.val))\n}\n\nfunc (d *AtomicDuration) MarshalText() ([]byte, error) {\n\ts := d.Load().String()\n\treturn dry.UnsafeStringBytes(&s), nil\n}\n\nfunc (d *AtomicDuration) UnmarshalText(b []byte) error {\n\tval, err := time.ParseDuration(dry.UnsafeString(b))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\td.Store(val)\n\n\treturn nil\n}\n\ntype HttpReverseProxy struct {\n\tDelay AtomicDuration `toml:\"delay\"`\n\tBreakWithStatusCode int64 `toml:\"break_with_status_code\"`\n\n\tsrv *httptest.Server\n\tremote *url.URL\n\twg sync.WaitGroup\n}\n\nfunc (p *HttpReverseProxy) Start(remoteURL string) (err error) {\n\tif p.srv != nil {\n\t\terr = errors.New(\"reverse proxy already started\")\n\t\treturn\n\t}\n\n\tif p.BreakWithStatusCode < 0 {\n\t\tp.BreakWithStatusCode = 0\n\t}\n\n\tif p.remote, err = url.Parse(remoteURL); err != nil {\n\t\terr = errors.New(\"reverse proxy already started\")\n\t\treturn\n\t}\n\n\tp.srv = httptest.NewUnstartedServer(p)\n\n\tp.wg.Add(1)\n\n\tgo func() {\n\t\tdefer p.wg.Done()\n\n\t\tp.srv.Start()\n\t}()\n\n\treturn\n}\n\nfunc (p *HttpReverseProxy) Stop() {\n\tif p.srv == nil {\n\t\treturn\n\t}\n\n\tp.srv.CloseClientConnections()\n\tp.srv.Close()\n\tp.wg.Wait()\n\tp.srv = nil\n}\n\nfunc (p *HttpReverseProxy) URL() string {\n\treturn p.srv.URL\n}\n\nfunc (p *HttpReverseProxy) SetDelay(delay time.Duration) {\n\tp.Delay.Store(delay)\n}\n\nfunc (p *HttpReverseProxy) GetDelay() time.Duration {\n\treturn p.Delay.Load()\n}\n\nfunc (p *HttpReverseProxy) SetBreakStatusCode(statusCode int) {\n\tatomic.StoreInt64(&p.BreakWithStatusCode, int64(statusCode))\n}\n\nfunc (p *HttpReverseProxy) GetBreakStatusCode() int {\n\treturn int(atomic.LoadInt64(&p.BreakWithStatusCode))\n}\n\nfunc (p *HttpReverseProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {\n\tr.Host = p.remote.Host\n\n\tdelay := p.GetDelay()\n\tif delay != 0 {\n\t\ttime.Sleep(delay)\n\t}\n\n\tbreakWithStatusCode := p.GetBreakStatusCode()\n\tif breakWithStatusCode != 0 {\n\t\thttp.Error(w, \"\", breakWithStatusCode)\n\t\treturn\n\t}\n\n\tproxy := httputil.NewSingleHostReverseProxy(p.remote)\n\tproxy.ServeHTTP(w, r)\n}\n" }, { "path": "cmd/e2e-test/utils.go", "content": "package main\n\nimport \"os/exec\"\n\nfunc cmdExec(programm string, args ...string) (string, error) {\n\tcmd := exec.Command(programm, args...)\n\tout, err := cmd.CombinedOutput()\n\n\treturn string(out), err\n}\n" }, { "path": "cmd/graphite-clickhouse-client/main.go", "content": "package main\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"os\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/go-graphite/protocol/carbonapi_v3_pb\"\n\t\"github.com/lomik/graphite-clickhouse/helper/client\"\n\t\"github.com/lomik/graphite-clickhouse/helper/datetime\"\n)\n\ntype StringSlice []string\n\nfunc (u *StringSlice) Set(value string) error {\n\t*u = append(*u, value)\n\treturn nil\n}\n\nfunc (u *StringSlice) String() string {\n\treturn \"[ \" + strings.Join(*u, \", \") + \" ]\"\n}\n\nfunc (u *StringSlice) Type() string {\n\treturn \"[]string\"\n}\n\nfunc main() {\n\taddress := flag.String(\"address\", \"http://127.0.0.1:9090\", \"Address of graphite-clickhouse server\")\n\tfromStr := flag.String(\"from\", \"0\", \"from\")\n\tuntilStr := flag.String(\"until\", \"\", \"until\")\n\tmaxDataPointsStr := flag.String(\"maxDataPoints\", \"1048576\", \"Maximum amount of datapoints in response\")\n\n\tmetricsFind := flag.String(\"find\", \"\", \"Query for /metrics/find/ , valid formats are carbonapi_v3_pb. protobuf, pickle\")\n\n\ttagsValues := flag.String(\"tags_values\", \"\", \"Query for /tags/autoComplete/values (with query like 'searchTag[=valuePrefix];tag1=value1;tag2=~value*' or '<>' for empty)\")\n\ttagsNames := flag.String(\"tags_names\", \"\", \"Query for /tags/autoComplete/tags (with query like '[tagPrefix];tag1=value1;tag2=~value*[' or '<>' for empty)\")\n\tlimit := flag.Uint64(\"limit\", 0, \"limit for some queries (tags_values, tags_values)\")\n\n\ttimeout := flag.Duration(\"timeout\", time.Minute, \"request timeout\")\n\n\tvar targets StringSlice\n\n\tflag.Var(&targets, \"target\", \"Target for /render\")\n\n\tformat := client.FormatDefault\n\tflag.Var(&format, \"format\", fmt.Sprintf(\"Response format %v\", client.FormatTypes()))\n\n\tflag.Parse()\n\n\tec := 0\n\n\ttz, err := datetime.Timezone(\"\")\n\tif err != nil {\n\t\tfmt.Printf(\"can't get timezone: %s\\n\", err.Error())\n\t\tos.Exit(1)\n\t}\n\n\tnow := time.Now()\n\n\tfrom := datetime.DateParamToEpoch(*fromStr, tz, now, 0)\n\tif from == 0 && len(targets) > 0 {\n\t\tfmt.Printf(\"invalid from: %s\\n\", *fromStr)\n\t\tos.Exit(1)\n\t}\n\n\tvar until int64\n\n\tif *untilStr == \"\" && len(targets) > 0 {\n\t\t*untilStr = \"now\"\n\t}\n\n\tuntil = datetime.DateParamToEpoch(*untilStr, tz, now, 0)\n\tif until == 0 && len(targets) > 0 {\n\t\tfmt.Printf(\"invalid until: %s\\n\", *untilStr)\n\t\tos.Exit(1)\n\t}\n\n\tmaxDataPoints, err := strconv.ParseInt(*maxDataPointsStr, 10, 64)\n\tif err != nil {\n\t\tfmt.Printf(\"invalid maxDataPoints: %s\\n\", *maxDataPointsStr)\n\t\tos.Exit(1)\n\t}\n\n\thttpClient := http.Client{\n\t\tTimeout: *timeout,\n\t}\n\n\tif *metricsFind != \"\" {\n\t\tformatFind := format\n\t\tif formatFind == client.FormatDefault {\n\t\t\tformatFind = client.FormatPb_v3\n\t\t}\n\n\t\tqueryRaw, r, respHeader, err := client.MetricsFind(&httpClient, *address, formatFind, *metricsFind, from, until)\n\t\tif respHeader != nil {\n\t\t\tfmt.Printf(\"Responce header: %+v\\n\", respHeader)\n\t\t}\n\n\t\tfmt.Print(\"'\")\n\t\tfmt.Print(queryRaw)\n\t\tfmt.Print(\"' = \")\n\n\t\tif err == nil {\n\t\t\tif len(r) > 0 {\n\t\t\t\tfmt.Println(\"[\")\n\n\t\t\t\tfor i, m := range r {\n\t\t\t\t\tfmt.Printf(\" { Path: '%s', IsLeaf: %v }\", m.Path, m.IsLeaf)\n\n\t\t\t\t\tif i < len(r)-1 {\n\t\t\t\t\t\tfmt.Println(\",\")\n\t\t\t\t\t} else {\n\t\t\t\t\t\tfmt.Println(\"\")\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tfmt.Println(\"]\")\n\t\t\t} else {\n\t\t\t\tfmt.Println(\"[]\")\n\t\t\t}\n\t\t} else {\n\t\t\tec = 1\n\n\t\t\tfmt.Printf(\"'%s'\\n\", strings.TrimRight(err.Error(), \"\\n\"))\n\t\t}\n\t}\n\n\tif *tagsValues != \"\" {\n\t\tformatTags := format\n\t\tif formatTags == client.FormatDefault {\n\t\t\tformatTags = client.FormatJSON\n\t\t}\n\n\t\tqueryRaw, r, respHeader, err := client.TagsValues(&httpClient, *address, formatTags, *tagsValues, *limit, from, until)\n\t\tif respHeader != nil {\n\t\t\tfmt.Printf(\"Responce header: %+v\\n\", respHeader)\n\t\t}\n\n\t\tfmt.Print(\"'\")\n\t\tfmt.Print(queryRaw)\n\t\tfmt.Print(\"' = \")\n\n\t\tif err == nil {\n\t\t\tif len(r) > 0 {\n\t\t\t\tfmt.Println(\"[\")\n\n\t\t\t\tfor i, v := range r {\n\t\t\t\t\tfmt.Printf(\" { Value: '%s' }\", v)\n\n\t\t\t\t\tif i < len(r)-1 {\n\t\t\t\t\t\tfmt.Println(\",\")\n\t\t\t\t\t} else {\n\t\t\t\t\t\tfmt.Println(\"\")\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tfmt.Println(\"]\")\n\t\t\t} else {\n\t\t\t\tfmt.Println(\"[]\")\n\t\t\t}\n\t\t} else {\n\t\t\tec = 1\n\n\t\t\tfmt.Printf(\"'%s'\\n\", strings.TrimRight(err.Error(), \"\\n\"))\n\t\t}\n\t}\n\n\tif *tagsNames != \"\" {\n\t\tformatTags := format\n\t\tif formatTags == client.FormatDefault {\n\t\t\tformatTags = client.FormatJSON\n\t\t}\n\n\t\tqueryRaw, r, respHeader, err := client.TagsNames(&httpClient, *address, formatTags, *tagsNames, *limit, from, until)\n\t\tif respHeader != nil {\n\t\t\tfmt.Printf(\"Responce header: %+v\\n\", respHeader)\n\t\t}\n\n\t\tfmt.Print(\"'\")\n\t\tfmt.Print(queryRaw)\n\t\tfmt.Print(\"' = \")\n\n\t\tif err == nil {\n\t\t\tif len(r) > 0 {\n\t\t\t\tfmt.Println(\"[\")\n\n\t\t\t\tfor i, v := range r {\n\t\t\t\t\tfmt.Printf(\" { Tag: '%s' }\", v)\n\n\t\t\t\t\tif i < len(r)-1 {\n\t\t\t\t\t\tfmt.Println(\",\")\n\t\t\t\t\t} else {\n\t\t\t\t\t\tfmt.Println(\"\")\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tfmt.Println(\"]\")\n\t\t\t} else {\n\t\t\t\tfmt.Println(\"[]\")\n\t\t\t}\n\t\t} else {\n\t\t\tec = 1\n\n\t\t\tfmt.Printf(\"'%s'\\n\", strings.TrimRight(err.Error(), \"\\n\"))\n\t\t}\n\t}\n\n\tif len(targets) > 0 {\n\t\tformatRender := format\n\t\tif formatRender == client.FormatDefault {\n\t\t\tformatRender = client.FormatPb_v3\n\t\t}\n\n\t\tqueryRaw, r, respHeader, err := client.Render(&httpClient, *address, formatRender, targets, []*carbonapi_v3_pb.FilteringFunction{}, maxDataPoints, from, until)\n\t\tif respHeader != nil {\n\t\t\tfmt.Printf(\"Responce header: %+v\\n\", respHeader)\n\t\t}\n\n\t\tfmt.Print(\"'\")\n\t\tfmt.Print(queryRaw)\n\t\tfmt.Print(\"' = \")\n\n\t\tif err == nil {\n\t\t\tif len(r) > 0 {\n\t\t\t\tfmt.Println(\"[\")\n\n\t\t\t\tfor i, m := range r {\n\t\t\t\t\tfmt.Println(\" {\")\n\t\t\t\t\tfmt.Printf(\" Name: '%s', PathExpression: '%v',\\n\", m.Name, m.PathExpression)\n\t\t\t\t\tfmt.Printf(\" ConsolidationFunc: %s, XFilesFactor: %f, AppliedFunctions: %s,\\n\", m.ConsolidationFunc, m.XFilesFactor, m.AppliedFunctions)\n\t\t\t\t\tfmt.Printf(\" Start: %d, Stop: %d, Step: %d, RequestStart: %d, RequestStop: %d,\\n\", m.StartTime, m.StopTime, m.StepTime, m.RequestStartTime, m.RequestStopTime)\n\t\t\t\t\tfmt.Printf(\" Values: %+v\\n\", m.Values)\n\n\t\t\t\t\tif i == len(r) {\n\t\t\t\t\t\tfmt.Println(\" }\")\n\t\t\t\t\t} else {\n\t\t\t\t\t\tfmt.Println(\" },\")\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tfmt.Println(\"]\")\n\t\t\t} else {\n\t\t\t\tfmt.Println(\"[]\")\n\t\t\t}\n\t\t} else {\n\t\t\tec = 1\n\n\t\t\tfmt.Printf(\"'%s'\\n\", strings.TrimRight(err.Error(), \"\\n\"))\n\t\t}\n\t}\n\n\tos.Exit(ec)\n}\n" }, { "path": "config/.gitignore", "content": "tests_tmp/\n" }, { "path": "config/config.go", "content": "package config\n\nimport (\n\t\"bytes\"\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"net\"\n\t\"net/url\"\n\t\"os\"\n\t\"reflect\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/cactus/go-statsd-client/v5/statsd\"\n\t\"github.com/lomik/carbon-clickhouse/helper/config\"\n\t\"github.com/msaf1980/go-metrics/graphite\"\n\t\"github.com/msaf1980/go-timeutils/duration\"\n\ttoml \"github.com/pelletier/go-toml\"\n\t\"github.com/pkg/errors\"\n\t\"go.uber.org/zap\"\n\n\t\"github.com/lomik/zapwriter\"\n\n\t\"github.com/lomik/graphite-clickhouse/cache\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/helper/date\"\n\t\"github.com/lomik/graphite-clickhouse/helper/rollup\"\n\t\"github.com/lomik/graphite-clickhouse/limiter\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n)\n\ntype SDType uint8\n\nconst (\n\tSDNone SDType = iota\n\tSDNginx // https://github.com/weibocom/nginx-upsync-module\n)\n\nvar sdTypeStrings map[SDType]string = map[SDType]string{SDNone: \"\", SDNginx: \"nginx\"}\n\nfunc (a *SDType) Set(value string) error {\n\tswitch value {\n\tcase \"nginx\":\n\t\t*a = SDNginx\n\tcase \"\", \"0\":\n\t\t*a = SDNone\n\tdefault:\n\t\treturn fmt.Errorf(\"invalid sd type %q\", value)\n\t}\n\n\treturn nil\n}\n\nfunc (a *SDType) UnmarshalText(data []byte) error {\n\treturn a.Set(string(data))\n}\n\nfunc (a *SDType) MarshalText() ([]byte, error) {\n\treturn []byte(a.String()), nil\n}\n\nfunc (a *SDType) UnmarshalJSON(data []byte) error {\n\treturn a.Set(string(data))\n}\n\nfunc (a *SDType) MarshalJSON() ([]byte, error) {\n\treturn []byte(a.String()), nil\n}\n\nfunc (a *SDType) String() string {\n\treturn sdTypeStrings[*a]\n}\n\nfunc (a *SDType) Type() string {\n\treturn \"service_discovery_type\"\n}\n\n// Cache config\ntype CacheConfig struct {\n\tType string `toml:\"type\" json:\"type\" comment:\"cache type\"`\n\tSize int `toml:\"size-mb\" json:\"size-mb\" comment:\"cache size\"`\n\tMemcachedServers []string `toml:\"memcached-servers\" json:\"memcached-servers\" comment:\"memcached servers\"`\n\tDefaultTimeoutSec int32 `toml:\"default-timeout\" json:\"default-timeout\" comment:\"default cache ttl\"`\n\tDefaultTimeoutStr string `toml:\"-\" json:\"-\"`\n\tShortTimeoutSec int32 `toml:\"short-timeout\" json:\"short-timeout\" comment:\"short-time cache ttl\"`\n\tShortTimeoutStr string `toml:\"-\" json:\"-\"`\n\tFindTimeoutSec int32 `toml:\"find-timeout\" json:\"find-timeout\" comment:\"finder/tags autocompleter cache ttl\"`\n\tShortDuration time.Duration `toml:\"short-duration\" json:\"short-duration\" comment:\"maximum diration, used with short_timeout\"`\n\tShortUntilOffsetSec int64 `toml:\"short-offset\" json:\"short-offset\" comment:\"offset beetween now and until for select short cache timeout\"`\n}\n\n// Common config\ntype Common struct {\n\tListen string `toml:\"listen\" json:\"listen\" comment:\"general listener\"`\n\tPprofListen string `toml:\"pprof-listen\" json:\"pprof-listen\" comment:\"listener to serve /debug/pprof requests. '-pprof' argument overrides it\"`\n\tMaxCPU int `toml:\"max-cpu\" json:\"max-cpu\"`\n\tMaxMetricsInFindAnswer int `toml:\"max-metrics-in-find-answer\" json:\"max-metrics-in-find-answer\" comment:\"limit number of results from find query, 0=unlimited\"`\n\tMaxMetricsPerTarget int `toml:\"max-metrics-per-target\" json:\"max-metrics-per-target\" comment:\"limit numbers of queried metrics per target in /render requests, 0 or negative = unlimited\"`\n\tAppendEmptySeries bool `toml:\"append-empty-series\" json:\"append-empty-series\" comment:\"if true, always return points for all metrics, replacing empty results with list of NaN\"`\n\tTargetBlacklist []string `toml:\"target-blacklist\" json:\"target-blacklist\" comment:\"daemon returns empty response if query matches any of regular expressions\" commented:\"true\"`\n\tBlacklist []*regexp.Regexp `toml:\"-\" json:\"-\"` // compiled TargetBlacklist\n\tMemoryReturnInterval time.Duration `toml:\"memory-return-interval\" json:\"memory-return-interval\" comment:\"daemon will return the freed memory to the OS when it>0\"`\n\tHeadersToLog []string `toml:\"headers-to-log\" json:\"headers-to-log\" comment:\"additional request headers to log\"`\n\n\tBaseWeight int `toml:\"base_weight\" json:\"base_weight\" comment:\"service discovery base weight (on idle)\"`\n\tDegragedMultiply float64 `toml:\"degraged-multiply\" json:\"degraged-multiply\" comment:\"service discovery degraded load avg multiplier (if normalized load avg > degraged_load_avg) (default 4.0)\"`\n\tDegragedLoad float64 `toml:\"degraged-load-avg\" json:\"degraged-load-avg\" comment:\"service discovery normilized load avg degraded point (default 1.0)\"`\n\tSDType SDType `toml:\"service-discovery-type\" json:\"service-discovery-type\" comment:\"service discovery type\"`\n\tSD string `toml:\"service-discovery\" json:\"service-discovery\" comment:\"service discovery address (consul)\"`\n\tSDNamespace string `toml:\"service-discovery-ns\" json:\"service-discovery-ns\" comment:\"service discovery namespace (graphite by default)\"`\n\tSDDc []string `toml:\"service-discovery-ds\" json:\"service-discovery-ds\" comment:\"service discovery datacenters (first - is primary, in other register as backup)\"`\n\tSDExpire time.Duration `toml:\"service-discovery-expire\" json:\"service-discovery-expire\" comment:\"service discovery expire duration for cleanup (minimum is 24h, if enabled)\"`\n\n\tFindCacheConfig CacheConfig `toml:\"find-cache\" json:\"find-cache\" comment:\"find/tags cache config\"`\n\n\tFindCache cache.BytesCache `toml:\"-\" json:\"-\"`\n}\n\n// FeatureFlags contains feature flags that significantly change how gch responds to some requests\ntype FeatureFlags struct {\n\tUseCarbonBehavior bool `toml:\"use-carbon-behaviour\" json:\"use-carbon-behaviour\" comment:\"if true, prefers carbon's behaviour on how tags are treated\"`\n\tDontMatchMissingTags bool `toml:\"dont-match-missing-tags\" json:\"dont-match-missing-tags\" comment:\"if true, seriesByTag terms containing '!=' or '!=~' operators will not match metrics that don't have the tag at all\"`\n\tLogQueryProgress bool `toml:\"log-query-progress\" json:\"log-query-progress\" comment:\"if true, gch will log affected rows count by clickhouse query\"`\n}\n\n// IndexReverseRule contains rules to use direct or reversed request to index table\ntype IndexReverseRule struct {\n\tSuffix string `toml:\"suffix,omitempty\" json:\"suffix\" comment:\"rule is used when the target suffix is matched\"`\n\tPrefix string `toml:\"prefix,omitempty\" json:\"prefix\" comment:\"rule is used when the target prefix is matched\"`\n\tRegexStr string `toml:\"regex,omitempty\" json:\"regex\" comment:\"rule is used when the target regex is matched\"`\n\tRegex *regexp.Regexp `toml:\"-\" json:\"-\"`\n\tReverse string `toml:\"reverse\" json:\"reverse\" comment:\"same as index-reverse\"`\n}\n\ntype Costs struct {\n\tCost *int `toml:\"cost\" json:\"cost\" comment:\"default cost (for wildcarded equalence or matched with regex, or if no value cost set)\"`\n\tValuesCost map[string]int `toml:\"values-cost\" json:\"values-cost\" comment:\"cost with some value (for equalence without wildcards) (additional tuning, usually not needed)\"`\n}\n\n// IndexReverses is a slise of ptrs to IndexReverseRule\ntype IndexReverses []*IndexReverseRule\n\nconst (\n\tIndexAuto = iota\n\tIndexDirect = iota\n\tIndexReversed = iota\n)\n\n// IndexReverse maps setting name to value\nvar IndexReverse = map[string]uint8{\n\t\"direct\": IndexDirect,\n\t\"auto\": IndexAuto,\n\t\"reversed\": IndexReversed,\n}\n\n// IndexReverseNames contains valid names for index-reverse setting\nvar IndexReverseNames = []string{\"auto\", \"direct\", \"reversed\"}\n\ntype UserLimits struct {\n\tMaxQueries int `toml:\"max-queries\" json:\"max-queries\" comment:\"Max queries to fetch data\"`\n\tConcurrentQueries int `toml:\"concurrent-queries\" json:\"concurrent-queries\" comment:\"Concurrent queries to fetch data\"`\n\tAdaptiveQueries int `toml:\"adaptive-queries\" json:\"adaptive-queries\" comment:\"Adaptive queries (based on load average) for increase/decrease concurrent queries\"`\n\n\tLimiter limiter.ServerLimiter `toml:\"-\" json:\"-\"`\n}\n\ntype QueryParam struct {\n\tDuration time.Duration `toml:\"duration\" json:\"duration\" comment:\"minimal duration (beetween from/until) for select query params\"`\n\tURL string `toml:\"url\" json:\"url\" comment:\"url for queries with durations greater or equal than\"`\n\tDataTimeout time.Duration `toml:\"data-timeout\" json:\"data-timeout\" comment:\"total timeout to fetch data\"`\n\n\tMaxQueries int `toml:\"max-queries\" json:\"max-queries\" comment:\"Max queries to fetch data\"`\n\tConcurrentQueries int `toml:\"concurrent-queries\" json:\"concurrent-queries\" comment:\"Concurrent queries to fetch data\"`\n\tAdaptiveQueries int `toml:\"adaptive-queries\" json:\"adaptive-queries\" comment:\"Adaptive queries (based on load average) for increase/decrease concurrent queries\"`\n\n\tLimiter limiter.ServerLimiter `toml:\"-\" json:\"-\"`\n}\n\nfunc binarySearchQueryParamLe(a []QueryParam, duration time.Duration, start, end int) int {\n\tlength := end - start\n\tif length <= 0 {\n\t\treturn -1 // not found\n\t} else if length == 1 {\n\t\tif a[start].Duration > duration {\n\t\t\treturn -1\n\t\t}\n\n\t\treturn start\n\t}\n\n\tvar result int\n\n\tmid := start + length/2\n\tif a[mid].Duration > duration {\n\t\tresult = binarySearchQueryParamLe(a, duration, start, mid)\n\t} else {\n\t\tif result = binarySearchQueryParamLe(a, duration, mid+1, end); result == -1 {\n\t\t\tresult = mid\n\t\t}\n\t}\n\n\treturn result\n}\n\n// ClickHouse config\ntype ClickHouse struct {\n\tURL string `toml:\"url\" json:\"url\" comment:\"default url, see https://clickhouse.tech/docs/en/interfaces/http. Can be overwritten with query-params\"`\n\tDataTimeout time.Duration `toml:\"data-timeout\" json:\"data-timeout\" comment:\"default total timeout to fetch data, can be overwritten with query-params\"`\n\tQueryParams []QueryParam `toml:\"query-params\" json:\"query-params\" comment:\"customized query params (url, data timeout, limiters) for durations greater or equal\"`\n\n\tProgressSendingInterval time.Duration `toml:\"progress-sending-interval\" json:\"progress-sending-interval\" comment:\"time interval for ch query progress sending, it's equal to http_headers_progress_interval_ms header\"`\n\n\tRenderMaxQueries int `toml:\"render-max-queries\" json:\"render-max-queries\" comment:\"Max queries to render queiries\"`\n\tRenderConcurrentQueries int `toml:\"render-concurrent-queries\" json:\"render-concurrent-queries\" comment:\"Concurrent queries to render queiries\"`\n\tRenderAdaptiveQueries int `toml:\"render-adaptive-queries\" json:\"render-adaptive-queries\" comment:\"Render adaptive queries (based on load average) for increase/decrease concurrent queries\"`\n\n\tFindMaxQueries int `toml:\"find-max-queries\" json:\"find-max-queries\" comment:\"Max queries for find queries\"`\n\tFindConcurrentQueries int `toml:\"find-concurrent-queries\" json:\"find-concurrent-queries\" comment:\"Find concurrent queries for find queries\"`\n\tFindAdaptiveQueries int `toml:\"find-adaptive-queries\" json:\"find-adaptive-queries\" comment:\"Find adaptive queries (based on load average) for increase/decrease concurrent queries\"`\n\tFindLimiter limiter.ServerLimiter `toml:\"-\" json:\"-\"`\n\n\tTagsMaxQueries int `toml:\"tags-max-queries\" json:\"tags-max-queries\" comment:\"Max queries for tags queries\"`\n\tTagsConcurrentQueries int `toml:\"tags-concurrent-queries\" json:\"tags-concurrent-queries\" comment:\"Concurrent queries for tags queries\"`\n\tTagsAdaptiveQueries int `toml:\"tags-adaptive-queries\" json:\"tags-adaptive-queries\" comment:\"Tags adaptive queries (based on load average) for increase/decrease concurrent queries\"`\n\tTagsLimiter limiter.ServerLimiter `toml:\"-\" json:\"-\"`\n\n\tWildcardMinDistance int `toml:\"wildcard-min-distance\" json:\"wildcard-min-distance\" comment:\"If a wildcard appears both at the start and the end of a plain query at a distance (in terms of nodes) less than wildcard-min-distance, then it will be discarded. This parameter can be used to discard expensive queries.\"`\n\tTrySplitQuery bool `toml:\"try-split-query\" json:\"try-split-query\" comment:\"Plain queries like '{first,second}.custom.metric.*' are also a subject to wildcard-min-distance restriction. But can be split into 2 queries: 'first.custom.metric.*', 'second.custom.metric.*'. Note that: only one list will be split; if there are wildcard in query before (after) list then reverse (direct) notation will be preferred; if there are wildcards before and after list, then query will not be split\"`\n\tMaxNodeToSplitIndex int `toml:\"max-node-to-split-index\" json:\"max-node-to-split-index\" comment:\"Used only if try-split-query is true. Query that contains list will be split if its (list) node index is less or equal to max-node-to-split-index. By default is 0. It is recommended to have this value set to 2 or 3 and increase it very carefully, because 3 or 4 plain nodes without wildcards have good selectivity\"`\n\tTagsMinInQuery int `toml:\"tags-min-in-query\" json:\"tags-min-in-query\" comment:\"Minimum tags in seriesByTag query\"`\n\tTagsMinInAutocomplete int `toml:\"tags-min-in-autocomplete\" json:\"tags-min-in-autocomplete\" comment:\"Minimum tags in autocomplete query\"`\n\n\tUserLimits map[string]UserLimits `toml:\"user-limits\" json:\"user-limits\" comment:\"customized query limiter for some users\" commented:\"true\"`\n\tDateFormat string `toml:\"date-format\" json:\"date-format\" comment:\"Date format (default, utc, both)\"`\n\tIndexTable string `toml:\"index-table\" json:\"index-table\" comment:\"see doc/index-table.md\"`\n\tIndexUseDaily bool `toml:\"index-use-daily\" json:\"index-use-daily\"`\n\tIndexReverse string `toml:\"index-reverse\" json:\"index-reverse\" comment:\"see doc/config.md\"`\n\tIndexReverses IndexReverses `toml:\"index-reverses\" json:\"index-reverses\" comment:\"see doc/config.md\" commented:\"true\"`\n\tIndexTimeout time.Duration `toml:\"index-timeout\" json:\"index-timeout\" comment:\"total timeout to fetch series list from index\"`\n\tTaggedTable string `toml:\"tagged-table\" json:\"tagged-table\" comment:\"'tagged' table from carbon-clickhouse, required for seriesByTag\"`\n\tTagsCountTable string `toml:\"tags-count-table\" json:\"tags-count-table\" comment:\"Table that contains the total amounts of each tag-value pair. It is used to avoid usage of high cardinality tag-value pairs when querying TaggedTable. If left empty, basic sorting will be used. See more detailed description in doc/config.md\"`\n\tTaggedAutocompleDays int `toml:\"tagged-autocomplete-days\" json:\"tagged-autocomplete-days\" comment:\"or how long the daemon will query tags during autocomplete\"`\n\tTaggedUseDaily bool `toml:\"tagged-use-daily\" json:\"tagged-use-daily\" comment:\"whether to use date filter when searching for the metrics in the tagged-table\"`\n\tTaggedCosts map[string]*Costs `toml:\"tagged-costs\" json:\"tagged-costs\" comment:\"costs for tags (for tune which tag will be used as primary), by default is 0, increase for costly (with poor selectivity) tags\" commented:\"true\"`\n\tTreeTable string `toml:\"tree-table\" json:\"tree-table\" comment:\"old index table, DEPRECATED, see description in doc/config.md\" commented:\"true\"`\n\tReverseTreeTable string `toml:\"reverse-tree-table\" json:\"reverse-tree-table\" commented:\"true\"`\n\tDateTreeTable string `toml:\"date-tree-table\" json:\"date-tree-table\" commented:\"true\"`\n\tDateTreeTableVersion int `toml:\"date-tree-table-version\" json:\"date-tree-table-version\" commented:\"true\"`\n\tTreeTimeout time.Duration `toml:\"tree-timeout\" json:\"tree-timeout\" commented:\"true\"`\n\tTagTable string `toml:\"tag-table\" json:\"tag-table\" comment:\"is not recommended to use, https://github.com/lomik/graphite-clickhouse/wiki/TagsRU\" commented:\"true\"`\n\tExtraPrefix string `toml:\"extra-prefix\" json:\"extra-prefix\" comment:\"add extra prefix (directory in graphite) for all metrics, w/o trailing dot\"`\n\tConnectTimeout time.Duration `toml:\"connect-timeout\" json:\"connect-timeout\" comment:\"TCP connection timeout\"`\n\t// TODO: remove in v0.14\n\tDataTableLegacy string `toml:\"data-table\" json:\"data-table\" comment:\"will be removed in 0.14\" commented:\"true\"`\n\t// TODO: remove in v0.14\n\tRollupConfLegacy string `toml:\"rollup-conf\" json:\"-\" commented:\"true\"`\n\tMaxDataPoints int `toml:\"max-data-points\" json:\"max-data-points\" comment:\"max points per metric when internal-aggregation=true\"`\n\t// InternalAggregation controls if ClickHouse itself or graphite-clickhouse aggregates points to proper retention\n\tInternalAggregation bool `toml:\"internal-aggregation\" json:\"internal-aggregation\" comment:\"ClickHouse-side aggregation, see doc/aggregation.md\"`\n\n\tTLSParams config.TLS `toml:\"tls\" json:\"tls\" comment:\"mTLS HTTPS configuration for connecting to clickhouse server\" commented:\"true\"`\n\tTLSConfig *tls.Config `toml:\"-\" json:\"-\"`\n}\n\nfunc clickhouseURLValidate(chURL string) (*url.URL, error) {\n\tu, err := url.Parse(chURL)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"error %q in url %q\", err.Error(), chURL)\n\t} else if u.Scheme != \"http\" && u.Scheme != \"https\" {\n\t\treturn nil, fmt.Errorf(\"scheme not supported in url %q\", chURL)\n\t} else if strings.Contains(u.RawQuery, \" \") {\n\t\treturn nil, fmt.Errorf(\"space not allowed in url %q\", chURL)\n\t}\n\n\treturn u, nil\n}\n\n// Tags config\ntype Tags struct {\n\tRules string `toml:\"rules\" json:\"rules\"`\n\tDate string `toml:\"date\" json:\"date\"`\n\tExtraWhere string `toml:\"extra-where\" json:\"extra-where\"`\n\tInputFile string `toml:\"input-file\" json:\"input-file\"`\n\tOutputFile string `toml:\"output-file\" json:\"output-file\"`\n\tThreads int `toml:\"threads\" json:\"threads\" comment:\"number of threads for uploading tags to clickhouse (1 by default)\"`\n\tCompression clickhouse.ContentEncoding `toml:\"compression\" json:\"compression\" comment:\"compression method for tags before sending them to clickhouse (i.e. content encoding): gzip (default), none, zstd\"`\n\tVersion uint32 `toml:\"version\" json:\"version\" comment:\"fixed tags version for testing purposes (by default the current timestamp is used for each upload)\"`\n\tSelectChunksCount int `toml:\"select-chunks-count\" json:\"select-chunks-count\" comment:\"number of chunks for selecting metrics from clickhouse (10 by default)\"`\n}\n\n// Carbonlink configuration\ntype Carbonlink struct {\n\tServer string `toml:\"server\" json:\"server\"`\n\tThreads int `toml:\"threads-per-request\" json:\"threads-per-request\"`\n\tRetries int `toml:\"-\" json:\"-\"`\n\tConnectTimeout time.Duration `toml:\"connect-timeout\" json:\"connect-timeout\"`\n\tQueryTimeout time.Duration `toml:\"query-timeout\" json:\"query-timeout\"`\n\tTotalTimeout time.Duration `toml:\"total-timeout\" json:\"total-timeout\" comment:\"timeout for querying and parsing response\"`\n}\n\n// Prometheus configuration\ntype Prometheus struct {\n\tListen string `toml:\"listen\" json:\"listen\" comment:\"listen addr for prometheus ui and api\"`\n\tExternalURLRaw string `toml:\"external-url\" json:\"external-url\" comment:\"allows to set URL for redirect manually\"`\n\tExternalURL *url.URL `toml:\"-\" json:\"-\"`\n\tPageTitle string `toml:\"page-title\" json:\"page-title\"`\n\tLookbackDelta time.Duration `toml:\"lookback-delta\" json:\"lookback-delta\"`\n\tRemoteReadConcurrencyLimit int `toml:\"remote-read-concurrency-limit\" json:\"remote-read-concurrency-limit\" comment:\"concurrently handled remote read requests\"`\n}\n\nconst (\n\t// ContextGraphite for data tables\n\tContextGraphite = \"graphite\"\n\t// ContextPrometheus for data tables\n\tContextPrometheus = \"prometheus\"\n)\n\nvar knownDataTableContext = map[string]bool{\n\tContextGraphite: true,\n\tContextPrometheus: true,\n}\n\n// DataTable configs\ntype DataTable struct {\n\tTable string `toml:\"table\" json:\"table\" comment:\"data table from carbon-clickhouse\"`\n\tReverse bool `toml:\"reverse\" json:\"reverse\" comment:\"if it stores direct or reversed metrics\"`\n\tMaxAge time.Duration `toml:\"max-age\" json:\"max-age\" comment:\"maximum age stored in the table\"`\n\tMinAge time.Duration `toml:\"min-age\" json:\"min-age\" comment:\"minimum age stored in the table\"`\n\tMaxInterval time.Duration `toml:\"max-interval\" json:\"max-interval\" comment:\"maximum until-from interval allowed for the table\"`\n\tMinInterval time.Duration `toml:\"min-interval\" json:\"min-interval\" comment:\"minimum until-from interval allowed for the table\"`\n\tTargetMatchAny string `toml:\"target-match-any\" json:\"target-match-any\" comment:\"table allowed only if any metrics in target matches regexp\"`\n\tTargetMatchAll string `toml:\"target-match-all\" json:\"target-match-all\" comment:\"table allowed only if all metrics in target matches regexp\"`\n\tTargetMatchAnyRegexp *regexp.Regexp `toml:\"-\" json:\"-\"`\n\tTargetMatchAllRegexp *regexp.Regexp `toml:\"-\" json:\"-\"`\n\tRollupConf string `toml:\"rollup-conf\" json:\"-\" comment:\"custom rollup.xml file for table, 'auto' and 'none' are allowed as well\"`\n\tRollupAutoTable string `toml:\"rollup-auto-table\" json:\"rollup-auto-table\" comment:\"custom table for 'rollup-conf=auto', useful for Distributed or MatView\"`\n\tRollupAutoInterval *time.Duration `toml:\"rollup-auto-interval\" json:\"rollup-auto-interval\" comment:\"rollup update interval for 'rollup-conf=auto'\"`\n\tRollupDefaultPrecision uint32 `toml:\"rollup-default-precision\" json:\"rollup-default-precision\" comment:\"is used when none of rules match\"`\n\tRollupDefaultFunction string `toml:\"rollup-default-function\" json:\"rollup-default-function\" comment:\"is used when none of rules match\"`\n\tRollupUseReverted bool `toml:\"rollup-use-reverted\" json:\"rollup-use-reverted\" comment:\"should be set to true if you don't have reverted regexps in rollup-conf for reversed tables\"`\n\tContext []string `toml:\"context\" json:\"context\" comment:\"valid values are 'graphite' of 'prometheus'\"`\n\tContextMap map[string]bool `toml:\"-\" json:\"-\"`\n\tRollup *rollup.Rollup `toml:\"-\" json:\"rollup-conf\"`\n\tQueryMetrics *metrics.QueryMetrics `toml:\"-\" json:\"-\"`\n}\n\n// Debug config\ntype Debug struct {\n\tDirectory string `toml:\"directory\" json:\"directory\" comment:\"the directory for additional debug output\"`\n\tDirectoryPerm os.FileMode `toml:\"directory-perm\" json:\"directory-perm\" comment:\"permissions for directory, octal value is set as 0o755\"`\n\t// If ExternalDataPerm > 0 and X-Gch-Debug-Ext-Data HTTP header is set, the external data used in the query\n\t// will be saved in the DebugDir directory\n\tExternalDataPerm os.FileMode `toml:\"external-data-perm\" json:\"external-data-perm\" comment:\"permissions for directory, octal value is set as 0o640\"`\n}\n\n// Config is the daemon configuration\ntype Config struct {\n\tCommon Common `toml:\"common\" json:\"common\"`\n\tFeatureFlags FeatureFlags `toml:\"feature-flags\" json:\"feature-flags\"`\n\tMetrics metrics.Config `toml:\"metrics\" json:\"metrics\"`\n\tClickHouse ClickHouse `toml:\"clickhouse\" json:\"clickhouse\"`\n\tDataTable []DataTable `toml:\"data-table\" json:\"data-table\" comment:\"data tables, see doc/config.md for additional info\"`\n\tTags Tags `toml:\"tags\" json:\"tags\" comment:\"is not recommended to use, https://github.com/lomik/graphite-clickhouse/wiki/TagsRU\" commented:\"true\"`\n\tCarbonlink Carbonlink `toml:\"carbonlink\" json:\"carbonlink\"`\n\tPrometheus Prometheus `toml:\"prometheus\" json:\"prometheus\"`\n\tDebug Debug `toml:\"debug\" json:\"debug\" comment:\"see doc/debugging.md\"`\n\tLogging []zapwriter.Config `toml:\"logging\" json:\"logging\"`\n}\n\n// New returns *Config with default values\nfunc New() *Config {\n\tcfg := &Config{\n\t\tCommon: Common{\n\t\t\tListen: \":9090\",\n\t\t\tPprofListen: \"\",\n\t\t\t// MetricPrefix: \"carbon.graphite-clickhouse.{host}\",\n\t\t\t// MetricInterval: time.Minute,\n\t\t\t// MetricEndpoint: MetricEndpointLocal,\n\t\t\tMaxCPU: 1,\n\t\t\tMaxMetricsInFindAnswer: 0,\n\t\t\tMaxMetricsPerTarget: 15000, // This is arbitrary value to protect CH from overload\n\t\t\tMemoryReturnInterval: 0,\n\t\t\tFindCacheConfig: CacheConfig{\n\t\t\t\tType: \"null\",\n\t\t\t\tDefaultTimeoutSec: 0,\n\t\t\t\tShortTimeoutSec: 0,\n\t\t\t\tFindTimeoutSec: 0,\n\t\t\t},\n\t\t\tDegragedMultiply: 4.0,\n\t\t\tDegragedLoad: 1.0,\n\t\t},\n\t\tClickHouse: ClickHouse{\n\t\t\tURL: \"http://localhost:8123?cancel_http_readonly_queries_on_client_close=1\",\n\t\t\tDataTimeout: time.Minute,\n\t\t\tProgressSendingInterval: 10 * time.Second,\n\t\t\tIndexTable: \"graphite_index\",\n\t\t\tIndexUseDaily: true,\n\t\t\tTaggedUseDaily: true,\n\t\t\tIndexReverse: \"auto\",\n\t\t\tIndexReverses: IndexReverses{},\n\t\t\tIndexTimeout: time.Minute,\n\t\t\tTaggedTable: \"graphite_tagged\",\n\t\t\tTaggedAutocompleDays: 7,\n\t\t\tExtraPrefix: \"\",\n\t\t\tConnectTimeout: time.Second,\n\t\t\tDataTableLegacy: \"\",\n\t\t\tRollupConfLegacy: \"auto\",\n\t\t\tMaxDataPoints: 1048576,\n\t\t\tInternalAggregation: true,\n\t\t\tFindLimiter: limiter.NoopLimiter{},\n\t\t\tTagsLimiter: limiter.NoopLimiter{},\n\t\t},\n\t\tTags: Tags{\n\t\t\tThreads: 1,\n\t\t\tCompression: \"gzip\",\n\t\t},\n\t\tCarbonlink: Carbonlink{\n\t\t\tThreads: 10,\n\t\t\tRetries: 2,\n\t\t\tConnectTimeout: 50 * time.Millisecond,\n\t\t\tQueryTimeout: 50 * time.Millisecond,\n\t\t\tTotalTimeout: 500 * time.Millisecond,\n\t\t},\n\t\tPrometheus: Prometheus{\n\t\t\tExternalURLRaw: \"\",\n\t\t\tPageTitle: \"Prometheus Time Series Collection and Processing Server\",\n\t\t\tListen: \":9092\",\n\t\t\tLookbackDelta: 5 * time.Minute,\n\t\t\tRemoteReadConcurrencyLimit: 10,\n\t\t},\n\t\tDebug: Debug{\n\t\t\tDirectory: \"\",\n\t\t\tDirectoryPerm: 0755,\n\t\t\tExternalDataPerm: 0,\n\t\t},\n\t\tLogging: nil,\n\t}\n\n\treturn cfg\n}\n\n// Compile checks if IndexReverseRule are valid in the IndexReverses and compiles regexps if set\nfunc (ir IndexReverses) Compile() error {\n\tvar err error\n\n\tfor i, n := range ir {\n\t\tif len(n.RegexStr) > 0 {\n\t\t\tif n.Regex, err = regexp.Compile(n.RegexStr); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t} else if len(n.Prefix) == 0 && len(n.Suffix) == 0 {\n\t\t\treturn fmt.Errorf(\"empthy index-use-reverses[%d] rule\", i)\n\t\t}\n\n\t\tif _, ok := IndexReverse[n.Reverse]; !ok {\n\t\t\treturn fmt.Errorf(\"%s is not valid value for index-reverses.reverse\", n.Reverse)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc newLoggingConfig() zapwriter.Config {\n\tcfg := zapwriter.NewConfig()\n\tcfg.File = \"/var/log/graphite-clickhouse/graphite-clickhouse.log\"\n\n\treturn cfg\n}\n\nfunc DefaultConfig() (*Config, error) {\n\tcfg := New()\n\n\tif cfg.Logging == nil {\n\t\tcfg.Logging = make([]zapwriter.Config, 0)\n\t}\n\n\tif len(cfg.Logging) == 0 {\n\t\tcfg.Logging = append(cfg.Logging, newLoggingConfig())\n\t}\n\n\tif len(cfg.DataTable) == 0 {\n\t\tinterval := time.Minute\n\t\tcfg.DataTable = []DataTable{\n\t\t\t{\n\t\t\t\tTable: \"graphite_data\",\n\t\t\t\tRollupConf: \"auto\",\n\t\t\t\tRollupAutoInterval: &interval,\n\t\t\t},\n\t\t}\n\t}\n\n\tif len(cfg.ClickHouse.IndexReverses) == 0 {\n\t\tcfg.ClickHouse.IndexReverses = IndexReverses{\n\t\t\t&IndexReverseRule{Suffix: \"suffix\", Reverse: \"auto\"},\n\t\t\t&IndexReverseRule{Prefix: \"prefix\", Reverse: \"direct\"},\n\t\t\t&IndexReverseRule{RegexStr: \"regex\", Reverse: \"reversed\"},\n\t\t}\n\n\t\terr := cfg.ClickHouse.IndexReverses.Compile()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\treturn cfg, nil\n}\n\n// PrintDefaultConfig prints the default config with some additions to be useful\nfunc PrintDefaultConfig() error {\n\tbuf := new(bytes.Buffer)\n\n\tcfg, err := DefaultConfig()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tencoder := toml.NewEncoder(buf).Indentation(\" \").Order(toml.OrderPreserve).CompactComments(true)\n\n\tif err := encoder.Encode(cfg); err != nil {\n\t\treturn err\n\t}\n\n\tout := strings.Replace(buf.String(), \"\\n\", \"\", 1)\n\n\tfmt.Print(out)\n\n\treturn nil\n}\n\n// ReadConfig reads the content of the file with given name and process it to the *Config\nfunc ReadConfig(filename string, exactConfig bool) (*Config, []zap.Field, error) {\n\tvar err error\n\n\tvar body []byte\n\tif filename != \"\" {\n\t\tbody, err = os.ReadFile(filename)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t}\n\n\treturn Unmarshal(body, exactConfig)\n}\n\n// Unmarshal process the body to *Config\nfunc Unmarshal(body []byte, exactConfig bool) (cfg *Config, warns []zap.Field, err error) {\n\tdeprecations := make(map[string]error)\n\n\tcfg = New()\n\n\tif len(body) != 0 {\n\t\t// TODO: remove in v0.14\n\t\tif bytes.Index(body, []byte(\"\\n[logging]\\n\")) != -1 || bytes.Index(body, []byte(\"[logging]\")) == 0 {\n\t\t\tdeprecations[\"logging\"] = fmt.Errorf(\"single [logging] value became multivalue [[logging]]; please, adjust your config\")\n\n\t\t\tbody = bytes.ReplaceAll(body, []byte(\"\\n[logging]\\n\"), []byte(\"\\n[[logging]]\\n\"))\n\t\t\tif bytes.Index(body, []byte(\"[logging]\")) == 0 {\n\t\t\t\tbody = bytes.Replace(body, []byte(\"[logging]\"), []byte(\"[[logging]]\"), 1)\n\t\t\t}\n\t\t}\n\n\t\tdecoder := toml.NewDecoder(bytes.NewReader(body))\n\t\tdecoder.Strict(exactConfig)\n\n\t\terr := decoder.Decode(cfg)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t}\n\n\tif cfg.Logging == nil {\n\t\tcfg.Logging = make([]zapwriter.Config, 0)\n\t}\n\n\tif cfg.ClickHouse.RenderConcurrentQueries > cfg.ClickHouse.RenderMaxQueries && cfg.ClickHouse.RenderMaxQueries > 0 {\n\t\tcfg.ClickHouse.RenderConcurrentQueries = 0\n\t}\n\n\tchURL, err := clickhouseURLValidate(cfg.ClickHouse.URL)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tif !reflect.DeepEqual(cfg.ClickHouse.TLSParams, config.TLS{}) {\n\t\ttlsConfig, warnings, err := config.ParseClientTLSConfig(&cfg.ClickHouse.TLSParams)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\n\t\tif chURL.Scheme == \"https\" {\n\t\t\tcfg.ClickHouse.TLSConfig = tlsConfig\n\t\t} else {\n\t\t\twarnings = append(warnings, \"TLS configurations is ignored because scheme is not HTTPS\")\n\t\t}\n\n\t\twarns = append(warns, zap.Strings(\"tls-config\", warnings))\n\t}\n\n\tfor i := range cfg.ClickHouse.QueryParams {\n\t\tif cfg.ClickHouse.QueryParams[i].ConcurrentQueries > cfg.ClickHouse.QueryParams[i].MaxQueries && cfg.ClickHouse.QueryParams[i].MaxQueries > 0 {\n\t\t\tcfg.ClickHouse.QueryParams[i].ConcurrentQueries = 0\n\t\t}\n\n\t\tif cfg.ClickHouse.QueryParams[i].Duration == 0 {\n\t\t\treturn nil, nil, fmt.Errorf(\"query duration param not set for: %+v\", cfg.ClickHouse.QueryParams[i])\n\t\t}\n\n\t\tif cfg.ClickHouse.QueryParams[i].DataTimeout == 0 {\n\t\t\tcfg.ClickHouse.QueryParams[i].DataTimeout = cfg.ClickHouse.DataTimeout\n\t\t}\n\n\t\tif cfg.ClickHouse.QueryParams[i].URL == \"\" {\n\t\t\t// reuse default url\n\t\t\tcfg.ClickHouse.QueryParams[i].URL = cfg.ClickHouse.URL\n\t\t}\n\n\t\tif _, err = clickhouseURLValidate(cfg.ClickHouse.QueryParams[i].URL); err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t}\n\n\tcfg.ClickHouse.QueryParams = append(\n\t\t[]QueryParam{{\n\t\t\tURL: cfg.ClickHouse.URL, DataTimeout: cfg.ClickHouse.DataTimeout,\n\t\t\tMaxQueries: cfg.ClickHouse.RenderMaxQueries, ConcurrentQueries: cfg.ClickHouse.RenderConcurrentQueries,\n\t\t\tAdaptiveQueries: cfg.ClickHouse.RenderAdaptiveQueries,\n\t\t}},\n\t\tcfg.ClickHouse.QueryParams...,\n\t)\n\n\tsort.SliceStable(cfg.ClickHouse.QueryParams, func(i, j int) bool {\n\t\treturn cfg.ClickHouse.QueryParams[i].Duration < cfg.ClickHouse.QueryParams[j].Duration\n\t})\n\n\tif len(cfg.Logging) == 0 {\n\t\tcfg.Logging = append(cfg.Logging, newLoggingConfig())\n\t}\n\n\tif err = zapwriter.CheckConfig(cfg.Logging, nil); err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\t// Check if debug directory exists or could be created\n\tif cfg.Debug.Directory != \"\" {\n\t\tinfo, err := os.Stat(cfg.Debug.Directory)\n\t\tif os.IsNotExist(err) {\n\t\t\terr := os.MkdirAll(cfg.Debug.Directory, os.ModeDir|cfg.Debug.DirectoryPerm)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, nil, err\n\t\t\t}\n\t\t} else if !info.IsDir() {\n\t\t\treturn nil, nil, fmt.Errorf(\"the file for external data debug dumps exists and is not a directory: %v\", cfg.Debug.Directory)\n\t\t}\n\t}\n\n\tif _, ok := IndexReverse[cfg.ClickHouse.IndexReverse]; !ok {\n\t\treturn nil, nil, fmt.Errorf(\"%s is not valid value for index-reverse\", cfg.ClickHouse.IndexReverse)\n\t}\n\n\terr = cfg.ClickHouse.IndexReverses.Compile()\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tif cfg.Common.FindCache, err = CreateCache(\"index\", &cfg.Common.FindCacheConfig); err == nil {\n\t\tif cfg.Common.FindCacheConfig.Type != \"null\" {\n\t\t\twarns = append(warns, zap.Any(\"enable find cache\", zap.String(\"type\", cfg.Common.FindCacheConfig.Type)))\n\t\t}\n\t} else {\n\t\treturn nil, nil, err\n\t}\n\n\tl := len(cfg.Common.TargetBlacklist)\n\tif l > 0 {\n\t\tcfg.Common.Blacklist = make([]*regexp.Regexp, l)\n\t\tfor i := 0; i < l; i++ {\n\t\t\tr, err := regexp.Compile(cfg.Common.TargetBlacklist[i])\n\t\t\tif err != nil {\n\t\t\t\treturn nil, nil, err\n\t\t\t}\n\n\t\t\tcfg.Common.Blacklist[i] = r\n\t\t}\n\t}\n\n\terr = cfg.ProcessDataTables()\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\t// compute prometheus external url\n\trawURL := cfg.Prometheus.ExternalURLRaw\n\tif rawURL == \"\" {\n\t\thostname, err := os.Hostname()\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\n\t\t_, port, err := net.SplitHostPort(cfg.Common.Listen)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\n\t\trawURL = fmt.Sprintf(\"http://%s:%s/\", hostname, port)\n\t}\n\n\tcfg.Prometheus.ExternalURL, err = url.Parse(rawURL)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tcfg.Prometheus.ExternalURL.Path = strings.TrimRight(cfg.Prometheus.ExternalURL.Path, \"/\")\n\n\tcheckDeprecations(cfg, deprecations)\n\n\tif len(deprecations) != 0 {\n\t\tdeprecationList := make([]error, len(deprecations))\n\t\tfor name, message := range deprecations {\n\t\t\tdeprecationList = append(deprecationList, errors.Wrap(message, name))\n\t\t}\n\n\t\twarns = append(warns, zap.Errors(\"config deprecations\", deprecationList))\n\t}\n\n\tswitch strings.ToLower(cfg.ClickHouse.DateFormat) {\n\tcase \"utc\":\n\t\tdate.SetUTC()\n\tcase \"both\":\n\t\tdate.SetBoth()\n\tdefault:\n\t\tif cfg.ClickHouse.DateFormat != \"\" && cfg.ClickHouse.DateFormat != \"default\" {\n\t\t\treturn nil, nil, fmt.Errorf(\"unsupported date-format: %s\", cfg.ClickHouse.DateFormat)\n\t\t}\n\t}\n\n\tif cfg.ClickHouse.FindConcurrentQueries > cfg.ClickHouse.FindMaxQueries && cfg.ClickHouse.FindMaxQueries > 0 {\n\t\tcfg.ClickHouse.FindConcurrentQueries = 0\n\t}\n\n\tif cfg.ClickHouse.TagsConcurrentQueries > cfg.ClickHouse.TagsMaxQueries && cfg.ClickHouse.TagsMaxQueries > 0 {\n\t\tcfg.ClickHouse.TagsConcurrentQueries = 0\n\t}\n\n\tmetricsEnabled := cfg.setupGraphiteMetrics()\n\n\tcfg.ClickHouse.FindLimiter = limiter.NewALimiter(\n\t\tcfg.ClickHouse.FindMaxQueries, cfg.ClickHouse.FindConcurrentQueries, cfg.ClickHouse.FindAdaptiveQueries,\n\t\tmetricsEnabled, \"find\", \"all\",\n\t)\n\n\tcfg.ClickHouse.TagsLimiter = limiter.NewALimiter(\n\t\tcfg.ClickHouse.TagsMaxQueries, cfg.ClickHouse.TagsConcurrentQueries, cfg.ClickHouse.TagsAdaptiveQueries,\n\t\tmetricsEnabled, \"tags\", \"all\",\n\t)\n\n\tfor i := range cfg.ClickHouse.QueryParams {\n\t\tcfg.ClickHouse.QueryParams[i].Limiter = limiter.NewALimiter(\n\t\t\tcfg.ClickHouse.QueryParams[i].MaxQueries, cfg.ClickHouse.QueryParams[i].ConcurrentQueries,\n\t\t\tcfg.ClickHouse.QueryParams[i].AdaptiveQueries,\n\t\t\tmetricsEnabled, \"render\", duration.String(cfg.ClickHouse.QueryParams[i].Duration),\n\t\t)\n\t}\n\n\tfor u, q := range cfg.ClickHouse.UserLimits {\n\t\tq.Limiter = limiter.NewALimiter(\n\t\t\tq.MaxQueries, q.ConcurrentQueries, q.AdaptiveQueries, metricsEnabled, u, \"all\",\n\t\t)\n\t\tcfg.ClickHouse.UserLimits[u] = q\n\t}\n\n\treturn cfg, warns, nil\n}\n\n// NeedLoadAvgColect check if load avg collect is neeeded\nfunc (c *Config) NeedLoadAvgColect() bool {\n\tif c.Common.SD != \"\" {\n\t\tif c.Common.DegragedMultiply <= 0 {\n\t\t\tc.Common.DegragedMultiply = 4.0\n\t\t}\n\n\t\tif c.Common.DegragedLoad <= 0 {\n\t\t\tc.Common.DegragedLoad = 1.0\n\t\t}\n\n\t\tif c.Common.BaseWeight <= 0 {\n\t\t\tc.Common.BaseWeight = 100\n\t\t}\n\n\t\tif c.Common.SDNamespace == \"\" {\n\t\t\tc.Common.SDNamespace = \"graphite\"\n\t\t}\n\n\t\tif c.Common.SDExpire < 24*time.Hour {\n\t\t\tc.Common.SDExpire = 24 * time.Hour\n\t\t}\n\n\t\treturn true\n\t}\n\n\tif c.ClickHouse.RenderAdaptiveQueries > 0 {\n\t\treturn true\n\t}\n\n\tif c.ClickHouse.FindAdaptiveQueries > 0 {\n\t\treturn true\n\t}\n\n\tif c.ClickHouse.TagsAdaptiveQueries > 0 {\n\t\treturn true\n\t}\n\n\tfor _, u := range c.ClickHouse.UserLimits {\n\t\tif u.AdaptiveQueries > 0 {\n\t\t\treturn true\n\t\t}\n\t}\n\n\treturn false\n}\n\n// ProcessDataTables checks if legacy `data`-table config is used, compiles regexps for `target-match-any` and `target-match-all`\n// parameters, sets the rollup configuration and proper context.\nfunc (c *Config) ProcessDataTables() (err error) {\n\tif c.ClickHouse.DataTableLegacy != \"\" {\n\t\tc.DataTable = append(c.DataTable, DataTable{\n\t\t\tTable: c.ClickHouse.DataTableLegacy,\n\t\t\tRollupConf: c.ClickHouse.RollupConfLegacy,\n\t\t})\n\t}\n\n\tfor i := 0; i < len(c.DataTable); i++ {\n\t\tif c.DataTable[i].TargetMatchAny != \"\" {\n\t\t\tr, err := regexp.Compile(c.DataTable[i].TargetMatchAny)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tc.DataTable[i].TargetMatchAnyRegexp = r\n\t\t}\n\n\t\tif c.DataTable[i].TargetMatchAll != \"\" {\n\t\t\tr, err := regexp.Compile(c.DataTable[i].TargetMatchAll)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tc.DataTable[i].TargetMatchAllRegexp = r\n\t\t}\n\n\t\trdp := c.DataTable[i].RollupDefaultPrecision\n\t\trdf := c.DataTable[i].RollupDefaultFunction\n\n\t\tif c.DataTable[i].RollupConf == \"auto\" || c.DataTable[i].RollupConf == \"\" {\n\t\t\ttable := c.DataTable[i].Table\n\t\t\tinterval := time.Minute\n\n\t\t\tif c.DataTable[i].RollupAutoTable != \"\" {\n\t\t\t\ttable = c.DataTable[i].RollupAutoTable\n\t\t\t}\n\n\t\t\tif c.DataTable[i].RollupAutoInterval != nil {\n\t\t\t\tinterval = *c.DataTable[i].RollupAutoInterval\n\t\t\t}\n\n\t\t\tc.DataTable[i].Rollup, err = rollup.NewAuto(\n\t\t\t\tc.ClickHouse.URL,\n\t\t\t\tc.ClickHouse.TLSConfig,\n\t\t\t\ttable,\n\t\t\t\tinterval,\n\t\t\t\trdp,\n\t\t\t\trdf,\n\t\t\t)\n\t\t} else if c.DataTable[i].RollupConf == \"none\" {\n\t\t\tc.DataTable[i].Rollup, err = rollup.NewDefault(rdp, rdf)\n\t\t} else {\n\t\t\tc.DataTable[i].Rollup, err = rollup.NewXMLFile(c.DataTable[i].RollupConf, rdp, rdf)\n\t\t}\n\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif len(c.DataTable[i].Context) == 0 {\n\t\t\tc.DataTable[i].ContextMap = knownDataTableContext\n\t\t} else {\n\t\t\tc.DataTable[i].ContextMap = make(map[string]bool)\n\t\t\tfor _, ctx := range c.DataTable[i].Context {\n\t\t\t\tif !knownDataTableContext[ctx] {\n\t\t\t\t\treturn fmt.Errorf(\"unknown context %#v\", ctx)\n\t\t\t\t}\n\n\t\t\t\tc.DataTable[i].ContextMap[ctx] = true\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc checkDeprecations(cfg *Config, d map[string]error) {\n\tif cfg.ClickHouse.DataTableLegacy != \"\" {\n\t\td[\"data-table\"] = fmt.Errorf(\"data-table parameter in [clickhouse] is deprecated; use [[data-table]]\")\n\t}\n}\n\nfunc CreateCache(cacheName string, cacheConfig *CacheConfig) (cache.BytesCache, error) {\n\tif cacheConfig.DefaultTimeoutSec <= 0 && cacheConfig.ShortTimeoutSec <= 0 && cacheConfig.FindTimeoutSec <= 0 {\n\t\treturn nil, nil\n\t}\n\n\tif cacheConfig.DefaultTimeoutSec < cacheConfig.ShortTimeoutSec {\n\t\tcacheConfig.DefaultTimeoutSec = cacheConfig.ShortTimeoutSec\n\t}\n\n\tif cacheConfig.ShortTimeoutSec < 0 || cacheConfig.DefaultTimeoutSec == cacheConfig.ShortTimeoutSec {\n\t\t// broken value or short timeout not need due to equal\n\t\tcacheConfig.ShortTimeoutSec = 0\n\t}\n\n\tif cacheConfig.DefaultTimeoutSec < cacheConfig.ShortTimeoutSec {\n\t\tcacheConfig.DefaultTimeoutSec = cacheConfig.ShortTimeoutSec\n\t}\n\n\tif cacheConfig.ShortDuration == 0 {\n\t\tcacheConfig.ShortDuration = 3 * time.Hour\n\t}\n\n\tif cacheConfig.ShortUntilOffsetSec == 0 {\n\t\tcacheConfig.ShortUntilOffsetSec = 120\n\t}\n\n\tcacheConfig.DefaultTimeoutStr = strconv.Itoa(int(cacheConfig.DefaultTimeoutSec))\n\tcacheConfig.ShortTimeoutStr = strconv.Itoa(int(cacheConfig.ShortTimeoutSec))\n\n\tswitch cacheConfig.Type {\n\tcase \"memcache\":\n\t\tif len(cacheConfig.MemcachedServers) == 0 {\n\t\t\treturn nil, fmt.Errorf(cacheName + \": memcache cache requested but no memcache servers provided\")\n\t\t}\n\n\t\treturn cache.NewMemcached(\"gch-\"+cacheName, cacheConfig.MemcachedServers...), nil\n\tcase \"mem\":\n\t\treturn cache.NewExpireCache(uint64(cacheConfig.Size * 1024 * 1024)), nil\n\tcase \"null\":\n\t\t// defaults\n\t\treturn nil, nil\n\tdefault:\n\t\treturn nil, fmt.Errorf(\n\t\t\t\"%s: unknown cache type '%s', known_cache_types 'null', 'mem', 'memcache'\",\n\t\t\tcacheName,\n\t\t\tcacheConfig.Type,\n\t\t)\n\t}\n}\n\nfunc (c *Config) setupGraphiteMetrics() bool {\n\tif c.Metrics.MetricEndpoint == \"\" {\n\t\tmetrics.DisableMetrics()\n\t} else {\n\t\tif c.Metrics.MetricInterval == 0 {\n\t\t\tc.Metrics.MetricInterval = 60 * time.Second\n\t\t}\n\n\t\tif c.Metrics.MetricTimeout == 0 {\n\t\t\tc.Metrics.MetricTimeout = time.Second\n\t\t}\n\n\t\thostname, _ := os.Hostname()\n\t\tfqdn := strings.ReplaceAll(hostname, \".\", \"_\")\n\t\thostname = strings.Split(hostname, \".\")[0]\n\n\t\tc.Metrics.MetricPrefix = strings.ReplaceAll(c.Metrics.MetricPrefix, \"{prefix}\", c.Metrics.MetricPrefix)\n\t\tc.Metrics.MetricPrefix = strings.ReplaceAll(c.Metrics.MetricPrefix, \"{fqdn}\", fqdn)\n\t\tc.Metrics.MetricPrefix = strings.ReplaceAll(c.Metrics.MetricPrefix, \"{host}\", hostname)\n\n\t\t// register our metrics with graphite\n\t\tmetrics.Graphite = graphite.New(c.Metrics.MetricInterval, c.Metrics.MetricPrefix, c.Metrics.MetricEndpoint, c.Metrics.MetricTimeout)\n\n\t\tif c.Metrics.Statsd != \"\" && c.Metrics.ExtendedStat {\n\t\t\tvar err error\n\n\t\t\tconfig := &statsd.ClientConfig{\n\t\t\t\tAddress: c.Metrics.Statsd,\n\t\t\t\tPrefix: c.Metrics.MetricPrefix,\n\t\t\t\tResInterval: 5 * time.Minute,\n\t\t\t\tUseBuffered: true,\n\t\t\t\tFlushInterval: 300 * time.Millisecond,\n\t\t\t}\n\n\t\t\tmetrics.Gstatsd, err = statsd.NewClientWithConfig(config)\n\t\t\tif err != nil {\n\t\t\t\tmetrics.Gstatsd = metrics.NullSender{}\n\n\t\t\t\tfmt.Fprintf(os.Stderr, \"statsd init: %v\\n\", err)\n\t\t\t}\n\t\t}\n\n\t\tmetrics.InitMetrics(&c.Metrics, c.ClickHouse.FindMaxQueries > 0, c.ClickHouse.TagsMaxQueries > 0)\n\t}\n\n\tmetrics.AutocompleteQMetric = metrics.InitQueryMetrics(\"tags\", &c.Metrics)\n\tmetrics.FindQMetric = metrics.InitQueryMetrics(\"find\", &c.Metrics)\n\n\tfor i := 0; i < len(c.DataTable); i++ {\n\t\tc.DataTable[i].QueryMetrics = metrics.InitQueryMetrics(c.DataTable[i].Table, &c.Metrics)\n\t}\n\n\tif c.ClickHouse.IndexTable != \"\" {\n\t\tmetrics.InitQueryMetrics(c.ClickHouse.IndexTable, &c.Metrics)\n\t}\n\n\tif c.ClickHouse.TaggedTable != \"\" {\n\t\tmetrics.InitQueryMetrics(c.ClickHouse.TaggedTable, &c.Metrics)\n\t}\n\n\tif c.ClickHouse.TagsCountTable != \"\" {\n\t\tmetrics.InitQueryMetrics(c.ClickHouse.TagsCountTable, &c.Metrics)\n\t}\n\n\treturn metrics.Graphite != nil\n}\n\nfunc (c *Config) GetUserFindLimiter(username string) limiter.ServerLimiter {\n\tif username != \"\" && len(c.ClickHouse.UserLimits) > 0 {\n\t\tif q, ok := c.ClickHouse.UserLimits[username]; ok {\n\t\t\treturn q.Limiter\n\t\t}\n\t}\n\n\treturn c.ClickHouse.FindLimiter\n}\n\nfunc (c *Config) GetUserTagsLimiter(username string) limiter.ServerLimiter {\n\tif username != \"\" && len(c.ClickHouse.UserLimits) > 0 {\n\t\tif q, ok := c.ClickHouse.UserLimits[username]; ok {\n\t\t\treturn q.Limiter\n\t\t}\n\t}\n\n\treturn c.ClickHouse.TagsLimiter\n}\n\n// search on sorted slice\nfunc GetQueryParam(a []QueryParam, duration time.Duration) int {\n\tif indx := binarySearchQueryParamLe(a, duration, 0, len(a)); indx == -1 {\n\t\treturn 0\n\t} else {\n\t\treturn indx\n\t}\n}\n" }, { "path": "config/config_test.go", "content": "package config\n\nimport (\n\t\"fmt\"\n\t\"io/fs\"\n\t\"math\"\n\t\"net/url\"\n\t\"os\"\n\t\"regexp\"\n\t\"regexp/syntax\"\n\t\"syscall\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/zapwriter\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n\n\t\"github.com/lomik/graphite-clickhouse/limiter\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n)\n\nfunc TestProcessDataTables(t *testing.T) {\n\ttype in struct {\n\t\ttable DataTable\n\t\ttableLegacy string\n\t}\n\n\ttype out struct {\n\t\ttables []DataTable\n\t\terr error\n\t}\n\n\ttype ctx map[string]bool\n\n\tregexpCompileWrapper := func(re string) *regexp.Regexp {\n\t\tr, _ := regexp.Compile(re)\n\t\treturn r\n\t}\n\n\ttests := []struct {\n\t\tname string\n\t\tin in\n\t\tout out\n\t}{\n\t\t{\n\t\t\tname: \"legacy table only\",\n\t\t\tin: in{\n\t\t\t\ttableLegacy: \"graphite.data\",\n\t\t\t},\n\t\t\tout: out{\n\t\t\t\t[]DataTable{\n\t\t\t\t\t{\n\t\t\t\t\t\tTable: \"graphite.data\",\n\t\t\t\t\t\tRollupConf: \"auto\",\n\t\t\t\t\t\tContextMap: ctx{\"graphite\": true, \"prometheus\": true},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tnil,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"legacy and normal tables\",\n\t\t\tin: in{\n\t\t\t\ttable: DataTable{Table: \"graphite.new_data\"},\n\t\t\t\ttableLegacy: \"graphite.data\",\n\t\t\t},\n\t\t\tout: out{\n\t\t\t\t[]DataTable{\n\t\t\t\t\t{\n\t\t\t\t\t\tTable: \"graphite.new_data\",\n\t\t\t\t\t\tContextMap: ctx{\"graphite\": true, \"prometheus\": true},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tTable: \"graphite.data\",\n\t\t\t\t\t\tRollupConf: \"auto\",\n\t\t\t\t\t\tContextMap: ctx{\"graphite\": true, \"prometheus\": true},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tnil,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"fail to compile TargetMatchAll\",\n\t\t\tin: in{\n\t\t\t\ttable: DataTable{Table: \"graphite.data\", TargetMatchAll: \"[2223\"},\n\t\t\t},\n\t\t\tout: out{\n\t\t\t\t[]DataTable{{Table: \"graphite.data\", TargetMatchAll: \"[2223\"}},\n\t\t\t\t&syntax.Error{Code: syntax.ErrMissingBracket, Expr: \"[2223\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"fail to compile TargetMatchAny\",\n\t\t\tin: in{\n\t\t\t\ttable: DataTable{Table: \"graphite.data\", TargetMatchAny: \"[2223\"},\n\t\t\t},\n\t\t\tout: out{\n\t\t\t\t[]DataTable{{Table: \"graphite.data\", TargetMatchAny: \"[2223\"}},\n\t\t\t\t&syntax.Error{Code: syntax.ErrMissingBracket, Expr: \"[2223\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"fail to compile TargetMatchAny\",\n\t\t\tin: in{\n\t\t\t\ttable: DataTable{Table: \"graphite.data\", TargetMatchAny: \"[2223\"},\n\t\t\t},\n\t\t\tout: out{\n\t\t\t\t[]DataTable{{Table: \"graphite.data\", TargetMatchAny: \"[2223\"}},\n\t\t\t\t&syntax.Error{Code: syntax.ErrMissingBracket, Expr: \"[2223\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"fail to read xml rollup\",\n\t\t\tin: in{\n\t\t\t\ttable: DataTable{Table: \"graphite.data\", RollupConf: \"/some/file/that/does/not/hopefully/exists/on/the/disk\"},\n\t\t\t},\n\t\t\tout: out{\n\t\t\t\t[]DataTable{{Table: \"graphite.data\", RollupConf: \"/some/file/that/does/not/hopefully/exists/on/the/disk\"}},\n\t\t\t\t&fs.PathError{Op: \"open\", Path: \"/some/file/that/does/not/hopefully/exists/on/the/disk\", Err: syscall.ENOENT},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"unknown context\",\n\t\t\tin: in{\n\t\t\t\ttable: DataTable{Table: \"graphite.data\", Context: []string{\"unexpected\"}},\n\t\t\t},\n\t\t\tout: out{\n\t\t\t\t[]DataTable{\n\t\t\t\t\t{\n\t\t\t\t\t\tTable: \"graphite.data\",\n\t\t\t\t\t\tContext: []string{\"unexpected\"},\n\t\t\t\t\t\tContextMap: ctx{},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tfmt.Errorf(\"unknown context \\\"unexpected\\\"\"),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"check all works\",\n\t\t\tin: in{\n\t\t\t\ttable: DataTable{\n\t\t\t\t\tTable: \"graphite.data\",\n\t\t\t\t\tReverse: true,\n\t\t\t\t\tTargetMatchAll: \"^.*[asdf][.].*\",\n\t\t\t\t\tTargetMatchAny: \"^.*{a|s|d|f}[.].*\",\n\t\t\t\t\tRollupConf: \"none\",\n\t\t\t\t\tRollupDefaultFunction: \"any\",\n\t\t\t\t\tRollupDefaultPrecision: 61,\n\t\t\t\t\tRollupUseReverted: true,\n\t\t\t\t\tContext: []string{\"prometheus\"},\n\t\t\t\t},\n\t\t\t\ttableLegacy: \"table\",\n\t\t\t},\n\t\t\tout: out{\n\t\t\t\t[]DataTable{\n\t\t\t\t\t{\n\t\t\t\t\t\tTable: \"graphite.data\",\n\t\t\t\t\t\tReverse: true,\n\t\t\t\t\t\tTargetMatchAll: \"^.*[asdf][.].*\",\n\t\t\t\t\t\tTargetMatchAny: \"^.*{a|s|d|f}[.].*\",\n\t\t\t\t\t\tTargetMatchAllRegexp: regexpCompileWrapper(\"^.*[asdf][.].*\"),\n\t\t\t\t\t\tTargetMatchAnyRegexp: regexpCompileWrapper(\"^.*{a|s|d|f}[.].*\"),\n\t\t\t\t\t\tRollupConf: \"none\",\n\t\t\t\t\t\tRollupDefaultFunction: \"any\",\n\t\t\t\t\t\tRollupDefaultPrecision: 61,\n\t\t\t\t\t\tRollupUseReverted: true,\n\t\t\t\t\t\tContext: []string{\"prometheus\"},\n\t\t\t\t\t\tContextMap: ctx{\"prometheus\": true},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tTable: \"table\",\n\t\t\t\t\t\tRollupConf: \"auto\",\n\t\t\t\t\t\tContextMap: ctx{\"graphite\": true, \"prometheus\": true},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tnil,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"unknown context\",\n\t\t\tin: in{\n\t\t\t\ttable: DataTable{Table: \"graphite.data\", Context: []string{\"unexpected\"}},\n\t\t\t},\n\t\t\tout: out{\n\t\t\t\t[]DataTable{\n\t\t\t\t\t{\n\t\t\t\t\t\tTable: \"graphite.data\",\n\t\t\t\t\t\tContext: []string{\"unexpected\"},\n\t\t\t\t\t\tContextMap: ctx{},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tfmt.Errorf(\"unknown context \\\"unexpected\\\"\"),\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tcfg := New()\n\t\t\tif test.in.table.Table != \"\" {\n\t\t\t\tcfg.DataTable = []DataTable{test.in.table}\n\t\t\t}\n\n\t\t\tif test.in.tableLegacy != \"\" {\n\t\t\t\tcfg.ClickHouse.DataTableLegacy = test.in.tableLegacy\n\t\t\t}\n\n\t\t\terr := cfg.ProcessDataTables()\n\t\t\tif err != nil {\n\t\t\t\tassert.Equal(t, test.out.err, err)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tassert.Equal(t, len(test.out.tables), len(cfg.DataTable))\n\t\t\t// it's difficult to check rollup.Rollup because Rules.updated field\n\t\t\t// We explicitly don't check it here\n\t\t\tfor i := range cfg.DataTable {\n\t\t\t\ttest.out.tables[i].Rollup = nil\n\t\t\t\tcfg.DataTable[i].Rollup = nil\n\t\t\t}\n\n\t\t\tassert.Equal(t, test.out.tables, cfg.DataTable)\n\t\t})\n\t}\n}\n\nfunc TestKnownDataTableContext(t *testing.T) {\n\tassert.Equal(t, map[string]bool{ContextGraphite: true, ContextPrometheus: true}, knownDataTableContext)\n}\n\nfunc TestReadConfig(t *testing.T) {\n\tbody := []byte(\n\t\t`[common]\nlisten = \"[::1]:9090\"\npprof-listen = \"127.0.0.1:9091\"\nmax-cpu = 15\nmax-metrics-in-find-answer = 13\nmax-metrics-per-target = 16\ntarget-blacklist = ['^blacklisted']\nmemory-return-interval = \"12s150ms\"\n\n\n[clickhouse]\nurl = \"http://somehost:8123\"\nindex-table = \"graphite_index\"\nindex-use-daily = false\nindex-reverse = \"direct\"\nindex-reverses = [\n {suffix = \"suf\", prefix = \"pref\", reverse = \"direct\"},\n {regex = \"^reg$\", reverse = \"reversed\"},\n]\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 5\ntagged-use-daily = false\ntree-table = \"tree\"\nreverse-tree-table = \"reversed_tree\"\ndate-tree-table = \"data_tree\"\ndate-tree-table-version = 2\ntag-table = \"tag_table\"\nextra-prefix = \"tum.pu-dum\"\ndata-table = \"data\"\nrollup-conf = \"none\"\nmax-data-points = 8000\ninternal-aggregation = true\ndata-timeout = \"64s\"\nindex-timeout = \"4s\"\ntree-timeout = \"5s\"\nconnect-timeout = \"2s\"\n\n# DataTable is tested in TestProcessDataTables\n# [[data-table]]\n# table = \"another_data\"\n# rollup-conf = \"auto\"\n# rollup-conf-table = \"another_table\"\n\n[tags]\nrules = \"filename\"\ndate = \"2012-12-12\"\nextra-where = \"AND case\"\ninput-file = \"input\"\noutput-file = \"output\"\nthreads = 5\ncompression = \"zstd\"\nversion = 42\nselect-chunks-count = 15\n\n[carbonlink]\nserver = \"server:3333\"\nthreads-per-request = 5\nconnect-timeout = \"250ms\"\nquery-timeout = \"350ms\"\ntotal-timeout = \"800ms\"\n\n[prometheus]\nlisten = \":9092\"\nexternal-url = \"https://server:3456/uri\"\npage-title = \"Prometheus Time Series\"\nlookback-delta = \"5m\"\n\n[debug]\ndirectory = \"tests_tmp\"\ndirectory-perm = 0o755\nexternal-data-perm = 0o640\n\n[[logging]]\nlogger = \"debugger\"\nfile = \"stdout\"\nlevel = \"debug\"\nencoding = \"console\"\nencoding-time = \"iso8601\"\nencoding-duration = \"string\"\nsample-tick = \"5ms\"\nsample-initial = 1\nsample-thereafter = 2\n\n[[logging]]\nlogger = \"logger\"\nfile = \"tests_tmp/logger.txt\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"epoch\"\nencoding-duration = \"seconds\"\nsample-tick = \"50ms\"\nsample-initial = 10\nsample-thereafter = 12\n`,\n\t)\n\tconfig, _, err := Unmarshal(body, false)\n\texpected := New()\n\n\trequire.NoError(t, err)\n\n\t// Common\n\texpected.Common = Common{\n\t\tListen: \"[::1]:9090\",\n\t\tPprofListen: \"127.0.0.1:9091\",\n\t\tMaxCPU: 15,\n\t\tMaxMetricsInFindAnswer: 13,\n\t\tMaxMetricsPerTarget: 16,\n\t\tTargetBlacklist: []string{\"^blacklisted\"},\n\t\tBlacklist: make([]*regexp.Regexp, 1),\n\t\tMemoryReturnInterval: 12150000000,\n\t\tFindCacheConfig: CacheConfig{\n\t\t\tType: \"null\",\n\t\t\tDefaultTimeoutSec: 0,\n\t\t\tShortTimeoutSec: 0,\n\t\t},\n\t\tDegragedMultiply: 4.0,\n\t\tDegragedLoad: 1.0,\n\t}\n\texpected.Metrics = metrics.Config{}\n\n\tr, _ := regexp.Compile(expected.Common.TargetBlacklist[0])\n\texpected.Common.Blacklist[0] = r\n\tassert.Equal(t, expected.Common, config.Common)\n\tassert.Equal(t, expected.Metrics, config.Metrics)\n\n\t// ClickHouse\n\texpected.ClickHouse = ClickHouse{\n\t\tURL: \"http://somehost:8123\",\n\t\tDataTimeout: 64000000000,\n\t\tQueryParams: []QueryParam{\n\t\t\t{\n\t\t\t\tDuration: 0,\n\t\t\t\tURL: \"http://somehost:8123\",\n\t\t\t\tDataTimeout: 64000000000,\n\t\t\t\tLimiter: limiter.NoopLimiter{},\n\t\t\t},\n\t\t},\n\t\tProgressSendingInterval: 10 * time.Second,\n\t\tFindLimiter: limiter.NoopLimiter{},\n\t\tTagsLimiter: limiter.NoopLimiter{},\n\t\tIndexTable: \"graphite_index\",\n\t\tIndexReverse: \"direct\",\n\t\tIndexReverses: make(IndexReverses, 2),\n\t\tIndexTimeout: 4000000000,\n\t\tTaggedTable: \"graphite_tags\",\n\t\tTaggedAutocompleDays: 5,\n\t\tTreeTable: \"tree\",\n\t\tReverseTreeTable: \"reversed_tree\",\n\t\tDateTreeTable: \"data_tree\",\n\t\tDateTreeTableVersion: 2,\n\t\tTreeTimeout: 5000000000,\n\t\tTagTable: \"tag_table\",\n\t\tExtraPrefix: \"tum.pu-dum\",\n\t\tConnectTimeout: 2000000000,\n\t\tDataTableLegacy: \"data\",\n\t\tRollupConfLegacy: \"none\",\n\t\tMaxDataPoints: 8000,\n\t\tInternalAggregation: true,\n\t}\n\texpected.ClickHouse.IndexReverses[0] = &IndexReverseRule{\"suf\", \"pref\", \"\", nil, \"direct\"}\n\tr, _ = regexp.Compile(\"^reg$\")\n\texpected.ClickHouse.IndexReverses[1] = &IndexReverseRule{\"\", \"\", \"^reg$\", r, \"reversed\"}\n\tassert.Equal(t, expected.ClickHouse, config.ClickHouse)\n\n\t// Tags\n\texpected.Tags = Tags{\"filename\", \"2012-12-12\", \"AND case\", \"input\", \"output\", 5, \"zstd\", 42, 15}\n\tassert.Equal(t, expected.Tags, config.Tags)\n\n\t// Carbonlink\n\texpected.Carbonlink = Carbonlink{\"server:3333\", 5, 2, 250000000, 350000000, 800000000}\n\tassert.Equal(t, expected.Carbonlink, config.Carbonlink)\n\n\t// Prometheus\n\texpected.Prometheus = Prometheus{Listen: \":9092\", ExternalURLRaw: \"https://server:3456/uri\", PageTitle: \"Prometheus Time Series\", LookbackDelta: 5 * time.Minute, RemoteReadConcurrencyLimit: 10}\n\tu, _ := url.Parse(expected.Prometheus.ExternalURLRaw)\n\texpected.Prometheus.ExternalURL = u\n\tassert.Equal(t, expected.Prometheus, config.Prometheus)\n\n\t// Debug\n\texpected.Debug = Debug{\"tests_tmp\", os.FileMode(0755), os.FileMode(0640)}\n\tassert.Equal(t, expected.Debug, config.Debug)\n\tassert.DirExists(t, \"tests_tmp\")\n\n\t// Logger\n\texpected.Logging = make([]zapwriter.Config, 2)\n\texpected.Logging[0] = zapwriter.Config{\n\t\tLogger: \"debugger\",\n\t\tFile: \"stdout\",\n\t\tLevel: \"debug\",\n\t\tEncoding: \"console\",\n\t\tEncodingTime: \"iso8601\",\n\t\tEncodingDuration: \"string\",\n\t\tSampleTick: \"5ms\",\n\t\tSampleInitial: 1,\n\t\tSampleThereafter: 2,\n\t}\n\texpected.Logging[1] = zapwriter.Config{\n\t\tLogger: \"logger\",\n\t\tFile: \"tests_tmp/logger.txt\",\n\t\tLevel: \"info\",\n\t\tEncoding: \"json\",\n\t\tEncodingTime: \"epoch\",\n\t\tEncodingDuration: \"seconds\",\n\t\tSampleTick: \"50ms\",\n\t\tSampleInitial: 10,\n\t\tSampleThereafter: 12,\n\t}\n\tassert.Equal(t, expected.Logging, config.Logging)\n}\n\nfunc TestReadConfigGraphiteWithLimiter(t *testing.T) {\n\tbody := []byte(\n\t\t`[common]\nlisten = \"[::1]:9090\"\npprof-listen = \"127.0.0.1:9091\"\nmax-cpu = 15\nmax-metrics-in-find-answer = 13\nmax-metrics-per-target = 16\ntarget-blacklist = ['^blacklisted']\nmemory-return-interval = \"12s150ms\"\n\n[metrics]\nmetric-endpoint = \"127.0.0.1:2003\"\nmetric-interval = \"10s\"\nmetric-prefix = \"graphite\"\nranges = { \"1h\" = \"1h\", \"3d\" = \"72h\", \"7d\" = \"168h\", \"30d\" = \"720h\", \"90d\" = \"2160h\" }\n\n[clickhouse]\nurl = \"http://somehost:8123\"\nindex-table = \"graphite_index\"\nindex-use-daily = false\nindex-reverse = \"direct\"\nindex-reverses = [\n {suffix = \"suf\", prefix = \"pref\", reverse = \"direct\"},\n {regex = \"^reg$\", reverse = \"reversed\"},\n]\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 5\ntagged-use-daily = false\ntree-table = \"tree\"\nreverse-tree-table = \"reversed_tree\"\ndate-tree-table = \"data_tree\"\ndate-tree-table-version = 2\ntag-table = \"tag_table\"\nextra-prefix = \"tum.pu-dum\"\ndata-table = \"data\"\nrollup-conf = \"none\"\nmax-data-points = 8000\ninternal-aggregation = true\ndata-timeout = \"64s\"\nindex-timeout = \"4s\"\ntree-timeout = \"5s\"\nconnect-timeout = \"2s\"\n\nrender-max-queries = 1000\nrender-concurrent-queries = 10\nfind-max-queries = 200\nfind-concurrent-queries = 8\ntags-max-queries = 50\ntags-concurrent-queries = 4\n\nquery-params = [\n\t{\n\t\tduration = \"72h\",\n\t\turl = \"http://localhost:8123/?max_rows_to_read=20000\"\n\t}\n]\n\nuser-limits = {\n\t\"alert\" = {\n\t\tmax-queries = 200,\n\t\tconcurrent-queries = 10\n\t}\n}\n\n# DataTable is tested in TestProcessDataTables\n# [[data-table]]\n# table = \"another_data\"\n# rollup-conf = \"auto\"\n# rollup-conf-table = \"another_table\"\n\n[tags]\nrules = \"filename\"\ndate = \"2012-12-12\"\nextra-where = \"AND case\"\ninput-file = \"input\"\noutput-file = \"output\"\nthreads = 5\ncompression = \"zstd\"\nversion = 42\nselect-chunks-count = 15\n\n[carbonlink]\nserver = \"server:3333\"\nthreads-per-request = 5\nconnect-timeout = \"250ms\"\nquery-timeout = \"350ms\"\ntotal-timeout = \"800ms\"\n\n[prometheus]\nlisten = \":9092\"\nexternal-url = \"https://server:3456/uri\"\npage-title = \"Prometheus Time Series\"\nlookback-delta = \"5m\"\n\n[debug]\ndirectory = \"tests_tmp\"\ndirectory-perm = 0o755\nexternal-data-perm = 0o640\n\n[[logging]]\nlogger = \"debugger\"\nfile = \"stdout\"\nlevel = \"debug\"\nencoding = \"console\"\nencoding-time = \"iso8601\"\nencoding-duration = \"string\"\nsample-tick = \"5ms\"\nsample-initial = 1\nsample-thereafter = 2\n\n[[logging]]\nlogger = \"logger\"\nfile = \"tests_tmp/logger.txt\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"epoch\"\nencoding-duration = \"seconds\"\nsample-tick = \"50ms\"\nsample-initial = 10\nsample-thereafter = 12\n`,\n\t)\n\tconfig, _, err := Unmarshal(body, false)\n\texpected := New()\n\n\trequire.NoError(t, err)\n\tassert.NotNil(t, metrics.Graphite)\n\tmetrics.Graphite = nil\n\n\t// Common\n\texpected.Common = Common{\n\t\tListen: \"[::1]:9090\",\n\t\tPprofListen: \"127.0.0.1:9091\",\n\t\tMaxCPU: 15,\n\t\tMaxMetricsInFindAnswer: 13,\n\t\tMaxMetricsPerTarget: 16,\n\t\tTargetBlacklist: []string{\"^blacklisted\"},\n\t\tBlacklist: make([]*regexp.Regexp, 1),\n\t\tMemoryReturnInterval: 12150000000,\n\t\tFindCacheConfig: CacheConfig{\n\t\t\tType: \"null\",\n\t\t\tDefaultTimeoutSec: 0,\n\t\t\tShortTimeoutSec: 0,\n\t\t},\n\t\tDegragedMultiply: 4.0,\n\t\tDegragedLoad: 1.0,\n\t}\n\texpected.Metrics = metrics.Config{\n\t\tMetricEndpoint: \"127.0.0.1:2003\",\n\t\tMetricInterval: 10 * time.Second,\n\t\tMetricTimeout: time.Second,\n\t\tMetricPrefix: \"graphite\",\n\t\tBucketsWidth: []int64{200, 500, 1000, 2000, 3000, 5000, 7000, 10000, 15000, 20000, 25000, 30000, 40000, 50000, 60000},\n\t\tBucketsLabels: []string{\n\t\t\t\"_to_200ms\",\n\t\t\t\"_to_500ms\",\n\t\t\t\"_to_1000ms\",\n\t\t\t\"_to_2000ms\",\n\t\t\t\"_to_3000ms\",\n\t\t\t\"_to_5000ms\",\n\t\t\t\"_to_7000ms\",\n\t\t\t\"_to_10000ms\",\n\t\t\t\"_to_15000ms\",\n\t\t\t\"_to_20000ms\",\n\t\t\t\"_to_25000ms\",\n\t\t\t\"_to_30000ms\",\n\t\t\t\"_to_40000ms\",\n\t\t\t\"_to_50000ms\",\n\t\t\t\"_to_60000ms\",\n\t\t\t\"_to_inf\",\n\t\t},\n\t\t// until-from = { \"1h\" = \"1h\", \"3d\" = \"72h\", \"7d\" = \"168h\", \"30d\" = \"720h\", \"90d\" = \"2160h\" }\n\t\tRanges: map[string]time.Duration{\n\t\t\t\"1h\": time.Hour,\n\t\t\t\"3d\": 72 * time.Hour,\n\t\t\t\"7d\": 168 * time.Hour,\n\t\t\t\"30d\": 720 * time.Hour,\n\t\t\t\"90d\": 2160 * time.Hour,\n\t\t},\n\t\tRangeNames: []string{\"1h\", \"3d\", \"7d\", \"30d\", \"90d\", \"history\"},\n\t\tRangeS: []int64{3600, 259200, 604800, 2592000, 7776000, math.MaxInt64},\n\t}\n\tr, _ := regexp.Compile(expected.Common.TargetBlacklist[0])\n\texpected.Common.Blacklist[0] = r\n\tassert.Equal(t, expected.Common, config.Common)\n\tassert.Equal(t, expected.Metrics, config.Metrics)\n\n\t// ClickHouse\n\texpected.ClickHouse = ClickHouse{\n\t\tURL: \"http://somehost:8123\",\n\t\tDataTimeout: 64000000000,\n\t\tQueryParams: []QueryParam{\n\t\t\t{\n\t\t\t\tDuration: 0,\n\t\t\t\tURL: \"http://somehost:8123\",\n\t\t\t\tDataTimeout: 64000000000,\n\t\t\t\tMaxQueries: 1000,\n\t\t\t\tConcurrentQueries: 10,\n\t\t\t},\n\t\t\t{\n\t\t\t\tDuration: 72 * time.Hour,\n\t\t\t\tURL: \"http://localhost:8123/?max_rows_to_read=20000\",\n\t\t\t\tDataTimeout: 64000000000,\n\t\t\t\tLimiter: limiter.NoopLimiter{},\n\t\t\t},\n\t\t},\n\t\tProgressSendingInterval: 10 * time.Second,\n\t\tRenderMaxQueries: 1000,\n\t\tRenderConcurrentQueries: 10,\n\t\tFindMaxQueries: 200,\n\t\tFindConcurrentQueries: 8,\n\t\tTagsMaxQueries: 50,\n\t\tTagsConcurrentQueries: 4,\n\t\tUserLimits: map[string]UserLimits{\n\t\t\t\"alert\": {\n\t\t\t\tMaxQueries: 200,\n\t\t\t\tConcurrentQueries: 10,\n\t\t\t},\n\t\t},\n\t\tIndexTable: \"graphite_index\",\n\t\tIndexReverse: \"direct\",\n\t\tIndexReverses: make(IndexReverses, 2),\n\t\tIndexTimeout: 4000000000,\n\t\tTaggedTable: \"graphite_tags\",\n\t\tTaggedAutocompleDays: 5,\n\t\tTreeTable: \"tree\",\n\t\tReverseTreeTable: \"reversed_tree\",\n\t\tDateTreeTable: \"data_tree\",\n\t\tDateTreeTableVersion: 2,\n\t\tTreeTimeout: 5000000000,\n\t\tTagTable: \"tag_table\",\n\t\tExtraPrefix: \"tum.pu-dum\",\n\t\tConnectTimeout: 2000000000,\n\t\tDataTableLegacy: \"data\",\n\t\tRollupConfLegacy: \"none\",\n\t\tMaxDataPoints: 8000,\n\t\tInternalAggregation: true,\n\t}\n\texpected.ClickHouse.IndexReverses[0] = &IndexReverseRule{\"suf\", \"pref\", \"\", nil, \"direct\"}\n\tr, _ = regexp.Compile(\"^reg$\")\n\texpected.ClickHouse.IndexReverses[1] = &IndexReverseRule{\"\", \"\", \"^reg$\", r, \"reversed\"}\n\n\tfor i := range config.ClickHouse.QueryParams {\n\t\tif _, ok := config.ClickHouse.QueryParams[i].Limiter.(*limiter.WLimiter); ok && config.ClickHouse.QueryParams[i].MaxQueries > 0 && config.ClickHouse.QueryParams[i].ConcurrentQueries > 0 {\n\t\t\tconfig.ClickHouse.QueryParams[i].Limiter = nil\n\t\t}\n\t}\n\n\tif _, ok := config.ClickHouse.FindLimiter.(*limiter.WLimiter); ok && config.ClickHouse.FindMaxQueries > 0 && config.ClickHouse.FindConcurrentQueries > 0 {\n\t\tconfig.ClickHouse.FindLimiter = nil\n\t}\n\n\tif _, ok := config.ClickHouse.TagsLimiter.(*limiter.WLimiter); ok && config.ClickHouse.TagsMaxQueries > 0 && config.ClickHouse.TagsConcurrentQueries > 0 {\n\t\tconfig.ClickHouse.TagsLimiter = nil\n\t}\n\n\tfor u, q := range config.ClickHouse.UserLimits {\n\t\tif _, ok := q.Limiter.(*limiter.WLimiter); ok && q.MaxQueries > 0 && q.ConcurrentQueries > 0 {\n\t\t\tq.Limiter = nil\n\t\t\tconfig.ClickHouse.UserLimits[u] = q\n\t\t}\n\t}\n\n\tassert.Equal(t, expected.ClickHouse, config.ClickHouse)\n\n\t// Tags\n\texpected.Tags = Tags{\"filename\", \"2012-12-12\", \"AND case\", \"input\", \"output\", 5, \"zstd\", 42, 15}\n\tassert.Equal(t, expected.Tags, config.Tags)\n\n\t// Carbonlink\n\texpected.Carbonlink = Carbonlink{\"server:3333\", 5, 2, 250000000, 350000000, 800000000}\n\tassert.Equal(t, expected.Carbonlink, config.Carbonlink)\n\n\t// Prometheus\n\texpected.Prometheus = Prometheus{Listen: \":9092\", ExternalURLRaw: \"https://server:3456/uri\", PageTitle: \"Prometheus Time Series\", LookbackDelta: 5 * time.Minute, RemoteReadConcurrencyLimit: 10}\n\tu, _ := url.Parse(expected.Prometheus.ExternalURLRaw)\n\texpected.Prometheus.ExternalURL = u\n\tassert.Equal(t, expected.Prometheus, config.Prometheus)\n\n\t// Debug\n\texpected.Debug = Debug{\"tests_tmp\", os.FileMode(0755), os.FileMode(0640)}\n\tassert.Equal(t, expected.Debug, config.Debug)\n\tassert.DirExists(t, \"tests_tmp\")\n\n\t// Logger\n\texpected.Logging = make([]zapwriter.Config, 2)\n\texpected.Logging[0] = zapwriter.Config{\n\t\tLogger: \"debugger\",\n\t\tFile: \"stdout\",\n\t\tLevel: \"debug\",\n\t\tEncoding: \"console\",\n\t\tEncodingTime: \"iso8601\",\n\t\tEncodingDuration: \"string\",\n\t\tSampleTick: \"5ms\",\n\t\tSampleInitial: 1,\n\t\tSampleThereafter: 2,\n\t}\n\texpected.Logging[1] = zapwriter.Config{\n\t\tLogger: \"logger\",\n\t\tFile: \"tests_tmp/logger.txt\",\n\t\tLevel: \"info\",\n\t\tEncoding: \"json\",\n\t\tEncodingTime: \"epoch\",\n\t\tEncodingDuration: \"seconds\",\n\t\tSampleTick: \"50ms\",\n\t\tSampleInitial: 10,\n\t\tSampleThereafter: 12,\n\t}\n\tassert.Equal(t, expected.Logging, config.Logging)\n\n\tmetrics.FindRequestMetric = nil\n\tmetrics.TagsRequestMetric = nil\n\tmetrics.RenderRequestMetric = nil\n\tmetrics.UnregisterAll()\n}\n\nfunc TestReadConfigGraphiteWithALimiter(t *testing.T) {\n\tbody := []byte(\n\t\t`[common]\nlisten = \"[::1]:9090\"\npprof-listen = \"127.0.0.1:9091\"\nmax-cpu = 15\nmax-metrics-in-find-answer = 13\nmax-metrics-per-target = 16\ntarget-blacklist = ['^blacklisted']\nmemory-return-interval = \"12s150ms\"\n\n[metrics]\nmetric-endpoint = \"127.0.0.1:2003\"\nmetric-interval = \"10s\"\nmetric-prefix = \"graphite\"\nranges = { \"1h\" = \"1h\", \"3d\" = \"72h\", \"7d\" = \"168h\", \"30d\" = \"720h\", \"90d\" = \"2160h\" }\n\n[clickhouse]\nurl = \"http://somehost:8123\"\nindex-table = \"graphite_index\"\nindex-use-daily = false\nindex-reverse = \"direct\"\nindex-reverses = [\n {suffix = \"suf\", prefix = \"pref\", reverse = \"direct\"},\n {regex = \"^reg$\", reverse = \"reversed\"},\n]\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 5\ntagged-use-daily = false\ntree-table = \"tree\"\nreverse-tree-table = \"reversed_tree\"\ndate-tree-table = \"data_tree\"\ndate-tree-table-version = 2\ntag-table = \"tag_table\"\nextra-prefix = \"tum.pu-dum\"\ndata-table = \"data\"\nrollup-conf = \"none\"\nmax-data-points = 8000\ninternal-aggregation = true\ndata-timeout = \"64s\"\nindex-timeout = \"4s\"\ntree-timeout = \"5s\"\nconnect-timeout = \"2s\"\n\nrender-max-queries = 1000\nrender-concurrent-queries = 10\nrender-adaptive-queries = 4\nfind-max-queries = 200\nfind-concurrent-queries = 8\ntags-max-queries = 50\ntags-concurrent-queries = 4\ntags-adaptive-queries = 3\n\nquery-params = [\n\t{\n\t\tduration = \"72h\",\n\t\turl = \"http://localhost:8123/?max_rows_to_read=20000\",\n\t\tconcurrent-queries = 4,\n\t\tadaptive-queries = 6\n\t}\n]\n\nuser-limits = {\n\t\"alert\" = {\n\t\tmax-queries = 200,\n\t\tconcurrent-queries = 10,\n\t\tadaptive-queries = 5\n\t}\n}\n\n# DataTable is tested in TestProcessDataTables\n# [[data-table]]\n# table = \"another_data\"\n# rollup-conf = \"auto\"\n# rollup-conf-table = \"another_table\"\n\n[tags]\nrules = \"filename\"\ndate = \"2012-12-12\"\nextra-where = \"AND case\"\ninput-file = \"input\"\noutput-file = \"output\"\nthreads = 5\ncompression = \"zstd\"\nversion = 42\nselect-chunks-count = 15\n\n[carbonlink]\nserver = \"server:3333\"\nthreads-per-request = 5\nconnect-timeout = \"250ms\"\nquery-timeout = \"350ms\"\ntotal-timeout = \"800ms\"\n\n[prometheus]\nlisten = \":9092\"\nexternal-url = \"https://server:3456/uri\"\npage-title = \"Prometheus Time Series\"\nlookback-delta = \"5m\"\n\n[debug]\ndirectory = \"tests_tmp\"\ndirectory-perm = 0o755\nexternal-data-perm = 0o640\n\n[[logging]]\nlogger = \"debugger\"\nfile = \"stdout\"\nlevel = \"debug\"\nencoding = \"console\"\nencoding-time = \"iso8601\"\nencoding-duration = \"string\"\nsample-tick = \"5ms\"\nsample-initial = 1\nsample-thereafter = 2\n\n[[logging]]\nlogger = \"logger\"\nfile = \"tests_tmp/logger.txt\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"epoch\"\nencoding-duration = \"seconds\"\nsample-tick = \"50ms\"\nsample-initial = 10\nsample-thereafter = 12\n`,\n\t)\n\tconfig, _, err := Unmarshal(body, false)\n\texpected := New()\n\n\trequire.NoError(t, err)\n\tassert.NotNil(t, metrics.Graphite)\n\tmetrics.Graphite = nil\n\n\t// Common\n\texpected.Common = Common{\n\t\tListen: \"[::1]:9090\",\n\t\tPprofListen: \"127.0.0.1:9091\",\n\t\tMaxCPU: 15,\n\t\tMaxMetricsInFindAnswer: 13,\n\t\tMaxMetricsPerTarget: 16,\n\t\tTargetBlacklist: []string{\"^blacklisted\"},\n\t\tBlacklist: make([]*regexp.Regexp, 1),\n\t\tMemoryReturnInterval: 12150000000,\n\t\tFindCacheConfig: CacheConfig{\n\t\t\tType: \"null\",\n\t\t\tDefaultTimeoutSec: 0,\n\t\t\tShortTimeoutSec: 0,\n\t\t},\n\t\tDegragedMultiply: 4.0,\n\t\tDegragedLoad: 1.0,\n\t}\n\texpected.Metrics = metrics.Config{\n\t\tMetricEndpoint: \"127.0.0.1:2003\",\n\t\tMetricInterval: 10 * time.Second,\n\t\tMetricTimeout: time.Second,\n\t\tMetricPrefix: \"graphite\",\n\t\tBucketsWidth: []int64{200, 500, 1000, 2000, 3000, 5000, 7000, 10000, 15000, 20000, 25000, 30000, 40000, 50000, 60000},\n\t\tBucketsLabels: []string{\n\t\t\t\"_to_200ms\",\n\t\t\t\"_to_500ms\",\n\t\t\t\"_to_1000ms\",\n\t\t\t\"_to_2000ms\",\n\t\t\t\"_to_3000ms\",\n\t\t\t\"_to_5000ms\",\n\t\t\t\"_to_7000ms\",\n\t\t\t\"_to_10000ms\",\n\t\t\t\"_to_15000ms\",\n\t\t\t\"_to_20000ms\",\n\t\t\t\"_to_25000ms\",\n\t\t\t\"_to_30000ms\",\n\t\t\t\"_to_40000ms\",\n\t\t\t\"_to_50000ms\",\n\t\t\t\"_to_60000ms\",\n\t\t\t\"_to_inf\",\n\t\t},\n\t\t// until-from = { \"1h\" = \"1h\", \"3d\" = \"72h\", \"7d\" = \"168h\", \"30d\" = \"720h\", \"90d\" = \"2160h\" }\n\t\tRanges: map[string]time.Duration{\n\t\t\t\"1h\": time.Hour,\n\t\t\t\"3d\": 72 * time.Hour,\n\t\t\t\"7d\": 168 * time.Hour,\n\t\t\t\"30d\": 720 * time.Hour,\n\t\t\t\"90d\": 2160 * time.Hour,\n\t\t},\n\t\tRangeNames: []string{\"1h\", \"3d\", \"7d\", \"30d\", \"90d\", \"history\"},\n\t\tRangeS: []int64{3600, 259200, 604800, 2592000, 7776000, math.MaxInt64},\n\t}\n\tr, _ := regexp.Compile(expected.Common.TargetBlacklist[0])\n\texpected.Common.Blacklist[0] = r\n\tassert.Equal(t, expected.Common, config.Common)\n\tassert.Equal(t, expected.Metrics, config.Metrics)\n\n\t// ClickHouse\n\texpected.ClickHouse = ClickHouse{\n\t\tURL: \"http://somehost:8123\",\n\t\tDataTimeout: 64000000000,\n\t\tQueryParams: []QueryParam{\n\t\t\t{\n\t\t\t\tDuration: 0,\n\t\t\t\tURL: \"http://somehost:8123\",\n\t\t\t\tDataTimeout: 64000000000,\n\t\t\t\tMaxQueries: 1000,\n\t\t\t\tConcurrentQueries: 10,\n\t\t\t\tAdaptiveQueries: 4,\n\t\t\t},\n\t\t\t{\n\t\t\t\tDuration: 72 * time.Hour,\n\t\t\t\tURL: \"http://localhost:8123/?max_rows_to_read=20000\",\n\t\t\t\tDataTimeout: 64000000000,\n\t\t\t\tConcurrentQueries: 4,\n\t\t\t\tAdaptiveQueries: 6,\n\t\t\t},\n\t\t},\n\t\tProgressSendingInterval: 10 * time.Second,\n\t\tRenderMaxQueries: 1000,\n\t\tRenderConcurrentQueries: 10,\n\t\tRenderAdaptiveQueries: 4,\n\t\tFindMaxQueries: 200,\n\t\tFindConcurrentQueries: 8,\n\t\tTagsMaxQueries: 50,\n\t\tTagsConcurrentQueries: 4,\n\t\tTagsAdaptiveQueries: 3,\n\t\tUserLimits: map[string]UserLimits{\n\t\t\t\"alert\": {\n\t\t\t\tMaxQueries: 200,\n\t\t\t\tConcurrentQueries: 10,\n\t\t\t\tAdaptiveQueries: 5,\n\t\t\t},\n\t\t},\n\t\tIndexTable: \"graphite_index\",\n\t\tIndexReverse: \"direct\",\n\t\tIndexReverses: make(IndexReverses, 2),\n\t\tIndexTimeout: 4000000000,\n\t\tTaggedTable: \"graphite_tags\",\n\t\tTaggedAutocompleDays: 5,\n\t\tTreeTable: \"tree\",\n\t\tReverseTreeTable: \"reversed_tree\",\n\t\tDateTreeTable: \"data_tree\",\n\t\tDateTreeTableVersion: 2,\n\t\tTreeTimeout: 5000000000,\n\t\tTagTable: \"tag_table\",\n\t\tExtraPrefix: \"tum.pu-dum\",\n\t\tConnectTimeout: 2000000000,\n\t\tDataTableLegacy: \"data\",\n\t\tRollupConfLegacy: \"none\",\n\t\tMaxDataPoints: 8000,\n\t\tInternalAggregation: true,\n\t}\n\texpected.ClickHouse.IndexReverses[0] = &IndexReverseRule{\"suf\", \"pref\", \"\", nil, \"direct\"}\n\tr, _ = regexp.Compile(\"^reg$\")\n\texpected.ClickHouse.IndexReverses[1] = &IndexReverseRule{\"\", \"\", \"^reg$\", r, \"reversed\"}\n\n\tfor i := range config.ClickHouse.QueryParams {\n\t\tif _, ok := config.ClickHouse.QueryParams[i].Limiter.(*limiter.ALimiter); ok {\n\t\t\tconfig.ClickHouse.QueryParams[i].Limiter = nil\n\t\t}\n\t}\n\n\tif _, ok := config.ClickHouse.FindLimiter.(*limiter.WLimiter); ok {\n\t\tconfig.ClickHouse.FindLimiter = nil\n\t}\n\n\tif _, ok := config.ClickHouse.TagsLimiter.(*limiter.ALimiter); ok {\n\t\tconfig.ClickHouse.TagsLimiter = nil\n\t}\n\n\tfor u, q := range config.ClickHouse.UserLimits {\n\t\tif _, ok := q.Limiter.(*limiter.ALimiter); ok {\n\t\t\tq.Limiter = nil\n\t\t\tconfig.ClickHouse.UserLimits[u] = q\n\t\t}\n\t}\n\n\tassert.Equal(t, expected.ClickHouse, config.ClickHouse)\n\n\t// Tags\n\texpected.Tags = Tags{\"filename\", \"2012-12-12\", \"AND case\", \"input\", \"output\", 5, \"zstd\", 42, 15}\n\tassert.Equal(t, expected.Tags, config.Tags)\n\n\t// Carbonlink\n\texpected.Carbonlink = Carbonlink{\"server:3333\", 5, 2, 250000000, 350000000, 800000000}\n\tassert.Equal(t, expected.Carbonlink, config.Carbonlink)\n\n\t// Prometheus\n\texpected.Prometheus = Prometheus{Listen: \":9092\", ExternalURLRaw: \"https://server:3456/uri\", PageTitle: \"Prometheus Time Series\", LookbackDelta: 5 * time.Minute, RemoteReadConcurrencyLimit: 10}\n\tu, _ := url.Parse(expected.Prometheus.ExternalURLRaw)\n\texpected.Prometheus.ExternalURL = u\n\tassert.Equal(t, expected.Prometheus, config.Prometheus)\n\n\t// Debug\n\texpected.Debug = Debug{\"tests_tmp\", os.FileMode(0755), os.FileMode(0640)}\n\tassert.Equal(t, expected.Debug, config.Debug)\n\tassert.DirExists(t, \"tests_tmp\")\n\n\t// Logger\n\texpected.Logging = make([]zapwriter.Config, 2)\n\texpected.Logging[0] = zapwriter.Config{\n\t\tLogger: \"debugger\",\n\t\tFile: \"stdout\",\n\t\tLevel: \"debug\",\n\t\tEncoding: \"console\",\n\t\tEncodingTime: \"iso8601\",\n\t\tEncodingDuration: \"string\",\n\t\tSampleTick: \"5ms\",\n\t\tSampleInitial: 1,\n\t\tSampleThereafter: 2,\n\t}\n\texpected.Logging[1] = zapwriter.Config{\n\t\tLogger: \"logger\",\n\t\tFile: \"tests_tmp/logger.txt\",\n\t\tLevel: \"info\",\n\t\tEncoding: \"json\",\n\t\tEncodingTime: \"epoch\",\n\t\tEncodingDuration: \"seconds\",\n\t\tSampleTick: \"50ms\",\n\t\tSampleInitial: 10,\n\t\tSampleThereafter: 12,\n\t}\n\tassert.Equal(t, expected.Logging, config.Logging)\n\n\tmetrics.FindRequestMetric = nil\n\tmetrics.TagsRequestMetric = nil\n\tmetrics.RenderRequestMetric = nil\n\tmetrics.UnregisterAll()\n}\n\nfunc TestGetQueryParamBroken(t *testing.T) {\n\tconfig :=\n\t\t[]byte(`\n\t\t\t[clickhouse]\n\t\t\turl = \"http://localhost:8123/?max_rows_to_read=1000\"\n\t\t\tdata-timeout = \"20s\"\n\t\t\tquery-params = [\n\t\t\t {\n\t\t\t\tduration = \"72h\",\n\t\t\t\turl = \"http://localhost:8123/?max_rows_to_read=20000\",\n\t\t\t },\n\t\t\t]`)\n\n\t_, _, err := Unmarshal(config, false)\n\tassert.Error(t, err)\n\n\tconfig =\n\t\t[]byte(`\n\t\t\t[clickhouse]\n\t\t\turl = \"http://localhost:8123/?max_rows_to_read=1000\"\n\t\t\tdata-timeout = \"20s\"\n\t\t\tquery-params = [\n\t\t\t {\n\t\t\t\turl = \"http://localhost:8123/?max_rows_to_read=20000\",\n\t\t\t\tdata-timeout = \"60s\"\n\t\t\t },\n\t\t\t]`)\n\n\t_, _, err = Unmarshal(config, false)\n\tassert.Error(t, err)\n}\n\nfunc TestGetQueryParam(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tconfig []byte\n\t\tdurations []time.Duration\n\t\twantParams []QueryParam\n\t\twantUserParams map[string]QueryParam\n\t}{\n\t\t{\n\t\t\tname: \"Only default\",\n\t\t\tconfig: []byte(`\n\t\t\t[clickhouse]\n\t\t\turl = \"http://localhost:8123/?max_rows_to_read=1000\"\n\t\t\tdata-timeout = \"20s\"\n\t\t\t`),\n\t\t\tdurations: []time.Duration{\n\t\t\t\t-time.Minute, // only for safety\n\t\t\t\t0, // only for safety\n\t\t\t\ttime.Minute,\n\t\t\t},\n\t\t\twantParams: []QueryParam{\n\t\t\t\t{\n\t\t\t\t\tDuration: 0,\n\t\t\t\t\tURL: \"http://localhost:8123/?max_rows_to_read=1000\",\n\t\t\t\t\tDataTimeout: 20 * time.Second,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tDuration: 0,\n\t\t\t\t\tURL: \"http://localhost:8123/?max_rows_to_read=1000\",\n\t\t\t\t\tDataTimeout: 20 * time.Second,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tDuration: 0,\n\t\t\t\t\tURL: \"http://localhost:8123/?max_rows_to_read=1000\",\n\t\t\t\t\tDataTimeout: 20 * time.Second,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"two params\",\n\t\t\tconfig: []byte(`\n\t\t\t[clickhouse]\n\t\t\turl = \"http://localhost:8123/?max_rows_to_read=1000\"\n\t\t\tdata-timeout = \"20s\"\n\t\t\tquery-params = [\n\t\t\t {\n\t\t\t\tduration = \"72h\",\n\t\t\t\turl = \"http://localhost:8123/?max_rows_to_read=20000\",\n\t\t\t\tdata-timeout = \"40s\"\n\t\t\t },\n\t\t\t]`),\n\t\t\tdurations: []time.Duration{\n\t\t\t\t-time.Minute, // only for safety\n\t\t\t\t0, // only for safety\n\t\t\t\ttime.Minute,\n\t\t\t\t72*time.Hour - time.Second,\n\t\t\t\t72 * time.Hour,\n\t\t\t\t2160 * time.Hour,\n\t\t\t},\n\t\t\twantParams: []QueryParam{\n\t\t\t\t{\n\t\t\t\t\tDuration: 0,\n\t\t\t\t\tURL: \"http://localhost:8123/?max_rows_to_read=1000\",\n\t\t\t\t\tDataTimeout: 20 * time.Second,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tDuration: 0,\n\t\t\t\t\tURL: \"http://localhost:8123/?max_rows_to_read=1000\",\n\t\t\t\t\tDataTimeout: 20 * time.Second,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tDuration: 0,\n\t\t\t\t\tURL: \"http://localhost:8123/?max_rows_to_read=1000\",\n\t\t\t\t\tDataTimeout: 20 * time.Second,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tDuration: 0,\n\t\t\t\t\tURL: \"http://localhost:8123/?max_rows_to_read=1000\",\n\t\t\t\t\tDataTimeout: 20 * time.Second,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tDuration: 72 * time.Hour,\n\t\t\t\t\tURL: \"http://localhost:8123/?max_rows_to_read=20000\",\n\t\t\t\t\tDataTimeout: 40 * time.Second,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tDuration: 72 * time.Hour,\n\t\t\t\t\tURL: \"http://localhost:8123/?max_rows_to_read=20000\",\n\t\t\t\t\tDataTimeout: 40 * time.Second,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"serveral params\",\n\t\t\tconfig: []byte(`\n\t\t\t[clickhouse]\n\t\t\turl = \"http://localhost:8123/?max_rows_to_read=1000\"\n\t\t\tdata-timeout = \"20s\"\n\t\t\tquery-params = [\n\t\t\t {\n\t\t\t\tduration = \"72h\",\n\t\t\t\turl = \"http://localhost:8123/?max_rows_to_read=20000\",\n\t\t\t\tdata-timeout = \"40s\"\n\t\t\t },\n\t\t\t {\n\t\t\t\tduration = \"2160h\",\n\t\t\t\tdata-timeout = \"60s\"\n\t\t\t }\n\t\t\t]`),\n\t\t\tdurations: []time.Duration{\n\t\t\t\t-time.Minute, // only for safety\n\t\t\t\t0, // only for safety\n\t\t\t\ttime.Minute,\n\t\t\t\t72*time.Hour - time.Second,\n\t\t\t\t72 * time.Hour,\n\t\t\t\t2160 * time.Hour,\n\t\t\t\t4000 * time.Hour,\n\t\t\t},\n\t\t\twantParams: []QueryParam{\n\t\t\t\t{\n\t\t\t\t\tDuration: 0,\n\t\t\t\t\tURL: \"http://localhost:8123/?max_rows_to_read=1000\",\n\t\t\t\t\tDataTimeout: 20 * time.Second,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tDuration: 0,\n\t\t\t\t\tURL: \"http://localhost:8123/?max_rows_to_read=1000\",\n\t\t\t\t\tDataTimeout: 20 * time.Second,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tDuration: 0,\n\t\t\t\t\tURL: \"http://localhost:8123/?max_rows_to_read=1000\",\n\t\t\t\t\tDataTimeout: 20 * time.Second,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tDuration: 0,\n\t\t\t\t\tURL: \"http://localhost:8123/?max_rows_to_read=1000\",\n\t\t\t\t\tDataTimeout: 20 * time.Second,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tDuration: 72 * time.Hour,\n\t\t\t\t\tURL: \"http://localhost:8123/?max_rows_to_read=20000\",\n\t\t\t\t\tDataTimeout: 40 * time.Second,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tDuration: 2160 * time.Hour,\n\t\t\t\t\tURL: \"http://localhost:8123/?max_rows_to_read=1000\",\n\t\t\t\t\tDataTimeout: 60 * time.Second,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tDuration: 2160 * time.Hour,\n\t\t\t\t\tURL: \"http://localhost:8123/?max_rows_to_read=1000\",\n\t\t\t\t\tDataTimeout: 60 * time.Second,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tif config, _, err := Unmarshal(tt.config, false); err == nil {\n\t\t\t\tfor i := range config.ClickHouse.QueryParams {\n\t\t\t\t\tconfig.ClickHouse.QueryParams[i].Limiter = nil\n\t\t\t\t}\n\n\t\t\t\tfor i, duration := range tt.durations {\n\t\t\t\t\tgot := GetQueryParam(config.ClickHouse.QueryParams, duration)\n\t\t\t\t\tif config.ClickHouse.QueryParams[got] != tt.wantParams[i] {\n\t\t\t\t\t\tt.Errorf(\"[%d] GetQueryParam(%v) = %+v, want %+v\", i, duration, config.ClickHouse.QueryParams[got], tt.wantParams[i])\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tt.Errorf(\"Load config error = %v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestClickHouse_Validate(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tch ClickHouse\n\t\twantErr string\n\t}{\n\t\t{\n\t\t\tname: \"url with spaces\",\n\t\t\tch: ClickHouse{\n\t\t\t\tURL: \"http://localhost:8123/?max_rows_to_read=600 &max_threads=2&skip_unavailable_shards=1&log_queries=1\",\n\t\t\t},\n\t\t\twantErr: `space not allowed in url \"http://localhost:8123/?max_rows_to_read=600 &max_threads=2&skip_unavailable_shards=1&log_queries=1\"`,\n\t\t},\n\t\t{\n\t\t\tname: \"valid url\",\n\t\t\tch: ClickHouse{\n\t\t\t\tURL: \"http://localhost:8123/?max_rows_to_read=600&max_threads=2&skip_unavailable_shards=1&log_queries=1\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t_, err := clickhouseURLValidate(tt.ch.URL)\n\t\t\tif err == nil {\n\t\t\t\tif tt.wantErr != \"\" {\n\t\t\t\t\tt.Errorf(\"ClickHouse.Validate() error = nil, wantErr %q\", tt.wantErr)\n\t\t\t\t}\n\t\t\t} else if err.Error() != tt.wantErr {\n\t\t\t\tt.Errorf(\"ClickHouse.Validate() error = %v, wantErr %q\", err, tt.wantErr)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "config/json.go", "content": "package config\n\nimport (\n\t\"encoding/json\"\n\t\"net/url\"\n)\n\nfunc (c *ClickHouse) MarshalJSON() ([]byte, error) {\n\ttype ClickHouseRaw ClickHouse\n\n\t// make copy\n\ta := *c\n\n\tu, err := url.Parse(a.URL)\n\tif err != nil {\n\t\ta.URL = \"\"\n\t} else {\n\t\tif _, isSet := u.User.Password(); isSet {\n\t\t\tu.User = url.UserPassword(u.User.Username(), \"xxxxxx\")\n\t\t}\n\n\t\ta.URL = u.String()\n\t}\n\n\treturn json.Marshal((*ClickHouseRaw)(&a))\n}\n" }, { "path": "config/json_test.go", "content": "package config\n\nimport (\n\t\"encoding/json\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestClickhouseUrlPassword(t *testing.T) {\n\tassert := assert.New(t)\n\n\tresult := make(map[string]interface{})\n\n\tc := &ClickHouse{URL: \"http://user:qwerty@localhost:8123/?param=value\"}\n\tb, err := json.Marshal(c)\n\tassert.NoError(err)\n\n\tassert.NoError(json.Unmarshal(b, &result))\n\tassert.Equal(\"http://user:xxxxxx@localhost:8123/?param=value\", result[\"url\"].(string))\n\tassert.Equal(\"http://user:qwerty@localhost:8123/?param=value\", c.URL)\n}\n" }, { "path": "deploy/doc/.gitignore", "content": "# autogenerated config for documentation\ngraphite-clickhouse.conf\n" }, { "path": "deploy/doc/config.md", "content": "# Configuration\n\n## Common `[common]`\n\n### Finder cache\n\nSpecify what storage to use for finder cache. This cache stores finder results (metrics find/tags autocomplete/render).\n\nSupported cache types:\n - `mem` - will use integrated in-memory cache. Not distributed. Fast.\n - `memcache` - will use specified memcache servers. Could be shared. Slow.\n - `null` - disable cache\n\nExtra options:\n - `size_mb` - specify max size of cache, in MiB\n - `defaultTimeoutSec` - specify default cache ttl.\n - `shortTimeoutSec` - cache ttl for short duration intervals of render queries (duration <= shortDuration && now-until <= 61) (if 0, disable this cache)\n - `findTimeoutSec` - cache ttl for finder/tags autocompleter queries (if 0, disable this cache)\n - `shortDuration` - maximum duration for render queries, which use shortTimeoutSec duration\n\n### Example\n```yaml\n[common.find-cache]\ntype = \"memcache\"\nsize_mb = 0\nmemcachedServers = [ \"127.0.0.1:1234\", \"127.0.0.2:1235\" ]\ndefaultTimeoutSec = 10800\nshortTimeoutSec = 300\nfindTimeoutSec = 600\n```\n\n## Feature flags `[feature-flags]`\n\n`use-carbon-behaviour=true`.\n\n- Tagged terms with `=` operator and empty value (e.g. `t=`) match all metrics that don't have that tag.\n\n`dont-match-missing-tags=true`.\n\n- Tagged terms with `!=`, `!=~` operators only match metrics that have that tag.\n\n### Examples\n\nGiven tagged metrics:\n```\nmetric.two;env=prod\nmetric.one;env=stage;dc=mydc1\nmetric.one;env=prod;dc=otherdc1\n```\n| Target | use-carbon-behaviour | Matched metrics |\n|-----------------------------|----------------------|---------------------------------------------------|\n| seriesByTag('dc=') | false | - |\n| seriesByTag('dc=') | true | metric.two;env=prod |\n\n| Target | dont-match-missing-tags | Matched metrics |\n|--------------------------|-------------------------|--------------------------------------------------------|\n| seriesByTag('dc!=mydc1') | false | metric.two;env=prod
metric.one;env=prod;dc=otherdc1 |\n| seriesByTag('dc!=mydc1') | true | metric.one;env=prod;dc=otherdc1 |\n| seriesByTag('dc!=~otherdc') | false | metric.two;env=prod
metric.one;env=stage;dc=mydc1 |\n| seriesByTag('dc!=~otherdc') | true | metric.one;env=stage;dc=mydc1 |\n\n## ClickHouse `[clickhouse]`\n\n### URL `url`\nDetailed explanation of ClickHouse HTTP interface is given in [documentation](https://clickhouse.tech/docs/en/interfaces/http). It's recommended to create a dedicated read-only user for graphite-clickhouse.\n\nExample: `url = \"http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1\"`\n\nSome useful parameters:\n\n- [log_queries=1](https://clickhouse.tech/docs/en/operations/settings/settings/#settings-log-queries): all queries will be logged in the `system.query_log` table. Useful for debug.\n- [readonly=2](https://clickhouse.tech/docs/en/operations/settings/permissions-for-queries/#settings_readonly): do not change data on the server\n- [max_rows_to_read=200000000](https://clickhouse.tech/docs/en/operations/settings/query-complexity/#max-rows-to-read): useful if you want to prevent too broad requests\n- [cancel_http_readonly_queries_on_client_close=1](https://clickhouse.tech/docs/en/operations/settings/settings/#cancel-http-readonly-queries-on-client-close): cancel DB query when request is canceled.\n\nAll these and more settings can be set in clickhouse-server configuration as user's profile settings.\n\nUseless settings:\n\n- `max_query_size`: at the moment [external data](https://clickhouse.tech/docs/en/engines/table-engines/special/external-data/) is used, the query length is relatively small and always less than the default [262144](https://clickhouse.tech/docs/en/operations/settings/settings/#settings-max_query_size)\n- `max_ast_elements`: the same\n- `max_execution_time`: with `cancel_http_readonly_queries_on_client_close=1` and `data-timeout = \"1m\"` it's already covered.\n\n### Query multi parameters (for overwrite default url and data-timeout)\n\nFor queries with duration (until - from) >= 72 hours, use custom url and data-timeout\n\n```\nurl = \"http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=102400000&max_result_bytes=12800000&max_threads=2\"\ndata-timeout = \"30s\"\n\nquery-params = [\n {\n duration = \"72h\",\n url = \"http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=1024000000&max_result_bytes=128000000&max_threads=1\",\n data-timeout = \"60s\"\n }\n]\n```\n\n### Query limiter for prevent database overloading (limit concurrent/maximum incomming requests)\n\nFor prevent database overloading incomming requests (render/find/autocomplete) can be limited.\nIf wait max-queries requests, for new request error returned immediately.\nIf executing concurrent-queries requests, next request will be wait for free slot until index-timeout reached\nadaptive-queries prevent overload with load average check if graphite-clickhouse run on one host with clickhouse\nReal queries will be concurrent-queries + adaptive-queries * (1 / normalized_load_avg - 1).\nIf normalized_load_avg > 0.9, limit will be concurrent-queries.\n```\nurl = \"http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=102400000&max_result_bytes=12800000&max_threads=2\"\nrender-max-queries = 500\nrender-max-concurrent = 10\nfind-max-queries = 100\nfind-concurrent-queries = 10\ntags-max-queries = 100\ntags-max-concurrent = 10\n\nquery-params = [\n {\n duration = \"72h\",\n url = \"http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=1024000000&max_result_bytes=128000000&max_threads=1\",\n data-timeout = \"60s\"\n max-queries = 100,\n max-concurrent = 4\n }\n]\n\nuser-limits = {\n \"alerting\" = {\n max-queries = 100,\n max-concurrent = 5\n }\n}\n\n```\n\n### Index table\nSee [index table](./index-table.md) documentation for details.\n\n### Index reversed queries tuning\nBy default the daemon decides to make a direct or reversed request to the [index table](./index-table.md) based on a first and last glob node in the metric. It choose the most long path to reduce readings. Additional examples can be found in [tests](../finder/index_test.go).\n\nYou can overwrite automatic behavior with `index-reverse`. Valid values are `\"auto\", direct, \"reversed\"`\n\nIf you need fine tuning for different paths, you can use `[[clickhouse.index-reverses]]` to set behavior per metrics' `prefix`, `suffix` or `regexp`.\n\n### Tags table\nBy default, tags are stored in the tagged-table on the daily basis. If a metric set doesn't change much, that leads to situation when the same data stored multiple times.\nTo prevent uncontrolled growth and reduce the amount of data stored in the tagged-table, the `tagged-use-daily` parameter could be set to `false` and table definition could be changed to something like:\n```\nCREATE TABLE graphite_tagged (\n Date Date,\n Tag1 String,\n Path String,\n Tags Array(String),\n Version UInt32\n) ENGINE = ReplacingMergeTree(Date)\nORDER BY (Tag1, Path);\n```\n\nFor restrict costly seriesByTag (may be like `seriesByTag('name=~test.*.*.rabbitmq_overview.connections')` or `seriesByTag('name=test.*.*.rabbitmq_overview.connections')`) use tags-min-in-query parameter.\nFor restrict costly autocomplete queries use tags-min-in-autocomplete parameter.\n\nset for require at minimum 1 eq argument (without wildcards)\n`tags-min-in-query=1`\n\n\n`ReplacingMergeTree(Date)` prevent broken tags autocomplete with default `ReplacingMergeTree(Version)`, when write to the past.\n\n### ClickHouse aggregation\nFor detailed description of `max-data-points` and `internal-aggregation` see [aggregation documentation](./aggregation.md).\n\n## Data tables `[[data-table]]`\n\n### Rollup\nThe rollup configuration is used for a proper metrics pre-aggregation. It contains two rules types:\n\n- retention for point per time range\n- aggregation function for a values\n\nHistorically, the way to define the config was `rollup-conf = \"/path/to/the/conf/with/graphite_rollup.xml\"`. The format is the same as [graphite_rollup](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/graphitemergetree/#rollup-configuration) scheme for ClickHouse server.\n\nFor a quite long time it's recommended to use `rollup-conf = \"auto\"` to get the configuration from remote ClickHouse server. It will update itself on each `rollup-auto-interval` (1 minute by default) or once on startup if set to \"0s\".\n\nIf you don't use a `GraphiteMergeTree` family engine, you can still use `rollup-conf = \"auto\"` by setting `rollup-auto-table=\"graphiteMergeTreeTable\"` and get the proper config. In this case `graphiteMergeTreeTable` is a dummy table associated with proper [graphite_rollup](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/graphitemergetree/#rollup-configuration). The cases when you may need it:\n\n- ReplacingMergeTree engine\n- Distributed engine\n- Materialized view\n\nIt's possible as well to set `rollup-conf = \"none\"`. Then values from `rollup-default-precision` and `rollup-default-function` will be used.\n\n#### Additional rollup tuning for reversed data tables\nWhen `reverse = true` is set for data-table, there are two possibles cases for [graphite_rollup](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/graphitemergetree/#rollup-configuration):\n\n- Original regexps are used, like `^level_one.level_two.suffix$`\n- Reversed regexps are used, like `^suffix.level_two.level_one$`\n\nDepends on it for having a proper retention and aggregation you must additionally set `rollup-use-reverted = true` for the first case and `rollup-use-reverted = false` for the second.\n\n#### Additional tuning tagged find for seriesByTag and autocomplete\nOnly one tag used as filter for index field Tag1, see graphite_tagged table [structure](https://github.com/lomik/\n\nTo always choose the best Tag1 you can set the parameter `tag1-count-table = `. The value should be a table in clickhouse that has columns (Date, Tag1, Count) similar to the graphite_tagged table. The table can be defined like this:\n\n```\nCREATE TABLE IF NOT EXISTS default.tag1_count_per_day\n(\n Date Date,\n Tag1 String,\n Count UInt64\n)\nENGINE = SummingMergeTree\nORDER BY (Date, Tag1);\n\nCREATE MATERIALIZED VIEW IF NOT EXISTS default.tag1_count_per_day_mv TO default.tag1_count_per_day AS\nSELECT Date AS Date,\n Tag1 AS Tag1,\n count(*) AS Count\nFROM default.graphite_tags\nGROUP BY (Date, Tag1);\n```\n\nHere we additionally create a materialized view to automatically save the quantities of rows with each unique Tag1 as the metrics are being written.\ngraphite-clickhouse will query this table when it tries to decide which tag should be used when querying graphite_tagged table.\nOverall using this parameter will somewhat increase writing load but can improve reading tagged metrics greatly in some cases.\n\nNote that this option only works for terms with '=' operator in them. Using it will also override tag costs that were set manually with tagged-costs option.\n" }, { "path": "deploy/root/usr/lib/systemd/system/graphite-clickhouse.service", "content": "[Unit]\nDescription=Graphite cluster backend with ClickHouse support\nDocumentation=https://github.com/lomik/graphite-clickhouse\nAfter=network.target\n\n[Service]\nType=simple\nPermissionsStartOnly=true\nExecStart=/usr/bin/graphite-clickhouse -config /etc/graphite-clickhouse/graphite-clickhouse.conf\nRestart=on-failure\nKillMode=control-group\n\n[Install]\nWantedBy=multi-user.target\n" }, { "path": "doc/aggregation.md", "content": "# Enable ClickHouse aggregation\n\nThe feature was added in [v0.12.0](https://github.com/lomik/graphite-clickhouse/releases/tag/v0.12.0). It's enabled by default since [v0.13.0](https://github.com/lomik/graphite-clickhouse/releases/tag/v0.13.0) ([#157](https://github.com/lomik/graphite-clickhouse/pull/157)). You can disable it be setting `internal-aggregation = false` to use aggregation in graphite-clickhouse.\n\n```\n[clickhouse]\n# ClickHouse-side aggregation\ninternal-aggregation = true\n# maximum number of points per metric. It should be set to 4096 or less for ClickHouse older than 20.8\n# https://github.com/ClickHouse/ClickHouse/commit/d7871f3976e4f066c6efb419db22725f476fd9fa\nmax-data-points = 1048576\n```\n\nThe only known _frontend_ supporting passing `maxDataPoints` from requests is [carbonapi>=0.14](https://github.com/go-graphite/carbonapi/releases/tag/0.14.0). Protocol should be set to `carbonapi_v3_pb` for this feature to fully work, see [config](https://github.com/go-graphite/carbonapi/blob/main/doc/configuration.md#upstreams)->backendv2->backends->protocol.\n\nBut even without mentioned adjustments, `internal-aggregation` improves the whole picture by implementing whisper-like aggregation behavior (see below).\n\n## Compatible ClickHouse versions\nThe feature uses ClickHouse aggregation combinator [-Resample](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/combinators/#agg-functions-combinator-resample). This aggregator is available since version [19.11](https://github.com/ClickHouse/ClickHouse/commit/57db1fac5990a7227e720c9dd438d88a381d298f)\n\n*Note*: version 0.12 is compatible only with CH 20.1.13.105, 20.3.10.75, 20.4.5.36, 20.5.2.7 or newer since it uses -OrNull modifier.\n\nGenerally, it's a good idea to always use the latest [LTS](https://repo.clickhouse.tech/deb/lts/main/) ClickHouse release to have the actual version.\n\n## Upgrade\n\n- Upgrade the carbonapi to version 0.14.0 or greater\n- Upgrade graphite-clickhouse to version 0.12.0 or greater\n- Set the `backendv2->backends->protocol: carbonapi_v3_pb` in carbonapi *only after graphite-clickhouse is upgraded*\n- Upgrade ClickHouse\n- Enable `internal-aggregation` in graphite-clickhouse\n\n# Historical remark: schemes and changes overview\n## Classic whisper scheme\n\n```\nheader:\nxFilesFactor: [0, 1]\naggregation: {avg,sum,min,max,...}\nretention: 1d:1m,1w:5m,1y:1h\ndata:\narchive1: 1440 points\narchive2: 2016 points\narchive3: 8760 points\n```\n\n- Retention description:\n - Stores point/1m for one day\n - Stores point/5m for one week\n - Stores point/1h for one year\n - Older points than any of mentioned age are overwriten by new incoming points\n- Each archive filled up simultaneously\n- Aggregation on the fly during writing\n- `xFilesFactor` controls if points from archive(N) should be aggregated into archive(N+1)\n- Points are selected only from one archive, with the most precision:\n - from <= now-1d -> archive1\n - from <= now-7d -> archive2\n - else -> archive3\n\n## Historical graphite-clickhouse way\n\n### Storing: GraphiteMergeTree table engine in ClickHouse (CH)\n\nCompletely another principle of data storage.\n\n- Retention scheme looks slightly different: \n`retention: 0:60,1d:5m,1w:1h,1y:1d`\n - Stores point/minute, if the age of point is at least 0sec\n - Stores point/5min, if the age of point is at least one day\n - Stores point/1h, if the age of point is at least one week\n - GraphiteMergeTree doesn't drop metrics after some particular age, so after one year we would store it with the minimum possible resolution point/day\n- Retention and aggregation policies are applied only when point becomes older than X (1d,1w,1y)\n- There is no such thing as `archive`, each point is stored only once\n- No `xFilesFactor` entity: each point will be aggregated\n\n### Fetching data: before September 2019 (current `internal-aggregation = false` behavior)\n\n```sql\nSELECT Path, Time, Value, Timestamp\nFROM data WHERE ...\n```\n\nLogic:\n\n- Select all points\n- Aggregate them on the fly to the proper `archive` step\n- Pass further to *graphite-web*/*carbonapi*\n\nProblems:\n\n- A huge overhead for Path (the heaviest part)\n- Extremely inefficient in terms of network traffic, especially when CH cluster is used\n - The CH node `query-initiator` must collect the whole data (in memory or on the disk), and only then points will be passed further\n\n### Fetching data: after September 2019 ([#61](https://github.com/lomik/graphite-clickhouse/pull/61), [#62](https://github.com/lomik/graphite-clickhouse/pull/62), [#65](https://github.com/lomik/graphite-clickhouse/pull/65)) (between v0.11.7 and v0.12.0)\n\n```sql\nSELECT Path,\n groupArray(Time),\n groupArray(Value),\n groupArray(Timestamp)\nFROM data WHERE ... GROUP BY Path\n```\n\n- Up to 6 time less network load\n- But still selects all points and aggregates in *graphite-clickhouse*\n\n### Fetching data: September 2020 ([#88](https://github.com/lomik/graphite-clickhouse/pull/88)) (v0.12.0)\n\n```sql\nSELECT Path,\n arrayFilter(x->isNotNull(x),\n anyOrNullResample($from, $until, $step)\n (toUInt32(intDiv(Time, $step)*$step), Time)\n ),\n arrayFilter(x->isNotNull(x),\n ${func}OrNullResample($from, $until, $step)\n (Value, Time)\n )\nFROM data WHERE ... GROUP BY Path\n```\n\n- This solution implements `archive` analog on CH side\n- Most of the data is aggregated on CH shards and doesn't leave them, so `query-initiator` consumes much less memory\n- When *carbonapi* with `format=carbonapi_v3_pb` is used, the `/render?maxDataPoints=x` parameter processed on CH side too\n\n### Fetching data: April 2021 ([#145](https://github.com/lomik/graphite-clickhouse/pull/145))\n\n```sql\nWITH anyResample($from, $until, $step)(toUInt32(intDiv(Time, $step)*$step), Time) AS mask\nSELECT Path,\n arrayFilter(m->m!=0, mask) AS times,\n arrayFilter((v,m)->m!=0, ${func}Resample($from, $until, $step)(Value, Time), mask) AS values\nFROM data WHERE ... GROUP BY Path\n```\n\n- Query improved a bit: dropped the use of `-OrNull` improved compatibility with different CH versions.\n\n## Fetching data: concepts' difference\n\nFor small requests, the difference is not so big, but for the heavy one the amount of data was decreased up to 100 times:\n\n```\ntarget=${986_metrics_60s_precision}\nfrom=-7d\nmaxDataPoints=100\n```\n\n| method | rows | points | data (binary) | time (s) |\n| - | - | - | - | - |\n| row/point | 9887027 | 9887027 | 556378258 (530M) | 16.486 |\n| groupArray | 986 | 9887027 | 158180388 (150M) | 35.498 |\n| -Resample | 986 | 98553 | 1421418 (1M) | 13.181 |\n\n*note*: it's localhost, so with slow network effect may be even more significant.\n\n### The maxDataPoints processing\n\nThe classical pipeline:\n\n- Fetch the data in *graphite-web*/*carbonapi*\n- Apply all functions from `target`\n- Compare the result with `maxDataPoints` URI parameter and adjust them\n\nCurrent:\n\n- Get data, aggregated with the proper function directly from CH\n- Fetch pre-aggregated data with a proper functions from ClickHouse\n- Apply all functions to the pre-aggregated data\n\n" }, { "path": "doc/config.md", "content": "[//]: # (This file is built out of deploy/doc/config.md, please do not edit it manually) \n[//]: # (To rebuild it run `make config`)\n\n# Configuration\n\n## Common `[common]`\n\n### Finder cache\n\nSpecify what storage to use for finder cache. This cache stores finder results (metrics find/tags autocomplete/render).\n\nSupported cache types:\n - `mem` - will use integrated in-memory cache. Not distributed. Fast.\n - `memcache` - will use specified memcache servers. Could be shared. Slow.\n - `null` - disable cache\n\nExtra options:\n - `size_mb` - specify max size of cache, in MiB\n - `defaultTimeoutSec` - specify default cache ttl.\n - `shortTimeoutSec` - cache ttl for short duration intervals of render queries (duration <= shortDuration && now-until <= 61) (if 0, disable this cache)\n - `findTimeoutSec` - cache ttl for finder/tags autocompleter queries (if 0, disable this cache)\n - `shortDuration` - maximum duration for render queries, which use shortTimeoutSec duration\n\n### Example\n```yaml\n[common.find-cache]\ntype = \"memcache\"\nsize_mb = 0\nmemcachedServers = [ \"127.0.0.1:1234\", \"127.0.0.2:1235\" ]\ndefaultTimeoutSec = 10800\nshortTimeoutSec = 300\nfindTimeoutSec = 600\n```\n\n## Feature flags `[feature-flags]`\n\n`use-carbon-behaviour=true`.\n\n- Tagged terms with `=` operator and empty value (e.g. `t=`) match all metrics that don't have that tag.\n\n`dont-match-missing-tags=true`.\n\n- Tagged terms with `!=`, `!=~` operators only match metrics that have that tag.\n\n### Examples\n\nGiven tagged metrics:\n```\nmetric.two;env=prod\nmetric.one;env=stage;dc=mydc1\nmetric.one;env=prod;dc=otherdc1\n```\n| Target | use-carbon-behaviour | Matched metrics |\n|-----------------------------|----------------------|---------------------------------------------------|\n| seriesByTag('dc=') | false | - |\n| seriesByTag('dc=') | true | metric.two;env=prod |\n\n| Target | dont-match-missing-tags | Matched metrics |\n|--------------------------|-------------------------|--------------------------------------------------------|\n| seriesByTag('dc!=mydc1') | false | metric.two;env=prod
metric.one;env=prod;dc=otherdc1 |\n| seriesByTag('dc!=mydc1') | true | metric.one;env=prod;dc=otherdc1 |\n| seriesByTag('dc!=~otherdc') | false | metric.two;env=prod
metric.one;env=stage;dc=mydc1 |\n| seriesByTag('dc!=~otherdc') | true | metric.one;env=stage;dc=mydc1 |\n\n## ClickHouse `[clickhouse]`\n\n### URL `url`\nDetailed explanation of ClickHouse HTTP interface is given in [documentation](https://clickhouse.tech/docs/en/interfaces/http). It's recommended to create a dedicated read-only user for graphite-clickhouse.\n\nExample: `url = \"http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1\"`\n\nSome useful parameters:\n\n- [log_queries=1](https://clickhouse.tech/docs/en/operations/settings/settings/#settings-log-queries): all queries will be logged in the `system.query_log` table. Useful for debug.\n- [readonly=2](https://clickhouse.tech/docs/en/operations/settings/permissions-for-queries/#settings_readonly): do not change data on the server\n- [max_rows_to_read=200000000](https://clickhouse.tech/docs/en/operations/settings/query-complexity/#max-rows-to-read): useful if you want to prevent too broad requests\n- [cancel_http_readonly_queries_on_client_close=1](https://clickhouse.tech/docs/en/operations/settings/settings/#cancel-http-readonly-queries-on-client-close): cancel DB query when request is canceled.\n\nAll these and more settings can be set in clickhouse-server configuration as user's profile settings.\n\nUseless settings:\n\n- `max_query_size`: at the moment [external data](https://clickhouse.tech/docs/en/engines/table-engines/special/external-data/) is used, the query length is relatively small and always less than the default [262144](https://clickhouse.tech/docs/en/operations/settings/settings/#settings-max_query_size)\n- `max_ast_elements`: the same\n- `max_execution_time`: with `cancel_http_readonly_queries_on_client_close=1` and `data-timeout = \"1m\"` it's already covered.\n\n### Query multi parameters (for overwrite default url and data-timeout)\n\nFor queries with duration (until - from) >= 72 hours, use custom url and data-timeout\n\n```\nurl = \"http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=102400000&max_result_bytes=12800000&max_threads=2\"\ndata-timeout = \"30s\"\n\nquery-params = [\n {\n duration = \"72h\",\n url = \"http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=1024000000&max_result_bytes=128000000&max_threads=1\",\n data-timeout = \"60s\"\n }\n]\n```\n\n### Query limiter for prevent database overloading (limit concurrent/maximum incomming requests)\n\nFor prevent database overloading incomming requests (render/find/autocomplete) can be limited.\nIf wait max-queries requests, for new request error returned immediately.\nIf executing concurrent-queries requests, next request will be wait for free slot until index-timeout reached\nadaptive-queries prevent overload with load average check if graphite-clickhouse run on one host with clickhouse\nReal queries will be concurrent-queries + adaptive-queries * (1 / normalized_load_avg - 1).\nIf normalized_load_avg > 0.9, limit will be concurrent-queries.\n```\nurl = \"http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=102400000&max_result_bytes=12800000&max_threads=2\"\nrender-max-queries = 500\nrender-max-concurrent = 10\nfind-max-queries = 100\nfind-concurrent-queries = 10\ntags-max-queries = 100\ntags-max-concurrent = 10\n\nquery-params = [\n {\n duration = \"72h\",\n url = \"http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=1024000000&max_result_bytes=128000000&max_threads=1\",\n data-timeout = \"60s\"\n max-queries = 100,\n max-concurrent = 4\n }\n]\n\nuser-limits = {\n \"alerting\" = {\n max-queries = 100,\n max-concurrent = 5\n }\n}\n\n```\n\n### Index table\nSee [index table](./index-table.md) documentation for details.\n\n### Index reversed queries tuning\nBy default the daemon decides to make a direct or reversed request to the [index table](./index-table.md) based on a first and last glob node in the metric. It choose the most long path to reduce readings. Additional examples can be found in [tests](../finder/index_test.go).\n\nYou can overwrite automatic behavior with `index-reverse`. Valid values are `\"auto\", direct, \"reversed\"`\n\nIf you need fine tuning for different paths, you can use `[[clickhouse.index-reverses]]` to set behavior per metrics' `prefix`, `suffix` or `regexp`.\n\n### Tags table\nBy default, tags are stored in the tagged-table on the daily basis. If a metric set doesn't change much, that leads to situation when the same data stored multiple times.\nTo prevent uncontrolled growth and reduce the amount of data stored in the tagged-table, the `tagged-use-daily` parameter could be set to `false` and table definition could be changed to something like:\n```\nCREATE TABLE graphite_tagged (\n Date Date,\n Tag1 String,\n Path String,\n Tags Array(String),\n Version UInt32\n) ENGINE = ReplacingMergeTree(Date)\nORDER BY (Tag1, Path);\n```\n\nFor restrict costly seriesByTag (may be like `seriesByTag('name=~test.*.*.rabbitmq_overview.connections')` or `seriesByTag('name=test.*.*.rabbitmq_overview.connections')`) use tags-min-in-query parameter.\nFor restrict costly autocomplete queries use tags-min-in-autocomplete parameter.\n\nset for require at minimum 1 eq argument (without wildcards)\n`tags-min-in-query=1`\n\n\n`ReplacingMergeTree(Date)` prevent broken tags autocomplete with default `ReplacingMergeTree(Version)`, when write to the past.\n\n### ClickHouse aggregation\nFor detailed description of `max-data-points` and `internal-aggregation` see [aggregation documentation](./aggregation.md).\n\n## Data tables `[[data-table]]`\n\n### Rollup\nThe rollup configuration is used for a proper metrics pre-aggregation. It contains two rules types:\n\n- retention for point per time range\n- aggregation function for a values\n\nHistorically, the way to define the config was `rollup-conf = \"/path/to/the/conf/with/graphite_rollup.xml\"`. The format is the same as [graphite_rollup](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/graphitemergetree/#rollup-configuration) scheme for ClickHouse server.\n\nFor a quite long time it's recommended to use `rollup-conf = \"auto\"` to get the configuration from remote ClickHouse server. It will update itself on each `rollup-auto-interval` (1 minute by default) or once on startup if set to \"0s\".\n\nIf you don't use a `GraphiteMergeTree` family engine, you can still use `rollup-conf = \"auto\"` by setting `rollup-auto-table=\"graphiteMergeTreeTable\"` and get the proper config. In this case `graphiteMergeTreeTable` is a dummy table associated with proper [graphite_rollup](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/graphitemergetree/#rollup-configuration). The cases when you may need it:\n\n- ReplacingMergeTree engine\n- Distributed engine\n- Materialized view\n\nIt's possible as well to set `rollup-conf = \"none\"`. Then values from `rollup-default-precision` and `rollup-default-function` will be used.\n\n#### Additional rollup tuning for reversed data tables\nWhen `reverse = true` is set for data-table, there are two possibles cases for [graphite_rollup](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/graphitemergetree/#rollup-configuration):\n\n- Original regexps are used, like `^level_one.level_two.suffix$`\n- Reversed regexps are used, like `^suffix.level_two.level_one$`\n\nDepends on it for having a proper retention and aggregation you must additionally set `rollup-use-reverted = true` for the first case and `rollup-use-reverted = false` for the second.\n\n#### Additional tuning tagged find for seriesByTag and autocomplete\nOnly one tag used as filter for index field Tag1, see graphite_tagged table [structure](https://github.com/lomik/\n\nTo always choose the best Tag1 you can set the parameter `tag1-count-table = `. The value should be a table in clickhouse that has columns (Date, Tag1, Count) similar to the graphite_tagged table. The table can be defined like this:\n\n```\nCREATE TABLE IF NOT EXISTS default.tag1_count_per_day\n(\n Date Date,\n Tag1 String,\n Count UInt64\n)\nENGINE = SummingMergeTree\nORDER BY (Date, Tag1);\n\nCREATE MATERIALIZED VIEW IF NOT EXISTS default.tag1_count_per_day_mv TO default.tag1_count_per_day AS\nSELECT Date AS Date,\n Tag1 AS Tag1,\n count(*) AS Count\nFROM default.graphite_tags\nGROUP BY (Date, Tag1);\n```\n\nHere we additionally create a materialized view to automatically save the quantities of rows with each unique Tag1 as the metrics are being written.\ngraphite-clickhouse will query this table when it tries to decide which tag should be used when querying graphite_tagged table.\nOverall using this parameter will somewhat increase writing load but can improve reading tagged metrics greatly in some cases.\n\nNote that this option only works for terms with '=' operator in them. Using it will also override tag costs that were set manually with tagged-costs option.\n\n```toml\n[common]\n # general listener\n listen = \":9090\"\n # listener to serve /debug/pprof requests. '-pprof' argument overrides it\n pprof-listen = \"\"\n max-cpu = 1\n # limit number of results from find query, 0=unlimited\n max-metrics-in-find-answer = 0\n # limit numbers of queried metrics per target in /render requests, 0 or negative = unlimited\n max-metrics-per-target = 15000\n # if true, always return points for all metrics, replacing empty results with list of NaN\n append-empty-series = false\n # daemon returns empty response if query matches any of regular expressions\n # target-blacklist = []\n # daemon will return the freed memory to the OS when it>0\n memory-return-interval = \"0s\"\n # additional request headers to log\n headers-to-log = []\n # service discovery base weight (on idle)\n base_weight = 0\n # service discovery degraded load avg multiplier (if normalized load avg > degraged_load_avg) (default 4.0)\n degraged-multiply = 4.0\n # service discovery normilized load avg degraded point (default 1.0)\n degraged-load-avg = 1.0\n # service discovery type\n service-discovery-type = 0\n # service discovery address (consul)\n service-discovery = \"\"\n # service discovery namespace (graphite by default)\n service-discovery-ns = \"\"\n # service discovery datacenters (first - is primary, in other register as backup)\n service-discovery-ds = []\n # service discovery expire duration for cleanup (minimum is 24h, if enabled)\n service-discovery-expire = \"0s\"\n\n # find/tags cache config\n [common.find-cache]\n # cache type\n type = \"null\"\n # cache size\n size-mb = 0\n # memcached servers\n memcached-servers = []\n # default cache ttl\n default-timeout = 0\n # short-time cache ttl\n short-timeout = 0\n # finder/tags autocompleter cache ttl\n find-timeout = 0\n # maximum diration, used with short_timeout\n short-duration = \"0s\"\n # offset beetween now and until for select short cache timeout\n short-offset = 0\n\n[feature-flags]\n # if true, prefers carbon's behaviour on how tags are treated\n use-carbon-behaviour = false\n # if true, seriesByTag terms containing '!=' or '!=~' operators will not match metrics that don't have the tag at all\n dont-match-missing-tags = false\n # if true, gch will log affected rows count by clickhouse query\n log-query-progress = false\n\n[metrics]\n # graphite relay address\n metric-endpoint = \"\"\n # statsd server address\n statsd-endpoint = \"\"\n # Extended metrics\n extended-stat = false\n # graphite metrics send interval\n metric-interval = \"0s\"\n # graphite metrics send timeout\n metric-timeout = \"0s\"\n # graphite metrics prefix\n metric-prefix = \"\"\n # Request historgram buckets widths\n request-buckets = []\n # Request historgram buckets labels\n request-labels = []\n\n # Additional separate stats for until-from ranges\n [metrics.ranges]\n\n # Additional separate stats for until-from find ranges\n [metrics.find-ranges]\n\n[clickhouse]\n # default url, see https://clickhouse.tech/docs/en/interfaces/http. Can be overwritten with query-params\n url = \"http://localhost:8123?cancel_http_readonly_queries_on_client_close=1\"\n # default total timeout to fetch data, can be overwritten with query-params\n data-timeout = \"1m0s\"\n # time interval for ch query progress sending, it's equal to http_headers_progress_interval_ms header\n progress-sending-interval = \"10s\"\n # Max queries to render queiries\n render-max-queries = 0\n # Concurrent queries to render queiries\n render-concurrent-queries = 0\n # Render adaptive queries (based on load average) for increase/decrease concurrent queries\n render-adaptive-queries = 0\n # Max queries for find queries\n find-max-queries = 0\n # Find concurrent queries for find queries\n find-concurrent-queries = 0\n # Find adaptive queries (based on load average) for increase/decrease concurrent queries\n find-adaptive-queries = 0\n # Max queries for tags queries\n tags-max-queries = 0\n # Concurrent queries for tags queries\n tags-concurrent-queries = 0\n # Tags adaptive queries (based on load average) for increase/decrease concurrent queries\n tags-adaptive-queries = 0\n # If a wildcard appears both at the start and the end of a plain query at a distance (in terms of nodes) less than wildcard-min-distance, then it will be discarded. This parameter can be used to discard expensive queries.\n wildcard-min-distance = 0\n # Plain queries like '{first,second}.custom.metric.*' are also a subject to wildcard-min-distance restriction. But can be split into 2 queries: 'first.custom.metric.*', 'second.custom.metric.*'. Note that: only one list will be split; if there are wildcard in query before (after) list then reverse (direct) notation will be preferred; if there are wildcards before and after list, then query will not be split\n try-split-query = false\n # Used only if try-split-query is true. Query that contains list will be split if its (list) node index is less or equal to max-node-to-split-index. By default is 0. It is recommended to have this value set to 2 or 3 and increase it very carefully, because 3 or 4 plain nodes without wildcards have good selectivity\n max-node-to-split-index = 0\n # Minimum tags in seriesByTag query\n tags-min-in-query = 0\n # Minimum tags in autocomplete query\n tags-min-in-autocomplete = 0\n\n # customized query limiter for some users\n # [clickhouse.user-limits]\n # Date format (default, utc, both)\n date-format = \"\"\n # see doc/index-table.md\n index-table = \"graphite_index\"\n index-use-daily = true\n # see doc/config.md\n index-reverse = \"auto\"\n\n # [[clickhouse.index-reverses]]\n # rule is used when the target suffix is matched\n # suffix = \"suffix\"\n # same as index-reverse\n # reverse = \"auto\"\n\n # [[clickhouse.index-reverses]]\n # rule is used when the target prefix is matched\n # prefix = \"prefix\"\n # same as index-reverse\n # reverse = \"direct\"\n\n # [[clickhouse.index-reverses]]\n # rule is used when the target regex is matched\n # regex = \"regex\"\n # same as index-reverse\n # reverse = \"reversed\"\n # total timeout to fetch series list from index\n index-timeout = \"1m0s\"\n # 'tagged' table from carbon-clickhouse, required for seriesByTag\n tagged-table = \"graphite_tagged\"\n # Table that contains the total amounts of each tag-value pair. It is used to avoid usage of high cardinality tag-value pairs when querying TaggedTable. If left empty, basic sorting will be used. See more detailed description in doc/config.md\n tags-count-table = \"\"\n # or how long the daemon will query tags during autocomplete\n tagged-autocomplete-days = 7\n # whether to use date filter when searching for the metrics in the tagged-table\n tagged-use-daily = true\n\n # costs for tags (for tune which tag will be used as primary), by default is 0, increase for costly (with poor selectivity) tags\n # [clickhouse.tagged-costs]\n # old index table, DEPRECATED, see description in doc/config.md\n # tree-table = \"\"\n # reverse-tree-table = \"\"\n # date-tree-table = \"\"\n # date-tree-table-version = 0\n # tree-timeout = \"0s\"\n # is not recommended to use, https://github.com/lomik/graphite-clickhouse/wiki/TagsRU\n # tag-table = \"\"\n # add extra prefix (directory in graphite) for all metrics, w/o trailing dot\n extra-prefix = \"\"\n # TCP connection timeout\n connect-timeout = \"1s\"\n # will be removed in 0.14\n # data-table = \"\"\n # rollup-conf = \"auto\"\n # max points per metric when internal-aggregation=true\n max-data-points = 1048576\n # ClickHouse-side aggregation, see doc/aggregation.md\n internal-aggregation = true\n\n # mTLS HTTPS configuration for connecting to clickhouse server\n # [clickhouse.tls]\n # ca-cert = []\n # client-auth = \"\"\n # server-name = \"\"\n # min-version = \"\"\n # max-version = \"\"\n # insecure-skip-verify = false\n # curves = []\n # cipher-suites = []\n\n[[data-table]]\n # data table from carbon-clickhouse\n table = \"graphite_data\"\n # if it stores direct or reversed metrics\n reverse = false\n # maximum age stored in the table\n max-age = \"0s\"\n # minimum age stored in the table\n min-age = \"0s\"\n # maximum until-from interval allowed for the table\n max-interval = \"0s\"\n # minimum until-from interval allowed for the table\n min-interval = \"0s\"\n # table allowed only if any metrics in target matches regexp\n target-match-any = \"\"\n # table allowed only if all metrics in target matches regexp\n target-match-all = \"\"\n # custom rollup.xml file for table, 'auto' and 'none' are allowed as well\n rollup-conf = \"auto\"\n # custom table for 'rollup-conf=auto', useful for Distributed or MatView\n rollup-auto-table = \"\"\n # rollup update interval for 'rollup-conf=auto'\n rollup-auto-interval = \"1m0s\"\n # is used when none of rules match\n rollup-default-precision = 0\n # is used when none of rules match\n rollup-default-function = \"\"\n # should be set to true if you don't have reverted regexps in rollup-conf for reversed tables\n rollup-use-reverted = false\n # valid values are 'graphite' of 'prometheus'\n context = []\n\n# is not recommended to use, https://github.com/lomik/graphite-clickhouse/wiki/TagsRU\n# [tags]\n # rules = \"\"\n # date = \"\"\n # extra-where = \"\"\n # input-file = \"\"\n # output-file = \"\"\n # number of threads for uploading tags to clickhouse (1 by default)\n # threads = 1\n # compression method for tags before sending them to clickhouse (i.e. content encoding): gzip (default), none, zstd\n # compression = \"gzip\"\n # fixed tags version for testing purposes (by default the current timestamp is used for each upload)\n # version = 0\n # number of chunks for selecting metrics from clickhouse (10 by default)\n # select-chunks-count = 0\n\n[carbonlink]\n server = \"\"\n threads-per-request = 10\n connect-timeout = \"50ms\"\n query-timeout = \"50ms\"\n # timeout for querying and parsing response\n total-timeout = \"500ms\"\n\n[prometheus]\n # listen addr for prometheus ui and api\n listen = \":9092\"\n # allows to set URL for redirect manually\n external-url = \"\"\n page-title = \"Prometheus Time Series Collection and Processing Server\"\n lookback-delta = \"5m0s\"\n # concurrently handled remote read requests\n remote-read-concurrency-limit = 10\n\n# see doc/debugging.md\n[debug]\n # the directory for additional debug output\n directory = \"\"\n # permissions for directory, octal value is set as 0o755\n directory-perm = 493\n # permissions for directory, octal value is set as 0o640\n external-data-perm = 0\n\n[[logging]]\n # handler name, default empty\n logger = \"\"\n # '/path/to/filename', 'stderr', 'stdout', 'empty' (=='stderr'), 'none'\n file = \"/var/log/graphite-clickhouse/graphite-clickhouse.log\"\n # 'debug', 'info', 'warn', 'error', 'dpanic', 'panic', and 'fatal'\n level = \"info\"\n # 'json' or 'console'\n encoding = \"mixed\"\n # 'millis', 'nanos', 'epoch', 'iso8601'\n encoding-time = \"iso8601\"\n # 'seconds', 'nanos', 'string'\n encoding-duration = \"seconds\"\n # passed to time.ParseDuration\n sample-tick = \"\"\n # first n messages logged per tick\n sample-initial = 0\n # every m-th message logged thereafter per tick\n sample-thereafter = 0\n```\n" }, { "path": "doc/debugging.md", "content": "# Debug graphite-clickhouse\n## General config\nThe `debug` section contains common parameters:\n\n```toml\n[debug]\ndirectory = '/var/log/graphite-clickhouse/debug' # where the additional debug information will be dumped.\ndirectory-perm = '0644' # file mode for the directory. It's applied only if directory does not exist.\n```\n\n## Debug queries with external data\nAll queries to the `data-table` tables use external data. It reduces the SQL parsing time and allows to query big number of metrics without generating 100k+ characters SQL query.\n\nUnfortunately, it requires some additional effort to reproduce the query in case of problems.\n\nIn PR [#126](https://github.com/lomik/graphite-clickhouse/pull/126) it's solved. All you need to do is set the additional config parameter in `[debug]` (see `General config` above):\n\n```toml\n[debug]\nexternal-data-perm = '0640' # to not read the metrics by anybody\n```\n\nAnd pass the HTTP header `X-Gch-Debug-External-Data` with any value in the `/render` or Prometheus request. It will produce the external data dump files in the debug directory and generate a `curl` command in the log on INFO level.\n\nE.g. `[2021-01-26T09:57:33.548+0100] INFO [render] external-data {\"request_id\": \"7994db164f6eef7f2e4da20c54c089f2\", \"debug command\": \"curl -F 'metrics_list=@/tmp/ext-data-debug/ext-metrics_list:7994db164f6eef7f2e4da20c54c089f2.TSV;' 'http://graphite:xxxxx@clickhouse-hostname.tld:8123/?cancel_http_readonly_queries_on_client_close=1&metrics_list_format=TSV&metrics_list_structure=Path+String&query=SELECT+Path%2C%0A%09arrayFilter%28x-%3EisNotNull%28x%29%2C+anyOrNullResample%281611590400%2C+1611594059%2C+60%29%28toUInt32%28intDiv%28Time%2C+60%29%2A60%29%2C+Time%29%29%2C%0A%09arrayFilter%28x-%3EisNotNull%28x%29%2C+avgOrNullResample%281611590400%2C+1611594059%2C+60%29%28Value%2C+Time%29%29%0AFROM+graphite.data%0APREWHERE+Date+%3E%3D%272021-01-25%27+AND+Date+%3C%3D+%272021-01-25%27%0AWHERE+%28Path+in+metrics_list%29+AND+%28Time+%3E%3D+1611590400+AND+Time+%3C%3D+1611594059%29%0AGROUP+BY+Path%0AFORMAT+RowBinary&query_id=7994db164f6eef7f2e4da20c54c089f2%3Adebug'\"}`\n\nIf URL contains user and password, it will be redacted to not expose the credentials.\n\n## Debug render data\nAll supported formats of `/render` handler are binary and may be difficult to debug. Although it's possible.\n\n### format=pickle\nTo get the data in text format you may pipe the output to the following command: \n`curl 'localhost:9090/render/?format=pickle&target=metric.name&from=1619777413&until=1619778013' | python3 -c 'import pickle, sys; print(pickle.loads(sys.stdin.buffer.read()))'`\n\n### format=protobuf (or format=carbonapi_v2_pb)\nThe format is relatively easy to debug. You should have `protoc` binary installed. It's usually available in `protobuf` package. Then you can run the following command from the root of the repository: \n`curl 'localhost:9090/render/?format=protobuf&target=metric.name&from=1619777413&until=1619778013' | protoc --decode carbonapi_v2_pb.MultiFetchResponse -Ivendor/ vendor/github.com/go-graphite/protocol/carbonapi_v2_pb/carbonapi_v2_pb.proto`\n\nIf the repository is not available, there's still a way to run `protoc --decode_raw`, but it's much less readable.\n\n### format=carbonapi_v3_pb\nThe format is the most efficient in the meaning of network traffic and memory. At the same time it is the least debug-able. The request itself is done by sending a POST body with `carbonapi_v3_pb.MultiFetchRequest` protobuf message. So, first one have to generate the request itself, pipe it to curl, and then decode the request. To do it one should run the following command in the root of the repository:\n\n```\necho 'metrics{startTime: 1619777413, stopTime: 1619778013, pathExpression: \"metric.name\"}' | \\\n protoc --encode=carbonapi_v3_pb.MultiFetchRequest -Ivendor/ vendor/github.com/go-graphite/protocol/carbonapi_v3_pb/carbonapi_v3_pb.proto | \\\n curl -XGET --data-binary @- 'localhost:9090/render/?format=carbonapi_v3_pb' | \\\n protoc --decode=carbonapi_v3_pb.MultiFetchResponse -Ivendor/ vendor/github.com/go-graphite/protocol/carbonapi_v3_pb/carbonapi_v3_pb.proto\n```\n\nTo make it a little bit easier the JSON format is implemented.\n\n### format=json\nThe format exists only for debugging purpose and enabled by passing a header `X-Gch-Debug-Output: any string`. Here is a general way to debug the data:\n\n- Optional: make a request to the frontend (carbonapi) with additional header `X-Gch-Debug-Output: a`. Then in log a similar line will be generated: \n `INFO [render.pb3parser] v3pb_request {\"request_id\": \"051fe964d78d9f3d33827397df779ba0\", \"json\": \"{\\\"metrics\\\":[{\\\"name\\\":\\\"metric.name\\\",\\\"startTime\\\":1619777413,\\\"stopTime\\\":1619778013,\\\"pathExpression\\\":\\\"metric.name\\\",\\\"maxDataPoints\\\":700}]}\"}`\n- Get the request ID from the responses request, for example: `X-Gch-Request-Id: 051fe964d78d9f3d33827397df779ba0`\n- In logs either see the JSON body itself for the query ID, or look for `[render.pb3parser] pb3_target` record.\n- Now to make a request just run: \n`curl -H 'Content-Type: application/json' -H 'Content-Type: application/json' -d \"{\\\"metrics\\\":[{\\\"name\\\":\\\"metric.name\\\",\\\"startTime\\\":1619777413,\\\"stopTime\\\":1619778013,\\\"pathExpression\\\":\\\"metric.name\\\",\\\"maxDataPoints\\\":700}]}\" 'localhost:9090/render/?format=json'`\n\n### Marshal protobuf data with original marshallers\nBoth `carbonapi_v2_pb` and `carbonapi_v3_proto` have the optimized marshallers to convert ClickHouse data points to the protobuf response. But when it's necessary, it's possible to debug if the proper data is produced by passing `X-Gch-Debug-Protobuf: 1` header.\n" }, { "path": "doc/graphite_clickhouse.gliffy", "content": "{\"contentType\":\"application/gliffy+json\",\"version\":\"1.3\",\"stage\":{\"background\":\"#FFFFFF\",\"width\":700,\"height\":710,\"nodeIndex\":146,\"autoFit\":true,\"exportBorder\":false,\"gridOn\":true,\"snapToGrid\":true,\"drawingGuidesOn\":true,\"pageBreaksOn\":false,\"printGridOn\":false,\"printPaper\":\"LETTER\",\"printShrinkToFit\":false,\"printPortrait\":true,\"maxWidth\":5000,\"maxHeight\":5000,\"themeData\":null,\"imageCache\":null,\"viewportType\":\"default\",\"fitBB\":{\"min\":{\"x\":30,\"y\":10},\"max\":{\"x\":700,\"y\":710}},\"printModel\":{\"pageSize\":\"Letter\",\"portrait\":true,\"fitToOnePage\":false,\"displayPageBreaks\":false},\"objects\":[{\"x\":362.0,\"y\":708.0,\"rotation\":0.0,\"id\":140,\"width\":159.0,\"height\":86.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":136,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":137,\"py\":0.29289321881345237,\"px\":1.0}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":121,\"py\":1.0,\"px\":0.5}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#666666\",\"fillColor\":\"none\",\"dashStyle\":\"1.0,1.0\",\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[-2.0,-19.213203435596483],[196.5,-19.213203435596483],[196.5,-88.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":262.0,\"y\":715.0,\"rotation\":0.0,\"id\":139,\"width\":141.0,\"height\":123.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":135,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":137,\"py\":0.2928932188134525,\"px\":1.1102230246251563E-16}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":119,\"py\":1.0,\"px\":0.5}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#666666\",\"fillColor\":\"none\",\"dashStyle\":\"1.0,1.0\",\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[-2.0,-26.21320343559637],[-142.0,-26.21320343559637],[-142.0,-125.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":260.0,\"y\":680.0,\"rotation\":0.0,\"id\":137,\"width\":100.0,\"height\":30.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":132,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#666666\",\"fillColor\":\"#FFFFFF\",\"gradient\":false,\"dashStyle\":\"2,2\",\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":2.0,\"y\":0.0,\"rotation\":0.0,\"id\":138,\"width\":96.0,\"height\":14.0,\"uid\":null,\"order\":\"auto\",\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

grafana

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":122.0,\"y\":592.0,\"rotation\":0.0,\"id\":133,\"width\":101.0,\"height\":41.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":131,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":119,\"py\":1.0,\"px\":0.7071067811865476}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":123,\"py\":0.5,\"px\":0.0}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#666666\",\"fillColor\":\"none\",\"dashStyle\":\"1.0,1.0\",\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[35.27922061357856,-2.0],[35.27922061357856,48.0],[98.0,48.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":521.0,\"y\":623.0,\"rotation\":0.0,\"id\":132,\"width\":119.0,\"height\":10.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":130,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":121,\"py\":0.5,\"px\":0.0}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":123,\"py\":0.5,\"px\":1.0}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#666666\",\"fillColor\":\"none\",\"dashStyle\":\"1.0,1.0\",\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[-31.0,-23.0],[-76.0,-23.0],[-76.0,17.0],[-121.0,17.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":329.0,\"y\":620.0,\"rotation\":0.0,\"id\":131,\"width\":326.0,\"height\":160.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":129,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":123,\"py\":0.0,\"px\":0.7071067811865476}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":68,\"py\":1.0,\"px\":0.5}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#666666\",\"fillColor\":\"none\",\"dashStyle\":\"1.0,1.0\",\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[18.27922061357856,0.0],[18.27922061357856,-80.0],[235.5,-80.0],[235.5,-160.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":498.0,\"y\":562.0,\"rotation\":0.0,\"id\":130,\"width\":211.0,\"height\":99.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":128,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":121,\"py\":0.0,\"px\":0.7071067811865476}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":68,\"py\":1.0,\"px\":0.7071067811865476}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#666666\",\"fillColor\":\"none\",\"dashStyle\":\"1.0,1.0\",\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[88.873629022557,18.0],[88.873629022557,-42.0],[118.06958851545028,-42.0],[118.06958851545028,-102.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":423.0,\"y\":560.0,\"rotation\":0.0,\"id\":129,\"width\":196.0,\"height\":97.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":127,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":121,\"py\":0.0,\"px\":0.2928932188134524}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":46,\"py\":1.0,\"px\":0.7071067811865476}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#666666\",\"fillColor\":\"none\",\"dashStyle\":\"1.0,1.0\",\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[107.126370977443,20.0],[107.126370977443,-40.0],[-196.93041148454967,-40.0],[-196.93041148454967,-100.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":402.0,\"y\":620.0,\"rotation\":0.0,\"id\":128,\"width\":177.0,\"height\":162.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":126,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":123,\"py\":0.0,\"px\":0.2928932188134524}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":46,\"py\":1.0,\"px\":0.5}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#666666\",\"fillColor\":\"none\",\"dashStyle\":\"1.0,1.0\",\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[-129.27922061357856,0.0],[-129.27922061357856,-80.0],[-227.5,-80.0],[-227.5,-160.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":164.0,\"y\":561.0,\"rotation\":0.0,\"id\":125,\"width\":13.0,\"height\":99.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":124,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":119,\"py\":0.0,\"px\":0.5}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":46,\"py\":0.9999999999999998,\"px\":0.29289321881345254}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#000000\",\"fillColor\":\"none\",\"dashStyle\":null,\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[-41.069588515450334,-10.952305351726068],[-41.069588515450334,-40.96820356781734],[-41.06958851545032,-70.98410178390867],[-41.06958851545032,-101.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":30.0,\"y\":550.0,\"rotation\":0.0,\"id\":119,\"width\":180.0,\"height\":40.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":115,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#333333\",\"fillColor\":\"#e6b8af\",\"gradient\":false,\"dashStyle\":null,\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":3.6,\"y\":0.0,\"rotation\":0.0,\"id\":120,\"width\":172.79999999999998,\"height\":14.0,\"uid\":null,\"order\":\"auto\",\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

graphite-web

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":463.0,\"y\":42.0,\"rotation\":0.0,\"id\":118,\"width\":188.0,\"height\":36.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":114,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":112,\"py\":1.0,\"px\":0.7071067811865476}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":98,\"py\":0.0,\"px\":0.5}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#666666\",\"fillColor\":\"none\",\"dashStyle\":\"1.0,1.0\",\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":53.66972977941759,\"endArrowRotation\":63.66014952022448,\"interpolationType\":\"quadratic\",\"cornerRadius\":null,\"controlPath\":[[-43.29437251522859,-2.0],[-43.29437251522859,18.0],[97.0,18.0],[97.0,38.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":360.0,\"y\":43.0,\"rotation\":0.0,\"id\":116,\"width\":193.0,\"height\":37.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":113,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":112,\"py\":0.9999999999999998,\"px\":0.29289321881345254}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":41,\"py\":0.0,\"px\":0.5}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#666666\",\"fillColor\":\"none\",\"dashStyle\":\"1.0,1.0\",\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":127.52419086511517,\"endArrowRotation\":117.54599022518734,\"interpolationType\":\"quadratic\",\"cornerRadius\":null,\"controlPath\":[[-39.70562748477141,-3.000000000000007],[-39.70562748477141,17.0],[-190.0,17.0],[-190.0,37.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":250.0,\"y\":10.0,\"rotation\":0.0,\"id\":112,\"width\":240.0,\"height\":30.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":110,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#666666\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":\"2,2\",\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":2.0,\"y\":0.0,\"rotation\":0.0,\"id\":113,\"width\":236.0,\"height\":14.0,\"uid\":null,\"order\":\"auto\",\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

Balancer / Mirroring

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":298.0,\"y\":300.0,\"rotation\":0.0,\"id\":108,\"width\":140.0,\"height\":50.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.double_arrow\",\"order\":107,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.double_arrow.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#666666\",\"fillColor\":\"#FFFFFF\",\"gradient\":false,\"dashStyle\":\"2,2\",\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":1.4,\"y\":0.0,\"rotation\":0.0,\"id\":110,\"width\":137.20000000000002,\"height\":14.0,\"uid\":null,\"order\":\"auto\",\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

Apache ZooKeeper

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":490.0,\"y\":580.0,\"rotation\":0.0,\"id\":121,\"width\":137.0,\"height\":40.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":118,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#666666\",\"fillColor\":\"#FFFFFF\",\"gradient\":false,\"dashStyle\":\"2,2\",\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":2.74,\"y\":0.0,\"rotation\":0.0,\"id\":122,\"width\":131.51999999999998,\"height\":14.0,\"uid\":null,\"order\":120,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

carbonapi

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":220.0,\"y\":620.0,\"rotation\":0.0,\"id\":123,\"width\":180.0,\"height\":40.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":121,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#666666\",\"fillColor\":\"#FFFFFF\",\"gradient\":false,\"dashStyle\":\"2,2\",\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":3.6,\"y\":0.0,\"rotation\":0.0,\"id\":124,\"width\":172.79999999999998,\"height\":14.0,\"uid\":null,\"order\":123,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

carbonzipper

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":174.0,\"y\":571.0,\"rotation\":0.0,\"id\":127,\"width\":13.0,\"height\":99.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":125,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":119,\"py\":0.0,\"px\":0.7071067811865476}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":68,\"py\":0.9999999999999998,\"px\":0.29289321881345254}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#666666\",\"fillColor\":\"none\",\"dashStyle\":\"1.0,1.0\",\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[-16.72077938642144,-21.0],[-16.72077938642144,-66.0],[338.9304114845497,-66.0],[338.9304114845497,-111.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":169.0,\"y\":243.0,\"rotation\":0.0,\"id\":39,\"width\":55.0,\"height\":77.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":43,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":19,\"py\":1.0,\"px\":0.5}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":31,\"py\":0.0,\"px\":0.5}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#000000\",\"fillColor\":\"none\",\"dashStyle\":null,\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[1.0,-3.0],[1.0,39.5],[-59.00000000000003,39.5],[-59.00000000000003,82.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":219.0,\"y\":242.0,\"rotation\":0.0,\"id\":40,\"width\":32.0,\"height\":86.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":44,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":19,\"py\":1.0,\"px\":0.5}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":35,\"py\":0.0,\"px\":0.5}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#000000\",\"fillColor\":\"none\",\"dashStyle\":null,\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[-49.0,-2.0],[-49.0,40.5],[11.0,40.5],[11.0,83.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":50.0,\"y\":290.0,\"rotation\":0.0,\"id\":45,\"width\":245.0,\"height\":70.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.group\",\"order\":46,\"lockAspectRatio\":false,\"lockShape\":false,\"children\":[{\"x\":46.0,\"y\":10.0,\"rotation\":0.0,\"id\":38,\"width\":150.0,\"height\":14.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.text\",\"order\":42,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":2,\"paddingRight\":2,\"paddingBottom\":2,\"paddingLeft\":2,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

ClickHouse

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":130.0,\"y\":35.0,\"rotation\":0.0,\"id\":35,\"width\":100.0,\"height\":20.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":37,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#000000\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":null,\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":1.0,\"y\":0.0,\"rotation\":0.0,\"id\":36,\"width\":97.99999999999997,\"height\":14.0,\"uid\":null,\"order\":40,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

graphite_tree

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":14.999999999999972,\"y\":35.0,\"rotation\":0.0,\"id\":31,\"width\":90.0,\"height\":20.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":32,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#000000\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":null,\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":0.9,\"y\":0.0,\"rotation\":0.0,\"id\":32,\"width\":88.19999999999999,\"height\":14.0,\"uid\":null,\"order\":\"auto\",\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

graphite

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":0.0,\"y\":0.0,\"rotation\":0.0,\"id\":29,\"width\":245.0,\"height\":70.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":30,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#000000\",\"fillColor\":\"#fff2cc\",\"gradient\":false,\"dashStyle\":null,\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":111.0,\"y\":399.0,\"rotation\":0.0,\"id\":56,\"width\":1.0,\"height\":53.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":61,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":52,\"py\":0.0,\"px\":0.5}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":31,\"py\":1.0,\"px\":0.5}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#000000\",\"fillColor\":\"none\",\"dashStyle\":null,\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[-1.0000000000000426,11.06919393998976],[-1.0000000000000426,-10.620537373340142],[-1.0000000000000284,-32.3102686866701],[-1.0000000000000284,-54.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":106.0,\"y\":398.0,\"rotation\":0.0,\"id\":57,\"width\":128.0,\"height\":49.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":62,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":52,\"py\":0.0,\"px\":0.7071067811865476}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":35,\"py\":0.9999999999999998,\"px\":0.29289321881345254}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#000000\",\"fillColor\":\"none\",\"dashStyle\":null,\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[27.710678118654727,12.0],[27.710678118654727,-20.5],[103.28932188134524,-20.5],[103.28932188134524,-53.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":228.0,\"y\":399.0,\"rotation\":0.0,\"id\":58,\"width\":3.0,\"height\":49.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":63,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":48,\"py\":0.0,\"px\":0.5}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":35,\"py\":1.0,\"px\":0.5}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#000000\",\"fillColor\":\"none\",\"dashStyle\":null,\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[2.0,11.06919393998976],[2.0,-10.620537373340142],[2.0,-32.3102686866701],[2.0,-54.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":50.0,\"y\":400.0,\"rotation\":0.0,\"id\":59,\"width\":249.0,\"height\":60.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.group\",\"order\":64,\"lockAspectRatio\":false,\"lockShape\":false,\"children\":[{\"x\":45.0,\"y\":39.0,\"rotation\":0.0,\"id\":55,\"width\":150.0,\"height\":14.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.text\",\"order\":60,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":2,\"paddingRight\":2,\"paddingBottom\":2,\"paddingLeft\":2,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

graphite-clickhouse

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":12.999999999999972,\"y\":10.0,\"rotation\":0.0,\"id\":52,\"width\":100.0,\"height\":20.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":55,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#000000\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":null,\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":2.5,\"y\":0.0,\"rotation\":0.0,\"id\":53,\"width\":95.0,\"height\":14.0,\"uid\":null,\"order\":58,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

/render/

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":133.0,\"y\":10.0,\"rotation\":0.0,\"id\":48,\"width\":100.0,\"height\":20.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":50,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#000000\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":null,\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":2.5,\"y\":0.0,\"rotation\":0.0,\"id\":49,\"width\":95.0,\"height\":14.0,\"uid\":null,\"order\":\"auto\",\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

/metrics/find/

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":0.0,\"y\":0.0,\"rotation\":0.0,\"id\":46,\"width\":249.0,\"height\":60.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":48,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#000000\",\"fillColor\":\"#d9d2e9\",\"gradient\":false,\"dashStyle\":null,\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":30.0,\"y\":80.0,\"rotation\":0.0,\"id\":41,\"width\":280.0,\"height\":170.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":0,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#000000\",\"fillColor\":\"#d0e0e3\",\"gradient\":false,\"dashStyle\":null,\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":105.0,\"y\":180.0,\"rotation\":0.0,\"id\":11,\"width\":130.0,\"height\":20.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":18,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#000000\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":null,\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":1.625,\"y\":0.0,\"rotation\":0.0,\"id\":12,\"width\":126.75,\"height\":14.0,\"uid\":null,\"order\":\"auto\",\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

disk buffer

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":50.0,\"y\":220.0,\"rotation\":0.0,\"id\":19,\"width\":240.0,\"height\":20.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":24,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#000000\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":null,\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":2.0,\"y\":0.0,\"rotation\":0.0,\"id\":20,\"width\":236.0,\"height\":14.0,\"uid\":null,\"order\":\"auto\",\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

Batch upload to clickhouse

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":170.0,\"y\":148.0,\"rotation\":0.0,\"id\":23,\"width\":5.0,\"height\":32.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":27,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":13,\"py\":1.0,\"px\":0.5}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":11,\"py\":0.0,\"px\":0.5}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#000000\",\"fillColor\":\"none\",\"dashStyle\":null,\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[0.0,9.0],[0.0,16.666666666666657],[0.0,24.333333333333343],[0.0,32.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":170.0,\"y\":200.0,\"rotation\":0.0,\"id\":26,\"width\":1.0,\"height\":19.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":28,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":11,\"py\":1.0,\"px\":0.5}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":19,\"py\":0.0,\"px\":0.5}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#000000\",\"fillColor\":\"none\",\"dashStyle\":null,\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[0.0,0.0],[0.0,6.666666666666657],[0.0,13.333333333333343],[0.0,20.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":97.0,\"y\":83.0,\"rotation\":0.0,\"id\":42,\"width\":150.0,\"height\":14.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.text\",\"order\":45,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":2,\"paddingRight\":2,\"paddingBottom\":2,\"paddingLeft\":2,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

carbon-clickhouse

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":420.0,\"y\":80.0,\"rotation\":0.0,\"id\":98,\"width\":280.0,\"height\":170.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":65,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#666666\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":\"2,2\",\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":440.0,\"y\":104.0,\"rotation\":0.0,\"id\":95,\"width\":240.0,\"height\":53.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":66,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#666666\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":\"2,2\",\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":460.0,\"y\":130.0,\"rotation\":0.0,\"id\":93,\"width\":93.00000000000001,\"height\":19.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":67,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#000000\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":\"2,2\",\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":3.795918367346939,\"y\":0.0,\"rotation\":0.0,\"id\":94,\"width\":85.40816326530613,\"height\":14.0,\"uid\":null,\"order\":69,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

UDP

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":569.0,\"y\":130.0,\"rotation\":0.0,\"id\":89,\"width\":94.0,\"height\":20.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":73,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#000000\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":\"2,2\",\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":3.1333333333333337,\"y\":0.0,\"rotation\":0.0,\"id\":90,\"width\":87.73333333333336,\"height\":14.0,\"uid\":null,\"order\":75,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

TCP

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":495.0,\"y\":180.0,\"rotation\":0.0,\"id\":96,\"width\":130.0,\"height\":20.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":76,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#666666\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":\"2,2\",\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":1.625,\"y\":0.0,\"rotation\":0.0,\"id\":97,\"width\":126.75,\"height\":14.0,\"uid\":null,\"order\":78,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

disk buffer

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":487.0,\"y\":107.5,\"rotation\":0.0,\"id\":88,\"width\":150.0,\"height\":14.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.text\",\"order\":79,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":2,\"paddingRight\":2,\"paddingBottom\":2,\"paddingLeft\":2,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

Receiver

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":440.0,\"y\":220.0,\"rotation\":0.0,\"id\":85,\"width\":240.0,\"height\":20.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":80,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#666666\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":\"2,2\",\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":2.0,\"y\":0.0,\"rotation\":0.0,\"id\":86,\"width\":236.0,\"height\":14.0,\"uid\":null,\"order\":82,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

Batch upload to clickhouse

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":560.0,\"y\":148.0,\"rotation\":0.0,\"id\":83,\"width\":5.0,\"height\":32.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":83,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":95,\"py\":1.0,\"px\":0.5}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":96,\"py\":0.0,\"px\":0.5}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#666666\",\"fillColor\":\"none\",\"dashStyle\":\"1.0,1.0\",\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":null,\"controlPath\":[[0.0,9.0],[0.0,32.0]],\"lockSegments\":{},\"ortho\":false}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":560.0,\"y\":200.0,\"rotation\":0.0,\"id\":81,\"width\":1.0,\"height\":19.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":84,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":96,\"py\":1.0,\"px\":0.5}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":85,\"py\":0.0,\"px\":0.5}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#666666\",\"fillColor\":\"none\",\"dashStyle\":\"1.0,1.0\",\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[0.0,0.0],[0.0,6.666666666666657],[0.0,13.333333333333343],[0.0,20.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":440.0,\"y\":290.0,\"rotation\":0.0,\"id\":78,\"width\":245.0,\"height\":70.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":85,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#666666\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":\"2,2\",\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":455.0,\"y\":325.0,\"rotation\":0.0,\"id\":76,\"width\":90.0,\"height\":20.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":86,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#666666\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":\"2,2\",\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":0.9,\"y\":0.0,\"rotation\":0.0,\"id\":77,\"width\":88.19999999999999,\"height\":14.0,\"uid\":null,\"order\":88,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

graphite

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":570.0,\"y\":325.0,\"rotation\":0.0,\"id\":74,\"width\":100.0,\"height\":20.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":89,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#666666\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":\"2,2\",\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":1.0,\"y\":0.0,\"rotation\":0.0,\"id\":75,\"width\":97.99999999999997,\"height\":14.0,\"uid\":null,\"order\":91,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

graphite_tree

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":486.0,\"y\":300.0,\"rotation\":0.0,\"id\":73,\"width\":150.0,\"height\":14.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.text\",\"order\":92,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":2,\"paddingRight\":2,\"paddingBottom\":2,\"paddingLeft\":2,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

ClickHouse

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":559.0,\"y\":243.0,\"rotation\":0.0,\"id\":100,\"width\":55.0,\"height\":77.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":93,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":85,\"py\":1.0,\"px\":0.5}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":76,\"py\":0.0,\"px\":0.5}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#666666\",\"fillColor\":\"none\",\"dashStyle\":\"1.0,1.0\",\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[1.0,-3.0],[1.0,39.5],[-59.0,39.5],[-59.0,82.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":609.0,\"y\":242.0,\"rotation\":0.0,\"id\":99,\"width\":32.0,\"height\":86.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":94,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":85,\"py\":1.0,\"px\":0.5}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":74,\"py\":0.0,\"px\":0.5}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#666666\",\"fillColor\":\"none\",\"dashStyle\":\"1.0,1.0\",\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[-49.0,-2.0],[-49.0,40.5],[11.0,40.5],[11.0,83.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":487.0,\"y\":83.0,\"rotation\":0.0,\"id\":80,\"width\":150.0,\"height\":14.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.text\",\"order\":95,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":2,\"paddingRight\":2,\"paddingBottom\":2,\"paddingLeft\":2,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

carbon-clickhouse

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":440.0,\"y\":400.0,\"rotation\":0.0,\"id\":68,\"width\":249.0,\"height\":60.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":96,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#666666\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":\"2,2\",\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":573.0,\"y\":410.0,\"rotation\":0.0,\"id\":66,\"width\":100.0,\"height\":20.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":97,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#666666\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":\"2,2\",\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":2.5,\"y\":0.0,\"rotation\":0.0,\"id\":67,\"width\":95.0,\"height\":14.0,\"uid\":null,\"order\":99,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

/metrics/find/

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":453.0,\"y\":410.0,\"rotation\":0.0,\"id\":64,\"width\":100.0,\"height\":20.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":100,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#666666\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":\"2,2\",\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":2.5,\"y\":0.0,\"rotation\":0.0,\"id\":65,\"width\":95.0,\"height\":14.0,\"uid\":null,\"order\":102,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

/render/

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":485.0,\"y\":439.0,\"rotation\":0.0,\"id\":63,\"width\":150.0,\"height\":14.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.text\",\"order\":103,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":2,\"paddingRight\":2,\"paddingBottom\":2,\"paddingLeft\":2,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

graphite-clickhouse

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":501.0,\"y\":399.0,\"rotation\":0.0,\"id\":71,\"width\":1.0,\"height\":53.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":104,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":64,\"py\":0.0,\"px\":0.5}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":76,\"py\":1.0,\"px\":0.5}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#666666\",\"fillColor\":\"none\",\"dashStyle\":\"1.0,1.0\",\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[-1.0,11.06919393998976],[-1.0,-10.620537373340142],[-1.0,-32.3102686866701],[-1.0,-54.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":496.0,\"y\":398.0,\"rotation\":0.0,\"id\":70,\"width\":128.0,\"height\":49.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":105,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":64,\"py\":0.0,\"px\":0.7071067811865476}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":74,\"py\":0.9999999999999998,\"px\":0.29289321881345254}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#666666\",\"fillColor\":\"none\",\"dashStyle\":\"1.0,1.0\",\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[27.710678118654755,12.0],[27.710678118654755,-20.5],[103.28932188134524,-20.5],[103.28932188134524,-53.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":618.0,\"y\":399.0,\"rotation\":0.0,\"id\":69,\"width\":3.0,\"height\":49.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.line\",\"order\":106,\"lockAspectRatio\":false,\"lockShape\":false,\"constraints\":{\"constraints\":[],\"startConstraint\":{\"type\":\"StartPositionConstraint\",\"StartPositionConstraint\":{\"nodeId\":66,\"py\":0.0,\"px\":0.5}},\"endConstraint\":{\"type\":\"EndPositionConstraint\",\"EndPositionConstraint\":{\"nodeId\":74,\"py\":1.0,\"px\":0.5}}},\"graphic\":{\"type\":\"Line\",\"Line\":{\"strokeWidth\":2.0,\"strokeColor\":\"#666666\",\"fillColor\":\"none\",\"dashStyle\":\"1.0,1.0\",\"startArrow\":0,\"endArrow\":1,\"startArrowRotation\":\"auto\",\"endArrowRotation\":\"auto\",\"interpolationType\":\"linear\",\"cornerRadius\":10.0,\"controlPath\":[[2.0,11.06919393998976],[2.0,-10.620537373340142],[2.0,-32.3102686866701],[2.0,-54.0]],\"lockSegments\":{},\"ortho\":true}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":50.0,\"y\":104.0,\"rotation\":0.0,\"id\":13,\"width\":240.0,\"height\":53.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":2,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#000000\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":null,\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":70.0,\"y\":130.0,\"rotation\":0.0,\"id\":0,\"width\":93.0,\"height\":19.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":4,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#000000\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":null,\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":3.795918367346938,\"y\":0.0,\"rotation\":0.0,\"id\":1,\"width\":85.4081632653061,\"height\":14.0,\"uid\":null,\"order\":\"auto\",\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

UDP

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":180.0,\"y\":130.0,\"rotation\":0.0,\"id\":9,\"width\":90.0,\"height\":20.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.rectangle\",\"order\":14,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Shape\",\"Shape\":{\"tid\":\"com.gliffy.stencil.rectangle.basic_v1\",\"strokeWidth\":1.0,\"strokeColor\":\"#000000\",\"fillColor\":\"#ffffff\",\"gradient\":false,\"dashStyle\":null,\"dropShadow\":false,\"state\":0,\"opacity\":1.0,\"shadowX\":0.0,\"shadowY\":0.0}},\"linkMap\":[],\"children\":[{\"x\":3.0,\"y\":0.0,\"rotation\":0.0,\"id\":10,\"width\":84.00000000000003,\"height\":14.0,\"uid\":null,\"order\":\"auto\",\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":8,\"paddingRight\":8,\"paddingBottom\":8,\"paddingLeft\":8,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

TCP

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"},{\"x\":97.0,\"y\":107.5,\"rotation\":0.0,\"id\":16,\"width\":150.0,\"height\":14.0,\"uid\":\"com.gliffy.shape.basic.basic_v1.default.text\",\"order\":22,\"lockAspectRatio\":false,\"lockShape\":false,\"graphic\":{\"type\":\"Text\",\"Text\":{\"overflow\":\"none\",\"paddingTop\":2,\"paddingRight\":2,\"paddingBottom\":2,\"paddingLeft\":2,\"outerPaddingTop\":6,\"outerPaddingRight\":6,\"outerPaddingBottom\":2,\"outerPaddingLeft\":6,\"type\":\"fixed\",\"lineTValue\":null,\"linePerpValue\":null,\"cardinalityType\":null,\"html\":\"

Receiver

\",\"tid\":null,\"valign\":\"middle\",\"vposition\":\"none\",\"hposition\":\"none\"}},\"linkMap\":[],\"children\":[],\"hidden\":false,\"layerId\":\"dv8wPdfFedCX\"}],\"layers\":[{\"guid\":\"dv8wPdfFedCX\",\"order\":0,\"name\":\"Layer 0\",\"active\":true,\"locked\":false,\"visible\":true,\"nodeIndex\":142}],\"shapeStyles\":{},\"lineStyles\":{\"global\":{\"stroke\":\"#666666\",\"strokeWidth\":2,\"dashStyle\":\"1.0,1.0\",\"endArrow\":1,\"orthoMode\":1}},\"textStyles\":{\"global\":{\"color\":\"#666666\"}}},\"metadata\":{\"title\":\"untitled\",\"revision\":0,\"exportBorder\":false,\"loadPosition\":\"default\",\"libraries\":[\"com.gliffy.libraries.basic.basic_v1.default\",\"com.gliffy.libraries.flowchart.flowchart_v1.default\",\"com.gliffy.libraries.swimlanes.swimlanes_v1.default\",\"com.gliffy.libraries.uml.uml_v1.default\",\"com.gliffy.libraries.erd.erd_v1.default\",\"com.gliffy.libraries.ui.ui_v2.forms_components\",\"com.gliffy.libraries.network.network_v3.home\",\"com.gliffy.libraries.images\"],\"autosaveDisabled\":false,\"lastSerialized\":1481700752646,\"analyticsProduct\":\"Online\",\"editorVersion\":\"2.16.2\"},\"embeddedResources\":{\"index\":0,\"resources\":[]}}" }, { "path": "doc/index-table.md", "content": "# Index table\nThe `index` type table is used to look up metrics that [match the query](https://graphite.readthedocs.io/en/latest/render_api.html#paths-and-wildcards).\n\n```sql\nCREATE TABLE graphite_index (\n Date Date,\n Level UInt32,\n Path String,\n Version UInt32\n) ENGINE = ReplacingMergeTree(Version)\nPARTITION BY toYYYYMM(Date)\nORDER BY (Level, Path, Date);\n```\n\nWhere:\n* `Date` - date from received point. Or constant date for full metric list (`1970-02-12` (`toDate(42)`) by default)\n* `Level` - metrics depth, see description below\n* `Version` - unix timestamp when the last point was received, only the last one is stored in ReplacingMergeTree table engine\n\nEach metric creates multiple entries in a table:\n* daily with direct Path and plain level\n* daily with reversed Path and Level = 10000+OriginalLevel\n* records with constant Date and Level = 20000+OriginalLevel for metric itself and all it parents\n* record with constant Date, reversed Path and Level = 30000+OriginalLevel\n\nFor example, getting the metric `lorem.ipsum.dolor.sit.amet` adds the following entries to the table:\n\n| Date | Level | Path | Version |\n| ------------- | ------| -------------------------- | ---------- |\n| 2019-05-14 | 5 | lorem.ipsum.dolor.sit.amet | 1557827619 |\n| 2019-05-14 | 10005 | amet.sit.dolor.ipsum.lorem | 1557827619 |\n| 1970-02-12 | 20001 | lorem. | 1557827619 |\n| 1970-02-12 | 20002 | lorem.ipsum. | 1557827619 |\n| 1970-02-12 | 20003 | lorem.ipsum.dolor. | 1557827619 |\n| 1970-02-12 | 20004 | lorem.ipsum.dolor.sit. | 1557827619 |\n| 1970-02-12 | 20005 | lorem.ipsum.dolor.sit.amet | 1557827619 |\n| 1970-02-12 | 30005 | amet.sit.dolor.ipsum.lorem | 1557827619 |\n\nIf you'd like to use only fixed date for index, `index-use-daily = false` can be set in `[clickhouse]` configuration. To prevent continuous growing up of index table, parameter `disable-daily-index = false` should be set in carbon-clickhouse.\n\n### Migrate `tree` table\n\n```sql\n-- direct Path and parents\nINSERT INTO graphite_index (Date, Level, Path, Version)\nSELECT\n '1970-02-12',\n Level+20000,\n Path,\n Version\nFROM graphite_tree;\n\n-- reversed Path without parents\nINSERT INTO graphite_index (Date, Level, Path, Version)\nSELECT\n '1970-02-12',\n Level+30000,\n arrayStringConcat(arrayMap(x->reverse(x), splitByChar('.', reverse(Path))), '.'),\n Version\nFROM graphite_tree\nWHERE NOT Path LIKE '%.';\n```\n\n### Migrate `series` table\n```sql\n-- direct Path\nINSERT INTO graphite_index (Date, Level, Path, Version)\nSELECT\n Date,\n Level,\n Path,\n Version\nFROM graphite_series;\n\n-- reverse Path\nINSERT INTO graphite_index (Date, Level, Path, Version)\nSELECT\n Date,\n Level+10000,\n arrayStringConcat(arrayMap(x->reverse(x), splitByChar('.', reverse(Path))), '.'),\n Version\nFROM graphite_series;\n```\n\n### Migrate `series-reverse` table\n```sql\n-- direct Path\nINSERT INTO graphite_index (Date, Level, Path, Version)\nSELECT\n Date,\n Level,\n arrayStringConcat(arrayMap(x->reverse(x), splitByChar('.', reverse(Path))), '.'),\n Version\nFROM graphite_series_reverse;\n\n-- reverse Path\nINSERT INTO graphite_index (Date, Level, Path, Version)\nSELECT\n Date,\n Level+10000,\n Path,\n Version\nFROM graphite_series_reverse;\n```\n" }, { "path": "doc/release.md", "content": "# New release\n\n- Update `const Version` in graphite-clickhouse.go\n" }, { "path": "find/find.go", "content": "package find\n\nimport (\n\t\"context\"\n\t\"io\"\n\n\t\"github.com/gogo/protobuf/proto\"\n\t\"github.com/msaf1980/go-stringutils\"\n\n\tv2pb \"github.com/go-graphite/protocol/carbonapi_v2_pb\"\n\tv3pb \"github.com/go-graphite/protocol/carbonapi_v3_pb\"\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/finder\"\n\t\"github.com/lomik/graphite-clickhouse/helper/pickle\"\n)\n\ntype Find struct {\n\tconfig *config.Config\n\tcontext context.Context\n\tquery string // original query\n\tresult finder.Result\n}\n\nfunc NewCached(config *config.Config, body []byte) *Find {\n\treturn &Find{\n\t\tconfig: config,\n\t\tresult: finder.NewCachedIndex(body),\n\t}\n}\n\nfunc New(config *config.Config, ctx context.Context, query string) (*Find, error) {\n\tres, err := finder.Find(config, ctx, query, 0, 0)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &Find{\n\t\tquery: query,\n\t\tconfig: config,\n\t\tcontext: ctx,\n\t\tresult: res,\n\t}, nil\n}\n\nfunc (f *Find) isResultsLimitExceeded(numResults int) bool {\n\treturn f.config.Common.MaxMetricsInFindAnswer != 0 &&\n\t\tnumResults >= f.config.Common.MaxMetricsInFindAnswer\n}\n\nfunc (f *Find) WritePickle(w io.Writer) error {\n\trows := f.result.List()\n\n\tif len(rows) == 0 { // empty\n\t\tw.Write(pickle.EmptyList)\n\t\treturn nil\n\t}\n\n\tp := pickle.NewWriter(w)\n\n\tp.List()\n\n\tvar numResults = 0\n\n\tfor i := 0; i < len(rows); i++ {\n\t\tif len(rows[i]) == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tp.Dict()\n\n\t\tpath, isLeaf := finder.Leaf(rows[i])\n\n\t\tp.String(\"metric_path\")\n\t\tp.Bytes(path)\n\t\tp.SetItem()\n\n\t\tp.String(\"isLeaf\")\n\t\tp.Bool(isLeaf)\n\t\tp.SetItem()\n\n\t\tp.Append()\n\n\t\tnumResults++\n\t\tif f.isResultsLimitExceeded(numResults) {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tp.Stop()\n\n\treturn nil\n}\n\nfunc (f *Find) WriteProtobuf(w io.Writer) error {\n\trows := f.result.List()\n\n\tif len(rows) == 0 { // empty\n\t\treturn nil\n\t}\n\n\t// message GlobMatch {\n\t// required string path = 1;\n\t// required bool isLeaf = 2;\n\t// }\n\n\t// message GlobResponse {\n\t// required string name = 1;\n\t// repeated GlobMatch matches = 2;\n\t// }\n\n\tvar response v2pb.GlobResponse\n\tresponse.Name = f.query\n\n\tvar numResults = 0\n\n\tfor i := 0; i < len(rows); i++ {\n\t\tif len(rows[i]) == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tpath, isLeaf := finder.Leaf(rows[i])\n\n\t\tresponse.Matches = append(response.Matches, v2pb.GlobMatch{\n\t\t\tPath: string(path),\n\t\t\tIsLeaf: isLeaf,\n\t\t})\n\n\t\tnumResults++\n\t\tif f.isResultsLimitExceeded(numResults) {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tbody, err := proto.Marshal(&response)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tw.Write(body)\n\n\treturn nil\n}\n\nfunc (f *Find) WriteProtobufV3(w io.Writer) error {\n\trows := f.result.List()\n\n\tif len(rows) == 0 { // empty\n\t\treturn nil\n\t}\n\n\t// message GlobMatch {\n\t// required string path = 1;\n\t// required bool isLeaf = 2;\n\t// }\n\n\t// message GlobResponse {\n\t// required string name = 1;\n\t// repeated GlobMatch matches = 2;\n\t// }\n\n\tvar response v3pb.GlobResponse\n\tresponse.Name = f.query\n\n\tvar numResults = 0\n\n\tfor i := 0; i < len(rows); i++ {\n\t\tif len(rows[i]) == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tpath, isLeaf := finder.Leaf(rows[i])\n\n\t\tresponse.Matches = append(response.Matches, v3pb.GlobMatch{\n\t\t\tPath: string(path),\n\t\t\tIsLeaf: isLeaf,\n\t\t})\n\n\t\tnumResults++\n\t\tif f.isResultsLimitExceeded(numResults) {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tmultiGlobResponse := v3pb.MultiGlobResponse{\n\t\tMetrics: []v3pb.GlobResponse{\n\t\t\tresponse,\n\t\t},\n\t}\n\n\tbody, err := proto.Marshal(&multiGlobResponse)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tw.Write(body)\n\n\treturn nil\n}\n\nfunc (f *Find) WriteJSON(w io.Writer) error {\n\trows := f.result.List()\n\n\tif len(rows) == 0 { // empty\n\t\treturn nil\n\t}\n\n\tvar numResults int\n\n\tvar sb stringutils.Builder\n\n\tsb.WriteString(\"[\")\n\n\tfor i := 0; i < len(rows); i++ {\n\t\tif len(rows[i]) == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tpath, isLeaf := finder.Leaf(rows[i])\n\n\t\tif numResults == 0 {\n\t\t\tsb.WriteString(\"{path=\\\"\")\n\t\t} else {\n\t\t\tsb.WriteString(\",{path=\\\"\")\n\t\t}\n\n\t\tsb.Write(path)\n\n\t\tif isLeaf {\n\t\t\tsb.WriteString(\"\\\",leaf=1}\")\n\t\t} else {\n\t\t\tsb.WriteString(\"\\\"}\")\n\t\t}\n\n\t\tnumResults++\n\t\tif f.isResultsLimitExceeded(numResults) {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tsb.WriteString(\"]\\r\\n\")\n\n\tw.Write(sb.Bytes())\n\n\treturn nil\n}\n" }, { "path": "find/handler.go", "content": "package find\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/go-graphite/carbonapi/pkg/parser\"\n\tv3pb \"github.com/go-graphite/protocol/carbonapi_v3_pb\"\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/helper/utils\"\n\t\"github.com/lomik/graphite-clickhouse/logs\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"go.uber.org/zap\"\n)\n\ntype Handler struct {\n\tconfig *config.Config\n\tqMetric *metrics.QueryMetrics\n}\n\nfunc NewHandler(config *config.Config) *Handler {\n\treturn &Handler{\n\t\tconfig: config,\n\t\tqMetric: metrics.InitQueryMetrics(\"find\", &config.Metrics),\n\t}\n}\n\nfunc (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {\n\tstart := time.Now()\n\tstatus := http.StatusOK\n\taccessLogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named(\"http\")\n\tlogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named(\"metrics-find\")\n\tr = r.WithContext(scope.WithLogger(r.Context(), logger))\n\n\tvar (\n\t\tmetricsCount int64\n\t\tstat metrics.FinderStat\n\t\tqueueFail bool\n\t\tqueueDuration time.Duration\n\t\tfindCache bool\n\t\tquery string\n\t)\n\n\tusername := r.Header.Get(\"X-Forwarded-User\")\n\tlimiter := h.config.GetUserTagsLimiter(username)\n\n\tdefer func() {\n\t\tif rec := recover(); rec != nil {\n\t\t\tstatus = http.StatusInternalServerError\n\n\t\t\tlogger.Error(\"panic during eval:\",\n\t\t\t\tzap.String(\"requestID\", scope.String(r.Context(), \"requestID\")),\n\t\t\t\tzap.Any(\"reason\", rec),\n\t\t\t\tzap.Stack(\"stack\"),\n\t\t\t)\n\n\t\t\tanswer := fmt.Sprintf(\"%v\\nStack trace: %v\", rec, zap.Stack(\"\").String)\n\t\t\thttp.Error(w, answer, status)\n\t\t}\n\n\t\td := time.Since(start)\n\t\tdMS := d.Milliseconds()\n\t\tlogs.AccessLog(accessLogger, h.config, r, status, d, queueDuration, findCache, queueFail)\n\t\tlimiter.SendDuration(queueDuration.Milliseconds())\n\t\tmetrics.SendFindMetrics(metrics.FindRequestMetric, status, dMS, 0, h.config.Metrics.ExtendedStat, metricsCount)\n\n\t\tif stat.ChReadRows > 0 && stat.ChReadBytes > 0 {\n\t\t\terrored := status != http.StatusOK && status != http.StatusNotFound\n\t\t\tmetrics.SendQueryRead(metrics.FindQMetric, 0, 0, dMS, metricsCount, stat.ReadBytes, stat.ChReadRows, stat.ChReadBytes, errored)\n\t\t}\n\t}()\n\n\tr.ParseMultipartForm(1024 * 1024)\n\n\tformat := r.FormValue(\"format\")\n\tif format == \"carbonapi_v3_pb\" {\n\t\tbody, err := io.ReadAll(r.Body)\n\t\tif err != nil {\n\t\t\tstatus = http.StatusBadRequest\n\t\t\thttp.Error(w, fmt.Sprintf(\"Failed to read request body: %v\", err), status)\n\n\t\t\treturn\n\t\t}\n\n\t\tvar pv3Request v3pb.MultiGlobRequest\n\t\tif err := pv3Request.Unmarshal(body); err != nil {\n\t\t\tstatus = http.StatusBadRequest\n\t\t\thttp.Error(w, fmt.Sprintf(\"Failed to unmarshal request: %v\", err), status)\n\n\t\t\treturn\n\t\t}\n\n\t\tif len(pv3Request.Metrics) != 1 {\n\t\t\tstatus = http.StatusBadRequest\n\t\t\thttp.Error(w, fmt.Sprintf(\"Multiple metrics in same find request is not supported yet: %v\", err), status)\n\n\t\t\treturn\n\t\t}\n\n\t\tquery = pv3Request.Metrics[0]\n\t\tq := r.URL.Query()\n\t\tq.Set(\"query\", query)\n\t\tr.URL.RawQuery = q.Encode()\n\t} else {\n\t\tswitch r.FormValue(\"format\") {\n\t\tcase \"json\":\n\t\tcase \"pickle\":\n\t\tcase \"protobuf\":\n\t\tdefault:\n\t\t\tlogger.Error(\"unsupported formatter\")\n\n\t\t\tstatus = http.StatusBadRequest\n\t\t\thttp.Error(w, \"Failed to parse request: unsupported formatter\", status)\n\n\t\t\treturn\n\t\t}\n\n\t\tquery = r.FormValue(\"query\")\n\t}\n\n\tif len(query) == 0 {\n\t\tstatus = http.StatusBadRequest\n\t\thttp.Error(w, \"Query not set\", status)\n\n\t\treturn\n\t}\n\n\tvar key string\n\t// params := []string{query}\n\tuseCache := h.config.Common.FindCache != nil && h.config.Common.FindCacheConfig.FindTimeoutSec > 0 && !parser.TruthyBool(r.FormValue(\"noCache\"))\n\tif useCache {\n\t\tts := utils.TimestampTruncate(time.Now().Unix(), time.Duration(h.config.Common.FindCacheConfig.FindTimeoutSec)*time.Second)\n\t\tkey = \"1970-02-12;query=\" + query + \";ts=\" + strconv.FormatInt(ts, 10)\n\n\t\tbody, err := h.config.Common.FindCache.Get(key)\n\t\tif err == nil {\n\t\t\tif metrics.FinderCacheMetrics != nil {\n\t\t\t\tmetrics.FinderCacheMetrics.CacheHits.Add(1)\n\t\t\t}\n\n\t\t\tfindCache = true\n\n\t\t\tw.Header().Set(\"X-Cached-Find\", strconv.Itoa(int(h.config.Common.FindCacheConfig.FindTimeoutSec)))\n\t\t\tf := NewCached(h.config, body)\n\t\t\tmetricsCount = int64(len(f.result.List()))\n\t\t\tlogger.Info(\"finder\", zap.String(\"get_cache\", key),\n\t\t\t\tzap.Int64(\"metrics\", metricsCount), zap.Bool(\"find_cached\", true),\n\t\t\t\tzap.Int32(\"ttl\", h.config.Common.FindCacheConfig.FindTimeoutSec))\n\n\t\t\th.Reply(w, r, f)\n\n\t\t\treturn\n\t\t}\n\t}\n\n\tvar (\n\t\tentered bool\n\t\tctx context.Context\n\t\tcancel context.CancelFunc\n\t)\n\n\tif limiter.Enabled() {\n\t\tctx, cancel = context.WithTimeout(context.Background(), h.config.ClickHouse.IndexTimeout)\n\t\tdefer cancel()\n\n\t\terr := limiter.Enter(ctx, \"find\")\n\t\tqueueDuration = time.Since(start)\n\n\t\tif err != nil {\n\t\t\tstatus = http.StatusServiceUnavailable\n\t\t\tqueueFail = true\n\n\t\t\tlogger.Error(err.Error())\n\t\t\thttp.Error(w, err.Error(), status)\n\n\t\t\treturn\n\t\t}\n\n\t\tqueueDuration = time.Since(start)\n\t\tentered = true\n\n\t\tdefer func() {\n\t\t\tif entered {\n\t\t\t\tlimiter.Leave(ctx, \"find\")\n\n\t\t\t\tentered = false\n\t\t\t}\n\t\t}()\n\t}\n\n\tf, err := New(h.config, r.Context(), query)\n\n\tif entered {\n\t\t// release early as possible\n\t\tlimiter.Leave(ctx, \"find\")\n\n\t\tentered = false\n\t}\n\n\tif err != nil {\n\t\tstatus, _ = clickhouse.HandleError(w, err)\n\t\treturn\n\t}\n\n\tif useCache {\n\t\tif body, err := f.result.Bytes(); err == nil {\n\t\t\tif metrics.FinderCacheMetrics != nil {\n\t\t\t\tmetrics.FinderCacheMetrics.CacheMisses.Add(1)\n\t\t\t}\n\n\t\t\th.config.Common.FindCache.Set(key, body, h.config.Common.FindCacheConfig.FindTimeoutSec)\n\t\t\tlogger.Info(\"finder\", zap.String(\"set_cache\", key),\n\t\t\t\tzap.Int(\"metrics\", len(f.result.List())), zap.Bool(\"find_cached\", false),\n\t\t\t\tzap.Int32(\"ttl\", h.config.Common.FindCacheConfig.FindTimeoutSec))\n\t\t}\n\t}\n\n\tmetricsCount = int64(len(f.result.List()))\n\tstatus = h.Reply(w, r, f)\n}\n\nfunc (h *Handler) Reply(w http.ResponseWriter, r *http.Request, f *Find) (status int) {\n\tstatus = http.StatusOK\n\n\tswitch r.FormValue(\"format\") {\n\tcase \"json\":\n\t\tf.WriteJSON(w)\n\tcase \"pickle\":\n\t\tf.WritePickle(w)\n\tcase \"protobuf\":\n\t\tw.Header().Set(\"Content-Type\", \"application/x-protobuf\")\n\t\tf.WriteProtobuf(w)\n\tcase \"carbonapi_v3_pb\":\n\t\tw.Header().Set(\"Content-Type\", \"application/x-protobuf\")\n\t\tf.WriteProtobufV3(w)\n\tdefault:\n\t\tstatus = http.StatusInternalServerError\n\t\thttp.Error(w, \"Failed to parse request: unhandled formatter\", status)\n\t}\n\n\treturn\n}\n" }, { "path": "find/handler_json_test.go", "content": "package find\n\nimport (\n\t\"io\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/tests/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc NewRequest(method, url string, body io.Reader) *http.Request {\n\tr, _ := http.NewRequest(method, url, body)\n\n\treturn r\n}\n\ntype testStruct struct {\n\trequest *http.Request\n\twantCode int\n\twant string\n\twantContent string\n}\n\nfunc testResponce(t *testing.T, step int, h *Handler, tt *testStruct, wantCachedFind string) {\n\tw := httptest.NewRecorder()\n\n\th.ServeHTTP(w, tt.request)\n\n\ts := w.Body.String()\n\n\tassert.Equalf(t, tt.wantCode, w.Code, \"code mismatch step %d\\n,%s\", step, s)\n\n\tif w.Code == http.StatusOK {\n\t\tif tt.wantContent != \"\" {\n\t\t\tcontentType := w.Result().Header[\"Content-Type\"]\n\t\t\tassert.Equalf(t, []string{tt.wantContent}, contentType, \"content type mismatch, step %d\", step)\n\t\t}\n\n\t\tcachedFind := w.Result().Header.Get(\"X-Cached-Find\")\n\t\tassert.Equalf(t, cachedFind, wantCachedFind, \"cached find mismatch, step %d\", step)\n\n\t\tassert.Equalf(t, tt.want, s, \"Step %d\", step)\n\t}\n}\n\nfunc TestHandler_ServeValuesJSON(t *testing.T) {\n\tmetrics.DisableMetrics()\n\n\tsrv := clickhouse.NewTestServer()\n\tdefer srv.Close()\n\n\tcfg, _ := config.DefaultConfig()\n\tcfg.ClickHouse.URL = srv.URL\n\n\th := NewHandler(cfg)\n\n\tsrv.AddResponce(\n\t\t\"SELECT Path FROM graphite_index WHERE ((Level=20003) AND (Path LIKE 'DB.postgres.%')) AND (Date='1970-02-12') GROUP BY Path FORMAT TabSeparatedRaw\",\n\t\t&clickhouse.TestResponse{\n\t\t\tBody: []byte(\"DB.postgres.host1.\\nDB.postgres.host2.\\n\"),\n\t\t})\n\n\tsrv.AddResponce(\n\t\t\"SELECT Path FROM graphite_index WHERE ((Level=20005) AND (Path LIKE 'DB.postgres.%' AND match(Path, '^DB[.]postgres[.]([^.]*?)[.]cpu[.]load_avg[.]?$'))) AND (Date='1970-02-12') GROUP BY Path FORMAT TabSeparatedRaw\",\n\t\t&clickhouse.TestResponse{\n\t\t\tBody: []byte(\"DB.postgres.host1.cpu.load_avg\\nDB.postgres.host2.cpu.load_avg\\n\"),\n\t\t})\n\n\ttests := []testStruct{\n\t\t{\n\t\t\trequest: NewRequest(\"GET\", srv.URL+\"/metrics/find/?format=json&query=DB.postgres.%2A\", nil),\n\t\t\twantCode: http.StatusOK,\n\t\t\twant: \"[{path=\\\"DB.postgres.host1\\\"},{path=\\\"DB.postgres.host2\\\"}]\\r\\n\",\n\t\t\twantContent: \"text/plain; charset=utf-8\",\n\t\t},\n\t\t{\n\t\t\trequest: NewRequest(\"GET\", srv.URL+\"/metrics/find/?format=json&query=DB.postgres.%2A.cpu.load_avg\", nil),\n\t\t\twantCode: http.StatusOK,\n\t\t\twant: \"[{path=\\\"DB.postgres.host1.cpu.load_avg\\\",leaf=1},{path=\\\"DB.postgres.host2.cpu.load_avg\\\",leaf=1}]\\r\\n\",\n\t\t\twantContent: \"text/plain; charset=utf-8\",\n\t\t},\n\t}\n\n\tvar queries uint64\n\n\tfor i, tt := range tests {\n\t\tt.Run(tt.request.URL.RawQuery+\"#\"+strconv.Itoa(i), func(t *testing.T) {\n\t\t\tfor i := 0; i < 2; i++ {\n\t\t\t\ttestResponce(t, i, h, &tt, \"\")\n\t\t\t}\n\n\t\t\tassert.Equal(t, uint64(2), srv.Queries()-queries)\n\t\t\tqueries = srv.Queries()\n\t\t})\n\t}\n}\n\nfunc TestHandler_ServeValuesCachedJSON(t *testing.T) {\n\tsrv := clickhouse.NewTestServer()\n\tdefer srv.Close()\n\n\tcfg, _ := config.DefaultConfig()\n\tcfg.ClickHouse.URL = srv.URL\n\n\t// find cache config\n\tcfg.Common.FindCacheConfig = config.CacheConfig{\n\t\tType: \"mem\",\n\t\tSize: 8192,\n\t\tFindTimeoutSec: 1,\n\t}\n\n\tvar err error\n\n\tcfg.Common.FindCache, err = config.CreateCache(\"metric-finder\", &cfg.Common.FindCacheConfig)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create find cache: %v\", err)\n\t}\n\n\th := NewHandler(cfg)\n\n\tsrv.AddResponce(\n\t\t\"SELECT Path FROM graphite_index WHERE ((Level=20003) AND (Path LIKE 'DB.postgres.%')) AND (Date='1970-02-12') GROUP BY Path FORMAT TabSeparatedRaw\",\n\t\t&clickhouse.TestResponse{\n\t\t\tBody: []byte(\"DB.postgres.host1.\\nDB.postgres.host2.\\n\"),\n\t\t})\n\n\tsrv.AddResponce(\n\t\t\"SELECT Path FROM graphite_index WHERE ((Level=20005) AND (Path LIKE 'DB.postgres.%' AND match(Path, '^DB[.]postgres[.]([^.]*?)[.]cpu[.]load_avg[.]?$'))) AND (Date='1970-02-12') GROUP BY Path FORMAT TabSeparatedRaw\",\n\t\t&clickhouse.TestResponse{\n\t\t\tBody: []byte(\"DB.postgres.host1.cpu.load_avg\\nDB.postgres.host2.cpu.load_avg\\n\"),\n\t\t})\n\n\ttests := []testStruct{\n\t\t{\n\t\t\trequest: NewRequest(\"GET\", srv.URL+\"/metrics/find/?format=json&query=DB.postgres.%2A\", nil),\n\t\t\twantCode: http.StatusOK,\n\t\t\twant: \"[{path=\\\"DB.postgres.host1\\\"},{path=\\\"DB.postgres.host2\\\"}]\\r\\n\",\n\t\t\twantContent: \"text/plain; charset=utf-8\",\n\t\t},\n\t\t{\n\t\t\trequest: NewRequest(\"GET\", srv.URL+\"/metrics/find/?format=json&query=DB.postgres.%2A.cpu.load_avg\", nil),\n\t\t\twantCode: http.StatusOK,\n\t\t\twant: \"[{path=\\\"DB.postgres.host1.cpu.load_avg\\\",leaf=1},{path=\\\"DB.postgres.host2.cpu.load_avg\\\",leaf=1}]\\r\\n\",\n\t\t\twantContent: \"text/plain; charset=utf-8\",\n\t\t},\n\t}\n\n\tvar queries uint64\n\n\tfor i, tt := range tests {\n\t\tt.Run(tt.request.URL.RawQuery+\"#\"+strconv.Itoa(i), func(t *testing.T) {\n\t\t\ttestResponce(t, 0, h, &tt, \"\")\n\t\t\tassert.Equal(t, uint64(1), srv.Queries()-queries)\n\n\t\t\t// query from cache\n\t\t\ttestResponce(t, 1, h, &tt, \"1\")\n\t\t\tassert.Equal(t, uint64(1), srv.Queries()-queries)\n\n\t\t\t// wait for expire cache\n\t\t\ttime.Sleep(time.Second * 2)\n\t\t\ttestResponce(t, 2, h, &tt, \"\")\n\n\t\t\tassert.Equal(t, uint64(2), srv.Queries()-queries)\n\t\t\tqueries = srv.Queries()\n\t\t})\n\t}\n}\n" }, { "path": "find/handler_test.go", "content": "package find\n\nimport (\n\t\"io\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"testing\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n)\n\ntype clickhouseMock struct {\n\trequestLog chan []byte\n}\n\nfunc (m *clickhouseMock) ServeHTTP(w http.ResponseWriter, r *http.Request) {\n\tbody, _ := io.ReadAll(r.Body)\n\n\tif m.requestLog != nil {\n\t\tm.requestLog <- body\n\t}\n}\n\nfunc TestFind(t *testing.T) {\n\ttestCase := func(findQuery, expectedClickHouseQuery string) {\n\t\trequestLog := make(chan []byte, 1)\n\t\tm := &clickhouseMock{\n\t\t\trequestLog: requestLog,\n\t\t}\n\n\t\tsrv := httptest.NewServer(m)\n\t\tdefer srv.Close()\n\n\t\tcfg := config.New()\n\t\tcfg.ClickHouse.URL = srv.URL\n\n\t\thandler := NewHandler(cfg)\n\t\tw := httptest.NewRecorder()\n\t\tr := httptest.NewRequest(\n\t\t\thttp.MethodGet,\n\t\t\t\"http://localhost/metrics/find/?local=1&format=pickle&query=\"+findQuery,\n\t\t\tnil,\n\t\t)\n\n\t\thandler.ServeHTTP(w, r)\n\n\t\tchQuery := <-requestLog\n\n\t\tif string(chQuery) != expectedClickHouseQuery {\n\t\t\tt.Fatalf(\"%#v (actual) != %#v (expected)\", string(chQuery), expectedClickHouseQuery)\n\t\t}\n\t}\n\n\ttestCase(\n\t\t\"host.top.cpu.cpu%2A\",\n\t\t\"SELECT Path FROM graphite_index WHERE ((Level=20004) AND (Path LIKE 'host.top.cpu.cpu%')) AND (Date='1970-02-12') GROUP BY Path FORMAT TabSeparatedRaw\",\n\t)\n\n\ttestCase(\n\t\t\"host.?cpu\",\n\t\t\"SELECT Path FROM graphite_index WHERE ((Level=20002) AND (Path LIKE 'host.%' AND match(Path, '^host[.][^.]cpu[.]?$'))) AND (Date='1970-02-12') GROUP BY Path FORMAT TabSeparatedRaw\",\n\t)\n}\n" }, { "path": "finder/base.go", "content": "package finder\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/where\"\n)\n\nvar ErrNotImplemented = errors.New(\"not implemented\")\n\ntype BaseFinder struct {\n\turl string // clickhouse dsn\n\ttable string // graphite_tree table\n\topts clickhouse.Options // timeout, connectTimeout\n\tbody []byte // clickhouse response body\n\tstats []metrics.FinderStat\n}\n\nfunc NewBase(url string, table string, opts clickhouse.Options) Finder {\n\treturn &BaseFinder{\n\t\turl: url,\n\t\ttable: table,\n\t\topts: opts,\n\t\tstats: make([]metrics.FinderStat, 0),\n\t}\n}\n\nfunc (b *BaseFinder) where(query string) *where.Where {\n\tlevel := strings.Count(query, \".\") + 1\n\n\tw := where.New()\n\tw.And(where.Eq(\"Level\", level))\n\tw.And(where.TreeGlob(\"Path\", query))\n\n\treturn w\n}\n\nfunc (b *BaseFinder) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) (err error) {\n\tw := b.where(query)\n\n\tb.stats = append(b.stats, metrics.FinderStat{})\n\tstat := &b.stats[len(b.stats)-1]\n\n\tb.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query(\n\t\tscope.WithTable(ctx, b.table),\n\t\tb.url,\n\t\t// TODO: consider consistent query generator\n\t\tfmt.Sprintf(\"SELECT Path FROM %s WHERE %s GROUP BY Path FORMAT TabSeparatedRaw\", b.table, w),\n\t\tb.opts,\n\t\tnil,\n\t)\n\tstat.Table = b.table\n\tstat.ReadBytes = int64(len(b.body))\n\n\treturn\n}\n\nfunc (b *BaseFinder) makeList(onlySeries bool) [][]byte {\n\tif b.body == nil {\n\t\treturn [][]byte{}\n\t}\n\n\trows := bytes.Split(b.body, []byte{'\\n'})\n\n\tskip := 0\n\n\tfor i := 0; i < len(rows); i++ {\n\t\tif len(rows[i]) == 0 {\n\t\t\tskip++\n\t\t\tcontinue\n\t\t}\n\n\t\tif onlySeries && rows[i][len(rows[i])-1] == '.' {\n\t\t\tskip++\n\t\t\tcontinue\n\t\t}\n\n\t\tif skip > 0 {\n\t\t\trows[i-skip] = rows[i]\n\t\t}\n\t}\n\n\trows = rows[:len(rows)-skip]\n\n\treturn rows\n}\n\nfunc (b *BaseFinder) List() [][]byte {\n\treturn b.makeList(false)\n}\n\nfunc (b *BaseFinder) Series() [][]byte {\n\treturn b.makeList(true)\n}\n\nfunc (b *BaseFinder) Abs(v []byte) []byte {\n\treturn v\n}\n\nfunc (b *BaseFinder) Bytes() ([]byte, error) {\n\treturn b.body, nil\n}\n\nfunc (b *BaseFinder) Stats() []metrics.FinderStat {\n\treturn b.stats\n}\n" }, { "path": "finder/blacklist.go", "content": "package finder\n\nimport (\n\t\"context\"\n\t\"regexp\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n)\n\ntype BlacklistFinder struct {\n\twrapped Finder\n\tblacklist []*regexp.Regexp // config\n\tmatched bool\n}\n\nfunc WrapBlacklist(f Finder, blacklist []*regexp.Regexp) *BlacklistFinder {\n\treturn &BlacklistFinder{\n\t\twrapped: f,\n\t\tblacklist: blacklist,\n\t}\n}\n\nfunc (p *BlacklistFinder) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) (err error) {\n\tfor i := 0; i < len(p.blacklist); i++ {\n\t\tif p.blacklist[i].MatchString(query) {\n\t\t\tp.matched = true\n\t\t\treturn\n\t\t}\n\t}\n\n\treturn p.wrapped.Execute(ctx, config, query, from, until)\n}\n\nfunc (p *BlacklistFinder) List() [][]byte {\n\tif p.matched {\n\t\treturn [][]byte{}\n\t}\n\n\treturn p.wrapped.List()\n}\n\n// For Render\nfunc (p *BlacklistFinder) Series() [][]byte {\n\tif p.matched {\n\t\treturn [][]byte{}\n\t}\n\n\treturn p.wrapped.Series()\n}\n\nfunc (p *BlacklistFinder) Abs(v []byte) []byte {\n\treturn p.wrapped.Abs(v)\n}\n\nfunc (p *BlacklistFinder) Bytes() ([]byte, error) {\n\treturn nil, ErrNotImplemented\n}\n\nfunc (p *BlacklistFinder) Stats() []metrics.FinderStat {\n\treturn p.wrapped.Stats()\n}\n" }, { "path": "finder/date.go", "content": "package finder\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/where\"\n)\n\ntype DateFinder struct {\n\t*BaseFinder\n\ttableVersion int\n}\n\nfunc NewDateFinder(url string, table string, tableVersion int, opts clickhouse.Options) Finder {\n\tif tableVersion == 3 {\n\t\treturn NewDateFinderV3(url, table, opts)\n\t}\n\n\tb := &BaseFinder{\n\t\turl: url,\n\t\ttable: table,\n\t\topts: opts,\n\t}\n\n\treturn &DateFinder{b, tableVersion}\n}\n\nfunc (b *DateFinder) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) (err error) {\n\tw := b.where(query)\n\n\tdateWhere := where.New()\n\tdateWhere.Andf(\n\t\t\"Date >='%s' AND Date <= '%s'\",\n\t\ttime.Unix(from, 0).Format(\"2006-01-02\"),\n\t\ttime.Unix(until, 0).Format(\"2006-01-02\"),\n\t)\n\n\tb.stats = append(b.stats, metrics.FinderStat{})\n\tstat := &b.stats[len(b.stats)-1]\n\n\tif b.tableVersion == 2 {\n\t\tb.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query(\n\t\t\tscope.WithTable(ctx, b.table),\n\t\t\tb.url,\n\t\t\t// TODO: consider consistent query generator\n\t\t\tfmt.Sprintf(`SELECT Path FROM %s PREWHERE (%s) WHERE %s GROUP BY Path FORMAT TabSeparatedRaw`, b.table, dateWhere, w),\n\t\t\tb.opts,\n\t\t\tnil,\n\t\t)\n\t} else {\n\t\tb.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query(\n\t\t\tscope.WithTable(ctx, b.table),\n\t\t\tb.url,\n\t\t\t// TODO: consider consistent query generator\n\t\t\tfmt.Sprintf(`SELECT DISTINCT Path FROM %s PREWHERE (%s) WHERE (%s) FORMAT TabSeparatedRaw`, b.table, dateWhere, w),\n\t\t\tb.opts,\n\t\t\tnil,\n\t\t)\n\t}\n\n\tstat.ReadBytes = int64(len(b.body))\n\tstat.Table = b.table\n\n\treturn\n}\n" }, { "path": "finder/date_reverse.go", "content": "package finder\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/helper/date\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/where\"\n)\n\ntype DateFinderV3 struct {\n\t*BaseFinder\n}\n\n// Same as v2, but reversed\nfunc NewDateFinderV3(url string, table string, opts clickhouse.Options) Finder {\n\tb := &BaseFinder{\n\t\turl: url,\n\t\ttable: table,\n\t\topts: opts,\n\t}\n\n\treturn &DateFinderV3{b}\n}\n\nfunc (f *DateFinderV3) whereFilter(query string, from int64, until int64) (*where.Where, *where.Where) {\n\tw := f.where(ReverseString(query))\n\n\tdateWhere := where.New()\n\tdateWhere.Andf(\n\t\t\"Date >='%s' AND Date <= '%s'\",\n\t\tdate.FromTimestampToDaysFormat(from),\n\t\tdate.UntilTimestampToDaysFormat(until),\n\t)\n\n\treturn w, dateWhere\n}\n\nfunc (f *DateFinderV3) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) (err error) {\n\tw, dateWhere := f.whereFilter(query, from, until)\n\n\tf.stats = append(f.stats, metrics.FinderStat{})\n\tstat := &f.stats[len(f.stats)-1]\n\n\tf.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query(\n\t\tscope.WithTable(ctx, f.table),\n\t\tf.url,\n\t\t// TODO: consider consistent query generator\n\t\tfmt.Sprintf(`SELECT Path FROM %s WHERE (%s) AND (%s) GROUP BY Path FORMAT TabSeparatedRaw`, f.table, dateWhere, w),\n\t\tf.opts,\n\t\tnil,\n\t)\n\tstat.Table = f.table\n\tstat.ReadBytes = int64(len(f.body))\n\n\treturn\n}\n\nfunc (f *DateFinderV3) List() [][]byte {\n\tlist := f.BaseFinder.List()\n\tfor i := 0; i < len(list); i++ {\n\t\tlist[i] = ReverseBytes(list[i])\n\t}\n\n\treturn list\n}\n\nfunc (f *DateFinderV3) Series() [][]byte {\n\tlist := f.BaseFinder.Series()\n\tfor i := 0; i < len(list); i++ {\n\t\tlist[i] = ReverseBytes(list[i])\n\t}\n\n\treturn list\n}\n" }, { "path": "finder/date_reverse_test.go", "content": "package finder\n\nimport (\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/helper/date\"\n)\n\nfunc TestDateFinderV3_whereFilter(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tquery string\n\t\tfrom int64\n\t\tuntil int64\n\t\twant string\n\t\twantDate string\n\t}{\n\t\t{\n\t\t\tname: \"midnight at utc (direct)\",\n\t\t\tquery: \"test.metric*\",\n\t\t\tfrom: 1668124800, // 2022-11-11 00:00:00 UTC\n\t\t\tuntil: 1668124810, // 2022-11-11 00:00:10 UTC\n\t\t\twant: \"(Level=2) AND (Path LIKE 'metric%' AND match(Path, '^metric([^.]*?)[.]test[.]?$'))\",\n\t\t\twantDate: \"Date >='\" + date.FromTimestampToDaysFormat(1668124800) + \"' AND Date <= '\" + date.UntilTimestampToDaysFormat(1668124810) + \"'\",\n\t\t},\n\t\t{\n\t\t\tname: \"midnight at utc (reverse)\",\n\t\t\tquery: \"*test.metric\",\n\t\t\tfrom: 1668124800, // 2022-11-11 00:00:00 UTC\n\t\t\tuntil: 1668124810, // 2022-11-11 00:00:10 UTC\n\t\t\twant: \"(Level=2) AND (Path LIKE 'metric.%' AND match(Path, '^metric[.]([^.]*?)test[.]?$'))\",\n\t\t\twantDate: \"Date >='\" + date.FromTimestampToDaysFormat(1668124800) + \"' AND Date <= '\" + date.UntilTimestampToDaysFormat(1668124810) + \"'\",\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name+\" \"+time.Unix(tt.from, 0).Format(time.RFC3339), func(t *testing.T) {\n\t\t\tf := NewDateFinderV3(\"http://localhost:8123/\", \"graphite_index\", clickhouse.Options{}).(*DateFinderV3)\n\n\t\t\tgot, gotDate := f.whereFilter(tt.query, tt.from, tt.until)\n\t\t\tif got.String() != tt.want {\n\t\t\t\tt.Errorf(\"DateFinderV3.whereFilter()[0] = %v, want %v\", got, tt.want)\n\t\t\t}\n\n\t\t\tif gotDate.String() != tt.wantDate {\n\t\t\t\tt.Errorf(\"DateFinderV3.whereFilter()[1] = %v, want %v\", gotDate, tt.wantDate)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "finder/finder.go", "content": "package finder\n\nimport (\n\t\"context\"\n\t\"strings\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n)\n\ntype Result interface {\n\tList() [][]byte\n\tSeries() [][]byte\n\tAbs([]byte) []byte\n\tBytes() ([]byte, error)\n\tStats() []metrics.FinderStat\n}\ntype Finder interface {\n\tResult\n\tExecute(ctx context.Context, config *config.Config, query string, from int64, until int64) error\n}\n\nfunc newPlainFinder(ctx context.Context, config *config.Config, query string, from int64, until int64, useCache bool) Finder {\n\topts := clickhouse.Options{\n\t\tTLSConfig: config.ClickHouse.TLSConfig,\n\t\tTimeout: config.ClickHouse.IndexTimeout,\n\t\tConnectTimeout: config.ClickHouse.ConnectTimeout,\n\t\tCheckRequestProgress: config.FeatureFlags.LogQueryProgress,\n\t\tProgressSendingInterval: config.ClickHouse.ProgressSendingInterval,\n\t}\n\n\tvar f Finder\n\n\tif config.ClickHouse.TaggedTable != \"\" && strings.HasPrefix(strings.TrimSpace(query), \"seriesByTag\") {\n\t\tf = NewTagged(\n\t\t\tconfig.ClickHouse.URL,\n\t\t\tconfig.ClickHouse.TaggedTable,\n\t\t\tconfig.ClickHouse.TagsCountTable,\n\t\t\tconfig.ClickHouse.TaggedUseDaily,\n\t\t\tconfig.FeatureFlags.UseCarbonBehavior,\n\t\t\tconfig.FeatureFlags.DontMatchMissingTags,\n\t\t\tfalse,\n\t\t\topts,\n\t\t\tconfig.ClickHouse.TaggedCosts,\n\t\t)\n\n\t\tif len(config.Common.Blacklist) > 0 {\n\t\t\tf = WrapBlacklist(f, config.Common.Blacklist)\n\t\t}\n\n\t\treturn f\n\t}\n\n\tif config.ClickHouse.IndexTable != \"\" {\n\t\tf = NewIndex(\n\t\t\tconfig.ClickHouse.URL,\n\t\t\tconfig.ClickHouse.IndexTable,\n\t\t\tconfig.ClickHouse.IndexUseDaily,\n\t\t\tconfig.ClickHouse.IndexReverse,\n\t\t\tconfig.ClickHouse.IndexReverses,\n\t\t\topts,\n\t\t\tuseCache,\n\t\t)\n\n\t\tif config.ClickHouse.TrySplitQuery {\n\t\t\tf = WrapSplitIndex(\n\t\t\t\tf,\n\t\t\t\tconfig.ClickHouse.WildcardMinDistance,\n\t\t\t\tconfig.ClickHouse.URL,\n\t\t\t\tconfig.ClickHouse.IndexTable,\n\t\t\t\tconfig.ClickHouse.IndexUseDaily,\n\t\t\t\tconfig.ClickHouse.IndexReverse,\n\t\t\t\tconfig.ClickHouse.IndexReverses,\n\t\t\t\topts,\n\t\t\t\tuseCache,\n\t\t\t)\n\t\t}\n\t} else {\n\t\tif from > 0 && until > 0 && config.ClickHouse.DateTreeTable != \"\" {\n\t\t\tf = NewDateFinder(config.ClickHouse.URL, config.ClickHouse.DateTreeTable, config.ClickHouse.DateTreeTableVersion, opts)\n\t\t} else {\n\t\t\tf = NewBase(config.ClickHouse.URL, config.ClickHouse.TreeTable, opts)\n\t\t}\n\n\t\tif config.ClickHouse.ReverseTreeTable != \"\" {\n\t\t\tf = WrapReverse(f, config.ClickHouse.URL, config.ClickHouse.ReverseTreeTable, opts)\n\t\t}\n\t}\n\n\tif config.ClickHouse.TagTable != \"\" {\n\t\tf = WrapTag(f, config.ClickHouse.URL, config.ClickHouse.TagTable, opts)\n\t}\n\n\tif config.ClickHouse.ExtraPrefix != \"\" {\n\t\tf = WrapPrefix(f, config.ClickHouse.ExtraPrefix)\n\t}\n\n\tif len(config.Common.Blacklist) > 0 {\n\t\tf = WrapBlacklist(f, config.Common.Blacklist)\n\t}\n\n\treturn f\n}\n\nfunc Find(config *config.Config, ctx context.Context, query string, from int64, until int64) (Result, error) {\n\tfnd := newPlainFinder(ctx, config, query, from, until, config.Common.FindCache != nil)\n\n\terr := fnd.Execute(ctx, config, query, from, until)\n\n\treturn fnd.(Result), err\n}\n\n// Leaf strips last dot and detect IsLeaf\nfunc Leaf(value []byte) ([]byte, bool) {\n\tif len(value) > 0 && value[len(value)-1] == '.' {\n\t\treturn value[:len(value)-1], false\n\t}\n\n\treturn value, true\n}\n\nfunc FindTagged(ctx context.Context, config *config.Config, terms []TaggedTerm, from int64, until int64) (Result, error) {\n\topts := clickhouse.Options{\n\t\tTimeout: config.ClickHouse.IndexTimeout,\n\t\tConnectTimeout: config.ClickHouse.ConnectTimeout,\n\t\tTLSConfig: config.ClickHouse.TLSConfig,\n\t\tCheckRequestProgress: config.FeatureFlags.LogQueryProgress,\n\t\tProgressSendingInterval: config.ClickHouse.ProgressSendingInterval,\n\t}\n\n\tuseCache := config.Common.FindCache != nil\n\n\tplain := makePlainFromTagged(terms)\n\tif plain != nil {\n\t\tplain.wrappedPlain = newPlainFinder(ctx, config, plain.Target(), from, until, useCache)\n\n\t\terr := plain.Execute(ctx, config, plain.Target(), from, until)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\treturn Result(plain), nil\n\t}\n\n\tfnd := NewTagged(\n\t\tconfig.ClickHouse.URL,\n\t\tconfig.ClickHouse.TaggedTable,\n\t\tconfig.ClickHouse.TagsCountTable,\n\t\tconfig.ClickHouse.TaggedUseDaily,\n\t\tconfig.FeatureFlags.UseCarbonBehavior,\n\t\tconfig.FeatureFlags.DontMatchMissingTags,\n\t\ttrue,\n\t\topts,\n\t\tconfig.ClickHouse.TaggedCosts,\n\t)\n\n\terr := fnd.ExecutePrepared(ctx, terms, from, until)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn Result(fnd), nil\n}\n" }, { "path": "finder/index.go", "content": "package finder\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"strings\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/helper/date\"\n\t\"github.com/lomik/graphite-clickhouse/helper/errs\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/where\"\n)\n\nconst ReverseLevelOffset = 10000\nconst TreeLevelOffset = 20000\nconst ReverseTreeLevelOffset = 30000\n\nconst DefaultTreeDate = \"1970-02-12\"\n\nconst (\n\tqueryAuto = config.IndexAuto\n\tqueryDirect = config.IndexDirect\n\tqueryReversed = config.IndexReversed\n)\n\ntype IndexFinder struct {\n\turl string // clickhouse dsn\n\ttable string // graphite_tree table\n\topts clickhouse.Options // timeout, connectTimeout\n\tdailyEnabled bool\n\tconfReverse uint8\n\tconfReverses config.IndexReverses\n\treverse uint8 // calculated in IndexFinder.useReverse only once\n\tbody []byte // clickhouse response body\n\trows [][]byte\n\tstats []metrics.FinderStat\n\tuseCache bool // rotate body if needed (for store in cache)\n\tuseDaily bool\n}\n\nfunc NewCachedIndex(body []byte) Finder {\n\tidx := &IndexFinder{\n\t\tbody: body,\n\t\treverse: queryDirect,\n\t}\n\tidx.bodySplit()\n\n\treturn idx\n}\n\nfunc NewIndex(url string, table string, dailyEnabled bool, reverse string, reverses config.IndexReverses, opts clickhouse.Options, useCache bool) Finder {\n\treturn &IndexFinder{\n\t\turl: url,\n\t\ttable: table,\n\t\topts: opts,\n\t\tdailyEnabled: dailyEnabled,\n\t\tconfReverse: config.IndexReverse[reverse],\n\t\tconfReverses: reverses,\n\t\tstats: make([]metrics.FinderStat, 0),\n\t\tuseCache: useCache,\n\t}\n}\n\nfunc (idx *IndexFinder) where(query string, levelOffset int) *where.Where {\n\tlevel := strings.Count(query, \".\") + 1\n\n\tw := where.New()\n\n\tw.And(where.Eq(\"Level\", level+levelOffset))\n\tw.And(where.TreeGlob(\"Path\", query))\n\n\treturn w\n}\n\nfunc (idx *IndexFinder) checkReverses(query string) uint8 {\n\tfor _, rule := range idx.confReverses {\n\t\tif len(rule.Prefix) > 0 && !strings.HasPrefix(query, rule.Prefix) {\n\t\t\tcontinue\n\t\t}\n\n\t\tif len(rule.Suffix) > 0 && !strings.HasSuffix(query, rule.Suffix) {\n\t\t\tcontinue\n\t\t}\n\n\t\tif rule.Regex != nil && rule.Regex.FindStringIndex(query) == nil {\n\t\t\tcontinue\n\t\t}\n\n\t\treturn config.IndexReverse[rule.Reverse]\n\t}\n\n\treturn idx.confReverse\n}\n\nfunc (idx *IndexFinder) useReverse(query string) bool {\n\tif idx.reverse == queryDirect {\n\t\treturn false\n\t} else if idx.reverse == queryReversed {\n\t\treturn true\n\t}\n\n\tif idx.reverse = idx.checkReverses(query); idx.reverse != queryAuto {\n\t\treturn idx.useReverse(query)\n\t}\n\n\tw := where.IndexWildcard(query)\n\tif w == -1 {\n\t\tidx.reverse = queryDirect\n\t\treturn idx.useReverse(query)\n\t}\n\n\tfirstWildcardNode := strings.Count(query[:w], \".\")\n\n\tw = where.IndexLastWildcard(query)\n\tlastWildcardNode := strings.Count(query[w:], \".\")\n\n\tif firstWildcardNode < lastWildcardNode {\n\t\tidx.reverse = queryReversed\n\t\treturn idx.useReverse(query)\n\t}\n\n\tidx.reverse = queryDirect\n\n\treturn idx.useReverse(query)\n}\n\nfunc useDaily(dailyEnabled bool, from, until int64) bool {\n\treturn dailyEnabled && from > 0 && until > 0\n}\n\nfunc calculateIndexLevelOffset(useDaily, reverse bool) int {\n\tvar levelOffset int\n\n\tif useDaily {\n\t\tif reverse {\n\t\t\tlevelOffset = ReverseLevelOffset\n\t\t}\n\t} else if reverse {\n\t\tlevelOffset = ReverseTreeLevelOffset\n\t} else {\n\t\tlevelOffset = TreeLevelOffset\n\t}\n\n\treturn levelOffset\n}\n\nfunc addDatesToWhere(w *where.Where, useDaily bool, from, until int64) {\n\tif useDaily {\n\t\tw.Andf(\n\t\t\t\"Date >='%s' AND Date <= '%s'\",\n\t\t\tdate.FromTimestampToDaysFormat(from),\n\t\t\tdate.UntilTimestampToDaysFormat(until),\n\t\t)\n\t} else {\n\t\tw.And(where.Eq(\"Date\", DefaultTreeDate))\n\t}\n}\n\nfunc (idx *IndexFinder) whereFilter(query string, from int64, until int64) *where.Where {\n\treverse := idx.useReverse(query)\n\tif reverse {\n\t\tquery = ReverseString(query)\n\t}\n\n\tidx.useDaily = useDaily(idx.dailyEnabled, from, until)\n\n\tlevelOffset := calculateIndexLevelOffset(idx.useDaily, reverse)\n\n\tw := idx.where(query, levelOffset)\n\taddDatesToWhere(w, idx.useDaily, from, until)\n\n\treturn w\n}\n\nfunc validatePlainQuery(query string, wildcardMinDistance int) error {\n\tif where.HasUnmatchedBrackets(query) {\n\t\treturn errs.NewErrorWithCode(\"query has unmatched brackets\", http.StatusBadRequest)\n\t}\n\n\tvar maxDist = where.MaxWildcardDistance(query)\n\n\t// If the amount of nodes in a plain query is equal to 1,\n\t// then make an exception\n\t// This allows to check which root nodes exist\n\tmoreThanOneNode := strings.Count(query, \".\") >= 1\n\n\tif maxDist != -1 && maxDist < wildcardMinDistance && moreThanOneNode {\n\t\treturn errs.NewErrorWithCode(\"query has wildcards way too early at the start and at the end of it\", http.StatusBadRequest)\n\t}\n\n\treturn nil\n}\n\nfunc (idx *IndexFinder) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) (err error) {\n\terr = validatePlainQuery(query, config.ClickHouse.WildcardMinDistance)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tw := idx.whereFilter(query, from, until)\n\n\tidx.stats = append(idx.stats, metrics.FinderStat{})\n\tstat := &idx.stats[len(idx.stats)-1]\n\n\tidx.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query(\n\t\tscope.WithTable(ctx, idx.table),\n\t\tidx.url,\n\t\t// TODO: consider consistent query generator\n\t\tfmt.Sprintf(\"SELECT Path FROM %s WHERE %s GROUP BY Path FORMAT TabSeparatedRaw\", idx.table, w),\n\t\tidx.opts,\n\t\tnil,\n\t)\n\tstat.Table = idx.table\n\n\tif err == nil {\n\t\tstat.ReadBytes = int64(len(idx.body))\n\t\tidx.bodySplit()\n\t}\n\n\treturn\n}\n\nfunc (idx *IndexFinder) Abs(v []byte) []byte {\n\treturn v\n}\n\nfunc splitIndexBody(body []byte, useReverse, useCache bool) ([]byte, [][]byte, bool) {\n\tif len(body) == 0 {\n\t\treturn body, [][]byte{}, false\n\t}\n\n\trows := bytes.Split(bytes.TrimSuffix(body, []byte{'\\n'}), []byte{'\\n'})\n\tsetDirect := false\n\n\tif useReverse {\n\t\tvar buf bytes.Buffer\n\t\tif useCache {\n\t\t\tbuf.Grow(len(body))\n\t\t}\n\n\t\tfor i := range rows {\n\t\t\trows[i] = ReverseBytes(rows[i])\n\t\t\tif useCache {\n\t\t\t\tbuf.Write(rows[i])\n\t\t\t\tbuf.WriteByte('\\n')\n\t\t\t}\n\t\t}\n\n\t\tif useCache {\n\t\t\tbody = buf.Bytes()\n\t\t\tsetDirect = true\n\t\t}\n\t}\n\n\treturn body, rows, setDirect\n}\n\nfunc (idx *IndexFinder) bodySplit() {\n\tsetDirect := false\n\tidx.body, idx.rows, setDirect = splitIndexBody(idx.body, idx.useReverse(\"\"), idx.useCache)\n\n\tif setDirect {\n\t\tidx.reverse = queryDirect\n\t}\n}\n\nfunc makeList(rows [][]byte, onlySeries bool) [][]byte {\n\tif len(rows) == 0 {\n\t\treturn [][]byte{}\n\t}\n\n\tresRows := make([][]byte, len(rows))\n\n\tfor i := 0; i < len(rows); i++ {\n\t\tresRows[i] = rows[i]\n\t}\n\n\treturn resRows\n}\n\nfunc (idx *IndexFinder) makeList(onlySeries bool) [][]byte {\n\treturn makeList(idx.rows, onlySeries)\n}\n\nfunc (idx *IndexFinder) List() [][]byte {\n\treturn idx.makeList(false)\n}\n\nfunc (idx *IndexFinder) Series() [][]byte {\n\treturn idx.makeList(true)\n}\n\nfunc (idx *IndexFinder) Bytes() ([]byte, error) {\n\treturn idx.body, nil\n}\n\nfunc (idx *IndexFinder) Stats() []metrics.FinderStat {\n\treturn idx.stats\n}\n" }, { "path": "finder/index_test.go", "content": "package finder\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/helper/date\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc Test_useReverse(t *testing.T) {\n\tassert := assert.New(t)\n\n\ttable := []struct {\n\t\tquery string\n\t\tresult bool\n\t}{\n\t\t{\"a.b.c.d.e\", false},\n\t\t{\"a.b*\", false},\n\t\t{\"a.b.c.d.e*\", false},\n\t\t{\"a.b.c.d*.e\", false},\n\t\t{\"a.b*.c*.d.e\", true},\n\t\t{\"a.b*.c.d.e\", true},\n\t}\n\n\tfor _, tt := range table {\n\t\tidx := IndexFinder{confReverse: queryAuto}\n\t\tassert.Equal(tt.result, idx.useReverse(tt.query), tt.query)\n\t}\n}\n\nfunc Test_useReverseWithSetConfig(t *testing.T) {\n\tassert := assert.New(t)\n\n\ttable := []struct {\n\t\tquery string\n\t\treverse uint8\n\t\tresult bool\n\t}{\n\t\t{\"a.b.c.d.e\", queryReversed, true},\n\t\t{\"a.b.c.d.e\", queryAuto, false},\n\t\t{\"a.b.c.d.e\", queryDirect, false},\n\t\t{\"a.b.c.d.e\", queryDirect, false},\n\t\t{\"a.b.c.d.e*\", queryDirect, false},\n\t\t{\"a.b.c.d*.e\", queryDirect, false},\n\t\t{\"a.b.c.d*.e\", queryReversed, true},\n\t\t{\"a*.b.c.d*.e\", queryReversed, true}, // Wildcard at first level, use reverse if possible\n\t\t{\"a.b*.c.d*.e\", queryReversed, true},\n\t\t{\"a.*.c.*.e.*.j\", queryReversed, true},\n\t\t{\"a.*.c.*.e.*.j\", queryDirect, false},\n\t\t{\"a.b*.c.*d.e\", queryReversed, true},\n\t}\n\n\tfor _, tt := range table {\n\t\tidx := IndexFinder{confReverse: tt.reverse}\n\t\tassert.Equal(tt.result, idx.useReverse(tt.query), fmt.Sprintf(\"%s with iota %d\", tt.query, tt.reverse))\n\t}\n}\n\nfunc Test_checkReverses(t *testing.T) {\n\tassert := assert.New(t)\n\n\treverses := config.IndexReverses{\n\t\t{Suffix: \".sum\", Reverse: \"direct\"},\n\t\t{Prefix: \"test.\", Suffix: \".alloc\", Reverse: \"direct\"},\n\t\t{Prefix: \"test2.\", Reverse: \"reversed\"},\n\t\t{RegexStr: `^a\\..*\\.max$`, Reverse: \"reversed\"},\n\t}\n\n\ttable := []struct {\n\t\tquery string\n\t\treverse uint8\n\t\tresult uint8\n\t}{\n\t\t{\"a.b.c.d*.sum\", queryAuto, queryDirect},\n\t\t{\"a*.b.c.d.sum\", queryAuto, queryDirect},\n\t\t{\"test.b.c*.d*.alloc\", queryAuto, queryDirect},\n\t\t{\"test.b.c*.d.alloc\", queryAuto, queryDirect},\n\t\t{\"test2.b.c*.d*.e\", queryAuto, queryReversed},\n\t\t{\"test2.b.c*.d.e\", queryAuto, queryReversed},\n\t\t{\"a.b.c.d*.max\", queryAuto, queryReversed}, // regex test\n\t\t{\"a.b.c*.d.max\", queryAuto, queryReversed}, // regex test\n\t}\n\n\tassert.NoError(reverses.Compile())\n\n\tfor _, tt := range table {\n\t\tidx := IndexFinder{confReverse: tt.reverse, confReverses: reverses}\n\t\tassert.Equal(tt.result, idx.checkReverses(tt.query), fmt.Sprintf(\"%s with iota %d\", tt.query, tt.reverse))\n\t}\n}\n\nfunc Benchmark_useReverseDepth(b *testing.B) {\n\treverses := config.IndexReverses{\n\t\t{Prefix: \"test2.\", Reverse: \"reversed\"},\n\t}\n\tif err := reverses.Compile(); err != nil {\n\t\tb.Fatal(\"failed to compile reverses\")\n\t}\n\n\tidx := IndexFinder{confReverses: reverses}\n\n\tfor i := 0; i < b.N; i++ {\n\t\t_ = idx.checkReverses(\"test2.b.c*.d.e\")\n\t}\n}\n\nfunc Benchmark_useReverseDepthPrefixSuffix(b *testing.B) {\n\treverses := config.IndexReverses{\n\t\t{Prefix: \"test2.\", Suffix: \".e\", Reverse: \"direct\"},\n\t}\n\tif err := reverses.Compile(); err != nil {\n\t\tb.Fatal(\"failed to compile reverses\")\n\t}\n\n\tidx := IndexFinder{confReverses: reverses}\n\n\tfor i := 0; i < b.N; i++ {\n\t\t_ = idx.checkReverses(\"test2.b.c*.d.e\")\n\t}\n}\n\nfunc Benchmark_useReverseDepthRegex(b *testing.B) {\n\treverses := config.IndexReverses{\n\t\t{RegexStr: `^a\\..*\\.max$`, Reverse: \"auto\"},\n\t}\n\tif err := reverses.Compile(); err != nil {\n\t\tb.Fatal(\"failed to compile reverses\")\n\t}\n\n\tidx := IndexFinder{confReverses: reverses}\n\n\tfor i := 0; i < b.N; i++ {\n\t\t_ = idx.checkReverses(\"a.b.c*.d.max\")\n\t}\n}\n\nfunc TestIndexFinder_whereFilter(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tquery string\n\t\tfrom int64\n\t\tuntil int64\n\t\tdailyEnabled bool\n\t\tindexReverse string\n\t\tindexReverses config.IndexReverses\n\t\twant string\n\t}{\n\t\t{\n\t\t\tname: \"nodaily (direct)\",\n\t\t\tquery: \"test.metric*\",\n\t\t\tfrom: 1668106860,\n\t\t\tuntil: 1668106870,\n\t\t\tdailyEnabled: false,\n\t\t\twant: \"((Level=20002) AND (Path LIKE 'test.metric%')) AND (Date='1970-02-12')\",\n\t\t},\n\t\t{\n\t\t\tname: \"nodaily (reverse)\",\n\t\t\tquery: \"*test.metric\",\n\t\t\tfrom: 1668106860,\n\t\t\tuntil: 1668106870,\n\t\t\tdailyEnabled: false,\n\t\t\twant: \"((Level=30002) AND (Path LIKE 'metric.%' AND match(Path, '^metric[.]([^.]*?)test[.]?$'))) AND (Date='1970-02-12')\",\n\t\t},\n\t\t{\n\t\t\tname: \"midnight at utc (direct)\",\n\t\t\tquery: \"test.metric*\",\n\t\t\tfrom: 1668124800, // 2022-11-11 00:00:00 UTC\n\t\t\tuntil: 1668124810, // 2022-11-11 00:00:10 UTC\n\t\t\tdailyEnabled: true,\n\t\t\twant: \"((Level=2) AND (Path LIKE 'test.metric%')) AND (Date >='\" +\n\t\t\t\tdate.FromTimestampToDaysFormat(1668124800) + \"' AND Date <= '\" + date.UntilTimestampToDaysFormat(1668124810) + \"')\",\n\t\t},\n\t\t{\n\t\t\tname: \"midnight at utc (reverse)\",\n\t\t\tquery: \"*test.metric\",\n\t\t\tfrom: 1668124800, // 2022-11-11 00:00:00 UTC\n\t\t\tuntil: 1668124810, // 2022-11-11 00:00:10 UTC\n\t\t\tdailyEnabled: true,\n\t\t\twant: \"((Level=10002) AND (Path LIKE 'metric.%' AND match(Path, '^metric[.]([^.]*?)test[.]?$'))) AND (Date >='\" +\n\t\t\t\tdate.FromTimestampToDaysFormat(1668124800) + \"' AND Date <= '\" + date.UntilTimestampToDaysFormat(1668124810) + \"')\",\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name+\" \"+time.Unix(tt.from, 0).Format(time.RFC3339), func(t *testing.T) {\n\t\t\tif tt.indexReverse == \"\" {\n\t\t\t\ttt.indexReverse = \"auto\"\n\t\t\t}\n\n\t\t\tidx := NewIndex(\"http://localhost:8123/\", \"graphite_index\", tt.dailyEnabled, tt.indexReverse, tt.indexReverses, clickhouse.Options{}, false).(*IndexFinder)\n\t\t\tif got := idx.whereFilter(tt.query, tt.from, tt.until); got.String() != tt.want {\n\t\t\t\tt.Errorf(\"IndexFinder.whereFilter() = %v, want %v\", got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "finder/mock.go", "content": "package finder\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"strings\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n)\n\n// MockFinder is used for testing purposes\ntype MockFinder struct {\n\tfnd Finder\n\tquery string // logged from execute\n}\n\n// NewMockFinder returns new MockFinder object with given result\nfunc NewMockFinder(result [][]byte) *MockFinder {\n\treturn &MockFinder{\n\t\tfnd: NewCachedIndex(bytes.Join(result, []byte{'\\n'})),\n\t}\n}\n\n// NewMockTagged returns new MockFinder object with given result\nfunc NewMockTagged(result [][]byte) *MockFinder {\n\treturn &MockFinder{\n\t\tfnd: NewCachedTags(bytes.Join(result, []byte{'\\n'})),\n\t}\n}\n\n// Execute assigns given query to the query field\nfunc (m *MockFinder) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) (err error) {\n\tm.query = query\n\treturn\n}\n\n// List returns the result\nfunc (m *MockFinder) List() [][]byte {\n\treturn m.fnd.List()\n}\n\n// Series returns the result\nfunc (m *MockFinder) Series() [][]byte {\n\treturn m.fnd.Series()\n}\n\n// Abs returns the same given v\nfunc (m *MockFinder) Abs(v []byte) []byte {\n\treturn m.fnd.Abs(v)\n}\n\nfunc (m *MockFinder) Bytes() ([]byte, error) {\n\treturn m.fnd.Bytes()\n}\n\n// Strings returns the result converted to []string\nfunc (m *MockFinder) Strings() []string {\n\tbody, _ := m.fnd.Bytes()\n\treturn strings.Split(string(body), \"\\n\")\n}\n\nfunc (m *MockFinder) Stats() []metrics.FinderStat {\n\treturn m.fnd.Stats()\n}\n" }, { "path": "finder/plain_from_tagged.go", "content": "package finder\n\nimport (\n\t\"context\"\n\t\"net/url\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n)\n\n// Special finder for query plain graphite from prometheus\n// graphite{target=\"telegraf.*.cpu.avg\"}\ntype plainFromTaggedFinder struct {\n\twrappedPlain Finder\n\ttarget string\n\tnodeLabel map[int]string\n\tmetricName string\n}\n\nfunc makePlainFromTagged(matchers []TaggedTerm) *plainFromTaggedFinder {\n\tvar isMetricNameFound bool\n\n\tvar target string\n\n\tfor _, m := range matchers {\n\t\tif m.Key == \"__name__\" && m.Value == \"graphite\" && m.Op == TaggedTermEq {\n\t\t\tisMetricNameFound = true\n\t\t}\n\n\t\tif m.Key == \"target\" && m.Op == TaggedTermEq && m.Value != \"\" {\n\t\t\ttarget = m.Value\n\t\t}\n\t}\n\n\t// not plain graphite query\n\tif !isMetricNameFound || target == \"\" {\n\t\treturn nil\n\t}\n\n\tq := &plainFromTaggedFinder{target: target}\n\t// fill additional params\n\tfor _, m := range matchers {\n\t\tif m.Key == \"rename\" && m.Op == TaggedTermEq && m.Value != \"\" {\n\t\t\tq.metricName = m.Value\n\t\t}\n\n\t\tif strings.HasPrefix(m.Key, \"node\") && m.Op == TaggedTermEq && m.Value != \"\" {\n\t\t\tv, err := strconv.Atoi(m.Key[4:])\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif q.nodeLabel == nil {\n\t\t\t\tq.nodeLabel = make(map[int]string)\n\t\t\t}\n\n\t\t\tq.nodeLabel[v] = m.Value\n\t\t}\n\t}\n\n\treturn q\n}\n\nfunc (f *plainFromTaggedFinder) Target() string {\n\treturn f.target\n}\n\nfunc (f *plainFromTaggedFinder) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) error {\n\treturn f.wrappedPlain.Execute(ctx, config, query, from, until)\n}\n\n// For Render\nfunc (f *plainFromTaggedFinder) Series() [][]byte {\n\treturn f.wrappedPlain.Series()\n}\n\ntype taggedLabel struct {\n\tname string\n\tvalue string\n}\n\nfunc (f *plainFromTaggedFinder) Abs(value []byte) []byte {\n\tname := \"graphite\"\n\tpath := string(value)\n\tlb := []taggedLabel{\n\t\t{\"metric\", path},\n\t}\n\n\tif f.metricName != \"\" {\n\t\tname = f.metricName\n\t}\n\n\tif len(f.nodeLabel) > 0 {\n\t\ta := strings.Split(path, \".\")\n\t\tfor n, v := range a {\n\t\t\tl := f.nodeLabel[n]\n\t\t\tif l != \"\" {\n\t\t\t\tlb = append(lb, taggedLabel{l, v})\n\t\t\t}\n\t\t}\n\t}\n\n\tsort.Slice(lb, func(i, j int) bool { return lb[i].name < lb[j].name })\n\n\tvar buf strings.Builder\n\n\tbuf.WriteString(name)\n\tbuf.WriteByte('?')\n\n\tfor i, l := range lb {\n\t\tif i > 0 {\n\t\t\tbuf.WriteByte('&')\n\t\t}\n\n\t\tbuf.WriteString(url.QueryEscape(l.name))\n\t\tbuf.WriteByte('=')\n\t\tbuf.WriteString(url.QueryEscape(l.value))\n\t}\n\n\treturn []byte(buf.String())\n}\n\nfunc (f *plainFromTaggedFinder) List() [][]byte {\n\treturn f.wrappedPlain.List()\n}\n\nfunc (f *plainFromTaggedFinder) Bytes() ([]byte, error) {\n\treturn nil, ErrNotImplemented\n}\n\nfunc (f *plainFromTaggedFinder) Stats() []metrics.FinderStat {\n\treturn f.wrappedPlain.Stats()\n}\n" }, { "path": "finder/plain_from_tagged_test.go", "content": "package finder\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestPlainFromTaggedFinderAbs(t *testing.T) {\n\tassert := assert.New(t)\n\n\teq := func(name, value string) TaggedTerm {\n\t\treturn TaggedTerm{Op: TaggedTermEq, Key: name, Value: value}\n\t}\n\n\tjoin := func(terms ...TaggedTerm) []TaggedTerm {\n\t\treturn terms\n\t}\n\n\tf := makePlainFromTagged(join(\n\t\teq(\"__name__\", \"graphite\"),\n\t\teq(\"rename\", \"cpu_usage\"),\n\t\teq(\"target\", \"telegraf.*.cpu.usage\"),\n\t\teq(\"node1\", \"host\"),\n\t))\n\n\tassert.NotNil(f)\n\n\ttable := [][2]string{\n\t\t{\n\t\t\t\"telegraf.localhost.cpu.usage\",\n\t\t\t`cpu_usage?host=localhost&metric=telegraf.localhost.cpu.usage`,\n\t\t},\n\t}\n\n\tfor _, c := range table {\n\t\tassert.Equal(c[1], string(f.Abs([]byte(c[0]))))\n\t}\n}\n" }, { "path": "finder/prefix.go", "content": "package finder\n\nimport (\n\t\"context\"\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/where\"\n)\n\ntype PrefixMatchResult int\n\nconst (\n\tPrefixNotMatched PrefixMatchResult = iota\n\tPrefixMatched\n\tPrefixPartialMathed\n)\n\ntype PrefixFinder struct {\n\twrapped Finder\n\tprefix string // config\n\tprefixBytes []byte // same prefix with []bytes type\n\tmatched PrefixMatchResult // request\n\tpart string // request. partially matched part\n}\n\nfunc bytesConcat(s1 []byte, s2 []byte) []byte {\n\tret := make([]byte, len(s1)+len(s2))\n\tcopy(ret, s1)\n\tcopy(ret[len(s1):], s2)\n\n\treturn ret\n}\n\nfunc WrapPrefix(f Finder, prefix string) *PrefixFinder {\n\treturn &PrefixFinder{\n\t\twrapped: f,\n\t\tprefix: prefix,\n\t\tprefixBytes: bytesConcat([]byte(prefix), []byte{'.'}),\n\t\tmatched: PrefixNotMatched,\n\t}\n}\n\nfunc (p *PrefixFinder) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) error {\n\tqs := strings.Split(query, \".\")\n\n\t// check regexp\n\tfor _, queryPart := range qs {\n\t\tif _, err := regexp.Compile(where.GlobToRegexp(queryPart)); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tps := strings.Split(p.prefix, \".\")\n\n\tvar i int\n\tfor i = 0; i < len(qs) && i < len(ps); i++ {\n\t\tm, err := regexp.MatchString(\"^\"+where.GlobToRegexp(qs[i])+\"$\", ps[i])\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif !m { // not matched\n\t\t\treturn nil\n\t\t}\n\t}\n\n\tif len(qs) <= len(ps) {\n\t\t// prefix matched, but not finished\n\t\tp.part = strings.Join(ps[:len(qs)], \".\") + \".\"\n\t\tp.matched = PrefixPartialMathed\n\n\t\treturn nil\n\t}\n\n\tp.matched = PrefixMatched\n\n\treturn p.wrapped.Execute(ctx, config, strings.Join(qs[len(ps):], \".\"), from, until)\n}\n\nfunc (p *PrefixFinder) List() [][]byte {\n\tif p.matched == PrefixNotMatched {\n\t\treturn [][]byte{}\n\t}\n\n\tif p.matched == PrefixPartialMathed {\n\t\treturn [][]byte{[]byte(p.part)}\n\t}\n\n\tlist := p.wrapped.List()\n\tresult := make([][]byte, len(list))\n\n\tfor i := 0; i < len(list); i++ {\n\t\tresult[i] = bytesConcat(p.prefixBytes, list[i])\n\t}\n\n\treturn result\n}\n\n// For Render\nfunc (p *PrefixFinder) Series() [][]byte {\n\tif p.matched == PrefixNotMatched {\n\t\treturn [][]byte{}\n\t}\n\n\tif p.matched != PrefixMatched {\n\t\treturn [][]byte{}\n\t}\n\n\treturn p.wrapped.Series()\n}\n\nfunc (p *PrefixFinder) Abs(value []byte) []byte {\n\treturn bytesConcat(p.prefixBytes, p.wrapped.Abs(value))\n}\n\nfunc (p *PrefixFinder) Bytes() ([]byte, error) {\n\treturn nil, ErrNotImplemented\n}\n\nfunc (p *PrefixFinder) Stats() []metrics.FinderStat {\n\treturn p.wrapped.Stats()\n}\n" }, { "path": "finder/prefix_test.go", "content": "package finder\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestPrefixFinderExecute(t *testing.T) {\n\tassert := assert.New(t)\n\n\ttable := []struct {\n\t\tprefix string\n\t\tquery string\n\t\texpectedMatched PrefixMatchResult\n\t\texpectedQ string\n\t\texpectedPart string\n\t\texpectedError bool\n\t}{\n\t\t{\"ch\", \"*\", PrefixPartialMathed, \"\", \"ch.\", false},\n\t\t{\"ch.data\", \"*\", PrefixPartialMathed, \"\", \"ch.\", false},\n\t\t{\"ch.data\", \"ch.*\", PrefixPartialMathed, \"\", \"ch.data.\", false},\n\t\t{\"ch.data\", \"ch.data.*\", PrefixMatched, \"*\", \"\", false},\n\t\t{\"ch.data\", \"epta.*\", PrefixNotMatched, \"\", \"\", false},\n\t\t{\"ch.data\", \"ch.data._tag.daemon.h.hostname.top.cpu_avg\", PrefixMatched, \"_tag.daemon.h.hostname.top.cpu_avg\", \"\", false},\n\t\t{\"ch.data\", \"ch.d[a]\", PrefixNotMatched, \"\", \"\", false},\n\t}\n\n\tfor _, test := range table {\n\t\ttestName := fmt.Sprintf(\"prefix: %#v, query: %#v\", test.prefix, test.query)\n\n\t\tm := NewMockFinder([][]byte{})\n\n\t\tf := WrapPrefix(m, test.prefix)\n\n\t\tconfig := config.New()\n\t\terr := f.Execute(context.Background(), config, test.query, 0, 0)\n\n\t\tif test.expectedError {\n\t\t\tassert.Error(err, testName)\n\t\t} else {\n\t\t\tassert.NoError(err, testName)\n\t\t}\n\n\t\tassert.Equal(test.expectedQ, m.query, testName)\n\t\tassert.Equal(test.expectedMatched, f.matched, testName)\n\t\tassert.Equal(test.expectedPart, f.part, testName)\n\t}\n}\n\nfunc TestPrefixFinderAbs(t *testing.T) {\n\tassert := assert.New(t)\n\n\tm := NewMockFinder([][]byte{})\n\tf := WrapPrefix(m, \"hello\")\n\n\tassert.Equal(\"hello.world\", string(f.Abs([]byte(\"world\"))))\n}\n\nfunc TestPrefixFinderList(t *testing.T) {\n\tassert := assert.New(t)\n\n\tmockData := [][]byte{[]byte(\"world\")}\n\tprefix := \"hello\"\n\n\ttable := []struct {\n\t\tquery string\n\t\texpectedList []string\n\t\texpectedSeries []string\n\t}{\n\t\t{\"*\", []string{\"hello.\"}, []string{}},\n\t\t{\"hello\", []string{\"hello.\"}, []string{}},\n\t\t{\"hello.*\", []string{\"hello.world\"}, []string{\"world\"}},\n\t\t{\"*.*\", []string{\"hello.world\"}, []string{\"world\"}},\n\t\t{\"*404*\", []string{}, []string{}},\n\t\t{\"*404*.*\", []string{}, []string{}},\n\t\t{\"hello.[bad regexp\", []string{}, []string{}},\n\t}\n\n\tfor _, test := range table {\n\t\ttestName := fmt.Sprintf(\"query: %#v\", test.query)\n\n\t\tm := NewMockFinder(mockData)\n\t\tf := WrapPrefix(m, prefix)\n\n\t\tconfig := config.New()\n\t\tf.Execute(context.Background(), config, test.query, 0, 0)\n\n\t\tlist := make([]string, 0)\n\t\tfor _, r := range f.List() {\n\t\t\tlist = append(list, string(r))\n\t\t}\n\n\t\tseries := make([]string, 0)\n\t\tfor _, r := range f.Series() {\n\t\t\tseries = append(series, string(r))\n\t\t}\n\n\t\tassert.Equal(test.expectedList, list, testName+\", list\")\n\t\tassert.Equal(test.expectedSeries, series, testName+\", series\")\n\t}\n}\n" }, { "path": "finder/reverse.go", "content": "package finder\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"strings\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/where\"\n)\n\ntype ReverseFinder struct {\n\twrapped Finder\n\tbaseFinder Finder\n\turl string // clickhouse dsn\n\ttable string // graphite_reverse_tree table\n\tisUsed bool // use reverse table\n}\n\nfunc ReverseString(target string) string {\n\ta := strings.Split(target, \".\")\n\tl := len(a)\n\n\tfor i := 0; i < l/2; i++ {\n\t\ta[i], a[l-i-1] = a[l-i-1], a[i]\n\t}\n\n\treturn strings.Join(a, \".\")\n}\n\nfunc ReverseBytes(target []byte) []byte {\n\t// @TODO: check performance\n\ta := bytes.Split(target, []byte{'.'})\n\n\tl := len(a)\n\tfor i := 0; i < l/2; i++ {\n\t\ta[i], a[l-i-1] = a[l-i-1], a[i]\n\t}\n\n\treturn bytes.Join(a, []byte{'.'})\n}\n\nfunc WrapReverse(f Finder, url string, table string, opts clickhouse.Options) *ReverseFinder {\n\treturn &ReverseFinder{\n\t\twrapped: f,\n\t\tbaseFinder: NewBase(url, table, opts),\n\t\turl: url,\n\t\ttable: table,\n\t}\n}\n\nfunc (r *ReverseFinder) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) (err error) {\n\tp := strings.LastIndexByte(query, '.')\n\tif p < 0 || p >= len(query)-1 {\n\t\treturn r.wrapped.Execute(ctx, config, query, from, until)\n\t}\n\n\tif where.HasWildcard(query[p+1:]) {\n\t\treturn r.wrapped.Execute(ctx, config, query, from, until)\n\t}\n\n\tr.isUsed = true\n\n\treturn r.baseFinder.Execute(ctx, config, ReverseString(query), from, until)\n}\n\nfunc (r *ReverseFinder) List() [][]byte {\n\tif !r.isUsed {\n\t\treturn r.wrapped.List()\n\t}\n\n\tlist := r.baseFinder.List()\n\tfor i := 0; i < len(list); i++ {\n\t\tlist[i] = ReverseBytes(list[i])\n\t}\n\n\treturn list\n}\n\nfunc (r *ReverseFinder) Series() [][]byte {\n\tif !r.isUsed {\n\t\treturn r.wrapped.Series()\n\t}\n\n\tlist := r.baseFinder.Series()\n\tfor i := 0; i < len(list); i++ {\n\t\tlist[i] = ReverseBytes(list[i])\n\t}\n\n\treturn list\n}\n\nfunc (r *ReverseFinder) Abs(v []byte) []byte {\n\treturn v\n}\n\nfunc (f *ReverseFinder) Bytes() ([]byte, error) {\n\treturn f.wrapped.Bytes()\n}\n\nfunc (f *ReverseFinder) Stats() []metrics.FinderStat {\n\treturn f.wrapped.Stats()\n}\n" }, { "path": "finder/reverse_test.go", "content": "package finder\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestReverse(t *testing.T) {\n\tassert := assert.New(t)\n\n\ttable := []string{\n\t\t\"hello.world\", \"world.hello\",\n\t\t\"hello.\", \".hello\",\n\t\t\"hello\", \"hello\",\n\t\t\".\", \".\",\n\t\t\"a1.b2.c3\", \"c3.b2.a1\",\n\t}\n\n\tfor i := 0; i < len(table); i += 2 {\n\t\tassert.Equal(table[i+1], ReverseString(table[i]))\n\n\t\tassert.Equal([]byte(table[i+1]), ReverseBytes([]byte(table[i])))\n\t}\n}\n" }, { "path": "finder/split.go", "content": "package finder\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"strings\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/helper/errs\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/where\"\n)\n\ntype indexFinderParams struct {\n\turl string\n\ttable string\n\topts clickhouse.Options\n\tdailyEnabled bool\n\tuseCache bool\n\treverse string\n\tconfReverses config.IndexReverses\n}\n\n// SplitIndexFinder will try to split queries like {first,second}.some.metric into n queries (n - number of cases inside {}).\n// No matter if '{}' in first node or not. Only one {} will be split.\ntype SplitIndexFinder struct {\n\tindexFinderParams\n\t// wrapped finder will be called if we can't split query.\n\twrapped Finder\n\tbody []byte\n\trows [][]byte\n\tstats []metrics.FinderStat\n\t// useWrapped indicated if we should use wrapped Finder.\n\tuseWrapped bool\n\tuseReverse bool\n\twildcardMinDistance int\n}\n\n// WrapSplitIndex wraps given finder with SplitIndexFinder logic.\nfunc WrapSplitIndex(\n\tf Finder,\n\twildcardMinDistance int,\n\turl string,\n\ttable string,\n\tdailyEnabled bool,\n\treverse string,\n\treverses config.IndexReverses,\n\topts clickhouse.Options,\n\tuseCache bool,\n) *SplitIndexFinder {\n\treturn &SplitIndexFinder{\n\t\twrapped: f,\n\t\tuseWrapped: false,\n\t\tuseReverse: false,\n\t\twildcardMinDistance: wildcardMinDistance,\n\t\tindexFinderParams: indexFinderParams{\n\t\t\turl: url,\n\t\t\ttable: table,\n\t\t\tdailyEnabled: dailyEnabled,\n\t\t\treverse: reverse,\n\t\t\tconfReverses: reverses,\n\t\t\topts: opts,\n\t\t\tuseCache: useCache,\n\t\t},\n\t}\n}\n\n// Execute will try to split query if it contains list in it. If query can't be split wrapped Finder will be used.\n// Use List, Series or Bytes after calling Execute to get data.\nfunc (splitFinder *SplitIndexFinder) Execute(\n\tctx context.Context,\n\tconfig *config.Config,\n\tquery string,\n\tfrom int64,\n\tuntil int64,\n) error {\n\tif where.HasUnmatchedBrackets(query) {\n\t\treturn errs.NewErrorWithCode(\"query has unmatched brackets\", http.StatusBadRequest)\n\t}\n\n\tquery = where.ClearGlob(query)\n\n\tidx := strings.IndexAny(query, \"{}\")\n\tif idx == -1 {\n\t\tsplitFinder.useWrapped = true\n\t\treturn splitFinder.wrapped.Execute(ctx, config, query, from, until)\n\t}\n\n\tsplitQueries, err := splitQuery(query, config.ClickHouse.MaxNodeToSplitIndex)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif len(splitQueries) <= 1 {\n\t\tsplitFinder.useWrapped = true\n\t\treturn splitFinder.wrapped.Execute(ctx, config, query, from, until)\n\t}\n\n\tw, err := splitFinder.whereFilter(splitQueries, from, until)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tsplitFinder.stats = append(splitFinder.stats, metrics.FinderStat{})\n\tstat := &splitFinder.stats[len(splitFinder.stats)-1]\n\n\tsplitFinder.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query(\n\t\tscope.WithTable(ctx, splitFinder.table),\n\t\tsplitFinder.url,\n\t\t// TODO: consider consistent query generator\n\t\tfmt.Sprintf(\"SELECT Path FROM %s WHERE %s GROUP BY Path FORMAT TabSeparatedRaw\", splitFinder.table, w),\n\t\tsplitFinder.opts,\n\t\tnil,\n\t)\n\tstat.Table = splitFinder.table\n\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tstat.ReadBytes = int64(len(splitFinder.body))\n\tsplitFinder.body, splitFinder.rows, _ = splitIndexBody(splitFinder.body, splitFinder.useReverse, splitFinder.useCache)\n\n\treturn nil\n}\n\nfunc splitQuery(query string, maxNodeToSplitIdx int) ([]string, error) {\n\tsplitQueries := make([]string, 0, 1)\n\n\tfirstClosingBracketIndex := strings.Index(query, \"}\")\n\tlastOpenBracketIndex := strings.LastIndex(query, \"{\")\n\n\tfirstOpenBracketsIndex := strings.Index(query, \"{\")\n\tdirectNodeCount := strings.Count(query[:firstOpenBracketsIndex], \".\")\n\tdirectWildcardIndex := where.IndexWildcard(query[:firstOpenBracketsIndex])\n\n\tlastClosingBracketIndex := strings.LastIndex(query, \"}\")\n\treverseNodeCount := strings.Count(query[lastClosingBracketIndex:], \".\")\n\n\tvar reverseWildcardIndex int\n\tif lastClosingBracketIndex == len(query)-1 {\n\t\treverseWildcardIndex = -1\n\t} else {\n\t\treverseWildcardIndex = where.IndexLastWildcard(query[lastClosingBracketIndex+1:])\n\t}\n\n\tuseDirect := true\n\n\tif directWildcardIndex >= 0 && reverseWildcardIndex >= 0 {\n\t\treturn []string{query}, nil\n\t} else if directWildcardIndex < 0 && reverseWildcardIndex >= 0 {\n\t\tif directNodeCount > maxNodeToSplitIdx {\n\t\t\treturn []string{query}, nil\n\t\t}\n\n\t\tuseDirect = true\n\t} else if directWildcardIndex >= 0 && reverseWildcardIndex < 0 {\n\t\tif reverseNodeCount > maxNodeToSplitIdx {\n\t\t\treturn []string{query}, nil\n\t\t}\n\n\t\tuseDirect = false\n\t} else {\n\t\tif directNodeCount > maxNodeToSplitIdx && reverseNodeCount > maxNodeToSplitIdx {\n\t\t\treturn []string{query}, nil\n\t\t}\n\t}\n\n\tif lastOpenBracketIndex < firstClosingBracketIndex {\n\t\t// we have only one bracket in query\n\t\terr := where.GlobExpandSimple(query, \"\", &splitQueries)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\treturn splitQueries, nil\n\t}\n\n\tchoicesInLeftMost := strings.Count(query[firstOpenBracketsIndex:firstClosingBracketIndex], \",\")\n\tchoicesInRightMost := strings.Count(query[lastOpenBracketIndex:lastClosingBracketIndex], \",\")\n\n\tif directWildcardIndex < 0 && reverseWildcardIndex < 0 {\n\t\tif directNodeCount > reverseNodeCount {\n\t\t\tif directNodeCount > maxNodeToSplitIdx {\n\t\t\t\treturn []string{query}, nil\n\t\t\t}\n\n\t\t\tuseDirect = true\n\t\t} else if reverseNodeCount > directNodeCount {\n\t\t\tif reverseNodeCount > maxNodeToSplitIdx {\n\t\t\t\treturn []string{query}, nil\n\t\t\t}\n\n\t\t\tuseDirect = false\n\t\t} else {\n\t\t\tif choicesInLeftMost >= choicesInRightMost {\n\t\t\t\tuseDirect = true\n\t\t\t} else {\n\t\t\t\tuseDirect = false\n\t\t\t}\n\t\t}\n\t}\n\n\tvar prefix, suffix, queryPart string\n\tif useDirect {\n\t\tprefix = \"\"\n\t\tqueryPart = query[:firstClosingBracketIndex+1]\n\t\tsuffix = query[firstClosingBracketIndex+1:]\n\t} else {\n\t\tprefix = query[:lastOpenBracketIndex]\n\t\tqueryPart = query[lastOpenBracketIndex:]\n\t\tsuffix = \"\"\n\t}\n\n\tsplitQueries, err := splitPartOfQuery(prefix, queryPart, suffix)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn splitQueries, nil\n}\n\nfunc splitPartOfQuery(prefix, queryPart, suffix string) ([]string, error) {\n\tsplitQueries := make([]string, 0)\n\n\terr := where.GlobExpandSimple(queryPart, \"\", &splitQueries)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfor i := range splitQueries {\n\t\tsplitQueries[i] = prefix + splitQueries[i] + suffix\n\t}\n\n\treturn splitQueries, nil\n}\n\nfunc (splitFinder *SplitIndexFinder) whereFilter(queries []string, from, until int64) (*where.Where, error) {\n\tqueryWithWildcardIdx := -1\n\n\tfor i, q := range queries {\n\t\terr := validatePlainQuery(q, splitFinder.wildcardMinDistance)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tif queryWithWildcardIdx < 0 && where.HasWildcard(q) {\n\t\t\tqueryWithWildcardIdx = i\n\t\t}\n\t}\n\n\tif queryWithWildcardIdx >= 0 {\n\t\tsplitFinder.useReverse = (&IndexFinder{\n\t\t\tconfReverses: splitFinder.confReverses,\n\t\t\tconfReverse: config.IndexReverse[splitFinder.reverse],\n\t\t}).useReverse(queries[queryWithWildcardIdx])\n\t} else {\n\t\tsplitFinder.useReverse = false\n\t}\n\n\tnonWildcardQueries := make([]string, 0)\n\taggregatedWhere := where.New()\n\n\tfor _, q := range queries {\n\t\tif splitFinder.useReverse {\n\t\t\tq = ReverseString(q)\n\t\t}\n\n\t\tif !where.HasWildcard(q) {\n\t\t\tnonWildcardQueries = append(nonWildcardQueries, q, q+\".\")\n\t\t} else {\n\t\t\taggregatedWhere.Or(where.TreeGlob(\"Path\", q))\n\t\t}\n\t}\n\n\tif len(nonWildcardQueries) > 0 {\n\t\taggregatedWhere.Or(where.In(\"Path\", nonWildcardQueries))\n\t}\n\n\tuseDates := useDaily(splitFinder.dailyEnabled, from, until)\n\tlevelOffset := calculateIndexLevelOffset(useDates, splitFinder.useReverse)\n\tlevel := strings.Count(queries[0], \".\") + 1\n\n\taggregatedWhere.And(where.Eq(\"Level\", level+levelOffset))\n\taddDatesToWhere(aggregatedWhere, useDates, from, until)\n\n\treturn aggregatedWhere, nil\n}\n\n// List returns clickhouse response split by delimiter.\n// If there was no split, wrapped.List will be used.\nfunc (splitFinder *SplitIndexFinder) List() [][]byte {\n\tif splitFinder.useWrapped {\n\t\treturn splitFinder.wrapped.List()\n\t}\n\n\treturn makeList(splitFinder.rows, false)\n}\n\n// Series same as List. If there was no split, wrapped.Series will be used.\nfunc (splitFinder *SplitIndexFinder) Series() [][]byte {\n\tif splitFinder.useWrapped {\n\t\treturn splitFinder.wrapped.Series()\n\t}\n\n\treturn makeList(splitFinder.rows, true)\n}\n\n// Abs for this implementation returns given v.\n// If there was no split, wrapped.Abs will be used.\nfunc (splitFinder *SplitIndexFinder) Abs(v []byte) []byte {\n\tif splitFinder.useWrapped {\n\t\treturn splitFinder.wrapped.Abs(v)\n\t}\n\n\treturn v\n}\n\n// Bytes returns clickhouse response bytes.\n// If there was no split, wrapped.Bytes will be used.\nfunc (splitFinder *SplitIndexFinder) Bytes() ([]byte, error) {\n\tif splitFinder.useWrapped {\n\t\treturn splitFinder.wrapped.Bytes()\n\t}\n\n\treturn splitFinder.body, nil\n}\n\nfunc (splitFinder *SplitIndexFinder) Stats() []metrics.FinderStat {\n\tif splitFinder.useWrapped {\n\t\treturn splitFinder.wrapped.Stats()\n\t}\n\n\treturn splitFinder.stats\n}\n" }, { "path": "finder/split_test.go", "content": "package finder\n\nimport (\n\t\"fmt\"\n\t\"net/http\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/helper/date\"\n\t\"github.com/lomik/graphite-clickhouse/helper/errs\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc Test_splitQuery(t *testing.T) {\n\ttype testcase struct {\n\t\tgivenQuery string\n\t\tgivenMaxNodeToSplitIndex int\n\t\texpectedQueries []string\n\t\texpectedErr error\n\t\tdesc string\n\t}\n\n\tcases := []testcase{\n\t\t{\n\t\t\tgivenQuery: \"some.*.{a,b,c}.{first,second}.*.test.metric\",\n\t\t\tgivenMaxNodeToSplitIndex: 3,\n\t\t\texpectedQueries: []string{\n\t\t\t\t\"some.*.{a,b,c}.{first,second}.*.test.metric\",\n\t\t\t},\n\t\t\texpectedErr: nil,\n\t\t\tdesc: \"have wildcards on both direct and reverse, so no split\",\n\t\t},\n\t\t{\n\t\t\tgivenQuery: \"some.long.{a,b,c}.{first,second}.*.metric\",\n\t\t\tgivenMaxNodeToSplitIndex: 1,\n\t\t\texpectedQueries: []string{\n\t\t\t\t\"some.long.{a,b,c}.{first,second}.*.metric\",\n\t\t\t},\n\t\t\texpectedErr: nil,\n\t\t\tdesc: \"have wildcard on reverse but index of list is greater than given in config\",\n\t\t},\n\t\t{\n\t\t\tgivenQuery: \"some.long.{a,b,c}.{first,second}.*.metric\",\n\t\t\tgivenMaxNodeToSplitIndex: 2,\n\t\t\texpectedQueries: []string{\n\t\t\t\t\"some.long.a.{first,second}.*.metric\",\n\t\t\t\t\"some.long.b.{first,second}.*.metric\",\n\t\t\t\t\"some.long.c.{first,second}.*.metric\",\n\t\t\t},\n\t\t\texpectedErr: nil,\n\t\t\tdesc: \"have wildcard on reverse and index of list is less or equal than given in config\",\n\t\t},\n\t\t{\n\t\t\tgivenQuery: \"some.*.{a,b,c}.{first,second}.test.metric\",\n\t\t\tgivenMaxNodeToSplitIndex: 1,\n\t\t\texpectedQueries: []string{\n\t\t\t\t\"some.*.{a,b,c}.{first,second}.test.metric\",\n\t\t\t},\n\t\t\texpectedErr: nil,\n\t\t\tdesc: \"have wildcard on direct but index of list is greater than given in config\",\n\t\t},\n\t\t{\n\t\t\tgivenQuery: \"some.*.{a,b,c}.{first,second}.test.metric\",\n\t\t\tgivenMaxNodeToSplitIndex: 2,\n\t\t\texpectedQueries: []string{\n\t\t\t\t\"some.*.{a,b,c}.first.test.metric\",\n\t\t\t\t\"some.*.{a,b,c}.second.test.metric\",\n\t\t\t},\n\t\t\texpectedErr: nil,\n\t\t\tdesc: \"have wildcard on direct and index of list is less or equal than given in config\",\n\t\t},\n\t\t{\n\t\t\tgivenQuery: \"some.long.{a,b,c}.{first,second}.test.metric\",\n\t\t\tgivenMaxNodeToSplitIndex: 1,\n\t\t\texpectedQueries: []string{\n\t\t\t\t\"some.long.{a,b,c}.{first,second}.test.metric\",\n\t\t\t},\n\t\t\texpectedErr: nil,\n\t\t\tdesc: \"no wildcards on both but indexes of lists are greater than given in config\",\n\t\t},\n\t\t{\n\t\t\tgivenQuery: \"{first,second}.some.metric.*\",\n\t\t\tgivenMaxNodeToSplitIndex: 3,\n\t\t\texpectedQueries: []string{\n\t\t\t\t\"first.some.metric.*\",\n\t\t\t\t\"second.some.metric.*\",\n\t\t\t},\n\t\t\texpectedErr: nil,\n\t\t\tdesc: \"only one bracket with wildcard on reverse\",\n\t\t},\n\t\t{\n\t\t\tgivenQuery: \"*.some.metric.{first,second}\",\n\t\t\tgivenMaxNodeToSplitIndex: 3,\n\t\t\texpectedQueries: []string{\n\t\t\t\t\"*.some.metric.first\",\n\t\t\t\t\"*.some.metric.second\",\n\t\t\t},\n\t\t\texpectedErr: nil,\n\t\t\tdesc: \"only one bracket and wildcard on direct\",\n\t\t},\n\t\t{\n\t\t\tgivenQuery: \"some.very.long.{a,b}.*.{first,second}.metric\",\n\t\t\tgivenMaxNodeToSplitIndex: 2,\n\t\t\texpectedQueries: []string{\n\t\t\t\t\"some.very.long.{a,b}.*.{first,second}.metric\",\n\t\t\t},\n\t\t\texpectedErr: nil,\n\t\t\tdesc: \"no wildcards, but direct has more nodes than max-node-to-split-index\",\n\t\t},\n\t\t{\n\t\t\tgivenQuery: \"some.very.long.{a,b}.*.{first,second}.metric\",\n\t\t\tgivenMaxNodeToSplitIndex: 3,\n\t\t\texpectedQueries: []string{\n\t\t\t\t\"some.very.long.a.*.{first,second}.metric\",\n\t\t\t\t\"some.very.long.b.*.{first,second}.metric\",\n\t\t\t},\n\t\t\texpectedErr: nil,\n\t\t\tdesc: \"no wildcards, direct has more nodes than reverse\",\n\t\t},\n\t\t{\n\t\t\tgivenQuery: \"some.{a,b}.*.{first,second}.long.test.metric\",\n\t\t\tgivenMaxNodeToSplitIndex: 2,\n\t\t\texpectedQueries: []string{\n\t\t\t\t\"some.{a,b}.*.{first,second}.long.test.metric\",\n\t\t\t},\n\t\t\texpectedErr: nil,\n\t\t\tdesc: \"no wildcards, but reverse has more nodes than max-node-to-split-index\",\n\t\t},\n\t\t{\n\t\t\tgivenQuery: \"some.{a,b}.*.{first,second}.long.test.metric\",\n\t\t\tgivenMaxNodeToSplitIndex: 3,\n\t\t\texpectedQueries: []string{\n\t\t\t\t\"some.{a,b}.*.first.long.test.metric\",\n\t\t\t\t\"some.{a,b}.*.second.long.test.metric\",\n\t\t\t},\n\t\t\texpectedErr: nil,\n\t\t\tdesc: \"no wildcards, reverse has more nodes than direct\",\n\t\t},\n\t\t{\n\t\t\tgivenQuery: \"some.very.long.{a,b,c}.*.{first,second}.long.test.metric\",\n\t\t\tgivenMaxNodeToSplitIndex: 3,\n\t\t\texpectedQueries: []string{\n\t\t\t\t\"some.very.long.a.*.{first,second}.long.test.metric\",\n\t\t\t\t\"some.very.long.b.*.{first,second}.long.test.metric\",\n\t\t\t\t\"some.very.long.c.*.{first,second}.long.test.metric\",\n\t\t\t},\n\t\t\texpectedErr: nil,\n\t\t\tdesc: \"no wildcards, direct nd reverse has equal nodes, but leftmost has more choices\",\n\t\t},\n\t\t{\n\t\t\tgivenQuery: \"some.very.long.{a,b}.*.{first,second,third}.long.test.metric\",\n\t\t\tgivenMaxNodeToSplitIndex: 3,\n\t\t\texpectedQueries: []string{\n\t\t\t\t\"some.very.long.{a,b}.*.first.long.test.metric\",\n\t\t\t\t\"some.very.long.{a,b}.*.second.long.test.metric\",\n\t\t\t\t\"some.very.long.{a,b}.*.third.long.test.metric\",\n\t\t\t},\n\t\t\texpectedErr: nil,\n\t\t\tdesc: \"no wildcards, direct nd reverse has equal nodes, but leftmost has more choices\",\n\t\t},\n\t\t{\n\t\t\tgivenQuery: \"query.{a,b}\",\n\t\t\tgivenMaxNodeToSplitIndex: -1,\n\t\t\texpectedQueries: []string{\n\t\t\t\t\"query.{a,b}\",\n\t\t\t},\n\t\t\texpectedErr: nil,\n\t\t\tdesc: \"not split query\",\n\t\t},\n\t\t{\n\t\t\tgivenQuery: \"*.query.{a,b}\",\n\t\t\tgivenMaxNodeToSplitIndex: -1,\n\t\t\texpectedQueries: []string{\n\t\t\t\t\"*.query.{a,b}\",\n\t\t\t},\n\t\t\texpectedErr: nil,\n\t\t\tdesc: \"not split query\",\n\t\t},\n\t\t{\n\t\t\tgivenQuery: \"*.query.{a,b}\",\n\t\t\tgivenMaxNodeToSplitIndex: 20,\n\t\t\texpectedQueries: []string{\n\t\t\t\t\"*.query.a\",\n\t\t\t\t\"*.query.b\",\n\t\t\t},\n\t\t\texpectedErr: nil,\n\t\t\tdesc: \"query split if MaxNodeToSplitIndex is greater than nodes amount in query\",\n\t\t},\n\t}\n\n\tfor i, singleCase := range cases {\n\t\tt.Run(fmt.Sprintf(\"case %v: %s\", i+1, singleCase.desc), func(t *testing.T) {\n\t\t\tgotQueries, gotErr := splitQuery(singleCase.givenQuery, singleCase.givenMaxNodeToSplitIndex)\n\n\t\t\tassert.Equal(t, singleCase.expectedQueries, gotQueries, singleCase.desc)\n\t\t\tassert.Equal(t, singleCase.expectedErr, gotErr, singleCase.desc)\n\t\t})\n\t}\n}\n\nfunc TestSplitIndexFinder_whereFilter(t *testing.T) {\n\ttype testcase struct {\n\t\tname string\n\t\tgivenQueries []string\n\t\tgivenFrom int64\n\t\tgivenUntil int64\n\t\tdailyEnabled bool\n\t\twildcardMinDistance int\n\t\treverse string\n\t\tconfReverses config.IndexReverses\n\t\texpectedWhereStr string\n\t\texpectedErr error\n\t}\n\n\tsomeFrom := time.Now().Unix() - 120\n\tsomeUntil := time.Now().Unix()\n\n\tcases := []testcase{\n\t\t{\n\t\t\tname: \"no wildcards in queries, no daily\",\n\t\t\tgivenQueries: []string{\n\t\t\t\t\"first.metric\",\n\t\t\t\t\"second.metric\",\n\t\t\t},\n\t\t\tdailyEnabled: false,\n\t\t\texpectedWhereStr: \"((Path IN ('first.metric','first.metric.','second.metric','second.metric.')) AND (Level=20002)) AND (Date='1970-02-12')\",\n\t\t},\n\t\t{\n\t\t\tname: \"wildcard in queries, reverse preferred, no daily\",\n\t\t\tgivenQueries: []string{\n\t\t\t\t\"*.first.metric\",\n\t\t\t\t\"*.second.metric\",\n\t\t\t},\n\t\t\tdailyEnabled: false,\n\t\t\texpectedWhereStr: \"(((Path LIKE 'metric.first.%') OR (Path LIKE 'metric.second.%')) AND (Level=30003)) AND (Date='1970-02-12')\",\n\t\t},\n\t\t{\n\t\t\tname: \"no wildcards in queries, daily enabled, but no from and until\",\n\t\t\tgivenQueries: []string{\n\t\t\t\t\"first.metric\",\n\t\t\t\t\"second.metric\",\n\t\t\t},\n\t\t\tdailyEnabled: true,\n\t\t\texpectedWhereStr: \"((Path IN ('first.metric','first.metric.','second.metric','second.metric.')) AND (Level=20002)) AND (Date='1970-02-12')\",\n\t\t},\n\t\t{\n\t\t\tname: \"no wildcards in queries, daily enabled, has from, until\",\n\t\t\tgivenQueries: []string{\n\t\t\t\t\"first.metric\",\n\t\t\t\t\"second.metric\",\n\t\t\t},\n\t\t\tgivenFrom: someFrom,\n\t\t\tgivenUntil: someFrom,\n\t\t\tdailyEnabled: true,\n\t\t\texpectedWhereStr: \"((Path IN ('first.metric','first.metric.','second.metric','second.metric.')) AND (Level=2)) AND (Date >='\" +\n\t\t\t\tdate.FromTimestampToDaysFormat(someFrom) + \"' AND Date <= '\" + date.UntilTimestampToDaysFormat(someUntil) + \"')\",\n\t\t},\n\t\t{\n\t\t\tname: \"wildcard in queries, reverse preferred, daily enabled, no from, until\",\n\t\t\tgivenQueries: []string{\n\t\t\t\t\"*.first.metric\",\n\t\t\t\t\"*.second.metric\",\n\t\t\t},\n\t\t\tdailyEnabled: true,\n\t\t\texpectedWhereStr: \"(((Path LIKE 'metric.first.%') OR (Path LIKE 'metric.second.%')) AND (Level=30003)) AND (Date='1970-02-12')\",\n\t\t},\n\t\t{\n\t\t\tname: \"wildcard in queries, reverse preferred, daily enabled, has from, until\",\n\t\t\tgivenQueries: []string{\n\t\t\t\t\"*.first.metric\",\n\t\t\t\t\"*.second.metric\",\n\t\t\t},\n\t\t\tdailyEnabled: true,\n\t\t\tgivenFrom: someFrom,\n\t\t\tgivenUntil: someUntil,\n\t\t\texpectedWhereStr: \"(((Path LIKE 'metric.first.%') OR (Path LIKE 'metric.second.%')) AND (Level=10003)) AND (Date >='\" +\n\t\t\t\tdate.FromTimestampToDaysFormat(someFrom) + \"' AND Date <= '\" + date.UntilTimestampToDaysFormat(someUntil) + \"')\",\n\t\t},\n\t\t{\n\t\t\tname: \"some queries have wildcard, daily enabled, has from, until\",\n\t\t\tgivenQueries: []string{\n\t\t\t\t\"help.*first.metric\",\n\t\t\t\t\"help.second.metric\",\n\t\t\t\t\"help.th*rd.metric\",\n\t\t\t\t\"help.forth.metric\",\n\t\t\t},\n\t\t\tdailyEnabled: true,\n\t\t\tgivenFrom: someFrom,\n\t\t\tgivenUntil: someUntil,\n\t\t\texpectedWhereStr: \"((((Path LIKE 'help.%' AND match(Path, '^help[.]([^.]*?)first[.]metric[.]?$')) OR (Path LIKE 'help.th%' AND match(Path, '^help[.]th([^.]*?)rd[.]metric[.]?$'))) OR (Path IN ('help.second.metric','help.second.metric.','help.forth.metric','help.forth.metric.'))) AND (Level=3)) AND (Date >='\" +\n\t\t\t\tdate.FromTimestampToDaysFormat(someFrom) + \"' AND Date <= '\" + date.UntilTimestampToDaysFormat(someUntil) + \"')\",\n\t\t},\n\t\t{\n\t\t\tname: \"some queries have wildcard, daily enabled, has from, until, but reverse preferred\",\n\t\t\tgivenQueries: []string{\n\t\t\t\t\"help.*first.metric.count\",\n\t\t\t\t\"help.second.metric.count\",\n\t\t\t\t\"help.th*rd.metric.count\",\n\t\t\t\t\"help.forth.metric.count\",\n\t\t\t},\n\t\t\tdailyEnabled: true,\n\t\t\tgivenFrom: someFrom,\n\t\t\tgivenUntil: someUntil,\n\t\t\texpectedWhereStr: \"((((Path LIKE 'count.metric.%' AND match(Path, '^count[.]metric[.]([^.]*?)first[.]help[.]?$')) OR (Path LIKE 'count.metric.th%' AND match(Path, '^count[.]metric[.]th([^.]*?)rd[.]help[.]?$'))) OR (Path IN ('count.metric.second.help','count.metric.second.help.','count.metric.forth.help','count.metric.forth.help.'))) AND (Level=10004)) AND (Date >='\" +\n\t\t\t\tdate.FromTimestampToDaysFormat(someFrom) + \"' AND Date <= '\" + date.UntilTimestampToDaysFormat(someUntil) + \"')\",\n\t\t\texpectedErr: nil,\n\t\t},\n\t\t{\n\t\t\tname: \"some queries have wildcard, daily enabled, has from, until, but reverse preferred, first query has no wildcard\",\n\t\t\tgivenQueries: []string{\n\t\t\t\t\"help.second.metric.count\",\n\t\t\t\t\"help.*first.metric.count\",\n\t\t\t\t\"help.th*rd.metric.count\",\n\t\t\t\t\"help.forth.metric.count\",\n\t\t\t},\n\t\t\tdailyEnabled: true,\n\t\t\tgivenFrom: someFrom,\n\t\t\tgivenUntil: someUntil,\n\t\t\texpectedWhereStr: \"((((Path LIKE 'count.metric.%' AND match(Path, '^count[.]metric[.]([^.]*?)first[.]help[.]?$')) OR (Path LIKE 'count.metric.th%' AND match(Path, '^count[.]metric[.]th([^.]*?)rd[.]help[.]?$'))) OR (Path IN ('count.metric.second.help','count.metric.second.help.','count.metric.forth.help','count.metric.forth.help.'))) AND (Level=10004)) AND (Date >='\" +\n\t\t\t\tdate.FromTimestampToDaysFormat(someFrom) + \"' AND Date <= '\" + date.UntilTimestampToDaysFormat(someUntil) + \"')\",\n\t\t},\n\t\t{\n\t\t\tname: \"queries do not satisfy wildcard min distance\",\n\t\t\tgivenQueries: []string{\n\t\t\t\t\"a*.first.metric.*\",\n\t\t\t\t\"b*.second.metric.*\",\n\t\t\t},\n\t\t\twildcardMinDistance: 1,\n\t\t\texpectedWhereStr: \"\",\n\t\t\texpectedErr: errs.NewErrorWithCode(\"query has wildcards way too early at the start and at the end of it\", http.StatusBadRequest),\n\t\t},\n\t}\n\n\tfor i, tc := range cases {\n\t\tt.Run(fmt.Sprintf(\"Case %v: %s\", i+1, tc.name), func(t *testing.T) {\n\t\t\tf := WrapSplitIndex(\n\t\t\t\t&IndexFinder{},\n\t\t\t\ttc.wildcardMinDistance,\n\t\t\t\t\"http://localhost:8123/\",\n\t\t\t\t\"graphite_index\",\n\t\t\t\ttc.dailyEnabled,\n\t\t\t\ttc.reverse,\n\t\t\t\ttc.confReverses,\n\t\t\t\tclickhouse.Options{},\n\t\t\t\tfalse)\n\n\t\t\tgot, err := f.whereFilter(tc.givenQueries, tc.givenFrom, tc.givenUntil)\n\t\t\tassert.Equal(t, tc.expectedErr, err)\n\n\t\t\tif err == nil {\n\t\t\t\tassert.Equal(t, tc.expectedWhereStr, got.String())\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "finder/tag.go", "content": "package finder\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/where\"\n)\n\ntype TagState int\n\nconst (\n\tTagRoot TagState = iota // query = \"*\"\n\tTagSkip // not _tag prefix\n\tTagInfoRoot // query = \"_tag\"\n\tTagList\n\tTagListSeriesRoot\n\tTagListSeries\n\tTagListParam\n)\n\ntype TagQ struct {\n\tParam *string\n\tValue *string\n}\n\nfunc (q TagQ) String() string {\n\tif q.Param != nil && q.Value != nil {\n\t\treturn fmt.Sprintf(\"{\\\"param\\\"=%#v, \\\"value\\\"=%#v}\", *q.Param, *q.Value)\n\t}\n\n\tif q.Param != nil {\n\t\treturn fmt.Sprintf(\"{\\\"param\\\"=%#v}\", *q.Param)\n\t}\n\n\tif q.Value != nil {\n\t\treturn fmt.Sprintf(\"{\\\"value\\\"=%#v}\", *q.Value)\n\t}\n\n\treturn \"{}\"\n}\n\nfunc (q *TagQ) Where(field string) string {\n\tif q.Param != nil && q.Value != nil && *q.Value != \"*\" {\n\t\treturn where.Eq(field, *q.Param+*q.Value)\n\t}\n\n\tif q.Param != nil {\n\t\treturn where.HasPrefix(field, *q.Param)\n\t}\n\n\tif q.Value != nil && *q.Value != \"*\" {\n\t\treturn where.Eq(field, *q.Value)\n\t}\n\n\treturn \"\"\n}\n\ntype TagFinder struct {\n\twrapped Finder\n\turl string // clickhouse dsn\n\ttable string // graphite_tag table\n\topts clickhouse.Options // clickhouse timeout, connectTimeout, etc\n\tstats []metrics.FinderStat\n\tstate TagState\n\ttagQuery []TagQ\n\tseriesQuery string\n\ttagPrefix []byte\n\tuseWrapped bool\n\tbody []byte // clickhouse response\n}\n\nvar EmptyList [][]byte = [][]byte{}\n\nfunc WrapTag(f Finder, url string, table string, opts clickhouse.Options) *TagFinder {\n\treturn &TagFinder{\n\t\twrapped: f,\n\t\turl: url,\n\t\ttable: table,\n\t\topts: opts,\n\t\ttagQuery: make([]TagQ, 0),\n\t\tuseWrapped: true,\n\t}\n}\n\nfunc (t *TagFinder) tagListSQL() (string, error) {\n\tif len(t.tagQuery) == 0 {\n\t\treturn \"\", nil\n\t}\n\n\tw := where.New()\n\n\t// first\n\tw.And(t.tagQuery[0].Where(\"Tag1\"))\n\n\tif len(t.tagQuery) == 1 {\n\t\tw.And(where.Eq(\"Level\", 1))\n\t\treturn fmt.Sprintf(\"SELECT Tag1 FROM %s WHERE %s GROUP BY Tag1\", t.table, w), nil\n\t}\n\n\t// 1..(n-1)\n\tfor i := 1; i < len(t.tagQuery)-1; i++ {\n\t\tcond := t.tagQuery[i].Where(\"x\")\n\t\tif cond != \"\" {\n\t\t\tw.Andf(\"arrayExists((x) -> %s, Tags)\", cond)\n\t\t}\n\t}\n\n\t// last\n\tw.And(t.tagQuery[len(t.tagQuery)-1].Where(\"TagN\"))\n\n\tw.And(where.Eq(\"IsLeaf\", 1))\n\n\treturn fmt.Sprintf(\"SELECT TagN FROM %s ARRAY JOIN Tags AS TagN WHERE %s GROUP BY TagN\", t.table, w), nil\n}\n\nfunc (t *TagFinder) seriesSQL() (string, error) {\n\tif len(t.tagQuery) == 0 {\n\t\treturn \"\", nil\n\t}\n\n\tw := where.New()\n\n\tw.Andf(\"Version>=(SELECT Max(Version) FROM %s WHERE Tag1='' AND Level=0 AND Path='')\", t.table)\n\t// first\n\tw.And(t.tagQuery[0].Where(\"Tag1\"))\n\n\t// 1..(n-1)\n\tfor i := 1; i < len(t.tagQuery); i++ {\n\t\tcond := t.tagQuery[i].Where(\"x\")\n\t\tif cond != \"\" {\n\t\t\tw.Andf(\"arrayExists((x) -> %s, Tags)\", cond)\n\t\t}\n\t}\n\n\tbase := &BaseFinder{}\n\tw.And(base.where(t.seriesQuery).String())\n\n\t// TODO: consider consistent query generator\n\treturn fmt.Sprintf(\"SELECT Path FROM %s WHERE %s GROUP BY Path FORMAT TabSeparatedRaw\", t.table, w), nil\n}\n\nfunc (t *TagFinder) MakeSQL(query string) (string, error) {\n\tif query == \"_tag\" {\n\t\tt.state = TagInfoRoot\n\t\treturn \"\", nil\n\t}\n\n\tqs0 := strings.Split(query, \".\")\n\tqs := qs0\n\n\tt.tagQuery = make([]TagQ, 0)\n\n\tfor {\n\t\tif len(qs) == 0 {\n\t\t\tbreak\n\t\t}\n\n\t\tif qs[0] == \"_tag\" {\n\t\t\tif len(qs) >= 2 {\n\t\t\t\tv := qs[1]\n\t\t\t\tif len(v) > 0 && v[len(v)-1] == '=' {\n\t\t\t\t\tif len(qs) >= 3 {\n\t\t\t\t\t\tt.tagQuery = append(t.tagQuery, TagQ{Param: &v, Value: &qs[2]})\n\t\t\t\t\t\tqs = qs[3:]\n\t\t\t\t\t} else {\n\t\t\t\t\t\tt.tagQuery = append(t.tagQuery, TagQ{Param: &v})\n\t\t\t\t\t\tqs = qs[2:]\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tt.tagQuery = append(t.tagQuery, TagQ{Value: &v})\n\t\t\t\t\tqs = qs[2:]\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tt.tagQuery = append(t.tagQuery, TagQ{})\n\t\t\t\tqs = qs[1:]\n\t\t\t}\n\t\t} else {\n\t\t\tt.seriesQuery = strings.Join(qs, \".\")\n\t\t\tbreak\n\t\t}\n\t}\n\n\tif len(qs0) > len(qs) {\n\t\tt.tagPrefix = append([]byte(strings.Join(qs0[:len(qs0)-len(qs)], \".\")), '.')\n\t}\n\n\tif t.seriesQuery == \"\" {\n\t\tif len(t.tagQuery) > 0 && t.tagQuery[len(t.tagQuery)-1].Param != nil {\n\t\t\tt.state = TagListParam\n\t\t} else {\n\t\t\tt.state = TagList\n\t\t}\n\n\t\treturn t.tagListSQL()\n\t}\n\n\tif t.seriesQuery == \"*\" {\n\t\tt.state = TagListSeriesRoot\n\t\treturn t.seriesSQL()\n\t}\n\n\tt.state = TagListSeries\n\n\treturn t.seriesSQL()\n}\n\nfunc (t *TagFinder) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) (err error) {\n\tt.state = TagSkip\n\n\tif query == \"\" {\n\t\treturn t.wrapped.Execute(ctx, config, query, from, until)\n\t}\n\n\tif query == \"*\" {\n\t\tt.state = TagRoot\n\t\treturn t.wrapped.Execute(ctx, config, query, from, until)\n\t}\n\n\tif !strings.HasPrefix(query, \"_tag.\") && query != \"_tag\" {\n\t\treturn t.wrapped.Execute(ctx, config, query, from, until)\n\t}\n\n\tt.useWrapped = false\n\n\tvar sql string\n\n\tsql, err = t.MakeSQL(query)\n\tif err != nil || sql == \"\" {\n\t\treturn\n\t}\n\n\tt.stats = append(t.stats, metrics.FinderStat{})\n\tstat := &t.stats[len(t.stats)-1]\n\n\tt.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query(scope.WithTable(ctx, t.table), t.url, sql, t.opts, nil)\n\tstat.Table = t.table\n\tstat.ReadBytes = int64(len(t.body))\n\n\treturn\n}\n\nfunc (t *TagFinder) List() [][]byte {\n\tswitch t.state {\n\tcase TagSkip:\n\t\treturn t.wrapped.List()\n\tcase TagInfoRoot:\n\t\treturn [][]byte{[]byte(\"_tag.\")}\n\tcase TagRoot:\n\t\t// pass\n\t\treturn append([][]byte{[]byte(\"_tag.\")}, t.wrapped.List()...)\n\t}\n\n\tif t.body == nil {\n\t\treturn [][]byte{}\n\t}\n\n\trows := bytes.Split(t.body, []byte{'\\n'})\n\n\tskip := 0\n\n\tfor i := 0; i < len(rows); i++ {\n\t\tif len(rows[i]) == 0 {\n\t\t\tskip++\n\t\t\tcontinue\n\t\t}\n\n\t\tif skip > 0 {\n\t\t\trows[i-skip] = rows[i]\n\t\t}\n\t}\n\n\trows = rows[:len(rows)-skip]\n\n\tif t.state == TagList || t.state == TagListParam {\n\t\t// add dots\n\t\tfor i := 0; i < len(rows); i++ {\n\t\t\teqIndex := bytes.IndexByte(rows[i], '=')\n\t\t\tif eqIndex > 0 && eqIndex < len(rows[i])-1 {\n\t\t\t\tif t.state == TagListParam {\n\t\t\t\t\trows[i] = append(rows[i][eqIndex+1:], '.')\n\t\t\t\t} else {\n\t\t\t\t\trows[i][eqIndex+1] = '.'\n\t\t\t\t\trows[i] = rows[i][:eqIndex+2]\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\trows[i] = append(rows[i], '.')\n\t\t\t}\n\t\t}\n\t}\n\n\tif t.state == TagListSeriesRoot {\n\t\trows = append(rows, []byte(\"_tag.\"))\n\t}\n\n\treturn rows\n}\n\nfunc (t *TagFinder) Series() [][]byte {\n\tswitch t.state {\n\tcase TagSkip:\n\t\treturn t.wrapped.Series()\n\tcase TagInfoRoot:\n\t\treturn EmptyList\n\tcase TagRoot:\n\t\treturn t.wrapped.Series()\n\t}\n\n\trows := t.List()\n\n\tskip := 0\n\n\tfor i := 0; i < len(rows); i++ {\n\t\tif len(rows[i]) == 0 {\n\t\t\tskip++\n\t\t\tcontinue\n\t\t}\n\n\t\tif rows[i][len(rows[i])-1] == '.' {\n\t\t\tskip++\n\t\t\tcontinue\n\t\t}\n\n\t\tif skip > 0 {\n\t\t\trows[i-skip] = rows[i]\n\t\t}\n\t}\n\n\treturn rows\n}\n\nfunc (t *TagFinder) Abs(v []byte) []byte {\n\tif t.state == TagSkip {\n\t\treturn t.wrapped.Abs(v)\n\t}\n\n\treturn bytesConcat(t.tagPrefix, v)\n}\n\nfunc (t *TagFinder) Bytes() ([]byte, error) {\n\treturn nil, ErrNotImplemented\n}\n\nfunc (t *TagFinder) Stats() []metrics.FinderStat {\n\tif t.useWrapped {\n\t\treturn t.wrapped.Stats()\n\t}\n\n\treturn t.stats\n}\n" }, { "path": "finder/tag_test.go", "content": "package finder\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\tchtest \"github.com/lomik/graphite-clickhouse/helper/tests/clickhouse\"\n)\n\nfunc TestTagsMakeSQL(t *testing.T) {\n\tassert := assert.New(t)\n\n\ttag1Base := \"SELECT Tag1 FROM table WHERE \"\n\ttag1Group := \" GROUP BY Tag1\"\n\n\ttagNBase := \"SELECT TagN FROM table ARRAY JOIN Tags AS TagN WHERE \"\n\ttagNGroup := \" GROUP BY TagN\"\n\n\ttable := []struct {\n\t\tquery string\n\t\tsql string\n\t\terror bool\n\t}{\n\t\t// SELECT Tag1 FROM graphite_tag WHERE Version >= (SELECT Max(Version) FROM graphite_tag WHERE Tag1='' AND Level=0 AND Path='') AND Level=1 GROUP BY Tag1;\n\t\t{\"_tag\", \"\", false},\n\t\t{\"_tag.*\", tag1Base + \"Level=1\" + tag1Group, false},\n\t\t{\"_tag.t1\", tag1Base + \"(Tag1='t1') AND (Level=1)\" + tag1Group, false},\n\t\t{\"_tag.p1=\", tag1Base + \"(Tag1 LIKE 'p1=%') AND (Level=1)\" + tag1Group, false},\n\t\t{\"_tag.p1=.*\", tag1Base + \"(Tag1 LIKE 'p1=%') AND (Level=1)\" + tag1Group, false},\n\t\t{\"_tag.p1=.v1\", tag1Base + \"(Tag1='p1=v1') AND (Level=1)\" + tag1Group, false},\n\t\t{\"_tag.t2._tag.*\", tagNBase + \"(Tag1='t2') AND (IsLeaf=1)\" + tagNGroup, false},\n\t\t{\"_tag.t2._tag.t2._tag.p3=.*\", tagNBase + \"(((Tag1='t2') AND (arrayExists((x) -> x='t2', Tags))) AND (TagN LIKE 'p3=%')) AND (IsLeaf=1)\" + tagNGroup, false},\n\t}\n\n\tfor _, test := range table {\n\t\ttestName := fmt.Sprintf(\"query: %#v\", test.query)\n\n\t\tm := NewMockFinder([][]byte{[]byte(\"mock\")})\n\t\tf := WrapTag(m, \"http://localhost:8123/\", \"table\", clickhouse.Options{Timeout: time.Second, ConnectTimeout: time.Second})\n\n\t\tsql, err := f.MakeSQL(test.query)\n\n\t\tif test.error {\n\t\t\tassert.Error(err)\n\t\t} else {\n\t\t\tassert.NoError(err)\n\t\t}\n\n\t\tassert.Equal(test.sql, sql, testName)\n\t}\n}\n\nfunc _TestTags(t *testing.T) {\n\tassert := assert.New(t)\n\n\tmockData := [][]byte{[]byte(\"mock\")}\n\n\ttype w []string\n\n\tmock := w{\"mock\"}\n\tempty := w{}\n\n\ttable := []struct {\n\t\tquery string\n\t\texpectedList []string\n\t\texpectedSeries []string\n\t}{\n\t\t// not tagged query\n\t\t{\"\", mock, mock},\n\t\t{\"t*\", mock, mock},\n\t\t{\"hello.*\", mock, mock},\n\n\t\t// list root\n\t\t{\"*\", w{\"_tag.\", \"mock\"}, mock},\n\n\t\t// info about _tag \"directory\"\n\t\t{\"_tag\", w{\"_tag.\"}, empty},\n\t\t{\"_tag.*\", w{\"_tag.t1.\", \"_tag.t2.\"}, empty},\n\t\t{\"_tag.t1\", w{\"_tag.t1.\", \"_tag.t2.\"}, empty},\n\t\t{\"_tag.t1.*\", w{\"_tag.t1.\", \"_tag.t2.\"}, empty},\n\t\t{\"_tag.t1._tag.*\", w{\"_tag.t1.\", \"_tag.t2.\"}, empty},\n\t\t{\"_tag.t1._tag.param=\", w{\"_tag.t1.\", \"_tag.t2.\"}, empty},\n\t\t{\"_tag.t1._tag.param=.value\", w{\"_tag.t1.\", \"_tag.t2.\"}, empty},\n\t\t{\"_tag.t1._tag.param=.value.*\", w{\"_tag.t1.\", \"_tag.t2.\"}, empty},\n\n\t\t// {\"hello\", []string{\"hello.\"}, []string{}},\n\t\t// {\"hello.*\", []string{\"hello.world\"}, []string{\"world\"}},\n\t\t// {\"*.*\", []string{\"hello.world\"}, []string{\"world\"}},\n\t\t// {\"*404*\", []string{}, []string{}},\n\t\t// {\"*404*.*\", []string{}, []string{}},\n\t\t// {\"hello.[bad regexp\", []string{}, []string{}},\n\t}\n\n\tfor _, test := range table {\n\t\ttestName := fmt.Sprintf(\"query: %#v\", test.query)\n\n\t\tsrv := chtest.NewTestServer()\n\n\t\tm := NewMockFinder(mockData)\n\t\tf := WrapTag(m, srv.URL, \"graphite_tag\", clickhouse.Options{Timeout: time.Second, ConnectTimeout: time.Second})\n\n\t\tconfig := config.New()\n\t\tf.Execute(context.Background(), config, test.query, 0, 0)\n\n\t\tlist := make([]string, 0)\n\t\tfor _, r := range f.List() {\n\t\t\tlist = append(list, string(r))\n\t\t}\n\n\t\tseries := make([]string, 0)\n\t\tfor _, r := range f.Series() {\n\t\t\tseries = append(series, string(r))\n\t\t}\n\n\t\tassert.Equal(test.expectedList, list, testName+\", list\")\n\t\tassert.Equal(test.expectedSeries, series, testName+\", series\")\n\n\t\tsrv.Close()\n\t}\n}\n" }, { "path": "finder/tagged.go", "content": "package finder\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"sort\"\n\t\"strings\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/helper/date\"\n\t\"github.com/lomik/graphite-clickhouse/helper/errs\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/where\"\n\n\t\"github.com/msaf1980/go-stringutils\"\n)\n\nvar (\n\tErrCostlySeriesByTag = errs.NewErrorWithCode(\"seriesByTag argument has too much wildcard and regex terms\", http.StatusForbidden)\n\tErrSyntaxSeriesByTag = errs.NewErrorWithCode(\"invalid seriesByTag syntax\", http.StatusBadRequest)\n\tErrNotEnoughArgsSeriesByTag = errs.NewErrorWithCode(\"not enough arguments in seriesByTag\", http.StatusBadRequest)\n)\n\ntype TaggedTermOp int\n\nconst (\n\tTaggedTermEq TaggedTermOp = 1\n\tTaggedTermMatch TaggedTermOp = 2\n\tTaggedTermNe TaggedTermOp = 3\n\tTaggedTermNotMatch TaggedTermOp = 4\n)\n\ntype TaggedTerm struct {\n\tKey string\n\tOp TaggedTermOp\n\tValue string\n\tHasWildcard bool // only for TaggedTermEq\n\n\tNonDefaultCost bool\n\tCost int // tag cost for use ad primary filter (use tag with maximal selectivity). 0 by default, minimal is better.\n\t// __name__ tag is prefered, if some tag has better selectivity than name, set it cost to < 0\n\t// values with wildcards or regex matching also has lower priority, set if needed it cost to < 0\n}\n\ntype TaggedTermList []TaggedTerm\n\nfunc (s TaggedTermList) Len() int {\n\treturn len(s)\n}\nfunc (s TaggedTermList) Swap(i, j int) {\n\ts[i], s[j] = s[j], s[i]\n}\nfunc (s TaggedTermList) Less(i, j int) bool {\n\tif s[i].Op < s[j].Op {\n\t\treturn true\n\t}\n\n\tif s[i].Op > s[j].Op {\n\t\treturn false\n\t}\n\n\tif s[i].Op == TaggedTermEq && !s[i].HasWildcard && s[j].HasWildcard {\n\t\t// globs as fist eq might be have a bad perfomance\n\t\treturn true\n\t}\n\n\tif s[i].Key == \"__name__\" && s[j].Key != \"__name__\" {\n\t\treturn true\n\t}\n\n\treturn false\n}\n\ntype TaggedFinder struct {\n\turl string // clickhouse dsn\n\ttable string // graphite_tag table\n\ttcq *TagCountQuerier // An object for querying tag weights from clickhouse. See doc/config.md for details.\n\tabsKeepEncoded bool // Abs returns url encoded value. For queries from prometheus\n\topts clickhouse.Options // clickhouse query timeout\n\tconfiguredTagCosts map[string]*config.Costs // costs for taggs (sor tune index search)\n\tdailyEnabled bool\n\tuseCarbonBehavior bool\n\tdontMatchMissingTags bool\n\tmetricMightExists bool // if false, skip all subsequent queries because we determined that result will be empty anyway\n\tstats []metrics.FinderStat\n\tbody []byte // clickhouse response\n}\n\nfunc NewTagged(url string, table, tag1CountTable string, dailyEnabled, useCarbonBehavior, dontMatchMissingTags, absKeepEncoded bool, opts clickhouse.Options, taggedCosts map[string]*config.Costs) *TaggedFinder {\n\tfnd := &TaggedFinder{\n\t\turl: url,\n\t\ttable: table,\n\t\ttcq: nil,\n\t\tabsKeepEncoded: absKeepEncoded,\n\t\topts: opts,\n\t\tconfiguredTagCosts: taggedCosts,\n\t\tdailyEnabled: dailyEnabled,\n\t\tuseCarbonBehavior: useCarbonBehavior,\n\t\tdontMatchMissingTags: dontMatchMissingTags,\n\t\tmetricMightExists: true,\n\t\tstats: make([]metrics.FinderStat, 0),\n\t}\n\tif tag1CountTable != \"\" {\n\t\tfnd.tcq = NewTagCountQuerier(\n\t\t\turl,\n\t\t\ttag1CountTable,\n\t\t\topts,\n\t\t\tuseCarbonBehavior,\n\t\t\tdontMatchMissingTags,\n\t\t\tdailyEnabled,\n\t\t)\n\t}\n\n\treturn fnd\n}\n\nfunc (term *TaggedTerm) concat() string {\n\treturn term.Key + \"=\" + term.Value\n}\n\nfunc (term *TaggedTerm) concatMask() string {\n\tv := strings.ReplaceAll(term.Value, \"*\", \"%\")\n\treturn fmt.Sprintf(\"%s=%s\", term.Key, v)\n}\n\nfunc TaggedTermWhere1(term *TaggedTerm, useCarbonBehaviour, dontMatchMissingTags bool) (string, error) {\n\t// positive expression check only in Tag1\n\t// negative check in all Tags\n\tswitch term.Op {\n\tcase TaggedTermEq:\n\t\tif useCarbonBehaviour && term.Value == \"\" {\n\t\t\t// special case\n\t\t\t// container_name=\"\" ==> response should not contain container_name\n\t\t\treturn fmt.Sprintf(\"NOT arrayExists((x) -> %s, Tags)\", where.HasPrefix(\"x\", term.Key+\"=\")), nil\n\t\t}\n\n\t\tif strings.Contains(term.Value, \"*\") {\n\t\t\treturn where.Like(\"Tag1\", term.concatMask()), nil\n\t\t}\n\n\t\tvar values []string\n\t\tif err := where.GlobExpandSimple(term.Value, term.Key+\"=\", &values); err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\n\t\tif len(values) == 1 {\n\t\t\treturn where.Eq(\"Tag1\", values[0]), nil\n\t\t} else if len(values) > 1 {\n\t\t\treturn where.In(\"Tag1\", values), nil\n\t\t} else {\n\t\t\treturn where.Eq(\"Tag1\", term.concat()), nil\n\t\t}\n\tcase TaggedTermNe:\n\t\tif term.Value == \"\" {\n\t\t\t// special case\n\t\t\t// container_name!=\"\" ==> container_name exists and it is not empty\n\t\t\treturn where.HasPrefixAndNotEq(\"Tag1\", term.Key+\"=\"), nil\n\t\t}\n\n\t\tvar whereLikeAnyVal string\n\t\tif dontMatchMissingTags {\n\t\t\twhereLikeAnyVal = where.HasPrefix(\"Tag1\", term.Key+\"=\") + \" AND \"\n\t\t}\n\n\t\tif strings.Contains(term.Value, \"*\") {\n\t\t\twhereLike := where.Like(\"x\", term.concatMask())\n\t\t\treturn fmt.Sprintf(\"%sNOT arrayExists((x) -> %s, Tags)\", whereLikeAnyVal, whereLike), nil\n\t\t}\n\n\t\tvar values []string\n\t\tif err := where.GlobExpandSimple(term.Value, term.Key+\"=\", &values); err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\n\t\tif len(values) == 1 {\n\t\t\twhereEq := where.Eq(\"x\", values[0])\n\t\t\treturn fmt.Sprintf(\"%sNOT arrayExists((x) -> %s, Tags)\", whereLikeAnyVal, whereEq), nil\n\t\t} else if len(values) > 1 {\n\t\t\twhereIn := where.In(\"x\", values)\n\t\t\treturn fmt.Sprintf(\"%sNOT arrayExists((x) -> %s, Tags)\", whereLikeAnyVal, whereIn), nil\n\t\t} else {\n\t\t\twhereEq := where.Eq(\"x\", term.concat())\n\t\t\treturn fmt.Sprintf(\"%sNOT arrayExists((x) -> %s, Tags)\", whereLikeAnyVal, whereEq), nil\n\t\t}\n\tcase TaggedTermMatch:\n\t\treturn where.Match(\"Tag1\", term.Key, term.Value), nil\n\tcase TaggedTermNotMatch:\n\t\tvar whereLikeAnyVal string\n\t\tif dontMatchMissingTags {\n\t\t\twhereLikeAnyVal = where.HasPrefix(\"Tag1\", term.Key+\"=\") + \" AND \"\n\t\t}\n\n\t\twhereMatch := where.Match(\"x\", term.Key, term.Value)\n\n\t\treturn fmt.Sprintf(\"%sNOT arrayExists((x) -> %s, Tags)\", whereLikeAnyVal, whereMatch), nil\n\tdefault:\n\t\treturn \"\", nil\n\t}\n}\n\nfunc TaggedTermWhereN(term *TaggedTerm, useCarbonBehaviour, dontMatchMissingTags bool) (string, error) {\n\t// arrayExists((x) -> %s, Tags)\n\tswitch term.Op {\n\tcase TaggedTermEq:\n\t\tif useCarbonBehaviour && term.Value == \"\" {\n\t\t\t// special case\n\t\t\t// container_name=\"\" ==> response should not contain container_name\n\t\t\treturn fmt.Sprintf(\"NOT arrayExists((x) -> %s, Tags)\", where.HasPrefix(\"x\", term.Key+\"=\")), nil\n\t\t}\n\n\t\tif strings.Contains(term.Value, \"*\") {\n\t\t\treturn fmt.Sprintf(\"arrayExists((x) -> %s, Tags)\", where.Like(\"x\", term.concatMask())), nil\n\t\t}\n\n\t\tvar values []string\n\t\tif err := where.GlobExpandSimple(term.Value, term.Key+\"=\", &values); err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\n\t\tif len(values) == 1 {\n\t\t\treturn where.ArrayHas(\"Tags\", values[0]), nil\n\t\t} else if len(values) > 1 {\n\t\t\tw := where.New()\n\t\t\tfor _, v := range values {\n\t\t\t\tw.Or(where.ArrayHas(\"Tags\", v))\n\t\t\t}\n\n\t\t\treturn w.String(), nil\n\t\t} else {\n\t\t\treturn where.ArrayHas(\"Tags\", term.concat()), nil\n\t\t}\n\tcase TaggedTermNe:\n\t\tif term.Value == \"\" {\n\t\t\t// special case\n\t\t\t// container_name!=\"\" ==> container_name exists and it is not empty\n\t\t\treturn fmt.Sprintf(\"arrayExists((x) -> %s, Tags)\", where.HasPrefixAndNotEq(\"x\", term.Key+\"=\")), nil\n\t\t}\n\n\t\tvar whereLikeAnyVal string\n\t\tif dontMatchMissingTags {\n\t\t\twhereLikeAnyVal = fmt.Sprintf(\"arrayExists((x) -> %s, Tags) AND \", where.HasPrefix(\"x\", term.Key+\"=\"))\n\t\t}\n\n\t\tif strings.Contains(term.Value, \"*\") {\n\t\t\twhereLike := where.Like(\"x\", term.concatMask())\n\t\t\treturn fmt.Sprintf(\"%sNOT arrayExists((x) -> %s, Tags)\", whereLikeAnyVal, whereLike), nil\n\t\t}\n\n\t\tvar values []string\n\t\tif err := where.GlobExpandSimple(term.Value, term.Key+\"=\", &values); err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\n\t\tif len(values) == 1 {\n\t\t\twhereEq := where.Eq(\"x\", values[0])\n\t\t\treturn fmt.Sprintf(\"%sNOT arrayExists((x) -> %s, Tags)\", whereLikeAnyVal, whereEq), nil\n\t\t} else if len(values) > 1 {\n\t\t\twhereIn := where.In(\"x\", values)\n\t\t\treturn fmt.Sprintf(\"%sNOT arrayExists((x) -> %s, Tags)\", whereLikeAnyVal, whereIn), nil\n\t\t} else {\n\t\t\twhereEq := where.Eq(\"x\", term.concat())\n\t\t\treturn fmt.Sprintf(\"%sNOT arrayExists((x) -> %s, Tags)\", whereLikeAnyVal, whereEq), nil\n\t\t}\n\tcase TaggedTermMatch:\n\t\treturn fmt.Sprintf(\"arrayExists((x) -> %s, Tags)\", where.Match(\"x\", term.Key, term.Value)), nil\n\tcase TaggedTermNotMatch:\n\t\tvar whereLikeAnyVal string\n\t\tif dontMatchMissingTags {\n\t\t\twhereLikeAnyVal = fmt.Sprintf(\"arrayExists((x) -> %s, Tags) AND \", where.HasPrefix(\"x\", term.Key+\"=\"))\n\t\t}\n\n\t\twhereMatch := where.Match(\"x\", term.Key, term.Value)\n\n\t\treturn fmt.Sprintf(\"%sNOT arrayExists((x) -> %s, Tags)\", whereLikeAnyVal, whereMatch), nil\n\tdefault:\n\t\treturn \"\", nil\n\t}\n}\n\nfunc setCost(term *TaggedTerm, costs *config.Costs) {\n\tif term.Op == TaggedTermEq || term.Op == TaggedTermMatch {\n\t\tif len(costs.ValuesCost) > 0 {\n\t\t\tif cost, ok := costs.ValuesCost[term.Value]; ok {\n\t\t\t\tterm.Cost = cost\n\t\t\t\tterm.NonDefaultCost = true\n\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\n\t\tif term.Op == TaggedTermEq && !term.HasWildcard && costs.Cost != nil {\n\t\t\tterm.Cost = *costs.Cost // only for non-wildcared eq\n\t\t\tterm.NonDefaultCost = true\n\t\t}\n\t}\n}\n\nfunc ParseTaggedConditions(conditions []string, config *config.Config, autocomplete bool) ([]TaggedTerm, error) {\n\tnonWildcards := 0\n\tterms := make([]TaggedTerm, len(conditions))\n\n\tfor i := 0; i < len(conditions); i++ {\n\t\ts := conditions[i]\n\n\t\ta := strings.SplitN(s, \"=\", 2)\n\t\tif len(a) != 2 {\n\t\t\treturn nil, fmt.Errorf(\"wrong seriesByTag expr: %#v\", s)\n\t\t}\n\n\t\ta[0] = strings.TrimSpace(a[0])\n\t\ta[1] = strings.TrimSpace(a[1])\n\n\t\top := \"=\"\n\n\t\tif len(a[0]) > 0 && a[0][len(a[0])-1] == '!' {\n\t\t\top = \"!\" + op\n\t\t\ta[0] = strings.TrimSpace(a[0][:len(a[0])-1])\n\t\t}\n\n\t\tif len(a[1]) > 0 && a[1][0] == '~' {\n\t\t\top = op + \"~\"\n\t\t\ta[1] = strings.TrimSpace(a[1][1:])\n\t\t}\n\n\t\tterms[i].Key = a[0]\n\t\tterms[i].Value = a[1]\n\n\t\tif terms[i].Key == \"name\" {\n\t\t\tterms[i].Key = \"__name__\"\n\t\t}\n\n\t\tswitch op {\n\t\tcase \"=\":\n\t\t\tterms[i].Op = TaggedTermEq\n\t\t\tterms[i].HasWildcard = where.HasWildcard(terms[i].Value)\n\t\t\t// special case when using useCarbonBehaviour = true\n\t\t\t// which matches everything that does not have that tag\n\t\t\tterms[i].HasWildcard = terms[i].HasWildcard || config.FeatureFlags.UseCarbonBehavior && terms[i].Value == \"\"\n\t\t\tif !terms[i].HasWildcard {\n\t\t\t\tnonWildcards++\n\t\t\t}\n\t\tcase \"!=\":\n\t\t\tterms[i].Op = TaggedTermNe\n\t\tcase \"=~\":\n\t\t\tterms[i].Op = TaggedTermMatch\n\t\tcase \"!=~\":\n\t\t\tterms[i].Op = TaggedTermNotMatch\n\t\tdefault:\n\t\t\treturn nil, fmt.Errorf(\"wrong seriesByTag expr: %#v\", s)\n\t\t}\n\t}\n\n\tif autocomplete {\n\t\tif config.ClickHouse.TagsMinInAutocomplete > 0 && nonWildcards < config.ClickHouse.TagsMinInAutocomplete {\n\t\t\treturn nil, ErrCostlySeriesByTag\n\t\t}\n\t} else if config.ClickHouse.TagsMinInQuery > 0 && nonWildcards < config.ClickHouse.TagsMinInQuery {\n\t\treturn nil, ErrCostlySeriesByTag\n\t}\n\n\treturn terms, nil\n}\n\nfunc parseString(s string) (string, string, error) {\n\tif s[0] != '\\'' && s[0] != '\"' {\n\t\tpanic(\"string should start with open quote\")\n\t}\n\n\tmatch := s[0]\n\n\ts = s[1:]\n\n\tvar i int\n\tfor i < len(s) && s[i] != match {\n\t\ti++\n\t}\n\n\tif i == len(s) {\n\t\treturn \"\", \"\", errs.NewErrorfWithCode(http.StatusBadRequest, \"seriesByTag arg missing quote %q'\", s)\n\t}\n\n\treturn s[:i], s[i+1:], nil\n}\n\nfunc seriesByTagArgs(query string) ([]string, error) {\n\tvar err error\n\n\targs := make([]string, 0, 8)\n\n\t// trim spaces\n\te := strings.Trim(query, \" \")\n\tif !strings.HasPrefix(e, \"seriesByTag(\") {\n\t\treturn nil, ErrSyntaxSeriesByTag\n\t}\n\n\tif e[len(e)-1] != ')' {\n\t\treturn nil, ErrSyntaxSeriesByTag\n\t}\n\n\te = e[12 : len(e)-1]\n\n\tfor len(e) > 0 {\n\t\tvar arg string\n\n\t\tif e[0] == '\\'' || e[0] == '\"' {\n\t\t\tif arg, e, err = parseString(e); err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\t// skip empty arg\n\t\t\tif arg != \"\" {\n\t\t\t\targs = append(args, arg)\n\t\t\t}\n\t\t} else if e[0] == ' ' || e[0] == ',' {\n\t\t\te = e[1:]\n\t\t} else {\n\t\t\treturn nil, errs.NewErrorfWithCode(http.StatusBadRequest, \"seriesByTag arg missing quote %q\", e)\n\t\t}\n\t}\n\n\treturn args, nil\n}\n\nfunc ParseSeriesByTag(query string, config *config.Config) ([]TaggedTerm, error) {\n\tconditions, err := seriesByTagArgs(query)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif len(conditions) < 1 {\n\t\treturn nil, ErrNotEnoughArgsSeriesByTag\n\t}\n\n\treturn ParseTaggedConditions(conditions, config, false)\n}\n\nfunc TaggedWhere(terms []TaggedTerm, useCarbonBehaviour, dontMatchMissingTags bool) (*where.Where, *where.Where, error) {\n\tw := where.New()\n\tpw := where.New()\n\n\tx, err := TaggedTermWhere1(&terms[0], useCarbonBehaviour, dontMatchMissingTags)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tif terms[0].Op == TaggedTermMatch {\n\t\tpw.And(x)\n\t}\n\n\tw.And(x)\n\n\tfor i := 1; i < len(terms); i++ {\n\t\tand, err := TaggedTermWhereN(&terms[i], useCarbonBehaviour, dontMatchMissingTags)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\n\t\tw.And(and)\n\t}\n\n\treturn w, pw, nil\n}\n\nfunc NewCachedTags(body []byte) *TaggedFinder {\n\treturn &TaggedFinder{\n\t\tbody: body,\n\t}\n}\n\nfunc (t *TaggedFinder) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) error {\n\tterms, err := t.PrepareTaggedTerms(ctx, config, query, from, until)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn t.ExecutePrepared(ctx, terms, from, until)\n}\n\nfunc (t *TaggedFinder) whereFilter(terms []TaggedTerm, from int64, until int64) (*where.Where, *where.Where, error) {\n\tw, pw, err := TaggedWhere(terms, t.useCarbonBehavior, t.dontMatchMissingTags)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tif t.dailyEnabled {\n\t\tw.Andf(\n\t\t\t\"Date >='%s' AND Date <= '%s'\",\n\t\t\tdate.FromTimestampToDaysFormat(from),\n\t\t\tdate.UntilTimestampToDaysFormat(until),\n\t\t)\n\t} else {\n\t\tw.Andf(\n\t\t\t\"Date >='%s'\",\n\t\t\tdate.FromTimestampToDaysFormat(from),\n\t\t)\n\t}\n\n\treturn w, pw, nil\n}\n\nfunc (t *TaggedFinder) ExecutePrepared(ctx context.Context, terms []TaggedTerm, from int64, until int64) error {\n\tw, pw, err := t.whereFilter(terms, from, until)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tt.stats = append(t.stats, metrics.FinderStat{})\n\tstat := &t.stats[len(t.stats)-1]\n\n\t// TODO: consider consistent query generator\n\tsql := fmt.Sprintf(\"SELECT Path FROM %s %s %s GROUP BY Path FORMAT TabSeparatedRaw\", t.table, pw.PreWhereSQL(), w.SQL())\n\tt.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query(scope.WithTable(ctx, t.table), t.url, sql, t.opts, nil)\n\tstat.Table = t.table\n\tstat.ReadBytes = int64(len(t.body))\n\n\treturn err\n}\n\nfunc (t *TaggedFinder) List() [][]byte {\n\tif t.body == nil {\n\t\treturn [][]byte{}\n\t}\n\n\trows := bytes.Split(t.body, []byte{'\\n'})\n\n\tskip := 0\n\n\tfor i := 0; i < len(rows); i++ {\n\t\tif len(rows[i]) == 0 {\n\t\t\tskip++\n\t\t\tcontinue\n\t\t}\n\n\t\tif skip > 0 {\n\t\t\trows[i-skip] = rows[i]\n\t\t}\n\t}\n\n\trows = rows[:len(rows)-skip]\n\n\treturn rows\n}\n\nfunc (t *TaggedFinder) Series() [][]byte {\n\treturn t.List()\n}\n\nfunc tagsParse(path string) (string, []string, error) {\n\tname, args, n := stringutils.Split2(path, \"?\")\n\tif n == 1 || args == \"\" {\n\t\treturn name, nil, fmt.Errorf(\"incomplete tags in '%s'\", path)\n\t}\n\n\ttags := strings.Split(args, \"&\")\n\tfor i := range tags {\n\t\ttags[i] = unescape(tags[i])\n\t}\n\n\treturn unescape(name), tags, nil\n}\n\nfunc TaggedDecode(v []byte) []byte {\n\ts := stringutils.UnsafeString(v)\n\n\tname, tags, err := tagsParse(s)\n\tif err != nil {\n\t\treturn v\n\t}\n\n\tif len(tags) == 0 {\n\t\treturn stringutils.UnsafeStringBytes(&name)\n\t}\n\n\tsort.Strings(tags)\n\n\tvar sb stringutils.Builder\n\n\tlength := len(name)\n\tfor _, tag := range tags {\n\t\tlength += len(tag) + 1\n\t}\n\n\tsb.Grow(length)\n\n\tsb.WriteString(name)\n\n\tfor _, tag := range tags {\n\t\tsb.WriteString(\";\")\n\t\tsb.WriteString(tag)\n\t}\n\n\treturn sb.Bytes()\n}\n\nfunc (t *TaggedFinder) Abs(v []byte) []byte {\n\tif t.absKeepEncoded {\n\t\treturn v\n\t}\n\n\treturn TaggedDecode(v)\n}\n\nfunc (t *TaggedFinder) Bytes() ([]byte, error) {\n\treturn nil, ErrNotImplemented\n}\n\nfunc (t *TaggedFinder) Stats() []metrics.FinderStat {\n\treturn t.stats\n}\n\nfunc (t *TaggedFinder) PrepareTaggedTerms(ctx context.Context, cfg *config.Config, query string, from int64, until int64) (terms []TaggedTerm, err error) {\n\tterms, err = ParseSeriesByTag(query, cfg)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar tagCounts map[string]*config.Costs = nil\n\tif t.tcq != nil {\n\t\ttagCounts, err = t.tcq.GetCostsFromCountTable(ctx, terms, from, until)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif tagCounts != nil {\n\t\tSetCosts(terms, tagCounts)\n\t} else if len(t.configuredTagCosts) != 0 {\n\t\tSetCosts(terms, t.configuredTagCosts)\n\t}\n\n\tSortTaggedTermsByCost(terms)\n\n\treturn terms, nil\n}\n\nfunc SortTaggedTermsByCost(terms []TaggedTerm) {\n\t// compare with taggs costs\n\tsort.Slice(terms, func(i, j int) bool {\n\t\t// compare taggs costs, if all of TaggegTerms has custom cost.\n\t\t// this is allow overwrite operators order (Eq with or without wildcards/Match), use with carefully\n\t\tif terms[i].Cost != terms[j].Cost {\n\t\t\tif terms[i].NonDefaultCost && terms[j].NonDefaultCost ||\n\t\t\t\t(terms[i].NonDefaultCost && terms[j].Op == TaggedTermEq && !terms[j].HasWildcard) ||\n\t\t\t\t(terms[j].NonDefaultCost && terms[i].Op == TaggedTermEq && !terms[i].HasWildcard) {\n\t\t\t\treturn terms[i].Cost < terms[j].Cost\n\t\t\t}\n\t\t}\n\n\t\tif terms[i].Op == terms[j].Op {\n\t\t\tif terms[i].Op == TaggedTermEq && !terms[i].HasWildcard && terms[j].HasWildcard {\n\t\t\t\t// globs as fist eq might be have a bad perfomance\n\t\t\t\treturn true\n\t\t\t}\n\n\t\t\tif terms[i].Key == \"__name__\" && terms[j].Key != \"__name__\" {\n\t\t\t\treturn true\n\t\t\t}\n\n\t\t\tif terms[i].Cost != terms[j].Cost && terms[i].HasWildcard == terms[j].HasWildcard {\n\t\t\t\t// compare taggs costs\n\t\t\t\treturn terms[i].Cost < terms[j].Cost\n\t\t\t}\n\n\t\t\treturn false\n\t\t} else {\n\t\t\treturn terms[i].Op < terms[j].Op\n\t\t}\n\t})\n}\n\nfunc SetCosts(terms []TaggedTerm, costs map[string]*config.Costs) {\n\tfor i := 0; i < len(terms); i++ {\n\t\tif cost, ok := costs[terms[i].Key]; ok {\n\t\t\tsetCost(&terms[i], cost)\n\t\t}\n\t}\n}\n" }, { "path": "finder/tagged_test.go", "content": "package finder\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net/url\"\n\t\"sort\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/helper/date\"\n\tchtest \"github.com/lomik/graphite-clickhouse/helper/tests/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/where\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestTaggedWhere(t *testing.T) {\n\tassert := assert.New(t)\n\trequire := require.New(t)\n\n\ttable := []struct {\n\t\tquery string\n\t\tminTags int\n\t\twhere string\n\t\tprewhere string\n\t\tisErr bool\n\t}{\n\t\t// test for issue #195\n\t\t{\"seriesByTag()\", 0, \"\", \"\", true},\n\t\t{\"seriesByTag('')\", 0, \"\", \"\", true},\n\t\t// incomplete\n\t\t{\"seriesByTag('key=value)\", 0, \"\", \"\", true},\n\t\t// missing quote\n\t\t{\"seriesByTag(key=value)\", 0, \"\", \"\", true},\n\t\t// info about _tag \"directory\"\n\t\t{\"seriesByTag('key=value')\", 0, \"Tag1='key=value'\", \"\", false},\n\t\t{\"seriesByTag('key=value')\", 1, \"Tag1='key=value'\", \"\", false},\n\t\t{\"seriesByTag('key=value')\", 2, \"\", \"\", true},\n\t\t// test case for wildcarded name, must be not first check\n\t\t{\"seriesByTag('name=*', 'key=value')\", 0, \"(Tag1='key=value') AND (arrayExists((x) -> x LIKE '__name__=%', Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=*', 'key=value')\", 1, \"(Tag1='key=value') AND (arrayExists((x) -> x LIKE '__name__=%', Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=*', 'key=value')\", 2, \"\", \"\", true},\n\t\t{\"seriesByTag('name=*', 'key=value*')\", 0, \"(Tag1 LIKE '__name__=%') AND (arrayExists((x) -> x LIKE 'key=value%', Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=rps')\", 0, \"Tag1='__name__=rps'\", \"\", false},\n\t\t{\"seriesByTag('name=~cpu.usage')\", 0, \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*cpu.usage')\", \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*cpu.usage')\", false},\n\t\t{\"seriesByTag('name=~cpu.usage')\", 1, \"\", \"\", true},\n\t\t{\"seriesByTag('name=~cpu|mem')\", 0, \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem)')\", \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem)')\", false},\n\t\t{\"seriesByTag('name=~cpu|mem$')\", 0, \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem$)')\", \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem$)')\", false},\n\t\t{\"seriesByTag('name=~^cpu|mem')\", 0, \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem)')\", \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem)')\", false},\n\t\t{\"seriesByTag('name=~^cpu|mem$')\", 0, \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem$)')\", \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem$)')\", false},\n\t\t{\"seriesByTag('name=rps', 'key=~value')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=%' AND match(x, '^key=.*value'), Tags))\", \"\", false},\n\t\t// test for issue #244\n\t\t{\"seriesByTag('name=rps', 'key=~^value')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=value%' AND match(x, '^key=value'), Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=rps', 'key=~^value.*')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=value%' AND match(x, '^key=value.*'), Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=rps', 'key=~^valu[a-e]')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=valu%' AND match(x, '^key=valu[a-e]'), Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=rps', 'key=~^value$')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x='key=value', Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=rps', 'key=~hello.world')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=%' AND match(x, '^key=.*hello.world'), Tags))\", \"\", false},\n\t\t{`seriesByTag('cpu=cpu-total','host=~Vladimirs-MacBook-Pro\\.local')`, 0, `(Tag1='cpu=cpu-total') AND (arrayExists((x) -> x LIKE 'host=%' AND match(x, '^host=.*Vladimirs-MacBook-Pro\\\\.local'), Tags))`, \"\", false},\n\t\t// grafana multi-value variable produce this\n\t\t{\"seriesByTag('name=value','what=*')\", 0, \"(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))\", \"\", false}, // If All masked to value with *\n\t\t{\"seriesByTag('name=value','what=*x')\", 0, \"(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%x', Tags))\", \"\", false}, // If All masked to value with *\n\t\t{\"seriesByTag('name=value','what!=*x')\", 0, \"(Tag1='__name__=value') AND (NOT arrayExists((x) -> x LIKE 'what=%x', Tags))\", \"\", false}, // If All masked to value with *\n\t\t{\"seriesByTag('name={avg,max}')\", 0, \"Tag1 IN ('__name__=avg','__name__=max')\", \"\", false},\n\t\t{\"seriesByTag('name=m{in}')\", 0, \"Tag1='__name__=min'\", \"\", false},\n\t\t{\"seriesByTag('name=m{in,ax}')\", 0, \"Tag1 IN ('__name__=min','__name__=max')\", \"\", false},\n\t\t{\"seriesByTag('name=m{in,ax')\", 0, \"\", \"\", true},\n\t\t{\"seriesByTag('name=value','what={avg,max}')\", 0, \"(Tag1='__name__=value') AND ((has(Tags, 'what=avg')) OR (has(Tags, 'what=max')))\", \"\", false},\n\t\t{\"seriesByTag('name=value','what!={avg,max}')\", 0, \"(Tag1='__name__=value') AND (NOT arrayExists((x) -> x IN ('what=avg','what=max'), Tags))\", \"\", false},\n\t\t// grafana workaround for multi-value variables default, masked with *\n\t\t{\"seriesByTag('name=value','what=~*')\", 0, \"(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))\", \"\", false}, // If All masked to value with *\n\t\t// empty tag value during autocompletion\n\t\t{\"seriesByTag('name=value','what=~')\", 0, \"(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))\", \"\", false}, // If All masked to value with *\n\t\t// testcases for useCarbonBehaviour=false\n\t\t{\"seriesByTag('what=')\", 0, \"Tag1='what='\", \"\", false},\n\t\t{\"seriesByTag('name=value','what=')\", 0, \"(Tag1='__name__=value') AND (has(Tags, 'what='))\", \"\", false},\n\t\t// testcases for dontMatchMissingTags=false\n\t\t{\"seriesByTag('key!=value')\", 0, \"NOT arrayExists((x) -> x='key=value', Tags)\", \"\", false},\n\t\t{\"seriesByTag('dc!=de', 'cpu=cpu-total')\", 0, \"(Tag1='cpu=cpu-total') AND (NOT arrayExists((x) -> x='dc=de', Tags))\", \"\", false},\n\t\t{\"seriesByTag('dc!=de', 'cpu!=cpu-total')\", 0, \"(NOT arrayExists((x) -> x='dc=de', Tags)) AND (NOT arrayExists((x) -> x='cpu=cpu-total', Tags))\", \"\", false},\n\t\t{\"seriesByTag('dc!=~de|us', 'cpu=cpu-total')\", 0, \"(Tag1='cpu=cpu-total') AND (NOT arrayExists((x) -> x LIKE 'dc=%' AND match(x, '^dc=.*(de|us)'), Tags))\", \"\", false},\n\t}\n\n\tfor i, test := range table {\n\t\tt.Run(test.query+\"#\"+strconv.Itoa(i), func(t *testing.T) {\n\t\t\ttestName := fmt.Sprintf(\"query: %#v\", test.query)\n\n\t\t\tconfig := config.New()\n\t\t\tconfig.ClickHouse.TagsMinInQuery = test.minTags\n\t\t\tterms, err := ParseSeriesByTag(test.query, config)\n\t\t\tsort.Sort(TaggedTermList(terms))\n\n\t\t\tif test.isErr {\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\n\t\t\trequire.NoError(err, testName+\", err\")\n\n\t\t\tvar w, pw *where.Where\n\t\t\tif err == nil {\n\t\t\t\tw, pw, err = TaggedWhere(terms, false, false)\n\t\t\t}\n\n\t\t\tif test.isErr {\n\t\t\t\trequire.Error(err, testName+\", err\")\n\t\t\t\treturn\n\t\t\t} else {\n\t\t\t\tassert.NoError(err, testName+\", err\")\n\t\t\t}\n\n\t\t\tassert.Equal(test.where, w.String(), testName+\", where\")\n\t\t\tassert.Equal(test.prewhere, pw.String(), testName+\", prewhere\")\n\t\t})\n\t}\n}\n\nfunc TestTaggedWhere_UseCarbonBehaviourFlag(t *testing.T) {\n\tassert := assert.New(t)\n\trequire := require.New(t)\n\n\ttable := []struct {\n\t\tquery string\n\t\tminTags int\n\t\twhere string\n\t\tprewhere string\n\t\tisErr bool\n\t}{\n\t\t// test for issue #195\n\t\t{\"seriesByTag()\", 0, \"\", \"\", true},\n\t\t{\"seriesByTag('')\", 0, \"\", \"\", true},\n\t\t// incomplete\n\t\t{\"seriesByTag('key=value)\", 0, \"\", \"\", true},\n\t\t// missing quote\n\t\t{\"seriesByTag(key=value)\", 0, \"\", \"\", true},\n\t\t// info about _tag \"directory\"\n\t\t{\"seriesByTag('key=value')\", 0, \"Tag1='key=value'\", \"\", false},\n\t\t{\"seriesByTag('key=value')\", 1, \"Tag1='key=value'\", \"\", false},\n\t\t{\"seriesByTag('key=value')\", 2, \"\", \"\", true},\n\t\t// test case for wildcarded name, must be not first check\n\t\t{\"seriesByTag('name=*', 'key=value')\", 0, \"(Tag1='key=value') AND (arrayExists((x) -> x LIKE '__name__=%', Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=*', 'key=value')\", 1, \"(Tag1='key=value') AND (arrayExists((x) -> x LIKE '__name__=%', Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=*', 'key=value')\", 2, \"\", \"\", true},\n\t\t{\"seriesByTag('name=*', 'key=value*')\", 0, \"(Tag1 LIKE '__name__=%') AND (arrayExists((x) -> x LIKE 'key=value%', Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=rps')\", 0, \"Tag1='__name__=rps'\", \"\", false},\n\t\t{\"seriesByTag('name=~cpu.usage')\", 0, \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*cpu.usage')\", \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*cpu.usage')\", false},\n\t\t{\"seriesByTag('name=~cpu.usage')\", 1, \"\", \"\", true},\n\t\t{\"seriesByTag('name=~cpu|mem')\", 0, \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem)')\", \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem)')\", false},\n\t\t{\"seriesByTag('name=~cpu|mem$')\", 0, \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem$)')\", \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem$)')\", false},\n\t\t{\"seriesByTag('name=~^cpu|mem')\", 0, \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem)')\", \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem)')\", false},\n\t\t{\"seriesByTag('name=~^cpu|mem$')\", 0, \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem$)')\", \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem$)')\", false},\n\t\t{\"seriesByTag('name=rps', 'key=~value')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=%' AND match(x, '^key=.*value'), Tags))\", \"\", false},\n\t\t// test for issue #244\n\t\t{\"seriesByTag('name=rps', 'key=~^value')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=value%' AND match(x, '^key=value'), Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=rps', 'key=~^value.*')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=value%' AND match(x, '^key=value.*'), Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=rps', 'key=~^valu[a-e]')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=valu%' AND match(x, '^key=valu[a-e]'), Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=rps', 'key=~^value$')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x='key=value', Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=rps', 'key=~hello.world')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=%' AND match(x, '^key=.*hello.world'), Tags))\", \"\", false},\n\t\t{`seriesByTag('cpu=cpu-total','host=~Vladimirs-MacBook-Pro\\.local')`, 0, `(Tag1='cpu=cpu-total') AND (arrayExists((x) -> x LIKE 'host=%' AND match(x, '^host=.*Vladimirs-MacBook-Pro\\\\.local'), Tags))`, \"\", false},\n\t\t// grafana multi-value variable produce this\n\t\t{\"seriesByTag('name=value','what=*')\", 0, \"(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))\", \"\", false}, // If All masked to value with *\n\t\t{\"seriesByTag('name=value','what=*x')\", 0, \"(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%x', Tags))\", \"\", false}, // If All masked to value with *\n\t\t{\"seriesByTag('name=value','what!=*x')\", 0, \"(Tag1='__name__=value') AND (NOT arrayExists((x) -> x LIKE 'what=%x', Tags))\", \"\", false}, // If All masked to value with *\n\t\t{\"seriesByTag('name={avg,max}')\", 0, \"Tag1 IN ('__name__=avg','__name__=max')\", \"\", false},\n\t\t{\"seriesByTag('name=m{in}')\", 0, \"Tag1='__name__=min'\", \"\", false},\n\t\t{\"seriesByTag('name=m{in,ax}')\", 0, \"Tag1 IN ('__name__=min','__name__=max')\", \"\", false},\n\t\t{\"seriesByTag('name=m{in,ax')\", 0, \"\", \"\", true},\n\t\t{\"seriesByTag('name=value','what={avg,max}')\", 0, \"(Tag1='__name__=value') AND ((has(Tags, 'what=avg')) OR (has(Tags, 'what=max')))\", \"\", false},\n\t\t{\"seriesByTag('name=value','what!={avg,max}')\", 0, \"(Tag1='__name__=value') AND (NOT arrayExists((x) -> x IN ('what=avg','what=max'), Tags))\", \"\", false},\n\t\t// grafana workaround for multi-value variables default, masked with *\n\t\t{\"seriesByTag('name=value','what=~*')\", 0, \"(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))\", \"\", false}, // If All masked to value with *\n\t\t// empty tag value during autocompletion\n\t\t{\"seriesByTag('name=value','what=~')\", 0, \"(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))\", \"\", false}, // If All masked to value with *\n\t\t// testcases for useCarbonBehaviour=true\n\t\t{\"seriesByTag('what=')\", 0, \"NOT arrayExists((x) -> x LIKE 'what=%', Tags)\", \"\", false},\n\t\t{\"seriesByTag('name=value','what=')\", 0, \"(Tag1='__name__=value') AND (NOT arrayExists((x) -> x LIKE 'what=%', Tags))\", \"\", false},\n\t\t// testcases for dontMatchMissingTags=false\n\t\t{\"seriesByTag('key!=value')\", 0, \"NOT arrayExists((x) -> x='key=value', Tags)\", \"\", false},\n\t\t{\"seriesByTag('dc!=de', 'cpu=cpu-total')\", 0, \"(Tag1='cpu=cpu-total') AND (NOT arrayExists((x) -> x='dc=de', Tags))\", \"\", false},\n\t\t{\"seriesByTag('dc!=de', 'cpu!=cpu-total')\", 0, \"(NOT arrayExists((x) -> x='dc=de', Tags)) AND (NOT arrayExists((x) -> x='cpu=cpu-total', Tags))\", \"\", false},\n\t\t{\"seriesByTag('dc!=~de|us', 'cpu=cpu-total')\", 0, \"(Tag1='cpu=cpu-total') AND (NOT arrayExists((x) -> x LIKE 'dc=%' AND match(x, '^dc=.*(de|us)'), Tags))\", \"\", false},\n\t}\n\n\tfor i, test := range table {\n\t\tt.Run(test.query+\"#\"+strconv.Itoa(i), func(t *testing.T) {\n\t\t\ttestName := fmt.Sprintf(\"query: %#v\", test.query)\n\n\t\t\tconfig := config.New()\n\t\t\tconfig.ClickHouse.TagsMinInQuery = test.minTags\n\t\t\tterms, err := ParseSeriesByTag(test.query, config)\n\t\t\tsort.Sort(TaggedTermList(terms))\n\n\t\t\tif test.isErr {\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\n\t\t\trequire.NoError(err, testName+\", err\")\n\n\t\t\tvar w, pw *where.Where\n\t\t\tif err == nil {\n\t\t\t\tw, pw, err = TaggedWhere(terms, true, false)\n\t\t\t}\n\n\t\t\tif test.isErr {\n\t\t\t\trequire.Error(err, testName+\", err\")\n\t\t\t\treturn\n\t\t\t} else {\n\t\t\t\tassert.NoError(err, testName+\", err\")\n\t\t\t}\n\n\t\t\tassert.Equal(test.where, w.String(), testName+\", where\")\n\t\t\tassert.Equal(test.prewhere, pw.String(), testName+\", prewhere\")\n\t\t})\n\t}\n}\n\nfunc TestTaggedWhere_DontMatchMissingTagsFlag(t *testing.T) {\n\tassert := assert.New(t)\n\trequire := require.New(t)\n\n\ttable := []struct {\n\t\tquery string\n\t\tminTags int\n\t\twhere string\n\t\tprewhere string\n\t\tisErr bool\n\t}{\n\t\t// test for issue #195\n\t\t{\"seriesByTag()\", 0, \"\", \"\", true},\n\t\t{\"seriesByTag('')\", 0, \"\", \"\", true},\n\t\t// incomplete\n\t\t{\"seriesByTag('key=value)\", 0, \"\", \"\", true},\n\t\t// missing quote\n\t\t{\"seriesByTag(key=value)\", 0, \"\", \"\", true},\n\t\t// info about _tag \"directory\"\n\t\t{\"seriesByTag('key=value')\", 0, \"Tag1='key=value'\", \"\", false},\n\t\t{\"seriesByTag('key=value')\", 1, \"Tag1='key=value'\", \"\", false},\n\t\t{\"seriesByTag('key=value')\", 2, \"\", \"\", true},\n\t\t// test case for wildcarded name, must be not first check\n\t\t{\"seriesByTag('name=*', 'key=value')\", 0, \"(Tag1='key=value') AND (arrayExists((x) -> x LIKE '__name__=%', Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=*', 'key=value')\", 1, \"(Tag1='key=value') AND (arrayExists((x) -> x LIKE '__name__=%', Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=*', 'key=value')\", 2, \"\", \"\", true},\n\t\t{\"seriesByTag('name=*', 'key=value*')\", 0, \"(Tag1 LIKE '__name__=%') AND (arrayExists((x) -> x LIKE 'key=value%', Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=rps')\", 0, \"Tag1='__name__=rps'\", \"\", false},\n\t\t{\"seriesByTag('name=~cpu.usage')\", 0, \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*cpu.usage')\", \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*cpu.usage')\", false},\n\t\t{\"seriesByTag('name=~cpu.usage')\", 1, \"\", \"\", true},\n\t\t{\"seriesByTag('name=~cpu|mem')\", 0, \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem)')\", \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem)')\", false},\n\t\t{\"seriesByTag('name=~cpu|mem$')\", 0, \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem$)')\", \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem$)')\", false},\n\t\t{\"seriesByTag('name=~^cpu|mem')\", 0, \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem)')\", \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem)')\", false},\n\t\t{\"seriesByTag('name=~^cpu|mem$')\", 0, \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem$)')\", \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem$)')\", false},\n\t\t{\"seriesByTag('name=rps', 'key=~value')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=%' AND match(x, '^key=.*value'), Tags))\", \"\", false},\n\t\t// test for issue #244\n\t\t{\"seriesByTag('name=rps', 'key=~^value')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=value%' AND match(x, '^key=value'), Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=rps', 'key=~^value.*')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=value%' AND match(x, '^key=value.*'), Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=rps', 'key=~^valu[a-e]')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=valu%' AND match(x, '^key=valu[a-e]'), Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=rps', 'key=~^value$')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x='key=value', Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=rps', 'key=~hello.world')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=%' AND match(x, '^key=.*hello.world'), Tags))\", \"\", false},\n\t\t{`seriesByTag('cpu=cpu-total','host=~Vladimirs-MacBook-Pro\\.local')`, 0, `(Tag1='cpu=cpu-total') AND (arrayExists((x) -> x LIKE 'host=%' AND match(x, '^host=.*Vladimirs-MacBook-Pro\\\\.local'), Tags))`, \"\", false},\n\t\t// grafana multi-value variable produce this\n\t\t{\"seriesByTag('name=value','what=*')\", 0, \"(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))\", \"\", false}, // If All masked to value with *\n\t\t{\"seriesByTag('name=value','what=*x')\", 0, \"(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%x', Tags))\", \"\", false}, // If All masked to value with *\n\t\t// {\"seriesByTag('name=value','what!=*x')\", 0, \"(Tag1='__name__=value') AND (NOT arrayExists((x) -> x LIKE 'what=%x', Tags))\", \"\", false}, // If All masked to value with *\n\t\t{\"seriesByTag('name=value','what!=*x')\", 0, \"(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags) AND NOT arrayExists((x) -> x LIKE 'what=%x', Tags))\", \"\", false},\n\t\t{\"seriesByTag('name={avg,max}')\", 0, \"Tag1 IN ('__name__=avg','__name__=max')\", \"\", false},\n\t\t{\"seriesByTag('name=m{in}')\", 0, \"Tag1='__name__=min'\", \"\", false},\n\t\t{\"seriesByTag('name=m{in,ax}')\", 0, \"Tag1 IN ('__name__=min','__name__=max')\", \"\", false},\n\t\t{\"seriesByTag('name=m{in,ax')\", 0, \"\", \"\", true},\n\t\t{\"seriesByTag('name=value','what={avg,max}')\", 0, \"(Tag1='__name__=value') AND ((has(Tags, 'what=avg')) OR (has(Tags, 'what=max')))\", \"\", false},\n\t\t// {\"seriesByTag('name=value','what!={avg,max}')\", 0, \"(Tag1='__name__=value') AND (NOT arrayExists((x) -> x IN ('what=avg','what=max'), Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=value','what!={avg,max}')\", 0, \"(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags) AND NOT arrayExists((x) -> x IN ('what=avg','what=max'), Tags))\", \"\", false},\n\t\t// grafana workaround for multi-value variables default, masked with *\n\t\t{\"seriesByTag('name=value','what=~*')\", 0, \"(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))\", \"\", false}, // If All masked to value with *\n\t\t// empty tag value during autocompletion\n\t\t{\"seriesByTag('name=value','what=~')\", 0, \"(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))\", \"\", false}, // If All masked to value with *\n\t\t// testcases for useCarbonBehaviour=false\n\t\t{\"seriesByTag('what=')\", 0, \"Tag1='what='\", \"\", false},\n\t\t{\"seriesByTag('name=value','what=')\", 0, \"(Tag1='__name__=value') AND (has(Tags, 'what='))\", \"\", false},\n\t\t// testcases for dontMatchMissingTags=true\n\t\t{\"seriesByTag('key!=value')\", 0, \"Tag1 LIKE 'key=%' AND NOT arrayExists((x) -> x='key=value', Tags)\", \"\", false},\n\t\t{\"seriesByTag('dc!=de', 'cpu=cpu-total')\", 0, \"(Tag1='cpu=cpu-total') AND (arrayExists((x) -> x LIKE 'dc=%', Tags) AND NOT arrayExists((x) -> x='dc=de', Tags))\", \"\", false},\n\t\t{\"seriesByTag('dc!=de', 'cpu!=cpu-total')\", 0, \"(Tag1 LIKE 'dc=%' AND NOT arrayExists((x) -> x='dc=de', Tags)) AND (arrayExists((x) -> x LIKE 'cpu=%', Tags) AND NOT arrayExists((x) -> x='cpu=cpu-total', Tags))\", \"\", false},\n\t\t{\"seriesByTag('dc!=~de|us', 'cpu=cpu-total')\", 0, \"(Tag1='cpu=cpu-total') AND (arrayExists((x) -> x LIKE 'dc=%', Tags) AND NOT arrayExists((x) -> x LIKE 'dc=%' AND match(x, '^dc=.*(de|us)'), Tags))\", \"\", false},\n\t}\n\n\tfor i, test := range table {\n\t\tt.Run(test.query+\"#\"+strconv.Itoa(i), func(t *testing.T) {\n\t\t\ttestName := fmt.Sprintf(\"query: %#v\", test.query)\n\n\t\t\tconfig := config.New()\n\t\t\tconfig.ClickHouse.TagsMinInQuery = test.minTags\n\t\t\tterms, err := ParseSeriesByTag(test.query, config)\n\t\t\tsort.Sort(TaggedTermList(terms))\n\n\t\t\tif test.isErr {\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\n\t\t\trequire.NoError(err, testName+\", err\")\n\n\t\t\tvar w, pw *where.Where\n\t\t\tif err == nil {\n\t\t\t\tw, pw, err = TaggedWhere(terms, false, true)\n\t\t\t}\n\n\t\t\tif test.isErr {\n\t\t\t\trequire.Error(err, testName+\", err\")\n\t\t\t\treturn\n\t\t\t} else {\n\t\t\t\tassert.NoError(err, testName+\", err\")\n\t\t\t}\n\n\t\t\tassert.Equal(test.where, w.String(), testName+\", where\")\n\t\t\tassert.Equal(test.prewhere, pw.String(), testName+\", prewhere\")\n\t\t})\n\t}\n}\n\nfunc TestTaggedWhere_BothFeatureFlags(t *testing.T) {\n\tassert := assert.New(t)\n\trequire := require.New(t)\n\n\ttable := []struct {\n\t\tquery string\n\t\tminTags int\n\t\twhere string\n\t\tprewhere string\n\t\tisErr bool\n\t}{\n\t\t// test for issue #195\n\t\t{\"seriesByTag()\", 0, \"\", \"\", true},\n\t\t{\"seriesByTag('')\", 0, \"\", \"\", true},\n\t\t// incomplete\n\t\t{\"seriesByTag('key=value)\", 0, \"\", \"\", true},\n\t\t// missing quote\n\t\t{\"seriesByTag(key=value)\", 0, \"\", \"\", true},\n\t\t// info about _tag \"directory\"\n\t\t{\"seriesByTag('key=value')\", 0, \"Tag1='key=value'\", \"\", false},\n\t\t{\"seriesByTag('key=value')\", 1, \"Tag1='key=value'\", \"\", false},\n\t\t{\"seriesByTag('key=value')\", 2, \"\", \"\", true},\n\t\t// test case for wildcarded name, must be not first check\n\t\t{\"seriesByTag('name=*', 'key=value')\", 0, \"(Tag1='key=value') AND (arrayExists((x) -> x LIKE '__name__=%', Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=*', 'key=value')\", 1, \"(Tag1='key=value') AND (arrayExists((x) -> x LIKE '__name__=%', Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=*', 'key=value')\", 2, \"\", \"\", true},\n\t\t{\"seriesByTag('name=*', 'key=value*')\", 0, \"(Tag1 LIKE '__name__=%') AND (arrayExists((x) -> x LIKE 'key=value%', Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=rps')\", 0, \"Tag1='__name__=rps'\", \"\", false},\n\t\t{\"seriesByTag('name=~cpu.usage')\", 0, \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*cpu.usage')\", \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*cpu.usage')\", false},\n\t\t{\"seriesByTag('name=~cpu.usage')\", 1, \"\", \"\", true},\n\t\t{\"seriesByTag('name=~cpu|mem')\", 0, \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem)')\", \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem)')\", false},\n\t\t{\"seriesByTag('name=~cpu|mem$')\", 0, \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem$)')\", \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem$)')\", false},\n\t\t{\"seriesByTag('name=~^cpu|mem')\", 0, \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem)')\", \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem)')\", false},\n\t\t{\"seriesByTag('name=~^cpu|mem$')\", 0, \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem$)')\", \"Tag1 LIKE '\\\\\\\\_\\\\\\\\_name\\\\\\\\_\\\\\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem$)')\", false},\n\t\t{\"seriesByTag('name=rps', 'key=~value')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=%' AND match(x, '^key=.*value'), Tags))\", \"\", false},\n\t\t// test for issue #244\n\t\t{\"seriesByTag('name=rps', 'key=~^value')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=value%' AND match(x, '^key=value'), Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=rps', 'key=~^value.*')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=value%' AND match(x, '^key=value.*'), Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=rps', 'key=~^valu[a-e]')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=valu%' AND match(x, '^key=valu[a-e]'), Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=rps', 'key=~^value$')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x='key=value', Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=rps', 'key=~hello.world')\", 0, \"(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=%' AND match(x, '^key=.*hello.world'), Tags))\", \"\", false},\n\t\t{`seriesByTag('cpu=cpu-total','host=~Vladimirs-MacBook-Pro\\.local')`, 0, `(Tag1='cpu=cpu-total') AND (arrayExists((x) -> x LIKE 'host=%' AND match(x, '^host=.*Vladimirs-MacBook-Pro\\\\.local'), Tags))`, \"\", false},\n\t\t// grafana multi-value variable produce this\n\t\t{\"seriesByTag('name=value','what=*')\", 0, \"(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))\", \"\", false}, // If All masked to value with *\n\t\t{\"seriesByTag('name=value','what=*x')\", 0, \"(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%x', Tags))\", \"\", false}, // If All masked to value with *\n\t\t// {\"seriesByTag('name=value','what!=*x')\", 0, \"(Tag1='__name__=value') AND (NOT arrayExists((x) -> x LIKE 'what=%x', Tags))\", \"\", false}, // If All masked to value with *\n\t\t{\"seriesByTag('name=value','what!=*x')\", 0, \"(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags) AND NOT arrayExists((x) -> x LIKE 'what=%x', Tags))\", \"\", false},\n\t\t{\"seriesByTag('name={avg,max}')\", 0, \"Tag1 IN ('__name__=avg','__name__=max')\", \"\", false},\n\t\t{\"seriesByTag('name=m{in}')\", 0, \"Tag1='__name__=min'\", \"\", false},\n\t\t{\"seriesByTag('name=m{in,ax}')\", 0, \"Tag1 IN ('__name__=min','__name__=max')\", \"\", false},\n\t\t{\"seriesByTag('name=m{in,ax')\", 0, \"\", \"\", true},\n\t\t{\"seriesByTag('name=value','what={avg,max}')\", 0, \"(Tag1='__name__=value') AND ((has(Tags, 'what=avg')) OR (has(Tags, 'what=max')))\", \"\", false},\n\t\t// {\"seriesByTag('name=value','what!={avg,max}')\", 0, \"(Tag1='__name__=value') AND (NOT arrayExists((x) -> x IN ('what=avg','what=max'), Tags))\", \"\", false},\n\t\t{\"seriesByTag('name=value','what!={avg,max}')\", 0, \"(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags) AND NOT arrayExists((x) -> x IN ('what=avg','what=max'), Tags))\", \"\", false},\n\t\t// grafana workaround for multi-value variables default, masked with *\n\t\t{\"seriesByTag('name=value','what=~*')\", 0, \"(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))\", \"\", false}, // If All masked to value with *\n\t\t// empty tag value during autocompletion\n\t\t{\"seriesByTag('name=value','what=~')\", 0, \"(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))\", \"\", false}, // If All masked to value with *\n\t\t// testcases for useCarbonBehaviour=true\n\t\t{\"seriesByTag('what=')\", 0, \"NOT arrayExists((x) -> x LIKE 'what=%', Tags)\", \"\", false},\n\t\t{\"seriesByTag('name=value','what=')\", 0, \"(Tag1='__name__=value') AND (NOT arrayExists((x) -> x LIKE 'what=%', Tags))\", \"\", false},\n\t\t// testcases for dontMatchMissingTags=true\n\t\t{\"seriesByTag('key!=value')\", 0, \"Tag1 LIKE 'key=%' AND NOT arrayExists((x) -> x='key=value', Tags)\", \"\", false},\n\t\t{\"seriesByTag('dc!=de', 'cpu=cpu-total')\", 0, \"(Tag1='cpu=cpu-total') AND (arrayExists((x) -> x LIKE 'dc=%', Tags) AND NOT arrayExists((x) -> x='dc=de', Tags))\", \"\", false},\n\t\t{\"seriesByTag('dc!=de', 'cpu!=cpu-total')\", 0, \"(Tag1 LIKE 'dc=%' AND NOT arrayExists((x) -> x='dc=de', Tags)) AND (arrayExists((x) -> x LIKE 'cpu=%', Tags) AND NOT arrayExists((x) -> x='cpu=cpu-total', Tags))\", \"\", false},\n\t\t{\"seriesByTag('dc!=~de|us', 'cpu=cpu-total')\", 0, \"(Tag1='cpu=cpu-total') AND (arrayExists((x) -> x LIKE 'dc=%', Tags) AND NOT arrayExists((x) -> x LIKE 'dc=%' AND match(x, '^dc=.*(de|us)'), Tags))\", \"\", false},\n\t}\n\n\tfor i, test := range table {\n\t\tt.Run(test.query+\"#\"+strconv.Itoa(i), func(t *testing.T) {\n\t\t\ttestName := fmt.Sprintf(\"query: %#v\", test.query)\n\n\t\t\tconfig := config.New()\n\t\t\tconfig.ClickHouse.TagsMinInQuery = test.minTags\n\t\t\tterms, err := ParseSeriesByTag(test.query, config)\n\t\t\tsort.Sort(TaggedTermList(terms))\n\n\t\t\tif test.isErr {\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\n\t\t\trequire.NoError(err, testName+\", err\")\n\n\t\t\tvar w, pw *where.Where\n\t\t\tif err == nil {\n\t\t\t\tw, pw, err = TaggedWhere(terms, true, true)\n\t\t\t}\n\n\t\t\tif test.isErr {\n\t\t\t\trequire.Error(err, testName+\", err\")\n\t\t\t\treturn\n\t\t\t} else {\n\t\t\t\tassert.NoError(err, testName+\", err\")\n\t\t\t}\n\n\t\t\tassert.Equal(test.where, w.String(), testName+\", where\")\n\t\t\tassert.Equal(test.prewhere, pw.String(), testName+\", prewhere\")\n\t\t})\n\t}\n}\n\nfunc TestParseSeriesByTag(t *testing.T) {\n\tassert := assert.New(t)\n\n\tok := func(query string, expected []TaggedTerm) {\n\t\tconfig := config.New()\n\t\tp, err := ParseSeriesByTag(query, config)\n\t\tassert.NoError(err)\n\t\tassert.Equal(len(expected), len(p))\n\n\t\tlength := len(expected)\n\t\tif length < len(p) {\n\t\t\tlength = len(p)\n\t\t}\n\n\t\tfor i := 0; i < length; i++ {\n\t\t\tif i >= len(p) {\n\t\t\t\tt.Errorf(\"%s\\n- [%d]=%+v\", query, i, expected[i])\n\t\t\t} else if i >= len(expected) {\n\t\t\t\tt.Errorf(\"%s\\n+ [%d]=%+v\", query, i, p[i])\n\t\t\t} else if p[i] != expected[i] {\n\t\t\t\tt.Errorf(\"%s\\n- [%d]=%+v\\n+ [%d]=%+v\", query, i, expected[i], i, p[i])\n\t\t\t}\n\t\t}\n\t}\n\n\tok(`seriesByTag('key=value')`, []TaggedTerm{\n\t\t{Op: TaggedTermEq, Key: \"key\", Value: \"value\"},\n\t})\n\n\tok(`seriesByTag('name=rps')`, []TaggedTerm{\n\t\t{Op: TaggedTermEq, Key: \"__name__\", Value: \"rps\"},\n\t})\n\n\tok(`seriesByTag('name=~cpu.usage')`, []TaggedTerm{\n\t\t{Op: TaggedTermMatch, Key: \"__name__\", Value: \"cpu.usage\"},\n\t})\n\n\tok(`seriesByTag('name!=cpu.usage')`, []TaggedTerm{\n\t\t{Op: TaggedTermNe, Key: \"__name__\", Value: \"cpu.usage\"},\n\t})\n\n\tok(`seriesByTag('name!=~cpu.usage')`, []TaggedTerm{\n\t\t{Op: TaggedTermNotMatch, Key: \"__name__\", Value: \"cpu.usage\"},\n\t})\n\n\tok(`seriesByTag('cpu=cpu-total','host=~Vladimirs-MacBook-Pro\\.local')`, []TaggedTerm{\n\t\t{Op: TaggedTermEq, Key: \"cpu\", Value: \"cpu-total\"},\n\t\t{Op: TaggedTermMatch, Key: \"host\", Value: `Vladimirs-MacBook-Pro\\.local`},\n\t})\n}\n\nfunc newInt(i int) *int {\n\tp := new(int)\n\t*p = i\n\n\treturn p\n}\n\nfunc TestParseSeriesByTagWithCosts(t *testing.T) {\n\tassert := assert.New(t)\n\n\ttaggedCosts := map[string]*config.Costs{\n\t\t\"environment\": {Cost: newInt(100)},\n\t\t\"dc\": {Cost: newInt(60)},\n\t\t\"project\": {Cost: newInt(50)},\n\t\t\"__name__\": {Cost: newInt(0), ValuesCost: map[string]int{\"high_cost\": 70}},\n\t\t\"key\": {ValuesCost: map[string]int{\"value2\": 70, \"value3\": -1, \"val*4\": -1, \"^val.*4$\": -1}},\n\t}\n\n\tok := func(query string, expected []TaggedTerm) {\n\t\tconfig := config.New()\n\t\tconfig.ClickHouse.TaggedCosts = taggedCosts\n\t\tterms, err := ParseSeriesByTag(query, config)\n\t\tSetCosts(terms, config.ClickHouse.TaggedCosts)\n\t\tSortTaggedTermsByCost(terms)\n\t\tassert.NoError(err)\n\n\t\tlength := len(expected)\n\t\tif length < len(terms) {\n\t\t\tlength = len(terms)\n\t\t}\n\n\t\tfor i := 0; i < length; i++ {\n\t\t\tif i >= len(terms) {\n\t\t\t\tt.Errorf(\"%s\\n- [%d]=%+v\", query, i, expected[i])\n\t\t\t} else if i >= len(expected) {\n\t\t\t\tt.Errorf(\"%s\\n+ [%d]=%+v\", query, i, terms[i])\n\t\t\t} else if terms[i] != expected[i] {\n\t\t\t\tt.Errorf(\"%s\\n- [%d]=%+v\\n+ [%d]=%+v\", query, i, expected[i], i, terms[i])\n\t\t\t}\n\t\t}\n\t}\n\n\tok(`seriesByTag('environment=production', 'dc=west', 'key=value')`, []TaggedTerm{\n\t\t{Op: TaggedTermEq, Key: \"key\", Value: \"value\"},\n\t\t{Op: TaggedTermEq, Key: \"dc\", Value: \"west\", Cost: 60, NonDefaultCost: true},\n\t\t{Op: TaggedTermEq, Key: \"environment\", Value: \"production\", Cost: 100, NonDefaultCost: true},\n\t})\n\n\t// Check for values cost (key=value2)\n\tok(`seriesByTag('environment=production', 'dc=west', 'key=value2')`, []TaggedTerm{\n\t\t{Op: TaggedTermEq, Key: \"dc\", Value: \"west\", Cost: 60, NonDefaultCost: true},\n\t\t{Op: TaggedTermEq, Key: \"key\", Value: \"value2\", Cost: 70, NonDefaultCost: true},\n\t\t{Op: TaggedTermEq, Key: \"environment\", Value: \"production\", Cost: 100, NonDefaultCost: true},\n\t})\n\n\t// Check for __name_ preference\n\tok(`seriesByTag('environment=production', 'dc=west', 'key=value', 'name=cpu.load_avg')`, []TaggedTerm{\n\t\t{Op: TaggedTermEq, Key: \"__name__\", Value: \"cpu.load_avg\", Cost: 0, NonDefaultCost: true},\n\t\t{Op: TaggedTermEq, Key: \"key\", Value: \"value\"},\n\t\t{Op: TaggedTermEq, Key: \"dc\", Value: \"west\", Cost: 60, NonDefaultCost: true},\n\t\t{Op: TaggedTermEq, Key: \"environment\", Value: \"production\", Cost: 100, NonDefaultCost: true},\n\t})\n\n\t// Check for __name_ preference overrided\n\tok(`seriesByTag('environment=production', 'dc=west', 'name=cpu.load_avg', 'key=value3')`, []TaggedTerm{\n\t\t{Op: TaggedTermEq, Key: \"key\", Value: \"value3\", Cost: -1, NonDefaultCost: true},\n\t\t{Op: TaggedTermEq, Key: \"__name__\", Value: \"cpu.load_avg\", Cost: 0, NonDefaultCost: true},\n\t\t{Op: TaggedTermEq, Key: \"dc\", Value: \"west\", Cost: 60, NonDefaultCost: true},\n\t\t{Op: TaggedTermEq, Key: \"environment\", Value: \"production\", Cost: 100, NonDefaultCost: true},\n\t})\n\n\t// wildcard (dc=west*)\n\tok(`seriesByTag('environment=production', 'dc=west*', 'name=cpu.load_avg', 'key=value3')`, []TaggedTerm{\n\t\t{Op: TaggedTermEq, Key: \"key\", Value: \"value3\", Cost: -1, NonDefaultCost: true},\n\t\t{Op: TaggedTermEq, Key: \"__name__\", Value: \"cpu.load_avg\", Cost: 0, NonDefaultCost: true},\n\t\t{Op: TaggedTermEq, Key: \"environment\", Value: \"production\", Cost: 100, NonDefaultCost: true},\n\t\t{Op: TaggedTermEq, Key: \"dc\", Value: \"west*\", HasWildcard: true},\n\t})\n\n\t// wildcard cost -1\n\tok(`seriesByTag('dc=west*', 'environment=production', 'name=cpu.load_avg', 'key=val*4')`, []TaggedTerm{\n\t\t{Op: TaggedTermEq, Key: \"key\", Value: \"val*4\", Cost: -1, HasWildcard: true, NonDefaultCost: true},\n\t\t{Op: TaggedTermEq, Key: \"__name__\", Value: \"cpu.load_avg\", Cost: 0, NonDefaultCost: true},\n\t\t{Op: TaggedTermEq, Key: \"environment\", Value: \"production\", Cost: 100, NonDefaultCost: true},\n\t\t{Op: TaggedTermEq, Key: \"dc\", Value: \"west*\", HasWildcard: true},\n\t})\n\n\t// match cost -1 - not as wildcard\n\tok(`seriesByTag('dc=~west.*', 'environment=production', 'name=cpu.load_avg', 'key=~^val.*4$')`, []TaggedTerm{\n\t\t{Op: TaggedTermMatch, Key: \"key\", Value: \"^val.*4$\", Cost: -1, NonDefaultCost: true},\n\t\t{Op: TaggedTermEq, Key: \"__name__\", Value: \"cpu.load_avg\", Cost: 0, NonDefaultCost: true},\n\t\t{Op: TaggedTermEq, Key: \"environment\", Value: \"production\", Cost: 100, NonDefaultCost: true},\n\t\t{Op: TaggedTermMatch, Key: \"dc\", Value: \"west.*\"},\n\t})\n\n\t// match cost -1 - and no cost\n\tok(`seriesByTag('dc=~west.*', 'environment=production', 'Name=cpu.load_avg', 'key=~^val.*4$')`, []TaggedTerm{\n\t\t{Op: TaggedTermMatch, Key: \"key\", Value: \"^val.*4$\", Cost: -1, NonDefaultCost: true},\n\t\t{Op: TaggedTermEq, Key: \"Name\", Value: \"cpu.load_avg\"},\n\t\t{Op: TaggedTermEq, Key: \"environment\", Value: \"production\", Cost: 100, NonDefaultCost: true},\n\t\t{Op: TaggedTermMatch, Key: \"dc\", Value: \"west.*\"},\n\t})\n\n\t// reduce cost for __name__\n\tok(`seriesByTag('dc=~west.*', 'environment=production', 'name=high_cost', 'key=~^val.*4$', 'key2=~^val.*4$', 'key3=val.*4')`, []TaggedTerm{\n\t\t{Op: TaggedTermMatch, Key: \"key\", Value: \"^val.*4$\", Cost: -1, NonDefaultCost: true},\n\t\t{Op: TaggedTermEq, Key: \"__name__\", Value: \"high_cost\", Cost: 70, NonDefaultCost: true},\n\t\t{Op: TaggedTermEq, Key: \"environment\", Value: \"production\", Cost: 100, NonDefaultCost: true},\n\t\t{Op: TaggedTermEq, Key: \"key3\", Value: \"val.*4\", HasWildcard: true},\n\t\t{Op: TaggedTermMatch, Key: \"dc\", Value: \"west.*\"},\n\t\t{Op: TaggedTermMatch, Key: \"key2\", Value: \"^val.*4$\"},\n\t})\n}\n\nfunc BenchmarkParseSeriesByTag(b *testing.B) {\n\tbenchmarks := []string{\n\t\t\"seriesByTag('key=value')\",\n\t\t\"seriesByTag('name=*', 'key=value')\",\n\t\t\"seriesByTag('name=value', '')\",\n\t}\n\tfor _, bm := range benchmarks {\n\t\tb.Run(bm, func(b *testing.B) {\n\t\t\tfor i := 0; i < b.N; i++ {\n\t\t\t\t_, _ = ParseSeriesByTag(bm, nil)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestParseSeriesByTagWithCostsFromCountTable(t *testing.T) {\n\tassert := assert.New(t)\n\n\tvar from, until int64\n\t// 2022-11-11 00:01:00 +05:00 && 2022-11-11 00:01:10 +05:00\n\tfrom, until = 1668106860, 1668106870\n\n\ttaggedCosts := map[string]*config.Costs{\n\t\t\"environment\": {Cost: newInt(100)},\n\t\t\"dc\": {Cost: newInt(60)},\n\t\t\"project\": {Cost: newInt(50)},\n\t\t\"__name__\": {Cost: newInt(0), ValuesCost: map[string]int{\"high_cost\": 70}},\n\t\t\"key\": {ValuesCost: map[string]int{\"value2\": 70, \"value3\": -1, \"val*4\": -1, \"^val.*4$\": -1}},\n\t}\n\n\tok := func(\n\t\ttestName, query, sql string,\n\t\tresponse *chtest.TestResponse,\n\t\texpected []TaggedTerm,\n\t\texpectedErr error,\n\t\tuseTagCostsFromConfig bool,\n\t) {\n\t\tsrv := chtest.NewTestServer()\n\t\tdefer srv.Close()\n\n\t\tcfg, _ := config.DefaultConfig()\n\t\tcfg.ClickHouse.URL = srv.URL\n\n\t\tif useTagCostsFromConfig {\n\t\t\tcfg.ClickHouse.TaggedCosts = taggedCosts\n\t\t}\n\n\t\tsrv.AddResponce(sql, response)\n\n\t\topts := clickhouse.Options{\n\t\t\tTimeout: cfg.ClickHouse.IndexTimeout,\n\t\t\tConnectTimeout: cfg.ClickHouse.ConnectTimeout,\n\t\t\tTLSConfig: cfg.ClickHouse.TLSConfig,\n\t\t\tCheckRequestProgress: cfg.FeatureFlags.LogQueryProgress,\n\t\t\tProgressSendingInterval: cfg.ClickHouse.ProgressSendingInterval,\n\t\t}\n\n\t\ttaggedFinder := NewTagged(\n\t\t\tcfg.ClickHouse.URL,\n\t\t\tcfg.ClickHouse.TaggedTable,\n\t\t\t\"tag1_count_table\", // non-empty string, tagged finder will query clickhouse for costs\n\t\t\ttrue,\n\t\t\tcfg.FeatureFlags.UseCarbonBehavior,\n\t\t\tcfg.FeatureFlags.DontMatchMissingTags,\n\t\t\tfalse,\n\t\t\topts,\n\t\t\tcfg.ClickHouse.TaggedCosts,\n\t\t)\n\n\t\tterms, err := taggedFinder.PrepareTaggedTerms(context.Background(), cfg, query, from, until)\n\t\tif expectedErr != nil {\n\t\t\tassert.Equal(expectedErr, err, testName+\", err\")\n\t\t\treturn\n\t\t}\n\n\t\tassert.NoError(err)\n\n\t\tlength := len(expected)\n\t\tif length < len(terms) {\n\t\t\tlength = len(terms)\n\t\t}\n\n\t\tfor i := 0; i < length; i++ {\n\t\t\tif i >= len(terms) {\n\t\t\t\tt.Errorf(\"%s\\n- [%d]=%+v\", testName, i, expected[i])\n\t\t\t} else if i >= len(expected) {\n\t\t\t\tt.Errorf(\"%s\\n+ [%d]=%+v\", testName, i, terms[i])\n\t\t\t} else if terms[i] != expected[i] {\n\t\t\t\tt.Errorf(\"%s\\n- [%d]=%+v\\n+ [%d]=%+v\", testName, i, expected[i], i, terms[i])\n\t\t\t}\n\t\t}\n\t}\n\n\tok(\n\t\t`3 TaggedTermEq, database contains all of them`,\n\t\t`seriesByTag('environment=production', 'dc=west', 'key=value')`,\n\t\t`SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+\n\t\t\t`(((Tag1='environment=production') OR (Tag1='dc=west')) OR (Tag1='key=value')) `+\n\t\t\t`AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+\n\t\t\t`GROUP BY Tag1 FORMAT TabSeparatedRaw`,\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"environment=production\\t100\\ndc=west\\t10\\nkey=value\\t1\\n\"),\n\t\t},\n\t\t[]TaggedTerm{\n\t\t\t{Op: TaggedTermEq, Key: \"key\", Value: \"value\", Cost: 1, NonDefaultCost: true},\n\t\t\t{Op: TaggedTermEq, Key: \"dc\", Value: \"west\", Cost: 10, NonDefaultCost: true},\n\t\t\t{Op: TaggedTermEq, Key: \"environment\", Value: \"production\", Cost: 100, NonDefaultCost: true},\n\t\t},\n\t\tnil,\n\t\tfalse,\n\t)\n\n\tok(\n\t\t`3 TaggedTermEq, database doesn't contain 1 of them`,\n\t\t`seriesByTag('environment=production', 'dc=west', 'key=value')`,\n\t\t`SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+\n\t\t\t`(((Tag1='environment=production') OR (Tag1='dc=west')) OR (Tag1='key=value')) `+\n\t\t\t`AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+\n\t\t\t`GROUP BY Tag1 FORMAT TabSeparatedRaw`,\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"environment=production\\t100\\nkey=value\\t1\\n\"),\n\t\t},\n\t\t[]TaggedTerm{\n\t\t\t{Op: TaggedTermEq, Key: \"environment\", Value: \"production\", Cost: 0, NonDefaultCost: false},\n\t\t\t{Op: TaggedTermEq, Key: \"dc\", Value: \"west\", Cost: 0, NonDefaultCost: false},\n\t\t\t{Op: TaggedTermEq, Key: \"key\", Value: \"value\", Cost: 0, NonDefaultCost: false},\n\t\t},\n\t\tnil,\n\t\tfalse,\n\t)\n\n\tok(\n\t\t`3 TaggedTermEq, database doesn't contain any of them`,\n\t\t`seriesByTag('environment=production', 'dc=west', 'key=value')`,\n\t\t`SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+\n\t\t\t`(((Tag1='environment=production') OR (Tag1='dc=west')) OR (Tag1='key=value')) `+\n\t\t\t`AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+\n\t\t\t`GROUP BY Tag1 FORMAT TabSeparatedRaw`,\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"\"),\n\t\t},\n\t\t[]TaggedTerm{\n\t\t\t{Op: TaggedTermEq, Key: \"environment\", Value: \"production\", Cost: 0, NonDefaultCost: false},\n\t\t\t{Op: TaggedTermEq, Key: \"dc\", Value: \"west\", Cost: 0, NonDefaultCost: false},\n\t\t\t{Op: TaggedTermEq, Key: \"key\", Value: \"value\", Cost: 0, NonDefaultCost: false},\n\t\t},\n\t\tnil,\n\t\tfalse,\n\t)\n\n\tok(\n\t\t`3 TaggedTermEq, one of them has a wildcard`,\n\t\t`seriesByTag('environment=production', 'dc=*', 'key=value')`,\n\t\t`SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+\n\t\t\t`((Tag1='environment=production') OR (Tag1='key=value')) `+\n\t\t\t`AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+\n\t\t\t`GROUP BY Tag1 FORMAT TabSeparatedRaw`,\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"environment=production\\t100\\nkey=value\\t2\\n\"),\n\t\t},\n\t\t[]TaggedTerm{\n\t\t\t{Op: TaggedTermEq, Key: \"key\", Value: \"value\", Cost: 2, NonDefaultCost: true},\n\t\t\t{Op: TaggedTermEq, Key: \"environment\", Value: \"production\", Cost: 100, NonDefaultCost: true},\n\t\t\t{Op: TaggedTermEq, Key: \"dc\", Value: \"*\", Cost: 0, NonDefaultCost: false, HasWildcard: true},\n\t\t},\n\t\tnil,\n\t\tfalse,\n\t)\n\n\tok(\n\t\t`2 TaggedTermEq, 2 TaggedTermMatch`,\n\t\t`seriesByTag('environment=production', 'dc=west', 'status=~^o.*', 'key=~val.*')`,\n\t\t`SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+\n\t\t\t`((Tag1='environment=production') OR (Tag1='dc=west')) `+\n\t\t\t`AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+\n\t\t\t`GROUP BY Tag1 FORMAT TabSeparatedRaw`,\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"environment=production\\t100\\ndc=west\\t10\\n\"),\n\t\t},\n\t\t[]TaggedTerm{\n\t\t\t{Op: TaggedTermEq, Key: \"dc\", Value: \"west\", Cost: 10, NonDefaultCost: true},\n\t\t\t{Op: TaggedTermEq, Key: \"environment\", Value: \"production\", Cost: 100, NonDefaultCost: true},\n\t\t\t{Op: TaggedTermMatch, Key: \"status\", Value: \"^o.*\", Cost: 0},\n\t\t\t{Op: TaggedTermMatch, Key: \"key\", Value: \"val.*\", Cost: 0},\n\t\t},\n\t\tnil,\n\t\tfalse,\n\t)\n\n\tok(\n\t\t`2 TaggedTermEq, 2 TaggedTermNe`,\n\t\t`seriesByTag('environment=production', 'dc=west', 'status!=on', 'key!=value')`,\n\t\t`SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+\n\t\t\t`((Tag1='environment=production') OR (Tag1='dc=west')) `+\n\t\t\t`AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+\n\t\t\t`GROUP BY Tag1 FORMAT TabSeparatedRaw`,\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"environment=production\\t100\\ndc=west\\t10\\n\"),\n\t\t},\n\t\t[]TaggedTerm{\n\t\t\t{Op: TaggedTermEq, Key: \"dc\", Value: \"west\", Cost: 10, NonDefaultCost: true},\n\t\t\t{Op: TaggedTermEq, Key: \"environment\", Value: \"production\", Cost: 100, NonDefaultCost: true},\n\t\t\t{Op: TaggedTermNe, Key: \"status\", Value: \"on\", Cost: 0},\n\t\t\t{Op: TaggedTermNe, Key: \"key\", Value: \"value\", Cost: 0},\n\t\t},\n\t\tnil,\n\t\tfalse,\n\t)\n\n\tok(\n\t\t`2 TaggedTermEq, 2 TaggedTermNotMatch`,\n\t\t`seriesByTag('environment=production', 'dc=west', 'status!=~^o.*', 'key!=~val.*')`,\n\t\t`SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+\n\t\t\t`((Tag1='environment=production') OR (Tag1='dc=west')) `+\n\t\t\t`AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+\n\t\t\t`GROUP BY Tag1 FORMAT TabSeparatedRaw`,\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"environment=production\\t100\\ndc=west\\t10\\n\"),\n\t\t},\n\t\t[]TaggedTerm{\n\t\t\t{Op: TaggedTermEq, Key: \"dc\", Value: \"west\", Cost: 10, NonDefaultCost: true},\n\t\t\t{Op: TaggedTermEq, Key: \"environment\", Value: \"production\", Cost: 100, NonDefaultCost: true},\n\t\t\t{Op: TaggedTermNotMatch, Key: \"status\", Value: \"^o.*\", Cost: 0},\n\t\t\t{Op: TaggedTermNotMatch, Key: \"key\", Value: \"val.*\", Cost: 0},\n\t\t},\n\t\tnil,\n\t\tfalse,\n\t)\n\n\tok(\n\t\t`3 TaggedTermMatch`,\n\t\t`seriesByTag('environment=~prod', 'dc=~west', 'key=~^val')`,\n\t\t``,\n\t\tnil,\n\t\t[]TaggedTerm{\n\t\t\t{Op: TaggedTermMatch, Key: \"environment\", Value: \"prod\", Cost: 0},\n\t\t\t{Op: TaggedTermMatch, Key: \"dc\", Value: \"west\", Cost: 0},\n\t\t\t{Op: TaggedTermMatch, Key: \"key\", Value: \"^val\", Cost: 0},\n\t\t},\n\t\tnil,\n\t\tfalse,\n\t)\n\n\tok(\n\t\t`3 TaggedTermEq, 2 have same tag`,\n\t\t`seriesByTag('environment=production', 'dc=west', 'dc=east')`,\n\t\t`SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+\n\t\t\t`(((Tag1='environment=production') OR (Tag1='dc=west')) OR (Tag1='dc=east')) `+\n\t\t\t`AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+\n\t\t\t`GROUP BY Tag1 FORMAT TabSeparatedRaw`,\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"environment=production\\t100\\ndc=west\\t10\\ndc=east\\t5\\n\"),\n\t\t},\n\t\t[]TaggedTerm{\n\t\t\t{Op: TaggedTermEq, Key: \"dc\", Value: \"east\", Cost: 5, NonDefaultCost: true},\n\t\t\t{Op: TaggedTermEq, Key: \"dc\", Value: \"west\", Cost: 10, NonDefaultCost: true},\n\t\t\t{Op: TaggedTermEq, Key: \"environment\", Value: \"production\", Cost: 100, NonDefaultCost: true},\n\t\t},\n\t\tnil,\n\t\tfalse,\n\t)\n\n\tok(\n\t\t`3 TaggedTermEq, 1 of them has __name__ key`,\n\t\t`seriesByTag('name=load.avg', 'environment=production', 'dc=west')`,\n\t\t`SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+\n\t\t\t`(((Tag1='__name__=load.avg') OR (Tag1='environment=production')) OR (Tag1='dc=west')) `+\n\t\t\t`AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+\n\t\t\t`GROUP BY Tag1 FORMAT TabSeparatedRaw`,\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"environment=production\\t100\\ndc=west\\t10\\n__name__=load.avg\\t10000\\n\"),\n\t\t},\n\t\t[]TaggedTerm{\n\t\t\t{Op: TaggedTermEq, Key: \"dc\", Value: \"west\", Cost: 10, NonDefaultCost: true},\n\t\t\t{Op: TaggedTermEq, Key: \"environment\", Value: \"production\", Cost: 100, NonDefaultCost: true},\n\t\t\t{Op: TaggedTermEq, Key: \"__name__\", Value: \"load.avg\", Cost: 10000, NonDefaultCost: true},\n\t\t},\n\t\tnil,\n\t\tfalse,\n\t)\n\n\tok(\n\t\t`3 TaggedTermEq, 1 of them has __name__ key, 1 does not exist in count table`,\n\t\t`seriesByTag('environment=production', 'dc=west', 'name=load.avg')`,\n\t\t`SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+\n\t\t\t`(((Tag1='environment=production') OR (Tag1='dc=west')) OR (Tag1='__name__=load.avg')) `+\n\t\t\t`AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+\n\t\t\t`GROUP BY Tag1 FORMAT TabSeparatedRaw`,\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"environment=production\\t100\\n__name__=load.avg\\t10000\\n\"),\n\t\t},\n\t\t[]TaggedTerm{\n\t\t\t{Op: TaggedTermEq, Key: \"__name__\", Value: \"load.avg\", Cost: 0, NonDefaultCost: false},\n\t\t\t{Op: TaggedTermEq, Key: \"environment\", Value: \"production\", Cost: 0, NonDefaultCost: false},\n\t\t\t{Op: TaggedTermEq, Key: \"dc\", Value: \"west\", Cost: 0, NonDefaultCost: false},\n\t\t},\n\t\tnil,\n\t\tfalse,\n\t)\n\n\tok(\n\t\t`Clickhouse returned broken response`,\n\t\t`seriesByTag('environment=production', 'dc=west', 'key=value')`,\n\t\t`SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+\n\t\t\t`(((Tag1='environment=production') OR (Tag1='dc=west')) OR (Tag1='key=value')) `+\n\t\t\t`AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+\n\t\t\t`GROUP BY Tag1 FORMAT TabSeparatedRaw`,\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"broken_response\"),\n\t\t},\n\t\tnil,\n\t\tfmt.Errorf(\"failed to parse result from clickhouse while querying for tag costs: no tag count\"),\n\t\tfalse,\n\t)\n\n\tok(\n\t\t`3 TaggedTermEq, 1 of them has __name__ key, 1 does not exist in count table, fallback to config costs`,\n\t\t`seriesByTag('name=high_cost', 'environment=production', 'dc=west')`,\n\t\t`SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+\n\t\t\t`(((Tag1='__name__=high_cost') OR (Tag1='environment=production')) OR (Tag1='dc=west')) `+\n\t\t\t`AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+\n\t\t\t`GROUP BY Tag1 FORMAT TabSeparatedRaw`,\n\t\t&chtest.TestResponse{\n\t\t\tBody: []byte(\"environment=production\\t100\\n__name__=load.avg\\t10000\\n\"),\n\t\t},\n\t\t[]TaggedTerm{\n\t\t\t{Op: TaggedTermEq, Key: \"dc\", Value: \"west\", Cost: 60, NonDefaultCost: true},\n\t\t\t{Op: TaggedTermEq, Key: \"__name__\", Value: \"high_cost\", Cost: 70, NonDefaultCost: true},\n\t\t\t{Op: TaggedTermEq, Key: \"environment\", Value: \"production\", Cost: 100, NonDefaultCost: true},\n\t\t},\n\t\tnil,\n\t\ttrue,\n\t)\n}\n\nfunc TestTaggedFinder_whereFilter(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tquery string\n\t\tfrom int64\n\t\tuntil int64\n\t\tdailyEnabled bool\n\t\tuseCarbonBehavior bool\n\t\tdontMatchMissingTags bool\n\t\ttaggedCosts map[string]*config.Costs\n\t\twant string\n\t\twantPre string\n\t}{\n\t\t{\n\t\t\tname: \"nodaily\",\n\t\t\tquery: \"seriesByTag('name=metric')\",\n\t\t\tfrom: 1668106860, // 2022-11-11 00:01:00 +05:00\n\t\t\tuntil: 1668106870, // 2022-11-11 00:01:10 +05:00\n\t\t\tdailyEnabled: false,\n\t\t\twant: \"(Tag1='__name__=metric') AND (Date >='\" +\n\t\t\t\tdate.FromTimestampToDaysFormat(1668106860) + \"')\",\n\t\t\twantPre: \"\",\n\t\t},\n\t\t{\n\t\t\tname: \"midnight at utc (direct)\",\n\t\t\tquery: \"seriesByTag('name=metric')\",\n\t\t\tfrom: 1668124800, // 2022-11-11 00:00:00 UTC\n\t\t\tuntil: 1668124810, // 2022-11-11 00:00:10 UTC\n\t\t\tdailyEnabled: true,\n\t\t\twant: \"(Tag1='__name__=metric') AND (Date >='\" +\n\t\t\t\tdate.FromTimestampToDaysFormat(1668124800) + \"' AND Date <= '\" + date.UntilTimestampToDaysFormat(1668124810) + \"')\",\n\t\t\twantPre: \"\",\n\t\t},\n\t\t{\n\t\t\tname: \"\",\n\t\t\tquery: \"seriesByTag('emptyval=', 'what=value')\",\n\t\t\tfrom: 1668124800, // 2022-11-11 00:00:00 UTC\n\t\t\tuntil: 1668124810, // 2022-11-11 00:00:10 UTC\n\t\t\tdailyEnabled: true,\n\t\t\tuseCarbonBehavior: true,\n\t\t\twant: \"((Tag1='what=value') AND (NOT arrayExists((x) -> x LIKE 'emptyval=%', Tags))) AND (Date >='\" +\n\t\t\t\tdate.FromTimestampToDaysFormat(1668124800) + \"' AND Date <= '\" + date.UntilTimestampToDaysFormat(1668124810) + \"')\",\n\t\t\twantPre: \"\",\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name+\" \"+time.Unix(tt.from, 0).Format(time.RFC3339), func(t *testing.T) {\n\t\t\tconfig := config.New()\n\t\t\tconfig.ClickHouse.TaggedCosts = tt.taggedCosts\n\t\t\tconfig.FeatureFlags.UseCarbonBehavior = tt.useCarbonBehavior\n\t\t\tconfig.FeatureFlags.DontMatchMissingTags = tt.dontMatchMissingTags\n\n\t\t\tf := NewTagged(\n\t\t\t\t\"http://localhost:8123/\",\n\t\t\t\t\"graphite_tags\",\n\t\t\t\t\"\",\n\t\t\t\ttt.dailyEnabled,\n\t\t\t\ttt.useCarbonBehavior,\n\t\t\t\ttt.dontMatchMissingTags,\n\t\t\t\tfalse,\n\t\t\t\tclickhouse.Options{},\n\t\t\t\ttt.taggedCosts,\n\t\t\t)\n\n\t\t\tterms, err := f.PrepareTaggedTerms(context.Background(), config, tt.query, tt.from, tt.until)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tgot, gotDate, err := f.whereFilter(terms, tt.from, tt.until)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tif got.String() != tt.want {\n\t\t\t\tt.Errorf(\"TaggedFinder.whereFilter()[0] = %v, want %v\", got, tt.want)\n\t\t\t}\n\n\t\t\tif gotDate.String() != tt.wantPre {\n\t\t\t\tt.Errorf(\"TaggedFinder.whereFilter()[1] = %v, want %v\", gotDate, tt.wantPre)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestTaggedFinder_Abs(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tv []byte\n\t\tcached bool\n\t\twant []byte\n\t}{\n\t\t{\n\t\t\tname: \"cached\",\n\t\t\tv: []byte(\"test_metric;colon=:;forward=/;hash=#;host=127.0.0.1;minus=-;percent=%;plus=+;underscore=_\"),\n\t\t\tcached: true,\n\t\t\twant: []byte(\"test_metric;colon=:;forward=/;hash=#;host=127.0.0.1;minus=-;percent=%;plus=+;underscore=_\"),\n\t\t},\n\t\t{\n\t\t\tname: \"escaped\",\n\t\t\tv: []byte(url.QueryEscape(\"instance:cpu_utilization?ratio_avg\") +\n\t\t\t\t\"?\" + url.QueryEscape(\"dc\") + \"=\" + url.QueryEscape(\"qwe+1\") +\n\t\t\t\t\"&\" + url.QueryEscape(\"fqdn\") + \"=\" + url.QueryEscape(\"asd&a\") +\n\t\t\t\t\"&\" + url.QueryEscape(\"instance\") + \"=\" + url.QueryEscape(\"10.33.10.10:9100\") +\n\t\t\t\t\"&\" + url.QueryEscape(\"job\") + \"=\" + url.QueryEscape(\"node&a\")),\n\t\t\twant: []byte(\"instance:cpu_utilization?ratio_avg;dc=qwe+1;fqdn=asd&a;instance=10.33.10.10:9100;job=node&a\"),\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tvar tf *TaggedFinder\n\t\t\tif tt.cached {\n\t\t\t\ttf = NewCachedTags(nil)\n\t\t\t} else {\n\t\t\t\ttf = NewTagged(\"http:/127.0.0.1:8123\", \"graphite_tags\", \"\", true, false, false, false, clickhouse.Options{}, nil)\n\t\t\t}\n\n\t\t\tif got := string(tf.Abs(tt.v)); got != string(tt.want) {\n\t\t\t\tt.Errorf(\"TaggedDecode() =\\n%q\\nwant\\n%q\", got, string(tt.want))\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "finder/tags_count_querier.go", "content": "package finder\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"strconv\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/helper/date\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/where\"\n\t\"github.com/msaf1980/go-stringutils\"\n)\n\ntype TagCountQuerier struct {\n\turl string\n\ttable string\n\topts clickhouse.Options\n\tuseCarbonBehavior bool\n\tdontMatchMissingTags bool\n\tdailyEnabled bool\n\tbody []byte\n\tstats []metrics.FinderStat\n}\n\nfunc NewTagCountQuerier(url, table string, opts clickhouse.Options, useCarbonBehavior, dontMatchMissingTags, dailyEnabled bool) *TagCountQuerier {\n\treturn &TagCountQuerier{\n\t\turl: url,\n\t\ttable: table,\n\t\topts: opts,\n\t\tuseCarbonBehavior: useCarbonBehavior,\n\t\tdontMatchMissingTags: dontMatchMissingTags,\n\t\tdailyEnabled: dailyEnabled,\n\t}\n}\n\nfunc (tcq *TagCountQuerier) GetCostsFromCountTable(ctx context.Context, terms []TaggedTerm, from int64, until int64) (map[string]*config.Costs, error) {\n\tif len(terms) < 2 {\n\t\treturn nil, nil\n\t}\n\n\tw := where.New()\n\teqTermCount := 0\n\n\tfor i := 0; i < len(terms); i++ {\n\t\tif terms[i].Op == TaggedTermEq && !terms[i].HasWildcard && terms[i].Value != \"\" {\n\t\t\tsqlTerm, err := TaggedTermWhere1(&terms[i], tcq.useCarbonBehavior, tcq.dontMatchMissingTags)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\tw.Or(sqlTerm)\n\n\t\t\teqTermCount++\n\t\t}\n\t}\n\n\tif w.SQL() == \"\" {\n\t\treturn nil, nil\n\t}\n\n\tif tcq.dailyEnabled {\n\t\tw.Andf(\n\t\t\t\"Date >= '%s' AND Date <= '%s'\",\n\t\t\tdate.FromTimestampToDaysFormat(from),\n\t\t\tdate.UntilTimestampToDaysFormat(until),\n\t\t)\n\t} else {\n\t\tw.Andf(\n\t\t\t\"Date >= '%s'\",\n\t\t\tdate.FromTimestampToDaysFormat(from),\n\t\t)\n\t}\n\n\tsql := fmt.Sprintf(\"SELECT Tag1, sum(Count) as cnt FROM %s %s GROUP BY Tag1 FORMAT TabSeparatedRaw\", tcq.table, w.SQL())\n\n\tvar err error\n\n\ttcq.stats = append(tcq.stats, metrics.FinderStat{})\n\tstat := &tcq.stats[len(tcq.stats)-1]\n\tstat.Table = tcq.table\n\n\ttcq.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query(scope.WithTable(ctx, tcq.table), tcq.url, sql, tcq.opts, nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\trows := tcq.List()\n\n\t// create cost var to validate CH response without writing to t.taggedCosts\n\tvar costs map[string]*config.Costs\n\n\tcosts, err = chResultToCosts(rows)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// The metric does not exist if the response has less rows\n\t// than there were tags with '=' op in the initial request\n\t// This is due to each tag-value pair of a metric being written\n\t// exactly one time as Tag1\n\tif len(rows) < eqTermCount {\n\t\ttcq.body = []byte{}\n\t\treturn nil, nil\n\t}\n\n\treturn costs, nil\n}\n\nfunc chResultToCosts(body [][]byte) (map[string]*config.Costs, error) {\n\tcosts := make(map[string]*config.Costs, 0)\n\n\tfor i := 0; i < len(body); i++ {\n\t\ts := stringutils.UnsafeString(body[i])\n\n\t\ttag, val, count, err := parseTag1CountRow(s)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"failed to parse result from clickhouse while querying for tag costs: %s\", err.Error())\n\t\t}\n\n\t\tif costs[tag] == nil {\n\t\t\tcosts[tag] = &config.Costs{Cost: nil, ValuesCost: make(map[string]int, 0)}\n\t\t}\n\n\t\tcosts[tag].ValuesCost[val] = count\n\t}\n\n\treturn costs, nil\n}\n\nfunc parseTag1CountRow(s string) (string, string, int, error) {\n\tvar (\n\t\ttag1, count, tag, val string\n\t\tcnt, n int\n\t\terr error\n\t)\n\n\tif tag1, count, n = stringutils.Split2(s, \"\\t\"); n != 2 {\n\t\treturn \"\", \"\", 0, fmt.Errorf(\"no tag count\")\n\t}\n\n\tif tag, val, n = stringutils.Split2(tag1, \"=\"); n != 2 {\n\t\treturn \"\", \"\", 0, fmt.Errorf(\"no '=' in Tag1\")\n\t}\n\n\tif cnt, err = strconv.Atoi(count); err != nil {\n\t\treturn \"\", \"\", 0, fmt.Errorf(\"can't convert count to int\")\n\t}\n\n\treturn tag, val, cnt, nil\n}\n\nfunc (t *TagCountQuerier) List() [][]byte {\n\tif t.body == nil {\n\t\treturn [][]byte{}\n\t}\n\n\trows := bytes.Split(t.body, []byte{'\\n'})\n\n\tskip := 0\n\n\tfor i := 0; i < len(rows); i++ {\n\t\tif len(rows[i]) == 0 {\n\t\t\tskip++\n\t\t\tcontinue\n\t\t}\n\n\t\tif skip > 0 {\n\t\t\trows[i-skip] = rows[i]\n\t\t}\n\t}\n\n\trows = rows[:len(rows)-skip]\n\n\treturn rows\n}\n\nfunc (tcq *TagCountQuerier) Stats() []metrics.FinderStat {\n\treturn tcq.stats\n}\n" }, { "path": "finder/unescape.go", "content": "package finder\n\nimport \"strings\"\n\nfunc ishex(c byte) bool {\n\tswitch {\n\tcase '0' <= c && c <= '9':\n\t\treturn true\n\tcase 'a' <= c && c <= 'f':\n\t\treturn true\n\tcase 'A' <= c && c <= 'F':\n\t\treturn true\n\t}\n\n\treturn false\n}\n\nfunc unhex(c byte) byte {\n\tswitch {\n\tcase '0' <= c && c <= '9':\n\t\treturn c - '0'\n\tcase 'a' <= c && c <= 'f':\n\t\treturn c - 'a' + 10\n\tcase 'A' <= c && c <= 'F':\n\t\treturn c - 'A' + 10\n\t}\n\n\treturn 0\n}\n\nfunc isPercentEscape(s string, i int) bool {\n\treturn i+2 < len(s) && ishex(s[i+1]) && ishex(s[i+2])\n}\n\n// unescape unescapes a string.\nfunc unescape(s string) string {\n\tfirst := strings.IndexByte(s, '%')\n\tif first == -1 {\n\t\treturn s\n\t}\n\n\tvar t strings.Builder\n\n\tt.Grow(len(s))\n\tt.WriteString(s[:first])\n\nLOOP:\n\tfor i := first; i < len(s); i++ {\n\t\tswitch s[i] {\n\t\tcase '%':\n\t\t\tif len(s) < i+3 {\n\t\t\t\tt.WriteString(s[i:])\n\t\t\t\tbreak LOOP\n\t\t\t}\n\t\t\tif !isPercentEscape(s, i) {\n\t\t\t\tt.WriteString(s[i : i+3])\n\t\t\t} else {\n\t\t\t\tt.WriteByte(unhex(s[i+1])<<4 | unhex(s[i+2]))\n\t\t\t}\n\t\t\ti += 2\n\t\tdefault:\n\t\t\tt.WriteByte(s[i])\n\t\t}\n\t}\n\n\treturn t.String()\n}\n" }, { "path": "go.mod", "content": "module github.com/lomik/graphite-clickhouse\n\ngo 1.23.1\n\nrequire (\n\tgithub.com/BurntSushi/toml v0.3.1\n\tgithub.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874\n\tgithub.com/cactus/go-statsd-client/v5 v5.1.0\n\tgithub.com/go-graphite/carbonapi v0.16.1\n\tgithub.com/go-graphite/protocol v1.0.0\n\tgithub.com/gogo/protobuf v1.3.2\n\tgithub.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc\n\tgithub.com/json-iterator/go v1.1.12\n\tgithub.com/klauspost/compress v1.17.9\n\tgithub.com/lomik/carbon-clickhouse v0.11.8\n\tgithub.com/lomik/graphite-pickle v0.0.0-20171221213606-614e8df42119\n\tgithub.com/lomik/og-rek v0.0.0-20170411191824-628eefeb8d80\n\tgithub.com/lomik/prometheus-ui-static v0.2.54-1.1\n\tgithub.com/lomik/zapwriter v0.0.0-20210624082824-c1161d1eb463\n\tgithub.com/msaf1980/go-expirecache v0.0.2\n\tgithub.com/msaf1980/go-metrics v0.0.14\n\tgithub.com/msaf1980/go-stringutils v0.1.6\n\tgithub.com/msaf1980/go-syncutils v0.0.3\n\tgithub.com/msaf1980/go-timeutils v0.0.4\n\tgithub.com/pelletier/go-toml v1.9.5\n\tgithub.com/pkg/errors v0.9.1\n\tgithub.com/prometheus/client_golang v1.20.3\n\tgithub.com/prometheus/client_model v0.6.1\n\tgithub.com/prometheus/common/assets v0.2.0\n\tgithub.com/prometheus/prometheus v0.0.0-20240827104400-e6cfa720fbe6\n\tgithub.com/stretchr/testify v1.9.0\n\tgo.uber.org/zap v1.24.0\n\tgolang.org/x/sync v0.12.0\n)\n\nrequire (\n\tgithub.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 // indirect\n\tgithub.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 // indirect\n\tgithub.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect\n\tgithub.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect\n\tgithub.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 // indirect\n\tgithub.com/ansel1/merry v1.6.2 // indirect\n\tgithub.com/ansel1/merry/v2 v2.0.2 // indirect\n\tgithub.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect\n\tgithub.com/aws/aws-sdk-go v1.55.5 // indirect\n\tgithub.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect\n\tgithub.com/beorn7/perks v1.0.1 // indirect\n\tgithub.com/cespare/xxhash/v2 v2.3.0 // indirect\n\tgithub.com/coreos/go-systemd/v22 v22.5.0 // indirect\n\tgithub.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect\n\tgithub.com/dennwc/varint v1.0.0 // indirect\n\tgithub.com/digitalocean/godo v1.122.0 // indirect\n\tgithub.com/docker/docker v27.2.0+incompatible // indirect\n\tgithub.com/edsrzf/mmap-go v1.1.0 // indirect\n\tgithub.com/envoyproxy/go-control-plane v0.13.0 // indirect\n\tgithub.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect\n\tgithub.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect\n\tgithub.com/felixge/httpsnoop v1.0.4 // indirect\n\tgithub.com/go-kit/log v0.2.1 // indirect\n\tgithub.com/go-logfmt/logfmt v0.6.0 // indirect\n\tgithub.com/go-logr/logr v1.4.2 // indirect\n\tgithub.com/go-logr/stdr v1.2.2 // indirect\n\tgithub.com/go-openapi/analysis v0.22.2 // indirect\n\tgithub.com/go-openapi/errors v0.22.0 // indirect\n\tgithub.com/go-openapi/jsonpointer v0.20.2 // indirect\n\tgithub.com/go-openapi/jsonreference v0.20.4 // indirect\n\tgithub.com/go-openapi/loads v0.21.5 // indirect\n\tgithub.com/go-openapi/spec v0.20.14 // indirect\n\tgithub.com/go-openapi/strfmt v0.23.0 // indirect\n\tgithub.com/go-openapi/swag v0.22.9 // indirect\n\tgithub.com/go-openapi/validate v0.23.0 // indirect\n\tgithub.com/go-zookeeper/zk v1.0.4 // indirect\n\tgithub.com/golang-jwt/jwt/v5 v5.2.2 // indirect\n\tgithub.com/golang/snappy v0.0.4 // indirect\n\tgithub.com/google/go-cmp v0.6.0 // indirect\n\tgithub.com/google/uuid v1.6.0 // indirect\n\tgithub.com/gophercloud/gophercloud v1.14.0 // indirect\n\tgithub.com/hashicorp/consul/api v1.29.4 // indirect\n\tgithub.com/hetznercloud/hcloud-go/v2 v2.13.1 // indirect\n\tgithub.com/ionos-cloud/sdk-go/v6 v6.2.1 // indirect\n\tgithub.com/jmespath/go-jmespath v0.4.0 // indirect\n\tgithub.com/josharian/intern v1.0.0 // indirect\n\tgithub.com/jpillora/backoff v1.0.0 // indirect\n\tgithub.com/julienschmidt/httprouter v1.3.0 // indirect\n\tgithub.com/kylelemons/godebug v1.1.0 // indirect\n\tgithub.com/linode/linodego v1.40.0 // indirect\n\tgithub.com/lomik/stop v0.0.0-20161127103810-188e98d969bd // indirect\n\tgithub.com/mailru/easyjson v0.7.7 // indirect\n\tgithub.com/mdlayher/socket v0.4.1 // indirect\n\tgithub.com/mdlayher/vsock v1.2.1 // indirect\n\tgithub.com/miekg/dns v1.1.62 // indirect\n\tgithub.com/mitchellh/mapstructure v1.5.0 // indirect\n\tgithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect\n\tgithub.com/modern-go/reflect2 v1.0.2 // indirect\n\tgithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect\n\tgithub.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect\n\tgithub.com/oklog/ulid v1.3.1 // indirect\n\tgithub.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect\n\tgithub.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect\n\tgithub.com/prometheus/alertmanager v0.27.0 // indirect\n\tgithub.com/prometheus/common v0.59.1 // indirect\n\tgithub.com/prometheus/common/sigv4 v0.1.0 // indirect\n\tgithub.com/prometheus/exporter-toolkit v0.12.0 // indirect\n\tgithub.com/prometheus/procfs v0.15.1 // indirect\n\tgithub.com/scaleway/scaleway-sdk-go v1.0.0-beta.30 // indirect\n\tgithub.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c // indirect\n\tgithub.com/stretchr/objx v0.5.2 // indirect\n\tgo.mongodb.org/mongo-driver v1.14.0 // indirect\n\tgo.opentelemetry.io/collector/pdata v1.14.1 // indirect\n\tgo.opentelemetry.io/collector/semconv v0.108.1 // indirect\n\tgo.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect\n\tgo.opentelemetry.io/otel v1.29.0 // indirect\n\tgo.opentelemetry.io/otel/metric v1.29.0 // indirect\n\tgo.opentelemetry.io/otel/trace v1.29.0 // indirect\n\tgo.uber.org/atomic v1.11.0 // indirect\n\tgo.uber.org/goleak v1.3.0 // indirect\n\tgo.uber.org/multierr v1.11.0 // indirect\n\tgolang.org/x/crypto v0.36.0 // indirect\n\tgolang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect\n\tgolang.org/x/net v0.38.0 // indirect\n\tgolang.org/x/oauth2 v0.27.0 // indirect\n\tgolang.org/x/sys v0.31.0 // indirect\n\tgolang.org/x/text v0.23.0 // indirect\n\tgolang.org/x/time v0.6.0 // indirect\n\tgolang.org/x/tools v0.24.0 // indirect\n\tgoogle.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed // indirect\n\tgoogle.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect\n\tgoogle.golang.org/grpc v1.66.0 // indirect\n\tgoogle.golang.org/protobuf v1.34.2 // indirect\n\tgopkg.in/yaml.v2 v2.4.0 // indirect\n\tgopkg.in/yaml.v3 v3.0.1 // indirect\n\tk8s.io/apimachinery v0.31.0 // indirect\n\tk8s.io/client-go v0.31.0 // indirect\n\tk8s.io/klog/v2 v2.130.1 // indirect\n\tk8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect\n)\n\nreplace (\n\tk8s.io/klog => github.com/simonpasquier/klog-gokit v0.3.0\n\tk8s.io/klog/v2 => github.com/simonpasquier/klog-gokit/v3 v3.5.0\n)\n\n// Exclude linodego v1.0.0 as it is no longer published on github.\nexclude github.com/linode/linodego v1.0.0\n\n// Exclude grpc v1.30.0 because of breaking changes. See #7621.\nexclude (\n\tgithub.com/grpc-ecosystem/grpc-gateway v1.14.7\n\tgoogle.golang.org/api v0.30.0\n)\n" }, { "path": "go.sum", "content": "cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=\ncloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=\ncloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=\ncloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=\ncloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=\ncloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=\ncloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=\ncloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=\ncloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=\ncloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=\ncloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=\ncloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=\ncloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=\ncloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=\ncloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=\ncloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=\ncloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=\ncloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=\ncloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=\ncloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=\ncloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=\ncloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=\ncloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=\ncloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=\ncloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=\ncloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=\ncloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=\ncloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=\ncloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=\ncloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=\ncloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=\ndmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8=\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0=\ngithub.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=\ngithub.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=\ngithub.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=\ngithub.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=\ngithub.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 h1:LkHbJbgF3YyvC53aqYGR+wWQDn2Rdp9AQdGndf9QvY4=\ngithub.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0/go.mod h1:QyiQdW4f4/BIfB8ZutZ2s+28RAgfa/pT+zS++ZHyM1I=\ngithub.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0 h1:bXwSugBiSbgtz7rOtbfGf+woewp4f06orW9OP5BjHLA=\ngithub.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0/go.mod h1:Y/HgrePTmGy9HjdSGTqZNa+apUpTVIEVKXJyARP2lrk=\ngithub.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=\ngithub.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=\ngithub.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=\ngithub.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=\ngithub.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=\ngithub.com/Code-Hex/go-generics-cache v1.5.1 h1:6vhZGc5M7Y/YD8cIUcY8kcuQLB4cHR7U+0KMqAA0KcU=\ngithub.com/Code-Hex/go-generics-cache v1.5.1/go.mod h1:qxcC9kRVrct9rHeiYpFWSoW1vxyillCVzX13KZG8dl4=\ngithub.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=\ngithub.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=\ngithub.com/Shopify/sarama v1.29.0/go.mod h1:2QpgD79wpdAESqNQMxNc0KYMkycd4slxGdV3TWSVqrU=\ngithub.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=\ngithub.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=\ngithub.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=\ngithub.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=\ngithub.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=\ngithub.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=\ngithub.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 h1:t3eaIm0rUkzbrIewtiFmMK5RXHej2XnoXNhxVsAYUfg=\ngithub.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs=\ngithub.com/ansel1/merry v1.5.0/go.mod h1:wUy/yW0JX0ix9GYvUbciq+bi3jW/vlKPlbpI7qdZpOw=\ngithub.com/ansel1/merry v1.5.1/go.mod h1:wUy/yW0JX0ix9GYvUbciq+bi3jW/vlKPlbpI7qdZpOw=\ngithub.com/ansel1/merry v1.6.2 h1:0xr40haRrfVzmOH/JVOu7KOKGEI1c/7q5EmgTEbn+Ng=\ngithub.com/ansel1/merry v1.6.2/go.mod h1:pAcMW+2uxIgpzEON021vMtFsrymREY6faJWiiz1QGVQ=\ngithub.com/ansel1/merry/v2 v2.0.1/go.mod h1:dD5OhpiPrVkvgseRYd+xgYlx7s6ytU3v9BTTJlDA7FM=\ngithub.com/ansel1/merry/v2 v2.0.2 h1:xPHMhTp2iOkGCN1q+vqetL1Ww692cFuiASHjUMMjAiY=\ngithub.com/ansel1/merry/v2 v2.0.2/go.mod h1:dD5OhpiPrVkvgseRYd+xgYlx7s6ytU3v9BTTJlDA7FM=\ngithub.com/ansel1/vespucci/v4 v4.1.1/go.mod h1:zzdrO4IgBfgcGMbGTk/qNGL8JPslmW3nPpcBHKReFYY=\ngithub.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=\ngithub.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=\ngithub.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=\ngithub.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=\ngithub.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=\ngithub.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=\ngithub.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=\ngithub.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=\ngithub.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps=\ngithub.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0=\ngithub.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o=\ngithub.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=\ngithub.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=\ngithub.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=\ngithub.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=\ngithub.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=\ngithub.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 h1:N7oVaKyGp8bttX0bfZGmcGkjz7DLQXhAn3DNd3T0ous=\ngithub.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874/go.mod h1:r5xuitiExdLAJ09PR7vBVENGvp4ZuTBeWTGtxuX3K+c=\ngithub.com/cactus/go-statsd-client/v5 v5.1.0 h1:sbbdfIl9PgisjEoXzvXI1lwUKWElngsjJKaZeC021P4=\ngithub.com/cactus/go-statsd-client/v5 v5.1.0/go.mod h1:COEvJ1E+/E2L4q6QE5CkjWPi4eeDw9maJBMIuMPBZbY=\ngithub.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=\ngithub.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=\ngithub.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=\ngithub.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=\ngithub.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=\ngithub.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=\ngithub.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=\ngithub.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=\ngithub.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=\ngithub.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=\ngithub.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=\ngithub.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=\ngithub.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=\ngithub.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=\ngithub.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnThWgvH2wg8376yUJmPhEH4H3kw=\ngithub.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=\ngithub.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=\ngithub.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=\ngithub.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=\ngithub.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=\ngithub.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE=\ngithub.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA=\ngithub.com/digitalocean/godo v1.122.0 h1:ziytLQi8QKtDp2K1A+YrYl2dWLHLh2uaMzWvcz9HkKg=\ngithub.com/digitalocean/godo v1.122.0/go.mod h1:WQVH83OHUy6gC4gXpEVQKtxTd4L5oCp+5OialidkPLY=\ngithub.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=\ngithub.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=\ngithub.com/docker/docker v27.2.0+incompatible h1:Rk9nIVdfH3+Vz4cyI/uhbINhEZ/oLmc+CBXmH6fbNk4=\ngithub.com/docker/docker v27.2.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=\ngithub.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=\ngithub.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=\ngithub.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=\ngithub.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=\ngithub.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=\ngithub.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=\ngithub.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=\ngithub.com/eclipse/paho.mqtt.golang v1.3.5/go.mod h1:eTzb4gxwwyWpqBUHGQZ4ABAV7+Jgm1PklsYT/eo8Hcc=\ngithub.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ=\ngithub.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=\ngithub.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=\ngithub.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=\ngithub.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=\ngithub.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=\ngithub.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=\ngithub.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=\ngithub.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=\ngithub.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=\ngithub.com/envoyproxy/go-control-plane v0.13.0 h1:HzkeUz1Knt+3bK+8LG1bxOO/jzWZmdxpwC51i202les=\ngithub.com/envoyproxy/go-control-plane v0.13.0/go.mod h1:GRaKG3dwvFoTg4nj7aXdZnvMg4d7nvT/wl9WgVXn3Q8=\ngithub.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=\ngithub.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM=\ngithub.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4=\ngithub.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb h1:IT4JYU7k4ikYg1SCxNI1/Tieq/NFvh6dzLdgi7eu0tM=\ngithub.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb/go.mod h1:bH6Xx7IW64qjjJq8M2u4dxNaBiDfKK+z/3eGDpXEQhc=\ngithub.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=\ngithub.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=\ngithub.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=\ngithub.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=\ngithub.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=\ngithub.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=\ngithub.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=\ngithub.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=\ngithub.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=\ngithub.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=\ngithub.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=\ngithub.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=\ngithub.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg=\ngithub.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs=\ngithub.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=\ngithub.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=\ngithub.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=\ngithub.com/go-graphite/carbonapi v0.16.1 h1:pbFmR0CqQH9tWYAehOXKZjfbaJRKVx0yJBs4dGFtSdg=\ngithub.com/go-graphite/carbonapi v0.16.1/go.mod h1:vefIVMyWf471SFMpHS4CSbySjGaMPtNxmMTnTlU2tSc=\ngithub.com/go-graphite/protocol v1.0.0 h1:Fqb0mkVVtfMrn6vw6Ntm3raf3gVVZCOVdZu4JosW5qE=\ngithub.com/go-graphite/protocol v1.0.0/go.mod h1:eonkg/0UGhJUYu+PshOg1NzWSUcXskr/yHeQXJHJr8Y=\ngithub.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=\ngithub.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=\ngithub.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=\ngithub.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=\ngithub.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=\ngithub.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=\ngithub.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=\ngithub.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=\ngithub.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=\ngithub.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=\ngithub.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=\ngithub.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=\ngithub.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=\ngithub.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=\ngithub.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=\ngithub.com/go-openapi/analysis v0.22.2 h1:ZBmNoP2h5omLKr/srIC9bfqrUGzT6g6gNv03HE9Vpj0=\ngithub.com/go-openapi/analysis v0.22.2/go.mod h1:pDF4UbZsQTo/oNuRfAWWd4dAh4yuYf//LYorPTjrpvo=\ngithub.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w=\ngithub.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE=\ngithub.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q=\ngithub.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs=\ngithub.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU=\ngithub.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4=\ngithub.com/go-openapi/loads v0.21.5 h1:jDzF4dSoHw6ZFADCGltDb2lE4F6De7aWSpe+IcsRzT0=\ngithub.com/go-openapi/loads v0.21.5/go.mod h1:PxTsnFBoBe+z89riT+wYt3prmSBP6GDAQh2l9H1Flz8=\ngithub.com/go-openapi/spec v0.20.14 h1:7CBlRnw+mtjFGlPDRZmAMnq35cRzI91xj03HVyUi/Do=\ngithub.com/go-openapi/spec v0.20.14/go.mod h1:8EOhTpBoFiask8rrgwbLC3zmJfz4zsCUueRuPM6GNkw=\ngithub.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c=\ngithub.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4=\ngithub.com/go-openapi/swag v0.22.9 h1:XX2DssF+mQKM2DHsbgZK74y/zj4mo9I99+89xUmuZCE=\ngithub.com/go-openapi/swag v0.22.9/go.mod h1:3/OXnFfnMAwBD099SwYRk7GD3xOrr1iL7d/XNLXVVwE=\ngithub.com/go-openapi/validate v0.23.0 h1:2l7PJLzCis4YUGEoW6eoQw3WhyM65WSIcjX6SQnlfDw=\ngithub.com/go-openapi/validate v0.23.0/go.mod h1:EeiAZ5bmpSIOJV1WLfyYF9qp/B1ZgSaEpHTJHtN5cbE=\ngithub.com/go-resty/resty/v2 v2.13.1 h1:x+LHXBI2nMB1vqndymf26quycC4aggYJ7DECYbiz03g=\ngithub.com/go-resty/resty/v2 v2.13.1/go.mod h1:GznXlLxkq6Nh4sU59rPmUw3VtgpO3aS96ORAI6Q7d+0=\ngithub.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=\ngithub.com/go-zookeeper/zk v1.0.4 h1:DPzxraQx7OrPyXq2phlGlNSIyWEsAox0RJmjTseMV6I=\ngithub.com/go-zookeeper/zk v1.0.4/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=\ngithub.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=\ngithub.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=\ngithub.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=\ngithub.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=\ngithub.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=\ngithub.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=\ngithub.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=\ngithub.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=\ngithub.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=\ngithub.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=\ngithub.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=\ngithub.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=\ngithub.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=\ngithub.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=\ngithub.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=\ngithub.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=\ngithub.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=\ngithub.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=\ngithub.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=\ngithub.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=\ngithub.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=\ngithub.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=\ngithub.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=\ngithub.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=\ngithub.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=\ngithub.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=\ngithub.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=\ngithub.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=\ngithub.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=\ngithub.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=\ngithub.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=\ngithub.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=\ngithub.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=\ngithub.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=\ngithub.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=\ngithub.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=\ngithub.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=\ngithub.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=\ngithub.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=\ngithub.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=\ngithub.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=\ngithub.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=\ngithub.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=\ngithub.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=\ngithub.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=\ngithub.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=\ngithub.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=\ngithub.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=\ngithub.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=\ngithub.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=\ngithub.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=\ngithub.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=\ngithub.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=\ngithub.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=\ngithub.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=\ngithub.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=\ngithub.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=\ngithub.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=\ngithub.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=\ngithub.com/gophercloud/gophercloud v1.14.0 h1:Bt9zQDhPrbd4qX7EILGmy+i7GP35cc+AAL2+wIJpUE8=\ngithub.com/gophercloud/gophercloud v1.14.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM=\ngithub.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=\ngithub.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=\ngithub.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=\ngithub.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=\ngithub.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=\ngithub.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrRoT6yV5+wkrOpcszoIsO4+4ds248=\ngithub.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk=\ngithub.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=\ngithub.com/hashicorp/consul/api v1.29.4 h1:P6slzxDLBOxUSj3fWo2o65VuKtbtOXFi7TSSgtXutuE=\ngithub.com/hashicorp/consul/api v1.29.4/go.mod h1:HUlfw+l2Zy68ceJavv2zAyArl2fqhGWnMycyt56sBgg=\ngithub.com/hashicorp/cronexpr v1.1.2 h1:wG/ZYIKT+RT3QkOdgYc+xsKWVRgnxJ1OJtjjy84fJ9A=\ngithub.com/hashicorp/cronexpr v1.1.2/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4=\ngithub.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=\ngithub.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=\ngithub.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=\ngithub.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=\ngithub.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=\ngithub.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=\ngithub.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=\ngithub.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=\ngithub.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=\ngithub.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=\ngithub.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=\ngithub.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=\ngithub.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=\ngithub.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=\ngithub.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=\ngithub.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=\ngithub.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=\ngithub.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4=\ngithub.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=\ngithub.com/hashicorp/nomad/api v0.0.0-20240717122358-3d93bd3778f3 h1:fgVfQ4AC1avVOnu2cfms8VAiD8lUq3vWI8mTocOXN/w=\ngithub.com/hashicorp/nomad/api v0.0.0-20240717122358-3d93bd3778f3/go.mod h1:svtxn6QnrQ69P23VvIWMR34tg3vmwLz4UdUzm1dSCgE=\ngithub.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=\ngithub.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=\ngithub.com/hetznercloud/hcloud-go/v2 v2.13.1 h1:jq0GP4QaYE5d8xR/Zw17s9qoaESRJMXfGmtD1a/qckQ=\ngithub.com/hetznercloud/hcloud-go/v2 v2.13.1/go.mod h1:dhix40Br3fDiBhwaSG/zgaYOFFddpfBm/6R1Zz0IiF0=\ngithub.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=\ngithub.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=\ngithub.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=\ngithub.com/ionos-cloud/sdk-go/v6 v6.2.1 h1:mxxN+frNVmbFrmmFfXnBC3g2USYJrl6mc1LW2iNYbFY=\ngithub.com/ionos-cloud/sdk-go/v6 v6.2.1/go.mod h1:SXrO9OGyWjd2rZhAhEpdYN6VUAODzzqRdqA9BCviQtI=\ngithub.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=\ngithub.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=\ngithub.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=\ngithub.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=\ngithub.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc=\ngithub.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=\ngithub.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=\ngithub.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=\ngithub.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=\ngithub.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=\ngithub.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=\ngithub.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=\ngithub.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=\ngithub.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=\ngithub.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=\ngithub.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=\ngithub.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=\ngithub.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=\ngithub.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=\ngithub.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=\ngithub.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=\ngithub.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=\ngithub.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=\ngithub.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=\ngithub.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=\ngithub.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=\ngithub.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg=\ngithub.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=\ngithub.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=\ngithub.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=\ngithub.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=\ngithub.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=\ngithub.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJis0QP7YMxobob6zhzq6Yre00=\ngithub.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM=\ngithub.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=\ngithub.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=\ngithub.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=\ngithub.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=\ngithub.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=\ngithub.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=\ngithub.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=\ngithub.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=\ngithub.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=\ngithub.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=\ngithub.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=\ngithub.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=\ngithub.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=\ngithub.com/linode/linodego v1.40.0 h1:7ESY0PwK94hoggoCtIroT1Xk6b1flrFBNZ6KwqbTqlI=\ngithub.com/linode/linodego v1.40.0/go.mod h1:NsUw4l8QrLdIofRg1NYFBbW5ZERnmbZykVBszPZLORM=\ngithub.com/lomik/carbon-clickhouse v0.11.8 h1:Kgix46vequoJ60Qd4+3k2ocEh0hLBrdOO7df/G3D1aY=\ngithub.com/lomik/carbon-clickhouse v0.11.8/go.mod h1:c9H5G+2CwVqOcxxUc2WOL6i+rlOTmbd9G+AqIeLttgs=\ngithub.com/lomik/graphite-pickle v0.0.0-20171221213606-614e8df42119 h1:9kRJjaYdyzqGcGMeWeVif1vkToJvqzPEe5Vqx4IDXBg=\ngithub.com/lomik/graphite-pickle v0.0.0-20171221213606-614e8df42119/go.mod h1:C0xsTshsU0n/LkhSbjZx2UkLuWSa3uFmq9D35Ch4rNE=\ngithub.com/lomik/og-rek v0.0.0-20170411191824-628eefeb8d80 h1:KVyDGUXjVOdHQt24wIgY4ZdGFXHtQHLWw0L/MAK3Kb0=\ngithub.com/lomik/og-rek v0.0.0-20170411191824-628eefeb8d80/go.mod h1:T7SQVaLtK7mcQIEVzveZVJzsDQpAtzTs2YoezrIBdvI=\ngithub.com/lomik/prometheus-ui-static v0.2.54-1.1 h1:ilVQwP1nKxW85y3BD41mttIInQ/3pCNrTJvqI3znP3A=\ngithub.com/lomik/prometheus-ui-static v0.2.54-1.1/go.mod h1:aYbUFvTr1G39aW1WMI4Bp1nKhI9u/DsBcLaR5nKz0KE=\ngithub.com/lomik/stop v0.0.0-20161127103810-188e98d969bd h1:hUNpVzZOYNANa5s8XMBEt8IBj3m1GWcUN6ewUXPVA6c=\ngithub.com/lomik/stop v0.0.0-20161127103810-188e98d969bd/go.mod h1:3pLqdYIrxHYk+VsfIlrTcBD9J34YkGq8iN9yzJuhrP0=\ngithub.com/lomik/zapwriter v0.0.0-20210624082824-c1161d1eb463 h1:SN/0TEkyYpp8tit79JPUnecebCGZsXiYYPxN8i3I6Rk=\ngithub.com/lomik/zapwriter v0.0.0-20210624082824-c1161d1eb463/go.mod h1:rWIJAUD2hPOAyOzc3jBShAhN4CAZeLAyzUA/n8tE8ak=\ngithub.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=\ngithub.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=\ngithub.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=\ngithub.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=\ngithub.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=\ngithub.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=\ngithub.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=\ngithub.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=\ngithub.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=\ngithub.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U=\ngithub.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA=\ngithub.com/mdlayher/vsock v1.2.1 h1:pC1mTJTvjo1r9n9fbm7S1j04rCgCzhCOS5DY0zqHlnQ=\ngithub.com/mdlayher/vsock v1.2.1/go.mod h1:NRfCibel++DgeMD8z/hP+PPTjlNJsdPOmxcnENvE+SE=\ngithub.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ=\ngithub.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ=\ngithub.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=\ngithub.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=\ngithub.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=\ngithub.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=\ngithub.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=\ngithub.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=\ngithub.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=\ngithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=\ngithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=\ngithub.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=\ngithub.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=\ngithub.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=\ngithub.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=\ngithub.com/msaf1980/go-expirecache v0.0.2 h1:lkxQMd/cXnz/WTS5IO1HC399dxR9DrqNAhaET7gPKLE=\ngithub.com/msaf1980/go-expirecache v0.0.2/go.mod h1:AVemStNEitwcK0IDFtGBQ9GZJesybwaTe8mG1pCCajM=\ngithub.com/msaf1980/go-metrics v0.0.14 h1:gD0kCG5MDbon33Nkz49yW6kz3yu0DHzDN0SxjGTWlTA=\ngithub.com/msaf1980/go-metrics v0.0.14/go.mod h1:8VcR8MdyvIJpcVLOVFKbhb27+60tXy0M+zq7Ag8a6Pw=\ngithub.com/msaf1980/go-stringutils v0.1.2/go.mod h1:AxmV/6JuQUAtZJg5XmYATB5ZwCWgtpruVHY03dswRf8=\ngithub.com/msaf1980/go-stringutils v0.1.6 h1:qri8o+4XLJCJYemHcvJY6xJhrGTmllUoPwayKEj4NSg=\ngithub.com/msaf1980/go-stringutils v0.1.6/go.mod h1:xpicaTIpLAVzL0gUQkciB1zjypDGKsOCI25cKQbRQYA=\ngithub.com/msaf1980/go-syncutils v0.0.3 h1:bd6+yTSB8/CmpG7M6j1gq5sJMyPqecjJcBf19s2Y6u4=\ngithub.com/msaf1980/go-syncutils v0.0.3/go.mod h1:zoZwQNkDATcfKq5lQPK6dmJT7Z01COxw/vd8bcJyC9w=\ngithub.com/msaf1980/go-timeutils v0.0.4 h1:qdWcThz2gMTb73d3uDjwfXNsIzhpIjMlBQCnqc4pa6M=\ngithub.com/msaf1980/go-timeutils v0.0.4/go.mod h1:r252j2O/ZLuwNMp/rlSYhbQdxg6glZ3MzgvskE/ItGY=\ngithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=\ngithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=\ngithub.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=\ngithub.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=\ngithub.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=\ngithub.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=\ngithub.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=\ngithub.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=\ngithub.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=\ngithub.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=\ngithub.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=\ngithub.com/ovh/go-ovh v1.6.0 h1:ixLOwxQdzYDx296sXcgS35TOPEahJkpjMGtzPadCjQI=\ngithub.com/ovh/go-ovh v1.6.0/go.mod h1:cTVDnl94z4tl8pP1uZ/8jlVxntjSIf09bNcQ5TJSC7c=\ngithub.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=\ngithub.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=\ngithub.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=\ngithub.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=\ngithub.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=\ngithub.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=\ngithub.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=\ngithub.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=\ngithub.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/prometheus/alertmanager v0.27.0 h1:V6nTa2J5V4s8TG4C4HtrBP/WNSebCCTYGGv4qecA/+I=\ngithub.com/prometheus/alertmanager v0.27.0/go.mod h1:8Ia/R3urPmbzJ8OsdvmZvIprDwvwmYCmUbwBL+jlPOE=\ngithub.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=\ngithub.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=\ngithub.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=\ngithub.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=\ngithub.com/prometheus/client_golang v1.20.3 h1:oPksm4K8B+Vt35tUhw6GbSNSgVlVSBH0qELP/7u83l4=\ngithub.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=\ngithub.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=\ngithub.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=\ngithub.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=\ngithub.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=\ngithub.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=\ngithub.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=\ngithub.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=\ngithub.com/prometheus/common v0.59.1 h1:LXb1quJHWm1P6wq/U824uxYi4Sg0oGvNeUm1z5dJoX0=\ngithub.com/prometheus/common v0.59.1/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0=\ngithub.com/prometheus/common/assets v0.2.0 h1:0P5OrzoHrYBOSM1OigWL3mY8ZvV2N4zIE/5AahrSrfM=\ngithub.com/prometheus/common/assets v0.2.0/go.mod h1:D17UVUE12bHbim7HzwUvtqm6gwBEaDQ0F+hIGbFbccI=\ngithub.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4=\ngithub.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI=\ngithub.com/prometheus/exporter-toolkit v0.12.0 h1:DkE5RcEZR3lQA2QD5JLVQIf41dFKNsVMXFhgqcif7fo=\ngithub.com/prometheus/exporter-toolkit v0.12.0/go.mod h1:fQH0KtTn0yrrS0S82kqppRjDDiwMfIQUwT+RBRRhwUc=\ngithub.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=\ngithub.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=\ngithub.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=\ngithub.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=\ngithub.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=\ngithub.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=\ngithub.com/prometheus/prometheus v0.0.0-20240827104400-e6cfa720fbe6 h1:T1gXvCkxUmoQYSlrM+eXRrLfzHAVDyONlWVXtfWugoo=\ngithub.com/prometheus/prometheus v0.0.0-20240827104400-e6cfa720fbe6/go.mod h1:xlLByHhk2g3ycakQGrMaU8K7OySZx98BzeCR99991NY=\ngithub.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=\ngithub.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=\ngithub.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=\ngithub.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=\ngithub.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=\ngithub.com/scaleway/scaleway-sdk-go v1.0.0-beta.30 h1:yoKAVkEVwAqbGbR8n87rHQ1dulL25rKloGadb3vm770=\ngithub.com/scaleway/scaleway-sdk-go v1.0.0-beta.30/go.mod h1:sH0u6fq6x4R5M7WxkoQFY/o7UaiItec0o1LinLCJNq8=\ngithub.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c h1:aqg5Vm5dwtvL+YgDpBcK1ITf3o96N/K7/wsRXQnUTEs=\ngithub.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c/go.mod h1:owqhoLW1qZoYLZzLnBw+QkPP9WZnjlSWihhxAJC1+/M=\ngithub.com/simonpasquier/klog-gokit/v3 v3.5.0 h1:ewnk+ickph0hkQFgdI4pffKIbruAxxWcg0Fe/vQmLOM=\ngithub.com/simonpasquier/klog-gokit/v3 v3.5.0/go.mod h1:S9flvRzzpaYLYtXI2w8jf9R/IU/Cy14NrbvDUevNP1E=\ngithub.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=\ngithub.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=\ngithub.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=\ngithub.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=\ngithub.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=\ngithub.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=\ngithub.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=\ngithub.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=\ngithub.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=\ngithub.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=\ngithub.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=\ngithub.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=\ngithub.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=\ngithub.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=\ngithub.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=\ngithub.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=\ngithub.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=\ngithub.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs=\ngithub.com/vultr/govultr/v2 v2.17.2/go.mod h1:ZFOKGWmgjytfyjeyAdhQlSWwTjh2ig+X49cAp50dzXI=\ngithub.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=\ngithub.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=\ngithub.com/xdg/scram v1.0.3/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=\ngithub.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=\ngithub.com/xhhuango/json v1.19.0/go.mod h1:ynZo8WeuBMtTh7LMR1ljdu/8QxceUVbYEcAsPJ7iUb8=\ngithub.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngithub.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngithub.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngithub.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngo.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80=\ngo.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c=\ngo.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=\ngo.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=\ngo.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=\ngo.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=\ngo.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=\ngo.opentelemetry.io/collector/pdata v1.14.1 h1:wXZjtQA7Vy5HFqco+yA95ENyMQU5heBB1IxMHQf6mUk=\ngo.opentelemetry.io/collector/pdata v1.14.1/go.mod h1:z1dTjwwtcoXxZx2/nkHysjxMeaxe9pEmYTEr4SMNIx8=\ngo.opentelemetry.io/collector/semconv v0.108.1 h1:Txk9tauUnamZaxS5vlf1O0uZ4VD6nioRBR0nX8L/fU4=\ngo.opentelemetry.io/collector/semconv v0.108.1/go.mod h1:zCJ5njhWpejR+A40kiEoeFm1xq1uzyZwMnRNX6/D82A=\ngo.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA=\ngo.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg=\ngo.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw=\ngo.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8=\ngo.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc=\ngo.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8=\ngo.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4=\ngo.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ=\ngo.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=\ngo.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=\ngo.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=\ngo.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=\ngo.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=\ngo.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=\ngo.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=\ngo.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=\ngo.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=\ngo.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=\ngo.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=\ngo.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=\ngolang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=\ngolang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=\ngolang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=\ngolang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=\ngolang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=\ngolang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=\ngolang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=\ngolang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=\ngolang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=\ngolang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=\ngolang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=\ngolang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=\ngolang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=\ngolang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=\ngolang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=\ngolang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=\ngolang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=\ngolang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=\ngolang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA=\ngolang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=\ngolang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=\ngolang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=\ngolang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=\ngolang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=\ngolang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=\ngolang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=\ngolang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=\ngolang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=\ngolang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=\ngolang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=\ngolang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=\ngolang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=\ngolang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=\ngolang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=\ngolang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=\ngolang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=\ngolang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=\ngolang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=\ngolang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=\ngolang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=\ngolang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=\ngolang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=\ngolang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=\ngolang.org/x/net v0.0.0-20210427231257-85d9c07bbe3a/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=\ngolang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=\ngolang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=\ngolang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=\ngolang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=\ngolang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=\ngolang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=\ngolang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=\ngolang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=\ngolang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=\ngolang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=\ngolang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M=\ngolang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=\ngolang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=\ngolang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=\ngolang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=\ngolang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=\ngolang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=\ngolang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y=\ngolang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g=\ngolang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=\ngolang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=\ngolang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=\ngolang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=\ngolang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=\ngolang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=\ngolang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=\ngolang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=\ngolang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=\ngolang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=\ngolang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=\ngolang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=\ngolang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=\ngolang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=\ngolang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=\ngolang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=\ngolang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=\ngolang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=\ngolang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=\ngolang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=\ngolang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=\ngolang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=\ngolang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngoogle.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=\ngoogle.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=\ngoogle.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=\ngoogle.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=\ngoogle.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=\ngoogle.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=\ngoogle.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=\ngoogle.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=\ngoogle.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=\ngoogle.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=\ngoogle.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=\ngoogle.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=\ngoogle.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=\ngoogle.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=\ngoogle.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=\ngoogle.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=\ngoogle.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=\ngoogle.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=\ngoogle.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=\ngoogle.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=\ngoogle.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=\ngoogle.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=\ngoogle.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=\ngoogle.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=\ngoogle.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=\ngoogle.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=\ngoogle.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=\ngoogle.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=\ngoogle.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=\ngoogle.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=\ngoogle.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=\ngoogle.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=\ngoogle.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=\ngoogle.golang.org/genproto v0.0.0-20220208230804-65c12eb4c068/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=\ngoogle.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed h1:3RgNmBoI9MZhsj3QxC+AP/qQhNwpCLOvYDYYsFrhFt0=\ngoogle.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo=\ngoogle.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ=\ngoogle.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=\ngoogle.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=\ngoogle.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=\ngoogle.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=\ngoogle.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=\ngoogle.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=\ngoogle.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=\ngoogle.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=\ngoogle.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=\ngoogle.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=\ngoogle.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=\ngoogle.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=\ngoogle.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=\ngoogle.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=\ngoogle.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=\ngoogle.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=\ngoogle.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c=\ngoogle.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y=\ngoogle.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=\ngoogle.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=\ngoogle.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=\ngoogle.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=\ngoogle.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=\ngoogle.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=\ngoogle.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=\ngoogle.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=\ngoogle.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=\ngoogle.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=\ngoogle.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=\ngoogle.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=\ngopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=\ngopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=\ngopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=\ngopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=\ngopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=\ngopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=\ngopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=\ngopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=\ngopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\ngopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\ngopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=\ngopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\nhonnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=\nhonnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=\nhonnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=\nk8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo=\nk8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE=\nk8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc=\nk8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=\nk8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8=\nk8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU=\nk8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag=\nk8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98=\nk8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A=\nk8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=\nrsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=\nrsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=\nrsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=\nsigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=\nsigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=\nsigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=\nsigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=\nsigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=\nsigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=\n" }, { "path": "graphite-clickhouse.go", "content": "package main\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"flag\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"math/rand\"\n\t\"net/http\"\n\t_ \"net/http/pprof\"\n\t\"os\"\n\t\"os/signal\"\n\t\"runtime\"\n\t\"runtime/debug\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/lomik/zapwriter\"\n\t\"go.uber.org/zap\"\n\n\t\"github.com/lomik/graphite-clickhouse/autocomplete\"\n\t\"github.com/lomik/graphite-clickhouse/capabilities\"\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/find\"\n\t\"github.com/lomik/graphite-clickhouse/healthcheck\"\n\t\"github.com/lomik/graphite-clickhouse/helper/rollup\"\n\t\"github.com/lomik/graphite-clickhouse/index\"\n\t\"github.com/lomik/graphite-clickhouse/logs\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"github.com/lomik/graphite-clickhouse/prometheus\"\n\t\"github.com/lomik/graphite-clickhouse/render\"\n\t\"github.com/lomik/graphite-clickhouse/sd\"\n\t\"github.com/lomik/graphite-clickhouse/tagger\"\n)\n\n// Version of graphite-clickhouse\nconst Version = \"0.14.0\"\n\nfunc init() {\n\tscope.Version = Version\n}\n\ntype LogResponseWriter struct {\n\thttp.ResponseWriter\n\tstatus int\n\tcached bool\n}\n\nfunc (w *LogResponseWriter) WriteHeader(status int) {\n\tw.status = status\n\tw.ResponseWriter.WriteHeader(status)\n}\n\nfunc (w *LogResponseWriter) Status() int {\n\tif w.status == 0 {\n\t\treturn http.StatusOK\n\t}\n\n\treturn w.status\n}\n\nfunc WrapResponseWriter(w http.ResponseWriter) *LogResponseWriter {\n\tif wrapped, ok := w.(*LogResponseWriter); ok {\n\t\treturn wrapped\n\t}\n\n\treturn &LogResponseWriter{ResponseWriter: w}\n}\n\ntype App struct {\n\tconfig *config.Config\n}\n\nfunc (app *App) Handler(handler http.Handler) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\twriter := WrapResponseWriter(w)\n\n\t\tr = scope.HttpRequest(r)\n\n\t\tw.Header().Add(\"X-Gch-Request-ID\", scope.RequestID(r.Context()))\n\n\t\thandler.ServeHTTP(writer, r)\n\t})\n}\n\nvar (\n\tBuildVersion = \"(development build)\"\n\tsrv *http.Server\n)\n\nfunc sdList(name string, args []string) {\n\tdescr := \"List registered nodes in SD\"\n\tflagName := \"sd-list\"\n\tflagSet := flag.NewFlagSet(descr, flag.ExitOnError)\n\thelp := flagSet.Bool(\"help\", false, \"Print help\")\n\tconfigFile := flagSet.String(\"config\", \"/etc/graphite-clickhouse/graphite-clickhouse.conf\", \"Filename of config\")\n\texactConfig := flagSet.Bool(\"exact-config\", false, \"Ensure that all config params are contained in the target struct.\")\n\tflagSet.Usage = func() {\n\t\tfmt.Fprintf(os.Stderr, \"Usage of %s %s:\\n\", name, flagName)\n\t\tflagSet.PrintDefaults()\n\t}\n\tflagSet.Parse(args)\n\n\tif *help || flagSet.NArg() > 0 {\n\t\tflagSet.Usage()\n\t\treturn\n\t}\n\n\tcfg, _, err := config.ReadConfig(*configFile, *exactConfig)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tif cfg.Common.SD != \"\" && cfg.NeedLoadAvgColect() {\n\t\tvar s sd.SD\n\n\t\tlogger := zapwriter.Default()\n\t\tif s, err = sd.New(&cfg.Common, \"\", logger); err != nil {\n\t\t\tfmt.Fprintf(os.Stderr, \"service discovery type %q can be registered\", cfg.Common.SDType.String())\n\t\t\tos.Exit(1)\n\t\t}\n\n\t\tif nodes, err := s.Nodes(); err == nil {\n\t\t\tfor _, node := range nodes {\n\t\t\t\tfmt.Printf(\"%s/%s: %s (%s)\\n\", s.Namespace(), node.Key, node.Value, time.Unix(node.Flags, 0).UTC().Format(time.RFC3339Nano))\n\t\t\t}\n\t\t} else {\n\t\t\tfmt.Fprintln(os.Stderr, err.Error())\n\t\t\tos.Exit(1)\n\t\t}\n\t}\n}\n\nfunc sdDelete(name string, args []string) {\n\tdescr := \"Delete registered nodes for local hostname in SD\"\n\tflagName := \"sd-delete\"\n\tflagSet := flag.NewFlagSet(descr, flag.ExitOnError)\n\thelp := flagSet.Bool(\"help\", false, \"Print help\")\n\tconfigFile := flagSet.String(\"config\", \"/etc/graphite-clickhouse/graphite-clickhouse.conf\", \"Filename of config\")\n\texactConfig := flagSet.Bool(\"exact-config\", false, \"Ensure that all config params are contained in the target struct.\")\n\tflagSet.Usage = func() {\n\t\tfmt.Fprintf(os.Stderr, \"Usage of %s %s:\\n\", name, flagName)\n\t\tflagSet.PrintDefaults()\n\t}\n\tflagSet.Parse(args)\n\n\tif *help || flagSet.NArg() > 0 {\n\t\tflagSet.Usage()\n\t\treturn\n\t}\n\n\tcfg, _, err := config.ReadConfig(*configFile, *exactConfig)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tif cfg.Common.SD != \"\" && cfg.NeedLoadAvgColect() {\n\t\tvar s sd.SD\n\n\t\tlogger := zapwriter.Default()\n\t\tif s, err = sd.New(&cfg.Common, \"\", logger); err != nil {\n\t\t\tfmt.Fprintf(os.Stderr, \"service discovery type %q can be registered\", cfg.Common.SDType.String())\n\t\t\tos.Exit(1)\n\t\t}\n\n\t\thostname, _ := os.Hostname()\n\t\thostname, _, _ = strings.Cut(hostname, \".\")\n\n\t\tif err = s.Clear(\"\", \"\"); err != nil {\n\t\t\tfmt.Fprintln(os.Stderr, err.Error())\n\t\t\tos.Exit(1)\n\t\t}\n\t}\n}\n\nfunc sdEvict(name string, args []string) {\n\tdescr := \"Delete registered nodes for hostnames in SD\"\n\tflagName := \"sd-evict\"\n\tflagSet := flag.NewFlagSet(descr, flag.ExitOnError)\n\thelp := flagSet.Bool(\"help\", false, \"Print help\")\n\tconfigFile := flagSet.String(\"config\", \"/etc/graphite-clickhouse/graphite-clickhouse.conf\", \"Filename of config\")\n\texactConfig := flagSet.Bool(\"exact-config\", false, \"Ensure that all config params are contained in the target struct.\")\n\tflagSet.Usage = func() {\n\t\tfmt.Fprintf(os.Stderr, \"Usage of %s %s:\\n\", name, flagName)\n\t\tflagSet.PrintDefaults()\n\t\tfmt.Fprintf(os.Stderr, \" HOST []string\\n \tList of hostnames\\n\")\n\t}\n\tflagSet.Parse(args)\n\n\tif *help {\n\t\tflagSet.Usage()\n\t\treturn\n\t}\n\n\tcfg, _, err := config.ReadConfig(*configFile, *exactConfig)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tif cfg.Common.SD != \"\" && cfg.NeedLoadAvgColect() {\n\t\tfor _, host := range flagSet.Args() {\n\t\t\tvar s sd.SD\n\n\t\t\tlogger := zapwriter.Default()\n\t\t\tif s, err = sd.New(&cfg.Common, host, logger); err != nil {\n\t\t\t\tfmt.Fprintf(os.Stderr, \"service discovery type %q can be registered\", cfg.Common.SDType.String())\n\t\t\t\tos.Exit(1)\n\t\t\t}\n\n\t\t\terr = s.Clear(\"\", \"\")\n\t\t}\n\t}\n}\n\nfunc sdExpired(name string, args []string) {\n\tdescr := \"List expired registered nodes in SD\"\n\tflagName := \"sd-expired\"\n\tflagSet := flag.NewFlagSet(descr, flag.ExitOnError)\n\thelp := flagSet.Bool(\"help\", false, \"Print help\")\n\tconfigFile := flagSet.String(\"config\", \"/etc/graphite-clickhouse/graphite-clickhouse.conf\", \"Filename of config\")\n\texactConfig := flagSet.Bool(\"exact-config\", false, \"Ensure that all config params are contained in the target struct.\")\n\tflagSet.Usage = func() {\n\t\tfmt.Fprintf(os.Stderr, \"Usage of %s %s:\\n\", name, flagName)\n\t\tflagSet.PrintDefaults()\n\t}\n\tflagSet.Parse(args)\n\n\tif *help || flagSet.NArg() > 0 {\n\t\tflagSet.Usage()\n\t\treturn\n\t}\n\n\tcfg, _, err := config.ReadConfig(*configFile, *exactConfig)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tif cfg.Common.SD != \"\" && cfg.NeedLoadAvgColect() {\n\t\tvar s sd.SD\n\n\t\tlogger := zapwriter.Default()\n\t\tif s, err = sd.New(&cfg.Common, \"\", logger); err != nil {\n\t\t\tfmt.Fprintf(os.Stderr, \"service discovery type %q can be registered\", cfg.Common.SDType.String())\n\t\t\tos.Exit(1)\n\t\t}\n\n\t\tif err = sd.Cleanup(&cfg.Common, s, true); err != nil {\n\t\t\tfmt.Fprintln(os.Stderr, err.Error())\n\t\t\tos.Exit(1)\n\t\t}\n\t}\n}\n\nfunc sdClean(name string, args []string) {\n\tdescr := \"Cleanup expired registered nodes in SD\"\n\tflagName := \"sd-clean\"\n\tflagSet := flag.NewFlagSet(descr, flag.ExitOnError)\n\thelp := flagSet.Bool(\"help\", false, \"Print help\")\n\tconfigFile := flagSet.String(\"config\", \"/etc/graphite-clickhouse/graphite-clickhouse.conf\", \"Filename of config\")\n\texactConfig := flagSet.Bool(\"exact-config\", false, \"Ensure that all config params are contained in the target struct.\")\n\tflagSet.Usage = func() {\n\t\tfmt.Fprintf(os.Stderr, \"Usage of %s %s:\\n\", name, flagName)\n\t\tflagSet.PrintDefaults()\n\t}\n\tflagSet.Parse(args)\n\n\tif *help || flagSet.NArg() > 0 {\n\t\tflagSet.Usage()\n\t\treturn\n\t}\n\n\tcfg, _, err := config.ReadConfig(*configFile, *exactConfig)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tif cfg.Common.SD != \"\" && cfg.NeedLoadAvgColect() {\n\t\tvar s sd.SD\n\n\t\tlogger := zapwriter.Default()\n\t\tif s, err = sd.New(&cfg.Common, \"\", logger); err != nil {\n\t\t\tfmt.Fprintf(os.Stderr, \"service discovery type %q can be registered\", cfg.Common.SDType.String())\n\t\t\tos.Exit(1)\n\t\t}\n\n\t\tif err = sd.Cleanup(&cfg.Common, s, false); err != nil {\n\t\t\tfmt.Fprintln(os.Stderr, err.Error())\n\t\t\tos.Exit(1)\n\t\t}\n\t}\n}\n\nfunc printMatchedRollupRules(metric string, age uint32, rollupRules *rollup.Rules) {\n\t// check metric rollup rules\n\tprec, aggr, aggrPattern, retentionPattern := rollupRules.Lookup(metric, age, true)\n\tfmt.Printf(\" metric %q, age %d -> precision=%d, aggr=%s\\n\", metric, age, prec, aggr.Name())\n\n\tif aggrPattern != nil {\n\t\tfmt.Printf(\" aggr pattern: type=%s, regexp=%q, function=%s\", aggrPattern.RuleType.String(), aggrPattern.Regexp, aggrPattern.Function)\n\n\t\tif len(aggrPattern.Retention) > 0 {\n\t\t\tfmt.Print(\", retentions:\\n\")\n\n\t\t\tfor i := range aggrPattern.Retention {\n\t\t\t\tfmt.Printf(\" [age: %d, precision: %d]\\n\", aggrPattern.Retention[i].Age, aggrPattern.Retention[i].Precision)\n\t\t\t}\n\t\t} else {\n\t\t\tfmt.Print(\"\\n\")\n\t\t}\n\t}\n\n\tif retentionPattern != nil {\n\t\tfmt.Printf(\" retention pattern: type=%s, regexp=%q, function=%s, retentions:\\n\", retentionPattern.RuleType.String(), retentionPattern.Regexp, retentionPattern.Function)\n\n\t\tfor i := range retentionPattern.Retention {\n\t\t\tfmt.Printf(\" [age: %d, precision: %d]\\n\", retentionPattern.Retention[i].Age, retentionPattern.Retention[i].Precision)\n\t\t}\n\t}\n}\n\nfunc checkRollupMatch(name string, args []string) {\n\tdescr := \"Match metric against rollup rules\"\n\tflagName := \"match\"\n\tflagSet := flag.NewFlagSet(descr, flag.ExitOnError)\n\thelp := flagSet.Bool(\"help\", false, \"Print help\")\n\n\trollupFile := flagSet.String(\"rollup\", \"\", \"Filename of rollup rules file\")\n\tconfigFile := flagSet.String(\"config\", \"\", \"Filename of config\")\n\texactConfig := flagSet.Bool(\"exact-config\", false, \"Ensure that all config params are contained in the target struct.\")\n\ttable := flagSet.String(\"table\", \"\", \"Table in config for lookup rules\")\n\n\tage := flagSet.Uint64(\"age\", 0, \"Age\")\n\tflagSet.Usage = func() {\n\t\tfmt.Fprintf(os.Stderr, \"Usage of %s %s:\\n\", name, flagName)\n\t\tflagSet.PrintDefaults()\n\t\tfmt.Fprintf(os.Stderr, \" METRIC []string\\n \tList of metric names\\n\")\n\t}\n\tflagSet.Parse(args)\n\n\tif *help {\n\t\tflagSet.Usage()\n\t\treturn\n\t}\n\n\tif *rollupFile == \"\" && *configFile == \"\" {\n\t\tfmt.Fprint(os.Stderr, \"set rollup and/or config file\\n\")\n\t\tos.Exit(1)\n\t}\n\n\tif *rollupFile != \"\" {\n\t\tfmt.Printf(\"rollup file %q\\n\", *rollupFile)\n\n\t\tif rollup, err := rollup.NewXMLFile(*rollupFile, 0, \"\"); err == nil {\n\t\t\tfor _, metric := range flagSet.Args() {\n\t\t\t\tprintMatchedRollupRules(metric, uint32(*age), rollup.Rules())\n\t\t\t}\n\t\t} else {\n\t\t\tlog.Fatal(err)\n\t\t}\n\t}\n\n\tif *configFile != \"\" {\n\t\tcfg, _, err := config.ReadConfig(*configFile, *exactConfig)\n\t\tif err != nil {\n\t\t\tlog.Fatal(err)\n\t\t}\n\n\t\tec := 0\n\n\t\tfor i := range cfg.DataTable {\n\t\t\tvar rulesTable string\n\n\t\t\tif *table == \"\" || *table == cfg.DataTable[i].Table {\n\t\t\t\tif cfg.DataTable[i].RollupConf == \"auto\" || cfg.DataTable[i].RollupConf == \"\" {\n\t\t\t\t\trulesTable = cfg.DataTable[i].Table\n\t\t\t\t\tif cfg.DataTable[i].RollupAutoTable != \"\" {\n\t\t\t\t\t\trulesTable = cfg.DataTable[i].RollupAutoTable\n\t\t\t\t\t}\n\n\t\t\t\t\tfmt.Printf(\"table %q, rollup rules table %q in Clickhouse\\n\", cfg.DataTable[i].Table, rulesTable)\n\t\t\t\t} else {\n\t\t\t\t\tfmt.Printf(\"rollup file %q\\n\", cfg.DataTable[i].RollupConf)\n\t\t\t\t}\n\n\t\t\t\trules := cfg.DataTable[i].Rollup.Rules()\n\t\t\t\tif rules == nil {\n\t\t\t\t\tif cfg.DataTable[i].RollupConf == \"auto\" || cfg.DataTable[i].RollupConf == \"\" {\n\t\t\t\t\t\trules, err = rollup.RemoteLoad(cfg.ClickHouse.URL,\n\t\t\t\t\t\t\tcfg.ClickHouse.TLSConfig, rulesTable)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\tec = 1\n\n\t\t\t\t\t\t\tfmt.Fprintf(os.Stderr, \"%v\\n\", err)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif rules != nil {\n\t\t\t\t\tfor _, metric := range flagSet.Args() {\n\t\t\t\t\t\tprintMatchedRollupRules(metric, uint32(*age), rules)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tos.Exit(ec)\n\t}\n}\n\nfunc main() {\n\trand.Seed(time.Now().UnixNano())\n\n\tvar err error\n\n\t/* CONFIG start */\n\n\tconfigFile := flag.String(\"config\", \"/etc/graphite-clickhouse/graphite-clickhouse.conf\", \"Filename of config\")\n\tprintDefaultConfig := flag.Bool(\"config-print-default\", false, \"Print default config\")\n\tcheckConfig := flag.Bool(\"check-config\", false, \"Check config and exit\")\n\texactConfig := flag.Bool(\"exact-config\", false, \"Ensure that all config params are contained in the target struct.\")\n\tbuildTags := flag.Bool(\"tags\", false, \"Build tags table\")\n\tpprof := flag.String(\n\t\t\"pprof\",\n\t\t\"\",\n\t\t\"Additional pprof listen addr for non-server modes (tagger, etc..), overrides pprof-listen from common \",\n\t)\n\n\tprintVersion := flag.Bool(\"version\", false, \"Print version\")\n\tverbose := flag.Bool(\"verbose\", false, \"Verbose (print config on startup)\")\n\n\tflag.Usage = func() {\n\t\tfmt.Fprintf(os.Stderr, \"Usage of %s:\\n\", os.Args[0])\n\n\t\tflag.PrintDefaults()\n\n\t\tfmt.Fprintf(os.Stderr, \"\\n\\nAdditional commands:\\n\")\n\t\tfmt.Fprintf(os.Stderr, \"\tsd-list\tList registered nodes in SD\\n\")\n\t\tfmt.Fprintf(os.Stderr, \"\tsd-delete\tDelete registered nodes for local hostname in SD\\n\")\n\t\tfmt.Fprintf(os.Stderr, \"\tsd-evict\tDelete registered nodes for hostnames in SD\\n\")\n\t\tfmt.Fprintf(os.Stderr, \"\tsd-clean\tCleanup expired registered nodes in SD\\n\")\n\t\tfmt.Fprintf(os.Stderr, \"\tsd-expired\tList expired registered nodes in SD\\n\")\n\t\tfmt.Fprintf(os.Stderr, \"\tmatch\tMatch metric against rollup rules\\n\")\n\t}\n\n\tif len(os.Args) > 1 {\n\t\tswitch os.Args[1] {\n\t\tcase \"sd-list\", \"-sd-list\":\n\t\t\tsdList(os.Args[0], os.Args[2:])\n\t\t\treturn\n\t\tcase \"sd-delete\", \"-sd-delete\":\n\t\t\tsdDelete(os.Args[0], os.Args[2:])\n\t\t\treturn\n\t\tcase \"sd-evict\", \"-sd-evict\":\n\t\t\tsdEvict(os.Args[0], os.Args[2:])\n\t\t\treturn\n\t\tcase \"sd-clean\", \"-sd-clean\":\n\t\t\tsdClean(os.Args[0], os.Args[2:])\n\t\t\treturn\n\t\tcase \"sd-expired\", \"-sd-expired\":\n\t\t\tsdExpired(os.Args[0], os.Args[2:])\n\t\t\treturn\n\t\tcase \"match\", \"-match\":\n\t\t\tcheckRollupMatch(os.Args[0], os.Args[2:])\n\t\t\treturn\n\t\t}\n\t}\n\n\tflag.Parse()\n\n\tif *printVersion {\n\t\tfmt.Print(Version)\n\t\treturn\n\t}\n\n\tif *printDefaultConfig {\n\t\tif err = config.PrintDefaultConfig(); err != nil {\n\t\t\tlog.Fatal(err)\n\t\t}\n\n\t\treturn\n\t}\n\n\tcfg, warns, err := config.ReadConfig(*configFile, *exactConfig)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\t// config parsed successfully. Exit in check-only mode\n\tif *checkConfig {\n\t\treturn\n\t}\n\n\tif err = zapwriter.ApplyConfig(cfg.Logging); err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tlocalManager, err := zapwriter.NewManager(cfg.Logging)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tlogger := localManager.Logger(\"start\")\n\n\tif len(warns) > 0 {\n\t\tzapwriter.Logger(\"config\").Warn(\"warnings\", warns...)\n\t}\n\n\tif *verbose {\n\t\tlogger.Info(\"starting graphite-clickhouse\",\n\t\t\tzap.String(\"build_version\", BuildVersion),\n\t\t\tzap.Any(\"config\", cfg),\n\t\t)\n\t} else {\n\t\tlogger.Info(\"starting graphite-clickhouse\",\n\t\t\tzap.String(\"build_version\", BuildVersion),\n\t\t)\n\t}\n\n\truntime.GOMAXPROCS(cfg.Common.MaxCPU)\n\n\tif cfg.Common.MemoryReturnInterval > 0 {\n\t\tgo func() {\n\t\t\tt := time.NewTicker(cfg.Common.MemoryReturnInterval)\n\n\t\t\tfor {\n\t\t\t\t<-t.C\n\t\t\t\tdebug.FreeOSMemory()\n\t\t\t}\n\t\t}()\n\t}\n\n\t/* CONFIG end */\n\n\tif pprof != nil && *pprof != \"\" || cfg.Common.PprofListen != \"\" {\n\t\tlisten := cfg.Common.PprofListen\n\t\tif *pprof != \"\" {\n\t\t\tlisten = *pprof\n\t\t}\n\n\t\tgo func() { log.Fatal(http.ListenAndServe(listen, nil)) }()\n\t}\n\n\t/* CONSOLE COMMANDS start */\n\tif *buildTags {\n\t\tif err := tagger.Make(cfg); err != nil {\n\t\t\tlog.Fatal(err)\n\t\t}\n\n\t\treturn\n\t}\n\n\t/* CONSOLE COMMANDS end */\n\n\tapp := App{config: cfg}\n\n\tmux := http.NewServeMux()\n\tmux.Handle(\"/_internal/capabilities/\", app.Handler(capabilities.NewHandler(cfg)))\n\tmux.Handle(\"/metrics/find/\", app.Handler(find.NewHandler(cfg)))\n\tmux.Handle(\"/metrics/index.json\", app.Handler(index.NewHandler(cfg)))\n\tmux.Handle(\"/render/\", app.Handler(render.NewHandler(cfg)))\n\tmux.Handle(\"/tags/autoComplete/tags\", app.Handler(autocomplete.NewTags(cfg)))\n\tmux.Handle(\"/tags/autoComplete/values\", app.Handler(autocomplete.NewValues(cfg)))\n\tmux.HandleFunc(\"/alive\", func(w http.ResponseWriter, r *http.Request) {\n\t\tw.WriteHeader(http.StatusOK)\n\t\tio.WriteString(w, \"Graphite-clickhouse is alive.\\n\")\n\t})\n\tmux.Handle(\"/health\", app.Handler(healthcheck.NewHandler(cfg)))\n\tmux.HandleFunc(\"/debug/config\", func(w http.ResponseWriter, r *http.Request) {\n\t\tstatus := http.StatusOK\n\t\tstart := time.Now()\n\n\t\taccessLogger := scope.LoggerWithHeaders(r.Context(), r, app.config.Common.HeadersToLog)\n\n\t\tdefer func() {\n\t\t\td := time.Since(start)\n\t\t\tlogs.AccessLog(accessLogger, app.config, r, status, d, time.Duration(0), false, false)\n\t\t}()\n\n\t\tb, err := json.MarshalIndent(cfg, \"\", \" \")\n\t\tif err != nil {\n\t\t\tstatus = http.StatusInternalServerError\n\t\t\thttp.Error(w, err.Error(), status)\n\n\t\t\treturn\n\t\t}\n\n\t\tw.Write(b)\n\t})\n\n\tif cfg.Prometheus.Listen != \"\" {\n\t\tif err := prometheus.Run(cfg); err != nil {\n\t\t\tlog.Fatal(err)\n\t\t}\n\t}\n\n\tif metrics.Graphite != nil {\n\t\tmetrics.Graphite.Start(nil)\n\t}\n\n\tvar exitWait sync.WaitGroup\n\n\tsrv = &http.Server{\n\t\tAddr: cfg.Common.Listen,\n\t\tHandler: mux,\n\t}\n\n\texitWait.Add(1)\n\n\tgo func() {\n\t\tdefer exitWait.Done()\n\n\t\tif err := srv.ListenAndServe(); err != http.ErrServerClosed {\n\t\t\t// unexpected error. port in use?\n\t\t\tlog.Fatalf(\"ListenAndServe(): %v\", err)\n\t\t}\n\t}()\n\n\tif cfg.Common.SD != \"\" && cfg.NeedLoadAvgColect() {\n\t\tgo func() {\n\t\t\ttime.Sleep(time.Millisecond * 100)\n\n\t\t\tsdLogger := localManager.Logger(\"service discovery\")\n\t\t\tsd.Register(&cfg.Common, sdLogger)\n\t\t}()\n\t}\n\n\tgo func() {\n\t\tstop := make(chan os.Signal, 1)\n\t\tsignal.Notify(stop, syscall.SIGTERM, syscall.SIGINT)\n\t\t<-stop\n\t\tlogger.Info(\"stoping graphite-clickhouse\")\n\n\t\tif cfg.Common.SD != \"\" {\n\t\t\t// unregister SD\n\t\t\tsd.Stop()\n\t\t\ttime.Sleep(10 * time.Second)\n\t\t}\n\t\t// initiating the shutdown\n\t\tctx, cancel := context.WithTimeout(context.Background(), time.Second*10)\n\t\tsrv.Shutdown(ctx)\n\t\tcancel()\n\t}()\n\n\texitWait.Wait()\n\n\tlogger.Info(\"stop graphite-clickhouse\")\n}\n" }, { "path": "healthcheck/healthcheck.go", "content": "package healthcheck\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strings\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"github.com/msaf1980/go-stringutils\"\n\t\"go.uber.org/zap\"\n)\n\n// Handler serves /render requests\ntype Handler struct {\n\tconfig *config.Config\n\tlast int64\n\tfailed int32\n}\n\n// NewHandler generates new *Handler\nfunc NewHandler(config *config.Config) *Handler {\n\th := &Handler{\n\t\tconfig: config,\n\t\tfailed: 1,\n\t}\n\n\treturn h\n}\n\nfunc (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {\n\tvar (\n\t\tquery string\n\t\tfailed int32\n\t)\n\n\tif h.config.ClickHouse.IndexTable != \"\" {\n\t\t// non-existing name with wrong level\n\t\tquery = \"SELECT Path FROM \" + h.config.ClickHouse.IndexTable + \" WHERE ((Level=20002) AND (Path IN ('NonExistient','NonExistient.'))) AND (Date='1970-02-12') GROUP BY Path FORMAT TabSeparatedRaw\"\n\t} else if h.config.ClickHouse.TaggedTable != \"\" {\n\t\t// non-existing partition\n\t\tquery = \"SELECT Path FROM \" + h.config.ClickHouse.TaggedTable + \" WHERE (Tag1='__name__=NonExistient') AND (Date='1970-02-12') GROUP BY Path FORMAT TabSeparatedRaw\"\n\t}\n\n\tif query != \"\" {\n\t\tfailed = 1\n\t\tnow := time.Now().Unix()\n\n\t\tfor {\n\t\t\tlast := atomic.LoadInt64(&h.last)\n\t\t\tif now-last < 10 {\n\t\t\t\tfailed = atomic.LoadInt32(&h.failed)\n\t\t\t\tbreak\n\t\t\t}\n\t\t\t// one query in 10 seconds for prevent overloading\n\t\t\tif !atomic.CompareAndSwapInt64(&h.last, last, now) {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tlogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named(\"healthcheck\")\n\n\t\t\tclient := http.Client{\n\t\t\t\tTimeout: 2 * time.Second,\n\t\t\t}\n\n\t\t\tvar u string\n\t\t\tif pos := strings.Index(h.config.ClickHouse.URL, \"/?\"); pos > 0 {\n\t\t\t\tu = h.config.ClickHouse.URL[:pos+2] + \"query=\" + url.QueryEscape(query)\n\t\t\t} else {\n\t\t\t\tu = h.config.ClickHouse.URL + \"/?query=\" + url.QueryEscape(query)\n\t\t\t}\n\n\t\t\treq, _ := http.NewRequest(http.MethodGet, u, nil)\n\n\t\t\tresp, err := client.Do(req)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Error(\"healthcheck error\",\n\t\t\t\t\tzap.Error(err),\n\t\t\t\t)\n\t\t\t}\n\n\t\t\tif resp.Body != nil {\n\t\t\t\tif body, err := io.ReadAll(resp.Body); err == nil {\n\t\t\t\t\tif resp.StatusCode == http.StatusOK {\n\t\t\t\t\t\tfailed = 0\n\t\t\t\t\t} else {\n\t\t\t\t\t\tfailed = 1\n\n\t\t\t\t\t\tlogger.Error(\"healthcheck error\",\n\t\t\t\t\t\t\tzap.String(\"error\", stringutils.UnsafeString(body)),\n\t\t\t\t\t\t)\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tfailed = 1\n\n\t\t\t\t\tlogger.Error(\"healthcheck error\",\n\t\t\t\t\t\tzap.Error(err),\n\t\t\t\t\t)\n\t\t\t\t}\n\n\t\t\t\tresp.Body.Close()\n\t\t\t} else {\n\t\t\t\tfailed = 1\n\n\t\t\t\tlogger.Error(\"healthcheck error\",\n\t\t\t\t\tzap.Error(err),\n\t\t\t\t)\n\t\t\t}\n\n\t\t\tatomic.StoreInt32(&h.failed, failed)\n\n\t\t\tbreak\n\t\t}\n\t}\n\n\tif failed > 0 {\n\t\thttp.Error(w, \"Storage healthcheck failed\", http.StatusServiceUnavailable)\n\t} else {\n\t\tw.WriteHeader(http.StatusOK)\n\t\tfmt.Fprintf(w, \"Graphite-clickhouse is alive.\\n\")\n\t}\n}\n" }, { "path": "helper/RowBinary/encode.go", "content": "package RowBinary\n\nimport (\n\t\"encoding/binary\"\n\t\"io\"\n\t\"math\"\n\t\"time\"\n)\n\nconst NullUint32 = ^uint32(0)\n\nfunc DateToUint16(t time.Time) uint16 {\n\treturn uint16(time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC).Unix() / 86400)\n}\n\ntype Encoder struct {\n\twrapped io.Writer\n\tbuffer []byte\n}\n\nfunc NewEncoder(w io.Writer) *Encoder {\n\treturn &Encoder{\n\t\twrapped: w,\n\t\tbuffer: make([]byte, 256),\n\t}\n}\n\nfunc (w *Encoder) Date(value time.Time) error {\n\treturn w.Uint16(DateToUint16(value))\n}\n\nfunc (w *Encoder) Uint8(value uint8) error {\n\t_, err := w.wrapped.Write([]byte{value})\n\treturn err\n}\n\nfunc (w *Encoder) Uint16(value uint16) error {\n\tbinary.LittleEndian.PutUint16(w.buffer, value)\n\t_, err := w.wrapped.Write(w.buffer[:2])\n\n\treturn err\n}\n\nfunc (w *Encoder) Uint32(value uint32) error {\n\tbinary.LittleEndian.PutUint32(w.buffer, value)\n\t_, err := w.wrapped.Write(w.buffer[:4])\n\n\treturn err\n}\n\nfunc (w *Encoder) NullableUint32(value uint32) error {\n\tif value == NullUint32 {\n\t\t_, err := w.wrapped.Write([]byte{1})\n\t\treturn err\n\t}\n\n\t_, err := w.wrapped.Write([]byte{0})\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn w.Uint32(value)\n}\n\nfunc (w *Encoder) Uint64(value uint64) error {\n\tbinary.LittleEndian.PutUint64(w.buffer, value)\n\t_, err := w.wrapped.Write(w.buffer[:8])\n\n\treturn err\n}\n\nfunc (w *Encoder) Float64(value float64) error {\n\treturn w.Uint64(math.Float64bits(value))\n}\n\nfunc (w *Encoder) NullableFloat64(value float64) error {\n\tif math.IsNaN(value) {\n\t\t_, err := w.wrapped.Write([]byte{1})\n\t\treturn err\n\t}\n\n\t_, err := w.wrapped.Write([]byte{0})\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn w.Float64(value)\n}\n\nfunc (w *Encoder) Bytes(value []byte) error {\n\tn := binary.PutUvarint(w.buffer, uint64(len(value)))\n\n\t_, err := w.wrapped.Write(w.buffer[:n])\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t_, err = w.wrapped.Write(value)\n\n\treturn err\n}\n\nfunc (w *Encoder) String(value string) error {\n\treturn w.Bytes([]byte(value))\n}\n\nfunc (w *Encoder) StringList(value []string) error {\n\tn := binary.PutUvarint(w.buffer, uint64(len(value)))\n\n\t_, err := w.wrapped.Write(w.buffer[:n])\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor i := 0; i < len(value); i++ {\n\t\terr = w.String(value[i])\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (w *Encoder) Uint32List(value []uint32) error {\n\tn := binary.PutUvarint(w.buffer, uint64(len(value)))\n\n\t_, err := w.wrapped.Write(w.buffer[:n])\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor i := 0; i < len(value); i++ {\n\t\terr = w.Uint32(value[i])\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (w *Encoder) NullableUint32List(value []uint32) error {\n\tn := binary.PutUvarint(w.buffer, uint64(len(value)))\n\n\t_, err := w.wrapped.Write(w.buffer[:n])\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor i := 0; i < len(value); i++ {\n\t\terr = w.NullableUint32(value[i])\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (w *Encoder) Float64List(value []float64) error {\n\tn := binary.PutUvarint(w.buffer, uint64(len(value)))\n\n\t_, err := w.wrapped.Write(w.buffer[:n])\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor i := 0; i < len(value); i++ {\n\t\terr = w.Float64(value[i])\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (w *Encoder) NullableFloat64List(value []float64) error {\n\tn := binary.PutUvarint(w.buffer, uint64(len(value)))\n\n\t_, err := w.wrapped.Write(w.buffer[:n])\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor i := 0; i < len(value); i++ {\n\t\terr = w.NullableFloat64(value[i])\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n" }, { "path": "helper/clickhouse/clickhouse.go", "content": "package clickhouse\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"encoding/binary\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"html\"\n\t\"io\"\n\t\"math/rand\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/errs\"\n\thttpHelper \"github.com/lomik/graphite-clickhouse/helper/http\"\n\t\"github.com/lomik/graphite-clickhouse/limiter\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\n\t\"go.uber.org/zap\"\n\t\"go.uber.org/zap/zapcore\"\n)\n\ntype ErrWithDescr struct {\n\terr string\n\tdata string\n}\n\ntype ContentEncoding string\n\nconst (\n\tContentEncodingNone ContentEncoding = \"none\"\n\tContentEncodingGzip ContentEncoding = \"gzip\"\n\tContentEncodingZstd ContentEncoding = \"zstd\"\n)\n\nconst (\n\tClickHouseProgressHeader string = \"X-Clickhouse-Progress\"\n\tClickHouseSummaryHeader string = \"X-Clickhouse-Summary\"\n)\n\nfunc NewErrWithDescr(err string, data string) error {\n\treturn &ErrWithDescr{err, data}\n}\n\nfunc (e *ErrWithDescr) Error() string {\n\treturn e.err + \": \" + e.data\n}\n\nfunc (e *ErrWithDescr) PrependDescription(test string) {\n\te.data = test + e.data\n}\n\nvar ErrInvalidTimeRange = errors.New(\"Invalid or empty time range\")\nvar ErrUvarintRead = errors.New(\"ReadUvarint: Malformed array\")\nvar ErrUvarintOverflow = errors.New(\"ReadUvarint: varint overflows a 64-bit integer\")\nvar ErrClickHouseResponse = errors.New(\"Malformed response from clickhouse\")\n\nfunc extractClickhouseError(e string) (int, string) {\n\tif strings.HasPrefix(e, \"clickhouse response status 500: Code:\") || strings.HasPrefix(e, \"Malformed response from clickhouse\") {\n\t\tif start := strings.Index(e, \": Limit for \"); start != -1 {\n\t\t\te := e[start+8:]\n\t\t\tif end := strings.Index(e, \" (version \"); end != -1 {\n\t\t\t\te = e[0:end]\n\t\t\t}\n\n\t\t\treturn http.StatusForbidden, \"Storage read limit \" + e\n\t\t} else if start := strings.Index(e, \": Memory limit \"); start != -1 {\n\t\t\treturn http.StatusForbidden, \"Storage read limit for memory\"\n\t\t} else if strings.HasPrefix(e, \"clickhouse response status 500: Code: 170,\") {\n\t\t\t// distributed table configuration error\n\t\t\t// clickhouse response status 500: Code: 170, e.displayText() = DB::Exception: Requested cluster 'cluster' not found\n\t\t\treturn http.StatusServiceUnavailable, \"Storage configuration error\"\n\t\t}\n\t}\n\n\tif strings.HasPrefix(e, \"clickhouse response status 404: Code: 60. DB::Exception: Table default.\") {\n\t\treturn http.StatusServiceUnavailable, \"Storage default tables damaged\"\n\t}\n\n\tif strings.HasPrefix(e, \"clickhouse response status 500: Code: 427\") || strings.HasPrefix(e, \"clickhouse response status 400: Code: 427.\") {\n\t\treturn http.StatusBadRequest, \"Incorrect regex syntax\"\n\t}\n\n\treturn http.StatusServiceUnavailable, \"Storage unavailable\"\n}\n\nfunc HandleError(w http.ResponseWriter, err error) (status int, queueFail bool) {\n\tstatus = http.StatusOK\n\terrStr := err.Error()\n\n\tif err == ErrInvalidTimeRange {\n\t\tstatus = http.StatusBadRequest\n\t\thttp.Error(w, errStr, status)\n\n\t\treturn\n\t}\n\n\tif err == limiter.ErrTimeout || err == limiter.ErrOverflow {\n\t\tqueueFail = true\n\t\tstatus = http.StatusServiceUnavailable\n\t\thttp.Error(w, err.Error(), status)\n\n\t\treturn\n\t}\n\n\tif _, ok := err.(*ErrWithDescr); ok {\n\t\tstatus, errStr = extractClickhouseError(errStr)\n\t\thttp.Error(w, errStr, status)\n\n\t\treturn\n\t}\n\n\tnetErr, ok := err.(net.Error)\n\tif ok {\n\t\tif netErr.Timeout() {\n\t\t\tstatus = http.StatusGatewayTimeout\n\t\t\thttp.Error(w, \"Storage read timeout\", status)\n\t\t} else if strings.HasSuffix(errStr, \"connect: no route to host\") ||\n\t\t\tstrings.HasPrefix(errStr, \"dial tcp: lookup \") { // DNS lookup\n\t\t\tstatus = http.StatusServiceUnavailable\n\t\t\thttp.Error(w, \"Storage route error\", status)\n\t\t} else if strings.HasSuffix(errStr, \"connect: connection refused\") ||\n\t\t\tstrings.HasSuffix(errStr, \": connection reset by peer\") {\n\t\t\tstatus = http.StatusServiceUnavailable\n\t\t\thttp.Error(w, \"Storage connect error\", status)\n\t\t} else {\n\t\t\tstatus = http.StatusServiceUnavailable\n\t\t\thttp.Error(w, \"Storage network error\", status)\n\t\t}\n\n\t\treturn\n\t}\n\n\terrCode, ok := err.(errs.ErrorWithCode)\n\tif ok {\n\t\tif (errCode.Code > 500 && errCode.Code < 512) ||\n\t\t\terrCode.Code == http.StatusBadRequest || errCode.Code == http.StatusForbidden {\n\t\t\tstatus = errCode.Code\n\t\t\thttp.Error(w, html.EscapeString(errStr), status)\n\t\t} else {\n\t\t\tstatus = http.StatusInternalServerError\n\t\t\thttp.Error(w, html.EscapeString(errStr), status)\n\t\t}\n\n\t\treturn\n\t}\n\n\tif errors.Is(err, context.Canceled) {\n\t\tstatus = http.StatusGatewayTimeout\n\t\thttp.Error(w, \"Storage read context canceled\", status)\n\t} else {\n\t\t//logger.Debug(\"query\", zap.Error(err))\n\t\tstatus = http.StatusInternalServerError\n\t\thttp.Error(w, html.EscapeString(errStr), status)\n\t}\n\n\treturn\n}\n\ntype Options struct {\n\tTLSConfig *tls.Config\n\tTimeout time.Duration\n\tConnectTimeout time.Duration\n\tProgressSendingInterval time.Duration\n\tCheckRequestProgress bool\n}\n\ntype LoggedReader struct {\n\treader io.ReadCloser\n\tlogger *zap.Logger\n\tstart time.Time\n\tfinished bool\n\tqueryID string\n\tread_rows int64\n\tread_bytes int64\n}\n\nfunc (r *LoggedReader) Read(p []byte) (int, error) {\n\tn, err := r.reader.Read(p)\n\tif err != nil && !r.finished {\n\t\tr.finished = true\n\t\tr.logger.Info(\"query\", zap.String(\"query_id\", r.queryID), zap.Duration(\"time\", time.Since(r.start)))\n\t}\n\n\treturn n, err\n}\n\nfunc (r *LoggedReader) Close() error {\n\terr := r.reader.Close()\n\n\tif !r.finished {\n\t\tr.finished = true\n\t\tr.logger.Info(\"query\", zap.String(\"query_id\", r.queryID), zap.Duration(\"time\", time.Since(r.start)))\n\t}\n\n\treturn err\n}\n\nfunc (r *LoggedReader) ChReadRows() int64 {\n\treturn r.read_rows\n}\n\nfunc (r *LoggedReader) ChReadBytes() int64 {\n\treturn r.read_bytes\n}\n\ntype queryStats struct {\n\treadRows int64\n\treadBytes int64\n\tloggerFields []zapcore.Field\n\trawHeader string\n}\n\nfunc formatSQL(q string) string {\n\ts := strings.Split(q, \"\\n\")\n\tfor i := 0; i < len(s); i++ {\n\t\ts[i] = strings.TrimSpace(s[i])\n\t}\n\n\treturn strings.Join(s, \" \")\n}\n\nfunc Query(ctx context.Context, dsn string, query string, opts Options, extData *ExternalData) ([]byte, int64, int64, error) {\n\treturn Post(ctx, dsn, query, nil, opts, extData)\n}\n\nfunc Post(ctx context.Context, dsn string, query string, postBody io.Reader, opts Options, extData *ExternalData) ([]byte, int64, int64, error) {\n\treturn do(ctx, dsn, query, postBody, ContentEncodingNone, opts, extData)\n}\n\n// Deprecated: use PostWithEncoding instead\nfunc PostGzip(ctx context.Context, dsn string, query string, postBody io.Reader, opts Options, extData *ExternalData) ([]byte, int64, int64, error) {\n\treturn do(ctx, dsn, query, postBody, ContentEncodingGzip, opts, extData)\n}\n\nfunc PostWithEncoding(ctx context.Context, dsn string, query string, postBody io.Reader, encoding ContentEncoding, opts Options, extData *ExternalData) ([]byte, int64, int64, error) {\n\treturn do(ctx, dsn, query, postBody, encoding, opts, extData)\n}\n\nfunc Reader(ctx context.Context, dsn string, query string, opts Options, extData *ExternalData) (*LoggedReader, error) {\n\treturn reader(ctx, dsn, query, nil, ContentEncodingNone, opts, extData)\n}\n\nfunc reader(ctx context.Context, dsn string, query string, postBody io.Reader, encoding ContentEncoding, opts Options, extData *ExternalData) (bodyReader *LoggedReader, err error) {\n\tif postBody != nil && extData != nil {\n\t\terr = fmt.Errorf(\"postBody and extData could not be passed in one request\")\n\t\treturn\n\t}\n\n\tvar chQueryID string\n\n\tstart := time.Now()\n\n\trequestID := scope.RequestID(ctx)\n\n\tqueryForLogger := query\n\tif len(queryForLogger) > 500 {\n\t\tqueryForLogger = queryForLogger[:395] + \"<...>\" + queryForLogger[len(queryForLogger)-100:]\n\t}\n\n\tlogger := scope.Logger(ctx).With(zap.String(\"query\", formatSQL(queryForLogger)))\n\n\tdefer func() {\n\t\t// fmt.Println(time.Since(start), formatSQL(queryForLogger))\n\t\tif err != nil {\n\t\t\tlogger.Error(\"query\", zap.Error(err), zap.Duration(\"time\", time.Since(start)))\n\t\t}\n\t}()\n\n\tp, err := url.Parse(dsn)\n\tif err != nil {\n\t\treturn\n\t}\n\n\tvar b [8]byte\n\n\tbinary.LittleEndian.PutUint64(b[:], rand.Uint64())\n\tqueryID := fmt.Sprintf(\"%x\", b)\n\n\tq := p.Query()\n\tq.Set(\"query_id\", fmt.Sprintf(\"%s::%s\", requestID, queryID))\n\t// Get X-Clickhouse-Summary header\n\t// TODO: remove when https://github.com/ClickHouse/ClickHouse/issues/16207 is done\n\tq.Set(\"send_progress_in_http_headers\", \"1\")\n\tq.Set(\"http_headers_progress_interval_ms\", strconv.FormatInt(opts.ProgressSendingInterval.Milliseconds(), 10))\n\tp.RawQuery = q.Encode()\n\n\tvar contentHeader string\n\n\tif postBody != nil {\n\t\tq := p.Query()\n\t\tq.Set(\"query\", query)\n\t\tp.RawQuery = q.Encode()\n\t} else if extData != nil {\n\t\tq := p.Query()\n\t\tq.Set(\"query\", query)\n\t\tp.RawQuery = q.Encode()\n\n\t\tpostBody, contentHeader, err = extData.buildBody(ctx, p)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t} else {\n\t\tpostBody = strings.NewReader(query)\n\t}\n\n\turl := p.String()\n\n\treq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, postBody)\n\tif err != nil {\n\t\treturn\n\t}\n\n\treq.Header.Add(\"User-Agent\", scope.ClickhouseUserAgent(ctx))\n\n\tif contentHeader != \"\" {\n\t\treq.Header.Add(\"Content-Type\", contentHeader)\n\t}\n\n\tswitch encoding {\n\tcase ContentEncodingNone:\n\t\t// no encoding\n\tcase ContentEncodingGzip:\n\t\treq.Header.Add(\"Content-Encoding\", \"gzip\")\n\tcase ContentEncodingZstd:\n\t\treq.Header.Add(\"Content-Encoding\", \"zstd\")\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"unknown encoding: %s\", encoding)\n\t}\n\n\tvar resp *http.Response\n\tif opts.CheckRequestProgress {\n\t\tresp, err = sendRequestWithProgressCheck(req, &opts)\n\t} else {\n\t\tresp, err = sendRequestViaDefaultClient(req, &opts)\n\t}\n\n\tif err != nil {\n\t\tif opts.CheckRequestProgress && resp != nil {\n\t\t\tstats, parse_err := getQueryStats(resp, ClickHouseProgressHeader)\n\t\t\tif parse_err != nil {\n\t\t\t\tlogger.Warn(\"query\", zap.Error(err), zap.String(\"clickhouse-progress\", stats.rawHeader))\n\t\t\t}\n\n\t\t\tlogger = logger.With(stats.loggerFields...)\n\t\t}\n\n\t\treturn\n\t}\n\n\t// chproxy overwrite our query id. So read it again\n\tchQueryID = resp.Header.Get(\"X-ClickHouse-Query-Id\")\n\n\tstats, err := getQueryStats(resp, ClickHouseSummaryHeader)\n\tif err != nil {\n\t\tsummaryHeader := resp.Header.Get(ClickHouseSummaryHeader)\n\t\tlogger.Warn(\"query\",\n\t\t\tzap.Error(err),\n\t\t\tzap.String(\"clickhouse-summary\", summaryHeader))\n\n\t\terr = nil\n\t}\n\n\tread_rows, read_bytes, fields := stats.readRows, stats.readBytes, stats.loggerFields\n\n\tif len(fields) > 0 {\n\t\tsort.Slice(fields, func(i, j int) bool {\n\t\t\treturn fields[i].Key < fields[j].Key\n\t\t})\n\n\t\tlogger = logger.With(fields...)\n\t}\n\n\t// check for return 5xx error, may be 502 code if clickhouse accesed via reverse proxy\n\tif resp.StatusCode > http.StatusInternalServerError && resp.StatusCode < 512 {\n\t\tbody, _ := io.ReadAll(resp.Body)\n\t\tresp.Body.Close()\n\t\terr = errs.NewErrorWithCode(string(body), resp.StatusCode)\n\n\t\treturn\n\t} else if resp.StatusCode != http.StatusOK {\n\t\tbody, _ := io.ReadAll(resp.Body)\n\t\tresp.Body.Close()\n\t\terr = NewErrWithDescr(\"clickhouse response status \"+strconv.Itoa(resp.StatusCode), string(body))\n\n\t\treturn\n\t}\n\n\tbodyReader = &LoggedReader{\n\t\treader: resp.Body,\n\t\tlogger: logger,\n\t\tstart: start,\n\t\tqueryID: chQueryID,\n\t\tread_rows: read_rows,\n\t\tread_bytes: read_bytes,\n\t}\n\n\treturn\n}\n\nfunc getQueryStats(resp *http.Response, statsHeaderName string) (queryStats, error) {\n\tread_rows := int64(-1)\n\tread_bytes := int64(-1)\n\n\tif resp == nil {\n\t\treturn queryStats{\n\t\t\treadRows: read_rows,\n\t\t\treadBytes: read_bytes,\n\t\t\tloggerFields: []zapcore.Field{},\n\t\t}, nil\n\t}\n\n\tstatsHeader := \"\"\n\tstatsHeaders := resp.Header.Values(statsHeaderName)\n\n\tif len(statsHeaders) > 0 {\n\t\tstatsHeader = statsHeaders[len(statsHeaders)-1]\n\t} else {\n\t\treturn queryStats{\n\t\t\treadRows: read_rows,\n\t\t\treadBytes: read_bytes,\n\t\t\tloggerFields: []zapcore.Field{},\n\t\t}, nil\n\t}\n\n\tstats := make(map[string]string)\n\n\terr := json.Unmarshal([]byte(statsHeader), &stats)\n\tif err != nil {\n\t\treturn queryStats{\n\t\t\treadRows: read_rows,\n\t\t\treadBytes: read_bytes,\n\t\t\tloggerFields: []zapcore.Field{},\n\t\t\trawHeader: statsHeader,\n\t\t}, err\n\t}\n\n\t// TODO: use in carbon metrics sender when it will be implemented\n\tfields := make([]zapcore.Field, 0, len(stats))\n\tfor k, v := range stats {\n\t\tfields = append(fields, zap.String(k, v))\n\n\t\tswitch k {\n\t\tcase \"read_rows\":\n\t\t\tread_rows, _ = strconv.ParseInt(v, 10, 64)\n\t\tcase \"read_bytes\":\n\t\t\tread_bytes, _ = strconv.ParseInt(v, 10, 64)\n\t\t}\n\t}\n\n\tsort.Slice(fields, func(i int, j int) bool {\n\t\treturn fields[i].Key < fields[j].Key\n\t})\n\n\treturn queryStats{\n\t\treadRows: read_rows,\n\t\treadBytes: read_bytes,\n\t\tloggerFields: fields,\n\t\trawHeader: statsHeader,\n\t}, nil\n}\n\nfunc sendRequestViaDefaultClient(request *http.Request, opts *Options) (*http.Response, error) {\n\tclient := &http.Client{\n\t\tTimeout: opts.Timeout,\n\t\tTransport: &http.Transport{\n\t\t\tDialContext: (&net.Dialer{\n\t\t\t\tTimeout: opts.ConnectTimeout,\n\t\t\t}).DialContext,\n\t\t\tTLSClientConfig: opts.TLSConfig,\n\t\t\tDisableKeepAlives: true,\n\t\t},\n\t}\n\n\treturn client.Do(request)\n}\n\nfunc sendRequestWithProgressCheck(request *http.Request, opts *Options) (*http.Response, error) {\n\ttransport := &http.Transport{\n\t\tDialContext: (&net.Dialer{\n\t\t\tTimeout: opts.ConnectTimeout,\n\t\t}).DialContext,\n\t\tTLSClientConfig: opts.TLSConfig,\n\t\tDisableKeepAlives: true,\n\t}\n\n\treturn httpHelper.DoHTTPOverTCP(request.Context(), transport, request, opts.Timeout)\n}\n\nfunc do(ctx context.Context, dsn string, query string, postBody io.Reader, encoding ContentEncoding, opts Options, extData *ExternalData) ([]byte, int64, int64, error) {\n\tbodyReader, err := reader(ctx, dsn, query, postBody, encoding, opts, extData)\n\tif err != nil {\n\t\treturn nil, 0, 0, err\n\t}\n\n\tbody, err := io.ReadAll(bodyReader)\n\tbodyReader.Close()\n\n\tif err != nil {\n\t\treturn nil, bodyReader.ChReadRows(), bodyReader.ChReadBytes(), err\n\t}\n\n\treturn body, bodyReader.ChReadRows(), bodyReader.ChReadBytes(), nil\n}\n\nfunc ReadUvarint(array []byte) (uint64, int, error) {\n\tvar x uint64\n\n\tvar s uint\n\n\tl := len(array) - 1\n\n\tfor i := 0; ; i++ {\n\t\tif i > l {\n\t\t\treturn x, i + 1, ErrUvarintRead\n\t\t}\n\n\t\tif array[i] < 0x80 {\n\t\t\tif i > 9 || i == 9 && array[i] > 1 {\n\t\t\t\treturn x, i + 1, ErrUvarintOverflow\n\t\t\t}\n\n\t\t\treturn x | uint64(array[i])< match(x, '^t=.**a*') UInt8 : 1': while executing 'FUNCTION arrayExists(__lambda_11 :: 7, Tags :: 3) -> arrayExists(lambda(tuple(x), and(like(x, 't=%'), match(x, '^t=.**a*'))), Tags) UInt8 : 6' (version 21.3.20.1 (official build))`,\n\t\t\twantStatus: http.StatusBadRequest,\n\t\t\twantMessage: \"Incorrect regex syntax\",\n\t\t},\n\t\t{\n\t\t\terrStr: `clickhouse response status 500: Code: 427. DB::Exception: OptimizedRegularExpression: cannot compile re2: ^t=.**a*, error: bad repetition operator: **. Look at https://github.com/google/re2/wiki/Syntax for reference. Please note that if you specify regex as an SQL string literal, the slashes have to be additionally escaped. For example, to match an opening brace, write '\\(' -- the first slash is for SQL and the second one is for regex: while executing 'FUNCTION and(like(x, 't=%') :: 3, match(x, '^t=.**a*') :: 1) -> and(like(x, 't=%'), match(x, '^t=.**a*')) UInt8 : 2': while executing 'FUNCTION and(and(greaterOrEquals(Date, '2024-10-28'), lessOrEquals(Date, '2024-10-28')) :: 0, and(equals(Tag1, '__name__=request_success_total.counter'), arrayExists(lambda(tuple(x), equals(x, 'app=test')), Tags), arrayExists(lambda(tuple(x), equals(x, 'project=Test')), Tags), arrayExists(lambda(tuple(x), equals(x, 'environment=TEST')), Tags), arrayExists(lambda(tuple(x), and(like(x, 't=%'), match(x, '^t=.**a*'))), Tags)) :: 3) -> and(and(greaterOrEquals(Date, '2024-10-28'), lessOrEquals(Date, '2024-10-28')), and(equals(Tag1, '__name__=request_success_total.counter'), arrayExists(lambda(tuple(x), equals(x, 'app=test')), Tags), arrayExists(lambda(tuple(x), equals(x, 'project=Test')), Tags), arrayExists(lambda(tuple(x), equals(x, 'environment=TEST')), Tags), arrayExists(lambda(tuple(x), and(like(x, 't=%'), match(x, '^t=.**a*'))), Tags))) UInt8 : 6'. (CANNOT_COMPILE_REGEXP) (version 22.8.21.38 (official build))`,\n\t\t\twantStatus: http.StatusBadRequest,\n\t\t\twantMessage: \"Incorrect regex syntax\",\n\t\t},\n\t\t{\n\t\t\terrStr: \"Other error\",\n\t\t\twantStatus: http.StatusServiceUnavailable,\n\t\t\twantMessage: \"Storage unavailable\",\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.errStr, func(t *testing.T) {\n\t\t\tgotStatus, gotMessage := extractClickhouseError(tt.errStr)\n\t\t\tassert.Equal(t, tt.wantStatus, gotStatus)\n\t\t\tassert.Equal(t, tt.wantMessage, gotMessage)\n\t\t})\n\t}\n}\n" }, { "path": "helper/clickhouse/external-data.go", "content": "package clickhouse\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"mime/multipart\"\n\t\"net/url\"\n\t\"os\"\n\t\"path\"\n\t\"strings\"\n\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"go.uber.org/zap\"\n)\n\n// ExternalTable is a structure to use ClickHouse feature that creates a temporary table for a query\ntype ExternalTable struct {\n\t// Table name\n\tName string\n\tColumns []Column\n\t// ClickHouse input/output format\n\tFormat string\n\tData []byte\n}\n\n// Column is a pair of Name and Type for temporary table structure\ntype Column struct {\n\tName string\n\t// ClickHouse data type\n\tType string\n}\n\nfunc (c *Column) String() string {\n\treturn c.Name + \" \" + c.Type\n}\n\n// ExternalData is a type to use ClickHouse external data feature. You could use it to pass multiple\n// temporary tables for a query.\ntype ExternalData struct {\n\tTables []ExternalTable\n\tdebug *extDataDebug\n}\n\ntype extDataDebug struct {\n\tdir string\n\tperm os.FileMode\n}\n\n// NewExternalData returns the `*ExternalData` object for `tables`\nfunc NewExternalData(tables ...ExternalTable) *ExternalData {\n\treturn &ExternalData{Tables: tables, debug: nil}\n}\n\n// SetDebug sets the directory and file permission for an external table data dump. Works only if\n// both `debugDir` and `perm` are set\nfunc (e *ExternalData) SetDebug(debugDir string, perm os.FileMode) {\n\tif debugDir == \"\" || perm == 0 {\n\t\te.debug = nil\n\t}\n\n\te.debug = &extDataDebug{debugDir, perm}\n}\n\n// buildBody returns multiform body, content type header and error\nfunc (e *ExternalData) buildBody(ctx context.Context, u *url.URL) (*bytes.Buffer, string, error) {\n\tbody := new(bytes.Buffer)\n\theader := \"\"\n\twriter := multipart.NewWriter(body)\n\n\tfor _, t := range e.Tables {\n\t\tpart, err := writer.CreateFormFile(t.Name, t.Name)\n\t\tif err != nil {\n\t\t\treturn nil, header, err\n\t\t}\n\n\t\t// Send each table in separated form\n\t\t_, err = part.Write(t.Data)\n\t\tif err != nil {\n\t\t\treturn nil, header, err\n\t\t}\n\n\t\t// Set name_format and name_structure for the table\n\t\tq := u.Query()\n\t\tif t.Format != \"\" {\n\t\t\tq.Set(t.Name+\"_format\", t.Format)\n\t\t}\n\n\t\tstructure := make([]string, 0, len(t.Columns))\n\t\tfor _, c := range t.Columns {\n\t\t\tstructure = append(structure, c.String())\n\t\t}\n\n\t\tq.Set(t.Name+\"_structure\", strings.Join(structure, \",\"))\n\t\tu.RawQuery = q.Encode()\n\t}\n\n\terr := writer.Close()\n\tif err != nil {\n\t\treturn nil, header, err\n\t}\n\n\theader = writer.FormDataContentType()\n\tdu := *u\n\t// Do not lock the execution by debugging process\n\tgo e.debugDump(ctx, du)\n\n\treturn body, header, nil\n}\n\nfunc (e *ExternalData) debugDump(ctx context.Context, u url.URL) {\n\tif e.debug == nil || !scope.Debug(ctx, \"External-Data\") {\n\t\t// Do not dump if the settings are not set\n\t\treturn\n\t}\n\n\trequestID := scope.RequestID(ctx)\n\tlogger := scope.Logger(ctx)\n\tcommand := \"curl \"\n\n\tfor _, t := range e.Tables {\n\t\tfilename := path.Join(e.debug.dir, fmt.Sprintf(\"ext-%v:%v.%v\", t.Name, requestID, t.Format))\n\n\t\terr := os.WriteFile(filename, t.Data, e.debug.perm)\n\t\tif err != nil {\n\t\t\tlogger.Warn(\"external-data\", zap.Error(err))\n\t\t\t// The debug command couldn't be built w/o all external tables\n\t\t\treturn\n\t\t}\n\n\t\tcommand += fmt.Sprintf(\"-F '%v=@%v;' \", t.Name, filename)\n\t}\n\n\t// Change query_id to not interfere with the original one\n\tq := u.Query()\n\tq[\"query_id\"] = []string{fmt.Sprintf(\"%v:debug\", requestID)}\n\tu.RawQuery = q.Encode()\n\n\tcommand += \"'\" + u.Redacted() + \"'\"\n\n\tlogger.Info(\"external-data\", zap.String(\"debug command\", command))\n}\n" }, { "path": "helper/clickhouse/external-data_test.go", "content": "package clickhouse\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"net/url\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc getTestCases() (tables []ExternalTable) {\n\ttables = []ExternalTable{\n\t\t{\n\t\t\tName: \"test1\",\n\t\t\tColumns: []Column{\n\t\t\t\t{\n\t\t\t\t\tName: \"aString\",\n\t\t\t\t\tType: \"String\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tName: \"anInt\",\n\t\t\t\t\tType: \"Int32\",\n\t\t\t\t},\n\t\t\t},\n\t\t\tFormat: \"TSV\",\n\t\t\tData: []byte(`f\t3`),\n\t\t},\n\t\t{\n\t\t\tName: \"test2\",\n\t\t\tColumns: []Column{\n\t\t\t\t{\n\t\t\t\t\tName: \"aFloat\",\n\t\t\t\t\tType: \"Float32\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tName: \"aDate\",\n\t\t\t\t\tType: \"Date\",\n\t\t\t\t},\n\t\t\t},\n\t\t\tFormat: \"TSKV\",\n\t\t\tData: []byte(`aFloat=13.13\taDate=2013-12-13`),\n\t\t},\n\t}\n\n\treturn\n}\n\nfunc TestColumnString(t *testing.T) {\n\ttables := getTestCases()\n\tfor _, table := range tables {\n\t\tfor _, c := range table.Columns {\n\t\t\tassert.Equal(t, c.Name+\" \"+c.Type, c.String(), \"Column.String doesn't work\")\n\t\t}\n\t}\n}\n\nfunc TestNewExternalData(t *testing.T) {\n\ttables := getTestCases()\n\tfor _, tt := range [][]ExternalTable{tables, tables[0:1], tables[1:]} {\n\t\textData := NewExternalData(tt...)\n\t\tassert.ElementsMatch(t, extData.Tables, tt, \"tables don't match ExternalData\")\n\t}\n}\n\nfunc TestBuildBody(t *testing.T) {\n\ttables := getTestCases()\n\tfor _, tt := range [][]ExternalTable{tables, tables[0:1], tables[1:]} {\n\t\textData := NewExternalData(tt...)\n\t\tu := &url.URL{}\n\t\tbody, header, err := extData.buildBody(context.Background(), u)\n\t\tassert.NoError(t, err, \"body is not built\")\n\t\tassert.Regexp(t, \"^multipart/form-data; boundary=[A-Fa-f0-9]+$\", header, \"header does not match\")\n\t\tcontentID := strings.TrimPrefix(header, \"multipart/form-data; boundary=\")\n\n\t\tvar b string\n\n\t\tvals := make(url.Values)\n\n\t\tfor _, table := range tt {\n\t\t\tb += \"--\" + contentID\n\t\t\tb += \"\\r\\nContent-Disposition: form-data; name=\\\"\" + table.Name + \"\\\"; filename=\\\"\" + table.Name + \"\\\"\\r\\n\"\n\t\t\tb += \"Content-Type: application/octet-stream\\r\\n\\r\\n\" + string(table.Data) + \"\\r\\n\"\n\t\t\tvals[table.Name+\"_format\"] = []string{table.Format}\n\t\t\tvals[table.Name+\"_structure\"] = make([]string, 0)\n\n\t\t\tfor _, c := range table.Columns {\n\t\t\t\tvals[table.Name+\"_structure\"] = append(vals[table.Name+\"_structure\"], c.String())\n\t\t\t}\n\t\t}\n\n\t\tb += \"--\" + contentID + \"--\\r\\n\"\n\t\tassert.Equal(t, b, body.String(), \"built body and expected body don't match\")\n\t}\n}\n\nfunc TestDebugDump(t *testing.T) {\n\textData := NewExternalData(getTestCases()...)\n\n\tdir, err := os.MkdirTemp(\".\", \"external-data\")\n\tif err != nil {\n\t\tt.Fatalf(\"unable to create directory %s: %v\", dir, err)\n\t}\n\n\tdefer os.RemoveAll(dir)\n\n\treqID := fmt.Sprintf(\"%x\", rand.Uint32())\n\tctx := scope.WithRequestID(context.Background(), reqID)\n\tctx = scope.WithDebug(ctx, \"External-Data\")\n\n\textData.SetDebug(dir, 0640)\n\n\tu := url.URL{}\n\textData.debugDump(ctx, u)\n\n\tfor _, table := range extData.Tables {\n\t\tdumpFile := filepath.Join(dir, fmt.Sprintf(\"ext-%v:%v.%v\", table.Name, reqID, table.Format))\n\t\tassert.FileExists(t, dumpFile)\n\t\tdata, err := os.ReadFile(dumpFile)\n\t\tassert.NoError(t, err, \"unable to read dump file: %w\", err)\n\t\tassert.Equal(t, table.Data, data, \"data in the file and source are different\")\n\t}\n}\n" }, { "path": "helper/client/datetime.go", "content": "package client\n\nimport (\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/datetime\"\n)\n\nfunc MetricsTimestampTruncate(metrics []Metric, precision time.Duration) {\n\tif precision == 0 {\n\t\treturn\n\t}\n\n\tfor i := range metrics {\n\t\tmetrics[i].StartTime = datetime.TimestampTruncate(metrics[i].StartTime, precision)\n\t\tmetrics[i].StopTime = datetime.TimestampTruncate(metrics[i].StopTime, precision)\n\t\tmetrics[i].RequestStartTime = datetime.TimestampTruncate(metrics[i].RequestStartTime, precision)\n\t\tmetrics[i].RequestStopTime = datetime.TimestampTruncate(metrics[i].RequestStopTime, precision)\n\t}\n}\n" }, { "path": "helper/client/errros.go", "content": "package client\n\nimport \"strconv\"\n\ntype HttpError struct {\n\tstatusCode int\n\tmessage string\n}\n\nfunc NewHttpError(statusCode int, message string) *HttpError {\n\treturn &HttpError{\n\t\tstatusCode: statusCode,\n\t\tmessage: message,\n\t}\n}\n\nfunc (e *HttpError) Error() string {\n\treturn strconv.Itoa(e.statusCode) + \": \" + e.message\n}\n" }, { "path": "helper/client/find.go", "content": "package client\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\n\tprotov2 \"github.com/go-graphite/protocol/carbonapi_v2_pb\"\n\tprotov3 \"github.com/go-graphite/protocol/carbonapi_v3_pb\"\n\n\tpickle \"github.com/lomik/og-rek\"\n)\n\ntype FindMatch struct {\n\tPath string `toml:\"path\"`\n\tIsLeaf bool `toml:\"is_leaf\"`\n}\n\n// MetricsFind do /metrics/find/ request\n// Valid formats are carbonapi_v3_pb. protobuf, pickle\nfunc MetricsFind(client *http.Client, address string, format FormatType, query string, from, until int64) (string, []FindMatch, http.Header, error) {\n\tif format == FormatDefault {\n\t\tformat = FormatPb_v3\n\t}\n\n\trUrl := \"/metrics/find/\"\n\n\tqueryParams := fmt.Sprintf(\"%s?format=%s, from=%d, until=%d, query %s\", rUrl, format.String(), from, until, query)\n\n\tvar fromStr, untilStr string\n\n\tu, err := url.Parse(address + rUrl)\n\tif err != nil {\n\t\treturn queryParams, nil, nil, err\n\t}\n\n\tv := url.Values{\n\t\t\"format\": []string{format.String()},\n\t}\n\n\tvar reader io.Reader\n\n\tswitch format {\n\tcase FormatPb_v3:\n\t\tvar body []byte\n\n\t\tr := protov3.MultiGlobRequest{\n\t\t\tMetrics: []string{query},\n\t\t\tStartTime: from,\n\t\t\tStopTime: until,\n\t\t}\n\n\t\tbody, err = r.Marshal()\n\t\tif err != nil {\n\t\t\treturn query, nil, nil, err\n\t\t}\n\n\t\tif body != nil {\n\t\t\treader = bytes.NewReader(body)\n\t\t}\n\tcase FormatProtobuf, FormatPickle:\n\t\tv[\"query\"] = []string{query}\n\t\tif from > 0 {\n\t\t\tv[\"from\"] = []string{fromStr}\n\t\t}\n\n\t\tif until > 0 {\n\t\t\tv[\"until\"] = []string{untilStr}\n\t\t}\n\tdefault:\n\t\treturn queryParams, nil, nil, ErrUnsupportedFormat\n\t}\n\n\tu.RawQuery = v.Encode()\n\n\treq, err := http.NewRequest(http.MethodGet, u.String(), reader)\n\tif err != nil {\n\t\treturn queryParams, nil, nil, err\n\t}\n\n\tresp, err := client.Do(req)\n\tif err != nil {\n\t\treturn queryParams, nil, nil, err\n\t}\n\n\tdefer resp.Body.Close()\n\n\tb, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\treturn queryParams, nil, nil, err\n\t}\n\n\tif resp.StatusCode == http.StatusNotFound {\n\t\treturn queryParams, nil, resp.Header, nil\n\t} else if resp.StatusCode != http.StatusOK {\n\t\treturn queryParams, nil, resp.Header, NewHttpError(resp.StatusCode, string(b))\n\t}\n\n\tvar globs []FindMatch\n\n\tswitch format {\n\tcase FormatProtobuf:\n\t\tvar globsv2 protov2.GlobResponse\n\t\tif err = globsv2.Unmarshal(b); err != nil {\n\t\t\treturn queryParams, nil, resp.Header, err\n\t\t}\n\n\t\tfor _, m := range globsv2.Matches {\n\t\t\tglobs = append(globs, FindMatch{Path: m.Path, IsLeaf: m.IsLeaf})\n\t\t}\n\tcase FormatPb_v3:\n\t\tvar globsv3 protov3.MultiGlobResponse\n\t\tif err = globsv3.Unmarshal(b); err != nil {\n\t\t\treturn queryParams, nil, resp.Header, err\n\t\t}\n\n\t\tfor _, m := range globsv3.Metrics {\n\t\t\tfor _, v := range m.Matches {\n\t\t\t\tglobs = append(globs, FindMatch{Path: v.Path, IsLeaf: v.IsLeaf})\n\t\t\t}\n\t\t}\n\tcase FormatPickle:\n\t\treader := bytes.NewReader(b)\n\t\tdecoder := pickle.NewDecoder(reader)\n\n\t\tp, err := decoder.Decode()\n\t\tif err != nil {\n\t\t\treturn queryParams, nil, resp.Header, err\n\t\t}\n\n\t\tfor _, v := range p.([]interface{}) {\n\t\t\tm := v.(map[interface{}]interface{})\n\t\t\tpath := m[\"metric_path\"].(string)\n\t\t\tisLeaf := m[\"isLeaf\"].(bool)\n\t\t\tglobs = append(globs, FindMatch{Path: path, IsLeaf: isLeaf})\n\t\t}\n\tdefault:\n\t\treturn queryParams, nil, resp.Header, ErrUnsupportedFormat\n\t}\n\n\treturn queryParams, globs, resp.Header, nil\n}\n" }, { "path": "helper/client/render.go", "content": "package client\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\n\tprotov2 \"github.com/go-graphite/protocol/carbonapi_v2_pb\"\n\tprotov3 \"github.com/go-graphite/protocol/carbonapi_v3_pb\"\n\tpickle \"github.com/lomik/og-rek\"\n)\n\nvar (\n\tErrInvalidFrom = errors.New(\"invalid from\")\n\tErrInvalidUntil = errors.New(\"invalid until\")\n)\n\ntype Metric struct {\n\tName string `toml:\"name\"`\n\tPathExpression string `toml:\"path\"`\n\tConsolidationFunc string `toml:\"consolidation\"`\n\tStartTime int64 `toml:\"start\"`\n\tStopTime int64 `toml:\"stop\"`\n\tStepTime int64 `toml:\"step\"`\n\tXFilesFactor float32 `toml:\"xfiles\"`\n\tHighPrecisionTimestamps bool `toml:\"precision\"`\n\tValues []float64 `toml:\"values\"`\n\tAppliedFunctions []string `toml:\"applied_functions\"`\n\tRequestStartTime int64 `toml:\"req_start\"`\n\tRequestStopTime int64 `toml:\"req_stop\"`\n}\n\n// Render do /metrics/find/ request\n// Valid formats are carbonapi_v3_pb. protobuf, pickle, json\nfunc Render(client *http.Client, address string, format FormatType, targets []string, filteringFunctions []*protov3.FilteringFunction, maxDataPoints, from, until int64) (string, []Metric, http.Header, error) {\n\trUrl := \"/render/\"\n\n\tif format == FormatDefault {\n\t\tformat = FormatPb_v3\n\t}\n\n\tqueryParams := fmt.Sprintf(\"%s?format=%s, from=%d, until=%d, targets [%s]\", rUrl, format.String(), from, until, strings.Join(targets, \",\"))\n\tif len(targets) == 0 {\n\t\treturn queryParams, nil, nil, nil\n\t}\n\n\tif from <= 0 {\n\t\treturn queryParams, nil, nil, ErrInvalidFrom\n\t}\n\n\tif until <= 0 {\n\t\treturn queryParams, nil, nil, ErrInvalidUntil\n\t}\n\n\tfromStr := strconv.FormatInt(from, 10)\n\tuntilStr := strconv.FormatInt(until, 10)\n\tmaxDataPointsStr := strconv.FormatInt(maxDataPoints, 10)\n\n\tu, err := url.Parse(address + rUrl)\n\tif err != nil {\n\t\treturn queryParams, nil, nil, err\n\t}\n\n\tvar v url.Values\n\n\tvar reader io.Reader\n\n\tswitch format {\n\tcase FormatPb_v3:\n\t\tv = url.Values{\n\t\t\t\"format\": []string{format.String()},\n\t\t}\n\t\tu.RawQuery = v.Encode()\n\n\t\tvar body []byte\n\n\t\tr := protov3.MultiFetchRequest{\n\t\t\tMetrics: make([]protov3.FetchRequest, len(targets)),\n\t\t}\n\t\tfor i, target := range targets {\n\t\t\tr.Metrics[i] = protov3.FetchRequest{\n\t\t\t\tName: target,\n\t\t\t\tStartTime: from,\n\t\t\t\tStopTime: until,\n\t\t\t\tPathExpression: target,\n\t\t\t\tFilterFunctions: filteringFunctions,\n\t\t\t\tMaxDataPoints: maxDataPoints,\n\t\t\t}\n\t\t}\n\n\t\tbody, err = r.Marshal()\n\t\tif err != nil {\n\t\t\treturn queryParams, nil, nil, err\n\t\t}\n\n\t\tif body != nil {\n\t\t\treader = bytes.NewReader(body)\n\t\t}\n\tcase FormatPb_v2, FormatProtobuf, FormatPickle, FormatJSON:\n\t\tv := url.Values{\n\t\t\t\"format\": []string{format.String()},\n\t\t\t\"from\": []string{fromStr},\n\t\t\t\"until\": []string{untilStr},\n\t\t\t\"target\": targets,\n\t\t\t\"maxDataPoints\": []string{maxDataPointsStr},\n\t\t}\n\t\tu.RawQuery = v.Encode()\n\tdefault:\n\t\treturn queryParams, nil, nil, ErrUnsupportedFormat\n\t}\n\n\treq, err := http.NewRequest(http.MethodGet, u.String(), reader)\n\tif err != nil {\n\t\treturn queryParams, nil, nil, err\n\t}\n\n\tresp, err := client.Do(req)\n\tif err != nil {\n\t\treturn queryParams, nil, nil, err\n\t}\n\n\tdefer resp.Body.Close()\n\n\tb, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\treturn queryParams, nil, nil, err\n\t}\n\n\tif resp.StatusCode == http.StatusNotFound {\n\t\treturn queryParams, nil, resp.Header, nil\n\t} else if resp.StatusCode != http.StatusOK {\n\t\treturn queryParams, nil, resp.Header, NewHttpError(resp.StatusCode, string(b))\n\t}\n\n\tmetrics, err := Decode(b, format)\n\tif err != nil {\n\t\treturn queryParams, nil, resp.Header, err\n\t}\n\n\treturn queryParams, metrics, resp.Header, nil\n}\n\n// Decode converts data in the give format to a Metric\nfunc Decode(b []byte, format FormatType) ([]Metric, error) {\n\tvar (\n\t\tmetrics []Metric\n\t\terr error\n\t)\n\n\tswitch format {\n\tcase FormatPb_v3:\n\t\tvar r protov3.MultiFetchResponse\n\n\t\terr = r.Unmarshal(b)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tmetrics = make([]Metric, 0, len(r.Metrics))\n\t\tfor _, m := range r.Metrics {\n\t\t\tmetrics = append(metrics, Metric{\n\t\t\t\tName: m.Name,\n\t\t\t\tPathExpression: m.PathExpression,\n\t\t\t\tConsolidationFunc: m.ConsolidationFunc,\n\t\t\t\tStartTime: m.StartTime,\n\t\t\t\tStopTime: m.StopTime,\n\t\t\t\tStepTime: m.StepTime,\n\t\t\t\tXFilesFactor: m.XFilesFactor,\n\t\t\t\tHighPrecisionTimestamps: m.HighPrecisionTimestamps,\n\t\t\t\tValues: m.Values,\n\t\t\t\tAppliedFunctions: m.AppliedFunctions,\n\t\t\t\tRequestStartTime: m.RequestStartTime,\n\t\t\t\tRequestStopTime: m.StopTime,\n\t\t\t})\n\t\t}\n\tcase FormatPb_v2, FormatProtobuf:\n\t\tvar r protov2.MultiFetchResponse\n\n\t\terr = r.Unmarshal(b)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tmetrics = make([]Metric, 0, len(r.Metrics))\n\n\t\tfor _, m := range r.Metrics {\n\t\t\tfor i, a := range m.IsAbsent {\n\t\t\t\tif a {\n\t\t\t\t\tm.Values[i] = math.NaN()\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tmetrics = append(metrics, Metric{\n\t\t\t\tName: m.Name,\n\t\t\t\tStartTime: int64(m.StartTime),\n\t\t\t\tStopTime: int64(m.StopTime),\n\t\t\t\tStepTime: int64(m.StepTime),\n\t\t\t\tValues: m.Values,\n\t\t\t})\n\t\t}\n\tcase FormatPickle:\n\t\treader := bytes.NewReader(b)\n\t\tdecoder := pickle.NewDecoder(reader)\n\n\t\tp, err := decoder.Decode()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tfor _, v := range p.([]interface{}) {\n\t\t\tm := v.(map[interface{}]interface{})\n\t\t\tvals := m[\"values\"].([]interface{})\n\t\t\tvalues := make([]float64, len(vals))\n\n\t\t\tfor i, vv := range vals {\n\t\t\t\tif _, isNaN := vv.(pickle.None); isNaN {\n\t\t\t\t\tvalues[i] = math.NaN()\n\t\t\t\t} else {\n\t\t\t\t\tvalues[i] = vv.(float64)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tmetrics = append(metrics, Metric{\n\t\t\t\tName: m[\"name\"].(string),\n\t\t\t\tPathExpression: m[\"pathExpression\"].(string),\n\t\t\t\tStartTime: m[\"start\"].(int64),\n\t\t\t\tStopTime: m[\"end\"].(int64),\n\t\t\t\tStepTime: m[\"step\"].(int64),\n\t\t\t\tValues: values,\n\t\t\t})\n\t\t}\n\tcase FormatJSON:\n\t\tvar r jsonResponse\n\n\t\terr = json.Unmarshal(b, &r)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tmetrics = make([]Metric, 0, len(r.Metrics))\n\n\t\tfor _, m := range r.Metrics {\n\t\t\tvalues := make([]float64, len(m.Values))\n\n\t\t\tfor i, v := range m.Values {\n\t\t\t\tif v == nil {\n\t\t\t\t\tvalues[i] = math.NaN()\n\t\t\t\t} else {\n\t\t\t\t\tvalues[i] = *v\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tmetrics = append(metrics, Metric{\n\t\t\t\tName: m.Name,\n\t\t\t\tPathExpression: m.PathExpression,\n\t\t\t\tStartTime: m.StartTime,\n\t\t\t\tStopTime: m.StopTime,\n\t\t\t\tStepTime: m.StepTime,\n\t\t\t\tValues: values,\n\t\t\t})\n\t\t}\n\tdefault:\n\t\treturn nil, ErrUnsupportedFormat\n\t}\n\n\treturn metrics, nil\n}\n\n// jsonResponse is a simple struct to decode JSON responses for testing purposes\ntype jsonResponse struct {\n\tMetrics []jsonMetric `json:\"metrics\"`\n}\n\ntype jsonMetric struct {\n\tName string `json:\"name\"`\n\tPathExpression string `json:\"pathExpression\"`\n\tValues []*float64 `json:\"values\"`\n\tStartTime int64 `json:\"startTime\"`\n\tStopTime int64 `json:\"stopTime\"`\n\tStepTime int64 `json:\"stepTime\"`\n}\n" }, { "path": "helper/client/requests.go", "content": "package client\n\nimport protov3 \"github.com/go-graphite/protocol/carbonapi_v3_pb\"\n\ntype MultiGlobRequestV3 struct {\n\tprotov3.MultiGlobRequest\n}\n\nfunc (r *MultiGlobRequestV3) Marshal() ([]byte, error) {\n\treturn r.MultiGlobRequest.Marshal()\n}\n\nfunc (r *MultiGlobRequestV3) LogInfo() interface{} {\n\treturn r.MultiGlobRequest\n}\n" }, { "path": "helper/client/tags.go", "content": "package client\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/msaf1980/go-stringutils\"\n)\n\n// TagsNames do /tags/autoComplete/tags request with query like [tagPrefix];tag1=value1;tag2=~value*\n// Valid formats are json\nfunc TagsNames(client *http.Client, address string, format FormatType, query string, limit uint64, from, until int64) (string, []string, http.Header, error) {\n\trTags := \"/tags/autoComplete/tags\"\n\n\tif format == FormatDefault {\n\t\tformat = FormatJSON\n\t}\n\n\tvar queryParams string\n\n\tswitch format {\n\tcase FormatJSON:\n\t\tbreak\n\tdefault:\n\t\tqueryParams = fmt.Sprintf(\"%s?format=%s, from=%d, until=%d, limits=%d, query %s\", rTags, format.String(), from, until, limit, query)\n\t\treturn queryParams, nil, nil, ErrUnsupportedFormat\n\t}\n\n\tu, err := url.Parse(address + rTags)\n\tif err != nil {\n\t\treturn queryParams, nil, nil, err\n\t}\n\n\tvar tagPrefix string\n\n\tvar exprs []string\n\n\tif query != \"\" && query != \"<>\" {\n\t\targs := strings.Split(query, \";\")\n\t\tif len(args) < 1 {\n\t\t\treturn queryParams, nil, nil, ErrInvalidQuery\n\t\t}\n\n\t\texprs = make([]string, 0, len(args))\n\n\t\tfor i, arg := range args {\n\t\t\tdelim := strings.IndexRune(arg, '=')\n\t\t\tif i == 0 && delim == -1 {\n\t\t\t\ttagPrefix = arg\n\t\t\t} else if delim <= 0 {\n\t\t\t\treturn queryParams, nil, nil, errors.New(\"invalid expr: \" + arg)\n\t\t\t} else {\n\t\t\t\texprs = append(exprs, arg)\n\t\t\t}\n\t\t}\n\t}\n\n\tv := make([]string, 0, 2+len(exprs))\n\n\tvar rawQuery stringutils.Builder\n\n\trawQuery.Grow(128)\n\n\tv = append(v, \"format=\"+format.String())\n\n\trawQuery.WriteString(\"format=\")\n\trawQuery.WriteString(url.QueryEscape(format.String()))\n\n\tif tagPrefix != \"\" {\n\t\tv = append(v, \"tagPrefix=\"+tagPrefix)\n\n\t\trawQuery.WriteString(\"&tagPrefix=\")\n\t\trawQuery.WriteString(url.QueryEscape(tagPrefix))\n\t}\n\n\tfor _, expr := range exprs {\n\t\tv = append(v, \"expr=\"+expr)\n\n\t\trawQuery.WriteString(\"&expr=\")\n\t\trawQuery.WriteString(url.QueryEscape(expr))\n\t}\n\n\tif from > 0 {\n\t\tfromStr := strconv.FormatInt(from, 10)\n\t\tv = append(v, \"from=\"+fromStr)\n\n\t\trawQuery.WriteString(\"&from=\")\n\t\trawQuery.WriteString(fromStr)\n\t}\n\n\tif until > 0 {\n\t\tuntilStr := strconv.FormatInt(until, 10)\n\t\tv = append(v, \"until=\"+untilStr)\n\n\t\trawQuery.WriteString(\"&until=\")\n\t\trawQuery.WriteString(untilStr)\n\t}\n\n\tif limit > 0 {\n\t\tlimitStr := strconv.FormatUint(limit, 10)\n\t\tv = append(v, \"limit=\"+limitStr)\n\n\t\trawQuery.WriteString(\"&limit=\")\n\t\trawQuery.WriteString(limitStr)\n\t}\n\n\tqueryParams = fmt.Sprintf(\"%s %q\", rTags, v)\n\n\tu.RawQuery = rawQuery.String()\n\n\treq, err := http.NewRequest(http.MethodGet, u.String(), nil)\n\tif err != nil {\n\t\treturn queryParams, nil, nil, err\n\t}\n\n\tresp, err := client.Do(req)\n\tif err != nil {\n\t\treturn queryParams, nil, nil, err\n\t}\n\n\tdefer resp.Body.Close()\n\n\tb, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\treturn queryParams, nil, nil, err\n\t}\n\n\tif resp.StatusCode == http.StatusNotFound {\n\t\treturn u.RawQuery, nil, resp.Header, nil\n\t} else if resp.StatusCode != http.StatusOK {\n\t\treturn queryParams, nil, resp.Header, NewHttpError(resp.StatusCode, string(b))\n\t}\n\n\tvar values []string\n\n\terr = json.Unmarshal(b, &values)\n\tif err != nil {\n\t\treturn queryParams, nil, resp.Header, errors.New(err.Error() + \": \" + string(b))\n\t}\n\n\treturn queryParams, values, resp.Header, nil\n}\n\n// TagsValues do /tags/autoComplete/values request with query like searchTag[=valuePrefix];tag1=value1;tag2=~value*\n// Valid formats are json\nfunc TagsValues(client *http.Client, address string, format FormatType, query string, limit uint64, from, until int64) (string, []string, http.Header, error) {\n\trTags := \"/tags/autoComplete/values\"\n\n\tif format == FormatDefault {\n\t\tformat = FormatJSON\n\t}\n\n\tvar queryParams string\n\n\tswitch format {\n\tcase FormatJSON:\n\t\tbreak\n\tdefault:\n\t\tqueryParams = fmt.Sprintf(\"%s?format=%s, from=%d, until=%d, limits=%d, query %s\", rTags, format.String(), from, until, limit, query)\n\t\treturn queryParams, nil, nil, ErrUnsupportedFormat\n\t}\n\n\tu, err := url.Parse(address + rTags)\n\tif err != nil {\n\t\treturn queryParams, nil, nil, err\n\t}\n\n\tvar (\n\t\ttag string\n\t\tvaluePrefix string\n\t\texprs []string\n\t)\n\n\tif query != \"\" && query != \"<>\" {\n\t\targs := strings.Split(query, \";\")\n\t\tif len(args) < 2 {\n\t\t\treturn queryParams, nil, nil, ErrInvalidQuery\n\t\t}\n\n\t\tvals := strings.Split(args[0], \"=\")\n\t\ttag = vals[0]\n\n\t\tif len(vals) > 2 {\n\t\t\treturn queryParams, nil, nil, errors.New(\"invalid tag: \" + args[0])\n\t\t} else if len(vals) == 2 {\n\t\t\tvaluePrefix = vals[1]\n\t\t}\n\n\t\texprs = make([]string, 0, len(args)-1)\n\n\t\tfor i := 1; i < len(args); i++ {\n\t\t\texpr := args[i]\n\t\t\tif strings.IndexRune(expr, '=') <= 0 {\n\t\t\t\treturn queryParams, nil, nil, errors.New(\"invalid expr: \" + expr)\n\t\t\t}\n\n\t\t\texprs = append(exprs, expr)\n\t\t}\n\t}\n\n\tv := make([]string, 0, 2+len(exprs))\n\n\tvar rawQuery stringutils.Builder\n\n\trawQuery.Grow(128)\n\n\tv = append(v, \"format=\"+format.String())\n\n\trawQuery.WriteString(\"format=\")\n\trawQuery.WriteString(url.QueryEscape(format.String()))\n\n\tif tag != \"\" {\n\t\tv = append(v, \"tag=\"+tag)\n\n\t\trawQuery.WriteString(\"&tag=\")\n\t\trawQuery.WriteString(url.QueryEscape(tag))\n\t}\n\n\tif valuePrefix != \"\" {\n\t\tv = append(v, \"valuePrefix=\"+valuePrefix)\n\n\t\trawQuery.WriteString(\"&valuePrefix=\")\n\t\trawQuery.WriteString(url.QueryEscape(valuePrefix))\n\t}\n\n\tfor _, expr := range exprs {\n\t\tv = append(v, \"expr=\"+expr)\n\n\t\trawQuery.WriteString(\"&expr=\")\n\t\trawQuery.WriteString(url.QueryEscape(expr))\n\t}\n\n\tif from > 0 {\n\t\tfromStr := strconv.FormatInt(from, 10)\n\t\tv = append(v, \"from=\"+fromStr)\n\n\t\trawQuery.WriteString(\"&from=\")\n\t\trawQuery.WriteString(fromStr)\n\t}\n\n\tif until > 0 {\n\t\tuntilStr := strconv.FormatInt(until, 10)\n\t\tv = append(v, \"until=\"+untilStr)\n\n\t\trawQuery.WriteString(\"&until=\")\n\t\trawQuery.WriteString(untilStr)\n\t}\n\n\tif limit > 0 {\n\t\tlimitStr := strconv.FormatUint(limit, 10)\n\t\tv = append(v, \"limit=\"+limitStr)\n\n\t\trawQuery.WriteString(\"&limit=\")\n\t\trawQuery.WriteString(limitStr)\n\t}\n\n\tqueryParams = fmt.Sprintf(\"%s %q\", rTags, v)\n\n\tu.RawQuery = rawQuery.String()\n\n\treq, err := http.NewRequest(http.MethodGet, u.String(), nil)\n\tif err != nil {\n\t\treturn queryParams, nil, nil, err\n\t}\n\n\tresp, err := client.Do(req)\n\tif err != nil {\n\t\treturn u.RawQuery, nil, nil, err\n\t}\n\n\tdefer resp.Body.Close()\n\n\tb, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\treturn queryParams, nil, nil, err\n\t}\n\n\tif resp.StatusCode == http.StatusNotFound {\n\t\treturn queryParams, nil, resp.Header, nil\n\t} else if resp.StatusCode != http.StatusOK {\n\t\treturn queryParams, nil, resp.Header, NewHttpError(resp.StatusCode, string(b))\n\t}\n\n\tvar values []string\n\n\terr = json.Unmarshal(b, &values)\n\tif err != nil {\n\t\treturn queryParams, nil, resp.Header, errors.New(err.Error() + \": \" + string(b))\n\t}\n\n\treturn queryParams, values, resp.Header, nil\n}\n" }, { "path": "helper/client/types.go", "content": "package client\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n)\n\ntype FormatType int\n\nconst (\n\tFormatDefault FormatType = iota\n\tFormatJSON\n\tFormatProtobuf\n\tFormatPb_v2 // alias for FormatProtobuf\n\tFormatPb_v3\n\tFormatPickle\n)\n\nvar formatStrings []string = []string{\"default\", \"json\", \"protobuf\", \"carbonapi_v2_pb\", \"carbonapi_v3_pb\", \"pickle\"}\n\nfunc (a *FormatType) String() string {\n\treturn formatStrings[*a]\n}\n\nfunc FormatTypes() []string {\n\treturn formatStrings\n}\n\nfunc (a *FormatType) Set(value string) error {\n\tswitch value {\n\tcase \"json\":\n\t\t*a = FormatJSON\n\tcase \"protobuf\":\n\t\t*a = FormatProtobuf\n\tcase \"carbonapi_v2_pb\":\n\t\t*a = FormatPb_v2\n\tcase \"carbonapi_v3_pb\":\n\t\t*a = FormatPb_v3\n\tcase \"pickle\":\n\t\t*a = FormatPickle\n\tdefault:\n\t\treturn fmt.Errorf(\"invalid format type %s\", value)\n\t}\n\n\treturn nil\n}\n\nfunc (a *FormatType) UnmarshalText(text []byte) error {\n\treturn a.Set(string(text))\n}\n\nvar (\n\tErrUnsupportedFormat = errors.New(\"unsupported format\")\n\tErrInvalidQuery = errors.New(\"invalid query\")\n\n\t//ErrEmptyQuery = errors.New(\"missing query\")\n)\n" }, { "path": "helper/date/date.go", "content": "package date\n\nimport \"time\"\n\nvar FromTimestampToDaysFormat func(int64) string\nvar FromTimeToDaysFormat func(time.Time) string\nvar UntilTimestampToDaysFormat func(int64) string\nvar UntilTimeToDaysFormat func(time.Time) string\n\n// SetDefault() is for broken SlowTimestampToDays in carbon-clickhouse\nfunc SetDefault() {\n\tFromTimestampToDaysFormat = DefaultTimestampToDaysFormat\n\tFromTimeToDaysFormat = DefaultTimeToDaysFormat\n\tUntilTimestampToDaysFormat = DefaultTimestampToDaysFormat\n\tUntilTimeToDaysFormat = DefaultTimeToDaysFormat\n}\n\n// SetUTC() is for UTCTimestampToDays in carbon-clickhouse (see https://github.com/go-graphite/carbon-clickhouse/pull/114)\nfunc SetUTC() {\n\tFromTimestampToDaysFormat = UTCTimestampToDaysFormat\n\tFromTimeToDaysFormat = UTCTimeToDaysFormat\n\tUntilTimestampToDaysFormat = UTCTimestampToDaysFormat\n\tUntilTimeToDaysFormat = UTCTimeToDaysFormat\n}\n\n// SetBoth() is for mixed SlowTimestampToDays/UTCTimestampToDays (before rebuild tables complete)\nfunc SetBoth() {\n\tFromTimestampToDaysFormat = MinTimestampToDaysFormat\n\tFromTimeToDaysFormat = MinTimeToDaysFormat\n\tUntilTimestampToDaysFormat = MaxTimestampToDaysFormat\n\tUntilTimeToDaysFormat = MaxTimeToDaysFormat\n}\n\nfunc init() {\n\tSetDefault()\n}\n\n// from carbon-clickhouse, port of SlowTimestampToDays, broken symmetic, not always UTC\nfunc DefaultTimestampToDaysFormat(ts int64) string {\n\tt := time.Unix(ts, 0)\n\treturn time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC).Format(\"2006-01-02\")\n}\n\n// from carbon-clickhouse, port of SlowTimestampToDays, broken symmetic, not always UTC\nfunc DefaultTimeToDaysFormat(t time.Time) string {\n\treturn time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC).Format(\"2006-01-02\")\n}\n\nfunc UTCTimestampToDaysFormat(timestamp int64) string {\n\treturn time.Unix(timestamp, 0).UTC().Format(\"2006-01-02\")\n}\n\nfunc UTCTimeToDaysFormat(t time.Time) string {\n\treturn t.UTC().Format(\"2006-01-02\")\n}\n\nfunc defaultDate(t time.Time) time.Time {\n\treturn time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC)\n}\n\nfunc minLocalAndUTC(t time.Time) time.Time {\n\ttu := defaultDate(t.UTC())\n\ttd := defaultDate(t)\n\n\tif tu.Unix() < td.Unix() {\n\t\treturn tu\n\t} else {\n\t\treturn td\n\t}\n}\n\n// MinTimestampToDaysFormat return formatted minimum (local, UTC) date\nfunc MinTimestampToDaysFormat(ts int64) string {\n\tt := minLocalAndUTC(time.Unix(ts, 0))\n\treturn t.Format(\"2006-01-02\")\n}\n\n// MinTimeToDaysFormat return formatted minimum (local, UTC) date\nfunc MinTimeToDaysFormat(t time.Time) string {\n\tt = minLocalAndUTC(t)\n\treturn t.Format(\"2006-01-02\")\n}\n\nfunc maxLocalAndUTC(t time.Time) time.Time {\n\ttu := defaultDate(t.UTC())\n\ttd := defaultDate(t)\n\n\tif tu.Unix() > td.Unix() {\n\t\treturn tu\n\t} else {\n\t\treturn td\n\t}\n}\n\n// MaxTimestampToDaysFormat return formatted maximum (local, UTC) date\nfunc MaxTimestampToDaysFormat(ts int64) string {\n\tt := maxLocalAndUTC(time.Unix(ts, 0))\n\treturn t.Format(\"2006-01-02\")\n}\n\n// MaxTimeToDaysFormat return formatted maximum (local, UTC) date\nfunc MaxTimeToDaysFormat(t time.Time) string {\n\tt = maxLocalAndUTC(t)\n\treturn t.Format(\"2006-01-02\")\n}\n" }, { "path": "helper/date/date_test.go", "content": "package date\n\nimport (\n\t\"os\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n)\n\nvar verbose bool\n\nfunc isVerbose() bool {\n\tfor _, arg := range os.Args {\n\t\tif arg == \"-test.v=true\" {\n\t\t\treturn true\n\t\t}\n\t}\n\n\treturn false\n}\n\nfunc init() {\n\tverbose = isVerbose()\n}\n\n// TimestampDaysFormat is broken symmetic with carbon-clickhouse of SlowTimestampToDays, not always UTC\n// $ TZ=Etc/GMT-5 go test -v -timeout 30s -run ^TestTimestampDaysFormat$ github.com/lomik/graphite-clickhouse/helper/date\n// === RUN TestTimestampDaysFormat\n// === RUN TestTimestampDaysFormat/1668106870_2022-11-11T00:01:10+05:00_2022-11-10T19:01:10Z_[0]\n//\n//\tdate_test.go:62: Warning (TimestampDaysFormat broken) TimestampDaysFormat(1668106870) = 2022-11-11, want UTC 2022-11-10\n//\n// --- FAIL: TestTimestampDaysFormat (0.00s)\n//\n//\t--- FAIL: TestTimestampDaysFormat/1668106870_2022-11-11T00:01:10+05:00_2022-11-10T19:01:10Z_[0] (0.00s)\n//\n// FAIL\n// FAIL\tgithub.com/lomik/graphite-clickhouse/helper/date\t0.001s\n//\n// $ TZ=Etc/GMT+5 go test -v -timeout 30s -run ^TestTimestampDaysFormat$ github.com/lomik/graphite-clickhouse/helper/date\n// === RUN TestTimestampDaysFormat\n// === RUN TestTimestampDaysFormat/1668124800_2022-11-10T19:00:00-05:00_2022-11-11T00:00:00Z_[1]\n//\n//\tdate_test.go:62: Warning (TimestampDaysFormat broken) TimestampDaysFormat(1668124800) = 2022-11-10, want UTC 2022-11-11\n//\n// === RUN TestTimestampDaysFormat/1668142799_2022-11-10T23:59:59-05:00_2022-11-11T04:59:59Z_[2]\n//\n//\tdate_test.go:62: Warning (TimestampDaysFormat broken) TimestampDaysFormat(1668142799) = 2022-11-10, want UTC 2022-11-11\n//\n// === RUN TestTimestampDaysFormat/1650776160_2022-04-23T23:56:00-05:00_2022-04-24T04:56:00Z_[3]\n//\n//\tdate_test.go:62: Warning (TimestampDaysFormat broken) TimestampDaysFormat(1650776160) = 2022-04-23, want UTC 2022-04-24\n//\n// --- FAIL: TestTimestampDaysFormat (0.00s)\nfunc TestDefaultTimestampToDaysFormat(t *testing.T) {\n\ttests := []struct {\n\t\tts int64\n\t\twant string\n\t}{\n\t\t{\n\t\t\tts: 1668106870, // 2022-11-11 00:01:10 +05:00 ; 2022-11-10 19:01:10 UTC\n\t\t\t// select toDate(1650776160,'UTC')\n\t\t\t// 2022-11-10\n\t\t\twant: time.Unix(1668106870, 0).Format(\"2006-01-02\"),\n\t\t},\n\t\t{\n\t\t\tts: 1668124800, // 2022-11-11 00:00:00 UTC\n\t\t\twant: time.Unix(1668124800, 0).Format(\"2006-01-02\"),\n\t\t},\n\t\t{\n\t\t\tts: 1668142799, // 2022-11-10 23:59:59 -05:00; 2022-11-11 04:59:59 UTC\n\t\t\twant: time.Unix(1668142799, 0).Format(\"2006-01-02\"),\n\t\t},\n\t\t{\n\t\t\tts: 1650776160, // graphite-clickhouse issue #184, graphite-clickhouse in UTC, clickhouse in PDT(UTC-7)\n\t\t\t// 2022-04-24 4:56:00\n\t\t\t// select toDate(1650776160,'UTC')\n\t\t\t// 2022-04-24\n\t\t\t// select toDate(1650776160,'Etc/GMT+7')\n\t\t\t// 2022-04-23\n\t\t\twant: time.Unix(1650776160, 0).Format(\"2006-01-02\"),\n\t\t},\n\t}\n\tfor i, tt := range tests {\n\t\tt.Run(strconv.FormatInt(tt.ts, 10)+\" \"+time.Unix(tt.ts, 0).Format(time.RFC3339)+\" \"+time.Unix(tt.ts, 0).UTC().Format(time.RFC3339)+\" [\"+strconv.Itoa(i)+\"]\", func(t *testing.T) {\n\t\t\tif got := DefaultTimestampToDaysFormat(tt.ts); got != tt.want {\n\t\t\t\tt.Errorf(\"DefaultTimestampDaysFormat(%d) = %s, want %s\", tt.ts, got, tt.want)\n\t\t\t} else if gotUTC := UTCTimestampToDaysFormat(tt.ts); got != gotUTC {\n\t\t\t\t// Run to see a warning\n\t\t\t\t// go test -v -timeout 30s -run ^TestTimestampDaysFormat$ github.com/lomik/graphite-clickhouse/helper/date\n\t\t\t\tif verbose {\n\t\t\t\t\tt.Errorf(\"Warning (DefaultTimestampDaysFormat broken) DefaultTimestampDaysFormat(%d) = %s, want UTC %s\", tt.ts, got, gotUTC)\n\t\t\t\t} else {\n\t\t\t\t\tt.Logf(\"Warning (DefaultTimestampDaysFormat broken) DefaultTimestampDaysFormat(%d) = %s, want UTC %s\", tt.ts, got, gotUTC)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestDefaultTimeToDaysFormat(t *testing.T) {\n\ttests := []struct {\n\t\tts int64\n\t\twant string\n\t}{\n\t\t{\n\t\t\tts: 1668106870, // 2022-11-11 00:01:10 +05:00 ; 2022-11-10 19:01:10 UTC\n\t\t\t// select toDate(1650776160,'UTC')\n\t\t\t// 2022-11-10\n\t\t\twant: time.Unix(1668106870, 0).Format(\"2006-01-02\"),\n\t\t},\n\t\t{\n\t\t\tts: 1668124800, // 2022-11-11 00:00:00 UTC\n\t\t\twant: time.Unix(1668124800, 0).Format(\"2006-01-02\"),\n\t\t},\n\t\t{\n\t\t\tts: 1668142799, // 2022-11-10 23:59:59 -05:00; 2022-11-11 04:59:59 UTC\n\t\t\twant: time.Unix(1668142799, 0).Format(\"2006-01-02\"),\n\t\t},\n\t\t{\n\t\t\tts: 1650776160, // graphite-clickhouse issue #184, graphite-clickhouse in UTC, clickhouse in PDT(UTC-7)\n\t\t\t// 2022-04-24 4:56:00\n\t\t\t// select toDate(1650776160,'UTC')\n\t\t\t// 2022-04-24\n\t\t\t// select toDate(1650776160,'Etc/GMT+7')\n\t\t\t// 2022-04-23\n\t\t\twant: time.Unix(1650776160, 0).Format(\"2006-01-02\"),\n\t\t},\n\t}\n\tfor i, tt := range tests {\n\t\tt.Run(strconv.FormatInt(tt.ts, 10)+\" \"+time.Unix(tt.ts, 0).UTC().Format(time.RFC3339)+\" [\"+strconv.Itoa(i)+\"]\", func(t *testing.T) {\n\t\t\tif got := DefaultTimeToDaysFormat(time.Unix(tt.ts, 0)); got != tt.want {\n\t\t\t\tt.Errorf(\"DefaultTimeDaysFormat() = %s, want %s\", got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestUTCTimestampToDaysFormat(t *testing.T) {\n\ttests := []struct {\n\t\tts int64\n\t\twant string\n\t}{\n\t\t{\n\t\t\tts: 1668106870, // 2022-11-11 00:01:10 +05:00 ; 2022-11-10 19:01:10 UTC\n\t\t\t// select toDate(1650776160,'UTC')\n\t\t\t// 2022-11-10\n\t\t\twant: \"2022-11-10\",\n\t\t},\n\t\t{\n\t\t\tts: 1668124800, // 2022-11-11 00:00:00 UTC\n\t\t\twant: \"2022-11-11\",\n\t\t},\n\t\t{\n\t\t\tts: 1668142799, // 2022-11-10 23:59:59 -05:00; 2022-11-11 04:59:59 UTC\n\t\t\twant: \"2022-11-11\",\n\t\t},\n\t\t{\n\t\t\tts: 1650776160, // graphite-clickhouse issue #184, graphite-clickhouse in UTC, clickhouse in PDT(UTC-7)\n\t\t\t// 2022-04-24 4:56:00\n\t\t\t// select toDate(1650776160,'UTC')\n\t\t\t// 2022-04-24\n\t\t\t// select toDate(1650776160,'Etc/GMT+7')\n\t\t\t// 2022-04-23\n\t\t\twant: \"2022-04-24\",\n\t\t},\n\t}\n\tfor i, tt := range tests {\n\t\tt.Run(strconv.FormatInt(tt.ts, 10)+\" \"+time.Unix(tt.ts, 0).Format(time.RFC3339)+\" \"+time.Unix(tt.ts, 0).UTC().Format(time.RFC3339)+\" [\"+strconv.Itoa(i)+\"]\", func(t *testing.T) {\n\t\t\tif got := UTCTimestampToDaysFormat(tt.ts); got != tt.want {\n\t\t\t\tt.Errorf(\"UTCTimestampDaysFormat(%d) = %s, want %s\", tt.ts, got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestUTCTimeToDaysFormat(t *testing.T) {\n\ttests := []struct {\n\t\tts int64\n\t\twant string\n\t}{\n\t\t{\n\t\t\tts: 1668106870, // 2022-11-11 00:01:10 +05:00 ; 2022-11-10 19:01:10 UTC\n\t\t\t// select toDate(1650776160,'UTC')\n\t\t\t// 2022-11-10\n\t\t\twant: \"2022-11-10\",\n\t\t},\n\t\t{\n\t\t\tts: 1668124800, // 2022-11-11 00:00:00 UTC\n\t\t\twant: \"2022-11-11\",\n\t\t},\n\t\t{\n\t\t\tts: 1668142799, // 2022-11-10 23:59:59 -05:00; 2022-11-11 04:59:59 UTC\n\t\t\twant: \"2022-11-11\",\n\t\t},\n\t\t{\n\t\t\tts: 1650776160, // graphite-clickhouse issue #184, graphite-clickhouse in UTC, clickhouse in PDT(UTC-7)\n\t\t\t// 2022-04-24 4:56:00\n\t\t\t// select toDate(1650776160,'UTC')\n\t\t\t// 2022-04-24\n\t\t\t// select toDate(1650776160,'Etc/GMT+7')\n\t\t\t// 2022-04-23\n\t\t\twant: \"2022-04-24\",\n\t\t},\n\t}\n\tfor i, tt := range tests {\n\t\tt.Run(strconv.FormatInt(tt.ts, 10)+\" \"+time.Unix(tt.ts, 0).UTC().Format(time.RFC3339)+\" [\"+strconv.Itoa(i)+\"]\", func(t *testing.T) {\n\t\t\tif got := UTCTimeToDaysFormat(time.Unix(tt.ts, 0)); got != tt.want {\n\t\t\t\tt.Errorf(\"UTCTimeDaysFormat() = %s, want %s\", got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMinMaxTimestampToDaysFormat(t *testing.T) {\n\ttests := []struct {\n\t\tts int64\n\t}{\n\t\t{\n\t\t\tts: 1668106870, // 2022-11-11 00:01:10 +05:00 ; 2022-11-10 19:01:10 UTC\n\t\t\t// select toDate(1650776160,'UTC')\n\t\t\t// 2022-11-10\n\t\t},\n\t\t{\n\t\t\tts: 1668124800, // 2022-11-11 00:00:00 UTC\n\t\t},\n\t\t{\n\t\t\tts: 1668142799, // 2022-11-10 23:59:59 -05:00; 2022-11-11 04:59:59 UTC\n\t\t},\n\t\t{\n\t\t\tts: 1650776160, // graphite-clickhouse issue #184, graphite-clickhouse in UTC, clickhouse in PDT(UTC-7)\n\t\t\t// 2022-04-24 4:56:00\n\t\t\t// select toDate(1650776160,'UTC')\n\t\t\t// 2022-04-24\n\t\t\t// select toDate(1650776160,'Etc/GMT+7')\n\t\t\t// 2022-04-23\n\t\t},\n\t}\n\tfor i, tt := range tests {\n\t\tt.Run(strconv.FormatInt(tt.ts, 10)+\" \"+time.Unix(tt.ts, 0).UTC().Format(time.RFC3339)+\" [\"+strconv.Itoa(i)+\"]\", func(t *testing.T) {\n\t\t\tgotMin := MinTimestampToDaysFormat(tt.ts)\n\t\t\ttimeMin, _ := time.Parse(\"2006-01-02\", gotMin)\n\t\t\tgotMax := MaxTimestampToDaysFormat(tt.ts)\n\t\t\ttimeMax, _ := time.Parse(\"2006-01-02\", gotMax)\n\t\t\tgot := DefaultTimestampToDaysFormat(tt.ts)\n\t\t\ttm, _ := time.Parse(\"2006-01-02\", got)\n\n\t\t\tif timeMin.UnixNano() > timeMax.UnixNano() || tm.UnixNano() > timeMax.UnixNano() || tm.UnixNano() < timeMin.UnixNano() {\n\t\t\t\tt.Errorf(\"MinTimeDaysFormat() = %s > MaxTimeDaysFormat() = %s, DefaultTimeDaysFormat() = %s\", gotMin, gotMax, got)\n\t\t\t} else {\n\t\t\t\tt.Logf(\"MinTimeDaysFormat() = %s, MaxTimeDaysFormat() = %s, DefaultTimeDaysFormat() = %s\", gotMin, gotMax, got)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "helper/datetime/datetime.go", "content": "package datetime\n\nimport (\n\t\"errors\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/go-graphite/carbonapi/pkg/parser\"\n)\n\nvar ErrBadTime = errors.New(\"bad time\")\n\n// parseTime parses a time and returns hours and minutes\nfunc parseTime(s string) (hour, minute int, err error) {\n\tswitch s {\n\tcase \"midnight\":\n\t\treturn 0, 0, nil\n\tcase \"noon\":\n\t\treturn 12, 0, nil\n\tcase \"teatime\":\n\t\treturn 16, 0, nil\n\t}\n\n\tparts := strings.Split(s, \":\")\n\n\tif len(parts) != 2 {\n\t\treturn 0, 0, ErrBadTime\n\t}\n\n\thour, err = strconv.Atoi(parts[0])\n\tif err != nil {\n\t\treturn 0, 0, ErrBadTime\n\t}\n\n\tminute, err = strconv.Atoi(parts[1])\n\tif err != nil {\n\t\treturn 0, 0, ErrBadTime\n\t}\n\n\treturn hour, minute, nil\n}\n\nvar TimeFormats = []string{\"20060102\", \"01/02/06\"}\n\n// DateParamToEpoch turns a passed string parameter into a unix epoch\nfunc DateParamToEpoch(s string, tz *time.Location, now time.Time, truncate time.Duration) int64 {\n\tif s == \"\" {\n\t\t// return the default if nothing was passed\n\t\treturn 0\n\t}\n\n\t// relative timestamp\n\tif s[0] == '-' {\n\t\toffset, err := parser.IntervalString(s, -1)\n\t\tif err != nil {\n\t\t\treturn 0\n\t\t}\n\n\t\treturn now.Add(time.Duration(offset) * time.Second).Unix()\n\t} else if s[0] == '+' {\n\t\toffset, err := parser.IntervalString(s, 1)\n\t\tif err != nil {\n\t\t\treturn 0\n\t\t}\n\n\t\treturn now.Add(time.Duration(offset) * time.Second).Unix()\n\t}\n\n\tswitch s {\n\tcase \"now\":\n\t\treturn now.Unix()\n\tcase \"rnow\":\n\t\treturn TimeTruncate(now, truncate).Unix()\n\tcase \"midnight\", \"noon\", \"teatime\":\n\t\tyy, mm, dd := now.Date()\n\t\thh, min, _ := parseTime(s) // error ignored, we know it's valid\n\t\tdt := time.Date(yy, mm, dd, hh, min, 0, 0, tz)\n\n\t\treturn dt.Unix()\n\t}\n\n\tsint, err := strconv.Atoi(s)\n\t// need to check that len(s) != 8 to avoid turning 20060102 into seconds\n\tif err == nil && len(s) != 8 {\n\t\treturn int64(sint) // We got a timestamp so returning it\n\t}\n\n\ts = strings.Replace(s, \"_\", \" \", 1) // Go can't parse _ in date strings\n\n\tvar ts, ds string\n\n\tsplit := strings.Fields(s)\n\n\tvar t time.Time\n\n\tswitch {\n\tcase len(split) == 1:\n\t\tdelim := strings.IndexAny(s, \"+-\")\n\t\tif delim == -1 {\n\t\t\tds = s\n\t\t} else {\n\t\t\tds = s[:delim]\n\t\t\tswitch ds {\n\t\t\tcase \"now\", \"today\":\n\t\t\t\tt = now\n\t\t\tcase \"rnow\", \"rtoday\":\n\t\t\t\tt = TimeTruncate(now, truncate)\n\t\t\t\t// nothing\n\t\t\tcase \"midnight\", \"noon\", \"teatime\":\n\t\t\t\tyy, mm, dd := now.Date()\n\t\t\t\thh, min, _ := parseTime(s) // error ignored, we know it's valid\n\t\t\t\tt = time.Date(yy, mm, dd, hh, min, 0, 0, tz)\n\t\t\tcase \"yesterday\":\n\t\t\t\tt = now.AddDate(0, 0, -1)\n\t\t\tcase \"tomorrow\":\n\t\t\t\tt = now.AddDate(0, 0, 1)\n\t\t\tdefault:\n\t\t\t\treturn 0\n\t\t\t}\n\n\t\t\ts = s[delim:]\n\t\t\tfor len(s) > 0 {\n\t\t\t\tdelim := strings.IndexAny(s[1:], \"+-\")\n\t\t\t\tif delim == -1 {\n\t\t\t\t\tts = s\n\t\t\t\t\ts = s[:0]\n\t\t\t\t} else {\n\t\t\t\t\tts = s[:delim+1]\n\t\t\t\t\ts = s[delim+1:]\n\t\t\t\t}\n\n\t\t\t\toffset, err := parser.IntervalString(ts, 1)\n\t\t\t\tif err != nil {\n\t\t\t\t\toffset64, err := strconv.ParseInt(ts, 10, 32)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn 0\n\t\t\t\t\t}\n\n\t\t\t\t\toffset = int32(offset64)\n\t\t\t\t}\n\n\t\t\t\tt = t.Add(time.Duration(offset) * time.Second)\n\t\t\t}\n\n\t\t\treturn t.Unix()\n\t\t}\n\tcase len(split) == 2:\n\t\tts, ds = split[0], split[1]\n\tcase len(split) > 2:\n\t\treturn 0\n\t}\n\ndateStringSwitch:\n\tswitch ds {\n\tcase \"now\", \"today\":\n\t\tt = now\n\tcase \"rnow\", \"rtoday\":\n\t\tt = TimeTruncate(now, truncate)\n\tcase \"midnight\", \"noon\", \"teatime\":\n\t\tyy, mm, dd := now.Date()\n\t\thh, min, _ := parseTime(s) // error ignored, we know it's valid\n\t\tt = time.Date(yy, mm, dd, hh, min, 0, 0, tz)\n\tcase \"yesterday\":\n\t\tt = now.AddDate(0, 0, -1)\n\tcase \"ryesterday\":\n\t\tt = TimeTruncate(now, truncate).AddDate(0, 0, -1)\n\tcase \"tomorrow\":\n\t\tt = now.AddDate(0, 0, 1)\n\tcase \"rtomorrow\":\n\t\tt = TimeTruncate(now, truncate).AddDate(0, 0, 1)\n\tdefault:\n\t\tfor _, format := range TimeFormats {\n\t\t\tt, err = time.ParseInLocation(format, ds, tz)\n\t\t\tif err == nil {\n\t\t\t\tbreak dateStringSwitch\n\t\t\t}\n\t\t}\n\n\t\treturn 0\n\t}\n\n\tvar hour, minute int\n\tif ts != \"\" {\n\t\t// defaults to hour=0, minute=0 on error, which is midnight, which is fine for now\n\t\thour, minute, _ = parseTime(ts)\n\t}\n\n\tyy, mm, dd := t.Date()\n\tt = time.Date(yy, mm, dd, hour, minute, 0, 0, tz)\n\n\treturn t.Unix()\n}\n\nfunc Timezone(qtz string) (*time.Location, error) {\n\tif qtz == \"\" {\n\t\tqtz = \"Local\"\n\t}\n\n\treturn time.LoadLocation(qtz)\n}\n\nfunc TimestampTruncate(ts int64, truncate time.Duration) int64 {\n\tif ts == 0 || truncate == 0 {\n\t\treturn ts\n\t}\n\n\ttm := time.Unix(ts, 0).UTC()\n\n\treturn tm.Truncate(truncate).UTC().Unix()\n}\n\nfunc TimeTruncate(tm time.Time, truncate time.Duration) time.Time {\n\tif truncate == 0 {\n\t\treturn tm\n\t}\n\n\treturn tm.Truncate(truncate)\n}\n" }, { "path": "helper/datetime/datetime_test.go", "content": "package datetime\n\nimport (\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestDateParamToEpoch(t *testing.T) {\n\ttimeZone := time.Local\n\t//16 Aug 1994 15:30\n\tnow := time.Date(1994, time.August, 16, 15, 30, 0, 100, timeZone)\n\n\tconst shortForm = \"15:04:05 2006-Jan-02\"\n\n\tvar tests = []struct {\n\t\tinput string\n\t\toutput string\n\t}{\n\t\t{\"midnight\", \"00:00:00 1994-Aug-16\"},\n\t\t{\"noon\", \"12:00:00 1994-Aug-16\"},\n\t\t{\"teatime\", \"16:00:00 1994-Aug-16\"},\n\t\t{\"tomorrow\", \"00:00:00 1994-Aug-17\"},\n\n\t\t{\"noon 08/12/94\", \"12:00:00 1994-Aug-12\"},\n\t\t{\"midnight 20060812\", \"00:00:00 2006-Aug-12\"},\n\t\t{\"noon tomorrow\", \"12:00:00 1994-Aug-17\"},\n\n\t\t{\"17:04 19940812\", \"17:04:00 1994-Aug-12\"},\n\t\t{\"-1day\", \"15:30:00 1994-Aug-15\"},\n\t\t{\"19940812\", \"00:00:00 1994-Aug-12\"},\n\n\t\t{\"midnight-10\", \"23:59:50 1994-Aug-15\"},\n\t\t{\"midnight-1s\", \"23:59:59 1994-Aug-15\"},\n\t\t{\"midnight-1day\", \"00:00:00 1994-Aug-15\"},\n\t\t{\"midnight-1day+1s\", \"00:00:01 1994-Aug-15\"},\n\t}\n\n\tfor _, tt := range tests {\n\t\tvar (\n\t\t\twant int64\n\t\t\twantTime string\n\t\t)\n\n\t\tif tt.output != \"\" {\n\t\t\tts, err := time.ParseInLocation(shortForm, tt.output, timeZone)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"error parsing time: %q: %v\", tt.output, err)\n\t\t\t}\n\n\t\t\twant = ts.Unix()\n\t\t\twantTime = ts.Format(time.RFC3339Nano)\n\t\t}\n\n\t\tgot := DateParamToEpoch(tt.input, timeZone, now, 0)\n\t\tif got != want {\n\t\t\tgotTime := time.Unix(got, 0).Format(time.RFC3339Nano)\n\t\t\tt.Errorf(\"dateParamToEpoch(%q, local)=\\n%v (%s)\\nwant\\n%v (%s)\", tt.input, got, gotTime, want, wantTime)\n\t\t}\n\t}\n}\n\nfunc TestDateParamToEpochTruncate(t *testing.T) {\n\ttimeZone := time.Local\n\t//16 Aug 1994 15:30\n\tnow := time.Date(1994, time.August, 16, 15, 30, 0, 100, timeZone)\n\n\tconst shortForm = \"15:04:05 2006-Jan-02\"\n\n\tvar tests = []struct {\n\t\tinput string\n\t\toutput string\n\t}{\n\t\t{\"midnight\", \"00:00:00 1994-Aug-16\"},\n\t\t{\"noon\", \"12:00:00 1994-Aug-16\"},\n\t\t{\"teatime\", \"16:00:00 1994-Aug-16\"},\n\t\t{\"tomorrow\", \"00:00:00 1994-Aug-17\"},\n\n\t\t{\"noon 08/12/94\", \"12:00:00 1994-Aug-12\"},\n\t\t{\"midnight 20060812\", \"00:00:00 2006-Aug-12\"},\n\t\t{\"noon tomorrow\", \"12:00:00 1994-Aug-17\"},\n\n\t\t{\"17:04 19940812\", \"17:04:00 1994-Aug-12\"},\n\t\t{\"-1day\", \"15:30:00 1994-Aug-15\"},\n\t\t{\"19940812\", \"00:00:00 1994-Aug-12\"},\n\n\t\t{\"midnight-10\", \"23:59:50 1994-Aug-15\"},\n\t\t{\"midnight-1s\", \"23:59:59 1994-Aug-15\"},\n\t\t{\"midnight-1day\", \"00:00:00 1994-Aug-15\"},\n\n\t\t// truncate\n\t\t{\"now-1\", \"15:29:59 1994-Aug-16\"},\n\t\t{\"now-45s\", \"15:29:15 1994-Aug-16\"},\n\t}\n\n\tfor _, tt := range tests {\n\t\tvar (\n\t\t\twant int64\n\t\t\twantTime string\n\t\t)\n\n\t\tif tt.output != \"\" {\n\t\t\tts, err := time.ParseInLocation(shortForm, tt.output, timeZone)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"error parsing time: %q: %v\", tt.output, err)\n\t\t\t}\n\n\t\t\twant = ts.Unix()\n\t\t\twantTime = ts.Format(time.RFC3339Nano)\n\t\t}\n\n\t\tgot := DateParamToEpoch(tt.input, timeZone, now, 10*time.Second)\n\t\tif got != want {\n\t\t\tgotTime := time.Unix(got, 0).Format(time.RFC3339Nano)\n\t\t\tt.Errorf(\"dateParamToEpoch(%q, local)=\\n%v (%s)\\nwant\\n%v (%s)\", tt.input, got, gotTime, want, wantTime)\n\t\t}\n\t}\n}\n" }, { "path": "helper/errs/errors.go", "content": "package errs\n\nimport \"fmt\"\n\ntype ErrorWithCode struct {\n\terr string\n\tCode int // error code\n}\n\nfunc NewErrorWithCode(err string, code int) error {\n\treturn ErrorWithCode{err, code}\n}\n\nfunc NewErrorfWithCode(code int, f string, args ...interface{}) error {\n\treturn ErrorWithCode{fmt.Sprintf(f, args...), code}\n}\n\nfunc (e ErrorWithCode) Error() string { return e.err }\n" }, { "path": "helper/headers/headers.go", "content": "package headers\n\nimport \"net/http\"\n\nfunc GetHeaders(header *http.Header, keys []string) map[string]string {\n\tif len(keys) > 0 {\n\t\theaders := make(map[string]string)\n\n\t\tfor _, key := range keys {\n\t\t\tvalue := header.Get(key)\n\t\t\tif len(value) > 0 {\n\t\t\t\theaders[key] = value\n\t\t\t}\n\t\t}\n\n\t\treturn headers\n\t}\n\n\treturn nil\n}\n" }, { "path": "helper/http/live-http-client.go", "content": "package http\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"time\"\n)\n\nconst TCPNetwork string = \"tcp\"\n\nfunc DoHTTPOverTCP(ctx context.Context, transport *http.Transport, req *http.Request, timeout time.Duration) (*http.Response, error) {\n\tconn, err := transport.DialContext(ctx, TCPNetwork, req.URL.Host)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\terr = conn.SetDeadline(time.Now().Add(timeout))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\terr = req.Write(conn)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar backup_buf bytes.Buffer\n\treader := bufio.NewReader(io.TeeReader(conn, &backup_buf))\n\n\tfor {\n\t\tline, err := reader.ReadString('\\n')\n\t\tif err != nil {\n\t\t\tif netErr, ok := err.(net.Error); ok && netErr.Timeout() {\n\t\t\t\tfake_body_delimer := bytes.NewBuffer([]byte{'\\r', '\\n', '\\r', '\\n'})\n\n\t\t\t\tresp, err := http.ReadResponse(bufio.NewReader(io.MultiReader(&backup_buf, fake_body_delimer)), nil)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\n\t\t\t\treturn resp, netErr\n\t\t\t}\n\n\t\t\treturn nil, err\n\t\t}\n\n\t\tif line == \"\\r\\n\" {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tfull_resp_stream := io.MultiReader(&backup_buf, conn)\n\tresp, err := http.ReadResponse(bufio.NewReader(full_resp_stream), nil)\n\n\treturn resp, err\n}\n" }, { "path": "helper/pickle/pickle.go", "content": "package pickle\n\nimport (\n\t\"encoding/binary\"\n\t\"io\"\n\t\"math\"\n)\n\nvar EmptyList = []byte{0x28, 0x6c, 0x70, 0x30, 0xa, 0x2e}\n\n// Pickle encoder\ntype Writer struct {\n\tw io.Writer\n}\n\nfunc NewWriter(w io.Writer) *Writer {\n\treturn &Writer{w: w}\n}\n\nfunc (p *Writer) Mark() {\n\tp.w.Write([]byte{'('})\n}\n\nfunc (p *Writer) Stop() {\n\tp.w.Write([]byte{'.'})\n}\n\nfunc (p *Writer) Append() {\n\tp.w.Write([]byte{'a'})\n}\n\nfunc (p *Writer) SetItem() {\n\tp.w.Write([]byte{'s'})\n}\n\nfunc (p *Writer) List() {\n\tp.w.Write([]byte{'(', 'l'})\n}\n\nfunc (p *Writer) Dict() {\n\tp.w.Write([]byte{'(', 'd'})\n}\n\nfunc (p *Writer) TupleEnd() {\n\tp.w.Write([]byte{'t'})\n}\n\nfunc (p *Writer) Bytes(byt []byte) {\n\tl := len(byt)\n\n\tif l < 256 {\n\t\tp.w.Write([]byte{'U', byte(l)})\n\t} else {\n\t\tvar b [5]byte\n\t\tb[0] = 'T'\n\t\tbinary.LittleEndian.PutUint32(b[1:5], uint32(l))\n\t\tp.w.Write(b[:])\n\t}\n\n\tp.w.Write(byt)\n}\n\nfunc (p *Writer) String(v string) {\n\tp.Bytes([]byte(v))\n}\n\nfunc (p *Writer) Uint32(v uint32) {\n\tp.w.Write([]byte{'J'})\n\n\tvar b [4]byte\n\n\tbinary.LittleEndian.PutUint32(b[:], v)\n\tp.w.Write(b[:])\n}\n\nfunc (p *Writer) AppendFloat64(v float64) {\n\tu := math.Float64bits(v)\n\n\tvar b [10]byte\n\tb[0] = 'G'\n\tb[9] = 'a'\n\n\tbinary.BigEndian.PutUint64(b[1:10], u)\n\n\tp.w.Write(b[:])\n}\n\nfunc (p *Writer) AppendNulls(count int) {\n\tfor i := 0; i < count; i++ {\n\t\tp.w.Write([]byte{'N', 'a'})\n\t}\n}\n\nfunc (p *Writer) Bool(b bool) {\n\tif b {\n\t\tp.w.Write([]byte{'I', '0', '1', '\\n'})\n\t} else {\n\t\tp.w.Write([]byte{'I', '0', '0', '\\n'})\n\t}\n}\n" }, { "path": "helper/point/func.go", "content": "package point\n\nimport (\n\t\"fmt\"\n\t\"math\"\n)\n\n// CleanUp removes points with empty metric\n// for run after Deduplicate, Merge, etc for result cleanup\nfunc CleanUp(points []Point) []Point {\n\tl := len(points)\n\tsquashed := 0\n\n\tfor i := 0; i < l; i++ {\n\t\tif points[i].MetricID == 0 || math.IsNaN(points[i].Value) {\n\t\t\tsquashed++\n\t\t\tcontinue\n\t\t}\n\n\t\tif squashed > 0 {\n\t\t\tpoints[i-squashed] = points[i]\n\t\t}\n\t}\n\n\treturn points[:l-squashed]\n}\n\n// Uniq removes points with equal metric and time\nfunc Uniq(points []Point) []Point {\n\tl := len(points)\n\n\tvar i, n int\n\t// i - current position of iterator\n\t// n - position on first record with current key (metric + time)\n\n\tfor i = 1; i < l; i++ {\n\t\tif points[i].MetricID != points[n].MetricID ||\n\t\t\tpoints[i].Time != points[n].Time {\n\t\t\tn = i\n\t\t\tcontinue\n\t\t}\n\n\t\tif points[i].Timestamp > points[n].Timestamp {\n\t\t\tpoints[n] = points[i]\n\t\t}\n\n\t\tpoints[i].MetricID = 0 // mark for remove\n\t}\n\n\treturn CleanUp(points)\n}\n\n// FillNulls accepts an ordered []Point for one metric and returns a generator that will return all points for specific\n// interval. Generator returns EmptyPoint when it's finished\nfunc FillNulls(points []Point, from, until, step uint32) (start, stop, count uint32, getter GetValueOrNaN) {\n\tstart = from - (from % step)\n\tif start < from {\n\t\tstart += step\n\t}\n\n\tstop = until - (until % step) + step\n\tcount = (stop - start) / step\n\tlast := start - step\n\tcurrentPoint := 0\n\n\tvar metricID uint32\n\tif len(points) > 0 {\n\t\tmetricID = points[0].MetricID\n\t}\n\n\tgetter = func() (float64, error) {\n\t\tif stop <= last {\n\t\t\treturn 0, ErrTimeGreaterStop\n\t\t}\n\n\t\tfor i := currentPoint; i < len(points); i++ {\n\t\t\tpoint := points[i]\n\t\t\tif metricID != point.MetricID {\n\t\t\t\treturn 0, fmt.Errorf(\"the point MetricID %d differs from other %d: %w\", point.MetricID, metricID, ErrWrongMetricID)\n\t\t\t}\n\n\t\t\tif point.Time < start {\n\t\t\t\t// Points begin before request's start\n\t\t\t\tcurrentPoint++\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif point.Time <= last {\n\t\t\t\t// This is definitely an error. Possible reason is unsorted points\n\t\t\t\treturn 0, fmt.Errorf(\"the time is less or equal to previous %d < %d: %w\", point.Time, last, ErrPointsUnsorted)\n\t\t\t}\n\n\t\t\tif stop <= point.Time {\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\tif last+step < point.Time {\n\t\t\t\t// There are nulls in slice\n\t\t\t\tlast += step\n\t\t\t\treturn math.NaN(), nil\n\t\t\t}\n\n\t\t\tlast = point.Time\n\t\t\tcurrentPoint = i + 1\n\n\t\t\treturn point.Value, nil\n\t\t}\n\n\t\tif last+step < stop {\n\t\t\tlast += step\n\t\t\treturn math.NaN(), nil\n\t\t}\n\n\t\treturn 0, ErrTimeGreaterStop\n\t}\n\n\treturn\n}\n" }, { "path": "helper/point/func_test.go", "content": "package point\n\nimport (\n\t\"errors\"\n\t\"math\"\n\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nvar nan = math.NaN()\n\nfunc TestUniq(t *testing.T) {\n\ttests := [][2][]Point{\n\t\t{\n\t\t\t{ // in\n\t\t\t\tPoint{MetricID: 1, Time: 1478025152, Timestamp: 1, Value: 1},\n\t\t\t\tPoint{MetricID: 1, Time: 1478025152, Timestamp: 2, Value: 2},\n\t\t\t\tPoint{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1},\n\t\t\t},\n\t\t\t{ // out\n\t\t\t\tPoint{MetricID: 1, Time: 1478025152, Timestamp: 2, Value: 2},\n\t\t\t\tPoint{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t{ // in\n\t\t\t\tPoint{MetricID: 1, Time: 1478025152, Timestamp: 3, Value: 1},\n\t\t\t\tPoint{MetricID: 1, Time: 1478025152, Timestamp: 2, Value: 2},\n\t\t\t\tPoint{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1},\n\t\t\t},\n\t\t\t{ // out\n\t\t\t\tPoint{MetricID: 1, Time: 1478025152, Timestamp: 3, Value: 1},\n\t\t\t\tPoint{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t{ // in\n\t\t\t\tPoint{MetricID: 1, Time: 1478025152, Timestamp: 3, Value: nan},\n\t\t\t\tPoint{MetricID: 1, Time: 1478025152, Timestamp: 2, Value: 2},\n\t\t\t\tPoint{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1},\n\t\t\t},\n\t\t\t{ // out\n\t\t\t\tPoint{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tresult := Uniq(test[0])\n\t\tassert.Equal(t, test[1], result)\n\t}\n}\n\nfunc TestCleanUp(t *testing.T) {\n\ttests := [][2][]Point{\n\t\t{\n\t\t\t{ // in\n\t\t\t\tPoint{MetricID: 1, Time: 1478025152, Timestamp: 1, Value: 1},\n\t\t\t\tPoint{MetricID: 0, Time: 1478025152, Timestamp: 2, Value: 2},\n\t\t\t\tPoint{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1},\n\t\t\t},\n\t\t\t{ // out\n\t\t\t\tPoint{MetricID: 1, Time: 1478025152, Timestamp: 1, Value: 1},\n\t\t\t\tPoint{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t{ // in\n\t\t\t\tPoint{MetricID: 0, Time: 1478025152, Timestamp: 3, Value: 1},\n\t\t\t\tPoint{MetricID: 0, Time: 1478025152, Timestamp: 2, Value: 2},\n\t\t\t\tPoint{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1},\n\t\t\t},\n\t\t\t{ // out\n\t\t\t\tPoint{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t{ // in\n\t\t\t\tPoint{MetricID: 0, Time: 1478025152, Timestamp: 3, Value: 1},\n\t\t\t\tPoint{MetricID: 0, Time: 1478025152, Timestamp: 2, Value: 2},\n\t\t\t\tPoint{MetricID: 0, Time: 1478025155, Timestamp: 1, Value: 1},\n\t\t\t},\n\t\t\t{ // out\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t{ // in\n\t\t\t\tPoint{MetricID: 1, Time: 1478025152, Timestamp: 3, Value: nan},\n\t\t\t\tPoint{MetricID: 1, Time: 1478025152, Timestamp: 2, Value: 2},\n\t\t\t\tPoint{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: nan},\n\t\t\t},\n\t\t\t{ // out\n\t\t\t\tPoint{MetricID: 1, Time: 1478025152, Timestamp: 2, Value: 2},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tresult := CleanUp(test[0])\n\t\tassert.Equal(t, test[1], result)\n\t}\n}\n\nfunc TestFillNulls(t *testing.T) {\n\ttype in struct {\n\t\tpoints []Point\n\t\tfrom uint32\n\t\tuntil uint32\n\t\tstep uint32\n\t}\n\n\ttype expected struct {\n\t\tvalues []float64\n\t\tstart uint32\n\t\tstop uint32\n\t\tcount uint32\n\t\terr error\n\t}\n\n\ttests := []struct {\n\t\tname string\n\t\tin\n\t\texpected expected\n\t}{\n\t\t{\n\t\t\tname: \"shorter with NaNs\",\n\t\t\tin: in{\n\t\t\t\t[]Point{\n\t\t\t\t\t{1, 1, 0, 0},\n\t\t\t\t\t{1, 12, 2, 0},\n\t\t\t\t\t{1, 2, 4, 0},\n\t\t\t\t\t{1, 4, 8, 0},\n\t\t\t\t},\n\t\t\t\t1,\n\t\t\t\t13,\n\t\t\t\t2,\n\t\t\t},\n\t\t\texpected: expected{[]float64{12, 2, nan, 4, nan, nan}, 2, 14, 6, nil},\n\t\t},\n\t\t{\n\t\t\tname: \"longer than time interval, but wrong step\",\n\t\t\tin: in{\n\t\t\t\t[]Point{\n\t\t\t\t\t{1, 1, 0, 0},\n\t\t\t\t\t{1, 12, 2, 0},\n\t\t\t\t\t{1, 2, 4, 0},\n\t\t\t\t\t{1, 3, 6, 0},\n\t\t\t\t\t{1, 4, 8, 0},\n\t\t\t\t},\n\t\t\t\t2,\n\t\t\t\t7,\n\t\t\t\t1,\n\t\t\t},\n\t\t\texpected: expected{[]float64{12, nan, 2, nan, 3, nan}, 2, 8, 6, nil},\n\t\t},\n\t\t{\n\t\t\tname: \"wrong metric ID\",\n\t\t\tin: in{\n\t\t\t\t[]Point{\n\t\t\t\t\t{1, 1, 0, 0},\n\t\t\t\t\t{1, 12, 2, 0},\n\t\t\t\t\t{2, 12, 4, 0},\n\t\t\t\t},\n\t\t\t\t1,\n\t\t\t\t13,\n\t\t\t\t2,\n\t\t\t},\n\t\t\texpected: expected{[]float64{12}, 2, 14, 6, ErrWrongMetricID},\n\t\t},\n\t\t{\n\t\t\tname: \"unsorted points cause error\",\n\t\t\tin: in{\n\t\t\t\t[]Point{\n\t\t\t\t\t{1, 12, 4, 0},\n\t\t\t\t\t{1, 2, 2, 0},\n\t\t\t\t\t{1, 1, 6, 0},\n\t\t\t\t},\n\t\t\t\t1,\n\t\t\t\t13,\n\t\t\t\t2,\n\t\t\t},\n\t\t\texpected: expected{[]float64{nan, 12}, 2, 14, 6, ErrPointsUnsorted},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tresult := expected{}\n\n\t\t\tvar (\n\t\t\t\tgetter GetValueOrNaN\n\t\t\t\tvalue float64\n\t\t\t)\n\n\t\t\tresult.start, result.stop, result.count, getter = FillNulls(test.points, test.from, test.until, test.step)\n\n\t\t\tresult.values = make([]float64, 0, result.count)\n\n\t\t\tfor {\n\t\t\t\tvalue, result.err = getter()\n\t\t\t\tif result.err != nil {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\n\t\t\t\tresult.values = append(result.values, value)\n\t\t\t}\n\n\t\t\tif !errors.Is(result.err, ErrTimeGreaterStop) {\n\t\t\t\tassert.ErrorIs(t, result.err, test.expected.err)\n\t\t\t}\n\n\t\t\tresult.err = nil\n\t\t\ttest.expected.err = nil\n\n\t\t\t// Comparing values requires work around NaNs\n\t\t\tassert.Equal(t, len(result.values), len(test.expected.values), \"the length of expected and got values are different\")\n\n\t\t\tfor i := range result.values {\n\t\t\t\tif math.IsNaN(test.expected.values[i]) {\n\t\t\t\t\tassert.True(t, math.IsNaN(result.values[i]))\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tassert.Equal(t, test.expected.values[i], result.values[i])\n\t\t\t}\n\n\t\t\tresult.values = []float64{}\n\t\t\ttest.expected.values = []float64{}\n\n\t\t\tassert.Equal(t, test.expected, result)\n\t\t})\n\t}\n}\n" }, { "path": "helper/point/point.go", "content": "package point\n\nimport \"fmt\"\n\ntype Point struct {\n\tMetricID uint32\n\tValue float64\n\tTime uint32\n\tTimestamp uint32 // keep max if metric and time equal on two points\n}\n\n// GetValueOrNaN returns Value for the next point or NaN if the value is omited. ErrTimeGreaterStop shows the normal ending. Any else error is considered as real error\ntype GetValueOrNaN func() (float64, error)\n\n// ErrTimeGreaterStop shows the correct over for GetValueOrNaN\nvar ErrTimeGreaterStop = fmt.Errorf(\"the points for time interval are rover\")\n\n// ErrWrongMetricID shows the Point.MetricID is wrong somehow\nvar ErrWrongMetricID = fmt.Errorf(\"the point MetricID is wrong\")\n\n// ErrPointsUnsorted returns for unsorted []Point or Points\nvar ErrPointsUnsorted = fmt.Errorf(\"the points are unsorted\")\n" }, { "path": "helper/point/points.go", "content": "package point\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n)\n\n// Points is a structure that stores points and additional information about them, e.g. steps, aggregating functions and names.\ntype Points struct {\n\tlist []Point\n\tidMap map[string]uint32\n\tmetrics []string\n\tsteps []uint32\n\taggs []*string\n\tuniqAgg []string\n}\n\n// NextMetric returns the list of points for one metric name\ntype NextMetric func() []Point\n\n// NewPoints return new empty Points\nfunc NewPoints() *Points {\n\treturn &Points{\n\t\tlist: make([]Point, 0),\n\t\tidMap: make(map[string]uint32),\n\t\tmetrics: make([]string, 0),\n\t}\n}\n\n// AppendPoint creates a Point with given values and appends it to list\nfunc (pp *Points) AppendPoint(metricID uint32, value float64, time uint32, version uint32) {\n\tpp.list = append(pp.list, Point{\n\t\tMetricID: metricID,\n\t\tValue: value,\n\t\tTime: time,\n\t\tTimestamp: version,\n\t})\n}\n\n// MetricID checks if metric name already exists and returns the ID for it. If not, it creates it first.\nfunc (pp *Points) MetricID(metricName string) uint32 {\n\tid := pp.idMap[metricName]\n\tif id == 0 {\n\t\tpp.metrics = append(pp.metrics, metricName)\n\t\tid = uint32(len(pp.metrics))\n\t\tpp.idMap[metricName] = id\n\t}\n\n\treturn id\n}\n\n// MetricIDBytes checks if metric name already exists and returns the ID for it. If not, it creates it first.\nfunc (pp *Points) MetricIDBytes(metricNameBytes []byte) uint32 {\n\t// @TODO: optimize?\n\treturn pp.MetricID(string(metricNameBytes))\n}\n\n// MetricName returns name for metric with given metricID or empty string when ID does not exist\nfunc (pp *Points) MetricName(metricID uint32) string {\n\ti := int(metricID)\n\tif i < 1 || len(pp.metrics) < i {\n\t\treturn \"\"\n\t}\n\n\treturn pp.metrics[i-1]\n}\n\n// List returns list of points\nfunc (pp *Points) List() []Point {\n\treturn pp.list\n}\n\n// ReplaceList replaces list of points\nfunc (pp *Points) ReplaceList(list []Point) {\n\tpp.list = list\n}\n\n// GetStep returns uint32 step for given metric id.\nfunc (pp *Points) GetStep(id uint32) (uint32, error) {\n\ti := int(id)\n\tif i < 1 || len(pp.steps) < i {\n\t\treturn 0, fmt.Errorf(\"wrong id %d for given steps %d: %w\", i, len(pp.steps), ErrWrongMetricID)\n\t}\n\n\treturn pp.steps[i-1], nil\n}\n\n// SetSteps accepts map of metric name as keys and step as values and sets slice of uint32 steps for existing metrics in Data.Points\nfunc (pp *Points) SetSteps(steps map[uint32][]string) {\n\tif len(steps) == 0 {\n\t\treturn\n\t}\n\n\tpp.steps = make([]uint32, len(pp.metrics))\n\n\tfor step, mm := range steps {\n\t\tfor _, m := range mm {\n\t\t\tif id, ok := pp.idMap[m]; ok {\n\t\t\t\tpp.steps[id-1] = step\n\t\t\t}\n\t\t}\n\t}\n}\n\n// GetAggregation returns string function for given metric id.\nfunc (pp *Points) GetAggregation(id uint32) (string, error) {\n\ti := int(id)\n\tif i < 1 || len(pp.aggs) < i {\n\t\treturn \"\", fmt.Errorf(\"wrong id %d for given functions %d: %w\", i, len(pp.aggs), ErrWrongMetricID)\n\t}\n\n\treturn *pp.aggs[i-1], nil\n}\n\n// SetAggregations accepts map of metric name as keys and function as values and sets slice of functions for existing metrics in Data.Points\nfunc (pp *Points) SetAggregations(functions map[string][]string) {\n\tpp.aggs = make([]*string, len(pp.metrics))\n\tpp.uniqAgg = make([]string, 0, len(functions))\n\n\tfor f := range functions {\n\t\tpp.uniqAgg = append(pp.uniqAgg, f)\n\t}\n\n\tfor i, f := range pp.uniqAgg {\n\t\tfor _, m := range functions[f] {\n\t\t\tif id, ok := pp.idMap[m]; ok {\n\t\t\t\tpp.aggs[id-1] = &pp.uniqAgg[i]\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (pp *Points) Len() int {\n\treturn len(pp.list)\n}\n\nfunc (pp *Points) Less(i, j int) bool {\n\tif pp.list[i].MetricID == pp.list[j].MetricID {\n\t\treturn pp.list[i].Time < pp.list[j].Time\n\t}\n\n\treturn pp.list[i].MetricID < pp.list[j].MetricID\n}\n\nfunc (pp *Points) Swap(i, j int) {\n\tpp.list[i], pp.list[j] = pp.list[j], pp.list[i]\n}\n\n// Sort sorts the points list by ID, Time\nfunc (pp *Points) Sort() {\n\tsort.Sort(pp)\n}\n\n// Uniq cleans up the points\nfunc (pp *Points) Uniq() {\n\tpp.list = Uniq(pp.list)\n}\n\n// GroupByMetric returns NextMetric function, that by each call returns points for one next metric.\n// It should be called only on sorted and cleaned Points.\nfunc (pp *Points) GroupByMetric() NextMetric {\n\tvar i, n int\n\n\tl := pp.Len()\n\t// i - current position of iterator\n\t// n - position of the first record with current metric\n\treturn func() []Point {\n\t\tif n == l {\n\t\t\treturn []Point{}\n\t\t}\n\n\t\tfor i = n; i < l; i++ {\n\t\t\tif pp.list[i].MetricID != pp.list[n].MetricID {\n\t\t\t\tpoints := pp.list[n:i]\n\t\t\t\tn = i\n\n\t\t\t\treturn points\n\t\t\t}\n\t\t}\n\n\t\tpoints := pp.list[n:i]\n\t\tn = i\n\n\t\treturn points\n\t}\n}\n" }, { "path": "helper/rollup/aggr.go", "content": "package rollup\n\nimport (\n\t\"github.com/lomik/graphite-clickhouse/helper/point\"\n)\n\nvar AggrMap = map[string]*Aggr{\n\t\"avg\": {\"avg\", AggrAvg},\n\t\"max\": {\"max\", AggrMax},\n\t\"min\": {\"min\", AggrMin},\n\t\"sum\": {\"sum\", AggrSum},\n\t\"any\": {\"any\", AggrAny},\n\t\"anyLast\": {\"anyLast\", AggrAnyLast},\n}\n\ntype Aggr struct {\n\tname string\n\tf func(points []point.Point) (r float64)\n}\n\nfunc (ag *Aggr) Name() string {\n\tif ag == nil {\n\t\treturn \"\"\n\t}\n\n\treturn ag.name\n}\n\nfunc (ag *Aggr) String() string {\n\tif ag == nil {\n\t\treturn \"\"\n\t}\n\n\treturn ag.name\n}\n\nfunc (ag *Aggr) Do(points []point.Point) (r float64) {\n\tif ag == nil || ag.f == nil {\n\t\treturn 0\n\t}\n\n\treturn ag.f(points)\n}\n\nfunc AggrSum(points []point.Point) (r float64) {\n\tfor _, p := range points {\n\t\tr += p.Value\n\t}\n\n\treturn\n}\n\nfunc AggrMax(points []point.Point) (r float64) {\n\tif len(points) > 0 {\n\t\tr = points[0].Value\n\t}\n\n\tfor _, p := range points {\n\t\tif p.Value > r {\n\t\t\tr = p.Value\n\t\t}\n\t}\n\n\treturn\n}\n\nfunc AggrMin(points []point.Point) (r float64) {\n\tif len(points) > 0 {\n\t\tr = points[0].Value\n\t}\n\n\tfor _, p := range points {\n\t\tif p.Value < r {\n\t\t\tr = p.Value\n\t\t}\n\t}\n\n\treturn\n}\n\nfunc AggrAvg(points []point.Point) (r float64) {\n\tif len(points) == 0 {\n\t\treturn\n\t}\n\n\tr = AggrSum(points) / float64(len(points))\n\n\treturn\n}\n\nfunc AggrAny(points []point.Point) (r float64) {\n\tif len(points) > 0 {\n\t\tr = points[0].Value\n\t}\n\n\treturn\n}\n\nfunc AggrAnyLast(points []point.Point) (r float64) {\n\tif len(points) > 0 {\n\t\tr = points[len(points)-1].Value\n\t}\n\n\treturn\n}\n" }, { "path": "helper/rollup/compact.go", "content": "package rollup\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n)\n\n/*\ncompact form of rollup rules for tests\n\nregexp;function;age:precision,age:precision,...\n*/\n\nfunc parseCompact(body string) (*Rules, error) {\n\tlines := strings.Split(body, \"\\n\")\n\tpatterns := make([]Pattern, 0)\n\n\tfor _, line := range lines {\n\t\tif strings.TrimSpace(line) == \"\" {\n\t\t\tcontinue\n\t\t}\n\n\t\tp2 := strings.LastIndexByte(line, ';')\n\t\tif p2 < 0 {\n\t\t\treturn nil, fmt.Errorf(\"can't parse line: %#v\", line)\n\t\t}\n\n\t\tp1 := strings.LastIndexByte(line[:p2], ';')\n\t\tif p1 < 0 {\n\t\t\treturn nil, fmt.Errorf(\"can't parse line: %#v\", line)\n\t\t}\n\n\t\tregexp := strings.TrimSpace(line[:p1])\n\t\tfunction := strings.TrimSpace(line[p1+1 : p2])\n\t\tretention := make([]Retention, 0)\n\n\t\tif strings.TrimSpace(line[p2+1:]) != \"\" {\n\t\t\tarr := strings.Split(line[p2+1:], \",\")\n\n\t\t\tfor _, r := range arr {\n\t\t\t\tp := strings.Split(r, \":\")\n\t\t\t\tif len(p) != 2 {\n\t\t\t\t\treturn nil, fmt.Errorf(\"can't parse line: %#v\", line)\n\t\t\t\t}\n\n\t\t\t\tage, err := strconv.ParseUint(strings.TrimSpace(p[0]), 10, 32)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\n\t\t\t\tprecision, err := strconv.ParseUint(strings.TrimSpace(p[1]), 10, 32)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\n\t\t\t\tretention = append(retention, Retention{Age: uint32(age), Precision: uint32(precision)})\n\t\t\t}\n\t\t}\n\n\t\tpatterns = append(patterns, Pattern{Regexp: regexp, Function: function, Retention: retention})\n\t}\n\n\treturn (&Rules{Pattern: patterns}).compile()\n}\n" }, { "path": "helper/rollup/compact_test.go", "content": "package rollup\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestParseCompact(t *testing.T) {\n\tconfig := `\n\tclick_cost;any;0:3600,86400:60\n\t;max;0:60,3600:300,86400:3600`\n\n\texpected, _ := (&Rules{\n\t\tPattern: []Pattern{\n\t\t\t{Regexp: \"click_cost\", Function: \"any\", Retention: []Retention{\n\t\t\t\t{Age: 0, Precision: 3600},\n\t\t\t\t{Age: 86400, Precision: 60},\n\t\t\t}},\n\t\t\t{Regexp: \"\", Function: \"max\", Retention: []Retention{\n\t\t\t\t{Age: 0, Precision: 60},\n\t\t\t\t{Age: 3600, Precision: 300},\n\t\t\t\t{Age: 86400, Precision: 3600},\n\t\t\t}},\n\t\t},\n\t}).compile()\n\n\tassert := assert.New(t)\n\tr, err := parseCompact(config)\n\tassert.NoError(err)\n\tassert.Equal(expected, r)\n}\n" }, { "path": "helper/rollup/remote.go", "content": "package rollup\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/lomik/zapwriter\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n)\n\ntype rollupRulesResponseRecord struct {\n\tRuleType RuleType `json:\"rule_type\"`\n\tRegexp string `json:\"regexp\"`\n\tFunction string `json:\"function\"`\n\tAge string `json:\"age\"`\n\tPrecision string `json:\"precision\"`\n\tIsDefault int `json:\"is_default\"`\n}\ntype rollupRulesResponse struct {\n\tData []rollupRulesResponseRecord `json:\"data\"`\n}\n\nfunc parseJson(body []byte) (*Rules, error) {\n\tresp := &rollupRulesResponse{}\n\n\terr := json.Unmarshal(body, resp)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tr := &Rules{\n\t\tPattern: make([]Pattern, 0),\n\t}\n\n\tmakeRetention := func(d *rollupRulesResponseRecord) (Retention, error) {\n\t\tage, err := strconv.ParseInt(d.Age, 10, 32)\n\t\tif err != nil {\n\t\t\treturn Retention{}, err\n\t\t}\n\n\t\tprec, err := strconv.ParseInt(d.Precision, 10, 32)\n\t\tif err != nil {\n\t\t\treturn Retention{}, err\n\t\t}\n\n\t\treturn Retention{Age: uint32(age), Precision: uint32(prec)}, nil\n\t}\n\n\tlast := func() *Pattern {\n\t\tif len(r.Pattern) == 0 {\n\t\t\treturn nil\n\t\t}\n\n\t\treturn &r.Pattern[len(r.Pattern)-1]\n\t}\n\n\tdefaultFunction := \"\"\n\tdefaultRetention := make([]Retention, 0)\n\n\t// var last *Pattern\n\tfor _, d := range resp.Data {\n\t\tif d.IsDefault == 1 {\n\t\t\tif d.Function != \"\" {\n\t\t\t\tdefaultFunction = d.Function\n\t\t\t}\n\n\t\t\tif d.Age != \"\" && d.Precision != \"\" && d.Precision != \"0\" {\n\t\t\t\trt, err := makeRetention(&d)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\n\t\t\t\tdefaultRetention = append(defaultRetention, rt)\n\t\t\t}\n\t\t} else {\n\t\t\tif last() == nil || last().Regexp != d.Regexp || last().Function != d.Function {\n\t\t\t\tr.Pattern = append(r.Pattern, Pattern{\n\t\t\t\t\tRuleType: d.RuleType,\n\t\t\t\t\tRetention: make([]Retention, 0),\n\t\t\t\t\tRegexp: d.Regexp,\n\t\t\t\t\tFunction: d.Function,\n\t\t\t\t})\n\t\t\t}\n\n\t\t\tif d.Age != \"\" && d.Precision != \"\" && d.Precision != \"0\" {\n\t\t\t\trt, err := makeRetention(&d)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\n\t\t\t\tlast().Retention = append(last().Retention, rt)\n\t\t\t}\n\t\t}\n\t}\n\n\tif defaultFunction != \"\" || len(defaultRetention) != 0 {\n\t\tr.Pattern = append(r.Pattern, Pattern{\n\t\t\tRegexp: \"\",\n\t\t\tFunction: defaultFunction,\n\t\t\tRetention: defaultRetention,\n\t\t})\n\t}\n\n\treturn r.compile()\n}\n\nvar timeoutRulesLoad = 10 * time.Second\n\nfunc RemoteLoad(addr string, tlsConf *tls.Config, table string) (*Rules, error) {\n\tvar db string\n\n\tarr := strings.SplitN(table, \".\", 2)\n\tif len(arr) == 1 {\n\t\tdb = \"default\"\n\t} else {\n\t\tdb, table = arr[0], arr[1]\n\t}\n\n\tquery := `SELECT\n\t rule_type,\n\t regexp,\n\t function,\n\t age,\n\t precision,\n\t is_default\n\tFROM system.graphite_retentions\n\tARRAY JOIN Tables AS table\n\tWHERE (table.database = '` + db + `') AND (table.table = '` + table + `')\n\tORDER BY\n\t is_default ASC,\n\t priority ASC,\n\t regexp ASC,\n\t\tage ASC\n\tFORMAT JSON`\n\n\tbody, _, _, err := clickhouse.Query(\n\t\tscope.New(context.Background()).WithLogger(zapwriter.Logger(\"rollup\")).WithTable(\"system.graphite_retentions\"),\n\t\taddr,\n\t\tquery,\n\t\tclickhouse.Options{\n\t\t\tTimeout: timeoutRulesLoad,\n\t\t\tConnectTimeout: timeoutRulesLoad,\n\t\t\tTLSConfig: tlsConf,\n\t\t\tCheckRequestProgress: false,\n\t\t\tProgressSendingInterval: 10 * time.Second,\n\t\t},\n\t\tnil,\n\t)\n\tif err != nil && strings.Contains(err.Error(), \" Missing columns: 'rule_type' \") {\n\t\t// for old version\n\t\tquery = `SELECT\n\t\t\tregexp,\n\t\t\tfunction,\n\t\t\tage,\n\t\t\tprecision,\n\t\t\tis_default\n\t\tFROM system.graphite_retentions\n\t\tARRAY JOIN Tables AS table\n\t\tWHERE (table.database = '` + db + `') AND (table.table = '` + table + `')\n\t\tORDER BY\n\t\t\tis_default ASC,\n\t\t\tpriority ASC,\n\t\t\tregexp ASC,\n\t\t\tage ASC\n\t\tFORMAT JSON`\n\n\t\tbody, _, _, err = clickhouse.Query(\n\t\t\tscope.New(context.Background()).WithLogger(zapwriter.Logger(\"rollup\")).WithTable(\"system.graphite_retentions\"),\n\t\t\taddr,\n\t\t\tquery,\n\t\t\tclickhouse.Options{\n\t\t\t\tTimeout: timeoutRulesLoad,\n\t\t\t\tConnectTimeout: timeoutRulesLoad,\n\t\t\t\tTLSConfig: tlsConf,\n\t\t\t\tCheckRequestProgress: false,\n\t\t\t\tProgressSendingInterval: 10 * time.Second,\n\t\t\t},\n\t\t\tnil,\n\t\t)\n\t}\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tr, err := parseJson(body)\n\tif r != nil {\n\t\tr.Updated = time.Now().Unix()\n\t}\n\n\treturn r, err\n}\n" }, { "path": "helper/rollup/remote_test.go", "content": "package rollup\n\nimport (\n\t\"encoding/json\"\n\t\"regexp\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc assertJsonEqual(t *testing.T, expected string, actual string) {\n\tvar e, a interface{}\n\n\tassert := assert.New(t)\n\tassert.NoError(json.Unmarshal([]byte(expected), &e))\n\tassert.NoError(json.Unmarshal([]byte(actual), &a))\n\n\tassert.Equal(e, a)\n}\n\nfunc TestParseJson(t *testing.T) {\n\tresponse := `{\n\t\"meta\":\n\t[\n\t\t{\n\t\t\t\"name\": \"regexp\",\n\t\t\t\"type\": \"String\"\n\t\t},\n\t\t{\n\t\t\t\"name\": \"function\",\n\t\t\t\"type\": \"String\"\n\t\t},\n\t\t{\n\t\t\t\"name\": \"age\",\n\t\t\t\"type\": \"UInt64\"\n\t\t},\n\t\t{\n\t\t\t\"name\": \"precision\",\n\t\t\t\"type\": \"UInt64\"\n\t\t},\n\t\t{\n\t\t\t\"name\": \"is_default\",\n\t\t\t\"type\": \"UInt8\"\n\t\t}\n\t],\n\n\t\"data\":\n\t[\n\t\t{\n\t\t\t\"regexp\": \"^hourly\",\n\t\t\t\"function\": \"\",\n\t\t\t\"age\": \"0\",\n\t\t\t\"precision\": \"3600\",\n\t\t\t\"is_default\": 0\n\t\t},\n\t\t{\n\t\t\t\"regexp\": \"^hourly\",\n\t\t\t\"function\": \"\",\n\t\t\t\"age\": \"3600\",\n\t\t\t\"precision\": \"13600\",\n\t\t\t\"is_default\": 0\n\t\t},\n\t\t{\n\t\t\t\"regexp\": \"^live\",\n\t\t\t\"function\": \"\",\n\t\t\t\"age\": \"0\",\n\t\t\t\"precision\": \"1\",\n\t\t\t\"is_default\": 0\n\t\t},\n\t\t{\n\t\t\t\"regexp\": \"total$\",\n\t\t\t\"function\": \"sum\",\n\t\t\t\"age\": \"0\",\n\t\t\t\"precision\": \"0\",\n\t\t\t\"is_default\": 0\n\t\t},\n\t\t{\n\t\t\t\"regexp\": \"min$\",\n\t\t\t\"function\": \"min\",\n\t\t\t\"age\": \"0\",\n\t\t\t\"precision\": \"0\",\n\t\t\t\"is_default\": 0\n\t\t},\n\t\t{\n\t\t\t\"regexp\": \"max$\",\n\t\t\t\"function\": \"max\",\n\t\t\t\"age\": \"0\",\n\t\t\t\"precision\": \"0\",\n\t\t\t\"is_default\": 0\n\t\t},\n\t\t{\n\t\t\t\"regexp\": \"\",\n\t\t\t\"function\": \"max\",\n\t\t\t\"age\": \"0\",\n\t\t\t\"precision\": \"60\",\n\t\t\t\"is_default\": 1\n\t\t}\n\t],\n\n\t\"rows\": 7,\n\n\t\"statistics\":\n\t{\n\t\t\"elapsed\": 0.00053715,\n\t\t\"rows_read\": 7,\n\t\t\"bytes_read\": 1158\n\t}\n}`\n\n\tcompact := `\n\t^hourly;;0:3600,3600:13600\n\t^live;;0:1\n\ttotal$;sum;\n\tmin$;min;\n\tmax$;max;\n\t;max;0:60\n\t`\n\n\tassert := assert.New(t)\n\texpected, err := parseCompact(compact)\n\tassert.NoError(err)\n\n\tr, err := parseJson([]byte(response))\n\tassert.NotNil(r)\n\tassert.NoError(err)\n\tassert.Equal(expected, r)\n}\n\nfunc TestParseJsonTyped(t *testing.T) {\n\tresponse := `{\n\t\"meta\":\n\t[\n\t\t{\n\t\t\t\"name\": \"rule_type\",\n\t\t\t\"type\": \"String\"\n\t\t},\t\t\n\t\t{\n\t\t\t\"name\": \"regexp\",\n\t\t\t\"type\": \"String\"\n\t\t},\n\t\t{\n\t\t\t\"name\": \"function\",\n\t\t\t\"type\": \"String\"\n\t\t},\n\t\t{\n\t\t\t\"name\": \"age\",\n\t\t\t\"type\": \"UInt64\"\n\t\t},\n\t\t{\n\t\t\t\"name\": \"precision\",\n\t\t\t\"type\": \"UInt64\"\n\t\t},\n\t\t{\n\t\t\t\"name\": \"is_default\",\n\t\t\t\"type\": \"UInt8\"\n\t\t}\n\t],\n\n\t\"data\":\n\t[\n\t\t{\n\t\t\t\"rule_type\": \"all\",\n\t\t\t\"regexp\": \"^hourly\",\n\t\t\t\"function\": \"\",\n\t\t\t\"age\": \"0\",\n\t\t\t\"precision\": \"3600\",\n\t\t\t\"is_default\": 0\n\t\t},\n\t\t{\n\t\t\t\"rule_type\": \"all\",\n\t\t\t\"regexp\": \"^hourly\",\n\t\t\t\"function\": \"\",\n\t\t\t\"age\": \"3600\",\n\t\t\t\"precision\": \"13600\",\n\t\t\t\"is_default\": 0\n\t\t},\n\t\t{\n\t\t\t\"rule_type\": \"all\",\n\t\t\t\"regexp\": \"^live\",\n\t\t\t\"function\": \"\",\n\t\t\t\"age\": \"0\",\n\t\t\t\"precision\": \"1\",\n\t\t\t\"is_default\": 0\n\t\t},\n\t\t{\n\t\t\t\"rule_type\": \"plain\",\n\t\t\t\"regexp\": \"total$\",\n\t\t\t\"function\": \"sum\",\n\t\t\t\"age\": \"0\",\n\t\t\t\"precision\": \"0\",\n\t\t\t\"is_default\": 0\n\t\t},\n\t\t{\n\t\t\t\"rule_type\": \"plain\",\n\t\t\t\"regexp\": \"min$\",\n\t\t\t\"function\": \"min\",\n\t\t\t\"age\": \"0\",\n\t\t\t\"precision\": \"0\",\n\t\t\t\"is_default\": 0\n\t\t},\n\t\t{\n\t\t\t\"rule_type\": \"plain\",\n\t\t\t\"regexp\": \"max$\",\n\t\t\t\"function\": \"max\",\n\t\t\t\"age\": \"0\",\n\t\t\t\"precision\": \"0\",\n\t\t\t\"is_default\": 0\n\t\t},\n\t\t{\n\t\t\t\"rule_type\": \"tagged\",\n\t\t\t\"regexp\": \"^tag_name\\\\?\",\n\t\t\t\"function\": \"min\"\n\t\t},\n\t\t{\n\t\t\t\"rule_type\": \"tag_list\",\n\t\t\t\"regexp\": \"fake3;tag=Fake3\",\n\t\t\t\"function\": \"sum\"\n\t\t},\n\t\t{\n\t\t\t\"rule_type\": \"all\",\n\t\t\t\"regexp\": \"\",\n\t\t\t\"function\": \"max\",\n\t\t\t\"age\": \"0\",\n\t\t\t\"precision\": \"60\",\n\t\t\t\"is_default\": 1\n\t\t}\n\t],\n\n\t\"rows\": 7,\n\n\t\"statistics\":\n\t{\n\t\t\"elapsed\": 0.00053715,\n\t\t\"rows_read\": 7,\n\t\t\"bytes_read\": 1158\n\t}\n}`\n\n\texpected := &Rules{\n\t\tSeparated: true,\n\t\tPattern: []Pattern{\n\t\t\t{\n\t\t\t\tRegexp: \"^hourly\",\n\t\t\t\tRetention: []Retention{\n\t\t\t\t\t{Age: 0, Precision: 3600},\n\t\t\t\t\t{Age: 3600, Precision: 13600},\n\t\t\t\t},\n\t\t\t\tre: regexp.MustCompile(\"^hourly\"),\n\t\t\t},\n\t\t\t{\n\t\t\t\tRegexp: \"^live\",\n\t\t\t\tRetention: []Retention{\n\t\t\t\t\t{Age: 0, Precision: 1},\n\t\t\t\t},\n\t\t\t\tre: regexp.MustCompile(\"^live\"),\n\t\t\t},\n\t\t\t{\n\t\t\t\tRuleType: RulePlain,\n\t\t\t\tRegexp: \"total$\",\n\t\t\t\tFunction: \"sum\",\n\t\t\t\tre: regexp.MustCompile(\"total$\"),\n\t\t\t\taggr: AggrMap[\"sum\"],\n\t\t\t},\n\t\t\t{\n\t\t\t\tRuleType: RulePlain,\n\t\t\t\tRegexp: \"min$\",\n\t\t\t\tFunction: \"min\",\n\t\t\t\tre: regexp.MustCompile(\"min$\"),\n\t\t\t\taggr: AggrMap[\"min\"],\n\t\t\t},\n\t\t\t{\n\t\t\t\tRuleType: RulePlain,\n\t\t\t\tRegexp: \"max$\",\n\t\t\t\tFunction: \"max\",\n\t\t\t\tre: regexp.MustCompile(\"max$\"),\n\t\t\t\taggr: AggrMap[\"max\"],\n\t\t\t},\n\t\t\t{\n\t\t\t\tRuleType: RuleTagged,\n\t\t\t\tRegexp: `^tag_name\\?`,\n\t\t\t\tFunction: \"min\",\n\t\t\t\tre: regexp.MustCompile(`^tag_name\\?`),\n\t\t\t\taggr: AggrMap[\"min\"],\n\t\t\t},\n\t\t\t{\n\t\t\t\tRuleType: RuleTagged,\n\t\t\t\tRegexp: `^fake3\\?(.*&)?tag=Fake3(&.*)?$`,\n\t\t\t\tFunction: \"sum\",\n\t\t\t\tre: regexp.MustCompile(`^fake3\\?(.*&)?tag=Fake3(&.*)?$`),\n\t\t\t\taggr: AggrMap[\"sum\"],\n\t\t\t},\n\t\t\t{\n\t\t\t\tRegexp: \".*\",\n\t\t\t\tFunction: \"max\",\n\t\t\t\tRetention: []Retention{\n\t\t\t\t\t{Age: 0, Precision: 60},\n\t\t\t\t},\n\t\t\t\taggr: AggrMap[\"max\"],\n\t\t\t},\n\t\t},\n\t\tPatternPlain: []Pattern{\n\t\t\t{\n\t\t\t\tRegexp: \"^hourly\",\n\t\t\t\tRetention: []Retention{\n\t\t\t\t\t{Age: 0, Precision: 3600},\n\t\t\t\t\t{Age: 3600, Precision: 13600},\n\t\t\t\t},\n\t\t\t\tre: regexp.MustCompile(\"^hourly\"),\n\t\t\t},\n\t\t\t{\n\t\t\t\tRegexp: \"^live\",\n\t\t\t\tRetention: []Retention{\n\t\t\t\t\t{Age: 0, Precision: 1},\n\t\t\t\t},\n\t\t\t\tre: regexp.MustCompile(\"^live\"),\n\t\t\t},\n\t\t\t{\n\t\t\t\tRuleType: RulePlain,\n\t\t\t\tRegexp: \"total$\",\n\t\t\t\tFunction: \"sum\",\n\t\t\t\tre: regexp.MustCompile(\"total$\"),\n\t\t\t\taggr: AggrMap[\"sum\"],\n\t\t\t},\n\t\t\t{\n\t\t\t\tRuleType: RulePlain,\n\t\t\t\tRegexp: \"min$\",\n\t\t\t\tFunction: \"min\",\n\t\t\t\tre: regexp.MustCompile(\"min$\"),\n\t\t\t\taggr: AggrMap[\"min\"],\n\t\t\t},\n\t\t\t{\n\t\t\t\tRuleType: RulePlain,\n\t\t\t\tRegexp: \"max$\",\n\t\t\t\tFunction: \"max\",\n\t\t\t\tre: regexp.MustCompile(\"max$\"),\n\t\t\t\taggr: AggrMap[\"max\"],\n\t\t\t},\n\t\t\t{\n\t\t\t\tRegexp: \".*\",\n\t\t\t\tFunction: \"max\",\n\t\t\t\tRetention: []Retention{\n\t\t\t\t\t{Age: 0, Precision: 60},\n\t\t\t\t},\n\t\t\t\taggr: AggrMap[\"max\"],\n\t\t\t},\n\t\t},\n\t\tPatternTagged: []Pattern{\n\t\t\t{\n\t\t\t\tRegexp: \"^hourly\",\n\t\t\t\tRetention: []Retention{\n\t\t\t\t\t{Age: 0, Precision: 3600},\n\t\t\t\t\t{Age: 3600, Precision: 13600},\n\t\t\t\t},\n\t\t\t\tre: regexp.MustCompile(\"^hourly\"),\n\t\t\t},\n\t\t\t{\n\t\t\t\tRegexp: \"^live\",\n\t\t\t\tRetention: []Retention{\n\t\t\t\t\t{Age: 0, Precision: 1},\n\t\t\t\t},\n\t\t\t\tre: regexp.MustCompile(\"^live\"),\n\t\t\t},\n\t\t\t{\n\t\t\t\tRuleType: RuleTagged,\n\t\t\t\tRegexp: `^tag_name\\?`,\n\t\t\t\tFunction: \"min\",\n\t\t\t\tre: regexp.MustCompile(`^tag_name\\?`),\n\t\t\t\taggr: AggrMap[\"min\"],\n\t\t\t},\n\t\t\t{\n\t\t\t\tRuleType: RuleTagged,\n\t\t\t\tRegexp: `^fake3\\?(.*&)?tag=Fake3(&.*)?$`,\n\t\t\t\tFunction: \"sum\",\n\t\t\t\tre: regexp.MustCompile(`^fake3\\?(.*&)?tag=Fake3(&.*)?$`),\n\t\t\t\taggr: AggrMap[\"sum\"],\n\t\t\t},\n\t\t\t{\n\t\t\t\tRegexp: \".*\",\n\t\t\t\tFunction: \"max\",\n\t\t\t\tRetention: []Retention{\n\t\t\t\t\t{Age: 0, Precision: 60},\n\t\t\t\t},\n\t\t\t\taggr: AggrMap[\"max\"],\n\t\t\t},\n\t\t},\n\t}\n\n\tassert := assert.New(t)\n\n\tr, err := parseJson([]byte(response))\n\tassert.NotNil(r)\n\tassert.NoError(err)\n\tassert.Equal(expected, r)\n}\n" }, { "path": "helper/rollup/rollup.go", "content": "package rollup\n\nimport (\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"os\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/lomik/zapwriter\"\n\t\"go.uber.org/zap\"\n)\n\ntype Rollup struct {\n\tmu sync.RWMutex\n\trules *Rules\n\ttlsConfig *tls.Config\n\taddr string\n\ttable string\n\tdefaultPrecision uint32\n\tdefaultFunction string\n\tinterval time.Duration\n}\n\nfunc NewAuto(addr string, tlsConfig *tls.Config, table string, interval time.Duration, defaultPrecision uint32, defaultFunction string) (*Rollup, error) {\n\tr := &Rollup{\n\t\taddr: addr,\n\t\ttlsConfig: tlsConfig,\n\t\ttable: table,\n\t\tinterval: interval,\n\t\tdefaultPrecision: defaultPrecision,\n\t\tdefaultFunction: defaultFunction,\n\t}\n\n\tgo r.updateWorker()\n\n\treturn r, nil\n}\n\nfunc NewXMLFile(filename string, defaultPrecision uint32, defaultFunction string) (*Rollup, error) {\n\trollupConfBody, err := os.ReadFile(filename)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\trules, err := parseXML(rollupConfBody)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\trules, err = rules.prepare(defaultPrecision, defaultFunction)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn (&Rollup{\n\t\trules: rules,\n\t\tdefaultPrecision: defaultPrecision,\n\t\tdefaultFunction: defaultFunction,\n\t}), nil\n}\n\nfunc NewDefault(defaultPrecision uint32, defaultFunction string) (*Rollup, error) {\n\trules := &Rules{Pattern: []Pattern{}}\n\n\trules, err := rules.prepare(defaultPrecision, defaultFunction)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn (&Rollup{\n\t\trules: rules,\n\t\tdefaultPrecision: defaultPrecision,\n\t\tdefaultFunction: defaultFunction,\n\t}), nil\n}\n\nfunc (r *Rollup) Rules() *Rules {\n\tr.mu.RLock()\n\trules := r.rules\n\tr.mu.RUnlock()\n\n\treturn rules\n}\n\nfunc (r *Rollup) update() error {\n\trules, err := RemoteLoad(r.addr, r.tlsConfig, r.table)\n\tif err != nil {\n\t\tzapwriter.Logger(\"rollup\").Error(fmt.Sprintf(\"rollup rules update failed for table %#v\", r.table), zap.Error(err))\n\t\treturn err\n\t}\n\n\trules, err = rules.prepare(r.defaultPrecision, r.defaultFunction)\n\tif err != nil {\n\t\tzapwriter.Logger(\"rollup\").Error(fmt.Sprintf(\"rollup rules update failed for table %#v\", r.table), zap.Error(err))\n\t\treturn err\n\t}\n\n\tr.mu.Lock()\n\tr.rules = rules\n\tr.mu.Unlock()\n\n\treturn nil\n}\n\nfunc (r *Rollup) updateWorker() {\n\tfor {\n\t\tr.update()\n\n\t\t// If we still have no rules - try every second to fetch them\n\t\tif r.rules == nil {\n\t\t\ttime.Sleep(1 * time.Second)\n\t\t} else if r.interval != 0 {\n\t\t\ttime.Sleep(r.interval)\n\t\t} else {\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc (r *Rollup) MarshalJSON() ([]byte, error) {\n\treturn json.Marshal(r.Rules())\n}\n" }, { "path": "helper/rollup/rules.go", "content": "package rollup\n\nimport (\n\t\"encoding/xml\"\n\t\"fmt\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/pkg/dry\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/point\"\n)\n\ntype Retention struct {\n\tAge uint32 `json:\"age\"`\n\tPrecision uint32 `json:\"precision\"`\n}\n\ntype RuleType uint8\n\nconst (\n\tRuleAll RuleType = iota\n\tRulePlain\n\tRuleTagged\n\tRuleTagList\n)\n\nvar timeNow = time.Now\n\nvar ruleTypeStrings []string = []string{\"all\", \"plain\", \"tagged\", \"tag_list\"}\n\nfunc (r *RuleType) String() string {\n\treturn ruleTypeStrings[*r]\n}\n\nfunc (r *RuleType) Set(value string) error {\n\tswitch strings.ToLower(value) {\n\tcase \"all\":\n\t\t*r = RuleAll\n\tcase \"plain\":\n\t\t*r = RulePlain\n\tcase \"tagged\":\n\t\t*r = RuleTagged\n\tcase \"tag_list\":\n\t\t*r = RuleTagList\n\tdefault:\n\t\treturn fmt.Errorf(\"invalid rule type %s\", value)\n\t}\n\n\treturn nil\n}\n\nfunc (r *RuleType) UnmarshalJSON(data []byte) error {\n\ts := string(data)\n\tif strings.HasPrefix(s, `\"`) && strings.HasSuffix(s, `\"`) {\n\t\ts = s[1 : len(s)-1]\n\t}\n\n\treturn r.Set(s)\n}\n\nfunc (r *RuleType) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {\n\tvar s string\n\tif err := d.DecodeElement(&s, &start); err != nil {\n\t\treturn err\n\t}\n\n\treturn r.Set(s)\n}\n\nfunc splitTags(tagsStr string) (tags []string) {\n\tvals := strings.Split(tagsStr, \";\")\n\ttags = make([]string, 0, len(vals))\n\t// remove empthy elements\n\tfor _, v := range vals {\n\t\tif v != \"\" {\n\t\t\ttags = append(tags, v)\n\t\t}\n\t}\n\n\treturn\n}\n\nfunc buildTaggedRegex(regexpStr string) string {\n\t// see buildTaggedRegex in https://github.com/ClickHouse/ClickHouse/blob/780a1b2abea918d3205d149db7689a31fdff2f70/src/Processors/Merges/Algorithms/Graphite.cpp#L241\n\t//\n\t// * tags list in format (for name or any value can use regexp, alphabet sorting not needed)\n\t// * spaces are not stiped and used as tag and value part\n\t// * name must be first (if used)\n\t// *\n\t// * tag1=value1; tag2=VALUE2_REGEX;tag3=value3\n\t// * or\n\t// * name;tag1=value1;tag2=VALUE2_REGEX;tag3=value3\n\t// * or for one tag\n\t// * tag1=value1\n\t// *\n\t// * Resulting regex against metric like\n\t// * name?tag1=value1&tag2=value2\n\t// *\n\t// * So,\n\t// *\n\t// * name\n\t// * produce\n\t// * name\\?\n\t// *\n\t// * tag2=val2\n\t// * produce\n\t// * [\\?&]tag2=val2(&.*)?$\n\t// *\n\t// * nam.* ; tag1=val1 ; tag2=val2\n\t// * produce\n\t// * nam.*\\?(.*&)?tag1=val1&(.*&)?tag2=val2(&.*)?$\n\ttags := splitTags(regexpStr)\n\n\tif strings.Contains(tags[0], \"=\") {\n\t\tregexpStr = \"[\\\\?&]\"\n\t} else {\n\t\tif len(tags) == 1 {\n\t\t\t// only name\n\t\t\treturn \"^\" + tags[0] + \"\\\\?\"\n\t\t}\n\t\t// start with name value\n\t\tregexpStr = \"^\" + tags[0] + \"\\\\?(.*&)?\"\n\t\ttags = tags[1:]\n\t}\n\n\tsort.Strings(tags) // sorted tag keys\n\tregexpStr = regexpStr +\n\t\tstrings.Join(tags, \"&(.*&)?\") +\n\t\t\"(&.*)?$\" // close regex\n\n\treturn regexpStr\n}\n\ntype Pattern struct {\n\tRuleType RuleType `json:\"rule_type\"`\n\tRegexp string `json:\"regexp\"`\n\tFunction string `json:\"function\"`\n\tRetention []Retention `json:\"retention\"`\n\taggr *Aggr\n\tre *regexp.Regexp\n}\n\ntype Rules struct {\n\tPattern []Pattern `json:\"pattern\"`\n\tUpdated int64 `json:\"updated\"`\n\tSeparated bool `json:\"-\"`\n\tPatternPlain []Pattern `json:\"-\"`\n\tPatternTagged []Pattern `json:\"-\"`\n}\n\n// NewMockRulles creates mock rollup for tests\nfunc NewMockRules(pattern []Pattern, defaultPrecision uint32, defaultFunction string) (*Rules, error) {\n\trules, err := (&Rules{Pattern: pattern}).compile()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\trules, err = rules.prepare(defaultPrecision, defaultFunction)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn rules, nil\n}\n\n// should never be used in real conditions\nvar superDefaultFunction = AggrMap[\"avg\"]\n\nconst superDefaultPrecision = uint32(60)\n\nfunc (p *Pattern) compile() error {\n\tif p.RuleType == RuleTagList {\n\t\t// convert to tagged rule type\n\t\tp.RuleType = RuleTagged\n\t\tp.Regexp = buildTaggedRegex(p.Regexp)\n\t}\n\n\tvar err error\n\tif p.Regexp != \"\" && p.Regexp != \".*\" {\n\t\tp.re, err = regexp.Compile(p.Regexp)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t} else {\n\t\tp.Regexp = \".*\"\n\t\tp.re = nil\n\t}\n\n\tif p.Function != \"\" {\n\t\tvar exists bool\n\t\tp.aggr, exists = AggrMap[p.Function]\n\n\t\tif !exists {\n\t\t\treturn fmt.Errorf(\"unknown function %#v\", p.Function)\n\t\t}\n\t}\n\n\tif len(p.Retention) > 0 {\n\t\t// reverse sort by age\n\t\tsort.Slice(p.Retention, func(i, j int) bool { return p.Retention[i].Age < p.Retention[j].Age })\n\t} else {\n\t\tp.Retention = nil\n\t}\n\n\treturn nil\n}\n\nfunc (r *Rules) compile() (*Rules, error) {\n\tif r.Pattern == nil {\n\t\tr.Pattern = make([]Pattern, 0)\n\t}\n\n\tr.PatternPlain = make([]Pattern, 0)\n\tr.PatternTagged = make([]Pattern, 0)\n\n\tr.Separated = false\n\tfor i := range r.Pattern {\n\t\tif err := r.Pattern[i].compile(); err != nil {\n\t\t\treturn r, err\n\t\t}\n\n\t\tif !r.Separated && r.Pattern[i].RuleType != RuleAll {\n\t\t\tr.Separated = true\n\t\t}\n\t}\n\n\tif r.Separated {\n\t\tfor i := range r.Pattern {\n\t\t\tswitch r.Pattern[i].RuleType {\n\t\t\tcase RulePlain:\n\t\t\t\tr.PatternPlain = append(r.PatternPlain, r.Pattern[i])\n\t\t\tcase RuleTagged:\n\t\t\t\tr.PatternTagged = append(r.PatternTagged, r.Pattern[i])\n\t\t\tdefault:\n\t\t\t\tr.PatternPlain = append(r.PatternPlain, r.Pattern[i])\n\t\t\t\tr.PatternTagged = append(r.PatternTagged, r.Pattern[i])\n\t\t\t}\n\t\t}\n\t}\n\n\treturn r, nil\n}\n\nfunc (r *Rules) prepare(defaultPrecision uint32, defaultFunction string) (*Rules, error) {\n\tdefaultAggr := AggrMap[defaultFunction]\n\tif defaultFunction != \"\" && defaultAggr == nil {\n\t\treturn r, fmt.Errorf(\"unknown function %#v\", defaultFunction)\n\t}\n\n\treturn r.withDefault(defaultPrecision, defaultAggr).withSuperDefault().setUpdated(), nil\n}\n\nfunc (r *Rules) withDefault(defaultPrecision uint32, defaultFunction *Aggr) *Rules {\n\tpatterns := make([]Pattern, len(r.Pattern)+1)\n\tcopy(patterns, r.Pattern)\n\n\tvar retention []Retention\n\tif defaultPrecision != 0 {\n\t\tretention = []Retention{{Age: 0, Precision: defaultPrecision}}\n\t}\n\n\tpatterns = append(patterns, Pattern{\n\t\tRegexp: \".*\",\n\t\tFunction: defaultFunction.Name(),\n\t\tRetention: retention,\n\t})\n\tn, _ := (&Rules{Pattern: patterns, Updated: r.Updated}).compile()\n\n\treturn n\n}\n\nfunc (r *Rules) setUpdated() *Rules {\n\tr.Updated = timeNow().Unix()\n\treturn r\n}\n\nfunc (r *Rules) withSuperDefault() *Rules {\n\treturn r.withDefault(superDefaultPrecision, superDefaultFunction)\n}\n\n// Lookup returns precision and aggregate function for metric name and age\nfunc (r *Rules) Lookup(metric string, age uint32, verbose bool) (precision uint32, ag *Aggr, aggrPattern, retentionPattern *Pattern) {\n\tif r.Separated {\n\t\tif strings.Contains(metric, \"?\") {\n\t\t\treturn lookup(metric, age, r.PatternTagged, verbose)\n\t\t}\n\n\t\treturn lookup(metric, age, r.PatternPlain, verbose)\n\t}\n\n\treturn lookup(metric, age, r.Pattern, verbose)\n}\n\n// Lookup returns precision and aggregate function for metric name and age\nfunc lookup(metric string, age uint32, patterns []Pattern, verbose bool) (precision uint32, ag *Aggr, aggrPattern, retentionPattern *Pattern) {\n\tprecisionFound := false\n\n\tfor n, p := range patterns {\n\t\t// pattern hasn't interested data\n\t\tif (ag != nil || p.aggr == nil) && (precisionFound || len(p.Retention) == 0) {\n\t\t\tcontinue\n\t\t}\n\n\t\t// metric not matched regexp\n\t\tif p.re != nil && !p.re.MatchString(metric) {\n\t\t\tcontinue\n\t\t}\n\n\t\tif ag == nil && p.aggr != nil {\n\t\t\tif verbose {\n\t\t\t\taggrPattern = &patterns[n]\n\t\t\t}\n\n\t\t\tag = p.aggr\n\t\t}\n\n\t\tif !precisionFound && len(p.Retention) > 0 {\n\t\t\tfor i, r := range p.Retention {\n\t\t\t\tif age < r.Age {\n\t\t\t\t\tif i > 0 {\n\t\t\t\t\t\tprecision = p.Retention[i-1].Precision\n\t\t\t\t\t\tprecisionFound = true\n\n\t\t\t\t\t\tif verbose {\n\t\t\t\t\t\t\tretentionPattern = &patterns[n]\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\tbreak\n\t\t\t\t}\n\n\t\t\t\tif i == len(p.Retention)-1 {\n\t\t\t\t\tprecision = r.Precision\n\t\t\t\t\tprecisionFound = true\n\n\t\t\t\t\tif verbose {\n\t\t\t\t\t\tretentionPattern = &patterns[n]\n\t\t\t\t\t}\n\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// all found\n\t\tif ag != nil && precisionFound {\n\t\t\treturn\n\t\t}\n\t}\n\n\tif ag == nil {\n\t\tag = superDefaultFunction\n\t}\n\n\tif !precisionFound {\n\t\tprecision = superDefaultPrecision\n\t}\n\n\treturn\n}\n\n// LookupBytes returns precision and aggregate function for metric name and age\nfunc (r *Rules) LookupBytes(metric []byte, age uint32, verbose bool) (precision uint32, ag *Aggr, aggrPattern, retentionPattern *Pattern) {\n\treturn r.Lookup(dry.UnsafeString(metric), age, verbose)\n}\n\nfunc doMetricPrecision(points []point.Point, precision uint32, aggr *Aggr) []point.Point {\n\tl := len(points)\n\n\tvar i, n int\n\t// i - current position of iterator\n\t// n - position of the first record with time rounded to precision\n\n\tif l == 0 {\n\t\treturn points\n\t}\n\n\t// set first point time\n\tt := points[0].Time\n\tt = t - (t % precision)\n\tpoints[0].Time = t\n\n\tfor i = 1; i < l; i++ {\n\t\tt = points[i].Time\n\t\tt = t - (t % precision)\n\t\tpoints[i].Time = t\n\n\t\tif points[n].Time == t {\n\t\t\tpoints[i].MetricID = 0\n\t\t} else {\n\t\t\tif i > n+1 {\n\t\t\t\tpoints[n].Value = aggr.Do(points[n:i])\n\t\t\t}\n\n\t\t\tn = i\n\t\t}\n\t}\n\n\tif i > n+1 {\n\t\tpoints[n].Value = aggr.Do(points[n:i])\n\t}\n\n\treturn point.CleanUp(points)\n}\n\n// RollupMetricAge rolling up list of points of ONE metric sorted by key \"time\"\n// returns (new points slice, precision)\nfunc (r *Rules) RollupMetricAge(metricName string, age uint32, points []point.Point) ([]point.Point, uint32, error) {\n\tl := len(points)\n\tif l == 0 {\n\t\treturn points, 1, nil\n\t}\n\n\tprecision, ag, _, _ := r.Lookup(metricName, age, false)\n\tpoints = doMetricPrecision(points, precision, ag)\n\n\treturn points, precision, nil\n}\n\n// RollupMetric rolling up list of points of ONE metric sorted by key \"time\"\n// returns (new points slice, precision)\nfunc (r *Rules) RollupMetric(metricName string, from uint32, points []point.Point) ([]point.Point, uint32, error) {\n\tnow := uint32(timeNow().Unix())\n\tage := uint32(0)\n\n\tif now > from {\n\t\tage = now - from\n\t}\n\n\treturn r.RollupMetricAge(metricName, age, points)\n}\n\n// RollupPoints groups sorted Points by metric name and apply rollup one by one.\n// If the `step` parameter is 0, it will be got from the current *Rules, otherwise it will be used directly.\nfunc (r *Rules) RollupPoints(pp *point.Points, from int64, step int64) error {\n\tif from < 0 || step < 0 {\n\t\treturn fmt.Errorf(\"from and step must be >= 0: %v, %v\", from, step)\n\t}\n\n\tnow := timeNow().Unix()\n\tage := int64(0)\n\n\tif now > from {\n\t\tage = now - from\n\t}\n\n\tvar i, n int\n\t// i - current position of iterator\n\t// n - position of the first record with current metric\n\tl := pp.Len()\n\tif l == 0 {\n\t\treturn nil\n\t}\n\n\toldPoints := pp.List()\n\tnewPoints := make([]point.Point, 0, pp.Len())\n\trollup := func(p []point.Point) ([]point.Point, error) {\n\t\tmetricName := pp.MetricName(p[0].MetricID)\n\n\t\tvar err error\n\n\t\tif step == 0 {\n\t\t\tp, _, err = r.RollupMetricAge(metricName, uint32(age), p)\n\t\t} else {\n\t\t\t_, agg, _, _ := r.Lookup(metricName, uint32(from), false)\n\t\t\tp = doMetricPrecision(p, uint32(step), agg)\n\t\t}\n\n\t\tfor i := range p {\n\t\t\tp[i].MetricID = p[0].MetricID\n\t\t}\n\n\t\treturn p, err\n\t}\n\n\tfor i = 1; i < l; i++ {\n\t\tif oldPoints[i].MetricID != oldPoints[n].MetricID {\n\t\t\tpoints, err := rollup(oldPoints[n:i])\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tnewPoints = append(newPoints, points...)\n\t\t\tn = i\n\n\t\t\tcontinue\n\t\t}\n\t}\n\n\tpoints, err := rollup(oldPoints[n:i])\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tnewPoints = append(newPoints, points...)\n\tpp.ReplaceList(newPoints)\n\n\treturn nil\n}\n" }, { "path": "helper/rollup/rules_test.go", "content": "package rollup\n\nimport (\n\t\"fmt\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/point\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestMetricPrecision(t *testing.T) {\n\ttests := [][2][]point.Point{\n\t\t{\n\t\t\t{ // in\n\t\t\t\t{MetricID: 1, Time: 1478025152, Value: 3},\n\t\t\t\t{MetricID: 1, Time: 1478025154, Value: 2},\n\t\t\t\t{MetricID: 1, Time: 1478025255, Value: 1},\n\t\t\t},\n\t\t\t{ // out\n\t\t\t\t{MetricID: 1, Time: 1478025120, Value: 5},\n\t\t\t\t{MetricID: 1, Time: 1478025240, Value: 1},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tresult := doMetricPrecision(test[0], 60, AggrMap[\"sum\"])\n\t\tassert.Equal(t, test[1], result)\n\t}\n}\n\nfunc Test_buildTaggedRegex(t *testing.T) {\n\ttests := []struct {\n\t\ttagsStr string\n\t\twant string\n\t\tmatch string\n\t\tnomatch string\n\t}{\n\t\t{\n\t\t\ttagsStr: `cpu\\.loadavg;project=DB.*;env=st.*`, want: `^cpu\\.loadavg\\?(.*&)?env=st.*&(.*&)?project=DB.*(&.*)?$`,\n\t\t\tmatch: `cpu.loadavg?env=staging&project=DBAAS`,\n\t\t\tnomatch: `cpu.loadavg?env=staging&project=D`,\n\t\t},\n\t\t{\n\t\t\ttagsStr: `project=DB.*;env=staging;`, want: `[\\?&]env=staging&(.*&)?project=DB.*(&.*)?$`,\n\t\t\tmatch: `cpu.loadavg?env=staging&project=DBPG`,\n\t\t\tnomatch: `cpu.loadavg?env=stagingN&project=DBAAS`,\n\t\t},\n\t\t{\n\t\t\ttagsStr: \"env=staging;\", want: `[\\?&]env=staging(&.*)?$`,\n\t\t\tmatch: `cpu.loadavg?env=staging&project=DPG`,\n\t\t\tnomatch: `cpu.loadavg?env=stagingN`,\n\t\t},\n\t\t{\n\t\t\ttagsStr: \" env = staging ;\", // spaces are allowed,\n\t\t\twant: `[\\?&] env = staging (&.*)?$`,\n\t\t\tmatch: `cpu.loadavg? env = staging &project=DPG`,\n\t\t\tnomatch: `cpu.loadavg?env=stagingN`,\n\t\t},\n\t\t{\n\t\t\ttagsStr: \"name;\",\n\t\t\twant: `^name\\?`,\n\t\t\tmatch: `name?env=staging&project=DPG`,\n\t\t\tnomatch: `nameN?env=stagingN`,\n\t\t},\n\t\t{\n\t\t\ttagsStr: \"name\",\n\t\t\twant: `^name\\?`,\n\t\t\tmatch: `name?env=staging&project=DPG`,\n\t\t\tnomatch: `nameN?env=stagingN`,\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.tagsStr, func(t *testing.T) {\n\t\t\tif got := buildTaggedRegex(tt.tagsStr); got != tt.want {\n\t\t\t\tt.Errorf(\"buildTaggedRegex(%q) = %v, want %v\", tt.tagsStr, got, tt.want)\n\t\t\t} else {\n\t\t\t\tre := regexp.MustCompile(got)\n\t\t\t\tif tt.match != \"\" && !re.Match([]byte(tt.match)) {\n\t\t\t\t\tt.Errorf(\"match(%q, %q) must be true\", tt.tagsStr, tt.match)\n\t\t\t\t}\n\n\t\t\t\tif tt.nomatch != \"\" && re.Match([]byte(tt.nomatch)) {\n\t\t\t\t\tt.Errorf(\"match(%q, %q) must be false\", tt.tagsStr, tt.match)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestLookup(t *testing.T) {\n\tconfig := `\n\t^hourly;;3600:60,86400:3600\n\t^live;;0:1\n\ttotal$;sum;\n\tmin$;min;\n\tmax$;max;\n\t;avg;\n\t;;60:10\n\t;;0:42`\n\n\ttable := [][4]string{\n\t\t{\"hello.world\", \"0\", \"avg\", \"42\"},\n\t\t{\"hourly.rps\", \"0\", \"avg\", \"42\"},\n\t\t{\"hourly.rps_total\", \"0\", \"sum\", \"42\"},\n\t\t{\"live.rps_total\", \"0\", \"sum\", \"1\"},\n\t\t{\"hourly.rps_min\", \"0\", \"min\", \"42\"},\n\t\t{\"hourly.rps_min\", \"1\", \"min\", \"42\"},\n\t\t{\"hourly.rps_min\", \"59\", \"min\", \"42\"},\n\t\t{\"hourly.rps_min\", \"60\", \"min\", \"10\"},\n\t\t{\"hourly.rps_min\", \"61\", \"min\", \"10\"},\n\t\t{\"hourly.rps_min\", \"3599\", \"min\", \"10\"},\n\t\t{\"hourly.rps_min\", \"3600\", \"min\", \"60\"},\n\t\t{\"hourly.rps_min\", \"3601\", \"min\", \"60\"},\n\t\t{\"hourly.rps_min\", \"86399\", \"min\", \"60\"},\n\t\t{\"hourly.rps_min\", \"86400\", \"min\", \"3600\"},\n\t\t{\"hourly.rps_min\", \"86401\", \"min\", \"3600\"},\n\t}\n\n\tr, err := parseCompact(config)\n\trequire.NoError(t, err)\n\n\tfor _, c := range table {\n\t\tt.Run(fmt.Sprintf(\"%#v\", c[:]), func(t *testing.T) {\n\t\t\tassert := assert.New(t)\n\t\t\tage, err := strconv.Atoi(c[1])\n\t\t\tassert.NoError(err)\n\n\t\t\tprecision, ag, _, _ := r.Lookup(c[0], uint32(age), false)\n\t\t\tassert.Equal(c[2], ag.String())\n\t\t\tassert.Equal(c[3], fmt.Sprintf(\"%d\", precision))\n\t\t})\n\t}\n}\n\nfunc TestLookupTyped(t *testing.T) {\n\tconfig := `\n\t\n\t \t\n\t \t\t^hourly\n\t \t\t\n\t \t\t\t3600\n\t \t\t\t60\n\t \t\t\n\t \t\t\n\t \t\t\t86400\n\t \t\t\t3600\n\t \t\t\n\t\t\n\t\t\n\t \t\t^live\n\t \t\t\n\t \t\t\t0\n\t \t\t\t1\n\t \t\t\n\t\t\n\t\t\n\t\t\ttag_list\n\t\t\tfake3;tag3=Fake3\n\t\t\t\n\t \t\t\t0\n\t \t\t\t1\n\t \t\t\n \t\t\n\t\t \n\t\t tag_list\n\t\t tag5=Fake5;tag3=Fake3\n\t\t \n\t\t\t 0\n\t\t\t 90\n\t\t \n\t \n\t\t\n\t\t\ttag_list\n\t\t\tfake_name\n\t\t\t\n\t\t\t\t0\n\t\t\t\t20\n\t\t \t\n\t \t\n\t\t\n\t\t\tplain\n\t\t\ttotal$\n\t\t\tsum\n \t\t\n\t\t\n\t\t plain\n\t\t min$\n\t\t min\n\t\t\n\t\t\n\t\t plain\n\t\t max$\n\t\t max\n\t\t\n\t\t\n\t\t\ttagged\n\t\t\ttotal?\n\t\t\tsum\n \t\t\n\t\t\n\t\t tagged\n\t\t min\\?\n\t\t min\n\t\t\n\t\t\n\t\t tagged\n\t\t max\\?\n\t\t max\n\t\t\n\t\t\n\t\t\ttagged\n\t\t\t^hourly\n\t\t\tsum\n\t\t\n\t \t\n\t \t\tavg\n\t \t\t\n\t \t\t\t0\n\t \t\t\t42\n\t \t\t\n\t \t\t\n\t \t\t\t60\n\t \t\t\t10\n\t \t\t\n\t \t\n\t\n\t`\n\n\ttable := [][4]string{\n\t\t{\"hello.world\", \"0\", \"avg\", \"42\"},\n\t\t{\"hourly.rps\", \"0\", \"avg\", \"42\"},\n\t\t{\"hourly.rps?tag=value\", \"0\", \"sum\", \"42\"},\n\t\t{\"hourly.rps\", \"0\", \"avg\", \"42\"},\n\t\t{\"hourly.rps_total\", \"0\", \"sum\", \"42\"},\n\t\t{\"live.rps_total\", \"0\", \"sum\", \"1\"},\n\t\t{\"hourly.rps_min\", \"0\", \"min\", \"42\"},\n\t\t{\"hourly.rps_min?tag=value\", \"0\", \"min\", \"42\"},\n\t\t{\"hourly.rps_min\", \"1\", \"min\", \"42\"},\n\t\t{\"hourly.rps_min\", \"59\", \"min\", \"42\"},\n\t\t{\"hourly.rps_min?tag=value\", \"59\", \"min\", \"42\"},\n\t\t{\"hourly.rps_min\", \"60\", \"min\", \"10\"},\n\t\t{\"hourly.rps_min\", \"61\", \"min\", \"10\"},\n\t\t{\"hourly.rps_min\", \"3599\", \"min\", \"10\"},\n\t\t{\"hourly.rps_min\", \"3600\", \"min\", \"60\"},\n\t\t{\"hourly.rps_min\", \"3601\", \"min\", \"60\"},\n\t\t{\"hourly.rps_min\", \"86399\", \"min\", \"60\"},\n\t\t{\"hourly.rps_min\", \"86400\", \"min\", \"3600\"},\n\t\t{\"hourly.rps_min\", \"86401\", \"min\", \"3600\"},\n\t\t{\"fake3?tag3=Fake3\", \"0\", \"avg\", \"1\"},\n\t\t{\"fake3?tag1=Fake1&tag3=Fake3\", \"0\", \"avg\", \"1\"},\n\t\t{\"fake3?tag1=Fake1&tag3=Fake3&tag4=Fake4\", \"0\", \"avg\", \"1\"},\n\t\t{\"fake3?tag3=Fake\", \"0\", \"avg\", \"42\"},\n\t\t{\"fake3?tag1=Fake1&tag3=Fake\", \"0\", \"avg\", \"42\"},\n\t\t{\"fake3?tag1=Fake1&tag3=Fake&tag4=Fake4\", \"0\", \"avg\", \"42\"},\n\t\t{\"fake?tag3=Fake3\", \"0\", \"avg\", \"42\"},\n\t\t{\"fake_name?tag3=Fake3\", \"0\", \"avg\", \"20\"},\n\t\t{\"fake5?tag1=Fake1&tag3=Fake3&tag4=Fake4&tag5=Fake5\", \"0\", \"avg\", \"90\"},\n\t\t{\"fake5?tag3=Fake3&tag4=Fake4&tag5=Fake5&tag6=Fake6\", \"0\", \"avg\", \"90\"},\n\t\t{\"fake5?tag4=Fake4&tag5=Fake5&tag6=Fake6\", \"0\", \"avg\", \"42\"},\n\t}\n\n\tr, err := parseXML([]byte(config))\n\trequire.NoError(t, err)\n\n\tfor _, c := range table {\n\t\tt.Run(fmt.Sprintf(\"%#v\", c[:]), func(t *testing.T) {\n\t\t\tassert := assert.New(t)\n\t\t\tage, err := strconv.Atoi(c[1])\n\t\t\tassert.NoError(err)\n\n\t\t\tprecision, ag, _, _ := r.Lookup(c[0], uint32(age), false)\n\t\t\tassert.Equal(c[2], ag.String())\n\t\t\tassert.Equal(c[3], fmt.Sprintf(\"%d\", precision))\n\t\t})\n\t}\n}\n\nfunc TestRules_RollupPoints(t *testing.T) {\n\tconfig := `\n\t^10sec;;0:10,3600:60\n\t;max;0:20`\n\n\tr, err := parseCompact(config)\n\trequire.NoError(t, err)\n\n\ttimeNow = func() time.Time {\n\t\treturn time.Unix(10010, 0)\n\t}\n\n\tnewPoints := func() *point.Points {\n\t\tpp := point.NewPoints()\n\n\t\tid10Sec := pp.MetricID(\"10sec\")\n\t\tpp.AppendPoint(id10Sec, 1.0, 10, 0)\n\t\tpp.AppendPoint(id10Sec, 2.0, 20, 0)\n\t\tpp.AppendPoint(id10Sec, 3.0, 30, 0)\n\t\tpp.AppendPoint(id10Sec, 6.0, 60, 0)\n\t\tpp.AppendPoint(id10Sec, 7.0, 70, 0)\n\n\t\tidDefault := pp.MetricID(\"default\")\n\t\tpp.AppendPoint(idDefault, 2.0, 20, 0)\n\t\tpp.AppendPoint(idDefault, 4.0, 40, 0)\n\t\tpp.AppendPoint(idDefault, 6.0, 60, 0)\n\t\tpp.AppendPoint(idDefault, 8.0, 80, 0)\n\n\t\treturn pp\n\t}\n\n\tpointsTo60SecNoDefault := func() *point.Points {\n\t\tpp := point.NewPoints()\n\n\t\tid10Sec := pp.MetricID(\"10sec\")\n\t\tpp.AppendPoint(id10Sec, 3.0, 0, 0)\n\t\tpp.AppendPoint(id10Sec, 7.0, 60, 0)\n\n\t\tidDefault := pp.MetricID(\"default\")\n\t\tpp.AppendPoint(idDefault, 2.0, 20, 0)\n\t\tpp.AppendPoint(idDefault, 4.0, 40, 0)\n\t\tpp.AppendPoint(idDefault, 6.0, 60, 0)\n\t\tpp.AppendPoint(idDefault, 8.0, 80, 0)\n\n\t\treturn pp\n\t}\n\n\tpointsTo60Sec := func() *point.Points {\n\t\tpp := point.NewPoints()\n\n\t\tid10Sec := pp.MetricID(\"10sec\")\n\t\tpp.AppendPoint(id10Sec, 3.0, 0, 0)\n\t\tpp.AppendPoint(id10Sec, 7.0, 60, 0)\n\n\t\tidDefault := pp.MetricID(\"default\")\n\t\tpp.AppendPoint(idDefault, 4.0, 0, 0)\n\t\tpp.AppendPoint(idDefault, 8.0, 60, 0)\n\n\t\treturn pp\n\t}\n\n\ttests := []struct {\n\t\tname string\n\t\tpp *point.Points\n\t\tfrom int64\n\t\tstep int64\n\t\twant *point.Points\n\t\twantErr bool\n\t}{\n\t\t{\n\t\t\tname: \"without step and no rollup\",\n\t\t\tpp: newPoints(),\n\t\t\tfrom: int64(10000), step: int64(0),\n\t\t\twant: newPoints(),\n\t\t},\n\t\t{\n\t\t\tname: \"without step\",\n\t\t\tpp: newPoints(),\n\t\t\tfrom: int64(10), step: int64(0),\n\t\t\twant: pointsTo60SecNoDefault(),\n\t\t},\n\t\t{\n\t\t\tname: \"with step 10\",\n\t\t\tpp: newPoints(),\n\t\t\tfrom: int64(10000), step: int64(10),\n\t\t\twant: newPoints(),\n\t\t},\n\t\t{\n\t\t\tname: \"with step 60\",\n\t\t\tpp: newPoints(),\n\t\t\tfrom: int64(10), step: int64(60),\n\t\t\twant: pointsTo60Sec(),\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tif err := r.RollupPoints(tt.pp, tt.from, tt.step); (err != nil) != tt.wantErr {\n\t\t\t\tt.Errorf(\"Rules.RollupPoints() error = %v, wantErr %v\", err, tt.wantErr)\n\t\t\t} else if err == nil {\n\t\t\t\tassert.Equal(t, tt.want, tt.pp)\n\t\t\t}\n\t\t})\n\t}\n}\n\nvar benchConfig = `\n\t\n\t \t\n\t \t\t^hourly\n\t \t\t\n\t \t\t\t3600\n\t \t\t\t60\n\t \t\t\n\t \t\t\n\t \t\t\t86400\n\t \t\t\t3600\n\t \t\t\n\t\t\n\t\t\n\t \t\t^live\n\t \t\t\n\t \t\t\t0\n\t \t\t\t1\n\t \t\t\n\t\t\n\t\t\n\t\t\t\\.fake1\\..*\\.Fake1\\.\n\t\t\t\n\t \t\t\t3600\n\t \t\t\t60\n\t \t\t\n\t \t\t\n\t \t\t\t86400\n\t \t\t\t3600\n\t \t\t\n \t\t\n\t\t\n\t\t\t\n\t\t\t\n\t\t\t\t3600\n\t\t\t\t60\n\t\t\t\n\t\t\t\n\t\t\t\t86400\n\t\t\t\t3600\n\t\t\t\n\t \t\n\t\t\n\t\t\t\\.fake2\\..*\\.Fake2\\.\n\t\t\t\n\t\t\t\t3600\n\t\t\t\t60\n\t\t\t\n\t\t\t\n\t\t\t\t86400\n\t\t\t\t3600\n\t\t\t\n\t \t\n\t \t\n\t\t \n\t\t \n\t\t\t 3600\n\t\t\t 60\n\t\t \n\t\t \n\t\t\t 86400\n\t\t\t 3600\n\t\t \n\t\t\n\t\t\n\t\t\t\\.fake3\\..*\\.Fake3\\.\n\t\t\t\n\t\t\t\t3600\n\t\t\t\t60\n\t\t\t\n\t\t\t\n\t\t\t\t86400\n\t\t\t\t3600\n\t\t\t\n\t \t\n\t \t\n\t\t \n\t\t \n\t\t\t 3600\n\t\t\t 60\n\t\t \n\t\t \n\t\t\t 86400\n\t\t\t 3600\n\t\t \n\t\t\n\t\t\n\t\t\t\\.fake4\\..*\\.Fake4\\.\n\t\t\t\n\t\t\t\t3600\n\t\t\t\t60\n\t\t\t\n\t\t\t\n\t\t\t\t86400\n\t\t\t\t3600\n\t\t\t\n\t \t\n\t \t\n\t\t \n\t\t \n\t\t\t 3600\n\t\t\t 60\n\t\t \n\t\t \n\t\t\t 86400\n\t\t\t 3600\n\t\t \n\t\t\n\t\t\n\t\t\ttotal$\n\t\t\tsum\n \t\t\n\t\t\n\t\t min$\n\t\t min\n\t\t\n\t\t\n\t\t max$\n\t\t max\n\t\t\n\t\t\n\t\t\ttotal?\n\t\t\tsum\n \t\t\n\t\t\n\t\t min\\?\n\t\t min\n\t\t\n\t\t\n\t\t max\\?\n\t\t max\n\t\t\n\t\t\n\t\t\t^hourly\n\t\t\tsum\n\t\t\n\t \t\n\t \t\tavg\n\t \t\t\n\t \t\t\t0\n\t \t\t\t42\n\t \t\t\n\t \t\t\n\t \t\t\t60\n\t \t\t\t10\n\t \t\t\n\t \t\n\t\n\t`\n\nvar benchConfigSeparated = `\n\t\n\t \t\n\t\t\tplain\n\t \t\t^hourly\n\t \t\t\n\t \t\t\t3600\n\t \t\t\t60\n\t \t\t\n\t \t\t\n\t \t\t\t86400\n\t \t\t\t3600\n\t \t\t\n\t\t\n\t\t\n\t\t\tplain\n\t \t\t^live\n\t \t\t\n\t \t\t\t0\n\t \t\t\t1\n\t \t\t\n\t\t\n\t\t\n\t\t\tplain\n\t\t\t\\.fake1\\..*\\.Fake1\\.\n\t\t\t\n\t \t\t\t3600\n\t \t\t\t60\n\t \t\t\n\t \t\t\n\t \t\t\t86400\n\t \t\t\t3600\n\t \t\t\n \t\t\n\t\t\n\t\t\ttagged\n\t\t\t\n\t\t\t\n\t\t\t\t3600\n\t\t\t\t60\n\t\t\t\n\t\t\t\n\t\t\t\t86400\n\t\t\t\t3600\n\t\t\t\n\t \t\n\t\t\n\t\t\tplain\n\t\t\t\\.fake2\\..*\\.Fake2\\.\n\t\t\t\n\t\t\t\t3600\n\t\t\t\t60\n\t\t\t\n\t\t\t\n\t\t\t\t86400\n\t\t\t\t3600\n\t\t\t\n\t \t\n\t \t\n\t\t tagged\n\t\t \n\t\t \n\t\t\t 3600\n\t\t\t 60\n\t\t \n\t\t \n\t\t\t 86400\n\t\t\t 3600\n\t\t \n\t\t\n\t\t\n\t\t\tplain\n\t\t\t\\.fake3\\..*\\.Fake3\\.\n\t\t\t\n\t\t\t\t3600\n\t\t\t\t60\n\t\t\t\n\t\t\t\n\t\t\t\t86400\n\t\t\t\t3600\n\t\t\t\n\t \t\n\t \t\n\t\t tagged\n\t\t \n\t\t \n\t\t\t 3600\n\t\t\t 60\n\t\t \n\t\t \n\t\t\t 86400\n\t\t\t 3600\n\t\t \n\t\t\n\t\t\n\t\t\tplain\n\t\t\t\\.fake4\\..*\\.Fake4\\.\n\t\t\t\n\t\t\t\t3600\n\t\t\t\t60\n\t\t\t\n\t\t\t\n\t\t\t\t86400\n\t\t\t\t3600\n\t\t\t\n\t \t\n\t \t\n\t\t tagged\n\t\t \n\t\t \n\t\t\t 3600\n\t\t\t 60\n\t\t \n\t\t \n\t\t\t 86400\n\t\t\t 3600\n\t\t \n\t\t\n\t\t\n\t\t\tplain\n\t\t\ttotal$\n\t\t\tsum\n \t\t\n\t\t\n\t\t plain\n\t\t min$\n\t\t min\n\t\t\n\t\t\n\t\t plain\n\t\t max$\n\t\t max\n\t\t\n\t\t\n\t\t\ttagged\n\t\t\ttotal?\n\t\t\tsum\n \t\t\n\t\t\n\t\t tagged\n\t\t min\\?\n\t\t min\n\t\t\n\t\t\n\t\t tagged\n\t\t max\\?\n\t\t max\n\t\t\n\t\t\n\t\t\ttagged\n\t\t\t^hourly\n\t\t\tsum\n\t\t\n\t \t\n\t \t\tavg\n\t \t\t\n\t \t\t\t0\n\t \t\t\t42\n\t \t\t\n\t \t\t\n\t \t\t\t60\n\t \t\t\t10\n\t \t\t\n\t \t\n\t\n\t`\n\nfunc BenchmarkLookupSum(b *testing.B) {\n\tr, err := parseXML([]byte(benchConfig))\n\trequire.NoError(b, err)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tprecision, ag, _, _ := r.Lookup(\"test.sum\", 1, false)\n\t\t_ = precision\n\t\t_ = ag\n\t}\n}\n\nfunc BenchmarkLookupSumSeparated(b *testing.B) {\n\tr, err := parseXML([]byte(benchConfigSeparated))\n\trequire.NoError(b, err)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tprecision, ag, _, _ := r.Lookup(\"test.sum\", 1, false)\n\t\t_ = precision\n\t\t_ = ag\n\t}\n}\n\nfunc BenchmarkLookupSumTagged(b *testing.B) {\n\tr, err := parseXML([]byte(benchConfig))\n\trequire.NoError(b, err)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tprecision, ag, _, _ := r.Lookup(\"sum?env=test&tag=Fake5\", 1, false)\n\t\t_ = precision\n\t\t_ = ag\n\t}\n}\n\nfunc BenchmarkLookupSumTaggedSeparated(b *testing.B) {\n\tr, err := parseXML([]byte(benchConfigSeparated))\n\trequire.NoError(b, err)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tprecision, ag, _, _ := r.Lookup(\"sum?env=test&tag=Fake5\", 1, false)\n\t\t_ = precision\n\t\t_ = ag\n\t}\n}\n\nfunc BenchmarkLookupMax(b *testing.B) {\n\tr, err := parseXML([]byte(benchConfig))\n\trequire.NoError(b, err)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tprecision, ag, _, _ := r.Lookup(\"test.max\", 1, false)\n\t\t_ = precision\n\t\t_ = ag\n\t}\n}\n\nfunc BenchmarkLookupMaxSeparated(b *testing.B) {\n\tr, err := parseXML([]byte(benchConfigSeparated))\n\trequire.NoError(b, err)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tprecision, ag, _, _ := r.Lookup(\"test.max\", 1, false)\n\t\t_ = precision\n\t\t_ = ag\n\t}\n}\n\nfunc BenchmarkLookupMaxTagged(b *testing.B) {\n\tr, err := parseXML([]byte(benchConfig))\n\trequire.NoError(b, err)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tprecision, ag, _, _ := r.Lookup(\"max?env=test&tag=Fake5\", 1, false)\n\t\t_ = precision\n\t\t_ = ag\n\t}\n}\n\nfunc BenchmarkLookupMaxTaggedSeparated(b *testing.B) {\n\tr, err := parseXML([]byte(benchConfigSeparated))\n\trequire.NoError(b, err)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tprecision, ag, _, _ := r.Lookup(\"max?env=test&tag=Fake5\", 1, false)\n\t\t_ = precision\n\t\t_ = ag\n\t}\n}\n\nfunc BenchmarkLookupDefault(b *testing.B) {\n\tr, err := parseXML([]byte(benchConfig))\n\trequire.NoError(b, err)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tprecision, ag, _, _ := r.Lookup(\"test.p95\", 1, false)\n\t\t_ = precision\n\t\t_ = ag\n\t}\n}\n\nfunc BenchmarkLookupDefaultSeparated(b *testing.B) {\n\tr, err := parseXML([]byte(benchConfigSeparated))\n\trequire.NoError(b, err)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tprecision, ag, _, _ := r.Lookup(\"test.p95\", 1, false)\n\t\t_ = precision\n\t\t_ = ag\n\t}\n}\n\nfunc BenchmarkLookupDefaultTagged(b *testing.B) {\n\tr, err := parseXML([]byte(benchConfig))\n\trequire.NoError(b, err)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tprecision, ag, _, _ := r.Lookup(\"p95?env=test&tag=Fake5\", 1, false)\n\t\t_ = precision\n\t\t_ = ag\n\t}\n}\n\nfunc BenchmarkLookupDefaultTaggedSeparated(b *testing.B) {\n\tr, err := parseXML([]byte(benchConfigSeparated))\n\trequire.NoError(b, err)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tprecision, ag, _, _ := r.Lookup(\"p95?env=test&tag=Fake5\", 1, false)\n\t\t_ = precision\n\t\t_ = ag\n\t}\n}\n" }, { "path": "helper/rollup/xml.go", "content": "package rollup\n\nimport (\n\t\"encoding/xml\"\n)\n\n/*\n\n \t\n \t\tclick_cost\n \t\tany\n \t\t\n \t\t\t0\n \t\t\t3600\n \t\t\n \t\t\n \t\t\t86400\n \t\t\t60\n \t\t\n \t\n \t\n \t\tmax\n \t\t\n \t\t\t0\n \t\t\t60\n \t\t\n \t\t\n \t\t\t3600\n \t\t\t300\n \t\t\n \t\t\n \t\t\t86400\n \t\t\t3600\n \t\t\n \t\n\n*/\n\ntype ClickhouseRollupXML struct {\n\tRules RulesXML `xml:\"graphite_rollup\"`\n}\n\ntype RetentionXML struct {\n\tAge uint32 `xml:\"age\"`\n\tPrecision uint32 `xml:\"precision\"`\n}\n\ntype PatternXML struct {\n\tRuleType RuleType `xml:\"rule_type\"`\n\tRegexp string `xml:\"regexp\"`\n\tFunction string `xml:\"function\"`\n\tRetention []*RetentionXML `xml:\"retention\"`\n}\n\ntype RulesXML struct {\n\tPattern []*PatternXML `xml:\"pattern\"`\n\tDefault *PatternXML `xml:\"default\"`\n}\n\nfunc (r *RetentionXML) retention() Retention {\n\treturn Retention{Age: r.Age, Precision: r.Precision}\n}\n\nfunc (p *PatternXML) pattern() Pattern {\n\tresult := Pattern{\n\t\tRuleType: p.RuleType,\n\t\tRegexp: p.Regexp,\n\t\tFunction: p.Function,\n\t\tRetention: make([]Retention, 0, len(p.Retention)),\n\t}\n\n\tfor _, r := range p.Retention {\n\t\tresult.Retention = append(result.Retention, r.retention())\n\t}\n\n\treturn result\n}\n\nfunc parseXML(body []byte) (*Rules, error) {\n\tr := &RulesXML{}\n\n\terr := xml.Unmarshal(body, r)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Maybe we've got Clickhouse's graphite.xml?\n\tif r.Default == nil && r.Pattern == nil {\n\t\ty := &ClickhouseRollupXML{}\n\n\t\terr = xml.Unmarshal(body, y)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tr = &y.Rules\n\t}\n\n\tpatterns := make([]Pattern, 0, uint64(len(r.Pattern))+4)\n\tfor _, p := range r.Pattern {\n\t\tpatterns = append(patterns, p.pattern())\n\t}\n\n\tif r.Default != nil {\n\t\tpatterns = append(patterns, r.Default.pattern())\n\t}\n\n\treturn (&Rules{Pattern: patterns}).compile()\n}\n" }, { "path": "helper/rollup/xml_test.go", "content": "package rollup\n\nimport (\n\t\"fmt\"\n\t\"regexp\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestParseXML(t *testing.T) {\n\tconfig := `\n\n \t\n \t\tclick_cost\n \t\tany\n \t\t\n \t\t\t0\n \t\t\t3600\n \t\t\n \t\t\n \t\t\t86400\n \t\t\t60\n \t\t\n\t\n \t\n \t\twithout_function\n \t\t\n \t\t\t0\n \t\t\t3600\n \t\t\n \t\t\n \t\t\t86400\n \t\t\t60\n \t\t\n\t\n \t\n \t\twithout_retention\n \t\tmin\n \t\n \t\n \t\tmax\n \t\t\n \t\t\t0\n \t\t\t60\n \t\t\n \t\t\n \t\t\t3600\n \t\t\t300\n \t\t\n \t\t\n \t\t\t86400\n \t\t\t3600\n \t\t\n \t\n\n`\n\n\tcompact := `\n\tclick_cost;any;0:3600,86400:60\n\twithout_function;;0:3600,86400:60\n\twithout_retention;min;\n\t;max;0:60,3600:300,86400:3600\n\t`\n\n\texpected, _ := (&Rules{\n\t\tPattern: []Pattern{\n\t\t\t{Regexp: \"click_cost\", Function: \"any\", Retention: []Retention{\n\t\t\t\t{Age: 86400, Precision: 60},\n\t\t\t\t{Age: 0, Precision: 3600},\n\t\t\t}},\n\t\t\t{Regexp: \"without_function\", Function: \"\", Retention: []Retention{\n\t\t\t\t{Age: 0, Precision: 3600},\n\t\t\t\t{Age: 86400, Precision: 60},\n\t\t\t}},\n\t\t\t{Regexp: \"without_retention\", Function: \"min\", Retention: nil},\n\t\t\t{Regexp: \"\", Function: \"max\", Retention: []Retention{\n\t\t\t\t{Age: 0, Precision: 60},\n\t\t\t\t{Age: 3600, Precision: 300},\n\t\t\t\t{Age: 86400, Precision: 3600},\n\t\t\t}},\n\t\t},\n\t}).compile()\n\n\tt.Run(\"default\", func(t *testing.T) {\n\t\tassert := assert.New(t)\n\t\tr, err := parseXML([]byte(config))\n\t\tassert.NoError(err)\n\t\tassert.Equal(expected, r)\n\n\t\t// check sorting\n\t\tassert.Equal(uint32(0), r.Pattern[0].Retention[0].Age)\n\t\tassert.Equal(uint32(3600), r.Pattern[0].Retention[0].Precision)\n\t})\n\n\tt.Run(\"inside yandex tag\", func(t *testing.T) {\n\t\tassert := assert.New(t)\n\t\tr, err := parseXML([]byte(fmt.Sprintf(\"%s\", config)))\n\t\tassert.NoError(err)\n\t\tassert.Equal(expected, r)\n\t})\n\n\tt.Run(\"compare with compact\", func(t *testing.T) {\n\t\tassert := assert.New(t)\n\t\texpectedCompact, err := parseCompact(compact)\n\t\tassert.NoError(err)\n\n\t\tr, err := parseXML([]byte(fmt.Sprintf(\"%s\", config)))\n\t\tassert.NoError(err)\n\t\tassert.Equal(expectedCompact, r)\n\t})\n}\n\nfunc TestParseXMLTyped(t *testing.T) {\n\tconfig := `\n\n \t\n\t\tall>\n \t\tclick_cost\n \t\tany\n \t\t\n \t\t\t0\n \t\t\t3600\n \t\t\n \t\t\n \t\t\t86400\n \t\t\t60\n \t\t\n\t\n \t\n \t\twithout_function\n \t\t\n \t\t\t0\n \t\t\t3600\n \t\t\n \t\t\n \t\t\t86400\n \t\t\t60\n \t\t\n\t\n \t\n\t \tplain\n \t\twithout_retention\n \t\tmin\n \t\n\t\n\t\ttagged\n\t\t^((.*)|.)sum\\?\n\t\tsum\n \t\n\t\n\t\ttag_list\n\t\tfake3;tag=Fake3\n\t\tmin\n \t\n\t\n\t\ttagged\n\t\t\n\t\tmin\n \t\n \t\n \t\tmax\n \t\t\n \t\t\t0\n \t\t\t60\n \t\t\n \t\t\n \t\t\t3600\n \t\t\t300\n \t\t\n \t\t\n \t\t\t86400\n \t\t\t3600\n \t\t\n \t\n\n`\n\n\texpected := &Rules{\n\t\tSeparated: true,\n\t\tPattern: []Pattern{\n\t\t\t{\n\t\t\t\tRegexp: \"click_cost\", Function: \"any\", Retention: []Retention{\n\t\t\t\t\t{Age: 0, Precision: 3600},\n\t\t\t\t\t{Age: 86400, Precision: 60},\n\t\t\t\t},\n\t\t\t\taggr: AggrMap[\"any\"], re: regexp.MustCompile(\"click_cost\"),\n\t\t\t},\n\t\t\t{\n\t\t\t\tRegexp: \"without_function\", Function: \"\", Retention: []Retention{\n\t\t\t\t\t{Age: 0, Precision: 3600},\n\t\t\t\t\t{Age: 86400, Precision: 60},\n\t\t\t\t},\n\t\t\t\tre: regexp.MustCompile(\"without_function\"),\n\t\t\t},\n\t\t\t{\n\t\t\t\tRegexp: \"without_retention\", RuleType: RulePlain, Function: \"min\", Retention: nil,\n\t\t\t\taggr: AggrMap[\"min\"], re: regexp.MustCompile(\"without_retention\"),\n\t\t\t},\n\t\t\t{\n\t\t\t\tRegexp: `^((.*)|.)sum\\?`, RuleType: RuleTagged, Function: \"sum\", Retention: nil,\n\t\t\t\taggr: AggrMap[\"sum\"], re: regexp.MustCompile(`^((.*)|.)sum\\?`),\n\t\t\t},\n\t\t\t{\n\t\t\t\tRegexp: `^fake3\\?(.*&)?tag=Fake3(&.*)?$`, RuleType: RuleTagged, Function: \"min\", Retention: nil,\n\t\t\t\taggr: AggrMap[\"min\"], re: regexp.MustCompile(`^fake3\\?(.*&)?tag=Fake3(&.*)?$`),\n\t\t\t},\n\t\t\t{\n\t\t\t\tRegexp: `^fake4\\\\?(.*&)?tag4=Fake4(&.*)?$`, RuleType: RuleTagged, Function: \"min\", Retention: nil,\n\t\t\t\taggr: AggrMap[\"min\"], re: regexp.MustCompile(`^fake4\\\\?(.*&)?tag4=Fake4(&.*)?$`),\n\t\t\t},\n\t\t\t{\n\t\t\t\tRegexp: \".*\", Function: \"max\", Retention: []Retention{\n\t\t\t\t\t{Age: 0, Precision: 60},\n\t\t\t\t\t{Age: 3600, Precision: 300},\n\t\t\t\t\t{Age: 86400, Precision: 3600},\n\t\t\t\t},\n\t\t\t\taggr: AggrMap[\"max\"],\n\t\t\t},\n\t\t},\n\t\tPatternPlain: []Pattern{\n\t\t\t{\n\t\t\t\tRegexp: \"click_cost\", Function: \"any\", Retention: []Retention{\n\t\t\t\t\t{Age: 0, Precision: 3600},\n\t\t\t\t\t{Age: 86400, Precision: 60},\n\t\t\t\t},\n\t\t\t\taggr: AggrMap[\"any\"], re: regexp.MustCompile(\"click_cost\"),\n\t\t\t},\n\t\t\t{\n\t\t\t\tRegexp: \"without_function\", Function: \"\", Retention: []Retention{\n\t\t\t\t\t{Age: 0, Precision: 3600},\n\t\t\t\t\t{Age: 86400, Precision: 60},\n\t\t\t\t},\n\t\t\t\tre: regexp.MustCompile(\"without_function\"),\n\t\t\t},\n\t\t\t{\n\t\t\t\tRegexp: \"without_retention\", RuleType: RulePlain, Function: \"min\", Retention: nil,\n\t\t\t\taggr: AggrMap[\"min\"], re: regexp.MustCompile(\"without_retention\"),\n\t\t\t},\n\t\t\t{\n\t\t\t\tRegexp: \".*\", Function: \"max\", Retention: []Retention{\n\t\t\t\t\t{Age: 0, Precision: 60},\n\t\t\t\t\t{Age: 3600, Precision: 300},\n\t\t\t\t\t{Age: 86400, Precision: 3600},\n\t\t\t\t},\n\t\t\t\taggr: AggrMap[\"max\"],\n\t\t\t},\n\t\t},\n\t\tPatternTagged: []Pattern{\n\t\t\t{\n\t\t\t\tRegexp: \"click_cost\", Function: \"any\", Retention: []Retention{\n\t\t\t\t\t{Age: 0, Precision: 3600},\n\t\t\t\t\t{Age: 86400, Precision: 60},\n\t\t\t\t},\n\t\t\t\taggr: AggrMap[\"any\"], re: regexp.MustCompile(\"click_cost\"),\n\t\t\t},\n\t\t\t{\n\t\t\t\tRegexp: \"without_function\", Function: \"\", Retention: []Retention{\n\t\t\t\t\t{Age: 0, Precision: 3600},\n\t\t\t\t\t{Age: 86400, Precision: 60},\n\t\t\t\t},\n\t\t\t\tre: regexp.MustCompile(\"without_function\"),\n\t\t\t},\n\t\t\t{\n\t\t\t\tRegexp: `^((.*)|.)sum\\?`, RuleType: RuleTagged, Function: \"sum\", Retention: nil,\n\t\t\t\taggr: AggrMap[\"sum\"], re: regexp.MustCompile(`^((.*)|.)sum\\?`),\n\t\t\t},\n\t\t\t{\n\t\t\t\tRegexp: `^fake3\\?(.*&)?tag=Fake3(&.*)?$`, RuleType: RuleTagged, Function: \"min\", Retention: nil,\n\t\t\t\taggr: AggrMap[\"min\"], re: regexp.MustCompile(`^fake3\\?(.*&)?tag=Fake3(&.*)?$`),\n\t\t\t},\n\t\t\t{\n\t\t\t\tRegexp: `^fake4\\\\?(.*&)?tag4=Fake4(&.*)?$`, RuleType: RuleTagged, Function: \"min\", Retention: nil,\n\t\t\t\taggr: AggrMap[\"min\"], re: regexp.MustCompile(`^fake4\\\\?(.*&)?tag4=Fake4(&.*)?$`),\n\t\t\t},\n\t\t\t{\n\t\t\t\tRegexp: \".*\", Function: \"max\", Retention: []Retention{\n\t\t\t\t\t{Age: 0, Precision: 60},\n\t\t\t\t\t{Age: 3600, Precision: 300},\n\t\t\t\t\t{Age: 86400, Precision: 3600},\n\t\t\t\t},\n\t\t\t\taggr: AggrMap[\"max\"],\n\t\t\t},\n\t\t},\n\t}\n\n\tt.Run(\"default\", func(t *testing.T) {\n\t\tassert := assert.New(t)\n\t\tr, err := parseXML([]byte(config))\n\t\trequire.NoError(t, err)\n\t\tassert.Equal(expected, r)\n\n\t\t// check sorting\n\t\tassert.Equal(uint32(0), r.Pattern[0].Retention[0].Age)\n\t\tassert.Equal(uint32(3600), r.Pattern[0].Retention[0].Precision)\n\t})\n\n\tt.Run(\"inside yandex tag\", func(t *testing.T) {\n\t\tassert := assert.New(t)\n\t\tr, err := parseXML([]byte(fmt.Sprintf(\"%s\", config)))\n\t\tassert.NoError(err)\n\t\tassert.Equal(expected, r)\n\t})\n}\n" }, { "path": "helper/tests/clickhouse/server.go", "content": "package clickhouse\n\nimport (\n\t\"io\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"sync\"\n\t\"sync/atomic\"\n)\n\ntype TestResponse struct {\n\tHeaders map[string]string\n\tBody []byte\n\tCode int\n}\n\ntype TestHandler struct {\n\tsync.RWMutex\n\tresponceMap map[string]*TestResponse\n\tqueries uint64\n}\n\ntype TestServer struct {\n\t*httptest.Server\n\thandler *TestHandler\n}\n\nfunc (h *TestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {\n\tbody, _ := io.ReadAll(r.Body)\n\n\treq := string(body)\n\n\th.RLock()\n\tresp, ok := h.responceMap[req]\n\th.RUnlock()\n\n\tatomic.AddUint64(&h.queries, 1)\n\n\tif ok {\n\t\tfor k, v := range resp.Headers {\n\t\t\tw.Header().Set(k, v)\n\t\t}\n\n\t\tif resp.Code == 0 || resp.Code == http.StatusOK {\n\t\t\tw.Write(resp.Body)\n\t\t} else {\n\t\t\thttp.Error(w, string(resp.Body), http.StatusInternalServerError)\n\t\t}\n\t} else {\n\t\thttp.Error(w, \"Query not added: \"+req, http.StatusInternalServerError)\n\t}\n}\n\nfunc NewTestServer() *TestServer {\n\th := &TestHandler{responceMap: make(map[string]*TestResponse)}\n\n\tsrv := httptest.NewServer(h)\n\n\treturn &TestServer{Server: srv, handler: h}\n}\n\nfunc (s *TestServer) AddResponce(request string, response *TestResponse) {\n\ts.handler.Lock()\n\ts.handler.responceMap[request] = response\n\ts.handler.Unlock()\n}\n\nfunc (s *TestServer) Queries() uint64 {\n\treturn s.handler.queries\n}\n" }, { "path": "helper/tests/compare/compare.go", "content": "package compare\n\nimport \"math\"\n\nconst eps = 0.0000000001\n\nfunc NearlyEqualSlice(a, b []float64) bool {\n\tif len(a) != len(b) {\n\t\treturn false\n\t}\n\n\tfor i, v := range a {\n\t\t// \"same\"\n\t\tif math.IsNaN(a[i]) && math.IsNaN(b[i]) {\n\t\t\tcontinue\n\t\t}\n\n\t\tif math.IsNaN(a[i]) || math.IsNaN(b[i]) {\n\t\t\t// unexpected NaN\n\t\t\treturn false\n\t\t}\n\t\t// \"close enough\"\n\t\tif math.Abs(v-b[i]) > eps {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\nfunc NearlyEqual(a, b float64) bool {\n\tif math.IsNaN(a) && math.IsNaN(b) {\n\t\treturn true\n\t}\n\n\tif math.IsNaN(a) || math.IsNaN(b) {\n\t\t// unexpected NaN\n\t\treturn false\n\t}\n\n\tif math.Abs(a-b) > eps {\n\t\treturn false\n\t}\n\n\treturn true\n}\n\nfunc Max(a, b int) int {\n\tif a >= b {\n\t\treturn a\n\t}\n\n\treturn b\n}\n" }, { "path": "helper/tests/compare/expand/expand.go", "content": "package expand\n\nimport (\n\t\"go/token\"\n\t\"go/types\"\n\t\"strconv\"\n\t\"strings\"\n)\n\nfunc ExpandTimestamp(fs *token.FileSet, s string, replace map[string]string) (int64, error) {\n\tif s == \"\" {\n\t\treturn 0, nil\n\t}\n\n\tfor k, v := range replace {\n\t\ts = strings.ReplaceAll(s, k, v)\n\t}\n\n\tif tv, err := types.Eval(fs, nil, token.NoPos, s); err == nil {\n\t\treturn strconv.ParseInt(tv.Value.String(), 10, 32)\n\t} else {\n\t\treturn 0, err\n\t}\n}\n" }, { "path": "helper/utils/utils.go", "content": "package utils\n\nimport \"time\"\n\n// TimestampTruncate truncate timestamp with duration\nfunc TimestampTruncate(ts int64, duration time.Duration) int64 {\n\ttm := time.Unix(ts, 0).UTC()\n\treturn tm.Truncate(duration).UTC().Unix()\n}\n" }, { "path": "helper/utils/utils_test.go", "content": "package utils\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestTimestampTruncate(t *testing.T) {\n\t// reverse sorted\n\ttests := []struct {\n\t\tts int64\n\t\tduration time.Duration\n\t\twant int64\n\t}{\n\t\t{\n\t\t\tts: 1628876563,\n\t\t\tduration: 2 * time.Second,\n\t\t\twant: 1628876562,\n\t\t},\n\t\t{\n\t\t\tts: 1628876563,\n\t\t\tduration: 10 * time.Second,\n\t\t\twant: 1628876560,\n\t\t},\n\t\t{\n\t\t\tts: 1628876563,\n\t\t\tduration: time.Minute,\n\t\t\twant: 1628876520,\n\t\t},\n\t\t{\n\t\t\tts: 1628876563,\n\t\t\tduration: time.Hour,\n\t\t\twant: 1628874000,\n\t\t},\n\t\t{\n\t\t\tts: 1628876563,\n\t\t\tduration: 24 * time.Hour,\n\t\t\twant: 1628812800,\n\t\t},\n\t}\n\tfor i, tt := range tests {\n\t\tt.Run(fmt.Sprintf(\"#%d\", i), func(t *testing.T) {\n\t\t\tif got := TimestampTruncate(tt.ts, tt.duration); got != tt.want {\n\t\t\t\tt.Errorf(\"timestampTruncate(%d, %d) = %v, want %v\", tt.ts, tt.duration, got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "index/handler.go", "content": "package index\n\nimport (\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/logs\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n)\n\ntype Handler struct {\n\tconfig *config.Config\n}\n\nfunc NewHandler(config *config.Config) *Handler {\n\treturn &Handler{\n\t\tconfig: config,\n\t}\n}\n\nfunc (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {\n\taccessLogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named(\"http\")\n\tlogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named(\"index\")\n\tr = r.WithContext(scope.WithLogger(r.Context(), logger))\n\n\tstatus := http.StatusOK\n\tstart := time.Now()\n\n\tdefer func() {\n\t\td := time.Since(start)\n\t\tlogs.AccessLog(accessLogger, h.config, r, status, d, time.Duration(0), false, false)\n\t}()\n\n\ti, err := New(h.config, r.Context())\n\tif err != nil {\n\t\tstatus = http.StatusBadRequest\n\t\thttp.Error(w, err.Error(), status)\n\n\t\treturn\n\t}\n\n\ti.WriteJSON(w)\n\ti.Close()\n}\n" }, { "path": "index/index.go", "content": "package index\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/finder\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n)\n\ntype Index struct {\n\tconfig *config.Config\n\trowsReader io.ReadCloser\n}\n\nfunc New(config *config.Config, ctx context.Context) (*Index, error) {\n\tvar reader io.ReadCloser\n\n\tvar err error\n\n\topts := clickhouse.Options{\n\t\tTLSConfig: config.ClickHouse.TLSConfig,\n\t\tConnectTimeout: config.ClickHouse.ConnectTimeout,\n\t\tCheckRequestProgress: config.FeatureFlags.LogQueryProgress,\n\t\tProgressSendingInterval: config.ClickHouse.ProgressSendingInterval,\n\t}\n\tif config.ClickHouse.IndexTable != \"\" {\n\t\topts.Timeout = config.ClickHouse.IndexTimeout\n\t\treader, err = clickhouse.Reader(\n\t\t\tscope.WithTable(ctx, config.ClickHouse.IndexTable),\n\t\t\tconfig.ClickHouse.URL,\n\t\t\tfmt.Sprintf(\n\t\t\t\t\"SELECT Path FROM %s WHERE Date = '%s' AND Level >= %d AND Level < %d GROUP BY Path\",\n\t\t\t\tconfig.ClickHouse.IndexTable, finder.DefaultTreeDate, finder.TreeLevelOffset, finder.ReverseTreeLevelOffset,\n\t\t\t),\n\t\t\topts,\n\t\t\tnil,\n\t\t)\n\t} else {\n\t\topts.Timeout = config.ClickHouse.TreeTimeout\n\t\treader, err = clickhouse.Reader(\n\t\t\tscope.WithTable(ctx, config.ClickHouse.TreeTable),\n\t\t\tconfig.ClickHouse.URL,\n\t\t\tfmt.Sprintf(\"SELECT Path FROM %s GROUP BY Path\", config.ClickHouse.TreeTable),\n\t\t\topts,\n\t\t\tnil,\n\t\t)\n\t}\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &Index{\n\t\tconfig: config,\n\t\trowsReader: reader,\n\t}, nil\n}\n\nfunc (i *Index) Close() error {\n\treturn i.rowsReader.Close()\n}\n\nfunc (i *Index) WriteJSON(w http.ResponseWriter) error {\n\t_, err := w.Write([]byte(\"[\"))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ts := bufio.NewScanner(i.rowsReader)\n\tidx := 0\n\n\tfor s.Scan() {\n\t\tb := s.Bytes()\n\t\tif len(b) == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tif b[len(b)-1] == '.' {\n\t\t\tcontinue\n\t\t}\n\n\t\tjson_b, err := json.Marshal(string(b))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tjsonParts := [][]byte{\n\t\t\tnil,\n\t\t\tjson_b,\n\t\t}\n\t\tif idx != 0 {\n\t\t\tjsonParts[0] = []byte{','}\n\t\t}\n\n\t\tjsonified := bytes.Join(jsonParts, []byte(\"\"))\n\n\t\t_, err = w.Write(jsonified)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tidx++\n\t}\n\n\tif err := s.Err(); err != nil {\n\t\treturn err\n\t}\n\n\t_, err = w.Write([]byte(\"]\"))\n\n\treturn err\n}\n" }, { "path": "index/index_test.go", "content": "package index\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"io\"\n\t\"net/http/httptest\"\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestWriteJSONEmptyRows(t *testing.T) {\n\trows := []string{\n\t\t\"\",\n\t\t\"testing.leaf\",\n\t\t\"\",\n\t\t\"testing.leaf.node\",\n\t\t\"\",\n\t}\n\n\tmetrics, err := writeRows(rows)\n\tif err != nil {\n\t\tt.Fatalf(\"Error during transform or unmarshal: %s\", err)\n\t}\n\n\tif len(metrics) != 2 {\n\t\tt.Fatalf(\"Wrong metrics slice length = %d: %s\", len(metrics), metrics)\n\t}\n\n\tif metrics[0] != \"testing.leaf\" || metrics[1] != \"testing.leaf.node\" {\n\t\tt.Fatalf(\"Wrong metrics contents: %s\", metrics)\n\t}\n}\n\nfunc TestWriteJSONNonleafRows(t *testing.T) {\n\trows := []string{\n\t\t\"testing.leaf\",\n\t\t\"testing.nonleaf.\",\n\t\t\"testing.leaf.node\",\n\t\t\"testing.\\\"broken\\\".node\",\n\t}\n\n\tmetrics, err := writeRows(rows)\n\tif err != nil {\n\t\tt.Fatalf(\"Error during transform or unmarshal: %s\", err)\n\t}\n\n\tif len(metrics) != 3 {\n\t\tt.Fatalf(\"Wrong metrics slice length = %d: %s\", len(metrics), metrics)\n\t}\n\n\tif metrics[0] != \"testing.leaf\" || metrics[1] != \"testing.leaf.node\" || metrics[2] != \"testing.\\\"broken\\\".node\" {\n\t\tt.Fatalf(\"Wrong metrics contents: %s\", metrics)\n\t}\n}\n\nfunc TestWriteJSONEmptyIndex(t *testing.T) {\n\trows := []string{}\n\n\tmetrics, err := writeRows(rows)\n\tif err != nil {\n\t\tt.Fatalf(\"Error during transform or unmarshal: %s\", err)\n\t}\n\n\tif len(metrics) != 0 {\n\t\tt.Fatalf(\"Wrong metrics slice length = %d: %s\", len(metrics), metrics)\n\t}\n}\n\nfunc indexForBytes(b []byte) *Index {\n\tbuffer := bytes.NewBuffer(b)\n\n\treturn &Index{\n\t\tconfig: nil,\n\t\trowsReader: io.NopCloser(buffer),\n\t}\n}\n\nfunc writeRows(rows []string) ([]string, error) {\n\trowsBytes := []byte(strings.Join(rows, string('\\n')))\n\tindex := indexForBytes(rowsBytes)\n\tmockResponse := httptest.NewRecorder()\n\n\terr := index.WriteJSON(mockResponse)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar metrics []string\n\n\terr = json.Unmarshal(mockResponse.Body.Bytes(), &metrics)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn metrics, nil\n}\n" }, { "path": "issues/daytime/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "issues/daytime/graphite-clickhouse-internal-aggr.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = true\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\ndate-format = \"both\"\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "issues/daytime/graphite-clickhouse.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = false\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\ndate-format = \"both\"\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "issues/daytime/test.toml", "content": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\n\n[test.carbon_clickhouse]\n#version = \"v0.11.1\"\nimage = \"msaf1980/carbon-clickhouse\"\nversion = \"tz\"\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse-internal-aggr.conf.tpl\"\n\n[[test.input]]\nname = \"test.plain1\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 2.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"test.plain2\"\npoints = [{value = 2.0, time = \"rnow-30\"}, {value = 1.0, time = \"rnow-20\"}, {value = 1.5, time = \"rnow-10\"}, {value = 2.5, time = \"rnow\"}]\n\n[[test.input]]\nname = \"test2.plain\"\npoints = [{value = 1.0, time = \"rnow-30\"}, {value = 2.0, time = \"rnow-20\"}, {value = 2.5, time = \"rnow-10\"}, {value = 3.5, time = \"rnow\"}]\n\n[[test.input]]\nname = \"metric1;tag1=value1;tag2=value21;tag3=value3\"\npoints = [{value = 2.0, time = \"rnow-30\"}, {value = 2.5, time = \"rnow-20\"}, {value = 2.0, time = \"rnow-10\"}, {value = 3.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"metric1;tag2=value22;tag4=value4\"\npoints = [{value = 1.0, time = \"rnow-30\"}, {value = 2.0, time = \"rnow-20\"}, {value = 0.0, time = \"rnow-10\"}, {value = 1.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"metric1;tag1=value1;tag2=value23;tag3=value3\"\npoints = [{value = 0.5, time = \"rnow-30\"}, {value = 1.5, time = \"rnow-20\"}, {value = 4.0, time = \"rnow-10\"}, {value = 3.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"metric2;tag2=value21;tag4=value4\"\npoints = [{value = 2.0, time = \"rnow-30\"}, {value = 1.0, time = \"rnow-20\"}, {value = 0.0, time = \"rnow-10\"}, {value = 1.0, time = \"rnow\"}]\n\n######################################\n# Check metrics find\n\n[[test.find_checks]]\nformats = [ \"pickle\", \"protobuf\", \"carbonapi_v3_pb\" ]\nquery = \"test\"\nresult = [ \n { path = \"test\", is_leaf = false }\n]\n\n[[test.find_checks]]\nformats = [ \"pickle\", \"protobuf\", \"carbonapi_v3_pb\" ]\nquery = \"test.pl*\"\nresult = [\n { path = \"test.plain1\", is_leaf = true }, { path = \"test.plain2\", is_leaf = true }\n]\n\n# End - Check metrics find\n######################################\n# Check tags autocomplete\n\n[[test.tags_checks]]\nquery = \"tag1;tag2=value21\"\nresult = [\n \"value1\"\n]\n\n[[test.tags_checks]]\nquery = \"name;tag2=value21;tag1=~value\"\nresult = [\n \"metric1\",\n]\n\n# End - Check tags autocomplete\n##########################################################################\n# Plain metrics (carbonapi_v3_pb)\n\n# test.plain1\n# test.plain2\n# test2.plain\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow\"\ntargets = [ \n \"test.plain*\",\n \"test{1,2}.plain\"\n]\n\n[[test.render_checks.result]]\nname = \"test.plain1\"\npath = \"test.plain*\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, 2.0]\n\n[[test.render_checks.result]]\nname = \"test.plain2\"\npath = \"test.plain*\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.5, 2.5]\n\n[[test.render_checks.result]]\nname = \"test2.plain\"\npath = \"test{1,2}.plain\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [2.5, 3.5]\n\n# End - Plain metrics (carbonapi_v3_pb)\n##########################################################################\n# Plain metrics (carbonapi_v2_pb)\n\n[[test.render_checks]]\nformats = [ \"protobuf\", \"carbonapi_v2_pb\" ]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntargets = [ \n \"test.plain*\",\n \"test{1,2}.plain\"\n]\n\n[[test.render_checks.result]]\nname = \"test.plain1\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [1.0, 2.0]\n\n[[test.render_checks.result]]\nname = \"test.plain2\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [1.5, 2.5]\n\n[[test.render_checks.result]]\nname = \"test2.plain\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [2.5, 3.5]\n\n# End - Plain metrics (carbonapi_v2_pb)\n##########################################################################\n# Plain metrics (pickle)\n\n[[test.render_checks]]\nformats = [ \"pickle\" ]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntargets = [ \n \"test.plain*\",\n \"test{1,2}.plain\"\n]\n\n[[test.render_checks.result]]\nname = \"test.plain1\"\npath = \"test.plain*\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [1.0, 2.0]\n\n[[test.render_checks.result]]\nname = \"test.plain2\"\npath = \"test.plain*\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [1.5, 2.5]\n\n[[test.render_checks.result]]\nname = \"test2.plain\"\npath = \"test{1,2}.plain\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [2.5, 3.5]\n\n# End - Plain metrics (pickle)\n##########################################################################\n# Taged metrics (carbonapi_v3_pb)\n\n# metric1;tag1=value1;tag2=value21;tag3=value3\n# metric1;tag2=value22;tag4=value4\n# metric1;tag1=value1;tag2=value23;tag3=value3\n# metric2;tag2=value21;tag4=value4\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntargets = [ \n \"seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')\",\n \"seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')\"\n]\n\n[[test.render_checks.result]]\nname = \"metric1;tag1=value1;tag2=value21;tag3=value3\"\npath = \"seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [2.0, 3.0]\n\n[[test.render_checks.result]]\nname = \"metric1;tag1=value1;tag2=value23;tag3=value3\"\npath = \"seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [4.0, 3.0]\n\n[[test.render_checks.result]]\nname = \"metric2;tag2=value21;tag4=value4\"\npath = \"seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [0.0, 1.0]\n\n# End - Tagged metrics (carbonapi_v3_pb)\n##########################################################################\n# Tagged metrics (carbonapi_v2_pb)\n\n[[test.render_checks]]\nformats = [ \"protobuf\", \"carbonapi_v2_pb\" ]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntargets = [ \n \"seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')\",\n \"seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')\"\n]\n\n[[test.render_checks.result]]\nname = \"metric1;tag1=value1;tag2=value21;tag3=value3\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [2.0, 3.0]\n\n[[test.render_checks.result]]\nname = \"metric1;tag1=value1;tag2=value23;tag3=value3\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [4.0, 3.0]\n\n[[test.render_checks.result]]\nname = \"metric2;tag2=value21;tag4=value4\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [0.0, 1.0]\n\n# End - Tagged metrics (carbonapi_v2_pb)\n##########################################################################\n# Tagged metrics (pickle)\n\n[[test.render_checks]]\nformats = [ \"pickle\" ]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntargets = [ \n \"seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')\",\n \"seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')\"\n]\n\n[[test.render_checks.result]]\nname = \"metric1;tag1=value1;tag2=value21;tag3=value3\"\npath = \"seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [2.0, 3.0]\n\n[[test.render_checks.result]]\nname = \"metric1;tag1=value1;tag2=value23;tag3=value3\"\npath = \"seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [4.0, 3.0]\n\n[[test.render_checks.result]]\nname = \"metric2;tag2=value21;tag4=value4\"\npath = \"seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [0.0, 1.0]\n\n# End - Tagged metrics (pickle)\n##########################################################################\n# Midnight\n\n# points for check https://github.com/go-graphite/graphite-clickhouse/issues/184\n[[test.input]]\nname = \"test.midnight\"\npoints = [{value = 3.0, time = \"midnight+60s\"}]\n\n[[test.input]]\nname = \"now;scope=midnight\"\npoints = [{value = 4.0, time = \"midnight+60s\"}]\n\n[[test.find_checks]]\nname = \"Midnight (direct)\"\nquery = \"test.midnight*\"\nresult = [{ path = \"test.midnight\", is_leaf = true }]\n\n[[test.find_checks]]\nname = \"Midnight\"\nquery = \"test.midnight\"\nfrom = \"midnight+60s\"\nuntil = \"midnight+70s\"\nresult = [{ path = \"test.midnight\", is_leaf = true }]\n\n[[test.find_checks]]\nname = \"Midnight (reverse)\"\nquery = \"*test.midnight\"\nresult = [{ path = \"test.midnight\", is_leaf = true }]\n\n[[test.find_checks]]\nname = \"Midnight\"\nquery = \"test.midnight\"\nfrom = \"midnight+60s\"\nuntil = \"midnight+70s\"\nresult = [{ path = \"test.midnight\", is_leaf = true }]\n\n[[test.tags_checks]]\nname = \"Midnight\"\nquery = \"name;scope=midnight\"\nresult = [\n \"now\",\n]\n\n[[test.render_checks]]\nname = \"Midnight (direct)\"\nformats = [ \"protobuf\" ]\nfrom = \"midnight+60s\"\nuntil = \"midnight+70s\"\ntargets = [ \n \"test.midnight*\",\n ]\n\n[[test.render_checks.result]]\nname = \"test.midnight\"\nstart = \"midnight+60s\"\nstop = \"midnight+80s\"\nstep = 10\nvalues = [3.0, nan]\n\n[[test.render_checks]]\nname = \"Midnight (reverse)\"\nformats = [ \"protobuf\" ]\nfrom = \"midnight+60s\"\nuntil = \"midnight+70s\"\ntargets = [ \n \"*test.midnight\",\n]\ndump_if_empty = [\n \"SELECT Date, Path FROM graphite_index WHERE ((Level=2) AND (Path LIKE 'test.midnight%')) GROUP BY Date, Path\"\n]\n\n[[test.render_checks.result]]\nname = \"test.midnight\"\nstart = \"midnight+60s\"\nstop = \"midnight+80s\"\nstep = 10\nvalues = [3.0, nan]\n\n[[test.render_checks]]\nname = \"Midnight\"\nformats = [ \"protobuf\" ]\nfrom = \"midnight+60s\"\nuntil = \"midnight+70s\"\ntargets = [ \n \"seriesByTag('name=now', 'scope=midnight')\",\n ]\n\n[[test.render_checks.result]]\nname = \"now;scope=midnight\"\nstart = \"midnight+60s\"\nstop = \"midnight+80s\"\nstep = 10\nvalues = [4.0, nan]\n\n# End - Midnight\n##########################################################################\n# Day end\n\n# points for check https://github.com/go-graphite/graphite-clickhouse/issues/184\n[[test.input]]\nname = \"test.23h\"\npoints = [{value = 3.0, time = \"midnight+1380m\"}]\n\n[[test.input]]\nname = \"now;scope=23h\"\npoints = [{value = 4.0, time = \"midnight+1380m\"}]\n\n[[test.find_checks]]\nname = \"Day end\"\nquery = \"test.23h\"\nfrom = \"midnight+1380m\"\nuntil = \"midnight+1381m\"\nresult = [{ path = \"test.23h\", is_leaf = true }]\n\n[[test.find_checks]]\nname = \"Day end\"\nquery = \"test.23h\"\nfrom = \"midnight+1380m\"\nuntil = \"midnight+1381m\"\nresult = [{ path = \"test.23h\", is_leaf = true }]\n\n[[test.tags_checks]]\nname = \"Day end\"\nquery = \"name;scope=23h\"\nresult = [\n \"now\",\n]\ndump_if_empty = [\n \"SELECT Date, Tags FROM graphite_tags WHERE ((Tag1='scope=23h') AND (arrayJoin(Tags) LIKE '__name__=%')) GROUP BY Date, Tags ORDER BY Date, Tags\"\n]\n\n[[test.render_checks]]\nname = \"Day end\"\nformats = [ \"protobuf\" ]\nfrom = \"midnight+1380m\"\nuntil = \"midnight+1380m+10s\"\ntargets = [ \n \"test.23h\",\n]\ndump_if_empty = [\n \"SELECT Date, Path FROM graphite_index WHERE ((Level=2) AND (Path IN ('test.23h','test.23h.'))) GROUP BY Date, Path\"\n]\n\n[[test.render_checks.result]]\nname = \"test.23h\"\nstart = \"midnight+1380m\"\nstop = \"midnight+1380m+20s\"\nstep = 10\nvalues = [3.0, nan]\n\n[[test.render_checks]]\nname = \"Day end\"\nformats = [ \"protobuf\" ]\nfrom = \"midnight+1380m\"\nuntil = \"midnight+1380m+10s\"\ntargets = [ \n \"seriesByTag('name=now', 'scope=23h')\",\n ]\n\n[[test.render_checks.result]]\nname = \"now;scope=23h\"\nstart = \"midnight+1380m\"\nstop = \"midnight+1380m+20s\"\nstep = 10\nvalues = [4.0, nan]\n\n# End - Day end\n##########################################################################\n" }, { "path": "limiter/alimiter.go", "content": "package limiter\n\nimport (\n\t\"context\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/load_avg\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n)\n\nvar (\n\tctxMain, Stop = context.WithCancel(context.Background())\n\tcheckDelay = time.Second * 60\n)\n\n// calc reserved slots count based on load average (for protect overload)\nfunc getWeighted(n, max int) int {\n\tif n <= 0 {\n\t\treturn 0\n\t}\n\n\tloadAvg := load_avg.Load()\n\tif loadAvg < 0.6 {\n\t\treturn 0\n\t}\n\n\tl := int(float64(n) * loadAvg)\n\tif l >= max {\n\t\tif max <= 1 {\n\t\t\treturn 1\n\t\t}\n\n\t\treturn max - 1\n\t}\n\n\treturn l\n}\n\n// ALimiter provide limiter amount of requests/concurrently executing requests (adaptive with load avg)\ntype ALimiter struct {\n\tlimiter limiter\n\tconcurrentLimiter limiter\n\tconcurrent int\n\tn int\n\n\tm metrics.WaitMetric\n}\n\n// NewServerLimiter creates a limiter for specific servers list.\nfunc NewALimiter(capacity, concurrent, n int, enableMetrics bool, scope, sub string) ServerLimiter {\n\tif capacity <= 0 && concurrent <= 0 {\n\t\treturn NoopLimiter{}\n\t}\n\n\tif n >= concurrent {\n\t\tn = concurrent - 1\n\t}\n\n\tif n <= 0 {\n\t\treturn NewWLimiter(capacity, concurrent, enableMetrics, scope, sub)\n\t}\n\n\ta := &ALimiter{\n\t\tm: metrics.NewWaitMetric(enableMetrics, scope, sub), concurrent: concurrent, n: n,\n\t}\n\ta.concurrentLimiter.ch = make(chan struct{}, concurrent)\n\ta.concurrentLimiter.cap = concurrent\n\n\tgo a.balance()\n\n\treturn a\n}\n\nfunc (sl *ALimiter) balance() int {\n\tvar last int\n\n\tfor {\n\t\tstart := time.Now()\n\n\t\tn := getWeighted(sl.n, sl.concurrent)\n\t\tif n > last {\n\t\t\tfor i := 0; i < n-last; i++ {\n\t\t\t\tif sl.concurrentLimiter.enter(ctxMain, \"balance\") != nil {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tlast = n\n\t\t} else if n < last {\n\t\t\tfor i := 0; i < last-n; i++ {\n\t\t\t\tsl.concurrentLimiter.leave(ctxMain, \"balance\")\n\t\t\t}\n\n\t\t\tlast = n\n\t\t}\n\n\t\tdelay := time.Since(start)\n\t\tif delay < checkDelay {\n\t\t\ttime.Sleep(checkDelay - delay)\n\t\t}\n\t}\n}\n\nfunc (sl *ALimiter) Capacity() int {\n\treturn sl.limiter.capacity()\n}\n\nfunc (sl *ALimiter) Enter(ctx context.Context, s string) (err error) {\n\tif sl.limiter.cap > 0 {\n\t\tif err = sl.limiter.tryEnter(ctx, s); err != nil {\n\t\t\tsl.m.WaitErrors.Add(1)\n\t\t\treturn\n\t\t}\n\t}\n\n\tif sl.concurrentLimiter.cap > 0 {\n\t\tif sl.concurrentLimiter.enter(ctx, s) != nil {\n\t\t\tif sl.limiter.cap > 0 {\n\t\t\t\tsl.limiter.leave(ctx, s)\n\t\t\t}\n\n\t\t\tsl.m.WaitErrors.Add(1)\n\n\t\t\terr = ErrTimeout\n\t\t}\n\t}\n\n\tsl.m.Requests.Add(1)\n\n\treturn\n}\n\n// TryEnter claims one of free slots without blocking.\nfunc (sl *ALimiter) TryEnter(ctx context.Context, s string) (err error) {\n\tif sl.limiter.cap > 0 {\n\t\tif err = sl.limiter.tryEnter(ctx, s); err != nil {\n\t\t\tsl.m.WaitErrors.Add(1)\n\t\t\treturn\n\t\t}\n\t}\n\n\tif sl.concurrentLimiter.cap > 0 {\n\t\tif sl.concurrentLimiter.tryEnter(ctx, s) != nil {\n\t\t\tif sl.limiter.cap > 0 {\n\t\t\t\tsl.limiter.leave(ctx, s)\n\t\t\t}\n\n\t\t\tsl.m.WaitErrors.Add(1)\n\n\t\t\terr = ErrTimeout\n\t\t}\n\t}\n\n\tsl.m.Requests.Add(1)\n\n\treturn\n}\n\n// Frees a slot in limiter\nfunc (sl *ALimiter) Leave(ctx context.Context, s string) {\n\tif sl.limiter.cap > 0 {\n\t\tsl.limiter.leave(ctx, s)\n\t}\n\n\tsl.concurrentLimiter.leave(ctx, s)\n}\n\n// SendDuration send StatsD duration iming\nfunc (sl *ALimiter) SendDuration(queueMs int64) {\n\tif sl.m.WaitTimeName != \"\" {\n\t\tmetrics.Gstatsd.Timing(sl.m.WaitTimeName, queueMs, 1.0)\n\t}\n}\n\n// Unregiter unregister graphite metric\nfunc (sl *ALimiter) Unregiter() {\n\tsl.m.Unregister()\n}\n\n// Enabled return enabled flag, if false - it's a noop limiter and can be safely skiped\nfunc (sl *ALimiter) Enabled() bool {\n\treturn true\n}\n" }, { "path": "limiter/alimiter_test.go", "content": "package limiter\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"strconv\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/load_avg\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc Test_getWeighted(t *testing.T) {\n\ttests := []struct {\n\t\tloadAvg float64\n\t\tn int\n\t\tmax int\n\t\twant int\n\t}{\n\t\t{loadAvg: 0, max: 100, n: 100, want: 0},\n\t\t{loadAvg: 0.2, max: 100, n: 100, want: 0},\n\t\t{loadAvg: 0.7, max: 100, n: 100, want: 70},\n\t\t{loadAvg: 0.8, max: 100, n: 100, want: 80},\n\t\t{loadAvg: 0.999, max: 100, n: 100, want: 99},\n\t\t{loadAvg: 0.999, max: 100, n: 1, want: 0},\n\t\t{loadAvg: 1, max: 1, n: 100, want: 1},\n\t\t{loadAvg: 1, max: 100, n: 100, want: 99},\n\t\t{loadAvg: 1, max: 101, n: 100, want: 100},\n\t\t{loadAvg: 1, max: 200, n: 100, want: 100},\n\t\t{loadAvg: 2, max: 100, n: 200, want: 99},\n\t\t{loadAvg: 2, max: 200, n: 200, want: 199},\n\t\t{loadAvg: 2, max: 300, n: 200, want: 299},\n\t\t{loadAvg: 2, max: 400, n: 200, want: 399},\n\t\t{loadAvg: 2, max: 401, n: 200, want: 400},\n\t\t{loadAvg: 2, max: 402, n: 200, want: 400},\n\t}\n\tfor n, tt := range tests {\n\t\tt.Run(strconv.Itoa(n), func(t *testing.T) {\n\t\t\tload_avg.Store(tt.loadAvg)\n\n\t\t\tif got := getWeighted(tt.n, tt.max); got != tt.want {\n\t\t\t\tt.Errorf(\"load avg = %f getWeighted(%d, %d) = %v, want %v\", tt.loadAvg, tt.n, tt.max, got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestNewALimiter(t *testing.T) {\n\tcapacity := 14\n\tconcurrent := 12\n\tn := 10\n\tcheckDelay = time.Millisecond * 10\n\tlimiter := NewALimiter(capacity, concurrent, n, false, \"\", \"\")\n\n\t// inital - load not collected\n\tload_avg.Store(0)\n\n\tvar i int\n\n\tctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*100)\n\n\tfor i = 0; i < concurrent; i++ {\n\t\trequire.NoError(t, limiter.Enter(ctx, \"render\"), \"try to lock with load_avg = 0 [%d]\", i)\n\t}\n\n\trequire.Error(t, limiter.Enter(ctx, \"render\"))\n\n\tfor i = 0; i < concurrent; i++ {\n\t\tlimiter.Leave(ctx, \"render\")\n\t}\n\n\tcancel()\n\n\t// load_avg 0.5\n\tload_avg.Store(0.5)\n\n\tk := getWeighted(n, concurrent)\n\trequire.Equal(t, 0, k)\n\n\t// load_avg 0.6\n\tload_avg.Store(0.6)\n\n\tk = getWeighted(n, concurrent)\n\trequire.Equal(t, n*6/10, k)\n\n\ttime.Sleep(checkDelay * 2)\n\n\tctx, cancel = context.WithTimeout(context.Background(), time.Millisecond*100)\n\tfor i = 0; i < concurrent-k; i++ {\n\t\trequire.NoError(t, limiter.Enter(ctx, \"render\"), \"try to lock with load_avg = 0.5 [%d]\", i)\n\t}\n\n\trequire.Error(t, limiter.Enter(ctx, \"render\"))\n\n\tfor i = 0; i < concurrent-k; i++ {\n\t\tlimiter.Leave(ctx, \"render\")\n\t}\n\n\tcancel()\n\n\t// // load_avg 1\n\tload_avg.Store(1)\n\n\tk = getWeighted(n, concurrent)\n\trequire.Equal(t, n, k)\n\n\ttime.Sleep(checkDelay * 2)\n\n\tctx, cancel = context.WithTimeout(context.Background(), time.Millisecond*10)\n\tfor i = 0; i < concurrent-n; i++ {\n\t\trequire.NoError(t, limiter.Enter(ctx, \"render\"), \"try to lock with load_avg = 1 [%d]\", i)\n\t}\n\n\trequire.Error(t, limiter.Enter(ctx, \"render\"))\n\n\tfor i = 0; i < concurrent-n; i++ {\n\t\tlimiter.Leave(ctx, \"render\")\n\t}\n\n\tcancel()\n}\n\ntype testLimiter struct {\n\tl int\n\tc int\n\tn int\n\tconcurrencyLevel int\n}\n\nfunc Benchmark_Limiter_Parallel(b *testing.B) {\n\ttests := []testLimiter{\n\t\t// WLimiter\n\t\t{l: 2000, c: 10, concurrencyLevel: 1},\n\t\t{l: 2000, c: 10, concurrencyLevel: 10},\n\t\t{l: 2000, c: 10, concurrencyLevel: 20},\n\t\t{l: 2000, c: 10, concurrencyLevel: 50},\n\t\t{l: 2000, c: 10, concurrencyLevel: 100},\n\t\t{l: 2000, c: 10, concurrencyLevel: 1000},\n\t\t// ALimiter\n\t\t{l: 2000, c: 10, n: 50, concurrencyLevel: 1},\n\t\t{l: 2000, c: 10, n: 50, concurrencyLevel: 10},\n\t\t{l: 2000, c: 10, n: 50, concurrencyLevel: 20},\n\t\t{l: 2000, c: 10, n: 50, concurrencyLevel: 50},\n\t\t{l: 2000, c: 10, n: 50, concurrencyLevel: 100},\n\t\t{l: 2000, c: 10, n: 50, concurrencyLevel: 1000},\n\t}\n\n\tload_avg.Store(0.5)\n\n\tfor _, tt := range tests {\n\t\tb.Run(fmt.Sprintf(\"L%d_C%d_N%d_CONCURRENCY%d\", tt.l, tt.c, tt.n, tt.concurrencyLevel), func(b *testing.B) {\n\t\t\tvar (\n\t\t\t\terr error\n\t\t\t)\n\n\t\t\tlimiter := NewALimiter(tt.l, tt.c, tt.n, false, \"\", \"\")\n\n\t\t\twgStart := sync.WaitGroup{}\n\t\t\twg := sync.WaitGroup{}\n\n\t\t\twgStart.Add(tt.concurrencyLevel)\n\n\t\t\tctx := context.Background()\n\n\t\t\tb.ResetTimer()\n\n\t\t\tfor i := 0; i < tt.concurrencyLevel; i++ {\n\t\t\t\twg.Add(1)\n\n\t\t\t\tgo func() {\n\t\t\t\t\twgStart.Done()\n\t\t\t\t\twgStart.Wait()\n\t\t\t\t\t// Test routine\n\t\t\t\t\tfor n := 0; n < b.N; n++ {\n\t\t\t\t\t\terrW := limiter.Enter(ctx, \"render\")\n\t\t\t\t\t\tif errW == nil {\n\t\t\t\t\t\t\tlimiter.Leave(ctx, \"render\")\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\terr = errW\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\t// End test routine\n\t\t\t\t\twg.Done()\n\t\t\t\t}()\n\t\t\t}\n\n\t\t\twg.Wait()\n\t\t\tb.StopTimer()\n\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(b, err)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "limiter/interface.go", "content": "package limiter\n\nimport (\n\t\"context\"\n\t\"errors\"\n)\n\nvar (\n\tErrTimeout = errors.New(\"timeout exceeded\")\n\tErrOverflow = errors.New(\"storage maximum queries exceeded\")\n)\n\ntype ServerLimiter interface {\n\tCapacity() int\n\tEnabled() bool\n\tTryEnter(ctx context.Context, s string) error\n\tEnter(ctx context.Context, s string) error\n\tLeave(ctx context.Context, s string)\n\tSendDuration(queueMs int64)\n\tUnregiter()\n}\n" }, { "path": "limiter/limiter.go", "content": "package limiter\n\nimport (\n\t\"context\"\n\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n)\n\ntype limiter struct {\n\tch chan struct{}\n\tcap int\n}\n\n// Limiter provides interface to limit amount of requests\ntype Limiter struct {\n\tlimiter limiter\n\tmetrics metrics.WaitMetric\n}\n\n// NewServerLimiter creates a limiter for specific servers list.\nfunc NewLimiter(capacity int, enableMetrics bool, scope, sub string) ServerLimiter {\n\tif capacity <= 0 {\n\t\treturn NoopLimiter{}\n\t}\n\n\treturn &Limiter{\n\t\tlimiter: limiter{\n\t\t\tch: make(chan struct{}, capacity),\n\t\t\tcap: capacity,\n\t\t},\n\t\tmetrics: metrics.NewWaitMetric(enableMetrics, scope, sub),\n\t}\n}\n\nfunc (sl *Limiter) Capacity() int {\n\treturn sl.limiter.capacity()\n}\n\n// Enter claims one of free slots or blocks until there is one.\nfunc (sl *Limiter) Enter(ctx context.Context, s string) (err error) {\n\tif err = sl.limiter.enter(ctx, s); err != nil {\n\t\tsl.metrics.WaitErrors.Add(1)\n\t}\n\n\tsl.metrics.Requests.Add(1)\n\n\treturn\n}\n\n// TryEnter claims one of free slots without blocking.\nfunc (sl *Limiter) TryEnter(ctx context.Context, s string) (err error) {\n\tif err = sl.limiter.tryEnter(ctx, s); err != nil {\n\t\tsl.metrics.WaitErrors.Add(1)\n\t}\n\n\tsl.metrics.Requests.Add(1)\n\n\treturn\n}\n\n// Frees a slot in limiter\nfunc (sl *Limiter) Leave(ctx context.Context, s string) {\n\tsl.limiter.leave(ctx, s)\n}\n\n// SendDuration send StatsD duration iming\nfunc (sl *Limiter) SendDuration(queueMs int64) {\n\tif sl.metrics.WaitTimeName != \"\" {\n\t\tmetrics.Gstatsd.Timing(sl.metrics.WaitTimeName, queueMs, 1.0)\n\t}\n}\n\n// Unregiter unregister graphite metric\nfunc (sl *Limiter) Unregiter() {\n\tsl.metrics.Unregister()\n}\n\n// Enabled return enabled flag, if false - it's a noop limiter and can be safely skiped\nfunc (sl *Limiter) Enabled() bool {\n\treturn true\n}\n\nfunc (sl *limiter) capacity() int {\n\treturn sl.cap\n}\n\n// Enter claims one of free slots or blocks until there is one.\nfunc (sl *limiter) enter(ctx context.Context, s string) error {\n\tselect {\n\tcase sl.ch <- struct{}{}:\n\t\treturn nil\n\tcase <-ctx.Done():\n\t\treturn ErrTimeout\n\t}\n}\n\n// TryEnter claims one of free slots without blocking.\nfunc (sl *limiter) tryEnter(ctx context.Context, s string) error {\n\tselect {\n\tcase sl.ch <- struct{}{}:\n\t\treturn nil\n\tdefault:\n\t\treturn ErrOverflow\n\t}\n}\n\n// Frees a slot in limiter\nfunc (sl *limiter) leave(ctx context.Context, s string) {\n\t<-sl.ch\n}\n" }, { "path": "limiter/noop.go", "content": "package limiter\n\nimport (\n\t\"context\"\n)\n\n// ServerLimiter provides interface to limit amount of requests\ntype NoopLimiter struct {\n}\n\nfunc (l NoopLimiter) Capacity() int {\n\treturn 0\n}\n\n// Enter claims one of free slots or blocks until there is one.\nfunc (l NoopLimiter) Enter(ctx context.Context, s string) error {\n\treturn nil\n}\n\n// TryEnter claims one of free slots without blocking\nfunc (l NoopLimiter) TryEnter(ctx context.Context, s string) error {\n\treturn nil\n}\n\n// Frees a slot in limiter\nfunc (l NoopLimiter) Leave(ctx context.Context, s string) {\n}\n\n// SendDuration send StatsD duration iming\nfunc (l NoopLimiter) SendDuration(queueMs int64) {\n}\n\n// Unregiter unregister graphite metric\nfunc (l NoopLimiter) Unregiter() {\n}\n\n// Enabled return enabled flag, if false - it's a noop limiter and can be safely skiped\nfunc (l NoopLimiter) Enabled() bool {\n\treturn false\n}\n" }, { "path": "limiter/wlimiter.go", "content": "package limiter\n\nimport (\n\t\"context\"\n\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n)\n\n// WLimiter provide limiter amount of requests/concurrently executing requests\ntype WLimiter struct {\n\tlimiter limiter\n\tconcurrentLimiter limiter\n\tmetrics metrics.WaitMetric\n}\n\n// NewServerLimiter creates a limiter for specific servers list.\nfunc NewWLimiter(capacity, concurrent int, enableMetrics bool, scope, sub string) ServerLimiter {\n\tif capacity <= 0 && concurrent <= 0 {\n\t\treturn NoopLimiter{}\n\t}\n\n\tif concurrent <= 0 {\n\t\treturn NewLimiter(capacity, enableMetrics, scope, sub)\n\t}\n\n\tw := &WLimiter{\n\t\tmetrics: metrics.NewWaitMetric(enableMetrics, scope, sub),\n\t}\n\tif capacity > 0 {\n\t\tw.limiter.ch = make(chan struct{}, capacity)\n\t\tw.limiter.cap = capacity\n\t}\n\n\tif concurrent > 0 {\n\t\tw.concurrentLimiter.ch = make(chan struct{}, concurrent)\n\t\tw.concurrentLimiter.cap = concurrent\n\t}\n\n\treturn w\n}\n\nfunc (sl *WLimiter) Capacity() int {\n\treturn sl.limiter.capacity()\n}\n\nfunc (sl *WLimiter) Enter(ctx context.Context, s string) (err error) {\n\tif sl.limiter.cap > 0 {\n\t\tif err = sl.limiter.tryEnter(ctx, s); err != nil {\n\t\t\tsl.metrics.WaitErrors.Add(1)\n\t\t\treturn\n\t\t}\n\t}\n\n\tif sl.concurrentLimiter.cap > 0 {\n\t\tif sl.concurrentLimiter.enter(ctx, s) != nil {\n\t\t\tif sl.limiter.cap > 0 {\n\t\t\t\tsl.limiter.leave(ctx, s)\n\t\t\t}\n\n\t\t\tsl.metrics.WaitErrors.Add(1)\n\n\t\t\terr = ErrTimeout\n\t\t}\n\t}\n\n\tsl.metrics.Requests.Add(1)\n\n\treturn\n}\n\n// TryEnter claims one of free slots without blocking.\nfunc (sl *WLimiter) TryEnter(ctx context.Context, s string) (err error) {\n\tif sl.limiter.cap > 0 {\n\t\tif err = sl.limiter.tryEnter(ctx, s); err != nil {\n\t\t\tsl.metrics.WaitErrors.Add(1)\n\t\t\treturn\n\t\t}\n\t}\n\n\tif sl.concurrentLimiter.cap > 0 {\n\t\tif sl.concurrentLimiter.tryEnter(ctx, s) != nil {\n\t\t\tif sl.limiter.cap > 0 {\n\t\t\t\tsl.limiter.leave(ctx, s)\n\t\t\t}\n\n\t\t\tsl.metrics.WaitErrors.Add(1)\n\n\t\t\terr = ErrTimeout\n\t\t}\n\t}\n\n\tsl.metrics.Requests.Add(1)\n\n\treturn\n}\n\n// Frees a slot in limiter\nfunc (sl *WLimiter) Leave(ctx context.Context, s string) {\n\tif sl.limiter.cap > 0 {\n\t\tsl.limiter.leave(ctx, s)\n\t}\n\n\tsl.concurrentLimiter.leave(ctx, s)\n}\n\n// SendDuration send StatsD duration iming\nfunc (sl *WLimiter) SendDuration(queueMs int64) {\n\tif sl.metrics.WaitTimeName != \"\" {\n\t\tmetrics.Gstatsd.Timing(sl.metrics.WaitTimeName, queueMs, 1.0)\n\t}\n}\n\n// Unregiter unregister graphite metric\nfunc (sl *WLimiter) Unregiter() {\n\tsl.metrics.Unregister()\n}\n\n// Enabled return enabled flag, if false - it's a noop limiter and can be safely skiped\nfunc (sl *WLimiter) Enabled() bool {\n\treturn true\n}\n" }, { "path": "load_avg/load_avg.go", "content": "package load_avg\n\nimport (\n\t\"math\"\n\n\t\"github.com/msaf1980/go-syncutils/atomic\"\n)\n\nvar loadAvgStore atomic.Float64\n\nfunc Load() float64 {\n\treturn loadAvgStore.Load()\n}\n\nfunc Store(f float64) {\n\tloadAvgStore.Store(f)\n}\n\nfunc Weight(weight int, degraged, degragedLoadAvg, normalizedLoadAvg float64) int64 {\n\tif weight <= 0 || degraged <= 1 || normalizedLoadAvg >= 2.0 {\n\t\treturn 1\n\t}\n\n\tif normalizedLoadAvg > degragedLoadAvg {\n\t\tnormalizedLoadAvg *= degraged\n\t}\n\n\tnormalizedLoadAvg = math.Round(10*normalizedLoadAvg) / 10\n\tif normalizedLoadAvg == 0 {\n\t\treturn 2 * int64(weight)\n\t}\n\n\tnormalizedLoadAvg = math.Log10(normalizedLoadAvg)\n\n\tw := int64(weight) - int64(float64(weight)*normalizedLoadAvg)\n\tif w <= 0 {\n\t\treturn 1\n\t}\n\n\treturn w\n}\n" }, { "path": "load_avg/load_avg_default.go", "content": "//go:build !linux\n// +build !linux\n\npackage load_avg\n\nfunc Normalized() (float64, error) {\n\treturn 0, nil\n}\n\nfunc CpuCount() (uint64, error) {\n\treturn 0, nil\n}\n" }, { "path": "load_avg/load_avg_linux.go", "content": "//go:build linux\n// +build linux\n\npackage load_avg\n\nimport (\n\t\"os\"\n\t\"strings\"\n\t\"syscall\"\n\n\t\"github.com/msaf1980/go-stringutils\"\n)\n\nfunc Normalized() (float64, error) {\n\tvar info syscall.Sysinfo_t\n\n\terr := syscall.Sysinfo(&info)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tcpus, err := CpuCount()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tconst si_load_shift = 16\n\tload := float64(info.Loads[0]) / float64(1< 0 {\n\t\t\trequestMetric.RangeS = c.FindRangeS\n\t\t\trequestMetric.RangeNames = c.FindRangeNames\n\t\t\trequestMetric.RangeMetrics = make([]ReqMetric, len(c.FindRangeS))\n\n\t\t\tfor i := range c.FindRangeS {\n\t\t\t\trequestMetric.RangeMetrics[i].RequestsH = metrics.NewVSumHistogram(c.BucketsWidth, c.BucketsLabels).SetNameTotal(\"\")\n\t\t\t\trequestMetric.RangeMetrics[i].Errors = metrics.NewCounter()\n\t\t\t\trequestMetric.RangeMetrics[i].MetricsCountName = scope + \".\" + requestMetric.RangeNames[i] + \".metrics\"\n\t\t\t\trequestMetric.RangeMetrics[i].PointsCountName = scope + \".\" + requestMetric.RangeNames[i] + \".points\"\n\t\t\t\tmetrics.Register(scope+\".\"+c.FindRangeNames[i]+\".requests\", requestMetric.RangeMetrics[i].RequestsH)\n\t\t\t\tmetrics.Register(scope+\".\"+c.FindRangeNames[i]+\".errors\", requestMetric.RangeMetrics[i].Errors)\n\n\t\t\t\tif c.ExtendedStat {\n\t\t\t\t\trequestMetric.RangeMetrics[i].Requests200 = metrics.NewCounter()\n\t\t\t\t\trequestMetric.RangeMetrics[i].Requests400 = metrics.NewCounter()\n\t\t\t\t\trequestMetric.RangeMetrics[i].Requests403 = metrics.NewCounter()\n\t\t\t\t\trequestMetric.RangeMetrics[i].Requests404 = metrics.NewCounter()\n\t\t\t\t\trequestMetric.RangeMetrics[i].Requests500 = metrics.NewCounter()\n\t\t\t\t\trequestMetric.RangeMetrics[i].Requests503 = metrics.NewCounter()\n\t\t\t\t\trequestMetric.RangeMetrics[i].Requests504 = metrics.NewCounter()\n\t\t\t\t\trequestMetric.RangeMetrics[i].Requests5xx = metrics.NewCounter()\n\t\t\t\t\trequestMetric.RangeMetrics[i].Requests4xx = metrics.NewCounter()\n\t\t\t\t\tmetrics.Register(scope+\".\"+c.FindRangeNames[i]+\".requests_status_code.200\", requestMetric.RangeMetrics[i].Requests200)\n\t\t\t\t\tmetrics.Register(scope+\".\"+c.FindRangeNames[i]+\".requests_status_code.400\", requestMetric.RangeMetrics[i].Requests400)\n\t\t\t\t\tmetrics.Register(scope+\".\"+c.FindRangeNames[i]+\".requests_status_code.403\", requestMetric.RangeMetrics[i].Requests403)\n\t\t\t\t\tmetrics.Register(scope+\".\"+c.FindRangeNames[i]+\".requests_status_code.404\", requestMetric.RangeMetrics[i].Requests404)\n\t\t\t\t\tmetrics.Register(scope+\".\"+c.FindRangeNames[i]+\".requests_status_code.4xx\", requestMetric.RangeMetrics[i].Requests4xx)\n\t\t\t\t\tmetrics.Register(scope+\".\"+c.FindRangeNames[i]+\".requests_status_code.500\", requestMetric.RangeMetrics[i].Requests500)\n\t\t\t\t\tmetrics.Register(scope+\".\"+c.FindRangeNames[i]+\".requests_status_code.503\", requestMetric.RangeMetrics[i].Requests503)\n\t\t\t\t\tmetrics.Register(scope+\".\"+c.FindRangeNames[i]+\".requests_status_code.504\", requestMetric.RangeMetrics[i].Requests504)\n\t\t\t\t\tmetrics.Register(scope+\".\"+c.FindRangeNames[i]+\".requests_status_code.5xx\", requestMetric.RangeMetrics[i].Requests5xx)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t} else {\n\t\trequestMetric.RequestsH = metrics.NilHistogram{}\n\t}\n\n\treturn requestMetric\n}\n\nfunc initRenderMetrics(scope string, c *Config) *RenderMetrics {\n\trequestMetric := &RenderMetrics{\n\t\tRenderMetric: RenderMetric{\n\t\t\tReqMetric: ReqMetric{\n\t\t\t\tErrors: metrics.NewCounter(),\n\t\t\t\tMetricsCountName: scope + \".all.metrics\",\n\t\t\t\tPointsCountName: scope + \".all.points\",\n\t\t\t},\n\t\t},\n\t}\n\n\tif c == nil || Graphite == nil || !c.ExtendedStat {\n\t\trequestMetric.Requests200 = metrics.NilCounter{}\n\t\trequestMetric.Requests400 = metrics.NilCounter{}\n\t\trequestMetric.Requests403 = metrics.NilCounter{}\n\t\trequestMetric.Requests404 = metrics.NilCounter{}\n\t\trequestMetric.Requests500 = metrics.NilCounter{}\n\t\trequestMetric.Requests503 = metrics.NilCounter{}\n\t\trequestMetric.Requests504 = metrics.NilCounter{}\n\t\trequestMetric.Requests5xx = metrics.NilCounter{}\n\t\trequestMetric.Requests4xx = metrics.NilCounter{}\n\t} else {\n\t\trequestMetric.Requests200 = metrics.NewCounter()\n\t\trequestMetric.Requests400 = metrics.NewCounter()\n\t\trequestMetric.Requests403 = metrics.NewCounter()\n\t\trequestMetric.Requests404 = metrics.NewCounter()\n\t\trequestMetric.Requests500 = metrics.NewCounter()\n\t\trequestMetric.Requests503 = metrics.NewCounter()\n\t\trequestMetric.Requests504 = metrics.NewCounter()\n\t\trequestMetric.Requests5xx = metrics.NewCounter()\n\t\trequestMetric.Requests4xx = metrics.NewCounter()\n\t}\n\n\tif c != nil && Graphite != nil {\n\t\trequestMetric.RequestsH = metrics.NewVSumHistogram(c.BucketsWidth, c.BucketsLabels).SetNameTotal(\"\")\n\t\trequestMetric.FinderH = metrics.NewVSumHistogram(c.BucketsWidth, c.BucketsLabels).SetNameTotal(\"\")\n\t\tmetrics.Register(scope+\".all.requests\", requestMetric.RequestsH)\n\t\tmetrics.Register(scope+\".all.requests_finder\", requestMetric.FinderH)\n\t\tmetrics.Register(scope+\".all.errors\", requestMetric.Errors)\n\n\t\tif c.ExtendedStat {\n\t\t\tmetrics.Register(scope+\".all.requests_status_code.200\", requestMetric.Requests200)\n\t\t\tmetrics.Register(scope+\".all.requests_status_code.400\", requestMetric.Requests400)\n\t\t\tmetrics.Register(scope+\".all.requests_status_code.403\", requestMetric.Requests403)\n\t\t\tmetrics.Register(scope+\".all.requests_status_code.404\", requestMetric.Requests404)\n\t\t\tmetrics.Register(scope+\".all.requests_status_code.4xx\", requestMetric.Requests4xx)\n\t\t\tmetrics.Register(scope+\".all.requests_status_code.500\", requestMetric.Requests500)\n\t\t\tmetrics.Register(scope+\".all.requests_status_code.503\", requestMetric.Requests503)\n\t\t\tmetrics.Register(scope+\".all.requests_status_code.504\", requestMetric.Requests504)\n\t\t\tmetrics.Register(scope+\".all.requests_status_code.5xx\", requestMetric.Requests5xx)\n\t\t}\n\n\t\tif len(c.RangeS) > 0 {\n\t\t\trequestMetric.RangeS = c.RangeS\n\t\t\trequestMetric.RangeNames = c.RangeNames\n\t\t\trequestMetric.RangeMetrics = make([]RenderMetric, len(c.RangeS))\n\n\t\t\tfor i := range c.RangeS {\n\t\t\t\trequestMetric.RangeMetrics[i].RequestsH = metrics.NewVSumHistogram(c.BucketsWidth, c.BucketsLabels).SetNameTotal(\"\")\n\t\t\t\trequestMetric.RangeMetrics[i].FinderH = metrics.NewVSumHistogram(c.BucketsWidth, c.BucketsLabels).SetNameTotal(\"\")\n\t\t\t\trequestMetric.RangeMetrics[i].Errors = metrics.NewCounter()\n\t\t\t\trequestMetric.RangeMetrics[i].MetricsCountName = scope + \".\" + requestMetric.RangeNames[i] + \".metrics\"\n\t\t\t\trequestMetric.RangeMetrics[i].PointsCountName = scope + \".\" + requestMetric.RangeNames[i] + \".points\"\n\t\t\t\tmetrics.Register(scope+\".\"+c.RangeNames[i]+\".requests\", requestMetric.RangeMetrics[i].RequestsH)\n\t\t\t\tmetrics.Register(scope+\".\"+c.RangeNames[i]+\".requests_finder\", requestMetric.RangeMetrics[i].FinderH)\n\t\t\t\tmetrics.Register(scope+\".\"+c.RangeNames[i]+\".errors\", requestMetric.RangeMetrics[i].Errors)\n\n\t\t\t\tif c.ExtendedStat {\n\t\t\t\t\trequestMetric.RangeMetrics[i].Requests200 = metrics.NewCounter()\n\t\t\t\t\trequestMetric.RangeMetrics[i].Requests400 = metrics.NewCounter()\n\t\t\t\t\trequestMetric.RangeMetrics[i].Requests403 = metrics.NewCounter()\n\t\t\t\t\trequestMetric.RangeMetrics[i].Requests404 = metrics.NewCounter()\n\t\t\t\t\trequestMetric.RangeMetrics[i].Requests500 = metrics.NewCounter()\n\t\t\t\t\trequestMetric.RangeMetrics[i].Requests503 = metrics.NewCounter()\n\t\t\t\t\trequestMetric.RangeMetrics[i].Requests504 = metrics.NewCounter()\n\t\t\t\t\trequestMetric.RangeMetrics[i].Requests5xx = metrics.NewCounter()\n\t\t\t\t\trequestMetric.RangeMetrics[i].Requests4xx = metrics.NewCounter()\n\t\t\t\t\tmetrics.Register(scope+\".\"+c.RangeNames[i]+\".requests_status_code.200\", requestMetric.RangeMetrics[i].Requests200)\n\t\t\t\t\tmetrics.Register(scope+\".\"+c.RangeNames[i]+\".requests_status_code.400\", requestMetric.RangeMetrics[i].Requests400)\n\t\t\t\t\tmetrics.Register(scope+\".\"+c.RangeNames[i]+\".requests_status_code.403\", requestMetric.RangeMetrics[i].Requests403)\n\t\t\t\t\tmetrics.Register(scope+\".\"+c.RangeNames[i]+\".requests_status_code.404\", requestMetric.RangeMetrics[i].Requests404)\n\t\t\t\t\tmetrics.Register(scope+\".\"+c.RangeNames[i]+\".requests_status_code.4xx\", requestMetric.RangeMetrics[i].Requests4xx)\n\t\t\t\t\tmetrics.Register(scope+\".\"+c.RangeNames[i]+\".requests_status_code.500\", requestMetric.RangeMetrics[i].Requests500)\n\t\t\t\t\tmetrics.Register(scope+\".\"+c.RangeNames[i]+\".requests_status_code.503\", requestMetric.RangeMetrics[i].Requests503)\n\t\t\t\t\tmetrics.Register(scope+\".\"+c.RangeNames[i]+\".requests_status_code.504\", requestMetric.RangeMetrics[i].Requests504)\n\t\t\t\t\tmetrics.Register(scope+\".\"+c.RangeNames[i]+\".requests_status_code.5xx\", requestMetric.RangeMetrics[i].Requests5xx)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t} else {\n\t\trequestMetric.RequestsH = metrics.NilHistogram{}\n\t\trequestMetric.FinderH = metrics.NilHistogram{}\n\t}\n\n\treturn requestMetric\n}\n\nfunc SendFindMetrics(r *FindMetrics, statusCode int, durationMs, untilFromS int64, extended bool, metricsCount int64) {\n\tfromPos := -1\n\tif len(r.RangeS) > 0 {\n\t\tfromPos = metrics.SearchInt64Le(r.RangeS, untilFromS)\n\t}\n\n\tr.RequestsH.Add(durationMs)\n\n\tif fromPos >= 0 {\n\t\tr.RangeMetrics[fromPos].RequestsH.Add(durationMs)\n\t}\n\n\tswitch statusCode {\n\tcase 200:\n\t\tif extended {\n\t\t\tr.Requests200.Add(1)\n\t\t\tGstatsd.Timing(r.MetricsCountName, metricsCount, 1.0)\n\n\t\t\tif fromPos >= 0 {\n\t\t\t\tr.RangeMetrics[fromPos].Requests200.Add(1)\n\t\t\t\tGstatsd.Timing(r.RangeMetrics[fromPos].MetricsCountName, metricsCount, 1.0)\n\t\t\t}\n\t\t}\n\tcase 400:\n\t\tr.Errors.Add(1)\n\n\t\tif extended {\n\t\t\tr.Requests400.Add(1)\n\n\t\t\tif fromPos >= 0 {\n\t\t\t\tr.RangeMetrics[fromPos].Requests400.Add(1)\n\t\t\t\tr.RangeMetrics[fromPos].Errors.Add(1)\n\t\t\t}\n\t\t}\n\tcase 403:\n\t\tr.Errors.Add(1)\n\n\t\tif extended {\n\t\t\tr.Requests403.Add(1)\n\n\t\t\tif fromPos >= 0 {\n\t\t\t\tr.RangeMetrics[fromPos].Requests403.Add(1)\n\t\t\t\tr.RangeMetrics[fromPos].Errors.Add(1)\n\t\t\t}\n\t\t}\n\tcase 404:\n\t\tif extended {\n\t\t\tr.Requests404.Add(1)\n\t\t\tGstatsd.Timing(r.MetricsCountName, metricsCount, 1.0)\n\n\t\t\tif fromPos >= 0 {\n\t\t\t\tr.RangeMetrics[fromPos].Requests404.Add(1)\n\t\t\t\tGstatsd.Timing(r.RangeMetrics[fromPos].MetricsCountName, metricsCount, 1.0)\n\t\t\t}\n\t\t}\n\tcase 500:\n\t\tr.Errors.Add(1)\n\n\t\tif extended {\n\t\t\tr.Requests500.Add(1)\n\n\t\t\tif fromPos >= 0 {\n\t\t\t\tr.RangeMetrics[fromPos].Requests500.Add(1)\n\t\t\t\tr.RangeMetrics[fromPos].Errors.Add(1)\n\t\t\t}\n\t\t}\n\tcase 503:\n\t\tr.Errors.Add(1)\n\n\t\tif extended {\n\t\t\tr.Requests503.Add(1)\n\n\t\t\tif fromPos >= 0 {\n\t\t\t\tr.RangeMetrics[fromPos].Requests503.Add(1)\n\t\t\t\tr.RangeMetrics[fromPos].Errors.Add(1)\n\t\t\t}\n\t\t}\n\tcase 504:\n\t\tr.Errors.Add(1)\n\n\t\tif extended {\n\t\t\tr.Requests504.Add(1)\n\n\t\t\tif fromPos >= 0 {\n\t\t\t\tr.RangeMetrics[fromPos].Requests504.Add(1)\n\t\t\t\tr.RangeMetrics[fromPos].Errors.Add(1)\n\t\t\t}\n\t\t}\n\tdefault:\n\t\tif extended {\n\t\t\tif statusCode > 500 {\n\t\t\t\tr.Requests5xx.Add(1)\n\n\t\t\t\tif fromPos >= 0 {\n\t\t\t\t\tr.RangeMetrics[fromPos].Requests5xx.Add(1)\n\t\t\t\t\tr.RangeMetrics[fromPos].Errors.Add(1)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tr.Requests4xx.Add(1)\n\n\t\t\t\tif fromPos >= 0 {\n\t\t\t\t\tr.RangeMetrics[fromPos].Requests4xx.Add(1)\n\t\t\t\t\tr.RangeMetrics[fromPos].Errors.Add(1)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tr.Errors.Add(1)\n\t}\n}\n\nfunc SendRenderMetrics(r *RenderMetrics, statusCode int, start, fetch, end time.Time, untilFromS int64, extended bool, metricsCount, points int64) {\n\tfromPos := -1\n\tif len(r.RangeS) > 0 {\n\t\tfromPos = metrics.SearchInt64Le(r.RangeS, untilFromS)\n\t}\n\n\tstartMs := start.UnixMilli()\n\tendMs := end.UnixMilli()\n\n\tvar (\n\t\tdurFinderMs int64\n\t\tdurFetchMs int64\n\t)\n\n\tdurMs := endMs - startMs\n\tif fetch.IsZero() {\n\t\tdurFinderMs = durMs\n\t} else {\n\t\tfetchMs := fetch.UnixMilli()\n\t\tdurFinderMs = fetchMs - startMs\n\t\tdurFetchMs = endMs - fetchMs\n\t}\n\n\tr.RequestsH.Add(durMs)\n\tr.FinderH.Add(durFinderMs)\n\n\tif fromPos >= 0 {\n\t\tr.RangeMetrics[fromPos].RequestsH.Add(durMs)\n\t\tr.RangeMetrics[fromPos].FinderH.Add(durFinderMs)\n\t}\n\n\tswitch statusCode {\n\tcase 200:\n\t\tif extended {\n\t\t\tr.Requests200.Add(1)\n\t\t\tGstatsd.Timing(r.MetricsCountName, metricsCount, 1.0)\n\t\t\tGstatsd.Timing(r.PointsCountName, points, 1.0)\n\n\t\t\tif fromPos >= 0 {\n\t\t\t\tr.RangeMetrics[fromPos].Requests200.Add(1)\n\t\t\t\tGstatsd.Timing(r.RangeMetrics[fromPos].MetricsCountName, metricsCount, 1.0)\n\n\t\t\t\tif durFetchMs > 0 {\n\t\t\t\t\tGstatsd.Timing(r.RangeMetrics[fromPos].PointsCountName, points, 1.0)\n\t\t\t\t}\n\n\t\t\t\tr.RangeMetrics[fromPos].FinderH.Add(durFinderMs)\n\t\t\t}\n\t\t}\n\tcase 400:\n\t\tr.Errors.Add(1)\n\n\t\tif extended {\n\t\t\tr.Requests400.Add(1)\n\n\t\t\tif fromPos >= 0 {\n\t\t\t\tr.RangeMetrics[fromPos].Requests400.Add(1)\n\t\t\t\tr.RangeMetrics[fromPos].Errors.Add(1)\n\t\t\t}\n\t\t}\n\tcase 403:\n\t\tr.Errors.Add(1)\n\n\t\tif extended {\n\t\t\tr.Requests403.Add(1)\n\n\t\t\tif fromPos >= 0 {\n\t\t\t\tr.RangeMetrics[fromPos].Requests403.Add(1)\n\t\t\t\tr.RangeMetrics[fromPos].Errors.Add(1)\n\t\t\t}\n\t\t}\n\tcase 404:\n\t\tif extended {\n\t\t\tr.Requests404.Add(1)\n\t\t\tGstatsd.Timing(r.MetricsCountName, metricsCount, 1.0)\n\n\t\t\tif fromPos >= 0 {\n\t\t\t\tr.RangeMetrics[fromPos].Requests404.Add(1)\n\t\t\t\tGstatsd.Timing(r.RangeMetrics[fromPos].MetricsCountName, metricsCount, 1.0)\n\t\t\t\tGstatsd.Timing(r.RangeMetrics[fromPos].PointsCountName, points, 1.0)\n\n\t\t\t\tif durFetchMs > 0 {\n\t\t\t\t\tGstatsd.Timing(r.RangeMetrics[fromPos].PointsCountName, points, 1.0)\n\t\t\t\t}\n\n\t\t\t\tr.RangeMetrics[fromPos].FinderH.Add(durFinderMs)\n\t\t\t}\n\t\t}\n\tcase 500:\n\t\tr.Errors.Add(1)\n\n\t\tif extended {\n\t\t\tr.Requests500.Add(1)\n\n\t\t\tif fromPos >= 0 {\n\t\t\t\tr.RangeMetrics[fromPos].Requests500.Add(1)\n\t\t\t\tr.RangeMetrics[fromPos].Errors.Add(1)\n\t\t\t}\n\t\t}\n\tcase 503:\n\t\tr.Errors.Add(1)\n\n\t\tif extended {\n\t\t\tr.Requests503.Add(1)\n\n\t\t\tif fromPos >= 0 {\n\t\t\t\tr.RangeMetrics[fromPos].Requests503.Add(1)\n\t\t\t\tr.RangeMetrics[fromPos].Errors.Add(1)\n\t\t\t}\n\t\t}\n\tcase 504:\n\t\tr.Errors.Add(1)\n\n\t\tif extended {\n\t\t\tr.Requests504.Add(1)\n\n\t\t\tif fromPos >= 0 {\n\t\t\t\tr.RangeMetrics[fromPos].Requests504.Add(1)\n\t\t\t\tr.RangeMetrics[fromPos].Errors.Add(1)\n\t\t\t}\n\t\t}\n\tdefault:\n\t\tif extended {\n\t\t\tif statusCode > 500 {\n\t\t\t\tr.Requests5xx.Add(1)\n\n\t\t\t\tif fromPos >= 0 {\n\t\t\t\t\tr.RangeMetrics[fromPos].Requests5xx.Add(1)\n\t\t\t\t\tr.RangeMetrics[fromPos].Errors.Add(1)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tr.Requests4xx.Add(1)\n\n\t\t\t\tif fromPos >= 0 {\n\t\t\t\t\tr.RangeMetrics[fromPos].Requests4xx.Add(1)\n\t\t\t\t\tr.RangeMetrics[fromPos].Errors.Add(1)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tr.Errors.Add(1)\n\t}\n}\n\ntype rangeName struct {\n\tname string\n\tv int64\n}\n\nfunc InitMetrics(c *Config, findWaitQueue, tagsWaitQueue bool) {\n\tif c != nil && Graphite != nil {\n\t\tmetrics.RegisterRuntimeMemStats(nil)\n\t\tgo metrics.CaptureRuntimeMemStats(c.MetricInterval)\n\n\t\tif len(c.BucketsWidth) == 0 {\n\t\t\tc.BucketsWidth = []int64{200, 500, 1000, 2000, 3000, 5000, 7000, 10000, 15000, 20000, 25000, 30000, 40000, 50000, 60000}\n\t\t}\n\n\t\tlabels := make([]string, len(c.BucketsWidth)+1)\n\n\t\tfor i := 0; i <= len(c.BucketsWidth); i++ {\n\t\t\tif i >= len(c.BucketsLabels) || c.BucketsLabels[i] == \"\" {\n\t\t\t\tif i < len(c.BucketsWidth) {\n\t\t\t\t\tlabels[i] = fmt.Sprintf(\"_to_%dms\", c.BucketsWidth[i])\n\t\t\t\t} else {\n\t\t\t\t\tlabels[i] = \"_to_inf\"\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tlabels[i] = c.BucketsLabels[i]\n\t\t\t}\n\t\t}\n\n\t\tc.BucketsLabels = labels\n\n\t\tif len(c.Ranges) > 0 {\n\t\t\t// c.RangeS = make([]int64, 0, len(c.Range)+1)\n\t\t\tuntilFrom := make([]rangeName, 0, len(c.Ranges)+1)\n\n\t\t\tfor name, v := range c.Ranges {\n\t\t\t\tif v <= 0 {\n\t\t\t\t\tuntilFrom = append(untilFrom, rangeName{name: name, v: math.MaxInt64})\n\t\t\t\t} else {\n\t\t\t\t\tuntilFrom = append(untilFrom, rangeName{name: name, v: int64(v.Seconds())})\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tsort.Slice(untilFrom, func(i, j int) bool {\n\t\t\t\treturn untilFrom[i].v < untilFrom[j].v\n\t\t\t})\n\n\t\t\tif untilFrom[len(untilFrom)-1].v != math.MaxInt64 {\n\t\t\t\tuntilFrom = append(untilFrom, rangeName{name: \"history\", v: math.MaxInt64})\n\t\t\t}\n\n\t\t\tc.RangeS = make([]int64, len(untilFrom))\n\t\t\tc.RangeNames = make([]string, len(untilFrom))\n\n\t\t\tfor i := range untilFrom {\n\t\t\t\tc.RangeNames[i] = untilFrom[i].name\n\t\t\t\tc.RangeS[i] = untilFrom[i].v\n\t\t\t}\n\t\t}\n\n\t\tif len(c.FindRanges) > 0 {\n\t\t\t// c.RangeS = make([]int64, 0, len(c.Range)+1)\n\t\t\tuntilFrom := make([]rangeName, 0, len(c.Ranges)+1)\n\n\t\t\tfor name, v := range c.FindRanges {\n\t\t\t\tif v <= 0 {\n\t\t\t\t\tuntilFrom = append(untilFrom, rangeName{name: name, v: math.MaxInt64})\n\t\t\t\t} else {\n\t\t\t\t\tuntilFrom = append(untilFrom, rangeName{name: name, v: int64(v.Seconds())})\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tsort.Slice(untilFrom, func(i, j int) bool {\n\t\t\t\treturn untilFrom[i].v < untilFrom[j].v\n\t\t\t})\n\n\t\t\tif untilFrom[len(untilFrom)-1].v != math.MaxInt64 {\n\t\t\t\tuntilFrom = append(untilFrom, rangeName{name: \"history\", v: math.MaxInt64})\n\t\t\t}\n\n\t\t\tc.FindRangeS = make([]int64, len(untilFrom))\n\t\t\tc.FindRangeNames = make([]string, len(untilFrom))\n\n\t\t\tfor i := range untilFrom {\n\t\t\t\tc.FindRangeNames[i] = untilFrom[i].name\n\t\t\t\tc.FindRangeS[i] = untilFrom[i].v\n\t\t\t}\n\t\t}\n\t}\n\n\tinitFindCacheMetrics(c)\n\tFindRequestMetric = initFindMetrics(\"find\", c, findWaitQueue)\n\tTagsRequestMetric = initFindMetrics(\"tags\", c, tagsWaitQueue)\n\tRenderRequestMetric = initRenderMetrics(\"render\", c)\n}\n\nfunc DisableMetrics() {\n\tmetrics.UseNilMetrics = true\n\n\tInitMetrics(nil, false, false)\n}\n\nfunc UnregisterAll() {\n\tmetrics.DefaultRegistry.UnregisterAll()\n}\n" }, { "path": "metrics/metrics_test.go", "content": "package metrics\n\nimport (\n\t\"math\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/msaf1980/go-metrics\"\n\t\"github.com/msaf1980/go-metrics/graphite\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc max(a, b int) int {\n\tif a > b {\n\t\treturn a\n\t}\n\n\treturn b\n}\n\nfunc compareInterface(t *testing.T, name string, i interface{}, notNil bool) {\n\tm := metrics.Get(name)\n\tif notNil {\n\t\tassert.Truef(t, i == m, name+\"\\nwant\\n%+v\\ngot\\n%+v\", i, m)\n\t} else {\n\t\tassert.Nilf(t, m, name)\n\t}\n}\n\nfunc TestInitMetrics(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tc Config\n\t\tfindWaitQueue bool\n\t\ttagsWaitQueue bool\n\t\twant Config\n\t\twantFindCountName string\n\t\twantFindRangesMetricsCountNames []string\n\t\twantRenderMetricsCountName string\n\t\twantRenderRangesMetricsCountNames []string\n\t\twantRenderPointsCountName string\n\t\twantRenderRangesPointsCountNames []string\n\t}{\n\t\t{\n\t\t\tname: \"labels (all)\",\n\t\t\tc: Config{\n\t\t\t\tMetricEndpoint: \"127.0.0.1:2003\",\n\t\t\t\tMetricInterval: 10 * time.Second,\n\t\t\t\tMetricTimeout: time.Second,\n\t\t\t\tMetricPrefix: \"graphite\",\n\t\t\t\tBucketsWidth: []int64{200, 500, 1000, 2000, 3000},\n\t\t\t\tBucketsLabels: []string{\n\t\t\t\t\t\"_to_200ms\",\n\t\t\t\t\t\"_to_500ms\",\n\t\t\t\t\t\"_to_1000ms\",\n\t\t\t\t\t\"_to_2000ms\",\n\t\t\t\t\t\"_to_3000ms\",\n\t\t\t\t\t\"_to_last\",\n\t\t\t\t},\n\t\t\t},\n\t\t\twant: Config{\n\t\t\t\tMetricEndpoint: \"127.0.0.1:2003\",\n\t\t\t\tMetricInterval: 10 * time.Second,\n\t\t\t\tMetricTimeout: time.Second,\n\t\t\t\tMetricPrefix: \"graphite\",\n\t\t\t\tBucketsWidth: []int64{200, 500, 1000, 2000, 3000},\n\t\t\t\tBucketsLabels: []string{\n\t\t\t\t\t\"_to_200ms\",\n\t\t\t\t\t\"_to_500ms\",\n\t\t\t\t\t\"_to_1000ms\",\n\t\t\t\t\t\"_to_2000ms\",\n\t\t\t\t\t\"_to_3000ms\",\n\t\t\t\t\t\"_to_last\",\n\t\t\t\t},\n\t\t\t},\n\t\t\twantFindCountName: \"find.all.metrics\",\n\t\t\twantRenderMetricsCountName: \"render.all.metrics\",\n\t\t\twantRenderPointsCountName: \"render.all.points\",\n\t\t},\n\t\t{\n\t\t\tname: \"labels (part)\",\n\t\t\tc: Config{\n\t\t\t\tMetricEndpoint: \"127.0.0.1:2003\",\n\t\t\t\tMetricInterval: 10 * time.Second,\n\t\t\t\tMetricTimeout: time.Second,\n\t\t\t\tMetricPrefix: \"graphite\",\n\t\t\t\tBucketsWidth: []int64{200, 500, 1000, 2000, 3000},\n\t\t\t\tBucketsLabels: []string{\n\t\t\t\t\t\"_to_200ms\",\n\t\t\t\t\t\"_to_500ms\",\n\t\t\t\t\t\"_to_1000ms\",\n\t\t\t\t\t\"_to_2000ms\",\n\t\t\t\t\t\"_to_3000ms\",\n\t\t\t\t},\n\t\t\t},\n\t\t\twant: Config{\n\t\t\t\tMetricEndpoint: \"127.0.0.1:2003\",\n\t\t\t\tMetricInterval: 10 * time.Second,\n\t\t\t\tMetricTimeout: time.Second,\n\t\t\t\tMetricPrefix: \"graphite\",\n\t\t\t\tBucketsWidth: []int64{200, 500, 1000, 2000, 3000},\n\t\t\t\tBucketsLabels: []string{\n\t\t\t\t\t\"_to_200ms\",\n\t\t\t\t\t\"_to_500ms\",\n\t\t\t\t\t\"_to_1000ms\",\n\t\t\t\t\t\"_to_2000ms\",\n\t\t\t\t\t\"_to_3000ms\",\n\t\t\t\t\t\"_to_inf\",\n\t\t\t\t},\n\t\t\t},\n\t\t\twantFindCountName: \"find.all.metrics\",\n\t\t\twantRenderMetricsCountName: \"render.all.metrics\",\n\t\t\twantRenderPointsCountName: \"render.all.points\",\n\t\t},\n\t\t{\n\t\t\tname: \"labels (default)\",\n\t\t\tc: Config{\n\t\t\t\tMetricEndpoint: \"127.0.0.1:2003\",\n\t\t\t\tMetricInterval: 10 * time.Second,\n\t\t\t\tMetricTimeout: time.Second,\n\t\t\t\tMetricPrefix: \"graphite\",\n\t\t\t\tRanges: map[string]time.Duration{\n\t\t\t\t\t\"1h\": time.Hour,\n\t\t\t\t\t\"3d\": 72 * time.Hour,\n\t\t\t\t\t\"7d\": 168 * time.Hour,\n\t\t\t\t\t\"30d\": 720 * time.Hour,\n\t\t\t\t\t\"90d\": 2160 * time.Hour,\n\t\t\t\t},\n\t\t\t},\n\t\t\twant: Config{\n\t\t\t\tMetricEndpoint: \"127.0.0.1:2003\",\n\t\t\t\tMetricInterval: 10 * time.Second,\n\t\t\t\tMetricTimeout: time.Second,\n\t\t\t\tMetricPrefix: \"graphite\",\n\t\t\t\tBucketsWidth: []int64{200, 500, 1000, 2000, 3000, 5000, 7000, 10000, 15000, 20000, 25000, 30000, 40000, 50000, 60000},\n\t\t\t\tBucketsLabels: []string{\n\t\t\t\t\t\"_to_200ms\",\n\t\t\t\t\t\"_to_500ms\",\n\t\t\t\t\t\"_to_1000ms\",\n\t\t\t\t\t\"_to_2000ms\",\n\t\t\t\t\t\"_to_3000ms\",\n\t\t\t\t\t\"_to_5000ms\",\n\t\t\t\t\t\"_to_7000ms\",\n\t\t\t\t\t\"_to_10000ms\",\n\t\t\t\t\t\"_to_15000ms\",\n\t\t\t\t\t\"_to_20000ms\",\n\t\t\t\t\t\"_to_25000ms\",\n\t\t\t\t\t\"_to_30000ms\",\n\t\t\t\t\t\"_to_40000ms\",\n\t\t\t\t\t\"_to_50000ms\",\n\t\t\t\t\t\"_to_60000ms\",\n\t\t\t\t\t\"_to_inf\",\n\t\t\t\t},\n\t\t\t\t// until-from = { \"1h\" = \"1h\", \"3d\" = \"72h\", \"7d\" = \"168h\", \"30d\" = \"720h\", \"90d\" = \"2160h\" }\n\t\t\t\tRanges: map[string]time.Duration{\n\t\t\t\t\t\"1h\": time.Hour,\n\t\t\t\t\t\"3d\": 72 * time.Hour,\n\t\t\t\t\t\"7d\": 168 * time.Hour,\n\t\t\t\t\t\"30d\": 720 * time.Hour,\n\t\t\t\t\t\"90d\": 2160 * time.Hour,\n\t\t\t\t},\n\t\t\t\tRangeNames: []string{\"1h\", \"3d\", \"7d\", \"30d\", \"90d\", \"history\"},\n\t\t\t\tRangeS: []int64{3600, 259200, 604800, 2592000, 7776000, math.MaxInt64},\n\t\t\t},\n\t\t\twantFindCountName: \"find.all.metrics\",\n\t\t\twantRenderMetricsCountName: \"render.all.metrics\",\n\t\t\twantRenderRangesMetricsCountNames: []string{\n\t\t\t\t\"render.1h.metrics\",\n\t\t\t\t\"render.3d.metrics\",\n\t\t\t\t\"render.7d.metrics\",\n\t\t\t\t\"render.30d.metrics\",\n\t\t\t\t\"render.90d.metrics\",\n\t\t\t\t\"render.history.metrics\",\n\t\t\t},\n\t\t\twantRenderPointsCountName: \"render.all.points\",\n\t\t\twantRenderRangesPointsCountNames: []string{\n\t\t\t\t\"render.1h.points\",\n\t\t\t\t\"render.3d.points\",\n\t\t\t\t\"render.7d.points\",\n\t\t\t\t\"render.30d.points\",\n\t\t\t\t\"render.90d.points\",\n\t\t\t\t\"render.history.points\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"ranges\",\n\t\t\tc: Config{\n\t\t\t\tMetricEndpoint: \"127.0.0.1:2003\",\n\t\t\t\tMetricInterval: 10 * time.Second,\n\t\t\t\tMetricTimeout: time.Second,\n\t\t\t\tMetricPrefix: \"graphite\",\n\t\t\t\tRanges: map[string]time.Duration{\n\t\t\t\t\t\"1h\": time.Hour,\n\t\t\t\t\t\"3d\": 72 * time.Hour,\n\t\t\t\t\t\"7d\": 168 * time.Hour,\n\t\t\t\t\t\"30d\": 720 * time.Hour,\n\t\t\t\t\t\"90d\": 2160 * time.Hour,\n\t\t\t\t\t\"last\": 0,\n\t\t\t\t},\n\t\t\t},\n\t\t\twant: Config{\n\t\t\t\tMetricEndpoint: \"127.0.0.1:2003\",\n\t\t\t\tMetricInterval: 10 * time.Second,\n\t\t\t\tMetricTimeout: time.Second,\n\t\t\t\tMetricPrefix: \"graphite\",\n\t\t\t\tBucketsWidth: []int64{200, 500, 1000, 2000, 3000, 5000, 7000, 10000, 15000, 20000, 25000, 30000, 40000, 50000, 60000},\n\t\t\t\tBucketsLabels: []string{\n\t\t\t\t\t\"_to_200ms\",\n\t\t\t\t\t\"_to_500ms\",\n\t\t\t\t\t\"_to_1000ms\",\n\t\t\t\t\t\"_to_2000ms\",\n\t\t\t\t\t\"_to_3000ms\",\n\t\t\t\t\t\"_to_5000ms\",\n\t\t\t\t\t\"_to_7000ms\",\n\t\t\t\t\t\"_to_10000ms\",\n\t\t\t\t\t\"_to_15000ms\",\n\t\t\t\t\t\"_to_20000ms\",\n\t\t\t\t\t\"_to_25000ms\",\n\t\t\t\t\t\"_to_30000ms\",\n\t\t\t\t\t\"_to_40000ms\",\n\t\t\t\t\t\"_to_50000ms\",\n\t\t\t\t\t\"_to_60000ms\",\n\t\t\t\t\t\"_to_inf\",\n\t\t\t\t},\n\t\t\t\t// until-from = { \"1h\" = \"1h\", \"3d\" = \"72h\", \"7d\" = \"168h\", \"30d\" = \"720h\", \"90d\" = \"2160h\" }\n\t\t\t\tRanges: map[string]time.Duration{\n\t\t\t\t\t\"1h\": time.Hour,\n\t\t\t\t\t\"3d\": 72 * time.Hour,\n\t\t\t\t\t\"7d\": 168 * time.Hour,\n\t\t\t\t\t\"30d\": 720 * time.Hour,\n\t\t\t\t\t\"90d\": 2160 * time.Hour,\n\t\t\t\t\t\"last\": 0,\n\t\t\t\t},\n\t\t\t\tRangeNames: []string{\"1h\", \"3d\", \"7d\", \"30d\", \"90d\", \"last\"},\n\t\t\t\tRangeS: []int64{3600, 259200, 604800, 2592000, 7776000, math.MaxInt64},\n\t\t\t},\n\t\t\twantFindCountName: \"find.all.metrics\",\n\t\t\twantRenderMetricsCountName: \"render.all.metrics\",\n\t\t\twantRenderRangesMetricsCountNames: []string{\n\t\t\t\t\"render.1h.metrics\",\n\t\t\t\t\"render.3d.metrics\",\n\t\t\t\t\"render.7d.metrics\",\n\t\t\t\t\"render.30d.metrics\",\n\t\t\t\t\"render.90d.metrics\",\n\t\t\t\t\"render.last.metrics\",\n\t\t\t},\n\t\t\twantRenderPointsCountName: \"render.all.points\",\n\t\t\twantRenderRangesPointsCountNames: []string{\n\t\t\t\t\"render.1h.points\",\n\t\t\t\t\"render.3d.points\",\n\t\t\t\t\"render.7d.points\",\n\t\t\t\t\"render.30d.points\",\n\t\t\t\t\"render.90d.points\",\n\t\t\t\t\"render.last.points\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"all\",\n\t\t\tc: Config{\n\t\t\t\tMetricEndpoint: \"127.0.0.1:2003\",\n\t\t\t\tStatsd: \"127.0.0.1:8125\",\n\t\t\t\tMetricInterval: 10 * time.Second,\n\t\t\t\tMetricTimeout: time.Second,\n\t\t\t\tMetricPrefix: \"graphite\",\n\t\t\t\tExtendedStat: true,\n\t\t\t\tRanges: map[string]time.Duration{\n\t\t\t\t\t\"1h\": time.Hour,\n\t\t\t\t\t\"3d\": 72 * time.Hour,\n\t\t\t\t\t\"7d\": 168 * time.Hour,\n\t\t\t\t\t\"30d\": 720 * time.Hour,\n\t\t\t\t\t\"90d\": 2160 * time.Hour,\n\t\t\t\t\t\"last\": 0,\n\t\t\t\t},\n\t\t\t},\n\t\t\twant: Config{\n\t\t\t\tMetricEndpoint: \"127.0.0.1:2003\",\n\t\t\t\tStatsd: \"127.0.0.1:8125\",\n\t\t\t\tMetricInterval: 10 * time.Second,\n\t\t\t\tMetricTimeout: time.Second,\n\t\t\t\tMetricPrefix: \"graphite\",\n\t\t\t\tExtendedStat: true,\n\t\t\t\tBucketsWidth: []int64{200, 500, 1000, 2000, 3000, 5000, 7000, 10000, 15000, 20000, 25000, 30000, 40000, 50000, 60000},\n\t\t\t\tBucketsLabels: []string{\n\t\t\t\t\t\"_to_200ms\",\n\t\t\t\t\t\"_to_500ms\",\n\t\t\t\t\t\"_to_1000ms\",\n\t\t\t\t\t\"_to_2000ms\",\n\t\t\t\t\t\"_to_3000ms\",\n\t\t\t\t\t\"_to_5000ms\",\n\t\t\t\t\t\"_to_7000ms\",\n\t\t\t\t\t\"_to_10000ms\",\n\t\t\t\t\t\"_to_15000ms\",\n\t\t\t\t\t\"_to_20000ms\",\n\t\t\t\t\t\"_to_25000ms\",\n\t\t\t\t\t\"_to_30000ms\",\n\t\t\t\t\t\"_to_40000ms\",\n\t\t\t\t\t\"_to_50000ms\",\n\t\t\t\t\t\"_to_60000ms\",\n\t\t\t\t\t\"_to_inf\",\n\t\t\t\t},\n\t\t\t\t// until-from = { \"1h\" = \"1h\", \"3d\" = \"72h\", \"7d\" = \"168h\", \"30d\" = \"720h\", \"90d\" = \"2160h\" }\n\t\t\t\tRanges: map[string]time.Duration{\n\t\t\t\t\t\"1h\": time.Hour,\n\t\t\t\t\t\"3d\": 72 * time.Hour,\n\t\t\t\t\t\"7d\": 168 * time.Hour,\n\t\t\t\t\t\"30d\": 720 * time.Hour,\n\t\t\t\t\t\"90d\": 2160 * time.Hour,\n\t\t\t\t\t\"last\": 0,\n\t\t\t\t},\n\t\t\t\tRangeNames: []string{\"1h\", \"3d\", \"7d\", \"30d\", \"90d\", \"last\"},\n\t\t\t\tRangeS: []int64{3600, 259200, 604800, 2592000, 7776000, math.MaxInt64},\n\t\t\t},\n\t\t\twantFindCountName: \"find.all.metrics\",\n\t\t\twantRenderMetricsCountName: \"render.all.metrics\",\n\t\t\twantRenderRangesMetricsCountNames: []string{\n\t\t\t\t\"render.1h.metrics\",\n\t\t\t\t\"render.3d.metrics\",\n\t\t\t\t\"render.7d.metrics\",\n\t\t\t\t\"render.30d.metrics\",\n\t\t\t\t\"render.90d.metrics\",\n\t\t\t\t\"render.last.metrics\",\n\t\t\t},\n\t\t\twantRenderPointsCountName: \"render.all.points\",\n\t\t\twantRenderRangesPointsCountNames: []string{\n\t\t\t\t\"render.1h.points\",\n\t\t\t\t\"render.3d.points\",\n\t\t\t\t\"render.7d.points\",\n\t\t\t\t\"render.30d.points\",\n\t\t\t\t\"render.90d.points\",\n\t\t\t\t\"render.last.points\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"all (with find-ranges)\",\n\t\t\tc: Config{\n\t\t\t\tMetricEndpoint: \"127.0.0.1:2003\",\n\t\t\t\tStatsd: \"127.0.0.1:8125\",\n\t\t\t\tMetricInterval: 10 * time.Second,\n\t\t\t\tMetricTimeout: time.Second,\n\t\t\t\tMetricPrefix: \"graphite\",\n\t\t\t\tExtendedStat: true,\n\t\t\t\tRanges: map[string]time.Duration{\n\t\t\t\t\t\"1h\": time.Hour,\n\t\t\t\t\t\"3d\": 72 * time.Hour,\n\t\t\t\t\t\"7d\": 168 * time.Hour,\n\t\t\t\t\t\"30d\": 720 * time.Hour,\n\t\t\t\t\t\"90d\": 2160 * time.Hour,\n\t\t\t\t\t\"last\": 0,\n\t\t\t\t},\n\t\t\t\tFindRanges: map[string]time.Duration{\n\t\t\t\t\t\"1h\": time.Hour,\n\t\t\t\t\t\"3d\": 72 * time.Hour,\n\t\t\t\t\t\"7d\": 168 * time.Hour,\n\t\t\t\t\t\"30d\": 720 * time.Hour,\n\t\t\t\t\t\"last\": 0,\n\t\t\t\t},\n\t\t\t\tFindRangeNames: []string{\"1h\", \"3d\", \"7d\", \"30d\", \"last\"},\n\t\t\t\tFindRangeS: []int64{3600, 259200, 604800, 2592000, math.MaxInt64},\n\t\t\t},\n\t\t\twant: Config{\n\t\t\t\tMetricEndpoint: \"127.0.0.1:2003\",\n\t\t\t\tStatsd: \"127.0.0.1:8125\",\n\t\t\t\tMetricInterval: 10 * time.Second,\n\t\t\t\tMetricTimeout: time.Second,\n\t\t\t\tMetricPrefix: \"graphite\",\n\t\t\t\tExtendedStat: true,\n\t\t\t\tBucketsWidth: []int64{200, 500, 1000, 2000, 3000, 5000, 7000, 10000, 15000, 20000, 25000, 30000, 40000, 50000, 60000},\n\t\t\t\tBucketsLabels: []string{\n\t\t\t\t\t\"_to_200ms\",\n\t\t\t\t\t\"_to_500ms\",\n\t\t\t\t\t\"_to_1000ms\",\n\t\t\t\t\t\"_to_2000ms\",\n\t\t\t\t\t\"_to_3000ms\",\n\t\t\t\t\t\"_to_5000ms\",\n\t\t\t\t\t\"_to_7000ms\",\n\t\t\t\t\t\"_to_10000ms\",\n\t\t\t\t\t\"_to_15000ms\",\n\t\t\t\t\t\"_to_20000ms\",\n\t\t\t\t\t\"_to_25000ms\",\n\t\t\t\t\t\"_to_30000ms\",\n\t\t\t\t\t\"_to_40000ms\",\n\t\t\t\t\t\"_to_50000ms\",\n\t\t\t\t\t\"_to_60000ms\",\n\t\t\t\t\t\"_to_inf\",\n\t\t\t\t},\n\t\t\t\t// until-from = { \"1h\" = \"1h\", \"3d\" = \"72h\", \"7d\" = \"168h\", \"30d\" = \"720h\", \"90d\" = \"2160h\" }\n\t\t\t\tRanges: map[string]time.Duration{\n\t\t\t\t\t\"1h\": time.Hour,\n\t\t\t\t\t\"3d\": 72 * time.Hour,\n\t\t\t\t\t\"7d\": 168 * time.Hour,\n\t\t\t\t\t\"30d\": 720 * time.Hour,\n\t\t\t\t\t\"90d\": 2160 * time.Hour,\n\t\t\t\t\t\"last\": 0,\n\t\t\t\t},\n\t\t\t\tRangeNames: []string{\"1h\", \"3d\", \"7d\", \"30d\", \"90d\", \"last\"},\n\t\t\t\tRangeS: []int64{3600, 259200, 604800, 2592000, 7776000, math.MaxInt64},\n\t\t\t\tFindRanges: map[string]time.Duration{\n\t\t\t\t\t\"1h\": time.Hour,\n\t\t\t\t\t\"3d\": 72 * time.Hour,\n\t\t\t\t\t\"7d\": 168 * time.Hour,\n\t\t\t\t\t\"30d\": 720 * time.Hour,\n\t\t\t\t\t\"last\": 0,\n\t\t\t\t},\n\t\t\t\tFindRangeNames: []string{\"1h\", \"3d\", \"7d\", \"30d\", \"last\"},\n\t\t\t\tFindRangeS: []int64{3600, 259200, 604800, 2592000, math.MaxInt64},\n\t\t\t},\n\t\t\twantFindCountName: \"find.all.metrics\",\n\t\t\twantFindRangesMetricsCountNames: []string{\n\t\t\t\t\"find.1h.metrics\",\n\t\t\t\t\"find.3d.metrics\",\n\t\t\t\t\"find.7d.metrics\",\n\t\t\t\t\"find.30d.metrics\",\n\t\t\t\t\"find.last.metrics\",\n\t\t\t},\n\t\t\twantRenderMetricsCountName: \"render.all.metrics\",\n\t\t\twantRenderRangesMetricsCountNames: []string{\n\t\t\t\t\"render.1h.metrics\",\n\t\t\t\t\"render.3d.metrics\",\n\t\t\t\t\"render.7d.metrics\",\n\t\t\t\t\"render.30d.metrics\",\n\t\t\t\t\"render.90d.metrics\",\n\t\t\t\t\"render.last.metrics\",\n\t\t\t},\n\t\t\twantRenderPointsCountName: \"render.all.points\",\n\t\t\twantRenderRangesPointsCountNames: []string{\n\t\t\t\t\"render.1h.points\",\n\t\t\t\t\"render.3d.points\",\n\t\t\t\t\"render.7d.points\",\n\t\t\t\t\"render.30d.points\",\n\t\t\t\t\"render.90d.points\",\n\t\t\t\t\"render.last.points\",\n\t\t\t},\n\t\t},\n\t}\n\tfor n, tt := range tests {\n\t\tt.Run(tt.name+\"#\"+strconv.Itoa(n), func(t *testing.T) {\n\t\t\tFindRequestMetric = nil\n\t\t\tTagsRequestMetric = nil\n\t\t\tRenderRequestMetric = nil\n\n\t\t\tUnregisterAll()\n\n\t\t\tc := tt.c\n\t\t\tGraphite = &graphite.Graphite{}\n\n\t\t\tInitMetrics(&c, tt.findWaitQueue, tt.tagsWaitQueue)\n\n\t\t\tGraphite = nil\n\n\t\t\tassert.Equal(t, tt.want, c)\n\t\t\t// FindRequestH\n\t\t\tcompareInterface(t, \"find.all.requests\", FindRequestMetric.RequestsH, true)\n\t\t\t// FindRequestCount\n\t\t\tcompareInterface(t, \"find.all.requests_status_code.200\", FindRequestMetric.Requests200, c.ExtendedStat)\n\t\t\tcompareInterface(t, \"find.all.requests_status_code.400\", FindRequestMetric.Requests400, c.ExtendedStat)\n\t\t\tcompareInterface(t, \"find.all.requests_status_code.403\", FindRequestMetric.Requests403, c.ExtendedStat)\n\t\t\tcompareInterface(t, \"find.all.requests_status_code.404\", FindRequestMetric.Requests404, c.ExtendedStat)\n\t\t\tcompareInterface(t, \"find.all.requests_status_code.4xx\", FindRequestMetric.Requests4xx, c.ExtendedStat)\n\t\t\tcompareInterface(t, \"find.all.requests_status_code.500\", FindRequestMetric.Requests500, c.ExtendedStat)\n\t\t\tcompareInterface(t, \"find.all.requests_status_code.503\", FindRequestMetric.Requests503, c.ExtendedStat)\n\t\t\tcompareInterface(t, \"find.all.requests_status_code.504\", FindRequestMetric.Requests504, c.ExtendedStat)\n\t\t\tcompareInterface(t, \"find.all.requests_status_code.5xx\", FindRequestMetric.Requests5xx, c.ExtendedStat)\n\t\t\t//FindRequestMetric\n\t\t\tassert.Equal(t, tt.wantFindCountName, FindRequestMetric.MetricsCountName)\n\n\t\t\tfor i := 0; i < max(len(c.FindRangeS), len(tt.wantFindRangesMetricsCountNames)); i++ {\n\t\t\t\tif i < len(c.FindRangeNames) {\n\t\t\t\t\t// FindRequestH\n\t\t\t\t\tcompareInterface(t, \"find.\"+c.FindRangeNames[i]+\".requests\", FindRequestMetric.RangeMetrics[i].RequestsH, true)\n\t\t\t\t\t// FindRequestCount\n\t\t\t\t\tcompareInterface(t, \"find.\"+c.FindRangeNames[i]+\".requests_status_code.200\", FindRequestMetric.RangeMetrics[i].Requests200, c.ExtendedStat)\n\t\t\t\t\tcompareInterface(t, \"find.\"+c.FindRangeNames[i]+\".requests_status_code.400\", FindRequestMetric.RangeMetrics[i].Requests400, c.ExtendedStat)\n\t\t\t\t\tcompareInterface(t, \"find.\"+c.FindRangeNames[i]+\".requests_status_code.403\", FindRequestMetric.RangeMetrics[i].Requests403, c.ExtendedStat)\n\t\t\t\t\tcompareInterface(t, \"find.\"+c.FindRangeNames[i]+\".requests_status_code.404\", FindRequestMetric.RangeMetrics[i].Requests404, c.ExtendedStat)\n\t\t\t\t\tcompareInterface(t, \"find.\"+c.FindRangeNames[i]+\".requests_status_code.4xx\", FindRequestMetric.RangeMetrics[i].Requests4xx, c.ExtendedStat)\n\t\t\t\t\tcompareInterface(t, \"find.\"+c.FindRangeNames[i]+\".requests_status_code.500\", FindRequestMetric.RangeMetrics[i].Requests500, c.ExtendedStat)\n\t\t\t\t\tcompareInterface(t, \"find.\"+c.FindRangeNames[i]+\".requests_status_code.503\", FindRequestMetric.RangeMetrics[i].Requests503, c.ExtendedStat)\n\t\t\t\t\tcompareInterface(t, \"find.\"+c.FindRangeNames[i]+\".requests_status_code.504\", FindRequestMetric.RangeMetrics[i].Requests504, c.ExtendedStat)\n\t\t\t\t\tcompareInterface(t, \"find.\"+c.FindRangeNames[i]+\".requests_status_code.5xx\", FindRequestMetric.RangeMetrics[i].Requests5xx, c.ExtendedStat)\n\t\t\t\t}\n\n\t\t\t\tvar want, got string\n\t\t\t\tif i < len(tt.wantFindRangesMetricsCountNames) {\n\t\t\t\t\twant = tt.wantFindRangesMetricsCountNames[i]\n\t\t\t\t}\n\n\t\t\t\tif i < len(FindRequestMetric.RangeMetrics) {\n\t\t\t\t\tgot = FindRequestMetric.RangeMetrics[i].MetricsCountName\n\t\t\t\t}\n\n\t\t\t\tassert.Equal(t, want, got)\n\t\t\t}\n\n\t\t\tassert.Equal(t, tt.want.FindRangeS, FindRequestMetric.RangeS)\n\t\t\tassert.Equal(t, tt.want.FindRangeNames, FindRequestMetric.RangeNames)\n\t\t\tassert.Equalf(t, len(tt.want.FindRangeS), len(FindRequestMetric.RangeMetrics), \"FindRequestMetric.RangeMetrics\")\n\t\t\t// RenderRequestH\n\t\t\tcompareInterface(t, \"render.all.requests\", RenderRequestMetric.RequestsH, true)\n\t\t\tcompareInterface(t, \"render.all.requests_finder\", RenderRequestMetric.FinderH, true)\n\t\t\t// RenderRequestCount\n\t\t\tcompareInterface(t, \"render.all.requests_status_code.200\", RenderRequestMetric.Requests200, c.ExtendedStat)\n\t\t\tcompareInterface(t, \"render.all.requests_status_code.400\", RenderRequestMetric.Requests400, c.ExtendedStat)\n\t\t\tcompareInterface(t, \"render.all.requests_status_code.403\", RenderRequestMetric.Requests403, c.ExtendedStat)\n\t\t\tcompareInterface(t, \"render.all.requests_status_code.404\", RenderRequestMetric.Requests404, c.ExtendedStat)\n\t\t\tcompareInterface(t, \"render.all.requests_status_code.4xx\", RenderRequestMetric.Requests4xx, c.ExtendedStat)\n\t\t\tcompareInterface(t, \"render.all.requests_status_code.500\", RenderRequestMetric.Requests500, c.ExtendedStat)\n\t\t\tcompareInterface(t, \"render.all.requests_status_code.503\", RenderRequestMetric.Requests503, c.ExtendedStat)\n\t\t\tcompareInterface(t, \"render.all.requests_status_code.504\", RenderRequestMetric.Requests504, c.ExtendedStat)\n\t\t\tcompareInterface(t, \"render.all.requests_status_code.5xx\", RenderRequestMetric.Requests5xx, c.ExtendedStat)\n\t\t\t// RenderRequestMetric\n\t\t\tassert.Equal(t, tt.wantRenderMetricsCountName, RenderRequestMetric.MetricsCountName)\n\t\t\tassert.Equal(t, tt.wantRenderPointsCountName, RenderRequestMetric.PointsCountName)\n\n\t\t\tfor i := 0; i < max(len(c.RangeS), len(tt.wantRenderRangesMetricsCountNames)); i++ {\n\t\t\t\tvar want, got string\n\n\t\t\t\tif i < len(c.RangeNames) {\n\t\t\t\t\t// FindRequestH\n\t\t\t\t\tcompareInterface(t, \"render.\"+c.RangeNames[i]+\".requests\", RenderRequestMetric.RangeMetrics[i].RequestsH, true)\n\t\t\t\t\tcompareInterface(t, \"render.\"+c.RangeNames[i]+\".requests_finder\", RenderRequestMetric.RangeMetrics[i].FinderH, true)\n\t\t\t\t\t// FindRequestCount\n\t\t\t\t\tcompareInterface(t, \"render.\"+c.RangeNames[i]+\".requests_status_code.200\", RenderRequestMetric.RangeMetrics[i].Requests200, c.ExtendedStat)\n\t\t\t\t\tcompareInterface(t, \"render.\"+c.RangeNames[i]+\".requests_status_code.400\", RenderRequestMetric.RangeMetrics[i].Requests400, c.ExtendedStat)\n\t\t\t\t\tcompareInterface(t, \"render.\"+c.RangeNames[i]+\".requests_status_code.403\", RenderRequestMetric.RangeMetrics[i].Requests403, c.ExtendedStat)\n\t\t\t\t\tcompareInterface(t, \"render.\"+c.RangeNames[i]+\".requests_status_code.404\", RenderRequestMetric.RangeMetrics[i].Requests404, c.ExtendedStat)\n\t\t\t\t\tcompareInterface(t, \"render.\"+c.RangeNames[i]+\".requests_status_code.4xx\", RenderRequestMetric.RangeMetrics[i].Requests4xx, c.ExtendedStat)\n\t\t\t\t\tcompareInterface(t, \"render.\"+c.RangeNames[i]+\".requests_status_code.500\", RenderRequestMetric.RangeMetrics[i].Requests500, c.ExtendedStat)\n\t\t\t\t\tcompareInterface(t, \"render.\"+c.RangeNames[i]+\".requests_status_code.503\", RenderRequestMetric.RangeMetrics[i].Requests503, c.ExtendedStat)\n\t\t\t\t\tcompareInterface(t, \"render.\"+c.RangeNames[i]+\".requests_status_code.504\", RenderRequestMetric.RangeMetrics[i].Requests504, c.ExtendedStat)\n\t\t\t\t\tcompareInterface(t, \"render.\"+c.RangeNames[i]+\".requests_status_code.5xx\", RenderRequestMetric.RangeMetrics[i].Requests5xx, c.ExtendedStat)\n\t\t\t\t}\n\n\t\t\t\tif i < len(tt.wantRenderRangesMetricsCountNames) {\n\t\t\t\t\twant = tt.wantRenderRangesMetricsCountNames[i]\n\t\t\t\t}\n\n\t\t\t\tif i < len(RenderRequestMetric.RangeMetrics) {\n\t\t\t\t\tgot = RenderRequestMetric.RangeMetrics[i].MetricsCountName\n\t\t\t\t}\n\n\t\t\t\tassert.Equalf(t, want, got, strconv.Itoa(i))\n\n\t\t\t\tif i < len(tt.wantRenderRangesPointsCountNames) {\n\t\t\t\t\twant = tt.wantRenderRangesPointsCountNames[i]\n\t\t\t\t}\n\n\t\t\t\tif i < len(tt.wantRenderRangesPointsCountNames) {\n\t\t\t\t\tgot = RenderRequestMetric.RangeMetrics[i].PointsCountName\n\t\t\t\t}\n\n\t\t\t\tassert.Equalf(t, want, got, strconv.Itoa(i))\n\t\t\t}\n\n\t\t\tassert.Equal(t, tt.want.RangeS, RenderRequestMetric.RangeS)\n\t\t\tassert.Equal(t, tt.want.RangeNames, RenderRequestMetric.RangeNames)\n\t\t\tassert.Equalf(t, len(tt.want.RangeS), len(RenderRequestMetric.RangeMetrics), \"RenderRequestMetric.RangeMetrics\")\n\t\t\t// cleanup global vars\n\t\t\tFindRequestMetric = nil\n\t\t\tRenderRequestMetric = nil\n\n\t\t\tUnregisterAll()\n\t\t})\n\t}\n}\n" }, { "path": "metrics/query_metrics.go", "content": "package metrics\n\nimport \"github.com/msaf1980/go-metrics\"\n\ntype QueryMetric struct {\n\tRequestsH metrics.Histogram\n\tErrors metrics.Counter\n\tReadRowsName string\n\tReadBytesName string\n\tChReadRowsName string\n\tChReadBytesName string\n}\n\ntype QueryMetrics struct {\n\tQueryMetric\n\tRangeNames []string\n\tRangeS []int64\n\tRangeMetrics []QueryMetric\n}\n\ntype FinderStat struct {\n\tReadBytes int64\n\tChReadRows int64\n\tChReadBytes int64\n\tTable string\n}\n\nvar (\n\tQMetrics map[string]*QueryMetrics = make(map[string]*QueryMetrics)\n\tAutocompleteQMetric *QueryMetrics\n\tFindQMetric *QueryMetrics\n)\n\nfunc InitQueryMetrics(table string, c *Config) *QueryMetrics {\n\tif table == \"\" {\n\t\ttable = \"default\"\n\t}\n\n\tif q, exist := QMetrics[table]; exist {\n\t\treturn q\n\t}\n\n\tqueryMetric := &QueryMetrics{\n\t\tQueryMetric: QueryMetric{\n\t\t\tErrors: metrics.NewCounter(),\n\t\t\tReadRowsName: \"query.\" + table + \".all.read_rows\",\n\t\t\tReadBytesName: \"query.\" + table + \".all.read_bytes\",\n\t\t\tChReadRowsName: \"query.\" + table + \".all.ch_read_rows\",\n\t\t\tChReadBytesName: \"query.\" + table + \".all.ch_read_bytes\",\n\t\t},\n\t}\n\n\tif c != nil && Graphite != nil {\n\t\tqueryMetric.RequestsH = metrics.NewVSumHistogram(c.BucketsWidth, c.BucketsLabels).SetNameTotal(\"\")\n\t\tmetrics.Register(\"query.\"+table+\".all.requests\", queryMetric.RequestsH)\n\t\tmetrics.Register(\"query.\"+table+\".all.errors\", queryMetric.Errors)\n\n\t\tif len(c.RangeS) > 0 {\n\t\t\tqueryMetric.RangeS = c.RangeS\n\t\t\tqueryMetric.RangeNames = c.RangeNames\n\t\t\tqueryMetric.RangeMetrics = make([]QueryMetric, len(c.RangeS))\n\n\t\t\tfor i := range c.RangeS {\n\t\t\t\tqueryMetric.RangeMetrics[i].RequestsH = metrics.NewVSumHistogram(c.BucketsWidth, c.BucketsLabels).SetNameTotal(\"\")\n\t\t\t\tmetrics.Register(\"query.\"+table+\".\"+queryMetric.RangeNames[i]+\".requests\", queryMetric.RangeMetrics[i].RequestsH)\n\t\t\t\tqueryMetric.RangeMetrics[i].Errors = metrics.NewCounter()\n\t\t\t\tmetrics.Register(\"query.\"+table+\".\"+queryMetric.RangeNames[i]+\".errors\", queryMetric.RangeMetrics[i].Errors)\n\t\t\t\tqueryMetric.RangeMetrics[i].ReadRowsName = \"query.\" + table + \".\" + queryMetric.RangeNames[i] + \".read_rows\"\n\t\t\t\tqueryMetric.RangeMetrics[i].ReadBytesName = \"query.\" + table + \".\" + queryMetric.RangeNames[i] + \".read_bytes\"\n\t\t\t\tqueryMetric.RangeMetrics[i].ChReadRowsName = \"query.\" + table + \".\" + queryMetric.RangeNames[i] + \".ch_read_rows\"\n\t\t\t\tqueryMetric.RangeMetrics[i].ChReadBytesName = \"query.\" + table + \".\" + queryMetric.RangeNames[i] + \".ch_read_bytes\"\n\t\t\t}\n\t\t}\n\t} else {\n\t\tqueryMetric.RequestsH = metrics.NilHistogram{}\n\t}\n\n\tQMetrics[table] = queryMetric\n\n\treturn queryMetric\n}\n\nfunc SendQueryRead(r *QueryMetrics, from, until, durationMs, read_rows, read_bytes, ch_read_rows, ch_read_bytes int64, err bool) {\n\tr.RequestsH.Add(durationMs)\n\n\tif ch_read_rows > 0 {\n\t\tGstatsd.Timing(r.ChReadBytesName, ch_read_bytes, 1.0)\n\t\tGstatsd.Timing(r.ChReadRowsName, ch_read_rows, 1.0)\n\t}\n\n\tif err {\n\t\tr.Errors.Add(1)\n\t} else {\n\t\tGstatsd.Timing(r.ReadBytesName, read_bytes, 1.0)\n\t\tGstatsd.Timing(r.ReadRowsName, read_rows, 1.0)\n\t}\n\n\tif len(r.RangeS) > 0 {\n\t\tfromPos := metrics.SearchInt64Le(r.RangeS, until-from)\n\t\tr.RangeMetrics[fromPos].RequestsH.Add(durationMs)\n\n\t\tif ch_read_rows > 0 {\n\t\t\tGstatsd.Timing(r.RangeMetrics[fromPos].ChReadBytesName, ch_read_bytes, 1.0)\n\t\t\tGstatsd.Timing(r.RangeMetrics[fromPos].ChReadRowsName, ch_read_rows, 1.0)\n\t\t}\n\n\t\tif err {\n\t\t\tr.RangeMetrics[fromPos].Errors.Add(1)\n\t\t} else {\n\t\t\tGstatsd.Timing(r.RangeMetrics[fromPos].ReadBytesName, read_bytes, 1.0)\n\t\t\tGstatsd.Timing(r.RangeMetrics[fromPos].ReadRowsName, read_rows, 1.0)\n\t\t}\n\t}\n}\n\nfunc SendQueryReadChecked(r *QueryMetrics, from, until, durationMs, read_rows, read_bytes, ch_read_rows, ch_read_bytes int64, err bool) {\n\tif r != nil {\n\t\tSendQueryRead(r, from, until, durationMs, read_rows, read_bytes, ch_read_rows, ch_read_bytes, err)\n\t}\n}\n\nfunc SendQueryReadByTable(from, until, durationMs, read_rows int64, stats []FinderStat, err bool) {\n\tfor _, stat := range stats {\n\t\tif r, ok := QMetrics[stat.Table]; ok {\n\t\t\tSendQueryRead(r, from, until, durationMs, read_rows, stat.ReadBytes, stat.ChReadRows, stat.ChReadBytes, err)\n\t\t}\n\t}\n}\n" }, { "path": "metrics/statsd.go", "content": "package metrics\n\nimport (\n\t\"time\"\n\n\t\"github.com/cactus/go-statsd-client/v5/statsd\"\n)\n\n// NullSender is disabled sender (if stat need to be disabled)\ntype NullSender struct{}\n\nfunc (NullSender) Inc(string, int64, float32, ...statsd.Tag) error { return nil }\nfunc (NullSender) Dec(string, int64, float32, ...statsd.Tag) error { return nil }\nfunc (NullSender) Gauge(string, int64, float32, ...statsd.Tag) error { return nil }\nfunc (NullSender) GaugeDelta(string, int64, float32, ...statsd.Tag) error { return nil }\nfunc (NullSender) Timing(string, int64, float32, ...statsd.Tag) error { return nil }\nfunc (NullSender) TimingDuration(string, time.Duration, float32, ...statsd.Tag) error { return nil }\nfunc (NullSender) Set(string, string, float32, ...statsd.Tag) error { return nil }\nfunc (NullSender) SetInt(string, int64, float32, ...statsd.Tag) error { return nil }\nfunc (NullSender) Raw(string, string, float32, ...statsd.Tag) error { return nil }\nfunc (NullSender) NewSubStatter(string) statsd.SubStatter { return NullSender{} }\nfunc (NullSender) SetPrefix(string) {}\nfunc (NullSender) SetSamplerFunc(statsd.SamplerFunc) {}\nfunc (NullSender) Close() error { return nil }\n\nvar Gstatsd statsd.Statter = NullSender{}\n" }, { "path": "nfpm.yaml", "content": "---\nname: ${NAME}\ndescription: ${DESCRIPTION}\n\n# Common packages config\narch: \"${ARCH}\" # amd64, arm64\nplatform: \"linux\"\nversion: \"${VERSION_STRING}\"\nmaintainer: &m \"Roman Lomonosov \"\nvendor: *m\nhomepage: \"https://github.com/go-graphite/${NAME}\"\nlicense: \"MIT\"\nsection: \"admin\"\npriority: \"optional\"\n\ncontents:\n - src: deploy/root/usr/\n dst: /usr\n expand: true\n - src: deploy/root/etc/logrotate.d/${NAME}\n dst: /etc/logrotate.d/${NAME}\n type: config|noreplace\n expand: true\n - src: out/root/etc/${NAME}/${NAME}.conf\n dst: /etc/${NAME}/${NAME}.conf\n type: config|noreplace\n expand: true\n - src: \"out/${NAME}-linux-${ARCH}\"\n dst: /usr/bin/${NAME}\n expand: true\n # docs\n - src: LICENSE\n dst: /usr/share/doc/${NAME}/LICENSE\n expand: true\n" }, { "path": "packages.sh", "content": "#!/bin/sh -e\n\ncd \"$( dirname \"$0\" )\"\nROOT=$PWD\n\ndocker run -i -e \"DEVEL=${DEVEL:-0}\" --rm -v \"$ROOT:/root/go/src/github.com/lomik/graphite-clickhouse\" golang bash -e << 'EOF'\n cd /root/\n export TZ=Europe/Moscow\n ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone\n\n go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.40.0\n\n cd /root/go/src/github.com/lomik/graphite-clickhouse\n\n # go reads the VCS state\n git config --global --add safe.directory \"$PWD\"\n\n make nfpm-deb nfpm-rpm\n chmod -R a+w *.deb *.rpm out/\nEOF\n" }, { "path": "pkg/alias/map.go", "content": "package alias\n\nimport (\n\t\"bytes\"\n\t\"sync\"\n\n\t\"github.com/lomik/graphite-clickhouse/finder\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/reverse\"\n)\n\n// Value of Map\ntype Value struct {\n\tTarget string\n\tDisplayName string\n}\n\n// Map from real metric name to display name and target\ntype Map struct {\n\tdata map[string][]Value\n\tlock sync.RWMutex\n}\n\n// New returns new Map\nfunc New() *Map {\n\treturn &Map{\n\t\tdata: make(map[string][]Value),\n\t\tlock: sync.RWMutex{},\n\t}\n}\n\n// Merge data from finder.Result into aliases map\nfunc (m *Map) Merge(r finder.Result, useCache bool) {\n\tm.MergeTarget(r, \"\", useCache)\n}\n\n// MergeTarget data from finder.Result into aliases map\nfunc (m *Map) MergeTarget(r finder.Result, target string, saveCache bool) []byte {\n\tvar buf bytes.Buffer\n\n\tseries := r.Series()\n\tbuf.Grow(len(series) * 24)\n\n\tfor i := 0; i < len(series); i++ {\n\t\tif saveCache {\n\t\t\tbuf.Write(series[i])\n\t\t\tbuf.WriteByte('\\n')\n\t\t}\n\n\t\tkey := string(series[i])\n\t\tif len(key) == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tabs := string(r.Abs(series[i]))\n\n\t\tm.lock.Lock()\n\t\tif x, ok := m.data[key]; ok {\n\t\t\tm.data[key] = append(x, Value{Target: target, DisplayName: abs})\n\t\t} else {\n\t\t\tm.data[key] = []Value{{Target: target, DisplayName: abs}}\n\t\t}\n\t\tm.lock.Unlock()\n\t}\n\n\tif saveCache {\n\t\treturn buf.Bytes()\n\t} else {\n\t\treturn nil\n\t}\n}\n\n// Len returns count of keys\nfunc (m *Map) Len() int {\n\tm.lock.RLock()\n\tdefer m.lock.RUnlock()\n\n\treturn len(m.data)\n}\n\n// Size returns count of values\nfunc (m *Map) Size() int {\n\ts := 0\n\n\tm.lock.RLock()\n\tdefer m.lock.RUnlock()\n\n\tfor _, v := range m.data {\n\t\ts += len(v)\n\t}\n\n\treturn s\n}\n\n// Series returns keys of aliases map\nfunc (m *Map) Series(isReverse bool) []string {\n\tseries := make([]string, 0, m.Len())\n\n\tfor k := range m.data {\n\t\tif isReverse {\n\t\t\tseries = append(series, reverse.String(k))\n\t\t} else {\n\t\t\tseries = append(series, k)\n\t\t}\n\t}\n\n\treturn series\n}\n\n// DisplayNames returns DisplayName from all Values\nfunc (m *Map) DisplayNames() []string {\n\tdn := make([]string, 0, m.Size())\n\n\tfor _, v := range m.data {\n\t\tfor _, a := range v {\n\t\t\tdn = append(dn, a.DisplayName)\n\t\t}\n\t}\n\n\treturn dn\n}\n\n// Get returns aliases for metric\nfunc (m *Map) Get(metric string) []Value {\n\treturn m.data[metric]\n}\n" }, { "path": "pkg/alias/map_tagged_test.go", "content": "package alias\n\nimport (\n\t\"sort\"\n\t\"sync\"\n\t\"testing\"\n\n\t\"github.com/lomik/graphite-clickhouse/finder\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nvar taggedResult *finder.MockFinder = finder.NewMockTagged([][]byte{\n\t[]byte(\"cpu.loadavg?env=test&host=host1\"),\n\t[]byte(\"cpu.loadavg?env=production&host=dc-host2\"),\n\t[]byte(\"cpu.loadavg?env=staging&host=stg-host3\"),\n})\n\nvar taggedTarget string = \"seriesByTag('name=cpu.loadavg)\"\n\nfunc createAMTagged() *Map {\n\tam := New()\n\tam.MergeTarget(taggedResult, taggedTarget, false)\n\n\treturn am\n}\n\nfunc TestCreationTagged(t *testing.T) {\n\tam := createAMTagged()\n\n\tfor _, m := range taggedResult.List() {\n\t\tmetric := string(m)\n\t\tv, ok := am.data[metric]\n\t\tassert.True(t, ok, \"metric %m is not found in Map\", metric)\n\t\tassert.Equal(t, taggedTarget, v[0].Target)\n\t\t// convert cpu.loadavg?env=test&host=host1 to cpu.loadavg;env=test;host=host1\n\t\tassert.Equal(t, string(finder.TaggedDecode(m)), v[0].DisplayName)\n\t}\n}\n\nfunc TestAsyncMergeTagged(t *testing.T) {\n\ttestEnvResult := [][]byte{\n\t\t[]byte(\"cpu.loadavg?env=test&host=host1\"),\n\t\t[]byte(\"cpu.loadavg?env=test&host=host2\"),\n\t}\n\ttargetTest := \"seriesByTag('name=cpu.loadavg', 'env=test')\"\n\n\tprodEnvResult := [][]byte{\n\t\t[]byte(\"cpu.loadavg?env=production&host=dc-host3\"),\n\t\t[]byte(\"cpu.loadavg?env=production&host=dc-host4\"),\n\t}\n\ttargetProd := \"seriesByTag('name=cpu.loadavg', 'env=prod')\"\n\n\tam := New()\n\twg := sync.WaitGroup{}\n\twg.Add(2)\n\n\tgo func() {\n\t\tam.MergeTarget(finder.NewMockTagged(testEnvResult), targetTest, false)\n\t\twg.Done()\n\t}()\n\tgo func() {\n\t\tam.MergeTarget(finder.NewMockTagged(prodEnvResult), targetProd, false)\n\t\twg.Done()\n\t}()\n\n\tresultAM := &Map{\n\t\tdata: map[string][]Value{\n\t\t\t\"cpu.loadavg?env=test&host=host1\": {\n\t\t\t\t{Target: \"seriesByTag('name=cpu.loadavg', 'env=test')\", DisplayName: \"cpu.loadavg;env=test;host=host1\"},\n\t\t\t},\n\t\t\t\"cpu.loadavg?env=test&host=host2\": {\n\t\t\t\t{Target: \"seriesByTag('name=cpu.loadavg', 'env=test')\", DisplayName: \"cpu.loadavg;env=test;host=host2\"},\n\t\t\t},\n\t\t\t\"cpu.loadavg?env=production&host=dc-host3\": {\n\t\t\t\t{Target: \"seriesByTag('name=cpu.loadavg', 'env=prod')\", DisplayName: \"cpu.loadavg;env=production;host=dc-host3\"},\n\t\t\t},\n\t\t\t\"cpu.loadavg?env=production&host=dc-host4\": {\n\t\t\t\t{Target: \"seriesByTag('name=cpu.loadavg', 'env=prod')\", DisplayName: \"cpu.loadavg;env=production;host=dc-host4\"},\n\t\t\t},\n\t\t},\n\t}\n\n\twg.Wait()\n\n\tif !assert.Equal(t, resultAM.Len(), am.Len()) {\n\t\tt.FailNow()\n\t}\n\n\tfor i := range am.data {\n\t\tvar dv Values = am.data[i]\n\n\t\tsort.Sort(&dv)\n\t\tam.data[i] = dv\n\t}\n\n\tassert.Equal(t, resultAM, am)\n}\n\nfunc Benchmark_MergeTargetTagged(b *testing.B) {\n\tresult := [][]byte{\n\t\t[]byte(\"cpu.loadavg?env=test&host=host1\"),\n\t\t[]byte(\"cpu.loadavg?env=production&host=dc-host2\"),\n\t}\n\n\tfor i := 0; i < b.N; i++ {\n\t\tam := createAM()\n\t\tam.MergeTarget(finder.NewMockTagged(result), taggedTarget, false)\n\t\t_ = am\n\t}\n}\n" }, { "path": "pkg/alias/map_test.go", "content": "package alias\n\nimport (\n\t\"sort\"\n\t\"sync\"\n\t\"testing\"\n\n\t\"github.com/lomik/graphite-clickhouse/finder\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\ntype Values []Value\n\nfunc (v *Values) Len() int {\n\treturn len(*v)\n}\n\nfunc (v *Values) Less(i, j int) bool {\n\treturn (*v)[i].Target < (*v)[j].Target\n}\n\nfunc (v *Values) Swap(i, j int) {\n\tvp := *v\n\tvp[i], vp[j] = vp[j], vp[i]\n}\n\nvar finderResult *finder.MockFinder = finder.NewMockFinder([][]byte{\n\t[]byte(\"5_sec.name.max\"),\n\t[]byte(\"1_min.name.avg\"),\n\t[]byte(\"5_min.name.min\"),\n\t[]byte(\"10_min.name.any\"), // defaults will be used\n})\n\nvar findTarget string = \"*.name.*\"\n\nfunc createAM() *Map {\n\tam := New()\n\tam.MergeTarget(finderResult, findTarget, false)\n\n\treturn am\n}\n\nfunc TestCreation(t *testing.T) {\n\tam := createAM()\n\n\tfor _, m := range finderResult.List() {\n\t\tmetric := string(m)\n\t\tv, ok := am.data[metric]\n\t\tassert.True(t, ok, \"metric %m is not found in Map\", metric)\n\t\tassert.Equal(t, findTarget, v[0].Target)\n\t\tassert.Equal(t, metric, v[0].DisplayName)\n\t}\n}\n\nfunc TestAsyncMerge(t *testing.T) {\n\tam := New()\n\ttarget2 := \"5*.name.*\"\n\twg := sync.WaitGroup{}\n\twg.Add(2)\n\n\tgo func() {\n\t\tam.MergeTarget(finderResult, findTarget, false)\n\t\twg.Done()\n\t}()\n\tgo func() {\n\t\tresult := [][]byte{\n\t\t\t[]byte(\"5_sec.name.max\"),\n\t\t\t[]byte(\"5_min.name.avg\"),\n\t\t\t[]byte(\"5_min.name.min\"),\n\t\t}\n\t\tam.MergeTarget(finder.NewMockFinder(result), target2, false)\n\t\twg.Done()\n\t}()\n\n\tresultAM := &Map{\n\t\tdata: map[string][]Value{\n\t\t\t\"5_sec.name.max\": {\n\t\t\t\t{Target: \"*.name.*\", DisplayName: \"5_sec.name.max\"},\n\t\t\t\t{Target: \"5*.name.*\", DisplayName: \"5_sec.name.max\"},\n\t\t\t},\n\t\t\t\"1_min.name.avg\": {\n\t\t\t\t{Target: \"*.name.*\", DisplayName: \"1_min.name.avg\"},\n\t\t\t},\n\t\t\t\"5_min.name.min\": {\n\t\t\t\t{Target: \"*.name.*\", DisplayName: \"5_min.name.min\"},\n\t\t\t\t{Target: \"5*.name.*\", DisplayName: \"5_min.name.min\"},\n\t\t\t},\n\t\t\t\"10_min.name.any\": {\n\t\t\t\t{Target: \"*.name.*\", DisplayName: \"10_min.name.any\"},\n\t\t\t},\n\t\t\t\"5_min.name.avg\": {\n\t\t\t\t{Target: \"5*.name.*\", DisplayName: \"5_min.name.avg\"},\n\t\t\t},\n\t\t},\n\t}\n\n\twg.Wait()\n\n\tif !assert.Equal(t, resultAM.Len(), am.Len()) {\n\t\tt.FailNow()\n\t}\n\n\tfor i := range am.data {\n\t\tvar dv Values = am.data[i]\n\n\t\tsort.Sort(&dv)\n\t\tam.data[i] = dv\n\t}\n\n\tassert.Equal(t, resultAM, am)\n}\n\nfunc TestLen(t *testing.T) {\n\tam := createAM()\n\tassert.Equal(t, 4, am.Len())\n\n\tresult := [][]byte{\n\t\t[]byte(\"5_sec.name.any\"),\n\t\t[]byte(\"5_min.name.min\"), // it's repeated\n\t}\n\tam.MergeTarget(finder.NewMockFinder(result), findTarget, false)\n\tassert.Equal(t, 5, am.Len())\n}\n\nfunc TestSize(t *testing.T) {\n\tam := createAM()\n\tassert.Equal(t, 4, am.Size())\n\n\tresult := [][]byte{\n\t\t[]byte(\"5_sec.name.any\"),\n\t\t[]byte(\"5_min.name.min\"), // it's repeated, but it increases Size\n\t}\n\tam.MergeTarget(finder.NewMockFinder(result), findTarget, false)\n\tassert.Equal(t, 6, am.Size())\n}\n\nfunc TestDisplayNames(t *testing.T) {\n\tam := createAM()\n\tsortedDisplayNames := am.DisplayNames()\n\tsort.Strings(sortedDisplayNames)\n\texpectedSeries := finderResult.Strings()\n\tsort.Strings(expectedSeries)\n\tassert.Equal(t, expectedSeries, sortedDisplayNames)\n\n\tanotherFinderResult := finder.NewMockFinder([][]byte{\n\t\t[]byte(\"5_sec.name.any\"),\n\t\t[]byte(\"5_min.name.min\"), // it's repeated, but it increases Size\n\t})\n\tam.MergeTarget(anotherFinderResult, findTarget, false)\n\tsortedDisplayNames = am.DisplayNames()\n\tsort.Strings(sortedDisplayNames)\n\texpectedSeries = append(expectedSeries, anotherFinderResult.Strings()...)\n\tsort.Strings(expectedSeries)\n\tassert.Equal(t, expectedSeries, sortedDisplayNames)\n}\n\nfunc TestGet(t *testing.T) {\n\tam := createAM()\n\tassert.Equal(t, []Value{{Target: \"*.name.*\", DisplayName: \"5_sec.name.max\"}}, am.Get(\"5_sec.name.max\"))\n}\n\nfunc Benchmark_MergeTargetFinder(b *testing.B) {\n\tresult := [][]byte{\n\t\t[]byte(\"5_sec.name.any\"),\n\t\t[]byte(\"5_min.name.min\"),\n\t}\n\n\tfor i := 0; i < b.N; i++ {\n\t\tam := createAM()\n\t\tam.MergeTarget(finder.NewMockFinder(result), findTarget, false)\n\t\t_ = am\n\t}\n}\n" }, { "path": "pkg/dry/math.go", "content": "package dry\n\n// Max returns the larger of x or y.\nfunc Max(x, y int64) int64 {\n\tif x > y {\n\t\treturn x\n\t}\n\n\treturn y\n}\n\n// Min returns the lower of x or y.\nfunc Min(x, y int64) int64 {\n\tif x < y {\n\t\treturn x\n\t}\n\n\treturn y\n}\n\n// Ceil returns integer greater or equal to x and denominator d division.\n// Works only with x >= 0 and d > 0. It returns 0 with other values.\nfunc Ceil(x, d int64) int64 {\n\tif x <= 0 || d <= 0 {\n\t\treturn int64(0)\n\t}\n\n\treturn (x + d - 1) / d\n}\n\n// CeilToMultiplier returns the integer greater or equal to x and multiplier m product.\n// Works only with x >= 0 and m > 0. It returns 0 with other values.\nfunc CeilToMultiplier(x, m int64) int64 {\n\treturn Ceil(x, m) * m\n}\n\n// FloorToMultiplier returns the integer less or equal to x and multiplier m product.\n// Works only with x >= 0 and m > 0. It returns 0 with other values.\nfunc FloorToMultiplier(x, m int64) int64 {\n\tif x <= 0 || m <= 0 {\n\t\treturn int64(0)\n\t}\n\n\treturn x / m * m\n}\n\n// GCD returns the absolute greatest common divisor calculated via Euclidean algorithm\nfunc GCD(a, b int64) int64 {\n\tif b < 0 {\n\t\tb = -b\n\t}\n\n\tvar t int64\n\tfor b != 0 {\n\t\tt = b\n\t\tb = a % b\n\t\ta = t\n\t}\n\n\treturn a\n}\n\n// LCM returns the absolute least common multiple of 2 integers via GDB\nfunc LCM(a, b int64) int64 {\n\tif a*b < 0 {\n\t\treturn -a / GCD(a, b) * b\n\t}\n\n\treturn a / GCD(a, b) * b\n}\n" }, { "path": "pkg/dry/math_test.go", "content": "package dry\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestMax(t *testing.T) {\n\tassert := assert.New(t)\n\n\tassert.Equal(int64(1), Max(1, -1))\n\tassert.Equal(int64(2), Max(1, 2))\n\tassert.Equal(int64(3), Max(3, 3))\n}\n\nfunc TestMin(t *testing.T) {\n\tassert := assert.New(t)\n\n\tassert.Equal(int64(-1), Min(1, -1))\n\tassert.Equal(int64(1), Min(1, 2))\n\tassert.Equal(int64(3), Min(3, 3))\n}\n\nfunc TestCeil(t *testing.T) {\n\tassert := assert.New(t)\n\n\tassert.Equal(int64(0), Ceil(0, -1))\n\tassert.Equal(int64(3), Ceil(5, 2))\n\tassert.Equal(int64(1), Ceil(5, 5)) // if quotient is integer we should get quotient without +1\n\tassert.Equal(int64(2), Ceil(100001, 100000)) // if quotient is any fraction bigger than integer then we get +1\n}\n\nfunc TestCeilToMultiplier(t *testing.T) {\n\tassert := assert.New(t)\n\n\tassert.Equal(int64(0), CeilToMultiplier(0, -1))\n\tassert.Equal(int64(0), CeilToMultiplier(1, 0))\n\tassert.Equal(int64(0), CeilToMultiplier(1, -1))\n\tassert.Equal(int64(2), CeilToMultiplier(1, 2))\n\tassert.Equal(int64(6), CeilToMultiplier(4, 3))\n\tassert.Equal(int64(6), CeilToMultiplier(6, 3))\n}\n\nfunc TestFloorToMultiplier(t *testing.T) {\n\tassert := assert.New(t)\n\n\tassert.Equal(int64(0), FloorToMultiplier(0, -1))\n\tassert.Equal(int64(0), FloorToMultiplier(1, 0))\n\tassert.Equal(int64(0), FloorToMultiplier(1, -1))\n\tassert.Equal(int64(0), FloorToMultiplier(1, 2))\n\tassert.Equal(int64(3), FloorToMultiplier(4, 3))\n\tassert.Equal(int64(6), FloorToMultiplier(6, 3))\n}\n\nfunc TestGCD(t *testing.T) {\n\tassert := assert.New(t)\n\n\tassert.Equal(int64(1), GCD(1, -1))\n\tassert.Equal(int64(1), GCD(-1, 1))\n\tassert.Equal(int64(1), GCD(-1, -1))\n\tassert.Equal(int64(1), GCD(1, 2))\n\tassert.Equal(int64(1), GCD(4, 3))\n\tassert.Equal(int64(3), GCD(6, 3))\n}\n\nfunc TestLCM(t *testing.T) {\n\tassert := assert.New(t)\n\n\tassert.Equal(int64(1), LCM(1, -1))\n\tassert.Equal(int64(1), LCM(-1, 1))\n\tassert.Equal(int64(1), LCM(-1, -1))\n\tassert.Equal(int64(2), LCM(1, 2))\n\tassert.Equal(int64(6), LCM(6, 3))\n\tassert.Equal(int64(12), LCM(4, 3))\n}\n" }, { "path": "pkg/dry/strings.go", "content": "package dry\n\n// RemoveEmptyStrings removes empty strings from list and returns truncated slice\nfunc RemoveEmptyStrings(stringList []string) []string {\n\trm := 0\n\n\tfor i := 0; i < len(stringList); i++ {\n\t\tif stringList[i] == \"\" {\n\t\t\trm++\n\t\t\tcontinue\n\t\t}\n\n\t\tif rm > 0 {\n\t\t\tstringList[i-rm] = stringList[i]\n\t\t}\n\t}\n\n\treturn stringList[:len(stringList)-rm]\n}\n" }, { "path": "pkg/dry/strings_test.go", "content": "package dry\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestRemoveEmptyStrings(t *testing.T) {\n\tassert := assert.New(t)\n\n\tassert.Equal([]string{\"lorem\", \" \", \"ipsum\"},\n\t\tRemoveEmptyStrings([]string{\"\", \"\", \"lorem\", \"\", \" \", \"ipsum\", \"\"}),\n\t)\n}\n" }, { "path": "pkg/dry/unsafe.go", "content": "package dry\n\nimport (\n\t\"reflect\"\n\t\"unsafe\"\n)\n\n// UnsafeString returns string object from byte slice without copying\nfunc UnsafeString(b []byte) string {\n\treturn *(*string)(unsafe.Pointer(&b))\n}\n\n// UnsafeStringBytes returns the string bytes\nfunc UnsafeStringBytes(s *string) []byte {\n\treturn *(*[]byte)(unsafe.Pointer((*reflect.SliceHeader)(unsafe.Pointer(s))))\n}\n" }, { "path": "pkg/dry/unsafe_test.go", "content": "package dry\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestUnsafeString(t *testing.T) {\n\tassert := assert.New(t)\n\tassert.Equal(\"hello\", UnsafeString([]byte{'h', 'e', 'l', 'l', 'o'}))\n\tassert.Equal(\"h\", UnsafeString([]byte{'h'}))\n\tassert.Equal(\"\", UnsafeString([]byte{}))\n\tassert.Equal(\"\", UnsafeString(nil))\n}\n" }, { "path": "pkg/reverse/reverse.go", "content": "package reverse\n\nimport (\n\t\"bytes\"\n\t\"strings\"\n)\n\nfunc String(path string) string {\n\t// don't reverse tagged path\n\tif strings.IndexByte(path, '?') >= 0 {\n\t\treturn path\n\t}\n\n\ta := strings.Split(path, \".\")\n\n\tl := len(a)\n\tfor i := 0; i < l/2; i++ {\n\t\ta[i], a[l-i-1] = a[l-i-1], a[i]\n\t}\n\n\treturn strings.Join(a, \".\")\n}\n\nfunc reverse(m []byte) {\n\ti := 0\n\tj := len(m) - 1\n\n\tfor i < j {\n\t\tm[i], m[j] = m[j], m[i]\n\t\ti++\n\t\tj--\n\t}\n}\n\nfunc Inplace(path []byte) {\n\tif bytes.IndexByte(path, '?') >= 0 {\n\t\treturn\n\t}\n\n\treverse(path)\n\n\tvar a, b int\n\n\tl := len(path)\n\tfor b = 0; b < l; b++ {\n\t\tif path[b] == '.' {\n\t\t\treverse(path[a:b])\n\t\t\ta = b + 1\n\t\t}\n\t}\n\n\treverse(path[a:b])\n}\n\nfunc Bytes(path []byte) []byte {\n\t// @TODO: test\n\t// don't reverse tagged path\n\tif bytes.IndexByte(path, '?') >= 0 {\n\t\treturn path\n\t}\n\n\tr := make([]byte, len(path))\n\tcopy(r, path)\n\tInplace(r)\n\n\treturn r\n}\n" }, { "path": "pkg/reverse/reverse_test.go", "content": "package reverse\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestReverse(t *testing.T) {\n\tassert := assert.New(t)\n\ttable := map[string]string{\n\t\t\"carbon.agents.carbon-clickhouse.graphite1.tcp.metricsReceived\": \"metricsReceived.tcp.graphite1.carbon-clickhouse.agents.carbon\",\n\t\t\"\": \"\",\n\t\t\".\": \".\",\n\t\t\"carbon..xx\": \"xx..carbon\",\n\t\t\".hello..world.\": \".world..hello.\",\n\t\t\"metric_name?label=value\": \"metric_name?label=value\",\n\t}\n\n\tfor k, expected := range table {\n\t\tassert.Equal(expected, String(k))\n\t\tp := k\n\t\tassert.Equal([]byte(expected), Bytes([]byte(k)))\n\t\t// check k is unchanged\n\t\tassert.Equal(p, k)\n\t\t// inplace\n\t\tb := make([]byte, len(k))\n\t\tcopy(b, k)\n\t\tInplace(b)\n\t\tassert.Equal(expected, string(b))\n\t}\n}\n" }, { "path": "pkg/scope/context.go", "content": "package scope\n\nimport (\n\t\"context\"\n\n\t\"go.uber.org/zap\"\n)\n\n// Context wrapper for context.Context with chain constructor\ntype Context struct {\n\tcontext.Context\n}\n\n// New ...\nfunc New(ctx context.Context) *Context {\n\treturn &Context{ctx}\n}\n\n// With ...\nfunc (c *Context) With(key string, value interface{}) *Context {\n\treturn New(With(c.Context, key, value))\n}\n\n// WithRequestID ...\nfunc (c *Context) WithRequestID(requestID string) *Context {\n\treturn New(WithRequestID(c.Context, requestID))\n}\n\n// WithLogger ...\nfunc (c *Context) WithLogger(logger *zap.Logger) *Context {\n\treturn New(WithLogger(c.Context, logger))\n}\n\n// WithTable ...\nfunc (c *Context) WithTable(table string) *Context {\n\treturn New(WithTable(c.Context, table))\n}\n" }, { "path": "pkg/scope/http_request.go", "content": "package scope\n\nimport (\n\t\"context\"\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"net\"\n\t\"net/http\"\n\t\"regexp\"\n\t\"strings\"\n)\n\nvar (\n\trequestIdRegexp = regexp.MustCompile(\"^[a-zA-Z0-9_.-]+$\")\n\tpassHeaders = []string{\n\t\t\"X-Dashboard-Id\",\n\t\t\"X-Grafana-Org-Id\",\n\t\t\"X-Panel-Id\",\n\t\t\"X-Forwarded-For\",\n\t}\n)\n\nfunc HttpRequest(r *http.Request) *http.Request {\n\trequestID := r.Header.Get(\"X-Request-Id\")\n\tif requestID == \"\" || !requestIdRegexp.MatchString(requestID) {\n\t\tvar b [16]byte\n\n\t\tbinary.LittleEndian.PutUint64(b[:], rand.Uint64())\n\t\tbinary.LittleEndian.PutUint64(b[8:], rand.Uint64())\n\t\trequestID = fmt.Sprintf(\"%x\", b)\n\t}\n\n\tctx := r.Context()\n\tctx = WithRequestID(ctx, requestID)\n\n\t// Process all X-Gch-Debug-* headers\n\tdebugPrefix := \"X-Gch-Debug-\"\n\tfor name, values := range r.Header {\n\t\tif strings.HasPrefix(name, debugPrefix) && len(values) != 0 && values[0] != \"\" {\n\t\t\tctx = WithDebug(ctx, strings.TrimPrefix(name, debugPrefix))\n\t\t}\n\t}\n\n\t// Append the server IP to X-Forwarded-For if exists, else ignore\n\tif xff := r.Header.Get(\"X-Forwarded-For\"); xff != \"\" {\n\t\tclientIP, _, _ := net.SplitHostPort(r.RemoteAddr)\n\t\tr.Header.Set(\"X-Forwarded-For\", fmt.Sprintf(\"%s, %s\", xff, clientIP))\n\t}\n\n\tfor _, h := range passHeaders {\n\t\thv := r.Header.Get(h)\n\t\tif hv != \"\" {\n\t\t\tctx = With(ctx, h, hv)\n\t\t}\n\t}\n\n\treturn r.WithContext(ctx)\n}\n\nfunc Grafana(ctx context.Context) string {\n\to, d, p := String(ctx, \"X-Grafana-Org-Id\"), String(ctx, \"X-Dashboard-Id\"), String(ctx, \"X-Panel-Id\")\n\tif o != \"\" || d != \"\" || p != \"\" {\n\t\treturn fmt.Sprintf(\"Org:%s; Dashboard:%s; Panel:%s\", o, d, p)\n\t}\n\n\treturn \"\"\n}\n" }, { "path": "pkg/scope/key.go", "content": "package scope\n\nimport (\n\t\"context\"\n\t\"fmt\"\n)\n\n// key is type for context.Value keys\ntype scopeKey string\n\n// With returns a copy of parent in which the value associated with key is val.\nfunc With(ctx context.Context, key string, value interface{}) context.Context {\n\treturn context.WithValue(ctx, scopeKey(key), value)\n}\n\n// String returns the string value associated with this context for key\nfunc String(ctx context.Context, key string) string {\n\tif value, ok := ctx.Value(scopeKey(key)).(string); ok {\n\t\treturn value\n\t}\n\n\treturn \"\"\n}\n\n// Bool returns the true if particular key of the context is set\nfunc Bool(ctx context.Context, key string) bool {\n\tif _, ok := ctx.Value(scopeKey(key)).(bool); ok {\n\t\treturn true\n\t}\n\n\treturn false\n}\n\n// WithRequestID ...\nfunc WithRequestID(ctx context.Context, requestID string) context.Context {\n\treturn With(ctx, \"requestID\", requestID)\n}\n\n// RequestID ...\nfunc RequestID(ctx context.Context) string {\n\treturn String(ctx, \"requestID\")\n}\n\n// WithTable ...\nfunc WithTable(ctx context.Context, table string) context.Context {\n\treturn With(ctx, \"table\", table)\n}\n\n// Table ...\nfunc Table(ctx context.Context) string {\n\treturn String(ctx, \"table\")\n}\n\n// WithDebug returns the context with debug-name\nfunc WithDebug(ctx context.Context, name string) context.Context {\n\treturn With(ctx, \"debug-\"+name, true)\n}\n\n// Debug returns true if debug-name should be enabled\nfunc Debug(ctx context.Context, name string) bool {\n\treturn Bool(ctx, \"debug-\"+name)\n}\n\n// ClickhouseUserAgent ...\nfunc ClickhouseUserAgent(ctx context.Context) string {\n\tgrafana := Grafana(ctx)\n\tif grafana != \"\" {\n\t\treturn fmt.Sprintf(\"Graphite-Clickhouse/%s (table:%s) Grafana(%s)\", Version, Table(ctx), grafana)\n\t}\n\n\treturn fmt.Sprintf(\"Graphite-Clickhouse/%s (table:%s)\", Version, Table(ctx))\n}\n" }, { "path": "pkg/scope/logger.go", "content": "package scope\n\nimport (\n\t\"context\"\n\t\"net/http\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/headers\"\n\t\"github.com/lomik/zapwriter\"\n\t\"go.uber.org/zap\"\n)\n\nvar (\n\tCarbonapiUUIDName = \"carbonapi_uuid\"\n\tRequestHeadersName = \"request_headers\"\n)\n\n// Logger returns zap.Logger instance\nfunc Logger(ctx context.Context) *zap.Logger {\n\tlogger := ctx.Value(scopeKey(\"logger\"))\n\n\tvar zapLogger *zap.Logger\n\n\tif logger != nil {\n\t\tif zl, ok := logger.(*zap.Logger); ok {\n\t\t\tzapLogger = zl\n\t\t\treturn zapLogger\n\t\t}\n\t}\n\n\tif zapLogger == nil {\n\t\tzapLogger = zapwriter.Default()\n\t}\n\n\trequestId := RequestID(ctx)\n\tif requestId != \"\" {\n\t\tzapLogger = zapLogger.With(zap.String(\"request_id\", requestId))\n\t}\n\n\treturn zapLogger\n}\n\n// Logger returns zap.Logger instance\nfunc LoggerWithHeaders(ctx context.Context, r *http.Request, headersToLog []string) *zap.Logger {\n\tlogger := ctx.Value(scopeKey(\"logger\"))\n\n\tvar zapLogger *zap.Logger\n\n\tif logger != nil {\n\t\tif zl, ok := logger.(*zap.Logger); ok {\n\t\t\tzapLogger = zl\n\t\t\treturn zapLogger\n\t\t}\n\t}\n\n\tif zapLogger == nil {\n\t\tzapLogger = zapwriter.Default()\n\t}\n\n\trequestId := RequestID(ctx)\n\tif requestId != \"\" {\n\t\tzapLogger = zapLogger.With(zap.String(\"request_id\", requestId))\n\t}\n\n\tcarbonapiUUID := r.Header.Get(\"X-Ctx-Carbonapi-Uuid\")\n\tif carbonapiUUID != \"\" {\n\t\tzapLogger = zapLogger.With(zap.String(\"carbonapi_uuid\", carbonapiUUID))\n\t}\n\n\trequestHeaders := headers.GetHeaders(&r.Header, headersToLog)\n\tif len(requestHeaders) > 0 {\n\t\tzapLogger = zapLogger.With(zap.Any(\"request_headers\", requestHeaders))\n\t}\n\n\treturn zapLogger\n}\n\n// WithLogger ...\nfunc WithLogger(ctx context.Context, logger *zap.Logger) context.Context {\n\treturn With(ctx, \"logger\", logger)\n}\n" }, { "path": "pkg/scope/version.go", "content": "package scope\n\nvar Version string\n" }, { "path": "pkg/where/match.go", "content": "package where\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\nvar opEq = \"=\"\n\n// ClearGlob cleanup grafana globs like {name}\nfunc ClearGlob(query string) string {\n\tp := 0\n\n\ts := strings.IndexAny(query, \"{[\")\n\tif s == -1 {\n\t\treturn query\n\t}\n\n\tfound := false\n\n\tvar builder strings.Builder\n\n\tfor {\n\t\tvar e int\n\t\tif query[s] == '{' {\n\t\t\te = strings.IndexAny(query[s:], \"}.\")\n\t\t\tif e == -1 || query[s+e] == '.' {\n\t\t\t\t// { not closed, glob with error\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\te += s + 1\n\n\t\t\tdelim := strings.IndexRune(query[s+1:e], ',')\n\t\t\tif delim == -1 {\n\t\t\t\tif !found {\n\t\t\t\t\tbuilder.Grow(len(query) - 2)\n\n\t\t\t\t\tfound = true\n\t\t\t\t}\n\n\t\t\t\tbuilder.WriteString(query[p:s])\n\t\t\t\tbuilder.WriteString(query[s+1 : e-1])\n\t\t\t\tp = e\n\t\t\t}\n\t\t} else {\n\t\t\te = strings.IndexAny(query[s+1:], \"].\")\n\t\t\tif e == -1 || query[s+e] == '.' {\n\t\t\t\t// [ not closed, glob with error\n\t\t\t\tbreak\n\t\t\t} else {\n\t\t\t\tsymbols := 0\n\n\t\t\t\tfor _, c := range query[s+1 : s+e+1] {\n\t\t\t\t\t_ = c // for loop over runes\n\n\t\t\t\t\tsymbols++\n\t\t\t\t\tif symbols == 2 {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif symbols <= 1 {\n\t\t\t\t\tif !found {\n\t\t\t\t\t\tbuilder.Grow(len(query) - 2)\n\n\t\t\t\t\t\tfound = true\n\t\t\t\t\t}\n\n\t\t\t\t\tbuilder.WriteString(query[p:s])\n\t\t\t\t\tbuilder.WriteString(query[s+1 : s+e+1])\n\t\t\t\t\tp = e + s + 2\n\t\t\t\t}\n\t\t\t}\n\n\t\t\te += s + 2\n\t\t}\n\n\t\tif e >= len(query) {\n\t\t\tbreak\n\t\t}\n\n\t\ts = strings.IndexAny(query[e:], \"{[\")\n\t\tif s == -1 {\n\t\t\tbreak\n\t\t}\n\n\t\ts += e\n\t}\n\n\tif found {\n\t\tif p < len(query) {\n\t\t\tbuilder.WriteString(query[p:])\n\t\t}\n\n\t\treturn builder.String()\n\t}\n\n\treturn query\n}\n\nfunc HasUnmatchedBrackets(query string) bool {\n\tmatchingBracket := map[rune]rune{\n\t\t'}': '{',\n\t\t']': '[',\n\t}\n\tstack := make([]rune, 0)\n\n\tnodeHasUnmatched := func(query string) bool {\n\t\tfor _, c := range query {\n\t\t\tif c == '{' || c == '[' {\n\t\t\t\tstack = append(stack, c)\n\t\t\t}\n\n\t\t\tif c == '}' || c == ']' {\n\t\t\t\tif len(stack) == 0 || stack[len(stack)-1] != matchingBracket[c] {\n\t\t\t\t\treturn true\n\t\t\t\t}\n\n\t\t\t\tstack = stack[:len(stack)-1]\n\t\t\t}\n\t\t}\n\n\t\treturn len(stack) != 0\n\t}\n\n\tfor _, node := range strings.Split(query, \".\") {\n\t\tif nodeHasUnmatched(node) {\n\t\t\treturn true\n\t\t}\n\t}\n\n\treturn false\n}\n\nfunc glob(field string, query string, optionalDotAtEnd bool) string {\n\tif query == \"*\" {\n\t\treturn \"\"\n\t}\n\n\tquery = ClearGlob(query)\n\n\tif !HasWildcard(query) {\n\t\tif optionalDotAtEnd {\n\t\t\treturn In(field, []string{query, query + \".\"})\n\t\t} else {\n\t\t\treturn Eq(field, query)\n\t\t}\n\t}\n\n\tw := New()\n\n\t// before any wildcard symbol\n\tsimplePrefix := query[:strings.IndexAny(query, \"[]{}*?\")]\n\n\tif len(simplePrefix) > 0 {\n\t\tw.And(HasPrefix(field, simplePrefix))\n\t}\n\n\t// prefix search like \"metric.name.xx*\"\n\tif len(simplePrefix) == len(query)-1 && query[len(query)-1] == '*' {\n\t\treturn HasPrefix(field, simplePrefix)\n\t}\n\n\t// Q() replaces \\ with \\\\, so using \\. does not work here.\n\t// work around with [.]\n\tpostfix := `$`\n\tif optionalDotAtEnd {\n\t\tpostfix = `[.]?$`\n\t}\n\n\tif simplePrefix == \"\" {\n\t\treturn fmt.Sprintf(\"match(%s, %s)\", field, quote(`^`+GlobToRegexp(query)+postfix))\n\t}\n\n\treturn fmt.Sprintf(\"%s AND match(%s, %s)\",\n\t\tHasPrefix(field, simplePrefix),\n\t\tfield, quote(`^`+GlobToRegexp(query)+postfix),\n\t)\n}\n\n// Glob ...\nfunc Glob(field string, query string) string {\n\treturn glob(field, query, false)\n}\n\n// TreeGlob ...\nfunc TreeGlob(field string, query string) string {\n\treturn glob(field, query, true)\n}\n\nfunc ConcatMatchKV(key, value string) string {\n\tstartLine := value[0] == '^'\n\tendLine := value[len(value)-1] == '$'\n\n\tif startLine && endLine {\n\t\treturn key + opEq + value[1:]\n\t} else if startLine {\n\t\treturn key + opEq + value[1:] + \"\\\\\\\\%\"\n\t}\n\n\treturn key + opEq + \"\\\\\\\\%\" + value\n}\n\nfunc Match(field string, key, value string) string {\n\tif len(value) == 0 || value == \"*\" {\n\t\treturn Like(field, key+\"=%\")\n\t}\n\n\texpr := ConcatMatchKV(key, value)\n\tsimplePrefix := NonRegexpPrefix(expr)\n\n\tif len(simplePrefix) == len(expr) {\n\t\treturn Eq(field, expr)\n\t} else if len(simplePrefix) == len(expr)-1 && expr[len(expr)-1] == '$' {\n\t\treturn Eq(field, simplePrefix)\n\t}\n\n\tif simplePrefix == \"\" {\n\t\treturn fmt.Sprintf(\"match(%s, %s)\", field, quoteRegex(key, value))\n\t}\n\n\treturn fmt.Sprintf(\"%s AND match(%s, %s)\",\n\t\tHasPrefix(field, simplePrefix),\n\t\tfield, quoteRegex(key, value),\n\t)\n}\n" }, { "path": "pkg/where/match_test.go", "content": "package where\n\nimport \"testing\"\n\nfunc Test_ClearGlob(t *testing.T) {\n\ttests := []struct {\n\t\tquery string\n\t\twant string\n\t}{\n\t\t{\"a.{a,b}.te{s}t.b\", \"a.{a,b}.test.b\"},\n\t\t{\"a.{a,b}.te{s,t}*.b\", \"a.{a,b}.te{s,t}*.b\"},\n\t\t{\"a.{a,b}.test*.b\", \"a.{a,b}.test*.b\"},\n\t\t{\"a.[b].te{s}t.b\", \"a.b.test.b\"},\n\t\t{\"a.[ab].te{s,t}*.b\", \"a.[ab].te{s,t}*.b\"},\n\t\t{\"a.{a,b.}.te{s,t}*.b\", \"a.{a,b.}.te{s,t}*.b\"}, // some broken\n\t\t{\"О.[Đą].Ņ‚Đĩ{s}t.b\", \"О.Đą.Ņ‚Đĩst.b\"}, // utf-8 string\n\t\t{\"О.[].Ņ‚Đĩ{}t.b\", \"О..Ņ‚Đĩt.b\"}, // utf-8 string with empthy blocks\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.query, func(t *testing.T) {\n\t\t\tif got := ClearGlob(tt.query); got != tt.want {\n\t\t\t\tt.Errorf(\"ClearGlob() = %v, want %v\", got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc Test_HasUnmatchedBrackets(t *testing.T) {\n\ttests := []struct {\n\t\tquery string\n\t\twant bool\n\t}{\n\t\t{\"a.{a,b.te{s}t.b\", true},\n\t\t{\"a.{a,b}.te{s}t.b\", false},\n\t\t{\"a.{a,b}.te{s,t}}*.b\", true},\n\t\t{\"a.{a,b}.test*.b\", false},\n\t\t{\"a.a,b}.test*.b\", true},\n\t\t{\"a.{a,b.test*.b}\", true},\n\t\t{\"a.[a,b.test*.b]\", true},\n\t\t{\"a.[a,b].test*.b\", false},\n\t\t{\"a.[b].te{s}t.b\", false},\n\t\t{\"a.{[cd],[ef]}.b\", false},\n\t\t{\"a.[ab].te{s,t}*.b\", false},\n\t\t{\"a.{a,b.}.te{s,t}*.b\", true}, // dots are not escaped inside curly brackets\n\t\t{\"О.[Đą].Ņ‚Đĩ{s}t.b\", false}, // utf-8 string\n\t\t{\"О.[Đą.Ņ‚Đĩs}t.b\", true},\n\t\t{\"О.[].Ņ‚Đĩ{}t.b\", false}, // utf-8 string with empthy blocks\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.query, func(t *testing.T) {\n\t\t\tif got := HasUnmatchedBrackets(tt.query); got != tt.want {\n\t\t\t\tt.Errorf(\"HasUnmatchedBrackets() = %v, want %v\", got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestGlob(t *testing.T) {\n\tfield := \"test\"\n\n\ttests := []struct {\n\t\tquery string\n\t\twant string\n\t}{\n\t\t{\"a.{a,b}.te{s}t.b\", \"test LIKE 'a.%' AND match(test, '^a[.](a|b)[.]test[.]b$')\"},\n\t\t{\"a.{a,b}.te{s,t}*.b\", \"test LIKE 'a.%' AND match(test, '^a[.](a|b)[.]te(s|t)([^.]*?)[.]b$')\"},\n\t\t{\"a.{a,b}.test*.b\", \"test LIKE 'a.%' AND match(test, '^a[.](a|b)[.]test([^.]*?)[.]b$')\"},\n\t\t{\"a.[b].te{s}t.b\", \"test='a.b.test.b'\"},\n\t\t{\"a.[ab].te{s,t}*.b\", \"test LIKE 'a.%' AND match(test, '^a[.][ab][.]te(s|t)([^.]*?)[.]b$')\"},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.query, func(t *testing.T) {\n\t\t\tif got := Glob(field, tt.query); got != tt.want {\n\t\t\t\tt.Errorf(\"Glob() = %v, want %v\", got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "pkg/where/where.go", "content": "package where\n\nimport (\n\t\"fmt\"\n\t\"net/http\"\n\t\"regexp\"\n\t\"strings\"\n\t\"unsafe\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/date\"\n\t\"github.com/lomik/graphite-clickhouse/helper/errs\"\n)\n\nfunc unsafeString(b []byte) string {\n\treturn *(*string)(unsafe.Pointer(&b))\n}\n\n// workaraund for Grafana multi-value variables, expand S{a,b,c}E to [SaE,SbE,ScE]\nfunc GlobExpandSimple(value, prefix string, result *[]string) error {\n\tif len(value) == 0 {\n\t\t// we at the end of glob\n\t\t*result = append(*result, prefix)\n\t\treturn nil\n\t}\n\n\tstart := strings.IndexAny(value, \"{}\")\n\tif start == -1 {\n\t\t*result = append(*result, prefix+value)\n\t} else {\n\t\tend := strings.Index(value[start:], \"}\")\n\t\tif end <= 1 {\n\t\t\treturn errs.NewErrorWithCode(\"malformed glob: \"+value, http.StatusBadRequest)\n\t\t}\n\n\t\tif end == -1 || strings.IndexAny(value[start+1:start+end], \"{}\") != -1 {\n\t\t\treturn errs.NewErrorWithCode(\"malformed glob: \"+value, http.StatusBadRequest)\n\t\t}\n\n\t\tif start > 0 {\n\t\t\tprefix = prefix + value[0:start]\n\t\t}\n\n\t\tg := value[start+1 : start+end]\n\t\tvalues := strings.Split(g, \",\")\n\n\t\tvar postfix string\n\t\tif end+start-1 < len(value) {\n\t\t\tpostfix = value[start+end+1:]\n\t\t}\n\n\t\tfor _, v := range values {\n\t\t\tif err := GlobExpandSimple(postfix, prefix+v, result); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc GlobToRegexp(g string) string {\n\ts := g\n\ts = strings.ReplaceAll(s, \".\", \"[.]\")\n\ts = strings.ReplaceAll(s, \"$\", \"[$]\")\n\ts = strings.ReplaceAll(s, \"{\", \"(\")\n\ts = strings.ReplaceAll(s, \"}\", \")\")\n\ts = strings.ReplaceAll(s, \"?\", \"[^.]\")\n\ts = strings.ReplaceAll(s, \",\", \"|\")\n\ts = strings.ReplaceAll(s, \"*\", \"([^.]*?)\")\n\n\treturn s\n}\n\nfunc HasWildcard(target string) bool {\n\treturn strings.IndexAny(target, \"[]{}*?\") > -1\n}\n\nfunc IndexLastWildcard(target string) int {\n\treturn strings.LastIndexAny(target, \"[]{}*?\")\n}\n\nfunc IndexWildcard(target string) int {\n\treturn strings.IndexAny(target, \"[]{}*?\")\n}\n\nfunc MaxWildcardDistance(query string) int {\n\tif !HasWildcard(query) {\n\t\treturn -1\n\t}\n\n\tw := IndexWildcard(query)\n\tfirstWildcardNode := strings.Count(query[:w], \".\")\n\tw = IndexLastWildcard(query)\n\tlastWildcardNode := strings.Count(query[w:], \".\")\n\n\treturn max(firstWildcardNode, lastWildcardNode)\n}\n\nfunc NonRegexpPrefix(expr string) string {\n\ts := regexp.QuoteMeta(expr)\n\tfor i := 0; i < len(expr); i++ {\n\t\tif expr[i] != s[i] || expr[i] == '\\\\' {\n\t\t\tif len(expr) > i+1 && expr[i] == '|' {\n\t\t\t\teq := strings.LastIndexAny(expr[:i], \"=~\")\n\t\t\t\tif eq > 0 {\n\t\t\t\t\treturn expr[:eq+1]\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn expr[:i]\n\t\t}\n\t}\n\n\treturn expr\n}\n\nfunc escape(s string) string {\n\ts = strings.ReplaceAll(s, `\\`, `\\\\`)\n\ts = strings.ReplaceAll(s, `'`, `\\'`)\n\n\treturn s\n}\n\nfunc escapeRegex(s string) string {\n\ts = escape(s)\n\tif strings.Contains(s, \"|\") {\n\t\ts = \"(\" + s + \")\"\n\t}\n\n\treturn s\n}\n\nfunc likeEscape(s string) string {\n\ts = strings.ReplaceAll(s, `_`, `\\_`)\n\ts = strings.ReplaceAll(s, `%`, `\\%`)\n\ts = strings.ReplaceAll(s, `\\`, `\\\\`)\n\ts = strings.ReplaceAll(s, `'`, `\\'`)\n\n\treturn s\n}\n\nfunc quote(value interface{}) string {\n\tswitch v := value.(type) {\n\tcase int:\n\t\treturn fmt.Sprintf(\"%#v\", v)\n\tcase uint32:\n\t\treturn fmt.Sprintf(\"%#v\", v)\n\tcase string:\n\t\treturn fmt.Sprintf(\"'%s'\", escape(v))\n\tcase []byte:\n\t\treturn fmt.Sprintf(\"'%s'\", escape(unsafeString(v)))\n\tdefault:\n\t\tpanic(\"not implemented\")\n\t}\n}\n\nfunc quoteRegex(key, value string) string {\n\tstartLine := value[0] == '^'\n\tif startLine {\n\t\treturn fmt.Sprintf(\"'^%s%s%s'\", key, opEq, escapeRegex(value[1:]))\n\t}\n\n\treturn fmt.Sprintf(\"'^%s%s.*%s'\", key, opEq, escapeRegex(value))\n}\n\nfunc Like(field, s string) string {\n\treturn fmt.Sprintf(\"%s LIKE '%s'\", field, s)\n}\n\nfunc Eq(field, value interface{}) string {\n\treturn fmt.Sprintf(\"%s=%s\", field, quote(value))\n}\n\nfunc HasPrefix(field, prefix string) string {\n\treturn fmt.Sprintf(\"%s LIKE '%s%%'\", field, likeEscape(prefix))\n}\n\nfunc HasPrefixAndNotEq(field, prefix string) string {\n\treturn fmt.Sprintf(\"%s LIKE '%s_%%'\", field, likeEscape(prefix))\n}\n\nfunc HasPrefixBytes(field, prefix []byte) string {\n\treturn fmt.Sprintf(\"%s LIKE '%s%%'\", field, likeEscape(unsafeString(prefix)))\n}\n\nfunc ArrayHas(field, element string) string {\n\treturn fmt.Sprintf(\"has(%s, %s)\", field, quote(element))\n}\n\nfunc In(field string, list []string) string {\n\tif len(list) == 1 {\n\t\treturn Eq(field, list[0])\n\t}\n\n\tvar buf strings.Builder\n\n\tbuf.WriteString(field)\n\tbuf.WriteString(\" IN (\")\n\n\tfor i, v := range list {\n\t\tif i > 0 {\n\t\t\tbuf.WriteByte(',')\n\t\t}\n\n\t\tbuf.WriteString(quote(v))\n\t}\n\n\tbuf.WriteByte(')')\n\n\treturn buf.String()\n}\n\nfunc InTable(field string, table string) string {\n\treturn fmt.Sprintf(\"%s in %s\", field, table)\n}\n\nfunc DateBetween(field string, from int64, until int64) string {\n\treturn fmt.Sprintf(\n\t\t\"%s >= '%s' AND %s <= '%s'\",\n\t\tfield, date.FromTimestampToDaysFormat(from), field, date.UntilTimestampToDaysFormat(until),\n\t)\n}\n\nfunc TimestampBetween(field string, from int64, until int64) string {\n\treturn fmt.Sprintf(\"%s >= %d AND %s <= %d\", field, from, field, until)\n}\n\ntype Where struct {\n\twhere string\n}\n\nfunc New() *Where {\n\treturn &Where{}\n}\n\nfunc (w *Where) And(exp string) {\n\tif exp == \"\" {\n\t\treturn\n\t}\n\n\tif w.where != \"\" {\n\t\tw.where = fmt.Sprintf(\"(%s) AND (%s)\", w.where, exp)\n\t} else {\n\t\tw.where = exp\n\t}\n}\n\nfunc (w *Where) Or(exp string) {\n\tif exp == \"\" {\n\t\treturn\n\t}\n\n\tif w.where != \"\" {\n\t\tw.where = fmt.Sprintf(\"(%s) OR (%s)\", w.where, exp)\n\t} else {\n\t\tw.where = exp\n\t}\n}\n\nfunc (w *Where) Andf(format string, obj ...interface{}) {\n\tw.And(fmt.Sprintf(format, obj...))\n}\n\nfunc (w *Where) String() string {\n\treturn w.where\n}\n\nfunc (w *Where) SQL() string {\n\tif w.where == \"\" {\n\t\treturn \"\"\n\t}\n\n\treturn \"WHERE \" + w.where\n}\n\nfunc (w *Where) PreWhereSQL() string {\n\tif w.where == \"\" {\n\t\treturn \"\"\n\t}\n\n\treturn \"PREWHERE \" + w.where\n}\n" }, { "path": "pkg/where/where_test.go", "content": "package where\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestGlobExpandSimple(t *testing.T) {\n\ttests := []struct {\n\t\tvalue string\n\t\twant []string\n\t\twantErr bool\n\t}{\n\t\t{\"{a,bc,d}\", []string{\"a\", \"bc\", \"d\"}, false},\n\t\t{\"S{a,bc,d}\", []string{\"Sa\", \"Sbc\", \"Sd\"}, false},\n\t\t{\"{a,bc,d}E\", []string{\"aE\", \"bcE\", \"dE\"}, false},\n\t\t{\"S{a,bc,d}E\", []string{\"SaE\", \"SbcE\", \"SdE\"}, false},\n\t\t{\"S{a,bc,d}E{f,h}\", []string{\"SaEf\", \"SaEh\", \"SbcEf\", \"SbcEh\", \"SdEf\", \"SdEh\"}, false},\n\t\t{\"test{a,b}\", []string{\"testa\", \"testb\"}, false},\n\t\t{\"S{a,bc,d}}E{f,h}\", nil, true}, //error\n\t\t{\"S{{a,bc,d}E{f,h}\", nil, true}, //error\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.value, func(t *testing.T) {\n\t\t\tvar got []string\n\n\t\t\terr := GlobExpandSimple(tt.value, \"\", &got)\n\t\t\tif tt.wantErr {\n\t\t\t\tassert.Error(t, err, \"Expand() not returns error for %v\", tt.value)\n\t\t\t} else {\n\t\t\t\tassert.NoErrorf(t, err, \"Expand() returns error %v for %v\", err, tt.value)\n\t\t\t}\n\n\t\t\tassert.Equal(t, tt.want, got, \"Expand() result\")\n\t\t})\n\t}\n}\n\nfunc TestGlobToRegexp(t *testing.T) {\n\ttable := []struct {\n\t\tglob string\n\t\tregexp string\n\t}{\n\t\t{`test.*.foo`, `test[.]([^.]*?)[.]foo`},\n\t\t{`test.{foo,bar}`, `test[.](foo|bar)`},\n\t\t{`test?.foo`, `test[^.][.]foo`},\n\t\t{`test?.$foo`, `test[^.][.][$]foo`},\n\t}\n\n\tfor _, test := range table {\n\t\ttestName := fmt.Sprintf(\"glob: %#v\", test.glob)\n\t\tregexp := GlobToRegexp(test.glob)\n\t\tassert.Equal(t, test.regexp, regexp, testName)\n\t}\n}\n\nfunc TestNonRegexpPrefix(t *testing.T) {\n\ttable := []struct {\n\t\texpr string\n\t\tprefix string\n\t}{\n\t\t{`test[.]([^.]*?)[.]foo`, `test`},\n\t\t{`__name__=cpu.load`, `__name__=cpu`},\n\t\t{`__name__=~(cpu|mem)`, `__name__=~`},\n\t\t{`__name__=~cpu|mem`, `__name__=~`},\n\t\t{`__name__=~^host`, `__name__=~`},\n\t}\n\n\tfor _, test := range table {\n\t\ttestName := fmt.Sprintf(\"expr: %#v\", test.expr)\n\t\tprefix := NonRegexpPrefix(test.expr)\n\t\tassert.Equal(t, test.prefix, prefix, testName)\n\t}\n}\n\nfunc TestMaxWildcardDistance(t *testing.T) {\n\ttable := []struct {\n\t\tglob string\n\t\tdist int\n\t}{\n\t\t{`a.b.c.d.e`, -1},\n\t\t{`test.*.foo.bar`, 2},\n\t\t{`test.foo.*.*.bar.count`, 2},\n\t\t{`test.foo.bar.*.bar.foo.test`, 3},\n\t\t{`test.foo.bar.foobar.*.middle.*.foobar.bar.foo.test`, 4},\n\t\t{`*.test.foo.bar.*`, 0},\n\t}\n\n\tfor _, test := range table {\n\t\ttestName := fmt.Sprintf(\"glob: %#v\", test.glob)\n\t\tdist := MaxWildcardDistance(test.glob)\n\t\tassert.Equal(t, test.dist, dist, testName)\n\t}\n}\n" }, { "path": "prometheus/.gitignore", "content": "tmp\n" }, { "path": "prometheus/empty_iterator.go", "content": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"github.com/prometheus/prometheus/model/histogram\"\n\t\"github.com/prometheus/prometheus/tsdb/chunkenc\"\n)\n\n// Iterator is a simple iterator that can only get the next value.\n// Iterator iterates over the samples of a time series, in timestamp-increasing order.\ntype emptyIterator struct{}\n\nvar emptyIteratorValue chunkenc.Iterator = &emptyIterator{}\n\n// Next advances the iterator by one and returns the type of the value\n// at the new position (or ValNone if the iterator is exhausted).\nfunc (it *emptyIterator) Next() chunkenc.ValueType { return chunkenc.ValNone }\n\n// Seek advances the iterator forward to the first sample with a\n// timestamp equal or greater than t. If the current sample found by a\n// previous `Next` or `Seek` operation already has this property, Seek\n// has no effect. If a sample has been found, Seek returns the type of\n// its value. Otherwise, it returns ValNone, after with the iterator is\n// exhausted.\nfunc (it *emptyIterator) Seek(t int64) chunkenc.ValueType { return chunkenc.ValNone }\n\n// At returns the current timestamp/value pair if the value is a float.\n// Before the iterator has advanced, the behaviour is unspecified.\nfunc (it *emptyIterator) At() (int64, float64) { return 0, 0 }\n\n// AtHistogram returns the current timestamp/value pair if the value is\n// a histogram with integer counts. Before the iterator has advanced,\n// the behaviour is unspecified.\nfunc (it *emptyIterator) AtHistogram(histogram *histogram.Histogram) (int64, *histogram.Histogram) {\n\treturn 0, nil\n}\n\n// AtFloatHistogram returns the current timestamp/value pair if the\n// value is a histogram with floating-point counts. It also works if the\n// value is a histogram with integer counts, in which case a\n// FloatHistogram copy of the histogram is returned. Before the iterator\n// has advanced, the behaviour is unspecified.\nfunc (it *emptyIterator) AtFloatHistogram(histogram *histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {\n\treturn 0, nil\n}\n\n// AtT returns the current timestamp.\n// Before the iterator has advanced, the behaviour is unspecified.\nfunc (it *emptyIterator) AtT() int64 { return 0 }\n\n// Err returns the current error. It should be used only after the\n// iterator is exhausted, i.e. `Next` or `Seek` have returned ValNone.\nfunc (it *emptyIterator) Err() error { return nil }\n" }, { "path": "prometheus/exemplar.go", "content": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"context\"\n\n\t\"github.com/prometheus/prometheus/model/exemplar\"\n\t\"github.com/prometheus/prometheus/model/labels\"\n\t\"github.com/prometheus/prometheus/storage\"\n)\n\ntype nopExemplarQueryable struct {\n}\n\ntype nopExemplarQuerier struct {\n}\n\nvar _ storage.ExemplarQueryable = &nopExemplarQueryable{}\nvar _ storage.ExemplarQuerier = &nopExemplarQuerier{}\n\nfunc (e *nopExemplarQueryable) ExemplarQuerier(ctx context.Context) (storage.ExemplarQuerier, error) {\n\treturn &nopExemplarQuerier{}, nil\n}\n\nfunc (e *nopExemplarQuerier) Select(start, end int64, matchers ...[]*labels.Matcher) ([]exemplar.QueryResult, error) {\n\treturn []exemplar.QueryResult{}, nil\n}\n" }, { "path": "prometheus/gatherer.go", "content": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"github.com/prometheus/client_golang/prometheus\"\n\tdto \"github.com/prometheus/client_model/go\"\n)\n\ntype nopGatherer struct{}\n\nvar _ prometheus.Gatherer = &nopGatherer{}\n\nfunc (*nopGatherer) Gather() ([]*dto.MetricFamily, error) {\n\treturn []*dto.MetricFamily{}, nil\n}\n" }, { "path": "prometheus/labels.go", "content": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"net/url\"\n\t\"sort\"\n\t\"strings\"\n\n\t\"github.com/prometheus/prometheus/model/labels\"\n)\n\nfunc urlParse(rawurl string) (*url.URL, error) {\n\tp := strings.IndexByte(rawurl, '?')\n\tif p < 0 {\n\t\treturn url.Parse(rawurl)\n\t}\n\n\tm, err := url.Parse(rawurl[p:])\n\tif m != nil {\n\t\tm.Path = rawurl[:p]\n\t}\n\n\treturn m, err\n}\n\nfunc Labels(path string) labels.Labels {\n\tu, err := urlParse(path)\n\tif err != nil {\n\t\treturn labels.Labels{labels.Label{Name: \"__name__\", Value: path}}\n\t}\n\n\tq := u.Query()\n\tlb := make(labels.Labels, len(q)+1)\n\tlb[0].Name = \"__name__\"\n\tlb[0].Value = u.Path\n\n\ti := 0\n\tfor k, v := range q {\n\t\ti++\n\t\tlb[i].Name = k\n\t\tlb[i].Value = v[0]\n\t}\n\n\tif len(lb) > 1 {\n\t\tsort.Slice(lb, func(i, j int) bool { return lb[i].Name < lb[j].Name })\n\t}\n\n\treturn lb\n}\n" }, { "path": "prometheus/labels_test.go", "content": "package prometheus\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestLabels(t *testing.T) {\n\tassert := assert.New(t)\n\n\ttable := [][2]string{\n\t\t{\n\t\t\t\"cpu_usage_system?cpu=cpu5&host=telegraf-b9468c8b5-g47xt&instance=telegraf.default%3A9273&job=telegraf\",\n\t\t\t`{__name__=\"cpu_usage_system\", cpu=\"cpu5\", host=\"telegraf-b9468c8b5-g47xt\", instance=\"telegraf.default:9273\", job=\"telegraf\"}`,\n\t\t},\n\t\t{\n\t\t\t\"cpu_usage_system\",\n\t\t\t`{__name__=\"cpu_usage_system\"}`,\n\t\t},\n\t\t{\n\t\t\t\":metric:?instance=localhost\",\n\t\t\t`{__name__=\":metric:\", instance=\"localhost\"}`,\n\t\t},\n\t}\n\n\tfor _, c := range table {\n\t\tassert.Equal(c[1], Labels(c[0]).String())\n\t}\n}\n" }, { "path": "prometheus/local_storage.go", "content": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"context\"\n\n\t\"github.com/prometheus/prometheus/model/labels\"\n\t\"github.com/prometheus/prometheus/tsdb\"\n\t\"github.com/prometheus/prometheus/tsdb/index\"\n\t\"github.com/prometheus/prometheus/web\"\n)\n\nvar _ web.LocalStorage = &storageImpl{}\n\nfunc (s *storageImpl) CleanTombstones() error {\n\treturn nil\n}\n\nfunc (s *storageImpl) Delete(ctx context.Context, mint, maxt int64, ms ...*labels.Matcher) error {\n\treturn nil\n}\n\nfunc (s *storageImpl) Snapshot(dir string, withHead bool) error {\n\treturn nil\n}\n\nfunc (s *storageImpl) Stats(statsByLabelName string, limit int) (*tsdb.Stats, error) {\n\treturn &tsdb.Stats{\n\t\tIndexPostingStats: &index.PostingsStats{},\n\t}, nil\n}\n\nfunc (s *storageImpl) WALReplayStatus() (tsdb.WALReplayStatus, error) {\n\treturn tsdb.WALReplayStatus{}, nil\n}\n" }, { "path": "prometheus/logger.go", "content": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"go.uber.org/zap\"\n)\n\ntype errorLevel interface {\n\tString() string\n}\n\ntype logger struct {\n\tz *zap.Logger\n}\n\nfunc (l *logger) Log(keyvals ...interface{}) error {\n\tlg := l.z\n\n\tvar msg string\n\n\tvar level errorLevel\n\n\tfor i := 1; i < len(keyvals); i += 2 {\n\t\tkeyObj := keyvals[i-1]\n\n\t\tkeyStr, ok := keyObj.(string)\n\t\tif !ok {\n\t\t\tl.z.Error(\"can't handle log, wrong key\", zap.Any(\"keyvals\", keyvals))\n\t\t\treturn nil\n\t\t}\n\n\t\tif keyStr == \"level\" {\n\t\t\tlevel, ok = keyvals[i].(errorLevel)\n\t\t\tif !ok {\n\t\t\t\tl.z.Error(\"can't handle log, wrong level\", zap.Any(\"keyvals\", keyvals))\n\t\t\t\treturn nil\n\t\t\t}\n\n\t\t\tcontinue\n\t\t}\n\n\t\tif keyStr == \"msg\" {\n\t\t\tmsg, ok = keyvals[i].(string)\n\t\t\tif !ok {\n\t\t\t\tl.z.Error(\"can't handle log, wrong msg\", zap.Any(\"keyvals\", keyvals))\n\t\t\t\treturn nil\n\t\t\t}\n\n\t\t\tcontinue\n\t\t}\n\n\t\tlg = lg.With(zap.Any(keyStr, keyvals[i]))\n\t}\n\n\tswitch level.String() {\n\tcase \"debug\":\n\t\tlg.Debug(msg)\n\tcase \"info\":\n\t\tlg.Info(msg)\n\tcase \"warn\":\n\t\tlg.Warn(msg)\n\tcase \"error\":\n\t\tlg.Error(msg)\n\tdefault:\n\t\tl.z.Error(\"can't handle log, unknown level\", zap.Any(\"keyvals\", keyvals))\n\t\treturn nil\n\t}\n\n\treturn nil\n}\n" }, { "path": "prometheus/matcher.go", "content": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\n\t\"github.com/lomik/graphite-clickhouse/finder\"\n\t\"github.com/prometheus/prometheus/model/labels\"\n\t\"github.com/prometheus/prometheus/prompb\"\n)\n\nvar prompbMatchMap = map[prompb.LabelMatcher_Type]finder.TaggedTermOp{\n\tprompb.LabelMatcher_EQ: finder.TaggedTermEq,\n\tprompb.LabelMatcher_RE: finder.TaggedTermMatch,\n\tprompb.LabelMatcher_NEQ: finder.TaggedTermNe,\n\tprompb.LabelMatcher_NRE: finder.TaggedTermNotMatch,\n}\n\nvar promqlMatchMap = map[labels.MatchType]finder.TaggedTermOp{\n\tlabels.MatchEqual: finder.TaggedTermEq,\n\tlabels.MatchNotEqual: finder.TaggedTermNe,\n\tlabels.MatchRegexp: finder.TaggedTermMatch,\n\tlabels.MatchNotRegexp: finder.TaggedTermNotMatch,\n}\n\nfunc makeTaggedFromPromPB(matchers []*prompb.LabelMatcher) ([]finder.TaggedTerm, error) {\n\tterms := make([]finder.TaggedTerm, 0, len(matchers))\n\n\tfor i := 0; i < len(matchers); i++ {\n\t\tif matchers[i] == nil {\n\t\t\tcontinue\n\t\t}\n\n\t\top, ok := prompbMatchMap[matchers[i].Type]\n\t\tif !ok {\n\t\t\treturn nil, fmt.Errorf(\"unknown matcher type %#v\", matchers[i].GetType())\n\t\t}\n\n\t\tterms = append(terms, finder.TaggedTerm{\n\t\t\tKey: matchers[i].Name,\n\t\t\tValue: matchers[i].Value,\n\t\t\tOp: op,\n\t\t})\n\t}\n\n\tsort.Sort(finder.TaggedTermList(terms))\n\n\treturn terms, nil\n}\n\nfunc makeTaggedFromPromQL(matchers []*labels.Matcher) ([]finder.TaggedTerm, error) {\n\tterms := make([]finder.TaggedTerm, 0, len(matchers))\n\n\tfor i := 0; i < len(matchers); i++ {\n\t\tif matchers[i] == nil {\n\t\t\tcontinue\n\t\t}\n\n\t\top, ok := promqlMatchMap[matchers[i].Type]\n\t\tif !ok {\n\t\t\treturn nil, fmt.Errorf(\"unknown matcher type %#v\", matchers[i].Type)\n\t\t}\n\n\t\tterms = append(terms, finder.TaggedTerm{\n\t\t\tKey: matchers[i].Name,\n\t\t\tValue: matchers[i].Value,\n\t\t\tOp: op,\n\t\t})\n\t}\n\n\tsort.Sort(finder.TaggedTermList(terms))\n\n\treturn terms, nil\n}\n\n// func wherePromPB(matchers []*prompb.LabelMatcher) (string, error) {\n// \tif len(matchers) == 0 {\n// \t\treturn \"\", nil\n// \t}\n\n// \tterms := make([]finder.TaggedTerm, 0, len(matchers))\n// \tfor i := 0; i < len(matchers); i++ {\n// \t\tif matchers[i] == nil {\n// \t\t\tcontinue\n// \t\t}\n// \t\top, ok := prompbMatchMap[matchers[i].Type]\n// \t\tif !ok {\n// \t\t\treturn \"\", fmt.Errorf(\"unknown matcher type %#v\", matchers[i].GetType())\n// \t\t}\n// \t\tterms = append(terms, finder.TaggedTerm{\n// \t\t\tKey: matchers[i].Name,\n// \t\t\tValue: matchers[i].Value,\n// \t\t\tOp: op,\n// \t\t})\n// \t}\n\n// \tsort.Sort(finder.TaggedTermList(terms))\n\n// \tw := where.New()\n// \tw.And(finder.TaggedTermWhere1(&terms[0]))\n\n// \tfor i := 1; i < len(terms); i++ {\n// \t\tw.And(finder.TaggedTermWhereN(&terms[i]))\n// \t}\n\n// \treturn w.String(), nil\n// }\n\n// func wherePromQL(matchers []*labels.Matcher) (string, error) {\n// \tif len(matchers) == 0 {\n// \t\treturn \"\", nil\n// \t}\n\n// \tterms := make([]finder.TaggedTerm, 0, len(matchers))\n// \tfor i := 0; i < len(matchers); i++ {\n// \t\tif matchers[i] == nil {\n// \t\t\tcontinue\n// \t\t}\n// \t\top, ok := promqlMatchMap[matchers[i].Type]\n// \t\tif !ok {\n// \t\t\treturn \"\", fmt.Errorf(\"unknown matcher type %#v\", matchers[i].Type)\n// \t\t}\n// \t\tterms = append(terms, finder.TaggedTerm{\n// \t\t\tKey: matchers[i].Name,\n// \t\t\tValue: matchers[i].Value,\n// \t\t\tOp: op,\n// \t\t})\n// \t}\n\n// \tsort.Sort(finder.TaggedTermList(terms))\n\n// \tw := where.New()\n// \tw.And(finder.TaggedTermWhere1(&terms[0]))\n\n// \tfor i := 1; i < len(terms); i++ {\n// \t\tw.And(finder.TaggedTermWhereN(&terms[i]))\n// \t}\n\n// \treturn w.String(), nil\n// }\n" }, { "path": "prometheus/metrics_set.go", "content": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"github.com/prometheus/prometheus/model/labels\"\n\t\"github.com/prometheus/prometheus/storage\"\n\t\"github.com/prometheus/prometheus/tsdb/chunkenc\"\n\t\"github.com/prometheus/prometheus/util/annotations\"\n)\n\n// SeriesSet contains a set of series.\ntype metricsSet struct {\n\tmetrics []string\n\tcurrent int\n}\n\ntype metric struct {\n\tname string\n}\n\nvar _ storage.SeriesSet = &metricsSet{}\n\nfunc (ms *metricsSet) At() storage.Series {\n\treturn &metric{name: ms.metrics[ms.current]}\n}\n\n// Iterator returns a new iterator of the data of the series.\nfunc (s *metric) Iterator(iterator chunkenc.Iterator) chunkenc.Iterator {\n\treturn emptyIteratorValue\n}\n\nfunc (s *metric) Labels() labels.Labels {\n\treturn Labels(s.name)\n}\n\n// Err returns the current error.\nfunc (ms *metricsSet) Err() error { return nil }\n\nfunc (ms *metricsSet) Next() bool {\n\tif ms.current < 0 {\n\t\tms.current = 0\n\t} else {\n\t\tms.current++\n\t}\n\n\treturn ms.current < len(ms.metrics)\n}\n\nfunc newMetricsSet(metrics []string) storage.SeriesSet {\n\treturn &metricsSet{metrics: metrics, current: -1}\n}\n\n// Warnings ...\nfunc (s *metricsSet) Warnings() annotations.Annotations { return nil }\n" }, { "path": "prometheus/querier.go", "content": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/prometheus/prometheus/storage\"\n\t\"github.com/prometheus/prometheus/util/annotations\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/where\"\n\t\"github.com/prometheus/prometheus/model/labels\"\n)\n\n// Querier provides reading access to time series data.\ntype Querier struct {\n\tconfig *config.Config\n\tmint int64\n\tmaxt int64\n}\n\n// Close releases the resources of the Querier.\nfunc (q *Querier) Close() error {\n\treturn nil\n}\n\n// LabelValues returns all potential values for a label name.\nfunc (q *Querier) LabelValues(ctx context.Context, label string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {\n\t// @TODO: support matchers\n\tw := where.New()\n\tw.And(where.HasPrefix(\"Tag1\", label+\"=\"))\n\n\tfromDate := timeNow().AddDate(0, 0, -q.config.ClickHouse.TaggedAutocompleDays)\n\tw.Andf(\"Date >= '%s'\", fromDate.Format(\"2006-01-02\"))\n\n\tsql := fmt.Sprintf(\"SELECT splitByChar('=', Tag1)[2] as value FROM %s %s GROUP BY value ORDER BY value\",\n\t\tq.config.ClickHouse.TaggedTable,\n\t\tw.SQL(),\n\t)\n\n\tbody, _, _, err := clickhouse.Query(\n\t\tscope.WithTable(ctx, q.config.ClickHouse.TaggedTable),\n\t\tq.config.ClickHouse.URL,\n\t\tsql,\n\t\tclickhouse.Options{\n\t\t\tTLSConfig: q.config.ClickHouse.TLSConfig,\n\t\t\tTimeout: q.config.ClickHouse.IndexTimeout,\n\t\t\tConnectTimeout: q.config.ClickHouse.ConnectTimeout,\n\t\t\tCheckRequestProgress: q.config.FeatureFlags.LogQueryProgress,\n\t\t\tProgressSendingInterval: q.config.ClickHouse.ProgressSendingInterval,\n\t\t},\n\t\tnil,\n\t)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\trows := strings.Split(string(body), \"\\n\")\n\tif len(rows) > 0 && rows[len(rows)-1] == \"\" {\n\t\trows = rows[:len(rows)-1]\n\t}\n\n\treturn rows, nil, nil\n}\n\n// LabelNames returns all the unique label names present in the block in sorted order.\nfunc (q *Querier) LabelNames(ctx context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {\n\t// @TODO support matchers\n\tw := where.New()\n\tfromDate := time.Now().AddDate(0, 0, -q.config.ClickHouse.TaggedAutocompleDays).UTC()\n\tw.Andf(\"Date >= '%s'\", fromDate.Format(\"2006-01-02\"))\n\n\tsql := fmt.Sprintf(\"SELECT splitByChar('=', Tag1)[1] as value FROM %s %s GROUP BY value ORDER BY value\",\n\t\tq.config.ClickHouse.TaggedTable,\n\t\tw.SQL(),\n\t)\n\n\tbody, _, _, err := clickhouse.Query(\n\t\tscope.WithTable(ctx, q.config.ClickHouse.TaggedTable),\n\t\tq.config.ClickHouse.URL,\n\t\tsql,\n\t\tclickhouse.Options{\n\t\t\tTimeout: q.config.ClickHouse.IndexTimeout,\n\t\t\tConnectTimeout: q.config.ClickHouse.ConnectTimeout,\n\t\t\tTLSConfig: q.config.ClickHouse.TLSConfig,\n\t\t\tCheckRequestProgress: q.config.FeatureFlags.LogQueryProgress,\n\t\t\tProgressSendingInterval: q.config.ClickHouse.ProgressSendingInterval,\n\t\t},\n\t\tnil,\n\t)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\trows := strings.Split(string(body), \"\\n\")\n\tif len(rows) > 0 && rows[len(rows)-1] == \"\" {\n\t\trows = rows[:len(rows)-1]\n\t}\n\n\treturn rows, nil, nil\n}\n" }, { "path": "prometheus/querier_select.go", "content": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"context\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/finder\"\n\t\"github.com/lomik/graphite-clickhouse/limiter\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/alias\"\n\t\"github.com/lomik/graphite-clickhouse/render/data\"\n\t\"github.com/prometheus/prometheus/model/labels\"\n\t\"github.com/prometheus/prometheus/storage\"\n)\n\n// override in unit tests for stable results\nvar timeNow = time.Now\n\nfunc (q *Querier) lookup(ctx context.Context, from, until int64, qlimiter limiter.ServerLimiter, queueDuration *time.Duration, labelsMatcher ...*labels.Matcher) (*alias.Map, error) {\n\tterms, err := makeTaggedFromPromQL(labelsMatcher)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar (\n\t\tlimitCtx context.Context\n\t\tcancel context.CancelFunc\n\t)\n\n\tif qlimiter.Enabled() {\n\t\tlimitCtx, cancel = context.WithTimeout(ctx, q.config.ClickHouse.IndexTimeout)\n\t\tdefer cancel()\n\n\t\tstart := time.Now()\n\t\terr = qlimiter.Enter(limitCtx, \"render\")\n\t\t*queueDuration += time.Since(start)\n\n\t\tif err != nil {\n\t\t\t// status = http.StatusServiceUnavailable\n\t\t\t// queueFail = true\n\t\t\t// http.Error(w, err.Error(), status)\n\t\t\treturn nil, err\n\t\t}\n\n\t\tdefer qlimiter.Leave(limitCtx, \"render\")\n\t}\n\t// TODO: implement use stat for Prometheus queries\n\tfndResult, err := finder.FindTagged(ctx, q.config, terms, from, until)\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tam := alias.New()\n\tam.Merge(fndResult, false)\n\n\treturn am, nil\n}\n\nfunc (q *Querier) timeRange(hints *storage.SelectHints) (int64, int64) {\n\tvar from, until time.Time\n\n\t// ClickHouse supported range of values by the Date type: [1970-01-01, 2149-06-06]\n\tif hints != nil && hints.Start > 0 && hints.Start < 5662310400000 {\n\t\tfrom = time.Unix(hints.Start/1000, (hints.Start%1000)*1000000)\n\t}\n\n\tif hints != nil && hints.End > 0 && hints.End < 5662310400000 {\n\t\tuntil = time.Unix(hints.End/1000, (hints.End%1000)*1000000)\n\t}\n\n\tif until.IsZero() {\n\t\tif q.maxt > 0 && q.maxt < 5662310400000 {\n\t\t\tuntil = time.Unix(q.maxt/1000, (q.maxt%1000)*1000000)\n\t\t} else {\n\t\t\tuntil = timeNow()\n\t\t}\n\t}\n\n\tif from.IsZero() {\n\t\tif q.mint > 0 && q.mint < 5662310400000 {\n\t\t\tfrom = time.Unix(q.mint/1000, (q.mint%1000)*1000000)\n\t\t} else {\n\t\t\tfrom = until.AddDate(0, 0, -q.config.ClickHouse.TaggedAutocompleDays)\n\t\t}\n\t}\n\n\treturn from.Unix(), until.Unix()\n}\n\n// Select returns a set of series that matches the given label matchers.\nfunc (q *Querier) Select(ctx context.Context, sortSeries bool, hints *storage.SelectHints, labelsMatcher ...*labels.Matcher) storage.SeriesSet {\n\tvar (\n\t\tqueueDuration time.Duration\n\t)\n\n\tfrom, until := q.timeRange(hints)\n\tqlimiter := data.GetQueryLimiterFrom(\"\", q.config, from, until)\n\n\tam, err := q.lookup(ctx, from, until, qlimiter, &queueDuration, labelsMatcher...)\n\tif err != nil {\n\t\treturn nil //, nil, err @TODO\n\t}\n\n\tif am.Len() == 0 {\n\t\treturn emptySeriesSet()\n\t}\n\n\tif hints != nil && hints.Func == \"series\" {\n\t\t// /api/v1/series?match[]=...\n\t\treturn newMetricsSet(am.DisplayNames()) //, nil, nil\n\t}\n\n\tvar step int64 = 60000\n\tif hints.Step != 0 {\n\t\tstep = hints.Step\n\t}\n\n\tmaxDataPoints := 1000 * (until - from) / step\n\n\tmultiTarget := data.MultiTarget{\n\t\tdata.TimeFrame{\n\t\t\tFrom: from,\n\t\t\tUntil: until,\n\t\t\tMaxDataPoints: maxDataPoints,\n\t\t}: data.NewTargets([]string{}, am),\n\t}\n\n\treply, err := multiTarget.Fetch(ctx, q.config, config.ContextPrometheus, qlimiter, &queueDuration)\n\tif err != nil {\n\t\treturn nil // , nil, err @TODO\n\t}\n\n\tif len(reply) == 0 {\n\t\treturn emptySeriesSet() //, nil, nil\n\t}\n\n\tss, err := makeSeriesSet(reply[0].Data, step)\n\tif err != nil {\n\t\treturn nil // , nil, err @TODO\n\t}\n\n\treturn ss //, nil, nil\n}\n" }, { "path": "prometheus/querier_select_test.go", "content": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/prometheus/prometheus/storage\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestQuerier_timeRange(t *testing.T) {\n\ttimeNow = func() time.Time {\n\t\t// 2022-11-29 09:30:47 UTC\n\t\treturn time.Unix(1669714247, 0)\n\t}\n\tcfg := &config.Config{\n\t\tClickHouse: config.ClickHouse{\n\t\t\tTaggedAutocompleDays: 4,\n\t\t},\n\t}\n\n\ttests := []struct {\n\t\tname string\n\n\t\tmint int64\n\t\tmaxt int64\n\t\thints *storage.SelectHints\n\n\t\twantFrom int64\n\t\twantUntil int64\n\t}{\n\t\t{\n\t\t\tname: \"default from/until\",\n\t\t\twantFrom: 1669368647, // timeNow() - config.Clickhouse.TaggedAutocompleDays\n\t\t\twantUntil: 1669714247, // timeNow() result\n\t\t},\n\t\t{\n\t\t\tname: \"start/end in SelectHints\",\n\t\t\thints: &storage.SelectHints{\n\t\t\t\tStart: 1669453200000,\n\t\t\t\tEnd: 1669626000000,\n\t\t\t},\n\t\t\twantFrom: 1669453200,\n\t\t\twantUntil: 1669626000,\n\t\t},\n\t\t{\n\t\t\tname: \"start/end in SelectHints overflow\",\n\t\t\thints: &storage.SelectHints{\n\t\t\t\t// ClickHouse supported range of values by the Date type: [1970-01-01, 2149-06-06]\n\t\t\t\tStart: 5662310400001,\n\t\t\t\tEnd: 5662310400100,\n\t\t\t},\n\t\t\twantFrom: 1669368647, // timeNow() - config.Clickhouse.TaggedAutocompleDays\n\t\t\twantUntil: 1669714247, // timeNow() result\n\t\t},\n\t\t{\n\t\t\tname: \"no start/end in SelectHints\",\n\t\t\thints: &storage.SelectHints{},\n\t\t\tmint: 1669194000000,\n\t\t\tmaxt: 1669280400000,\n\t\t\twantFrom: 1669194000,\n\t\t\twantUntil: 1669280400,\n\t\t},\n\t\t{\n\t\t\tname: \"no start/end in SelectHints, mint/maxt overflow\",\n\t\t\thints: &storage.SelectHints{},\n\t\t\t// ClickHouse supported range of values by the Date type: [1970-01-01, 2149-06-06]\n\t\t\tmint: 5662310400001,\n\t\t\tmaxt: 5662310400100,\n\t\t\twantFrom: 1669368647, // timeNow() - config.Clickhouse.TaggedAutocompleDays\n\t\t\twantUntil: 1669714247, // timeNow() result\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\ts := newStorage(cfg)\n\n\t\t\t// Querier returns a new Querier on the storage.\n\t\t\tsq, err := s.Querier(tt.mint, tt.maxt)\n\t\t\trequire.NoError(t, err)\n\n\t\t\tq := sq.(*Querier)\n\n\t\t\tgotFrom, gotUntil := q.timeRange(tt.hints)\n\t\t\tif gotFrom != tt.wantFrom {\n\t\t\t\tt.Errorf(\"Querier.timeRange().from got = %v, want %v\", gotFrom, tt.wantFrom)\n\t\t\t}\n\n\t\t\tif gotUntil != tt.wantUntil {\n\t\t\t\tt.Errorf(\"Querier.timeRange().until got = %v, want %v\", gotUntil, tt.wantUntil)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "prometheus/run.go", "content": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"context\"\n\t\"log\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/grafana/regexp\"\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/zapwriter\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\tpromConfig \"github.com/prometheus/prometheus/config\"\n\t\"github.com/prometheus/prometheus/notifier\"\n\t\"github.com/prometheus/prometheus/promql\"\n\t\"github.com/prometheus/prometheus/rules\"\n\t\"github.com/prometheus/prometheus/scrape\"\n\t\"github.com/prometheus/prometheus/web\"\n\t\"github.com/prometheus/prometheus/web/ui\"\n\n\tuiStatic \"github.com/lomik/prometheus-ui-static\"\n\t\"github.com/prometheus/common/assets\"\n)\n\nfunc Run(config *config.Config) error {\n\t// use precompiled static from github.com/lomik/prometheus-ui-static\n\tui.Assets = http.FS(assets.New(uiStatic.EmbedFS))\n\n\tzapLogger := &logger{\n\t\tz: zapwriter.Logger(\"prometheus\"),\n\t}\n\n\tstorage := newStorage(config)\n\n\tcorsOrigin, err := regexp.Compile(\"^$\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tqueryEngine := promql.NewEngine(promql.EngineOpts{\n\t\tLogger: zapLogger,\n\t\tTimeout: time.Minute,\n\t\tMaxSamples: 50000000,\n\t\tLookbackDelta: config.Prometheus.LookbackDelta,\n\t})\n\n\tscrapeManager, err := scrape.NewManager(&scrape.Options{}, zapLogger, storage, prometheus.DefaultRegisterer)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\trulesManager := rules.NewManager(&rules.ManagerOptions{\n\t\tLogger: zapLogger,\n\t\tAppendable: storage,\n\t\tQueryable: storage,\n\t})\n\n\tnotifierManager := notifier.NewManager(¬ifier.Options{}, zapLogger)\n\n\tpromHandler := web.New(zapLogger, &web.Options{\n\t\tListenAddress: config.Prometheus.Listen,\n\t\tMaxConnections: 500,\n\t\tStorage: storage,\n\t\tExemplarStorage: &nopExemplarQueryable{},\n\t\tExternalURL: config.Prometheus.ExternalURL,\n\t\tRoutePrefix: \"/\",\n\t\tQueryEngine: queryEngine,\n\t\tScrapeManager: scrapeManager,\n\t\tRuleManager: rulesManager,\n\t\tFlags: make(map[string]string),\n\t\tLocalStorage: storage,\n\t\tGatherer: &nopGatherer{},\n\t\tNotifier: notifierManager,\n\t\tCORSOrigin: corsOrigin,\n\t\tPageTitle: config.Prometheus.PageTitle,\n\t\tLookbackDelta: config.Prometheus.LookbackDelta,\n\t\tRemoteReadConcurrencyLimit: config.Prometheus.RemoteReadConcurrencyLimit,\n\t})\n\n\tpromHandler.ApplyConfig(&promConfig.Config{})\n\tpromHandler.SetReady(true)\n\n\tgo func() {\n\t\tlog.Fatal(promHandler.Run(context.Background(), nil, \"\"))\n\t}()\n\n\treturn nil\n}\n" }, { "path": "prometheus/run_dummy.go", "content": "//go:build noprom\n// +build noprom\n\npackage prometheus\n\nimport (\n\t\"github.com/lomik/graphite-clickhouse/config\"\n)\n\nfunc Run(config *config.Config) error {\n\treturn nil\n}\n" }, { "path": "prometheus/series_set.go", "content": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"log\"\n\t\"math\"\n\n\t\"github.com/prometheus/prometheus/util/annotations\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/point\"\n\n\t\"github.com/lomik/graphite-clickhouse/render/data\"\n\t\"github.com/prometheus/prometheus/model/histogram\"\n\t\"github.com/prometheus/prometheus/model/labels\"\n\t\"github.com/prometheus/prometheus/storage\"\n\t\"github.com/prometheus/prometheus/tsdb/chunkenc\"\n)\n\n// SeriesIterator iterates over the data of a time series.\ntype seriesIterator struct {\n\tmetricName string\n\tpoints []point.Point\n\tcurrent int\n\tstep int64\n}\n\n// Series represents a single time series.\ntype series struct {\n\tmetricName string\n\tpoints []point.Point\n\tstep int64\n}\n\n// SeriesSet contains a set of series.\ntype seriesSet struct {\n\tseries []series\n\tcurrent int\n}\n\nvar _ storage.SeriesSet = &seriesSet{}\n\nfunc makeSeriesSet(data *data.Data, step int64) (storage.SeriesSet, error) {\n\tss := &seriesSet{series: make([]series, 0), current: -1}\n\tif data == nil {\n\t\treturn ss, nil\n\t}\n\n\tif data.Len() == 0 {\n\t\treturn ss, nil\n\t}\n\n\tnextMetric := data.GroupByMetric()\n\n\tfor {\n\t\tpoints := nextMetric()\n\t\tif len(points) == 0 {\n\t\t\tbreak\n\t\t}\n\n\t\tmetricName := data.MetricName(points[0].MetricID)\n\t\tfor _, v := range data.AM.Get(metricName) {\n\t\t\tss.series = append(ss.series, series{metricName: v.DisplayName, points: points, step: step})\n\t\t}\n\t}\n\n\treturn ss, nil\n}\n\nfunc emptySeriesSet() storage.SeriesSet {\n\treturn &seriesSet{series: make([]series, 0), current: -1}\n}\n\n// func (sit *seriesIterator) logger() *zap.Logger {\n// \treturn zap.L() //.With(zap.String(\"metric\", sit.metricName))\n// }\n\n// Seek advances the iterator forward to the value at or after\n// the given timestamp.\nfunc (sit *seriesIterator) Seek(t int64) chunkenc.ValueType {\n\ttt := uint32(t / 1000)\n\tif t%1000 != 0 {\n\t\ttt++\n\t}\n\n\tfor ; sit.current < len(sit.points); sit.current++ {\n\t\tif sit.points[sit.current].Time >= tt {\n\t\t\t// sit.logger().Debug(\"seriesIterator.Seek\", zap.Int64(\"t\", t), zap.Bool(\"ret\", true))\n\t\t\treturn chunkenc.ValFloat\n\t\t}\n\t}\n\n\t// sit.logger().Debug(\"seriesIterator.Seek\", zap.Int64(\"t\", t), zap.Bool(\"ret\", false))\n\treturn chunkenc.ValNone\n}\n\n// At returns the current timestamp/value pair.\nfunc (sit *seriesIterator) At() (t int64, v float64) {\n\tindex := sit.current\n\tif index == len(sit.points) {\n\t\treturn int64(sit.points[len(sit.points)-1].Time)*1000 + sit.step, math.NaN()\n\t}\n\n\tif index < 0 || index >= len(sit.points) {\n\t\tindex = 0\n\t}\n\n\tp := sit.points[index]\n\t// sit.logger().Debug(\"seriesIterator.At\", zap.Int64(\"t\", int64(p.Time)*1000), zap.Float64(\"v\", p.Value))\n\treturn int64(p.Time) * 1000, p.Value\n}\n\n// AtHistogram returns the current timestamp/value pair if the value is\n// a histogram with integer counts. Before the iterator has advanced,\n// the behaviour is unspecified.\nfunc (sit *seriesIterator) AtHistogram(histogram *histogram.Histogram) (int64, *histogram.Histogram) {\n\tlog.Fatal(\"seriesIterator.AtHistogram not implemented\")\n\treturn 0, nil // @TODO\n}\n\n// AtFloatHistogram returns the current timestamp/value pair if the\n// value is a histogram with floating-point counts. It also works if the\n// value is a histogram with integer counts, in which case a\n// FloatHistogram copy of the histogram is returned. Before the iterator\n// has advanced, the behaviour is unspecified.\nfunc (sit *seriesIterator) AtFloatHistogram(histogram *histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {\n\tlog.Fatal(\"seriesIterator.AtFloatHistogram not implemented\")\n\treturn 0, nil // @TODO\n}\n\n// AtT returns the current timestamp.\n// Before the iterator has advanced, the behaviour is unspecified.\nfunc (sit *seriesIterator) AtT() int64 {\n\tt, _ := sit.At()\n\treturn t\n}\n\n// Next advances the iterator by one.\nfunc (sit *seriesIterator) Next() chunkenc.ValueType {\n\tif sit.current < len(sit.points) {\n\t\tif sit.step == 0 && sit.current == len(sit.points)-1 {\n\t\t\treturn chunkenc.ValNone\n\t\t}\n\n\t\tsit.current++\n\t\t// sit.logger().Debug(\"seriesIterator.Next\", zap.Bool(\"ret\", true))\n\t\treturn chunkenc.ValFloat\n\t}\n\t// sit.logger().Debug(\"seriesIterator.Next\", zap.Bool(\"ret\", false))\n\treturn chunkenc.ValNone\n}\n\n// Err returns the current error.\nfunc (sit *seriesIterator) Err() error { return nil }\n\n// Err returns the current error.\nfunc (ss *seriesSet) Err() error { return nil }\n\nfunc (ss *seriesSet) At() storage.Series {\n\tif ss == nil || ss.current < 0 || ss.current >= len(ss.series) {\n\t\t// zap.L().Debug(\"seriesSet.At\", zap.String(\"metricName\", \"nil\"))\n\t\treturn nil\n\t}\n\n\ts := &ss.series[ss.current]\n\t// zap.L().Debug(\"seriesSet.At\", zap.String(\"metricName\", s.name()))\n\treturn s\n}\n\nfunc (ss *seriesSet) Next() bool {\n\tif ss == nil || ss.current+1 >= len(ss.series) {\n\t\t// zap.L().Debug(\"seriesSet.Next\", zap.Bool(\"ret\", false))\n\t\treturn false\n\t}\n\n\tss.current++\n\t// zap.L().Debug(\"seriesSet.Next\", zap.Bool(\"ret\", true))\n\treturn true\n}\n\n// Warnings ...\nfunc (s *seriesSet) Warnings() annotations.Annotations {\n\treturn nil\n}\n\n// Iterator returns a new iterator of the data of the series.\nfunc (s *series) Iterator(iterator chunkenc.Iterator) chunkenc.Iterator {\n\treturn &seriesIterator{metricName: s.metricName, points: s.points, current: -1, step: s.step}\n}\n\nfunc (s *series) name() string {\n\treturn s.metricName\n}\n\nfunc (s *series) Labels() labels.Labels {\n\treturn Labels(s.name())\n}\n" }, { "path": "prometheus/storage.go", "content": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"context\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/prometheus/prometheus/storage\"\n)\n\ntype storageImpl struct {\n\tconfig *config.Config\n}\n\nvar _ storage.Storage = &storageImpl{}\n\nfunc newStorage(config *config.Config) *storageImpl {\n\treturn &storageImpl{config: config}\n}\n\n// Querier returns a new Querier on the storage.\nfunc (s *storageImpl) Querier(mint, maxt int64) (storage.Querier, error) {\n\treturn &Querier{\n\t\tconfig: s.config,\n\t\tmint: mint,\n\t\tmaxt: maxt,\n\t}, nil\n}\n\n// ChunkQuerier ...\nfunc (s *storageImpl) ChunkQuerier(mint, maxt int64) (storage.ChunkQuerier, error) {\n\treturn nil, nil\n}\n\n// Appender ...\nfunc (s *storageImpl) Appender(ctx context.Context) storage.Appender {\n\treturn nil\n}\n\n// StartTime ...\nfunc (s *storageImpl) StartTime() (int64, error) {\n\treturn 0, nil\n}\n\n// Close ...\nfunc (s *storageImpl) Close() error {\n\treturn nil\n}\n" }, { "path": "render/data/carbonlink.go", "content": "package data\n\nimport (\n\t\"context\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/point\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"go.uber.org/zap\"\n\n\tgraphitePickle \"github.com/lomik/graphite-pickle\"\n)\n\ntype carbonlinkFetcher interface {\n\tCacheQueryMulti(context.Context, []string) (map[string][]graphitePickle.DataPoint, error)\n}\n\n// carbonlink to get data from carbonlink server globally\ntype carbonlinkClient struct {\n\tcarbonlinkFetcher\n\ttotalTimeout time.Duration\n}\n\nvar carbonlink *carbonlinkClient = nil\n\n// setCarbonlinkClient setup the client once. Does nothing if Config.Carbonlink.Server is not set\nfunc setCarbonlinkClient(config *config.Carbonlink) {\n\tif carbonlink != nil {\n\t\treturn\n\t}\n\n\tif config.Server == \"\" {\n\t\treturn\n\t}\n\n\tcarbonlink = &carbonlinkClient{\n\t\tgraphitePickle.NewCarbonlinkClient(\n\t\t\tconfig.Server,\n\t\t\tconfig.Retries,\n\t\t\tconfig.Threads,\n\t\t\tconfig.ConnectTimeout,\n\t\t\tconfig.QueryTimeout,\n\t\t),\n\t\tconfig.TotalTimeout,\n\t}\n\n\treturn\n}\n\n// queryCarbonlink returns callable result fetcher\nfunc queryCarbonlink(parentCtx context.Context, carbonlink *carbonlinkClient, metrics []string) func() *point.Points {\n\tlogger := scope.Logger(parentCtx)\n\n\tif carbonlink == nil {\n\t\treturn func() *point.Points { return nil }\n\t}\n\n\tcarbonlinkResponseChan := make(chan *point.Points, 1)\n\n\tfetchResult := func() *point.Points {\n\t\tresult := <-carbonlinkResponseChan\n\t\treturn result\n\t}\n\n\tgo func() {\n\t\tctx, cancel := context.WithTimeout(parentCtx, carbonlink.totalTimeout)\n\t\tdefer cancel()\n\n\t\tres, err := carbonlink.CacheQueryMulti(ctx, metrics)\n\n\t\tif err != nil {\n\t\t\tlogger.Info(\"carbonlink failed\", zap.Error(err))\n\t\t}\n\n\t\tresult := point.NewPoints()\n\n\t\tif res != nil && len(res) > 0 {\n\t\t\ttm := uint32(time.Now().Unix())\n\n\t\t\tfor metric, points := range res {\n\t\t\t\tmetricID := result.MetricID(metric)\n\t\t\t\tfor _, p := range points {\n\t\t\t\t\tresult.AppendPoint(metricID, p.Value, uint32(p.Timestamp), tm)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tcarbonlinkResponseChan <- result\n\t}()\n\n\treturn fetchResult\n}\n" }, { "path": "render/data/carbonlink_test.go", "content": "package data\n\nimport (\n\t\"context\"\n\t\"sort\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/point\"\n\tgraphitePickle \"github.com/lomik/graphite-pickle\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/mock\"\n)\n\ntype carbonlinkMocked struct {\n\tmock.Mock\n}\n\nfunc (c *carbonlinkMocked) CacheQueryMulti(ctx context.Context, metrics []string) (map[string][]graphitePickle.DataPoint, error) {\n\targs := c.Called(ctx, metrics)\n\treturn args.Get(0).(map[string][]graphitePickle.DataPoint), args.Error(1)\n}\n\nfunc TestSetCarbonlingClient(t *testing.T) {\n\tassert.Nil(t, carbonlink, \"client is set in the begining of tests\")\n\n\tcfg := config.New()\n\tcfg.Carbonlink.Server = \"localhost:0\"\n\tsetCarbonlinkClient(&cfg.Carbonlink)\n\tassert.NotNil(t, carbonlink, \"client is not set aftert setCarbonlinkClient\")\n\tassert.IsType(t, &graphitePickle.CarbonlinkClient{}, carbonlink.carbonlinkFetcher, \"\")\n\tcarbonlink = nil\n}\n\nfunc TestQueryCarbonlink(t *testing.T) {\n\tcarbonlink = nil\n\n\tres := make(map[string][]graphitePickle.DataPoint)\n\tmetrics := []string{\"metric1\", \"metric2\"}\n\tdataPoints := []graphitePickle.DataPoint{\n\t\t{\n\t\t\tTimestamp: 1500000000,\n\t\t\tValue: 13,\n\t\t},\n\t\t{\n\t\t\tTimestamp: 1500000060,\n\t\t\tValue: 14,\n\t\t},\n\t}\n\n\tfor _, m := range metrics {\n\t\tres[m] = dataPoints\n\t}\n\n\ttestGrCarbonlinkClient := new(carbonlinkMocked)\n\ttestGrCarbonlinkClient.On(\"CacheQueryMulti\", mock.AnythingOfType(\"*context.timerCtx\"), metrics).Return(res, nil)\n\tcarbonlink = &carbonlinkClient{testGrCarbonlinkClient, time.Duration(0)}\n\n\tnow := uint32(time.Now().Unix())\n\tpoints := queryCarbonlink(context.Background(), carbonlink, metrics)()\n\t// Result points.metrics are not ordered\n\tpMetrics := []string{points.MetricName(1), points.MetricName(2)}\n\ti := 0\n\n\tfor _, m := range pMetrics {\n\t\tfor _, dp := range dataPoints {\n\t\t\t// There is a tiny chance that point will have greated Timestamp than now. Here we test it's at most the next second\n\t\t\tassert.GreaterOrEqual(t, uint32(1), (points.List()[i].Timestamp - now), \"difference between now and point.Timestamp is greater than 1\")\n\n\t\t\texpectedPoint := point.Point{MetricID: points.MetricID(m), Value: dp.Value, Time: uint32(dp.Timestamp), Timestamp: points.List()[i].Timestamp}\n\t\t\tassert.Equal(t, expectedPoint, points.List()[i], \"point is not correct\")\n\n\t\t\ti++\n\t\t}\n\t}\n\n\tsort.Strings(pMetrics)\n\tassert.Equal(t, metrics, pMetrics, \"sorted points.metrics is not the same as in request\")\n\n\tcarbonlink = nil\n\temptyPoints := queryCarbonlink(context.Background(), carbonlink, metrics)()\n\tassert.Nil(t, emptyPoints, \"points are not nil\")\n}\n" }, { "path": "render/data/ch_response.go", "content": "package data\n\nimport (\n\t\"errors\"\n\t\"math\"\n\n\tv2pb \"github.com/go-graphite/protocol/carbonapi_v2_pb\"\n\tv3pb \"github.com/go-graphite/protocol/carbonapi_v3_pb\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/point\"\n)\n\n// CHResponse contains the parsed Data and From/Until timestamps\ntype CHResponse struct {\n\tData *Data\n\tFrom int64\n\tUntil int64\n\t// if true, return points for all metrics, replacing empty results with list of NaN\n\tAppendOutEmptySeries bool\n\tAppliedFunctions map[string][]string\n}\n\n// CHResponses is a slice of CHResponse\ntype CHResponses []CHResponse\n\n// EmptyResponse returns an CHResponses with one element containing emptyData for the following encoding\nfunc EmptyResponse() CHResponses { return CHResponses{{Data: emptyData}} }\n\n// ToMultiFetchResponseV2 returns protobuf v2pb.MultiFetchResponse message for given CHResponse\nfunc (c *CHResponse) ToMultiFetchResponseV2() (*v2pb.MultiFetchResponse, error) {\n\tmfr := &v2pb.MultiFetchResponse{Metrics: make([]v2pb.FetchResponse, 0)}\n\tdata := c.Data\n\n\taddResponse := func(name string, step uint32, points []point.Point) error {\n\t\tfrom, until := uint32(c.From), uint32(c.Until)\n\t\tstart, stop, count, getValue := point.FillNulls(points, from, until, step)\n\t\tvalues := make([]float64, 0, count)\n\t\tisAbsent := make([]bool, 0, count)\n\n\t\tfor {\n\t\t\tvalue, err := getValue()\n\t\t\tif err != nil {\n\t\t\t\tif errors.Is(err, point.ErrTimeGreaterStop) {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\t// if err is not point.ErrTimeGreaterStop, the points are corrupted\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tif math.IsNaN(value) {\n\t\t\t\tvalues = append(values, 0)\n\t\t\t\tisAbsent = append(isAbsent, true)\n\t\t\t} else {\n\t\t\t\tvalues = append(values, value)\n\t\t\t\tisAbsent = append(isAbsent, false)\n\t\t\t}\n\t\t}\n\n\t\tfor _, a := range data.AM.Get(name) {\n\t\t\tfr := v2pb.FetchResponse{\n\t\t\t\tName: a.DisplayName,\n\t\t\t\tStartTime: int32(start),\n\t\t\t\tStopTime: int32(stop),\n\t\t\t\tStepTime: int32(step),\n\t\t\t\tValues: values,\n\t\t\t\tIsAbsent: isAbsent,\n\t\t\t}\n\t\t\tmfr.Metrics = append(mfr.Metrics, fr)\n\t\t}\n\n\t\treturn nil\n\t}\n\n\t// process metrics with points\n\twrittenMetrics := make(map[string]struct{})\n\tnextMetric := data.GroupByMetric()\n\n\tfor {\n\t\tpoints := nextMetric()\n\t\tif len(points) == 0 {\n\t\t\tbreak\n\t\t}\n\n\t\tid := points[0].MetricID\n\t\tname := data.MetricName(id)\n\t\twrittenMetrics[name] = struct{}{}\n\n\t\tstep, err := data.GetStep(id)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tif err := addResponse(name, step, points); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\t// process metrics with no points\n\tif c.AppendOutEmptySeries && len(writtenMetrics) < data.AM.Len() && data.CommonStep > 0 {\n\t\tfor _, metricName := range data.AM.Series(false) {\n\t\t\tif _, done := writtenMetrics[metricName]; !done {\n\t\t\t\terr := addResponse(metricName, uint32(data.CommonStep), []point.Point{})\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn mfr, nil\n}\n\n// ToMultiFetchResponseV2 returns protobuf v2pb.MultiFetchResponse message for given CHResponses\nfunc (cc *CHResponses) ToMultiFetchResponseV2() (*v2pb.MultiFetchResponse, error) {\n\tmfr := &v2pb.MultiFetchResponse{Metrics: make([]v2pb.FetchResponse, 0)}\n\n\tfor _, c := range *cc {\n\t\tm, err := c.ToMultiFetchResponseV2()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tmfr.Metrics = append(mfr.Metrics, m.Metrics...)\n\t}\n\n\treturn mfr, nil\n}\n\n// ToMultiFetchResponseV3 returns protobuf v3pb.MultiFetchResponse message for given CHResponse\nfunc (c *CHResponse) ToMultiFetchResponseV3() (*v3pb.MultiFetchResponse, error) {\n\tmfr := &v3pb.MultiFetchResponse{Metrics: make([]v3pb.FetchResponse, 0)}\n\tdata := c.Data\n\taddResponse := func(name, function string, step uint32, points []point.Point) error {\n\t\tfrom, until := uint32(c.From), uint32(c.Until)\n\t\tstart, stop, count, getValue := point.FillNulls(points, from, until, step)\n\t\tvalues := make([]float64, 0, count)\n\n\t\tfor {\n\t\t\tvalue, err := getValue()\n\t\t\tif err != nil {\n\t\t\t\tif errors.Is(err, point.ErrTimeGreaterStop) {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\t// if err is not point.ErrTimeGreaterStop, the points are corrupted\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tvalues = append(values, value)\n\t\t}\n\n\t\tfor _, a := range data.AM.Get(name) {\n\t\t\tfr := v3pb.FetchResponse{\n\t\t\t\tName: a.DisplayName,\n\t\t\t\tPathExpression: a.Target,\n\t\t\t\tConsolidationFunc: function,\n\t\t\t\tStartTime: int64(start),\n\t\t\t\tStopTime: int64(stop),\n\t\t\t\tStepTime: int64(step),\n\t\t\t\tXFilesFactor: 0,\n\t\t\t\tHighPrecisionTimestamps: false,\n\t\t\t\tValues: values,\n\t\t\t\tAppliedFunctions: c.AppliedFunctions[a.Target],\n\t\t\t\tRequestStartTime: c.From,\n\t\t\t\tRequestStopTime: c.Until,\n\t\t\t}\n\t\t\tmfr.Metrics = append(mfr.Metrics, fr)\n\t\t}\n\n\t\treturn nil\n\t}\n\n\t// process metrics with points\n\twrittenMetrics := make(map[string]struct{})\n\tnextMetric := data.GroupByMetric()\n\n\tfor {\n\t\tpoints := nextMetric()\n\t\tif len(points) == 0 {\n\t\t\tbreak\n\t\t}\n\n\t\tid := points[0].MetricID\n\t\tname := data.MetricName(id)\n\t\twrittenMetrics[name] = struct{}{}\n\n\t\tconsolidationFunc, err := data.GetAggregation(id)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tstep, err := data.GetStep(id)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tif err := addResponse(name, consolidationFunc, step, points); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\t// process metrics with no points\n\tif c.AppendOutEmptySeries && len(writtenMetrics) < data.AM.Len() && data.CommonStep > 0 {\n\t\tfor _, metricName := range data.AM.Series(false) {\n\t\t\tif _, done := writtenMetrics[metricName]; !done {\n\t\t\t\terr := addResponse(metricName, \"any\", uint32(data.CommonStep), []point.Point{})\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn mfr, nil\n}\n\n// ToMultiFetchResponseV3 returns protobuf v3pb.MultiFetchResponse message for given CHResponses\nfunc (cc *CHResponses) ToMultiFetchResponseV3() (*v3pb.MultiFetchResponse, error) {\n\tmfr := &v3pb.MultiFetchResponse{Metrics: make([]v3pb.FetchResponse, 0)}\n\n\tfor _, c := range *cc {\n\t\tm, err := c.ToMultiFetchResponseV3()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tmfr.Metrics = append(mfr.Metrics, m.Metrics...)\n\t}\n\n\treturn mfr, nil\n}\n" }, { "path": "render/data/common_step.go", "content": "package data\n\nimport (\n\t\"context\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/pkg/dry\"\n)\n\n// This is used to calculate lowest common multiplier of metrics for ClickHouse internal aggregation\n// Collect amount of targets;\n// Wait until all targets will send the step, and calculate the LCM on the fly\n// Return the calculated LCM()\ntype commonStep struct {\n\tresult int64\n\twg sync.WaitGroup\n\tlock sync.RWMutex\n}\n\nfunc (c *commonStep) addTargets(delta int) {\n\tc.wg.Add(delta)\n}\n\nfunc (c *commonStep) doneTarget() {\n\tc.wg.Done()\n}\n\nfunc (c *commonStep) calculateUnsafe(a, b int64) int64 {\n\tif a == 0 || b == 0 {\n\t\treturn dry.Max(a, b)\n\t}\n\n\treturn dry.LCM(a, b)\n}\n\nfunc (c *commonStep) calculate(value int64) {\n\tc.lock.Lock()\n\tc.result = c.calculateUnsafe(c.result, value)\n\tc.lock.Unlock()\n\tc.doneTarget()\n}\n\nfunc (c *commonStep) getResult() int64 {\n\tctx, cancel := context.WithTimeout(context.Background(), time.Second*2)\n\tdefer cancel()\n\n\tch := make(chan int64)\n\tgo func(ch chan int64) {\n\t\tc.wg.Wait()\n\t\tc.lock.RLock()\n\t\tdefer c.lock.RUnlock()\n\t\tch <- c.result\n\t}(ch)\n\tselect {\n\tcase r := <-ch:\n\t\treturn r\n\tcase <-ctx.Done():\n\t\t// -1 is a definitely wrong value, it will break following ClickHouse query\n\t\t// This possible, when one of the queries in request already returned error\n\t\treturn -1\n\t}\n}\n" }, { "path": "render/data/common_step_test.go", "content": "package data\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\ntype wrapper struct {\n\t*commonStep\n\tcalcCounter int\n\tcLock sync.RWMutex\n}\n\nfunc (w *wrapper) calc(step int64) {\n\tw.cLock.Lock()\n\tw.calcCounter++\n\tw.calculate(step)\n\tw.cLock.Unlock()\n}\n\nfunc newWrapper() *wrapper {\n\tc := &commonStep{\n\t\tresult: 0,\n\t\twg: sync.WaitGroup{},\n\t\tlock: sync.RWMutex{},\n\t}\n\n\treturn &wrapper{\n\t\tcommonStep: c,\n\t\tcLock: sync.RWMutex{},\n\t}\n}\n\nfunc TestCommonStepWorker(t *testing.T) {\n\tw := newWrapper()\n\tw.addTargets(4)\n\n\tgo func() {\n\t\tlastStep := int64(0)\n\t\tfor i := 0; i < 20000; i++ {\n\t\t\tw.calculateUnsafe(lastStep, 0)\n\t\t}\n\n\t\tw.calc(0)\n\t\tassert.Equal(t, int64(120), w.commonStep.getResult())\n\t}()\n\tgo func() {\n\t\tlastStep := int64(0)\n\t\tfor i := 0; i < 30000; i++ {\n\t\t\tw.calculateUnsafe(lastStep, 6)\n\t\t}\n\n\t\tw.calc(6)\n\t\tassert.Equal(t, int64(120), w.commonStep.getResult())\n\t}()\n\tgo func() {\n\t\tlastStep := int64(0)\n\t\tfor i := 0; i < 40000; i++ {\n\t\t\tw.calculateUnsafe(lastStep, 8)\n\t\t}\n\n\t\tw.calc(8)\n\t\tassert.Equal(t, int64(120), w.commonStep.getResult())\n\t}()\n\tgo func() {\n\t\tlastStep := int64(0)\n\t\tfor i := 0; i < 50000; i++ {\n\t\t\tw.calculateUnsafe(lastStep, 10)\n\t\t}\n\n\t\tw.calc(10)\n\t\tassert.Equal(t, int64(120), w.commonStep.getResult())\n\t}()\n\tassert.Equal(t, int64(120), w.commonStep.getResult())\n\tassert.Equal(t, 4, w.calcCounter)\n}\n" }, { "path": "render/data/data.go", "content": "package data\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"encoding/binary\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/helper/point\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/alias\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/reverse\"\n)\n\nvar errClickHouseResponse = errors.New(\"Malformed response from clickhouse\")\n\n// ReadUvarint reads unsigned int with variable length\nvar ReadUvarint = clickhouse.ReadUvarint\n\n// Data stores parsed response from ClickHouse server\ntype Data struct {\n\t*point.Points\n\tAM *alias.Map\n\tCommonStep int64\n}\n\nvar emptyData *Data = &Data{Points: point.NewPoints(), AM: alias.New()}\n\nfunc contextIsValid(ctx context.Context) error {\n\tselect {\n\tcase <-ctx.Done():\n\t\treturn ctx.Err()\n\tdefault:\n\t\treturn nil\n\t}\n}\n\n// GetStep returns the commonStep for all points or, if unset, step for metric ID id\nfunc (d *Data) GetStep(id uint32) (uint32, error) {\n\tif 0 < d.CommonStep {\n\t\treturn uint32(d.CommonStep), nil\n\t}\n\n\treturn d.Points.GetStep(id)\n}\n\n// GetAggregation returns the generic whisper compatible name for an aggregation of metric with ID id\nfunc (d *Data) GetAggregation(id uint32) (string, error) {\n\tfunction, err := d.Points.GetAggregation(id)\n\tif err != nil {\n\t\treturn function, err\n\t}\n\n\tswitch function {\n\tcase \"any\":\n\t\treturn \"first\", nil\n\tcase \"anyLast\":\n\t\treturn \"last\", nil\n\tdefault:\n\t\treturn function, nil\n\t}\n}\n\n// data wraps Data and adds asynchronous processing of data\n// data.wait() should be used with the same context as prepareData and parseResponse to check the error\ntype data struct {\n\t*Data\n\tlength int // readed bytes count\n\tspent time.Duration // time spent on parsing\n\tb chan io.ReadCloser\n\te chan error\n\tmut sync.RWMutex\n\twg sync.WaitGroup\n}\n\n// prepareData returns new data with asynchronous processing points from carbonlinkClient\nfunc prepareData(ctx context.Context, targets int, fetcher func() *point.Points) *data {\n\tdata := &data{\n\t\tData: &Data{Points: point.NewPoints()},\n\t\tb: make(chan io.ReadCloser, 1),\n\t\te: make(chan error, targets),\n\t\tmut: sync.RWMutex{},\n\t\twg: sync.WaitGroup{},\n\t}\n\tdata.wg.Add(1)\n\n\textraPoints := make(chan *point.Points, 1)\n\n\tgo func() {\n\t\t// add extraPoints. With NameToID\n\t\tdefer func() {\n\t\t\tdata.wg.Done()\n\t\t\tclose(extraPoints)\n\t\t}()\n\n\t\t// First check is context is already done\n\t\tif err := contextIsValid(ctx); err != nil {\n\t\t\tdata.e <- fmt.Errorf(\"prepareData failed: %w\", err)\n\t\t\treturn\n\t\t}\n\n\t\tselect {\n\t\tcase extraPoints <- fetcher():\n\t\t\tp := <-extraPoints\n\t\t\tif p != nil {\n\t\t\t\tdata.mut.Lock()\n\t\t\t\tdefer data.mut.Unlock()\n\n\t\t\t\textraList := p.List()\n\t\t\t\tfor i := 0; i < len(extraList); i++ {\n\t\t\t\t\tdata.Points.AppendPoint(\n\t\t\t\t\t\tdata.Points.MetricID(p.MetricName(extraList[i].MetricID)),\n\t\t\t\t\t\textraList[i].Value,\n\t\t\t\t\t\textraList[i].Time,\n\t\t\t\t\t\textraList[i].Timestamp,\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn\n\t\tcase <-ctx.Done():\n\t\t\tdata.e <- fmt.Errorf(\"prepareData failed: %w\", ctx.Err())\n\t\t\treturn\n\t\t}\n\t}()\n\n\treturn data\n}\n\n// setSteps sets commonStep for aggregated requests and per-metric step for non-aggregated\nfunc (d *data) setSteps(cond *conditions) {\n\tif cond.aggregated {\n\t\td.CommonStep = cond.step\n\t\treturn\n\t}\n\n\td.Points.SetSteps(cond.steps)\n}\n\n// Error handler for data splitting functions\nfunc splitErrorHandler(data *[]byte, atEOF bool, tokenLen int, err error) (int, []byte, error) {\n\tif err == clickhouse.ErrUvarintRead {\n\t\tif atEOF {\n\t\t\treturn 0, nil, clickhouse.NewErrWithDescr(errClickHouseResponse.Error(), string(*data))\n\t\t}\n\t\t// signal for read more\n\t\treturn 0, nil, nil\n\t} else if err != nil || (len(*data) < tokenLen && atEOF) {\n\t\treturn 0, nil, clickhouse.NewErrWithDescr(errClickHouseResponse.Error(), string(*data))\n\t}\n\t// signal for read more\n\treturn 0, nil, nil\n}\n\n// dataSplitAggregated is a split function for bufio.Scanner for read row binary response for queries of aggregated data\nfunc dataSplitAggregated(data []byte, atEOF bool) (advance int, token []byte, err error) {\n\tif len(data) == 0 && atEOF {\n\t\t// stop\n\t\treturn 0, nil, nil\n\t}\n\n\tnameLen, readBytes, err := ReadUvarint(data)\n\ttokenLen := readBytes + int(nameLen)\n\n\tif err != nil || len(data) < tokenLen {\n\t\treturn splitErrorHandler(&data, atEOF, tokenLen, err)\n\t}\n\n\ttimeLen, readBytes, err := ReadUvarint(data[tokenLen:])\n\ttokenLen += readBytes + int(timeLen)*4\n\n\tif err != nil || len(data) < tokenLen {\n\t\treturn splitErrorHandler(&data, atEOF, tokenLen, err)\n\t}\n\n\tvalueLen, readBytes, err := ReadUvarint(data[tokenLen:])\n\ttokenLen += readBytes + int(valueLen)*8\n\n\tif err != nil || len(data) < tokenLen {\n\t\treturn splitErrorHandler(&data, atEOF, tokenLen, err)\n\t}\n\n\tif timeLen != valueLen {\n\t\treturn 0, nil, clickhouse.NewErrWithDescr(errClickHouseResponse.Error()+\": Different amount of Times and Values\", string(data))\n\t}\n\n\treturn tokenLen, data[:tokenLen], nil\n}\n\n// dataSplitUnaggregated is a split function for bufio.Scanner for read row binary response for queries of unaggregated data\nfunc dataSplitUnaggregated(data []byte, atEOF bool) (advance int, token []byte, err error) {\n\tif len(data) == 0 && atEOF {\n\t\t// stop\n\t\treturn 0, nil, nil\n\t}\n\n\tnameLen, readBytes, err := ReadUvarint(data)\n\ttokenLen := readBytes + int(nameLen)\n\n\tif err != nil || len(data) < tokenLen {\n\t\treturn splitErrorHandler(&data, atEOF, tokenLen, err)\n\t}\n\n\ttimeLen, readBytes, err := ReadUvarint(data[tokenLen:])\n\ttokenLen += readBytes + int(timeLen)*4\n\n\tif err != nil || len(data) < tokenLen {\n\t\treturn splitErrorHandler(&data, atEOF, tokenLen, err)\n\t}\n\n\tvalueLen, readBytes, err := ReadUvarint(data[tokenLen:])\n\ttokenLen += readBytes + int(valueLen)*8\n\n\tif err != nil || len(data) < tokenLen {\n\t\treturn splitErrorHandler(&data, atEOF, tokenLen, err)\n\t}\n\n\ttimestampLen, readBytes, err := ReadUvarint(data[tokenLen:])\n\ttokenLen += readBytes + int(timestampLen)*4\n\n\tif err != nil || len(data) < tokenLen {\n\t\treturn splitErrorHandler(&data, atEOF, tokenLen, err)\n\t}\n\n\tif timeLen != valueLen || timeLen != timestampLen {\n\t\treturn 0, nil, clickhouse.NewErrWithDescr(errClickHouseResponse.Error()+\": Different amount of Values, Times and Timestamps\", string(data))\n\t}\n\n\treturn tokenLen, data[:tokenLen], nil\n}\n\n// readResponse reads the ClickHouse body into *Data and merges with extraPoints.\n// Expected, that on error the context will be cancelled on the upper level.\nfunc (d *data) parseResponse(ctx context.Context, bodyReader io.ReadCloser, cond *conditions) error {\n\tpp := d.Points\n\n\tdataSplit := dataSplitUnaggregated\n\tif cond.aggregated {\n\t\tdataSplit = dataSplitAggregated\n\t}\n\n\t// Prevent starting parser if context is done\n\tif err := contextIsValid(ctx); err != nil {\n\t\treturn ctx.Err()\n\t}\n\n\t// Then wait if there is an active parser working\n\tselect {\n\tcase d.b <- bodyReader:\n\tcase <-ctx.Done():\n\t\treturn fmt.Errorf(\"parseResponse failed: %w\", ctx.Err())\n\t}\n\n\tvar metricID uint32\n\n\td.mut.Lock()\n\tdefer func() {\n\t\td.mut.Unlock()\n\t\t<-d.b\n\t}()\n\n\t// Are we still good to go?\n\tif err := contextIsValid(ctx); err != nil {\n\t\treturn fmt.Errorf(\"parseResponse failed: %w\", ctx.Err())\n\t}\n\n\tstart := time.Now()\n\tscanner := bufio.NewScanner(bodyReader)\n\tscanner.Buffer(make([]byte, 1048576), 67108864)\n\tscanner.Split(dataSplit)\n\n\tvar rowStart []byte\n\tfor scanner.Scan() {\n\t\trowStart = scanner.Bytes()\n\n\t\td.length += len(rowStart)\n\n\t\tnameLen, readBytes, err := ReadUvarint(rowStart)\n\t\tif err != nil {\n\t\t\treturn errClickHouseResponse\n\t\t}\n\n\t\trow := rowStart[readBytes:]\n\n\t\tname := row[:int(nameLen)]\n\t\trow = row[int(nameLen):]\n\n\t\tif cond.isReverse {\n\t\t\tmetricID = pp.MetricIDBytes(reverse.Bytes(name))\n\t\t} else {\n\t\t\tmetricID = pp.MetricIDBytes(name)\n\t\t}\n\n\t\tarrayLen, readBytes, err := ReadUvarint(row)\n\t\tif err != nil {\n\t\t\treturn errClickHouseResponse\n\t\t}\n\n\t\ttimes := make([]uint32, 0, arrayLen)\n\t\tvalues := make([]float64, 0, arrayLen)\n\n\t\trow = row[readBytes:]\n\t\tfor i := uint64(0); i < arrayLen; i++ {\n\t\t\ttimes = append(times, binary.LittleEndian.Uint32(row[:4]))\n\t\t\trow = row[4:]\n\t\t}\n\n\t\trow = row[readBytes:]\n\t\tfor i := uint64(0); i < arrayLen; i++ {\n\t\t\tvalues = append(values, math.Float64frombits(binary.LittleEndian.Uint64(row[:8])))\n\t\t\trow = row[8:]\n\t\t}\n\n\t\ttimestamps := times\n\t\tif !cond.aggregated {\n\t\t\ttimestamps = make([]uint32, 0, arrayLen)\n\t\t\trow = row[readBytes:]\n\n\t\t\tfor i := uint64(0); i < arrayLen; i++ {\n\t\t\t\ttimestamps = append(timestamps, binary.LittleEndian.Uint32(row[:4]))\n\t\t\t\trow = row[4:]\n\t\t\t}\n\t\t}\n\n\t\tfor i := range times {\n\t\t\tpp.AppendPoint(metricID, values[i], times[i], timestamps[i])\n\t\t}\n\t}\n\n\td.spent += time.Since(start)\n\n\terr := scanner.Err()\n\tif err != nil {\n\t\tdataErr, ok := err.(*clickhouse.ErrWithDescr)\n\t\tif ok {\n\t\t\t// format full error string, sometimes parse not failed at start orf error string\n\t\t\tdataErr.PrependDescription(string(rowStart))\n\t\t}\n\n\t\tbodyReader.Close()\n\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc (d *data) wait(ctx context.Context) error {\n\t// First check is context is already done\n\tif err := contextIsValid(ctx); err != nil {\n\t\treturn fmt.Errorf(\"prepareData failed: %w\", err)\n\t}\n\n\t// if anything is already in error channel\n\tselect {\n\tcase err := <-d.e:\n\t\treturn err\n\tdefault:\n\t}\n\n\t// watch if workers are done\n\tparsersDone := make(chan struct{}, 1)\n\tgo func() {\n\t\td.wg.Wait()\n\t\tparsersDone <- struct{}{}\n\t}()\n\n\t// and watch for all channels\n\tselect {\n\tcase <-ctx.Done():\n\t\treturn ctx.Err()\n\tcase err := <-d.e:\n\t\treturn err\n\tcase <-parsersDone:\n\t\treturn nil\n\t}\n}\n" }, { "path": "render/data/data_parse_test.go", "content": "package data\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/RowBinary\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/helper/point\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/reverse\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\ntype pointValues struct {\n\tValues []float64\n\tTimes []uint32\n\tTimestamps []uint32\n}\n\ntype testPoint struct {\n\tMetric string\n\tPointValues *pointValues\n}\n\nfunc makeAggregatedBody(points []testPoint) []byte {\n\tbuf := new(bytes.Buffer)\n\tw := RowBinary.NewEncoder(buf)\n\n\tfor i := 0; i < len(points); i++ {\n\t\tw.String(points[i].Metric)\n\t\tw.Uint32List(points[i].PointValues.Times)\n\t\tw.Float64List(points[i].PointValues.Values)\n\t}\n\n\treturn buf.Bytes()\n}\n\nfunc makeUnaggregatedBody(points []testPoint) []byte {\n\tbuf := new(bytes.Buffer)\n\tw := RowBinary.NewEncoder(buf)\n\n\tfor i := 0; i < len(points); i++ {\n\t\tw.String(points[i].Metric)\n\t\tw.Uint32List(points[i].PointValues.Times)\n\t\tw.Float64List(points[i].PointValues.Values)\n\t\tw.Uint32List(points[i].PointValues.Timestamps)\n\t}\n\n\treturn buf.Bytes()\n}\n\nfunc testCarbonlinkReaderNil() *point.Points {\n\treturn nil\n}\n\nfunc TestUnaggregatedDataParse(t *testing.T) {\n\tctx := context.Background()\n\tcond := &conditions{Targets: &Targets{isReverse: false}, aggregated: false}\n\n\tt.Run(\"empty response\", func(t *testing.T) {\n\t\tbody := []byte{}\n\t\tr := io.NopCloser(bytes.NewReader(body))\n\t\td := prepareData(ctx, 1, testCarbonlinkReaderNil)\n\n\t\terr := d.parseResponse(ctx, r, cond)\n\t\tassert.NoError(t, err)\n\n\t\twerr := d.wait(ctx)\n\t\tassert.NoError(t, werr)\n\t\tassert.Empty(t, d.Points.List())\n\t})\n\n\ttable := [][]testPoint{\n\t\t{\n\t\t\t{\"hello.world\", &pointValues{[]float64{42.1}, []uint32{1520056686}, []uint32{1520056706}}},\n\t\t},\n\t\t{\n\t\t\t{\"hello.world\", &pointValues{[]float64{42.1}, []uint32{1520056686}, []uint32{1520056706}}},\n\t\t\t{\"foobar\", &pointValues{[]float64{42.2}, []uint32{1520056687}, []uint32{1520056707}}},\n\t\t},\n\t\t{\n\t\t\t{\"samelen1\", &pointValues{[]float64{42.1}, []uint32{1520056686}, []uint32{1520056706}}},\n\t\t\t{\"samelen2\", &pointValues{[]float64{42.2}, []uint32{1520056687}, []uint32{1520056707}}},\n\t\t},\n\t\t{\n\t\t\t{\"long.metric.with.points.key1\", &pointValues{[]float64{42.1, 42.2}, []uint32{1520056686, 1520056687}, []uint32{1520056706, 1520056687}}},\n\t\t\t{\"long.metric.with.points.key2\", &pointValues{[]float64{42.2}, []uint32{1520056687}, []uint32{1520056707}}},\n\t\t},\n\t}\n\n\tfor i := 0; i < len(table); i++ {\n\t\tt.Run(fmt.Sprintf(\"ok #%d\", i), func(t *testing.T) {\n\t\t\tbody := makeUnaggregatedBody(table[i])\n\n\t\t\tr := io.NopCloser(bytes.NewReader(body))\n\t\t\td := prepareData(ctx, 1, testCarbonlinkReaderNil)\n\n\t\t\terr := d.parseResponse(ctx, r, cond)\n\t\t\tassert.NoError(t, err)\n\n\t\t\twerr := d.wait(ctx)\n\t\t\tassert.NoError(t, werr)\n\t\t\t// point number\n\t\t\tp := 0\n\n\t\t\tfor j := 0; j < len(table[i]); j++ {\n\t\t\t\tfor m := 0; m < len(table[i][j].PointValues.Times); m++ {\n\t\t\t\t\tassert.Equal(t, table[i][j].Metric, d.Points.MetricName(d.Points.List()[p].MetricID))\n\t\t\t\t\tassert.Equal(t, table[i][j].PointValues.Times[m], d.Points.List()[p].Time)\n\t\t\t\t\tassert.Equal(t, table[i][j].PointValues.Values[m], d.Points.List()[p].Value)\n\t\t\t\t\tassert.Equal(t, table[i][j].PointValues.Timestamps[m], d.Points.List()[p].Timestamp)\n\n\t\t\t\t\tp++\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n\n\tfor i := 0; i < len(table); i++ {\n\t\tt.Run(fmt.Sprintf(\"reversed #%d\", i), func(t *testing.T) {\n\t\t\tcond := &conditions{Targets: &Targets{isReverse: true}, aggregated: false}\n\t\t\tbody := makeUnaggregatedBody(table[i])\n\n\t\t\tr := io.NopCloser(bytes.NewReader(body))\n\t\t\td := prepareData(ctx, 1, testCarbonlinkReaderNil)\n\n\t\t\terr := d.parseResponse(ctx, r, cond)\n\t\t\tassert.NoError(t, err)\n\n\t\t\twerr := d.wait(ctx)\n\t\t\tassert.NoError(t, werr)\n\t\t\t// point number\n\t\t\tp := 0\n\n\t\t\tfor j := 0; j < len(table[i]); j++ {\n\t\t\t\tfor m := 0; m < len(table[i][j].PointValues.Times); m++ {\n\t\t\t\t\tassert.Equal(t, table[i][j].Metric, reverse.String(d.Points.MetricName(d.Points.List()[p].MetricID)))\n\t\t\t\t\tassert.Equal(t, table[i][j].PointValues.Times[m], d.Points.List()[p].Time)\n\t\t\t\t\tassert.Equal(t, table[i][j].PointValues.Values[m], d.Points.List()[p].Value)\n\t\t\t\t\tassert.Equal(t, table[i][j].PointValues.Timestamps[m], d.Points.List()[p].Timestamp)\n\n\t\t\t\t\tp++\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n\n\tt.Run(\"malformed ClickHouse body\", func(t *testing.T) {\n\t\tbody := makeUnaggregatedBody([]testPoint{\n\t\t\t{\n\t\t\t\tMetric: \"hello.world\",\n\t\t\t\tPointValues: &pointValues{\n\t\t\t\t\tValues: []float64{42.1},\n\t\t\t\t\tTimes: []uint32{1520056686},\n\t\t\t\t\tTimestamps: []uint32{1520056706, 1520056707},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t\tr := io.NopCloser(bytes.NewReader(body))\n\t\td := prepareData(ctx, 1, testCarbonlinkReaderNil)\n\n\t\terr := d.parseResponse(ctx, r, cond)\n\t\tassert.Error(t, err)\n\t})\n\n\tt.Run(\"incomplete response\", func(t *testing.T) {\n\t\tpoints := []testPoint{\n\t\t\t{\n\t\t\t\tMetric: \"hello.world\",\n\t\t\t\tPointValues: &pointValues{\n\t\t\t\t\tValues: []float64{42.1},\n\t\t\t\t\tTimes: []uint32{1520056686},\n\t\t\t\t\tTimestamps: []uint32{1520056706},\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tMetric: \"bye-bye.sky\",\n\t\t\t\tPointValues: &pointValues{\n\t\t\t\t\tValues: []float64{42.42},\n\t\t\t\t\tTimes: []uint32{1520056686},\n\t\t\t\t\tTimestamps: []uint32{1520056706},\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\t\tbody := makeUnaggregatedBody(points)\n\n\t\tfirstMetricLength := len(makeUnaggregatedBody(points[:1]))\n\t\tfor i := 1; i < len(body)-1; i++ {\n\t\t\tif i == firstMetricLength {\n\t\t\t\t// length of the first metric\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tr := io.NopCloser(bytes.NewReader(body[:i]))\n\t\t\td := prepareData(ctx, 1, testCarbonlinkReaderNil)\n\n\t\t\terr := d.parseResponse(ctx, r, cond)\n\t\t\tassert.Error(t, err)\n\t\t\tassert.True(t, (d.length == 0 || d.length == firstMetricLength), \"length of read data is wrong\")\n\t\t}\n\t})\n}\n\nfunc TestAggregatedDataParse(t *testing.T) {\n\tctx := context.Background()\n\tcond := &conditions{Targets: &Targets{isReverse: false}, aggregated: true}\n\n\tt.Run(\"empty response\", func(t *testing.T) {\n\t\tbody := []byte{}\n\t\td := prepareData(ctx, 1, testCarbonlinkReaderNil)\n\t\tr := io.NopCloser(bytes.NewReader(body))\n\n\t\terr := d.parseResponse(ctx, r, cond)\n\t\tassert.NoError(t, err)\n\t\tassert.Empty(t, d.Points.List())\n\t})\n\n\tt.Run(\"incomplete response\", func(t *testing.T) {\n\t\tpoints := []testPoint{\n\t\t\t{\n\t\t\t\tMetric: \"hello.world\",\n\t\t\t\tPointValues: &pointValues{\n\t\t\t\t\tValues: []float64{42.1},\n\t\t\t\t\tTimes: []uint32{1520056686},\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tMetric: \"bye-bye.sky\",\n\t\t\t\tPointValues: &pointValues{\n\t\t\t\t\tValues: []float64{42.1},\n\t\t\t\t\tTimes: []uint32{1520056686},\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\t\tbody := makeAggregatedBody(points)\n\n\t\tfirstMetricLength := len(makeAggregatedBody(points[:1]))\n\t\tfor i := 1; i < len(body)-1; i++ {\n\t\t\tif i == firstMetricLength {\n\t\t\t\t// length of the first metric\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tr := io.NopCloser(bytes.NewReader(body[:i]))\n\n\t\t\td := prepareData(ctx, 1, testCarbonlinkReaderNil)\n\t\t\terr := d.parseResponse(ctx, r, cond)\n\t\t\tassert.Error(t, err)\n\t\t\tassert.True(t, (d.length == 0 || d.length == firstMetricLength), \"length of read data is wrong\")\n\t\t}\n\t})\n\n\tt.Run(\"malformed ClickHouse body\", func(t *testing.T) {\n\t\tpoints := []testPoint{\n\t\t\t{\n\t\t\t\t// different length of -Resample arrays\n\t\t\t\tMetric: \"different.arrays.in.body\",\n\t\t\t\tPointValues: &pointValues{\n\t\t\t\t\tValues: []float64{42.1},\n\t\t\t\t\tTimes: []uint32{1520056706, 1520056707},\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\t\tbody := makeAggregatedBody(points)\n\t\tr := io.NopCloser(bytes.NewReader(body))\n\n\t\td := prepareData(ctx, 1, testCarbonlinkReaderNil)\n\t\terr := d.parseResponse(ctx, r, cond)\n\t\tassert.Error(t, err)\n\t})\n\n\tt.Run(\"normal work\", func(t *testing.T) {\n\t\tpoints := []testPoint{\n\t\t\t{\n\t\t\t\tMetric: \"hello.world\",\n\t\t\t\tPointValues: &pointValues{\n\t\t\t\t\tValues: []float64{42.1},\n\t\t\t\t\tTimes: []uint32{1520056686},\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tMetric: \"null.in.the.middle\",\n\t\t\t\tPointValues: &pointValues{\n\t\t\t\t\tValues: []float64{42.1, 43},\n\t\t\t\t\tTimes: []uint32{1520056686, 1520056690},\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\t\tbody := makeAggregatedBody(points)\n\t\tr := io.NopCloser(bytes.NewReader(body))\n\n\t\td := prepareData(ctx, 1, testCarbonlinkReaderNil)\n\t\terr := d.parseResponse(ctx, r, cond)\n\t\tresult := []point.Point{\n\t\t\t{MetricID: 1, Value: 42.1, Time: 1520056686, Timestamp: 1520056686},\n\t\t\t{MetricID: 2, Value: 42.1, Time: 1520056686, Timestamp: 1520056686},\n\t\t\t{MetricID: 2, Value: 43, Time: 1520056690, Timestamp: 1520056690},\n\t\t}\n\n\t\tassert.NoError(t, err)\n\t\tassert.Equal(t, result, d.Points.List())\n\t})\n\n\tt.Run(\"reversed\", func(t *testing.T) {\n\t\tpoints := []testPoint{\n\t\t\t{\n\t\t\t\tMetric: \"hello.world\",\n\t\t\t\tPointValues: &pointValues{\n\t\t\t\t\tValues: []float64{42.1},\n\t\t\t\t\tTimes: []uint32{1520056686},\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tMetric: \"null.in.the.middle\",\n\t\t\t\tPointValues: &pointValues{\n\t\t\t\t\tValues: []float64{42.1, 43},\n\t\t\t\t\tTimes: []uint32{1520056686, 1520056690},\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\t\tbody := makeAggregatedBody(points)\n\t\tr := io.NopCloser(bytes.NewReader(body))\n\t\tcond := &conditions{Targets: &Targets{isReverse: true}, aggregated: true}\n\n\t\td := prepareData(ctx, 1, testCarbonlinkReaderNil)\n\t\terr := d.parseResponse(ctx, r, cond)\n\t\tassert.NoError(t, err)\n\t\tassert.Equal(t, \"world.hello\", d.Points.MetricName(1))\n\t\tassert.Equal(t, \"middle.the.in.null\", d.Points.MetricName(2))\n\t})\n}\n\nfunc TestPrepareDataParse(t *testing.T) {\n\tctx := context.Background()\n\n\tt.Run(\"empty datapoints\", func(t *testing.T) {\n\t\tdata := prepareData(ctx, 1, testCarbonlinkReaderNil)\n\t\terr := data.wait(ctx)\n\t\tassert.NoError(t, err)\n\t\tassert.Equal(t, &Data{Points: point.NewPoints()}, data.Data)\n\t})\n\n\tt.Run(\"cancelled context\", func(t *testing.T) {\n\t\tctx, cancel := context.WithCancel(ctx)\n\t\tcancel()\n\n\t\tdata := prepareData(ctx, 1, testCarbonlinkReaderNil)\n\t\terr := data.wait(ctx)\n\t\tassert.ErrorIs(t, err, context.Canceled)\n\t\tassert.Equal(t, &Data{Points: point.NewPoints()}, data.Data)\n\t})\n\n\tt.Run(\"data contains points\", func(t *testing.T) {\n\t\t//points := []point.Point{{1, 42.1, 1520056686, 1520056686}}\n\t\textraPoints := point.NewPoints()\n\t\textraPoints.MetricID(\"some.metric1\")\n\t\textraPoints.MetricID(\"some.metric2\")\n\t\textraPoints.AppendPoint(1, 1, 3, 3)\n\t\textraPoints.AppendPoint(2, 1, 3, 3)\n\n\t\treader := func() *point.Points {\n\t\t\ttime.Sleep(1 * time.Millisecond)\n\t\t\treturn extraPoints\n\t\t}\n\t\td := prepareData(ctx, 1, reader)\n\t\terr := d.wait(ctx)\n\t\tassert.NoError(t, err)\n\t\tassert.Equal(\n\t\t\tt, []point.Point{\n\t\t\t\t{MetricID: 1, Value: 1, Time: 3, Timestamp: 3},\n\t\t\t\t{MetricID: 2, Value: 1, Time: 3, Timestamp: 3},\n\t\t\t},\n\t\t\td.Points.List(),\n\t\t)\n\t\tassert.Equal(t, \"some.metric1\", d.Points.MetricName(1))\n\t\tassert.Equal(t, \"some.metric2\", d.Points.MetricName(2))\n\t})\n}\n\nfunc TestAsyncDataParse(t *testing.T) {\n\tctx := context.Background()\n\tcond := &conditions{Targets: &Targets{isReverse: false}, aggregated: false}\n\n\t// normal work is tested in other places\n\tt.Run(\"context deadline exceeded\", func(t *testing.T) {\n\t\textraPoints := point.NewPoints()\n\t\textraPoints.MetricID(\"some.metric1\")\n\t\textraPoints.MetricID(\"some.metric2\")\n\t\textraPoints.AppendPoint(1, 1, 3, 3)\n\t\textraPoints.AppendPoint(2, 1, 3, 3)\n\n\t\treader := func() *point.Points { return extraPoints }\n\n\t\tctx, cancel := context.WithTimeout(ctx, -1*time.Nanosecond)\n\t\tdefer cancel()\n\n\t\td := prepareData(ctx, 1, reader)\n\t\tassert.Len(t, d.Points.List(), 0, \"timeout should prevent points parsing\")\n\n\t\tbody := makeUnaggregatedBody([]testPoint{\n\t\t\t{\n\t\t\t\tMetric: \"hello.world\",\n\t\t\t\tPointValues: &pointValues{\n\t\t\t\t\tValues: []float64{42.1},\n\t\t\t\t\tTimes: []uint32{1520056686},\n\t\t\t\t\tTimestamps: []uint32{1520056706},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t\tr := io.NopCloser(bytes.NewReader(body))\n\n\t\terr := d.parseResponse(ctx, r, cond)\n\t\tassert.ErrorIs(t, err, context.DeadlineExceeded, \"parseResponse shouldn't return error on a done context\")\n\t\tassert.Len(t, d.Points.List(), 0, \"timeout should prevent points parsing\")\n\t\terr = d.wait(ctx)\n\t\tassert.ErrorIs(t, err, context.DeadlineExceeded, \"data.wait returns 'context dedline exceeded'\")\n\t})\n\n\tt.Run(\"context deadline faster than carbonlink reader\", func(t *testing.T) {\n\t\textraPoints := point.NewPoints()\n\t\textraPoints.MetricID(\"some.metric1\")\n\t\textraPoints.MetricID(\"some.metric2\")\n\t\textraPoints.AppendPoint(1, 1, 3, 3)\n\t\textraPoints.AppendPoint(2, 1, 3, 3)\n\n\t\treader := func() *point.Points {\n\t\t\ttime.Sleep(1 * time.Second)\n\t\t\treturn extraPoints\n\t\t}\n\n\t\tctx, cancel := context.WithTimeout(ctx, 50*time.Nanosecond)\n\t\tdefer cancel()\n\n\t\td := prepareData(ctx, 1, reader)\n\t\terr := d.wait(ctx)\n\t\tassert.Len(t, d.Points.List(), 0, \"timeout should prevent points parsing\")\n\t\tassert.ErrorIs(t, err, context.DeadlineExceeded, \"data.wait returns 'context dedline exceeded'\")\n\t})\n\n\tt.Run(\"cancel context before different steps\", func(t *testing.T) {\n\t\tctx, cancel := context.WithCancel(ctx)\n\t\tbody := []byte{}\n\t\t// works fine\n\t\td := prepareData(ctx, 1, testCarbonlinkReaderNil)\n\t\tr := io.NopCloser(bytes.NewReader(body))\n\t\terr := d.parseResponse(ctx, r, cond)\n\t\tassert.NoError(t, err)\n\t\terr = d.wait(ctx)\n\t\tassert.NoError(t, err)\n\t\tcancel()\n\t\t// fails after context is cancelled\n\t\terr = d.wait(ctx)\n\t\tassert.ErrorIs(t, err, context.Canceled)\n\n\t\tr = io.NopCloser(bytes.NewReader(body))\n\t\terr = d.parseResponse(ctx, r, cond)\n\t\tassert.ErrorIs(t, err, context.Canceled)\n\t})\n\n\tt.Run(\"wait fails on errors\", func(t *testing.T) {\n\t\td := prepareData(ctx, 1, testCarbonlinkReaderNil)\n\t\td.e <- clickhouse.ErrClickHouseResponse\n\t\terr := d.wait(ctx)\n\t\tassert.ErrorIs(t, err, clickhouse.ErrClickHouseResponse, \"err %v is not expected\", err)\n\t})\n}\n" }, { "path": "render/data/multi_target.go", "content": "package data\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"sync\"\n\t\"time\"\n\n\tv3pb \"github.com/go-graphite/protocol/carbonapi_v3_pb\"\n\t\"go.uber.org/zap\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/errs\"\n\t\"github.com/lomik/graphite-clickhouse/limiter\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/alias\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n)\n\n// TimeFrame contains information about fetch request time conditions\ntype TimeFrame struct {\n\tFrom int64\n\tUntil int64\n\tMaxDataPoints int64\n}\n\n// MultiTarget is a map of TimeFrame keys and targets slice of strings values\ntype MultiTarget map[TimeFrame]*Targets\n\nfunc MFRToMultiTarget(v3Request *v3pb.MultiFetchRequest) MultiTarget {\n\tmultiTarget := make(MultiTarget)\n\n\tif len(v3Request.Metrics) > 0 {\n\t\tfor _, m := range v3Request.Metrics {\n\t\t\ttf := TimeFrame{\n\t\t\t\tFrom: m.StartTime,\n\t\t\t\tUntil: m.StopTime,\n\t\t\t\tMaxDataPoints: m.MaxDataPoints,\n\t\t\t}\n\t\t\tif _, ok := multiTarget[tf]; ok {\n\t\t\t\ttarget := multiTarget[tf]\n\t\t\t\ttarget.Append(m.PathExpression)\n\t\t\t} else {\n\t\t\t\tmultiTarget[tf] = NewTargetsOne(m.PathExpression, len(v3Request.Metrics), alias.New())\n\t\t\t}\n\n\t\t\tif len(m.FilterFunctions) > 0 {\n\t\t\t\tmultiTarget[tf].SetFilteringFunctions(m.PathExpression, m.FilterFunctions)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn multiTarget\n}\n\nfunc (m *MultiTarget) checkMetricsLimitExceeded(num int) error {\n\tif num <= 0 {\n\t\t// zero or negative means unlimited\n\t\treturn nil\n\t}\n\n\tfor _, t := range *m {\n\t\tif num < t.AM.Len() {\n\t\t\treturn errs.NewErrorWithCode(fmt.Sprintf(\"metrics limit exceeded: %d < %d\", num, t.AM.Len()), http.StatusForbidden)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc getDataTimeout(cfg *config.Config, m *MultiTarget) time.Duration {\n\tdataTimeout := cfg.ClickHouse.DataTimeout\n\n\tif len(cfg.ClickHouse.QueryParams) > 1 {\n\t\tvar maxDuration time.Duration\n\n\t\tfor tf := range *m {\n\t\t\tduration := time.Second * time.Duration(tf.Until-tf.From)\n\t\t\tif duration >= maxDuration {\n\t\t\t\tmaxDuration = duration\n\t\t\t}\n\t\t}\n\n\t\tn := config.GetQueryParam(cfg.ClickHouse.QueryParams, maxDuration)\n\n\t\treturn cfg.ClickHouse.QueryParams[n].DataTimeout\n\t}\n\n\treturn dataTimeout\n}\n\nfunc GetQueryLimiter(username string, cfg *config.Config, m *MultiTarget) (string, limiter.ServerLimiter) {\n\tn := 0\n\n\tif username != \"\" && len(cfg.ClickHouse.UserLimits) > 0 {\n\t\tif u, ok := cfg.ClickHouse.UserLimits[username]; ok {\n\t\t\treturn username, u.Limiter\n\t\t}\n\t}\n\n\tif len(cfg.ClickHouse.QueryParams) > 1 {\n\t\tvar maxDuration time.Duration\n\n\t\tfor tf := range *m {\n\t\t\tduration := time.Second * time.Duration(tf.Until-tf.From)\n\t\t\tif duration >= maxDuration {\n\t\t\t\tmaxDuration = duration\n\t\t\t}\n\t\t}\n\n\t\tn = config.GetQueryParam(cfg.ClickHouse.QueryParams, maxDuration)\n\t}\n\n\treturn \"\", cfg.ClickHouse.QueryParams[n].Limiter\n}\n\nfunc GetQueryLimiterFrom(username string, cfg *config.Config, from, until int64) limiter.ServerLimiter {\n\tn := 0\n\n\tif username != \"\" && len(cfg.ClickHouse.UserLimits) > 0 {\n\t\tif u, ok := cfg.ClickHouse.UserLimits[username]; ok {\n\t\t\treturn u.Limiter\n\t\t}\n\t}\n\n\tif len(cfg.ClickHouse.QueryParams) > 1 {\n\t\tn = config.GetQueryParam(cfg.ClickHouse.QueryParams, time.Second*time.Duration(until-from))\n\t}\n\n\treturn cfg.ClickHouse.QueryParams[n].Limiter\n}\n\nfunc GetQueryParam(username string, cfg *config.Config, m *MultiTarget) (*config.QueryParam, int) {\n\tn := 0\n\n\tif len(cfg.ClickHouse.QueryParams) > 1 {\n\t\tvar maxDuration time.Duration\n\n\t\tfor tf := range *m {\n\t\t\tduration := time.Second * time.Duration(tf.Until-tf.From)\n\t\t\tif duration >= maxDuration {\n\t\t\t\tmaxDuration = duration\n\t\t\t}\n\t\t}\n\n\t\tn = config.GetQueryParam(cfg.ClickHouse.QueryParams, maxDuration)\n\t}\n\n\treturn &cfg.ClickHouse.QueryParams[n], n\n}\n\n// Fetch fetches the parsed ClickHouse data returns CHResponses\nfunc (m *MultiTarget) Fetch(ctx context.Context, cfg *config.Config, chContext string, qlimiter limiter.ServerLimiter, queueDuration *time.Duration) (CHResponses, error) {\n\tvar (\n\t\tlock sync.RWMutex\n\t\twg sync.WaitGroup\n\t\tentered int\n\t)\n\n\tlogger := scope.Logger(ctx)\n\n\tsetCarbonlinkClient(&cfg.Carbonlink)\n\n\terr := m.checkMetricsLimitExceeded(cfg.Common.MaxMetricsPerTarget)\n\tif err != nil {\n\t\tlogger.Error(\"data fetch\", zap.Error(err))\n\t\treturn nil, err\n\t}\n\n\tdataTimeout := getDataTimeout(cfg, m)\n\n\tctxTimeout, cancel := context.WithTimeout(ctx, dataTimeout)\n\tdefer func() {\n\t\tfor i := 0; i < entered; i++ {\n\t\t\tqlimiter.Leave(ctxTimeout, \"render\")\n\t\t}\n\n\t\tcancel()\n\t}()\n\n\terrors := make([]error, 0, len(*m))\n\tquery := newQuery(cfg, len(*m))\n\n\tfor tf, targets := range *m {\n\t\ttf, targets := tf, targets\n\n\t\tcond := &conditions{TimeFrame: &tf,\n\t\t\tTargets: targets,\n\t\t\taggregated: cfg.ClickHouse.InternalAggregation,\n\t\t\tappendEmptySeries: cfg.Common.AppendEmptySeries,\n\t\t}\n\t\tif cond.MaxDataPoints <= 0 || int64(cfg.ClickHouse.MaxDataPoints) < cond.MaxDataPoints {\n\t\t\tcond.MaxDataPoints = int64(cfg.ClickHouse.MaxDataPoints)\n\t\t}\n\n\t\terr := cond.selectDataTable(cfg, cond.TimeFrame, chContext)\n\t\tif err != nil {\n\t\t\tlock.Lock()\n\t\t\terrors = append(errors, err)\n\t\t\tlock.Unlock()\n\t\t\tlogger.Error(\"data tables is not specified\", zap.Error(err))\n\n\t\t\treturn EmptyResponse(), err\n\t\t}\n\n\t\tif qlimiter.Enabled() {\n\t\t\tstart := time.Now()\n\t\t\terr = qlimiter.Enter(ctxTimeout, \"render\")\n\t\t\t*queueDuration += time.Since(start)\n\n\t\t\tif err != nil {\n\t\t\t\t// status = http.StatusServiceUnavailable\n\t\t\t\t// queueFail = true\n\t\t\t\t// http.Error(w, err.Error(), status)\n\t\t\t\tlock.Lock()\n\t\t\t\terrors = append(errors, err)\n\t\t\t\tlock.Unlock()\n\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\tentered++\n\t\t}\n\n\t\twg.Add(1)\n\n\t\tgo func(cond *conditions) {\n\t\t\tdefer wg.Done()\n\n\t\t\terr := query.getDataPoints(ctxTimeout, cond)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrors = append(errors, err)\n\t\t\t\tlock.Unlock()\n\n\t\t\t\treturn\n\t\t\t}\n\t\t}(cond)\n\t}\n\n\twg.Wait()\n\n\tfor len(errors) != 0 {\n\t\treturn EmptyResponse(), errors[0]\n\t}\n\n\treturn query.CHResponses, nil\n}\n" }, { "path": "render/data/multi_target_test.go", "content": "package data\n\nimport (\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n)\n\nfunc Test_getDataTimeout(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tcfg *config.Config\n\t\tm *MultiTarget\n\t\twant time.Duration\n\t}{\n\t\t{\n\t\t\tname: \"one DataTimeout\",\n\t\t\tcfg: &config.Config{\n\t\t\t\tClickHouse: config.ClickHouse{\n\t\t\t\t\tDataTimeout: time.Second,\n\t\t\t\t\tQueryParams: []config.QueryParam{\n\t\t\t\t\t\t{ // default params\n\t\t\t\t\t\t\tDuration: 0,\n\t\t\t\t\t\t\tDataTimeout: time.Second,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tm: &MultiTarget{\n\t\t\t\tTimeFrame{\n\t\t\t\t\tFrom: 1647198000,\n\t\t\t\t\tUntil: 1647234000,\n\t\t\t\t}: &Targets{},\n\t\t\t},\n\t\t\twant: time.Second,\n\t\t},\n\t\t{\n\t\t\tname: \"default DataTimeout\",\n\t\t\tcfg: &config.Config{\n\t\t\t\tClickHouse: config.ClickHouse{\n\t\t\t\t\tDataTimeout: time.Second,\n\t\t\t\t\tQueryParams: []config.QueryParam{\n\t\t\t\t\t\t{ // default params\n\t\t\t\t\t\t\tDuration: 0,\n\t\t\t\t\t\t\tDataTimeout: time.Second,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tDuration: time.Hour,\n\t\t\t\t\t\t\tDataTimeout: time.Minute,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tm: &MultiTarget{\n\t\t\t\tTimeFrame{ // 1 hour - 1s\n\t\t\t\t\tFrom: 1647198000,\n\t\t\t\t\tUntil: 1647201600 - 1,\n\t\t\t\t}: &Targets{},\n\t\t\t},\n\t\t\twant: time.Second,\n\t\t},\n\t\t{\n\t\t\tname: \"1m DataTimeout (1 param), select 1h duration\",\n\t\t\tcfg: &config.Config{\n\t\t\t\tClickHouse: config.ClickHouse{\n\t\t\t\t\tDataTimeout: time.Second * 10,\n\t\t\t\t\tQueryParams: []config.QueryParam{\n\t\t\t\t\t\t{ // default params\n\t\t\t\t\t\t\tDuration: 0,\n\t\t\t\t\t\t\tDataTimeout: time.Second,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tDuration: time.Hour,\n\t\t\t\t\t\t\tDataTimeout: time.Minute,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tm: &MultiTarget{\n\t\t\t\tTimeFrame{ // 1 hour\n\t\t\t\t\tFrom: 1647198000,\n\t\t\t\t\tUntil: 1647201600,\n\t\t\t\t}: &Targets{},\n\t\t\t},\n\t\t\twant: time.Minute,\n\t\t},\n\t\t{\n\t\t\tname: \"1m DataTimeout (2 param), select 1h duration\",\n\t\t\tcfg: &config.Config{\n\t\t\t\tClickHouse: config.ClickHouse{\n\t\t\t\t\tDataTimeout: time.Second,\n\t\t\t\t\tQueryParams: []config.QueryParam{\n\t\t\t\t\t\t{ // default params\n\t\t\t\t\t\t\tDuration: 0,\n\t\t\t\t\t\t\tDataTimeout: time.Second,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tDuration: time.Hour,\n\t\t\t\t\t\t\tDataTimeout: time.Minute,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tDuration: time.Hour * 2,\n\t\t\t\t\t\t\tDataTimeout: 10 * time.Minute,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tm: &MultiTarget{\n\t\t\t\tTimeFrame{ // 1 hour\n\t\t\t\t\tFrom: 1647198000,\n\t\t\t\t\tUntil: 1647201600,\n\t\t\t\t}: &Targets{},\n\t\t\t},\n\t\t\twant: time.Minute,\n\t\t},\n\t\t{\n\t\t\tname: \"10m DataTimeout (2 param), select 2h1s duration\",\n\t\t\tcfg: &config.Config{\n\t\t\t\tClickHouse: config.ClickHouse{\n\t\t\t\t\tDataTimeout: time.Second,\n\t\t\t\t\tQueryParams: []config.QueryParam{\n\t\t\t\t\t\t{ // default params\n\t\t\t\t\t\t\tDuration: 0,\n\t\t\t\t\t\t\tDataTimeout: time.Second,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tDuration: time.Hour,\n\t\t\t\t\t\t\tDataTimeout: time.Minute,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tDuration: time.Hour * 2,\n\t\t\t\t\t\t\tDataTimeout: 10 * time.Minute,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tm: &MultiTarget{\n\t\t\t\tTimeFrame{ // 2 hour 1s\n\t\t\t\t\tFrom: 1647198000,\n\t\t\t\t\tUntil: 1647205201,\n\t\t\t\t}: &Targets{},\n\t\t\t},\n\t\t\twant: 10 * time.Minute,\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tif got := getDataTimeout(tt.cfg, tt.m); got != tt.want {\n\t\t\t\tt.Errorf(\"getDataTimeout() = %v, want %v\", got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "render/data/query.go", "content": "package data\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"os\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"go.uber.org/zap\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/helper/errs\"\n\t\"github.com/lomik/graphite-clickhouse/helper/rollup\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/dry\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/reverse\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/where\"\n)\n\n// from, until, step, function, table, prewhere, where\n// arrayFilter(x->isNotNull(x)) - do not pass nulls to client\n// -Resample - group time and values by time intervals and apply aggregation function\n// -OrNull - if there aren't points in an interval, null will be returned\n// intDiv(Time, x)*x - round Time down to step multiplier\n// TODO: support custom aggregating functions\nconst queryAggregated = `WITH anyResample(%[1]d, %[2]d, %[3]d)(toUInt32(intDiv(Time, %[3]d)*%[3]d), Time) AS mask\nSELECT Path,\n arrayFilter(m->m!=0, mask) AS times,\n arrayFilter((v,m)->m!=0, %[4]sResample(%[1]d, %[2]d, %[3]d)(Value, Time), mask) AS values\nFROM %[5]s\n%[6]s\n%[7]s\nGROUP BY Path\nFORMAT RowBinary`\n\n// table, prewhere, where\nconst queryUnaggregated = `SELECT Path, groupArray(Time), groupArray(Value), groupArray(Timestamp)\nFROM %s\n%s\n%s\nGROUP BY Path\nFORMAT RowBinary`\n\n// name of external-data table with metrics paths\nconst extTableName = \"metrics_list\"\n\ntype query struct {\n\tCHResponses\n\tcStep *commonStep\n\tchTLSConfig *tls.Config\n\tchQueryParams []config.QueryParam\n\n\tchConnectTimeout time.Duration\n\tchProgressSendingInterval time.Duration\n\tdebugDir string\n\tdebugExtDataPerm os.FileMode\n\tfeatureFlags *config.FeatureFlags\n\tlock sync.RWMutex\n}\n\ntype conditions struct {\n\t*TimeFrame\n\t*Targets\n\t// aggregated shows is it request with ClickHouse aggregation or not\n\taggregated bool\n\t// step is used in requests for proper until/from calculation. It's max(steps) for non-aggregated\n\t// requests and LCM(steps) for aggregated requests\n\tstep int64\n\t// from is aligned to step\n\tfrom int64\n\t// until is aligned to step\n\tuntil int64\n\t// metricUnreversed grouped by step\n\tsteps map[uint32][]string\n\t// prewhere contains PREWHERE condition\n\tprewhere string\n\t// where contains WHERE condition\n\twhere string\n\t// show list of NaN values instead of empty results\n\tappendEmptySeries bool\n\t// metricUnreversed grouped by aggregating function\n\taggregations map[string][]string\n\t// External-data bodies grouped by aggregatig function. For non-aggregated requests \"\" used as a key\n\textDataBodies map[string]*strings.Builder\n\tmetricsRequested []string\n\tmetricsUnreverse []string\n\tmetricsLookup []string\n\tappliedFunctions map[string][]string\n}\n\nfunc newQuery(cfg *config.Config, targets int) *query {\n\tvar cStep *commonStep = nil\n\tif cfg.ClickHouse.InternalAggregation {\n\t\tcStep = &commonStep{\n\t\t\tresult: 0,\n\t\t\twg: sync.WaitGroup{},\n\t\t\tlock: sync.RWMutex{},\n\t\t}\n\n\t\tcStep.addTargets(targets)\n\t}\n\n\tquery := &query{\n\t\tCHResponses: make([]CHResponse, 0, targets),\n\t\tcStep: cStep,\n\t\tchQueryParams: cfg.ClickHouse.QueryParams,\n\t\tchConnectTimeout: cfg.ClickHouse.ConnectTimeout,\n\t\tchProgressSendingInterval: cfg.ClickHouse.ProgressSendingInterval,\n\t\tchTLSConfig: cfg.ClickHouse.TLSConfig,\n\t\tdebugDir: cfg.Debug.Directory,\n\t\tdebugExtDataPerm: cfg.Debug.ExternalDataPerm,\n\t\tfeatureFlags: &cfg.FeatureFlags,\n\t\tlock: sync.RWMutex{},\n\t}\n\n\treturn query\n}\n\nfunc (q *query) appendReply(chr CHResponse) {\n\tq.lock.Lock()\n\tq.CHResponses = append(q.CHResponses, chr)\n\tq.lock.Unlock()\n}\n\nfunc (q *query) getParam(from, until int64) (string, time.Duration) {\n\tduration := time.Second * time.Duration(until-from)\n\n\tn := config.GetQueryParam(q.chQueryParams, duration)\n\n\treturn q.chQueryParams[n].URL, q.chQueryParams[n].DataTimeout\n}\n\nfunc (q *query) getDataPoints(ctx context.Context, cond *conditions) error {\n\tlogger := scope.Logger(ctx)\n\n\tvar err error\n\n\tcond.prepareMetricsLists()\n\n\tif len(cond.metricsRequested) == 0 {\n\t\tq.cStep.doneTarget()\n\t\treturn nil\n\t}\n\n\t// carbonlink request\n\tcarbonlinkResponseRead := queryCarbonlink(ctx, carbonlink, cond.metricsUnreverse)\n\n\terr = cond.prepareLookup()\n\tif err != nil {\n\t\tlogger.Error(\"prepare_lookup\", zap.Error(err))\n\t\treturn errs.NewErrorWithCode(err.Error(), http.StatusBadRequest)\n\t}\n\n\tcond.setStep(q.cStep)\n\n\tif cond.step < 1 {\n\t\treturn ErrSetStepTimeout\n\t}\n\n\tcond.setFromUntil()\n\tcond.setPrewhere()\n\tcond.setWhere()\n\n\tqueryContext, queryCancel := context.WithCancel(ctx)\n\tdefer queryCancel()\n\n\tdata := prepareData(queryContext, len(cond.extDataBodies), carbonlinkResponseRead)\n\n\tvar ch_read_bytes, ch_read_rows int64\n\n\tfor agg, extTableBody := range cond.extDataBodies {\n\t\textData := q.metricsListExtData(extTableBody)\n\t\tquery := cond.generateQuery(agg)\n\n\t\tdata.wg.Add(1)\n\n\t\tgo func() {\n\t\t\tdefer data.wg.Done()\n\n\t\t\tchURL, chDataTimeout := q.getParam(cond.from, cond.until)\n\n\t\t\tbody, err := clickhouse.Reader(\n\t\t\t\tscope.WithTable(ctx, cond.pointsTable),\n\t\t\t\tchURL,\n\t\t\t\tquery,\n\t\t\t\tclickhouse.Options{\n\t\t\t\t\tTimeout: chDataTimeout,\n\t\t\t\t\tConnectTimeout: q.chConnectTimeout,\n\t\t\t\t\tTLSConfig: q.chTLSConfig,\n\t\t\t\t\tCheckRequestProgress: q.featureFlags.LogQueryProgress,\n\t\t\t\t\tProgressSendingInterval: q.chProgressSendingInterval,\n\t\t\t\t},\n\t\t\t\textData,\n\t\t\t)\n\t\t\tif err == nil {\n\t\t\t\tatomic.AddInt64(&ch_read_bytes, body.ChReadBytes())\n\t\t\t\tatomic.AddInt64(&ch_read_rows, body.ChReadRows())\n\n\t\t\t\terr = data.parseResponse(queryContext, body, cond)\n\t\t\t\tif err != nil {\n\t\t\t\t\tlogger.Error(\"reader\", zap.Error(err))\n\t\t\t\t\tdata.e <- err\n\n\t\t\t\t\tqueryCancel()\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tlogger.Error(\"reader\", zap.Error(err))\n\t\t\t\tdata.e <- err\n\n\t\t\t\tqueryCancel()\n\t\t\t}\n\t\t}()\n\t}\n\n\terr = data.wait(queryContext)\n\tmetrics.SendQueryRead(cond.queryMetrics, cond.from, cond.until, data.spent.Milliseconds(), int64(data.Points.Len()), int64(data.length), ch_read_rows, ch_read_bytes, err != nil)\n\n\tif err != nil {\n\t\tlogger.Error(\n\t\t\t\"data_parser\", zap.Error(err), zap.Int(\"read_bytes\", data.length),\n\t\t\tzap.String(\"runtime\", data.spent.String()), zap.Duration(\"runtime_ns\", data.spent),\n\t\t)\n\n\t\treturn err\n\t}\n\n\tlogger.Info(\n\t\t\"data_parse\", zap.Int(\"read_bytes\", data.length), zap.Int(\"read_points\", data.Points.Len()),\n\t\tzap.String(\"runtime\", data.spent.String()), zap.Duration(\"runtime_ns\", data.spent),\n\t)\n\n\tdata.setSteps(cond)\n\tdata.Points.SetAggregations(cond.aggregations)\n\n\t// ClickHouse returns sorted and uniq values, when internal aggregation is used\n\t// But if carbonlink is used, we still need to sort, filter and rollup points\n\tif !cond.aggregated || carbonlink != nil {\n\t\tsortStart := time.Now()\n\n\t\tdata.Points.Sort()\n\n\t\td := time.Since(sortStart)\n\t\tlogger.Debug(\"sort\", zap.String(\"runtime\", d.String()), zap.Duration(\"runtime_ns\", d))\n\n\t\tdata.Points.Uniq()\n\n\t\trollupStart := time.Now()\n\n\t\terr = cond.rollupRules.RollupPoints(data.Points, cond.From, data.CommonStep)\n\t\tif err != nil {\n\t\t\tlogger.Error(\"rollup failed\", zap.Error(err))\n\t\t\treturn err\n\t\t}\n\n\t\trollupTime := time.Since(rollupStart)\n\t\tlogger.Debug(\n\t\t\t\"rollup\",\n\t\t\tzap.String(\"runtime\", rollupTime.String()),\n\t\t\tzap.Duration(\"runtime_ns\", rollupTime),\n\t\t)\n\t}\n\n\tdata.AM = cond.AM\n\n\tq.appendReply(CHResponse{\n\t\tData: data.Data,\n\t\tFrom: cond.From,\n\t\tUntil: cond.Until,\n\t\tAppendOutEmptySeries: cond.appendEmptySeries,\n\t\tAppliedFunctions: cond.appliedFunctions,\n\t})\n\n\treturn nil\n}\n\nfunc (q *query) metricsListExtData(body *strings.Builder) *clickhouse.ExternalData {\n\textTable := clickhouse.ExternalTable{\n\t\tName: extTableName,\n\t\tColumns: []clickhouse.Column{{\n\t\t\tName: \"Path\",\n\t\t\tType: \"String\",\n\t\t}},\n\t\tFormat: \"TSV\",\n\t\tData: []byte(body.String()),\n\t}\n\n\textData := clickhouse.NewExternalData(extTable)\n\textData.SetDebug(q.debugDir, q.debugExtDataPerm)\n\n\treturn extData\n}\n\nfunc (c *conditions) prepareMetricsLists() {\n\tc.metricsUnreverse = c.AM.Series(false)\n\tc.metricsRequested = c.metricsUnreverse\n\n\tif c.isReverse {\n\t\tc.metricsRequested = make([]string, len(c.metricsRequested))\n\t\tfor i := range c.metricsRequested {\n\t\t\tc.metricsRequested[i] = reverse.String(c.metricsUnreverse[i])\n\t\t}\n\t}\n\n\tc.metricsLookup = c.metricsRequested\n\tif c.rollupUseReverted {\n\t\tc.metricsLookup = c.metricsUnreverse\n\t}\n}\n\nfunc (c *conditions) prepareLookup() error {\n\tage := uint32(dry.Max(0, time.Now().Unix()-c.From))\n\tc.aggregations = make(map[string][]string)\n\tc.appliedFunctions = make(map[string][]string)\n\tc.extDataBodies = make(map[string]*strings.Builder)\n\tc.steps = make(map[uint32][]string)\n\taggName := \"\"\n\n\tfor i := range c.metricsRequested {\n\t\tstep, agg, _, _ := c.rollupRules.Lookup(c.metricsLookup[i], age, false)\n\n\t\t// Override agregation with an argument of consolidateBy function.\n\t\t// consolidateBy with its argument is passed through FilteringFunctions field of carbonapi_v3_pb protocol.\n\t\t// Currently it just finds the first target matching the metric\n\t\t// to avoid making multiple request for every type of aggregation for a given metric.\n\t\tfor _, alias := range c.AM.Get(c.metricsUnreverse[i]) {\n\t\t\trequestedAgg, err := c.GetRequestedAggregation(alias.Target)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"failed to choose appropriate aggregation for '%s': %s\", alias.Target, err.Error())\n\t\t\t}\n\n\t\t\tif requestedAgg != \"\" {\n\t\t\t\tagg = rollup.AggrMap[requestedAgg]\n\t\t\t\tc.appliedFunctions[alias.Target] = []string{graphiteConsolidationFunction}\n\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tif _, ok := c.steps[step]; !ok {\n\t\t\tc.steps[step] = make([]string, 0)\n\t\t}\n\t\t// Fill up metric names for steps only for non-aggregated requests.\n\t\t// Aggregated use commonStep\n\t\tif !c.aggregated {\n\t\t\tc.steps[step] = append(c.steps[step], c.metricsUnreverse[i])\n\t\t}\n\n\t\t// Fill up metric names for aggregations\n\t\tif mm, ok := c.aggregations[agg.Name()]; ok {\n\t\t\tc.aggregations[agg.Name()] = append(mm, c.metricsUnreverse[i])\n\t\t} else {\n\t\t\tc.aggregations[agg.Name()] = []string{c.metricsUnreverse[i]}\n\t\t}\n\n\t\t// Build external-data bodies. For non-aggregated requests there is only one request\n\t\tif c.aggregated {\n\t\t\taggName = agg.Name()\n\t\t}\n\n\t\tif mm, ok := c.extDataBodies[aggName]; ok {\n\t\t\tmm.WriteString(c.metricsRequested[i] + \"\\n\")\n\t\t} else {\n\t\t\tvar mm strings.Builder\n\t\t\tc.extDataBodies[aggName] = &mm\n\t\t\tmm.WriteString(c.metricsRequested[i] + \"\\n\")\n\t\t}\n\t}\n\n\treturn nil\n}\n\nvar ErrSetStepTimeout = errors.New(\"unexpected error, setStep timeout\")\n\nfunc (c *conditions) setStep(cStep *commonStep) {\n\tstep := int64(0)\n\n\tif !c.aggregated {\n\t\t// Use max(steps)\n\t\tfor s := range c.steps {\n\t\t\tstep = dry.Max(step, int64(s))\n\t\t}\n\n\t\tc.step = step\n\n\t\treturn\n\t}\n\n\t// Use LCM(steps)\n\t// XXX: This could cause problems, when MutliFetchRequest uses different MaxDataPoints,\n\t// but currently (2021-04-22) it's not possible\n\tfor s := range c.steps {\n\t\tstep = cStep.calculateUnsafe(step, int64(s))\n\t}\n\n\tcStep.calculate(step)\n\n\trStep := cStep.getResult()\n\tif rStep == -1 {\n\t\tc.step = -1\n\t\treturn\n\t}\n\n\tstep = dry.Max(rStep, dry.Ceil(c.Until-c.From, c.MaxDataPoints))\n\tc.step = dry.CeilToMultiplier(step, rStep)\n\n\treturn\n}\n\nfunc (c *conditions) setFromUntil() {\n\tc.from = dry.CeilToMultiplier(c.From, c.step)\n\tc.until = dry.FloorToMultiplier(c.Until, c.step) + c.step - 1\n}\n\nfunc (c *conditions) setPrewhere() {\n\tpw := where.New()\n\tpw.And(where.DateBetween(\"Date\", c.from, c.until))\n\tc.prewhere = pw.PreWhereSQL()\n}\n\nfunc (c *conditions) setWhere() {\n\twr := where.New()\n\twr.And(where.InTable(\"Path\", extTableName))\n\twr.And(where.TimestampBetween(\"Time\", c.from, c.until))\n\tc.where = wr.SQL()\n}\n\nfunc (c *conditions) generateQuery(agg string) string {\n\tif c.aggregated {\n\t\treturn c.generateQueryaAggregated(agg)\n\t}\n\n\treturn c.generateQueryUnaggregated()\n}\n\nfunc (c *conditions) generateQueryaAggregated(agg string) string {\n\treturn fmt.Sprintf(\n\t\tqueryAggregated,\n\t\tc.from, c.until, c.step, agg,\n\t\tc.pointsTable, c.prewhere, c.where,\n\t)\n}\n\nfunc (c *conditions) generateQueryUnaggregated() string {\n\treturn fmt.Sprintf(queryUnaggregated, c.pointsTable, c.prewhere, c.where)\n}\n" }, { "path": "render/data/query_test.go", "content": "package data\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\tv3pb \"github.com/go-graphite/protocol/carbonapi_v3_pb\"\n\t\"github.com/lomik/graphite-clickhouse/finder\"\n\t\"github.com/lomik/graphite-clickhouse/helper/date\"\n\t\"github.com/lomik/graphite-clickhouse/helper/rollup\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/alias\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/reverse\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc genPattern(regexp, function string, retention []rollup.Retention) rollup.Pattern {\n\treturn rollup.Pattern{Regexp: regexp, Function: function, Retention: retention}\n}\n\nvar finderResult *finder.MockFinder = finder.NewMockFinder([][]byte{\n\t[]byte(\"5_sec.name.max\"),\n\t[]byte(\"1_min.name.avg\"),\n\t[]byte(\"5_min.name.min\"),\n\t[]byte(\"10_min.name.any\"), // defaults will be used in rollup.Rules\n})\n\nfunc newAM() *alias.Map {\n\tam := alias.New()\n\tam.MergeTarget(finderResult, \"*.name.*\", false)\n\n\treturn am\n}\n\nfunc newRules(reversed bool) *rollup.Rules {\n\tfiveSec := []rollup.Retention{{Age: 0, Precision: 5}, {Age: 3600, Precision: 60}}\n\toneMin := []rollup.Retention{{Age: 0, Precision: 60}, {Age: 3600, Precision: 300}}\n\tfiveMin := []rollup.Retention{{Age: 0, Precision: 300}, {Age: 3600, Precision: 1200}}\n\temptyRet := make([]rollup.Retention, 0)\n\n\tvar pattern []rollup.Pattern\n\n\tif reversed {\n\t\tpattern = []rollup.Pattern{\n\t\t\tgenPattern(\"[.]5_sec$\", \"\", fiveSec),\n\t\t\tgenPattern(\"[.]1_min$\", \"\", oneMin),\n\t\t\tgenPattern(\"[.]5_min$\", \"\", fiveMin),\n\t\t\tgenPattern(\"^max[.]\", \"max\", emptyRet),\n\t\t\tgenPattern(\"^min[.]\", \"min\", emptyRet),\n\t\t\tgenPattern(\"^avg[.]\", \"avg\", emptyRet),\n\t\t}\n\t} else {\n\t\tpattern = []rollup.Pattern{\n\t\t\tgenPattern(\"^5_sec[.]\", \"\", fiveSec),\n\t\t\tgenPattern(\"^1_min[.]\", \"\", oneMin),\n\t\t\tgenPattern(\"^5_min[.]\", \"\", fiveMin),\n\t\t\tgenPattern(\"[.]max$\", \"max\", emptyRet),\n\t\t\tgenPattern(\"[.]min$\", \"min\", emptyRet),\n\t\t\tgenPattern(\"[.]avg$\", \"avg\", emptyRet),\n\t\t}\n\t}\n\n\trules, _ := rollup.NewMockRules(pattern, 30, \"avg\")\n\n\treturn rules\n}\n\nfunc ageToTimestamp(age int64) int64 {\n\treturn time.Now().Unix() - age\n}\n\n// fromAge and untilAge are relative age of timeframe\nfunc newCondition(fromAge, untilAge, maxDataPoints int64) *conditions {\n\ttf := TimeFrame{ageToTimestamp(fromAge), ageToTimestamp(untilAge), maxDataPoints}\n\ttt := NewTargets([]string{\"*.name.*\"}, newAM())\n\ttt.pointsTable = \"graphite.data\"\n\ttt.rollupRules = newRules(false)\n\n\treturn &conditions{TimeFrame: &tf, Targets: tt}\n}\n\nfunc extTableString(et map[string]*strings.Builder) map[string]string {\n\tett := make(map[string]string)\n\tfor a := range et {\n\t\tett[a] = et[a].String()\n\t}\n\n\treturn ett\n}\n\nfunc TestPrepareMetricsLists(t *testing.T) {\n\tt.Run(\"unreversed request\", func(t *testing.T) {\n\t\tcond := newCondition(0, 0, 60)\n\t\tcond.isReverse = false\n\t\tcond.rollupUseReverted = false\n\t\tcond.prepareMetricsLists()\n\n\t\texpectedSeries := finderResult.Strings()\n\t\tsort.Strings(expectedSeries)\n\t\tsort.Strings(cond.metricsLookup)\n\t\tsort.Strings(cond.metricsRequested)\n\t\tsort.Strings(cond.metricsUnreverse)\n\t\tassert.Equal(t, expectedSeries, cond.metricsUnreverse)\n\t\tassert.Equal(t, cond.metricsRequested, cond.metricsUnreverse)\n\t\tassert.Equal(t, cond.metricsLookup, cond.metricsRequested)\n\n\t\t// nothing should change in case of cond.isReverse == false\n\t\tcond.rollupUseReverted = true\n\t\tcond.prepareMetricsLists()\n\t\tsort.Strings(cond.metricsLookup)\n\t\tsort.Strings(cond.metricsRequested)\n\t\tsort.Strings(cond.metricsUnreverse)\n\t\tassert.Equal(t, expectedSeries, cond.metricsUnreverse)\n\t\tassert.Equal(t, cond.metricsRequested, cond.metricsUnreverse)\n\t\tassert.Equal(t, cond.metricsLookup, cond.metricsRequested)\n\t})\n\n\tt.Run(\"reversed request\", func(t *testing.T) {\n\t\tcond := newCondition(0, 0, 60)\n\t\tcond.isReverse = true\n\t\tcond.rollupUseReverted = false\n\t\tcond.prepareMetricsLists()\n\n\t\tfor i := range cond.metricsRequested {\n\t\t\tassert.Equal(t, cond.metricsRequested[i], reverse.String(cond.metricsUnreverse[i]))\n\t\t}\n\n\t\texpectedSeries := finderResult.Strings()\n\t\tsort.Strings(expectedSeries)\n\n\t\texpectedSeriesReversed := make([]string, len(expectedSeries))\n\t\tfor i := range expectedSeries {\n\t\t\texpectedSeriesReversed[i] = reverse.String(expectedSeries[i])\n\t\t}\n\n\t\tsort.Strings(expectedSeriesReversed)\n\t\tsort.Strings(cond.metricsLookup)\n\t\tsort.Strings(cond.metricsRequested)\n\t\tsort.Strings(cond.metricsUnreverse)\n\t\tassert.Equal(t, expectedSeries, cond.metricsUnreverse)\n\t\tassert.Equal(t, expectedSeriesReversed, cond.metricsRequested)\n\t\tassert.Equal(t, cond.metricsLookup, cond.metricsRequested)\n\n\t\tcond.rollupUseReverted = true\n\t\tcond.prepareMetricsLists()\n\n\t\tfor i := range cond.metricsRequested {\n\t\t\tassert.Equal(t, cond.metricsRequested[i], reverse.String(cond.metricsUnreverse[i]))\n\t\t}\n\n\t\tsort.Strings(cond.metricsLookup)\n\t\tsort.Strings(cond.metricsRequested)\n\t\tsort.Strings(cond.metricsUnreverse)\n\t\tassert.Equal(t, expectedSeries, cond.metricsUnreverse)\n\t\tassert.Equal(t, expectedSeriesReversed, cond.metricsRequested)\n\t\tassert.Equal(t, cond.metricsLookup, cond.metricsUnreverse)\n\t})\n}\n\nfunc TestPrepareLookup(t *testing.T) {\n\t// cases:\n\t// - aggregater / non-aggregated\n\t// - proper/inproper lookup rules for reversed table\n\t// testing:\n\t// - c.aggregations\n\t// - c.extDataBodies: only for c.isReverse=false\n\t// - c.steps\n\tt.Run(\"aggregated non-reverse query\", func(t *testing.T) {\n\t\tcond := newCondition(5400, 1800, 5)\n\t\tcond.aggregated = true\n\t\tcond.isReverse = false\n\t\tcond.prepareMetricsLists()\n\t\tsort.Strings(cond.metricsLookup)\n\t\tsort.Strings(cond.metricsRequested)\n\t\tsort.Strings(cond.metricsUnreverse)\n\t\tcond.prepareLookup()\n\n\t\taggregations := map[string][]string{\n\t\t\t\"avg\": {\"10_min.name.any\", \"1_min.name.avg\"},\n\t\t\t\"max\": {\"5_sec.name.max\"},\n\t\t\t\"min\": {\"5_min.name.min\"},\n\t\t}\n\t\tassert.Equal(t, aggregations, cond.aggregations)\n\t\t// Steps saves only values, not the metrics list\n\t\tsteps := map[uint32][]string{\n\t\t\t30: {},\n\t\t\t60: {},\n\t\t\t300: {},\n\t\t\t1200: {},\n\t\t}\n\t\tassert.Equal(t, steps, cond.steps)\n\n\t\tbodies := make(map[string]string)\n\t\tfor a, m := range aggregations {\n\t\t\tbodies[a] = strings.Join(m, \"\\n\") + \"\\n\"\n\t\t}\n\n\t\tassert.Equal(t, bodies, extTableString(cond.extDataBodies))\n\n\t\tcond.From = ageToTimestamp(1800)\n\t\tcond.Until = ageToTimestamp(0)\n\t\tcond.prepareLookup()\n\n\t\tsteps = map[uint32][]string{\n\t\t\t30: {},\n\t\t\t60: {},\n\t\t\t300: {},\n\t\t\t5: {},\n\t\t}\n\t\tassert.Equal(t, steps, cond.steps)\n\t\tassert.Equal(t, aggregations, cond.aggregations)\n\t\tassert.Equal(t, bodies, extTableString(cond.extDataBodies))\n\t})\n\n\tt.Run(\"non-aggregated non-reverse query\", func(t *testing.T) {\n\t\tcond := newCondition(5400, 1800, 5)\n\t\tcond.aggregated = false\n\t\tcond.isReverse = false\n\t\tcond.prepareMetricsLists()\n\t\tsort.Strings(cond.metricsLookup)\n\t\tsort.Strings(cond.metricsRequested)\n\t\tsort.Strings(cond.metricsUnreverse)\n\t\tcond.prepareLookup()\n\n\t\taggregations := map[string][]string{\n\t\t\t\"avg\": {\"10_min.name.any\", \"1_min.name.avg\"},\n\t\t\t\"max\": {\"5_sec.name.max\"},\n\t\t\t\"min\": {\"5_min.name.min\"},\n\t\t}\n\t\tassert.Equal(t, aggregations, cond.aggregations)\n\t\t// Steps saves only values, not the metrics list\n\t\tsteps := map[uint32][]string{\n\t\t\t30: {\"10_min.name.any\"},\n\t\t\t60: {\"5_sec.name.max\"},\n\t\t\t300: {\"1_min.name.avg\"},\n\t\t\t1200: {\"5_min.name.min\"},\n\t\t}\n\t\tassert.Equal(t, steps, cond.steps)\n\n\t\tbodies := map[string]string{\"\": \"10_min.name.any\\n1_min.name.avg\\n5_min.name.min\\n5_sec.name.max\\n\"}\n\t\tassert.Equal(t, bodies, extTableString(cond.extDataBodies))\n\n\t\tcond.From = ageToTimestamp(1800)\n\t\tcond.Until = ageToTimestamp(0)\n\t\tcond.prepareLookup()\n\n\t\tsteps = map[uint32][]string{\n\t\t\t5: {\"5_sec.name.max\"},\n\t\t\t30: {\"10_min.name.any\"},\n\t\t\t60: {\"1_min.name.avg\"},\n\t\t\t300: {\"5_min.name.min\"},\n\t\t}\n\t\tassert.Equal(t, steps, cond.steps)\n\t\tassert.Equal(t, aggregations, cond.aggregations)\n\t\tassert.Equal(t, bodies, extTableString(cond.extDataBodies))\n\t})\n\n\tt.Run(\"reverse query with improper rules\", func(t *testing.T) {\n\t\tcond := newCondition(5400, 1800, 5)\n\t\tcond.aggregated = false\n\t\tcond.isReverse = true\n\t\tcond.prepareMetricsLists()\n\t\tsort.Strings(cond.metricsUnreverse)\n\t\tsort.Strings(cond.metricsRequested)\n\t\tcond.prepareLookup()\n\n\t\taggregations := map[string][]string{\n\t\t\t\"avg\": {\"10_min.name.any\", \"1_min.name.avg\", \"5_min.name.min\", \"5_sec.name.max\"},\n\t\t}\n\t\tassert.Equal(t, aggregations, cond.aggregations)\n\t\t// Steps saves only values, not the metrics list\n\t\tsteps := map[uint32][]string{\n\t\t\t30: {\"10_min.name.any\", \"1_min.name.avg\", \"5_min.name.min\", \"5_sec.name.max\"},\n\t\t}\n\t\tassert.Equal(t, steps, cond.steps)\n\n\t\tbodies := map[string]string{\"\": \"any.name.10_min\\navg.name.1_min\\nmax.name.5_sec\\nmin.name.5_min\\n\"}\n\t\tassert.Equal(t, bodies, extTableString(cond.extDataBodies))\n\n\t\tcond.From = ageToTimestamp(1800)\n\t\tcond.Until = ageToTimestamp(0)\n\t\tcond.prepareLookup()\n\t\tassert.Equal(t, steps, cond.steps)\n\t\tassert.Equal(t, aggregations, cond.aggregations)\n\t\tassert.Equal(t, bodies, extTableString(cond.extDataBodies))\n\t})\n\n\tt.Run(\"reverse query with proper rules\", func(t *testing.T) {\n\t\tcond := newCondition(5400, 1800, 5)\n\t\tcond.rollupRules = newRules(true)\n\t\tcond.aggregated = false\n\t\tcond.isReverse = true\n\t\tcond.prepareMetricsLists()\n\t\tcond.prepareLookup()\n\n\t\tfor a := range cond.aggregations {\n\t\t\tsort.Strings(cond.aggregations[a])\n\t\t}\n\n\t\taggregations := map[string][]string{\n\t\t\t\"avg\": {\"10_min.name.any\", \"1_min.name.avg\"},\n\t\t\t\"max\": {\"5_sec.name.max\"},\n\t\t\t\"min\": {\"5_min.name.min\"},\n\t\t}\n\t\tassert.Equal(t, aggregations, cond.aggregations)\n\t\t// Steps saves only values, not the metrics list\n\t\tsteps := map[uint32][]string{\n\t\t\t30: {\"10_min.name.any\"},\n\t\t\t60: {\"5_sec.name.max\"},\n\t\t\t300: {\"1_min.name.avg\"},\n\t\t\t1200: {\"5_min.name.min\"},\n\t\t}\n\t\tassert.Equal(t, steps, cond.steps)\n\n\t\tcond.From = ageToTimestamp(1800)\n\t\tcond.Until = ageToTimestamp(0)\n\t\tcond.prepareLookup()\n\n\t\tsteps = map[uint32][]string{\n\t\t\t5: {\"5_sec.name.max\"},\n\t\t\t30: {\"10_min.name.any\"},\n\t\t\t60: {\"1_min.name.avg\"},\n\t\t\t300: {\"5_min.name.min\"},\n\t\t}\n\n\t\tfor a := range cond.aggregations {\n\t\t\tsort.Strings(cond.aggregations[a])\n\t\t}\n\n\t\tassert.Equal(t, steps, cond.steps)\n\t\tassert.Equal(t, aggregations, cond.aggregations)\n\t})\n\n\tt.Run(\"non-reverse query with override aggregation\", func(t *testing.T) {\n\t\tcond := newCondition(5400, 1800, 5)\n\n\t\tcond.aggregated = true\n\t\tcond.isReverse = false\n\t\tcond.prepareMetricsLists()\n\t\tsort.Strings(cond.metricsLookup)\n\t\tsort.Strings(cond.metricsRequested)\n\t\tsort.Strings(cond.metricsUnreverse)\n\n\t\tvar aggregations map[string][]string\n\n\t\tfor _, aggrStr := range []string{\"avg\", \"min\", \"max\", \"sum\"} {\n\t\t\tcond.SetFilteringFunctions(\n\t\t\t\t\"*.name.*\",\n\t\t\t\t[]*v3pb.FilteringFunction{{Name: \"consolidateBy\", Arguments: []string{aggrStr}}},\n\t\t\t)\n\t\t\tcond.prepareLookup()\n\n\t\t\taggregations = map[string][]string{\n\t\t\t\taggrStr: {\"10_min.name.any\", \"1_min.name.avg\", \"5_min.name.min\", \"5_sec.name.max\"},\n\t\t\t}\n\t\t\tassert.Equal(t, aggregations, cond.aggregations)\n\t\t}\n\n\t\t// Steps saves only values, not the metrics list\n\t\tsteps := map[uint32][]string{\n\t\t\t30: {},\n\t\t\t60: {},\n\t\t\t300: {},\n\t\t\t1200: {},\n\t\t}\n\t\tassert.Equal(t, steps, cond.steps)\n\n\t\tbodies := make(map[string]string)\n\t\tfor a, m := range aggregations {\n\t\t\tbodies[a] = strings.Join(m, \"\\n\") + \"\\n\"\n\t\t}\n\n\t\tassert.Equal(t, bodies, extTableString(cond.extDataBodies))\n\n\t\tcond.From = ageToTimestamp(1800)\n\t\tcond.Until = ageToTimestamp(0)\n\t\tcond.prepareLookup()\n\n\t\tsteps = map[uint32][]string{\n\t\t\t30: {},\n\t\t\t60: {},\n\t\t\t300: {},\n\t\t\t5: {},\n\t\t}\n\t\tassert.Equal(t, steps, cond.steps)\n\t\tassert.Equal(t, aggregations, cond.aggregations)\n\t\tassert.Equal(t, bodies, extTableString(cond.extDataBodies))\n\t})\n}\n\nfunc TestSetStep(t *testing.T) {\n\tt.Run(\"unaggregated max\", func(t *testing.T) {\n\t\tcond := newCondition(1800, 0, 1)\n\t\tcond.prepareMetricsLists()\n\t\tcond.prepareLookup()\n\t\tcond.setStep(nil)\n\n\t\tvar step int64 = 300\n\n\t\tassert.Equal(t, step, cond.step)\n\n\t\tcond.From = ageToTimestamp(5400)\n\t\tcond.prepareLookup()\n\t\tcond.setStep(nil)\n\n\t\tstep = 1200\n\t\tassert.Equal(t, step, cond.step)\n\t})\n\n\tt.Run(\"aggregated common step\", func(t *testing.T) {\n\t\tcStep := &commonStep{\n\t\t\tresult: 0,\n\t\t\twg: sync.WaitGroup{},\n\t\t\tlock: sync.RWMutex{},\n\t\t}\n\n\t\tcond := newCondition(1800, 0, 2)\n\t\tcond.aggregated = true\n\t\tcond.prepareMetricsLists()\n\t\tcond.prepareLookup()\n\n\t\tcStep.addTargets(1)\n\t\tcond.setStep(cStep)\n\n\t\tvar step int64 = 1800 / 2\n\n\t\tassert.Equal(t, step, cond.step)\n\n\t\tcStep.addTargets(1)\n\t\tcStep.result = 0\n\t\tcond.From = ageToTimestamp(1200)\n\t\tcond.Until = ageToTimestamp(700)\n\t\tcond.MaxDataPoints = 5\n\t\tcond.setStep(cStep)\n\n\t\tstep = 300\n\t\tassert.Equal(t, step, cond.step)\n\n\t\tcStep.addTargets(1)\n\t\tcStep.result = 0\n\t\tcond.MaxDataPoints = 10\n\t\tcond.steps = map[uint32][]string{1: {}, 5: {}, 3: {}, 4: {}}\n\t\tcond.setStep(cStep)\n\n\t\tstep = 60\n\t\tassert.Equal(t, step, cond.step)\n\n\t\tcStep.addTargets(1)\n\t\tcStep.result = 0\n\t\tcond.MaxDataPoints = 7\n\t\tcond.steps = map[uint32][]string{1: {}, 5: {}, 8: {}, 4: {}}\n\t\tcond.setStep(cStep)\n\n\t\tstep = 80\n\t\tassert.Equal(t, step, cond.step)\n\n\t\tcStep.addTargets(1)\n\n\t\tcond.MaxDataPoints = 6\n\t\tcond.setStep(cStep)\n\n\t\tstep = 120\n\t\tassert.Equal(t, step, cond.step)\n\t})\n}\n\nfunc TestSetFromUntil(t *testing.T) {\n\ttype in struct {\n\t\tfrom int64\n\t\tuntil int64\n\t\tstep int64\n\t}\n\n\ttype out struct {\n\t\tfrom int64\n\t\tuntil int64\n\t}\n\n\ttests := []struct {\n\t\tin in\n\t\tout out\n\t}{\n\t\t{in: in{4, 9, 2}, out: out{4, 9}},\n\t\t{in: in{4, 19, 3}, out: out{6, 20}},\n\t\t{in: in{4, 29, 5}, out: out{5, 29}},\n\t\t{in: in{7, 108, 7}, out: out{7, 111}},\n\t\t{in: in{7, 108, 13}, out: out{13, 116}},\n\t}\n\n\tfor tn, test := range tests {\n\t\tt.Run(fmt.Sprintf(\"setFromUntil %d\", tn), func(t *testing.T) {\n\t\t\tcond := &conditions{\n\t\t\t\tTimeFrame: &TimeFrame{From: test.in.from, Until: test.in.until},\n\t\t\t\tstep: test.in.step,\n\t\t\t}\n\t\t\tcond.setFromUntil()\n\t\t\tresult := out{cond.from, cond.until}\n\t\t\tassert.Equal(t, test.out, result)\n\t\t})\n\t}\n}\n\n// prewhere, where and both generators are checked here\nfunc TestGenerateQuery(t *testing.T) {\n\ttable := \"graphite.table\"\n\n\ttype in struct {\n\t\tfrom int64\n\t\tuntil int64\n\t\tstep int64\n\t\tagg string\n\t}\n\n\ttests := []struct {\n\t\tin in\n\t\taggregated string\n\t\tunaggregated string\n\t}{\n\t\t{\n\t\t\tin: in{1668124800, 1668325322, 1, \"avg\"},\n\t\t\taggregated: (\"WITH anyResample(1668124800, 1668325322, 1)(toUInt32(intDiv(Time, 1)*1), Time) AS mask\\n\" +\n\t\t\t\t\"SELECT Path,\\n arrayFilter(m->m!=0, mask) AS times,\\n\" +\n\t\t\t\t\" arrayFilter((v,m)->m!=0, avgResample(1668124800, 1668325322, 1)(Value, Time), mask) AS values\\n\" +\n\t\t\t\t\"FROM graphite.table\\n\" +\n\t\t\t\t\"PREWHERE Date >= '\" + date.FromTimestampToDaysFormat(1668124800) + \"' AND Date <= '\" + date.UntilTimestampToDaysFormat(1668325322) + \"'\\n\" +\n\t\t\t\t\"WHERE (Path in metrics_list) AND (Time >= 1668124800 AND Time <= 1668325322)\\n\" +\n\t\t\t\t\"GROUP BY Path\\n\" +\n\t\t\t\t\"FORMAT RowBinary\"),\n\t\t\tunaggregated: (\"SELECT Path, groupArray(Time), groupArray(Value), groupArray(Timestamp)\\n\" +\n\t\t\t\t\"FROM graphite.table\\n\" +\n\t\t\t\t\"PREWHERE Date >= '\" + date.FromTimestampToDaysFormat(1668124800) + \"' AND Date <= '\" + date.UntilTimestampToDaysFormat(1668325322) + \"'\\n\" +\n\t\t\t\t\"WHERE (Path in metrics_list) AND (Time >= 1668124800 AND Time <= 1668325322)\\n\" +\n\t\t\t\t\"GROUP BY Path\\n\" +\n\t\t\t\t\"FORMAT RowBinary\"),\n\t\t},\n\t\t{\n\t\t\tin: in{11111, 33333, 11111, \"min\"},\n\t\t\taggregated: (\"WITH anyResample(11111, 33333, 11111)(toUInt32(intDiv(Time, 11111)*11111), Time) AS mask\\n\" +\n\t\t\t\t\"SELECT Path,\\n arrayFilter(m->m!=0, mask) AS times,\\n\" +\n\t\t\t\t\" arrayFilter((v,m)->m!=0, minResample(11111, 33333, 11111)(Value, Time), mask) AS values\\n\" +\n\t\t\t\t\"FROM graphite.table\\n\" +\n\t\t\t\t\"PREWHERE Date >= '\" + date.FromTimestampToDaysFormat(11111) + \"' AND Date <= '\" + date.FromTimestampToDaysFormat(33333) + \"'\\n\" +\n\t\t\t\t\"WHERE (Path in metrics_list) AND (Time >= 11111 AND Time <= 33333)\\n\" +\n\t\t\t\t\"GROUP BY Path\\n\" +\n\t\t\t\t\"FORMAT RowBinary\"),\n\t\t\tunaggregated: (\"SELECT Path, groupArray(Time), groupArray(Value), groupArray(Timestamp)\\n\" +\n\t\t\t\t\"FROM graphite.table\\n\" +\n\t\t\t\t\"PREWHERE Date >= '\" + date.FromTimestampToDaysFormat(11111) + \"' AND Date <= '\" + date.UntilTimestampToDaysFormat(33333) + \"'\\n\" +\n\t\t\t\t\"WHERE (Path in metrics_list) AND (Time >= 11111 AND Time <= 33333)\\n\" +\n\t\t\t\t\"GROUP BY Path\\n\" +\n\t\t\t\t\"FORMAT RowBinary\"),\n\t\t},\n\t}\n\tfor tn, test := range tests {\n\t\tt.Run(fmt.Sprintf(\"generate query %d\", tn), func(t *testing.T) {\n\t\t\tcond := &conditions{\n\t\t\t\tTargets: &Targets{},\n\t\t\t\tfrom: test.in.from,\n\t\t\t\tuntil: test.in.until,\n\t\t\t\tstep: test.in.step,\n\t\t\t}\n\t\t\tcond.pointsTable = table\n\t\t\tcond.setPrewhere()\n\t\t\tcond.setWhere()\n\t\t\tunaggQuery := cond.generateQuery(test.in.agg)\n\t\t\tassert.Equal(t, test.unaggregated, unaggQuery)\n\n\t\t\tcond.aggregated = true\n\t\t\taggQuery := cond.generateQuery(test.in.agg)\n\t\t\tassert.Equal(t, test.aggregated, aggQuery)\n\t\t})\n\t}\n}\n" }, { "path": "render/data/targets.go", "content": "package data\n\nimport (\n\t\"fmt\"\n\t\"time\"\n\n\tv3pb \"github.com/go-graphite/protocol/carbonapi_v3_pb\"\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/rollup\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/alias\"\n)\n\nconst graphiteConsolidationFunction = \"consolidateBy\"\n\ntype FilteringFunctionsByTarget map[string][]*v3pb.FilteringFunction\ntype Cache struct {\n\tCached bool\n\tTS int64 // cached timestamp\n\tTimeout int32\n\tTimeoutStr string\n\tKey string // cache key\n\tM *metrics.CacheMetric\n}\n\n// Targets represents requested metrics\ntype Targets struct {\n\t// List contains queried metrics, e.g. [metric.{name1,name2}, metric.name[3-9]]\n\tList []string\n\tCache []Cache\n\tCached bool // all is cached\n\t// AM stores found expanded metrics\n\tAM *alias.Map\n\tfilteringFunctionsByTarget FilteringFunctionsByTarget\n\tpointsTable string\n\tisReverse bool\n\trollupRules *rollup.Rules\n\trollupUseReverted bool\n\tqueryMetrics *metrics.QueryMetrics\n}\n\nfunc NewTargets(list []string, am *alias.Map) *Targets {\n\ttargets := &Targets{\n\t\tList: list,\n\t\tCache: make([]Cache, len(list)),\n\t\tAM: am,\n\t\tfilteringFunctionsByTarget: make(FilteringFunctionsByTarget),\n\t}\n\n\treturn targets\n}\n\nfunc NewTargetsOne(target string, capacity int, am *alias.Map) *Targets {\n\tlist := make([]string, 1, capacity)\n\tlist[0] = target\n\ttargets := &Targets{\n\t\tList: list,\n\t\tCache: make([]Cache, len(list)),\n\t\tAM: am,\n\t\tfilteringFunctionsByTarget: make(FilteringFunctionsByTarget),\n\t}\n\n\treturn targets\n}\n\nfunc (tt *Targets) Append(target string) {\n\ttt.List = append(tt.List, target)\n\ttt.Cache = append(tt.Cache, Cache{})\n}\n\nfunc (tt *Targets) SetFilteringFunctions(target string, filteringFunctions []*v3pb.FilteringFunction) {\n\ttt.filteringFunctionsByTarget[target] = filteringFunctions\n}\n\nfunc (tt *Targets) selectDataTable(cfg *config.Config, tf *TimeFrame, context string) error {\n\tnow := time.Now().Unix()\n\nTableLoop:\n\tfor i := 0; i < len(cfg.DataTable); i++ {\n\t\tt := &cfg.DataTable[i]\n\n\t\tif !t.ContextMap[context] {\n\t\t\tcontinue TableLoop\n\t\t}\n\n\t\tif t.MaxInterval != 0 && (tf.Until-tf.From) > int64(t.MaxInterval.Seconds()) {\n\t\t\tcontinue TableLoop\n\t\t}\n\n\t\tif t.MinInterval != 0 && (tf.Until-tf.From) < int64(t.MinInterval.Seconds()) {\n\t\t\tcontinue TableLoop\n\t\t}\n\n\t\tif t.MaxAge != 0 && tf.From < now-int64(t.MaxAge.Seconds()) {\n\t\t\tcontinue TableLoop\n\t\t}\n\n\t\tif t.MinAge != 0 && tf.Until > now-int64(t.MinAge.Seconds()) {\n\t\t\tcontinue TableLoop\n\t\t}\n\n\t\tif t.TargetMatchAllRegexp != nil {\n\t\t\tfor j := 0; j < len(tt.List); j++ {\n\t\t\t\tif !t.TargetMatchAllRegexp.MatchString(tt.List[j]) {\n\t\t\t\t\tcontinue TableLoop\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif t.TargetMatchAnyRegexp != nil {\n\t\t\tmatched := false\n\t\tTargetsLoop:\n\t\t\tfor j := 0; j < len(tt.List); j++ {\n\t\t\t\tif t.TargetMatchAnyRegexp.MatchString(tt.List[j]) {\n\t\t\t\t\tmatched = true\n\t\t\t\t\tbreak TargetsLoop\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !matched {\n\t\t\t\tcontinue TableLoop\n\t\t\t}\n\t\t}\n\t\ttt.pointsTable = t.Table\n\t\ttt.isReverse = t.Reverse\n\t\ttt.rollupUseReverted = t.RollupUseReverted\n\t\ttt.rollupRules = t.Rollup.Rules()\n\t\ttt.queryMetrics = t.QueryMetrics\n\t\treturn nil\n\t}\n\n\treturn fmt.Errorf(\"data tables is not specified for %v\", tt.List[0])\n}\n\nfunc (tt *Targets) GetRequestedAggregation(target string) (string, error) {\n\tif ffs, ok := tt.filteringFunctionsByTarget[target]; !ok {\n\t\treturn \"\", nil\n\t} else {\n\t\tfor _, filteringFunc := range ffs {\n\t\t\tffName := filteringFunc.GetName()\n\t\t\tffArgs := filteringFunc.GetArguments()\n\n\t\t\tif ffName != graphiteConsolidationFunction {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif len(ffArgs) < 1 {\n\t\t\t\treturn \"\", fmt.Errorf(\"no argumets were provided to consolidateBy function\")\n\t\t\t}\n\n\t\t\tswitch ffArgs[0] {\n\t\t\t// 'last' in graphite == clickhouse aggregate function 'anyLast'\n\t\t\tcase \"last\":\n\t\t\t\treturn \"anyLast\", nil\n\t\t\t// 'first' in graphite == clickhouse aggregate function 'any'\n\t\t\tcase \"first\":\n\t\t\t\treturn \"any\", nil\n\t\t\t// Graphite standard supports both average and avg.\n\t\t\t// It is the only aggregation that has two aliases.\n\t\t\t// https://graphite.readthedocs.io/en/latest/functions.html#graphite.render.functions.consolidateBy\n\t\t\tcase \"average\":\n\t\t\t\treturn \"avg\", nil\n\t\t\t// avg, sum, max, min have the same name in clickhouse\n\t\t\tcase \"avg\", \"sum\", \"max\", \"min\":\n\t\t\t\treturn ffArgs[0], nil\n\t\t\tdefault:\n\t\t\t\treturn \"\",\n\t\t\t\t\tfmt.Errorf(\n\t\t\t\t\t\t\"unknown \\\"%s\\\" argument function (allowed argumets are: 'avg', 'average', 'sum', 'max', 'min', 'last', 'first'): recieved %s\",\n\t\t\t\t\t\tgraphiteConsolidationFunction,\n\t\t\t\t\t\tffArgs[0],\n\t\t\t\t\t)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn \"\", nil\n}\n" }, { "path": "render/data/targets_test.go", "content": "package data\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestSelectDataTableTime(t *testing.T) {\n\tcfg := config.New()\n\tcfg.DataTable = []config.DataTable{\n\t\t{\n\t\t\tTable: \"first_day\",\n\t\t\tMaxAge: 24 * time.Hour,\n\t\t},\n\t\t{\n\t\t\tTable: \"second_day\",\n\t\t\tMinAge: 24 * time.Hour,\n\t\t\tMaxAge: 48 * time.Hour,\n\t\t},\n\t\t{\n\t\t\tTable: \"two_days_min_interval\",\n\t\t\tMaxAge: 48 * time.Hour,\n\t\t\tMinInterval: 2 * time.Hour,\n\t\t},\n\t\t{\n\t\t\tTable: \"two_days_min_max_interval\",\n\t\t\tMaxAge: 48 * time.Hour,\n\t\t\tMinInterval: 30 * time.Minute,\n\t\t\tMaxInterval: 1 * time.Hour,\n\t\t},\n\t\t{\n\t\t\tTable: \"two_days_max_interval\",\n\t\t\tMaxAge: 48 * time.Hour,\n\t\t\tMaxInterval: 2 * time.Hour,\n\t\t},\n\t\t{\n\t\t\tTable: \"three_days\",\n\t\t\tMaxAge: 72 * time.Hour,\n\t\t},\n\t\t{\n\t\t\tTable: \"unlimited\",\n\t\t},\n\t}\n\terr := cfg.ProcessDataTables()\n\tassert.NoError(t, err)\n\n\ttg := NewTargets([]string{\"metric\"}, nil)\n\n\ttests := []struct {\n\t\t*TimeFrame\n\t\tconfig.DataTable\n\t\terr error\n\t}{\n\t\t{\n\t\t\t&TimeFrame{ageToTimestamp(3600*24 - 1), ageToTimestamp(1800), 1},\n\t\t\tcfg.DataTable[0],\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t&TimeFrame{ageToTimestamp(3600*48 - 1), ageToTimestamp(24*3600 + 1), 1},\n\t\t\tcfg.DataTable[1],\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t&TimeFrame{ageToTimestamp(3600 * 26), ageToTimestamp(3600 * 23), 1},\n\t\t\tcfg.DataTable[2],\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t&TimeFrame{ageToTimestamp(3600*24 + 1600), ageToTimestamp(3600*24 - 1600), 1},\n\t\t\tcfg.DataTable[3],\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t&TimeFrame{ageToTimestamp(3600*24 + 2000), ageToTimestamp(3600*24 - 2000), 1},\n\t\t\tcfg.DataTable[4],\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t&TimeFrame{ageToTimestamp(3600*72 - 1), ageToTimestamp(3600*11 - 1), 1},\n\t\t\tcfg.DataTable[5],\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\t&TimeFrame{ageToTimestamp(3600 * 100), ageToTimestamp(3600*11 - 1), 1},\n\t\t\tcfg.DataTable[6],\n\t\t\tnil,\n\t\t},\n\t}\n\n\tfor i, test := range tests {\n\t\tt.Run(fmt.Sprintf(\"%d-%s\", i+1, test.DataTable.Table), func(t *testing.T) {\n\t\t\terr := tg.selectDataTable(cfg, test.TimeFrame, config.ContextGraphite)\n\t\t\tassert.Equal(t, test.err, err)\n\t\t\tassert.Equal(t, test.DataTable.Table, tg.pointsTable)\n\t\t})\n\t}\n}\n\nfunc TestSelectDataTableMatch(t *testing.T) {\n\tcfg := config.New()\n\tcfg.DataTable = []config.DataTable{\n\t\t{\n\t\t\tTable: \"all\",\n\t\t\tTargetMatchAll: \"^all.*avg\",\n\t\t},\n\t\t{\n\t\t\tTable: \"any\",\n\t\t\tTargetMatchAny: \"^any.*avg\",\n\t\t},\n\t\t{\n\t\t\tTable: \"unlimited\",\n\t\t},\n\t}\n\terr := cfg.ProcessDataTables()\n\tassert.NoError(t, err)\n\n\ttf := &TimeFrame{ageToTimestamp(3600*24 - 1), ageToTimestamp(1800), 1}\n\n\ttests := []struct {\n\t\t*Targets\n\t\tconfig.DataTable\n\t\terr error\n\t}{\n\t\t{\n\t\t\tNewTargets([]string{\"allinclucive.in.avg\", \"all.metrics.for.avg\"}, nil),\n\t\t\tcfg.DataTable[0],\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\tNewTargets([]string{\"allinclucive.in.avg\", \"any.metrics.for.avg\"}, nil),\n\t\t\tcfg.DataTable[1],\n\t\t\tnil,\n\t\t},\n\t\t{\n\t\t\tNewTargets([]string{\"allinclucive.in.avg\", \"some.metrics.for.avg\"}, nil),\n\t\t\tcfg.DataTable[2],\n\t\t\tnil,\n\t\t},\n\t}\n\n\tfor i, test := range tests {\n\t\tt.Run(fmt.Sprintf(\"%d-%s\", i+1, test.DataTable.Table), func(t *testing.T) {\n\t\t\terr := test.Targets.selectDataTable(cfg, tf, config.ContextGraphite)\n\t\t\tassert.Equal(t, test.err, err)\n\t\t\tassert.Equal(t, test.DataTable.Table, test.Targets.pointsTable)\n\t\t})\n\t}\n}\n" }, { "path": "render/handler.go", "content": "package render\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"go.uber.org/zap\"\n\n\t\"github.com/go-graphite/carbonapi/pkg/parser\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/finder\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/helper/utils\"\n\t\"github.com/lomik/graphite-clickhouse/limiter\"\n\t\"github.com/lomik/graphite-clickhouse/logs\"\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"github.com/lomik/graphite-clickhouse/render/data\"\n\t\"github.com/lomik/graphite-clickhouse/render/reply\"\n)\n\n// Handler serves /render requests\ntype Handler struct {\n\tconfig *config.Config\n}\n\n// NewHandler generates new *Handler\nfunc NewHandler(config *config.Config) *Handler {\n\th := &Handler{\n\t\tconfig: config,\n\t}\n\n\treturn h\n}\n\nfunc targetKey(from, until int64, target, ttl string) string {\n\treturn time.Unix(from, 0).Format(\"2006-01-02\") + \";\" + time.Unix(until, 0).Format(\"2006-01-02\") + \";\" + target + \";ttl=\" + ttl\n}\n\nfunc getCacheTimeout(now time.Time, from, until int64, cacheConfig *config.CacheConfig) (int32, string, *metrics.CacheMetric) {\n\tif cacheConfig.ShortDuration == 0 {\n\t\treturn cacheConfig.DefaultTimeoutSec, cacheConfig.DefaultTimeoutStr, metrics.DefaultCacheMetrics\n\t}\n\n\tduration := time.Second * time.Duration(until-from)\n\tif duration > cacheConfig.ShortDuration || now.Unix()-until > cacheConfig.ShortUntilOffsetSec {\n\t\treturn cacheConfig.DefaultTimeoutSec, cacheConfig.DefaultTimeoutStr, metrics.DefaultCacheMetrics\n\t}\n\t// short cache ttl\n\treturn cacheConfig.ShortTimeoutSec, cacheConfig.ShortTimeoutStr, metrics.ShortCacheMetrics\n}\n\n// try to fetch cached finder queries\nfunc (h *Handler) finderCached(ts time.Time, fetchRequests data.MultiTarget, logger *zap.Logger, metricsLen *int) (cachedFind int, maxCacheTimeoutStr string, err error) {\n\tvar lock sync.RWMutex\n\n\tvar maxCacheTimeout int32\n\n\terrors := make([]error, 0, len(fetchRequests))\n\n\tvar wg sync.WaitGroup\n\n\tfor tf, targets := range fetchRequests {\n\t\tfor i, expr := range targets.List {\n\t\t\twg.Add(1)\n\n\t\t\tgo func(tf data.TimeFrame, target string, targets *data.Targets, n int) {\n\t\t\t\tdefer wg.Done()\n\n\t\t\t\ttargets.Cache[n].Timeout, targets.Cache[n].TimeoutStr, targets.Cache[n].M = getCacheTimeout(ts, tf.From, tf.Until, &h.config.Common.FindCacheConfig)\n\t\t\t\tif targets.Cache[n].Timeout > 0 {\n\t\t\t\t\tif maxCacheTimeout < targets.Cache[n].Timeout {\n\t\t\t\t\t\tmaxCacheTimeout = targets.Cache[n].Timeout\n\t\t\t\t\t\tmaxCacheTimeoutStr = targets.Cache[n].TimeoutStr\n\t\t\t\t\t}\n\n\t\t\t\t\ttargets.Cache[n].TS = utils.TimestampTruncate(ts.Unix(), time.Duration(targets.Cache[n].Timeout)*time.Second)\n\t\t\t\t\ttargets.Cache[n].Key = targetKey(tf.From, tf.Until, target, targets.Cache[n].TimeoutStr)\n\n\t\t\t\t\tbody, err := h.config.Common.FindCache.Get(targets.Cache[n].Key)\n\t\t\t\t\tif err == nil {\n\t\t\t\t\t\tif len(body) > 0 {\n\t\t\t\t\t\t\ttargets.Cache[n].M.CacheHits.Add(1)\n\n\t\t\t\t\t\t\tvar f finder.Finder\n\t\t\t\t\t\t\tif strings.HasPrefix(target, \"seriesByTag(\") {\n\t\t\t\t\t\t\t\tf = finder.NewCachedTags(body)\n\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\tf = finder.NewCachedIndex(body)\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\ttargets.AM.MergeTarget(f.(finder.Result), target, false)\n\t\t\t\t\t\t\tlock.Lock()\n\t\t\t\t\t\t\tamLen := targets.AM.Len()\n\t\t\t\t\t\t\t*metricsLen += amLen\n\t\t\t\t\t\t\tlock.Unlock()\n\n\t\t\t\t\t\t\ttargets.Cache[n].Cached = true\n\n\t\t\t\t\t\t\tlogger.Info(\"finder\", zap.String(\"get_cache\", targets.Cache[n].Key), zap.Time(\"timestamp_cached\", time.Unix(targets.Cache[n].TS, 0)),\n\t\t\t\t\t\t\t\tzap.Int(\"metrics\", amLen), zap.Bool(\"find_cached\", true),\n\t\t\t\t\t\t\t\tzap.String(\"ttl\", targets.Cache[n].TimeoutStr),\n\t\t\t\t\t\t\t\tzap.Int64(\"from\", tf.From), zap.Int64(\"until\", tf.Until))\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}(tf, expr, targets, i)\n\t\t}\n\t}\n\n\twg.Wait()\n\n\tif len(errors) != 0 {\n\t\terr = errors[0]\n\t\treturn\n\t}\n\n\tfor _, targets := range fetchRequests {\n\t\tvar cached int\n\n\t\tfor _, c := range targets.Cache {\n\t\t\tif c.Cached {\n\t\t\t\tcached++\n\t\t\t}\n\t\t}\n\n\t\tcachedFind += cached\n\n\t\tif cached == len(targets.Cache) {\n\t\t\ttargets.Cached = true\n\t\t}\n\t}\n\n\treturn\n}\n\n// try to fetch finder queries\nfunc (h *Handler) finder(fetchRequests data.MultiTarget, ctx context.Context, logger *zap.Logger, qlimiter limiter.ServerLimiter, metricsLen *int, queueDuration *time.Duration, useCache bool) (maxDuration int64, err error) {\n\tvar (\n\t\twg sync.WaitGroup\n\t\tlock sync.RWMutex\n\t\tentered int\n\t\tlimitCtx context.Context\n\t\tcancel context.CancelFunc\n\t)\n\n\tif qlimiter.Enabled() {\n\t\t// no reason wait longer than index-timeout\n\t\tlimitCtx, cancel = context.WithTimeout(ctx, h.config.ClickHouse.IndexTimeout)\n\t\tdefer func() {\n\t\t\tfor i := 0; i < entered; i++ {\n\t\t\t\tqlimiter.Leave(limitCtx, \"render\")\n\t\t\t}\n\n\t\t\tdefer cancel()\n\t\t}()\n\t}\n\n\terrors := make([]error, 0, len(fetchRequests))\n\n\tfor tf, targets := range fetchRequests {\n\t\tfor i, expr := range targets.List {\n\t\t\td := tf.Until - tf.From\n\t\t\tif maxDuration < d {\n\t\t\t\tmaxDuration = d\n\t\t\t}\n\n\t\t\tif targets.Cache[i].Cached {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif qlimiter.Enabled() {\n\t\t\t\tstart := time.Now()\n\t\t\t\terr = qlimiter.Enter(limitCtx, \"render\")\n\t\t\t\t*queueDuration += time.Since(start)\n\n\t\t\t\tif err != nil {\n\t\t\t\t\tlock.Lock()\n\t\t\t\t\terrors = append(errors, err)\n\t\t\t\t\tlock.Unlock()\n\n\t\t\t\t\tbreak\n\t\t\t\t}\n\n\t\t\t\tentered++\n\t\t\t}\n\n\t\t\twg.Add(1)\n\n\t\t\tgo func(tf data.TimeFrame, target string, targets *data.Targets, n int) {\n\t\t\t\tdefer wg.Done()\n\n\t\t\t\tvar fndResult finder.Result\n\n\t\t\t\tvar err error\n\n\t\t\t\tfStart := time.Now()\n\t\t\t\tfndResult, err = finder.Find(h.config, ctx, target, tf.From, tf.Until)\n\t\t\t\td := time.Since(fStart).Milliseconds()\n\n\t\t\t\tif err != nil {\n\t\t\t\t\tmetrics.SendQueryReadByTable(tf.From, tf.Until, d, 0, fndResult.Stats(), true)\n\t\t\t\t\tlogger.Error(\"find\", zap.Error(err))\n\t\t\t\t\tlock.Lock()\n\t\t\t\t\terrors = append(errors, err)\n\t\t\t\t\tlock.Unlock()\n\n\t\t\t\t\treturn\n\t\t\t\t}\n\n\t\t\t\tbody := targets.AM.MergeTarget(fndResult, target, useCache)\n\n\t\t\t\tcacheTimeout := targets.Cache[n].Timeout\n\t\t\t\tif useCache && cacheTimeout > 0 {\n\t\t\t\t\tcacheTimeoutStr := targets.Cache[n].TimeoutStr\n\t\t\t\t\tkey := targets.Cache[n].Key\n\t\t\t\t\ttargets.Cache[n].M.CacheMisses.Add(1)\n\t\t\t\t\th.config.Common.FindCache.Set(key, body, cacheTimeout)\n\t\t\t\t\tlogger.Info(\"finder\", zap.String(\"set_cache\", key), zap.Time(\"timestamp_cached\", time.Unix(targets.Cache[n].TS, 0)),\n\t\t\t\t\t\tzap.Int(\"metrics\", targets.AM.Len()), zap.Bool(\"find_cached\", false),\n\t\t\t\t\t\tzap.String(\"ttl\", cacheTimeoutStr),\n\t\t\t\t\t\tzap.Int64(\"from\", tf.From), zap.Int64(\"until\", tf.Until))\n\t\t\t\t}\n\n\t\t\t\tlock.Lock()\n\t\t\t\trows := targets.AM.Len()\n\t\t\t\tlock.Unlock()\n\n\t\t\t\t*metricsLen += rows\n\t\t\t\tmetrics.SendQueryReadByTable(tf.From, tf.Until, d, int64(rows), fndResult.Stats(), false)\n\t\t\t}(tf, expr, targets, i)\n\t\t}\n\t}\n\n\twg.Wait()\n\n\tif len(errors) != 0 {\n\t\terr = errors[0]\n\t}\n\n\treturn\n}\n\nfunc (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {\n\tvar (\n\t\tmaxDuration int64\n\t\ttargetsLen int\n\t\tmetricsLen int\n\t\tpointsCount int64\n\t\tfetchStart time.Time\n\t\tcachedFind bool\n\t\tqueueFail bool\n\t\tqueueDuration time.Duration\n\t\terr error\n\t\tfetchRequests data.MultiTarget\n\t\tluser string\n\t)\n\n\tstart := time.Now()\n\tstatus := http.StatusOK\n\taccessLogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named(\"http\")\n\tlogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named(\"render\")\n\n\tr = r.WithContext(scope.WithLogger(r.Context(), logger))\n\n\tusername := r.Header.Get(\"X-Forwarded-User\")\n\n\tvar qlimiter limiter.ServerLimiter = limiter.NoopLimiter{}\n\n\tdefer func() {\n\t\tif rec := recover(); rec != nil {\n\t\t\tstatus = http.StatusInternalServerError\n\n\t\t\tlogger.Error(\"panic during eval:\",\n\t\t\t\tzap.String(\"requestID\", scope.String(r.Context(), \"requestID\")),\n\t\t\t\tzap.Any(\"reason\", rec),\n\t\t\t\tzap.Stack(\"stack\"),\n\t\t\t)\n\n\t\t\tanswer := fmt.Sprintf(\"%v\\nStack trace: %v\", rec, zap.Stack(\"\").String)\n\t\t\thttp.Error(w, answer, status)\n\t\t}\n\n\t\tend := time.Now()\n\t\tlogs.AccessLog(accessLogger, h.config, r, status, end.Sub(start), queueDuration, cachedFind, queueFail)\n\t\tqlimiter.SendDuration(queueDuration.Milliseconds())\n\t\tmetrics.SendRenderMetrics(metrics.RenderRequestMetric, status, start, fetchStart, end, maxDuration, h.config.Metrics.ExtendedStat, int64(metricsLen), pointsCount)\n\t}()\n\n\tr.ParseMultipartForm(1024 * 1024)\n\n\tformatter, err := reply.GetFormatter(r)\n\tif err != nil {\n\t\tstatus = http.StatusBadRequest\n\n\t\tlogger.Error(\"formatter\", zap.Error(err))\n\t\thttp.Error(w, fmt.Sprintf(\"Failed to parse request: %v\", err.Error()), status)\n\n\t\treturn\n\t}\n\n\tfetchRequests, err = formatter.ParseRequest(r)\n\tif err != nil {\n\t\tstatus = http.StatusBadRequest\n\t\thttp.Error(w, fmt.Sprintf(\"Failed to parse request: %v\", err.Error()), status)\n\n\t\treturn\n\t}\n\n\tfor tf, targets := range fetchRequests {\n\t\tif tf.From >= tf.Until {\n\t\t\t// wrong duration\n\t\t\tif err != nil {\n\t\t\t\tstatus, _ = clickhouse.HandleError(w, clickhouse.ErrInvalidTimeRange)\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\n\t\ttargetsLen += len(targets.List)\n\t}\n\n\tluser, qlimiter = data.GetQueryLimiter(username, h.config, &fetchRequests)\n\tlogger.Debug(\"use user limiter\", zap.String(\"username\", username), zap.String(\"luser\", luser))\n\n\tvar maxCacheTimeoutStr string\n\n\tuseCache := h.config.Common.FindCache != nil && !parser.TruthyBool(r.FormValue(\"noCache\"))\n\n\tif useCache {\n\t\tvar cached int\n\n\t\tcached, maxCacheTimeoutStr, err = h.finderCached(start, fetchRequests, logger, &metricsLen)\n\t\tif err != nil {\n\t\t\tstatus, _ = clickhouse.HandleError(w, err)\n\t\t\treturn\n\t\t}\n\n\t\tif cached > 0 {\n\t\t\tif cached == targetsLen && metricsLen == 0 {\n\t\t\t\t// all from cache and no metric\n\t\t\t\tstatus = http.StatusNotFound\n\n\t\t\t\tformatter.Reply(w, r, data.EmptyResponse())\n\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tcachedFind = true\n\t\t}\n\t}\n\n\tmaxDuration, err = h.finder(fetchRequests, r.Context(), logger, qlimiter, &metricsLen, &queueDuration, useCache)\n\tif err != nil {\n\t\tstatus, queueFail = clickhouse.HandleError(w, err)\n\t\treturn\n\t}\n\n\tlogger.Info(\"finder\", zap.Int(\"metrics\", metricsLen), zap.Bool(\"find_cached\", cachedFind))\n\n\tif cachedFind {\n\t\tw.Header().Set(\"X-Cached-Find\", maxCacheTimeoutStr)\n\t}\n\n\tif metricsLen == 0 {\n\t\tstatus = http.StatusNotFound\n\n\t\tformatter.Reply(w, r, data.EmptyResponse())\n\n\t\treturn\n\t}\n\n\tfetchStart = time.Now()\n\n\treply, err := fetchRequests.Fetch(r.Context(), h.config, config.ContextGraphite, qlimiter, &queueDuration)\n\tif err != nil {\n\t\tstatus, queueFail = clickhouse.HandleError(w, err)\n\t\treturn\n\t}\n\n\tif len(reply) == 0 {\n\t\tstatus = http.StatusNotFound\n\n\t\tformatter.Reply(w, r, reply)\n\n\t\treturn\n\t}\n\n\tfor i := range reply {\n\t\tpointsCount += int64(reply[i].Data.Len())\n\t}\n\n\trStart := time.Now()\n\n\tformatter.Reply(w, r, reply)\n\n\td := time.Since(rStart)\n\tlogger.Debug(\"reply\", zap.String(\"runtime\", d.String()), zap.Duration(\"runtime_ns\", d))\n}\n" }, { "path": "render/handler_test.go", "content": "package render\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n)\n\nfunc Test_getCacheTimeout(t *testing.T) {\n\tcacheConfig := config.CacheConfig{\n\t\tShortTimeoutSec: 60,\n\t\tShortTimeoutStr: \"60\",\n\t\tDefaultTimeoutSec: 300,\n\t\tDefaultTimeoutStr: \"300\",\n\t\tShortDuration: 3 * time.Hour,\n\t\tShortUntilOffsetSec: 120,\n\t}\n\n\tnow := int64(1636985018)\n\n\ttests := []struct {\n\t\tname string\n\t\tnow time.Time\n\t\tfrom int64\n\t\tuntil int64\n\t\twant int32\n\t\twantStr string\n\t}{\n\t\t{\n\t\t\tname: \"short: from = now - 600, until = now - 120\",\n\t\t\tnow: time.Unix(now, 0),\n\t\t\tfrom: now - 600,\n\t\t\tuntil: now - 120,\n\t\t\twant: 60,\n\t\t\twantStr: \"60\",\n\t\t},\n\t\t{\n\t\t\tname: \"short: from = now - 10800\",\n\t\t\tnow: time.Unix(now, 0),\n\t\t\tfrom: now - 10800,\n\t\t\tuntil: now,\n\t\t\twant: 60,\n\t\t\twantStr: \"60\",\n\t\t},\n\t\t{\n\t\t\tname: \"short: from = now - 10810, until = now - 120\",\n\t\t\tnow: time.Unix(now, 0),\n\t\t\tfrom: now - 10800,\n\t\t\tuntil: now - 120,\n\t\t\twant: 60,\n\t\t\twantStr: \"60\",\n\t\t},\n\t\t{\n\t\t\tname: \"short: from = now - 10800, until now - 121\",\n\t\t\tnow: time.Unix(now, 0),\n\t\t\tfrom: now - 10800,\n\t\t\tuntil: now - 121,\n\t\t\twant: 300,\n\t\t\twantStr: \"300\",\n\t\t},\n\t\t{\n\t\t\tname: \"default: from = now - 10801\",\n\t\t\tnow: time.Unix(now, 0),\n\t\t\tfrom: now - 10801,\n\t\t\tuntil: now,\n\t\t\twant: 300,\n\t\t\twantStr: \"300\",\n\t\t},\n\t\t{\n\t\t\tname: \"short: from = now - 122, until = now - 121\",\n\t\t\tnow: time.Unix(now, 0),\n\t\t\tfrom: now - 122,\n\t\t\tuntil: now - 121,\n\t\t\twant: 300,\n\t\t\twantStr: \"300\",\n\t\t},\n\t}\n\tfor i, tt := range tests {\n\t\tt.Run(fmt.Sprintf(\"[%d] %s\", i, tt.name), func(t *testing.T) {\n\t\t\tgot, gotStr, _ := getCacheTimeout(tt.now, tt.from, tt.until, &cacheConfig)\n\t\t\tif got != tt.want {\n\t\t\t\tt.Errorf(\"getCacheTimeout() = %v, want %v\", got, tt.want)\n\t\t\t}\n\n\t\t\tif gotStr != tt.wantStr {\n\t\t\t\tt.Errorf(\"getCacheTimeout() = %q, want %q\", gotStr, tt.wantStr)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "render/reply/formatter.go", "content": "package reply\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"net/http\"\n\t\"strconv\"\n\n\t\"github.com/lomik/graphite-clickhouse/pkg/alias\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/dry\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"github.com/lomik/graphite-clickhouse/render/data\"\n\t\"go.uber.org/zap\"\n)\n\n// Formatter implements request parser and response generator\ntype Formatter interface {\n\t// Parse request\n\tParseRequest(r *http.Request) (data.MultiTarget, error)\n\t// Generate reply payload\n\tReply(http.ResponseWriter, *http.Request, data.CHResponses)\n}\n\n// GetFormatter returns a proper interface for render format\nfunc GetFormatter(r *http.Request) (Formatter, error) {\n\tformat := r.FormValue(\"format\")\n\tswitch format {\n\tcase \"carbonapi_v3_pb\":\n\t\treturn &V3PB{}, nil\n\tcase \"pickle\":\n\t\treturn &Pickle{}, nil\n\tcase \"protobuf\":\n\t\treturn &V2PB{}, nil\n\tcase \"carbonapi_v2_pb\":\n\t\treturn &V2PB{}, nil\n\t}\n\n\terr := fmt.Errorf(\"format %v is not supported, supported formats: carbonapi_v3_pb, pickle, protobuf (aka carbonapi_v2_pb)\", format)\n\tif !scope.Debug(r.Context(), \"Output\") {\n\t\treturn nil, err\n\t}\n\n\tswitch format {\n\tcase \"json\":\n\t\treturn &JSON{}, nil\n\t}\n\n\terr = fmt.Errorf(\"%w\\n(formats available for output debug: json)\", err)\n\n\treturn nil, err\n}\n\nfunc parseRequestForms(r *http.Request) (data.MultiTarget, error) {\n\tfromTimestamp, err := strconv.ParseInt(r.FormValue(\"from\"), 10, 32)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"cannot parse from\")\n\t}\n\n\tuntilTimestamp, err := strconv.ParseInt(r.FormValue(\"until\"), 10, 32)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"cannot parse until\")\n\t}\n\n\tmaxDataPoints, err := strconv.ParseInt(r.FormValue(\"maxDataPoints\"), 10, 32)\n\tif err != nil {\n\t\tmaxDataPoints = int64(math.MaxInt64)\n\t}\n\n\ttargets := dry.RemoveEmptyStrings(r.Form[\"target\"])\n\ttf := data.TimeFrame{\n\t\tFrom: fromTimestamp,\n\t\tUntil: untilTimestamp,\n\t\tMaxDataPoints: maxDataPoints,\n\t}\n\tmultiTarget := make(data.MultiTarget)\n\tmultiTarget[tf] = data.NewTargets(targets, alias.New())\n\n\tif len(targets) > 0 {\n\t\tlogger := scope.Logger(r.Context()).Named(\"form_parser\")\n\t\tfor _, t := range targets {\n\t\t\tlogger.Info(\n\t\t\t\t\"target\",\n\t\t\t\tzap.Int64(\"from\", tf.From),\n\t\t\t\tzap.Int64(\"until\", tf.Until),\n\t\t\t\tzap.Int64(\"maxDataPoints\", tf.MaxDataPoints),\n\t\t\t\tzap.String(\"target\", t),\n\t\t\t)\n\t\t}\n\t}\n\n\treturn multiTarget, nil\n}\n" }, { "path": "render/reply/formatter_test.go", "content": "package reply\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"sort\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n\n\t\"github.com/lomik/graphite-clickhouse/finder\"\n\t\"github.com/lomik/graphite-clickhouse/helper/client\"\n\t\"github.com/lomik/graphite-clickhouse/helper/point\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/alias\"\n\t\"github.com/lomik/graphite-clickhouse/render/data\"\n)\n\nvar results = []client.Metric{\n\t{\n\t\tName: \"test.metric1\",\n\t\tPathExpression: \"test.*\",\n\t\tStartTime: 1688990040,\n\t\tStepTime: 60,\n\t\tStopTime: 1688990520,\n\t\tValues: func() []float64 {\n\t\t\ttemp := emptyValues(8)\n\t\t\ttemp[2] = 3\n\t\t\treturn temp\n\t\t}(),\n\t},\n\t{\n\t\tName: \"test.metric2\",\n\t\tPathExpression: \"test.*\",\n\t\tStartTime: 1688990040,\n\t\tStepTime: 60,\n\t\tStopTime: 1688990520,\n\t\tValues: emptyValues(8),\n\t},\n\t{\n\t\tName: \"test.metric3\",\n\t\tPathExpression: \"test.*\",\n\t\tStartTime: 1688990040,\n\t\tStepTime: 60,\n\t\tStopTime: 1688990520,\n\t\tValues: emptyValues(8),\n\t},\n}\n\nfunc TestFormatterReply(t *testing.T) {\n\tformatters := []struct {\n\t\timpl Formatter\n\t\tname string\n\t\tformat client.FormatType\n\t}{\n\t\t{&V3PB{}, \"v3pb\", client.FormatPb_v3},\n\t\t{&V2PB{}, \"v2pb\", client.FormatPb_v2},\n\t\t{&JSON{}, \"json\", client.FormatJSON},\n\t\t{&Pickle{}, \"pickle\", client.FormatPickle},\n\t}\n\n\ttests := []struct {\n\t\tname string\n\t\tinput data.CHResponses\n\t\t// result when CHResponse.AppendOutEmptySeries is false\n\t\texpectedWithoutEmpty []client.Metric\n\t\t// result when CHResponse.AppendOutEmptySeries is true\n\t\texpectedWithEmpty []client.Metric\n\t}{\n\t\t{\n\t\t\tname: \"no index found\",\n\t\t\tinput: data.EmptyResponse(),\n\t\t\texpectedWithoutEmpty: []client.Metric{},\n\t\t\texpectedWithEmpty: []client.Metric{},\n\t\t},\n\t\t{\n\t\t\tname: \"three metrics; test.metric1 with points and other with NaN\",\n\t\t\tinput: prepareCHResponses(1688990000, 1688990460,\n\t\t\t\t[][]byte{[]byte(\"test.metric1\"), []byte(\"test.metric2\"), []byte(\"test.metric3\")},\n\t\t\t\tmap[string][]point.Point{\n\t\t\t\t\t\"test.metric1\": {{Value: 3, Time: 1688990160, Timestamp: 1688990204}},\n\t\t\t\t},\n\t\t\t),\n\t\t\texpectedWithoutEmpty: results[:1],\n\t\t\texpectedWithEmpty: results,\n\t\t},\n\t\t{\n\t\t\tname: \"three metrics, no points in all\",\n\t\t\tinput: prepareCHResponses(1688990000, 1688990460,\n\t\t\t\t[][]byte{[]byte(\"test.metric1\"), []byte(\"test.metric2\"), []byte(\"test.metric3\")},\n\t\t\t\tmap[string][]point.Point{},\n\t\t\t),\n\t\t\texpectedWithoutEmpty: []client.Metric{},\n\t\t\texpectedWithEmpty: append([]client.Metric{\n\t\t\t\t{\n\t\t\t\t\tName: results[0].Name,\n\t\t\t\t\tPathExpression: results[0].PathExpression,\n\t\t\t\t\tStartTime: results[0].StartTime,\n\t\t\t\t\tStopTime: results[0].StopTime,\n\t\t\t\t\tStepTime: results[0].StepTime,\n\t\t\t\t\tValues: emptyValues(8),\n\t\t\t\t},\n\t\t\t}, results[1:]...),\n\t\t},\n\t}\n\n\tfor _, formatter := range formatters {\n\t\tt.Run(fmt.Sprintf(\"format=%s\", formatter.name), func(t *testing.T) {\n\t\t\tfor _, tt := range tests {\n\t\t\t\t// case 0: test for AppendOutEmptySeries = false\n\t\t\t\t// case 1: test for AppendOutEmptySeries = true\n\t\t\t\tfor i := 0; i < 2; i++ {\n\t\t\t\t\tvar expected []client.Metric\n\n\t\t\t\t\tvar testName string\n\n\t\t\t\t\tswitch i {\n\t\t\t\t\tcase 0:\n\t\t\t\t\t\texpected = tt.expectedWithoutEmpty\n\t\t\t\t\t\ttestName = fmt.Sprintf(\"NoAppend: %s\", tt.name)\n\n\t\t\t\t\t\tfor j := range tt.input {\n\t\t\t\t\t\t\ttt.input[j].AppendOutEmptySeries = false\n\t\t\t\t\t\t}\n\t\t\t\t\tcase 1:\n\t\t\t\t\t\texpected = tt.expectedWithEmpty\n\t\t\t\t\t\ttestName = fmt.Sprintf(\"WithAppend: %s\", tt.name)\n\n\t\t\t\t\t\tfor j := range tt.input {\n\t\t\t\t\t\t\ttt.input[j].AppendOutEmptySeries = true\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\tt.Run(testName, func(t *testing.T) {\n\t\t\t\t\t\tctx := context.Background()\n\t\t\t\t\t\t// if tt.protobufDebug {\n\t\t\t\t\t\t// \tctx = scope.WithDebug(ctx, \"Protobuf\")\n\t\t\t\t\t\t// }\n\t\t\t\t\t\tw := httptest.NewRecorder()\n\n\t\t\t\t\t\tr, err := http.NewRequestWithContext(ctx, \"\", \"\", nil)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\trequire.NoErrorf(t, err, \"failed to create request\")\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tformatter.impl.Reply(w, r, tt.input)\n\n\t\t\t\t\t\tresponse := w.Result()\n\t\t\t\t\t\tdefer response.Body.Close()\n\n\t\t\t\t\t\t// then\n\t\t\t\t\t\trequire.Equal(t, http.StatusOK, response.StatusCode)\n\t\t\t\t\t\tdata, err := io.ReadAll(response.Body)\n\t\t\t\t\t\trequire.NoError(t, err)\n\t\t\t\t\t\tgot, err := client.Decode(data, formatter.format)\n\t\t\t\t\t\trequire.NoError(t, err)\n\n\t\t\t\t\t\tif !equalMetrics(expected, got) {\n\t\t\t\t\t\t\tt.Errorf(\"metrics not equal: expected:\\n%#v\\ngot:\\n%#v\\n\", expected, got)\n\t\t\t\t\t\t}\n\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\n// prepareCHResponses prepares CHResponses for tests.\nfunc prepareCHResponses(from, until int64, indices [][]byte, points map[string][]point.Point) data.CHResponses {\n\t// alias\n\tidx := finder.NewMockFinder(indices)\n\tm := alias.New()\n\tm.MergeTarget(idx, \"test.*\", false)\n\n\t// points\n\tpts := point.NewPoints()\n\n\tstringIndex := make([]string, 0, len(indices))\n\tfor _, each := range indices {\n\t\tstringIndex = append(stringIndex, string(each))\n\t}\n\n\tfor k, v := range points {\n\t\tid := pts.MetricID(k)\n\t\tfor _, eachPoint := range v {\n\t\t\tpts.AppendPoint(id, eachPoint.Value, eachPoint.Time, eachPoint.Timestamp)\n\t\t}\n\t}\n\n\tpts.SetAggregations(map[string][]string{\n\t\t\"avg\": stringIndex,\n\t})\n\tsort.Sort(pts)\n\n\treturn data.CHResponses{{\n\t\tData: &data.Data{\n\t\t\tPoints: pts,\n\t\t\tAM: m,\n\t\t\tCommonStep: 60,\n\t\t},\n\t\tFrom: from,\n\t\tUntil: until,\n\t}}\n}\n\n// emptyValues prefill slice of `size` with math.NaN\nfunc emptyValues(size int) []float64 {\n\tarr := make([]float64, 0, size)\n\tfor i := 0; i < size; i++ {\n\t\tarr = append(arr, math.NaN())\n\t}\n\n\treturn arr\n}\n\n// equalMetrics returns true if two slices of client.Metric are equal.\n// This function only compares important fields of client.Metric.\nfunc equalMetrics(m1, m2 []client.Metric) bool {\n\tif len(m1) != len(m2) {\n\t\treturn false\n\t}\n\n\tsort.Slice(m1, func(i, j int) bool {\n\t\treturn m1[i].Name < m1[j].Name\n\t})\n\tsort.Slice(m2, func(i, j int) bool {\n\t\treturn m2[i].Name < m2[j].Name\n\t})\n\n\tfor i := 0; i < len(m1); i++ {\n\t\t// compare props\n\t\tif m1[i].Name != m2[i].Name ||\n\t\t\tm1[i].StartTime != m2[i].StartTime ||\n\t\t\tm1[i].StopTime != m2[i].StopTime ||\n\t\t\tm1[i].StepTime != m2[i].StepTime {\n\t\t\treturn false\n\t\t}\n\t\t// compare values\n\t\tif len(m1[i].Values) != len(m2[i].Values) {\n\t\t\treturn false\n\t\t}\n\n\t\tfor j := 0; j < len(m1[i].Values); j++ {\n\t\t\ta, b := m1[i].Values[j], m2[i].Values[j]\n\t\t\tif math.IsNaN(a) && math.IsNaN(b) {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif a != b {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t}\n\n\treturn true\n}\n" }, { "path": "render/reply/json.go", "content": "package reply\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"net/http\"\n\n\tv3pb \"github.com/go-graphite/protocol/carbonapi_v3_pb\"\n\t\"go.uber.org/zap\"\n\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"github.com/lomik/graphite-clickhouse/render/data\"\n)\n\n// JSON is an implementation of carbonapi_v3_pb MultiGlobRequest and MultiFetchResponse interconnection. It accepts the\n// normal forms parser of `Content-Type: application/json` POST requests with JSON representation of MultiGlobRequest.\ntype JSON struct{}\n\nfunc marshalJSON(mfr *v3pb.MultiFetchResponse) []byte {\n\tbuf := bytes.Buffer{}\n\tbuf.WriteString(`{\"metrics\":[`)\n\n\tfor _, m := range mfr.Metrics {\n\t\tbuf.WriteRune('{')\n\n\t\tif m.Name != \"\" {\n\t\t\tbuf.WriteString(fmt.Sprintf(`\"name\":%q,`, m.Name))\n\t\t}\n\n\t\tif m.PathExpression != \"\" {\n\t\t\tbuf.WriteString(fmt.Sprintf(`\"pathExpression\":%q,`, m.PathExpression))\n\t\t}\n\n\t\tif m.ConsolidationFunc != \"\" {\n\t\t\tbuf.WriteString(fmt.Sprintf(`\"consolidationFunc\":%q,`, m.ConsolidationFunc))\n\t\t}\n\n\t\tbuf.WriteString(fmt.Sprintf(`\"startTime\":%d,`, m.StartTime))\n\t\tbuf.WriteString(fmt.Sprintf(`\"stopTime\":%d,`, m.StopTime))\n\t\tbuf.WriteString(fmt.Sprintf(`\"stepTime\":%d,`, m.StepTime))\n\t\tbuf.WriteString(fmt.Sprintf(`\"xFilesFactor\":%f,`, m.XFilesFactor))\n\n\t\tif m.HighPrecisionTimestamps {\n\t\t\tbuf.WriteString(`\"highPrecisionTimestamp\":true,`)\n\t\t}\n\n\t\tif len(m.Values) != 0 {\n\t\t\tbuf.WriteString(`\"values\":[`)\n\n\t\t\tfor _, v := range m.Values {\n\t\t\t\tif math.IsNaN(v) || math.IsInf(v, 0) {\n\t\t\t\t\tbuf.WriteString(\"null,\")\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tbuf.WriteString(fmt.Sprintf(\"%f,\", v))\n\t\t\t}\n\n\t\t\tbuf.Truncate(buf.Len() - 1)\n\t\t\tbuf.WriteString(\"],\")\n\t\t}\n\n\t\tbuf.WriteString(fmt.Sprintf(`\"requestStartTime\":%d,`, m.RequestStartTime))\n\t\tbuf.WriteString(fmt.Sprintf(`\"requestStopTime\":%d,`, m.RequestStopTime))\n\t\tbuf.Truncate(buf.Len() - 1)\n\t\tbuf.WriteString(\"},\")\n\t}\n\n\tif len(mfr.Metrics) != 0 {\n\t\tbuf.Truncate(buf.Len() - 1)\n\t}\n\n\tbuf.WriteString(\"]}\")\n\n\treturn buf.Bytes()\n}\n\nfunc parseJSONBody(r *http.Request) (data.MultiTarget, error) {\n\tlogger := scope.Logger(r.Context()).Named(\"json_parser\")\n\n\tvar pv3Request v3pb.MultiFetchRequest\n\n\terr := json.NewDecoder(r.Body).Decode(&pv3Request)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfetchRequests := data.MFRToMultiTarget(&pv3Request)\n\n\tif len(pv3Request.Metrics) > 0 {\n\t\tfor _, m := range pv3Request.Metrics {\n\t\t\tlogger.Info(\n\t\t\t\t\"json_target\",\n\t\t\t\tzap.Int64(\"from\", m.StartTime),\n\t\t\t\tzap.Int64(\"until\", m.StopTime),\n\t\t\t\tzap.Int64(\"maxDataPoints\", m.MaxDataPoints),\n\t\t\t\tzap.String(\"target\", m.PathExpression),\n\t\t\t)\n\t\t}\n\t}\n\n\treturn fetchRequests, nil\n}\n\n// ParseRequest first tries to get body for application/json and convert it to carbonapi_v3_pb.MultiFetchRequest. As a fail-over it\n// parses request forms.\nfunc (*JSON) ParseRequest(r *http.Request) (data.MultiTarget, error) {\n\tif !scope.Debug(r.Context(), \"Output\") {\n\t\treturn nil, errors.New(\"json format is only enabled for debugging purposes, pass 'X-Gch-Debug-Output: true' header\")\n\t}\n\n\tfetchRequests, err := parseJSONBody(r)\n\tif err == nil {\n\t\treturn fetchRequests, err\n\t}\n\n\treturn parseRequestForms(r)\n}\n\n// Reply response to request with JSON representation of carbonapi_v3_pb.MultiFetchResponse.\nfunc (*JSON) Reply(w http.ResponseWriter, r *http.Request, multiData data.CHResponses) {\n\tmfr, err := multiData.ToMultiFetchResponseV3()\n\tif err != nil {\n\t\thttp.Error(w, fmt.Sprintf(\"failed to convert response to v3pb.MultiFetchResponse: %v\", err), http.StatusInternalServerError)\n\t\treturn\n\t}\n\n\tresponse := marshalJSON(mfr)\n\tw.Write(response)\n}\n" }, { "path": "render/reply/pickle.go", "content": "package reply\n\nimport (\n\t\"bufio\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"net/http\"\n\t\"time\"\n\n\tgraphitePickle \"github.com/lomik/graphite-pickle\"\n\t\"go.uber.org/zap\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/point\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"github.com/lomik/graphite-clickhouse/render/data\"\n)\n\n// Pickle is a formatter for python object serialization format.\ntype Pickle struct{}\n\n// ParseRequest parses target/from/until/maxDataPoints URL forms values\nfunc (*Pickle) ParseRequest(r *http.Request) (data.MultiTarget, error) {\n\treturn parseRequestForms(r)\n}\n\n// Reply serializes ClickHouse response to pickle format\nfunc (*Pickle) Reply(w http.ResponseWriter, r *http.Request, multiData data.CHResponses) {\n\tvar pickleTime time.Duration\n\t// Pickle format always contain single request/response\n\tdata := multiData[0].Data\n\tfrom := uint32(multiData[0].From)\n\tuntil := uint32(multiData[0].Until)\n\n\tlogger := scope.Logger(r.Context())\n\n\tdefer func() {\n\t\tlogger.Debug(\"pickle\",\n\t\t\tzap.String(\"runtime\", pickleTime.String()),\n\t\t\tzap.Duration(\"runtime_ns\", pickleTime),\n\t\t)\n\t}()\n\n\tif data.AM.Len() == 0 {\n\t\tw.Write(graphitePickle.EmptyList)\n\t\treturn\n\t}\n\n\twriter := bufio.NewWriterSize(w, 1024*1024)\n\tp := graphitePickle.NewWriter(writer)\n\tdefer writer.Flush()\n\n\tp.List()\n\n\twriteAlias := func(name string, pathExpression string, points []point.Point, step uint32) {\n\t\tpickleStart := time.Now()\n\n\t\tp.Dict()\n\n\t\tp.String(\"name\")\n\t\tp.String(name)\n\t\tp.SetItem()\n\n\t\tp.String(\"pathExpression\")\n\t\tp.String(pathExpression)\n\t\tp.SetItem()\n\n\t\tp.String(\"step\")\n\t\tp.Uint32(step)\n\t\tp.SetItem()\n\n\t\tstart, end, _, getValue := point.FillNulls(points, from, until, step)\n\n\t\tp.String(\"values\")\n\t\tp.List()\n\n\t\tfor {\n\t\t\tvalue, err := getValue()\n\t\t\tif err != nil {\n\t\t\t\tif errors.Is(err, point.ErrTimeGreaterStop) {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\t// if err is not point.ErrTimeGreaterStop, the points are corrupted\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif !math.IsNaN(value) {\n\t\t\t\tp.AppendFloat64(value)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tp.AppendNulls(1)\n\t\t}\n\n\t\tp.SetItem()\n\n\t\tp.String(\"start\")\n\t\tp.Uint32(start)\n\t\tp.SetItem()\n\n\t\tp.String(\"end\")\n\t\tp.Uint32(end)\n\t\tp.SetItem()\n\n\t\tp.Append()\n\n\t\tpickleTime += time.Since(pickleStart)\n\t}\n\n\t// write points and mark as written in writeMap\n\twriteMetric := func(points []point.Point, writeMap map[string]struct{}) error {\n\t\tmetricName := data.MetricName(points[0].MetricID)\n\t\twriteMap[metricName] = struct{}{}\n\n\t\tstep, err := data.GetStep(points[0].MetricID)\n\t\tif err != nil {\n\t\t\tlogger.Error(\"fail to get step\", zap.Error(err))\n\t\t\thttp.Error(w, fmt.Sprintf(\"failed to get step for metric: %v\", data.MetricName(points[0].MetricID)), http.StatusInternalServerError)\n\n\t\t\treturn err\n\t\t}\n\n\t\tfor _, a := range data.AM.Get(metricName) {\n\t\t\twriteAlias(a.DisplayName, a.Target, points, step)\n\t\t}\n\n\t\treturn nil\n\t}\n\n\tnextMetric := data.GroupByMetric()\n\twrittenMetrics := make(map[string]struct{})\n\t// fill metrics with points\n\tfor {\n\t\tpoints := nextMetric()\n\t\tif len(points) == 0 {\n\t\t\tbreak\n\t\t}\n\n\t\tif err := writeMetric(points, writtenMetrics); err != nil {\n\t\t\treturn\n\t\t}\n\t}\n\t// fill metrics without points with NaN\n\tif multiData[0].AppendOutEmptySeries && len(writtenMetrics) != data.AM.Len() && data.CommonStep > 0 {\n\t\tfor _, metricName := range data.AM.Series(false) {\n\t\t\tif _, done := writtenMetrics[metricName]; !done {\n\t\t\t\tfor _, a := range data.AM.Get(metricName) {\n\t\t\t\t\twriteAlias(a.DisplayName, a.Target, []point.Point{}, uint32(data.CommonStep))\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tp.Stop()\n}\n" }, { "path": "render/reply/protobuf.go", "content": "package reply\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"net/http\"\n\n\t\"go.uber.org/zap\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/point\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"github.com/lomik/graphite-clickhouse/render/data\"\n)\n\nvar pbVarints []byte\n\nconst (\n\trepeated = 2\n\tflt32 = 5\n\tprotobufMaxVarintBytes = 10 // maximum length of a varint\n)\n\ntype pb interface {\n\tinitBuffer()\n\twriteBody(writer *bufio.Writer, target, name, function string, from, until, step uint32, points []point.Point)\n}\n\nfunc replyProtobuf(p pb, w http.ResponseWriter, r *http.Request, multiData data.CHResponses) {\n\tlogger := scope.Logger(r.Context())\n\n\t// var multiResponse carbonzipperpb.MultiFetchResponse\n\twriter := bufio.NewWriterSize(w, 1024*1024)\n\tdefer writer.Flush()\n\n\tp.initBuffer()\n\n\ttotalWritten := 0\n\n\tfor _, d := range multiData {\n\t\tdata := d.Data\n\t\tfrom := uint32(d.From)\n\t\tuntil := uint32(d.Until)\n\n\t\ttotalWritten++\n\n\t\tnextMetric := data.GroupByMetric()\n\t\twrittenMetrics := make(map[string]struct{})\n\n\t\t// fill metrics with points\n\t\tfor {\n\t\t\tpoints := nextMetric()\n\t\t\tif len(points) == 0 {\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\tmetricName := data.MetricName(points[0].MetricID)\n\t\t\twrittenMetrics[metricName] = struct{}{}\n\n\t\t\tstep, err := data.GetStep(points[0].MetricID)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Error(\"fail to get step\", zap.Error(err))\n\t\t\t\thttp.Error(w, fmt.Sprintf(\"failed to get step for metric: %v\", data.MetricName(points[0].MetricID)), http.StatusInternalServerError)\n\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tfunction, err := data.GetAggregation(points[0].MetricID)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Error(\"fail to get function\", zap.Error(err))\n\t\t\t\thttp.Error(w, fmt.Sprintf(\"failed to get function for metric: %v\", data.MetricName(points[0].MetricID)), http.StatusInternalServerError)\n\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tfor _, a := range data.AM.Get(metricName) {\n\t\t\t\tp.writeBody(writer, a.Target, a.DisplayName, function, from, until, step, points)\n\t\t\t}\n\t\t}\n\n\t\t// fill metrics without points with NaN\n\t\tif d.AppendOutEmptySeries && len(writtenMetrics) < data.AM.Len() && data.CommonStep > 0 {\n\t\t\tfor _, metricName := range data.AM.Series(false) {\n\t\t\t\tif _, done := writtenMetrics[metricName]; !done {\n\t\t\t\t\tfor _, a := range data.AM.Get(metricName) {\n\t\t\t\t\t\tp.writeBody(writer, a.Target, a.DisplayName, \"any\", from, until, uint32(data.CommonStep), []point.Point{})\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tif totalWritten == 0 {\n\t\tw.WriteHeader(http.StatusNotFound)\n\t\treturn\n\t}\n}\n\nfunc init() {\n\t// precalculate varints\n\tbuf := bytes.NewBuffer(nil)\n\n\tfor i := uint64(0); i < 16384; i++ {\n\t\tbuf.Write(VarintEncode(i))\n\t}\n\n\tpbVarints = buf.Bytes()\n}\n\nfunc VarintEncode(x uint64) []byte {\n\tvar buf [protobufMaxVarintBytes]byte\n\n\tvar n int\n\n\tfor n = 0; x > 127; n++ {\n\t\tbuf[n] = 0x80 | uint8(x&0x7F)\n\t\tx >>= 7\n\t}\n\n\tbuf[n] = uint8(x)\n\tn++\n\n\treturn buf[0:n]\n}\n\nfunc VarintWrite(w io.Writer, x uint64) {\n\t// for ResponseWriter. ignore write result\n\tif x < 128 {\n\t\tw.Write(pbVarints[x : x+1])\n\t} else if x < 16384 {\n\t\tw.Write(pbVarints[x*2-128 : x*2-126])\n\t} else {\n\t\tw.Write(VarintEncode(x))\n\t}\n}\n\nfunc VarintLen(x uint64) uint64 {\n\tif x < 128 {\n\t\treturn 1\n\t}\n\n\tif x < 16384 {\n\t\treturn 2\n\t}\n\n\tj := uint64(2)\n\tfor i := uint64(16384); i <= x; i *= 128 {\n\t\tj++\n\t}\n\n\treturn j\n}\n\nfunc WriteByteN(w *bufio.Writer, value byte, n int) {\n\t// @TODO: optimize\n\tfor i := 0; i < n; i++ {\n\t\tw.WriteByte(value)\n\t}\n}\n\nfunc Fixed64Encode(x uint64) []byte {\n\treturn []byte{\n\t\tuint8(x),\n\t\tuint8(x >> 8),\n\t\tuint8(x >> 16),\n\t\tuint8(x >> 24),\n\t\tuint8(x >> 32),\n\t\tuint8(x >> 40),\n\t\tuint8(x >> 48),\n\t\tuint8(x >> 56),\n\t}\n}\n\nfunc Fixed32Encode(x uint32) []byte {\n\treturn []byte{\n\t\tuint8(x),\n\t\tuint8(x >> 8),\n\t\tuint8(x >> 16),\n\t\tuint8(x >> 24),\n\t}\n}\n\nfunc ProtobufWriteSingle(w io.Writer, value float32) {\n\tw.Write(Fixed32Encode(math.Float32bits(value)))\n}\n\nfunc ProtobufWriteDouble(w io.Writer, value float64) {\n\tw.Write(Fixed64Encode(math.Float64bits(value)))\n}\n\nfunc ProtobufWriteDoubleN(w io.Writer, value float64, n int) {\n\tb := Fixed64Encode(math.Float64bits(value))\n\tfor i := 0; i < n; i++ {\n\t\tw.Write(b)\n\t}\n}\n" }, { "path": "render/reply/protobuf_test.go", "content": "package reply\n\nimport (\n\t\"encoding/binary\"\n\t\"testing\"\n)\n\nfunc TestVarintLen(t *testing.T) {\n\tbuf := make([]byte, binary.MaxVarintLen64)\n\n\tfor i := uint64(0); i < 1000000; i++ {\n\t\tn := binary.PutUvarint(buf, i)\n\t\tif VarintLen(i) != uint64(n) {\n\t\t\tt.FailNow()\n\t\t}\n\t}\n}\n" }, { "path": "render/reply/v2_pb.go", "content": "package reply\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"net/http\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/point\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"github.com/lomik/graphite-clickhouse/render/data\"\n)\n\n// V2PB is a formatter for carbonapi_v2_pb\ntype V2PB struct {\n\tb1 *bytes.Buffer\n\tb2 *bytes.Buffer\n}\n\n// ParseRequest parses target/from/until/maxDataPoints URL forms values\nfunc (*V2PB) ParseRequest(r *http.Request) (data.MultiTarget, error) {\n\treturn parseRequestForms(r)\n}\n\n// Reply serializes ClickHouse response to carbonapi_v2_pb.MultiFetchResponse format\nfunc (v *V2PB) Reply(w http.ResponseWriter, r *http.Request, multiData data.CHResponses) {\n\tif scope.Debug(r.Context(), \"Protobuf\") {\n\t\tv.replyDebug(w, r, multiData)\n\t}\n\n\treplyProtobuf(v, w, r, multiData)\n}\n\nfunc (v *V2PB) initBuffer() {\n\tv.b1 = new(bytes.Buffer)\n\tv.b2 = new(bytes.Buffer)\n}\n\nfunc (v *V2PB) replyDebug(w http.ResponseWriter, r *http.Request, multiData data.CHResponses) {\n\tmfr, err := multiData.ToMultiFetchResponseV2()\n\tif err != nil {\n\t\thttp.Error(w, fmt.Sprintf(\"failed to convert response to v2pb.MultiFetchResponse: %v\", err), http.StatusInternalServerError)\n\t}\n\n\tresponse, err := mfr.Marshal()\n\tif err != nil {\n\t\thttp.Error(w, fmt.Sprintf(\"failed to marshal v2pb.MultiFetchResponse: %v\", err), http.StatusInternalServerError)\n\t}\n\n\tw.Write(response)\n}\n\nfunc (v *V2PB) writeBody(writer *bufio.Writer, target, name, function string, from, until, step uint32, points []point.Point) {\n\tstart, stop, count, getValue := point.FillNulls(points, from, until, step)\n\n\tv.b1.Reset()\n\tv.b2.Reset()\n\n\t// name\n\tVarintWrite(v.b1, (1<<3)+repeated) // tag\n\tVarintWrite(v.b1, uint64(len(name)))\n\tv.b1.WriteString(name)\n\n\t// start\n\tVarintWrite(v.b1, 2<<3)\n\tVarintWrite(v.b1, uint64(start))\n\n\t// stop\n\tVarintWrite(v.b1, 3<<3)\n\tVarintWrite(v.b1, uint64(stop))\n\n\t// step\n\tVarintWrite(v.b1, 4<<3)\n\tVarintWrite(v.b1, uint64(step))\n\n\t// start write to output\n\t// Write values\n\tVarintWrite(v.b1, (5<<3)+repeated)\n\tVarintWrite(v.b1, uint64(8*count))\n\n\t// Write isAbsent\n\tVarintWrite(v.b2, (6<<3)+repeated)\n\tVarintWrite(v.b2, uint64(count))\n\n\tfor {\n\t\tvalue, err := getValue()\n\t\tif err != nil {\n\t\t\tif errors.Is(err, point.ErrTimeGreaterStop) {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\t// if err is not point.ErrTimeGreaterStop, the points are corrupted\n\t\t\treturn\n\t\t}\n\n\t\tif !math.IsNaN(value) {\n\t\t\tProtobufWriteDouble(v.b1, value)\n\t\t\tv.b2.WriteByte(0)\n\n\t\t\tcontinue\n\t\t}\n\n\t\tProtobufWriteDouble(v.b1, 0)\n\t\tv.b2.WriteByte(1)\n\t}\n\n\t// repeated FetchResponse metrics = 1;\n\t// write tag and len\n\tVarintWrite(writer, (1<<3)+repeated)\n\tVarintWrite(writer, uint64(v.b1.Len())+uint64(v.b2.Len()))\n\n\twriter.Write(v.b1.Bytes())\n\twriter.Write(v.b2.Bytes())\n}\n" }, { "path": "render/reply/v2_pb_test.go", "content": "package reply\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"math\"\n\t\"reflect\"\n\t\"testing\"\n\n\tv2pb \"github.com/go-graphite/protocol/carbonapi_v2_pb\"\n\t\"github.com/lomik/graphite-clickhouse/helper/point\"\n)\n\ntype testV2PB struct {\n\tname string\n\ttarget string\n\tfunction string\n\tresponse v2pb.MultiFetchResponse\n\tfrom uint32\n\tuntil uint32\n\tstep uint32\n\tpoints []point.Point\n}\n\nfunc TestV2PBWriteBody(t *testing.T) {\n\ttests := []testV2PB{\n\t\t{\n\t\t\tname: \"singlePoint\",\n\t\t\tfunction: \"avg\",\n\t\t\tfrom: 4,\n\t\t\tuntil: 13,\n\t\t\tstep: 5,\n\t\t\ttarget: \"*\",\n\t\t\tpoints: []point.Point{\n\t\t\t\t{\n\t\t\t\t\tMetricID: 0,\n\t\t\t\t\tValue: 1.0,\n\t\t\t\t\tTime: 5,\n\t\t\t\t\tTimestamp: 5,\n\t\t\t\t},\n\t\t\t},\n\t\t\tresponse: v2pb.MultiFetchResponse{\n\t\t\t\tMetrics: []v2pb.FetchResponse{\n\t\t\t\t\t{\n\t\t\t\t\t\tName: \"singlePoint\",\n\t\t\t\t\t\tStartTime: 5,\n\t\t\t\t\t\tStopTime: 10,\n\t\t\t\t\t\tStepTime: 5,\n\t\t\t\t\t\tValues: []float64{1.0},\n\t\t\t\t\t\tIsAbsent: []bool{false},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"multiPoint\",\n\t\t\tfunction: \"max\",\n\t\t\tfrom: 1,\n\t\t\tuntil: 5,\n\t\t\tstep: 1,\n\t\t\ttarget: \"multiPoint\",\n\t\t\tpoints: []point.Point{\n\t\t\t\t{\n\t\t\t\t\tMetricID: 0,\n\t\t\t\t\tValue: 1.0,\n\t\t\t\t\tTime: 2,\n\t\t\t\t\tTimestamp: 2,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tMetricID: 0,\n\t\t\t\t\tValue: math.NaN(),\n\t\t\t\t\tTime: 3,\n\t\t\t\t\tTimestamp: 3,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tMetricID: 0,\n\t\t\t\t\tValue: 3.0,\n\t\t\t\t\tTime: 4,\n\t\t\t\t\tTimestamp: 4,\n\t\t\t\t},\n\t\t\t},\n\t\t\tresponse: v2pb.MultiFetchResponse{\n\t\t\t\tMetrics: []v2pb.FetchResponse{\n\t\t\t\t\t{\n\t\t\t\t\t\tName: \"multiPoint\",\n\t\t\t\t\t\tStartTime: 1,\n\t\t\t\t\t\tStopTime: 6,\n\t\t\t\t\t\tStepTime: 1,\n\t\t\t\t\t\tValues: []float64{math.NaN(), 1.0, math.NaN(), 3.0, math.NaN()},\n\t\t\t\t\t\tIsAbsent: []bool{true, false, true, false, true},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\ttestName := tt.name\n\n\t\tt.Run(testName, func(t *testing.T) {\n\t\t\tcorrectResp, _ := tt.response.Marshal()\n\n\t\t\tb := bytes.Buffer{}\n\t\t\tw := bufio.NewWriter(&b)\n\n\t\t\tv := &V2PB{}\n\t\t\tv.initBuffer()\n\t\t\tv.writeBody(w, tt.target, tt.name, tt.function, tt.from, tt.until, tt.step, tt.points)\n\n\t\t\tw.Flush()\n\n\t\t\tvar resp v2pb.MultiFetchResponse\n\n\t\t\tdata := b.Bytes()\n\t\t\tif bytes.Compare(data, correctResp) != 0 {\n\t\t\t\tt.Logf(\"different byte response.\\ngot:\\n%v\\n\\nexpected:\\n%v\", data, correctResp)\n\t\t\t}\n\n\t\t\terr := resp.Unmarshal(data)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"failed to unmarshal reply, got '%v'\", err)\n\t\t\t}\n\n\t\t\tif len(resp.Metrics) != len(tt.response.Metrics) {\n\t\t\t\tt.Fatalf(\"incorrect amount of metrics, expected %v, got %v\", len(resp.Metrics), len(tt.response.Metrics))\n\t\t\t}\n\n\t\t\tfor i := range resp.Metrics {\n\t\t\t\tif resp.Metrics[i].Name != tt.response.Metrics[i].Name {\n\t\t\t\t\tif !reflect.DeepEqual(resp.Metrics[i], tt.response.Metrics[i]) {\n\t\t\t\t\t\tt.Fatalf(\"replies are not same.\\ngot:\\n%+v\\n\\nexpected:\\n%+v\", resp.Metrics[i], tt.response.Metrics[i])\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "render/reply/v3_pb.go", "content": "package reply\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\n\tv3pb \"github.com/go-graphite/protocol/carbonapi_v3_pb\"\n\t\"github.com/lomik/graphite-clickhouse/helper/point\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n\t\"github.com/lomik/graphite-clickhouse/render/data\"\n\t\"go.uber.org/zap\"\n)\n\n// V3PB is a formatter for carbonapi_v3_pb\ntype V3PB struct {\n\tb *bytes.Buffer\n}\n\n// ParseRequest reads the requests parameters from carbonapi_v3_pb.MultiFetchRequest\nfunc (*V3PB) ParseRequest(r *http.Request) (data.MultiTarget, error) {\n\tlogger := scope.Logger(r.Context()).Named(\"pb3parser\")\n\n\tbody, err := io.ReadAll(r.Body)\n\tif err != nil {\n\t\tlogger.Error(\"failed to read request\", zap.Error(err))\n\t\treturn nil, fmt.Errorf(\"failed to read request body: %w\", err)\n\t}\n\n\tvar pv3Request v3pb.MultiFetchRequest\n\tif err := pv3Request.Unmarshal(body); err != nil {\n\t\tlogger.Error(\"failed to unmarshal request\", zap.Error(err))\n\t\treturn nil, fmt.Errorf(\"failed to unmarshal request: %w\", err)\n\t}\n\n\tmultiTarget := data.MFRToMultiTarget(&pv3Request)\n\n\tif len(pv3Request.Metrics) > 0 {\n\t\tfor _, m := range pv3Request.Metrics {\n\t\t\tlogger.Info(\n\t\t\t\t\"pb3_target\",\n\t\t\t\tzap.Int64(\"from\", m.StartTime),\n\t\t\t\tzap.Int64(\"until\", m.StopTime),\n\t\t\t\tzap.Int64(\"maxDataPoints\", m.MaxDataPoints),\n\t\t\t\tzap.String(\"target\", m.PathExpression),\n\t\t\t)\n\t\t}\n\t}\n\n\tif scope.Debug(r.Context(), \"Output\") {\n\t\trequest, err := json.Marshal(pv3Request)\n\t\tif err == nil {\n\t\t\tlogger.Info(\"v3pb_request\", zap.ByteString(\"json\", request))\n\t\t}\n\t}\n\n\treturn multiTarget, nil\n}\n\n// Reply serializes ClickHouse response to carbonapi_v3_pb.MultiFetchResponse format\nfunc (v *V3PB) Reply(w http.ResponseWriter, r *http.Request, multiData data.CHResponses) {\n\tif scope.Debug(r.Context(), \"Protobuf\") {\n\t\tv.replyDebug(w, r, multiData)\n\t}\n\n\treplyProtobuf(v, w, r, multiData)\n}\n\nfunc (v *V3PB) initBuffer() {\n\tv.b = new(bytes.Buffer)\n}\n\nfunc (v *V3PB) replyDebug(w http.ResponseWriter, r *http.Request, multiData data.CHResponses) {\n\tmfr, err := multiData.ToMultiFetchResponseV3()\n\tif err != nil {\n\t\thttp.Error(w, fmt.Sprintf(\"failed to convert response to v3pb.MultiFetchResponse: %v\", err), http.StatusInternalServerError)\n\t}\n\n\tresponse, err := mfr.Marshal()\n\tif err != nil {\n\t\thttp.Error(w, fmt.Sprintf(\"failed to marshal v3pb.MultiFetchResponse: %v\", err), http.StatusInternalServerError)\n\t}\n\n\tw.Write(response)\n}\n\nfunc (v *V3PB) writeBody(writer *bufio.Writer, target, name, function string, from, until, step uint32, points []point.Point) {\n\tstart, stop, count, getValue := point.FillNulls(points, from, until, step)\n\n\tv.b.Reset()\n\n\t// First chunk\n\t// name\n\tVarintWrite(v.b, (1<<3)+repeated) // tag\n\tVarintWrite(v.b, uint64(len(name)))\n\tv.b.WriteString(name)\n\n\t// pathExpression\n\tVarintWrite(v.b, (2<<3)+repeated) // tag\n\tVarintWrite(v.b, uint64(len(target)))\n\tv.b.WriteString(target)\n\n\tconsolidationFunc := function\n\t// consolidationFunc\n\tVarintWrite(v.b, (3<<3)+repeated) // tag\n\tVarintWrite(v.b, uint64(len(consolidationFunc)))\n\tv.b.WriteString(consolidationFunc)\n\n\t// start\n\tVarintWrite(v.b, 4<<3) // tag\n\tVarintWrite(v.b, uint64(start))\n\n\t// stop\n\tVarintWrite(v.b, 5<<3) // tag\n\tVarintWrite(v.b, uint64(stop))\n\n\t// step\n\tVarintWrite(v.b, 6<<3) // tag\n\tVarintWrite(v.b, uint64(step))\n\n\t// xFilesFactor\n\tVarintWrite(v.b, (7<<3)+flt32) // tag\n\tProtobufWriteSingle(v.b, 0.0)\n\n\t// highPrecisionTimestamps\n\tVarintWrite(v.b, 8<<3) // tag\n\tv.b.WriteByte('\\x00') // False\n\n\t// Values header\n\tVarintWrite(v.b, (9<<3)+repeated) // tag\n\tVarintWrite(v.b, uint64(8*count))\n\n\tfor {\n\t\tvalue, err := getValue()\n\t\tif err != nil {\n\t\t\tif errors.Is(err, point.ErrTimeGreaterStop) {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\t// if err is not point.ErrTimeGreaterStop, the points are corrupted\n\t\t\treturn\n\t\t}\n\n\t\tProtobufWriteDouble(v.b, value)\n\t}\n\n\t// rest fields, that goes after values\n\n\t// Fields with default values are skipped, so this should be uncommented if support for appliedFunctions will be\n\t// implemented\n\t// appliedFunctions\n\t//VarintWrite(mb2, (10<<3)+Repeated) // tag\n\t//VarintWrite(mb2, VarintLen(0)) // currently not supported\n\n\t// requestStartTime\n\tVarintWrite(v.b, 11<<3)\n\tVarintWrite(v.b, uint64(from))\n\n\t// requestStopTime\n\tVarintWrite(v.b, 12<<3)\n\tVarintWrite(v.b, uint64(until))\n\n\t// start write to output\n\t// repeated FetchResponse metrics = 1;\n\t// write tag and len\n\tVarintWrite(writer, (1<<3)+2)\n\tVarintWrite(writer, uint64(v.b.Len()))\n\n\twriter.Write(v.b.Bytes())\n}\n" }, { "path": "render/reply/v3_pb_test.go", "content": "package reply\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"math\"\n\t\"reflect\"\n\t\"testing\"\n\n\tv3pb \"github.com/go-graphite/protocol/carbonapi_v3_pb\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/point\"\n)\n\ntype testV3PB struct {\n\tname string\n\ttarget string\n\tfunction string\n\tresponse v3pb.MultiFetchResponse\n\tfrom uint32\n\tuntil uint32\n\tstep uint32\n\tpoints []point.Point\n}\n\nfunc TestV3PBWriteBody(t *testing.T) {\n\ttests := []testV3PB{\n\t\t{\n\t\t\tname: \"singlePoint\",\n\t\t\tfunction: \"avg\",\n\t\t\tfrom: 4,\n\t\t\tuntil: 13,\n\t\t\tstep: 5,\n\t\t\ttarget: \"*\",\n\t\t\tpoints: []point.Point{\n\t\t\t\t{\n\t\t\t\t\tMetricID: 0,\n\t\t\t\t\tValue: 1.0,\n\t\t\t\t\tTime: 5,\n\t\t\t\t\tTimestamp: 5,\n\t\t\t\t},\n\t\t\t},\n\t\t\tresponse: v3pb.MultiFetchResponse{\n\t\t\t\tMetrics: []v3pb.FetchResponse{\n\t\t\t\t\t{\n\t\t\t\t\t\tName: \"singlePoint\",\n\t\t\t\t\t\tPathExpression: \"*\",\n\t\t\t\t\t\tConsolidationFunc: \"avg\",\n\t\t\t\t\t\tXFilesFactor: 0,\n\t\t\t\t\t\tHighPrecisionTimestamps: false,\n\t\t\t\t\t\tStartTime: 5,\n\t\t\t\t\t\tStopTime: 10,\n\t\t\t\t\t\tValues: []float64{1.0},\n\t\t\t\t\t\tAppliedFunctions: []string{},\n\t\t\t\t\t\tRequestStartTime: 4,\n\t\t\t\t\t\tRequestStopTime: 13,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"multiPoint\",\n\t\t\tfunction: \"max\",\n\t\t\tfrom: 1,\n\t\t\tuntil: 5,\n\t\t\tstep: 1,\n\t\t\ttarget: \"multiPoint\",\n\t\t\tpoints: []point.Point{\n\t\t\t\t{\n\t\t\t\t\tMetricID: 0,\n\t\t\t\t\tValue: 1.0,\n\t\t\t\t\tTime: 2,\n\t\t\t\t\tTimestamp: 2,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tMetricID: 0,\n\t\t\t\t\tValue: math.NaN(),\n\t\t\t\t\tTime: 3,\n\t\t\t\t\tTimestamp: 3,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tMetricID: 0,\n\t\t\t\t\tValue: 3.0,\n\t\t\t\t\tTime: 4,\n\t\t\t\t\tTimestamp: 4,\n\t\t\t\t},\n\t\t\t},\n\t\t\tresponse: v3pb.MultiFetchResponse{\n\t\t\t\tMetrics: []v3pb.FetchResponse{\n\t\t\t\t\t{\n\t\t\t\t\t\tName: \"multiPoint\",\n\t\t\t\t\t\tPathExpression: \"multiPoint\",\n\t\t\t\t\t\tConsolidationFunc: \"max\",\n\t\t\t\t\t\tXFilesFactor: 0,\n\t\t\t\t\t\tHighPrecisionTimestamps: false,\n\t\t\t\t\t\tStartTime: 1,\n\t\t\t\t\t\tStopTime: 6,\n\t\t\t\t\t\tValues: []float64{math.NaN(), 1.0, math.NaN(), 3.0, math.NaN()},\n\t\t\t\t\t\tAppliedFunctions: []string{},\n\t\t\t\t\t\tRequestStartTime: 1,\n\t\t\t\t\t\tRequestStopTime: 6,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\ttestName := tt.name\n\n\t\tt.Run(testName, func(t *testing.T) {\n\t\t\tcorrectResp, _ := tt.response.Marshal()\n\n\t\t\tb := bytes.Buffer{}\n\t\t\tw := bufio.NewWriter(&b)\n\n\t\t\tv := &V3PB{}\n\t\t\tv.initBuffer()\n\t\t\tv.writeBody(w, tt.target, tt.name, tt.function, tt.from, tt.until, tt.step, tt.points)\n\n\t\t\tw.Flush()\n\n\t\t\tvar resp v3pb.MultiFetchResponse\n\n\t\t\tdata := b.Bytes()\n\t\t\tif bytes.Compare(data, correctResp) != 0 {\n\t\t\t\tt.Logf(\"different byte response.\\ngot:\\n%v\\n\\nexpected:\\n%v\", data, correctResp)\n\t\t\t}\n\n\t\t\terr := resp.Unmarshal(data)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"failed to unmarshal reply, got '%v'\", err)\n\t\t\t}\n\n\t\t\tif len(resp.Metrics) != len(tt.response.Metrics) {\n\t\t\t\tt.Fatalf(\n\t\t\t\t\t\"incorrect amount of metrics, expected %v, got %v\",\n\t\t\t\t\tlen(resp.Metrics),\n\t\t\t\t\tlen(tt.response.Metrics),\n\t\t\t\t)\n\t\t\t}\n\n\t\t\tfor i := range resp.Metrics {\n\t\t\t\tif resp.Metrics[i].Name != tt.response.Metrics[i].Name {\n\t\t\t\t\tif !reflect.DeepEqual(resp.Metrics[i], tt.response.Metrics[i]) {\n\t\t\t\t\t\tt.Fatalf(\n\t\t\t\t\t\t\t\"replies are not same.\\ngot:\\n%+v\\n\\nexpected:\\n%+v\",\n\t\t\t\t\t\t\tresp.Metrics[i],\n\t\t\t\t\t\t\ttt.response.Metrics[i],\n\t\t\t\t\t\t)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "sd/nginx/nginx.go", "content": "package nginx\n\nimport (\n\t\"encoding/base64\"\n\t\"errors\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/sd/utils\"\n\n\tjsoniter \"github.com/json-iterator/go\"\n\t\"github.com/msaf1980/go-stringutils\"\n\t\"go.uber.org/zap\"\n)\n\ntype ErrInvalidKey struct {\n\tkey string\n\tval string\n}\n\nfunc (e ErrInvalidKey) Error() string {\n\treturn \"list key '\" + e.key + \"' is invalid: '\" + e.val + \"'\"\n}\n\nvar (\n\tjson = jsoniter.ConfigCompatibleWithStandardLibrary\n\tErrNoKey = errors.New(\"list key no found\")\n\ttimeNow = time.Now\n)\n\nfunc splitNode(node string) (dc, host, listen string, ok bool) {\n\tvar v string\n\n\tdc, v, ok = strings.Cut(node, \"/\")\n\tif !ok {\n\t\treturn\n\t}\n\n\thost, v, ok = strings.Cut(v, \"/\")\n\tif !ok {\n\t\treturn\n\t}\n\n\tlisten, _, ok = strings.Cut(v, \"/\")\n\tok = !ok\n\n\treturn\n}\n\n// Nginx register node for https://github.com/weibocom/nginx-upsync-module (with consul)\ntype Nginx struct {\n\tweight int64\n\thostname string\n\tnamespace string\n\tbody []byte\n\tbackupBody []byte\n\turl stringutils.Builder\n\tpos int // truncate offset for base address\n\tnsEnd string\n\n\tlogger *zap.Logger\n}\n\nfunc New(url, namespace, hostname string, logger *zap.Logger) *Nginx {\n\tif namespace == \"\" {\n\t\tnamespace = \"graphite\"\n\t}\n\n\tsd := &Nginx{\n\t\tlogger: logger,\n\t\tbody: make([]byte, 128),\n\t\tbackupBody: []byte(`{\"backup\":1,\"max_fails\":0}`),\n\t\tnsEnd: \"upstreams/\" + namespace + \"/\",\n\t\thostname: hostname,\n\t\tnamespace: namespace,\n\t}\n\tsd.setWeight(1)\n\n\tsd.url.WriteString(url)\n\tsd.url.WriteByte('/')\n\n\tif namespace != \"\" {\n\t\tsd.url.WriteString(namespace)\n\t\tsd.url.WriteByte('/')\n\t}\n\n\tsd.pos = sd.url.Len()\n\n\treturn sd\n}\n\nfunc (sd *Nginx) setWeight(weight int64) {\n\tif weight <= 0 {\n\t\tweight = 1\n\t}\n\n\tif sd.weight != weight {\n\t\tsd.weight = weight\n\t\tsd.body = sd.body[:0]\n\t\tsd.body = append(sd.body, `{\"weight\":`...)\n\t\tsd.body = strconv.AppendInt(sd.body, weight, 10)\n\t\tsd.body = append(sd.body, `,\"max_fails\":0}`...)\n\t}\n}\n\nfunc (sd *Nginx) Namespace() string {\n\treturn sd.namespace\n}\n\nfunc (sd *Nginx) List() (nodes []string, err error) {\n\tsd.url.Truncate(sd.pos)\n\tsd.url.WriteString(\"?recurse\")\n\n\tvar data []byte\n\n\tdata, err = utils.HttpGet(sd.url.String())\n\tif err != nil {\n\t\treturn\n\t}\n\n\tvar iNodes []interface{}\n\tif err = json.Unmarshal(data, &iNodes); err != nil {\n\t\treturn nil, err\n\t}\n\n\tnodes = make([]string, 0, len(iNodes))\n\n\tfor _, i := range iNodes {\n\t\tif jNode, ok := i.(map[string]interface{}); ok {\n\t\t\tif i, ok := jNode[\"Key\"]; ok {\n\t\t\t\tif s, ok := i.(string); ok {\n\t\t\t\t\tif strings.HasPrefix(s, sd.nsEnd) {\n\t\t\t\t\t\ts = s[len(sd.nsEnd):]\n\t\t\t\t\t\t_, host, _, ok := splitNode(s)\n\n\t\t\t\t\t\tif ok && host == sd.hostname {\n\t\t\t\t\t\t\tnodes = append(nodes, s)\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\treturn nil, ErrInvalidKey{key: sd.nsEnd, val: s}\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\treturn nil, ErrNoKey\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\treturn nil, ErrNoKey\n\t\t}\n\t}\n\n\treturn\n}\n\nfunc (sd *Nginx) ListMap() (nodes map[string]string, err error) {\n\tsd.url.Truncate(sd.pos)\n\tsd.url.WriteString(\"?recurse\")\n\n\tvar data []byte\n\n\tdata, err = utils.HttpGet(sd.url.String())\n\tif err != nil {\n\t\treturn\n\t}\n\n\tvar iNodes []interface{}\n\tif err = json.Unmarshal(data, &iNodes); err != nil {\n\t\treturn nil, err\n\t}\n\n\tnodes = make(map[string]string)\n\n\tfor _, i := range iNodes {\n\t\tif jNode, ok := i.(map[string]interface{}); ok {\n\t\t\tif i, ok := jNode[\"Key\"]; ok {\n\t\t\t\tif s, ok := i.(string); ok {\n\t\t\t\t\tif strings.HasPrefix(s, sd.nsEnd) {\n\t\t\t\t\t\ts = s[len(sd.nsEnd):]\n\n\t\t\t\t\t\t_, host, _, ok := splitNode(s)\n\t\t\t\t\t\tif ok && host == sd.hostname {\n\t\t\t\t\t\t\tif i, ok := jNode[\"Value\"]; ok {\n\t\t\t\t\t\t\t\tif v, ok := i.(string); ok {\n\t\t\t\t\t\t\t\t\td, err := base64.StdEncoding.DecodeString(v)\n\t\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\t\treturn nil, err\n\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\tnodes[s] = stringutils.UnsafeString(d)\n\t\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\t\tnodes[s] = \"\"\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\tnodes[s] = \"\"\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\treturn nil, ErrInvalidKey{key: sd.nsEnd, val: s}\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\treturn nil, ErrNoKey\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\treturn nil, ErrNoKey\n\t\t}\n\t}\n\n\treturn\n}\n\nfunc (sd *Nginx) Nodes() (nodes []utils.KV, err error) {\n\tsd.url.Truncate(sd.pos)\n\tsd.url.WriteString(\"?recurse\")\n\n\tvar data []byte\n\n\tdata, err = utils.HttpGet(sd.url.String())\n\tif err != nil {\n\t\treturn\n\t}\n\n\tvar iNodes []interface{}\n\tif err = json.Unmarshal(data, &iNodes); err != nil {\n\t\treturn nil, err\n\t}\n\n\tnodes = make([]utils.KV, 0, 3)\n\n\tfor _, i := range iNodes {\n\t\tif jNode, ok := i.(map[string]interface{}); ok {\n\t\t\tif i, ok := jNode[\"Key\"]; ok {\n\t\t\t\tif s, ok := i.(string); ok {\n\t\t\t\t\tif strings.HasPrefix(s, sd.nsEnd) {\n\t\t\t\t\t\ts = s[len(sd.nsEnd):]\n\t\t\t\t\t\tkv := utils.KV{Key: s}\n\n\t\t\t\t\t\tif i, ok := jNode[\"Value\"]; ok {\n\t\t\t\t\t\t\tif v, ok := i.(string); ok {\n\t\t\t\t\t\t\t\td, err := base64.StdEncoding.DecodeString(v)\n\t\t\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\t\t\treturn nil, err\n\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\tkv.Value = stringutils.UnsafeString(d)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tif i, ok := jNode[\"Flags\"]; ok {\n\t\t\t\t\t\t\tswitch v := i.(type) {\n\t\t\t\t\t\t\tcase float64:\n\t\t\t\t\t\t\t\tkv.Flags = int64(v)\n\t\t\t\t\t\t\tcase int:\n\t\t\t\t\t\t\t\tkv.Flags = int64(v)\n\t\t\t\t\t\t\tcase int64:\n\t\t\t\t\t\t\t\tkv.Flags = v\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tnodes = append(nodes, kv)\n\t\t\t\t\t} else {\n\t\t\t\t\t\treturn nil, ErrInvalidKey{key: sd.nsEnd, val: s}\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\treturn nil, ErrNoKey\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\treturn nil, ErrNoKey\n\t\t}\n\t}\n\n\treturn\n}\n\nfunc (sd *Nginx) update(ip, port string, dc []string) (err error) {\n\tif len(dc) == 0 {\n\t\tsd.url.Truncate(sd.pos)\n\t\tsd.url.WriteString(\"_/\")\n\t\tsd.url.WriteString(sd.hostname)\n\t\tsd.url.WriteByte('/')\n\n\t\tif ip != \"\" {\n\t\t\tsd.url.WriteString(ip)\n\t\t}\n\n\t\tsd.url.WriteString(port)\n\n\t\t// add custom query flags\n\t\tsd.url.WriteByte('?')\n\t\tsd.url.WriteString(\"flags=\")\n\t\tsd.url.WriteInt(timeNow().Unix(), 10)\n\n\t\tif err = utils.HttpPut(sd.url.String(), sd.body); err != nil {\n\t\t\tsd.logger.Error(\"put\", zap.String(\"address\", sd.url.String()[sd.pos:]), zap.Error(err))\n\t\t\treturn\n\t\t}\n\t} else {\n\t\tflags := make([]byte, 0, 32)\n\t\tflags = append(flags, \"?flags=\"...)\n\t\tflags = strconv.AppendInt(flags, timeNow().Unix(), 10)\n\n\t\tfor i := 0; i < len(dc); i++ {\n\t\t\t// cfg.Common.SDDc\n\t\t\tsd.url.Truncate(sd.pos)\n\t\t\tsd.url.WriteString(dc[i])\n\t\t\tsd.url.WriteByte('/')\n\t\t\tsd.url.WriteString(sd.hostname)\n\t\t\tsd.url.WriteByte('/')\n\t\t\tn := sd.url.Len()\n\n\t\t\tif ip != \"\" {\n\t\t\t\tsd.url.WriteString(ip)\n\t\t\t}\n\n\t\t\tsd.url.WriteString(port)\n\n\t\t\t// add custom query flags\n\t\t\tsd.url.Write(flags)\n\n\t\t\tif i == 0 {\n\t\t\t\tif nErr := utils.HttpPut(sd.url.String(), sd.body); nErr != nil {\n\t\t\t\t\tsd.logger.Error(\n\t\t\t\t\t\t\"put\", zap.String(\"address\", sd.url.String()[n:]), zap.String(\"dc\", dc[i]), zap.Error(nErr),\n\t\t\t\t\t)\n\n\t\t\t\t\terr = nErr\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif nErr := utils.HttpPut(sd.url.String(), sd.backupBody); nErr != nil {\n\t\t\t\t\tsd.logger.Error(\n\t\t\t\t\t\t\"put\", zap.String(\"address\", sd.url.String()[n:]), zap.String(\"dc\", dc[i]), zap.Error(nErr),\n\t\t\t\t\t)\n\n\t\t\t\t\terr = nErr\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn\n}\n\nfunc (sd *Nginx) Update(ip, port string, dc []string, weight int64) error {\n\tsd.setWeight(weight)\n\n\treturn sd.update(ip, port, dc)\n}\n\nfunc (sd *Nginx) DeleteNode(node string) (err error) {\n\tsd.url.Truncate(sd.pos)\n\tsd.url.WriteString(node)\n\n\tif err = utils.HttpDelete(sd.url.String()); err != nil {\n\t\tsd.logger.Error(\"delete\", zap.String(\"address\", sd.url.String()[sd.pos:]), zap.Error(err))\n\t}\n\n\treturn\n}\n\nfunc (sd *Nginx) Delete(ip, port string, dc []string) (err error) {\n\tif len(dc) == 0 {\n\t\tsd.url.Truncate(sd.pos)\n\t\tsd.url.WriteString(\"_/\")\n\t\tsd.url.WriteString(sd.hostname)\n\t\tsd.url.WriteByte('/')\n\n\t\tif ip != \"\" {\n\t\t\tsd.url.WriteString(ip)\n\t\t}\n\n\t\tsd.url.WriteString(port)\n\n\t\tif err = utils.HttpDelete(sd.url.String()); err != nil {\n\t\t\tsd.logger.Error(\"delete\", zap.String(\"address\", sd.url.String()[sd.pos:]), zap.Error(err))\n\t\t}\n\t} else {\n\t\tfor i := 0; i < len(dc); i++ {\n\t\t\t// cfg.Common.SDDc\n\t\t\tsd.url.Truncate(sd.pos)\n\t\t\tsd.url.WriteString(dc[i])\n\t\t\tsd.url.WriteByte('/')\n\t\t\tsd.url.WriteString(sd.hostname)\n\t\t\tsd.url.WriteByte('/')\n\t\t\tn := sd.url.Len()\n\n\t\t\tif ip != \"\" {\n\t\t\t\tsd.url.WriteString(ip)\n\t\t\t}\n\n\t\t\tsd.url.WriteString(port)\n\n\t\t\tif nErr := utils.HttpDelete(sd.url.String()); nErr != nil {\n\t\t\t\tsd.logger.Error(\n\t\t\t\t\t\"delete\", zap.String(\"address\", sd.url.String()[n:]), zap.String(\"dc\", dc[i]), zap.Error(nErr),\n\t\t\t\t)\n\n\t\t\t\terr = nErr\n\t\t\t}\n\t\t}\n\t}\n\n\treturn\n}\n\nfunc (sd *Nginx) Clear(preserveIP, preservePort string) (err error) {\n\tvar nodes []string\n\n\tnodes, err = sd.List()\n\tif err != nil {\n\t\tsd.logger.Error(\n\t\t\t\"list\", zap.String(\"address\", sd.url.String()[sd.pos:]), zap.Error(err),\n\t\t)\n\n\t\treturn\n\t}\n\n\tif len(nodes) == 0 {\n\t\treturn\n\t}\n\n\tpreserveListen := preserveIP + preservePort\n\n\tsd.url.WriteByte('/')\n\n\tfor _, node := range nodes {\n\t\tsd.url.Truncate(sd.pos)\n\n\t\t_, host, listen, _ := splitNode(node)\n\t\tif host == sd.hostname && listen != preserveListen {\n\t\t\tsd.url.WriteString(node)\n\n\t\t\tif nErr := utils.HttpDelete(sd.url.String()); nErr != nil {\n\t\t\t\tsd.logger.Error(\n\t\t\t\t\t\"delete\", zap.String(\"address\", sd.url.String()), zap.Error(nErr),\n\t\t\t\t)\n\n\t\t\t\terr = nErr\n\t\t\t}\n\t\t}\n\t}\n\n\treturn\n}\n" }, { "path": "sd/nginx/nginx_test.go", "content": "//go:build test_sd\n// +build test_sd\n\npackage nginx\n\nimport (\n\t\"sort\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/sd/utils\"\n\t\"github.com/lomik/zapwriter\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nvar (\n\thostname1 = \"test_host1\"\n\tip1 = \"192.168.0.1\"\n\thostname2 = \"test_host2\"\n\tip2 = \"192.168.1.25\"\n\tport = \":9090\"\n\tdc1 = []string{\"dc1\", \"dc2\", \"dc3\"}\n\tdc2 = []string{\"dc2\", \"dc1\", \"dc3\"}\n\thostname3 = \"test_host3\"\n\n\tnilStringSlice []string\n)\n\nfunc TestNginx(t *testing.T) {\n\ttimeNow = func() time.Time {\n\t\treturn time.Unix(1682408721, 0)\n\t}\n\n\tlogger := zapwriter.Default()\n\n\tsd1 := New(\"http://127.0.0.1:8500/v1/kv/upstreams\", \"graphite\", hostname1, logger)\n\tsd2 := New(\"http://127.0.0.1:8500/v1/kv/upstreams\", \"\", hostname2, logger)\n\n\terr := sd1.Clear(\"\", \"\")\n\trequire.True(t, err == nil || err == utils.ErrNotFound, err)\n\terr = sd2.Clear(\"\", \"\")\n\trequire.True(t, err == nil || err == utils.ErrNotFound, err)\n\n\tnodes, err := sd1.List()\n\trequire.True(t, err == nil || err == utils.ErrNotFound, err)\n\tassert.Equal(t, nilStringSlice, nodes)\n\tnodes, err = sd2.List()\n\trequire.True(t, err == nil || err == utils.ErrNotFound, err)\n\tassert.Equal(t, nilStringSlice, nodes)\n\n\t// register new\n\trequire.NoError(t, sd1.Update(ip1, port, nil, 10))\n\tnodes, err = sd1.List()\n\trequire.NoError(t, err)\n\tsort.Strings(nodes)\n\tassert.Equal(\n\t\tt, []string{\n\t\t\t\"_/test_host1/192.168.0.1:9090\",\n\t\t}, nodes,\n\t)\n\tnodesMap, err := sd1.ListMap()\n\trequire.NoError(t, err)\n\tassert.Equal(\n\t\tt, map[string]string{\n\t\t\t\"_/test_host1/192.168.0.1:9090\": `{\"weight\":10,\"max_fails\":0}`,\n\t\t}, nodesMap,\n\t)\n\n\t// register new\n\trequire.NoError(t, sd2.Update(ip2, port, nil, 21))\n\tnodes, err = sd2.List()\n\tsort.Strings(nodes)\n\trequire.NoError(t, err)\n\tassert.Equal(\n\t\tt, []string{\n\t\t\t\"_/test_host2/192.168.1.25:9090\",\n\t\t},\n\t\tnodes,\n\t)\n\tnodesMap, err = sd2.ListMap()\n\trequire.NoError(t, err)\n\tassert.Equal(\n\t\tt, map[string]string{\n\t\t\t\"_/test_host2/192.168.1.25:9090\": `{\"weight\":21,\"max_fails\":0}`,\n\t\t}, nodesMap,\n\t)\n\n\t// update\n\trequire.NoError(t, sd2.Update(ip2, port, nil, 25))\n\tnodes, err = sd2.List()\n\tsort.Strings(nodes)\n\trequire.NoError(t, err)\n\tassert.Equal(\n\t\tt, []string{\n\t\t\t\"_/test_host2/192.168.1.25:9090\",\n\t\t},\n\t\tnodes,\n\t)\n\tnodesMap, err = sd2.ListMap()\n\trequire.NoError(t, err)\n\tassert.Equal(\n\t\tt, map[string]string{\n\t\t\t\"_/test_host2/192.168.1.25:9090\": `{\"weight\":25,\"max_fails\":0}`,\n\t\t}, nodesMap,\n\t)\n\n\t// delete\n\trequire.NoError(t, sd2.Delete(ip2, port, nil))\n\tnodes, err = sd2.List()\n\tsort.Strings(nodes)\n\trequire.NoError(t, err)\n\tassert.Equal(t, []string{}, nodes)\n\n\tnodesMap, err = sd1.ListMap()\n\trequire.NoError(t, err)\n\tassert.Equal(\n\t\tt, map[string]string{\n\t\t\t\"_/test_host1/192.168.0.1:9090\": `{\"weight\":10,\"max_fails\":0}`,\n\t\t}, nodesMap,\n\t)\n\n\t// cleanup\n\trequire.NoError(t, sd2.Update(ip2, port, nil, 25))\n\trequire.NoError(t, sd2.Update(ip1, port, nil, 25))\n\tnodesMap, err = sd2.ListMap()\n\trequire.NoError(t, err)\n\tassert.Equal(\n\t\tt, map[string]string{\n\t\t\t\"_/test_host2/192.168.1.25:9090\": `{\"weight\":25,\"max_fails\":0}`,\n\t\t\t\"_/test_host2/192.168.0.1:9090\": `{\"weight\":25,\"max_fails\":0}`,\n\t\t}, nodesMap,\n\t)\n\n\tnodesV, err := sd2.Nodes()\n\trequire.NoError(t, err)\n\tassert.Equal(\n\t\tt, []utils.KV{\n\t\t\t{Key: \"_/test_host1/192.168.0.1:9090\", Value: `{\"weight\":10,\"max_fails\":0}`, Flags: 1682408721},\n\t\t\t{Key: \"_/test_host2/192.168.0.1:9090\", Value: `{\"weight\":25,\"max_fails\":0}`, Flags: 1682408721},\n\t\t\t{Key: \"_/test_host2/192.168.1.25:9090\", Value: `{\"weight\":25,\"max_fails\":0}`, Flags: 1682408721},\n\t\t}, nodesV,\n\t)\n\n\trequire.NoError(t, sd2.Clear(ip2, port))\n\tnodesMap, err = sd2.ListMap()\n\trequire.NoError(t, err)\n\tassert.Equal(\n\t\tt, map[string]string{\n\t\t\t\"_/test_host2/192.168.1.25:9090\": `{\"weight\":25,\"max_fails\":0}`,\n\t\t}, nodesMap,\n\t)\n\n\t// clear all\n\trequire.NoError(t, sd1.Clear(\"\", \"\"))\n\tnodes, err = sd1.List()\n\trequire.NoError(t, err)\n\tassert.Equal(t, []string{}, nodes)\n\n\trequire.NoError(t, sd2.Clear(\"\", \"\"))\n\tnodes, err = sd2.List()\n\trequire.True(t, err == nil || err == utils.ErrNotFound, err)\n\tassert.Equal(t, nilStringSlice, nodes)\n}\n\nfunc TestNginxDC(t *testing.T) {\n\ttimeNow = func() time.Time {\n\t\treturn time.Unix(1682408721, 0)\n\t}\n\n\tlogger := zapwriter.Default()\n\n\tsd1 := New(\"http://127.0.0.1:8500/v1/kv/upstreams\", \"\", hostname1, logger)\n\tsd2 := New(\"http://127.0.0.1:8500/v1/kv/upstreams\", \"graphite\", hostname2, logger)\n\n\terr := sd1.Clear(\"\", \"\")\n\trequire.True(t, err == nil || err == utils.ErrNotFound, err)\n\terr = sd2.Clear(\"\", \"\")\n\trequire.True(t, err == nil || err == utils.ErrNotFound, err)\n\n\tnodes, err := sd1.List()\n\trequire.True(t, err == nil || err == utils.ErrNotFound, err)\n\tassert.Equal(t, nilStringSlice, nodes)\n\tnodes, err = sd2.List()\n\trequire.True(t, err == nil || err == utils.ErrNotFound, err)\n\tassert.Equal(t, nilStringSlice, nodes)\n\n\t// register new\n\trequire.NoError(t, sd1.Update(ip1, port, dc1, 10))\n\tnodes, err = sd1.List()\n\trequire.NoError(t, err)\n\tsort.Strings(nodes)\n\tassert.Equal(\n\t\tt, []string{\n\t\t\t\"dc1/test_host1/192.168.0.1:9090\",\n\t\t\t\"dc2/test_host1/192.168.0.1:9090\",\n\t\t\t\"dc3/test_host1/192.168.0.1:9090\",\n\t\t}, nodes,\n\t)\n\tnodesMap, err := sd1.ListMap()\n\trequire.NoError(t, err)\n\tassert.Equal(\n\t\tt, map[string]string{\n\t\t\t\"dc1/test_host1/192.168.0.1:9090\": `{\"weight\":10,\"max_fails\":0}`,\n\t\t\t\"dc2/test_host1/192.168.0.1:9090\": `{\"backup\":1,\"max_fails\":0}`,\n\t\t\t\"dc3/test_host1/192.168.0.1:9090\": `{\"backup\":1,\"max_fails\":0}`,\n\t\t}, nodesMap,\n\t)\n\n\t// register new\n\trequire.NoError(t, sd2.Update(ip2, port, dc2, 21))\n\tnodes, err = sd2.List()\n\tsort.Strings(nodes)\n\trequire.NoError(t, err)\n\tassert.Equal(\n\t\tt, []string{\n\t\t\t\"dc1/test_host2/192.168.1.25:9090\",\n\t\t\t\"dc2/test_host2/192.168.1.25:9090\",\n\t\t\t\"dc3/test_host2/192.168.1.25:9090\",\n\t\t},\n\t\tnodes,\n\t)\n\tnodesMap, err = sd2.ListMap()\n\trequire.NoError(t, err)\n\tassert.Equal(\n\t\tt, map[string]string{\n\t\t\t\"dc2/test_host2/192.168.1.25:9090\": `{\"weight\":21,\"max_fails\":0}`,\n\t\t\t\"dc1/test_host2/192.168.1.25:9090\": `{\"backup\":1,\"max_fails\":0}`,\n\t\t\t\"dc3/test_host2/192.168.1.25:9090\": `{\"backup\":1,\"max_fails\":0}`,\n\t\t}, nodesMap,\n\t)\n\n\t// update\n\trequire.NoError(t, sd2.Update(ip2, port, dc2, 25))\n\tnodes, err = sd2.List()\n\tsort.Strings(nodes)\n\trequire.NoError(t, err)\n\tassert.Equal(\n\t\tt, []string{\n\t\t\t\"dc1/test_host2/192.168.1.25:9090\",\n\t\t\t\"dc2/test_host2/192.168.1.25:9090\",\n\t\t\t\"dc3/test_host2/192.168.1.25:9090\",\n\t\t},\n\t\tnodes,\n\t)\n\tnodesMap, err = sd2.ListMap()\n\trequire.NoError(t, err)\n\tassert.Equal(\n\t\tt, map[string]string{\n\t\t\t\"dc2/test_host2/192.168.1.25:9090\": `{\"weight\":25,\"max_fails\":0}`,\n\t\t\t\"dc1/test_host2/192.168.1.25:9090\": `{\"backup\":1,\"max_fails\":0}`,\n\t\t\t\"dc3/test_host2/192.168.1.25:9090\": `{\"backup\":1,\"max_fails\":0}`,\n\t\t}, nodesMap,\n\t)\n\n\t// delete\n\trequire.NoError(t, sd2.Delete(ip2, port, dc2))\n\tnodes, err = sd2.List()\n\tsort.Strings(nodes)\n\trequire.NoError(t, err)\n\tassert.Equal(t, []string{}, nodes)\n\n\tnodesMap, err = sd1.ListMap()\n\trequire.NoError(t, err)\n\tassert.Equal(\n\t\tt, map[string]string{\n\t\t\t\"dc1/test_host1/192.168.0.1:9090\": `{\"weight\":10,\"max_fails\":0}`,\n\t\t\t\"dc2/test_host1/192.168.0.1:9090\": `{\"backup\":1,\"max_fails\":0}`,\n\t\t\t\"dc3/test_host1/192.168.0.1:9090\": `{\"backup\":1,\"max_fails\":0}`,\n\t\t}, nodesMap,\n\t)\n\n\t// cleanup\n\trequire.NoError(t, sd2.Update(ip2, port, dc2, 25))\n\trequire.NoError(t, sd2.Update(ip1, port, dc2, 25))\n\tnodesMap, err = sd2.ListMap()\n\trequire.NoError(t, err)\n\tassert.Equal(\n\t\tt, map[string]string{\n\t\t\t\"dc2/test_host2/192.168.1.25:9090\": `{\"weight\":25,\"max_fails\":0}`,\n\t\t\t\"dc1/test_host2/192.168.1.25:9090\": `{\"backup\":1,\"max_fails\":0}`,\n\t\t\t\"dc3/test_host2/192.168.1.25:9090\": `{\"backup\":1,\"max_fails\":0}`,\n\t\t\t\"dc2/test_host2/192.168.0.1:9090\": `{\"weight\":25,\"max_fails\":0}`,\n\t\t\t\"dc1/test_host2/192.168.0.1:9090\": `{\"backup\":1,\"max_fails\":0}`,\n\t\t\t\"dc3/test_host2/192.168.0.1:9090\": `{\"backup\":1,\"max_fails\":0}`,\n\t\t}, nodesMap,\n\t)\n\n\tnodesV, err := sd2.Nodes()\n\trequire.NoError(t, err)\n\tassert.Equal(\n\t\tt, []utils.KV{\n\t\t\t{Key: \"dc1/test_host1/192.168.0.1:9090\", Value: `{\"weight\":10,\"max_fails\":0}`, Flags: 1682408721},\n\t\t\t{Key: \"dc1/test_host2/192.168.0.1:9090\", Value: `{\"backup\":1,\"max_fails\":0}`, Flags: 1682408721},\n\t\t\t{Key: \"dc1/test_host2/192.168.1.25:9090\", Value: `{\"backup\":1,\"max_fails\":0}`, Flags: 1682408721},\n\t\t\t{Key: \"dc2/test_host1/192.168.0.1:9090\", Value: `{\"backup\":1,\"max_fails\":0}`, Flags: 1682408721},\n\t\t\t{Key: \"dc2/test_host2/192.168.0.1:9090\", Value: `{\"weight\":25,\"max_fails\":0}`, Flags: 1682408721},\n\t\t\t{Key: \"dc2/test_host2/192.168.1.25:9090\", Value: `{\"weight\":25,\"max_fails\":0}`, Flags: 1682408721},\n\t\t\t{Key: \"dc3/test_host1/192.168.0.1:9090\", Value: `{\"backup\":1,\"max_fails\":0}`, Flags: 1682408721},\n\t\t\t{Key: \"dc3/test_host2/192.168.0.1:9090\", Value: `{\"backup\":1,\"max_fails\":0}`, Flags: 1682408721},\n\t\t\t{Key: \"dc3/test_host2/192.168.1.25:9090\", Value: `{\"backup\":1,\"max_fails\":0}`, Flags: 1682408721},\n\t\t}, nodesV,\n\t)\n\n\trequire.NoError(t, sd2.Clear(ip2, port))\n\tnodesMap, err = sd2.ListMap()\n\trequire.NoError(t, err)\n\tassert.Equal(\n\t\tt, map[string]string{\n\t\t\t\"dc2/test_host2/192.168.1.25:9090\": `{\"weight\":25,\"max_fails\":0}`,\n\t\t\t\"dc1/test_host2/192.168.1.25:9090\": `{\"backup\":1,\"max_fails\":0}`,\n\t\t\t\"dc3/test_host2/192.168.1.25:9090\": `{\"backup\":1,\"max_fails\":0}`,\n\t\t}, nodesMap,\n\t)\n\n\t// clear all\n\trequire.NoError(t, sd1.Clear(\"\", \"\"))\n\tnodes, err = sd1.List()\n\trequire.NoError(t, err)\n\tassert.Equal(t, []string{}, nodes)\n\n\trequire.NoError(t, sd2.Clear(\"\", \"\"))\n\tnodes, err = sd2.List()\n\tassert.Equal(t, nilStringSlice, nodes)\n\tassert.Equal(t, nilStringSlice, nodes)\n}\n" }, { "path": "sd/nginx/tests/nginx_cleanup_test.go", "content": "//go:build test_sd\n// +build test_sd\n\npackage nginx_test\n\nimport (\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/sd\"\n\t\"github.com/lomik/graphite-clickhouse/sd/utils\"\n\t\"github.com/lomik/zapwriter\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nvar (\n\thostname1 = \"test_host1\"\n\tip1 = \"192.168.0.1\"\n\thostname2 = \"test_host2\"\n\tip2 = \"192.168.1.25\"\n\tport = \":9090\"\n\tdc1 = []string{\"dc1\", \"dc2\", \"dc3\"}\n\tdc2 = []string{\"dc2\", \"dc1\", \"dc3\"}\n\thostname3 = \"test_host3\"\n\n\tnilStringSlice []string\n)\n\nfunc cleanup(nodes []utils.KV, start, end int64) {\n\tfor i := range nodes {\n\t\tif nodes[i].Flags >= start && nodes[i].Flags <= end {\n\t\t\tnodes[i].Flags = start\n\t\t}\n\t}\n}\n\nfunc TestNginxExpire(t *testing.T) {\n\tlogger := zapwriter.Default()\n\n\tcfg := &config.Common{\n\t\tSDType: config.SDNginx,\n\t\tSD: \"http://127.0.0.1:8500/v1/kv/upstreams\",\n\t\tSDNamespace: \"graphite\", //default\n\t\tSDExpire: time.Second * 5,\n\t}\n\n\tsd1, _ := sd.New(cfg, hostname1, logger)\n\tsd2, _ := sd.New(cfg, hostname2, logger)\n\n\terr := sd1.Clear(\"\", \"\")\n\trequire.True(t, err == nil || err == utils.ErrNotFound, err)\n\terr = sd2.Clear(\"\", \"\")\n\trequire.True(t, err == nil || err == utils.ErrNotFound, err)\n\n\tnodes, err := sd1.List()\n\trequire.True(t, err == nil || err == utils.ErrNotFound, err)\n\tassert.Equal(t, nilStringSlice, nodes)\n\tnodes, err = sd2.List()\n\trequire.True(t, err == nil || err == utils.ErrNotFound, err)\n\tassert.Equal(t, nilStringSlice, nodes)\n\n\t// check cleanup expired\n\tstart := time.Now().Unix()\n\trequire.NoError(t, sd1.Update(ip1, port, nil, 10))\n\ttime.Sleep(cfg.SDExpire + time.Second)\n\trequire.NoError(t, sd2.Update(ip2, port, nil, 10))\n\tnodesV, err := sd1.Nodes()\n\tend := time.Now().Unix()\n\trequire.NoError(t, err)\n\t// reset timestamp for compare\n\tcleanup(nodesV, start, end)\n\tassert.Equal(\n\t\tt,\n\t\t[]utils.KV{\n\t\t\t{Key: \"_/test_host1/192.168.0.1:9090\", Value: \"{\\\"weight\\\":10,\\\"max_fails\\\":0}\", Flags: start},\n\t\t\t{Key: \"_/test_host2/192.168.1.25:9090\", Value: \"{\\\"weight\\\":10,\\\"max_fails\\\":0}\", Flags: start},\n\t\t},\n\t\tnodesV,\n\t\t\"start = %d, end = %d\", start, end,\n\t)\n\n\tsd.Cleanup(cfg, sd1, false)\n\tnodesV, err = sd1.Nodes()\n\trequire.NoError(t, err)\n\t// reset timestamp for compare\n\tcleanup(nodesV, start, end)\n\tassert.Equal(\n\t\tt,\n\t\t[]utils.KV{\n\t\t\t{Key: \"_/test_host2/192.168.1.25:9090\", Value: \"{\\\"weight\\\":10,\\\"max_fails\\\":0}\", Flags: start},\n\t\t},\n\t\tnodesV,\n\t\t\"start = %d, end = %d\", start, end,\n\t)\n}\n\nfunc TestNginxExpireDC(t *testing.T) {\n\tlogger := zapwriter.Default()\n\n\tcfg1 := &config.Common{\n\t\tSDType: config.SDNginx,\n\t\tSD: \"http://127.0.0.1:8500/v1/kv/upstreams\",\n\t\tSDNamespace: \"graphite\", //default\n\t\tSDDc: dc1,\n\t\tSDExpire: time.Second * 5,\n\t}\n\tsd1, _ := sd.New(cfg1, hostname1, logger)\n\n\tcfg2 := &config.Common{\n\t\tSDType: config.SDNginx,\n\t\tSD: \"http://127.0.0.1:8500/v1/kv/upstreams\",\n\t\tSDNamespace: \"\", //default\n\t\tSDDc: dc2,\n\t\tSDExpire: time.Second * 5,\n\t}\n\tsd2, _ := sd.New(cfg2, hostname2, logger)\n\n\terr := sd1.Clear(\"\", \"\")\n\trequire.True(t, err == nil || err == utils.ErrNotFound, err)\n\terr = sd2.Clear(\"\", \"\")\n\trequire.True(t, err == nil || err == utils.ErrNotFound, err)\n\n\tnodes, err := sd1.List()\n\trequire.True(t, err == nil || err == utils.ErrNotFound, err)\n\tassert.Equal(t, nilStringSlice, nodes)\n\tnodes, err = sd2.List()\n\trequire.True(t, err == nil || err == utils.ErrNotFound, err)\n\tassert.Equal(t, nilStringSlice, nodes)\n\n\t// check cleanup expired\n\tstart := time.Now().Unix()\n\trequire.NoError(t, sd1.Update(ip1, port, dc1, 10))\n\ttime.Sleep(cfg1.SDExpire + time.Second)\n\trequire.NoError(t, sd2.Update(ip2, port, dc2, 10))\n\tnodesV, err := sd1.Nodes()\n\tend := time.Now().Unix()\n\trequire.NoError(t, err)\n\t// reset timestamp for compare\n\tcleanup(nodesV, start, end)\n\tassert.Equal(\n\t\tt,\n\t\t[]utils.KV{\n\t\t\t{Key: \"dc1/test_host1/192.168.0.1:9090\", Value: \"{\\\"weight\\\":10,\\\"max_fails\\\":0}\", Flags: start},\n\t\t\t{Key: \"dc1/test_host2/192.168.1.25:9090\", Value: \"{\\\"backup\\\":1,\\\"max_fails\\\":0}\", Flags: start},\n\t\t\t{Key: \"dc2/test_host1/192.168.0.1:9090\", Value: \"{\\\"backup\\\":1,\\\"max_fails\\\":0}\", Flags: start},\n\t\t\t{Key: \"dc2/test_host2/192.168.1.25:9090\", Value: \"{\\\"weight\\\":10,\\\"max_fails\\\":0}\", Flags: start},\n\t\t\t{Key: \"dc3/test_host1/192.168.0.1:9090\", Value: \"{\\\"backup\\\":1,\\\"max_fails\\\":0}\", Flags: start},\n\t\t\t{Key: \"dc3/test_host2/192.168.1.25:9090\", Value: \"{\\\"backup\\\":1,\\\"max_fails\\\":0}\", Flags: start},\n\t\t},\n\t\tnodesV,\n\t\t\"start = %d, end = %d\", start, end,\n\t)\n\n\tsd.Cleanup(cfg1, sd1, false)\n\tnodesV, err = sd1.Nodes()\n\trequire.NoError(t, err)\n\t// reset timestamp for compare\n\tcleanup(nodesV, start, end)\n\tassert.Equal(\n\t\tt,\n\t\t[]utils.KV{\n\t\t\t{Key: \"dc1/test_host2/192.168.1.25:9090\", Value: \"{\\\"backup\\\":1,\\\"max_fails\\\":0}\", Flags: start},\n\t\t\t{Key: \"dc2/test_host2/192.168.1.25:9090\", Value: \"{\\\"weight\\\":10,\\\"max_fails\\\":0}\", Flags: start},\n\t\t\t{Key: \"dc3/test_host2/192.168.1.25:9090\", Value: \"{\\\"backup\\\":1,\\\"max_fails\\\":0}\", Flags: start},\n\t\t},\n\t\tnodesV,\n\t\t\"start = %d, end = %d\", start, end,\n\t)\n}\n" }, { "path": "sd/register.go", "content": "package sd\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/load_avg\"\n\t\"github.com/lomik/graphite-clickhouse/sd/nginx\"\n\t\"github.com/lomik/graphite-clickhouse/sd/utils\"\n\t\"go.uber.org/zap\"\n)\n\nvar (\n\t// ctxMain, Stop = context.WithCancel(context.Background())\n\tstop chan struct{} = make(chan struct{}, 1)\n\tdelay = time.Second * 10\n\thostname string\n)\n\ntype SD interface {\n\t// Update update node record\n\tUpdate(listenIP, listenPort string, dc []string, weight int64) error\n\t// Delete delete node record (with ip/port/dcs)\n\tDelete(ip, port string, dcs []string) error\n\t// Delete delete node record\n\tDeleteNode(node string) (err error)\n\t// Clear clear node record (all except with current listen IP/port)\n\tClear(listenIP, listenPort string) error\n\t// Nodes return all registered nodes (for all hostnames in namespace)\n\tNodes() (nodes []utils.KV, err error)\n\t// List return all registered nodes for hostname\n\tList() (nodes []string, err error)\n\t// Namespace return namespace\n\tNamespace() string\n}\n\nfunc New(cfg *config.Common, hostname string, logger *zap.Logger) (SD, error) {\n\tswitch cfg.SDType {\n\tcase config.SDNginx:\n\t\tsd := nginx.New(cfg.SD, cfg.SDNamespace, hostname, logger)\n\t\treturn sd, nil\n\tdefault:\n\t\treturn nil, errors.New(\"serive discovery type not registered\")\n\t}\n}\n\nfunc Register(cfg *config.Common, logger *zap.Logger) {\n\tvar (\n\t\tlistenIP string\n\t\tprevIP string\n\t\tregisterFirst bool\n\t\tsd SD\n\t\terr error\n\t\tload float64\n\t\tw int64\n\t)\n\n\tif cfg.SD != \"\" {\n\t\tif strings.HasPrefix(cfg.Listen, \":\") {\n\t\t\tregisterFirst = true\n\t\t\tlistenIP = utils.GetLocalIP()\n\t\t\tprevIP = listenIP\n\t\t}\n\n\t\thostname, _ = os.Hostname()\n\t\thostname, _, _ = strings.Cut(hostname, \".\")\n\n\t\tsd, err = New(cfg, hostname, logger)\n\t\tif err != nil {\n\t\t\tpanic(\"serive discovery type not registered\")\n\t\t}\n\n\t\tload, err = load_avg.Normalized()\n\t\tif err == nil {\n\t\t\tload_avg.Store(load)\n\t\t}\n\n\t\tlogger.Info(\"init sd\",\n\t\t\tzap.String(\"hostname\", hostname),\n\t\t)\n\n\t\tw = load_avg.Weight(cfg.BaseWeight, cfg.DegragedMultiply, cfg.DegragedLoad, load)\n\t\tsd.Update(listenIP, cfg.Listen, cfg.SDDc, w)\n\t\tsd.Clear(listenIP, cfg.Listen)\n\t}\nLOOP:\n\tfor {\n\t\tload, err = load_avg.Normalized()\n\t\tif err == nil {\n\t\t\tload_avg.Store(load)\n\t\t}\n\t\tif sd != nil {\n\t\t\tw = load_avg.Weight(cfg.BaseWeight, cfg.DegragedMultiply, cfg.DegragedLoad, load)\n\n\t\t\tif registerFirst {\n\t\t\t\t// if listen on all ip, try to register with first ip\n\t\t\t\tlistenIP = utils.GetLocalIP()\n\t\t\t}\n\n\t\t\tsd.Update(listenIP, cfg.Listen, cfg.SDDc, w)\n\n\t\t\tif prevIP != listenIP {\n\t\t\t\tsd.Delete(prevIP, cfg.Listen, cfg.SDDc)\n\t\t\t\tprevIP = listenIP\n\t\t\t}\n\t\t}\n\t\tt := time.After(delay)\n\t\tselect {\n\t\tcase <-t:\n\t\t\tcontinue\n\t\tcase <-stop:\n\t\t\tbreak LOOP\n\t\t}\n\t}\n\n\tif sd != nil {\n\t\tif err := sd.Clear(\"\", \"\"); err == nil {\n\t\t\tlogger.Info(\"cleanup sd\",\n\t\t\t\tzap.String(\"hostname\", hostname),\n\t\t\t)\n\t\t} else {\n\t\t\tlogger.Warn(\"cleanup sd\",\n\t\t\t\tzap.String(\"hostname\", hostname),\n\t\t\t\tzap.Error(err),\n\t\t\t)\n\t\t}\n\t}\n}\n\nfunc Stop() {\n\tstop <- struct{}{}\n}\n\nfunc Cleanup(cfg *config.Common, sd SD, checkOnly bool) error {\n\tif cfg.SD != \"\" && cfg.SDExpire > 0 {\n\t\tts := time.Now().Unix() - int64(cfg.SDExpire.Seconds())\n\n\t\tif nodes, err := sd.Nodes(); err == nil {\n\t\t\tfor _, node := range nodes {\n\t\t\t\tif node.Flags > 0 {\n\t\t\t\t\tif ts > node.Flags {\n\t\t\t\t\t\tif checkOnly {\n\t\t\t\t\t\t\tfmt.Printf(\"%s: %s (%s), expired\\n\", node.Key, node.Value, time.Unix(node.Flags, 0).UTC().Format(time.RFC3339Nano))\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tif err = sd.DeleteNode(node.Key); err != nil {\n\t\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tfmt.Printf(\"%s: %s (%s), deleted\\n\", node.Key, node.Value, time.Unix(node.Flags, 0).UTC().Format(time.RFC3339Nano))\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tfmt.Printf(\"%s: %s (%s)\\n\", node.Key, node.Value, time.Unix(node.Flags, 0).UTC().Format(time.RFC3339Nano))\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n" }, { "path": "sd/utils/utils.go", "content": "package utils\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/errs\"\n)\n\nvar ErrNotFound = errors.New(\"entry not found\")\n\ntype KV struct {\n\tKey string\n\tValue string\n\tFlags int64\n}\n\nfunc HttpGet(url string) ([]byte, error) {\n\tclient := &http.Client{Timeout: 2 * time.Second}\n\n\tresp, err := client.Get(url)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tdata, err := io.ReadAll(resp.Body)\n\tresp.Body.Close()\n\n\tif resp.StatusCode == http.StatusNotFound {\n\t\treturn nil, ErrNotFound\n\t}\n\n\tif resp.StatusCode != http.StatusOK {\n\t\treturn nil, errs.NewErrorWithCode(string(data), resp.StatusCode)\n\t}\n\n\treturn data, err\n}\n\nfunc HttpPut(url string, body []byte) error {\n\treq, err := http.NewRequest(http.MethodPut, url, bytes.NewBuffer(body))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treq.Header.Set(\"Content-Type\", \"application/json\")\n\n\tclient := &http.Client{Timeout: 2 * time.Second}\n\n\tresp, err := client.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode == http.StatusNotFound {\n\t\treturn ErrNotFound\n\t}\n\n\tif resp.StatusCode != http.StatusOK {\n\t\tdata, _ := io.ReadAll(resp.Body)\n\t\treturn errs.NewErrorWithCode(string(data), resp.StatusCode)\n\t}\n\n\treturn nil\n}\n\nfunc HttpDelete(url string) error {\n\treq, err := http.NewRequest(http.MethodDelete, url, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tclient := &http.Client{Timeout: 2 * time.Second}\n\n\tresp, err := client.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode == http.StatusNotFound {\n\t\treturn ErrNotFound\n\t}\n\n\tif resp.StatusCode != http.StatusOK {\n\t\tdata, _ := io.ReadAll(resp.Body)\n\t\treturn errs.NewErrorWithCode(string(data), resp.StatusCode)\n\t}\n\n\treturn nil\n}\n\n// GetLocalIP returns the non loopback local IP of the host\nfunc GetLocalIP() string {\n\taddrs, err := net.InterfaceAddrs()\n\tif err != nil {\n\t\treturn \"\"\n\t}\n\n\tfor _, address := range addrs {\n\t\t// check the address type and if it is not a loopback the display it\n\t\tif ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {\n\t\t\tif ipnet.IP.To4() != nil {\n\t\t\t\treturn ipnet.IP.String()\n\t\t\t}\n\t\t}\n\t}\n\n\treturn \"\"\n}\n" }, { "path": "tagger/metric.go", "content": "package tagger\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\n\t\"github.com/lomik/graphite-clickhouse/pkg/dry\"\n)\n\ntype Metric struct {\n\tPath []byte\n\tLevel int\n\tParentIndex int\n\tTags *Set\n}\n\nfunc (m *Metric) ParentPath() []byte {\n\tif len(m.Path) == 0 {\n\t\treturn nil\n\t}\n\n\tindex := bytes.LastIndexByte(m.Path[:len(m.Path)-1], '.')\n\tif index < 0 {\n\t\treturn nil\n\t}\n\n\treturn m.Path[:index+1]\n}\n\nfunc (m *Metric) IsLeaf() uint8 {\n\tif len(m.Path) > 0 && m.Path[len(m.Path)-1] == '.' {\n\t\treturn 0\n\t}\n\n\treturn 1\n}\n\nfunc (m *Metric) MarshalJSON() ([]byte, error) {\n\treturn json.Marshal(map[string]interface{}{\n\t\t\"Path\": dry.UnsafeString(m.Path),\n\t\t\"Level\": m.Level,\n\t\t\"Tags\": m.Tags,\n\t\t\"IsLeaf\": m.IsLeaf(),\n\t})\n}\n" }, { "path": "tagger/rule.go", "content": "package tagger\n\nimport (\n\t\"bytes\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"regexp\"\n\n\t\"github.com/BurntSushi/toml\"\n)\n\ntype Rule struct {\n\tSingle string `toml:\"tag\"`\n\tList []string `toml:\"tags\"`\n\tre *regexp.Regexp `toml:\"-\"`\n\tEqual string `toml:\"equal\"`\n\tHasPrefix string `toml:\"has-prefix\"`\n\tHasSuffix string `toml:\"has-suffix\"`\n\tContains string `toml:\"contains\"`\n\tRegexp string `toml:\"regexp\"`\n\tBytesEqual []byte `toml:\"-\"`\n\tBytesHasPrefix []byte `toml:\"-\"`\n\tBytesHasSuffix []byte `toml:\"-\"`\n\tBytesContains []byte `toml:\"-\"`\n\tTags *Set `toml:\"-\"`\n}\n\ntype Rules struct {\n\tRule []Rule `toml:\"rule\"`\n\tprefix *Tree `toml:\"-\"`\n\tsuffix *Tree `toml:\"-\"` // @TODO\n\tcontains *Tree `toml:\"-\"`\n\tother []*Rule `toml:\"-\"`\n}\n\nfunc ParseFile(filename string) (*Rules, error) {\n\tc, err := os.ReadFile(filename)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn Parse(string(c))\n}\n\nfunc ParseGlob(glob string) (*Rules, error) {\n\tcontent := []byte{}\n\n\tfiles, err := filepath.Glob(glob)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfor i := 0; i < len(files); i++ {\n\t\tc, err := os.ReadFile(files[i])\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tcontent = append(append(content, '\\n'), c...)\n\t}\n\n\treturn Parse(string(content))\n}\n\nfunc Parse(content string) (*Rules, error) {\n\trules := &Rules{\n\t\tprefix: &Tree{},\n\t\tsuffix: &Tree{},\n\t\tcontains: &Tree{},\n\t\tother: make([]*Rule, 0),\n\t}\n\n\tif _, err := toml.Decode(content, rules); err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar err error\n\n\tfor i := 0; i < len(rules.Rule); i++ {\n\t\trule := &rules.Rule[i]\n\t\trule.Tags = EmptySet\n\n\t\tif rule.Single != \"\" {\n\t\t\trule.Tags = rule.Tags.Add(rule.Single)\n\t\t}\n\n\t\tif rule.List != nil {\n\t\t\trule.Tags = rule.Tags.Add(rule.List...)\n\t\t}\n\n\t\t// compile and check regexp\n\t\trule.re, err = regexp.Compile(rule.Regexp)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tif rule.Equal != \"\" {\n\t\t\trule.BytesEqual = []byte(rule.Equal)\n\t\t}\n\n\t\tif rule.Contains != \"\" {\n\t\t\trule.BytesContains = []byte(rule.Contains)\n\t\t}\n\n\t\tif rule.HasPrefix != \"\" {\n\t\t\trule.BytesHasPrefix = []byte(rule.HasPrefix)\n\t\t}\n\n\t\tif rule.HasSuffix != \"\" {\n\t\t\trule.BytesHasSuffix = []byte(rule.HasSuffix)\n\t\t}\n\n\t\tif rule.BytesHasPrefix != nil {\n\t\t\trules.prefix.Add(rule.BytesHasPrefix, rule)\n\t\t} else if rule.BytesEqual != nil {\n\t\t\trules.prefix.Add(rule.BytesEqual, rule)\n\t\t} else if rule.BytesContains != nil {\n\t\t\trules.contains.Add(rule.BytesContains, rule)\n\t\t} else if rule.BytesHasSuffix != nil {\n\t\t\trules.suffix.AddSuffix(rule.BytesHasSuffix, rule)\n\t\t} else {\n\t\t\trules.other = append(rules.other, rule)\n\t\t}\n\t}\n\n\treturn rules, nil\n}\n\nfunc (r *Rules) Match(m *Metric) {\n\tr.matchPrefix(m)\n\tr.matchSuffix(m)\n\tr.matchContains(m)\n\tr.matchOther(m)\n}\n\nfunc matchByPrefix(path []byte, tree *Tree, m *Metric) {\n\tx := tree\n\ti := 0\n\n\tfor {\n\t\tif i >= len(path) {\n\t\t\tbreak\n\t\t}\n\n\t\tx = x.Next[path[i]]\n\t\tif x == nil {\n\t\t\tbreak\n\t\t}\n\n\t\tif x.Rules != nil {\n\t\t\tfor _, rule := range x.Rules {\n\t\t\t\trule.Match(m)\n\t\t\t}\n\t\t}\n\n\t\ti++\n\t}\n}\n\nfunc matchBySuffix(path []byte, tree *Tree, m *Metric) {\n\tx := tree\n\ti := len(path) - 1\n\n\tfor {\n\t\tif i <= 0 {\n\t\t\tbreak\n\t\t}\n\n\t\tx = x.Next[path[i]]\n\t\tif x == nil {\n\t\t\tbreak\n\t\t}\n\n\t\tif x.Rules != nil {\n\t\t\tfor _, rule := range x.Rules {\n\t\t\t\trule.Match(m)\n\t\t\t}\n\t\t}\n\n\t\ti--\n\t}\n}\n\nfunc (r *Rules) matchPrefix(m *Metric) {\n\tmatchByPrefix(m.Path, r.prefix, m)\n}\n\nfunc (r *Rules) matchSuffix(m *Metric) {\n\tmatchBySuffix(m.Path, r.suffix, m)\n}\n\nfunc (r *Rules) matchContains(m *Metric) {\n\tfor i := 0; i < len(m.Path); i++ {\n\t\tmatchByPrefix(m.Path[i:], r.contains, m)\n\t}\n}\n\nfunc (r *Rules) matchOther(m *Metric) {\n\tfor _, rule := range r.other {\n\t\trule.Match(m)\n\t}\n}\n\nfunc (r *Rule) Match(m *Metric) {\n\tif r.BytesEqual != nil && !bytes.Equal(m.Path, r.BytesEqual) {\n\t\treturn\n\t}\n\n\tif r.BytesHasPrefix != nil && !bytes.HasPrefix(m.Path, r.BytesHasPrefix) {\n\t\treturn\n\t}\n\n\tif r.BytesHasSuffix != nil && !bytes.HasSuffix(m.Path, r.BytesHasSuffix) {\n\t\treturn\n\t}\n\n\tif r.BytesContains != nil && !bytes.Contains(m.Path, r.BytesContains) {\n\t\treturn\n\t}\n\n\tif r.re != nil && !r.re.Match(m.Path) {\n\t\treturn\n\t}\n\n\tm.Tags = m.Tags.Merge(r.Tags)\n}\n" }, { "path": "tagger/rule_test.go", "content": "package tagger\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nvar RulesConf = `\n[[rule]]\ntag = \"prefix\"\nhas-prefix = \"prefix\"\n\n[[rule]]\ntag = \"suffix\"\nhas-suffix = \"suffix\"\n\n[[rule]]\ntag = \"contains\"\ncontains = \"contains\"\n\n[[rule]]\ntag = \"equal\"\nequal = \"equal\"\n\n[[rule]]\ntag = \"regexp\"\nregexp = \"reg[e]xp\"\n`\n\nfunc TestRules(t *testing.T) {\n\tassert := assert.New(t)\n\trules, err := Parse(RulesConf)\n\n\tassert.NoError(err)\n\n\ttable := []struct {\n\t\tpath string\n\t\tmethod string // \"\" for all, \"prefix\", \"suffix\", \"contains\" for use only specified tree\n\t\texpectedTags []string\n\t}{\n\t\t{\"prefix.metric\", \"\", []string{\"prefix\"}},\n\t\t{\"prefix.metric\", \"prefix\", []string{\"prefix\"}},\n\t\t{\"prefix.metric\", \"suffix\", nil},\n\t\t{\"prefix.metric\", \"contains\", nil},\n\t\t{\"prefix.metric\", \"other\", nil},\n\n\t\t{\"metric.suffix\", \"\", []string{\"suffix\"}},\n\t\t{\"metric.suffix\", \"prefix\", nil},\n\t\t{\"metric.suffix\", \"suffix\", []string{\"suffix\"}},\n\t\t{\"metric.suffix\", \"contains\", nil},\n\t\t{\"metric.suffix\", \"other\", nil},\n\n\t\t{\"hello.contains.world\", \"\", []string{\"contains\"}},\n\t\t{\"hello.contains.world\", \"prefix\", nil},\n\t\t{\"hello.contains.world\", \"suffix\", nil},\n\t\t{\"hello.contains.world\", \"contains\", []string{\"contains\"}},\n\t\t{\"hello.contains.world\", \"other\", nil},\n\n\t\t{\"hello.regexp.world\", \"\", []string{\"regexp\"}},\n\t\t{\"hello.regexp.world\", \"prefix\", nil},\n\t\t{\"hello.regexp.world\", \"suffix\", nil},\n\t\t{\"hello.regexp.world\", \"contains\", nil},\n\t\t{\"hello.regexp.world\", \"other\", []string{\"regexp\"}},\n\n\t\t{\"prefix.suffix\", \"\", []string{\"prefix\", \"suffix\"}},\n\t}\n\n\tfor i := 0; i < len(table); i++ {\n\t\tt := table[i]\n\n\t\tm := Metric{Path: []byte(t.path), Tags: EmptySet}\n\n\t\tswitch t.method {\n\t\tcase \"\":\n\t\t\trules.Match(&m)\n\t\tcase \"prefix\":\n\t\t\trules.matchPrefix(&m)\n\t\tcase \"suffix\":\n\t\t\trules.matchSuffix(&m)\n\t\tcase \"contains\":\n\t\t\trules.matchContains(&m)\n\t\tcase \"other\":\n\t\t\trules.matchOther(&m)\n\t\t}\n\n\t\texpected := t.expectedTags\n\t\tif expected == nil {\n\t\t\texpected = []string{}\n\t\t}\n\n\t\tsort.Strings(expected)\n\n\t\ttags := m.Tags.List()\n\t\tsort.Strings(tags)\n\n\t\tassert.Equal(expected, tags, fmt.Sprintf(\"path: %s, method: %s\", t.path, t.method))\n\t}\n}\n" }, { "path": "tagger/set.go", "content": "package tagger\n\nimport (\n\t\"encoding/json\"\n)\n\n// set with copy-on-write\ntype Set struct {\n\tdata map[string]bool\n\tlist []string\n\tjson []byte\n}\n\nvar EmptySet = &Set{\n\tdata: make(map[string]bool),\n\tlist: make([]string, 0),\n}\n\nfunc (s *Set) Add(tag ...string) *Set {\n\tvar newList []string\n\n\tfor _, t := range tag {\n\t\tif !s.data[t] {\n\t\t\tif newList == nil {\n\t\t\t\tnewList = make([]string, len(s.list)+1)\n\t\t\t\tcopy(newList, s.list)\n\t\t\t\tnewList[len(newList)-1] = t\n\t\t\t} else {\n\t\t\t\tnewList = append(newList, t)\n\t\t\t}\n\t\t}\n\t}\n\n\t// no new tags\n\tif newList == nil {\n\t\treturn s\n\t}\n\n\t// new tag\n\tn := &Set{\n\t\tdata: make(map[string]bool),\n\t\tlist: newList,\n\t}\n\n\tfor _, t := range n.list {\n\t\tn.data[t] = true\n\t}\n\n\treturn n\n}\n\nfunc (s *Set) Merge(other *Set) *Set {\n\treturn s.Add(other.list...)\n}\n\nfunc (s *Set) Len() int {\n\treturn len(s.list)\n}\n\nfunc (s *Set) List() []string {\n\treturn s.list\n}\n\nfunc (s *Set) MarshalJSON() ([]byte, error) {\n\tif s.json != nil {\n\t\treturn s.json, nil\n\t}\n\n\tvar err error\n\n\ts.json, err = json.Marshal(s.list)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn s.json, nil\n}\n" }, { "path": "tagger/tagger.go", "content": "package tagger\n\nimport (\n\t\"bytes\"\n\t\"compress/gzip\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"runtime\"\n\t\"sort\"\n\t\"time\"\n\n\t\"go.uber.org/zap\"\n\t\"go.uber.org/zap/zapcore\"\n\t\"golang.org/x/sync/errgroup\"\n\n\t\"github.com/lomik/zapwriter\"\n\n\t\"github.com/klauspost/compress/zstd\"\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-clickhouse/helper/RowBinary\"\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomik/graphite-clickhouse/pkg/scope\"\n)\n\nconst SelectChunksCount = 10\n\ntype nopCloser struct {\n\tio.Writer\n}\n\nfunc (nopCloser) Close() error { return nil }\n\nfunc countMetrics(body []byte) (int, error) {\n\tvar namelen uint64\n\n\tbodyLen := len(body)\n\n\tvar count, offset, readBytes int\n\n\tvar err error\n\n\tfor {\n\t\tif offset >= bodyLen {\n\t\t\tif offset == bodyLen {\n\t\t\t\treturn count, nil\n\t\t\t}\n\n\t\t\treturn 0, clickhouse.ErrClickHouseResponse\n\t\t}\n\n\t\tnamelen, readBytes, err = clickhouse.ReadUvarint(body[offset:])\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\n\t\toffset += readBytes + int(namelen)\n\t\tcount++\n\t}\n}\n\nfunc pathLevel(path []byte) int {\n\tif len(path) == 0 {\n\t\treturn 0\n\t}\n\n\tif path[len(path)-1] == '.' {\n\t\treturn bytes.Count(path, []byte{'.'})\n\t}\n\n\treturn bytes.Count(path, []byte{'.'}) + 1\n}\n\nfunc Make(cfg *config.Config) error {\n\tvar start time.Time\n\n\tvar block string\n\n\tlogger := zapwriter.Logger(\"tagger\")\n\tchOpts := clickhouse.Options{\n\t\tTLSConfig: cfg.ClickHouse.TLSConfig,\n\t\tTimeout: cfg.ClickHouse.IndexTimeout,\n\t\tConnectTimeout: cfg.ClickHouse.ConnectTimeout,\n\t\tCheckRequestProgress: cfg.FeatureFlags.LogQueryProgress,\n\t\tProgressSendingInterval: cfg.ClickHouse.ProgressSendingInterval,\n\t}\n\n\tbegin := func(b string, fields ...zapcore.Field) {\n\t\tblock = b\n\t\tstart = time.Now()\n\n\t\tlogger.Info(fmt.Sprintf(\"begin %s\", block), fields...)\n\t}\n\n\tend := func() {\n\t\tvar m runtime.MemStats\n\n\t\truntime.ReadMemStats(&m)\n\n\t\td := time.Since(start)\n\t\tlogger.Info(fmt.Sprintf(\"end %s\", block),\n\t\t\tzap.Duration(\"time\", d),\n\t\t\tzap.Uint64(\"mem_rss_mb\", (m.Sys-m.HeapReleased)/1048576),\n\t\t)\n\t}\n\n\tversion := uint32(time.Now().Unix())\n\n\tif cfg.Tags.Version != 0 {\n\t\tversion = cfg.Tags.Version\n\t}\n\n\tlogger.Info(\"start\", zap.Uint32(\"version\", version))\n\n\t// Parse rules\n\tbegin(\"parse rules\")\n\n\trules, err := ParseGlob(cfg.Tags.Rules)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdate, err := time.ParseInLocation(\"2006-01-02\", cfg.Tags.Date, time.Local)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tend()\n\n\tselectChunksCount := SelectChunksCount\n\tif cfg.Tags.SelectChunksCount != 0 {\n\t\tselectChunksCount = cfg.Tags.SelectChunksCount\n\t}\n\n\t// Read clickhouse\n\tbegin(\"read metrics\", zap.Int(\"chunks_count\", selectChunksCount))\n\n\tvar bodies [][]byte\n\n\tif cfg.Tags.InputFile != \"\" {\n\t\tbody, err := os.ReadFile(cfg.Tags.InputFile)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tbodies = [][]byte{body}\n\t} else {\n\t\tbodies = make([][]byte, selectChunksCount)\n\n\t\textraWhere := \"\"\n\t\tif cfg.Tags.ExtraWhere != \"\" {\n\t\t\textraWhere = fmt.Sprintf(\"AND (%s)\", cfg.Tags.ExtraWhere)\n\t\t}\n\n\t\tfor i := 0; i < selectChunksCount; i++ {\n\t\t\tbodies[i], _, _, err = clickhouse.Query(\n\t\t\t\tscope.New(context.Background()).WithLogger(logger).WithTable(cfg.ClickHouse.IndexTable),\n\t\t\t\tcfg.ClickHouse.URL,\n\t\t\t\tfmt.Sprintf(\n\t\t\t\t\t\"SELECT Path FROM %s WHERE cityHash64(Path) %% %d = %d %s AND Level > 20000 AND Level < 30000 AND Date = '1970-02-12' GROUP BY Path FORMAT RowBinary\",\n\t\t\t\t\tcfg.ClickHouse.IndexTable,\n\t\t\t\t\tselectChunksCount,\n\t\t\t\t\ti,\n\t\t\t\t\textraWhere,\n\t\t\t\t),\n\t\t\t\tchOpts,\n\t\t\t\tnil,\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\n\tend()\n\n\tbegin(\"parse metrics\")\n\n\tvar count int\n\n\tfor i := 0; i < len(bodies); i++ {\n\t\tc, err := countMetrics(bodies[i])\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tcount += c\n\t}\n\n\tmetricList := make([]Metric, count)\n\n\tindex := 0\n\n\tvar maxLevel int\n\n\tfor i := 0; i < len(bodies); i++ {\n\t\tbody := bodies[i]\n\n\t\tvar namelen uint64\n\n\t\tbodyLen := len(body)\n\n\t\tvar offset, readBytes int\n\n\t\tfor ; ; index++ {\n\t\t\tif offset >= bodyLen {\n\t\t\t\tif offset == bodyLen {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\n\t\t\t\treturn clickhouse.ErrClickHouseResponse\n\t\t\t}\n\n\t\t\tnamelen, readBytes, err = clickhouse.ReadUvarint(body[offset:])\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tmetricList[index].Path = body[offset+readBytes : offset+readBytes+int(namelen)]\n\t\t\tmetricList[index].Level = pathLevel(metricList[index].Path)\n\n\t\t\tif metricList[index].Level > maxLevel {\n\t\t\t\tmaxLevel = metricList[index].Level\n\t\t\t}\n\n\t\t\toffset += readBytes + int(namelen)\n\t\t}\n\t}\n\n\tend()\n\n\tbegin(\"sort\")\n\n\tstart = time.Now()\n\n\tsort.Slice(metricList, func(i, j int) bool { return bytes.Compare(metricList[i].Path, metricList[j].Path) < 0 })\n\tend()\n\n\tbegin(\"make map\")\n\n\tlevelMap := make([]int, maxLevel+1)\n\n\tfor index := 0; index < len(metricList); index++ {\n\t\tm := &metricList[index]\n\t\tlevelMap[m.Level] = index\n\n\t\tif m.Level > 0 {\n\t\t\tparentIndex := levelMap[m.Level-1]\n\t\t\tif bytes.Equal(m.ParentPath(), metricList[parentIndex].Path) {\n\t\t\t\tm.ParentIndex = parentIndex\n\t\t\t} else {\n\t\t\t\tm.ParentIndex = -1\n\t\t\t}\n\t\t}\n\t}\n\n\tend()\n\n\tbegin(\"match\", zap.Int(\"metrics_count\", len(metricList)))\n\n\tfor index := 0; index < count; index++ {\n\t\tm := &metricList[index]\n\n\t\tif m.ParentIndex < 0 {\n\t\t\tm.Tags = EmptySet\n\t\t} else {\n\t\t\tm.Tags = metricList[m.ParentIndex].Tags\n\t\t}\n\n\t\trules.Match(m)\n\t}\n\n\tend()\n\n\t// copy from childs to parents\n\tbegin(\"copy tags from childs to parents\")\n\n\tfor index := 0; index < count; index++ {\n\t\tm := &metricList[index]\n\n\t\tfor p := m.ParentIndex; p >= 0; p = metricList[p].ParentIndex {\n\t\t\tmetricList[p].Tags = metricList[p].Tags.Merge(m.Tags)\n\t\t}\n\t}\n\n\tend()\n\n\tbegin(\"remove metrics without tags\", zap.Int(\"metrics_count\", len(metricList)))\n\n\ti := 0\n\n\tfor _, m := range metricList {\n\t\tif m.Tags == nil || m.Tags.Len() == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tmetricList[i] = m\n\t\ti++\n\t}\n\n\tmetricList = metricList[:i]\n\n\tend()\n\n\tif len(metricList) == 0 {\n\t\tlogger.Info(\"nothing to do\", zap.Int(\"metrics_count\", len(metricList)))\n\t\treturn nil\n\t}\n\n\tbegin(\"cut metrics into parts\", zap.Int(\"metrics_count\", len(metricList)))\n\tmetricListParts, tagsCount := cutMetricsIntoParts(metricList, cfg.Tags.Threads)\n\tthreads := len(metricListParts)\n\n\tend()\n\n\tbegin(\"marshal RowBinary\",\n\t\tzap.String(\"compression\", string(cfg.Tags.Compression)),\n\t\tzap.Int(\"tags_count\", tagsCount),\n\t\tzap.Int(\"threads\", threads),\n\t\tzap.Int(\"max_cpu\", cfg.Common.MaxCPU))\n\n\tbinaryParts := make([]*bytes.Buffer, threads)\n\n\teg := new(errgroup.Group)\n\teg.SetLimit(cfg.Common.MaxCPU)\n\n\tfor i := 0; i < threads; i++ {\n\t\tbinaryParts[i] = new(bytes.Buffer)\n\n\t\twc, err := wrapWithCompressor(cfg, binaryParts[i])\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tmetricList := metricListParts[i]\n\n\t\teg.Go(func() error {\n\t\t\treturn encodeMetricsToRowBinary(metricList, date, version, wc)\n\t\t})\n\t}\n\n\terr = eg.Wait()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\temptyRecord := new(bytes.Buffer)\n\n\twc, err := wrapWithCompressor(cfg, emptyRecord)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = encodeEmptyMetricToRowBinary(date, version, wc)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tend()\n\n\tif cfg.Tags.OutputFile != \"\" {\n\t\tbegin(fmt.Sprintf(\"write to %#v\", cfg.Tags.OutputFile))\n\n\t\tf, err := os.Create(cfg.Tags.OutputFile)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tfor i := 0; i < threads; i++ { // just concatenate the parts because zstd and gzip allow it\n\t\t\t_, err = binaryParts[i].WriteTo(f)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\t_, err = emptyRecord.WriteTo(f)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\terr = f.Close()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tend()\n\t} else {\n\t\tbegin(\"upload to clickhouse\", zap.Int(\"threads\", threads))\n\n\t\tupload := func(outBuf *bytes.Buffer) error {\n\t\t\t_, _, _, err := clickhouse.PostWithEncoding(\n\t\t\t\tscope.New(context.Background()).WithLogger(logger).WithTable(cfg.ClickHouse.TagTable),\n\t\t\t\tcfg.ClickHouse.URL,\n\t\t\t\tfmt.Sprintf(\"INSERT INTO %s (Date,Version,Level,Path,IsLeaf,Tags,Tag1) FORMAT RowBinary\", cfg.ClickHouse.TagTable),\n\t\t\t\toutBuf,\n\t\t\t\tcfg.Tags.Compression,\n\t\t\t\tchOpts,\n\t\t\t\tnil,\n\t\t\t)\n\n\t\t\treturn err\n\t\t}\n\t\teg := new(errgroup.Group)\n\n\t\tfor i := 0; i < threads; i++ {\n\t\t\toutBuf := binaryParts[i]\n\n\t\t\teg.Go(func() error {\n\t\t\t\treturn upload(outBuf)\n\t\t\t})\n\t\t}\n\n\t\terr = eg.Wait()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\terr = upload(emptyRecord)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tend()\n\t}\n\n\treturn nil\n}\n\nfunc cutMetricsIntoParts(metricList []Metric, threads int) ([][]Metric, int) {\n\ttagsCount := 0\n\tfor _, m := range metricList {\n\t\ttagsCount += m.Tags.Len()\n\t}\n\n\tif threads < 2 {\n\t\treturn [][]Metric{metricList}, tagsCount\n\t}\n\n\tparts := make([][]Metric, 0, threads)\n\ti := 0\n\tpartSize := (tagsCount-1)/threads + 1 // round up for cases like 99/50\n\n\tcnt := 0\n\tfor j, m := range metricList {\n\t\t// assert m.Tags != nil && m.Tags.Len() != 0\n\t\tcnt += m.Tags.Len()\n\t\tif cnt >= partSize {\n\t\t\tparts = append(parts, metricList[i:j+1])\n\t\t\ti = j + 1\n\t\t\tcnt = 0\n\t\t}\n\t}\n\n\tif i < len(metricList) {\n\t\tparts = append(parts, metricList[i:])\n\t}\n\n\treturn parts, tagsCount\n}\n\nfunc wrapWithCompressor(cfg *config.Config, writer io.Writer) (io.WriteCloser, error) {\n\tvar wc io.WriteCloser\n\n\tvar err error\n\n\tswitch cfg.Tags.Compression {\n\tcase clickhouse.ContentEncodingNone:\n\t\twc = nopCloser{writer}\n\tcase clickhouse.ContentEncodingGzip:\n\t\twc = gzip.NewWriter(writer)\n\tcase clickhouse.ContentEncodingZstd:\n\t\twc, err = zstd.NewWriter(writer, zstd.WithEncoderConcurrency(1))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"unknown compression: %s\", cfg.Tags.Compression)\n\t}\n\n\treturn wc, nil\n}\n\nfunc encodeMetricsToRowBinary(metricList []Metric, date time.Time, version uint32, wc io.WriteCloser) error {\n\tencoder := RowBinary.NewEncoder(wc)\n\tdays := RowBinary.DateToUint16(date)\n\tmetricBuffer := new(bytes.Buffer)\n\tmetricEncoder := RowBinary.NewEncoder(metricBuffer)\n\n\tfor i := 0; i < len(metricList); i++ {\n\t\tm := &metricList[i]\n\n\t\tif m.Tags == nil || m.Tags.Len() == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tmetricBuffer.Reset()\n\n\t\t// Date\n\t\terr := metricEncoder.Uint16(days)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// Version\n\t\terr = metricEncoder.Uint32(version)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// Level\n\t\terr = metricEncoder.Uint32(uint32(m.Level))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// Path\n\t\terr = metricEncoder.Bytes(m.Path)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// IsLeaf\n\t\terr = metricEncoder.Uint8(m.IsLeaf())\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// Tags\n\t\terr = metricEncoder.StringList(m.Tags.List())\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tfor _, tag := range m.Tags.List() {\n\t\t\t_, err = wc.Write(metricBuffer.Bytes())\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\t// Tag1\n\t\t\terr = encoder.String(tag)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\n\twc.Close()\n\n\treturn nil\n}\n\n// Empty record With Level=0, Path=0 and Without Tags\n// It is needed to filter current tags\nfunc encodeEmptyMetricToRowBinary(date time.Time, version uint32, wc io.WriteCloser) error {\n\tencoder := RowBinary.NewEncoder(wc)\n\tdays := RowBinary.DateToUint16(date)\n\n\t// Date\n\terr := encoder.Uint16(days)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// Version\n\terr = encoder.Uint32(version)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// Level=0\n\terr = encoder.Uint32(0)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// Path=\"\"\n\terr = encoder.Bytes([]byte{})\n\tif err != nil {\n\t\treturn err\n\t}\n\t// IsLeaf=0\n\terr = encoder.Uint8(0)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// Tags=[]\n\terr = encoder.StringList([]string{})\n\tif err != nil {\n\t\treturn err\n\t}\n\t// Tag1=\"\"\n\terr = encoder.String(\"\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\twc.Close()\n\n\treturn nil\n}\n" }, { "path": "tagger/tagger_test.go", "content": "package tagger\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestCutMetricsIntoParts(t *testing.T) {\n\trequire := assert.New(t)\n\n\tmetricList1 := []Metric{\n\t\t{Tags: new(Set).Add(\"tag1\", \"tag2\")},\n\t\t{Tags: new(Set).Add(\"tag3\", \"tag4\", \"tag5\")},\n\t\t{Tags: new(Set).Add(\"tag6\")},\n\t}\n\n\tmetricList2 := []Metric{\n\t\t{Tags: new(Set).Add(\"tag1\")},\n\t\t{Tags: new(Set).Add(\"tag2\")},\n\t\t{Tags: new(Set).Add(\"tag3\", \"tag4\", \"tag5\", \"tag6\")},\n\t}\n\n\tmetricList3 := []Metric{\n\t\t{Tags: new(Set).Add(\"tag1\")},\n\t\t{Tags: new(Set).Add(\"tag2\")},\n\t\t{Tags: new(Set).Add(\"tag3\")},\n\t\t{Tags: new(Set).Add(\"tag4\")},\n\t\t{Tags: new(Set).Add(\"tag5\", \"tag6\", \"tag7\", \"tag8\", \"tag9\")},\n\t}\n\n\tmetricList4 := []Metric{\n\t\t{Tags: new(Set).Add(\"tag1\")},\n\t\t{Tags: new(Set).Add(\"tag2\")},\n\t\t{Tags: new(Set).Add(\"tag3\")},\n\t\t{Tags: new(Set).Add(\"tag4\")},\n\t\t{Tags: new(Set).Add(\"tag5\")},\n\t\t{Tags: new(Set).Add(\"tag6\")},\n\t}\n\n\tmetricList5 := []Metric{\n\t\t{Tags: new(Set).Add(\"tag1\", \"tag2\", \"tag3\", \"tag4\")},\n\t\t{Tags: new(Set).Add(\"tag5\")},\n\t\t{Tags: new(Set).Add(\"tag6\")},\n\t\t{Tags: new(Set).Add(\"tag7\")},\n\t\t{Tags: new(Set).Add(\"tag8\")},\n\t\t{Tags: new(Set).Add(\"tag9\")},\n\t}\n\n\tmetricList6 := []Metric{\n\t\t{Tags: new(Set).Add(\"tag0\", \"tag1\")},\n\t\t{Tags: new(Set).Add(\"tag0\")},\n\t\t{Tags: new(Set).Add(\"tag0\")},\n\t\t{Tags: new(Set).Add(\"tag0\", \"tag1\")},\n\t\t{Tags: new(Set).Add(\"tag0\", \"tag1\")},\n\t\t{Tags: new(Set).Add(\"tag0\", \"tag1\")},\n\t\t{Tags: new(Set).Add(\"tag0\", \"tag1\")},\n\t\t{Tags: new(Set).Add(\"tag0\", \"tag1\")},\n\t}\n\n\ttestCases := []struct {\n\t\tname string\n\t\tmetricList []Metric\n\t\tthreads int\n\t\twant [][]Metric\n\t}{\n\t\t{\"case 0.0\", []Metric{}, 0, [][]Metric{{}}},\n\t\t{\"case 1.1\", metricList1, 0, [][]Metric{\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag1\", \"tag2\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag3\", \"tag4\", \"tag5\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag6\")},\n\t\t\t},\n\t\t}},\n\t\t{\"case 1.2\", metricList1, 1, [][]Metric{\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag1\", \"tag2\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag3\", \"tag4\", \"tag5\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag6\")},\n\t\t\t},\n\t\t}},\n\t\t{\"case 1.3\", metricList1, 2, [][]Metric{\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag1\", \"tag2\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag3\", \"tag4\", \"tag5\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag6\")},\n\t\t\t},\n\t\t}},\n\t\t{\"case 1.4\", metricList1, 3, [][]Metric{\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag1\", \"tag2\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag3\", \"tag4\", \"tag5\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag6\")},\n\t\t\t},\n\t\t}},\n\t\t{\"case 2.1\", metricList2, 2, [][]Metric{\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag1\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag2\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag3\", \"tag4\", \"tag5\", \"tag6\")},\n\t\t\t},\n\t\t}},\n\t\t{\"case 2.2\", metricList2, 3, [][]Metric{\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag1\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag2\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag3\", \"tag4\", \"tag5\", \"tag6\")},\n\t\t\t},\n\t\t}},\n\t\t{\"case 3.1\", metricList3, 2, [][]Metric{\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag1\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag2\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag3\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag4\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag5\", \"tag6\", \"tag7\", \"tag8\", \"tag9\")},\n\t\t\t},\n\t\t}},\n\t\t{\"case 3.2\", metricList3, 3, [][]Metric{\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag1\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag2\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag3\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag4\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag5\", \"tag6\", \"tag7\", \"tag8\", \"tag9\")},\n\t\t\t},\n\t\t}},\n\t\t{\"case 4.1\", metricList4, 2, [][]Metric{\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag1\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag2\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag3\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag4\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag5\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag6\")},\n\t\t\t},\n\t\t}},\n\t\t{\"case 4.2\", metricList4, 3, [][]Metric{\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag1\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag2\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag3\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag4\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag5\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag6\")},\n\t\t\t},\n\t\t}},\n\t\t{\"case 4.3\", metricList4, 4, [][]Metric{\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag1\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag2\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag3\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag4\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag5\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag6\")},\n\t\t\t},\n\t\t}},\n\t\t{\"case 5.1\", metricList5, 2, [][]Metric{\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag1\", \"tag2\", \"tag3\", \"tag4\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag5\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag6\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag7\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag8\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag9\")},\n\t\t\t},\n\t\t}},\n\t\t{\"case 5.2\", metricList5, 3, [][]Metric{\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag1\", \"tag2\", \"tag3\", \"tag4\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag5\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag6\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag7\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag8\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag9\")},\n\t\t\t},\n\t\t}},\n\t\t{\"case 5.3\", metricList5, 4, [][]Metric{\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag1\", \"tag2\", \"tag3\", \"tag4\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag5\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag6\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag7\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag8\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag9\")},\n\t\t\t},\n\t\t}},\n\t\t{\"case 5.4\", metricList5, 5, [][]Metric{\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag1\", \"tag2\", \"tag3\", \"tag4\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag5\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag6\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag7\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag8\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag9\")},\n\t\t\t},\n\t\t}},\n\t\t{\"case 6.1\", metricList6, 5, [][]Metric{\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag0\", \"tag1\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag0\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag0\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag0\", \"tag1\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag0\", \"tag1\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag0\", \"tag1\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag0\", \"tag1\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag0\", \"tag1\")},\n\t\t\t},\n\t\t}},\n\t\t{\"case 6.2\", metricList6, 7, [][]Metric{\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag0\", \"tag1\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag0\")},\n\t\t\t\t{Tags: new(Set).Add(\"tag0\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag0\", \"tag1\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag0\", \"tag1\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag0\", \"tag1\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag0\", \"tag1\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\t{Tags: new(Set).Add(\"tag0\", \"tag1\")},\n\t\t\t},\n\t\t}},\n\t}\n\tfor _, tc := range testCases {\n\t\t// if !strings.HasPrefix(tc.name, \"case 6.\") {\n\t\t// \tcontinue\n\t\t// }\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tgot, _ := cutMetricsIntoParts(tc.metricList, tc.threads)\n\t\t\trequire.Equal(len(tc.want), len(got), \"unexpected number of parts\")\n\t\t\trequire.Equal(tc.want, got)\n\t\t})\n\t}\n}\n\nfunc TestCutMetricsIntoPartsRandom(t *testing.T) {\n\trequire := require.New(t)\n\n\trand.Seed(time.Now().UnixNano())\n\n\tfor n := 0; n < 1000; n++ {\n\t\tmetricList := make([]Metric, rand.Intn(100))\n\t\ttagsMax := rand.Intn(100) + 1\n\t\ttagsCnt := 0\n\n\t\tfor i := range metricList {\n\t\t\ttags := make([]string, rand.Intn(tagsMax)+1)\n\t\t\ttagsCnt += len(tags)\n\n\t\t\tfor j := range tags {\n\t\t\t\ttags[j] = fmt.Sprintf(\"tag%d\", j)\n\t\t\t}\n\n\t\t\tmetricList[i].Tags = new(Set).Add(tags...)\n\t\t}\n\n\t\tthreads := rand.Intn(110)\n\t\tparts, _ := cutMetricsIntoParts(metricList, threads)\n\n\t\tif threads == 0 {\n\t\t\tthreads = 1\n\t\t}\n\n\t\tif len(parts) > threads {\n\t\t\tv, _ := json.MarshalIndent(parts, \"\", \" \")\n\t\t\tfmt.Println(string(v))\n\t\t}\n\n\t\trequire.LessOrEqual(len(parts), threads, fmt.Sprint(tagsCnt, len(metricList), len(parts), threads))\n\n\t\tif len(metricList) > 0 {\n\t\t\trequire.LessOrEqual(len(parts), len(metricList), fmt.Sprint(tagsCnt, len(metricList), len(parts), threads))\n\t\t}\n\n\t\ti := 0\n\n\t\tfor _, p := range parts {\n\t\t\tfor _, m := range p {\n\t\t\t\trequire.Equal(metricList[i], m)\n\n\t\t\t\ti++\n\t\t\t}\n\t\t}\n\n\t\trequire.Equal(len(metricList), i)\n\t}\n}\n" }, { "path": "tagger/tree.go", "content": "package tagger\n\ntype Tree struct {\n\tNext [256]*Tree\n\tRules []*Rule\n}\n\nfunc (t *Tree) Add(prefix []byte, rule *Rule) {\n\tx := t\n\n\tfor i := 0; i < len(prefix); i++ {\n\t\tif x.Next[prefix[i]] == nil {\n\t\t\tx.Next[prefix[i]] = &Tree{}\n\t\t}\n\n\t\tx = x.Next[prefix[i]]\n\t}\n\n\tif x.Rules == nil {\n\t\tx.Rules = make([]*Rule, 0)\n\t}\n\n\tx.Rules = append(x.Rules, rule)\n}\n\nfunc (t *Tree) AddSuffix(suffix []byte, rule *Rule) {\n\tx := t\n\n\tfor i := len(suffix) - 1; i >= 0; i-- {\n\t\tif x.Next[suffix[i]] == nil {\n\t\t\tx.Next[suffix[i]] = &Tree{}\n\t\t}\n\n\t\tx = x.Next[suffix[i]]\n\t}\n\n\tif x.Rules == nil {\n\t\tx.Rules = make([]*Rule, 0)\n\t}\n\n\tx.Rules = append(x.Rules, rule)\n}\n" }, { "path": "tests/agg_internal/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "tests/agg_internal/graphite-clickhouse-internal-aggr.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = true\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/agg_internal/test.toml", "content": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"21.3\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"22.8\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\n\n[test.carbon_clickhouse]\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse-internal-aggr.conf.tpl\"\n\n[[test.input]]\nname = \"test.avg\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 2.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"test.sum\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 2.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"test.min\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 2.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"test.max\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 2.0, time = \"rnow\", delay = \"5s\"}]\n\n# Test aggregations\n[[test.input]]\nname = \"test.avg\"\npoints = [{value = 0.0, time = \"rnow-1\"}, {value = 4.0, time = \"rnow+1\"}]\n\n# Test aggregations\n[[test.input]]\nname = \"test.sum\"\npoints = [{value = 0.0, time = \"rnow-1\"}, {value = 4.0, time = \"rnow+1\"}]\n\n# Test aggregations\n[[test.input]]\nname = \"test.min\"\npoints = [{value = 0.0, time = \"rnow-1\"}, {value = 4.0, time = \"rnow+1\"}]\n\n# Test aggregations\n[[test.input]]\nname = \"test.max\"\npoints = [{value = 0.0, time = \"rnow-1\"}, {value = 4.0, time = \"rnow+1\"}]\n\n##########################################################################\n# Aggregated, Deduplication not work with internal aggregation\n##########################################################################\n\n[[test.render_checks]]\nname = \"Test rollup\"\nfrom = \"rnow-10\"\nuntil = \"rnow+10\"\ntargets = [ \n \"test.{avg,min,max,sum}\"\n]\n\n[[test.render_checks.result]]\nname = \"test.avg\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [0.5, 3.0, nan]\n\n[[test.render_checks.result]]\nname = \"test.sum\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"sum\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [1.0, 6.0, nan]\n\n[[test.render_checks.result]]\nname = \"test.min\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"min\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [0.0, 2.0, nan]\n\n[[test.render_checks.result]]\nname = \"test.max\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"max\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [1.0, 4.0, nan]\n\n# End - Test rollup\n##########################################################################\n" }, { "path": "tests/agg_latest/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "tests/agg_latest/graphite-clickhouse.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = false\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/agg_latest/test.toml", "content": "[test]\nprecision = \"10s\"\n\n#[[test.clickhouse]]\n#version = \"20.3\"\n#dir = \"tests/clickhouse/rollup\"\n\n#[[test.clickhouse]]\n#version = \"20.8\"\n#dir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"21.3\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"22.8\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\n\n[test.carbon_clickhouse]\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse.conf.tpl\"\n\n[[test.input]]\nname = \"test.avg\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 2.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"test.sum\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 2.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"test.min\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 2.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"test.max\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 2.0, time = \"rnow\", delay = \"5s\"}]\n\n# Test aggregations\n[[test.input]]\nname = \"test.avg\"\npoints = [{value = 0.0, time = \"rnow-1\"}, {value = 4.0, time = \"rnow+1\"}]\n\n# Test aggregations\n[[test.input]]\nname = \"test.sum\"\npoints = [{value = 0.0, time = \"rnow-1\"}, {value = 4.0, time = \"rnow+1\"}]\n\n# Test aggregations\n[[test.input]]\nname = \"test.min\"\npoints = [{value = 0.0, time = \"rnow-1\"}, {value = 4.0, time = \"rnow+1\"}]\n\n# Test aggregations\n[[test.input]]\nname = \"test.max\"\npoints = [{value = 0.0, time = \"rnow-1\"}, {value = 4.0, time = \"rnow+1\"}]\n\n##########################################################################\n# Not aggregated, Deduplication work with version >= 21.3\n##########################################################################\n\n[[test.render_checks]]\nname = \"Test rollup\"\nfrom = \"rnow-10\"\nuntil = \"rnow+10\"\ntargets = [ \n \"test.{avg,min,max,sum}\"\n]\n\n[[test.render_checks.result]]\nname = \"test.avg\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [0.0, 4.0, nan]\n\n[[test.render_checks.result]]\nname = \"test.sum\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"sum\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [0.0, 4.0, nan]\n\n[[test.render_checks.result]]\nname = \"test.min\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"min\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [0.0, 4.0, nan]\n\n[[test.render_checks.result]]\nname = \"test.max\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"max\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [0.0, 4.0, nan]\n\n# End - Test rollup\n##########################################################################\n" }, { "path": "tests/agg_merge/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "tests/agg_merge/graphite-clickhouse-internal-aggr.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = true\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/agg_merge/graphite-clickhouse.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = false\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/agg_merge/test.toml", "content": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"21.3\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"22.8\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\n\n[test.carbon_clickhouse]\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse-internal-aggr.conf.tpl\"\n\n[[test.input]]\nname = \"test.avg\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 0.0, time = \"rnow-1\"}, {value = 2.0, time = \"rnow\"}, {value = 4.0, time = \"rnow+1\"}]\n\n[[test.input]]\nname = \"test.sum\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 0.0, time = \"rnow-1\"}, {value = 2.0, time = \"rnow\"}, {value = 4.0, time = \"rnow+1\"}]\n\n[[test.input]]\nname = \"test.min\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 0.0, time = \"rnow-1\"}, {value = 2.0, time = \"rnow\"}, {value = 4.0, time = \"rnow+1\"}]\n\n[[test.input]]\nname = \"test.max\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 0.0, time = \"rnow-1\"}, {value = 2.0, time = \"rnow\"}, {value = 4.0, time = \"rnow+1\"}]\n\n##########################################################################\n# Aggregated at merge\n##########################################################################\n\n[[test.render_checks]]\noptimize = [ \"graphite\", \"graphite_reverse\" ]\nfrom = \"rnow-10\"\nuntil = \"rnow+10\"\ntargets = [ \n \"test.{avg,min,max,sum}\"\n]\n\n[[test.render_checks.result]]\nname = \"test.avg\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [0.5, 3.0, nan]\n\n[[test.render_checks.result]]\nname = \"test.sum\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"sum\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [1.0, 6.0, nan]\n\n[[test.render_checks.result]]\nname = \"test.min\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"min\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [0.0, 2.0, nan]\n\n[[test.render_checks.result]]\nname = \"test.max\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"max\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [1.0, 4.0, nan]\n\n# End - Test rollup\n##########################################################################\n" }, { "path": "tests/agg_oneblock/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "tests/agg_oneblock/graphite-clickhouse-internal-aggr.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = true\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/agg_oneblock/graphite-clickhouse.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = false\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/agg_oneblock/test.toml", "content": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"21.3\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"22.8\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\n\n[test.carbon_clickhouse]\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse-internal-aggr.conf.tpl\"\n\n[[test.input]]\nname = \"test.avg\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 0.0, time = \"rnow-1\"}, {value = 2.0, time = \"rnow\"}, {value = 4.0, time = \"rnow+1\"}]\n\n[[test.input]]\nname = \"test.sum\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 0.0, time = \"rnow-1\"}, {value = 2.0, time = \"rnow\"}, {value = 4.0, time = \"rnow+1\"}]\n\n[[test.input]]\nname = \"test.min\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 0.0, time = \"rnow-1\"}, {value = 2.0, time = \"rnow\"}, {value = 4.0, time = \"rnow+1\"}]\n\n[[test.input]]\nname = \"test.max\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 0.0, time = \"rnow-1\"}, {value = 2.0, time = \"rnow\"}, {value = 4.0, time = \"rnow+1\"}]\n\n##########################################################################\n# Aggregated, Deduplication not work at one block\n##########################################################################\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+10\"\ntargets = [ \n \"test.{avg,min,max,sum}\"\n]\ndump_if_empty = [\n \"SELECT Date, Path FROM graphite_index WHERE ((Level=2) AND (Path LIKE 'test.%' AND match(Path, '^test[.](avg|min|max|sum)[.]?$'))) GROUP BY Date, Path\"\n]\n\n[[test.render_checks.result]]\nname = \"test.avg\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [0.5, 3.0, nan]\n\n[[test.render_checks.result]]\nname = \"test.sum\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"sum\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [1.0, 6.0, nan]\n\n[[test.render_checks.result]]\nname = \"test.min\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"min\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [0.0, 2.0, nan]\n\n[[test.render_checks.result]]\nname = \"test.max\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"max\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [1.0, 4.0, nan]\n\n# End - Test rollup\n##########################################################################\n" }, { "path": "tests/clickhouse/rollup/config.xml", "content": "\n\n \n debug\n /var/log/clickhouse-server/clickhouse-server.log\n /var/log/clickhouse-server/clickhouse-server.err.log\n 2000M\n 20\n \n\n 8123\n 9000\n\n \n 9009\n\n test-clickhouse-s1\n \n\n \n \n \n 0.0.0.0\n \n \n 1073741824\n\n \n 1073741824\n\n \n /var/lib/clickhouse/\n\n \n /var/lib/clickhouse/tmp/\n\n \n users.xml\n\n \n default\n\n \n default\n\n \n \n \n system\n query_log
\n\n \n 7500\n \n\n\n \n \n system\n part_log
\n\n 7500\n \n \n\n\n" }, { "path": "tests/clickhouse/rollup/init.sql", "content": "CREATE TABLE IF NOT EXISTS default.graphite_reverse (\n Path String,\n Value Float64,\n Time UInt32,\n Date Date,\n Timestamp UInt32\n) ENGINE = GraphiteMergeTree('graphite_rollup')\nPARTITION BY Date\nORDER BY (Path, Time);\n\nCREATE TABLE IF NOT EXISTS default.graphite (\n Path String,\n Value Float64,\n Time UInt32,\n Date Date,\n Timestamp UInt32\n) ENGINE = GraphiteMergeTree('graphite_rollup')\nPARTITION BY Date\nORDER BY (Path, Time);\n\nCREATE TABLE IF NOT EXISTS default.graphite_index (\n Date Date,\n Level UInt32,\n Path String,\n Version UInt32\n) ENGINE = ReplacingMergeTree(Version)\nPARTITION BY Date\nORDER BY (Level, Path, Date);\n\nCREATE TABLE IF NOT EXISTS default.graphite_tags (\n Date Date,\n Tag1 String,\n Path String,\n Tags Array(String),\n Version UInt32\n) ENGINE = ReplacingMergeTree(Version)\nPARTITION BY Date\nORDER BY (Tag1, Path, Date);\n\nCREATE TABLE IF NOT EXISTS default.tag1_count_per_day\n(\n Date Date,\n Tag1 String,\n Count UInt64\n)\nENGINE = SummingMergeTree\nORDER BY (Date, Tag1);\n\nCREATE MATERIALIZED VIEW IF NOT EXISTS default.tag1_count_per_day_mv TO default.tag1_count_per_day AS\nSELECT Date AS Date,\n Tag1 AS Tag1,\n count(*) AS Count\nFROM default.graphite_tags\nGROUP BY (Date, Tag1);" }, { "path": "tests/clickhouse/rollup/rollup.xml", "content": "\n\n \n avg\n \n 0\n 10\n \n \n \n \\.sum$\n sum\n \n \n \\.sum\\?\n sum\n \n \n \\.min$\n min\n \n \n \\.min\\?\n min\n \n \n \\.max$\n max\n \n \n \\.max\\?\n max\n \n\n\n" }, { "path": "tests/clickhouse/rollup/users.xml", "content": "\n\n \n \n \n \n \t 1\n \t 0\n\n \n 40000000000\n\n \n 1\n\n \n\n\t \n\t \n\t 1048576000\n\t 1000000\n\t 1000000\n\t 720000\n\t \n\t 6000000000000\n\t \n 40\n \n 16\n \n\t 25571520\n\t 1073741824\n\t \n random\n\t 1000\n \n\n \n \n 1\n \n \n\n \n \n \n \n \n \n\n \n \n ::/0\n \n\n \n default\n\n \n default\n \n\n \n \n \n \n ::1\n 127.0.0.1\n \n readonly\n default\n \n \n\n \n \n \n \n \n \n \n 3600\n\n \n 0\n 0\n 0\n 0\n 0\n \n \n \n\n" }, { "path": "tests/clickhouse/rollup_tls/config.xml", "content": "\n\n \n debug\n /var/log/clickhouse-server/clickhouse-server.log\n /var/log/clickhouse-server/clickhouse-server.err.log\n 2000M\n 20\n \n \n 8123\n 9000\n 8443\n 9440\n \n \n none\n false\n /etc/clickhouse-server/rootCA.crt\n /etc/clickhouse-server/server.crt\n /etc/clickhouse-server/server.key\n true\n \n true\n \n \n /etc/clickhouse-server/rootCA.crt\n \n \n\n \n 9009\n\n test-clickhouse-s1\n\n\n \n \n \n 0.0.0.0\n\n\n 1073741824\n\n \n 1073741824\n\n \n /var/lib/clickhouse/\n\n \n /var/lib/clickhouse/tmp/\n\n \n users.xml\n\n \n default\n\n \n default\n\n \n \n \n system\n query_log
\n\n \n 7500\n \n\n\n \n \n system\n part_log
\n\n 7500\n \n \n\n\n" }, { "path": "tests/clickhouse/rollup_tls/init.sql", "content": "CREATE TABLE IF NOT EXISTS default.graphite_reverse (\n Path String,\n Value Float64,\n Time UInt32,\n Date Date,\n Timestamp UInt32\n) ENGINE = GraphiteMergeTree('graphite_rollup')\nPARTITION BY toYYYYMM(Date)\nORDER BY (Path, Time);\n\nCREATE TABLE IF NOT EXISTS default.graphite (\n Path String,\n Value Float64,\n Time UInt32,\n Date Date,\n Timestamp UInt32\n) ENGINE = GraphiteMergeTree('graphite_rollup')\nPARTITION BY toYYYYMM(Date)\nORDER BY (Path, Time);\n\nCREATE TABLE IF NOT EXISTS default.graphite_index (\n Date Date,\n Level UInt32,\n Path String,\n Version UInt32\n) ENGINE = ReplacingMergeTree(Version)\nPARTITION BY toYYYYMM(Date)\nORDER BY (Level, Path, Date);\n\nCREATE TABLE IF NOT EXISTS default.graphite_tags (\n Date Date,\n Tag1 String,\n Path String,\n Tags Array(String),\n Version UInt32\n) ENGINE = ReplacingMergeTree(Version)\nPARTITION BY toYYYYMM(Date)\nORDER BY (Tag1, Path, Date);\n\nCREATE TABLE IF NOT EXISTS default.tag1_count_per_day\n(\n Date Date,\n Tag1 String,\n Count UInt64\n)\nENGINE = SummingMergeTree\nORDER BY (Date, Tag1);\n\nCREATE MATERIALIZED VIEW IF NOT EXISTS default.tag1_count_per_day_mv TO default.tag1_count_per_day AS\nSELECT Date AS Date,\n Tag1 AS Tag1,\n count(*) AS Count\nFROM default.graphite_tags\nGROUP BY (Date, Tag1);" }, { "path": "tests/clickhouse/rollup_tls/rollup.xml", "content": "\n\n \n avg\n \n 0\n 10\n \n \n \n \\.sum$\n sum\n \n \n \\.sum\\?\n sum\n \n \n \\.min$\n min\n \n \n \\.min\\?\n min\n \n \n \\.max$\n max\n \n \n \\.max\\?\n max\n \n\n\n\n" }, { "path": "tests/clickhouse/rollup_tls/rootCA.crt", "content": "-----BEGIN CERTIFICATE-----\nMIIDHTCCAgWgAwIBAgIURx5itXwLHeiQES1LzCHF7F8RNEkwDQYJKoZIhvcNAQEL\nBQAwHjEcMBoGA1UEAwwTbG9yZHZpcmRleC5sb2NhbCBDQTAeFw0yNDA4MDkxMjMy\nMzJaFw0zNDA4MDcxMjMyMzJaMB4xHDAaBgNVBAMME2xvcmR2aXJkZXgubG9jYWwg\nQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDuiK4tBYzNtROmhuXD\n80HsVVk2/+/TXV85Aey7oo2gxxJJ09iARnjJadNrbBUdoL42XtmBCkYY+pXYUWPD\nhvals2AbXiAePg7DlAHJfpaQTzHlsPvAUMjqbD6cFaQ7DfNQHcz2emmFhcRYzlQM\nh0Ob3v2yhogG7PuKaiTLTKYcHnRKfEIobQEIq16ABaaCFKzR6tpvrUJFYtkJ8EUz\njhrSg67qy7yiHiMmGQVq526X2oZYhMbSGjiPkaMZHdFkxZgJF5iQhANG9djvcopO\njdFfsJYM9rVxAjwO/P3fq5dpuQxWLLo6ZmholsixPZs1s8paEnonSDtyoNLsykwD\n2mFdAgMBAAGjUzBRMB0GA1UdDgQWBBS6BlL90Mo/+aHonqIqaewM8CyxnTAfBgNV\nHSMEGDAWgBS6BlL90Mo/+aHonqIqaewM8CyxnTAPBgNVHRMBAf8EBTADAQH/MA0G\nCSqGSIb3DQEBCwUAA4IBAQAIwTN3II6HdPfMsLvYoOmzcvUE9Y6QndI20eLqp3p8\n6KnU+lgLdSkLjc9BKwLh/Jhuy4H3u1nHpW8Jkgy/8irG2uaUvgKlutfApFQshAo7\n/k9xdH36ER0LF/bW5hQ535H76OaE+eaexx2zU50kPVuntal577d8HBfrKVI41KU8\nCVdqYTwEqHwjSyRhmmRqLi7Yo+i0o0hRwH39LxYXY2rup/V6uRyLXSIDUZ9VeqVt\nK8XDAbLV1s4kzR/OdpYcJuTWX9gFUlNHpGDkOSy9ggc5zxKaHlwGGZsvVSb4f+VF\nC89ABPZs+26EvExIih+civiC1XWIghP8RsiNyBOK3TOf\n-----END CERTIFICATE-----\n" }, { "path": "tests/clickhouse/rollup_tls/server.crt", "content": "-----BEGIN CERTIFICATE-----\nMIIDYDCCAkigAwIBAgIUElpBaTpcXsRXXP5YNhdKwu+lmE0wDQYJKoZIhvcNAQEL\nBQAwHjEcMBoGA1UEAwwTbG9yZHZpcmRleC5sb2NhbCBDQTAeFw0yNDA4MDkxMjM2\nNTJaFw0zNDA4MDcxMjM2NTJaMFYxCzAJBgNVBAYTAlJVMRIwEAYDVQQIDAlUYXRh\ncnN0YW4xDjAMBgNVBAcMBUthemFuMQ8wDQYDVQQKDAZLb250dXIxEjAQBgNVBAMM\nCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKdkGsVL\nlK0ecS7+pEzEFpKmKOrSMKGCfOkqIVNO4f21njvg3rOx+j/1G8+D1eFHJJkotsx/\nHfJg2sgMosltIlR19f1CzV6ewQLYN7fw4d8aMq1B1lnzzvfUjjygdxB353RiaCHI\neQ1xkTPPmdZMEgaYwto2nrNrCOTb/kZig6pQeQ1YLV4c1daiI9L7OJhwKIb9yqT6\ngT+jXrrRZWE5o0sSKBw16h+iFXy/niO+2+VLuAHXturTg8m0U+NagexJZzkM4wt1\n9pXAlODxu6y0en3lU2ngfGVV22HGSYsyKjBWAzA4HM3wQ6D6DrEa2W2ezV1MQfrS\nFYGVO8DBziaV41cCAwEAAaNeMFwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAAB\nMB0GA1UdDgQWBBRTabpN+c8bOE2B5o214IliRqUIqjAfBgNVHSMEGDAWgBS6BlL9\n0Mo/+aHonqIqaewM8CyxnTANBgkqhkiG9w0BAQsFAAOCAQEAJal0TGS91yRK4ATZ\nsifjon3w7Q47WAbhXNFasuFdEdEexcWmc+gzhYW+snnVUlHT9y1J675i/Le6ry7y\n/pkzzdSoyx7CVHlU81gZLCts1lzufDl/cE5vDG4Sjnx1SepumUy9IrXhCaAaH19s\nEiywBsZ1uPC3XqAlaXLUYQglmtzXzeOMDXVRz4n3+SujkZ+DD2UMmTvWe9P1D8Ss\ngMkg8iMvNtm90MjVFgddLf9QjHYEJjNvzaRdQXvsnCOBwR/kyKimaZxV34QCC7cl\nQMhc6SzEmGqz+NflrzJyAIOMkDxotyqZX6v5kYFwEMhvBa6tXi+mmxx4lQ6VElT9\npB9GKA==\n-----END CERTIFICATE-----\n" }, { "path": "tests/clickhouse/rollup_tls/server.key", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCnZBrFS5StHnEu\n/qRMxBaSpijq0jChgnzpKiFTTuH9tZ474N6zsfo/9RvPg9XhRySZKLbMfx3yYNrI\nDKLJbSJUdfX9Qs1ensEC2De38OHfGjKtQdZZ88731I48oHcQd+d0YmghyHkNcZEz\nz5nWTBIGmMLaNp6zawjk2/5GYoOqUHkNWC1eHNXWoiPS+ziYcCiG/cqk+oE/o166\n0WVhOaNLEigcNeofohV8v54jvtvlS7gB17bq04PJtFPjWoHsSWc5DOMLdfaVwJTg\n8bustHp95VNp4HxlVdthxkmLMiowVgMwOBzN8EOg+g6xGtltns1dTEH60hWBlTvA\nwc4mleNXAgMBAAECggEAMrhgbDvUlwhMX2MFQcWA2XrDlzONTMMPOk9rvaR/UbMA\neUBP+r8JBuwsOxrFafd2nXn6ucgiuNikMk2x3brV1iXQHadqNyt/bG87ot64cjOr\n+1ehrav0oJ+lYbV1nmXWmitfRi1KkMpCpyJWiNqP87PCBwDZ4Z+jGEWYrJcZMjem\ngAFoaUw0hrLc7FJe8sogC9j3gyIfjVq7k+epPlW6VsRW7h+aZowq2Bbik2VYz1Py\naIdpaZwf8Jrhn7Qo0V39OVEr/VVLKFzNlyLpp+XeXmXT8sinmWNtTWKUB2rzL9bR\noa+OeRTJIyzJXwpIBKte4TIKhtnmWANEWjuGf16m0QKBgQDFjMFwIYt6+8LEmL3Y\nxfpI9Hgy/PGLA/Y49ZR52OmEpXvQO12SQsDxMLIMXECSnjIIew+aJgly9HR3i+uK\neFjLjKBABx5xmdfzLT05YcTASRWoj4tZrLrLM4Vytcs4xCSbiWbYFsgbbls38Abs\nPbmcD7oU/n/F+GrGJCHRKEtNRQKBgQDY6v8nvMDS46WLou2VzAA25mGOUlY5jzW1\nWR0WxU2cLZwl+2upLj5UYRHXQtCOrVIGaMXEUdQgf+w3rvGnet14LeBByFQBs3wP\nTnluBEwG/ByZfjOwqAOULfIHJq75HyCZ5XR5H8tIm4hf8rb3BiJ1fe2bJSgJbst4\nTLmOPljx6wKBgQCHOP3//zY2jLaZU+Q/yeS0o4LThAjim2ejPZbQgQX3Yj8KHljC\nkSb48dguVcdtlROycmoPnhHBuksuuXwVYKOHUU8wBK92G1SShFjwOlgvNte4delx\nDKcgCLhD+OSOitR0Eu1u5Mk83aFa/NYAR5ARn0JEtKBJpu2Pi5QKU4aX8QKBgQCo\nufnozfBq2bouKHiHmVvdWEwv6Sm6sgOD4SI4URZyUiPwg2WV/itrdOnst8MECBsS\nczLJ5yCKexahpYnAzVgxn/WdFZcKj7MDMPZRNjRxBm+0kS7hzX6jJy3olBVsH+M6\n8fksMifsfVaR03iwIuxw2ZgVosxGshDArWV0GFkVKwKBgFnn1YGbKQM/9EnmO79d\nWrNsj3P9uzngeoi9eYQbzU8ow0qBkFch36iY9tVwxwpG4k8Job9q3LBBRtbdu0eE\n/snY1cMd9PVuh3HaBg/hFgA2cMnk3CG+7+wLmkonRSpdIhuQjQXbB7xMmCFImypK\nRO4iqN/2YDPT1z5UGJEDTkBH\n-----END PRIVATE KEY-----\n" }, { "path": "tests/clickhouse/rollup_tls/users.xml", "content": "\n\n \n \n \n \n \t 1\n \t 0\n\n \n 40000000000\n\n \n 1\n\n \n\n\t \n\t \n\t 1048576000\n\t 1000000\n\t 1000000\n\t 720000\n\t \n\t 6000000000000\n\t \n 40\n \n 16\n \n\t 25571520\n\t 1073741824\n\t \n random\n\t 1000\n \n\n \n \n 1\n \n \n\n \n \n \n \n \n \n\n \n \n ::/0\n \n\n \n default\n\n \n default\n \n\n \n \n \n \n ::1\n 127.0.0.1\n \n readonly\n default\n \n \n\n \n \n \n \n \n \n \n 3600\n\n \n 0\n 0\n 0\n 0\n 0\n \n \n \n\n" }, { "path": "tests/consolidateBy/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "tests/consolidateBy/graphite-clickhouse.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[feature-flags]\nuse-carbon-behaviour = false\ndont-match-missing-tags = true\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = true\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/consolidateBy/test.toml", "content": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"21.3\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"22.8\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\n\n[test.carbon_clickhouse]\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse.conf.tpl\"\n\n#######################################################################################\n\n[[test.input]]\nname = \"request_success_total.counter;app=test;project=Test;environment=TEST\"\npoints = [{value = 3.0, time = \"1000\"}, {value = 0.0, time = \"1010\"}, {value = 1.0, time = \"1020\"}, {value = 2.0, time = \"1030\"}]\n\n[[test.input]]\nname = \"request_success_total.counter;app=test;project=Test;environment=TEST;t=q\"\npoints = [{value = 3.0, time = \"1000\"}, {value = 0.0, time = \"1010\"}, {value = 1.0, time = \"1020\"}, {value = 2.0, time = \"1030\"}]\n\n[[test.input]]\nname = \"test;env=prod\"\npoints = [{value = 3.0, time = \"1000\"}, {value = 0.0, time = \"1010\"}, {value = 1.0, time = \"1020\"}, {value = 2.0, time = \"1030\"}]\n\n[[test.input]]\nname = \"test;env=dr\"\npoints = [{value = 3.0, time = \"1000\"}, {value = 0.0, time = \"1010\"}, {value = 1.0, time = \"1020\"}, {value = 2.0, time = \"1030\"}]\n\n# consolidateBy('max')\n\n[[test.render_checks]]\nfrom = \"1000\"\nuntil = \"1030\"\nmax_data_points = 2\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\", \n]\nfiltering_functions = [\n \"consolidateBy('max')\"\n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"max\"\nstart = \"1000\"\nstop = \"1040\"\nstep = 20\nreq_start = \"1000\"\nreq_stop = \"1040\"\nvalues = [3.0, 2.0]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"max\"\nstart = \"1000\"\nstop = \"1040\"\nstep = 20\nreq_start = \"1000\"\nreq_stop = \"1040\"\nvalues = [3.0, 2.0]\n\n# consolidateBy('min')\n\n[[test.render_checks]]\nfrom = \"1000\"\nuntil = \"1030\"\nmax_data_points = 2\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\", \n]\nfiltering_functions = [\n \"consolidateBy('min')\"\n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"min\"\nstart = \"1000\"\nstop = \"1040\"\nstep = 20\nreq_start = \"1000\"\nreq_stop = \"1040\"\nvalues = [0.0, 1.0]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"min\"\nstart = \"1000\"\nstop = \"1040\"\nstep = 20\nreq_start = \"1000\"\nreq_stop = \"1040\"\nvalues = [0.0, 1.0]\n\n\n# consolidateBy('sum')\n\n[[test.render_checks]]\nfrom = \"1000\"\nuntil = \"1030\"\nmax_data_points = 2\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\", \n]\nfiltering_functions = [\n \"consolidateBy('sum')\"\n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"sum\"\nstart = \"1000\"\nstop = \"1040\"\nstep = 20\nreq_start = \"1000\"\nreq_stop = \"1040\"\nvalues = [3.0, 3.0]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"sum\"\nstart = \"1000\"\nstop = \"1040\"\nstep = 20\nreq_start = \"1000\"\nreq_stop = \"1040\"\nvalues = [3.0, 3.0]\n\n# consolidateBy('avg')\n\n[[test.render_checks]]\nfrom = \"1000\"\nuntil = \"1030\"\nmax_data_points = 2\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\", \n]\nfiltering_functions = [\n \"consolidateBy('avg')\"\n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"avg\"\nstart = \"1000\"\nstop = \"1040\"\nstep = 20\nreq_start = \"1000\"\nreq_stop = \"1040\"\nvalues = [1.5, 1.5]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"avg\"\nstart = \"1000\"\nstop = \"1040\"\nstep = 20\nreq_start = \"1000\"\nreq_stop = \"1040\"\nvalues = [1.5, 1.5]\n\n# consolidateBy('average')\n\n[[test.render_checks]]\nfrom = \"1000\"\nuntil = \"1030\"\nmax_data_points = 2\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\", \n]\nfiltering_functions = [\n \"consolidateBy('average')\"\n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"avg\"\nstart = \"1000\"\nstop = \"1040\"\nstep = 20\nreq_start = \"1000\"\nreq_stop = \"1040\"\nvalues = [1.5, 1.5]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"avg\"\nstart = \"1000\"\nstop = \"1040\"\nstep = 20\nreq_start = \"1000\"\nreq_stop = \"1040\"\nvalues = [1.5, 1.5]\n\n# consolidateBy('last')\n\n[[test.render_checks]]\nfrom = \"1000\"\nuntil = \"1030\"\nmax_data_points = 2\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\", \n]\nfiltering_functions = [\n \"consolidateBy('last')\"\n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"last\"\nstart = \"1000\"\nstop = \"1040\"\nstep = 20\nreq_start = \"1000\"\nreq_stop = \"1040\"\nvalues = [0.0, 2.0]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"last\"\nstart = \"1000\"\nstop = \"1040\"\nstep = 20\nreq_start = \"1000\"\nreq_stop = \"1040\"\nvalues = [0.0, 2.0]\n\n# consolidateBy('first')\n\n[[test.render_checks]]\nfrom = \"1000\"\nuntil = \"1030\"\nmax_data_points = 2\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\", \n]\nfiltering_functions = [\n \"consolidateBy('first')\"\n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"first\"\nstart = \"1000\"\nstop = \"1040\"\nstep = 20\nreq_start = \"1000\"\nreq_stop = \"1040\"\nvalues = [3.0, 1.0]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"first\"\nstart = \"1000\"\nstop = \"1040\"\nstep = 20\nreq_start = \"1000\"\nreq_stop = \"1040\"\nvalues = [3.0, 1.0]\n\n# consolidateBy('invalid')\n\n[[test.render_checks]]\nfrom = \"1000\"\nuntil = \"1030\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\", \n]\nfiltering_functions = [\n \"consolidateBy('invalid')\"\n]\nerror_regexp = \"^400: failed to choose appropriate aggregation\"\n" }, { "path": "tests/consul.sh", "content": "#!/usr/bin/env bash\n\nif [ \"$1\" != \"\" ]; then\n wget -q https://releases.hashicorp.com/consul/${1}/consul_${1}_linux_amd64.zip || exit 1\n unzip consul_${1}_linux_amd64.zip || exit 1\nfi\n\n./consul agent -server -bootstrap -data-dir=/tmp/consul -bind=127.0.0.1\n" }, { "path": "tests/emptyseries_append/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "tests/emptyseries_append/graphite-clickhouse.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\nappend-empty-series = true\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = true\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/emptyseries_append/test.toml", "content": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"21.3\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"22.8\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\n\n[test.carbon_clickhouse]\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse.conf.tpl\"\n\n[[test.input]]\nname = \"test.avg\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow+21\"}, {value = 2.0, time = \"rnow+30\"}]\n\n[[test.input]]\nname = \"test.sum\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 2.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"test.min\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 2.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"test.max\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 2.0, time = \"rnow\", delay = \"5s\"}]\n\n# Test aggregations\n[[test.input]]\nname = \"test.avg\"\npoints = [{value = 0.0, time = \"rnow-30\"}, {value = 4.0, time = \"rnow+30\"}]\n\n# Test aggregations\n[[test.input]]\nname = \"test.sum\"\npoints = [{value = 0.0, time = \"rnow-1\"}, {value = 4.0, time = \"rnow+1\"}]\n\n# Test aggregations\n[[test.input]]\nname = \"test.min\"\npoints = [{value = 0.0, time = \"rnow-1\"}, {value = 4.0, time = \"rnow+1\"}]\n\n# Test aggregations\n[[test.input]]\nname = \"test.max\"\npoints = [{value = 0.0, time = \"rnow-1\"}, {value = 4.0, time = \"rnow+1\"}]\n\n##########################################################################\n# Aggregated, Deduplication not work with internal aggregation\n##########################################################################\n\n[[test.render_checks]]\nname = \"Test rollup\"\nfrom = \"rnow-10\"\nuntil = \"rnow+10\"\ntargets = [ \n \"test.{avg,min,max,sum}\"\n]\n\n[[test.render_checks.result]]\nname = \"test.avg\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"any\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [nan, nan, nan]\n\n[[test.render_checks.result]]\nname = \"test.sum\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"sum\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [1.0, 6.0, nan]\n\n[[test.render_checks.result]]\nname = \"test.min\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"min\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [0.0, 2.0, nan]\n\n[[test.render_checks.result]]\nname = \"test.max\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"max\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [1.0, 4.0, nan]\n\n# End - Test rollup\n##########################################################################\n" }, { "path": "tests/emptyseries_noappend/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "tests/emptyseries_noappend/graphite-clickhouse.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\nappend-empty-series = false\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = true\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/emptyseries_noappend/test.toml", "content": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"21.3\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"22.8\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\n\n[test.carbon_clickhouse]\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse.conf.tpl\"\n\n[[test.input]]\nname = \"test.avg\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow+21\"}, {value = 2.0, time = \"rnow+30\"}]\n\n[[test.input]]\nname = \"test.sum\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 2.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"test.min\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 2.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"test.max\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 2.0, time = \"rnow\", delay = \"5s\"}]\n\n# Test aggregations\n[[test.input]]\nname = \"test.avg\"\npoints = [{value = 0.0, time = \"rnow-30\"}, {value = 4.0, time = \"rnow+30\"}]\n\n# Test aggregations\n[[test.input]]\nname = \"test.sum\"\npoints = [{value = 0.0, time = \"rnow-1\"}, {value = 4.0, time = \"rnow+1\"}]\n\n# Test aggregations\n[[test.input]]\nname = \"test.min\"\npoints = [{value = 0.0, time = \"rnow-1\"}, {value = 4.0, time = \"rnow+1\"}]\n\n# Test aggregations\n[[test.input]]\nname = \"test.max\"\npoints = [{value = 0.0, time = \"rnow-1\"}, {value = 4.0, time = \"rnow+1\"}]\n\n##########################################################################\n# Aggregated, Deduplication not work with internal aggregation\n##########################################################################\n\n[[test.render_checks]]\nname = \"Test rollup\"\nfrom = \"rnow-10\"\nuntil = \"rnow+10\"\ntargets = [ \n \"test.{avg,min,max,sum}\"\n]\n\n[[test.render_checks.result]]\nname = \"test.sum\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"sum\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [1.0, 6.0, nan]\n\n[[test.render_checks.result]]\nname = \"test.min\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"min\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [0.0, 2.0, nan]\n\n[[test.render_checks.result]]\nname = \"test.max\"\npath = \"test.{avg,min,max,sum}\"\nconsolidation = \"max\"\nstart = \"rnow-10\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+20\"\nvalues = [1.0, 4.0, nan]\n\n# End - Test rollup\n##########################################################################\n" }, { "path": "tests/error_handling/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "tests/error_handling/graphite-clickhouse.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[clickhouse]\nurl = \"{{ .PROXY_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"1s\"\n\nquery-params = [\n {\n duration = \"1h\",\n url = \"{{ .PROXY_URL }}/?max_rows_to_read=1&max_result_bytes=1&readonly=2&log_queries=1\",\n data-timeout = \"5s\"\n },\n {\n duration = \"7h\",\n url = \"{{ .PROXY_URL }}/?max_memory_usage=1&max_memory_usage_for_user=1&readonly=2&log_queries=1\",\n data-timeout = \"5s\"\n }\n]\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1s\"\ninternal-aggregation = false\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/error_handling/test.toml", "content": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\n\n[test.carbon_clickhouse]\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse.conf.tpl\"\n\n[[test.input]]\nname = \"test.plain1\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 2.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"test.plain2\"\npoints = [{value = 2.0, time = \"rnow-30\"}, {value = 1.0, time = \"rnow-20\"}, {value = 1.5, time = \"rnow-10\"}, {value = 2.5, time = \"rnow\"}]\n\n[[test.input]]\nname = \"test2.plain\"\npoints = [{value = 1.0, time = \"rnow-30\"}, {value = 2.0, time = \"rnow-20\"}, {value = 2.5, time = \"rnow-10\"}, {value = 3.5, time = \"rnow\"}]\n\n[[test.input]]\nname = \"metric1;tag1=value1;tag2=value21;tag3=value3\"\npoints = [{value = 2.0, time = \"rnow-30\"}, {value = 2.5, time = \"rnow-20\"}, {value = 2.0, time = \"rnow-10\"}, {value = 3.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"metric1;tag2=value22;tag4=value4\"\npoints = [{value = 1.0, time = \"rnow-30\"}, {value = 2.0, time = \"rnow-20\"}, {value = 0.0, time = \"rnow-10\"}, {value = 1.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"metric1;tag1=value1;tag2=value23;tag3=value3\"\npoints = [{value = 0.5, time = \"rnow-30\"}, {value = 1.5, time = \"rnow-20\"}, {value = 4.0, time = \"rnow-10\"}, {value = 3.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"metric2;tag2=value21;tag4=value4\"\npoints = [{value = 2.0, time = \"rnow-30\"}, {value = 1.0, time = \"rnow-20\"}, {value = 0.0, time = \"rnow-10\"}, {value = 1.0, time = \"rnow\"}]\n\n[[test.find_checks]]\nquery = \"test\"\nresult = [ \n { path = \"test\", is_leaf = false }\n]\n\n# Check index-timeout\n[[test.find_checks]]\nquery = \"test\"\ntimeout = \"2s\"\nproxy_delay = \"1500ms\"\nerror_regexp = \"^504: Storage read timeout\"\n\n[[test.tags_checks]]\nquery = \"tag1;tag2=value21\"\nresult = [\n \"value1\"\n]\n\n# Check index-timeout\n[[test.tags_checks]]\nquery = \"tag1;tag2=value21\"\ntimeout = \"2s\"\nproxy_delay = \"1500ms\"\nerror_regexp = \"^504: Storage read timeout\"\n\n[[test.input]]\nname = \"test.long\"\npoints = [\n {value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 2.0, time = \"rnow\"},\n {value = 3.0, time = \"rnow-3600\"}, {value = 3.0, time = \"rnow-3590\"}, {value = 3.0, time = \"rnow-3580\"}, {value = 3.0, time = \"rnow-3570\"},\n {value = 3.0, time = \"rnow-3560\"}, {value = 3.0, time = \"rnow-3550\"}, {value = 3.0, time = \"rnow-3540\"}, {value = 3.0, time = \"rnow-3530\"},\n {value = 3.0, time = \"rnow-3520\"}, {value = 3.0, time = \"rnow-3510\"}, {value = 3.0, time = \"rnow-3500\"}, {value = 3.0, time = \"rnow-3490\"},\n {value = 3.0, time = \"rnow-14200\"}, {value = 3.0, time = \"rnow-14400\"},\n {value = 3.0, time = \"rnow-86200\"}, {value = 3.0, time = \"rnow-86400\"},\n {value = 3.0, time = \"rnow-172600\"}, {value = 3.0, time = \"rnow-172800\"},\n {value = 2.0, time = \"rnow-345200\"}, {value = 2.0, time = \"rnow-345210\"}, {value = 2.0, time = \"rnow-345220\"}, {value = 2.0, time = \"rnow-345230\"}, {value = 2.0, time = \"rnow-345240\"}, {value = 2.0, time = \"rnow-345250\"}, {value = 2.0, time = \"rnow-345260\"}, {value = 2.0, time = \"rnow-345270\"}, {value = 2.0, time = \"rnow-345280\"}, {value = 2.0, time = \"rnow-345290\"}, {value = 2.0, time = \"rnow-345300\"}, {value = 2.0, time = \"rnow-345310\"}, {value = 2.0, time = \"rnow-345320\"}, {value = 2.0, time = \"rnow-345330\"}, {value = 2.0, time = \"rnow-345340\"}, {value = 2.0, time = \"rnow-345350\"}, {value = 2.0, time = \"rnow-345360\"}, {value = 2.0, time = \"rnow-345370\"}, {value = 2.0, time = \"rnow-345380\"}, {value = 2.0, time = \"rnow-345390\"}, {value = 2.0, time = \"rnow-345400\"}, {value = 2.0, time = \"rnow-345410\"}, {value = 2.0, time = \"rnow-345420\"}, {value = 2.0, time = \"rnow-345430\"}, {value = 2.0, time = \"rnow-345440\"}, {value = 2.0, time = \"rnow-345450\"}, {value = 2.0, time = \"rnow-345460\"}, {value = 2.0, time = \"rnow-345470\"}, {value = 2.0, time = \"rnow-345480\"}, {value = 2.0, time = \"rnow-345490\"}, {value = 2.0, time = \"rnow-345500\"}, {value = 2.0, time = \"rnow-345510\"}, {value = 2.0, time = \"rnow-345520\"}, {value = 2.0, time = \"rnow-345530\"}, {value = 2.0, time = \"rnow-345540\"}, {value = 2.0, time = \"rnow-345550\"}, {value = 2.0, time = \"rnow-345560\"}, {value = 2.0, time = \"rnow-345570\"}, {value = 2.0, time = \"rnow-345580\"}, {value = 2.0, time = \"rnow-345590\"}, {value = 2.0, time = \"rnow-345600\"},\n {value = 2.0, time = \"rnow-431800\"}, {value = 2.0, time = \"rnow-431810\"}, {value = 2.0, time = \"rnow-431820\"}, {value = 2.0, time = \"rnow-431830\"}, {value = 2.0, time = \"rnow-431840\"}, {value = 2.0, time = \"rnow-431850\"}, {value = 2.0, time = \"rnow-431860\"}, {value = 2.0, time = \"rnow-431870\"}, {value = 2.0, time = \"rnow-431880\"}, {value = 2.0, time = \"rnow-431890\"}, {value = 2.0, time = \"rnow-431900\"}, {value = 2.0, time = \"rnow-431910\"}, {value = 2.0, time = \"rnow-431920\"}, {value = 2.0, time = \"rnow-431930\"}, {value = 2.0, time = \"rnow-431940\"}, {value = 2.0, time = \"rnow-431950\"}, {value = 2.0, time = \"rnow-431960\"}, {value = 2.0, time = \"rnow-431970\"}, {value = 2.0, time = \"rnow-431980\"}, {value = 2.0, time = \"rnow-431990\"}, {value = 2.0, time = \"rnow-432000\"}, {value = 2.0, time = \"rnow-432010\"}, {value = 2.0, time = \"rnow-432020\"}, {value = 2.0, time = \"rnow-432030\"}, {value = 2.0, time = \"rnow-432040\"}, {value = 2.0, time = \"rnow-432050\"}, {value = 2.0, time = \"rnow-432060\"}, {value = 2.0, time = \"rnow-432070\"}, {value = 2.0, time = \"rnow-432080\"}, {value = 2.0, time = \"rnow-432090\"}, {value = 2.0, time = \"rnow-432100\"}, {value = 2.0, time = \"rnow-432110\"}, {value = 2.0, time = \"rnow-432120\"}, {value = 2.0, time = \"rnow-432130\"}, {value = 2.0, time = \"rnow-432140\"}, {value = 2.0, time = \"rnow-432150\"}, {value = 2.0, time = \"rnow-432160\"}, {value = 2.0, time = \"rnow-432170\"}, {value = 2.0, time = \"rnow-432180\"}, {value = 2.0, time = \"rnow-432190\"}, {value = 2.0, time = \"rnow-432200\"},\n {value = 2.0, time = \"rnow-518000\"}, {value = 2.0, time = \"rnow-518010\"}, {value = 2.0, time = \"rnow-518020\"}, {value = 2.0, time = \"rnow-518030\"}, {value = 2.0, time = \"rnow-518040\"}, {value = 2.0, time = \"rnow-518050\"}, {value = 2.0, time = \"rnow-518060\"}, {value = 2.0, time = \"rnow-518070\"}, {value = 2.0, time = \"rnow-518080\"}, {value = 2.0, time = \"rnow-518090\"}, {value = 2.0, time = \"rnow-518100\"}, {value = 2.0, time = \"rnow-518110\"}, {value = 2.0, time = \"rnow-518120\"}, {value = 2.0, time = \"rnow-518130\"}, {value = 2.0, time = \"rnow-518140\"}, {value = 2.0, time = \"rnow-518150\"}, {value = 2.0, time = \"rnow-518160\"}, {value = 2.0, time = \"rnow-518170\"}, {value = 2.0, time = \"rnow-518180\"}, {value = 2.0, time = \"rnow-518190\"}, {value = 2.0, time = \"rnow-518200\"}, {value = 2.0, time = \"rnow-518210\"}, {value = 2.0, time = \"rnow-518220\"}, {value = 2.0, time = \"rnow-518230\"}, {value = 2.0, time = \"rnow-518240\"}, {value = 2.0, time = \"rnow-518250\"}, {value = 2.0, time = \"rnow-518260\"}, {value = 2.0, time = \"rnow-518270\"}, {value = 2.0, time = \"rnow-518280\"}, {value = 2.0, time = \"rnow-518290\"}, {value = 2.0, time = \"rnow-518300\"}, {value = 2.0, time = \"rnow-518310\"}, {value = 2.0, time = \"rnow-518320\"}, {value = 2.0, time = \"rnow-518330\"}, {value = 2.0, time = \"rnow-518340\"}, {value = 2.0, time = \"rnow-518350\"}, {value = 2.0, time = \"rnow-518360\"}, {value = 2.0, time = \"rnow-518370\"}, {value = 2.0, time = \"rnow-518380\"}, {value = 2.0, time = \"rnow-518390\"}, {value = 2.0, time = \"rnow-518400\"},\n {value = 3.0, time = \"rnow-604400\"}, {value = 3.0, time = \"rnow-604410\"}, {value = 3.0, time = \"rnow-604420\"}, {value = 3.0, time = \"rnow-604430\"}, {value = 3.0, time = \"rnow-604440\"}, {value = 3.0, time = \"rnow-604450\"}, {value = 3.0, time = \"rnow-604460\"}, {value = 3.0, time = \"rnow-604470\"}, {value = 3.0, time = \"rnow-604480\"}, {value = 3.0, time = \"rnow-604490\"}, {value = 3.0, time = \"rnow-604500\"}, {value = 3.0, time = \"rnow-604510\"}, {value = 3.0, time = \"rnow-604520\"}, {value = 3.0, time = \"rnow-604530\"}, {value = 3.0, time = \"rnow-604540\"}, {value = 3.0, time = \"rnow-604550\"}, {value = 3.0, time = \"rnow-604560\"}, {value = 3.0, time = \"rnow-604570\"}, {value = 3.0, time = \"rnow-604580\"}, {value = 3.0, time = \"rnow-604590\"}, {value = 3.0, time = \"rnow-604600\"}, {value = 3.0, time = \"rnow-604610\"}, {value = 3.0, time = \"rnow-604620\"}, {value = 3.0, time = \"rnow-604630\"}, {value = 3.0, time = \"rnow-604640\"}, {value = 3.0, time = \"rnow-604650\"}, {value = 3.0, time = \"rnow-604660\"}, {value = 3.0, time = \"rnow-604670\"}, {value = 3.0, time = \"rnow-604680\"}, {value = 3.0, time = \"rnow-604690\"}, {value = 3.0, time = \"rnow-604700\"}, {value = 3.0, time = \"rnow-604710\"}, {value = 3.0, time = \"rnow-604720\"}, {value = 3.0, time = \"rnow-604730\"}, {value = 3.0, time = \"rnow-604740\"}, {value = 3.0, time = \"rnow-604750\"}, {value = 3.0, time = \"rnow-604760\"}, {value = 3.0, time = \"rnow-604770\"}, {value = 3.0, time = \"rnow-604780\"}, {value = 3.0, time = \"rnow-604790\"}, {value = 3.0, time = \"rnow-604800\"},\n {value = 2.0, time = \"rnow-690800\"}, {value = 2.0, time = \"rnow-690810\"}, {value = 2.0, time = \"rnow-690820\"}, {value = 2.0, time = \"rnow-690830\"}, {value = 2.0, time = \"rnow-690840\"}, {value = 2.0, time = \"rnow-690850\"}, {value = 2.0, time = \"rnow-690860\"}, {value = 2.0, time = \"rnow-690870\"}, {value = 2.0, time = \"rnow-690880\"}, {value = 2.0, time = \"rnow-690890\"}, {value = 2.0, time = \"rnow-690900\"}, {value = 2.0, time = \"rnow-690910\"}, {value = 2.0, time = \"rnow-690920\"}, {value = 2.0, time = \"rnow-690930\"}, {value = 2.0, time = \"rnow-690940\"}, {value = 2.0, time = \"rnow-690950\"}, {value = 2.0, time = \"rnow-690960\"}, {value = 2.0, time = \"rnow-690970\"}, {value = 2.0, time = \"rnow-690980\"}, {value = 2.0, time = \"rnow-690990\"}, {value = 2.0, time = \"rnow-691000\"}, {value = 2.0, time = \"rnow-691010\"}, {value = 2.0, time = \"rnow-691020\"}, {value = 2.0, time = \"rnow-691030\"}, {value = 2.0, time = \"rnow-691040\"}, {value = 2.0, time = \"rnow-691050\"}, {value = 2.0, time = \"rnow-691060\"}, {value = 2.0, time = \"rnow-691070\"}, {value = 2.0, time = \"rnow-691080\"}, {value = 2.0, time = \"rnow-691090\"}, {value = 2.0, time = \"rnow-691100\"}, {value = 2.0, time = \"rnow-691110\"}, {value = 2.0, time = \"rnow-691120\"}, {value = 2.0, time = \"rnow-691130\"}, {value = 2.0, time = \"rnow-691140\"}, {value = 2.0, time = \"rnow-691150\"}, {value = 2.0, time = \"rnow-691160\"}, {value = 2.0, time = \"rnow-691170\"}, {value = 2.0, time = \"rnow-691180\"}, {value = 2.0, time = \"rnow-691190\"}, {value = 2.0, time = \"rnow-691200\"},\n {value = 2.0, time = \"rnow-777200\"}, {value = 2.0, time = \"rnow-777210\"}, {value = 2.0, time = \"rnow-777220\"}, {value = 2.0, time = \"rnow-777230\"}, {value = 2.0, time = \"rnow-777240\"}, {value = 2.0, time = \"rnow-777250\"}, {value = 2.0, time = \"rnow-777260\"}, {value = 2.0, time = \"rnow-777270\"}, {value = 2.0, time = \"rnow-777280\"}, {value = 2.0, time = \"rnow-777290\"}, {value = 2.0, time = \"rnow-777300\"}, {value = 2.0, time = \"rnow-777310\"}, {value = 2.0, time = \"rnow-777320\"}, {value = 2.0, time = \"rnow-777330\"}, {value = 2.0, time = \"rnow-777340\"}, {value = 2.0, time = \"rnow-777350\"}, {value = 2.0, time = \"rnow-777360\"}, {value = 2.0, time = \"rnow-777370\"}, {value = 2.0, time = \"rnow-777380\"}, {value = 2.0, time = \"rnow-777390\"}, {value = 2.0, time = \"rnow-777400\"}, {value = 2.0, time = \"rnow-777410\"}, {value = 2.0, time = \"rnow-777420\"}, {value = 2.0, time = \"rnow-777430\"}, {value = 2.0, time = \"rnow-777440\"}, {value = 2.0, time = \"rnow-777450\"}, {value = 2.0, time = \"rnow-777460\"}, {value = 2.0, time = \"rnow-777470\"}, {value = 2.0, time = \"rnow-777480\"}, {value = 2.0, time = \"rnow-777490\"}, {value = 2.0, time = \"rnow-777500\"}, {value = 2.0, time = \"rnow-777510\"}, {value = 2.0, time = \"rnow-777520\"}, {value = 2.0, time = \"rnow-777530\"}, {value = 2.0, time = \"rnow-777540\"}, {value = 2.0, time = \"rnow-777550\"}, {value = 2.0, time = \"rnow-777560\"}, {value = 2.0, time = \"rnow-777570\"}, {value = 2.0, time = \"rnow-777580\"}, {value = 2.0, time = \"rnow-777590\"}, {value = 2.0, time = \"rnow-777600\"},\n {value = 3.0, time = \"rnow-863800\"}, {value = 3.0, time = \"rnow-864000\"}\n]\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntargets = [ \"test.long\" ]\n\n[[test.render_checks.result]]\nname = \"test.long\"\npath = \"test.long\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, 2.0]\n\n# Check addional queryparam (storage read limit)\n[[test.render_checks]]\nfrom = \"rnow-21600\"\nuntil = \"rnow\"\ntargets = [ \"test.long\" ]\ntimeout = \"5s\"\nerror_regexp = \"^403: Storage read limit for rows\"\n\n# Check data-timeout on addional queryparam\n[[test.render_checks]]\nfrom = \"rnow-14200\"\nuntil = \"rnow\"\ntargets = [ \"test.long\" ]\ntimeout = \"2s\"\nproxy_delay = \"1500ms\"\nerror_regexp = \"^504: Storage read timeout\"\n\n# Check addional queryparam (storage read limit)\n[[test.render_checks]]\nfrom = \"rnow-864000\"\nuntil = \"rnow\"\ntargets = [ \"test.long\" ]\ntimeout = \"40s\"\nerror_regexp = \"^403: Storage read limit for memory\"\n" }, { "path": "tests/feature_flags_both_true/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "tests/feature_flags_both_true/graphite-clickhouse.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[feature-flags]\nuse-carbon-behaviour = true\ndont-match-missing-tags = true\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = true\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/feature_flags_both_true/test.toml", "content": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"21.3\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"22.8\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\n\n[test.carbon_clickhouse]\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse.conf.tpl\"\n\n#######################################################################################\n\n[[test.input]]\nname = \"request_success_total.counter;app=test;project=Test;environment=TEST\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"request_success_total.counter;app=test;project=Test;environment=TEST;t=q\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"request_success_total.counter;app=test;project=Test;environment=TEST;t=qac\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"request_success_total.counter;app=test;project=Test;environment=TEST;t=cqa\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"test;env=prod\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"test;env=dr\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n## seriesByTag('t=')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('t=')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('t=')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=dr\"\npath = \"seriesByTag('t=')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=prod\"\npath = \"seriesByTag('t=')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~')\", \n]\n\n### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n### seriesByTag('dc!=ru') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('dc!=ru')\", \n]\n\n## seriesByTag('name=request_success_total.counter', 'dc!=ru') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'dc!=ru')\", \n]\n\n## seriesByTag('dc!=~ru') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('dc!=~ru')\", \n]\n\n## seriesByTag('name=request_success_total.counter','dc!=~ru')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter','dc!=~ru')\", \n]\n\n### seriesByTag('t!=~qac') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('t!=~qac')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('t!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('t!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n## seriesByTag('name=request_success_total.counter', 't!=~qac')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 't!=~qac')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 't!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 't!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')\", \n]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')\", \n]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~q*') ###\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ## seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=test') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=test')\", \n]\n\n[[test.render_checks.result]]\nname = \"test;env=dr\"\npath = \"seriesByTag('name=test')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=prod\"\npath = \"seriesByTag('name=test')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=test','env!=~stage|env') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=test','env!=~stage|env')\", \n]\n\n[[test.render_checks.result]]\nname = \"test;env=dr\"\npath = \"seriesByTag('name=test','env!=~stage|env')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=prod\"\npath = \"seriesByTag('name=test','env!=~stage|env')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*a*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*a*')\", \n]\nerror_regexp = \"^400: Incorrect regex syntax\"\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# # End - Test no flags\n# #########################################################################\n\n" }, { "path": "tests/feature_flags_dont_match_missing_tags/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "tests/feature_flags_dont_match_missing_tags/graphite-clickhouse.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[feature-flags]\ndont-match-missing-tags = true\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = true\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/feature_flags_dont_match_missing_tags/test.toml", "content": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"21.3\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"22.8\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\n\n[test.carbon_clickhouse]\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse.conf.tpl\"\n\n#######################################################################################\n\n[[test.input]]\nname = \"request_success_total.counter;app=test;project=Test;environment=TEST\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"request_success_total.counter;app=test;project=Test;environment=TEST;t=q\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"request_success_total.counter;app=test;project=Test;environment=TEST;t=qac\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"request_success_total.counter;app=test;project=Test;environment=TEST;t=cqa\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"test;env=prod\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"test;env=dr\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=')\", \n]\n\n## seriesByTag('t=')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('t=')\", \n]\n\n# seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~')\", \n]\n\n### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n### seriesByTag('dc!=ru') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('dc!=ru')\", \n]\n\n## seriesByTag('name=request_success_total.counter', 'dc!=ru') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'dc!=ru')\", \n]\n\n## seriesByTag('dc!=~ru') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('dc!=~ru')\", \n]\n\n## seriesByTag('name=request_success_total.counter','dc!=~ru')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter','dc!=~ru')\", \n]\n\n### seriesByTag('t!=~qac') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('t!=~qac')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('t!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('t!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n## seriesByTag('name=request_success_total.counter', 't!=~qac')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 't!=~qac')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 't!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 't!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')\", \n]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')\", \n]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~q*') ###\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ## seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=test') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=test')\", \n]\n\n[[test.render_checks.result]]\nname = \"test;env=dr\"\npath = \"seriesByTag('name=test')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=prod\"\npath = \"seriesByTag('name=test')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=test','env!=~stage|env') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=test','env!=~stage|env')\", \n]\n\n[[test.render_checks.result]]\nname = \"test;env=dr\"\npath = \"seriesByTag('name=test','env!=~stage|env')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=prod\"\npath = \"seriesByTag('name=test','env!=~stage|env')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*a*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*a*')\", \n]\nerror_regexp = \"^400: Incorrect regex syntax\"\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# # End - Test no flags\n# #########################################################################\n\n" }, { "path": "tests/feature_flags_false/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "tests/feature_flags_false/graphite-clickhouse.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = true\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/feature_flags_false/test.toml", "content": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"21.3\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"22.8\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\n\n[test.carbon_clickhouse]\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse.conf.tpl\"\n\n#######################################################################################\n\n[[test.input]]\nname = \"request_success_total.counter;app=test;project=Test;environment=TEST\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"request_success_total.counter;app=test;project=Test;environment=TEST;t=q\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"request_success_total.counter;app=test;project=Test;environment=TEST;t=qac\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"request_success_total.counter;app=test;project=Test;environment=TEST;t=cqa\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"test;env=prod\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"test;env=dr\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=')\", \n]\n\n## seriesByTag('t=')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('t=')\", \n]\n\n# seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n### seriesByTag('dc!=ru') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('dc!=ru')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('dc!=ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('dc!=ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('dc!=ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('dc!=ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=dr\"\npath = \"seriesByTag('dc!=ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=prod\"\npath = \"seriesByTag('dc!=ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n## seriesByTag('name=request_success_total.counter', 'dc!=ru') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'dc!=ru')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 'dc!=ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'dc!=ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'dc!=ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'dc!=ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n## seriesByTag('dc!=~ru') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('dc!=~ru')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('dc!=~ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('dc!=~ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('dc!=~ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('dc!=~ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=dr\"\npath = \"seriesByTag('dc!=~ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=prod\"\npath = \"seriesByTag('dc!=~ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n## seriesByTag('name=request_success_total.counter','dc!=~ru')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter','dc!=~ru')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter','dc!=~ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter','dc!=~ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter','dc!=~ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter','dc!=~ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n### seriesByTag('t!=~qac') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('t!=~qac')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('t!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('t!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('t!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=dr\"\npath = \"seriesByTag('t!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=prod\"\npath = \"seriesByTag('t!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n## seriesByTag('name=request_success_total.counter', 't!=~qac')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 't!=~qac')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 't!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 't!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 't!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ## seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=test') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=test')\", \n]\n\n[[test.render_checks.result]]\nname = \"test;env=dr\"\npath = \"seriesByTag('name=test')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=prod\"\npath = \"seriesByTag('name=test')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=test','env!=~stage|env') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=test','env!=~stage|env')\", \n]\n\n[[test.render_checks.result]]\nname = \"test;env=dr\"\npath = \"seriesByTag('name=test','env!=~stage|env')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=prod\"\npath = \"seriesByTag('name=test','env!=~stage|env')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*a*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*a*')\", \n]\nerror_regexp = \"^400: Incorrect regex syntax\"\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# # End - Test no flags\n# #########################################################################\n\n" }, { "path": "tests/feature_flags_use_carbon_behaviour/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "tests/feature_flags_use_carbon_behaviour/graphite-clickhouse.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[feature-flags]\nuse-carbon-behaviour = true\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = true\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/feature_flags_use_carbon_behaviour/test.toml", "content": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"21.3\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"22.8\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\n\n[test.carbon_clickhouse]\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse.conf.tpl\"\n\n#######################################################################################\n\n[[test.input]]\nname = \"request_success_total.counter;app=test;project=Test;environment=TEST\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"request_success_total.counter;app=test;project=Test;environment=TEST;t=q\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"request_success_total.counter;app=test;project=Test;environment=TEST;t=qac\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"request_success_total.counter;app=test;project=Test;environment=TEST;t=cqa\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"test;env=prod\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"test;env=dr\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n## seriesByTag('t=')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('t=')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('t=')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=dr\"\npath = \"seriesByTag('t=')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=prod\"\npath = \"seriesByTag('t=')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n### seriesByTag('dc!=ru') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('dc!=ru')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('dc!=ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('dc!=ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('dc!=ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('dc!=ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=dr\"\npath = \"seriesByTag('dc!=ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=prod\"\npath = \"seriesByTag('dc!=ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n## seriesByTag('name=request_success_total.counter', 'dc!=ru') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'dc!=ru')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 'dc!=ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'dc!=ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'dc!=ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'dc!=ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n## seriesByTag('dc!=~ru') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('dc!=~ru')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('dc!=~ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('dc!=~ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('dc!=~ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('dc!=~ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=dr\"\npath = \"seriesByTag('dc!=~ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=prod\"\npath = \"seriesByTag('dc!=~ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n## seriesByTag('name=request_success_total.counter','dc!=~ru')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter','dc!=~ru')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter','dc!=~ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter','dc!=~ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter','dc!=~ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter','dc!=~ru')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n### seriesByTag('t!=~qac') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('t!=~qac')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('t!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('t!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('t!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=dr\"\npath = \"seriesByTag('t!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=prod\"\npath = \"seriesByTag('t!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n## seriesByTag('name=request_success_total.counter', 't!=~qac')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 't!=~qac')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 't!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 't!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 't!=~qac')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ## seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=test') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=test')\", \n]\n\n[[test.render_checks.result]]\nname = \"test;env=dr\"\npath = \"seriesByTag('name=test')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=prod\"\npath = \"seriesByTag('name=test')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=test','env!=~stage|env') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=test','env!=~stage|env')\", \n]\n\n[[test.render_checks.result]]\nname = \"test;env=dr\"\npath = \"seriesByTag('name=test','env!=~stage|env')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=prod\"\npath = \"seriesByTag('name=test','env!=~stage|env')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*a*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*a*')\", \n]\nerror_regexp = \"^400: Incorrect regex syntax\"\n\n# ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*') ###\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')\", \n]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=q\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"request_success_total.counter;app=test;environment=TEST;project=Test;t=qac\"\npath = \"seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# # End - Test no flags\n# #########################################################################\n\n" }, { "path": "tests/find_cache/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "tests/find_cache/graphite-clickhouse-cached.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[common.find-cache]\ntype = \"mem\"\nsize-mb = 1\ndefault-timeout = 300\nshort-timeout = 60\nshort-duration = \"240s\"\nfind-timeout = 120\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = false\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/find_cache/graphite-clickhouse-internal-aggr-cached.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[common.find-cache]\ntype = \"mem\"\nsize-mb = 1\ndefault-timeout = 300\nshort-timeout = 60\nshort-duration = \"240s\"\nfind-timeout = 120\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = true\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/find_cache/test.toml", "content": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\ndelay = \"10s\"\n\n[test.carbon_clickhouse]\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse-cached.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse-internal-aggr-cached.conf.tpl\"\n\n##########################################################################\n[[test.input]]\nname = \"test.cache\"\npoints = [{value = 1.0, time = \"midnight-270s\"}, {value = 3.0, time = \"now\"}]\n\n[[test.input]]\nname = \"cache;scope=test\"\npoints = [{value = 2.0, time = \"midnight-270s\"}, {value = 4.0, time = \"now\"}]\n\n##########################################################################\n[[test.find_checks]]\nquery = \"test\"\ncache_ttl = 120\nresult = [{ path = \"test\", is_leaf = false }]\n\n[[test.find_checks]]\nquery = \"test.cache\"\ncache_ttl = 120\nresult = [{ path = \"test.cache\", is_leaf = true }]\n\n##########################################################################\n\n[[test.tags_checks]]\nquery = \"name;scope=test\"\ncache_ttl = 120\nresult = [\n \"cache\",\n]\n\n##########################################################################\n# Short cache TTL\n\n[[test.render_checks]]\nfrom = \"rnow\"\nuntil = \"rnow+10\"\ncache_ttl = 60\ntargets = [ \"test.cache\" ]\n\n[[test.render_checks.result]]\nname = \"test.cache\"\npath = \"test.cache\"\nconsolidation = \"avg\"\nstart = \"rnow\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow\"\nreq_stop = \"rnow+20\"\nvalues = [3.0, nan]\n\n# Already in find cache\n[[test.render_checks]]\nfrom = \"rnow\"\nuntil = \"rnow+20\"\nin_cache = true\ncache_ttl = 60\ntargets = [ \"test.cache\" ]\n\n[[test.render_checks.result]]\nname = \"test.cache\"\npath = \"test.cache\"\nconsolidation = \"avg\"\nstart = \"rnow\"\nstop = \"rnow+30\"\nstep = 10\nreq_start = \"rnow\"\nreq_stop = \"rnow+30\"\nvalues = [3.0, nan, nan]\n\n##########################################################################\n# Short cache TTL\n\n[[test.render_checks]]\nfrom = \"rnow\"\nuntil = \"rnow+10\"\ncache_ttl = 60\ntargets = [ \n \"seriesByTag('scope=test')\"\n]\n\n[[test.render_checks.result]]\nname = \"cache;scope=test\"\npath = \"seriesByTag('scope=test')\"\nconsolidation = \"avg\"\nstart = \"rnow\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow\"\nreq_stop = \"rnow+20\"\nvalues = [4.0, nan]\n\n# Already in find cache\n[[test.render_checks]]\nfrom = \"rnow\"\nuntil = \"rnow+20\"\nin_cache = true\ncache_ttl = 60\ntargets = [ \n \"seriesByTag('scope=test')\"\n]\n\n[[test.render_checks.result]]\nname = \"cache;scope=test\"\npath = \"seriesByTag('scope=test')\"\nconsolidation = \"avg\"\nstart = \"rnow\"\nstop = \"rnow+30\"\nstep = 10\nreq_start = \"rnow\"\nreq_stop = \"rnow+30\"\nvalues = [4.0, nan, nan]\n\n##########################################################################\n# Default cache TTL\n\n[[test.render_checks]]\nfrom = \"midnight-270s\"\nuntil = \"midnight-20s\"\ncache_ttl = 300\ntargets = [ \"test.cache\" ]\n\n[[test.render_checks.result]]\nname = \"test.cache\"\npath = \"test.cache\"\nconsolidation = \"avg\"\nstart = \"midnight-270s\"\nstop = \"midnight-10s\"\nstep = 10\nreq_start = \"midnight-270s\"\nreq_stop = \"midnight-10s\"\nvalues = [1.0, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan]\n\n# Already in find cache\n[[test.render_checks]]\nfrom = \"midnight-270s\"\nuntil = \"midnight-10s\"\nin_cache = true\ncache_ttl = 300\ntargets = [ \"test.cache\" ]\n\n[[test.render_checks.result]]\nname = \"test.cache\"\npath = \"test.cache\"\nconsolidation = \"avg\"\nstart = \"midnight-270s\"\nstop = \"midnight\"\nstep = 10\nreq_start = \"midnight-270s\"\nreq_stop = \"midnight\"\nvalues = [1.0, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan]\n\n# Fetch points from 2 days, not in cache\n[[test.render_checks]]\nfrom = \"midnight-270s\"\nuntil = \"midnight\"\ncache_ttl = 300\ntargets = [ \"test.cache\" ]\n\n[[test.render_checks.result]]\nname = \"test.cache\"\npath = \"test.cache\"\nconsolidation = \"avg\"\nstart = \"midnight-270s\"\nstop = \"midnight+10s\"\nstep = 10\nreq_start = \"midnight-270s\"\nreq_stop = \"midnight+10s\"\nvalues = [1.0, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan]\n\n##########################################################################\n" }, { "path": "tests/limitera/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "tests/limitera/graphite-clickhouse-internal-aggr-cached.conf.tpl", "content": "# Adaptive limiter with throttle queries and limit max queries\n\n[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[common.find-cache]\ntype = \"mem\"\nsize-mb = 1\ndefault-timeout = 300\nshort-timeout = 60\nshort-duration = \"240s\"\nfind-timeout = 120\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = true\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\nrender-max-concurrent = 6\nrender-adaptive-queries = 2\nfind-max-concurrent = 4\nfind-adaptive-queries = 2\ntags-max-concurrent = 4\ntags-adaptive-queries = 2\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/limitera/test.toml", "content": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\ndelay = \"10s\"\n\n[test.carbon_clickhouse]\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse-internal-aggr-cached.conf.tpl\"\n\n##########################################################################\n[[test.input]]\nname = \"test.cache\"\npoints = [{value = 1.0, time = \"midnight-270s\"}, {value = 3.0, time = \"now\"}]\n\n[[test.input]]\nname = \"cache;scope=test\"\npoints = [{value = 2.0, time = \"midnight-270s\"}, {value = 4.0, time = \"now\"}]\n\n##########################################################################\n[[test.find_checks]]\nquery = \"test\"\nresult = [{ path = \"test\", is_leaf = false }]\n\n[[test.find_checks]]\nquery = \"test.cache\"\nresult = [{ path = \"test.cache\", is_leaf = true }]\n\n##########################################################################\n\n[[test.tags_checks]]\nquery = \"name;scope=test\"\nresult = [\n \"cache\",\n]\n\n##########################################################################\n\n[[test.render_checks]]\nfrom = \"rnow\"\nuntil = \"rnow+10\"\ntargets = [ \"test.cache\" ]\n\n[[test.render_checks.result]]\nname = \"test.cache\"\npath = \"test.cache\"\nconsolidation = \"avg\"\nstart = \"rnow\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow\"\nreq_stop = \"rnow+20\"\nvalues = [3.0, nan]\n\n##########################################################################\n" }, { "path": "tests/limitermax/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "tests/limitermax/graphite-clickhouse-internal-aggr-cached.conf.tpl", "content": "# Limiter with limit max connections\n\n[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[common.find-cache]\ntype = \"mem\"\nsize-mb = 1\ndefault-timeout = 300\nshort-timeout = 60\nshort-duration = \"240s\"\nfind-timeout = 120\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = true\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\nrender-max-queries = 100\nfind-max-queries = 50\ntags-max-queries = 50\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/limitermax/test.toml", "content": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\ndelay = \"10s\"\n\n[test.carbon_clickhouse]\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse-internal-aggr-cached.conf.tpl\"\n\n##########################################################################\n[[test.input]]\nname = \"test.cache\"\npoints = [{value = 1.0, time = \"midnight-270s\"}, {value = 3.0, time = \"now\"}]\n\n[[test.input]]\nname = \"cache;scope=test\"\npoints = [{value = 2.0, time = \"midnight-270s\"}, {value = 4.0, time = \"now\"}]\n\n##########################################################################\n[[test.find_checks]]\nquery = \"test\"\nresult = [{ path = \"test\", is_leaf = false }]\n\n[[test.find_checks]]\nquery = \"test.cache\"\nresult = [{ path = \"test.cache\", is_leaf = true }]\n\n##########################################################################\n\n[[test.tags_checks]]\nquery = \"name;scope=test\"\nresult = [\n \"cache\",\n]\n\n##########################################################################\n\n[[test.render_checks]]\nfrom = \"rnow\"\nuntil = \"rnow+10\"\ntargets = [ \"test.cache\" ]\n\n[[test.render_checks.result]]\nname = \"test.cache\"\npath = \"test.cache\"\nconsolidation = \"avg\"\nstart = \"rnow\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow\"\nreq_stop = \"rnow+20\"\nvalues = [3.0, nan]\n\n##########################################################################\n" }, { "path": "tests/limiterw/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "tests/limiterw/graphite-clickhouse-internal-aggr-cached.conf.tpl", "content": "# Limiter with throttle queries\n\n[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[common.find-cache]\ntype = \"mem\"\nsize-mb = 1\ndefault-timeout = 300\nshort-timeout = 60\nshort-duration = \"240s\"\nfind-timeout = 120\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = true\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\nrender-max-concurrent = 6\nfind-max-concurrent = 4\ntags-max-concurrent = 4\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/limiterw/test.toml", "content": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\ndelay = \"10s\"\n\n[test.carbon_clickhouse]\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse-internal-aggr-cached.conf.tpl\"\n\n##########################################################################\n[[test.input]]\nname = \"test.cache\"\npoints = [{value = 1.0, time = \"midnight-270s\"}, {value = 3.0, time = \"now\"}]\n\n[[test.input]]\nname = \"cache;scope=test\"\npoints = [{value = 2.0, time = \"midnight-270s\"}, {value = 4.0, time = \"now\"}]\n\n##########################################################################\n[[test.find_checks]]\nquery = \"test\"\nresult = [{ path = \"test\", is_leaf = false }]\n\n[[test.find_checks]]\nquery = \"test.cache\"\nresult = [{ path = \"test.cache\", is_leaf = true }]\n\n##########################################################################\n\n[[test.tags_checks]]\nquery = \"name;scope=test\"\nresult = [\n \"cache\",\n]\n\n##########################################################################\n\n[[test.render_checks]]\nfrom = \"rnow\"\nuntil = \"rnow+10\"\ntargets = [ \"test.cache\" ]\n\n[[test.render_checks.result]]\nname = \"test.cache\"\npath = \"test.cache\"\nconsolidation = \"avg\"\nstart = \"rnow\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow\"\nreq_stop = \"rnow+20\"\nvalues = [3.0, nan]\n\n##########################################################################\n" }, { "path": "tests/limiterwn/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "tests/limiterwn/graphite-clickhouse-internal-aggr-cached.conf.tpl", "content": "# Limiter with throttle queries and limit max queries\n\n[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[common.find-cache]\ntype = \"mem\"\nsize-mb = 1\ndefault-timeout = 300\nshort-timeout = 60\nshort-duration = \"240s\"\nfind-timeout = 120\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = true\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\nrender-max-queries = 100\nrender-max-concurrent = 6\nfind-max-queries = 50\nfind-max-concurrent = 4\ntags-max-queries = 50\ntags-max-concurrent = 4\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/limiterwn/test.toml", "content": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\ndelay = \"10s\"\n\n[test.carbon_clickhouse]\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse-internal-aggr-cached.conf.tpl\"\n\n##########################################################################\n[[test.input]]\nname = \"test.cache\"\npoints = [{value = 1.0, time = \"midnight-270s\"}, {value = 3.0, time = \"now\"}]\n\n[[test.input]]\nname = \"cache;scope=test\"\npoints = [{value = 2.0, time = \"midnight-270s\"}, {value = 4.0, time = \"now\"}]\n\n##########################################################################\n[[test.find_checks]]\nquery = \"test\"\nresult = [{ path = \"test\", is_leaf = false }]\n\n[[test.find_checks]]\nquery = \"test.cache\"\nresult = [{ path = \"test.cache\", is_leaf = true }]\n\n##########################################################################\n\n[[test.tags_checks]]\nquery = \"name;scope=test\"\nresult = [\n \"cache\",\n]\n\n##########################################################################\n\n[[test.render_checks]]\nfrom = \"rnow\"\nuntil = \"rnow+10\"\ntargets = [ \"test.cache\" ]\n\n[[test.render_checks.result]]\nname = \"test.cache\"\npath = \"test.cache\"\nconsolidation = \"avg\"\nstart = \"rnow\"\nstop = \"rnow+20\"\nstep = 10\nreq_start = \"rnow\"\nreq_stop = \"rnow+20\"\nvalues = [3.0, nan]\n\n##########################################################################\n" }, { "path": "tests/one_table/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "tests/one_table/graphite-clickhouse-internal-aggr.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = true\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/one_table/graphite-clickhouse.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = false\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/one_table/test.toml", "content": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"21.3\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"22.8\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\n\n[test.carbon_clickhouse]\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse-internal-aggr.conf.tpl\"\n\n[[test.input]]\nname = \"test.plain1\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 2.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"test.plain2\"\npoints = [{value = 2.0, time = \"rnow-30\"}, {value = 1.0, time = \"rnow-20\"}, {value = 1.5, time = \"rnow-10\"}, {value = 2.5, time = \"rnow\"}]\n\n[[test.input]]\nname = \"test2.plain\"\npoints = [{value = 1.0, time = \"rnow-30\"}, {value = 2.0, time = \"rnow-20\"}, {value = 2.5, time = \"rnow-10\"}, {value = 3.5, time = \"rnow\"}]\n\n[[test.input]]\nname = \"metric1;tag1=value1;tag2=value21;tag3=value3\"\npoints = [{value = 2.0, time = \"rnow-30\"}, {value = 2.5, time = \"rnow-20\"}, {value = 2.0, time = \"rnow-10\"}, {value = 3.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"metric1;tag2=value22;tag4=value4\"\npoints = [{value = 1.0, time = \"rnow-30\"}, {value = 2.0, time = \"rnow-20\"}, {value = 0.0, time = \"rnow-10\"}, {value = 1.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"metric1;tag1=value1;tag2=value23;tag3=value3\"\npoints = [{value = 0.5, time = \"rnow-30\"}, {value = 1.5, time = \"rnow-20\"}, {value = 4.0, time = \"rnow-10\"}, {value = 3.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"metric2;tag2=value21;tag4=value4\"\npoints = [{value = 2.0, time = \"rnow-30\"}, {value = 1.0, time = \"rnow-20\"}, {value = 0.0, time = \"rnow-10\"}, {value = 1.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"test_metric;minus=-;plus=+;percent=%;underscore=_;colon=:;hash=#;forward=/;host=127.0.0.1\"\npoints = [{value = 2.1, time = \"rnow-30\"}, {value = 0.1, time = \"rnow-20\"}, {value = 0.2, time = \"rnow-10\"}, {value = 1.5, time = \"rnow\"}]\n\n######################################\n# Check metrics find\n\n[[test.find_checks]]\nformats = [ \"pickle\", \"protobuf\", \"carbonapi_v3_pb\" ]\nquery = \"test\"\nresult = [ \n { path = \"test\", is_leaf = false }\n]\n\n[[test.find_checks]]\nformats = [ \"pickle\", \"protobuf\", \"carbonapi_v3_pb\" ]\nquery = \"test.pl*\"\nresult = [\n { path = \"test.plain1\", is_leaf = true }, { path = \"test.plain2\", is_leaf = true }\n]\n\n# End - Check metrics find\n######################################\n# Check tags autocomplete\n\n[[test.tags_checks]]\nquery = \"tag1;tag2=value21\"\nresult = [\n \"value1\"\n]\n\n[[test.tags_checks]]\nquery = \"name;tag2=value21;tag1=~value\"\nresult = [\n \"metric1\",\n]\n\n[[test.tags_checks]]\nquery = \"colon;percent=%\"\nresult = [\n \":\",\n]\n\n# End - Check tags autocomplete\n##########################################################################\n# Plain metrics (carbonapi_v3_pb)\n\n# test.plain1\n# test.plain2\n# test2.plain\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow\"\ntargets = [ \n \"test.plain*\",\n \"test{1,2}.plain\"\n]\n\n[[test.render_checks.result]]\nname = \"test.plain1\"\npath = \"test.plain*\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, 2.0]\n\n[[test.render_checks.result]]\nname = \"test.plain2\"\npath = \"test.plain*\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.5, 2.5]\n\n[[test.render_checks.result]]\nname = \"test2.plain\"\npath = \"test{1,2}.plain\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [2.5, 3.5]\n\n# End - Plain metrics (carbonapi_v3_pb)\n##########################################################################\n# Plain metrics (carbonapi_v2_pb)\n\n[[test.render_checks]]\nformats = [ \"protobuf\", \"carbonapi_v2_pb\" ]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntargets = [ \n \"test.plain*\",\n \"test{1,2}.plain\"\n]\n\n[[test.render_checks.result]]\nname = \"test.plain1\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [1.0, 2.0]\n\n[[test.render_checks.result]]\nname = \"test.plain2\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [1.5, 2.5]\n\n[[test.render_checks.result]]\nname = \"test2.plain\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [2.5, 3.5]\n\n# End - Plain metrics (carbonapi_v2_pb)\n##########################################################################\n# Plain metrics (pickle)\n\n[[test.render_checks]]\nformats = [ \"pickle\" ]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntargets = [ \n \"test.plain*\",\n \"test{1,2}.plain\"\n]\n\n[[test.render_checks.result]]\nname = \"test.plain1\"\npath = \"test.plain*\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [1.0, 2.0]\n\n[[test.render_checks.result]]\nname = \"test.plain2\"\npath = \"test.plain*\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [1.5, 2.5]\n\n[[test.render_checks.result]]\nname = \"test2.plain\"\npath = \"test{1,2}.plain\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [2.5, 3.5]\n\n# End - Plain metrics (pickle)\n##########################################################################\n# Taged metrics (carbonapi_v3_pb)\n\n# metric1;tag1=value1;tag2=value21;tag3=value3\n# metric1;tag2=value22;tag4=value4\n# metric1;tag1=value1;tag2=value23;tag3=value3\n# metric2;tag2=value21;tag4=value4\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntargets = [ \n \"seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')\",\n \"seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')\"\n]\n\n[[test.render_checks.result]]\nname = \"metric1;tag1=value1;tag2=value21;tag3=value3\"\npath = \"seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [2.0, 3.0]\n\n[[test.render_checks.result]]\nname = \"metric1;tag1=value1;tag2=value23;tag3=value3\"\npath = \"seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [4.0, 3.0]\n\n[[test.render_checks.result]]\nname = \"metric2;tag2=value21;tag4=value4\"\npath = \"seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [0.0, 1.0]\n\n# End - Tagged metrics (carbonapi_v3_pb)\n##########################################################################\n# Tagged metrics (carbonapi_v2_pb)\n\n[[test.render_checks]]\nformats = [ \"protobuf\", \"carbonapi_v2_pb\" ]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntargets = [ \n \"seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')\",\n \"seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')\"\n]\n\n[[test.render_checks.result]]\nname = \"metric1;tag1=value1;tag2=value21;tag3=value3\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [2.0, 3.0]\n\n[[test.render_checks.result]]\nname = \"metric1;tag1=value1;tag2=value23;tag3=value3\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [4.0, 3.0]\n\n[[test.render_checks.result]]\nname = \"metric2;tag2=value21;tag4=value4\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [0.0, 1.0]\n\n# End - Tagged metrics (carbonapi_v2_pb)\n##########################################################################\n# Tagged metrics (pickle)\n\n[[test.render_checks]]\nformats = [ \"pickle\" ]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntargets = [ \n \"seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')\",\n \"seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')\"\n]\n\n[[test.render_checks.result]]\nname = \"metric1;tag1=value1;tag2=value21;tag3=value3\"\npath = \"seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [2.0, 3.0]\n\n[[test.render_checks.result]]\nname = \"metric1;tag1=value1;tag2=value23;tag3=value3\"\npath = \"seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [4.0, 3.0]\n\n[[test.render_checks.result]]\nname = \"metric2;tag2=value21;tag4=value4\"\npath = \"seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [0.0, 1.0]\n\n# End - Tagged metrics (pickle)\n##########################################################################\n# Unescape\n\n[[test.render_checks]]\nformats = [ \"protobuf\", \"carbonapi_v2_pb\" ]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntargets = [ \n \"seriesByTag('percent=%')\",\n]\n\n[[test.render_checks.result]]\nname = \"test_metric;colon=:;forward=/;hash=#;host=127.0.0.1;minus=-;percent=%;plus=+;underscore=_\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [0.2, 1.5]\n\n# End - Tagged metrics (pickle)\n##########################################################################\n# Midnight\n\n# points for check https://github.com/go-graphite/graphite-clickhouse/issues/184\n[[test.input]]\nname = \"test.midnight\"\npoints = [{value = 3.0, time = \"midnight+60s\"}]\n\n[[test.input]]\nname = \"now;scope=midnight\"\npoints = [{value = 4.0, time = \"midnight+60s\"}]\n\n[[test.find_checks]]\nname = \"Midnight (direct)\"\nquery = \"test.midnight*\"\nresult = [{ path = \"test.midnight\", is_leaf = true }]\n\n[[test.find_checks]]\nname = \"Midnight\"\nquery = \"test.midnight\"\nfrom = \"midnight+60s\"\nuntil = \"midnight+70s\"\nresult = [{ path = \"test.midnight\", is_leaf = true }]\n\n[[test.find_checks]]\nname = \"Midnight (reverse)\"\nquery = \"*test.midnight\"\nresult = [{ path = \"test.midnight\", is_leaf = true }]\n\n[[test.find_checks]]\nname = \"Midnight\"\nquery = \"test.midnight\"\nfrom = \"midnight+60s\"\nuntil = \"midnight+70s\"\nresult = [{ path = \"test.midnight\", is_leaf = true }]\n\n[[test.tags_checks]]\nname = \"Midnight\"\nquery = \"name;scope=midnight\"\nresult = [\n \"now\",\n]\n\n[[test.render_checks]]\nname = \"Midnight (direct)\"\nformats = [ \"protobuf\" ]\nfrom = \"midnight+60s\"\nuntil = \"midnight+70s\"\ntargets = [ \n \"test.midnight*\",\n ]\n\n[[test.render_checks.result]]\nname = \"test.midnight\"\nstart = \"midnight+60s\"\nstop = \"midnight+80s\"\nstep = 10\nvalues = [3.0, nan]\n\n[[test.render_checks]]\nname = \"Midnight (reverse)\"\nformats = [ \"protobuf\" ]\nfrom = \"midnight+60s\"\nuntil = \"midnight+70s\"\ntargets = [ \n \"*test.midnight\",\n ]\n\n[[test.render_checks.result]]\nname = \"test.midnight\"\nstart = \"midnight+60s\"\nstop = \"midnight+80s\"\nstep = 10\nvalues = [3.0, nan]\n\n[[test.render_checks]]\nname = \"Midnight\"\nformats = [ \"protobuf\" ]\nfrom = \"midnight+60s\"\nuntil = \"midnight+70s\"\ntargets = [ \n \"seriesByTag('name=now', 'scope=midnight')\",\n ]\n\n[[test.render_checks.result]]\nname = \"now;scope=midnight\"\nstart = \"midnight+60s\"\nstop = \"midnight+80s\"\nstep = 10\nvalues = [4.0, nan]\n\n# End - Midnight\n##########################################################################\n# Day end\n\n# points for check https://github.com/go-graphite/graphite-clickhouse/issues/184\n[[test.input]]\nname = \"test.23h\"\npoints = [{value = 3.0, time = \"midnight+1380m\"}]\n\n[[test.input]]\nname = \"now;scope=23h\"\npoints = [{value = 4.0, time = \"midnight+1380m\"}]\n\n[[test.find_checks]]\nname = \"Day end\"\nquery = \"test.23h\"\nfrom = \"midnight+1380m\"\nuntil = \"midnight+1381m\"\nresult = [{ path = \"test.23h\", is_leaf = true }]\n\n[[test.find_checks]]\nname = \"Day end\"\nquery = \"test.23h\"\nfrom = \"midnight+1380m\"\nuntil = \"midnight+1381m\"\nresult = [{ path = \"test.23h\", is_leaf = true }]\n\n[[test.tags_checks]]\nname = \"Day end\"\nquery = \"name;scope=23h\"\nresult = [\n \"now\",\n]\n\n[[test.render_checks]]\nname = \"Day end\"\nformats = [ \"protobuf\" ]\nfrom = \"midnight+1380m\"\nuntil = \"midnight+1380m+10s\"\ntargets = [ \n \"test.23h\",\n ]\n\n[[test.render_checks.result]]\nname = \"test.23h\"\nstart = \"midnight+1380m\"\nstop = \"midnight+1380m+20s\"\nstep = 10\nvalues = [3.0, nan]\n\n[[test.render_checks]]\nname = \"Day end\"\nformats = [ \"protobuf\" ]\nfrom = \"midnight+1380m\"\nuntil = \"midnight+1380m+10s\"\ntargets = [ \n \"seriesByTag('name=now', 'scope=23h')\",\n ]\n\n[[test.render_checks.result]]\nname = \"now;scope=23h\"\nstart = \"midnight+1380m\"\nstop = \"midnight+1380m+20s\"\nstep = 10\nvalues = [4.0, nan]\n\n# End - Day end\n##########################################################################\n" }, { "path": "tests/tags_min_in_query/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "tests/tags_min_in_query/graphite-clickhouse.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[feature-flags]\nuse-carbon-behaviour = true\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = true\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\ntags-min-in-query = 1\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/tags_min_in_query/test.toml", "content": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"21.3\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"22.8\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\n\n[test.carbon_clickhouse]\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse.conf.tpl\"\n\n#######################################################################################\n\n[[test.input]]\nname = \"test;env=prod\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"test;env=dev\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"test;env=stage\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('name=test')\", \n]\n\n[[test.render_checks.result]]\nname = \"test;env=prod\"\npath = \"seriesByTag('name=test')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=dev\"\npath = \"seriesByTag('name=test')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"test;env=stage\"\npath = \"seriesByTag('name=test')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('env=dev')\", \n]\n\n[[test.render_checks.result]]\nname = \"test;env=dev\"\npath = \"seriesByTag('env=dev')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n# due to 'use-carbon-behaviour = true'\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('env=')\", \n]\nerror_regexp = \"^403: seriesByTag argument has too much wildcard and regex terms\"\n\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('env!=prod')\", \n]\nerror_regexp = \"^403: seriesByTag argument has too much wildcard and regex terms\"\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('env!=')\", \n]\nerror_regexp = \"^403: seriesByTag argument has too much wildcard and regex terms\"\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('env=~')\", \n]\nerror_regexp = \"^403: seriesByTag argument has too much wildcard and regex terms\"\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('env=~pr')\", \n]\nerror_regexp = \"^403: seriesByTag argument has too much wildcard and regex terms\"\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('env!=~')\", \n]\nerror_regexp = \"^403: seriesByTag argument has too much wildcard and regex terms\"\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"seriesByTag('env!=~pr')\", \n]\nerror_regexp = \"^403: seriesByTag argument has too much wildcard and regex terms\"\n" }, { "path": "tests/tls/ca.crt", "content": "-----BEGIN CERTIFICATE-----\nMIIDHTCCAgWgAwIBAgIURx5itXwLHeiQES1LzCHF7F8RNEkwDQYJKoZIhvcNAQEL\nBQAwHjEcMBoGA1UEAwwTbG9yZHZpcmRleC5sb2NhbCBDQTAeFw0yNDA4MDkxMjMy\nMzJaFw0zNDA4MDcxMjMyMzJaMB4xHDAaBgNVBAMME2xvcmR2aXJkZXgubG9jYWwg\nQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDuiK4tBYzNtROmhuXD\n80HsVVk2/+/TXV85Aey7oo2gxxJJ09iARnjJadNrbBUdoL42XtmBCkYY+pXYUWPD\nhvals2AbXiAePg7DlAHJfpaQTzHlsPvAUMjqbD6cFaQ7DfNQHcz2emmFhcRYzlQM\nh0Ob3v2yhogG7PuKaiTLTKYcHnRKfEIobQEIq16ABaaCFKzR6tpvrUJFYtkJ8EUz\njhrSg67qy7yiHiMmGQVq526X2oZYhMbSGjiPkaMZHdFkxZgJF5iQhANG9djvcopO\njdFfsJYM9rVxAjwO/P3fq5dpuQxWLLo6ZmholsixPZs1s8paEnonSDtyoNLsykwD\n2mFdAgMBAAGjUzBRMB0GA1UdDgQWBBS6BlL90Mo/+aHonqIqaewM8CyxnTAfBgNV\nHSMEGDAWgBS6BlL90Mo/+aHonqIqaewM8CyxnTAPBgNVHRMBAf8EBTADAQH/MA0G\nCSqGSIb3DQEBCwUAA4IBAQAIwTN3II6HdPfMsLvYoOmzcvUE9Y6QndI20eLqp3p8\n6KnU+lgLdSkLjc9BKwLh/Jhuy4H3u1nHpW8Jkgy/8irG2uaUvgKlutfApFQshAo7\n/k9xdH36ER0LF/bW5hQ535H76OaE+eaexx2zU50kPVuntal577d8HBfrKVI41KU8\nCVdqYTwEqHwjSyRhmmRqLi7Yo+i0o0hRwH39LxYXY2rup/V6uRyLXSIDUZ9VeqVt\nK8XDAbLV1s4kzR/OdpYcJuTWX9gFUlNHpGDkOSy9ggc5zxKaHlwGGZsvVSb4f+VF\nC89ABPZs+26EvExIih+civiC1XWIghP8RsiNyBOK3TOf\n-----END CERTIFICATE-----\n" }, { "path": "tests/tls/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "tests/tls/client.crt", "content": "-----BEGIN CERTIFICATE-----\nMIIDXTCCAkWgAwIBAgIUH+CPx0invXJZGZk7WQ0TOl2duV4wDQYJKoZIhvcNAQEL\nBQAwHjEcMBoGA1UEAwwTbG9yZHZpcmRleC5sb2NhbCBDQTAeFw0yNDA4MDkxMjM4\nNTdaFw0zNDA4MDcxMjM4NTdaMFMxCzAJBgNVBAYTAlJVMRIwEAYDVQQIDAlUYXRh\ncnN0YW4xDjAMBgNVBAcMBUthemFuMQ8wDQYDVQQKDAZLb250dXIxDzANBgNVBAMM\nBmNsaWVudDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAN4ksMPzkoon\nSBbAIutgLjpEJOoEVb5iHbBzkAn9c9EDwkHVrUGFlx69QwkBncoomV09WW3dCMlf\nFX8ClHZ5/vEpJAxQVHYTyeNpzRE+gtDuun0NN+TPgTv3Q/wBrBds/4xl1UxtuwQW\nQkrZtREi71SYcdkuWnMv4OiA6EZnhJUBuPW6oV0Sa12PeEcmQJuliHGGDd72l50d\nZsYJi/WhVgmJ5FlUkED8cVxKDbXhk3rGkXpkU/eyfEh12sNr0nX6BpPNCts3puM9\n8lkzJ2luSkfwtp46s/pQwgs0aADVd37WaV1DbNGT2iqSnnPlVHR+DoGfb8+c3AGP\nzKzuvMNH2xMCAwEAAaNeMFwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMB0G\nA1UdDgQWBBRK5htFqO/QAQp6O2yvQOjwIFaXDDAfBgNVHSMEGDAWgBS6BlL90Mo/\n+aHonqIqaewM8CyxnTANBgkqhkiG9w0BAQsFAAOCAQEABl6sVpN1O/fRF1RFKfvc\npYpzFdqQpH2lva26Ove3PMMn3gYD3fgH3JKt1JHJ8mejJ/fJDReM1hD5MtR8buuF\nP/UHEg0cJ47ljLFHjnjJX4IobuxAVRkkt+1mx7/HLQoJjPEyzDwuKazz1XcXQd4c\n4F3oa/nmo7/Nzf7NnnSEvNkwv3Anc18qAnwxCaONR0mkEWfJ0sZlcnxS1FlVEVtG\nkSymZJa6VsRMqgRDsrTyaOF0WcYuL7+onlywc2+A7fjbPzlFhTL83/yiZA+IDcrV\nOC81JN67uh4aK3nXlCHBDU1jFdr9u0jCGwo+wmWfKea7r6KG/J0a2IEokfIdSnmh\nwQ==\n-----END CERTIFICATE-----\n" }, { "path": "tests/tls/client.key", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDeJLDD85KKJ0gW\nwCLrYC46RCTqBFW+Yh2wc5AJ/XPRA8JB1a1BhZcevUMJAZ3KKJldPVlt3QjJXxV/\nApR2ef7xKSQMUFR2E8njac0RPoLQ7rp9DTfkz4E790P8AawXbP+MZdVMbbsEFkJK\n2bURIu9UmHHZLlpzL+DogOhGZ4SVAbj1uqFdEmtdj3hHJkCbpYhxhg3e9pedHWbG\nCYv1oVYJieRZVJBA/HFcSg214ZN6xpF6ZFP3snxIddrDa9J1+gaTzQrbN6bjPfJZ\nMydpbkpH8LaeOrP6UMILNGgA1Xd+1mldQ2zRk9oqkp5z5VR0fg6Bn2/PnNwBj8ys\n7rzDR9sTAgMBAAECggEACA7kqqGx9c5UUHRKeqdT20vlhZJVWev35RK2wuYCxtjl\nZGX6kZ869XkqxATe/cUDQIfyhTMOF9/vHlsFmlaf54z1KyKELdRXRSdCsmwbarYo\n5ahjwqpppwyNLB2+FGDL1Ff3/icXhZ/Dv9tYih/uS+9LvJ5wgYUsb21dqlAkVb4C\nSK03xmOQ/osaDUZpVj1E6uhiNcs3hc1z5nTLZXFeGjVdtJUePXoAzO6saJOcbcsy\n4iSzmCcT/WJ1T3crlXU6v+v7gc1L1/7uAq5yDTVHtwlxoU+SqcmbKKlUECH86cTs\nxT16UtU71SlmQtsnwDYavdb017vhB4+6JKOAFcyTwQKBgQDq8rje8qnacw1Nz1xD\nJS+p0R4dkE8uD8usQaTtajpd0lGci28zcl8pr/fPpVwlF8VSePDrMjaf8tqs5Dq/\nKRri3NEnXLhoKsGOn9PiGDwchDG57Lya4OnJ9dDa/FWUzCvxyttBpQZwocaMTEBU\nC0nD2G+SxVMdEjzhXKfQFVn7eQKBgQDyDD/hU8tFT28Bge7lRYG73JGZfr4ZeBu0\nEOGeu/402fOECngVK1b39VDOTD9me3QJKbKKRtjiUJq+0oFLXyl9nUbBIV8xhBF+\nr9jNd3W0aClzR4u9oxCTnyvodpElWChBWnTZu1EcCASz8KUm0IY+dnbu12I3K4uX\nti8n+xpb6wKBgC2zgRp9AWUotBHKoBu/hAH4V29QvtYq5GdhbX9xBmFxo8ZbqQnM\n2Y32WLHfbIkakpt0Qwi8/7slNjwjOPouOLigU17gvk4k4vmnRUPZivfRDwsnbZiC\n33cVhcbTBqKnBHVIDFY8j4AhN8namzi96V9bHnjiQUSKY6VCrLHhNVuhAoGBAO1Q\nI0WKAW8oLV7eBNrXZhZJcJt9D2crQoYuUvdtvBQXaNEZ7pha0L71z08kpLiW67Kc\nJke6pKRngQD8pPXADI7zJ87tKEcFBJ4gTMFOkaHaymETUagRe4ww8DzQGwjxQS6q\nQIzFQgXouqutkk7W/fe58GvF0q7iy89oOR3K7RIXAoGAVPk+MFC5cjyYc+srSoGg\nK66BhwVyhsjF+7n2qptSFa8OXTtIVV/TBnpeW0l2lD1EGs4RLNz7wmgqa7eU6co3\ntzFJqhGQPm0785QfSk3aOSS3OGzR3TqkDUt8LLK8rXIoUuFgNyRKnsp4ncSN75zu\nFL/drPzHeFzuHozRy8DxxBE=\n-----END PRIVATE KEY-----\n" }, { "path": "tests/tls/graphite-clickhouse.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\nappend-empty-series = false\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_TLS_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = true\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n[clickhouse.tls]\nca-cert = [\"{{- .TEST_DIR -}}/ca.crt\"]\nserver-name = \"localhost\"\n[[clickhouse.tls.certificates]]\nkey = \"{{- .TEST_DIR -}}/client.key\"\ncert = \"{{- .TEST_DIR -}}/client.crt\"\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/tls/test.toml", "content": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"21.3\"\ntls = true\ndir = \"tests/clickhouse/rollup_tls\"\n\n[[test.clickhouse]]\nversion = \"22.8\"\ntls = true\ndir = \"tests/clickhouse/rollup_tls\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ntls = true\ndir = \"tests/clickhouse/rollup_tls\"\n\n[test.carbon_clickhouse]\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse.conf.tpl\"\n\n[[test.input]]\nname = \"test.plain1\"\npoints = [{value = 3.0, time = \"rnow-30\"}, {value = 0.0, time = \"rnow-20\"}, {value = 1.0, time = \"rnow-10\"}, {value = 2.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"test.plain2\"\npoints = [{value = 2.0, time = \"rnow-30\"}, {value = 1.0, time = \"rnow-20\"}, {value = 1.5, time = \"rnow-10\"}, {value = 2.5, time = \"rnow\"}]\n\n[[test.input]]\nname = \"test2.plain\"\npoints = [{value = 1.0, time = \"rnow-30\"}, {value = 2.0, time = \"rnow-20\"}, {value = 2.5, time = \"rnow-10\"}, {value = 3.5, time = \"rnow\"}]\n\n[[test.input]]\nname = \"metric1;tag1=value1;tag2=value21;tag3=value3\"\npoints = [{value = 2.0, time = \"rnow-30\"}, {value = 2.5, time = \"rnow-20\"}, {value = 2.0, time = \"rnow-10\"}, {value = 3.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"metric1;tag2=value22;tag4=value4\"\npoints = [{value = 1.0, time = \"rnow-30\"}, {value = 2.0, time = \"rnow-20\"}, {value = 0.0, time = \"rnow-10\"}, {value = 1.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"metric1;tag1=value1;tag2=value23;tag3=value3\"\npoints = [{value = 0.5, time = \"rnow-30\"}, {value = 1.5, time = \"rnow-20\"}, {value = 4.0, time = \"rnow-10\"}, {value = 3.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"metric2;tag2=value21;tag4=value4\"\npoints = [{value = 2.0, time = \"rnow-30\"}, {value = 1.0, time = \"rnow-20\"}, {value = 0.0, time = \"rnow-10\"}, {value = 1.0, time = \"rnow\"}]\n\n[[test.input]]\nname = \"test_metric;minus=-;plus=+;percent=%;underscore=_;colon=:;hash=#;forward=/;host=127.0.0.1\"\npoints = [{value = 2.1, time = \"rnow-30\"}, {value = 0.1, time = \"rnow-20\"}, {value = 0.2, time = \"rnow-10\"}, {value = 1.5, time = \"rnow\"}]\n\n######################################\n# Check metrics find\n\n[[test.find_checks]]\nformats = [ \"pickle\", \"protobuf\", \"carbonapi_v3_pb\" ]\nquery = \"test\"\nresult = [ \n { path = \"test\", is_leaf = false }\n]\n\n[[test.find_checks]]\nformats = [ \"pickle\", \"protobuf\", \"carbonapi_v3_pb\" ]\nquery = \"test.pl*\"\nresult = [\n { path = \"test.plain1\", is_leaf = true }, { path = \"test.plain2\", is_leaf = true }\n]\n\n# End - Check metrics find\n######################################\n# Check tags autocomplete\n\n[[test.tags_checks]]\nquery = \"tag1;tag2=value21\"\nresult = [\n \"value1\"\n]\n\n[[test.tags_checks]]\nquery = \"name;tag2=value21;tag1=~value\"\nresult = [\n \"metric1\",\n]\n\n[[test.tags_checks]]\nquery = \"colon;percent=%\"\nresult = [\n \":\",\n]\n\n# End - Check tags autocomplete\n##########################################################################\n# Plain metrics (carbonapi_v3_pb)\n\n# test.plain1\n# test.plain2\n# test2.plain\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow\"\ntargets = [ \n \"test.plain*\",\n \"test{1,2}.plain\"\n]\n\n[[test.render_checks.result]]\nname = \"test.plain1\"\npath = \"test.plain*\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, 2.0]\n\n[[test.render_checks.result]]\nname = \"test.plain2\"\npath = \"test.plain*\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.5, 2.5]\n\n[[test.render_checks.result]]\nname = \"test2.plain\"\npath = \"test{1,2}.plain\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [2.5, 3.5]\n\n# End - Plain metrics (carbonapi_v3_pb)\n##########################################################################\n# Plain metrics (carbonapi_v2_pb)\n\n[[test.render_checks]]\nformats = [ \"protobuf\", \"carbonapi_v2_pb\" ]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntargets = [ \n \"test.plain*\",\n \"test{1,2}.plain\"\n]\n\n[[test.render_checks.result]]\nname = \"test.plain1\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [1.0, 2.0]\n\n[[test.render_checks.result]]\nname = \"test.plain2\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [1.5, 2.5]\n\n[[test.render_checks.result]]\nname = \"test2.plain\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [2.5, 3.5]\n\n# End - Plain metrics (carbonapi_v2_pb)\n##########################################################################\n# Plain metrics (pickle)\n\n[[test.render_checks]]\nformats = [ \"pickle\" ]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntargets = [ \n \"test.plain*\",\n \"test{1,2}.plain\"\n]\n\n[[test.render_checks.result]]\nname = \"test.plain1\"\npath = \"test.plain*\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [1.0, 2.0]\n\n[[test.render_checks.result]]\nname = \"test.plain2\"\npath = \"test.plain*\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [1.5, 2.5]\n\n[[test.render_checks.result]]\nname = \"test2.plain\"\npath = \"test{1,2}.plain\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [2.5, 3.5]\n\n# End - Plain metrics (pickle)\n##########################################################################\n# Taged metrics (carbonapi_v3_pb)\n\n# metric1;tag1=value1;tag2=value21;tag3=value3\n# metric1;tag2=value22;tag4=value4\n# metric1;tag1=value1;tag2=value23;tag3=value3\n# metric2;tag2=value21;tag4=value4\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntargets = [ \n \"seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')\",\n \"seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')\"\n]\n\n[[test.render_checks.result]]\nname = \"metric1;tag1=value1;tag2=value21;tag3=value3\"\npath = \"seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [2.0, 3.0]\n\n[[test.render_checks.result]]\nname = \"metric1;tag1=value1;tag2=value23;tag3=value3\"\npath = \"seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [4.0, 3.0]\n\n[[test.render_checks.result]]\nname = \"metric2;tag2=value21;tag4=value4\"\npath = \"seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [0.0, 1.0]\n\n# End - Tagged metrics (carbonapi_v3_pb)\n##########################################################################\n# Tagged metrics (carbonapi_v2_pb)\n\n[[test.render_checks]]\nformats = [ \"protobuf\", \"carbonapi_v2_pb\" ]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntargets = [ \n \"seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')\",\n \"seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')\"\n]\n\n[[test.render_checks.result]]\nname = \"metric1;tag1=value1;tag2=value21;tag3=value3\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [2.0, 3.0]\n\n[[test.render_checks.result]]\nname = \"metric1;tag1=value1;tag2=value23;tag3=value3\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [4.0, 3.0]\n\n[[test.render_checks.result]]\nname = \"metric2;tag2=value21;tag4=value4\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [0.0, 1.0]\n\n# End - Tagged metrics (carbonapi_v2_pb)\n##########################################################################\n# Tagged metrics (pickle)\n\n[[test.render_checks]]\nformats = [ \"pickle\" ]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntargets = [ \n \"seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')\",\n \"seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')\"\n]\n\n[[test.render_checks.result]]\nname = \"metric1;tag1=value1;tag2=value21;tag3=value3\"\npath = \"seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [2.0, 3.0]\n\n[[test.render_checks.result]]\nname = \"metric1;tag1=value1;tag2=value23;tag3=value3\"\npath = \"seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [4.0, 3.0]\n\n[[test.render_checks.result]]\nname = \"metric2;tag2=value21;tag4=value4\"\npath = \"seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [0.0, 1.0]\n\n# End - Tagged metrics (pickle)\n##########################################################################\n# Unescape\n\n[[test.render_checks]]\nformats = [ \"protobuf\", \"carbonapi_v2_pb\" ]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntargets = [ \n \"seriesByTag('percent=%')\",\n]\n\n[[test.render_checks.result]]\nname = \"test_metric;colon=:;forward=/;hash=#;host=127.0.0.1;minus=-;percent=%;plus=+;underscore=_\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nvalues = [0.2, 1.5]\n\n# End - Tagged metrics (pickle)\n##########################################################################\n# Midnight\n\n# points for check https://github.com/go-graphite/graphite-clickhouse/issues/184\n[[test.input]]\nname = \"test.midnight\"\npoints = [{value = 3.0, time = \"midnight+60s\"}]\n\n[[test.input]]\nname = \"now;scope=midnight\"\npoints = [{value = 4.0, time = \"midnight+60s\"}]\n\n[[test.find_checks]]\nname = \"Midnight (direct)\"\nquery = \"test.midnight*\"\nresult = [{ path = \"test.midnight\", is_leaf = true }]\n\n[[test.find_checks]]\nname = \"Midnight\"\nquery = \"test.midnight\"\nfrom = \"midnight+60s\"\nuntil = \"midnight+70s\"\nresult = [{ path = \"test.midnight\", is_leaf = true }]\n\n[[test.find_checks]]\nname = \"Midnight (reverse)\"\nquery = \"*test.midnight\"\nresult = [{ path = \"test.midnight\", is_leaf = true }]\n\n[[test.find_checks]]\nname = \"Midnight\"\nquery = \"test.midnight\"\nfrom = \"midnight+60s\"\nuntil = \"midnight+70s\"\nresult = [{ path = \"test.midnight\", is_leaf = true }]\n\n[[test.tags_checks]]\nname = \"Midnight\"\nquery = \"name;scope=midnight\"\nresult = [\n \"now\",\n]\n\n[[test.render_checks]]\nname = \"Midnight (direct)\"\nformats = [ \"protobuf\" ]\nfrom = \"midnight+60s\"\nuntil = \"midnight+70s\"\ntargets = [ \n \"test.midnight*\",\n ]\n\n[[test.render_checks.result]]\nname = \"test.midnight\"\nstart = \"midnight+60s\"\nstop = \"midnight+80s\"\nstep = 10\nvalues = [3.0, nan]\n\n[[test.render_checks]]\nname = \"Midnight (reverse)\"\nformats = [ \"protobuf\" ]\nfrom = \"midnight+60s\"\nuntil = \"midnight+70s\"\ntargets = [ \n \"*test.midnight\",\n ]\n\n[[test.render_checks.result]]\nname = \"test.midnight\"\nstart = \"midnight+60s\"\nstop = \"midnight+80s\"\nstep = 10\nvalues = [3.0, nan]\n\n[[test.render_checks]]\nname = \"Midnight\"\nformats = [ \"protobuf\" ]\nfrom = \"midnight+60s\"\nuntil = \"midnight+70s\"\ntargets = [ \n \"seriesByTag('name=now', 'scope=midnight')\",\n ]\n\n[[test.render_checks.result]]\nname = \"now;scope=midnight\"\nstart = \"midnight+60s\"\nstop = \"midnight+80s\"\nstep = 10\nvalues = [4.0, nan]\n\n# End - Midnight\n##########################################################################\n# Day end\n\n# points for check https://github.com/go-graphite/graphite-clickhouse/issues/184\n[[test.input]]\nname = \"test.23h\"\npoints = [{value = 3.0, time = \"midnight+1380m\"}]\n\n[[test.input]]\nname = \"now;scope=23h\"\npoints = [{value = 4.0, time = \"midnight+1380m\"}]\n\n[[test.find_checks]]\nname = \"Day end\"\nquery = \"test.23h\"\nfrom = \"midnight+1380m\"\nuntil = \"midnight+1381m\"\nresult = [{ path = \"test.23h\", is_leaf = true }]\n\n[[test.find_checks]]\nname = \"Day end\"\nquery = \"test.23h\"\nfrom = \"midnight+1380m\"\nuntil = \"midnight+1381m\"\nresult = [{ path = \"test.23h\", is_leaf = true }]\n\n[[test.tags_checks]]\nname = \"Day end\"\nquery = \"name;scope=23h\"\nresult = [\n \"now\",\n]\n\n[[test.render_checks]]\nname = \"Day end\"\nformats = [ \"protobuf\" ]\nfrom = \"midnight+1380m\"\nuntil = \"midnight+1380m+10s\"\ntargets = [ \n \"test.23h\",\n ]\n\n[[test.render_checks.result]]\nname = \"test.23h\"\nstart = \"midnight+1380m\"\nstop = \"midnight+1380m+20s\"\nstep = 10\nvalues = [3.0, nan]\n\n[[test.render_checks]]\nname = \"Day end\"\nformats = [ \"protobuf\" ]\nfrom = \"midnight+1380m\"\nuntil = \"midnight+1380m+10s\"\ntargets = [ \n \"seriesByTag('name=now', 'scope=23h')\",\n ]\n\n[[test.render_checks.result]]\nname = \"now;scope=23h\"\nstart = \"midnight+1380m\"\nstop = \"midnight+1380m+20s\"\nstep = 10\nvalues = [4.0, nan]\n\n# End - Day end\n##########################################################################\n" }, { "path": "tests/wildcard_min_distance/carbon-clickhouse.conf.tpl", "content": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_index]\ntype = \"index\"\ntable = \"graphite_index\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_tags]\ntype = \"tagged\"\ntable = \"graphite_tags\"\nthreads = 3\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\ncache-ttl = \"1h\"\n\n[upload.graphite_reverse]\ntype = \"points-reverse\"\ntable = \"graphite_reverse\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[upload.graphite]\ntype = \"points\"\ntable = \"graphite\"\nurl = \"{{ .CLICKHOUSE_URL }}/\"\ntimeout = \"2m30s\"\nzero-timestamp = false\n\n[tcp]\nlisten = \":2003\"\nenabled = true\ndrop-future = \"0s\"\ndrop-past = \"0s\"\n\n[logging]\nfile = \"/etc/carbon-clickhouse/carbon-clickhouse.log\"\nlevel = \"debug\"\n" }, { "path": "tests/wildcard_min_distance/graphite-clickhouse.conf.tpl", "content": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nheaders-to-log = [ \"X-Ctx-Carbonapi-Uuid\" ]\n\n[clickhouse]\nurl = \"{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1\"\ndata-timeout = \"30s\"\n\nwildcard-min-distance = 1\n\nindex-table = \"graphite_index\"\nindex-use-daily = true\nindex-timeout = \"1m\"\ninternal-aggregation = true\n\ntagged-table = \"graphite_tags\"\ntagged-autocomplete-days = 1\n\n[[data-table]]\n# # clickhouse table name\ntable = \"graphite\"\n# # points in table are stored with reverse path\nreverse = false\nrollup-conf = \"auto\"\n\n[[logging]]\nlogger = \"\"\nfile = \"{{ .GCH_DIR }}/graphite-clickhouse.log\"\nlevel = \"info\"\nencoding = \"json\"\nencoding-time = \"iso8601\"\nencoding-duration = \"seconds\"\n" }, { "path": "tests/wildcard_min_distance/test.toml", "content": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"21.3\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"22.8\"\ndir = \"tests/clickhouse/rollup\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\n\n[test.carbon_clickhouse]\ntemplate = \"carbon-clickhouse.conf.tpl\"\n\n[[test.graphite_clickhouse]]\ntemplate = \"graphite-clickhouse.conf.tpl\"\n\n[[test.input]]\nname = \"team_one.prod.test.metric_one\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"team_two.stage.test.metric_one\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"team_one.dev.test.metric_two\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.input]]\nname = \"team_one.dev.nontest.metric_one\"\npoints = [{value = 1.0, time = \"rnow-10\"}]\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"team_one.prod.test.metric_one\", \n]\n\n[[test.render_checks.result]]\nname = \"team_one.prod.test.metric_one\"\npath = \"team_one.prod.test.metric_one\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"*.dev.test.metric_two\", \n]\n\n[[test.render_checks.result]]\nname = \"team_one.dev.test.metric_two\"\npath = \"*.dev.test.metric_two\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"*.*.test.metric_one\", \n]\n\n[[test.render_checks.result]]\nname = \"team_one.prod.test.metric_one\"\npath = \"*.*.test.metric_one\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"team_two.stage.test.metric_one\"\npath = \"*.*.test.metric_one\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"team_two.stage.test.*\", \n]\n\n[[test.render_checks.result]]\nname = \"team_two.stage.test.metric_one\"\npath = \"team_two.stage.test.*\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"team_one.*.test.*\", \n]\n\n[[test.render_checks.result]]\nname = \"team_one.prod.test.metric_one\"\npath = \"team_one.*.test.*\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks.result]]\nname = \"team_one.dev.test.metric_two\"\npath = \"team_one.*.test.*\"\nconsolidation = \"avg\"\nstart = \"rnow-10\"\nstop = \"rnow+10\"\nstep = 10\nreq_start = \"rnow-10\"\nreq_stop = \"rnow+10\"\nvalues = [1.0, nan]\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"*.prod.test.*\", \n]\nerror_regexp = \"^400: query has wildcards way too early at the start and at the end of it\"\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"*.*.test.*\", \n]\nerror_regexp = \"^400: query has wildcards way too early at the start and at the end of it\"\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"*.*.*.*\", \n]\nerror_regexp = \"^400: query has wildcards way too early at the start and at the end of it\"\n\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"*.*\", \n]\nerror_regexp = \"^400: query has wildcards way too early at the start and at the end of it\"\n\n[[test.render_checks]]\nfrom = \"rnow-10\"\nuntil = \"rnow+1\"\ntimeout = \"1h\"\ntargets = [ \n \"*\", \n]" } ]