Repository: lomik/graphite-clickhouse Branch: master Commit: b816c69a5c9a Files: 293 Total size: 1.3 MB Directory structure: gitextract_n919tixz/ ├── .gitattributes ├── .github/ │ └── workflows/ │ ├── codeql.yml │ ├── docker.yml │ ├── lint.yml │ ├── release.yml │ ├── tests-sd.yml │ └── tests.yml ├── .gitignore ├── .golangci.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── autocomplete/ │ ├── autocomplete.go │ └── autocomplete_test.go ├── cache/ │ └── cache.go ├── capabilities/ │ └── handler.go ├── cmd/ │ ├── e2e-test/ │ │ ├── carbon-clickhouse.go │ │ ├── checks.go │ │ ├── clickhouse.go │ │ ├── container.go │ │ ├── e2etesting.go │ │ ├── errors.go │ │ ├── graphite-clickhouse.go │ │ ├── main.go │ │ ├── rproxy.go │ │ └── utils.go │ └── graphite-clickhouse-client/ │ └── main.go ├── config/ │ ├── .gitignore │ ├── config.go │ ├── config_test.go │ ├── json.go │ └── json_test.go ├── deploy/ │ ├── doc/ │ │ ├── .gitignore │ │ └── config.md │ └── root/ │ └── usr/ │ └── lib/ │ └── systemd/ │ └── system/ │ └── graphite-clickhouse.service ├── doc/ │ ├── aggregation.md │ ├── config.md │ ├── debugging.md │ ├── graphite_clickhouse.gliffy │ ├── index-table.md │ └── release.md ├── find/ │ ├── find.go │ ├── handler.go │ ├── handler_json_test.go │ └── handler_test.go ├── finder/ │ ├── base.go │ ├── blacklist.go │ ├── date.go │ ├── date_reverse.go │ ├── date_reverse_test.go │ ├── finder.go │ ├── index.go │ ├── index_test.go │ ├── mock.go │ ├── plain_from_tagged.go │ ├── plain_from_tagged_test.go │ ├── prefix.go │ ├── prefix_test.go │ ├── reverse.go │ ├── reverse_test.go │ ├── split.go │ ├── split_test.go │ ├── tag.go │ ├── tag_test.go │ ├── tagged.go │ ├── tagged_test.go │ ├── tags_count_querier.go │ └── unescape.go ├── go.mod ├── go.sum ├── graphite-clickhouse.go ├── healthcheck/ │ └── healthcheck.go ├── helper/ │ ├── RowBinary/ │ │ └── encode.go │ ├── clickhouse/ │ │ ├── clickhouse.go │ │ ├── clickhouse_test.go │ │ ├── external-data.go │ │ └── external-data_test.go │ ├── client/ │ │ ├── datetime.go │ │ ├── errros.go │ │ ├── find.go │ │ ├── render.go │ │ ├── requests.go │ │ ├── tags.go │ │ └── types.go │ ├── date/ │ │ ├── date.go │ │ └── date_test.go │ ├── datetime/ │ │ ├── datetime.go │ │ └── datetime_test.go │ ├── errs/ │ │ └── errors.go │ ├── headers/ │ │ └── headers.go │ ├── http/ │ │ └── live-http-client.go │ ├── pickle/ │ │ └── pickle.go │ ├── point/ │ │ ├── func.go │ │ ├── func_test.go │ │ ├── point.go │ │ └── points.go │ ├── rollup/ │ │ ├── aggr.go │ │ ├── compact.go │ │ ├── compact_test.go │ │ ├── remote.go │ │ ├── remote_test.go │ │ ├── rollup.go │ │ ├── rules.go │ │ ├── rules_test.go │ │ ├── xml.go │ │ └── xml_test.go │ ├── tests/ │ │ ├── clickhouse/ │ │ │ └── server.go │ │ └── compare/ │ │ ├── compare.go │ │ └── expand/ │ │ └── expand.go │ └── utils/ │ ├── utils.go │ └── utils_test.go ├── index/ │ ├── handler.go │ ├── index.go │ └── index_test.go ├── issues/ │ └── daytime/ │ ├── carbon-clickhouse.conf.tpl │ ├── graphite-clickhouse-internal-aggr.conf.tpl │ ├── graphite-clickhouse.conf.tpl │ └── test.toml ├── limiter/ │ ├── alimiter.go │ ├── alimiter_test.go │ ├── interface.go │ ├── limiter.go │ ├── noop.go │ └── wlimiter.go ├── load_avg/ │ ├── load_avg.go │ ├── load_avg_default.go │ ├── load_avg_linux.go │ └── load_avg_test.go ├── logs/ │ └── logger.go ├── metrics/ │ ├── limiter_metrics.go │ ├── metrics.go │ ├── metrics_test.go │ ├── query_metrics.go │ └── statsd.go ├── nfpm.yaml ├── packages.sh ├── pkg/ │ ├── alias/ │ │ ├── map.go │ │ ├── map_tagged_test.go │ │ └── map_test.go │ ├── dry/ │ │ ├── math.go │ │ ├── math_test.go │ │ ├── strings.go │ │ ├── strings_test.go │ │ ├── unsafe.go │ │ └── unsafe_test.go │ ├── reverse/ │ │ ├── reverse.go │ │ └── reverse_test.go │ ├── scope/ │ │ ├── context.go │ │ ├── http_request.go │ │ ├── key.go │ │ ├── logger.go │ │ └── version.go │ └── where/ │ ├── match.go │ ├── match_test.go │ ├── where.go │ └── where_test.go ├── prometheus/ │ ├── .gitignore │ ├── empty_iterator.go │ ├── exemplar.go │ ├── gatherer.go │ ├── labels.go │ ├── labels_test.go │ ├── local_storage.go │ ├── logger.go │ ├── matcher.go │ ├── metrics_set.go │ ├── querier.go │ ├── querier_select.go │ ├── querier_select_test.go │ ├── run.go │ ├── run_dummy.go │ ├── series_set.go │ └── storage.go ├── render/ │ ├── data/ │ │ ├── carbonlink.go │ │ ├── carbonlink_test.go │ │ ├── ch_response.go │ │ ├── common_step.go │ │ ├── common_step_test.go │ │ ├── data.go │ │ ├── data_parse_test.go │ │ ├── multi_target.go │ │ ├── multi_target_test.go │ │ ├── query.go │ │ ├── query_test.go │ │ ├── targets.go │ │ └── targets_test.go │ ├── handler.go │ ├── handler_test.go │ └── reply/ │ ├── formatter.go │ ├── formatter_test.go │ ├── json.go │ ├── pickle.go │ ├── protobuf.go │ ├── protobuf_test.go │ ├── v2_pb.go │ ├── v2_pb_test.go │ ├── v3_pb.go │ └── v3_pb_test.go ├── sd/ │ ├── nginx/ │ │ ├── nginx.go │ │ ├── nginx_test.go │ │ └── tests/ │ │ └── nginx_cleanup_test.go │ ├── register.go │ └── utils/ │ └── utils.go ├── tagger/ │ ├── metric.go │ ├── rule.go │ ├── rule_test.go │ ├── set.go │ ├── tagger.go │ ├── tagger_test.go │ └── tree.go └── tests/ ├── agg_internal/ │ ├── carbon-clickhouse.conf.tpl │ ├── graphite-clickhouse-internal-aggr.conf.tpl │ └── test.toml ├── agg_latest/ │ ├── carbon-clickhouse.conf.tpl │ ├── graphite-clickhouse.conf.tpl │ └── test.toml ├── agg_merge/ │ ├── carbon-clickhouse.conf.tpl │ ├── graphite-clickhouse-internal-aggr.conf.tpl │ ├── graphite-clickhouse.conf.tpl │ └── test.toml ├── agg_oneblock/ │ ├── carbon-clickhouse.conf.tpl │ ├── graphite-clickhouse-internal-aggr.conf.tpl │ ├── graphite-clickhouse.conf.tpl │ └── test.toml ├── clickhouse/ │ ├── rollup/ │ │ ├── config.xml │ │ ├── init.sql │ │ ├── rollup.xml │ │ └── users.xml │ └── rollup_tls/ │ ├── config.xml │ ├── init.sql │ ├── rollup.xml │ ├── rootCA.crt │ ├── server.crt │ ├── server.key │ └── users.xml ├── consolidateBy/ │ ├── carbon-clickhouse.conf.tpl │ ├── graphite-clickhouse.conf.tpl │ └── test.toml ├── consul.sh ├── emptyseries_append/ │ ├── carbon-clickhouse.conf.tpl │ ├── graphite-clickhouse.conf.tpl │ └── test.toml ├── emptyseries_noappend/ │ ├── carbon-clickhouse.conf.tpl │ ├── graphite-clickhouse.conf.tpl │ └── test.toml ├── error_handling/ │ ├── carbon-clickhouse.conf.tpl │ ├── graphite-clickhouse.conf.tpl │ └── test.toml ├── feature_flags_both_true/ │ ├── carbon-clickhouse.conf.tpl │ ├── graphite-clickhouse.conf.tpl │ └── test.toml ├── feature_flags_dont_match_missing_tags/ │ ├── carbon-clickhouse.conf.tpl │ ├── graphite-clickhouse.conf.tpl │ └── test.toml ├── feature_flags_false/ │ ├── carbon-clickhouse.conf.tpl │ ├── graphite-clickhouse.conf.tpl │ └── test.toml ├── feature_flags_use_carbon_behaviour/ │ ├── carbon-clickhouse.conf.tpl │ ├── graphite-clickhouse.conf.tpl │ └── test.toml ├── find_cache/ │ ├── carbon-clickhouse.conf.tpl │ ├── graphite-clickhouse-cached.conf.tpl │ ├── graphite-clickhouse-internal-aggr-cached.conf.tpl │ └── test.toml ├── limitera/ │ ├── carbon-clickhouse.conf.tpl │ ├── graphite-clickhouse-internal-aggr-cached.conf.tpl │ └── test.toml ├── limitermax/ │ ├── carbon-clickhouse.conf.tpl │ ├── graphite-clickhouse-internal-aggr-cached.conf.tpl │ └── test.toml ├── limiterw/ │ ├── carbon-clickhouse.conf.tpl │ ├── graphite-clickhouse-internal-aggr-cached.conf.tpl │ └── test.toml ├── limiterwn/ │ ├── carbon-clickhouse.conf.tpl │ ├── graphite-clickhouse-internal-aggr-cached.conf.tpl │ └── test.toml ├── one_table/ │ ├── carbon-clickhouse.conf.tpl │ ├── graphite-clickhouse-internal-aggr.conf.tpl │ ├── graphite-clickhouse.conf.tpl │ └── test.toml ├── tags_min_in_query/ │ ├── carbon-clickhouse.conf.tpl │ ├── graphite-clickhouse.conf.tpl │ └── test.toml ├── tls/ │ ├── ca.crt │ ├── carbon-clickhouse.conf.tpl │ ├── client.crt │ ├── client.key │ ├── graphite-clickhouse.conf.tpl │ └── test.toml └── wildcard_min_distance/ ├── carbon-clickhouse.conf.tpl ├── graphite-clickhouse.conf.tpl └── test.toml ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitattributes ================================================ ================================================ FILE: .github/workflows/codeql.yml ================================================ # For most projects, this workflow file will not need changing; you simply need # to commit it to your repository. # # You may wish to alter this file to override the set of languages analyzed, # or to provide custom queries or build logic. name: "CodeQL" on: push: branches: [master] pull_request: # The branches below must be a subset of the branches above branches: [master] jobs: analyze: name: Analyze runs-on: ubuntu-latest strategy: fail-fast: false matrix: # Override automatic language detection by changing the below list # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] language: ['go'] # Learn more... # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection steps: - name: Checkout repository uses: actions/checkout@v4 with: # We must fetch at least the immediate parents so that if this is # a pull request then we can checkout the head. fetch-depth: 2 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # queries: ./path/to/local/query, your-org/your-repo/queries@main # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v3 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines # and modify them (or add more) to build your code if your project # uses a compiled language #- run: | # make bootstrap # make release - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 ================================================ FILE: .github/workflows/docker.yml ================================================ name: Docker images on: push: branches: [ master ] tags: [ 'v*' ] pull_request: branches: [ master ] jobs: docker: name: Build image runs-on: ubuntu-latest steps: - name: Check out code uses: actions/checkout@v4 - name: Docker meta id: meta uses: docker/metadata-action@v3 with: images: ghcr.io/${{ github.repository }} # create latest tag for branch events flavor: | latest=${{ github.event_name == 'push' && github.ref_type == 'branch' }} tags: | type=ref,event=branch type=ref,event=pr type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} type=semver,pattern={{major}}.{{minor}}.{{patch}} - name: Login to DockerHub if: github.event_name != 'pull_request' uses: docker/login-action@v1 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push id: docker_build uses: docker/build-push-action@v2 with: # push for non-pr events push: ${{ github.event_name != 'pull_request' }} context: . tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} ================================================ FILE: .github/workflows/lint.yml ================================================ name: Lint on: pull_request: jobs: golangci: name: lint runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v4 with: go-version-file: go.mod - name: Run linter uses: golangci/golangci-lint-action@v7 with: version: v2.0.2 ================================================ FILE: .github/workflows/release.yml ================================================ name: Upload Packages to new release on: release: types: - published jobs: build: name: Build runs-on: ubuntu-latest env: BINARY: ${{ github.event.repository.name }} CGO_ENABLED: 0 outputs: matrix: ${{ steps.build.outputs.matrix }} steps: - name: Set up Go uses: actions/setup-go@v5 with: go-version: ^1 - uses: actions/checkout@v4 name: Checkout - name: Test run: make test env: CGO_ENABLED: 1 - name: Build packages id: build run: | go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.40.0 make nfpm-deb nfpm-rpm make sum-files ARTIFACTS= # Upload all deb and rpm packages for package in *deb *rpm; do ARTIFACTS=${ARTIFACTS}\"$package\",\ ; done echo ::set-output name=matrix::{\"file\": [${ARTIFACTS} \"sha256sum\", \"md5sum\"]} - name: Check version id: check_version run: | ./out/${BINARY}-linux-amd64 -version [ v$(./out/${BINARY}-linux-amd64 -version) = ${{ github.event.release.tag_name }} ] - name: Artifact id: artifact uses: actions/upload-artifact@v4 with: name: packages retention-days: 1 path: | *.deb *.rpm sha256sum md5sum - name: Push packages to the stable repo run: make packagecloud-stable env: PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} upload: needs: build runs-on: ubuntu-latest strategy: matrix: ${{fromJson(needs.build.outputs.matrix)}} steps: - name: Download artifact uses: actions/download-artifact@v4.1.7 with: name: packages - name: Upload ${{ matrix.file }} id: upload uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ github.event.release.upload_url }} asset_path: ${{ matrix.file }} asset_name: ${{ matrix.file }} asset_content_type: application/octet-stream ================================================ FILE: .github/workflows/tests-sd.yml ================================================ name: Tests register in SD on: push: branches: [ master ] pull_request: branches: [ master ] jobs: tests: env: CGO_ENABLED: 0 name: Test register in SD runs-on: ubuntu-latest strategy: matrix: go: - ^1 steps: - name: Set up Go uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Check out code uses: actions/checkout@v4 - name: Start consul run: | ./tests/consul.sh 1.15.2 > /tmp/consul.log & sleep 30 shell: bash - name: Test run: go test ./sd/nginx -tags=test_sd -v ================================================ FILE: .github/workflows/tests.yml ================================================ name: Tests on: push: branches: [ master ] pull_request: branches: [ master ] jobs: tests: env: CGO_ENABLED: 0 name: Test code runs-on: ubuntu-latest strategy: matrix: go: - ^1.20 - ^1.21 - ^1 steps: - name: Set up Go uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Check out code uses: actions/checkout@v4 - name: Checkout to the latest tag run: | # Fetch all tags git fetch --depth=1 --tags # Get the latest tag VERS=$(git tag -l | sort -Vr | head -n1) # Fetch everything to the latest tag git fetch --shallow-since=$(git log $VERS -1 --format=%at) if: ${{ github.event_name == 'push' }} # only when built from master - name: Build project run: make - name: Validate default configs run: | ./graphite-clickhouse -config-print-default > /tmp/graphite-clickhouse.conf ./graphite-clickhouse -config /tmp/graphite-clickhouse.conf -check-config - name: Check documentation consistency run: | make config git diff --exit-code - name: Test run: make test env: CGO_ENABLED: 1 - name: Test (with GMT-5) run: | go clean -testcache TZ=Etc/GMT-5 make test env: CGO_ENABLED: 1 - name: Test (with GMT+5) run: | go clean -testcache TZ=Etc/GMT+5 make test env: CGO_ENABLED: 1 - name: Integration tests run: | make e2e-test ./e2e-test -config tests -abort -rmi # TODO (msaf1980): find a way to set TZ in carbon-clickhouse docker (or run locally) # run with clickhouse.date-format = "both" # - name: Integration tests (with Etc/GMT-5) # run: | # make e2e-test # sudo timedatectl set-timezone Etc/GMT-5 # ./e2e-test -config issues/daytime # - name: Integration tests (with Etc/GMT+5) # run: | # make e2e-test # sudo timedatectl set-timezone Etc/GMT+5 # ./e2e-test -config issues/daytime - name: Check packaging run: | go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.40.0 make DEVEL=1 nfpm-deb nfpm-rpm make sum-files - name: Artifact id: artifact uses: actions/upload-artifact@v4 with: name: packages-${{ matrix.go }} path: | *.deb *.rpm sha256sum md5sum - name: Push packages to the autobuilds repo if: ${{ github.event_name == 'push' && matrix.go == '^1' }} # only when built from master with latest go run: make DEVEL=1 packagecloud-autobuilds env: PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} ================================================ FILE: .gitignore ================================================ # Compiled Object files, Static and Dynamic libs (Shared Objects) *.o *.a *.so graphite-clickhouse .vscode /out/ # Folders _obj _test # Architecture specific extensions/prefixes *.[568vq] [568vq].out *.cgo1.go *.cgo2.c _cgo_defun.c _cgo_gotypes.go _cgo_export.* _testmain.go *.exe *.test *.prof ================================================ FILE: .golangci.yml ================================================ version: "2" linters: default: none enable: - asasalint - asciicheck - bidichk - bodyclose # - contextcheck - decorder # - dogsled - durationcheck # - errcheck # - errorlint # - fatcontext - ginkgolinter - gocheckcompilerdirectives - gochecksumtype # - goconst # - gocyclo # - godot - goheader - govet - grouper # - ineffassign - loggercheck # - makezero # - misspell # - mnd # - nilerr # - noctx # - nosprintfhostport - prealloc # - predeclared - promlinter - protogetter - reassign # - revive - rowserrcheck - sloglint - spancheck - sqlclosecheck # - staticcheck # - testifylint - tparallel - unconvert # - unparam # - unused - usestdlibvars # - wastedassign - whitespace - wsl settings: gocyclo: min-complexity: 15 govet: settings: printf: funcs: - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf unparam: check-exported: false exclusions: generated: lax presets: - comments - common-false-positives - legacy - std-error-handling rules: - linters: - errcheck - contextcheck - goconst - mnd path: _test\.go - linters: - godot path: notifier/registrator.go paths: - third_party$ - builtin$ - examples$ formatters: enable: - gofmt # - gofumpt - goimports exclusions: generated: lax paths: - third_party$ - builtin$ - examples$ ================================================ FILE: Dockerfile ================================================ FROM golang:alpine as builder WORKDIR /go/src/github.com/lomik/graphite-clickhouse COPY . . ENV GOPATH=/go RUN apk add git --no-cache RUN go build -ldflags '-extldflags "-static"' github.com/lomik/graphite-clickhouse FROM alpine:latest RUN apk --no-cache add ca-certificates WORKDIR / COPY --from=builder /go/src/github.com/lomik/graphite-clickhouse/graphite-clickhouse /usr/bin/graphite-clickhouse CMD ["graphite-clickhouse"] ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2016 Roman Lomonosov Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Makefile ================================================ NAME:=graphite-clickhouse DESCRIPTION:="Graphite cluster backend with ClickHouse support" MODULE:=github.com/lomik/graphite-clickhouse GO ?= go export GO111MODULE := on TEMPDIR:=$(shell mktemp -d) DEVEL ?= 0 ifeq ($(DEVEL), 0) VERSION:=$(shell sh -c 'grep "const Version" $(NAME).go | cut -d\" -f2') else VERSION:=$(shell sh -c 'git describe --always --tags | sed -e "s/^v//i"') endif SRCS:=$(shell find . -name '*.go') all: $(NAME) .PHONY: clean clean: rm -f $(NAME) $(NAME)-client rm -rf out rm -f *deb *rpm rm -f sha256sum md5sum $(NAME): $(SRCS) $(GO) build -ldflags '-X main.BuildVersion=$(VERSION)' $(MODULE) debug: $(SRCS) $(GO) build -ldflags '-X main.BuildVersion=$(VERSION)' -gcflags=all='-N -l' $(MODULE) deploy/doc/graphite-clickhouse.conf: $(NAME) ./$(NAME) -config-print-default > $@ doc/config.md: deploy/doc/graphite-clickhouse.conf deploy/doc/config.md @echo 'Generating $@...' @printf '[//]: # (This file is built out of deploy/doc/config.md, please do not edit it manually) \n' > $@ @printf '[//]: # (To rebuild it run `make config`)\n\n' >> $@ @cat deploy/doc/config.md >> $@ @printf '\n```toml\n' >> $@ @cat deploy/doc/graphite-clickhouse.conf >> $@ @printf '```\n' >> $@ config: doc/config.md # run after prometheus upgrade prometheus/ui: vendor_prometheus_ui.sh test: $(GO) test -race ./... e2e-test: $(NAME) $(GO) build $(MODULE)/cmd/e2e-test client: $(NAME) $(GO) build $(MODULE)/cmd/graphite-clickhouse-client gox-build: out/$(NAME)-linux-amd64 out/$(NAME)-linux-arm64 out/root/etc/$(NAME)/$(NAME).conf ARCH = amd64 arm64 out/$(NAME)-linux-%: out $(SRCS) GOOS=linux GOARCH=$* $(GO) build -ldflags '-X main.BuildVersion=$(VERSION)' -o $@ $(MODULE) out: mkdir -p out out/root/etc/$(NAME)/$(NAME).conf: $(NAME) mkdir -p "$(shell dirname $@)" ./$(NAME) -config-print-default > $@ nfpm-deb: gox-build $(MAKE) nfpm-build-deb ARCH=amd64 $(MAKE) nfpm-build-deb ARCH=arm64 nfpm-rpm: gox-build $(MAKE) nfpm-build-rpm ARCH=amd64 $(MAKE) nfpm-build-rpm ARCH=arm64 nfpm-build-%: nfpm.yaml NAME=$(NAME) DESCRIPTION=$(DESCRIPTION) ARCH=$(ARCH) VERSION_STRING=$(VERSION) nfpm package --packager $* .ONESHELL: RPM_VERSION:=$(subst -,_,$(VERSION)) packagecloud-push-rpm: $(wildcard $(NAME)-$(RPM_VERSION)-1.*.rpm) for pkg in $^; do package_cloud push $(REPO)/el/7 $${pkg} || true package_cloud push $(REPO)/el/8 $${pkg} || true package_cloud push $(REPO)/el/9 $${pkg} || true done .ONESHELL: packagecloud-push-deb: $(wildcard $(NAME)_$(VERSION)_*.deb) for pkg in $^; do package_cloud push $(REPO)/ubuntu/xenial $${pkg} || true package_cloud push $(REPO)/ubuntu/bionic $${pkg} || true package_cloud push $(REPO)/ubuntu/focal $${pkg} || true package_cloud push $(REPO)/debian/stretch $${pkg} || true package_cloud push $(REPO)/debian/buster $${pkg} || true package_cloud push $(REPO)/debian/bullseye $${pkg} || true done packagecloud-push: @$(MAKE) packagecloud-push-rpm @$(MAKE) packagecloud-push-deb packagecloud-autobuilds: $(MAKE) packagecloud-push REPO=go-graphite/autobuilds packagecloud-stable: $(MAKE) packagecloud-push REPO=go-graphite/stable sum-files: | sha256sum md5sum md5sum: md5sum $(wildcard $(NAME)_$(VERSION)*.deb) $(wildcard $(NAME)-$(VERSION)*.rpm) > md5sum sha256sum: sha256sum $(wildcard $(NAME)_$(VERSION)*.deb) $(wildcard $(NAME)-$(VERSION)*.rpm) > sha256sum .PHONY: lint lint: golangci-lint run ================================================ FILE: README.md ================================================ [![deb](https://img.shields.io/badge/deb-packagecloud.io-844fec.svg)](https://packagecloud.io/go-graphite/stable) [![rpm](https://img.shields.io/badge/rpm-packagecloud.io-844fec.svg)](https://packagecloud.io/go-graphite/stable) # graphite-clickhouse Graphite cluster backend with ClickHouse support ## Work scheme ![stack.png](doc/stack.png?v3) Gray components are optional or alternative ## TL;DR [Preconfigured docker-compose](https://github.com/lomik/graphite-clickhouse-tldr) ### Docker Docker images are available on [packages](https://github.com/lomik/graphite-clickhouse/pkgs/container/graphite-clickhouse) page. ## Compatibility - [x] [graphite-web 1.1.0](https://github.com/graphite-project/graphite-web) - [x] [graphite-web 0.9.15](https://github.com/graphite-project/graphite-web/tree/0.9.15) - [x] [graphite-web 1.0.0](https://github.com/graphite-project/graphite-web) - [x] [carbonapi 0.14.1+](https://github.com/go-graphite/carbonapi) - [x] [carbonzipper](https://github.com/go-graphite/carbonzipper) (DEPRECATED, is part of carbonapi currently) ## Build Required golang 1.18+ ```sh # build binary git clone https://github.com/lomik/graphite-clickhouse.git cd graphite-clickhouse make ``` ## Installation 1. Setup [Yandex ClickHouse](https://github.com/yandex/ClickHouse) and [carbon-clickhouse](https://github.com/lomik/carbon-clickhouse) 2. Setup and configure `graphite-clickhouse` 3. Add graphite-clickhouse `host:port` to graphite-web [CLUSTER_SERVERS](http://graphite.readthedocs.io/en/latest/config-local-settings.html#cluster-configuration) ## Configuration See [configuration documentation](./doc/config.md). ### Special headers processing Some HTTP headers are processed specially by the service #### Request headers *Grafana headers*: `X-Dashboard-Id`, `X-Grafana-Org-Id`, and `X-Panel-Id` are logged and passed further to the ClickHouse. *Debug headers* (see [debugging.md](./doc/debugging.md) for details): - `X-Gch-Debug-External-Data` - when this header is set to anything and every of `directory`, `directory-perm`, and `external-data-perm` parameters in `[debug]` is set and valid, service will save the dump of external data tables in the directory for debug output. - `X-Gch-Debug-Output` - header to enable special processing for `format=carbonapi_v3_pb` and `format=json` render output. - `X-Gch-Debug-Protobuf` - header enables the original marshallers for `protobuf` and `carbonapi_v3_pb` to check the binary data integrity. #### Response headers - `X-Gch-Request-Id` - the current request ID. - `X-Cached-Find` - Flag for find cache hit. ## Run on same host with old graphite-web 0.9.x By default graphite-web won't connect to CLUSTER_SERVER on localhost. Cheat: ```python class ForceLocal(str): def split(self, *args, **kwargs): return ["8.8.8.8", "8080"] CLUSTER_SERVERS = [ForceLocal("127.0.0.1:9090")] ``` ================================================ FILE: autocomplete/autocomplete.go ================================================ package autocomplete import ( "context" "encoding/json" "fmt" "net/http" "sort" "strconv" "strings" "time" "github.com/go-graphite/carbonapi/pkg/parser" "github.com/msaf1980/go-stringutils" "go.uber.org/zap" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/finder" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/helper/date" "github.com/lomik/graphite-clickhouse/helper/utils" "github.com/lomik/graphite-clickhouse/logs" "github.com/lomik/graphite-clickhouse/metrics" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/pkg/where" ) // override in unit tests for stable results var timeNow = time.Now type Handler struct { config *config.Config isValues bool } func NewTags(config *config.Config) *Handler { h := &Handler{ config: config, } return h } func NewValues(config *config.Config) *Handler { h := &Handler{ config: config, isValues: true, } return h } func dateString(autocompleteDays int, tm time.Time) (string, string) { fromDate := date.FromTimeToDaysFormat(tm.AddDate(0, 0, -autocompleteDays)) untilDate := date.UntilTimeToDaysFormat(tm) return fromDate, untilDate } func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Don't process, if the tagged table is not set if h.config.ClickHouse.TaggedTable == "" { w.Write([]byte{'[', ']'}) return } if h.isValues { h.ServeValues(w, r) } else { h.ServeTags(w, r) } } func getTagCountQuerier(config *config.Config, opts clickhouse.Options) *finder.TagCountQuerier { var tcq *finder.TagCountQuerier = nil if config.ClickHouse.TagsCountTable != "" { tcq = finder.NewTagCountQuerier( config.ClickHouse.URL, config.ClickHouse.TagsCountTable, opts, config.FeatureFlags.UseCarbonBehavior, config.FeatureFlags.DontMatchMissingTags, config.ClickHouse.TaggedUseDaily, ) } return tcq } func (h *Handler) requestExpr(r *http.Request, tcq *finder.TagCountQuerier, from, until int64) (*where.Where, *where.Where, map[string]bool, error) { formExpr := r.Form["expr"] expr := make([]string, 0, len(formExpr)) for i := 0; i < len(formExpr); i++ { if formExpr[i] != "" { expr = append(expr, formExpr[i]) } } usedTags := make(map[string]bool) wr := where.New() pw := where.New() if len(expr) == 0 { return wr, pw, usedTags, nil } terms, err := finder.ParseTaggedConditions(expr, h.config, true) if err != nil { return wr, pw, usedTags, err } if tcq != nil { tagValuesCosts, err := tcq.GetCostsFromCountTable(r.Context(), terms, from, until) if err != nil { return wr, pw, usedTags, err } if tagValuesCosts != nil { finder.SetCosts(terms, tagValuesCosts) } else if len(h.config.ClickHouse.TaggedCosts) != 0 { finder.SetCosts(terms, h.config.ClickHouse.TaggedCosts) } } finder.SortTaggedTermsByCost(terms) wr, pw, err = finder.TaggedWhere(terms, h.config.FeatureFlags.UseCarbonBehavior, h.config.FeatureFlags.DontMatchMissingTags) if err != nil { return wr, pw, usedTags, err } for i := 0; i < len(expr); i++ { a := strings.Split(expr[i], "=") usedTags[a[0]] = true } return wr, pw, usedTags, nil } func taggedKey(typ string, truncateSec int32, fromDate, untilDate string, tag string, exprs []string, tagPrefix string, limit int) (string, string) { ts := utils.TimestampTruncate(timeNow().Unix(), time.Duration(truncateSec)*time.Second) var sb stringutils.Builder sb.Grow(128) sb.WriteString(typ) sb.WriteString(fromDate) sb.WriteByte(';') sb.WriteString(untilDate) sb.WriteString(";limit=") sb.WriteInt(int64(limit), 10) tagStart := sb.Len() if tagPrefix != "" { sb.WriteString(";tagPrefix=") sb.WriteString(tagPrefix) } if tag != "" { sb.WriteString(";tag=") sb.WriteString(tag) } for _, expr := range exprs { sb.WriteString(";expr='") sb.WriteString(strings.Replace(expr, " = ", "=", 1)) sb.WriteByte('\'') } exprEnd := sb.Len() sb.WriteString(";ts=") sb.WriteString(strconv.FormatInt(ts, 10)) s := sb.String() return s, s[tagStart:exprEnd] } func taggedValuesKey(typ string, truncateSec int32, fromDate, untilDate string, tag string, exprs []string, valuePrefix string, limit int) (string, string) { ts := utils.TimestampTruncate(timeNow().Unix(), time.Duration(truncateSec)*time.Second) var sb stringutils.Builder sb.Grow(128) sb.WriteString(typ) sb.WriteString(fromDate) sb.WriteByte(';') sb.WriteString(untilDate) sb.WriteString(";limit=") sb.WriteInt(int64(limit), 10) tagStart := sb.Len() if valuePrefix != "" { sb.WriteString(";valuePrefix=") sb.WriteString(valuePrefix) } if tag != "" { sb.WriteString(";tag=") sb.WriteString(tag) } for _, expr := range exprs { sb.WriteString(";expr='") sb.WriteString(strings.Replace(expr, " = ", "=", 1)) sb.WriteByte('\'') } exprEnd := sb.Len() sb.WriteString(";ts=") sb.WriteString(strconv.FormatInt(ts, 10)) s := sb.String() return s, s[tagStart:exprEnd] } // func taggedTagsQuery(exprs []string, tagPrefix string, limit int) []string { // query := make([]string, 0, 3+len(exprs)) // if tagPrefix != "" { // query = append(query, "tagPrefix="+tagPrefix) // } // for _, expr := range exprs { // query = append(query, "expr='"+expr+"'") // } // query = append(query, "limit="+strconv.Itoa(limit)) // return query // } func (h *Handler) ServeTags(w http.ResponseWriter, r *http.Request) { start := timeNow() status := http.StatusOK accessLogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("http") logger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("autocomplete") r = r.WithContext(scope.WithLogger(r.Context(), logger)) var ( err error chReadRows int64 chReadBytes int64 metricsCount int64 readBytes int64 queueFail bool queueDuration time.Duration findCache bool opts clickhouse.Options ) username := r.Header.Get("X-Forwarded-User") limiter := h.config.GetUserTagsLimiter(username) defer func() { if rec := recover(); rec != nil { status = http.StatusInternalServerError logger.Error("panic during eval:", zap.String("requestID", scope.String(r.Context(), "requestID")), zap.Any("reason", rec), zap.Stack("stack"), ) answer := fmt.Sprintf("%v\nStack trace: %v", rec, zap.Stack("").String) http.Error(w, answer, status) } d := time.Since(start) dMS := d.Milliseconds() logs.AccessLog(accessLogger, h.config, r, status, d, queueDuration, findCache, queueFail) limiter.SendDuration(queueDuration.Milliseconds()) metrics.SendFindMetrics(metrics.TagsRequestMetric, status, dMS, 0, h.config.Metrics.ExtendedStat, metricsCount) if !findCache && chReadRows > 0 && chReadBytes > 0 { errored := status != http.StatusOK && status != http.StatusNotFound metrics.SendQueryRead(metrics.AutocompleteQMetric, 0, 0, dMS, metricsCount, readBytes, chReadRows, chReadBytes, errored) } }() r.ParseMultipartForm(1024 * 1024) tagPrefix := r.FormValue("tagPrefix") limitStr := r.FormValue("limit") limit := 10000 var body []byte if limitStr != "" { limit, err = strconv.Atoi(limitStr) if err == finder.ErrCostlySeriesByTag { status = http.StatusForbidden http.Error(w, err.Error(), status) return } else if err != nil { status = http.StatusBadRequest http.Error(w, err.Error(), status) return } } fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, start) var key string exprs := r.Form["expr"] // params := taggedTagsQuery(exprs, tagPrefix, limit) useCache := h.config.Common.FindCache != nil && h.config.Common.FindCacheConfig.FindTimeoutSec > 0 && !parser.TruthyBool(r.FormValue("noCache")) if useCache { key, _ = taggedKey("tags;", h.config.Common.FindCacheConfig.FindTimeoutSec, fromDate, untilDate, "", exprs, tagPrefix, limit) body, err = h.config.Common.FindCache.Get(key) if err == nil { if metrics.FinderCacheMetrics != nil { metrics.FinderCacheMetrics.CacheHits.Add(1) } findCache = true w.Header().Set("X-Cached-Find", strconv.Itoa(int(h.config.Common.FindCacheConfig.FindTimeoutSec))) } } opts = clickhouse.Options{ TLSConfig: h.config.ClickHouse.TLSConfig, Timeout: h.config.ClickHouse.IndexTimeout, ConnectTimeout: h.config.ClickHouse.ConnectTimeout, CheckRequestProgress: h.config.FeatureFlags.LogQueryProgress, ProgressSendingInterval: h.config.ClickHouse.ProgressSendingInterval, } wr, pw, usedTags, err := h.requestExpr( r, getTagCountQuerier(h.config, opts), start.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Unix(), start.Unix(), ) if err != nil { status = http.StatusBadRequest http.Error(w, err.Error(), status) return } if !findCache { var valueSQL string if len(usedTags) == 0 { valueSQL = "splitByChar('=', Tag1)[1] AS value" if tagPrefix != "" { wr.And(where.HasPrefix("Tag1", tagPrefix)) } } else { valueSQL = "splitByChar('=', arrayJoin(Tags))[1] AS value" if tagPrefix != "" { wr.And(where.HasPrefix("arrayJoin(Tags)", tagPrefix)) } } queryLimit := limit + len(usedTags) wr.Andf("Date >= '%s' AND Date <= '%s'", fromDate, untilDate) sql := fmt.Sprintf("SELECT %s FROM %s %s %s GROUP BY value ORDER BY value LIMIT %d", valueSQL, h.config.ClickHouse.TaggedTable, pw.PreWhereSQL(), wr.SQL(), queryLimit, ) var ( entered bool ctx context.Context cancel context.CancelFunc ) if limiter.Enabled() { ctx, cancel = context.WithTimeout(context.Background(), h.config.ClickHouse.IndexTimeout) defer cancel() err = limiter.Enter(ctx, "tags") queueDuration = time.Since(start) if err != nil { status = http.StatusServiceUnavailable queueFail = true logger.Error(err.Error()) http.Error(w, err.Error(), status) return } queueDuration = time.Since(start) entered = true defer func() { if entered { limiter.Leave(ctx, "tags") entered = false } }() } body, chReadRows, chReadBytes, err = clickhouse.Query( scope.WithTable(r.Context(), h.config.ClickHouse.TaggedTable), h.config.ClickHouse.URL, sql, opts, nil, ) if entered { // release early as possible limiter.Leave(ctx, "tags") entered = false } if err != nil { status, _ = clickhouse.HandleError(w, err) return } readBytes = int64(len(body)) if useCache { if metrics.FinderCacheMetrics != nil { metrics.FinderCacheMetrics.CacheMisses.Add(1) } h.config.Common.FindCache.Set(key, body, h.config.Common.FindCacheConfig.FindTimeoutSec) } } rows := strings.Split(stringutils.UnsafeString(body), "\n") tags := make([]string, 0, uint64(len(rows))+1) // +1 - reserve for "name" tag hasName := false for i := 0; i < len(rows); i++ { if rows[i] == "" { continue } if rows[i] == "__name__" { rows[i] = "name" } if usedTags[rows[i]] { continue } tags = append(tags, rows[i]) if rows[i] == "name" { hasName = true } } if !hasName && !usedTags["name"] && (tagPrefix == "" || strings.HasPrefix("name", tagPrefix)) { tags = append(tags, "name") } sort.Strings(tags) if len(tags) > limit { tags = tags[:limit] } if useCache { if findCache { logger.Info("finder", zap.String("get_cache", key), zap.Int("metrics", len(rows)), zap.Bool("find_cached", true), zap.Int32("ttl", h.config.Common.FindCacheConfig.FindTimeoutSec)) } else { logger.Info("finder", zap.String("set_cache", key), zap.Int("metrics", len(rows)), zap.Bool("find_cached", false), zap.Int32("ttl", h.config.Common.FindCacheConfig.FindTimeoutSec)) } } b, err := json.Marshal(tags) if err != nil { status = http.StatusInternalServerError http.Error(w, err.Error(), status) return } metricsCount = int64(len(tags)) w.Write(b) } // func taggedValuesQuery(tag string, exprs []string, valuePrefix string, limit int) []string { // query := make([]string, 0, 3+len(exprs)) // if tag != "" { // query = append(query, "tag="+tag) // } // if valuePrefix != "" { // query = append(query, "valuePrefix="+valuePrefix) // } // for _, expr := range exprs { // query = append(query, "expr='"+expr+"'") // } // query = append(query, "limit="+strconv.Itoa(limit)) // return query // } func (h *Handler) ServeValues(w http.ResponseWriter, r *http.Request) { start := timeNow() status := http.StatusOK accessLogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("http") logger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("autocomplete") r = r.WithContext(scope.WithLogger(r.Context(), logger)) var ( err error body []byte chReadRows int64 chReadBytes int64 metricsCount int64 queueFail bool queueDuration time.Duration findCache bool opts clickhouse.Options ) username := r.Header.Get("X-Forwarded-User") limiter := h.config.GetUserTagsLimiter(username) defer func() { if rec := recover(); rec != nil { status = http.StatusInternalServerError logger.Error("panic during eval:", zap.String("requestID", scope.String(r.Context(), "requestID")), zap.Any("reason", rec), zap.Stack("stack"), ) answer := fmt.Sprintf("%v\nStack trace: %v", rec, zap.Stack("").String) http.Error(w, answer, status) } d := time.Since(start) dMS := d.Milliseconds() logs.AccessLog(accessLogger, h.config, r, status, d, queueDuration, findCache, queueFail) limiter.SendDuration(queueDuration.Milliseconds()) metrics.SendFindMetrics(metrics.TagsRequestMetric, status, dMS, 0, h.config.Metrics.ExtendedStat, metricsCount) if !findCache && chReadRows > 0 && chReadBytes > 0 { errored := status != http.StatusOK && status != http.StatusNotFound metrics.SendQueryRead(metrics.AutocompleteQMetric, 0, 0, dMS, metricsCount, int64(len(body)), chReadRows, chReadBytes, errored) } }() r.ParseMultipartForm(1024 * 1024) tag := r.FormValue("tag") if tag == "name" { tag = "__name__" } valuePrefix := r.FormValue("valuePrefix") limitStr := r.FormValue("limit") limit := 10000 if limitStr != "" { limit, err = strconv.Atoi(limitStr) if err != nil { status = http.StatusBadRequest http.Error(w, err.Error(), status) return } } fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, start) var key string exprs := r.Form["expr"] // params := taggedValuesQuery(tag, exprs, valuePrefix, limit) // taggedKey(tag, , "valuePrefix="+valuePrefix, limit) useCache := h.config.Common.FindCache != nil && h.config.Common.FindCacheConfig.FindTimeoutSec > 0 && !parser.TruthyBool(r.FormValue("noCache")) if useCache { // logger = logger.With(zap.String("use_cache", "true")) key, _ = taggedValuesKey("values;", h.config.Common.FindCacheConfig.FindTimeoutSec, fromDate, untilDate, tag, exprs, valuePrefix, limit) body, err = h.config.Common.FindCache.Get(key) if err == nil { if metrics.FinderCacheMetrics != nil { metrics.FinderCacheMetrics.CacheHits.Add(1) } findCache = true w.Header().Set("X-Cached-Find", strconv.Itoa(int(h.config.Common.FindCacheConfig.FindTimeoutSec))) } } opts = clickhouse.Options{ TLSConfig: h.config.ClickHouse.TLSConfig, Timeout: h.config.ClickHouse.IndexTimeout, ConnectTimeout: h.config.ClickHouse.ConnectTimeout, CheckRequestProgress: h.config.FeatureFlags.LogQueryProgress, ProgressSendingInterval: h.config.ClickHouse.ProgressSendingInterval, } if !findCache { wr, pw, usedTags, err := h.requestExpr( r, getTagCountQuerier(h.config, opts), start.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Unix(), start.Unix(), ) if err == finder.ErrCostlySeriesByTag { status = http.StatusForbidden http.Error(w, err.Error(), status) return } else if err != nil { status = http.StatusBadRequest http.Error(w, err.Error(), status) return } var valueSQL string if len(usedTags) == 0 { valueSQL = fmt.Sprintf("substr(Tag1, %d) AS value", len(tag)+2) wr.And(where.HasPrefix("Tag1", tag+"="+valuePrefix)) } else { prefixSelector := where.HasPrefix("x", tag+"="+valuePrefix) valueSQL = fmt.Sprintf("substr(arrayFilter(x -> %s, Tags)[1], %d) AS value", prefixSelector, len(tag)+2) wr.And("arrayExists(x -> " + prefixSelector + ", Tags)") } wr.Andf("Date >= '%s' AND Date <= '%s'", fromDate, untilDate) sql := fmt.Sprintf("SELECT %s FROM %s %s %s GROUP BY value ORDER BY value LIMIT %d", valueSQL, h.config.ClickHouse.TaggedTable, pw.PreWhereSQL(), wr.SQL(), limit, ) var ( entered bool ctx context.Context cancel context.CancelFunc ) if limiter.Enabled() { ctx, cancel = context.WithTimeout(context.Background(), h.config.ClickHouse.IndexTimeout) defer cancel() err = limiter.Enter(ctx, "tags") queueDuration = time.Since(start) if err != nil { status = http.StatusServiceUnavailable queueFail = true logger.Error(err.Error()) http.Error(w, err.Error(), status) return } queueDuration = time.Since(start) entered = true defer func() { if entered { limiter.Leave(ctx, "tags") entered = false } }() } body, chReadRows, chReadBytes, err = clickhouse.Query( scope.WithTable(r.Context(), h.config.ClickHouse.TaggedTable), h.config.ClickHouse.URL, sql, opts, nil, ) if entered { // release early as possible limiter.Leave(ctx, "tags") entered = false } if err != nil { status, _ = clickhouse.HandleError(w, err) return } if useCache { if metrics.FinderCacheMetrics != nil { metrics.FinderCacheMetrics.CacheMisses.Add(1) } h.config.Common.FindCache.Set(key, body, h.config.Common.FindCacheConfig.FindTimeoutSec) } } var rows []string if len(body) > 0 { rows = strings.Split(stringutils.UnsafeString(body), "\n") if len(rows) > 0 && rows[len(rows)-1] == "" { rows = rows[:len(rows)-1] } metricsCount = int64(len(rows)) } if useCache { if findCache { logger.Info("finder", zap.String("get_cache", key), zap.Int("metrics", len(rows)), zap.Bool("find_cached", true), zap.Int32("ttl", h.config.Common.FindCacheConfig.FindTimeoutSec)) } else { logger.Info("finder", zap.String("set_cache", key), zap.Int("metrics", len(rows)), zap.Bool("find_cached", false), zap.Int32("ttl", h.config.Common.FindCacheConfig.FindTimeoutSec)) } } b, err := json.Marshal(rows) if err != nil { status = http.StatusInternalServerError http.Error(w, err.Error(), status) return } w.Write(b) } ================================================ FILE: autocomplete/autocomplete_test.go ================================================ package autocomplete import ( "io" "net/http" "net/http/httptest" "strconv" "testing" "time" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/date" chtest "github.com/lomik/graphite-clickhouse/helper/tests/clickhouse" "github.com/lomik/graphite-clickhouse/metrics" "github.com/stretchr/testify/assert" ) func NewRequest(method, url string, body io.Reader) *http.Request { r, _ := http.NewRequest(method, url, body) return r } type testStruct struct { request *http.Request wantCode int want string wantContent string } func testResponce(t *testing.T, step int, h *Handler, tt *testStruct, wantCachedFind string) { w := httptest.NewRecorder() h.ServeHTTP(w, tt.request) s := w.Body.String() assert.Equalf(t, tt.wantCode, w.Code, "code mismatch step %d\n,%s", step, s) if w.Code == http.StatusOK { if tt.wantContent != "" { contentType := w.Header().Get("Content-Type") assert.Equalf(t, tt.wantContent, contentType, "content type mismatch, step %d", step) } cachedFindHeader := w.Header().Get("X-Cached-Find") assert.Equalf(t, cachedFindHeader, wantCachedFind, "cached find '%s' mismatch, want be %v, step %d", cachedFindHeader, wantCachedFind, step) assert.Equalf(t, tt.want, s, "Step %d", step) } } func TestHandler_ServeTags(t *testing.T) { timeNow = func() time.Time { return time.Unix(1669714247, 0) } metrics.DisableMetrics() srv := chtest.NewTestServer() defer srv.Close() cfg, _ := config.DefaultConfig() cfg.ClickHouse.URL = srv.URL cfg.ClickHouse.TaggedTable = "graphite_tagged" h := NewTags(cfg) now := timeNow() fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now) // Test 1: Get all tags without filters srv.AddResponce( "SELECT splitByChar('=', Tag1)[1] AS value FROM graphite_tagged WHERE "+ "Date >= '"+fromDate+"' AND Date <= '"+untilDate+"' GROUP BY value ORDER BY value LIMIT 10000", &chtest.TestResponse{ Body: []byte("__name__\nenvironment\nproject\nhost\n"), }) // Test 2: Get tags with prefix filter srv.AddResponce( "SELECT splitByChar('=', Tag1)[1] AS value FROM graphite_tagged WHERE "+ "(Tag1 LIKE 'pr%') AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000", &chtest.TestResponse{ Body: []byte("project\n"), }) // Test 3: Get tags with expr filters srv.AddResponce( "SELECT splitByChar('=', arrayJoin(Tags))[1] AS value FROM graphite_tagged WHERE "+ "(Tag1='environment=production') AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10001", &chtest.TestResponse{ Body: []byte("__name__\nhost\nproject\n"), }) // Test 4: Get tags with multiple expr filters srv.AddResponce( "SELECT splitByChar('=', arrayJoin(Tags))[1] AS value FROM graphite_tagged WHERE "+ "((Tag1='environment=production') AND (has(Tags, 'project=web'))) AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10002", &chtest.TestResponse{ Body: []byte("__name__\nhost\n"), }) // Test 5: Get tags with prefix and expr filters srv.AddResponce( "SELECT splitByChar('=', arrayJoin(Tags))[1] AS value FROM graphite_tagged WHERE "+ "((Tag1='environment=production') AND (arrayJoin(Tags) LIKE 'h%')) AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10001", &chtest.TestResponse{ Body: []byte("host\n"), }) tests := []testStruct{ { request: NewRequest("GET", srv.URL+"/tags/autoComplete/tags", nil), wantCode: http.StatusOK, want: `["environment","host","name","project"]`, }, { request: NewRequest("GET", srv.URL+"/tags/autoComplete/tags?tagPrefix=pr", nil), wantCode: http.StatusOK, want: `["project"]`, }, { request: NewRequest("GET", srv.URL+"/tags/autoComplete/tags?expr=environment%3Dproduction", nil), wantCode: http.StatusOK, want: `["host","name","project"]`, }, { request: NewRequest("GET", srv.URL+"/tags/autoComplete/tags?expr=environment%3Dproduction&expr=project%3Dweb", nil), wantCode: http.StatusOK, want: `["host","name"]`, }, { request: NewRequest("GET", srv.URL+"/tags/autoComplete/tags?expr=environment%3Dproduction&tagPrefix=h", nil), wantCode: http.StatusOK, want: `["host"]`, }, } for i, tt := range tests { t.Run("Test#"+strconv.Itoa(i), func(t *testing.T) { testResponce(t, i, h, &tt, "") }) } } func TestHandler_ServeTagsWithCache(t *testing.T) { timeNow = func() time.Time { return time.Unix(1669714247, 0) } metrics.DisableMetrics() srv := chtest.NewTestServer() defer srv.Close() cfg, _ := config.DefaultConfig() cfg.ClickHouse.URL = srv.URL cfg.ClickHouse.TaggedTable = "graphite_tagged" // Enable cache cfg.Common.FindCacheConfig = config.CacheConfig{ Type: "mem", Size: 8192, FindTimeoutSec: 1, } var err error cfg.Common.FindCache, err = config.CreateCache("autocomplete", &cfg.Common.FindCacheConfig) if err != nil { t.Fatalf("Failed to create find cache: %v", err) } h := NewTags(cfg) now := timeNow() fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now) srv.AddResponce( "SELECT splitByChar('=', Tag1)[1] AS value FROM graphite_tagged WHERE "+ "Date >= '"+fromDate+"' AND Date <= '"+untilDate+"' GROUP BY value ORDER BY value LIMIT 10000", &chtest.TestResponse{ Body: []byte("__name__\nenvironment\nproject\nhost\n"), }) test := testStruct{ request: NewRequest("GET", srv.URL+"/tags/autoComplete/tags", nil), wantCode: http.StatusOK, want: `["environment","host","name","project"]`, } // First request - should hit the database testResponce(t, 0, h, &test, "") assert.Equal(t, uint64(1), srv.Queries()) // Second request - should hit the cache testResponce(t, 1, h, &test, "1") assert.Equal(t, uint64(1), srv.Queries()) // No new queries // Wait for cache expiration time.Sleep(time.Second * 2) // Third request - should hit the database again testResponce(t, 2, h, &test, "") assert.Equal(t, uint64(2), srv.Queries()) } func TestHandler_ServeValues(t *testing.T) { timeNow = func() time.Time { return time.Unix(1669714247, 0) } metrics.DisableMetrics() srv := chtest.NewTestServer() defer srv.Close() cfg, _ := config.DefaultConfig() cfg.ClickHouse.URL = srv.URL cfg.ClickHouse.TaggedTable = "graphite_tagged" h := NewValues(cfg) now := timeNow() until := strconv.FormatInt(now.Unix(), 10) from := strconv.FormatInt(now.Add(-time.Minute).Unix(), 10) fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now) srv.AddResponce( "SELECT substr(arrayFilter(x -> x LIKE 'host=%', Tags)[1], 6) AS value FROM graphite_tagged WHERE (((Tag1='environment=production') AND (has(Tags, 'project=web'))) AND (arrayExists(x -> x LIKE 'host=%', Tags))) AND "+ "(Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000", &chtest.TestResponse{ Body: []byte("host1\nhost2\ndc-host2\ndc-host3\n"), }) tests := []testStruct{ { request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?"+ "expr=environment%3Dproduction"+"&"+"expr=project%3Dweb"+"&"+"tag=host"+ "&limit=10000&from="+from+"&until="+until, nil), wantCode: http.StatusOK, want: "[\"host1\",\"host2\",\"dc-host2\",\"dc-host3\"]", wantContent: "text/plain; charset=utf-8", }, } var queries uint64 for i, tt := range tests { t.Run(tt.request.URL.RawQuery+"#"+strconv.Itoa(i), func(t *testing.T) { for i := 0; i < 2; i++ { testResponce(t, i, h, &tt, "") } assert.Equal(t, uint64(2), srv.Queries()-queries) queries = srv.Queries() }) } } func TestHandler_ServeValuesWithValuePrefix(t *testing.T) { timeNow = func() time.Time { return time.Unix(1669714247, 0) } metrics.DisableMetrics() srv := chtest.NewTestServer() defer srv.Close() cfg, _ := config.DefaultConfig() cfg.ClickHouse.URL = srv.URL cfg.ClickHouse.TaggedTable = "graphite_tagged" h := NewValues(cfg) now := timeNow() fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now) // Test with valuePrefix srv.AddResponce( "SELECT substr(Tag1, 6) AS value FROM graphite_tagged WHERE "+ "(Tag1 LIKE 'host=dc-%') AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 100", &chtest.TestResponse{ Body: []byte("dc-host1\ndc-host2\ndc-host3\n"), }) test := testStruct{ request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=host&valuePrefix=dc-&limit=100", nil), wantCode: http.StatusOK, want: "[\"dc-host1\",\"dc-host2\",\"dc-host3\"]", } testResponce(t, 0, h, &test, "") } func TestHandler_ServeValuesNameTag(t *testing.T) { timeNow = func() time.Time { return time.Unix(1669714247, 0) } metrics.DisableMetrics() srv := chtest.NewTestServer() defer srv.Close() cfg, _ := config.DefaultConfig() cfg.ClickHouse.URL = srv.URL cfg.ClickHouse.TaggedTable = "graphite_tagged" h := NewValues(cfg) now := timeNow() fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now) // Test with name tag (which should be converted to __name__) srv.AddResponce( `SELECT substr(Tag1, 10) AS value FROM graphite_tagged WHERE `+ `(Tag1 LIKE '\\_\\_name\\_\\_=metric.%') AND (Date >= '`+fromDate+`' AND Date <= '`+untilDate+`') GROUP BY value ORDER BY value LIMIT 10000`, &chtest.TestResponse{ Body: []byte("metric.cpu.usage\nmetric.memory.used\nmetric.disk.io\n"), }) test := testStruct{ request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=name&valuePrefix=metric.", nil), wantCode: http.StatusOK, want: "[\"metric.cpu.usage\",\"metric.memory.used\",\"metric.disk.io\"]", } testResponce(t, 0, h, &test, "") } func TestHandler_ServeValuesWithCache(t *testing.T) { timeNow = func() time.Time { return time.Unix(1669714247, 0) } metrics.DisableMetrics() srv := chtest.NewTestServer() defer srv.Close() cfg, _ := config.DefaultConfig() cfg.ClickHouse.URL = srv.URL cfg.ClickHouse.TaggedTable = "graphite_tagged" // Enable cache cfg.Common.FindCacheConfig = config.CacheConfig{ Type: "mem", Size: 8192, FindTimeoutSec: 1, } var err error cfg.Common.FindCache, err = config.CreateCache("autocomplete", &cfg.Common.FindCacheConfig) if err != nil { t.Fatalf("Failed to create find cache: %v", err) } h := NewValues(cfg) now := timeNow() fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now) srv.AddResponce( "SELECT substr(Tag1, 6) AS value FROM graphite_tagged WHERE "+ "(Tag1 LIKE 'host=%') AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000", &chtest.TestResponse{ Body: []byte("host1\nhost2\nhost3\n"), }) test := testStruct{ request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=host", nil), wantCode: http.StatusOK, want: "[\"host1\",\"host2\",\"host3\"]", } // First request - should hit the database testResponce(t, 0, h, &test, "") assert.Equal(t, uint64(1), srv.Queries()) // Second request - should hit the cache testResponce(t, 1, h, &test, "1") assert.Equal(t, uint64(1), srv.Queries()) // No new queries // Wait for cache expiration time.Sleep(time.Second * 2) // Third request - should hit the database again testResponce(t, 2, h, &test, "") assert.Equal(t, uint64(2), srv.Queries()) } func TestHandler_ServeValuesWithCacheAndExpr(t *testing.T) { timeNow = func() time.Time { return time.Unix(1669714247, 0) } metrics.DisableMetrics() srv := chtest.NewTestServer() defer srv.Close() cfg, _ := config.DefaultConfig() cfg.ClickHouse.URL = srv.URL cfg.ClickHouse.TaggedTable = "graphite_tagged" // Enable cache cfg.Common.FindCacheConfig = config.CacheConfig{ Type: "mem", Size: 8192, FindTimeoutSec: 1, } var err error cfg.Common.FindCache, err = config.CreateCache("autocomplete", &cfg.Common.FindCacheConfig) if err != nil { t.Fatalf("Failed to create find cache: %v", err) } h := NewValues(cfg) now := timeNow() fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now) srv.AddResponce( "SELECT substr(arrayFilter(x -> x LIKE 'host=%', Tags)[1], 6) AS value FROM graphite_tagged WHERE "+ "((Tag1='environment=production') AND (arrayExists(x -> x LIKE 'host=%', Tags))) AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000", &chtest.TestResponse{ Body: []byte("prod-host1\nprod-host2\n"), }) test := testStruct{ request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=host&expr=environment%3Dproduction", nil), wantCode: http.StatusOK, want: "[\"prod-host1\",\"prod-host2\"]", } // First request - should hit the database testResponce(t, 0, h, &test, "") assert.Equal(t, uint64(1), srv.Queries()) // Second request - should hit the cache testResponce(t, 1, h, &test, "1") assert.Equal(t, uint64(1), srv.Queries()) // No new queries // Test with different valuePrefix - should not hit cache test2 := testStruct{ request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=host&expr=environment%3Dproduction&valuePrefix=prod-host1", nil), wantCode: http.StatusOK, want: "[\"prod-host1\"]", } srv.AddResponce( "SELECT substr(arrayFilter(x -> x LIKE 'host=prod-host1%', Tags)[1], 6) AS value FROM graphite_tagged WHERE "+ "((Tag1='environment=production') AND (arrayExists(x -> x LIKE 'host=prod-host1%', Tags))) AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000", &chtest.TestResponse{ Body: []byte("prod-host1\n"), }) // Should hit the database because valuePrefix is different testResponce(t, 2, h, &test2, "") assert.Equal(t, uint64(2), srv.Queries()) } func TestHandler_ServeValuesWithInvalidLimit(t *testing.T) { timeNow = func() time.Time { return time.Unix(1669714247, 0) } metrics.DisableMetrics() srv := chtest.NewTestServer() defer srv.Close() cfg, _ := config.DefaultConfig() cfg.ClickHouse.URL = srv.URL cfg.ClickHouse.TaggedTable = "graphite_tagged" h := NewValues(cfg) test := testStruct{ request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=host&limit=invalid", nil), wantCode: http.StatusBadRequest, want: "", // Error response } testResponce(t, 0, h, &test, "") } func TestHandler_ServeValuesWithMultipleExpr(t *testing.T) { timeNow = func() time.Time { return time.Unix(1669714247, 0) } metrics.DisableMetrics() srv := chtest.NewTestServer() defer srv.Close() cfg, _ := config.DefaultConfig() cfg.ClickHouse.URL = srv.URL cfg.ClickHouse.TaggedTable = "graphite_tagged" h := NewValues(cfg) now := timeNow() fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now) // Test with multiple expressions and valuePrefix srv.AddResponce( "SELECT substr(arrayFilter(x -> x LIKE 'host=dc-%', Tags)[1], 6) AS value FROM graphite_tagged WHERE "+ "(((Tag1='environment=production') AND (has(Tags, 'project=web'))) AND (arrayExists(x -> x LIKE 'host=dc-%', Tags))) AND "+ "(Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000", &chtest.TestResponse{ Body: []byte("dc-host1\ndc-host2\n"), }) test := testStruct{ request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=host&expr=environment%3Dproduction&expr=project%3Dweb&valuePrefix=dc-", nil), wantCode: http.StatusOK, want: "[\"dc-host1\",\"dc-host2\"]", } testResponce(t, 0, h, &test, "") } func TestHandler_ServeValuesNoCache(t *testing.T) { timeNow = func() time.Time { return time.Unix(1669714247, 0) } metrics.DisableMetrics() srv := chtest.NewTestServer() defer srv.Close() cfg, _ := config.DefaultConfig() cfg.ClickHouse.URL = srv.URL cfg.ClickHouse.TaggedTable = "graphite_tagged" // Enable cache cfg.Common.FindCacheConfig = config.CacheConfig{ Type: "mem", Size: 8192, FindTimeoutSec: 60, } var err error cfg.Common.FindCache, err = config.CreateCache("autocomplete", &cfg.Common.FindCacheConfig) if err != nil { t.Fatalf("Failed to create find cache: %v", err) } h := NewValues(cfg) now := timeNow() fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now) srv.AddResponce( "SELECT substr(Tag1, 6) AS value FROM graphite_tagged WHERE "+ "(Tag1 LIKE 'host=%') AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000", &chtest.TestResponse{ Body: []byte("host1\nhost2\n"), }) test := testStruct{ request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=host&noCache=true", nil), wantCode: http.StatusOK, want: "[\"host1\",\"host2\"]", } // First request with noCache=true - should always hit the database testResponce(t, 0, h, &test, "") assert.Equal(t, uint64(1), srv.Queries()) // Second request with noCache=true - should hit the database again testResponce(t, 1, h, &test, "") assert.Equal(t, uint64(2), srv.Queries()) // Should increase } func TestHandler_ServeValuesEmptyResult(t *testing.T) { timeNow = func() time.Time { return time.Unix(1669714247, 0) } metrics.DisableMetrics() srv := chtest.NewTestServer() defer srv.Close() cfg, _ := config.DefaultConfig() cfg.ClickHouse.URL = srv.URL cfg.ClickHouse.TaggedTable = "graphite_tagged" h := NewValues(cfg) now := timeNow() fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now) // Test empty result srv.AddResponce( "SELECT substr(Tag1, 13) AS value FROM graphite_tagged WHERE "+ "(Tag1 LIKE 'nonexistent=%') AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000", &chtest.TestResponse{ Body: []byte(""), // Empty response }) test := testStruct{ request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=nonexistent", nil), wantCode: http.StatusOK, want: "null", // Empty array } testResponce(t, 0, h, &test, "") } func TestHandler_ServeTagsWithCostOptimization(t *testing.T) { timeNow = func() time.Time { return time.Unix(1669714247, 0) } metrics.DisableMetrics() srv := chtest.NewTestServer() defer srv.Close() cfg, _ := config.DefaultConfig() cfg.ClickHouse.URL = srv.URL cfg.ClickHouse.TaggedTable = "graphite_tagged" cfg.ClickHouse.TagsCountTable = "tag1_count_per_day" h := NewTags(cfg) now := timeNow() fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now) from := now.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Unix() until := now.Unix() // Test case: Tags query with multiple expressions should use cost optimization // First response: tags count query to get costs srv.AddResponce( "SELECT Tag1, sum(Count) as cnt FROM tag1_count_per_day WHERE "+ "((Tag1='environment=production') OR (Tag1='project=web')) AND "+ "(Date >= '"+date.FromTimestampToDaysFormat(from)+"' AND Date <= '"+date.UntilTimestampToDaysFormat(until)+"') GROUP BY Tag1 FORMAT TabSeparatedRaw", &chtest.TestResponse{ Body: []byte("environment=production\t10000\nproject=web\t500\n"), }) // Second response: main tags query (should be ordered based on costs) srv.AddResponce( "SELECT splitByChar('=', arrayJoin(Tags))[1] AS value FROM graphite_tagged WHERE "+ "((Tag1='project=web') AND (has(Tags, 'environment=production'))) AND "+ // Lower cost term first "(Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10002", &chtest.TestResponse{ Body: []byte("__name__\nhost\nregion\n"), }) test := testStruct{ request: NewRequest("GET", srv.URL+"/tags/autoComplete/tags?expr=environment%3Dproduction&expr=project%3Dweb", nil), wantCode: http.StatusOK, want: `["host","name","region"]`, } testResponce(t, 0, h, &test, "") assert.Equal(t, uint64(2), srv.Queries()) // Should have 2 queries: cost query + main query } func TestHandler_ServeValuesWithCostOptimization(t *testing.T) { timeNow = func() time.Time { return time.Unix(1669714247, 0) } metrics.DisableMetrics() srv := chtest.NewTestServer() defer srv.Close() cfg, _ := config.DefaultConfig() cfg.ClickHouse.URL = srv.URL cfg.ClickHouse.TaggedTable = "graphite_tagged" cfg.ClickHouse.TagsCountTable = "tag1_count_per_day" h := NewValues(cfg) now := timeNow() fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now) from := now.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Unix() until := now.Unix() // First response: tags count query for cost optimization srv.AddResponce( "SELECT Tag1, sum(Count) as cnt FROM tag1_count_per_day WHERE "+ "((Tag1='environment=production') OR (Tag1='datacenter=us-east')) AND "+ "(Date >= '"+date.FromTimestampToDaysFormat(from)+"' AND Date <= '"+date.UntilTimestampToDaysFormat(until)+"') GROUP BY Tag1 FORMAT TabSeparatedRaw", &chtest.TestResponse{ Body: []byte("environment=production\t5000\ndatacenter=us-east\t100\n"), }) // Second response: values query (should use optimized order) srv.AddResponce( "SELECT substr(arrayFilter(x -> x LIKE 'host=%', Tags)[1], 6) AS value FROM graphite_tagged WHERE "+ "(((Tag1='datacenter=us-east') AND (has(Tags, 'environment=production'))) AND "+ // Lower cost first "(arrayExists(x -> x LIKE 'host=%', Tags))) AND "+ "(Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000", &chtest.TestResponse{ Body: []byte("host1\nhost2\nhost3\n"), }) test := testStruct{ request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=host&expr=environment%3Dproduction&expr=datacenter%3Dus-east", nil), wantCode: http.StatusOK, want: "[\"host1\",\"host2\",\"host3\"]", } testResponce(t, 0, h, &test, "") assert.Equal(t, uint64(2), srv.Queries()) } func TestHandler_ServeTagsWithWildcardExpressions(t *testing.T) { timeNow = func() time.Time { return time.Unix(1669714247, 0) } metrics.DisableMetrics() srv := chtest.NewTestServer() defer srv.Close() cfg, _ := config.DefaultConfig() cfg.ClickHouse.URL = srv.URL cfg.ClickHouse.TaggedTable = "graphite_tagged" cfg.ClickHouse.TagsCountTable = "tag1_count_per_day" h := NewTags(cfg) now := timeNow() fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now) // Test with wildcard expressions (should not query tags count table) srv.AddResponce( "SELECT splitByChar('=', arrayJoin(Tags))[1] AS value FROM graphite_tagged WHERE "+ "(Tag1 LIKE 'environment=prod%') AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10001", &chtest.TestResponse{ Body: []byte("__name__\nhost\nproject\n"), }) test := testStruct{ request: NewRequest("GET", srv.URL+"/tags/autoComplete/tags?expr=environment%3Dprod*", nil), wantCode: http.StatusOK, want: `["host","name","project"]`, } testResponce(t, 0, h, &test, "") assert.Equal(t, uint64(1), srv.Queries()) // Only 1 query since wildcards skip cost optimization } func TestHandler_ServeValuesWithNoEqualityTerms(t *testing.T) { timeNow = func() time.Time { return time.Unix(1669714247, 0) } metrics.DisableMetrics() srv := chtest.NewTestServer() defer srv.Close() cfg, _ := config.DefaultConfig() cfg.ClickHouse.URL = srv.URL cfg.ClickHouse.TaggedTable = "graphite_tagged" cfg.ClickHouse.TagsCountTable = "tag1_count_per_day" h := NewValues(cfg) now := timeNow() fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now) // Test with != operator (should not use tags count table) srv.AddResponce( "SELECT substr(arrayFilter(x -> x LIKE 'host=%', Tags)[1], 6) AS value FROM graphite_tagged WHERE "+ "((NOT arrayExists((x) -> x='environment=development', Tags)) AND (arrayExists(x -> x LIKE 'host=%', Tags))) AND "+ "(Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000", &chtest.TestResponse{ Body: []byte("host1\nhost2\n"), }) test := testStruct{ request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=host&expr=environment%21%3Ddevelopment", nil), wantCode: http.StatusOK, want: "[\"host1\",\"host2\"]", } testResponce(t, 0, h, &test, "") assert.Equal(t, uint64(1), srv.Queries()) // Only 1 query since no equality terms } func TestHandler_ServeTagsWithHighCostTags(t *testing.T) { timeNow = func() time.Time { return time.Unix(1669714247, 0) } metrics.DisableMetrics() srv := chtest.NewTestServer() defer srv.Close() cfg, _ := config.DefaultConfig() cfg.ClickHouse.URL = srv.URL cfg.ClickHouse.TaggedTable = "graphite_tagged" cfg.ClickHouse.TagsCountTable = "tag1_count_per_day" h := NewTags(cfg) now := timeNow() fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now) from := now.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Unix() until := now.Unix() // Test with high cardinality tags - tags should be reordered based on cost srv.AddResponce( "SELECT Tag1, sum(Count) as cnt FROM tag1_count_per_day WHERE "+ "(((Tag1='__name__=high.cost.metric') OR (Tag1='environment=production')) OR (Tag1='dc=west')) AND "+ "(Date >= '"+date.FromTimestampToDaysFormat(from)+"' AND Date <= '"+date.UntilTimestampToDaysFormat(until)+"') GROUP BY Tag1 FORMAT TabSeparatedRaw", &chtest.TestResponse{ Body: []byte("__name__=high.cost.metric\t1000000\nenvironment=production\t10000\ndc=west\t50\n"), }) // Query should use lowest cost tag first (dc=west) srv.AddResponce( "SELECT splitByChar('=', arrayJoin(Tags))[1] AS value FROM graphite_tagged WHERE "+ "(((Tag1='dc=west') AND (has(Tags, 'environment=production'))) AND (has(Tags, '__name__=high.cost.metric'))) AND "+ "(Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10003", &chtest.TestResponse{ Body: []byte("host\nproject\nregion\n"), }) test := testStruct{ request: NewRequest("GET", srv.URL+"/tags/autoComplete/tags?expr=name%3Dhigh.cost.metric&expr=environment%3Dproduction&expr=dc%3Dwest", nil), wantCode: http.StatusOK, want: `["host","project","region"]`, } testResponce(t, 0, h, &test, "") assert.Equal(t, uint64(2), srv.Queries()) } func TestHandler_ServeValuesWithMixedOperators(t *testing.T) { timeNow = func() time.Time { return time.Unix(1669714247, 0) } metrics.DisableMetrics() srv := chtest.NewTestServer() defer srv.Close() cfg, _ := config.DefaultConfig() cfg.ClickHouse.URL = srv.URL cfg.ClickHouse.TaggedTable = "graphite_tagged" cfg.ClickHouse.TagsCountTable = "tag1_count_per_day" h := NewValues(cfg) now := timeNow() fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now) from := now.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Unix() until := now.Unix() // Test with mixed operators (only equality operators should be in cost query) srv.AddResponce( "SELECT Tag1, sum(Count) as cnt FROM tag1_count_per_day WHERE "+ "((Tag1='environment=production') OR (Tag1='project=api')) AND "+ "(Date >= '"+date.FromTimestampToDaysFormat(from)+"' AND Date <= '"+date.UntilTimestampToDaysFormat(until)+"') GROUP BY Tag1 FORMAT TabSeparatedRaw", &chtest.TestResponse{ Body: []byte("environment=production\t8000\nproject=api\t200\n"), }) // Main query should include != operator but order by cost srv.AddResponce( "SELECT substr(arrayFilter(x -> x LIKE 'host=%', Tags)[1], 6) AS value FROM graphite_tagged WHERE "+ "((((Tag1='project=api') AND (has(Tags, 'environment=production'))) AND (NOT arrayExists((x) -> x='dc=east', Tags))) AND "+ "(arrayExists(x -> x LIKE 'host=%', Tags))) AND "+ "(Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000", &chtest.TestResponse{ Body: []byte("host1\nhost2\n"), }) test := testStruct{ request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=host&expr=environment%3Dproduction&expr=project%3Dapi&expr=dc%21%3Deast", nil), wantCode: http.StatusOK, want: "[\"host1\",\"host2\"]", } testResponce(t, 0, h, &test, "") assert.Equal(t, uint64(2), srv.Queries()) // Cost query only for equality terms } ================================================ FILE: cache/cache.go ================================================ package cache import ( "crypto/sha256" "encoding/hex" "errors" "sync/atomic" "time" "github.com/bradfitz/gomemcache/memcache" "github.com/msaf1980/go-expirecache" ) var ( ErrTimeout = errors.New("cache: timeout") ErrNotFound = errors.New("cache: not found") ) type BytesCache interface { Get(k string) ([]byte, error) Set(k string, v []byte, expire int32) } func NewExpireCache(maxsize uint64) BytesCache { ec := expirecache.New[string, []byte](maxsize) go ec.ApproximateCleaner(10 * time.Second) return &ExpireCache{ec: ec} } type ExpireCache struct { ec *expirecache.Cache[string, []byte] } func (ec ExpireCache) Get(k string) ([]byte, error) { v, ok := ec.ec.Get(k) if !ok { return nil, ErrNotFound } return v, nil } func (ec ExpireCache) Set(k string, v []byte, expire int32) { ec.ec.Set(k, v, uint64(len(v)), expire) } func NewMemcached(prefix string, servers ...string) BytesCache { return &MemcachedCache{prefix: prefix, client: memcache.New(servers...)} } type MemcachedCache struct { prefix string client *memcache.Client timeouts uint64 } func (m *MemcachedCache) Get(k string) ([]byte, error) { key := sha256.Sum256([]byte(k)) hk := hex.EncodeToString(key[:]) done := make(chan bool, 1) var err error var item *memcache.Item go func() { item, err = m.client.Get(m.prefix + hk) done <- true }() timeout := time.After(50 * time.Millisecond) select { case <-timeout: atomic.AddUint64(&m.timeouts, 1) return nil, ErrTimeout case <-done: } if err != nil { // translate to internal cache miss error if errors.Is(err, memcache.ErrCacheMiss) { err = ErrNotFound } return nil, err } return item.Value, nil } func (m *MemcachedCache) Set(k string, v []byte, expire int32) { key := sha256.Sum256([]byte(k)) hk := hex.EncodeToString(key[:]) go func() { _ = m.client.Set(&memcache.Item{Key: m.prefix + hk, Value: v, Expiration: expire}) }() } func (m *MemcachedCache) Timeouts() uint64 { return atomic.LoadUint64(&m.timeouts) } ================================================ FILE: capabilities/handler.go ================================================ package capabilities import ( "encoding/json" "io" "net/http" "os" "time" v3pb "github.com/go-graphite/protocol/carbonapi_v3_pb" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/logs" "github.com/lomik/graphite-clickhouse/pkg/scope" ) type Handler struct { config *config.Config } func NewHandler(config *config.Config) *Handler { return &Handler{ config: config, } } func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { accessLogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("http") logger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("capabilities") r = r.WithContext(scope.WithLogger(r.Context(), logger)) r.ParseMultipartForm(1024 * 1024) format := r.FormValue("format") accepts := r.Header["Accept"] for _, accept := range accepts { if accept == "application/x-carbonapi-v3-pb" { format = "carbonapi_v3_pb" break } } status := http.StatusOK start := time.Now() defer func() { d := time.Since(start) logs.AccessLog(accessLogger, h.config, r, status, d, time.Duration(0), false, false) }() if format == "carbonapi_v3_pb" || format == "json" { body, err := io.ReadAll(r.Body) if err != nil { status = http.StatusBadRequest http.Error(w, "Bad request (malformed body)", status) } var pv3Request v3pb.CapabilityRequest err = pv3Request.Unmarshal(body) if err != nil { status = http.StatusBadRequest http.Error(w, "Bad request (malformed body)", status) } hostname, err := os.Hostname() if err != nil { hostname = "(unknown)" } pvResponse := v3pb.CapabilityResponse{ SupportedProtocols: []string{"carbonapi_v3_pb", "carbonapi_v2_pb", "graphite-web-pickle"}, Name: hostname, HighPrecisionTimestamps: false, SupportFilteringFunctions: false, LikeSplittedRequests: false, SupportStreaming: false, } var data []byte contentType := "" switch format { case "json": contentType = "application/json" data, err = json.Marshal(pvResponse) if err != nil { status = http.StatusInternalServerError http.Error(w, err.Error(), status) return } case "carbonapi_v3_pb": contentType = "application/x-carbonapi-v3-pb" data, err = pvResponse.Marshal() if err != nil { status = http.StatusBadRequest http.Error(w, "Bad request (unsupported format)", status) return } } w.Header().Set("Content-Type", contentType) w.Write(data) } else { status = http.StatusBadRequest http.Error(w, "Bad request (unsupported format)", status) } } ================================================ FILE: cmd/e2e-test/carbon-clickhouse.go ================================================ package main import ( "fmt" "os" "os/exec" "path" "path/filepath" "strings" "text/template" ) var CchContainerName = "carbon-clickhouse-gch-test" type CarbonClickhouse struct { Version string `toml:"version"` DockerImage string `toml:"image"` Template string `toml:"template"` // carbon-clickhouse config template TZ string `toml:"tz"` // override timezone address string `toml:"-"` container string `toml:"-"` storeDir string `toml:"-"` } func (c *CarbonClickhouse) Start(testDir, clickhouseURL string) (string, error) { if len(c.Version) == 0 { c.Version = "latest" } if len(c.DockerImage) == 0 { c.DockerImage = "ghcr.io/go-graphite/carbon-clickhouse" } var err error c.address, err = getFreeTCPPort("") if err != nil { return "", err } c.container = CchContainerName c.storeDir, err = os.MkdirTemp("", "carbon-clickhouse") if err != nil { return "", err } c.address, err = getFreeTCPPort("") if err != nil { c.Cleanup() return "", err } name := filepath.Base(c.Template) tpl := path.Join(testDir, c.Template) tmpl, err := template.New(name).ParseFiles(tpl) if err != nil { c.Cleanup() return "", err } param := struct { CLICKHOUSE_URL string CCH_ADDR string }{ CLICKHOUSE_URL: clickhouseURL, CCH_ADDR: c.address, } configFile := path.Join(c.storeDir, "carbon-clickhouse.conf") f, err := os.OpenFile(configFile, os.O_WRONLY|os.O_CREATE, 0644) if err != nil { c.Cleanup() return "", err } err = tmpl.ExecuteTemplate(f, name, param) if err != nil { c.Cleanup() return "", err } tz := os.Getenv("TZ") cchStart := []string{"run", "-d", "--name", c.container, "-p", c.address + ":2003", "-v", c.storeDir + ":/etc/carbon-clickhouse", // TZ, need to be same as graphite-clickhouse for prevent bugs, ike issue #184 "-v", "/etc/timezone:/etc/timezone:ro", "-v", "/etc/localtime:/etc/localtime:ro", "-e", "TZ=" + tz, "--network", DockerNetwork, } cchStart = append(cchStart, c.DockerImage+":"+c.Version) cmd := exec.Command(DockerBinary, cchStart...) out, err := cmd.CombinedOutput() if err == nil { dateLocal, _ := exec.Command("date").Output() dateLocalStr := strings.TrimRight(string(dateLocal), "\n") _, dateOnCCH := containerExec(c.container, []string{"date"}) fmt.Printf("date local %s, on carbon-clickhouse %s\n", dateLocalStr, dateOnCCH) } return string(out), err } func (c *CarbonClickhouse) Stop(delete bool) (string, error) { if len(c.container) == 0 { return "", nil } chStop := []string{"stop", c.container} cmd := exec.Command(DockerBinary, chStop...) out, err := cmd.CombinedOutput() if err == nil && delete { return c.Delete() } return string(out), err } func (c *CarbonClickhouse) Delete() (string, error) { if len(c.container) == 0 { return "", nil } chDel := []string{"rm", c.container} cmd := exec.Command(DockerBinary, chDel...) out, err := cmd.CombinedOutput() if err == nil { c.container = "" } c.Cleanup() return string(out), err } func (c *CarbonClickhouse) Cleanup() { if c.storeDir != "" { os.RemoveAll(c.storeDir) c.storeDir = "" } } func (c *CarbonClickhouse) Address() string { return c.address } func (c *CarbonClickhouse) Container() string { return c.container } ================================================ FILE: cmd/e2e-test/checks.go ================================================ package main import ( "bufio" "fmt" "net/http" "os" "reflect" "sort" "strconv" "strings" "time" "github.com/go-graphite/protocol/carbonapi_v3_pb" "github.com/lomik/graphite-clickhouse/helper/client" "github.com/lomik/graphite-clickhouse/helper/datetime" "github.com/lomik/graphite-clickhouse/helper/tests/compare" ) func isFindCached(header http.Header) (string, bool) { if header == nil { return "", false } v, exist := header["X-Cached-Find"] if len(v) == 0 { return "", false } return v[0], exist } func requestId(header http.Header) string { if header == nil { return "" } v, exist := header["X-Gch-Request-Id"] if exist && len(v) > 0 { return v[0] } return "" } func compareFindMatch(errors *[]string, name, url string, actual, expected []client.FindMatch, findCached bool, cacheTTL int, header http.Header) { var cacheTTLStr string if findCached { cacheTTLStr = strconv.Itoa(cacheTTL) } id := requestId(header) if header != nil { v, actualFindCached := isFindCached(header) if actualFindCached != findCached || cacheTTLStr != v { *errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s: X-Cached-Find want '%s', got '%s'", name, id, url, cacheTTLStr, v)) } } maxLen := compare.Max(len(expected), len(actual)) for i := 0; i < maxLen; i++ { if i > len(actual)-1 { *errors = append(*errors, fmt.Sprintf("- TRY[%s] %s %s [%d] = %+v", name, id, url, i, expected[i])) } else if i > len(expected)-1 { *errors = append(*errors, fmt.Sprintf("+ TRY[%s] %s %s [%d] = %+v", name, id, url, i, actual[i])) } else if expected[i] != actual[i] { *errors = append(*errors, fmt.Sprintf("- TRY[%s] %s %s [%d] = %+v", name, id, url, i, expected[i])) *errors = append(*errors, fmt.Sprintf("+ TRY[%s] %s %s [%d] = %+v", name, id, url, i, actual[i])) } } } func verifyMetricsFind(ch *Clickhouse, gch *GraphiteClickhouse, check *MetricsFindCheck) []string { var errors []string httpClient := http.Client{ Timeout: check.Timeout, } address := gch.URL() for _, format := range check.Formats { name := "" if url, result, respHeader, err := client.MetricsFind(&httpClient, address, format, check.Query, check.from, check.until); err == nil { id := requestId(respHeader) if check.ErrorRegexp != "" { errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s'", "", id, url, check.ErrorRegexp)) } compareFindMatch(&errors, name, url, result, check.Result, check.InCache, check.CacheTTL, respHeader) if len(result) == 0 && len(check.Result) > 0 { gch.Grep(id) if len(check.DumpIfEmpty) > 0 { for _, q := range check.DumpIfEmpty { if out, err := ch.Query(q); err == nil { fmt.Fprintf(os.Stderr, "%s\n%s", q, out) } else { fmt.Fprintf(os.Stderr, "%s: %s\n", err.Error(), q) } } } } if check.CacheTTL > 0 && check.ErrorRegexp == "" { // second query must be find-cached name = "cache" if url, result, respHeader, err = client.MetricsFind(&httpClient, address, format, check.Query, check.from, check.until); err == nil { compareFindMatch(&errors, name, url, result, check.Result, true, check.CacheTTL, respHeader) } else { errStr := strings.TrimRight(err.Error(), "\n") errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: %s", name, requestId(respHeader), url, errStr)) } } } else { errStr := strings.TrimRight(err.Error(), "\n") if check.errorRegexp == nil || !check.errorRegexp.MatchString(errStr) { errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s', got '%s'", "", requestId(respHeader), url, check.ErrorRegexp, errStr)) } else { fmt.Printf("EXPECTED ERROR, SUCCESS %s : %s\n", url, errStr) } } } return errors } func compareTags(errors *[]string, name, url string, actual, expected []string, findCached bool, cacheTTL int, header http.Header) { var cacheTTLStr string if findCached { cacheTTLStr = strconv.Itoa(cacheTTL) } id := requestId(header) if header != nil { v, actualFindCached := isFindCached(header) if actualFindCached != findCached || cacheTTLStr != v { *errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s: X-Cached-Find want '%s', got '%s'", name, id, url, cacheTTLStr, v)) } } maxLen := compare.Max(len(expected), len(actual)) for i := 0; i < maxLen; i++ { if i > len(actual)-1 { *errors = append(*errors, fmt.Sprintf("- TRY[%s] %s %s [%d] = %+v", name, id, url, i, expected[i])) } else if i > len(expected)-1 { *errors = append(*errors, fmt.Sprintf("+ TRY[%s] %s %s [%d] = %+v", name, id, url, i, actual[i])) } else if expected[i] != actual[i] { *errors = append(*errors, fmt.Sprintf("- TRY[%s] %s %s [%d] = %+v", name, id, url, i, expected[i])) *errors = append(*errors, fmt.Sprintf("+ TRY[%s] %s %s [%d] = %+v", name, id, url, i, actual[i])) } } } func verifyTags(ch *Clickhouse, gch *GraphiteClickhouse, check *TagsCheck) []string { var errors []string httpClient := http.Client{ Timeout: check.Timeout, } address := gch.URL() for _, format := range check.Formats { var ( result []string err error url string respHeader http.Header ) name := "" if check.Names { url, result, respHeader, err = client.TagsNames(&httpClient, address, format, check.Query, check.Limits, check.from, check.until) } else { url, result, respHeader, err = client.TagsValues(&httpClient, address, format, check.Query, check.Limits, check.from, check.until) } if err == nil { id := requestId(respHeader) if check.ErrorRegexp != "" { errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s'", "", id, url, check.ErrorRegexp)) } compareTags(&errors, name, url, result, check.Result, check.InCache, check.CacheTTL, respHeader) if len(result) == 0 && len(check.Result) > 0 { gch.Grep(id) if len(check.DumpIfEmpty) > 0 { for _, q := range check.DumpIfEmpty { if out, err := ch.Query(q); err == nil { fmt.Fprintf(os.Stderr, "%s\n%s", q, out) } else { fmt.Fprintf(os.Stderr, "%s: %s\n", err.Error(), q) } } } } if check.CacheTTL > 0 && check.ErrorRegexp == "" { // second query must be find-cached name = "cache" if check.Names { url, result, respHeader, err = client.TagsNames(&httpClient, address, format, check.Query, check.Limits, check.from, check.until) } else { url, result, respHeader, err = client.TagsValues(&httpClient, address, format, check.Query, check.Limits, check.from, check.until) } if err == nil { compareTags(&errors, name, url, result, check.Result, true, check.CacheTTL, respHeader) } else { errStr := strings.TrimRight(err.Error(), "\n") errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: %s", name, requestId(respHeader), url, errStr)) } } } else { errStr := strings.TrimRight(err.Error(), "\n") if check.errorRegexp == nil || !check.errorRegexp.MatchString(errStr) { errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s', got '%s'", "", requestId(respHeader), url, check.ErrorRegexp, errStr)) } else { fmt.Printf("EXPECTED ERROR, SUCCESS %s : %s\n", url, errStr) } } } return errors } func compareRender(errors *[]string, name, url string, actual, expected []client.Metric, findCached bool, header http.Header, cacheTTL int) { var cacheTTLStr string if findCached { cacheTTLStr = strconv.Itoa(cacheTTL) } sort.Slice(actual, func(i, j int) bool { return actual[i].Name < actual[j].Name }) id := requestId(header) if header != nil { v, actualFindCached := isFindCached(header) if actualFindCached != findCached || cacheTTLStr != v { *errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s: X-Cached-Find want '%s', got '%s'", name, id, url, cacheTTLStr, v)) } } maxLen := compare.Max(len(expected), len(actual)) for i := 0; i < maxLen; i++ { if i > len(actual)-1 { *errors = append(*errors, fmt.Sprintf("- TRY[%s] %s %s [%d] = %+v", name, id, url, i, expected[i])) } else if i > len(expected)-1 { *errors = append(*errors, fmt.Sprintf("+ TRY[%s] %s %s [%d] = %+v", name, id, url, i, actual[i])) } else if actual[i].Name != expected[i].Name { *errors = append(*errors, fmt.Sprintf("- TRY[%s] %s %s [%d] = %+v", name, id, url, i, expected[i])) *errors = append(*errors, fmt.Sprintf("+ TRY[%s] %s %s [%d] = %+v", name, id, url, i, actual[i])) } else { if actual[i].PathExpression != expected[i].PathExpression { *errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].PathExpression, got '%s', want '%s'", name, id, url, actual[i].Name, i, actual[i].PathExpression, expected[i].PathExpression)) } if actual[i].ConsolidationFunc != expected[i].ConsolidationFunc { *errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].ConsolidationFunc, got '%s', want '%s'", name, id, url, actual[i].Name, i, actual[i].ConsolidationFunc, expected[i].ConsolidationFunc)) } if actual[i].ConsolidationFunc != expected[i].ConsolidationFunc { *errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].ConsolidationFunc, got '%s', want '%s'", name, id, url, actual[i].Name, i, actual[i].ConsolidationFunc, expected[i].ConsolidationFunc)) } if actual[i].StartTime != expected[i].StartTime { *errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].StartTime, got %d, want %d", name, id, url, actual[i].Name, i, actual[i].StartTime, expected[i].StartTime)) } if actual[i].StopTime != expected[i].StopTime { *errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].StopTime, got %d, want %d", name, id, url, actual[i].Name, i, actual[i].StopTime, expected[i].StopTime)) } if actual[i].StepTime != expected[i].StepTime { *errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].StepTime, got %d, want %d", name, id, url, actual[i].Name, i, actual[i].StepTime, expected[i].StepTime)) } if actual[i].RequestStartTime != expected[i].RequestStartTime { *errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].RequestStartTime, got %d, want %d", name, id, url, actual[i].Name, i, actual[i].RequestStartTime, expected[i].RequestStartTime)) } if actual[i].RequestStopTime != expected[i].RequestStopTime { *errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].RequestStopTime, got %d, want %d", name, id, url, actual[i].Name, i, actual[i].RequestStopTime, expected[i].RequestStopTime)) } if actual[i].HighPrecisionTimestamps != expected[i].HighPrecisionTimestamps { *errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].HighPrecisionTimestamps, got %v, want %v", name, id, url, actual[i].Name, i, actual[i].HighPrecisionTimestamps, expected[i].HighPrecisionTimestamps)) } if !reflect.DeepEqual(actual[i].AppliedFunctions, expected[i].AppliedFunctions) { *errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].AppliedFunctions, got '%s', want '%s'", name, id, url, actual[i].Name, i, actual[i].AppliedFunctions, expected[i].AppliedFunctions)) } if !compare.NearlyEqual(float64(actual[i].XFilesFactor), float64(expected[i].XFilesFactor)) { *errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].XFilesFactor, got %g, want %g", name, id, url, actual[i].Name, i, actual[i].XFilesFactor, expected[i].XFilesFactor)) } if !compare.NearlyEqualSlice(actual[i].Values, expected[i].Values) { *errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].Values, got %g, want %g", name, id, url, actual[i].Name, i, actual[i].Values, expected[i].Values)) } } } } func parseFilteringFunctions(strFilteringFuncs []string) ([]*carbonapi_v3_pb.FilteringFunction, error) { res := make([]*carbonapi_v3_pb.FilteringFunction, 0, len(strFilteringFuncs)) for _, strFF := range strFilteringFuncs { strFFSplit := strings.Split(strFF, "(") if len(strFFSplit) != 2 { return nil, fmt.Errorf("could not parse filtering function: %s", strFF) } name := strFFSplit[0] args := strings.Split(strFFSplit[1], ",") for i := range args { args[i] = strings.TrimSpace(args[i]) args[i] = strings.Trim(args[i], ")'") } res = append(res, &carbonapi_v3_pb.FilteringFunction{Name: name, Arguments: args}) } return res, nil } func verifyRender(ch *Clickhouse, gch *GraphiteClickhouse, check *RenderCheck, defaultPreision time.Duration) []string { var errors []string httpClient := http.Client{ Timeout: check.Timeout, } address := gch.URL() from := datetime.TimestampTruncate(check.from, defaultPreision) until := datetime.TimestampTruncate(check.until, defaultPreision) for _, format := range check.Formats { var filteringFunctions []*carbonapi_v3_pb.FilteringFunction if format == client.FormatPb_v3 { var err error filteringFunctions, err = parseFilteringFunctions(check.FilteringFunctions) if err != nil { errors = append(errors, err.Error()) continue } } if url, result, respHeader, err := client.Render(&httpClient, address, format, check.Targets, filteringFunctions, check.MaxDataPoints, from, until); err == nil { id := requestId(respHeader) name := "" if check.ErrorRegexp != "" { errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s'", "", id, url, check.ErrorRegexp)) } compareRender(&errors, name, url, result, check.result, check.InCache, respHeader, check.CacheTTL) if len(result) == 0 && len(check.result) > 0 { gch.Grep(id) if len(check.DumpIfEmpty) > 0 { for _, q := range check.DumpIfEmpty { if out, err := ch.Query(q); err == nil { fmt.Fprintf(os.Stderr, "%s\n%s", q, out) } else { fmt.Fprintf(os.Stderr, "%s: %s\n", err.Error(), q) } } } } if check.CacheTTL > 0 && check.ErrorRegexp == "" { // second query must be find-cached name = "cache" if url, result, respHeader, err = client.Render(&httpClient, address, format, check.Targets, filteringFunctions, check.MaxDataPoints, from, until); err == nil { compareRender(&errors, name, url, result, check.result, true, respHeader, check.CacheTTL) } else { errStr := strings.TrimRight(err.Error(), "\n") errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: %s", name, requestId(respHeader), url, errStr)) } } } else { errStr := strings.TrimRight(err.Error(), "\n") if check.errorRegexp == nil || !check.errorRegexp.MatchString(errStr) { errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s', got '%s'", "", requestId(respHeader), url, check.ErrorRegexp, errStr)) } else { fmt.Printf("EXPECTED ERROR, SUCCESS %s : %s\n", url, errStr) } } } return errors } func debug(test *TestSchema, ch *Clickhouse, gch *GraphiteClickhouse) { for { cmd := gch.Cmd() fmt.Println(cmd) fmt.Printf("graphite-clickhouse URL: %s , clickhouse URL: %s , proxy URL: %s (delay %v)\n", gch.URL(), ch.URL(), test.Proxy.URL(), test.Proxy.GetDelay()) fmt.Printf("graphite-clickhouse log: %s , clickhouse container: %s\n", gch.storeDir+"/graphite-clickhouse.log", ch.container) fmt.Println("Some queries was failed, press y for continue after debug test, k for kill graphite-clickhouse:") in := bufio.NewScanner(os.Stdin) in.Scan() s := in.Text() if s == "y" || s == "Y" { break } else if s == "k" || s == "K" { gch.Stop(false) } } } ================================================ FILE: cmd/e2e-test/clickhouse.go ================================================ package main import ( "bytes" "errors" "fmt" "io" "net/http" "os" "os/exec" "strconv" "strings" "github.com/msaf1980/go-stringutils" ) var ( ClickhouseContainerName = "clickhouse-server-gch-test" ClickhouseOldImage = "yandex/clickhouse-server" ClickhouseDefaultImage = "clickhouse/clickhouse-server" ) type Clickhouse struct { Version string `toml:"version"` Dir string `toml:"dir"` TLSEnabled bool `toml:"tls"` DockerImage string `toml:"image"` TZ string `toml:"tz"` // override timezone httpAddress string `toml:"-"` httpsAddress string `toml:"-"` url string `toml:"-"` tlsurl string `toml:"-"` container string `toml:"-"` } func (c *Clickhouse) CheckConfig(rootDir string) error { if c.Version == "" { c.Version = "latest" } if len(c.Dir) == 0 { return ErrNoSetDir } if !strings.HasPrefix(c.Dir, "/") { c.Dir = rootDir + "/" + c.Dir } if c.DockerImage == "" { if c.Version == "latest" { c.DockerImage = ClickhouseDefaultImage } else { splitV := strings.Split(c.Version, ".") majorV, err := strconv.Atoi(splitV[0]) if err != nil { c.DockerImage = ClickhouseDefaultImage } else if majorV >= 21 { c.DockerImage = ClickhouseDefaultImage } else { c.DockerImage = ClickhouseOldImage } } } return nil } func (c *Clickhouse) Key() string { return c.DockerImage + ":" + c.Version + " " + c.Dir + " TZ " + c.TZ } func (c *Clickhouse) Start() (string, error) { var err error c.httpAddress, err = getFreeTCPPort("") if err != nil { return "", err } port := strings.Split(c.httpAddress, ":")[1] c.url = "http://" + c.httpAddress c.container = ClickhouseContainerName // tz, _ := localTZLocationName() chStart := []string{"run", "-d", "--name", c.container, "--ulimit", "nofile=262144:262144", "-p", port + ":8123", // "-e", "TZ=" + tz, // workaround for TZ=":/etc/localtime" "-v", c.Dir + "/config.xml:/etc/clickhouse-server/config.xml", "-v", c.Dir + "/users.xml:/etc/clickhouse-server/users.xml", "-v", c.Dir + "/rollup.xml:/etc/clickhouse-server/config.d/rollup.xml", "-v", c.Dir + "/init.sql:/docker-entrypoint-initdb.d/init.sql", "--network", DockerNetwork, } if c.TLSEnabled { c.httpsAddress, err = getFreeTCPPort("") if err != nil { return "", err } port = strings.Split(c.httpsAddress, ":")[1] c.tlsurl = "https://" + c.httpsAddress chStart = append(chStart, "-v", c.Dir+"/server.crt:/etc/clickhouse-server/server.crt", "-v", c.Dir+"/server.key:/etc/clickhouse-server/server.key", "-v", c.Dir+"/rootCA.crt:/etc/clickhouse-server/rootCA.crt", "-p", port+":8443", ) } if c.TZ != "" { chStart = append(chStart, "-e", "TZ="+c.TZ) } chStart = append(chStart, c.DockerImage+":"+c.Version) cmd := exec.Command(DockerBinary, chStart...) out, err := cmd.CombinedOutput() return string(out), err } func (c *Clickhouse) Stop(delete bool) (string, error) { if len(c.container) == 0 { return "", nil } chStop := []string{"stop", c.container} cmd := exec.Command(DockerBinary, chStop...) out, err := cmd.CombinedOutput() if err == nil && delete { return c.Delete() } return string(out), err } func (c *Clickhouse) Delete() (string, error) { if len(c.container) == 0 { return "", nil } chDel := []string{"rm", c.container} cmd := exec.Command(DockerBinary, chDel...) out, err := cmd.CombinedOutput() if err == nil { c.container = "" } return string(out), err } func (c *Clickhouse) URL() string { return c.url } func (c *Clickhouse) TLSURL() string { return c.tlsurl } func (c *Clickhouse) Container() string { return c.container } func (c *Clickhouse) Exec(sql string) (bool, string) { return containerExec(c.container, []string{"sh", "-c", "clickhouse-client -q '" + sql + "'"}) } func (c *Clickhouse) Query(sql string) (string, error) { reader := strings.NewReader(sql) request, err := http.NewRequest(http.MethodPost, c.URL(), reader) if err != nil { return "", err } resp, err := http.DefaultClient.Do(request) if err != nil { return "", err } defer resp.Body.Close() msg, err := io.ReadAll(resp.Body) if err != nil { return "", err } if resp.StatusCode != http.StatusOK { return "", errors.New(resp.Status + ": " + string(bytes.TrimRight(msg, "\n"))) } return string(msg), nil } func (c *Clickhouse) Alive() bool { if len(c.container) == 0 { return false } req, err := http.DefaultClient.Get(c.URL()) if err != nil { return false } defer req.Body.Close() return req.StatusCode == http.StatusOK } func (c *Clickhouse) CopyLog(destDir string, tail uint64) error { if len(c.container) == 0 { return nil } dest := destDir + "/clickhouse-server.log" chArgs := []string{"cp", c.container + ":/var/log/clickhouse-server/clickhouse-server.log", dest} cmd := exec.Command(DockerBinary, chArgs...) out, err := cmd.CombinedOutput() if err != nil { return errors.New(err.Error() + ": " + string(bytes.TrimRight(out, "\n"))) } if tail > 0 { out, _ := exec.Command("tail", "-"+strconv.FormatUint(tail, 10), dest).Output() fmt.Fprintf(os.Stderr, "CLICKHOUSE-SERVER.LOG %s", stringutils.UnsafeString(out)) } return nil } func (c *Clickhouse) CopyErrLog(destDir string, tail uint64) error { if len(c.container) == 0 { return nil } dest := destDir + "/clickhouse-server.err.log" chArgs := []string{"cp", c.container + ":/var/log/clickhouse-server/clickhouse-server.err.log", dest} cmd := exec.Command(DockerBinary, chArgs...) out, err := cmd.CombinedOutput() if err != nil { return errors.New(err.Error() + ": " + string(bytes.TrimRight(out, "\n"))) } if tail > 0 { out, _ := exec.Command("tail", "-"+strconv.FormatUint(tail, 10), dest).Output() fmt.Fprintf(os.Stderr, "CLICKHOUSE-SERVER.ERR %s", stringutils.UnsafeString(out)) } return nil } ================================================ FILE: cmd/e2e-test/container.go ================================================ package main import ( "os/exec" "strings" ) var ( DockerBinary string DockerNetwork string = "graphite-ch-test" ) func imageDelete(image, version string) (bool, string) { if len(DockerBinary) == 0 { panic("docker not set") } chArgs := []string{"rmi", image + ":" + version} cmd := exec.Command(DockerBinary, chArgs...) out, err := cmd.CombinedOutput() s := strings.Trim(string(out), "\n") if err == nil { return true, s } return false, err.Error() + ": " + s } func containerExist(name string) (bool, string) { if len(DockerBinary) == 0 { panic("docker not set") } chInspect := []string{"inspect", "--format", "'{{.Name}}'", name} cmd := exec.Command(DockerBinary, chInspect...) out, err := cmd.CombinedOutput() s := strings.Trim(string(out), "\n") if err == nil { return true, s } return false, err.Error() + ": " + s } func containerRemove(name string) (bool, string) { if len(DockerBinary) == 0 { panic("docker not set") } chInspect := []string{"rm", "-f", name} cmd := exec.Command(DockerBinary, chInspect...) out, err := cmd.CombinedOutput() s := strings.Trim(string(out), "\n") if err == nil { return true, s } return false, err.Error() + ": " + s } func containerExec(name string, args []string) (bool, string) { if len(DockerBinary) == 0 { panic("docker not set") } dCmd := []string{"exec", name} dCmd = append(dCmd, args...) cmd := exec.Command(DockerBinary, dCmd...) out, err := cmd.CombinedOutput() s := strings.Trim(string(out), "\n") if err == nil { return true, s } return false, err.Error() + ": " + s } ================================================ FILE: cmd/e2e-test/e2etesting.go ================================================ package main import ( "bufio" "fmt" "net" "os" "path" "regexp" "sort" "strconv" "strings" "time" "go.uber.org/zap" "github.com/lomik/graphite-clickhouse/helper/client" "github.com/lomik/graphite-clickhouse/helper/datetime" "github.com/pelletier/go-toml" ) var ( preSQL = []string{ "TRUNCATE TABLE IF EXISTS graphite_reverse", "TRUNCATE TABLE IF EXISTS graphite", "TRUNCATE TABLE IF EXISTS graphite_index", "TRUNCATE TABLE IF EXISTS graphite_tags", } ) type Point struct { Value float64 `toml:"value"` Time string `toml:"time"` Delay time.Duration `toml:"delay"` time int64 `toml:"-"` } type InputMetric struct { Name string `toml:"name"` Points []Point `toml:"points"` Round time.Duration `toml:"round"` } type Metric struct { Name string `toml:"name"` PathExpression string `toml:"path"` ConsolidationFunc string `toml:"consolidation"` StartTime string `toml:"start"` StopTime string `toml:"stop"` StepTime int64 `toml:"step"` XFilesFactor float32 `toml:"xfiles"` HighPrecisionTimestamps bool `toml:"high_precision"` Values []float64 `toml:"values"` AppliedFunctions []string `toml:"applied_functions"` RequestStartTime string `toml:"req_start"` RequestStopTime string `toml:"req_stop"` } type RenderCheck struct { Name string `toml:"name"` Formats []client.FormatType `toml:"formats"` From string `toml:"from"` Until string `toml:"until"` Targets []string `toml:"targets"` MaxDataPoints int64 `toml:"max_data_points"` FilteringFunctions []string `toml:"filtering_functions"` Timeout time.Duration `toml:"timeout"` DumpIfEmpty []string `toml:"dump_if_empty"` Optimize []string `toml:"optimize"` // optimize tables before run tests InCache bool `toml:"in_cache"` // already in cache CacheTTL int `toml:"cache_ttl"` ProxyDelay time.Duration `toml:"proxy_delay"` ProxyBreakWithCode int `toml:"proxy_break_with_code"` Result []Metric `toml:"result"` ErrorRegexp string `toml:"error_regexp"` from int64 `toml:"-"` until int64 `toml:"-"` errorRegexp *regexp.Regexp `toml:"-"` result []client.Metric `toml:"-"` } type MetricsFindCheck struct { Name string `toml:"name"` Formats []client.FormatType `toml:"formats"` From string `toml:"from"` Until string `toml:"until"` Query string `toml:"query"` Timeout time.Duration `toml:"timeout"` DumpIfEmpty []string `toml:"dump_if_empty"` InCache bool `toml:"in_cache"` // already in cache CacheTTL int `toml:"cache_ttl"` ProxyDelay time.Duration `toml:"proxy_delay"` ProxyBreakWithCode int `toml:"proxy_break_with_code"` Result []client.FindMatch `toml:"result"` ErrorRegexp string `toml:"error_regexp"` from int64 `toml:"-"` until int64 `toml:"-"` errorRegexp *regexp.Regexp `toml:"-"` } type TagsCheck struct { Name string `toml:"name"` Names bool `toml:"names"` // TagNames or TagValues Formats []client.FormatType `toml:"formats"` From string `toml:"from"` Until string `toml:"until"` Query string `toml:"query"` Limits uint64 `toml:"limits"` Timeout time.Duration `toml:"timeout"` DumpIfEmpty []string `toml:"dump_if_empty"` InCache bool `toml:"in_cache"` // already in cache CacheTTL int `toml:"cache_ttl"` ProxyDelay time.Duration `toml:"proxy_delay"` ProxyBreakWithCode int `toml:"proxy_break_with_code"` Result []string `toml:"result"` ErrorRegexp string `toml:"error_regexp"` from int64 `toml:"-"` until int64 `toml:"-"` errorRegexp *regexp.Regexp `toml:"-"` } type TestSchema struct { Input []InputMetric `toml:"input"` // carbon-clickhouse input Clickhouse []Clickhouse `toml:"clickhouse"` Proxy HttpReverseProxy `toml:"clickhouse_proxy"` Cch CarbonClickhouse `toml:"carbon_clickhouse"` Gch []GraphiteClickhouse `toml:"graphite_clickhouse"` FindChecks []*MetricsFindCheck `toml:"find_checks"` TagsChecks []*TagsCheck `toml:"tags_checks"` RenderChecks []*RenderCheck `toml:"render_checks"` Precision time.Duration `toml:"precision"` dir string `toml:"-"` name string `toml:"-"` // test alias (from config name) chVersions map[string]bool `toml:"-"` // input map[string][]Point `toml:"-"` } func (schema *TestSchema) HasTLSSettings() bool { return strings.Contains(schema.dir, "tls") } func getFreeTCPPort(name string) (string, error) { if len(name) == 0 { name = "127.0.0.1:0" } else if !strings.Contains(name, ":") { name = name + ":0" } addr, err := net.ResolveTCPAddr("tcp", name) if err != nil { return name, err } l, err := net.ListenTCP("tcp", addr) if err != nil { return name, err } defer l.Close() return l.Addr().String(), nil } func sendPlain(network, address string, metrics []InputMetric) error { if conn, err := net.DialTimeout(network, address, time.Second); err != nil { return err } else { bw := bufio.NewWriter(conn) for _, m := range metrics { conn.SetDeadline(time.Now().Add(time.Second)) for _, point := range m.Points { if _, err = fmt.Fprintf(bw, "%s %f %d\n", m.Name, point.Value, point.time); err != nil { conn.Close() return err } if point.Delay > 0 { if err = bw.Flush(); err != nil { conn.Close() return err } time.Sleep(point.Delay) } } } if err = bw.Flush(); err != nil { conn.Close() return err } return conn.Close() } } func verifyGraphiteClickhouse(test *TestSchema, gch *GraphiteClickhouse, clickhouse *Clickhouse, testDir, clickhouseDir string, verbose, breakOnError bool, logger *zap.Logger) (testSuccess bool, verifyCount, verifyFailed int) { testSuccess = true err := gch.Start(testDir, clickhouse.URL(), test.Proxy.URL(), clickhouse.TLSURL()) if err != nil { logger.Error("starting graphite-clickhouse", zap.String("config", test.name), zap.String("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouseDir), zap.String("graphite-clickhouse config", gch.ConfigTpl), zap.Error(err), ) testSuccess = false return } for i := 100; i < 1000; i += 200 { time.Sleep(time.Duration(i) * time.Millisecond) if gch.Alive() { break } } // start tests for n, check := range test.FindChecks { verifyCount++ test.Proxy.SetDelay(check.ProxyDelay) test.Proxy.SetBreakStatusCode(check.ProxyBreakWithCode) if len(check.Formats) == 0 { check.Formats = []client.FormatType{client.FormatPb_v3} } if errs := verifyMetricsFind(clickhouse, gch, check); len(errs) > 0 { verifyFailed++ for _, e := range errs { fmt.Fprintln(os.Stderr, e) } logger.Error("verify metrics find", zap.String("config", test.name), zap.String("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouseDir), zap.String("graphite-clickhouse config", gch.ConfigTpl), zap.String("query", check.Query), zap.String("from_raw", check.From), zap.String("until_raw", check.Until), zap.Int64("from", check.from), zap.Int64("until", check.until), zap.String("name", check.Name+"["+strconv.Itoa(n)+"]"), ) if breakOnError { debug(test, clickhouse, gch) } } else if verbose { logger.Info("verify metrics find", zap.String("config", test.name), zap.String("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouseDir), zap.String("graphite-clickhouse config", gch.ConfigTpl), zap.String("query", check.Query), zap.String("from_raw", check.From), zap.String("until_raw", check.Until), zap.Int64("from", check.from), zap.Int64("until", check.until), zap.String("name", check.Name+"["+strconv.Itoa(n)+"]"), ) } } for n, check := range test.TagsChecks { verifyCount++ test.Proxy.SetDelay(check.ProxyDelay) test.Proxy.SetBreakStatusCode(check.ProxyBreakWithCode) if len(check.Formats) == 0 { check.Formats = []client.FormatType{client.FormatJSON} } if errs := verifyTags(clickhouse, gch, check); len(errs) > 0 { verifyFailed++ for _, e := range errs { fmt.Fprintln(os.Stderr, e) } logger.Error("verify tags", zap.String("config", test.name), zap.String("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouseDir), zap.String("graphite-clickhouse config", gch.ConfigTpl), zap.Bool("name", check.Names), zap.String("query", check.Query), zap.String("from_raw", check.From), zap.String("until_raw", check.Until), zap.Int64("from", check.from), zap.Int64("until", check.until), zap.String("name", check.Name+"["+strconv.Itoa(n)+"]"), ) if breakOnError { debug(test, clickhouse, gch) } } else if verbose { logger.Info("verify tags", zap.String("config", test.name), zap.String("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouseDir), zap.String("graphite-clickhouse config", gch.ConfigTpl), zap.Bool("name", check.Names), zap.String("query", check.Query), zap.String("from_raw", check.From), zap.String("until_raw", check.Until), zap.Int64("from", check.from), zap.Int64("until", check.until), zap.String("name", check.Name+"["+strconv.Itoa(n)+"]"), ) } } for n, check := range test.RenderChecks { verifyCount++ test.Proxy.SetDelay(check.ProxyDelay) test.Proxy.SetBreakStatusCode(check.ProxyBreakWithCode) if len(check.Formats) == 0 { check.Formats = []client.FormatType{client.FormatPb_v3} } if len(check.Optimize) > 0 { for _, table := range check.Optimize { if success, out := clickhouse.Exec("OPTIMIZE TABLE " + table + " FINAL"); !success { logger.Error("optimize table", zap.String("config", test.name), zap.String("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouseDir), zap.String("graphite-clickhouse config", gch.ConfigTpl), zap.Strings("targets", check.Targets), zap.Strings("filtering_functions", check.FilteringFunctions), zap.String("from_raw", check.From), zap.String("until_raw", check.Until), zap.Int64("from", check.from), zap.Int64("until", check.until), zap.String("name", check.Name+"["+strconv.Itoa(n)+"]"), zap.String("table", table), zap.String("out", out), ) time.Sleep(5 * time.Second) } } } if errs := verifyRender(clickhouse, gch, check, test.Precision); len(errs) > 0 { verifyFailed++ for _, e := range errs { fmt.Fprintln(os.Stderr, e) } logger.Error("verify render", zap.String("config", test.name), zap.String("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouseDir), zap.String("graphite-clickhouse config", gch.ConfigTpl), zap.Strings("targets", check.Targets), zap.Strings("filtering_functions", check.FilteringFunctions), zap.String("from_raw", check.From), zap.String("until_raw", check.Until), zap.Int64("from", check.from), zap.Int64("until", check.until), zap.String("name", check.Name+"["+strconv.Itoa(n)+"]"), ) if breakOnError { debug(test, clickhouse, gch) } } else if verbose { logger.Info("verify render", zap.String("config", test.name), zap.String("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouseDir), zap.String("graphite-clickhouse config", gch.ConfigTpl), zap.Strings("targets", check.Targets), zap.Strings("filtering_functions", check.FilteringFunctions), zap.String("from_raw", check.From), zap.String("until_raw", check.Until), zap.Int64("from", check.from), zap.Int64("until", check.until), zap.String("name", check.Name+"["+strconv.Itoa(n)+"]"), ) } } if verifyFailed > 0 { testSuccess = false logger.Error("verify", zap.String("config", test.name), zap.String("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouseDir), zap.String("graphite-clickhouse config", gch.ConfigTpl), zap.Int64("count", int64(verifyCount)), zap.Int64("failed", int64(verifyFailed)), ) } err = gch.Stop(true) if err != nil { logger.Error("stoping graphite-clickhouse", zap.String("config", test.name), zap.String("gch", gch.ConfigTpl), zap.String("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouseDir), zap.Error(err), ) testSuccess = false } return } func testGraphiteClickhouse(test *TestSchema, clickhouse *Clickhouse, testDir, rootDir string, verbose, breakOnError bool, logger *zap.Logger) (testSuccess bool, verifyCount, verifyFailed int) { testSuccess = true for _, sql := range preSQL { if success, out := clickhouse.Exec(sql); !success { logger.Error("pre-execute", zap.String("config", test.name), zap.Any("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouse.Dir), zap.String("sql", sql), zap.String("out", out), ) return } } if err := test.Proxy.Start(clickhouse.URL()); err != nil { logger.Error("starting clickhouse proxy", zap.String("config", test.name), zap.Any("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouse.Dir), zap.Error(err), ) return } out, err := test.Cch.Start(testDir, "http://"+clickhouse.Container()+":8123") if err != nil { logger.Error("starting carbon-clickhouse", zap.String("config", test.name), zap.String("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouse.Dir), zap.Error(err), zap.String("out", out), ) testSuccess = false } if testSuccess { logger.Info("starting e2e test", zap.String("config", test.name), zap.String("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouse.Dir), ) time.Sleep(200 * time.Millisecond) // Populate test data err = sendPlain("tcp", test.Cch.address, test.Input) if err != nil { logger.Error("send plain to carbon-clickhouse", zap.String("config", test.name), zap.String("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouse.Dir), zap.Error(err), ) testSuccess = false } if testSuccess { time.Sleep(2 * time.Second) } if testSuccess { for _, gch := range test.Gch { stepSuccess, vCount, vFailed := verifyGraphiteClickhouse(test, &gch, clickhouse, testDir, clickhouse.Dir, verbose, breakOnError, logger) verifyCount += vCount verifyFailed += vFailed if !stepSuccess { testSuccess = false } } } } out, err = test.Cch.Stop(true) if err != nil { logger.Error("stoping carbon-clickhouse", zap.String("config", test.name), zap.String("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouse.Dir), zap.Error(err), zap.String("out", out), ) testSuccess = false } test.Proxy.Stop() if testSuccess { logger.Info("end e2e test", zap.String("config", test.name), zap.String("status", "success"), zap.String("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouse.Dir), ) } else { logger.Error("end e2e test", zap.String("config", test.name), zap.String("status", "failed"), zap.String("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouse.Dir), ) } return } func runTest(cfg *MainConfig, clickhouse *Clickhouse, rootDir string, now time.Time, verbose, breakOnError bool, logger *zap.Logger) (failed, total, verifyCount, verifyFailed int) { var isRunning bool total++ if exist, out := containerExist(CchContainerName); exist { logger.Error("carbon-clickhouse already exist", zap.String("container", CchContainerName), zap.String("out", out), ) isRunning = true } if isRunning { failed++ return } success, vCount, vFailed := testGraphiteClickhouse(cfg.Test, clickhouse, cfg.Test.dir, rootDir, verbose, breakOnError, logger) if !success { failed++ } verifyCount += vCount verifyFailed += vFailed return } func clickhouseStart(clickhouse *Clickhouse, logger *zap.Logger) bool { out, err := clickhouse.Start() if err != nil { logger.Error("starting clickhouse", zap.Any("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouse.Dir), zap.Error(err), zap.String("out", out), ) clickhouse.Stop(true) return false } return true } func clickhouseStop(clickhouse *Clickhouse, logger *zap.Logger) (result bool) { result = true if !clickhouse.Alive() { clickhouse.CopyLog(os.TempDir(), 10) result = false } out, err := clickhouse.Stop(true) if err != nil { logger.Error("stoping clickhouse", zap.String("clickhouse version", clickhouse.Version), zap.String("clickhouse config", clickhouse.Dir), zap.Error(err), zap.String("out", out), ) result = false } return result } func initTest(cfg *MainConfig, rootDir string, now time.Time, verbose, breakOnError bool, logger *zap.Logger) bool { tz, err := datetime.Timezone("") if err != nil { fmt.Printf("can't get timezone: %s\n", err.Error()) os.Exit(1) } // prepare for n, m := range cfg.Test.Input { for i := range m.Points { m.Points[i].time = datetime.DateParamToEpoch(m.Points[i].Time, tz, now, cfg.Test.Precision) if m.Points[i].time == 0 { err = ErrTimestampInvalid } if err != nil { logger.Error("failed to read config", zap.String("config", cfg.Test.name), zap.Error(err), zap.String("input", m.Name), zap.Int("metric", n), zap.Int("point", i), zap.String("time", m.Points[i].Time), ) return false } } } for n, find := range cfg.Test.FindChecks { if find.Timeout == 0 { find.Timeout = 10 * time.Second } find.from = datetime.DateParamToEpoch(find.From, tz, now, cfg.Test.Precision) if find.from == 0 && find.From != "" { err = ErrTimestampInvalid } if err != nil { logger.Error("failed to read config", zap.String("config", cfg.Test.name), zap.Error(err), zap.String("query", find.Query), zap.String("from", find.From), zap.Int("step", n), ) return false } find.until = datetime.DateParamToEpoch(find.Until, tz, now, cfg.Test.Precision) if find.until == 0 && find.Until != "" { err = ErrTimestampInvalid } if err != nil { logger.Error("failed to read config", zap.String("config", cfg.Test.name), zap.Error(err), zap.String("query", find.Query), zap.String("until", find.Until), zap.Int("step", n), ) return false } if find.ErrorRegexp != "" { find.errorRegexp = regexp.MustCompile(find.ErrorRegexp) } } for n, tags := range cfg.Test.TagsChecks { if tags.Timeout == 0 { tags.Timeout = 10 * time.Second } tags.from = datetime.DateParamToEpoch(tags.From, tz, now, cfg.Test.Precision) if tags.from == 0 && tags.From != "" { err = ErrTimestampInvalid } if err != nil { logger.Error("failed to read config", zap.String("config", cfg.Test.name), zap.Error(err), zap.String("query", tags.Query), zap.String("from", tags.From), zap.Int("find", n), ) return false } tags.until = datetime.DateParamToEpoch(tags.Until, tz, now, cfg.Test.Precision) if tags.until == 0 && tags.Until != "" { err = ErrTimestampInvalid } if err != nil { logger.Error("failed to read config", zap.String("config", cfg.Test.name), zap.Error(err), zap.String("query", tags.Query), zap.String("until", tags.Until), zap.Int("tags", n), zap.Bool("names", tags.Names), ) return false } if tags.ErrorRegexp != "" { tags.errorRegexp = regexp.MustCompile(tags.ErrorRegexp) } } for n, r := range cfg.Test.RenderChecks { if r.Timeout == 0 { r.Timeout = 10 * time.Second } r.from = datetime.DateParamToEpoch(r.From, tz, now, cfg.Test.Precision) if r.from == 0 && r.From != "" { err = ErrTimestampInvalid } if err != nil { logger.Error("failed to read config", zap.String("config", cfg.Test.name), zap.Error(err), zap.Strings("targets", r.Targets), zap.String("from", r.From), zap.Int("render", n), ) return false } r.until = datetime.DateParamToEpoch(r.Until, tz, now, cfg.Test.Precision) if r.until == 0 && r.Until != "" { err = ErrTimestampInvalid } if err != nil { logger.Error("failed to read config", zap.String("config", cfg.Test.name), zap.Error(err), zap.Strings("targets", r.Targets), zap.String("until", r.Until), zap.Int("render", n), ) return false } if r.ErrorRegexp != "" { r.errorRegexp = regexp.MustCompile(r.ErrorRegexp) } sort.Slice(r.Result, func(i, j int) bool { return r.Result[i].Name < r.Result[j].Name }) r.result = make([]client.Metric, len(r.Result)) for i, result := range r.Result { r.result[i].StartTime = datetime.DateParamToEpoch(result.StartTime, tz, now, cfg.Test.Precision) if r.result[i].StartTime == 0 && result.StartTime != "" { err = ErrTimestampInvalid } if err != nil { logger.Error("failed to read config", zap.String("config", cfg.Test.name), zap.Error(err), zap.Strings("targets", r.Targets), zap.Int("render", n), zap.String("metric", result.Name), zap.String("start", result.StartTime), ) return false } r.result[i].StopTime = datetime.DateParamToEpoch(result.StopTime, tz, now, cfg.Test.Precision) if r.result[i].StopTime == 0 && result.StopTime != "" { err = ErrTimestampInvalid } if err != nil { logger.Error("failed to read config", zap.String("config", cfg.Test.name), zap.Error(err), zap.Strings("targets", r.Targets), zap.Int("render", n), zap.String("metric", result.Name), zap.String("stop", result.StopTime), ) return false } r.result[i].RequestStartTime = datetime.DateParamToEpoch(result.RequestStartTime, tz, now, cfg.Test.Precision) if r.result[i].RequestStartTime == 0 && result.RequestStartTime != "" { err = ErrTimestampInvalid } if err != nil { logger.Error("failed to read config", zap.String("config", cfg.Test.name), zap.Error(err), zap.Strings("targets", r.Targets), zap.Int("render", n), zap.String("metric", result.Name), zap.String("req_start", result.RequestStartTime), ) return false } r.result[i].RequestStopTime = datetime.DateParamToEpoch(result.RequestStopTime, tz, now, cfg.Test.Precision) if r.result[i].RequestStopTime == 0 && result.RequestStopTime != "" { err = ErrTimestampInvalid } if err != nil { logger.Error("failed to read config", zap.String("config", cfg.Test.name), zap.Error(err), zap.Strings("targets", r.Targets), zap.Int("render", n), zap.String("metric", result.Name), zap.String("req_stop", result.RequestStopTime), ) return false } r.result[i].StepTime = result.StepTime r.result[i].Name = result.Name r.result[i].PathExpression = result.PathExpression r.result[i].ConsolidationFunc = result.ConsolidationFunc r.result[i].XFilesFactor = result.XFilesFactor r.result[i].HighPrecisionTimestamps = result.HighPrecisionTimestamps r.result[i].AppliedFunctions = result.AppliedFunctions r.result[i].Values = result.Values } } return true } func loadConfig(config string, rootDir string) (*MainConfig, error) { d, err := os.ReadFile(config) if err != nil { return nil, err } confShort := strings.ReplaceAll(config, rootDir+"/", "") var cfg = &MainConfig{} if err := toml.Unmarshal(d, cfg); err != nil { return nil, err } cfg.Test.name = confShort cfg.Test.dir = path.Dir(config) if cfg.Test == nil { return nil, ErrNoTest } cfg.Test.chVersions = make(map[string]bool) for i := range cfg.Test.Clickhouse { if err := cfg.Test.Clickhouse[i].CheckConfig(rootDir); err == nil { cfg.Test.chVersions[cfg.Test.Clickhouse[i].Key()] = true } else { return nil, fmt.Errorf("[%d] %s", i, err.Error()) } } return cfg, nil } ================================================ FILE: cmd/e2e-test/errors.go ================================================ package main import "errors" var ( ErrTimestampInvalid = errors.New("invalid timestamp") ErrNoTest = errors.New("no test section") ErrNoSetDir = errors.New("dir not set") ) ================================================ FILE: cmd/e2e-test/graphite-clickhouse.go ================================================ package main import ( "errors" "fmt" "net/http" "os" "os/exec" "path" "path/filepath" "strings" "syscall" "text/template" "github.com/msaf1980/go-stringutils" "github.com/lomik/graphite-clickhouse/helper/client" ) type GraphiteClickhouse struct { Binary string `toml:"binary"` ConfigTpl string `toml:"template"` TestDir string `toml:"-"` TZ string `toml:"tz"` // override timezone storeDir string `toml:"-"` configFile string `toml:"-"` address string `toml:"-"` cmd *exec.Cmd `toml:"-"` } func (c *GraphiteClickhouse) Start(testDir, chURL, chProxyURL, chTLSURL string) error { if c.cmd != nil { return errors.New("carbon-clickhouse already started") } if len(c.Binary) == 0 { c.Binary = "./graphite-clickhouse" } if len(c.ConfigTpl) == 0 { return errors.New("graphite-clickhouse config template not set") } var err error c.storeDir, err = os.MkdirTemp("", "graphite-clickhouse") if err != nil { return err } c.address, err = getFreeTCPPort("") if err != nil { c.Cleanup() return err } c.TestDir, err = filepath.Abs(testDir) if err != nil { return err } name := filepath.Base(c.ConfigTpl) tmpl, err := template.New(name).ParseFiles(path.Join(testDir, c.ConfigTpl)) if err != nil { c.Cleanup() return err } param := struct { CLICKHOUSE_URL string CLICKHOUSE_TLS_URL string PROXY_URL string GCH_ADDR string GCH_DIR string TEST_DIR string }{ CLICKHOUSE_URL: chURL, CLICKHOUSE_TLS_URL: chTLSURL, PROXY_URL: chProxyURL, GCH_ADDR: c.address, GCH_DIR: c.storeDir, TEST_DIR: c.TestDir, } c.configFile = path.Join(c.storeDir, "graphite-clickhouse.conf") f, err := os.OpenFile(c.configFile, os.O_WRONLY|os.O_CREATE, 0644) if err != nil { c.Cleanup() return err } err = tmpl.ExecuteTemplate(f, name, param) if err != nil { c.Cleanup() return err } c.cmd = exec.Command(c.Binary, "-config", c.configFile) c.cmd.Stdout = os.Stdout c.cmd.Stderr = os.Stderr if c.TZ != "" { c.cmd.Env = append(c.cmd.Env, "TZ="+c.TZ) } err = c.cmd.Start() if err != nil { c.Cleanup() return err } return nil } func (c *GraphiteClickhouse) Alive() bool { if c.cmd == nil { return false } _, _, _, err := client.MetricsFind(http.DefaultClient, "http://"+c.address+"/alive", client.FormatDefault, "NonExistentTarget", 0, 0) return err == nil } func (c *GraphiteClickhouse) Stop(cleanup bool) error { if cleanup { defer c.Cleanup() } if c.cmd == nil { return nil } var err error if err = c.cmd.Process.Kill(); err == nil { if err = c.cmd.Wait(); err != nil { if exitErr, ok := err.(*exec.ExitError); ok { if status, ok := exitErr.Sys().(syscall.WaitStatus); ok { ec := status.ExitStatus() if ec == 0 || ec == -1 { return nil } } } } } return err } func (c *GraphiteClickhouse) Cleanup() { if len(c.storeDir) > 0 { os.RemoveAll(c.storeDir) c.storeDir = "" c.cmd = nil } } func (c *GraphiteClickhouse) URL() string { return "http://" + c.address } func (c *GraphiteClickhouse) Cmd() string { return strings.Join(c.cmd.Args, " ") } func (c *GraphiteClickhouse) Grep(s string) { out, _ := exec.Command("grep", "-F", s, c.storeDir+"/graphite-clickhouse.log").Output() fmt.Fprintf(os.Stderr, "GREP %s", stringutils.UnsafeString(out)) } ================================================ FILE: cmd/e2e-test/main.go ================================================ package main import ( "flag" "log" "os" "path" "runtime" "time" "go.uber.org/zap" ) type MainConfig struct { Test *TestSchema `toml:"test"` } func IsDir(filename string) (bool, error) { info, err := os.Stat(filename) if os.IsNotExist(err) { return false, nil } else if err != nil { return false, err } return info.IsDir(), nil } func expandDir(dirname string, paths *[]string) error { files, err := os.ReadDir(dirname) if err != nil { return err } for _, file := range files { if file.IsDir() { if err = expandDir(path.Join(dirname, file.Name()), paths); err != nil { return err } } else { ext := path.Ext(file.Name()) if ext == ".toml" { *paths = append(*paths, path.Join(dirname, file.Name())) } } } return nil } func expandFilename(filename string, paths *[]string) error { if len(filename) == 0 { return nil } isDir, err := IsDir(filename) if err == nil { if isDir { if err = expandDir(filename, paths); err != nil { return err } } else { *paths = append(*paths, filename) } } return err } func main() { _, filename, _, _ := runtime.Caller(0) rootDir := path.Dir(path.Dir(path.Dir(filename))) // carbon-clickhouse repositiry root dir config := flag.String("config", "", "toml configuration file or dir where toml files is searched (recursieve)") verbose := flag.Bool("verbose", false, "verbose") breakOnError := flag.Bool("break", false, "break and wait user response if request failed") abortOnError := flag.Bool("abort", false, "abort tests if test failed") cleanup := flag.Bool("cleanup", false, "delete containers if exists before start") rmi := flag.Bool("rmi", false, "delete images after test end (for low space usage))") flag.Parse() logger, err := zap.NewProduction() if err != nil { log.Fatal(err) } DockerBinary = os.Getenv("DOCKER_E2E") if DockerBinary == "" { DockerBinary = "docker" } if *cleanup { if exist, _ := containerExist(CchContainerName); exist { if ok, out := containerRemove(CchContainerName); !ok { logger.Fatal("failed to cleanup", zap.String("container", CchContainerName), zap.String("error", out), ) } } if exist, _ := containerExist(ClickhouseContainerName); exist { if ok, out := containerRemove(ClickhouseContainerName); !ok { logger.Fatal("failed to cleanup", zap.String("container", ClickhouseContainerName), zap.String("error", out), ) } } if *config == "" { return } } var allConfigs []string err = expandFilename(*config, &allConfigs) if err != nil { logger.Fatal( "config", zap.Error(err), ) } if len(allConfigs) == 0 { logger.Fatal("config should be non-null") } chVersions := make(map[string]Clickhouse) configs := make([]*MainConfig, 0, len(allConfigs)) for _, config := range allConfigs { cfg, err := loadConfig(config, rootDir) if err == nil { configs = append(configs, cfg) for _, ch := range cfg.Test.Clickhouse { chVersions[ch.Key()] = ch } now := time.Now() if !initTest(cfg, rootDir, now, *verbose, *breakOnError, logger) { os.Exit(1) } } else { logger.Error("failed to read config", zap.String("config", config), zap.Error(err), zap.Any("decode", cfg), ) } } failed := 0 total := 0 verifyCount := 0 verifyFailed := 0 _, err = cmdExec(DockerBinary, "network", "inspect", DockerNetwork) if err != nil { out, err := cmdExec(DockerBinary, "network", "create", DockerNetwork) if err != nil { logger.Error("failed to create network", zap.Error(err), zap.String("out", out), ) os.Exit(1) } } for chVersion := range chVersions { ch := chVersions[chVersion] if exist, out := containerExist(ClickhouseContainerName); exist { logger.Error("clickhouse already exist", zap.String("container", ClickhouseContainerName), zap.String("out", out), ) os.Exit(1) } logger.Info("clickhouse", zap.Any("clickhouse image", ch.DockerImage), zap.Any("clickhouse version", ch.Version), zap.String("clickhouse config", ch.Dir), zap.String("tz", ch.TZ), ) if clickhouseStart(&ch, logger) { time.Sleep(100 * time.Millisecond) for i := 200; i < 3000; i += 200 { if ch.Alive() { break } time.Sleep(time.Duration(i) * time.Millisecond) } if !ch.Alive() { logger.Error("starting clickhouse", zap.Any("clickhouse version", ch.Version), zap.String("clickhouse config", ch.Dir), zap.String("error", "clickhouse is down"), ) failed++ total++ verifyCount++ verifyFailed++ } else { for _, config := range configs { if config.Test.chVersions[chVersion] { now := time.Now() if initTest(config, rootDir, now, *verbose, *breakOnError, logger) { testFailed, testTotal, vCount, vFailed := runTest(config, &ch, rootDir, now, *verbose, *breakOnError, logger) failed += testFailed total += testTotal verifyCount += vCount verifyFailed += vFailed } else { failed++ total++ verifyCount++ verifyFailed++ } } } } if !clickhouseStop(&ch, logger) { failed++ verifyFailed++ } } else { failed++ total++ verifyCount++ verifyFailed++ } if *rmi { if success, out := imageDelete(ch.DockerImage, ch.Version); !success { logger.Error("docker remove image", zap.Any("clickhouse version", ch.Version), zap.String("clickhouse config", ch.Dir), zap.String("out", out), ) } } if *abortOnError && failed > 0 { break } } if failed > 0 { logger.Error("tests ended", zap.String("status", "failed"), zap.Int("test_count", total), zap.Int("test_failed", failed), zap.Int("checks", verifyCount), zap.Int("failed", verifyFailed), zap.Int("configs", len(allConfigs)), ) os.Exit(1) } else { logger.Info("tests ended", zap.String("status", "success"), zap.Int("test_count", total), zap.Int("test_failed", failed), zap.Int("checks", verifyCount), zap.Int("failed", verifyFailed), zap.Int("configs", len(allConfigs)), ) } } ================================================ FILE: cmd/e2e-test/rproxy.go ================================================ package main import ( "errors" "net/http" "net/http/httptest" "net/http/httputil" "net/url" "sync" "sync/atomic" "time" "github.com/lomik/graphite-clickhouse/pkg/dry" ) type AtomicDuration struct { val int64 } func (d *AtomicDuration) Store(duration time.Duration) { atomic.StoreInt64(&d.val, duration.Nanoseconds()) } func (d *AtomicDuration) Load() time.Duration { return time.Duration(atomic.LoadInt64(&d.val)) } func (d *AtomicDuration) MarshalText() ([]byte, error) { s := d.Load().String() return dry.UnsafeStringBytes(&s), nil } func (d *AtomicDuration) UnmarshalText(b []byte) error { val, err := time.ParseDuration(dry.UnsafeString(b)) if err != nil { return err } d.Store(val) return nil } type HttpReverseProxy struct { Delay AtomicDuration `toml:"delay"` BreakWithStatusCode int64 `toml:"break_with_status_code"` srv *httptest.Server remote *url.URL wg sync.WaitGroup } func (p *HttpReverseProxy) Start(remoteURL string) (err error) { if p.srv != nil { err = errors.New("reverse proxy already started") return } if p.BreakWithStatusCode < 0 { p.BreakWithStatusCode = 0 } if p.remote, err = url.Parse(remoteURL); err != nil { err = errors.New("reverse proxy already started") return } p.srv = httptest.NewUnstartedServer(p) p.wg.Add(1) go func() { defer p.wg.Done() p.srv.Start() }() return } func (p *HttpReverseProxy) Stop() { if p.srv == nil { return } p.srv.CloseClientConnections() p.srv.Close() p.wg.Wait() p.srv = nil } func (p *HttpReverseProxy) URL() string { return p.srv.URL } func (p *HttpReverseProxy) SetDelay(delay time.Duration) { p.Delay.Store(delay) } func (p *HttpReverseProxy) GetDelay() time.Duration { return p.Delay.Load() } func (p *HttpReverseProxy) SetBreakStatusCode(statusCode int) { atomic.StoreInt64(&p.BreakWithStatusCode, int64(statusCode)) } func (p *HttpReverseProxy) GetBreakStatusCode() int { return int(atomic.LoadInt64(&p.BreakWithStatusCode)) } func (p *HttpReverseProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { r.Host = p.remote.Host delay := p.GetDelay() if delay != 0 { time.Sleep(delay) } breakWithStatusCode := p.GetBreakStatusCode() if breakWithStatusCode != 0 { http.Error(w, "", breakWithStatusCode) return } proxy := httputil.NewSingleHostReverseProxy(p.remote) proxy.ServeHTTP(w, r) } ================================================ FILE: cmd/e2e-test/utils.go ================================================ package main import "os/exec" func cmdExec(programm string, args ...string) (string, error) { cmd := exec.Command(programm, args...) out, err := cmd.CombinedOutput() return string(out), err } ================================================ FILE: cmd/graphite-clickhouse-client/main.go ================================================ package main import ( "flag" "fmt" "net/http" "os" "strconv" "strings" "time" "github.com/go-graphite/protocol/carbonapi_v3_pb" "github.com/lomik/graphite-clickhouse/helper/client" "github.com/lomik/graphite-clickhouse/helper/datetime" ) type StringSlice []string func (u *StringSlice) Set(value string) error { *u = append(*u, value) return nil } func (u *StringSlice) String() string { return "[ " + strings.Join(*u, ", ") + " ]" } func (u *StringSlice) Type() string { return "[]string" } func main() { address := flag.String("address", "http://127.0.0.1:9090", "Address of graphite-clickhouse server") fromStr := flag.String("from", "0", "from") untilStr := flag.String("until", "", "until") maxDataPointsStr := flag.String("maxDataPoints", "1048576", "Maximum amount of datapoints in response") metricsFind := flag.String("find", "", "Query for /metrics/find/ , valid formats are carbonapi_v3_pb. protobuf, pickle") tagsValues := flag.String("tags_values", "", "Query for /tags/autoComplete/values (with query like 'searchTag[=valuePrefix];tag1=value1;tag2=~value*' or '<>' for empty)") tagsNames := flag.String("tags_names", "", "Query for /tags/autoComplete/tags (with query like '[tagPrefix];tag1=value1;tag2=~value*[' or '<>' for empty)") limit := flag.Uint64("limit", 0, "limit for some queries (tags_values, tags_values)") timeout := flag.Duration("timeout", time.Minute, "request timeout") var targets StringSlice flag.Var(&targets, "target", "Target for /render") format := client.FormatDefault flag.Var(&format, "format", fmt.Sprintf("Response format %v", client.FormatTypes())) flag.Parse() ec := 0 tz, err := datetime.Timezone("") if err != nil { fmt.Printf("can't get timezone: %s\n", err.Error()) os.Exit(1) } now := time.Now() from := datetime.DateParamToEpoch(*fromStr, tz, now, 0) if from == 0 && len(targets) > 0 { fmt.Printf("invalid from: %s\n", *fromStr) os.Exit(1) } var until int64 if *untilStr == "" && len(targets) > 0 { *untilStr = "now" } until = datetime.DateParamToEpoch(*untilStr, tz, now, 0) if until == 0 && len(targets) > 0 { fmt.Printf("invalid until: %s\n", *untilStr) os.Exit(1) } maxDataPoints, err := strconv.ParseInt(*maxDataPointsStr, 10, 64) if err != nil { fmt.Printf("invalid maxDataPoints: %s\n", *maxDataPointsStr) os.Exit(1) } httpClient := http.Client{ Timeout: *timeout, } if *metricsFind != "" { formatFind := format if formatFind == client.FormatDefault { formatFind = client.FormatPb_v3 } queryRaw, r, respHeader, err := client.MetricsFind(&httpClient, *address, formatFind, *metricsFind, from, until) if respHeader != nil { fmt.Printf("Responce header: %+v\n", respHeader) } fmt.Print("'") fmt.Print(queryRaw) fmt.Print("' = ") if err == nil { if len(r) > 0 { fmt.Println("[") for i, m := range r { fmt.Printf(" { Path: '%s', IsLeaf: %v }", m.Path, m.IsLeaf) if i < len(r)-1 { fmt.Println(",") } else { fmt.Println("") } } fmt.Println("]") } else { fmt.Println("[]") } } else { ec = 1 fmt.Printf("'%s'\n", strings.TrimRight(err.Error(), "\n")) } } if *tagsValues != "" { formatTags := format if formatTags == client.FormatDefault { formatTags = client.FormatJSON } queryRaw, r, respHeader, err := client.TagsValues(&httpClient, *address, formatTags, *tagsValues, *limit, from, until) if respHeader != nil { fmt.Printf("Responce header: %+v\n", respHeader) } fmt.Print("'") fmt.Print(queryRaw) fmt.Print("' = ") if err == nil { if len(r) > 0 { fmt.Println("[") for i, v := range r { fmt.Printf(" { Value: '%s' }", v) if i < len(r)-1 { fmt.Println(",") } else { fmt.Println("") } } fmt.Println("]") } else { fmt.Println("[]") } } else { ec = 1 fmt.Printf("'%s'\n", strings.TrimRight(err.Error(), "\n")) } } if *tagsNames != "" { formatTags := format if formatTags == client.FormatDefault { formatTags = client.FormatJSON } queryRaw, r, respHeader, err := client.TagsNames(&httpClient, *address, formatTags, *tagsNames, *limit, from, until) if respHeader != nil { fmt.Printf("Responce header: %+v\n", respHeader) } fmt.Print("'") fmt.Print(queryRaw) fmt.Print("' = ") if err == nil { if len(r) > 0 { fmt.Println("[") for i, v := range r { fmt.Printf(" { Tag: '%s' }", v) if i < len(r)-1 { fmt.Println(",") } else { fmt.Println("") } } fmt.Println("]") } else { fmt.Println("[]") } } else { ec = 1 fmt.Printf("'%s'\n", strings.TrimRight(err.Error(), "\n")) } } if len(targets) > 0 { formatRender := format if formatRender == client.FormatDefault { formatRender = client.FormatPb_v3 } queryRaw, r, respHeader, err := client.Render(&httpClient, *address, formatRender, targets, []*carbonapi_v3_pb.FilteringFunction{}, maxDataPoints, from, until) if respHeader != nil { fmt.Printf("Responce header: %+v\n", respHeader) } fmt.Print("'") fmt.Print(queryRaw) fmt.Print("' = ") if err == nil { if len(r) > 0 { fmt.Println("[") for i, m := range r { fmt.Println(" {") fmt.Printf(" Name: '%s', PathExpression: '%v',\n", m.Name, m.PathExpression) fmt.Printf(" ConsolidationFunc: %s, XFilesFactor: %f, AppliedFunctions: %s,\n", m.ConsolidationFunc, m.XFilesFactor, m.AppliedFunctions) fmt.Printf(" Start: %d, Stop: %d, Step: %d, RequestStart: %d, RequestStop: %d,\n", m.StartTime, m.StopTime, m.StepTime, m.RequestStartTime, m.RequestStopTime) fmt.Printf(" Values: %+v\n", m.Values) if i == len(r) { fmt.Println(" }") } else { fmt.Println(" },") } } fmt.Println("]") } else { fmt.Println("[]") } } else { ec = 1 fmt.Printf("'%s'\n", strings.TrimRight(err.Error(), "\n")) } } os.Exit(ec) } ================================================ FILE: config/.gitignore ================================================ tests_tmp/ ================================================ FILE: config/config.go ================================================ package config import ( "bytes" "crypto/tls" "fmt" "net" "net/url" "os" "reflect" "regexp" "sort" "strconv" "strings" "time" "github.com/cactus/go-statsd-client/v5/statsd" "github.com/lomik/carbon-clickhouse/helper/config" "github.com/msaf1980/go-metrics/graphite" "github.com/msaf1980/go-timeutils/duration" toml "github.com/pelletier/go-toml" "github.com/pkg/errors" "go.uber.org/zap" "github.com/lomik/zapwriter" "github.com/lomik/graphite-clickhouse/cache" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/helper/date" "github.com/lomik/graphite-clickhouse/helper/rollup" "github.com/lomik/graphite-clickhouse/limiter" "github.com/lomik/graphite-clickhouse/metrics" ) type SDType uint8 const ( SDNone SDType = iota SDNginx // https://github.com/weibocom/nginx-upsync-module ) var sdTypeStrings map[SDType]string = map[SDType]string{SDNone: "", SDNginx: "nginx"} func (a *SDType) Set(value string) error { switch value { case "nginx": *a = SDNginx case "", "0": *a = SDNone default: return fmt.Errorf("invalid sd type %q", value) } return nil } func (a *SDType) UnmarshalText(data []byte) error { return a.Set(string(data)) } func (a *SDType) MarshalText() ([]byte, error) { return []byte(a.String()), nil } func (a *SDType) UnmarshalJSON(data []byte) error { return a.Set(string(data)) } func (a *SDType) MarshalJSON() ([]byte, error) { return []byte(a.String()), nil } func (a *SDType) String() string { return sdTypeStrings[*a] } func (a *SDType) Type() string { return "service_discovery_type" } // Cache config type CacheConfig struct { Type string `toml:"type" json:"type" comment:"cache type"` Size int `toml:"size-mb" json:"size-mb" comment:"cache size"` MemcachedServers []string `toml:"memcached-servers" json:"memcached-servers" comment:"memcached servers"` DefaultTimeoutSec int32 `toml:"default-timeout" json:"default-timeout" comment:"default cache ttl"` DefaultTimeoutStr string `toml:"-" json:"-"` ShortTimeoutSec int32 `toml:"short-timeout" json:"short-timeout" comment:"short-time cache ttl"` ShortTimeoutStr string `toml:"-" json:"-"` FindTimeoutSec int32 `toml:"find-timeout" json:"find-timeout" comment:"finder/tags autocompleter cache ttl"` ShortDuration time.Duration `toml:"short-duration" json:"short-duration" comment:"maximum diration, used with short_timeout"` ShortUntilOffsetSec int64 `toml:"short-offset" json:"short-offset" comment:"offset beetween now and until for select short cache timeout"` } // Common config type Common struct { Listen string `toml:"listen" json:"listen" comment:"general listener"` PprofListen string `toml:"pprof-listen" json:"pprof-listen" comment:"listener to serve /debug/pprof requests. '-pprof' argument overrides it"` MaxCPU int `toml:"max-cpu" json:"max-cpu"` MaxMetricsInFindAnswer int `toml:"max-metrics-in-find-answer" json:"max-metrics-in-find-answer" comment:"limit number of results from find query, 0=unlimited"` MaxMetricsPerTarget int `toml:"max-metrics-per-target" json:"max-metrics-per-target" comment:"limit numbers of queried metrics per target in /render requests, 0 or negative = unlimited"` AppendEmptySeries bool `toml:"append-empty-series" json:"append-empty-series" comment:"if true, always return points for all metrics, replacing empty results with list of NaN"` TargetBlacklist []string `toml:"target-blacklist" json:"target-blacklist" comment:"daemon returns empty response if query matches any of regular expressions" commented:"true"` Blacklist []*regexp.Regexp `toml:"-" json:"-"` // compiled TargetBlacklist MemoryReturnInterval time.Duration `toml:"memory-return-interval" json:"memory-return-interval" comment:"daemon will return the freed memory to the OS when it>0"` HeadersToLog []string `toml:"headers-to-log" json:"headers-to-log" comment:"additional request headers to log"` BaseWeight int `toml:"base_weight" json:"base_weight" comment:"service discovery base weight (on idle)"` DegragedMultiply float64 `toml:"degraged-multiply" json:"degraged-multiply" comment:"service discovery degraded load avg multiplier (if normalized load avg > degraged_load_avg) (default 4.0)"` DegragedLoad float64 `toml:"degraged-load-avg" json:"degraged-load-avg" comment:"service discovery normilized load avg degraded point (default 1.0)"` SDType SDType `toml:"service-discovery-type" json:"service-discovery-type" comment:"service discovery type"` SD string `toml:"service-discovery" json:"service-discovery" comment:"service discovery address (consul)"` SDNamespace string `toml:"service-discovery-ns" json:"service-discovery-ns" comment:"service discovery namespace (graphite by default)"` SDDc []string `toml:"service-discovery-ds" json:"service-discovery-ds" comment:"service discovery datacenters (first - is primary, in other register as backup)"` SDExpire time.Duration `toml:"service-discovery-expire" json:"service-discovery-expire" comment:"service discovery expire duration for cleanup (minimum is 24h, if enabled)"` FindCacheConfig CacheConfig `toml:"find-cache" json:"find-cache" comment:"find/tags cache config"` FindCache cache.BytesCache `toml:"-" json:"-"` } // FeatureFlags contains feature flags that significantly change how gch responds to some requests type FeatureFlags struct { UseCarbonBehavior bool `toml:"use-carbon-behaviour" json:"use-carbon-behaviour" comment:"if true, prefers carbon's behaviour on how tags are treated"` DontMatchMissingTags bool `toml:"dont-match-missing-tags" json:"dont-match-missing-tags" comment:"if true, seriesByTag terms containing '!=' or '!=~' operators will not match metrics that don't have the tag at all"` LogQueryProgress bool `toml:"log-query-progress" json:"log-query-progress" comment:"if true, gch will log affected rows count by clickhouse query"` } // IndexReverseRule contains rules to use direct or reversed request to index table type IndexReverseRule struct { Suffix string `toml:"suffix,omitempty" json:"suffix" comment:"rule is used when the target suffix is matched"` Prefix string `toml:"prefix,omitempty" json:"prefix" comment:"rule is used when the target prefix is matched"` RegexStr string `toml:"regex,omitempty" json:"regex" comment:"rule is used when the target regex is matched"` Regex *regexp.Regexp `toml:"-" json:"-"` Reverse string `toml:"reverse" json:"reverse" comment:"same as index-reverse"` } type Costs struct { Cost *int `toml:"cost" json:"cost" comment:"default cost (for wildcarded equalence or matched with regex, or if no value cost set)"` ValuesCost map[string]int `toml:"values-cost" json:"values-cost" comment:"cost with some value (for equalence without wildcards) (additional tuning, usually not needed)"` } // IndexReverses is a slise of ptrs to IndexReverseRule type IndexReverses []*IndexReverseRule const ( IndexAuto = iota IndexDirect = iota IndexReversed = iota ) // IndexReverse maps setting name to value var IndexReverse = map[string]uint8{ "direct": IndexDirect, "auto": IndexAuto, "reversed": IndexReversed, } // IndexReverseNames contains valid names for index-reverse setting var IndexReverseNames = []string{"auto", "direct", "reversed"} type UserLimits struct { MaxQueries int `toml:"max-queries" json:"max-queries" comment:"Max queries to fetch data"` ConcurrentQueries int `toml:"concurrent-queries" json:"concurrent-queries" comment:"Concurrent queries to fetch data"` AdaptiveQueries int `toml:"adaptive-queries" json:"adaptive-queries" comment:"Adaptive queries (based on load average) for increase/decrease concurrent queries"` Limiter limiter.ServerLimiter `toml:"-" json:"-"` } type QueryParam struct { Duration time.Duration `toml:"duration" json:"duration" comment:"minimal duration (beetween from/until) for select query params"` URL string `toml:"url" json:"url" comment:"url for queries with durations greater or equal than"` DataTimeout time.Duration `toml:"data-timeout" json:"data-timeout" comment:"total timeout to fetch data"` MaxQueries int `toml:"max-queries" json:"max-queries" comment:"Max queries to fetch data"` ConcurrentQueries int `toml:"concurrent-queries" json:"concurrent-queries" comment:"Concurrent queries to fetch data"` AdaptiveQueries int `toml:"adaptive-queries" json:"adaptive-queries" comment:"Adaptive queries (based on load average) for increase/decrease concurrent queries"` Limiter limiter.ServerLimiter `toml:"-" json:"-"` } func binarySearchQueryParamLe(a []QueryParam, duration time.Duration, start, end int) int { length := end - start if length <= 0 { return -1 // not found } else if length == 1 { if a[start].Duration > duration { return -1 } return start } var result int mid := start + length/2 if a[mid].Duration > duration { result = binarySearchQueryParamLe(a, duration, start, mid) } else { if result = binarySearchQueryParamLe(a, duration, mid+1, end); result == -1 { result = mid } } return result } // ClickHouse config type ClickHouse struct { URL string `toml:"url" json:"url" comment:"default url, see https://clickhouse.tech/docs/en/interfaces/http. Can be overwritten with query-params"` DataTimeout time.Duration `toml:"data-timeout" json:"data-timeout" comment:"default total timeout to fetch data, can be overwritten with query-params"` QueryParams []QueryParam `toml:"query-params" json:"query-params" comment:"customized query params (url, data timeout, limiters) for durations greater or equal"` ProgressSendingInterval time.Duration `toml:"progress-sending-interval" json:"progress-sending-interval" comment:"time interval for ch query progress sending, it's equal to http_headers_progress_interval_ms header"` RenderMaxQueries int `toml:"render-max-queries" json:"render-max-queries" comment:"Max queries to render queiries"` RenderConcurrentQueries int `toml:"render-concurrent-queries" json:"render-concurrent-queries" comment:"Concurrent queries to render queiries"` RenderAdaptiveQueries int `toml:"render-adaptive-queries" json:"render-adaptive-queries" comment:"Render adaptive queries (based on load average) for increase/decrease concurrent queries"` FindMaxQueries int `toml:"find-max-queries" json:"find-max-queries" comment:"Max queries for find queries"` FindConcurrentQueries int `toml:"find-concurrent-queries" json:"find-concurrent-queries" comment:"Find concurrent queries for find queries"` FindAdaptiveQueries int `toml:"find-adaptive-queries" json:"find-adaptive-queries" comment:"Find adaptive queries (based on load average) for increase/decrease concurrent queries"` FindLimiter limiter.ServerLimiter `toml:"-" json:"-"` TagsMaxQueries int `toml:"tags-max-queries" json:"tags-max-queries" comment:"Max queries for tags queries"` TagsConcurrentQueries int `toml:"tags-concurrent-queries" json:"tags-concurrent-queries" comment:"Concurrent queries for tags queries"` TagsAdaptiveQueries int `toml:"tags-adaptive-queries" json:"tags-adaptive-queries" comment:"Tags adaptive queries (based on load average) for increase/decrease concurrent queries"` TagsLimiter limiter.ServerLimiter `toml:"-" json:"-"` WildcardMinDistance int `toml:"wildcard-min-distance" json:"wildcard-min-distance" comment:"If a wildcard appears both at the start and the end of a plain query at a distance (in terms of nodes) less than wildcard-min-distance, then it will be discarded. This parameter can be used to discard expensive queries."` TrySplitQuery bool `toml:"try-split-query" json:"try-split-query" comment:"Plain queries like '{first,second}.custom.metric.*' are also a subject to wildcard-min-distance restriction. But can be split into 2 queries: 'first.custom.metric.*', 'second.custom.metric.*'. Note that: only one list will be split; if there are wildcard in query before (after) list then reverse (direct) notation will be preferred; if there are wildcards before and after list, then query will not be split"` MaxNodeToSplitIndex int `toml:"max-node-to-split-index" json:"max-node-to-split-index" comment:"Used only if try-split-query is true. Query that contains list will be split if its (list) node index is less or equal to max-node-to-split-index. By default is 0. It is recommended to have this value set to 2 or 3 and increase it very carefully, because 3 or 4 plain nodes without wildcards have good selectivity"` TagsMinInQuery int `toml:"tags-min-in-query" json:"tags-min-in-query" comment:"Minimum tags in seriesByTag query"` TagsMinInAutocomplete int `toml:"tags-min-in-autocomplete" json:"tags-min-in-autocomplete" comment:"Minimum tags in autocomplete query"` UserLimits map[string]UserLimits `toml:"user-limits" json:"user-limits" comment:"customized query limiter for some users" commented:"true"` DateFormat string `toml:"date-format" json:"date-format" comment:"Date format (default, utc, both)"` IndexTable string `toml:"index-table" json:"index-table" comment:"see doc/index-table.md"` IndexUseDaily bool `toml:"index-use-daily" json:"index-use-daily"` IndexReverse string `toml:"index-reverse" json:"index-reverse" comment:"see doc/config.md"` IndexReverses IndexReverses `toml:"index-reverses" json:"index-reverses" comment:"see doc/config.md" commented:"true"` IndexTimeout time.Duration `toml:"index-timeout" json:"index-timeout" comment:"total timeout to fetch series list from index"` TaggedTable string `toml:"tagged-table" json:"tagged-table" comment:"'tagged' table from carbon-clickhouse, required for seriesByTag"` TagsCountTable string `toml:"tags-count-table" json:"tags-count-table" comment:"Table that contains the total amounts of each tag-value pair. It is used to avoid usage of high cardinality tag-value pairs when querying TaggedTable. If left empty, basic sorting will be used. See more detailed description in doc/config.md"` TaggedAutocompleDays int `toml:"tagged-autocomplete-days" json:"tagged-autocomplete-days" comment:"or how long the daemon will query tags during autocomplete"` TaggedUseDaily bool `toml:"tagged-use-daily" json:"tagged-use-daily" comment:"whether to use date filter when searching for the metrics in the tagged-table"` TaggedCosts map[string]*Costs `toml:"tagged-costs" json:"tagged-costs" comment:"costs for tags (for tune which tag will be used as primary), by default is 0, increase for costly (with poor selectivity) tags" commented:"true"` TreeTable string `toml:"tree-table" json:"tree-table" comment:"old index table, DEPRECATED, see description in doc/config.md" commented:"true"` ReverseTreeTable string `toml:"reverse-tree-table" json:"reverse-tree-table" commented:"true"` DateTreeTable string `toml:"date-tree-table" json:"date-tree-table" commented:"true"` DateTreeTableVersion int `toml:"date-tree-table-version" json:"date-tree-table-version" commented:"true"` TreeTimeout time.Duration `toml:"tree-timeout" json:"tree-timeout" commented:"true"` TagTable string `toml:"tag-table" json:"tag-table" comment:"is not recommended to use, https://github.com/lomik/graphite-clickhouse/wiki/TagsRU" commented:"true"` ExtraPrefix string `toml:"extra-prefix" json:"extra-prefix" comment:"add extra prefix (directory in graphite) for all metrics, w/o trailing dot"` ConnectTimeout time.Duration `toml:"connect-timeout" json:"connect-timeout" comment:"TCP connection timeout"` // TODO: remove in v0.14 DataTableLegacy string `toml:"data-table" json:"data-table" comment:"will be removed in 0.14" commented:"true"` // TODO: remove in v0.14 RollupConfLegacy string `toml:"rollup-conf" json:"-" commented:"true"` MaxDataPoints int `toml:"max-data-points" json:"max-data-points" comment:"max points per metric when internal-aggregation=true"` // InternalAggregation controls if ClickHouse itself or graphite-clickhouse aggregates points to proper retention InternalAggregation bool `toml:"internal-aggregation" json:"internal-aggregation" comment:"ClickHouse-side aggregation, see doc/aggregation.md"` TLSParams config.TLS `toml:"tls" json:"tls" comment:"mTLS HTTPS configuration for connecting to clickhouse server" commented:"true"` TLSConfig *tls.Config `toml:"-" json:"-"` } func clickhouseURLValidate(chURL string) (*url.URL, error) { u, err := url.Parse(chURL) if err != nil { return nil, fmt.Errorf("error %q in url %q", err.Error(), chURL) } else if u.Scheme != "http" && u.Scheme != "https" { return nil, fmt.Errorf("scheme not supported in url %q", chURL) } else if strings.Contains(u.RawQuery, " ") { return nil, fmt.Errorf("space not allowed in url %q", chURL) } return u, nil } // Tags config type Tags struct { Rules string `toml:"rules" json:"rules"` Date string `toml:"date" json:"date"` ExtraWhere string `toml:"extra-where" json:"extra-where"` InputFile string `toml:"input-file" json:"input-file"` OutputFile string `toml:"output-file" json:"output-file"` Threads int `toml:"threads" json:"threads" comment:"number of threads for uploading tags to clickhouse (1 by default)"` Compression clickhouse.ContentEncoding `toml:"compression" json:"compression" comment:"compression method for tags before sending them to clickhouse (i.e. content encoding): gzip (default), none, zstd"` Version uint32 `toml:"version" json:"version" comment:"fixed tags version for testing purposes (by default the current timestamp is used for each upload)"` SelectChunksCount int `toml:"select-chunks-count" json:"select-chunks-count" comment:"number of chunks for selecting metrics from clickhouse (10 by default)"` } // Carbonlink configuration type Carbonlink struct { Server string `toml:"server" json:"server"` Threads int `toml:"threads-per-request" json:"threads-per-request"` Retries int `toml:"-" json:"-"` ConnectTimeout time.Duration `toml:"connect-timeout" json:"connect-timeout"` QueryTimeout time.Duration `toml:"query-timeout" json:"query-timeout"` TotalTimeout time.Duration `toml:"total-timeout" json:"total-timeout" comment:"timeout for querying and parsing response"` } // Prometheus configuration type Prometheus struct { Listen string `toml:"listen" json:"listen" comment:"listen addr for prometheus ui and api"` ExternalURLRaw string `toml:"external-url" json:"external-url" comment:"allows to set URL for redirect manually"` ExternalURL *url.URL `toml:"-" json:"-"` PageTitle string `toml:"page-title" json:"page-title"` LookbackDelta time.Duration `toml:"lookback-delta" json:"lookback-delta"` RemoteReadConcurrencyLimit int `toml:"remote-read-concurrency-limit" json:"remote-read-concurrency-limit" comment:"concurrently handled remote read requests"` } const ( // ContextGraphite for data tables ContextGraphite = "graphite" // ContextPrometheus for data tables ContextPrometheus = "prometheus" ) var knownDataTableContext = map[string]bool{ ContextGraphite: true, ContextPrometheus: true, } // DataTable configs type DataTable struct { Table string `toml:"table" json:"table" comment:"data table from carbon-clickhouse"` Reverse bool `toml:"reverse" json:"reverse" comment:"if it stores direct or reversed metrics"` MaxAge time.Duration `toml:"max-age" json:"max-age" comment:"maximum age stored in the table"` MinAge time.Duration `toml:"min-age" json:"min-age" comment:"minimum age stored in the table"` MaxInterval time.Duration `toml:"max-interval" json:"max-interval" comment:"maximum until-from interval allowed for the table"` MinInterval time.Duration `toml:"min-interval" json:"min-interval" comment:"minimum until-from interval allowed for the table"` TargetMatchAny string `toml:"target-match-any" json:"target-match-any" comment:"table allowed only if any metrics in target matches regexp"` TargetMatchAll string `toml:"target-match-all" json:"target-match-all" comment:"table allowed only if all metrics in target matches regexp"` TargetMatchAnyRegexp *regexp.Regexp `toml:"-" json:"-"` TargetMatchAllRegexp *regexp.Regexp `toml:"-" json:"-"` RollupConf string `toml:"rollup-conf" json:"-" comment:"custom rollup.xml file for table, 'auto' and 'none' are allowed as well"` RollupAutoTable string `toml:"rollup-auto-table" json:"rollup-auto-table" comment:"custom table for 'rollup-conf=auto', useful for Distributed or MatView"` RollupAutoInterval *time.Duration `toml:"rollup-auto-interval" json:"rollup-auto-interval" comment:"rollup update interval for 'rollup-conf=auto'"` RollupDefaultPrecision uint32 `toml:"rollup-default-precision" json:"rollup-default-precision" comment:"is used when none of rules match"` RollupDefaultFunction string `toml:"rollup-default-function" json:"rollup-default-function" comment:"is used when none of rules match"` RollupUseReverted bool `toml:"rollup-use-reverted" json:"rollup-use-reverted" comment:"should be set to true if you don't have reverted regexps in rollup-conf for reversed tables"` Context []string `toml:"context" json:"context" comment:"valid values are 'graphite' of 'prometheus'"` ContextMap map[string]bool `toml:"-" json:"-"` Rollup *rollup.Rollup `toml:"-" json:"rollup-conf"` QueryMetrics *metrics.QueryMetrics `toml:"-" json:"-"` } // Debug config type Debug struct { Directory string `toml:"directory" json:"directory" comment:"the directory for additional debug output"` DirectoryPerm os.FileMode `toml:"directory-perm" json:"directory-perm" comment:"permissions for directory, octal value is set as 0o755"` // If ExternalDataPerm > 0 and X-Gch-Debug-Ext-Data HTTP header is set, the external data used in the query // will be saved in the DebugDir directory ExternalDataPerm os.FileMode `toml:"external-data-perm" json:"external-data-perm" comment:"permissions for directory, octal value is set as 0o640"` } // Config is the daemon configuration type Config struct { Common Common `toml:"common" json:"common"` FeatureFlags FeatureFlags `toml:"feature-flags" json:"feature-flags"` Metrics metrics.Config `toml:"metrics" json:"metrics"` ClickHouse ClickHouse `toml:"clickhouse" json:"clickhouse"` DataTable []DataTable `toml:"data-table" json:"data-table" comment:"data tables, see doc/config.md for additional info"` Tags Tags `toml:"tags" json:"tags" comment:"is not recommended to use, https://github.com/lomik/graphite-clickhouse/wiki/TagsRU" commented:"true"` Carbonlink Carbonlink `toml:"carbonlink" json:"carbonlink"` Prometheus Prometheus `toml:"prometheus" json:"prometheus"` Debug Debug `toml:"debug" json:"debug" comment:"see doc/debugging.md"` Logging []zapwriter.Config `toml:"logging" json:"logging"` } // New returns *Config with default values func New() *Config { cfg := &Config{ Common: Common{ Listen: ":9090", PprofListen: "", // MetricPrefix: "carbon.graphite-clickhouse.{host}", // MetricInterval: time.Minute, // MetricEndpoint: MetricEndpointLocal, MaxCPU: 1, MaxMetricsInFindAnswer: 0, MaxMetricsPerTarget: 15000, // This is arbitrary value to protect CH from overload MemoryReturnInterval: 0, FindCacheConfig: CacheConfig{ Type: "null", DefaultTimeoutSec: 0, ShortTimeoutSec: 0, FindTimeoutSec: 0, }, DegragedMultiply: 4.0, DegragedLoad: 1.0, }, ClickHouse: ClickHouse{ URL: "http://localhost:8123?cancel_http_readonly_queries_on_client_close=1", DataTimeout: time.Minute, ProgressSendingInterval: 10 * time.Second, IndexTable: "graphite_index", IndexUseDaily: true, TaggedUseDaily: true, IndexReverse: "auto", IndexReverses: IndexReverses{}, IndexTimeout: time.Minute, TaggedTable: "graphite_tagged", TaggedAutocompleDays: 7, ExtraPrefix: "", ConnectTimeout: time.Second, DataTableLegacy: "", RollupConfLegacy: "auto", MaxDataPoints: 1048576, InternalAggregation: true, FindLimiter: limiter.NoopLimiter{}, TagsLimiter: limiter.NoopLimiter{}, }, Tags: Tags{ Threads: 1, Compression: "gzip", }, Carbonlink: Carbonlink{ Threads: 10, Retries: 2, ConnectTimeout: 50 * time.Millisecond, QueryTimeout: 50 * time.Millisecond, TotalTimeout: 500 * time.Millisecond, }, Prometheus: Prometheus{ ExternalURLRaw: "", PageTitle: "Prometheus Time Series Collection and Processing Server", Listen: ":9092", LookbackDelta: 5 * time.Minute, RemoteReadConcurrencyLimit: 10, }, Debug: Debug{ Directory: "", DirectoryPerm: 0755, ExternalDataPerm: 0, }, Logging: nil, } return cfg } // Compile checks if IndexReverseRule are valid in the IndexReverses and compiles regexps if set func (ir IndexReverses) Compile() error { var err error for i, n := range ir { if len(n.RegexStr) > 0 { if n.Regex, err = regexp.Compile(n.RegexStr); err != nil { return err } } else if len(n.Prefix) == 0 && len(n.Suffix) == 0 { return fmt.Errorf("empthy index-use-reverses[%d] rule", i) } if _, ok := IndexReverse[n.Reverse]; !ok { return fmt.Errorf("%s is not valid value for index-reverses.reverse", n.Reverse) } } return nil } func newLoggingConfig() zapwriter.Config { cfg := zapwriter.NewConfig() cfg.File = "/var/log/graphite-clickhouse/graphite-clickhouse.log" return cfg } func DefaultConfig() (*Config, error) { cfg := New() if cfg.Logging == nil { cfg.Logging = make([]zapwriter.Config, 0) } if len(cfg.Logging) == 0 { cfg.Logging = append(cfg.Logging, newLoggingConfig()) } if len(cfg.DataTable) == 0 { interval := time.Minute cfg.DataTable = []DataTable{ { Table: "graphite_data", RollupConf: "auto", RollupAutoInterval: &interval, }, } } if len(cfg.ClickHouse.IndexReverses) == 0 { cfg.ClickHouse.IndexReverses = IndexReverses{ &IndexReverseRule{Suffix: "suffix", Reverse: "auto"}, &IndexReverseRule{Prefix: "prefix", Reverse: "direct"}, &IndexReverseRule{RegexStr: "regex", Reverse: "reversed"}, } err := cfg.ClickHouse.IndexReverses.Compile() if err != nil { return nil, err } } return cfg, nil } // PrintDefaultConfig prints the default config with some additions to be useful func PrintDefaultConfig() error { buf := new(bytes.Buffer) cfg, err := DefaultConfig() if err != nil { return err } encoder := toml.NewEncoder(buf).Indentation(" ").Order(toml.OrderPreserve).CompactComments(true) if err := encoder.Encode(cfg); err != nil { return err } out := strings.Replace(buf.String(), "\n", "", 1) fmt.Print(out) return nil } // ReadConfig reads the content of the file with given name and process it to the *Config func ReadConfig(filename string, exactConfig bool) (*Config, []zap.Field, error) { var err error var body []byte if filename != "" { body, err = os.ReadFile(filename) if err != nil { return nil, nil, err } } return Unmarshal(body, exactConfig) } // Unmarshal process the body to *Config func Unmarshal(body []byte, exactConfig bool) (cfg *Config, warns []zap.Field, err error) { deprecations := make(map[string]error) cfg = New() if len(body) != 0 { // TODO: remove in v0.14 if bytes.Index(body, []byte("\n[logging]\n")) != -1 || bytes.Index(body, []byte("[logging]")) == 0 { deprecations["logging"] = fmt.Errorf("single [logging] value became multivalue [[logging]]; please, adjust your config") body = bytes.ReplaceAll(body, []byte("\n[logging]\n"), []byte("\n[[logging]]\n")) if bytes.Index(body, []byte("[logging]")) == 0 { body = bytes.Replace(body, []byte("[logging]"), []byte("[[logging]]"), 1) } } decoder := toml.NewDecoder(bytes.NewReader(body)) decoder.Strict(exactConfig) err := decoder.Decode(cfg) if err != nil { return nil, nil, err } } if cfg.Logging == nil { cfg.Logging = make([]zapwriter.Config, 0) } if cfg.ClickHouse.RenderConcurrentQueries > cfg.ClickHouse.RenderMaxQueries && cfg.ClickHouse.RenderMaxQueries > 0 { cfg.ClickHouse.RenderConcurrentQueries = 0 } chURL, err := clickhouseURLValidate(cfg.ClickHouse.URL) if err != nil { return nil, nil, err } if !reflect.DeepEqual(cfg.ClickHouse.TLSParams, config.TLS{}) { tlsConfig, warnings, err := config.ParseClientTLSConfig(&cfg.ClickHouse.TLSParams) if err != nil { return nil, nil, err } if chURL.Scheme == "https" { cfg.ClickHouse.TLSConfig = tlsConfig } else { warnings = append(warnings, "TLS configurations is ignored because scheme is not HTTPS") } warns = append(warns, zap.Strings("tls-config", warnings)) } for i := range cfg.ClickHouse.QueryParams { if cfg.ClickHouse.QueryParams[i].ConcurrentQueries > cfg.ClickHouse.QueryParams[i].MaxQueries && cfg.ClickHouse.QueryParams[i].MaxQueries > 0 { cfg.ClickHouse.QueryParams[i].ConcurrentQueries = 0 } if cfg.ClickHouse.QueryParams[i].Duration == 0 { return nil, nil, fmt.Errorf("query duration param not set for: %+v", cfg.ClickHouse.QueryParams[i]) } if cfg.ClickHouse.QueryParams[i].DataTimeout == 0 { cfg.ClickHouse.QueryParams[i].DataTimeout = cfg.ClickHouse.DataTimeout } if cfg.ClickHouse.QueryParams[i].URL == "" { // reuse default url cfg.ClickHouse.QueryParams[i].URL = cfg.ClickHouse.URL } if _, err = clickhouseURLValidate(cfg.ClickHouse.QueryParams[i].URL); err != nil { return nil, nil, err } } cfg.ClickHouse.QueryParams = append( []QueryParam{{ URL: cfg.ClickHouse.URL, DataTimeout: cfg.ClickHouse.DataTimeout, MaxQueries: cfg.ClickHouse.RenderMaxQueries, ConcurrentQueries: cfg.ClickHouse.RenderConcurrentQueries, AdaptiveQueries: cfg.ClickHouse.RenderAdaptiveQueries, }}, cfg.ClickHouse.QueryParams..., ) sort.SliceStable(cfg.ClickHouse.QueryParams, func(i, j int) bool { return cfg.ClickHouse.QueryParams[i].Duration < cfg.ClickHouse.QueryParams[j].Duration }) if len(cfg.Logging) == 0 { cfg.Logging = append(cfg.Logging, newLoggingConfig()) } if err = zapwriter.CheckConfig(cfg.Logging, nil); err != nil { return nil, nil, err } // Check if debug directory exists or could be created if cfg.Debug.Directory != "" { info, err := os.Stat(cfg.Debug.Directory) if os.IsNotExist(err) { err := os.MkdirAll(cfg.Debug.Directory, os.ModeDir|cfg.Debug.DirectoryPerm) if err != nil { return nil, nil, err } } else if !info.IsDir() { return nil, nil, fmt.Errorf("the file for external data debug dumps exists and is not a directory: %v", cfg.Debug.Directory) } } if _, ok := IndexReverse[cfg.ClickHouse.IndexReverse]; !ok { return nil, nil, fmt.Errorf("%s is not valid value for index-reverse", cfg.ClickHouse.IndexReverse) } err = cfg.ClickHouse.IndexReverses.Compile() if err != nil { return nil, nil, err } if cfg.Common.FindCache, err = CreateCache("index", &cfg.Common.FindCacheConfig); err == nil { if cfg.Common.FindCacheConfig.Type != "null" { warns = append(warns, zap.Any("enable find cache", zap.String("type", cfg.Common.FindCacheConfig.Type))) } } else { return nil, nil, err } l := len(cfg.Common.TargetBlacklist) if l > 0 { cfg.Common.Blacklist = make([]*regexp.Regexp, l) for i := 0; i < l; i++ { r, err := regexp.Compile(cfg.Common.TargetBlacklist[i]) if err != nil { return nil, nil, err } cfg.Common.Blacklist[i] = r } } err = cfg.ProcessDataTables() if err != nil { return nil, nil, err } // compute prometheus external url rawURL := cfg.Prometheus.ExternalURLRaw if rawURL == "" { hostname, err := os.Hostname() if err != nil { return nil, nil, err } _, port, err := net.SplitHostPort(cfg.Common.Listen) if err != nil { return nil, nil, err } rawURL = fmt.Sprintf("http://%s:%s/", hostname, port) } cfg.Prometheus.ExternalURL, err = url.Parse(rawURL) if err != nil { return nil, nil, err } cfg.Prometheus.ExternalURL.Path = strings.TrimRight(cfg.Prometheus.ExternalURL.Path, "/") checkDeprecations(cfg, deprecations) if len(deprecations) != 0 { deprecationList := make([]error, len(deprecations)) for name, message := range deprecations { deprecationList = append(deprecationList, errors.Wrap(message, name)) } warns = append(warns, zap.Errors("config deprecations", deprecationList)) } switch strings.ToLower(cfg.ClickHouse.DateFormat) { case "utc": date.SetUTC() case "both": date.SetBoth() default: if cfg.ClickHouse.DateFormat != "" && cfg.ClickHouse.DateFormat != "default" { return nil, nil, fmt.Errorf("unsupported date-format: %s", cfg.ClickHouse.DateFormat) } } if cfg.ClickHouse.FindConcurrentQueries > cfg.ClickHouse.FindMaxQueries && cfg.ClickHouse.FindMaxQueries > 0 { cfg.ClickHouse.FindConcurrentQueries = 0 } if cfg.ClickHouse.TagsConcurrentQueries > cfg.ClickHouse.TagsMaxQueries && cfg.ClickHouse.TagsMaxQueries > 0 { cfg.ClickHouse.TagsConcurrentQueries = 0 } metricsEnabled := cfg.setupGraphiteMetrics() cfg.ClickHouse.FindLimiter = limiter.NewALimiter( cfg.ClickHouse.FindMaxQueries, cfg.ClickHouse.FindConcurrentQueries, cfg.ClickHouse.FindAdaptiveQueries, metricsEnabled, "find", "all", ) cfg.ClickHouse.TagsLimiter = limiter.NewALimiter( cfg.ClickHouse.TagsMaxQueries, cfg.ClickHouse.TagsConcurrentQueries, cfg.ClickHouse.TagsAdaptiveQueries, metricsEnabled, "tags", "all", ) for i := range cfg.ClickHouse.QueryParams { cfg.ClickHouse.QueryParams[i].Limiter = limiter.NewALimiter( cfg.ClickHouse.QueryParams[i].MaxQueries, cfg.ClickHouse.QueryParams[i].ConcurrentQueries, cfg.ClickHouse.QueryParams[i].AdaptiveQueries, metricsEnabled, "render", duration.String(cfg.ClickHouse.QueryParams[i].Duration), ) } for u, q := range cfg.ClickHouse.UserLimits { q.Limiter = limiter.NewALimiter( q.MaxQueries, q.ConcurrentQueries, q.AdaptiveQueries, metricsEnabled, u, "all", ) cfg.ClickHouse.UserLimits[u] = q } return cfg, warns, nil } // NeedLoadAvgColect check if load avg collect is neeeded func (c *Config) NeedLoadAvgColect() bool { if c.Common.SD != "" { if c.Common.DegragedMultiply <= 0 { c.Common.DegragedMultiply = 4.0 } if c.Common.DegragedLoad <= 0 { c.Common.DegragedLoad = 1.0 } if c.Common.BaseWeight <= 0 { c.Common.BaseWeight = 100 } if c.Common.SDNamespace == "" { c.Common.SDNamespace = "graphite" } if c.Common.SDExpire < 24*time.Hour { c.Common.SDExpire = 24 * time.Hour } return true } if c.ClickHouse.RenderAdaptiveQueries > 0 { return true } if c.ClickHouse.FindAdaptiveQueries > 0 { return true } if c.ClickHouse.TagsAdaptiveQueries > 0 { return true } for _, u := range c.ClickHouse.UserLimits { if u.AdaptiveQueries > 0 { return true } } return false } // ProcessDataTables checks if legacy `data`-table config is used, compiles regexps for `target-match-any` and `target-match-all` // parameters, sets the rollup configuration and proper context. func (c *Config) ProcessDataTables() (err error) { if c.ClickHouse.DataTableLegacy != "" { c.DataTable = append(c.DataTable, DataTable{ Table: c.ClickHouse.DataTableLegacy, RollupConf: c.ClickHouse.RollupConfLegacy, }) } for i := 0; i < len(c.DataTable); i++ { if c.DataTable[i].TargetMatchAny != "" { r, err := regexp.Compile(c.DataTable[i].TargetMatchAny) if err != nil { return err } c.DataTable[i].TargetMatchAnyRegexp = r } if c.DataTable[i].TargetMatchAll != "" { r, err := regexp.Compile(c.DataTable[i].TargetMatchAll) if err != nil { return err } c.DataTable[i].TargetMatchAllRegexp = r } rdp := c.DataTable[i].RollupDefaultPrecision rdf := c.DataTable[i].RollupDefaultFunction if c.DataTable[i].RollupConf == "auto" || c.DataTable[i].RollupConf == "" { table := c.DataTable[i].Table interval := time.Minute if c.DataTable[i].RollupAutoTable != "" { table = c.DataTable[i].RollupAutoTable } if c.DataTable[i].RollupAutoInterval != nil { interval = *c.DataTable[i].RollupAutoInterval } c.DataTable[i].Rollup, err = rollup.NewAuto( c.ClickHouse.URL, c.ClickHouse.TLSConfig, table, interval, rdp, rdf, ) } else if c.DataTable[i].RollupConf == "none" { c.DataTable[i].Rollup, err = rollup.NewDefault(rdp, rdf) } else { c.DataTable[i].Rollup, err = rollup.NewXMLFile(c.DataTable[i].RollupConf, rdp, rdf) } if err != nil { return err } if len(c.DataTable[i].Context) == 0 { c.DataTable[i].ContextMap = knownDataTableContext } else { c.DataTable[i].ContextMap = make(map[string]bool) for _, ctx := range c.DataTable[i].Context { if !knownDataTableContext[ctx] { return fmt.Errorf("unknown context %#v", ctx) } c.DataTable[i].ContextMap[ctx] = true } } } return nil } func checkDeprecations(cfg *Config, d map[string]error) { if cfg.ClickHouse.DataTableLegacy != "" { d["data-table"] = fmt.Errorf("data-table parameter in [clickhouse] is deprecated; use [[data-table]]") } } func CreateCache(cacheName string, cacheConfig *CacheConfig) (cache.BytesCache, error) { if cacheConfig.DefaultTimeoutSec <= 0 && cacheConfig.ShortTimeoutSec <= 0 && cacheConfig.FindTimeoutSec <= 0 { return nil, nil } if cacheConfig.DefaultTimeoutSec < cacheConfig.ShortTimeoutSec { cacheConfig.DefaultTimeoutSec = cacheConfig.ShortTimeoutSec } if cacheConfig.ShortTimeoutSec < 0 || cacheConfig.DefaultTimeoutSec == cacheConfig.ShortTimeoutSec { // broken value or short timeout not need due to equal cacheConfig.ShortTimeoutSec = 0 } if cacheConfig.DefaultTimeoutSec < cacheConfig.ShortTimeoutSec { cacheConfig.DefaultTimeoutSec = cacheConfig.ShortTimeoutSec } if cacheConfig.ShortDuration == 0 { cacheConfig.ShortDuration = 3 * time.Hour } if cacheConfig.ShortUntilOffsetSec == 0 { cacheConfig.ShortUntilOffsetSec = 120 } cacheConfig.DefaultTimeoutStr = strconv.Itoa(int(cacheConfig.DefaultTimeoutSec)) cacheConfig.ShortTimeoutStr = strconv.Itoa(int(cacheConfig.ShortTimeoutSec)) switch cacheConfig.Type { case "memcache": if len(cacheConfig.MemcachedServers) == 0 { return nil, fmt.Errorf(cacheName + ": memcache cache requested but no memcache servers provided") } return cache.NewMemcached("gch-"+cacheName, cacheConfig.MemcachedServers...), nil case "mem": return cache.NewExpireCache(uint64(cacheConfig.Size * 1024 * 1024)), nil case "null": // defaults return nil, nil default: return nil, fmt.Errorf( "%s: unknown cache type '%s', known_cache_types 'null', 'mem', 'memcache'", cacheName, cacheConfig.Type, ) } } func (c *Config) setupGraphiteMetrics() bool { if c.Metrics.MetricEndpoint == "" { metrics.DisableMetrics() } else { if c.Metrics.MetricInterval == 0 { c.Metrics.MetricInterval = 60 * time.Second } if c.Metrics.MetricTimeout == 0 { c.Metrics.MetricTimeout = time.Second } hostname, _ := os.Hostname() fqdn := strings.ReplaceAll(hostname, ".", "_") hostname = strings.Split(hostname, ".")[0] c.Metrics.MetricPrefix = strings.ReplaceAll(c.Metrics.MetricPrefix, "{prefix}", c.Metrics.MetricPrefix) c.Metrics.MetricPrefix = strings.ReplaceAll(c.Metrics.MetricPrefix, "{fqdn}", fqdn) c.Metrics.MetricPrefix = strings.ReplaceAll(c.Metrics.MetricPrefix, "{host}", hostname) // register our metrics with graphite metrics.Graphite = graphite.New(c.Metrics.MetricInterval, c.Metrics.MetricPrefix, c.Metrics.MetricEndpoint, c.Metrics.MetricTimeout) if c.Metrics.Statsd != "" && c.Metrics.ExtendedStat { var err error config := &statsd.ClientConfig{ Address: c.Metrics.Statsd, Prefix: c.Metrics.MetricPrefix, ResInterval: 5 * time.Minute, UseBuffered: true, FlushInterval: 300 * time.Millisecond, } metrics.Gstatsd, err = statsd.NewClientWithConfig(config) if err != nil { metrics.Gstatsd = metrics.NullSender{} fmt.Fprintf(os.Stderr, "statsd init: %v\n", err) } } metrics.InitMetrics(&c.Metrics, c.ClickHouse.FindMaxQueries > 0, c.ClickHouse.TagsMaxQueries > 0) } metrics.AutocompleteQMetric = metrics.InitQueryMetrics("tags", &c.Metrics) metrics.FindQMetric = metrics.InitQueryMetrics("find", &c.Metrics) for i := 0; i < len(c.DataTable); i++ { c.DataTable[i].QueryMetrics = metrics.InitQueryMetrics(c.DataTable[i].Table, &c.Metrics) } if c.ClickHouse.IndexTable != "" { metrics.InitQueryMetrics(c.ClickHouse.IndexTable, &c.Metrics) } if c.ClickHouse.TaggedTable != "" { metrics.InitQueryMetrics(c.ClickHouse.TaggedTable, &c.Metrics) } if c.ClickHouse.TagsCountTable != "" { metrics.InitQueryMetrics(c.ClickHouse.TagsCountTable, &c.Metrics) } return metrics.Graphite != nil } func (c *Config) GetUserFindLimiter(username string) limiter.ServerLimiter { if username != "" && len(c.ClickHouse.UserLimits) > 0 { if q, ok := c.ClickHouse.UserLimits[username]; ok { return q.Limiter } } return c.ClickHouse.FindLimiter } func (c *Config) GetUserTagsLimiter(username string) limiter.ServerLimiter { if username != "" && len(c.ClickHouse.UserLimits) > 0 { if q, ok := c.ClickHouse.UserLimits[username]; ok { return q.Limiter } } return c.ClickHouse.TagsLimiter } // search on sorted slice func GetQueryParam(a []QueryParam, duration time.Duration) int { if indx := binarySearchQueryParamLe(a, duration, 0, len(a)); indx == -1 { return 0 } else { return indx } } ================================================ FILE: config/config_test.go ================================================ package config import ( "fmt" "io/fs" "math" "net/url" "os" "regexp" "regexp/syntax" "syscall" "testing" "time" "github.com/lomik/zapwriter" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lomik/graphite-clickhouse/limiter" "github.com/lomik/graphite-clickhouse/metrics" ) func TestProcessDataTables(t *testing.T) { type in struct { table DataTable tableLegacy string } type out struct { tables []DataTable err error } type ctx map[string]bool regexpCompileWrapper := func(re string) *regexp.Regexp { r, _ := regexp.Compile(re) return r } tests := []struct { name string in in out out }{ { name: "legacy table only", in: in{ tableLegacy: "graphite.data", }, out: out{ []DataTable{ { Table: "graphite.data", RollupConf: "auto", ContextMap: ctx{"graphite": true, "prometheus": true}, }, }, nil, }, }, { name: "legacy and normal tables", in: in{ table: DataTable{Table: "graphite.new_data"}, tableLegacy: "graphite.data", }, out: out{ []DataTable{ { Table: "graphite.new_data", ContextMap: ctx{"graphite": true, "prometheus": true}, }, { Table: "graphite.data", RollupConf: "auto", ContextMap: ctx{"graphite": true, "prometheus": true}, }, }, nil, }, }, { name: "fail to compile TargetMatchAll", in: in{ table: DataTable{Table: "graphite.data", TargetMatchAll: "[2223"}, }, out: out{ []DataTable{{Table: "graphite.data", TargetMatchAll: "[2223"}}, &syntax.Error{Code: syntax.ErrMissingBracket, Expr: "[2223"}, }, }, { name: "fail to compile TargetMatchAny", in: in{ table: DataTable{Table: "graphite.data", TargetMatchAny: "[2223"}, }, out: out{ []DataTable{{Table: "graphite.data", TargetMatchAny: "[2223"}}, &syntax.Error{Code: syntax.ErrMissingBracket, Expr: "[2223"}, }, }, { name: "fail to compile TargetMatchAny", in: in{ table: DataTable{Table: "graphite.data", TargetMatchAny: "[2223"}, }, out: out{ []DataTable{{Table: "graphite.data", TargetMatchAny: "[2223"}}, &syntax.Error{Code: syntax.ErrMissingBracket, Expr: "[2223"}, }, }, { name: "fail to read xml rollup", in: in{ table: DataTable{Table: "graphite.data", RollupConf: "/some/file/that/does/not/hopefully/exists/on/the/disk"}, }, out: out{ []DataTable{{Table: "graphite.data", RollupConf: "/some/file/that/does/not/hopefully/exists/on/the/disk"}}, &fs.PathError{Op: "open", Path: "/some/file/that/does/not/hopefully/exists/on/the/disk", Err: syscall.ENOENT}, }, }, { name: "unknown context", in: in{ table: DataTable{Table: "graphite.data", Context: []string{"unexpected"}}, }, out: out{ []DataTable{ { Table: "graphite.data", Context: []string{"unexpected"}, ContextMap: ctx{}, }, }, fmt.Errorf("unknown context \"unexpected\""), }, }, { name: "check all works", in: in{ table: DataTable{ Table: "graphite.data", Reverse: true, TargetMatchAll: "^.*[asdf][.].*", TargetMatchAny: "^.*{a|s|d|f}[.].*", RollupConf: "none", RollupDefaultFunction: "any", RollupDefaultPrecision: 61, RollupUseReverted: true, Context: []string{"prometheus"}, }, tableLegacy: "table", }, out: out{ []DataTable{ { Table: "graphite.data", Reverse: true, TargetMatchAll: "^.*[asdf][.].*", TargetMatchAny: "^.*{a|s|d|f}[.].*", TargetMatchAllRegexp: regexpCompileWrapper("^.*[asdf][.].*"), TargetMatchAnyRegexp: regexpCompileWrapper("^.*{a|s|d|f}[.].*"), RollupConf: "none", RollupDefaultFunction: "any", RollupDefaultPrecision: 61, RollupUseReverted: true, Context: []string{"prometheus"}, ContextMap: ctx{"prometheus": true}, }, { Table: "table", RollupConf: "auto", ContextMap: ctx{"graphite": true, "prometheus": true}, }, }, nil, }, }, { name: "unknown context", in: in{ table: DataTable{Table: "graphite.data", Context: []string{"unexpected"}}, }, out: out{ []DataTable{ { Table: "graphite.data", Context: []string{"unexpected"}, ContextMap: ctx{}, }, }, fmt.Errorf("unknown context \"unexpected\""), }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { cfg := New() if test.in.table.Table != "" { cfg.DataTable = []DataTable{test.in.table} } if test.in.tableLegacy != "" { cfg.ClickHouse.DataTableLegacy = test.in.tableLegacy } err := cfg.ProcessDataTables() if err != nil { assert.Equal(t, test.out.err, err) return } assert.Equal(t, len(test.out.tables), len(cfg.DataTable)) // it's difficult to check rollup.Rollup because Rules.updated field // We explicitly don't check it here for i := range cfg.DataTable { test.out.tables[i].Rollup = nil cfg.DataTable[i].Rollup = nil } assert.Equal(t, test.out.tables, cfg.DataTable) }) } } func TestKnownDataTableContext(t *testing.T) { assert.Equal(t, map[string]bool{ContextGraphite: true, ContextPrometheus: true}, knownDataTableContext) } func TestReadConfig(t *testing.T) { body := []byte( `[common] listen = "[::1]:9090" pprof-listen = "127.0.0.1:9091" max-cpu = 15 max-metrics-in-find-answer = 13 max-metrics-per-target = 16 target-blacklist = ['^blacklisted'] memory-return-interval = "12s150ms" [clickhouse] url = "http://somehost:8123" index-table = "graphite_index" index-use-daily = false index-reverse = "direct" index-reverses = [ {suffix = "suf", prefix = "pref", reverse = "direct"}, {regex = "^reg$", reverse = "reversed"}, ] tagged-table = "graphite_tags" tagged-autocomplete-days = 5 tagged-use-daily = false tree-table = "tree" reverse-tree-table = "reversed_tree" date-tree-table = "data_tree" date-tree-table-version = 2 tag-table = "tag_table" extra-prefix = "tum.pu-dum" data-table = "data" rollup-conf = "none" max-data-points = 8000 internal-aggregation = true data-timeout = "64s" index-timeout = "4s" tree-timeout = "5s" connect-timeout = "2s" # DataTable is tested in TestProcessDataTables # [[data-table]] # table = "another_data" # rollup-conf = "auto" # rollup-conf-table = "another_table" [tags] rules = "filename" date = "2012-12-12" extra-where = "AND case" input-file = "input" output-file = "output" threads = 5 compression = "zstd" version = 42 select-chunks-count = 15 [carbonlink] server = "server:3333" threads-per-request = 5 connect-timeout = "250ms" query-timeout = "350ms" total-timeout = "800ms" [prometheus] listen = ":9092" external-url = "https://server:3456/uri" page-title = "Prometheus Time Series" lookback-delta = "5m" [debug] directory = "tests_tmp" directory-perm = 0o755 external-data-perm = 0o640 [[logging]] logger = "debugger" file = "stdout" level = "debug" encoding = "console" encoding-time = "iso8601" encoding-duration = "string" sample-tick = "5ms" sample-initial = 1 sample-thereafter = 2 [[logging]] logger = "logger" file = "tests_tmp/logger.txt" level = "info" encoding = "json" encoding-time = "epoch" encoding-duration = "seconds" sample-tick = "50ms" sample-initial = 10 sample-thereafter = 12 `, ) config, _, err := Unmarshal(body, false) expected := New() require.NoError(t, err) // Common expected.Common = Common{ Listen: "[::1]:9090", PprofListen: "127.0.0.1:9091", MaxCPU: 15, MaxMetricsInFindAnswer: 13, MaxMetricsPerTarget: 16, TargetBlacklist: []string{"^blacklisted"}, Blacklist: make([]*regexp.Regexp, 1), MemoryReturnInterval: 12150000000, FindCacheConfig: CacheConfig{ Type: "null", DefaultTimeoutSec: 0, ShortTimeoutSec: 0, }, DegragedMultiply: 4.0, DegragedLoad: 1.0, } expected.Metrics = metrics.Config{} r, _ := regexp.Compile(expected.Common.TargetBlacklist[0]) expected.Common.Blacklist[0] = r assert.Equal(t, expected.Common, config.Common) assert.Equal(t, expected.Metrics, config.Metrics) // ClickHouse expected.ClickHouse = ClickHouse{ URL: "http://somehost:8123", DataTimeout: 64000000000, QueryParams: []QueryParam{ { Duration: 0, URL: "http://somehost:8123", DataTimeout: 64000000000, Limiter: limiter.NoopLimiter{}, }, }, ProgressSendingInterval: 10 * time.Second, FindLimiter: limiter.NoopLimiter{}, TagsLimiter: limiter.NoopLimiter{}, IndexTable: "graphite_index", IndexReverse: "direct", IndexReverses: make(IndexReverses, 2), IndexTimeout: 4000000000, TaggedTable: "graphite_tags", TaggedAutocompleDays: 5, TreeTable: "tree", ReverseTreeTable: "reversed_tree", DateTreeTable: "data_tree", DateTreeTableVersion: 2, TreeTimeout: 5000000000, TagTable: "tag_table", ExtraPrefix: "tum.pu-dum", ConnectTimeout: 2000000000, DataTableLegacy: "data", RollupConfLegacy: "none", MaxDataPoints: 8000, InternalAggregation: true, } expected.ClickHouse.IndexReverses[0] = &IndexReverseRule{"suf", "pref", "", nil, "direct"} r, _ = regexp.Compile("^reg$") expected.ClickHouse.IndexReverses[1] = &IndexReverseRule{"", "", "^reg$", r, "reversed"} assert.Equal(t, expected.ClickHouse, config.ClickHouse) // Tags expected.Tags = Tags{"filename", "2012-12-12", "AND case", "input", "output", 5, "zstd", 42, 15} assert.Equal(t, expected.Tags, config.Tags) // Carbonlink expected.Carbonlink = Carbonlink{"server:3333", 5, 2, 250000000, 350000000, 800000000} assert.Equal(t, expected.Carbonlink, config.Carbonlink) // Prometheus expected.Prometheus = Prometheus{Listen: ":9092", ExternalURLRaw: "https://server:3456/uri", PageTitle: "Prometheus Time Series", LookbackDelta: 5 * time.Minute, RemoteReadConcurrencyLimit: 10} u, _ := url.Parse(expected.Prometheus.ExternalURLRaw) expected.Prometheus.ExternalURL = u assert.Equal(t, expected.Prometheus, config.Prometheus) // Debug expected.Debug = Debug{"tests_tmp", os.FileMode(0755), os.FileMode(0640)} assert.Equal(t, expected.Debug, config.Debug) assert.DirExists(t, "tests_tmp") // Logger expected.Logging = make([]zapwriter.Config, 2) expected.Logging[0] = zapwriter.Config{ Logger: "debugger", File: "stdout", Level: "debug", Encoding: "console", EncodingTime: "iso8601", EncodingDuration: "string", SampleTick: "5ms", SampleInitial: 1, SampleThereafter: 2, } expected.Logging[1] = zapwriter.Config{ Logger: "logger", File: "tests_tmp/logger.txt", Level: "info", Encoding: "json", EncodingTime: "epoch", EncodingDuration: "seconds", SampleTick: "50ms", SampleInitial: 10, SampleThereafter: 12, } assert.Equal(t, expected.Logging, config.Logging) } func TestReadConfigGraphiteWithLimiter(t *testing.T) { body := []byte( `[common] listen = "[::1]:9090" pprof-listen = "127.0.0.1:9091" max-cpu = 15 max-metrics-in-find-answer = 13 max-metrics-per-target = 16 target-blacklist = ['^blacklisted'] memory-return-interval = "12s150ms" [metrics] metric-endpoint = "127.0.0.1:2003" metric-interval = "10s" metric-prefix = "graphite" ranges = { "1h" = "1h", "3d" = "72h", "7d" = "168h", "30d" = "720h", "90d" = "2160h" } [clickhouse] url = "http://somehost:8123" index-table = "graphite_index" index-use-daily = false index-reverse = "direct" index-reverses = [ {suffix = "suf", prefix = "pref", reverse = "direct"}, {regex = "^reg$", reverse = "reversed"}, ] tagged-table = "graphite_tags" tagged-autocomplete-days = 5 tagged-use-daily = false tree-table = "tree" reverse-tree-table = "reversed_tree" date-tree-table = "data_tree" date-tree-table-version = 2 tag-table = "tag_table" extra-prefix = "tum.pu-dum" data-table = "data" rollup-conf = "none" max-data-points = 8000 internal-aggregation = true data-timeout = "64s" index-timeout = "4s" tree-timeout = "5s" connect-timeout = "2s" render-max-queries = 1000 render-concurrent-queries = 10 find-max-queries = 200 find-concurrent-queries = 8 tags-max-queries = 50 tags-concurrent-queries = 4 query-params = [ { duration = "72h", url = "http://localhost:8123/?max_rows_to_read=20000" } ] user-limits = { "alert" = { max-queries = 200, concurrent-queries = 10 } } # DataTable is tested in TestProcessDataTables # [[data-table]] # table = "another_data" # rollup-conf = "auto" # rollup-conf-table = "another_table" [tags] rules = "filename" date = "2012-12-12" extra-where = "AND case" input-file = "input" output-file = "output" threads = 5 compression = "zstd" version = 42 select-chunks-count = 15 [carbonlink] server = "server:3333" threads-per-request = 5 connect-timeout = "250ms" query-timeout = "350ms" total-timeout = "800ms" [prometheus] listen = ":9092" external-url = "https://server:3456/uri" page-title = "Prometheus Time Series" lookback-delta = "5m" [debug] directory = "tests_tmp" directory-perm = 0o755 external-data-perm = 0o640 [[logging]] logger = "debugger" file = "stdout" level = "debug" encoding = "console" encoding-time = "iso8601" encoding-duration = "string" sample-tick = "5ms" sample-initial = 1 sample-thereafter = 2 [[logging]] logger = "logger" file = "tests_tmp/logger.txt" level = "info" encoding = "json" encoding-time = "epoch" encoding-duration = "seconds" sample-tick = "50ms" sample-initial = 10 sample-thereafter = 12 `, ) config, _, err := Unmarshal(body, false) expected := New() require.NoError(t, err) assert.NotNil(t, metrics.Graphite) metrics.Graphite = nil // Common expected.Common = Common{ Listen: "[::1]:9090", PprofListen: "127.0.0.1:9091", MaxCPU: 15, MaxMetricsInFindAnswer: 13, MaxMetricsPerTarget: 16, TargetBlacklist: []string{"^blacklisted"}, Blacklist: make([]*regexp.Regexp, 1), MemoryReturnInterval: 12150000000, FindCacheConfig: CacheConfig{ Type: "null", DefaultTimeoutSec: 0, ShortTimeoutSec: 0, }, DegragedMultiply: 4.0, DegragedLoad: 1.0, } expected.Metrics = metrics.Config{ MetricEndpoint: "127.0.0.1:2003", MetricInterval: 10 * time.Second, MetricTimeout: time.Second, MetricPrefix: "graphite", BucketsWidth: []int64{200, 500, 1000, 2000, 3000, 5000, 7000, 10000, 15000, 20000, 25000, 30000, 40000, 50000, 60000}, BucketsLabels: []string{ "_to_200ms", "_to_500ms", "_to_1000ms", "_to_2000ms", "_to_3000ms", "_to_5000ms", "_to_7000ms", "_to_10000ms", "_to_15000ms", "_to_20000ms", "_to_25000ms", "_to_30000ms", "_to_40000ms", "_to_50000ms", "_to_60000ms", "_to_inf", }, // until-from = { "1h" = "1h", "3d" = "72h", "7d" = "168h", "30d" = "720h", "90d" = "2160h" } Ranges: map[string]time.Duration{ "1h": time.Hour, "3d": 72 * time.Hour, "7d": 168 * time.Hour, "30d": 720 * time.Hour, "90d": 2160 * time.Hour, }, RangeNames: []string{"1h", "3d", "7d", "30d", "90d", "history"}, RangeS: []int64{3600, 259200, 604800, 2592000, 7776000, math.MaxInt64}, } r, _ := regexp.Compile(expected.Common.TargetBlacklist[0]) expected.Common.Blacklist[0] = r assert.Equal(t, expected.Common, config.Common) assert.Equal(t, expected.Metrics, config.Metrics) // ClickHouse expected.ClickHouse = ClickHouse{ URL: "http://somehost:8123", DataTimeout: 64000000000, QueryParams: []QueryParam{ { Duration: 0, URL: "http://somehost:8123", DataTimeout: 64000000000, MaxQueries: 1000, ConcurrentQueries: 10, }, { Duration: 72 * time.Hour, URL: "http://localhost:8123/?max_rows_to_read=20000", DataTimeout: 64000000000, Limiter: limiter.NoopLimiter{}, }, }, ProgressSendingInterval: 10 * time.Second, RenderMaxQueries: 1000, RenderConcurrentQueries: 10, FindMaxQueries: 200, FindConcurrentQueries: 8, TagsMaxQueries: 50, TagsConcurrentQueries: 4, UserLimits: map[string]UserLimits{ "alert": { MaxQueries: 200, ConcurrentQueries: 10, }, }, IndexTable: "graphite_index", IndexReverse: "direct", IndexReverses: make(IndexReverses, 2), IndexTimeout: 4000000000, TaggedTable: "graphite_tags", TaggedAutocompleDays: 5, TreeTable: "tree", ReverseTreeTable: "reversed_tree", DateTreeTable: "data_tree", DateTreeTableVersion: 2, TreeTimeout: 5000000000, TagTable: "tag_table", ExtraPrefix: "tum.pu-dum", ConnectTimeout: 2000000000, DataTableLegacy: "data", RollupConfLegacy: "none", MaxDataPoints: 8000, InternalAggregation: true, } expected.ClickHouse.IndexReverses[0] = &IndexReverseRule{"suf", "pref", "", nil, "direct"} r, _ = regexp.Compile("^reg$") expected.ClickHouse.IndexReverses[1] = &IndexReverseRule{"", "", "^reg$", r, "reversed"} for i := range config.ClickHouse.QueryParams { if _, ok := config.ClickHouse.QueryParams[i].Limiter.(*limiter.WLimiter); ok && config.ClickHouse.QueryParams[i].MaxQueries > 0 && config.ClickHouse.QueryParams[i].ConcurrentQueries > 0 { config.ClickHouse.QueryParams[i].Limiter = nil } } if _, ok := config.ClickHouse.FindLimiter.(*limiter.WLimiter); ok && config.ClickHouse.FindMaxQueries > 0 && config.ClickHouse.FindConcurrentQueries > 0 { config.ClickHouse.FindLimiter = nil } if _, ok := config.ClickHouse.TagsLimiter.(*limiter.WLimiter); ok && config.ClickHouse.TagsMaxQueries > 0 && config.ClickHouse.TagsConcurrentQueries > 0 { config.ClickHouse.TagsLimiter = nil } for u, q := range config.ClickHouse.UserLimits { if _, ok := q.Limiter.(*limiter.WLimiter); ok && q.MaxQueries > 0 && q.ConcurrentQueries > 0 { q.Limiter = nil config.ClickHouse.UserLimits[u] = q } } assert.Equal(t, expected.ClickHouse, config.ClickHouse) // Tags expected.Tags = Tags{"filename", "2012-12-12", "AND case", "input", "output", 5, "zstd", 42, 15} assert.Equal(t, expected.Tags, config.Tags) // Carbonlink expected.Carbonlink = Carbonlink{"server:3333", 5, 2, 250000000, 350000000, 800000000} assert.Equal(t, expected.Carbonlink, config.Carbonlink) // Prometheus expected.Prometheus = Prometheus{Listen: ":9092", ExternalURLRaw: "https://server:3456/uri", PageTitle: "Prometheus Time Series", LookbackDelta: 5 * time.Minute, RemoteReadConcurrencyLimit: 10} u, _ := url.Parse(expected.Prometheus.ExternalURLRaw) expected.Prometheus.ExternalURL = u assert.Equal(t, expected.Prometheus, config.Prometheus) // Debug expected.Debug = Debug{"tests_tmp", os.FileMode(0755), os.FileMode(0640)} assert.Equal(t, expected.Debug, config.Debug) assert.DirExists(t, "tests_tmp") // Logger expected.Logging = make([]zapwriter.Config, 2) expected.Logging[0] = zapwriter.Config{ Logger: "debugger", File: "stdout", Level: "debug", Encoding: "console", EncodingTime: "iso8601", EncodingDuration: "string", SampleTick: "5ms", SampleInitial: 1, SampleThereafter: 2, } expected.Logging[1] = zapwriter.Config{ Logger: "logger", File: "tests_tmp/logger.txt", Level: "info", Encoding: "json", EncodingTime: "epoch", EncodingDuration: "seconds", SampleTick: "50ms", SampleInitial: 10, SampleThereafter: 12, } assert.Equal(t, expected.Logging, config.Logging) metrics.FindRequestMetric = nil metrics.TagsRequestMetric = nil metrics.RenderRequestMetric = nil metrics.UnregisterAll() } func TestReadConfigGraphiteWithALimiter(t *testing.T) { body := []byte( `[common] listen = "[::1]:9090" pprof-listen = "127.0.0.1:9091" max-cpu = 15 max-metrics-in-find-answer = 13 max-metrics-per-target = 16 target-blacklist = ['^blacklisted'] memory-return-interval = "12s150ms" [metrics] metric-endpoint = "127.0.0.1:2003" metric-interval = "10s" metric-prefix = "graphite" ranges = { "1h" = "1h", "3d" = "72h", "7d" = "168h", "30d" = "720h", "90d" = "2160h" } [clickhouse] url = "http://somehost:8123" index-table = "graphite_index" index-use-daily = false index-reverse = "direct" index-reverses = [ {suffix = "suf", prefix = "pref", reverse = "direct"}, {regex = "^reg$", reverse = "reversed"}, ] tagged-table = "graphite_tags" tagged-autocomplete-days = 5 tagged-use-daily = false tree-table = "tree" reverse-tree-table = "reversed_tree" date-tree-table = "data_tree" date-tree-table-version = 2 tag-table = "tag_table" extra-prefix = "tum.pu-dum" data-table = "data" rollup-conf = "none" max-data-points = 8000 internal-aggregation = true data-timeout = "64s" index-timeout = "4s" tree-timeout = "5s" connect-timeout = "2s" render-max-queries = 1000 render-concurrent-queries = 10 render-adaptive-queries = 4 find-max-queries = 200 find-concurrent-queries = 8 tags-max-queries = 50 tags-concurrent-queries = 4 tags-adaptive-queries = 3 query-params = [ { duration = "72h", url = "http://localhost:8123/?max_rows_to_read=20000", concurrent-queries = 4, adaptive-queries = 6 } ] user-limits = { "alert" = { max-queries = 200, concurrent-queries = 10, adaptive-queries = 5 } } # DataTable is tested in TestProcessDataTables # [[data-table]] # table = "another_data" # rollup-conf = "auto" # rollup-conf-table = "another_table" [tags] rules = "filename" date = "2012-12-12" extra-where = "AND case" input-file = "input" output-file = "output" threads = 5 compression = "zstd" version = 42 select-chunks-count = 15 [carbonlink] server = "server:3333" threads-per-request = 5 connect-timeout = "250ms" query-timeout = "350ms" total-timeout = "800ms" [prometheus] listen = ":9092" external-url = "https://server:3456/uri" page-title = "Prometheus Time Series" lookback-delta = "5m" [debug] directory = "tests_tmp" directory-perm = 0o755 external-data-perm = 0o640 [[logging]] logger = "debugger" file = "stdout" level = "debug" encoding = "console" encoding-time = "iso8601" encoding-duration = "string" sample-tick = "5ms" sample-initial = 1 sample-thereafter = 2 [[logging]] logger = "logger" file = "tests_tmp/logger.txt" level = "info" encoding = "json" encoding-time = "epoch" encoding-duration = "seconds" sample-tick = "50ms" sample-initial = 10 sample-thereafter = 12 `, ) config, _, err := Unmarshal(body, false) expected := New() require.NoError(t, err) assert.NotNil(t, metrics.Graphite) metrics.Graphite = nil // Common expected.Common = Common{ Listen: "[::1]:9090", PprofListen: "127.0.0.1:9091", MaxCPU: 15, MaxMetricsInFindAnswer: 13, MaxMetricsPerTarget: 16, TargetBlacklist: []string{"^blacklisted"}, Blacklist: make([]*regexp.Regexp, 1), MemoryReturnInterval: 12150000000, FindCacheConfig: CacheConfig{ Type: "null", DefaultTimeoutSec: 0, ShortTimeoutSec: 0, }, DegragedMultiply: 4.0, DegragedLoad: 1.0, } expected.Metrics = metrics.Config{ MetricEndpoint: "127.0.0.1:2003", MetricInterval: 10 * time.Second, MetricTimeout: time.Second, MetricPrefix: "graphite", BucketsWidth: []int64{200, 500, 1000, 2000, 3000, 5000, 7000, 10000, 15000, 20000, 25000, 30000, 40000, 50000, 60000}, BucketsLabels: []string{ "_to_200ms", "_to_500ms", "_to_1000ms", "_to_2000ms", "_to_3000ms", "_to_5000ms", "_to_7000ms", "_to_10000ms", "_to_15000ms", "_to_20000ms", "_to_25000ms", "_to_30000ms", "_to_40000ms", "_to_50000ms", "_to_60000ms", "_to_inf", }, // until-from = { "1h" = "1h", "3d" = "72h", "7d" = "168h", "30d" = "720h", "90d" = "2160h" } Ranges: map[string]time.Duration{ "1h": time.Hour, "3d": 72 * time.Hour, "7d": 168 * time.Hour, "30d": 720 * time.Hour, "90d": 2160 * time.Hour, }, RangeNames: []string{"1h", "3d", "7d", "30d", "90d", "history"}, RangeS: []int64{3600, 259200, 604800, 2592000, 7776000, math.MaxInt64}, } r, _ := regexp.Compile(expected.Common.TargetBlacklist[0]) expected.Common.Blacklist[0] = r assert.Equal(t, expected.Common, config.Common) assert.Equal(t, expected.Metrics, config.Metrics) // ClickHouse expected.ClickHouse = ClickHouse{ URL: "http://somehost:8123", DataTimeout: 64000000000, QueryParams: []QueryParam{ { Duration: 0, URL: "http://somehost:8123", DataTimeout: 64000000000, MaxQueries: 1000, ConcurrentQueries: 10, AdaptiveQueries: 4, }, { Duration: 72 * time.Hour, URL: "http://localhost:8123/?max_rows_to_read=20000", DataTimeout: 64000000000, ConcurrentQueries: 4, AdaptiveQueries: 6, }, }, ProgressSendingInterval: 10 * time.Second, RenderMaxQueries: 1000, RenderConcurrentQueries: 10, RenderAdaptiveQueries: 4, FindMaxQueries: 200, FindConcurrentQueries: 8, TagsMaxQueries: 50, TagsConcurrentQueries: 4, TagsAdaptiveQueries: 3, UserLimits: map[string]UserLimits{ "alert": { MaxQueries: 200, ConcurrentQueries: 10, AdaptiveQueries: 5, }, }, IndexTable: "graphite_index", IndexReverse: "direct", IndexReverses: make(IndexReverses, 2), IndexTimeout: 4000000000, TaggedTable: "graphite_tags", TaggedAutocompleDays: 5, TreeTable: "tree", ReverseTreeTable: "reversed_tree", DateTreeTable: "data_tree", DateTreeTableVersion: 2, TreeTimeout: 5000000000, TagTable: "tag_table", ExtraPrefix: "tum.pu-dum", ConnectTimeout: 2000000000, DataTableLegacy: "data", RollupConfLegacy: "none", MaxDataPoints: 8000, InternalAggregation: true, } expected.ClickHouse.IndexReverses[0] = &IndexReverseRule{"suf", "pref", "", nil, "direct"} r, _ = regexp.Compile("^reg$") expected.ClickHouse.IndexReverses[1] = &IndexReverseRule{"", "", "^reg$", r, "reversed"} for i := range config.ClickHouse.QueryParams { if _, ok := config.ClickHouse.QueryParams[i].Limiter.(*limiter.ALimiter); ok { config.ClickHouse.QueryParams[i].Limiter = nil } } if _, ok := config.ClickHouse.FindLimiter.(*limiter.WLimiter); ok { config.ClickHouse.FindLimiter = nil } if _, ok := config.ClickHouse.TagsLimiter.(*limiter.ALimiter); ok { config.ClickHouse.TagsLimiter = nil } for u, q := range config.ClickHouse.UserLimits { if _, ok := q.Limiter.(*limiter.ALimiter); ok { q.Limiter = nil config.ClickHouse.UserLimits[u] = q } } assert.Equal(t, expected.ClickHouse, config.ClickHouse) // Tags expected.Tags = Tags{"filename", "2012-12-12", "AND case", "input", "output", 5, "zstd", 42, 15} assert.Equal(t, expected.Tags, config.Tags) // Carbonlink expected.Carbonlink = Carbonlink{"server:3333", 5, 2, 250000000, 350000000, 800000000} assert.Equal(t, expected.Carbonlink, config.Carbonlink) // Prometheus expected.Prometheus = Prometheus{Listen: ":9092", ExternalURLRaw: "https://server:3456/uri", PageTitle: "Prometheus Time Series", LookbackDelta: 5 * time.Minute, RemoteReadConcurrencyLimit: 10} u, _ := url.Parse(expected.Prometheus.ExternalURLRaw) expected.Prometheus.ExternalURL = u assert.Equal(t, expected.Prometheus, config.Prometheus) // Debug expected.Debug = Debug{"tests_tmp", os.FileMode(0755), os.FileMode(0640)} assert.Equal(t, expected.Debug, config.Debug) assert.DirExists(t, "tests_tmp") // Logger expected.Logging = make([]zapwriter.Config, 2) expected.Logging[0] = zapwriter.Config{ Logger: "debugger", File: "stdout", Level: "debug", Encoding: "console", EncodingTime: "iso8601", EncodingDuration: "string", SampleTick: "5ms", SampleInitial: 1, SampleThereafter: 2, } expected.Logging[1] = zapwriter.Config{ Logger: "logger", File: "tests_tmp/logger.txt", Level: "info", Encoding: "json", EncodingTime: "epoch", EncodingDuration: "seconds", SampleTick: "50ms", SampleInitial: 10, SampleThereafter: 12, } assert.Equal(t, expected.Logging, config.Logging) metrics.FindRequestMetric = nil metrics.TagsRequestMetric = nil metrics.RenderRequestMetric = nil metrics.UnregisterAll() } func TestGetQueryParamBroken(t *testing.T) { config := []byte(` [clickhouse] url = "http://localhost:8123/?max_rows_to_read=1000" data-timeout = "20s" query-params = [ { duration = "72h", url = "http://localhost:8123/?max_rows_to_read=20000", }, ]`) _, _, err := Unmarshal(config, false) assert.Error(t, err) config = []byte(` [clickhouse] url = "http://localhost:8123/?max_rows_to_read=1000" data-timeout = "20s" query-params = [ { url = "http://localhost:8123/?max_rows_to_read=20000", data-timeout = "60s" }, ]`) _, _, err = Unmarshal(config, false) assert.Error(t, err) } func TestGetQueryParam(t *testing.T) { tests := []struct { name string config []byte durations []time.Duration wantParams []QueryParam wantUserParams map[string]QueryParam }{ { name: "Only default", config: []byte(` [clickhouse] url = "http://localhost:8123/?max_rows_to_read=1000" data-timeout = "20s" `), durations: []time.Duration{ -time.Minute, // only for safety 0, // only for safety time.Minute, }, wantParams: []QueryParam{ { Duration: 0, URL: "http://localhost:8123/?max_rows_to_read=1000", DataTimeout: 20 * time.Second, }, { Duration: 0, URL: "http://localhost:8123/?max_rows_to_read=1000", DataTimeout: 20 * time.Second, }, { Duration: 0, URL: "http://localhost:8123/?max_rows_to_read=1000", DataTimeout: 20 * time.Second, }, }, }, { name: "two params", config: []byte(` [clickhouse] url = "http://localhost:8123/?max_rows_to_read=1000" data-timeout = "20s" query-params = [ { duration = "72h", url = "http://localhost:8123/?max_rows_to_read=20000", data-timeout = "40s" }, ]`), durations: []time.Duration{ -time.Minute, // only for safety 0, // only for safety time.Minute, 72*time.Hour - time.Second, 72 * time.Hour, 2160 * time.Hour, }, wantParams: []QueryParam{ { Duration: 0, URL: "http://localhost:8123/?max_rows_to_read=1000", DataTimeout: 20 * time.Second, }, { Duration: 0, URL: "http://localhost:8123/?max_rows_to_read=1000", DataTimeout: 20 * time.Second, }, { Duration: 0, URL: "http://localhost:8123/?max_rows_to_read=1000", DataTimeout: 20 * time.Second, }, { Duration: 0, URL: "http://localhost:8123/?max_rows_to_read=1000", DataTimeout: 20 * time.Second, }, { Duration: 72 * time.Hour, URL: "http://localhost:8123/?max_rows_to_read=20000", DataTimeout: 40 * time.Second, }, { Duration: 72 * time.Hour, URL: "http://localhost:8123/?max_rows_to_read=20000", DataTimeout: 40 * time.Second, }, }, }, { name: "serveral params", config: []byte(` [clickhouse] url = "http://localhost:8123/?max_rows_to_read=1000" data-timeout = "20s" query-params = [ { duration = "72h", url = "http://localhost:8123/?max_rows_to_read=20000", data-timeout = "40s" }, { duration = "2160h", data-timeout = "60s" } ]`), durations: []time.Duration{ -time.Minute, // only for safety 0, // only for safety time.Minute, 72*time.Hour - time.Second, 72 * time.Hour, 2160 * time.Hour, 4000 * time.Hour, }, wantParams: []QueryParam{ { Duration: 0, URL: "http://localhost:8123/?max_rows_to_read=1000", DataTimeout: 20 * time.Second, }, { Duration: 0, URL: "http://localhost:8123/?max_rows_to_read=1000", DataTimeout: 20 * time.Second, }, { Duration: 0, URL: "http://localhost:8123/?max_rows_to_read=1000", DataTimeout: 20 * time.Second, }, { Duration: 0, URL: "http://localhost:8123/?max_rows_to_read=1000", DataTimeout: 20 * time.Second, }, { Duration: 72 * time.Hour, URL: "http://localhost:8123/?max_rows_to_read=20000", DataTimeout: 40 * time.Second, }, { Duration: 2160 * time.Hour, URL: "http://localhost:8123/?max_rows_to_read=1000", DataTimeout: 60 * time.Second, }, { Duration: 2160 * time.Hour, URL: "http://localhost:8123/?max_rows_to_read=1000", DataTimeout: 60 * time.Second, }, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if config, _, err := Unmarshal(tt.config, false); err == nil { for i := range config.ClickHouse.QueryParams { config.ClickHouse.QueryParams[i].Limiter = nil } for i, duration := range tt.durations { got := GetQueryParam(config.ClickHouse.QueryParams, duration) if config.ClickHouse.QueryParams[got] != tt.wantParams[i] { t.Errorf("[%d] GetQueryParam(%v) = %+v, want %+v", i, duration, config.ClickHouse.QueryParams[got], tt.wantParams[i]) } } } else { t.Errorf("Load config error = %v", err) } }) } } func TestClickHouse_Validate(t *testing.T) { tests := []struct { name string ch ClickHouse wantErr string }{ { name: "url with spaces", ch: ClickHouse{ URL: "http://localhost:8123/?max_rows_to_read=600 &max_threads=2&skip_unavailable_shards=1&log_queries=1", }, wantErr: `space not allowed in url "http://localhost:8123/?max_rows_to_read=600 &max_threads=2&skip_unavailable_shards=1&log_queries=1"`, }, { name: "valid url", ch: ClickHouse{ URL: "http://localhost:8123/?max_rows_to_read=600&max_threads=2&skip_unavailable_shards=1&log_queries=1", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { _, err := clickhouseURLValidate(tt.ch.URL) if err == nil { if tt.wantErr != "" { t.Errorf("ClickHouse.Validate() error = nil, wantErr %q", tt.wantErr) } } else if err.Error() != tt.wantErr { t.Errorf("ClickHouse.Validate() error = %v, wantErr %q", err, tt.wantErr) } }) } } ================================================ FILE: config/json.go ================================================ package config import ( "encoding/json" "net/url" ) func (c *ClickHouse) MarshalJSON() ([]byte, error) { type ClickHouseRaw ClickHouse // make copy a := *c u, err := url.Parse(a.URL) if err != nil { a.URL = "" } else { if _, isSet := u.User.Password(); isSet { u.User = url.UserPassword(u.User.Username(), "xxxxxx") } a.URL = u.String() } return json.Marshal((*ClickHouseRaw)(&a)) } ================================================ FILE: config/json_test.go ================================================ package config import ( "encoding/json" "testing" "github.com/stretchr/testify/assert" ) func TestClickhouseUrlPassword(t *testing.T) { assert := assert.New(t) result := make(map[string]interface{}) c := &ClickHouse{URL: "http://user:qwerty@localhost:8123/?param=value"} b, err := json.Marshal(c) assert.NoError(err) assert.NoError(json.Unmarshal(b, &result)) assert.Equal("http://user:xxxxxx@localhost:8123/?param=value", result["url"].(string)) assert.Equal("http://user:qwerty@localhost:8123/?param=value", c.URL) } ================================================ FILE: deploy/doc/.gitignore ================================================ # autogenerated config for documentation graphite-clickhouse.conf ================================================ FILE: deploy/doc/config.md ================================================ # Configuration ## Common `[common]` ### Finder cache Specify what storage to use for finder cache. This cache stores finder results (metrics find/tags autocomplete/render). Supported cache types: - `mem` - will use integrated in-memory cache. Not distributed. Fast. - `memcache` - will use specified memcache servers. Could be shared. Slow. - `null` - disable cache Extra options: - `size_mb` - specify max size of cache, in MiB - `defaultTimeoutSec` - specify default cache ttl. - `shortTimeoutSec` - cache ttl for short duration intervals of render queries (duration <= shortDuration && now-until <= 61) (if 0, disable this cache) - `findTimeoutSec` - cache ttl for finder/tags autocompleter queries (if 0, disable this cache) - `shortDuration` - maximum duration for render queries, which use shortTimeoutSec duration ### Example ```yaml [common.find-cache] type = "memcache" size_mb = 0 memcachedServers = [ "127.0.0.1:1234", "127.0.0.2:1235" ] defaultTimeoutSec = 10800 shortTimeoutSec = 300 findTimeoutSec = 600 ``` ## Feature flags `[feature-flags]` `use-carbon-behaviour=true`. - Tagged terms with `=` operator and empty value (e.g. `t=`) match all metrics that don't have that tag. `dont-match-missing-tags=true`. - Tagged terms with `!=`, `!=~` operators only match metrics that have that tag. ### Examples Given tagged metrics: ``` metric.two;env=prod metric.one;env=stage;dc=mydc1 metric.one;env=prod;dc=otherdc1 ``` | Target | use-carbon-behaviour | Matched metrics | |-----------------------------|----------------------|---------------------------------------------------| | seriesByTag('dc=') | false | - | | seriesByTag('dc=') | true | metric.two;env=prod | | Target | dont-match-missing-tags | Matched metrics | |--------------------------|-------------------------|--------------------------------------------------------| | seriesByTag('dc!=mydc1') | false | metric.two;env=prod
metric.one;env=prod;dc=otherdc1 | | seriesByTag('dc!=mydc1') | true | metric.one;env=prod;dc=otherdc1 | | seriesByTag('dc!=~otherdc') | false | metric.two;env=prod
metric.one;env=stage;dc=mydc1 | | seriesByTag('dc!=~otherdc') | true | metric.one;env=stage;dc=mydc1 | ## ClickHouse `[clickhouse]` ### URL `url` Detailed explanation of ClickHouse HTTP interface is given in [documentation](https://clickhouse.tech/docs/en/interfaces/http). It's recommended to create a dedicated read-only user for graphite-clickhouse. Example: `url = "http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1"` Some useful parameters: - [log_queries=1](https://clickhouse.tech/docs/en/operations/settings/settings/#settings-log-queries): all queries will be logged in the `system.query_log` table. Useful for debug. - [readonly=2](https://clickhouse.tech/docs/en/operations/settings/permissions-for-queries/#settings_readonly): do not change data on the server - [max_rows_to_read=200000000](https://clickhouse.tech/docs/en/operations/settings/query-complexity/#max-rows-to-read): useful if you want to prevent too broad requests - [cancel_http_readonly_queries_on_client_close=1](https://clickhouse.tech/docs/en/operations/settings/settings/#cancel-http-readonly-queries-on-client-close): cancel DB query when request is canceled. All these and more settings can be set in clickhouse-server configuration as user's profile settings. Useless settings: - `max_query_size`: at the moment [external data](https://clickhouse.tech/docs/en/engines/table-engines/special/external-data/) is used, the query length is relatively small and always less than the default [262144](https://clickhouse.tech/docs/en/operations/settings/settings/#settings-max_query_size) - `max_ast_elements`: the same - `max_execution_time`: with `cancel_http_readonly_queries_on_client_close=1` and `data-timeout = "1m"` it's already covered. ### Query multi parameters (for overwrite default url and data-timeout) For queries with duration (until - from) >= 72 hours, use custom url and data-timeout ``` url = "http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=102400000&max_result_bytes=12800000&max_threads=2" data-timeout = "30s" query-params = [ { duration = "72h", url = "http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=1024000000&max_result_bytes=128000000&max_threads=1", data-timeout = "60s" } ] ``` ### Query limiter for prevent database overloading (limit concurrent/maximum incomming requests) For prevent database overloading incomming requests (render/find/autocomplete) can be limited. If wait max-queries requests, for new request error returned immediately. If executing concurrent-queries requests, next request will be wait for free slot until index-timeout reached adaptive-queries prevent overload with load average check if graphite-clickhouse run on one host with clickhouse Real queries will be concurrent-queries + adaptive-queries * (1 / normalized_load_avg - 1). If normalized_load_avg > 0.9, limit will be concurrent-queries. ``` url = "http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=102400000&max_result_bytes=12800000&max_threads=2" render-max-queries = 500 render-max-concurrent = 10 find-max-queries = 100 find-concurrent-queries = 10 tags-max-queries = 100 tags-max-concurrent = 10 query-params = [ { duration = "72h", url = "http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=1024000000&max_result_bytes=128000000&max_threads=1", data-timeout = "60s" max-queries = 100, max-concurrent = 4 } ] user-limits = { "alerting" = { max-queries = 100, max-concurrent = 5 } } ``` ### Index table See [index table](./index-table.md) documentation for details. ### Index reversed queries tuning By default the daemon decides to make a direct or reversed request to the [index table](./index-table.md) based on a first and last glob node in the metric. It choose the most long path to reduce readings. Additional examples can be found in [tests](../finder/index_test.go). You can overwrite automatic behavior with `index-reverse`. Valid values are `"auto", direct, "reversed"` If you need fine tuning for different paths, you can use `[[clickhouse.index-reverses]]` to set behavior per metrics' `prefix`, `suffix` or `regexp`. ### Tags table By default, tags are stored in the tagged-table on the daily basis. If a metric set doesn't change much, that leads to situation when the same data stored multiple times. To prevent uncontrolled growth and reduce the amount of data stored in the tagged-table, the `tagged-use-daily` parameter could be set to `false` and table definition could be changed to something like: ``` CREATE TABLE graphite_tagged ( Date Date, Tag1 String, Path String, Tags Array(String), Version UInt32 ) ENGINE = ReplacingMergeTree(Date) ORDER BY (Tag1, Path); ``` For restrict costly seriesByTag (may be like `seriesByTag('name=~test.*.*.rabbitmq_overview.connections')` or `seriesByTag('name=test.*.*.rabbitmq_overview.connections')`) use tags-min-in-query parameter. For restrict costly autocomplete queries use tags-min-in-autocomplete parameter. set for require at minimum 1 eq argument (without wildcards) `tags-min-in-query=1` `ReplacingMergeTree(Date)` prevent broken tags autocomplete with default `ReplacingMergeTree(Version)`, when write to the past. ### ClickHouse aggregation For detailed description of `max-data-points` and `internal-aggregation` see [aggregation documentation](./aggregation.md). ## Data tables `[[data-table]]` ### Rollup The rollup configuration is used for a proper metrics pre-aggregation. It contains two rules types: - retention for point per time range - aggregation function for a values Historically, the way to define the config was `rollup-conf = "/path/to/the/conf/with/graphite_rollup.xml"`. The format is the same as [graphite_rollup](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/graphitemergetree/#rollup-configuration) scheme for ClickHouse server. For a quite long time it's recommended to use `rollup-conf = "auto"` to get the configuration from remote ClickHouse server. It will update itself on each `rollup-auto-interval` (1 minute by default) or once on startup if set to "0s". If you don't use a `GraphiteMergeTree` family engine, you can still use `rollup-conf = "auto"` by setting `rollup-auto-table="graphiteMergeTreeTable"` and get the proper config. In this case `graphiteMergeTreeTable` is a dummy table associated with proper [graphite_rollup](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/graphitemergetree/#rollup-configuration). The cases when you may need it: - ReplacingMergeTree engine - Distributed engine - Materialized view It's possible as well to set `rollup-conf = "none"`. Then values from `rollup-default-precision` and `rollup-default-function` will be used. #### Additional rollup tuning for reversed data tables When `reverse = true` is set for data-table, there are two possibles cases for [graphite_rollup](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/graphitemergetree/#rollup-configuration): - Original regexps are used, like `^level_one.level_two.suffix$` - Reversed regexps are used, like `^suffix.level_two.level_one$` Depends on it for having a proper retention and aggregation you must additionally set `rollup-use-reverted = true` for the first case and `rollup-use-reverted = false` for the second. #### Additional tuning tagged find for seriesByTag and autocomplete Only one tag used as filter for index field Tag1, see graphite_tagged table [structure](https://github.com/lomik/ To always choose the best Tag1 you can set the parameter `tag1-count-table = `. The value should be a table in clickhouse that has columns (Date, Tag1, Count) similar to the graphite_tagged table. The table can be defined like this: ``` CREATE TABLE IF NOT EXISTS default.tag1_count_per_day ( Date Date, Tag1 String, Count UInt64 ) ENGINE = SummingMergeTree ORDER BY (Date, Tag1); CREATE MATERIALIZED VIEW IF NOT EXISTS default.tag1_count_per_day_mv TO default.tag1_count_per_day AS SELECT Date AS Date, Tag1 AS Tag1, count(*) AS Count FROM default.graphite_tags GROUP BY (Date, Tag1); ``` Here we additionally create a materialized view to automatically save the quantities of rows with each unique Tag1 as the metrics are being written. graphite-clickhouse will query this table when it tries to decide which tag should be used when querying graphite_tagged table. Overall using this parameter will somewhat increase writing load but can improve reading tagged metrics greatly in some cases. Note that this option only works for terms with '=' operator in them. Using it will also override tag costs that were set manually with tagged-costs option. ================================================ FILE: deploy/root/usr/lib/systemd/system/graphite-clickhouse.service ================================================ [Unit] Description=Graphite cluster backend with ClickHouse support Documentation=https://github.com/lomik/graphite-clickhouse After=network.target [Service] Type=simple PermissionsStartOnly=true ExecStart=/usr/bin/graphite-clickhouse -config /etc/graphite-clickhouse/graphite-clickhouse.conf Restart=on-failure KillMode=control-group [Install] WantedBy=multi-user.target ================================================ FILE: doc/aggregation.md ================================================ # Enable ClickHouse aggregation The feature was added in [v0.12.0](https://github.com/lomik/graphite-clickhouse/releases/tag/v0.12.0). It's enabled by default since [v0.13.0](https://github.com/lomik/graphite-clickhouse/releases/tag/v0.13.0) ([#157](https://github.com/lomik/graphite-clickhouse/pull/157)). You can disable it be setting `internal-aggregation = false` to use aggregation in graphite-clickhouse. ``` [clickhouse] # ClickHouse-side aggregation internal-aggregation = true # maximum number of points per metric. It should be set to 4096 or less for ClickHouse older than 20.8 # https://github.com/ClickHouse/ClickHouse/commit/d7871f3976e4f066c6efb419db22725f476fd9fa max-data-points = 1048576 ``` The only known _frontend_ supporting passing `maxDataPoints` from requests is [carbonapi>=0.14](https://github.com/go-graphite/carbonapi/releases/tag/0.14.0). Protocol should be set to `carbonapi_v3_pb` for this feature to fully work, see [config](https://github.com/go-graphite/carbonapi/blob/main/doc/configuration.md#upstreams)->backendv2->backends->protocol. But even without mentioned adjustments, `internal-aggregation` improves the whole picture by implementing whisper-like aggregation behavior (see below). ## Compatible ClickHouse versions The feature uses ClickHouse aggregation combinator [-Resample](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/combinators/#agg-functions-combinator-resample). This aggregator is available since version [19.11](https://github.com/ClickHouse/ClickHouse/commit/57db1fac5990a7227e720c9dd438d88a381d298f) *Note*: version 0.12 is compatible only with CH 20.1.13.105, 20.3.10.75, 20.4.5.36, 20.5.2.7 or newer since it uses -OrNull modifier. Generally, it's a good idea to always use the latest [LTS](https://repo.clickhouse.tech/deb/lts/main/) ClickHouse release to have the actual version. ## Upgrade - Upgrade the carbonapi to version 0.14.0 or greater - Upgrade graphite-clickhouse to version 0.12.0 or greater - Set the `backendv2->backends->protocol: carbonapi_v3_pb` in carbonapi *only after graphite-clickhouse is upgraded* - Upgrade ClickHouse - Enable `internal-aggregation` in graphite-clickhouse # Historical remark: schemes and changes overview ## Classic whisper scheme ``` header: xFilesFactor: [0, 1] aggregation: {avg,sum,min,max,...} retention: 1d:1m,1w:5m,1y:1h data: archive1: 1440 points archive2: 2016 points archive3: 8760 points ``` - Retention description: - Stores point/1m for one day - Stores point/5m for one week - Stores point/1h for one year - Older points than any of mentioned age are overwriten by new incoming points - Each archive filled up simultaneously - Aggregation on the fly during writing - `xFilesFactor` controls if points from archive(N) should be aggregated into archive(N+1) - Points are selected only from one archive, with the most precision: - from <= now-1d -> archive1 - from <= now-7d -> archive2 - else -> archive3 ## Historical graphite-clickhouse way ### Storing: GraphiteMergeTree table engine in ClickHouse (CH) Completely another principle of data storage. - Retention scheme looks slightly different: `retention: 0:60,1d:5m,1w:1h,1y:1d` - Stores point/minute, if the age of point is at least 0sec - Stores point/5min, if the age of point is at least one day - Stores point/1h, if the age of point is at least one week - GraphiteMergeTree doesn't drop metrics after some particular age, so after one year we would store it with the minimum possible resolution point/day - Retention and aggregation policies are applied only when point becomes older than X (1d,1w,1y) - There is no such thing as `archive`, each point is stored only once - No `xFilesFactor` entity: each point will be aggregated ### Fetching data: before September 2019 (current `internal-aggregation = false` behavior) ```sql SELECT Path, Time, Value, Timestamp FROM data WHERE ... ``` Logic: - Select all points - Aggregate them on the fly to the proper `archive` step - Pass further to *graphite-web*/*carbonapi* Problems: - A huge overhead for Path (the heaviest part) - Extremely inefficient in terms of network traffic, especially when CH cluster is used - The CH node `query-initiator` must collect the whole data (in memory or on the disk), and only then points will be passed further ### Fetching data: after September 2019 ([#61](https://github.com/lomik/graphite-clickhouse/pull/61), [#62](https://github.com/lomik/graphite-clickhouse/pull/62), [#65](https://github.com/lomik/graphite-clickhouse/pull/65)) (between v0.11.7 and v0.12.0) ```sql SELECT Path, groupArray(Time), groupArray(Value), groupArray(Timestamp) FROM data WHERE ... GROUP BY Path ``` - Up to 6 time less network load - But still selects all points and aggregates in *graphite-clickhouse* ### Fetching data: September 2020 ([#88](https://github.com/lomik/graphite-clickhouse/pull/88)) (v0.12.0) ```sql SELECT Path, arrayFilter(x->isNotNull(x), anyOrNullResample($from, $until, $step) (toUInt32(intDiv(Time, $step)*$step), Time) ), arrayFilter(x->isNotNull(x), ${func}OrNullResample($from, $until, $step) (Value, Time) ) FROM data WHERE ... GROUP BY Path ``` - This solution implements `archive` analog on CH side - Most of the data is aggregated on CH shards and doesn't leave them, so `query-initiator` consumes much less memory - When *carbonapi* with `format=carbonapi_v3_pb` is used, the `/render?maxDataPoints=x` parameter processed on CH side too ### Fetching data: April 2021 ([#145](https://github.com/lomik/graphite-clickhouse/pull/145)) ```sql WITH anyResample($from, $until, $step)(toUInt32(intDiv(Time, $step)*$step), Time) AS mask SELECT Path, arrayFilter(m->m!=0, mask) AS times, arrayFilter((v,m)->m!=0, ${func}Resample($from, $until, $step)(Value, Time), mask) AS values FROM data WHERE ... GROUP BY Path ``` - Query improved a bit: dropped the use of `-OrNull` improved compatibility with different CH versions. ## Fetching data: concepts' difference For small requests, the difference is not so big, but for the heavy one the amount of data was decreased up to 100 times: ``` target=${986_metrics_60s_precision} from=-7d maxDataPoints=100 ``` | method | rows | points | data (binary) | time (s) | | - | - | - | - | - | | row/point | 9887027 | 9887027 | 556378258 (530M) | 16.486 | | groupArray | 986 | 9887027 | 158180388 (150M) | 35.498 | | -Resample | 986 | 98553 | 1421418 (1M) | 13.181 | *note*: it's localhost, so with slow network effect may be even more significant. ### The maxDataPoints processing The classical pipeline: - Fetch the data in *graphite-web*/*carbonapi* - Apply all functions from `target` - Compare the result with `maxDataPoints` URI parameter and adjust them Current: - Get data, aggregated with the proper function directly from CH - Fetch pre-aggregated data with a proper functions from ClickHouse - Apply all functions to the pre-aggregated data ================================================ FILE: doc/config.md ================================================ [//]: # (This file is built out of deploy/doc/config.md, please do not edit it manually) [//]: # (To rebuild it run `make config`) # Configuration ## Common `[common]` ### Finder cache Specify what storage to use for finder cache. This cache stores finder results (metrics find/tags autocomplete/render). Supported cache types: - `mem` - will use integrated in-memory cache. Not distributed. Fast. - `memcache` - will use specified memcache servers. Could be shared. Slow. - `null` - disable cache Extra options: - `size_mb` - specify max size of cache, in MiB - `defaultTimeoutSec` - specify default cache ttl. - `shortTimeoutSec` - cache ttl for short duration intervals of render queries (duration <= shortDuration && now-until <= 61) (if 0, disable this cache) - `findTimeoutSec` - cache ttl for finder/tags autocompleter queries (if 0, disable this cache) - `shortDuration` - maximum duration for render queries, which use shortTimeoutSec duration ### Example ```yaml [common.find-cache] type = "memcache" size_mb = 0 memcachedServers = [ "127.0.0.1:1234", "127.0.0.2:1235" ] defaultTimeoutSec = 10800 shortTimeoutSec = 300 findTimeoutSec = 600 ``` ## Feature flags `[feature-flags]` `use-carbon-behaviour=true`. - Tagged terms with `=` operator and empty value (e.g. `t=`) match all metrics that don't have that tag. `dont-match-missing-tags=true`. - Tagged terms with `!=`, `!=~` operators only match metrics that have that tag. ### Examples Given tagged metrics: ``` metric.two;env=prod metric.one;env=stage;dc=mydc1 metric.one;env=prod;dc=otherdc1 ``` | Target | use-carbon-behaviour | Matched metrics | |-----------------------------|----------------------|---------------------------------------------------| | seriesByTag('dc=') | false | - | | seriesByTag('dc=') | true | metric.two;env=prod | | Target | dont-match-missing-tags | Matched metrics | |--------------------------|-------------------------|--------------------------------------------------------| | seriesByTag('dc!=mydc1') | false | metric.two;env=prod
metric.one;env=prod;dc=otherdc1 | | seriesByTag('dc!=mydc1') | true | metric.one;env=prod;dc=otherdc1 | | seriesByTag('dc!=~otherdc') | false | metric.two;env=prod
metric.one;env=stage;dc=mydc1 | | seriesByTag('dc!=~otherdc') | true | metric.one;env=stage;dc=mydc1 | ## ClickHouse `[clickhouse]` ### URL `url` Detailed explanation of ClickHouse HTTP interface is given in [documentation](https://clickhouse.tech/docs/en/interfaces/http). It's recommended to create a dedicated read-only user for graphite-clickhouse. Example: `url = "http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1"` Some useful parameters: - [log_queries=1](https://clickhouse.tech/docs/en/operations/settings/settings/#settings-log-queries): all queries will be logged in the `system.query_log` table. Useful for debug. - [readonly=2](https://clickhouse.tech/docs/en/operations/settings/permissions-for-queries/#settings_readonly): do not change data on the server - [max_rows_to_read=200000000](https://clickhouse.tech/docs/en/operations/settings/query-complexity/#max-rows-to-read): useful if you want to prevent too broad requests - [cancel_http_readonly_queries_on_client_close=1](https://clickhouse.tech/docs/en/operations/settings/settings/#cancel-http-readonly-queries-on-client-close): cancel DB query when request is canceled. All these and more settings can be set in clickhouse-server configuration as user's profile settings. Useless settings: - `max_query_size`: at the moment [external data](https://clickhouse.tech/docs/en/engines/table-engines/special/external-data/) is used, the query length is relatively small and always less than the default [262144](https://clickhouse.tech/docs/en/operations/settings/settings/#settings-max_query_size) - `max_ast_elements`: the same - `max_execution_time`: with `cancel_http_readonly_queries_on_client_close=1` and `data-timeout = "1m"` it's already covered. ### Query multi parameters (for overwrite default url and data-timeout) For queries with duration (until - from) >= 72 hours, use custom url and data-timeout ``` url = "http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=102400000&max_result_bytes=12800000&max_threads=2" data-timeout = "30s" query-params = [ { duration = "72h", url = "http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=1024000000&max_result_bytes=128000000&max_threads=1", data-timeout = "60s" } ] ``` ### Query limiter for prevent database overloading (limit concurrent/maximum incomming requests) For prevent database overloading incomming requests (render/find/autocomplete) can be limited. If wait max-queries requests, for new request error returned immediately. If executing concurrent-queries requests, next request will be wait for free slot until index-timeout reached adaptive-queries prevent overload with load average check if graphite-clickhouse run on one host with clickhouse Real queries will be concurrent-queries + adaptive-queries * (1 / normalized_load_avg - 1). If normalized_load_avg > 0.9, limit will be concurrent-queries. ``` url = "http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=102400000&max_result_bytes=12800000&max_threads=2" render-max-queries = 500 render-max-concurrent = 10 find-max-queries = 100 find-concurrent-queries = 10 tags-max-queries = 100 tags-max-concurrent = 10 query-params = [ { duration = "72h", url = "http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=1024000000&max_result_bytes=128000000&max_threads=1", data-timeout = "60s" max-queries = 100, max-concurrent = 4 } ] user-limits = { "alerting" = { max-queries = 100, max-concurrent = 5 } } ``` ### Index table See [index table](./index-table.md) documentation for details. ### Index reversed queries tuning By default the daemon decides to make a direct or reversed request to the [index table](./index-table.md) based on a first and last glob node in the metric. It choose the most long path to reduce readings. Additional examples can be found in [tests](../finder/index_test.go). You can overwrite automatic behavior with `index-reverse`. Valid values are `"auto", direct, "reversed"` If you need fine tuning for different paths, you can use `[[clickhouse.index-reverses]]` to set behavior per metrics' `prefix`, `suffix` or `regexp`. ### Tags table By default, tags are stored in the tagged-table on the daily basis. If a metric set doesn't change much, that leads to situation when the same data stored multiple times. To prevent uncontrolled growth and reduce the amount of data stored in the tagged-table, the `tagged-use-daily` parameter could be set to `false` and table definition could be changed to something like: ``` CREATE TABLE graphite_tagged ( Date Date, Tag1 String, Path String, Tags Array(String), Version UInt32 ) ENGINE = ReplacingMergeTree(Date) ORDER BY (Tag1, Path); ``` For restrict costly seriesByTag (may be like `seriesByTag('name=~test.*.*.rabbitmq_overview.connections')` or `seriesByTag('name=test.*.*.rabbitmq_overview.connections')`) use tags-min-in-query parameter. For restrict costly autocomplete queries use tags-min-in-autocomplete parameter. set for require at minimum 1 eq argument (without wildcards) `tags-min-in-query=1` `ReplacingMergeTree(Date)` prevent broken tags autocomplete with default `ReplacingMergeTree(Version)`, when write to the past. ### ClickHouse aggregation For detailed description of `max-data-points` and `internal-aggregation` see [aggregation documentation](./aggregation.md). ## Data tables `[[data-table]]` ### Rollup The rollup configuration is used for a proper metrics pre-aggregation. It contains two rules types: - retention for point per time range - aggregation function for a values Historically, the way to define the config was `rollup-conf = "/path/to/the/conf/with/graphite_rollup.xml"`. The format is the same as [graphite_rollup](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/graphitemergetree/#rollup-configuration) scheme for ClickHouse server. For a quite long time it's recommended to use `rollup-conf = "auto"` to get the configuration from remote ClickHouse server. It will update itself on each `rollup-auto-interval` (1 minute by default) or once on startup if set to "0s". If you don't use a `GraphiteMergeTree` family engine, you can still use `rollup-conf = "auto"` by setting `rollup-auto-table="graphiteMergeTreeTable"` and get the proper config. In this case `graphiteMergeTreeTable` is a dummy table associated with proper [graphite_rollup](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/graphitemergetree/#rollup-configuration). The cases when you may need it: - ReplacingMergeTree engine - Distributed engine - Materialized view It's possible as well to set `rollup-conf = "none"`. Then values from `rollup-default-precision` and `rollup-default-function` will be used. #### Additional rollup tuning for reversed data tables When `reverse = true` is set for data-table, there are two possibles cases for [graphite_rollup](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/graphitemergetree/#rollup-configuration): - Original regexps are used, like `^level_one.level_two.suffix$` - Reversed regexps are used, like `^suffix.level_two.level_one$` Depends on it for having a proper retention and aggregation you must additionally set `rollup-use-reverted = true` for the first case and `rollup-use-reverted = false` for the second. #### Additional tuning tagged find for seriesByTag and autocomplete Only one tag used as filter for index field Tag1, see graphite_tagged table [structure](https://github.com/lomik/ To always choose the best Tag1 you can set the parameter `tag1-count-table = `. The value should be a table in clickhouse that has columns (Date, Tag1, Count) similar to the graphite_tagged table. The table can be defined like this: ``` CREATE TABLE IF NOT EXISTS default.tag1_count_per_day ( Date Date, Tag1 String, Count UInt64 ) ENGINE = SummingMergeTree ORDER BY (Date, Tag1); CREATE MATERIALIZED VIEW IF NOT EXISTS default.tag1_count_per_day_mv TO default.tag1_count_per_day AS SELECT Date AS Date, Tag1 AS Tag1, count(*) AS Count FROM default.graphite_tags GROUP BY (Date, Tag1); ``` Here we additionally create a materialized view to automatically save the quantities of rows with each unique Tag1 as the metrics are being written. graphite-clickhouse will query this table when it tries to decide which tag should be used when querying graphite_tagged table. Overall using this parameter will somewhat increase writing load but can improve reading tagged metrics greatly in some cases. Note that this option only works for terms with '=' operator in them. Using it will also override tag costs that were set manually with tagged-costs option. ```toml [common] # general listener listen = ":9090" # listener to serve /debug/pprof requests. '-pprof' argument overrides it pprof-listen = "" max-cpu = 1 # limit number of results from find query, 0=unlimited max-metrics-in-find-answer = 0 # limit numbers of queried metrics per target in /render requests, 0 or negative = unlimited max-metrics-per-target = 15000 # if true, always return points for all metrics, replacing empty results with list of NaN append-empty-series = false # daemon returns empty response if query matches any of regular expressions # target-blacklist = [] # daemon will return the freed memory to the OS when it>0 memory-return-interval = "0s" # additional request headers to log headers-to-log = [] # service discovery base weight (on idle) base_weight = 0 # service discovery degraded load avg multiplier (if normalized load avg > degraged_load_avg) (default 4.0) degraged-multiply = 4.0 # service discovery normilized load avg degraded point (default 1.0) degraged-load-avg = 1.0 # service discovery type service-discovery-type = 0 # service discovery address (consul) service-discovery = "" # service discovery namespace (graphite by default) service-discovery-ns = "" # service discovery datacenters (first - is primary, in other register as backup) service-discovery-ds = [] # service discovery expire duration for cleanup (minimum is 24h, if enabled) service-discovery-expire = "0s" # find/tags cache config [common.find-cache] # cache type type = "null" # cache size size-mb = 0 # memcached servers memcached-servers = [] # default cache ttl default-timeout = 0 # short-time cache ttl short-timeout = 0 # finder/tags autocompleter cache ttl find-timeout = 0 # maximum diration, used with short_timeout short-duration = "0s" # offset beetween now and until for select short cache timeout short-offset = 0 [feature-flags] # if true, prefers carbon's behaviour on how tags are treated use-carbon-behaviour = false # if true, seriesByTag terms containing '!=' or '!=~' operators will not match metrics that don't have the tag at all dont-match-missing-tags = false # if true, gch will log affected rows count by clickhouse query log-query-progress = false [metrics] # graphite relay address metric-endpoint = "" # statsd server address statsd-endpoint = "" # Extended metrics extended-stat = false # graphite metrics send interval metric-interval = "0s" # graphite metrics send timeout metric-timeout = "0s" # graphite metrics prefix metric-prefix = "" # Request historgram buckets widths request-buckets = [] # Request historgram buckets labels request-labels = [] # Additional separate stats for until-from ranges [metrics.ranges] # Additional separate stats for until-from find ranges [metrics.find-ranges] [clickhouse] # default url, see https://clickhouse.tech/docs/en/interfaces/http. Can be overwritten with query-params url = "http://localhost:8123?cancel_http_readonly_queries_on_client_close=1" # default total timeout to fetch data, can be overwritten with query-params data-timeout = "1m0s" # time interval for ch query progress sending, it's equal to http_headers_progress_interval_ms header progress-sending-interval = "10s" # Max queries to render queiries render-max-queries = 0 # Concurrent queries to render queiries render-concurrent-queries = 0 # Render adaptive queries (based on load average) for increase/decrease concurrent queries render-adaptive-queries = 0 # Max queries for find queries find-max-queries = 0 # Find concurrent queries for find queries find-concurrent-queries = 0 # Find adaptive queries (based on load average) for increase/decrease concurrent queries find-adaptive-queries = 0 # Max queries for tags queries tags-max-queries = 0 # Concurrent queries for tags queries tags-concurrent-queries = 0 # Tags adaptive queries (based on load average) for increase/decrease concurrent queries tags-adaptive-queries = 0 # If a wildcard appears both at the start and the end of a plain query at a distance (in terms of nodes) less than wildcard-min-distance, then it will be discarded. This parameter can be used to discard expensive queries. wildcard-min-distance = 0 # Plain queries like '{first,second}.custom.metric.*' are also a subject to wildcard-min-distance restriction. But can be split into 2 queries: 'first.custom.metric.*', 'second.custom.metric.*'. Note that: only one list will be split; if there are wildcard in query before (after) list then reverse (direct) notation will be preferred; if there are wildcards before and after list, then query will not be split try-split-query = false # Used only if try-split-query is true. Query that contains list will be split if its (list) node index is less or equal to max-node-to-split-index. By default is 0. It is recommended to have this value set to 2 or 3 and increase it very carefully, because 3 or 4 plain nodes without wildcards have good selectivity max-node-to-split-index = 0 # Minimum tags in seriesByTag query tags-min-in-query = 0 # Minimum tags in autocomplete query tags-min-in-autocomplete = 0 # customized query limiter for some users # [clickhouse.user-limits] # Date format (default, utc, both) date-format = "" # see doc/index-table.md index-table = "graphite_index" index-use-daily = true # see doc/config.md index-reverse = "auto" # [[clickhouse.index-reverses]] # rule is used when the target suffix is matched # suffix = "suffix" # same as index-reverse # reverse = "auto" # [[clickhouse.index-reverses]] # rule is used when the target prefix is matched # prefix = "prefix" # same as index-reverse # reverse = "direct" # [[clickhouse.index-reverses]] # rule is used when the target regex is matched # regex = "regex" # same as index-reverse # reverse = "reversed" # total timeout to fetch series list from index index-timeout = "1m0s" # 'tagged' table from carbon-clickhouse, required for seriesByTag tagged-table = "graphite_tagged" # Table that contains the total amounts of each tag-value pair. It is used to avoid usage of high cardinality tag-value pairs when querying TaggedTable. If left empty, basic sorting will be used. See more detailed description in doc/config.md tags-count-table = "" # or how long the daemon will query tags during autocomplete tagged-autocomplete-days = 7 # whether to use date filter when searching for the metrics in the tagged-table tagged-use-daily = true # costs for tags (for tune which tag will be used as primary), by default is 0, increase for costly (with poor selectivity) tags # [clickhouse.tagged-costs] # old index table, DEPRECATED, see description in doc/config.md # tree-table = "" # reverse-tree-table = "" # date-tree-table = "" # date-tree-table-version = 0 # tree-timeout = "0s" # is not recommended to use, https://github.com/lomik/graphite-clickhouse/wiki/TagsRU # tag-table = "" # add extra prefix (directory in graphite) for all metrics, w/o trailing dot extra-prefix = "" # TCP connection timeout connect-timeout = "1s" # will be removed in 0.14 # data-table = "" # rollup-conf = "auto" # max points per metric when internal-aggregation=true max-data-points = 1048576 # ClickHouse-side aggregation, see doc/aggregation.md internal-aggregation = true # mTLS HTTPS configuration for connecting to clickhouse server # [clickhouse.tls] # ca-cert = [] # client-auth = "" # server-name = "" # min-version = "" # max-version = "" # insecure-skip-verify = false # curves = [] # cipher-suites = [] [[data-table]] # data table from carbon-clickhouse table = "graphite_data" # if it stores direct or reversed metrics reverse = false # maximum age stored in the table max-age = "0s" # minimum age stored in the table min-age = "0s" # maximum until-from interval allowed for the table max-interval = "0s" # minimum until-from interval allowed for the table min-interval = "0s" # table allowed only if any metrics in target matches regexp target-match-any = "" # table allowed only if all metrics in target matches regexp target-match-all = "" # custom rollup.xml file for table, 'auto' and 'none' are allowed as well rollup-conf = "auto" # custom table for 'rollup-conf=auto', useful for Distributed or MatView rollup-auto-table = "" # rollup update interval for 'rollup-conf=auto' rollup-auto-interval = "1m0s" # is used when none of rules match rollup-default-precision = 0 # is used when none of rules match rollup-default-function = "" # should be set to true if you don't have reverted regexps in rollup-conf for reversed tables rollup-use-reverted = false # valid values are 'graphite' of 'prometheus' context = [] # is not recommended to use, https://github.com/lomik/graphite-clickhouse/wiki/TagsRU # [tags] # rules = "" # date = "" # extra-where = "" # input-file = "" # output-file = "" # number of threads for uploading tags to clickhouse (1 by default) # threads = 1 # compression method for tags before sending them to clickhouse (i.e. content encoding): gzip (default), none, zstd # compression = "gzip" # fixed tags version for testing purposes (by default the current timestamp is used for each upload) # version = 0 # number of chunks for selecting metrics from clickhouse (10 by default) # select-chunks-count = 0 [carbonlink] server = "" threads-per-request = 10 connect-timeout = "50ms" query-timeout = "50ms" # timeout for querying and parsing response total-timeout = "500ms" [prometheus] # listen addr for prometheus ui and api listen = ":9092" # allows to set URL for redirect manually external-url = "" page-title = "Prometheus Time Series Collection and Processing Server" lookback-delta = "5m0s" # concurrently handled remote read requests remote-read-concurrency-limit = 10 # see doc/debugging.md [debug] # the directory for additional debug output directory = "" # permissions for directory, octal value is set as 0o755 directory-perm = 493 # permissions for directory, octal value is set as 0o640 external-data-perm = 0 [[logging]] # handler name, default empty logger = "" # '/path/to/filename', 'stderr', 'stdout', 'empty' (=='stderr'), 'none' file = "/var/log/graphite-clickhouse/graphite-clickhouse.log" # 'debug', 'info', 'warn', 'error', 'dpanic', 'panic', and 'fatal' level = "info" # 'json' or 'console' encoding = "mixed" # 'millis', 'nanos', 'epoch', 'iso8601' encoding-time = "iso8601" # 'seconds', 'nanos', 'string' encoding-duration = "seconds" # passed to time.ParseDuration sample-tick = "" # first n messages logged per tick sample-initial = 0 # every m-th message logged thereafter per tick sample-thereafter = 0 ``` ================================================ FILE: doc/debugging.md ================================================ # Debug graphite-clickhouse ## General config The `debug` section contains common parameters: ```toml [debug] directory = '/var/log/graphite-clickhouse/debug' # where the additional debug information will be dumped. directory-perm = '0644' # file mode for the directory. It's applied only if directory does not exist. ``` ## Debug queries with external data All queries to the `data-table` tables use external data. It reduces the SQL parsing time and allows to query big number of metrics without generating 100k+ characters SQL query. Unfortunately, it requires some additional effort to reproduce the query in case of problems. In PR [#126](https://github.com/lomik/graphite-clickhouse/pull/126) it's solved. All you need to do is set the additional config parameter in `[debug]` (see `General config` above): ```toml [debug] external-data-perm = '0640' # to not read the metrics by anybody ``` And pass the HTTP header `X-Gch-Debug-External-Data` with any value in the `/render` or Prometheus request. It will produce the external data dump files in the debug directory and generate a `curl` command in the log on INFO level. E.g. `[2021-01-26T09:57:33.548+0100] INFO [render] external-data {"request_id": "7994db164f6eef7f2e4da20c54c089f2", "debug command": "curl -F 'metrics_list=@/tmp/ext-data-debug/ext-metrics_list:7994db164f6eef7f2e4da20c54c089f2.TSV;' 'http://graphite:xxxxx@clickhouse-hostname.tld:8123/?cancel_http_readonly_queries_on_client_close=1&metrics_list_format=TSV&metrics_list_structure=Path+String&query=SELECT+Path%2C%0A%09arrayFilter%28x-%3EisNotNull%28x%29%2C+anyOrNullResample%281611590400%2C+1611594059%2C+60%29%28toUInt32%28intDiv%28Time%2C+60%29%2A60%29%2C+Time%29%29%2C%0A%09arrayFilter%28x-%3EisNotNull%28x%29%2C+avgOrNullResample%281611590400%2C+1611594059%2C+60%29%28Value%2C+Time%29%29%0AFROM+graphite.data%0APREWHERE+Date+%3E%3D%272021-01-25%27+AND+Date+%3C%3D+%272021-01-25%27%0AWHERE+%28Path+in+metrics_list%29+AND+%28Time+%3E%3D+1611590400+AND+Time+%3C%3D+1611594059%29%0AGROUP+BY+Path%0AFORMAT+RowBinary&query_id=7994db164f6eef7f2e4da20c54c089f2%3Adebug'"}` If URL contains user and password, it will be redacted to not expose the credentials. ## Debug render data All supported formats of `/render` handler are binary and may be difficult to debug. Although it's possible. ### format=pickle To get the data in text format you may pipe the output to the following command: `curl 'localhost:9090/render/?format=pickle&target=metric.name&from=1619777413&until=1619778013' | python3 -c 'import pickle, sys; print(pickle.loads(sys.stdin.buffer.read()))'` ### format=protobuf (or format=carbonapi_v2_pb) The format is relatively easy to debug. You should have `protoc` binary installed. It's usually available in `protobuf` package. Then you can run the following command from the root of the repository: `curl 'localhost:9090/render/?format=protobuf&target=metric.name&from=1619777413&until=1619778013' | protoc --decode carbonapi_v2_pb.MultiFetchResponse -Ivendor/ vendor/github.com/go-graphite/protocol/carbonapi_v2_pb/carbonapi_v2_pb.proto` If the repository is not available, there's still a way to run `protoc --decode_raw`, but it's much less readable. ### format=carbonapi_v3_pb The format is the most efficient in the meaning of network traffic and memory. At the same time it is the least debug-able. The request itself is done by sending a POST body with `carbonapi_v3_pb.MultiFetchRequest` protobuf message. So, first one have to generate the request itself, pipe it to curl, and then decode the request. To do it one should run the following command in the root of the repository: ``` echo 'metrics{startTime: 1619777413, stopTime: 1619778013, pathExpression: "metric.name"}' | \ protoc --encode=carbonapi_v3_pb.MultiFetchRequest -Ivendor/ vendor/github.com/go-graphite/protocol/carbonapi_v3_pb/carbonapi_v3_pb.proto | \ curl -XGET --data-binary @- 'localhost:9090/render/?format=carbonapi_v3_pb' | \ protoc --decode=carbonapi_v3_pb.MultiFetchResponse -Ivendor/ vendor/github.com/go-graphite/protocol/carbonapi_v3_pb/carbonapi_v3_pb.proto ``` To make it a little bit easier the JSON format is implemented. ### format=json The format exists only for debugging purpose and enabled by passing a header `X-Gch-Debug-Output: any string`. Here is a general way to debug the data: - Optional: make a request to the frontend (carbonapi) with additional header `X-Gch-Debug-Output: a`. Then in log a similar line will be generated: `INFO [render.pb3parser] v3pb_request {"request_id": "051fe964d78d9f3d33827397df779ba0", "json": "{\"metrics\":[{\"name\":\"metric.name\",\"startTime\":1619777413,\"stopTime\":1619778013,\"pathExpression\":\"metric.name\",\"maxDataPoints\":700}]}"}` - Get the request ID from the responses request, for example: `X-Gch-Request-Id: 051fe964d78d9f3d33827397df779ba0` - In logs either see the JSON body itself for the query ID, or look for `[render.pb3parser] pb3_target` record. - Now to make a request just run: `curl -H 'Content-Type: application/json' -H 'Content-Type: application/json' -d "{\"metrics\":[{\"name\":\"metric.name\",\"startTime\":1619777413,\"stopTime\":1619778013,\"pathExpression\":\"metric.name\",\"maxDataPoints\":700}]}" 'localhost:9090/render/?format=json'` ### Marshal protobuf data with original marshallers Both `carbonapi_v2_pb` and `carbonapi_v3_proto` have the optimized marshallers to convert ClickHouse data points to the protobuf response. But when it's necessary, it's possible to debug if the proper data is produced by passing `X-Gch-Debug-Protobuf: 1` header. ================================================ FILE: doc/graphite_clickhouse.gliffy ================================================ {"contentType":"application/gliffy+json","version":"1.3","stage":{"background":"#FFFFFF","width":700,"height":710,"nodeIndex":146,"autoFit":true,"exportBorder":false,"gridOn":true,"snapToGrid":true,"drawingGuidesOn":true,"pageBreaksOn":false,"printGridOn":false,"printPaper":"LETTER","printShrinkToFit":false,"printPortrait":true,"maxWidth":5000,"maxHeight":5000,"themeData":null,"imageCache":null,"viewportType":"default","fitBB":{"min":{"x":30,"y":10},"max":{"x":700,"y":710}},"printModel":{"pageSize":"Letter","portrait":true,"fitToOnePage":false,"displayPageBreaks":false},"objects":[{"x":362.0,"y":708.0,"rotation":0.0,"id":140,"width":159.0,"height":86.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":136,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":137,"py":0.29289321881345237,"px":1.0}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":121,"py":1.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#666666","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-2.0,-19.213203435596483],[196.5,-19.213203435596483],[196.5,-88.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":262.0,"y":715.0,"rotation":0.0,"id":139,"width":141.0,"height":123.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":135,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":137,"py":0.2928932188134525,"px":1.1102230246251563E-16}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":119,"py":1.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#666666","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-2.0,-26.21320343559637],[-142.0,-26.21320343559637],[-142.0,-125.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":260.0,"y":680.0,"rotation":0.0,"id":137,"width":100.0,"height":30.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":132,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#666666","fillColor":"#FFFFFF","gradient":false,"dashStyle":"2,2","dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.0,"y":0.0,"rotation":0.0,"id":138,"width":96.0,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

grafana

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":122.0,"y":592.0,"rotation":0.0,"id":133,"width":101.0,"height":41.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":131,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":119,"py":1.0,"px":0.7071067811865476}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":123,"py":0.5,"px":0.0}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#666666","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[35.27922061357856,-2.0],[35.27922061357856,48.0],[98.0,48.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":521.0,"y":623.0,"rotation":0.0,"id":132,"width":119.0,"height":10.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":130,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":121,"py":0.5,"px":0.0}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":123,"py":0.5,"px":1.0}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#666666","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-31.0,-23.0],[-76.0,-23.0],[-76.0,17.0],[-121.0,17.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":329.0,"y":620.0,"rotation":0.0,"id":131,"width":326.0,"height":160.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":129,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":123,"py":0.0,"px":0.7071067811865476}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":68,"py":1.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#666666","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[18.27922061357856,0.0],[18.27922061357856,-80.0],[235.5,-80.0],[235.5,-160.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":498.0,"y":562.0,"rotation":0.0,"id":130,"width":211.0,"height":99.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":128,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":121,"py":0.0,"px":0.7071067811865476}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":68,"py":1.0,"px":0.7071067811865476}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#666666","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[88.873629022557,18.0],[88.873629022557,-42.0],[118.06958851545028,-42.0],[118.06958851545028,-102.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":423.0,"y":560.0,"rotation":0.0,"id":129,"width":196.0,"height":97.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":127,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":121,"py":0.0,"px":0.2928932188134524}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":46,"py":1.0,"px":0.7071067811865476}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#666666","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[107.126370977443,20.0],[107.126370977443,-40.0],[-196.93041148454967,-40.0],[-196.93041148454967,-100.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":402.0,"y":620.0,"rotation":0.0,"id":128,"width":177.0,"height":162.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":126,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":123,"py":0.0,"px":0.2928932188134524}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":46,"py":1.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#666666","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-129.27922061357856,0.0],[-129.27922061357856,-80.0],[-227.5,-80.0],[-227.5,-160.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":164.0,"y":561.0,"rotation":0.0,"id":125,"width":13.0,"height":99.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":124,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":119,"py":0.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":46,"py":0.9999999999999998,"px":0.29289321881345254}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-41.069588515450334,-10.952305351726068],[-41.069588515450334,-40.96820356781734],[-41.06958851545032,-70.98410178390867],[-41.06958851545032,-101.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":30.0,"y":550.0,"rotation":0.0,"id":119,"width":180.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":115,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#333333","fillColor":"#e6b8af","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":3.6,"y":0.0,"rotation":0.0,"id":120,"width":172.79999999999998,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

graphite-web

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":463.0,"y":42.0,"rotation":0.0,"id":118,"width":188.0,"height":36.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":114,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":112,"py":1.0,"px":0.7071067811865476}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":98,"py":0.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#666666","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":1,"startArrowRotation":53.66972977941759,"endArrowRotation":63.66014952022448,"interpolationType":"quadratic","cornerRadius":null,"controlPath":[[-43.29437251522859,-2.0],[-43.29437251522859,18.0],[97.0,18.0],[97.0,38.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":360.0,"y":43.0,"rotation":0.0,"id":116,"width":193.0,"height":37.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":113,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":112,"py":0.9999999999999998,"px":0.29289321881345254}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":41,"py":0.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#666666","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":1,"startArrowRotation":127.52419086511517,"endArrowRotation":117.54599022518734,"interpolationType":"quadratic","cornerRadius":null,"controlPath":[[-39.70562748477141,-3.000000000000007],[-39.70562748477141,17.0],[-190.0,17.0],[-190.0,37.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":250.0,"y":10.0,"rotation":0.0,"id":112,"width":240.0,"height":30.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":110,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#666666","fillColor":"#ffffff","gradient":false,"dashStyle":"2,2","dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.0,"y":0.0,"rotation":0.0,"id":113,"width":236.0,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

Balancer / Mirroring

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":298.0,"y":300.0,"rotation":0.0,"id":108,"width":140.0,"height":50.0,"uid":"com.gliffy.shape.basic.basic_v1.default.double_arrow","order":107,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.double_arrow.basic_v1","strokeWidth":1.0,"strokeColor":"#666666","fillColor":"#FFFFFF","gradient":false,"dashStyle":"2,2","dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":1.4,"y":0.0,"rotation":0.0,"id":110,"width":137.20000000000002,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

Apache ZooKeeper

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":490.0,"y":580.0,"rotation":0.0,"id":121,"width":137.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":118,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#666666","fillColor":"#FFFFFF","gradient":false,"dashStyle":"2,2","dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.74,"y":0.0,"rotation":0.0,"id":122,"width":131.51999999999998,"height":14.0,"uid":null,"order":120,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

carbonapi

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":220.0,"y":620.0,"rotation":0.0,"id":123,"width":180.0,"height":40.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":121,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#666666","fillColor":"#FFFFFF","gradient":false,"dashStyle":"2,2","dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":3.6,"y":0.0,"rotation":0.0,"id":124,"width":172.79999999999998,"height":14.0,"uid":null,"order":123,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

carbonzipper

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":174.0,"y":571.0,"rotation":0.0,"id":127,"width":13.0,"height":99.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":125,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":119,"py":0.0,"px":0.7071067811865476}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":68,"py":0.9999999999999998,"px":0.29289321881345254}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#666666","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-16.72077938642144,-21.0],[-16.72077938642144,-66.0],[338.9304114845497,-66.0],[338.9304114845497,-111.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":169.0,"y":243.0,"rotation":0.0,"id":39,"width":55.0,"height":77.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":43,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":19,"py":1.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":31,"py":0.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[1.0,-3.0],[1.0,39.5],[-59.00000000000003,39.5],[-59.00000000000003,82.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":219.0,"y":242.0,"rotation":0.0,"id":40,"width":32.0,"height":86.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":44,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":19,"py":1.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":35,"py":0.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-49.0,-2.0],[-49.0,40.5],[11.0,40.5],[11.0,83.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":50.0,"y":290.0,"rotation":0.0,"id":45,"width":245.0,"height":70.0,"uid":"com.gliffy.shape.basic.basic_v1.default.group","order":46,"lockAspectRatio":false,"lockShape":false,"children":[{"x":46.0,"y":10.0,"rotation":0.0,"id":38,"width":150.0,"height":14.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":42,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

ClickHouse

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":130.0,"y":35.0,"rotation":0.0,"id":35,"width":100.0,"height":20.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":37,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#000000","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":1.0,"y":0.0,"rotation":0.0,"id":36,"width":97.99999999999997,"height":14.0,"uid":null,"order":40,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

graphite_tree

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":14.999999999999972,"y":35.0,"rotation":0.0,"id":31,"width":90.0,"height":20.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":32,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#000000","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":0.9,"y":0.0,"rotation":0.0,"id":32,"width":88.19999999999999,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

graphite

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":0.0,"y":0.0,"rotation":0.0,"id":29,"width":245.0,"height":70.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":30,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#000000","fillColor":"#fff2cc","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":111.0,"y":399.0,"rotation":0.0,"id":56,"width":1.0,"height":53.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":61,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":52,"py":0.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":31,"py":1.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-1.0000000000000426,11.06919393998976],[-1.0000000000000426,-10.620537373340142],[-1.0000000000000284,-32.3102686866701],[-1.0000000000000284,-54.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":106.0,"y":398.0,"rotation":0.0,"id":57,"width":128.0,"height":49.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":62,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":52,"py":0.0,"px":0.7071067811865476}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":35,"py":0.9999999999999998,"px":0.29289321881345254}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[27.710678118654727,12.0],[27.710678118654727,-20.5],[103.28932188134524,-20.5],[103.28932188134524,-53.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":228.0,"y":399.0,"rotation":0.0,"id":58,"width":3.0,"height":49.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":63,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":48,"py":0.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":35,"py":1.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[2.0,11.06919393998976],[2.0,-10.620537373340142],[2.0,-32.3102686866701],[2.0,-54.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":50.0,"y":400.0,"rotation":0.0,"id":59,"width":249.0,"height":60.0,"uid":"com.gliffy.shape.basic.basic_v1.default.group","order":64,"lockAspectRatio":false,"lockShape":false,"children":[{"x":45.0,"y":39.0,"rotation":0.0,"id":55,"width":150.0,"height":14.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":60,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

graphite-clickhouse

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":12.999999999999972,"y":10.0,"rotation":0.0,"id":52,"width":100.0,"height":20.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":55,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#000000","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.5,"y":0.0,"rotation":0.0,"id":53,"width":95.0,"height":14.0,"uid":null,"order":58,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

/render/

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":133.0,"y":10.0,"rotation":0.0,"id":48,"width":100.0,"height":20.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":50,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#000000","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.5,"y":0.0,"rotation":0.0,"id":49,"width":95.0,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

/metrics/find/

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":0.0,"y":0.0,"rotation":0.0,"id":46,"width":249.0,"height":60.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":48,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#000000","fillColor":"#d9d2e9","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":30.0,"y":80.0,"rotation":0.0,"id":41,"width":280.0,"height":170.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":0,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#000000","fillColor":"#d0e0e3","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":105.0,"y":180.0,"rotation":0.0,"id":11,"width":130.0,"height":20.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":18,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#000000","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":1.625,"y":0.0,"rotation":0.0,"id":12,"width":126.75,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

disk buffer

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":50.0,"y":220.0,"rotation":0.0,"id":19,"width":240.0,"height":20.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":24,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#000000","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.0,"y":0.0,"rotation":0.0,"id":20,"width":236.0,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

Batch upload to clickhouse

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":170.0,"y":148.0,"rotation":0.0,"id":23,"width":5.0,"height":32.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":27,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":13,"py":1.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":11,"py":0.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[0.0,9.0],[0.0,16.666666666666657],[0.0,24.333333333333343],[0.0,32.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":170.0,"y":200.0,"rotation":0.0,"id":26,"width":1.0,"height":19.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":28,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":11,"py":1.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":19,"py":0.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[0.0,0.0],[0.0,6.666666666666657],[0.0,13.333333333333343],[0.0,20.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":97.0,"y":83.0,"rotation":0.0,"id":42,"width":150.0,"height":14.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":45,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

carbon-clickhouse

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":420.0,"y":80.0,"rotation":0.0,"id":98,"width":280.0,"height":170.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":65,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#666666","fillColor":"#ffffff","gradient":false,"dashStyle":"2,2","dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":440.0,"y":104.0,"rotation":0.0,"id":95,"width":240.0,"height":53.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":66,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#666666","fillColor":"#ffffff","gradient":false,"dashStyle":"2,2","dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":460.0,"y":130.0,"rotation":0.0,"id":93,"width":93.00000000000001,"height":19.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":67,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#000000","fillColor":"#ffffff","gradient":false,"dashStyle":"2,2","dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":3.795918367346939,"y":0.0,"rotation":0.0,"id":94,"width":85.40816326530613,"height":14.0,"uid":null,"order":69,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

UDP

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":569.0,"y":130.0,"rotation":0.0,"id":89,"width":94.0,"height":20.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":73,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#000000","fillColor":"#ffffff","gradient":false,"dashStyle":"2,2","dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":3.1333333333333337,"y":0.0,"rotation":0.0,"id":90,"width":87.73333333333336,"height":14.0,"uid":null,"order":75,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

TCP

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":495.0,"y":180.0,"rotation":0.0,"id":96,"width":130.0,"height":20.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":76,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#666666","fillColor":"#ffffff","gradient":false,"dashStyle":"2,2","dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":1.625,"y":0.0,"rotation":0.0,"id":97,"width":126.75,"height":14.0,"uid":null,"order":78,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

disk buffer

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":487.0,"y":107.5,"rotation":0.0,"id":88,"width":150.0,"height":14.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":79,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

Receiver

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":440.0,"y":220.0,"rotation":0.0,"id":85,"width":240.0,"height":20.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":80,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#666666","fillColor":"#ffffff","gradient":false,"dashStyle":"2,2","dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.0,"y":0.0,"rotation":0.0,"id":86,"width":236.0,"height":14.0,"uid":null,"order":82,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

Batch upload to clickhouse

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":560.0,"y":148.0,"rotation":0.0,"id":83,"width":5.0,"height":32.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":83,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":95,"py":1.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":96,"py":0.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#666666","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":null,"controlPath":[[0.0,9.0],[0.0,32.0]],"lockSegments":{},"ortho":false}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":560.0,"y":200.0,"rotation":0.0,"id":81,"width":1.0,"height":19.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":84,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":96,"py":1.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":85,"py":0.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#666666","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[0.0,0.0],[0.0,6.666666666666657],[0.0,13.333333333333343],[0.0,20.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":440.0,"y":290.0,"rotation":0.0,"id":78,"width":245.0,"height":70.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":85,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#666666","fillColor":"#ffffff","gradient":false,"dashStyle":"2,2","dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":455.0,"y":325.0,"rotation":0.0,"id":76,"width":90.0,"height":20.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":86,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#666666","fillColor":"#ffffff","gradient":false,"dashStyle":"2,2","dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":0.9,"y":0.0,"rotation":0.0,"id":77,"width":88.19999999999999,"height":14.0,"uid":null,"order":88,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

graphite

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":570.0,"y":325.0,"rotation":0.0,"id":74,"width":100.0,"height":20.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":89,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#666666","fillColor":"#ffffff","gradient":false,"dashStyle":"2,2","dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":1.0,"y":0.0,"rotation":0.0,"id":75,"width":97.99999999999997,"height":14.0,"uid":null,"order":91,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

graphite_tree

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":486.0,"y":300.0,"rotation":0.0,"id":73,"width":150.0,"height":14.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":92,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

ClickHouse

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":559.0,"y":243.0,"rotation":0.0,"id":100,"width":55.0,"height":77.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":93,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":85,"py":1.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":76,"py":0.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#666666","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[1.0,-3.0],[1.0,39.5],[-59.0,39.5],[-59.0,82.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":609.0,"y":242.0,"rotation":0.0,"id":99,"width":32.0,"height":86.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":94,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":85,"py":1.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":74,"py":0.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#666666","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-49.0,-2.0],[-49.0,40.5],[11.0,40.5],[11.0,83.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":487.0,"y":83.0,"rotation":0.0,"id":80,"width":150.0,"height":14.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":95,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

carbon-clickhouse

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":440.0,"y":400.0,"rotation":0.0,"id":68,"width":249.0,"height":60.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":96,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#666666","fillColor":"#ffffff","gradient":false,"dashStyle":"2,2","dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":573.0,"y":410.0,"rotation":0.0,"id":66,"width":100.0,"height":20.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":97,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#666666","fillColor":"#ffffff","gradient":false,"dashStyle":"2,2","dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.5,"y":0.0,"rotation":0.0,"id":67,"width":95.0,"height":14.0,"uid":null,"order":99,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

/metrics/find/

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":453.0,"y":410.0,"rotation":0.0,"id":64,"width":100.0,"height":20.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":100,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#666666","fillColor":"#ffffff","gradient":false,"dashStyle":"2,2","dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":2.5,"y":0.0,"rotation":0.0,"id":65,"width":95.0,"height":14.0,"uid":null,"order":102,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

/render/

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":485.0,"y":439.0,"rotation":0.0,"id":63,"width":150.0,"height":14.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":103,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

graphite-clickhouse

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":501.0,"y":399.0,"rotation":0.0,"id":71,"width":1.0,"height":53.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":104,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":64,"py":0.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":76,"py":1.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#666666","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[-1.0,11.06919393998976],[-1.0,-10.620537373340142],[-1.0,-32.3102686866701],[-1.0,-54.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":496.0,"y":398.0,"rotation":0.0,"id":70,"width":128.0,"height":49.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":105,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":64,"py":0.0,"px":0.7071067811865476}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":74,"py":0.9999999999999998,"px":0.29289321881345254}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#666666","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[27.710678118654755,12.0],[27.710678118654755,-20.5],[103.28932188134524,-20.5],[103.28932188134524,-53.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":618.0,"y":399.0,"rotation":0.0,"id":69,"width":3.0,"height":49.0,"uid":"com.gliffy.shape.basic.basic_v1.default.line","order":106,"lockAspectRatio":false,"lockShape":false,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":66,"py":0.0,"px":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":74,"py":1.0,"px":0.5}}},"graphic":{"type":"Line","Line":{"strokeWidth":2.0,"strokeColor":"#666666","fillColor":"none","dashStyle":"1.0,1.0","startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","interpolationType":"linear","cornerRadius":10.0,"controlPath":[[2.0,11.06919393998976],[2.0,-10.620537373340142],[2.0,-32.3102686866701],[2.0,-54.0]],"lockSegments":{},"ortho":true}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":50.0,"y":104.0,"rotation":0.0,"id":13,"width":240.0,"height":53.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":2,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#000000","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":70.0,"y":130.0,"rotation":0.0,"id":0,"width":93.0,"height":19.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":4,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#000000","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":3.795918367346938,"y":0.0,"rotation":0.0,"id":1,"width":85.4081632653061,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

UDP

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":180.0,"y":130.0,"rotation":0.0,"id":9,"width":90.0,"height":20.0,"uid":"com.gliffy.shape.basic.basic_v1.default.rectangle","order":14,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1.0,"strokeColor":"#000000","fillColor":"#ffffff","gradient":false,"dashStyle":null,"dropShadow":false,"state":0,"opacity":1.0,"shadowX":0.0,"shadowY":0.0}},"linkMap":[],"children":[{"x":3.0,"y":0.0,"rotation":0.0,"id":10,"width":84.00000000000003,"height":14.0,"uid":null,"order":"auto","lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":8,"paddingRight":8,"paddingBottom":8,"paddingLeft":8,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

TCP

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"hidden":false,"layerId":"dv8wPdfFedCX"},{"x":97.0,"y":107.5,"rotation":0.0,"id":16,"width":150.0,"height":14.0,"uid":"com.gliffy.shape.basic.basic_v1.default.text","order":22,"lockAspectRatio":false,"lockShape":false,"graphic":{"type":"Text","Text":{"overflow":"none","paddingTop":2,"paddingRight":2,"paddingBottom":2,"paddingLeft":2,"outerPaddingTop":6,"outerPaddingRight":6,"outerPaddingBottom":2,"outerPaddingLeft":6,"type":"fixed","lineTValue":null,"linePerpValue":null,"cardinalityType":null,"html":"

Receiver

","tid":null,"valign":"middle","vposition":"none","hposition":"none"}},"linkMap":[],"children":[],"hidden":false,"layerId":"dv8wPdfFedCX"}],"layers":[{"guid":"dv8wPdfFedCX","order":0,"name":"Layer 0","active":true,"locked":false,"visible":true,"nodeIndex":142}],"shapeStyles":{},"lineStyles":{"global":{"stroke":"#666666","strokeWidth":2,"dashStyle":"1.0,1.0","endArrow":1,"orthoMode":1}},"textStyles":{"global":{"color":"#666666"}}},"metadata":{"title":"untitled","revision":0,"exportBorder":false,"loadPosition":"default","libraries":["com.gliffy.libraries.basic.basic_v1.default","com.gliffy.libraries.flowchart.flowchart_v1.default","com.gliffy.libraries.swimlanes.swimlanes_v1.default","com.gliffy.libraries.uml.uml_v1.default","com.gliffy.libraries.erd.erd_v1.default","com.gliffy.libraries.ui.ui_v2.forms_components","com.gliffy.libraries.network.network_v3.home","com.gliffy.libraries.images"],"autosaveDisabled":false,"lastSerialized":1481700752646,"analyticsProduct":"Online","editorVersion":"2.16.2"},"embeddedResources":{"index":0,"resources":[]}} ================================================ FILE: doc/index-table.md ================================================ # Index table The `index` type table is used to look up metrics that [match the query](https://graphite.readthedocs.io/en/latest/render_api.html#paths-and-wildcards). ```sql CREATE TABLE graphite_index ( Date Date, Level UInt32, Path String, Version UInt32 ) ENGINE = ReplacingMergeTree(Version) PARTITION BY toYYYYMM(Date) ORDER BY (Level, Path, Date); ``` Where: * `Date` - date from received point. Or constant date for full metric list (`1970-02-12` (`toDate(42)`) by default) * `Level` - metrics depth, see description below * `Version` - unix timestamp when the last point was received, only the last one is stored in ReplacingMergeTree table engine Each metric creates multiple entries in a table: * daily with direct Path and plain level * daily with reversed Path and Level = 10000+OriginalLevel * records with constant Date and Level = 20000+OriginalLevel for metric itself and all it parents * record with constant Date, reversed Path and Level = 30000+OriginalLevel For example, getting the metric `lorem.ipsum.dolor.sit.amet` adds the following entries to the table: | Date | Level | Path | Version | | ------------- | ------| -------------------------- | ---------- | | 2019-05-14 | 5 | lorem.ipsum.dolor.sit.amet | 1557827619 | | 2019-05-14 | 10005 | amet.sit.dolor.ipsum.lorem | 1557827619 | | 1970-02-12 | 20001 | lorem. | 1557827619 | | 1970-02-12 | 20002 | lorem.ipsum. | 1557827619 | | 1970-02-12 | 20003 | lorem.ipsum.dolor. | 1557827619 | | 1970-02-12 | 20004 | lorem.ipsum.dolor.sit. | 1557827619 | | 1970-02-12 | 20005 | lorem.ipsum.dolor.sit.amet | 1557827619 | | 1970-02-12 | 30005 | amet.sit.dolor.ipsum.lorem | 1557827619 | If you'd like to use only fixed date for index, `index-use-daily = false` can be set in `[clickhouse]` configuration. To prevent continuous growing up of index table, parameter `disable-daily-index = false` should be set in carbon-clickhouse. ### Migrate `tree` table ```sql -- direct Path and parents INSERT INTO graphite_index (Date, Level, Path, Version) SELECT '1970-02-12', Level+20000, Path, Version FROM graphite_tree; -- reversed Path without parents INSERT INTO graphite_index (Date, Level, Path, Version) SELECT '1970-02-12', Level+30000, arrayStringConcat(arrayMap(x->reverse(x), splitByChar('.', reverse(Path))), '.'), Version FROM graphite_tree WHERE NOT Path LIKE '%.'; ``` ### Migrate `series` table ```sql -- direct Path INSERT INTO graphite_index (Date, Level, Path, Version) SELECT Date, Level, Path, Version FROM graphite_series; -- reverse Path INSERT INTO graphite_index (Date, Level, Path, Version) SELECT Date, Level+10000, arrayStringConcat(arrayMap(x->reverse(x), splitByChar('.', reverse(Path))), '.'), Version FROM graphite_series; ``` ### Migrate `series-reverse` table ```sql -- direct Path INSERT INTO graphite_index (Date, Level, Path, Version) SELECT Date, Level, arrayStringConcat(arrayMap(x->reverse(x), splitByChar('.', reverse(Path))), '.'), Version FROM graphite_series_reverse; -- reverse Path INSERT INTO graphite_index (Date, Level, Path, Version) SELECT Date, Level+10000, Path, Version FROM graphite_series_reverse; ``` ================================================ FILE: doc/release.md ================================================ # New release - Update `const Version` in graphite-clickhouse.go ================================================ FILE: find/find.go ================================================ package find import ( "context" "io" "github.com/gogo/protobuf/proto" "github.com/msaf1980/go-stringutils" v2pb "github.com/go-graphite/protocol/carbonapi_v2_pb" v3pb "github.com/go-graphite/protocol/carbonapi_v3_pb" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/finder" "github.com/lomik/graphite-clickhouse/helper/pickle" ) type Find struct { config *config.Config context context.Context query string // original query result finder.Result } func NewCached(config *config.Config, body []byte) *Find { return &Find{ config: config, result: finder.NewCachedIndex(body), } } func New(config *config.Config, ctx context.Context, query string) (*Find, error) { res, err := finder.Find(config, ctx, query, 0, 0) if err != nil { return nil, err } return &Find{ query: query, config: config, context: ctx, result: res, }, nil } func (f *Find) isResultsLimitExceeded(numResults int) bool { return f.config.Common.MaxMetricsInFindAnswer != 0 && numResults >= f.config.Common.MaxMetricsInFindAnswer } func (f *Find) WritePickle(w io.Writer) error { rows := f.result.List() if len(rows) == 0 { // empty w.Write(pickle.EmptyList) return nil } p := pickle.NewWriter(w) p.List() var numResults = 0 for i := 0; i < len(rows); i++ { if len(rows[i]) == 0 { continue } p.Dict() path, isLeaf := finder.Leaf(rows[i]) p.String("metric_path") p.Bytes(path) p.SetItem() p.String("isLeaf") p.Bool(isLeaf) p.SetItem() p.Append() numResults++ if f.isResultsLimitExceeded(numResults) { break } } p.Stop() return nil } func (f *Find) WriteProtobuf(w io.Writer) error { rows := f.result.List() if len(rows) == 0 { // empty return nil } // message GlobMatch { // required string path = 1; // required bool isLeaf = 2; // } // message GlobResponse { // required string name = 1; // repeated GlobMatch matches = 2; // } var response v2pb.GlobResponse response.Name = f.query var numResults = 0 for i := 0; i < len(rows); i++ { if len(rows[i]) == 0 { continue } path, isLeaf := finder.Leaf(rows[i]) response.Matches = append(response.Matches, v2pb.GlobMatch{ Path: string(path), IsLeaf: isLeaf, }) numResults++ if f.isResultsLimitExceeded(numResults) { break } } body, err := proto.Marshal(&response) if err != nil { return err } w.Write(body) return nil } func (f *Find) WriteProtobufV3(w io.Writer) error { rows := f.result.List() if len(rows) == 0 { // empty return nil } // message GlobMatch { // required string path = 1; // required bool isLeaf = 2; // } // message GlobResponse { // required string name = 1; // repeated GlobMatch matches = 2; // } var response v3pb.GlobResponse response.Name = f.query var numResults = 0 for i := 0; i < len(rows); i++ { if len(rows[i]) == 0 { continue } path, isLeaf := finder.Leaf(rows[i]) response.Matches = append(response.Matches, v3pb.GlobMatch{ Path: string(path), IsLeaf: isLeaf, }) numResults++ if f.isResultsLimitExceeded(numResults) { break } } multiGlobResponse := v3pb.MultiGlobResponse{ Metrics: []v3pb.GlobResponse{ response, }, } body, err := proto.Marshal(&multiGlobResponse) if err != nil { return err } w.Write(body) return nil } func (f *Find) WriteJSON(w io.Writer) error { rows := f.result.List() if len(rows) == 0 { // empty return nil } var numResults int var sb stringutils.Builder sb.WriteString("[") for i := 0; i < len(rows); i++ { if len(rows[i]) == 0 { continue } path, isLeaf := finder.Leaf(rows[i]) if numResults == 0 { sb.WriteString("{path=\"") } else { sb.WriteString(",{path=\"") } sb.Write(path) if isLeaf { sb.WriteString("\",leaf=1}") } else { sb.WriteString("\"}") } numResults++ if f.isResultsLimitExceeded(numResults) { break } } sb.WriteString("]\r\n") w.Write(sb.Bytes()) return nil } ================================================ FILE: find/handler.go ================================================ package find import ( "context" "fmt" "io" "net/http" "strconv" "time" "github.com/go-graphite/carbonapi/pkg/parser" v3pb "github.com/go-graphite/protocol/carbonapi_v3_pb" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/helper/utils" "github.com/lomik/graphite-clickhouse/logs" "github.com/lomik/graphite-clickhouse/metrics" "github.com/lomik/graphite-clickhouse/pkg/scope" "go.uber.org/zap" ) type Handler struct { config *config.Config qMetric *metrics.QueryMetrics } func NewHandler(config *config.Config) *Handler { return &Handler{ config: config, qMetric: metrics.InitQueryMetrics("find", &config.Metrics), } } func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { start := time.Now() status := http.StatusOK accessLogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("http") logger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("metrics-find") r = r.WithContext(scope.WithLogger(r.Context(), logger)) var ( metricsCount int64 stat metrics.FinderStat queueFail bool queueDuration time.Duration findCache bool query string ) username := r.Header.Get("X-Forwarded-User") limiter := h.config.GetUserTagsLimiter(username) defer func() { if rec := recover(); rec != nil { status = http.StatusInternalServerError logger.Error("panic during eval:", zap.String("requestID", scope.String(r.Context(), "requestID")), zap.Any("reason", rec), zap.Stack("stack"), ) answer := fmt.Sprintf("%v\nStack trace: %v", rec, zap.Stack("").String) http.Error(w, answer, status) } d := time.Since(start) dMS := d.Milliseconds() logs.AccessLog(accessLogger, h.config, r, status, d, queueDuration, findCache, queueFail) limiter.SendDuration(queueDuration.Milliseconds()) metrics.SendFindMetrics(metrics.FindRequestMetric, status, dMS, 0, h.config.Metrics.ExtendedStat, metricsCount) if stat.ChReadRows > 0 && stat.ChReadBytes > 0 { errored := status != http.StatusOK && status != http.StatusNotFound metrics.SendQueryRead(metrics.FindQMetric, 0, 0, dMS, metricsCount, stat.ReadBytes, stat.ChReadRows, stat.ChReadBytes, errored) } }() r.ParseMultipartForm(1024 * 1024) format := r.FormValue("format") if format == "carbonapi_v3_pb" { body, err := io.ReadAll(r.Body) if err != nil { status = http.StatusBadRequest http.Error(w, fmt.Sprintf("Failed to read request body: %v", err), status) return } var pv3Request v3pb.MultiGlobRequest if err := pv3Request.Unmarshal(body); err != nil { status = http.StatusBadRequest http.Error(w, fmt.Sprintf("Failed to unmarshal request: %v", err), status) return } if len(pv3Request.Metrics) != 1 { status = http.StatusBadRequest http.Error(w, fmt.Sprintf("Multiple metrics in same find request is not supported yet: %v", err), status) return } query = pv3Request.Metrics[0] q := r.URL.Query() q.Set("query", query) r.URL.RawQuery = q.Encode() } else { switch r.FormValue("format") { case "json": case "pickle": case "protobuf": default: logger.Error("unsupported formatter") status = http.StatusBadRequest http.Error(w, "Failed to parse request: unsupported formatter", status) return } query = r.FormValue("query") } if len(query) == 0 { status = http.StatusBadRequest http.Error(w, "Query not set", status) return } var key string // params := []string{query} useCache := h.config.Common.FindCache != nil && h.config.Common.FindCacheConfig.FindTimeoutSec > 0 && !parser.TruthyBool(r.FormValue("noCache")) if useCache { ts := utils.TimestampTruncate(time.Now().Unix(), time.Duration(h.config.Common.FindCacheConfig.FindTimeoutSec)*time.Second) key = "1970-02-12;query=" + query + ";ts=" + strconv.FormatInt(ts, 10) body, err := h.config.Common.FindCache.Get(key) if err == nil { if metrics.FinderCacheMetrics != nil { metrics.FinderCacheMetrics.CacheHits.Add(1) } findCache = true w.Header().Set("X-Cached-Find", strconv.Itoa(int(h.config.Common.FindCacheConfig.FindTimeoutSec))) f := NewCached(h.config, body) metricsCount = int64(len(f.result.List())) logger.Info("finder", zap.String("get_cache", key), zap.Int64("metrics", metricsCount), zap.Bool("find_cached", true), zap.Int32("ttl", h.config.Common.FindCacheConfig.FindTimeoutSec)) h.Reply(w, r, f) return } } var ( entered bool ctx context.Context cancel context.CancelFunc ) if limiter.Enabled() { ctx, cancel = context.WithTimeout(context.Background(), h.config.ClickHouse.IndexTimeout) defer cancel() err := limiter.Enter(ctx, "find") queueDuration = time.Since(start) if err != nil { status = http.StatusServiceUnavailable queueFail = true logger.Error(err.Error()) http.Error(w, err.Error(), status) return } queueDuration = time.Since(start) entered = true defer func() { if entered { limiter.Leave(ctx, "find") entered = false } }() } f, err := New(h.config, r.Context(), query) if entered { // release early as possible limiter.Leave(ctx, "find") entered = false } if err != nil { status, _ = clickhouse.HandleError(w, err) return } if useCache { if body, err := f.result.Bytes(); err == nil { if metrics.FinderCacheMetrics != nil { metrics.FinderCacheMetrics.CacheMisses.Add(1) } h.config.Common.FindCache.Set(key, body, h.config.Common.FindCacheConfig.FindTimeoutSec) logger.Info("finder", zap.String("set_cache", key), zap.Int("metrics", len(f.result.List())), zap.Bool("find_cached", false), zap.Int32("ttl", h.config.Common.FindCacheConfig.FindTimeoutSec)) } } metricsCount = int64(len(f.result.List())) status = h.Reply(w, r, f) } func (h *Handler) Reply(w http.ResponseWriter, r *http.Request, f *Find) (status int) { status = http.StatusOK switch r.FormValue("format") { case "json": f.WriteJSON(w) case "pickle": f.WritePickle(w) case "protobuf": w.Header().Set("Content-Type", "application/x-protobuf") f.WriteProtobuf(w) case "carbonapi_v3_pb": w.Header().Set("Content-Type", "application/x-protobuf") f.WriteProtobufV3(w) default: status = http.StatusInternalServerError http.Error(w, "Failed to parse request: unhandled formatter", status) } return } ================================================ FILE: find/handler_json_test.go ================================================ package find import ( "io" "net/http" "net/http/httptest" "strconv" "testing" "time" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/tests/clickhouse" "github.com/lomik/graphite-clickhouse/metrics" "github.com/stretchr/testify/assert" ) func NewRequest(method, url string, body io.Reader) *http.Request { r, _ := http.NewRequest(method, url, body) return r } type testStruct struct { request *http.Request wantCode int want string wantContent string } func testResponce(t *testing.T, step int, h *Handler, tt *testStruct, wantCachedFind string) { w := httptest.NewRecorder() h.ServeHTTP(w, tt.request) s := w.Body.String() assert.Equalf(t, tt.wantCode, w.Code, "code mismatch step %d\n,%s", step, s) if w.Code == http.StatusOK { if tt.wantContent != "" { contentType := w.Result().Header["Content-Type"] assert.Equalf(t, []string{tt.wantContent}, contentType, "content type mismatch, step %d", step) } cachedFind := w.Result().Header.Get("X-Cached-Find") assert.Equalf(t, cachedFind, wantCachedFind, "cached find mismatch, step %d", step) assert.Equalf(t, tt.want, s, "Step %d", step) } } func TestHandler_ServeValuesJSON(t *testing.T) { metrics.DisableMetrics() srv := clickhouse.NewTestServer() defer srv.Close() cfg, _ := config.DefaultConfig() cfg.ClickHouse.URL = srv.URL h := NewHandler(cfg) srv.AddResponce( "SELECT Path FROM graphite_index WHERE ((Level=20003) AND (Path LIKE 'DB.postgres.%')) AND (Date='1970-02-12') GROUP BY Path FORMAT TabSeparatedRaw", &clickhouse.TestResponse{ Body: []byte("DB.postgres.host1.\nDB.postgres.host2.\n"), }) srv.AddResponce( "SELECT Path FROM graphite_index WHERE ((Level=20005) AND (Path LIKE 'DB.postgres.%' AND match(Path, '^DB[.]postgres[.]([^.]*?)[.]cpu[.]load_avg[.]?$'))) AND (Date='1970-02-12') GROUP BY Path FORMAT TabSeparatedRaw", &clickhouse.TestResponse{ Body: []byte("DB.postgres.host1.cpu.load_avg\nDB.postgres.host2.cpu.load_avg\n"), }) tests := []testStruct{ { request: NewRequest("GET", srv.URL+"/metrics/find/?format=json&query=DB.postgres.%2A", nil), wantCode: http.StatusOK, want: "[{path=\"DB.postgres.host1\"},{path=\"DB.postgres.host2\"}]\r\n", wantContent: "text/plain; charset=utf-8", }, { request: NewRequest("GET", srv.URL+"/metrics/find/?format=json&query=DB.postgres.%2A.cpu.load_avg", nil), wantCode: http.StatusOK, want: "[{path=\"DB.postgres.host1.cpu.load_avg\",leaf=1},{path=\"DB.postgres.host2.cpu.load_avg\",leaf=1}]\r\n", wantContent: "text/plain; charset=utf-8", }, } var queries uint64 for i, tt := range tests { t.Run(tt.request.URL.RawQuery+"#"+strconv.Itoa(i), func(t *testing.T) { for i := 0; i < 2; i++ { testResponce(t, i, h, &tt, "") } assert.Equal(t, uint64(2), srv.Queries()-queries) queries = srv.Queries() }) } } func TestHandler_ServeValuesCachedJSON(t *testing.T) { srv := clickhouse.NewTestServer() defer srv.Close() cfg, _ := config.DefaultConfig() cfg.ClickHouse.URL = srv.URL // find cache config cfg.Common.FindCacheConfig = config.CacheConfig{ Type: "mem", Size: 8192, FindTimeoutSec: 1, } var err error cfg.Common.FindCache, err = config.CreateCache("metric-finder", &cfg.Common.FindCacheConfig) if err != nil { t.Fatalf("Failed to create find cache: %v", err) } h := NewHandler(cfg) srv.AddResponce( "SELECT Path FROM graphite_index WHERE ((Level=20003) AND (Path LIKE 'DB.postgres.%')) AND (Date='1970-02-12') GROUP BY Path FORMAT TabSeparatedRaw", &clickhouse.TestResponse{ Body: []byte("DB.postgres.host1.\nDB.postgres.host2.\n"), }) srv.AddResponce( "SELECT Path FROM graphite_index WHERE ((Level=20005) AND (Path LIKE 'DB.postgres.%' AND match(Path, '^DB[.]postgres[.]([^.]*?)[.]cpu[.]load_avg[.]?$'))) AND (Date='1970-02-12') GROUP BY Path FORMAT TabSeparatedRaw", &clickhouse.TestResponse{ Body: []byte("DB.postgres.host1.cpu.load_avg\nDB.postgres.host2.cpu.load_avg\n"), }) tests := []testStruct{ { request: NewRequest("GET", srv.URL+"/metrics/find/?format=json&query=DB.postgres.%2A", nil), wantCode: http.StatusOK, want: "[{path=\"DB.postgres.host1\"},{path=\"DB.postgres.host2\"}]\r\n", wantContent: "text/plain; charset=utf-8", }, { request: NewRequest("GET", srv.URL+"/metrics/find/?format=json&query=DB.postgres.%2A.cpu.load_avg", nil), wantCode: http.StatusOK, want: "[{path=\"DB.postgres.host1.cpu.load_avg\",leaf=1},{path=\"DB.postgres.host2.cpu.load_avg\",leaf=1}]\r\n", wantContent: "text/plain; charset=utf-8", }, } var queries uint64 for i, tt := range tests { t.Run(tt.request.URL.RawQuery+"#"+strconv.Itoa(i), func(t *testing.T) { testResponce(t, 0, h, &tt, "") assert.Equal(t, uint64(1), srv.Queries()-queries) // query from cache testResponce(t, 1, h, &tt, "1") assert.Equal(t, uint64(1), srv.Queries()-queries) // wait for expire cache time.Sleep(time.Second * 2) testResponce(t, 2, h, &tt, "") assert.Equal(t, uint64(2), srv.Queries()-queries) queries = srv.Queries() }) } } ================================================ FILE: find/handler_test.go ================================================ package find import ( "io" "net/http" "net/http/httptest" "testing" "github.com/lomik/graphite-clickhouse/config" ) type clickhouseMock struct { requestLog chan []byte } func (m *clickhouseMock) ServeHTTP(w http.ResponseWriter, r *http.Request) { body, _ := io.ReadAll(r.Body) if m.requestLog != nil { m.requestLog <- body } } func TestFind(t *testing.T) { testCase := func(findQuery, expectedClickHouseQuery string) { requestLog := make(chan []byte, 1) m := &clickhouseMock{ requestLog: requestLog, } srv := httptest.NewServer(m) defer srv.Close() cfg := config.New() cfg.ClickHouse.URL = srv.URL handler := NewHandler(cfg) w := httptest.NewRecorder() r := httptest.NewRequest( http.MethodGet, "http://localhost/metrics/find/?local=1&format=pickle&query="+findQuery, nil, ) handler.ServeHTTP(w, r) chQuery := <-requestLog if string(chQuery) != expectedClickHouseQuery { t.Fatalf("%#v (actual) != %#v (expected)", string(chQuery), expectedClickHouseQuery) } } testCase( "host.top.cpu.cpu%2A", "SELECT Path FROM graphite_index WHERE ((Level=20004) AND (Path LIKE 'host.top.cpu.cpu%')) AND (Date='1970-02-12') GROUP BY Path FORMAT TabSeparatedRaw", ) testCase( "host.?cpu", "SELECT Path FROM graphite_index WHERE ((Level=20002) AND (Path LIKE 'host.%' AND match(Path, '^host[.][^.]cpu[.]?$'))) AND (Date='1970-02-12') GROUP BY Path FORMAT TabSeparatedRaw", ) } ================================================ FILE: finder/base.go ================================================ package finder import ( "bytes" "context" "errors" "fmt" "strings" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/metrics" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/pkg/where" ) var ErrNotImplemented = errors.New("not implemented") type BaseFinder struct { url string // clickhouse dsn table string // graphite_tree table opts clickhouse.Options // timeout, connectTimeout body []byte // clickhouse response body stats []metrics.FinderStat } func NewBase(url string, table string, opts clickhouse.Options) Finder { return &BaseFinder{ url: url, table: table, opts: opts, stats: make([]metrics.FinderStat, 0), } } func (b *BaseFinder) where(query string) *where.Where { level := strings.Count(query, ".") + 1 w := where.New() w.And(where.Eq("Level", level)) w.And(where.TreeGlob("Path", query)) return w } func (b *BaseFinder) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) (err error) { w := b.where(query) b.stats = append(b.stats, metrics.FinderStat{}) stat := &b.stats[len(b.stats)-1] b.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query( scope.WithTable(ctx, b.table), b.url, // TODO: consider consistent query generator fmt.Sprintf("SELECT Path FROM %s WHERE %s GROUP BY Path FORMAT TabSeparatedRaw", b.table, w), b.opts, nil, ) stat.Table = b.table stat.ReadBytes = int64(len(b.body)) return } func (b *BaseFinder) makeList(onlySeries bool) [][]byte { if b.body == nil { return [][]byte{} } rows := bytes.Split(b.body, []byte{'\n'}) skip := 0 for i := 0; i < len(rows); i++ { if len(rows[i]) == 0 { skip++ continue } if onlySeries && rows[i][len(rows[i])-1] == '.' { skip++ continue } if skip > 0 { rows[i-skip] = rows[i] } } rows = rows[:len(rows)-skip] return rows } func (b *BaseFinder) List() [][]byte { return b.makeList(false) } func (b *BaseFinder) Series() [][]byte { return b.makeList(true) } func (b *BaseFinder) Abs(v []byte) []byte { return v } func (b *BaseFinder) Bytes() ([]byte, error) { return b.body, nil } func (b *BaseFinder) Stats() []metrics.FinderStat { return b.stats } ================================================ FILE: finder/blacklist.go ================================================ package finder import ( "context" "regexp" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/metrics" ) type BlacklistFinder struct { wrapped Finder blacklist []*regexp.Regexp // config matched bool } func WrapBlacklist(f Finder, blacklist []*regexp.Regexp) *BlacklistFinder { return &BlacklistFinder{ wrapped: f, blacklist: blacklist, } } func (p *BlacklistFinder) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) (err error) { for i := 0; i < len(p.blacklist); i++ { if p.blacklist[i].MatchString(query) { p.matched = true return } } return p.wrapped.Execute(ctx, config, query, from, until) } func (p *BlacklistFinder) List() [][]byte { if p.matched { return [][]byte{} } return p.wrapped.List() } // For Render func (p *BlacklistFinder) Series() [][]byte { if p.matched { return [][]byte{} } return p.wrapped.Series() } func (p *BlacklistFinder) Abs(v []byte) []byte { return p.wrapped.Abs(v) } func (p *BlacklistFinder) Bytes() ([]byte, error) { return nil, ErrNotImplemented } func (p *BlacklistFinder) Stats() []metrics.FinderStat { return p.wrapped.Stats() } ================================================ FILE: finder/date.go ================================================ package finder import ( "context" "fmt" "time" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/metrics" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/pkg/where" ) type DateFinder struct { *BaseFinder tableVersion int } func NewDateFinder(url string, table string, tableVersion int, opts clickhouse.Options) Finder { if tableVersion == 3 { return NewDateFinderV3(url, table, opts) } b := &BaseFinder{ url: url, table: table, opts: opts, } return &DateFinder{b, tableVersion} } func (b *DateFinder) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) (err error) { w := b.where(query) dateWhere := where.New() dateWhere.Andf( "Date >='%s' AND Date <= '%s'", time.Unix(from, 0).Format("2006-01-02"), time.Unix(until, 0).Format("2006-01-02"), ) b.stats = append(b.stats, metrics.FinderStat{}) stat := &b.stats[len(b.stats)-1] if b.tableVersion == 2 { b.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query( scope.WithTable(ctx, b.table), b.url, // TODO: consider consistent query generator fmt.Sprintf(`SELECT Path FROM %s PREWHERE (%s) WHERE %s GROUP BY Path FORMAT TabSeparatedRaw`, b.table, dateWhere, w), b.opts, nil, ) } else { b.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query( scope.WithTable(ctx, b.table), b.url, // TODO: consider consistent query generator fmt.Sprintf(`SELECT DISTINCT Path FROM %s PREWHERE (%s) WHERE (%s) FORMAT TabSeparatedRaw`, b.table, dateWhere, w), b.opts, nil, ) } stat.ReadBytes = int64(len(b.body)) stat.Table = b.table return } ================================================ FILE: finder/date_reverse.go ================================================ package finder import ( "context" "fmt" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/helper/date" "github.com/lomik/graphite-clickhouse/metrics" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/pkg/where" ) type DateFinderV3 struct { *BaseFinder } // Same as v2, but reversed func NewDateFinderV3(url string, table string, opts clickhouse.Options) Finder { b := &BaseFinder{ url: url, table: table, opts: opts, } return &DateFinderV3{b} } func (f *DateFinderV3) whereFilter(query string, from int64, until int64) (*where.Where, *where.Where) { w := f.where(ReverseString(query)) dateWhere := where.New() dateWhere.Andf( "Date >='%s' AND Date <= '%s'", date.FromTimestampToDaysFormat(from), date.UntilTimestampToDaysFormat(until), ) return w, dateWhere } func (f *DateFinderV3) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) (err error) { w, dateWhere := f.whereFilter(query, from, until) f.stats = append(f.stats, metrics.FinderStat{}) stat := &f.stats[len(f.stats)-1] f.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query( scope.WithTable(ctx, f.table), f.url, // TODO: consider consistent query generator fmt.Sprintf(`SELECT Path FROM %s WHERE (%s) AND (%s) GROUP BY Path FORMAT TabSeparatedRaw`, f.table, dateWhere, w), f.opts, nil, ) stat.Table = f.table stat.ReadBytes = int64(len(f.body)) return } func (f *DateFinderV3) List() [][]byte { list := f.BaseFinder.List() for i := 0; i < len(list); i++ { list[i] = ReverseBytes(list[i]) } return list } func (f *DateFinderV3) Series() [][]byte { list := f.BaseFinder.Series() for i := 0; i < len(list); i++ { list[i] = ReverseBytes(list[i]) } return list } ================================================ FILE: finder/date_reverse_test.go ================================================ package finder import ( "testing" "time" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/helper/date" ) func TestDateFinderV3_whereFilter(t *testing.T) { tests := []struct { name string query string from int64 until int64 want string wantDate string }{ { name: "midnight at utc (direct)", query: "test.metric*", from: 1668124800, // 2022-11-11 00:00:00 UTC until: 1668124810, // 2022-11-11 00:00:10 UTC want: "(Level=2) AND (Path LIKE 'metric%' AND match(Path, '^metric([^.]*?)[.]test[.]?$'))", wantDate: "Date >='" + date.FromTimestampToDaysFormat(1668124800) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(1668124810) + "'", }, { name: "midnight at utc (reverse)", query: "*test.metric", from: 1668124800, // 2022-11-11 00:00:00 UTC until: 1668124810, // 2022-11-11 00:00:10 UTC want: "(Level=2) AND (Path LIKE 'metric.%' AND match(Path, '^metric[.]([^.]*?)test[.]?$'))", wantDate: "Date >='" + date.FromTimestampToDaysFormat(1668124800) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(1668124810) + "'", }, } for _, tt := range tests { t.Run(tt.name+" "+time.Unix(tt.from, 0).Format(time.RFC3339), func(t *testing.T) { f := NewDateFinderV3("http://localhost:8123/", "graphite_index", clickhouse.Options{}).(*DateFinderV3) got, gotDate := f.whereFilter(tt.query, tt.from, tt.until) if got.String() != tt.want { t.Errorf("DateFinderV3.whereFilter()[0] = %v, want %v", got, tt.want) } if gotDate.String() != tt.wantDate { t.Errorf("DateFinderV3.whereFilter()[1] = %v, want %v", gotDate, tt.wantDate) } }) } } ================================================ FILE: finder/finder.go ================================================ package finder import ( "context" "strings" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/metrics" "github.com/lomik/graphite-clickhouse/config" ) type Result interface { List() [][]byte Series() [][]byte Abs([]byte) []byte Bytes() ([]byte, error) Stats() []metrics.FinderStat } type Finder interface { Result Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) error } func newPlainFinder(ctx context.Context, config *config.Config, query string, from int64, until int64, useCache bool) Finder { opts := clickhouse.Options{ TLSConfig: config.ClickHouse.TLSConfig, Timeout: config.ClickHouse.IndexTimeout, ConnectTimeout: config.ClickHouse.ConnectTimeout, CheckRequestProgress: config.FeatureFlags.LogQueryProgress, ProgressSendingInterval: config.ClickHouse.ProgressSendingInterval, } var f Finder if config.ClickHouse.TaggedTable != "" && strings.HasPrefix(strings.TrimSpace(query), "seriesByTag") { f = NewTagged( config.ClickHouse.URL, config.ClickHouse.TaggedTable, config.ClickHouse.TagsCountTable, config.ClickHouse.TaggedUseDaily, config.FeatureFlags.UseCarbonBehavior, config.FeatureFlags.DontMatchMissingTags, false, opts, config.ClickHouse.TaggedCosts, ) if len(config.Common.Blacklist) > 0 { f = WrapBlacklist(f, config.Common.Blacklist) } return f } if config.ClickHouse.IndexTable != "" { f = NewIndex( config.ClickHouse.URL, config.ClickHouse.IndexTable, config.ClickHouse.IndexUseDaily, config.ClickHouse.IndexReverse, config.ClickHouse.IndexReverses, opts, useCache, ) if config.ClickHouse.TrySplitQuery { f = WrapSplitIndex( f, config.ClickHouse.WildcardMinDistance, config.ClickHouse.URL, config.ClickHouse.IndexTable, config.ClickHouse.IndexUseDaily, config.ClickHouse.IndexReverse, config.ClickHouse.IndexReverses, opts, useCache, ) } } else { if from > 0 && until > 0 && config.ClickHouse.DateTreeTable != "" { f = NewDateFinder(config.ClickHouse.URL, config.ClickHouse.DateTreeTable, config.ClickHouse.DateTreeTableVersion, opts) } else { f = NewBase(config.ClickHouse.URL, config.ClickHouse.TreeTable, opts) } if config.ClickHouse.ReverseTreeTable != "" { f = WrapReverse(f, config.ClickHouse.URL, config.ClickHouse.ReverseTreeTable, opts) } } if config.ClickHouse.TagTable != "" { f = WrapTag(f, config.ClickHouse.URL, config.ClickHouse.TagTable, opts) } if config.ClickHouse.ExtraPrefix != "" { f = WrapPrefix(f, config.ClickHouse.ExtraPrefix) } if len(config.Common.Blacklist) > 0 { f = WrapBlacklist(f, config.Common.Blacklist) } return f } func Find(config *config.Config, ctx context.Context, query string, from int64, until int64) (Result, error) { fnd := newPlainFinder(ctx, config, query, from, until, config.Common.FindCache != nil) err := fnd.Execute(ctx, config, query, from, until) return fnd.(Result), err } // Leaf strips last dot and detect IsLeaf func Leaf(value []byte) ([]byte, bool) { if len(value) > 0 && value[len(value)-1] == '.' { return value[:len(value)-1], false } return value, true } func FindTagged(ctx context.Context, config *config.Config, terms []TaggedTerm, from int64, until int64) (Result, error) { opts := clickhouse.Options{ Timeout: config.ClickHouse.IndexTimeout, ConnectTimeout: config.ClickHouse.ConnectTimeout, TLSConfig: config.ClickHouse.TLSConfig, CheckRequestProgress: config.FeatureFlags.LogQueryProgress, ProgressSendingInterval: config.ClickHouse.ProgressSendingInterval, } useCache := config.Common.FindCache != nil plain := makePlainFromTagged(terms) if plain != nil { plain.wrappedPlain = newPlainFinder(ctx, config, plain.Target(), from, until, useCache) err := plain.Execute(ctx, config, plain.Target(), from, until) if err != nil { return nil, err } return Result(plain), nil } fnd := NewTagged( config.ClickHouse.URL, config.ClickHouse.TaggedTable, config.ClickHouse.TagsCountTable, config.ClickHouse.TaggedUseDaily, config.FeatureFlags.UseCarbonBehavior, config.FeatureFlags.DontMatchMissingTags, true, opts, config.ClickHouse.TaggedCosts, ) err := fnd.ExecutePrepared(ctx, terms, from, until) if err != nil { return nil, err } return Result(fnd), nil } ================================================ FILE: finder/index.go ================================================ package finder import ( "bytes" "context" "fmt" "net/http" "strings" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/helper/date" "github.com/lomik/graphite-clickhouse/helper/errs" "github.com/lomik/graphite-clickhouse/metrics" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/pkg/where" ) const ReverseLevelOffset = 10000 const TreeLevelOffset = 20000 const ReverseTreeLevelOffset = 30000 const DefaultTreeDate = "1970-02-12" const ( queryAuto = config.IndexAuto queryDirect = config.IndexDirect queryReversed = config.IndexReversed ) type IndexFinder struct { url string // clickhouse dsn table string // graphite_tree table opts clickhouse.Options // timeout, connectTimeout dailyEnabled bool confReverse uint8 confReverses config.IndexReverses reverse uint8 // calculated in IndexFinder.useReverse only once body []byte // clickhouse response body rows [][]byte stats []metrics.FinderStat useCache bool // rotate body if needed (for store in cache) useDaily bool } func NewCachedIndex(body []byte) Finder { idx := &IndexFinder{ body: body, reverse: queryDirect, } idx.bodySplit() return idx } func NewIndex(url string, table string, dailyEnabled bool, reverse string, reverses config.IndexReverses, opts clickhouse.Options, useCache bool) Finder { return &IndexFinder{ url: url, table: table, opts: opts, dailyEnabled: dailyEnabled, confReverse: config.IndexReverse[reverse], confReverses: reverses, stats: make([]metrics.FinderStat, 0), useCache: useCache, } } func (idx *IndexFinder) where(query string, levelOffset int) *where.Where { level := strings.Count(query, ".") + 1 w := where.New() w.And(where.Eq("Level", level+levelOffset)) w.And(where.TreeGlob("Path", query)) return w } func (idx *IndexFinder) checkReverses(query string) uint8 { for _, rule := range idx.confReverses { if len(rule.Prefix) > 0 && !strings.HasPrefix(query, rule.Prefix) { continue } if len(rule.Suffix) > 0 && !strings.HasSuffix(query, rule.Suffix) { continue } if rule.Regex != nil && rule.Regex.FindStringIndex(query) == nil { continue } return config.IndexReverse[rule.Reverse] } return idx.confReverse } func (idx *IndexFinder) useReverse(query string) bool { if idx.reverse == queryDirect { return false } else if idx.reverse == queryReversed { return true } if idx.reverse = idx.checkReverses(query); idx.reverse != queryAuto { return idx.useReverse(query) } w := where.IndexWildcard(query) if w == -1 { idx.reverse = queryDirect return idx.useReverse(query) } firstWildcardNode := strings.Count(query[:w], ".") w = where.IndexLastWildcard(query) lastWildcardNode := strings.Count(query[w:], ".") if firstWildcardNode < lastWildcardNode { idx.reverse = queryReversed return idx.useReverse(query) } idx.reverse = queryDirect return idx.useReverse(query) } func useDaily(dailyEnabled bool, from, until int64) bool { return dailyEnabled && from > 0 && until > 0 } func calculateIndexLevelOffset(useDaily, reverse bool) int { var levelOffset int if useDaily { if reverse { levelOffset = ReverseLevelOffset } } else if reverse { levelOffset = ReverseTreeLevelOffset } else { levelOffset = TreeLevelOffset } return levelOffset } func addDatesToWhere(w *where.Where, useDaily bool, from, until int64) { if useDaily { w.Andf( "Date >='%s' AND Date <= '%s'", date.FromTimestampToDaysFormat(from), date.UntilTimestampToDaysFormat(until), ) } else { w.And(where.Eq("Date", DefaultTreeDate)) } } func (idx *IndexFinder) whereFilter(query string, from int64, until int64) *where.Where { reverse := idx.useReverse(query) if reverse { query = ReverseString(query) } idx.useDaily = useDaily(idx.dailyEnabled, from, until) levelOffset := calculateIndexLevelOffset(idx.useDaily, reverse) w := idx.where(query, levelOffset) addDatesToWhere(w, idx.useDaily, from, until) return w } func validatePlainQuery(query string, wildcardMinDistance int) error { if where.HasUnmatchedBrackets(query) { return errs.NewErrorWithCode("query has unmatched brackets", http.StatusBadRequest) } var maxDist = where.MaxWildcardDistance(query) // If the amount of nodes in a plain query is equal to 1, // then make an exception // This allows to check which root nodes exist moreThanOneNode := strings.Count(query, ".") >= 1 if maxDist != -1 && maxDist < wildcardMinDistance && moreThanOneNode { return errs.NewErrorWithCode("query has wildcards way too early at the start and at the end of it", http.StatusBadRequest) } return nil } func (idx *IndexFinder) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) (err error) { err = validatePlainQuery(query, config.ClickHouse.WildcardMinDistance) if err != nil { return err } w := idx.whereFilter(query, from, until) idx.stats = append(idx.stats, metrics.FinderStat{}) stat := &idx.stats[len(idx.stats)-1] idx.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query( scope.WithTable(ctx, idx.table), idx.url, // TODO: consider consistent query generator fmt.Sprintf("SELECT Path FROM %s WHERE %s GROUP BY Path FORMAT TabSeparatedRaw", idx.table, w), idx.opts, nil, ) stat.Table = idx.table if err == nil { stat.ReadBytes = int64(len(idx.body)) idx.bodySplit() } return } func (idx *IndexFinder) Abs(v []byte) []byte { return v } func splitIndexBody(body []byte, useReverse, useCache bool) ([]byte, [][]byte, bool) { if len(body) == 0 { return body, [][]byte{}, false } rows := bytes.Split(bytes.TrimSuffix(body, []byte{'\n'}), []byte{'\n'}) setDirect := false if useReverse { var buf bytes.Buffer if useCache { buf.Grow(len(body)) } for i := range rows { rows[i] = ReverseBytes(rows[i]) if useCache { buf.Write(rows[i]) buf.WriteByte('\n') } } if useCache { body = buf.Bytes() setDirect = true } } return body, rows, setDirect } func (idx *IndexFinder) bodySplit() { setDirect := false idx.body, idx.rows, setDirect = splitIndexBody(idx.body, idx.useReverse(""), idx.useCache) if setDirect { idx.reverse = queryDirect } } func makeList(rows [][]byte, onlySeries bool) [][]byte { if len(rows) == 0 { return [][]byte{} } resRows := make([][]byte, len(rows)) for i := 0; i < len(rows); i++ { resRows[i] = rows[i] } return resRows } func (idx *IndexFinder) makeList(onlySeries bool) [][]byte { return makeList(idx.rows, onlySeries) } func (idx *IndexFinder) List() [][]byte { return idx.makeList(false) } func (idx *IndexFinder) Series() [][]byte { return idx.makeList(true) } func (idx *IndexFinder) Bytes() ([]byte, error) { return idx.body, nil } func (idx *IndexFinder) Stats() []metrics.FinderStat { return idx.stats } ================================================ FILE: finder/index_test.go ================================================ package finder import ( "fmt" "testing" "time" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/helper/date" "github.com/stretchr/testify/assert" ) func Test_useReverse(t *testing.T) { assert := assert.New(t) table := []struct { query string result bool }{ {"a.b.c.d.e", false}, {"a.b*", false}, {"a.b.c.d.e*", false}, {"a.b.c.d*.e", false}, {"a.b*.c*.d.e", true}, {"a.b*.c.d.e", true}, } for _, tt := range table { idx := IndexFinder{confReverse: queryAuto} assert.Equal(tt.result, idx.useReverse(tt.query), tt.query) } } func Test_useReverseWithSetConfig(t *testing.T) { assert := assert.New(t) table := []struct { query string reverse uint8 result bool }{ {"a.b.c.d.e", queryReversed, true}, {"a.b.c.d.e", queryAuto, false}, {"a.b.c.d.e", queryDirect, false}, {"a.b.c.d.e", queryDirect, false}, {"a.b.c.d.e*", queryDirect, false}, {"a.b.c.d*.e", queryDirect, false}, {"a.b.c.d*.e", queryReversed, true}, {"a*.b.c.d*.e", queryReversed, true}, // Wildcard at first level, use reverse if possible {"a.b*.c.d*.e", queryReversed, true}, {"a.*.c.*.e.*.j", queryReversed, true}, {"a.*.c.*.e.*.j", queryDirect, false}, {"a.b*.c.*d.e", queryReversed, true}, } for _, tt := range table { idx := IndexFinder{confReverse: tt.reverse} assert.Equal(tt.result, idx.useReverse(tt.query), fmt.Sprintf("%s with iota %d", tt.query, tt.reverse)) } } func Test_checkReverses(t *testing.T) { assert := assert.New(t) reverses := config.IndexReverses{ {Suffix: ".sum", Reverse: "direct"}, {Prefix: "test.", Suffix: ".alloc", Reverse: "direct"}, {Prefix: "test2.", Reverse: "reversed"}, {RegexStr: `^a\..*\.max$`, Reverse: "reversed"}, } table := []struct { query string reverse uint8 result uint8 }{ {"a.b.c.d*.sum", queryAuto, queryDirect}, {"a*.b.c.d.sum", queryAuto, queryDirect}, {"test.b.c*.d*.alloc", queryAuto, queryDirect}, {"test.b.c*.d.alloc", queryAuto, queryDirect}, {"test2.b.c*.d*.e", queryAuto, queryReversed}, {"test2.b.c*.d.e", queryAuto, queryReversed}, {"a.b.c.d*.max", queryAuto, queryReversed}, // regex test {"a.b.c*.d.max", queryAuto, queryReversed}, // regex test } assert.NoError(reverses.Compile()) for _, tt := range table { idx := IndexFinder{confReverse: tt.reverse, confReverses: reverses} assert.Equal(tt.result, idx.checkReverses(tt.query), fmt.Sprintf("%s with iota %d", tt.query, tt.reverse)) } } func Benchmark_useReverseDepth(b *testing.B) { reverses := config.IndexReverses{ {Prefix: "test2.", Reverse: "reversed"}, } if err := reverses.Compile(); err != nil { b.Fatal("failed to compile reverses") } idx := IndexFinder{confReverses: reverses} for i := 0; i < b.N; i++ { _ = idx.checkReverses("test2.b.c*.d.e") } } func Benchmark_useReverseDepthPrefixSuffix(b *testing.B) { reverses := config.IndexReverses{ {Prefix: "test2.", Suffix: ".e", Reverse: "direct"}, } if err := reverses.Compile(); err != nil { b.Fatal("failed to compile reverses") } idx := IndexFinder{confReverses: reverses} for i := 0; i < b.N; i++ { _ = idx.checkReverses("test2.b.c*.d.e") } } func Benchmark_useReverseDepthRegex(b *testing.B) { reverses := config.IndexReverses{ {RegexStr: `^a\..*\.max$`, Reverse: "auto"}, } if err := reverses.Compile(); err != nil { b.Fatal("failed to compile reverses") } idx := IndexFinder{confReverses: reverses} for i := 0; i < b.N; i++ { _ = idx.checkReverses("a.b.c*.d.max") } } func TestIndexFinder_whereFilter(t *testing.T) { tests := []struct { name string query string from int64 until int64 dailyEnabled bool indexReverse string indexReverses config.IndexReverses want string }{ { name: "nodaily (direct)", query: "test.metric*", from: 1668106860, until: 1668106870, dailyEnabled: false, want: "((Level=20002) AND (Path LIKE 'test.metric%')) AND (Date='1970-02-12')", }, { name: "nodaily (reverse)", query: "*test.metric", from: 1668106860, until: 1668106870, dailyEnabled: false, want: "((Level=30002) AND (Path LIKE 'metric.%' AND match(Path, '^metric[.]([^.]*?)test[.]?$'))) AND (Date='1970-02-12')", }, { name: "midnight at utc (direct)", query: "test.metric*", from: 1668124800, // 2022-11-11 00:00:00 UTC until: 1668124810, // 2022-11-11 00:00:10 UTC dailyEnabled: true, want: "((Level=2) AND (Path LIKE 'test.metric%')) AND (Date >='" + date.FromTimestampToDaysFormat(1668124800) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(1668124810) + "')", }, { name: "midnight at utc (reverse)", query: "*test.metric", from: 1668124800, // 2022-11-11 00:00:00 UTC until: 1668124810, // 2022-11-11 00:00:10 UTC dailyEnabled: true, want: "((Level=10002) AND (Path LIKE 'metric.%' AND match(Path, '^metric[.]([^.]*?)test[.]?$'))) AND (Date >='" + date.FromTimestampToDaysFormat(1668124800) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(1668124810) + "')", }, } for _, tt := range tests { t.Run(tt.name+" "+time.Unix(tt.from, 0).Format(time.RFC3339), func(t *testing.T) { if tt.indexReverse == "" { tt.indexReverse = "auto" } idx := NewIndex("http://localhost:8123/", "graphite_index", tt.dailyEnabled, tt.indexReverse, tt.indexReverses, clickhouse.Options{}, false).(*IndexFinder) if got := idx.whereFilter(tt.query, tt.from, tt.until); got.String() != tt.want { t.Errorf("IndexFinder.whereFilter() = %v, want %v", got, tt.want) } }) } } ================================================ FILE: finder/mock.go ================================================ package finder import ( "bytes" "context" "strings" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/metrics" ) // MockFinder is used for testing purposes type MockFinder struct { fnd Finder query string // logged from execute } // NewMockFinder returns new MockFinder object with given result func NewMockFinder(result [][]byte) *MockFinder { return &MockFinder{ fnd: NewCachedIndex(bytes.Join(result, []byte{'\n'})), } } // NewMockTagged returns new MockFinder object with given result func NewMockTagged(result [][]byte) *MockFinder { return &MockFinder{ fnd: NewCachedTags(bytes.Join(result, []byte{'\n'})), } } // Execute assigns given query to the query field func (m *MockFinder) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) (err error) { m.query = query return } // List returns the result func (m *MockFinder) List() [][]byte { return m.fnd.List() } // Series returns the result func (m *MockFinder) Series() [][]byte { return m.fnd.Series() } // Abs returns the same given v func (m *MockFinder) Abs(v []byte) []byte { return m.fnd.Abs(v) } func (m *MockFinder) Bytes() ([]byte, error) { return m.fnd.Bytes() } // Strings returns the result converted to []string func (m *MockFinder) Strings() []string { body, _ := m.fnd.Bytes() return strings.Split(string(body), "\n") } func (m *MockFinder) Stats() []metrics.FinderStat { return m.fnd.Stats() } ================================================ FILE: finder/plain_from_tagged.go ================================================ package finder import ( "context" "net/url" "sort" "strconv" "strings" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/metrics" ) // Special finder for query plain graphite from prometheus // graphite{target="telegraf.*.cpu.avg"} type plainFromTaggedFinder struct { wrappedPlain Finder target string nodeLabel map[int]string metricName string } func makePlainFromTagged(matchers []TaggedTerm) *plainFromTaggedFinder { var isMetricNameFound bool var target string for _, m := range matchers { if m.Key == "__name__" && m.Value == "graphite" && m.Op == TaggedTermEq { isMetricNameFound = true } if m.Key == "target" && m.Op == TaggedTermEq && m.Value != "" { target = m.Value } } // not plain graphite query if !isMetricNameFound || target == "" { return nil } q := &plainFromTaggedFinder{target: target} // fill additional params for _, m := range matchers { if m.Key == "rename" && m.Op == TaggedTermEq && m.Value != "" { q.metricName = m.Value } if strings.HasPrefix(m.Key, "node") && m.Op == TaggedTermEq && m.Value != "" { v, err := strconv.Atoi(m.Key[4:]) if err != nil { continue } if q.nodeLabel == nil { q.nodeLabel = make(map[int]string) } q.nodeLabel[v] = m.Value } } return q } func (f *plainFromTaggedFinder) Target() string { return f.target } func (f *plainFromTaggedFinder) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) error { return f.wrappedPlain.Execute(ctx, config, query, from, until) } // For Render func (f *plainFromTaggedFinder) Series() [][]byte { return f.wrappedPlain.Series() } type taggedLabel struct { name string value string } func (f *plainFromTaggedFinder) Abs(value []byte) []byte { name := "graphite" path := string(value) lb := []taggedLabel{ {"metric", path}, } if f.metricName != "" { name = f.metricName } if len(f.nodeLabel) > 0 { a := strings.Split(path, ".") for n, v := range a { l := f.nodeLabel[n] if l != "" { lb = append(lb, taggedLabel{l, v}) } } } sort.Slice(lb, func(i, j int) bool { return lb[i].name < lb[j].name }) var buf strings.Builder buf.WriteString(name) buf.WriteByte('?') for i, l := range lb { if i > 0 { buf.WriteByte('&') } buf.WriteString(url.QueryEscape(l.name)) buf.WriteByte('=') buf.WriteString(url.QueryEscape(l.value)) } return []byte(buf.String()) } func (f *plainFromTaggedFinder) List() [][]byte { return f.wrappedPlain.List() } func (f *plainFromTaggedFinder) Bytes() ([]byte, error) { return nil, ErrNotImplemented } func (f *plainFromTaggedFinder) Stats() []metrics.FinderStat { return f.wrappedPlain.Stats() } ================================================ FILE: finder/plain_from_tagged_test.go ================================================ package finder import ( "testing" "github.com/stretchr/testify/assert" ) func TestPlainFromTaggedFinderAbs(t *testing.T) { assert := assert.New(t) eq := func(name, value string) TaggedTerm { return TaggedTerm{Op: TaggedTermEq, Key: name, Value: value} } join := func(terms ...TaggedTerm) []TaggedTerm { return terms } f := makePlainFromTagged(join( eq("__name__", "graphite"), eq("rename", "cpu_usage"), eq("target", "telegraf.*.cpu.usage"), eq("node1", "host"), )) assert.NotNil(f) table := [][2]string{ { "telegraf.localhost.cpu.usage", `cpu_usage?host=localhost&metric=telegraf.localhost.cpu.usage`, }, } for _, c := range table { assert.Equal(c[1], string(f.Abs([]byte(c[0])))) } } ================================================ FILE: finder/prefix.go ================================================ package finder import ( "context" "regexp" "strings" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/metrics" "github.com/lomik/graphite-clickhouse/pkg/where" ) type PrefixMatchResult int const ( PrefixNotMatched PrefixMatchResult = iota PrefixMatched PrefixPartialMathed ) type PrefixFinder struct { wrapped Finder prefix string // config prefixBytes []byte // same prefix with []bytes type matched PrefixMatchResult // request part string // request. partially matched part } func bytesConcat(s1 []byte, s2 []byte) []byte { ret := make([]byte, len(s1)+len(s2)) copy(ret, s1) copy(ret[len(s1):], s2) return ret } func WrapPrefix(f Finder, prefix string) *PrefixFinder { return &PrefixFinder{ wrapped: f, prefix: prefix, prefixBytes: bytesConcat([]byte(prefix), []byte{'.'}), matched: PrefixNotMatched, } } func (p *PrefixFinder) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) error { qs := strings.Split(query, ".") // check regexp for _, queryPart := range qs { if _, err := regexp.Compile(where.GlobToRegexp(queryPart)); err != nil { return err } } ps := strings.Split(p.prefix, ".") var i int for i = 0; i < len(qs) && i < len(ps); i++ { m, err := regexp.MatchString("^"+where.GlobToRegexp(qs[i])+"$", ps[i]) if err != nil { return err } if !m { // not matched return nil } } if len(qs) <= len(ps) { // prefix matched, but not finished p.part = strings.Join(ps[:len(qs)], ".") + "." p.matched = PrefixPartialMathed return nil } p.matched = PrefixMatched return p.wrapped.Execute(ctx, config, strings.Join(qs[len(ps):], "."), from, until) } func (p *PrefixFinder) List() [][]byte { if p.matched == PrefixNotMatched { return [][]byte{} } if p.matched == PrefixPartialMathed { return [][]byte{[]byte(p.part)} } list := p.wrapped.List() result := make([][]byte, len(list)) for i := 0; i < len(list); i++ { result[i] = bytesConcat(p.prefixBytes, list[i]) } return result } // For Render func (p *PrefixFinder) Series() [][]byte { if p.matched == PrefixNotMatched { return [][]byte{} } if p.matched != PrefixMatched { return [][]byte{} } return p.wrapped.Series() } func (p *PrefixFinder) Abs(value []byte) []byte { return bytesConcat(p.prefixBytes, p.wrapped.Abs(value)) } func (p *PrefixFinder) Bytes() ([]byte, error) { return nil, ErrNotImplemented } func (p *PrefixFinder) Stats() []metrics.FinderStat { return p.wrapped.Stats() } ================================================ FILE: finder/prefix_test.go ================================================ package finder import ( "context" "fmt" "testing" "github.com/lomik/graphite-clickhouse/config" "github.com/stretchr/testify/assert" ) func TestPrefixFinderExecute(t *testing.T) { assert := assert.New(t) table := []struct { prefix string query string expectedMatched PrefixMatchResult expectedQ string expectedPart string expectedError bool }{ {"ch", "*", PrefixPartialMathed, "", "ch.", false}, {"ch.data", "*", PrefixPartialMathed, "", "ch.", false}, {"ch.data", "ch.*", PrefixPartialMathed, "", "ch.data.", false}, {"ch.data", "ch.data.*", PrefixMatched, "*", "", false}, {"ch.data", "epta.*", PrefixNotMatched, "", "", false}, {"ch.data", "ch.data._tag.daemon.h.hostname.top.cpu_avg", PrefixMatched, "_tag.daemon.h.hostname.top.cpu_avg", "", false}, {"ch.data", "ch.d[a]", PrefixNotMatched, "", "", false}, } for _, test := range table { testName := fmt.Sprintf("prefix: %#v, query: %#v", test.prefix, test.query) m := NewMockFinder([][]byte{}) f := WrapPrefix(m, test.prefix) config := config.New() err := f.Execute(context.Background(), config, test.query, 0, 0) if test.expectedError { assert.Error(err, testName) } else { assert.NoError(err, testName) } assert.Equal(test.expectedQ, m.query, testName) assert.Equal(test.expectedMatched, f.matched, testName) assert.Equal(test.expectedPart, f.part, testName) } } func TestPrefixFinderAbs(t *testing.T) { assert := assert.New(t) m := NewMockFinder([][]byte{}) f := WrapPrefix(m, "hello") assert.Equal("hello.world", string(f.Abs([]byte("world")))) } func TestPrefixFinderList(t *testing.T) { assert := assert.New(t) mockData := [][]byte{[]byte("world")} prefix := "hello" table := []struct { query string expectedList []string expectedSeries []string }{ {"*", []string{"hello."}, []string{}}, {"hello", []string{"hello."}, []string{}}, {"hello.*", []string{"hello.world"}, []string{"world"}}, {"*.*", []string{"hello.world"}, []string{"world"}}, {"*404*", []string{}, []string{}}, {"*404*.*", []string{}, []string{}}, {"hello.[bad regexp", []string{}, []string{}}, } for _, test := range table { testName := fmt.Sprintf("query: %#v", test.query) m := NewMockFinder(mockData) f := WrapPrefix(m, prefix) config := config.New() f.Execute(context.Background(), config, test.query, 0, 0) list := make([]string, 0) for _, r := range f.List() { list = append(list, string(r)) } series := make([]string, 0) for _, r := range f.Series() { series = append(series, string(r)) } assert.Equal(test.expectedList, list, testName+", list") assert.Equal(test.expectedSeries, series, testName+", series") } } ================================================ FILE: finder/reverse.go ================================================ package finder import ( "bytes" "context" "strings" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/metrics" "github.com/lomik/graphite-clickhouse/pkg/where" ) type ReverseFinder struct { wrapped Finder baseFinder Finder url string // clickhouse dsn table string // graphite_reverse_tree table isUsed bool // use reverse table } func ReverseString(target string) string { a := strings.Split(target, ".") l := len(a) for i := 0; i < l/2; i++ { a[i], a[l-i-1] = a[l-i-1], a[i] } return strings.Join(a, ".") } func ReverseBytes(target []byte) []byte { // @TODO: check performance a := bytes.Split(target, []byte{'.'}) l := len(a) for i := 0; i < l/2; i++ { a[i], a[l-i-1] = a[l-i-1], a[i] } return bytes.Join(a, []byte{'.'}) } func WrapReverse(f Finder, url string, table string, opts clickhouse.Options) *ReverseFinder { return &ReverseFinder{ wrapped: f, baseFinder: NewBase(url, table, opts), url: url, table: table, } } func (r *ReverseFinder) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) (err error) { p := strings.LastIndexByte(query, '.') if p < 0 || p >= len(query)-1 { return r.wrapped.Execute(ctx, config, query, from, until) } if where.HasWildcard(query[p+1:]) { return r.wrapped.Execute(ctx, config, query, from, until) } r.isUsed = true return r.baseFinder.Execute(ctx, config, ReverseString(query), from, until) } func (r *ReverseFinder) List() [][]byte { if !r.isUsed { return r.wrapped.List() } list := r.baseFinder.List() for i := 0; i < len(list); i++ { list[i] = ReverseBytes(list[i]) } return list } func (r *ReverseFinder) Series() [][]byte { if !r.isUsed { return r.wrapped.Series() } list := r.baseFinder.Series() for i := 0; i < len(list); i++ { list[i] = ReverseBytes(list[i]) } return list } func (r *ReverseFinder) Abs(v []byte) []byte { return v } func (f *ReverseFinder) Bytes() ([]byte, error) { return f.wrapped.Bytes() } func (f *ReverseFinder) Stats() []metrics.FinderStat { return f.wrapped.Stats() } ================================================ FILE: finder/reverse_test.go ================================================ package finder import ( "testing" "github.com/stretchr/testify/assert" ) func TestReverse(t *testing.T) { assert := assert.New(t) table := []string{ "hello.world", "world.hello", "hello.", ".hello", "hello", "hello", ".", ".", "a1.b2.c3", "c3.b2.a1", } for i := 0; i < len(table); i += 2 { assert.Equal(table[i+1], ReverseString(table[i])) assert.Equal([]byte(table[i+1]), ReverseBytes([]byte(table[i]))) } } ================================================ FILE: finder/split.go ================================================ package finder import ( "context" "fmt" "net/http" "strings" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/helper/errs" "github.com/lomik/graphite-clickhouse/metrics" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/pkg/where" ) type indexFinderParams struct { url string table string opts clickhouse.Options dailyEnabled bool useCache bool reverse string confReverses config.IndexReverses } // SplitIndexFinder will try to split queries like {first,second}.some.metric into n queries (n - number of cases inside {}). // No matter if '{}' in first node or not. Only one {} will be split. type SplitIndexFinder struct { indexFinderParams // wrapped finder will be called if we can't split query. wrapped Finder body []byte rows [][]byte stats []metrics.FinderStat // useWrapped indicated if we should use wrapped Finder. useWrapped bool useReverse bool wildcardMinDistance int } // WrapSplitIndex wraps given finder with SplitIndexFinder logic. func WrapSplitIndex( f Finder, wildcardMinDistance int, url string, table string, dailyEnabled bool, reverse string, reverses config.IndexReverses, opts clickhouse.Options, useCache bool, ) *SplitIndexFinder { return &SplitIndexFinder{ wrapped: f, useWrapped: false, useReverse: false, wildcardMinDistance: wildcardMinDistance, indexFinderParams: indexFinderParams{ url: url, table: table, dailyEnabled: dailyEnabled, reverse: reverse, confReverses: reverses, opts: opts, useCache: useCache, }, } } // Execute will try to split query if it contains list in it. If query can't be split wrapped Finder will be used. // Use List, Series or Bytes after calling Execute to get data. func (splitFinder *SplitIndexFinder) Execute( ctx context.Context, config *config.Config, query string, from int64, until int64, ) error { if where.HasUnmatchedBrackets(query) { return errs.NewErrorWithCode("query has unmatched brackets", http.StatusBadRequest) } query = where.ClearGlob(query) idx := strings.IndexAny(query, "{}") if idx == -1 { splitFinder.useWrapped = true return splitFinder.wrapped.Execute(ctx, config, query, from, until) } splitQueries, err := splitQuery(query, config.ClickHouse.MaxNodeToSplitIndex) if err != nil { return err } if len(splitQueries) <= 1 { splitFinder.useWrapped = true return splitFinder.wrapped.Execute(ctx, config, query, from, until) } w, err := splitFinder.whereFilter(splitQueries, from, until) if err != nil { return err } splitFinder.stats = append(splitFinder.stats, metrics.FinderStat{}) stat := &splitFinder.stats[len(splitFinder.stats)-1] splitFinder.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query( scope.WithTable(ctx, splitFinder.table), splitFinder.url, // TODO: consider consistent query generator fmt.Sprintf("SELECT Path FROM %s WHERE %s GROUP BY Path FORMAT TabSeparatedRaw", splitFinder.table, w), splitFinder.opts, nil, ) stat.Table = splitFinder.table if err != nil { return err } stat.ReadBytes = int64(len(splitFinder.body)) splitFinder.body, splitFinder.rows, _ = splitIndexBody(splitFinder.body, splitFinder.useReverse, splitFinder.useCache) return nil } func splitQuery(query string, maxNodeToSplitIdx int) ([]string, error) { splitQueries := make([]string, 0, 1) firstClosingBracketIndex := strings.Index(query, "}") lastOpenBracketIndex := strings.LastIndex(query, "{") firstOpenBracketsIndex := strings.Index(query, "{") directNodeCount := strings.Count(query[:firstOpenBracketsIndex], ".") directWildcardIndex := where.IndexWildcard(query[:firstOpenBracketsIndex]) lastClosingBracketIndex := strings.LastIndex(query, "}") reverseNodeCount := strings.Count(query[lastClosingBracketIndex:], ".") var reverseWildcardIndex int if lastClosingBracketIndex == len(query)-1 { reverseWildcardIndex = -1 } else { reverseWildcardIndex = where.IndexLastWildcard(query[lastClosingBracketIndex+1:]) } useDirect := true if directWildcardIndex >= 0 && reverseWildcardIndex >= 0 { return []string{query}, nil } else if directWildcardIndex < 0 && reverseWildcardIndex >= 0 { if directNodeCount > maxNodeToSplitIdx { return []string{query}, nil } useDirect = true } else if directWildcardIndex >= 0 && reverseWildcardIndex < 0 { if reverseNodeCount > maxNodeToSplitIdx { return []string{query}, nil } useDirect = false } else { if directNodeCount > maxNodeToSplitIdx && reverseNodeCount > maxNodeToSplitIdx { return []string{query}, nil } } if lastOpenBracketIndex < firstClosingBracketIndex { // we have only one bracket in query err := where.GlobExpandSimple(query, "", &splitQueries) if err != nil { return nil, err } return splitQueries, nil } choicesInLeftMost := strings.Count(query[firstOpenBracketsIndex:firstClosingBracketIndex], ",") choicesInRightMost := strings.Count(query[lastOpenBracketIndex:lastClosingBracketIndex], ",") if directWildcardIndex < 0 && reverseWildcardIndex < 0 { if directNodeCount > reverseNodeCount { if directNodeCount > maxNodeToSplitIdx { return []string{query}, nil } useDirect = true } else if reverseNodeCount > directNodeCount { if reverseNodeCount > maxNodeToSplitIdx { return []string{query}, nil } useDirect = false } else { if choicesInLeftMost >= choicesInRightMost { useDirect = true } else { useDirect = false } } } var prefix, suffix, queryPart string if useDirect { prefix = "" queryPart = query[:firstClosingBracketIndex+1] suffix = query[firstClosingBracketIndex+1:] } else { prefix = query[:lastOpenBracketIndex] queryPart = query[lastOpenBracketIndex:] suffix = "" } splitQueries, err := splitPartOfQuery(prefix, queryPart, suffix) if err != nil { return nil, err } return splitQueries, nil } func splitPartOfQuery(prefix, queryPart, suffix string) ([]string, error) { splitQueries := make([]string, 0) err := where.GlobExpandSimple(queryPart, "", &splitQueries) if err != nil { return nil, err } for i := range splitQueries { splitQueries[i] = prefix + splitQueries[i] + suffix } return splitQueries, nil } func (splitFinder *SplitIndexFinder) whereFilter(queries []string, from, until int64) (*where.Where, error) { queryWithWildcardIdx := -1 for i, q := range queries { err := validatePlainQuery(q, splitFinder.wildcardMinDistance) if err != nil { return nil, err } if queryWithWildcardIdx < 0 && where.HasWildcard(q) { queryWithWildcardIdx = i } } if queryWithWildcardIdx >= 0 { splitFinder.useReverse = (&IndexFinder{ confReverses: splitFinder.confReverses, confReverse: config.IndexReverse[splitFinder.reverse], }).useReverse(queries[queryWithWildcardIdx]) } else { splitFinder.useReverse = false } nonWildcardQueries := make([]string, 0) aggregatedWhere := where.New() for _, q := range queries { if splitFinder.useReverse { q = ReverseString(q) } if !where.HasWildcard(q) { nonWildcardQueries = append(nonWildcardQueries, q, q+".") } else { aggregatedWhere.Or(where.TreeGlob("Path", q)) } } if len(nonWildcardQueries) > 0 { aggregatedWhere.Or(where.In("Path", nonWildcardQueries)) } useDates := useDaily(splitFinder.dailyEnabled, from, until) levelOffset := calculateIndexLevelOffset(useDates, splitFinder.useReverse) level := strings.Count(queries[0], ".") + 1 aggregatedWhere.And(where.Eq("Level", level+levelOffset)) addDatesToWhere(aggregatedWhere, useDates, from, until) return aggregatedWhere, nil } // List returns clickhouse response split by delimiter. // If there was no split, wrapped.List will be used. func (splitFinder *SplitIndexFinder) List() [][]byte { if splitFinder.useWrapped { return splitFinder.wrapped.List() } return makeList(splitFinder.rows, false) } // Series same as List. If there was no split, wrapped.Series will be used. func (splitFinder *SplitIndexFinder) Series() [][]byte { if splitFinder.useWrapped { return splitFinder.wrapped.Series() } return makeList(splitFinder.rows, true) } // Abs for this implementation returns given v. // If there was no split, wrapped.Abs will be used. func (splitFinder *SplitIndexFinder) Abs(v []byte) []byte { if splitFinder.useWrapped { return splitFinder.wrapped.Abs(v) } return v } // Bytes returns clickhouse response bytes. // If there was no split, wrapped.Bytes will be used. func (splitFinder *SplitIndexFinder) Bytes() ([]byte, error) { if splitFinder.useWrapped { return splitFinder.wrapped.Bytes() } return splitFinder.body, nil } func (splitFinder *SplitIndexFinder) Stats() []metrics.FinderStat { if splitFinder.useWrapped { return splitFinder.wrapped.Stats() } return splitFinder.stats } ================================================ FILE: finder/split_test.go ================================================ package finder import ( "fmt" "net/http" "testing" "time" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/helper/date" "github.com/lomik/graphite-clickhouse/helper/errs" "github.com/stretchr/testify/assert" ) func Test_splitQuery(t *testing.T) { type testcase struct { givenQuery string givenMaxNodeToSplitIndex int expectedQueries []string expectedErr error desc string } cases := []testcase{ { givenQuery: "some.*.{a,b,c}.{first,second}.*.test.metric", givenMaxNodeToSplitIndex: 3, expectedQueries: []string{ "some.*.{a,b,c}.{first,second}.*.test.metric", }, expectedErr: nil, desc: "have wildcards on both direct and reverse, so no split", }, { givenQuery: "some.long.{a,b,c}.{first,second}.*.metric", givenMaxNodeToSplitIndex: 1, expectedQueries: []string{ "some.long.{a,b,c}.{first,second}.*.metric", }, expectedErr: nil, desc: "have wildcard on reverse but index of list is greater than given in config", }, { givenQuery: "some.long.{a,b,c}.{first,second}.*.metric", givenMaxNodeToSplitIndex: 2, expectedQueries: []string{ "some.long.a.{first,second}.*.metric", "some.long.b.{first,second}.*.metric", "some.long.c.{first,second}.*.metric", }, expectedErr: nil, desc: "have wildcard on reverse and index of list is less or equal than given in config", }, { givenQuery: "some.*.{a,b,c}.{first,second}.test.metric", givenMaxNodeToSplitIndex: 1, expectedQueries: []string{ "some.*.{a,b,c}.{first,second}.test.metric", }, expectedErr: nil, desc: "have wildcard on direct but index of list is greater than given in config", }, { givenQuery: "some.*.{a,b,c}.{first,second}.test.metric", givenMaxNodeToSplitIndex: 2, expectedQueries: []string{ "some.*.{a,b,c}.first.test.metric", "some.*.{a,b,c}.second.test.metric", }, expectedErr: nil, desc: "have wildcard on direct and index of list is less or equal than given in config", }, { givenQuery: "some.long.{a,b,c}.{first,second}.test.metric", givenMaxNodeToSplitIndex: 1, expectedQueries: []string{ "some.long.{a,b,c}.{first,second}.test.metric", }, expectedErr: nil, desc: "no wildcards on both but indexes of lists are greater than given in config", }, { givenQuery: "{first,second}.some.metric.*", givenMaxNodeToSplitIndex: 3, expectedQueries: []string{ "first.some.metric.*", "second.some.metric.*", }, expectedErr: nil, desc: "only one bracket with wildcard on reverse", }, { givenQuery: "*.some.metric.{first,second}", givenMaxNodeToSplitIndex: 3, expectedQueries: []string{ "*.some.metric.first", "*.some.metric.second", }, expectedErr: nil, desc: "only one bracket and wildcard on direct", }, { givenQuery: "some.very.long.{a,b}.*.{first,second}.metric", givenMaxNodeToSplitIndex: 2, expectedQueries: []string{ "some.very.long.{a,b}.*.{first,second}.metric", }, expectedErr: nil, desc: "no wildcards, but direct has more nodes than max-node-to-split-index", }, { givenQuery: "some.very.long.{a,b}.*.{first,second}.metric", givenMaxNodeToSplitIndex: 3, expectedQueries: []string{ "some.very.long.a.*.{first,second}.metric", "some.very.long.b.*.{first,second}.metric", }, expectedErr: nil, desc: "no wildcards, direct has more nodes than reverse", }, { givenQuery: "some.{a,b}.*.{first,second}.long.test.metric", givenMaxNodeToSplitIndex: 2, expectedQueries: []string{ "some.{a,b}.*.{first,second}.long.test.metric", }, expectedErr: nil, desc: "no wildcards, but reverse has more nodes than max-node-to-split-index", }, { givenQuery: "some.{a,b}.*.{first,second}.long.test.metric", givenMaxNodeToSplitIndex: 3, expectedQueries: []string{ "some.{a,b}.*.first.long.test.metric", "some.{a,b}.*.second.long.test.metric", }, expectedErr: nil, desc: "no wildcards, reverse has more nodes than direct", }, { givenQuery: "some.very.long.{a,b,c}.*.{first,second}.long.test.metric", givenMaxNodeToSplitIndex: 3, expectedQueries: []string{ "some.very.long.a.*.{first,second}.long.test.metric", "some.very.long.b.*.{first,second}.long.test.metric", "some.very.long.c.*.{first,second}.long.test.metric", }, expectedErr: nil, desc: "no wildcards, direct nd reverse has equal nodes, but leftmost has more choices", }, { givenQuery: "some.very.long.{a,b}.*.{first,second,third}.long.test.metric", givenMaxNodeToSplitIndex: 3, expectedQueries: []string{ "some.very.long.{a,b}.*.first.long.test.metric", "some.very.long.{a,b}.*.second.long.test.metric", "some.very.long.{a,b}.*.third.long.test.metric", }, expectedErr: nil, desc: "no wildcards, direct nd reverse has equal nodes, but leftmost has more choices", }, { givenQuery: "query.{a,b}", givenMaxNodeToSplitIndex: -1, expectedQueries: []string{ "query.{a,b}", }, expectedErr: nil, desc: "not split query", }, { givenQuery: "*.query.{a,b}", givenMaxNodeToSplitIndex: -1, expectedQueries: []string{ "*.query.{a,b}", }, expectedErr: nil, desc: "not split query", }, { givenQuery: "*.query.{a,b}", givenMaxNodeToSplitIndex: 20, expectedQueries: []string{ "*.query.a", "*.query.b", }, expectedErr: nil, desc: "query split if MaxNodeToSplitIndex is greater than nodes amount in query", }, } for i, singleCase := range cases { t.Run(fmt.Sprintf("case %v: %s", i+1, singleCase.desc), func(t *testing.T) { gotQueries, gotErr := splitQuery(singleCase.givenQuery, singleCase.givenMaxNodeToSplitIndex) assert.Equal(t, singleCase.expectedQueries, gotQueries, singleCase.desc) assert.Equal(t, singleCase.expectedErr, gotErr, singleCase.desc) }) } } func TestSplitIndexFinder_whereFilter(t *testing.T) { type testcase struct { name string givenQueries []string givenFrom int64 givenUntil int64 dailyEnabled bool wildcardMinDistance int reverse string confReverses config.IndexReverses expectedWhereStr string expectedErr error } someFrom := time.Now().Unix() - 120 someUntil := time.Now().Unix() cases := []testcase{ { name: "no wildcards in queries, no daily", givenQueries: []string{ "first.metric", "second.metric", }, dailyEnabled: false, expectedWhereStr: "((Path IN ('first.metric','first.metric.','second.metric','second.metric.')) AND (Level=20002)) AND (Date='1970-02-12')", }, { name: "wildcard in queries, reverse preferred, no daily", givenQueries: []string{ "*.first.metric", "*.second.metric", }, dailyEnabled: false, expectedWhereStr: "(((Path LIKE 'metric.first.%') OR (Path LIKE 'metric.second.%')) AND (Level=30003)) AND (Date='1970-02-12')", }, { name: "no wildcards in queries, daily enabled, but no from and until", givenQueries: []string{ "first.metric", "second.metric", }, dailyEnabled: true, expectedWhereStr: "((Path IN ('first.metric','first.metric.','second.metric','second.metric.')) AND (Level=20002)) AND (Date='1970-02-12')", }, { name: "no wildcards in queries, daily enabled, has from, until", givenQueries: []string{ "first.metric", "second.metric", }, givenFrom: someFrom, givenUntil: someFrom, dailyEnabled: true, expectedWhereStr: "((Path IN ('first.metric','first.metric.','second.metric','second.metric.')) AND (Level=2)) AND (Date >='" + date.FromTimestampToDaysFormat(someFrom) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(someUntil) + "')", }, { name: "wildcard in queries, reverse preferred, daily enabled, no from, until", givenQueries: []string{ "*.first.metric", "*.second.metric", }, dailyEnabled: true, expectedWhereStr: "(((Path LIKE 'metric.first.%') OR (Path LIKE 'metric.second.%')) AND (Level=30003)) AND (Date='1970-02-12')", }, { name: "wildcard in queries, reverse preferred, daily enabled, has from, until", givenQueries: []string{ "*.first.metric", "*.second.metric", }, dailyEnabled: true, givenFrom: someFrom, givenUntil: someUntil, expectedWhereStr: "(((Path LIKE 'metric.first.%') OR (Path LIKE 'metric.second.%')) AND (Level=10003)) AND (Date >='" + date.FromTimestampToDaysFormat(someFrom) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(someUntil) + "')", }, { name: "some queries have wildcard, daily enabled, has from, until", givenQueries: []string{ "help.*first.metric", "help.second.metric", "help.th*rd.metric", "help.forth.metric", }, dailyEnabled: true, givenFrom: someFrom, givenUntil: someUntil, expectedWhereStr: "((((Path LIKE 'help.%' AND match(Path, '^help[.]([^.]*?)first[.]metric[.]?$')) OR (Path LIKE 'help.th%' AND match(Path, '^help[.]th([^.]*?)rd[.]metric[.]?$'))) OR (Path IN ('help.second.metric','help.second.metric.','help.forth.metric','help.forth.metric.'))) AND (Level=3)) AND (Date >='" + date.FromTimestampToDaysFormat(someFrom) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(someUntil) + "')", }, { name: "some queries have wildcard, daily enabled, has from, until, but reverse preferred", givenQueries: []string{ "help.*first.metric.count", "help.second.metric.count", "help.th*rd.metric.count", "help.forth.metric.count", }, dailyEnabled: true, givenFrom: someFrom, givenUntil: someUntil, expectedWhereStr: "((((Path LIKE 'count.metric.%' AND match(Path, '^count[.]metric[.]([^.]*?)first[.]help[.]?$')) OR (Path LIKE 'count.metric.th%' AND match(Path, '^count[.]metric[.]th([^.]*?)rd[.]help[.]?$'))) OR (Path IN ('count.metric.second.help','count.metric.second.help.','count.metric.forth.help','count.metric.forth.help.'))) AND (Level=10004)) AND (Date >='" + date.FromTimestampToDaysFormat(someFrom) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(someUntil) + "')", expectedErr: nil, }, { name: "some queries have wildcard, daily enabled, has from, until, but reverse preferred, first query has no wildcard", givenQueries: []string{ "help.second.metric.count", "help.*first.metric.count", "help.th*rd.metric.count", "help.forth.metric.count", }, dailyEnabled: true, givenFrom: someFrom, givenUntil: someUntil, expectedWhereStr: "((((Path LIKE 'count.metric.%' AND match(Path, '^count[.]metric[.]([^.]*?)first[.]help[.]?$')) OR (Path LIKE 'count.metric.th%' AND match(Path, '^count[.]metric[.]th([^.]*?)rd[.]help[.]?$'))) OR (Path IN ('count.metric.second.help','count.metric.second.help.','count.metric.forth.help','count.metric.forth.help.'))) AND (Level=10004)) AND (Date >='" + date.FromTimestampToDaysFormat(someFrom) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(someUntil) + "')", }, { name: "queries do not satisfy wildcard min distance", givenQueries: []string{ "a*.first.metric.*", "b*.second.metric.*", }, wildcardMinDistance: 1, expectedWhereStr: "", expectedErr: errs.NewErrorWithCode("query has wildcards way too early at the start and at the end of it", http.StatusBadRequest), }, } for i, tc := range cases { t.Run(fmt.Sprintf("Case %v: %s", i+1, tc.name), func(t *testing.T) { f := WrapSplitIndex( &IndexFinder{}, tc.wildcardMinDistance, "http://localhost:8123/", "graphite_index", tc.dailyEnabled, tc.reverse, tc.confReverses, clickhouse.Options{}, false) got, err := f.whereFilter(tc.givenQueries, tc.givenFrom, tc.givenUntil) assert.Equal(t, tc.expectedErr, err) if err == nil { assert.Equal(t, tc.expectedWhereStr, got.String()) } }) } } ================================================ FILE: finder/tag.go ================================================ package finder import ( "bytes" "context" "fmt" "strings" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/metrics" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/pkg/where" ) type TagState int const ( TagRoot TagState = iota // query = "*" TagSkip // not _tag prefix TagInfoRoot // query = "_tag" TagList TagListSeriesRoot TagListSeries TagListParam ) type TagQ struct { Param *string Value *string } func (q TagQ) String() string { if q.Param != nil && q.Value != nil { return fmt.Sprintf("{\"param\"=%#v, \"value\"=%#v}", *q.Param, *q.Value) } if q.Param != nil { return fmt.Sprintf("{\"param\"=%#v}", *q.Param) } if q.Value != nil { return fmt.Sprintf("{\"value\"=%#v}", *q.Value) } return "{}" } func (q *TagQ) Where(field string) string { if q.Param != nil && q.Value != nil && *q.Value != "*" { return where.Eq(field, *q.Param+*q.Value) } if q.Param != nil { return where.HasPrefix(field, *q.Param) } if q.Value != nil && *q.Value != "*" { return where.Eq(field, *q.Value) } return "" } type TagFinder struct { wrapped Finder url string // clickhouse dsn table string // graphite_tag table opts clickhouse.Options // clickhouse timeout, connectTimeout, etc stats []metrics.FinderStat state TagState tagQuery []TagQ seriesQuery string tagPrefix []byte useWrapped bool body []byte // clickhouse response } var EmptyList [][]byte = [][]byte{} func WrapTag(f Finder, url string, table string, opts clickhouse.Options) *TagFinder { return &TagFinder{ wrapped: f, url: url, table: table, opts: opts, tagQuery: make([]TagQ, 0), useWrapped: true, } } func (t *TagFinder) tagListSQL() (string, error) { if len(t.tagQuery) == 0 { return "", nil } w := where.New() // first w.And(t.tagQuery[0].Where("Tag1")) if len(t.tagQuery) == 1 { w.And(where.Eq("Level", 1)) return fmt.Sprintf("SELECT Tag1 FROM %s WHERE %s GROUP BY Tag1", t.table, w), nil } // 1..(n-1) for i := 1; i < len(t.tagQuery)-1; i++ { cond := t.tagQuery[i].Where("x") if cond != "" { w.Andf("arrayExists((x) -> %s, Tags)", cond) } } // last w.And(t.tagQuery[len(t.tagQuery)-1].Where("TagN")) w.And(where.Eq("IsLeaf", 1)) return fmt.Sprintf("SELECT TagN FROM %s ARRAY JOIN Tags AS TagN WHERE %s GROUP BY TagN", t.table, w), nil } func (t *TagFinder) seriesSQL() (string, error) { if len(t.tagQuery) == 0 { return "", nil } w := where.New() w.Andf("Version>=(SELECT Max(Version) FROM %s WHERE Tag1='' AND Level=0 AND Path='')", t.table) // first w.And(t.tagQuery[0].Where("Tag1")) // 1..(n-1) for i := 1; i < len(t.tagQuery); i++ { cond := t.tagQuery[i].Where("x") if cond != "" { w.Andf("arrayExists((x) -> %s, Tags)", cond) } } base := &BaseFinder{} w.And(base.where(t.seriesQuery).String()) // TODO: consider consistent query generator return fmt.Sprintf("SELECT Path FROM %s WHERE %s GROUP BY Path FORMAT TabSeparatedRaw", t.table, w), nil } func (t *TagFinder) MakeSQL(query string) (string, error) { if query == "_tag" { t.state = TagInfoRoot return "", nil } qs0 := strings.Split(query, ".") qs := qs0 t.tagQuery = make([]TagQ, 0) for { if len(qs) == 0 { break } if qs[0] == "_tag" { if len(qs) >= 2 { v := qs[1] if len(v) > 0 && v[len(v)-1] == '=' { if len(qs) >= 3 { t.tagQuery = append(t.tagQuery, TagQ{Param: &v, Value: &qs[2]}) qs = qs[3:] } else { t.tagQuery = append(t.tagQuery, TagQ{Param: &v}) qs = qs[2:] } } else { t.tagQuery = append(t.tagQuery, TagQ{Value: &v}) qs = qs[2:] } } else { t.tagQuery = append(t.tagQuery, TagQ{}) qs = qs[1:] } } else { t.seriesQuery = strings.Join(qs, ".") break } } if len(qs0) > len(qs) { t.tagPrefix = append([]byte(strings.Join(qs0[:len(qs0)-len(qs)], ".")), '.') } if t.seriesQuery == "" { if len(t.tagQuery) > 0 && t.tagQuery[len(t.tagQuery)-1].Param != nil { t.state = TagListParam } else { t.state = TagList } return t.tagListSQL() } if t.seriesQuery == "*" { t.state = TagListSeriesRoot return t.seriesSQL() } t.state = TagListSeries return t.seriesSQL() } func (t *TagFinder) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) (err error) { t.state = TagSkip if query == "" { return t.wrapped.Execute(ctx, config, query, from, until) } if query == "*" { t.state = TagRoot return t.wrapped.Execute(ctx, config, query, from, until) } if !strings.HasPrefix(query, "_tag.") && query != "_tag" { return t.wrapped.Execute(ctx, config, query, from, until) } t.useWrapped = false var sql string sql, err = t.MakeSQL(query) if err != nil || sql == "" { return } t.stats = append(t.stats, metrics.FinderStat{}) stat := &t.stats[len(t.stats)-1] t.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query(scope.WithTable(ctx, t.table), t.url, sql, t.opts, nil) stat.Table = t.table stat.ReadBytes = int64(len(t.body)) return } func (t *TagFinder) List() [][]byte { switch t.state { case TagSkip: return t.wrapped.List() case TagInfoRoot: return [][]byte{[]byte("_tag.")} case TagRoot: // pass return append([][]byte{[]byte("_tag.")}, t.wrapped.List()...) } if t.body == nil { return [][]byte{} } rows := bytes.Split(t.body, []byte{'\n'}) skip := 0 for i := 0; i < len(rows); i++ { if len(rows[i]) == 0 { skip++ continue } if skip > 0 { rows[i-skip] = rows[i] } } rows = rows[:len(rows)-skip] if t.state == TagList || t.state == TagListParam { // add dots for i := 0; i < len(rows); i++ { eqIndex := bytes.IndexByte(rows[i], '=') if eqIndex > 0 && eqIndex < len(rows[i])-1 { if t.state == TagListParam { rows[i] = append(rows[i][eqIndex+1:], '.') } else { rows[i][eqIndex+1] = '.' rows[i] = rows[i][:eqIndex+2] } } else { rows[i] = append(rows[i], '.') } } } if t.state == TagListSeriesRoot { rows = append(rows, []byte("_tag.")) } return rows } func (t *TagFinder) Series() [][]byte { switch t.state { case TagSkip: return t.wrapped.Series() case TagInfoRoot: return EmptyList case TagRoot: return t.wrapped.Series() } rows := t.List() skip := 0 for i := 0; i < len(rows); i++ { if len(rows[i]) == 0 { skip++ continue } if rows[i][len(rows[i])-1] == '.' { skip++ continue } if skip > 0 { rows[i-skip] = rows[i] } } return rows } func (t *TagFinder) Abs(v []byte) []byte { if t.state == TagSkip { return t.wrapped.Abs(v) } return bytesConcat(t.tagPrefix, v) } func (t *TagFinder) Bytes() ([]byte, error) { return nil, ErrNotImplemented } func (t *TagFinder) Stats() []metrics.FinderStat { if t.useWrapped { return t.wrapped.Stats() } return t.stats } ================================================ FILE: finder/tag_test.go ================================================ package finder import ( "context" "fmt" "testing" "time" "github.com/stretchr/testify/assert" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/clickhouse" chtest "github.com/lomik/graphite-clickhouse/helper/tests/clickhouse" ) func TestTagsMakeSQL(t *testing.T) { assert := assert.New(t) tag1Base := "SELECT Tag1 FROM table WHERE " tag1Group := " GROUP BY Tag1" tagNBase := "SELECT TagN FROM table ARRAY JOIN Tags AS TagN WHERE " tagNGroup := " GROUP BY TagN" table := []struct { query string sql string error bool }{ // SELECT Tag1 FROM graphite_tag WHERE Version >= (SELECT Max(Version) FROM graphite_tag WHERE Tag1='' AND Level=0 AND Path='') AND Level=1 GROUP BY Tag1; {"_tag", "", false}, {"_tag.*", tag1Base + "Level=1" + tag1Group, false}, {"_tag.t1", tag1Base + "(Tag1='t1') AND (Level=1)" + tag1Group, false}, {"_tag.p1=", tag1Base + "(Tag1 LIKE 'p1=%') AND (Level=1)" + tag1Group, false}, {"_tag.p1=.*", tag1Base + "(Tag1 LIKE 'p1=%') AND (Level=1)" + tag1Group, false}, {"_tag.p1=.v1", tag1Base + "(Tag1='p1=v1') AND (Level=1)" + tag1Group, false}, {"_tag.t2._tag.*", tagNBase + "(Tag1='t2') AND (IsLeaf=1)" + tagNGroup, false}, {"_tag.t2._tag.t2._tag.p3=.*", tagNBase + "(((Tag1='t2') AND (arrayExists((x) -> x='t2', Tags))) AND (TagN LIKE 'p3=%')) AND (IsLeaf=1)" + tagNGroup, false}, } for _, test := range table { testName := fmt.Sprintf("query: %#v", test.query) m := NewMockFinder([][]byte{[]byte("mock")}) f := WrapTag(m, "http://localhost:8123/", "table", clickhouse.Options{Timeout: time.Second, ConnectTimeout: time.Second}) sql, err := f.MakeSQL(test.query) if test.error { assert.Error(err) } else { assert.NoError(err) } assert.Equal(test.sql, sql, testName) } } func _TestTags(t *testing.T) { assert := assert.New(t) mockData := [][]byte{[]byte("mock")} type w []string mock := w{"mock"} empty := w{} table := []struct { query string expectedList []string expectedSeries []string }{ // not tagged query {"", mock, mock}, {"t*", mock, mock}, {"hello.*", mock, mock}, // list root {"*", w{"_tag.", "mock"}, mock}, // info about _tag "directory" {"_tag", w{"_tag."}, empty}, {"_tag.*", w{"_tag.t1.", "_tag.t2."}, empty}, {"_tag.t1", w{"_tag.t1.", "_tag.t2."}, empty}, {"_tag.t1.*", w{"_tag.t1.", "_tag.t2."}, empty}, {"_tag.t1._tag.*", w{"_tag.t1.", "_tag.t2."}, empty}, {"_tag.t1._tag.param=", w{"_tag.t1.", "_tag.t2."}, empty}, {"_tag.t1._tag.param=.value", w{"_tag.t1.", "_tag.t2."}, empty}, {"_tag.t1._tag.param=.value.*", w{"_tag.t1.", "_tag.t2."}, empty}, // {"hello", []string{"hello."}, []string{}}, // {"hello.*", []string{"hello.world"}, []string{"world"}}, // {"*.*", []string{"hello.world"}, []string{"world"}}, // {"*404*", []string{}, []string{}}, // {"*404*.*", []string{}, []string{}}, // {"hello.[bad regexp", []string{}, []string{}}, } for _, test := range table { testName := fmt.Sprintf("query: %#v", test.query) srv := chtest.NewTestServer() m := NewMockFinder(mockData) f := WrapTag(m, srv.URL, "graphite_tag", clickhouse.Options{Timeout: time.Second, ConnectTimeout: time.Second}) config := config.New() f.Execute(context.Background(), config, test.query, 0, 0) list := make([]string, 0) for _, r := range f.List() { list = append(list, string(r)) } series := make([]string, 0) for _, r := range f.Series() { series = append(series, string(r)) } assert.Equal(test.expectedList, list, testName+", list") assert.Equal(test.expectedSeries, series, testName+", series") srv.Close() } } ================================================ FILE: finder/tagged.go ================================================ package finder import ( "bytes" "context" "fmt" "net/http" "sort" "strings" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/helper/date" "github.com/lomik/graphite-clickhouse/helper/errs" "github.com/lomik/graphite-clickhouse/metrics" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/pkg/where" "github.com/msaf1980/go-stringutils" ) var ( ErrCostlySeriesByTag = errs.NewErrorWithCode("seriesByTag argument has too much wildcard and regex terms", http.StatusForbidden) ErrSyntaxSeriesByTag = errs.NewErrorWithCode("invalid seriesByTag syntax", http.StatusBadRequest) ErrNotEnoughArgsSeriesByTag = errs.NewErrorWithCode("not enough arguments in seriesByTag", http.StatusBadRequest) ) type TaggedTermOp int const ( TaggedTermEq TaggedTermOp = 1 TaggedTermMatch TaggedTermOp = 2 TaggedTermNe TaggedTermOp = 3 TaggedTermNotMatch TaggedTermOp = 4 ) type TaggedTerm struct { Key string Op TaggedTermOp Value string HasWildcard bool // only for TaggedTermEq NonDefaultCost bool Cost int // tag cost for use ad primary filter (use tag with maximal selectivity). 0 by default, minimal is better. // __name__ tag is prefered, if some tag has better selectivity than name, set it cost to < 0 // values with wildcards or regex matching also has lower priority, set if needed it cost to < 0 } type TaggedTermList []TaggedTerm func (s TaggedTermList) Len() int { return len(s) } func (s TaggedTermList) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s TaggedTermList) Less(i, j int) bool { if s[i].Op < s[j].Op { return true } if s[i].Op > s[j].Op { return false } if s[i].Op == TaggedTermEq && !s[i].HasWildcard && s[j].HasWildcard { // globs as fist eq might be have a bad perfomance return true } if s[i].Key == "__name__" && s[j].Key != "__name__" { return true } return false } type TaggedFinder struct { url string // clickhouse dsn table string // graphite_tag table tcq *TagCountQuerier // An object for querying tag weights from clickhouse. See doc/config.md for details. absKeepEncoded bool // Abs returns url encoded value. For queries from prometheus opts clickhouse.Options // clickhouse query timeout configuredTagCosts map[string]*config.Costs // costs for taggs (sor tune index search) dailyEnabled bool useCarbonBehavior bool dontMatchMissingTags bool metricMightExists bool // if false, skip all subsequent queries because we determined that result will be empty anyway stats []metrics.FinderStat body []byte // clickhouse response } func NewTagged(url string, table, tag1CountTable string, dailyEnabled, useCarbonBehavior, dontMatchMissingTags, absKeepEncoded bool, opts clickhouse.Options, taggedCosts map[string]*config.Costs) *TaggedFinder { fnd := &TaggedFinder{ url: url, table: table, tcq: nil, absKeepEncoded: absKeepEncoded, opts: opts, configuredTagCosts: taggedCosts, dailyEnabled: dailyEnabled, useCarbonBehavior: useCarbonBehavior, dontMatchMissingTags: dontMatchMissingTags, metricMightExists: true, stats: make([]metrics.FinderStat, 0), } if tag1CountTable != "" { fnd.tcq = NewTagCountQuerier( url, tag1CountTable, opts, useCarbonBehavior, dontMatchMissingTags, dailyEnabled, ) } return fnd } func (term *TaggedTerm) concat() string { return term.Key + "=" + term.Value } func (term *TaggedTerm) concatMask() string { v := strings.ReplaceAll(term.Value, "*", "%") return fmt.Sprintf("%s=%s", term.Key, v) } func TaggedTermWhere1(term *TaggedTerm, useCarbonBehaviour, dontMatchMissingTags bool) (string, error) { // positive expression check only in Tag1 // negative check in all Tags switch term.Op { case TaggedTermEq: if useCarbonBehaviour && term.Value == "" { // special case // container_name="" ==> response should not contain container_name return fmt.Sprintf("NOT arrayExists((x) -> %s, Tags)", where.HasPrefix("x", term.Key+"=")), nil } if strings.Contains(term.Value, "*") { return where.Like("Tag1", term.concatMask()), nil } var values []string if err := where.GlobExpandSimple(term.Value, term.Key+"=", &values); err != nil { return "", err } if len(values) == 1 { return where.Eq("Tag1", values[0]), nil } else if len(values) > 1 { return where.In("Tag1", values), nil } else { return where.Eq("Tag1", term.concat()), nil } case TaggedTermNe: if term.Value == "" { // special case // container_name!="" ==> container_name exists and it is not empty return where.HasPrefixAndNotEq("Tag1", term.Key+"="), nil } var whereLikeAnyVal string if dontMatchMissingTags { whereLikeAnyVal = where.HasPrefix("Tag1", term.Key+"=") + " AND " } if strings.Contains(term.Value, "*") { whereLike := where.Like("x", term.concatMask()) return fmt.Sprintf("%sNOT arrayExists((x) -> %s, Tags)", whereLikeAnyVal, whereLike), nil } var values []string if err := where.GlobExpandSimple(term.Value, term.Key+"=", &values); err != nil { return "", err } if len(values) == 1 { whereEq := where.Eq("x", values[0]) return fmt.Sprintf("%sNOT arrayExists((x) -> %s, Tags)", whereLikeAnyVal, whereEq), nil } else if len(values) > 1 { whereIn := where.In("x", values) return fmt.Sprintf("%sNOT arrayExists((x) -> %s, Tags)", whereLikeAnyVal, whereIn), nil } else { whereEq := where.Eq("x", term.concat()) return fmt.Sprintf("%sNOT arrayExists((x) -> %s, Tags)", whereLikeAnyVal, whereEq), nil } case TaggedTermMatch: return where.Match("Tag1", term.Key, term.Value), nil case TaggedTermNotMatch: var whereLikeAnyVal string if dontMatchMissingTags { whereLikeAnyVal = where.HasPrefix("Tag1", term.Key+"=") + " AND " } whereMatch := where.Match("x", term.Key, term.Value) return fmt.Sprintf("%sNOT arrayExists((x) -> %s, Tags)", whereLikeAnyVal, whereMatch), nil default: return "", nil } } func TaggedTermWhereN(term *TaggedTerm, useCarbonBehaviour, dontMatchMissingTags bool) (string, error) { // arrayExists((x) -> %s, Tags) switch term.Op { case TaggedTermEq: if useCarbonBehaviour && term.Value == "" { // special case // container_name="" ==> response should not contain container_name return fmt.Sprintf("NOT arrayExists((x) -> %s, Tags)", where.HasPrefix("x", term.Key+"=")), nil } if strings.Contains(term.Value, "*") { return fmt.Sprintf("arrayExists((x) -> %s, Tags)", where.Like("x", term.concatMask())), nil } var values []string if err := where.GlobExpandSimple(term.Value, term.Key+"=", &values); err != nil { return "", err } if len(values) == 1 { return where.ArrayHas("Tags", values[0]), nil } else if len(values) > 1 { w := where.New() for _, v := range values { w.Or(where.ArrayHas("Tags", v)) } return w.String(), nil } else { return where.ArrayHas("Tags", term.concat()), nil } case TaggedTermNe: if term.Value == "" { // special case // container_name!="" ==> container_name exists and it is not empty return fmt.Sprintf("arrayExists((x) -> %s, Tags)", where.HasPrefixAndNotEq("x", term.Key+"=")), nil } var whereLikeAnyVal string if dontMatchMissingTags { whereLikeAnyVal = fmt.Sprintf("arrayExists((x) -> %s, Tags) AND ", where.HasPrefix("x", term.Key+"=")) } if strings.Contains(term.Value, "*") { whereLike := where.Like("x", term.concatMask()) return fmt.Sprintf("%sNOT arrayExists((x) -> %s, Tags)", whereLikeAnyVal, whereLike), nil } var values []string if err := where.GlobExpandSimple(term.Value, term.Key+"=", &values); err != nil { return "", err } if len(values) == 1 { whereEq := where.Eq("x", values[0]) return fmt.Sprintf("%sNOT arrayExists((x) -> %s, Tags)", whereLikeAnyVal, whereEq), nil } else if len(values) > 1 { whereIn := where.In("x", values) return fmt.Sprintf("%sNOT arrayExists((x) -> %s, Tags)", whereLikeAnyVal, whereIn), nil } else { whereEq := where.Eq("x", term.concat()) return fmt.Sprintf("%sNOT arrayExists((x) -> %s, Tags)", whereLikeAnyVal, whereEq), nil } case TaggedTermMatch: return fmt.Sprintf("arrayExists((x) -> %s, Tags)", where.Match("x", term.Key, term.Value)), nil case TaggedTermNotMatch: var whereLikeAnyVal string if dontMatchMissingTags { whereLikeAnyVal = fmt.Sprintf("arrayExists((x) -> %s, Tags) AND ", where.HasPrefix("x", term.Key+"=")) } whereMatch := where.Match("x", term.Key, term.Value) return fmt.Sprintf("%sNOT arrayExists((x) -> %s, Tags)", whereLikeAnyVal, whereMatch), nil default: return "", nil } } func setCost(term *TaggedTerm, costs *config.Costs) { if term.Op == TaggedTermEq || term.Op == TaggedTermMatch { if len(costs.ValuesCost) > 0 { if cost, ok := costs.ValuesCost[term.Value]; ok { term.Cost = cost term.NonDefaultCost = true return } } if term.Op == TaggedTermEq && !term.HasWildcard && costs.Cost != nil { term.Cost = *costs.Cost // only for non-wildcared eq term.NonDefaultCost = true } } } func ParseTaggedConditions(conditions []string, config *config.Config, autocomplete bool) ([]TaggedTerm, error) { nonWildcards := 0 terms := make([]TaggedTerm, len(conditions)) for i := 0; i < len(conditions); i++ { s := conditions[i] a := strings.SplitN(s, "=", 2) if len(a) != 2 { return nil, fmt.Errorf("wrong seriesByTag expr: %#v", s) } a[0] = strings.TrimSpace(a[0]) a[1] = strings.TrimSpace(a[1]) op := "=" if len(a[0]) > 0 && a[0][len(a[0])-1] == '!' { op = "!" + op a[0] = strings.TrimSpace(a[0][:len(a[0])-1]) } if len(a[1]) > 0 && a[1][0] == '~' { op = op + "~" a[1] = strings.TrimSpace(a[1][1:]) } terms[i].Key = a[0] terms[i].Value = a[1] if terms[i].Key == "name" { terms[i].Key = "__name__" } switch op { case "=": terms[i].Op = TaggedTermEq terms[i].HasWildcard = where.HasWildcard(terms[i].Value) // special case when using useCarbonBehaviour = true // which matches everything that does not have that tag terms[i].HasWildcard = terms[i].HasWildcard || config.FeatureFlags.UseCarbonBehavior && terms[i].Value == "" if !terms[i].HasWildcard { nonWildcards++ } case "!=": terms[i].Op = TaggedTermNe case "=~": terms[i].Op = TaggedTermMatch case "!=~": terms[i].Op = TaggedTermNotMatch default: return nil, fmt.Errorf("wrong seriesByTag expr: %#v", s) } } if autocomplete { if config.ClickHouse.TagsMinInAutocomplete > 0 && nonWildcards < config.ClickHouse.TagsMinInAutocomplete { return nil, ErrCostlySeriesByTag } } else if config.ClickHouse.TagsMinInQuery > 0 && nonWildcards < config.ClickHouse.TagsMinInQuery { return nil, ErrCostlySeriesByTag } return terms, nil } func parseString(s string) (string, string, error) { if s[0] != '\'' && s[0] != '"' { panic("string should start with open quote") } match := s[0] s = s[1:] var i int for i < len(s) && s[i] != match { i++ } if i == len(s) { return "", "", errs.NewErrorfWithCode(http.StatusBadRequest, "seriesByTag arg missing quote %q'", s) } return s[:i], s[i+1:], nil } func seriesByTagArgs(query string) ([]string, error) { var err error args := make([]string, 0, 8) // trim spaces e := strings.Trim(query, " ") if !strings.HasPrefix(e, "seriesByTag(") { return nil, ErrSyntaxSeriesByTag } if e[len(e)-1] != ')' { return nil, ErrSyntaxSeriesByTag } e = e[12 : len(e)-1] for len(e) > 0 { var arg string if e[0] == '\'' || e[0] == '"' { if arg, e, err = parseString(e); err != nil { return nil, err } // skip empty arg if arg != "" { args = append(args, arg) } } else if e[0] == ' ' || e[0] == ',' { e = e[1:] } else { return nil, errs.NewErrorfWithCode(http.StatusBadRequest, "seriesByTag arg missing quote %q", e) } } return args, nil } func ParseSeriesByTag(query string, config *config.Config) ([]TaggedTerm, error) { conditions, err := seriesByTagArgs(query) if err != nil { return nil, err } if len(conditions) < 1 { return nil, ErrNotEnoughArgsSeriesByTag } return ParseTaggedConditions(conditions, config, false) } func TaggedWhere(terms []TaggedTerm, useCarbonBehaviour, dontMatchMissingTags bool) (*where.Where, *where.Where, error) { w := where.New() pw := where.New() x, err := TaggedTermWhere1(&terms[0], useCarbonBehaviour, dontMatchMissingTags) if err != nil { return nil, nil, err } if terms[0].Op == TaggedTermMatch { pw.And(x) } w.And(x) for i := 1; i < len(terms); i++ { and, err := TaggedTermWhereN(&terms[i], useCarbonBehaviour, dontMatchMissingTags) if err != nil { return nil, nil, err } w.And(and) } return w, pw, nil } func NewCachedTags(body []byte) *TaggedFinder { return &TaggedFinder{ body: body, } } func (t *TaggedFinder) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64) error { terms, err := t.PrepareTaggedTerms(ctx, config, query, from, until) if err != nil { return err } return t.ExecutePrepared(ctx, terms, from, until) } func (t *TaggedFinder) whereFilter(terms []TaggedTerm, from int64, until int64) (*where.Where, *where.Where, error) { w, pw, err := TaggedWhere(terms, t.useCarbonBehavior, t.dontMatchMissingTags) if err != nil { return nil, nil, err } if t.dailyEnabled { w.Andf( "Date >='%s' AND Date <= '%s'", date.FromTimestampToDaysFormat(from), date.UntilTimestampToDaysFormat(until), ) } else { w.Andf( "Date >='%s'", date.FromTimestampToDaysFormat(from), ) } return w, pw, nil } func (t *TaggedFinder) ExecutePrepared(ctx context.Context, terms []TaggedTerm, from int64, until int64) error { w, pw, err := t.whereFilter(terms, from, until) if err != nil { return err } t.stats = append(t.stats, metrics.FinderStat{}) stat := &t.stats[len(t.stats)-1] // TODO: consider consistent query generator sql := fmt.Sprintf("SELECT Path FROM %s %s %s GROUP BY Path FORMAT TabSeparatedRaw", t.table, pw.PreWhereSQL(), w.SQL()) t.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query(scope.WithTable(ctx, t.table), t.url, sql, t.opts, nil) stat.Table = t.table stat.ReadBytes = int64(len(t.body)) return err } func (t *TaggedFinder) List() [][]byte { if t.body == nil { return [][]byte{} } rows := bytes.Split(t.body, []byte{'\n'}) skip := 0 for i := 0; i < len(rows); i++ { if len(rows[i]) == 0 { skip++ continue } if skip > 0 { rows[i-skip] = rows[i] } } rows = rows[:len(rows)-skip] return rows } func (t *TaggedFinder) Series() [][]byte { return t.List() } func tagsParse(path string) (string, []string, error) { name, args, n := stringutils.Split2(path, "?") if n == 1 || args == "" { return name, nil, fmt.Errorf("incomplete tags in '%s'", path) } tags := strings.Split(args, "&") for i := range tags { tags[i] = unescape(tags[i]) } return unescape(name), tags, nil } func TaggedDecode(v []byte) []byte { s := stringutils.UnsafeString(v) name, tags, err := tagsParse(s) if err != nil { return v } if len(tags) == 0 { return stringutils.UnsafeStringBytes(&name) } sort.Strings(tags) var sb stringutils.Builder length := len(name) for _, tag := range tags { length += len(tag) + 1 } sb.Grow(length) sb.WriteString(name) for _, tag := range tags { sb.WriteString(";") sb.WriteString(tag) } return sb.Bytes() } func (t *TaggedFinder) Abs(v []byte) []byte { if t.absKeepEncoded { return v } return TaggedDecode(v) } func (t *TaggedFinder) Bytes() ([]byte, error) { return nil, ErrNotImplemented } func (t *TaggedFinder) Stats() []metrics.FinderStat { return t.stats } func (t *TaggedFinder) PrepareTaggedTerms(ctx context.Context, cfg *config.Config, query string, from int64, until int64) (terms []TaggedTerm, err error) { terms, err = ParseSeriesByTag(query, cfg) if err != nil { return nil, err } var tagCounts map[string]*config.Costs = nil if t.tcq != nil { tagCounts, err = t.tcq.GetCostsFromCountTable(ctx, terms, from, until) if err != nil { return nil, err } } if tagCounts != nil { SetCosts(terms, tagCounts) } else if len(t.configuredTagCosts) != 0 { SetCosts(terms, t.configuredTagCosts) } SortTaggedTermsByCost(terms) return terms, nil } func SortTaggedTermsByCost(terms []TaggedTerm) { // compare with taggs costs sort.Slice(terms, func(i, j int) bool { // compare taggs costs, if all of TaggegTerms has custom cost. // this is allow overwrite operators order (Eq with or without wildcards/Match), use with carefully if terms[i].Cost != terms[j].Cost { if terms[i].NonDefaultCost && terms[j].NonDefaultCost || (terms[i].NonDefaultCost && terms[j].Op == TaggedTermEq && !terms[j].HasWildcard) || (terms[j].NonDefaultCost && terms[i].Op == TaggedTermEq && !terms[i].HasWildcard) { return terms[i].Cost < terms[j].Cost } } if terms[i].Op == terms[j].Op { if terms[i].Op == TaggedTermEq && !terms[i].HasWildcard && terms[j].HasWildcard { // globs as fist eq might be have a bad perfomance return true } if terms[i].Key == "__name__" && terms[j].Key != "__name__" { return true } if terms[i].Cost != terms[j].Cost && terms[i].HasWildcard == terms[j].HasWildcard { // compare taggs costs return terms[i].Cost < terms[j].Cost } return false } else { return terms[i].Op < terms[j].Op } }) } func SetCosts(terms []TaggedTerm, costs map[string]*config.Costs) { for i := 0; i < len(terms); i++ { if cost, ok := costs[terms[i].Key]; ok { setCost(&terms[i], cost) } } } ================================================ FILE: finder/tagged_test.go ================================================ package finder import ( "context" "fmt" "net/url" "sort" "strconv" "testing" "time" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/helper/date" chtest "github.com/lomik/graphite-clickhouse/helper/tests/clickhouse" "github.com/lomik/graphite-clickhouse/pkg/where" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestTaggedWhere(t *testing.T) { assert := assert.New(t) require := require.New(t) table := []struct { query string minTags int where string prewhere string isErr bool }{ // test for issue #195 {"seriesByTag()", 0, "", "", true}, {"seriesByTag('')", 0, "", "", true}, // incomplete {"seriesByTag('key=value)", 0, "", "", true}, // missing quote {"seriesByTag(key=value)", 0, "", "", true}, // info about _tag "directory" {"seriesByTag('key=value')", 0, "Tag1='key=value'", "", false}, {"seriesByTag('key=value')", 1, "Tag1='key=value'", "", false}, {"seriesByTag('key=value')", 2, "", "", true}, // test case for wildcarded name, must be not first check {"seriesByTag('name=*', 'key=value')", 0, "(Tag1='key=value') AND (arrayExists((x) -> x LIKE '__name__=%', Tags))", "", false}, {"seriesByTag('name=*', 'key=value')", 1, "(Tag1='key=value') AND (arrayExists((x) -> x LIKE '__name__=%', Tags))", "", false}, {"seriesByTag('name=*', 'key=value')", 2, "", "", true}, {"seriesByTag('name=*', 'key=value*')", 0, "(Tag1 LIKE '__name__=%') AND (arrayExists((x) -> x LIKE 'key=value%', Tags))", "", false}, {"seriesByTag('name=rps')", 0, "Tag1='__name__=rps'", "", false}, {"seriesByTag('name=~cpu.usage')", 0, "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu.usage')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu.usage')", false}, {"seriesByTag('name=~cpu.usage')", 1, "", "", true}, {"seriesByTag('name=~cpu|mem')", 0, "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem)')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem)')", false}, {"seriesByTag('name=~cpu|mem$')", 0, "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem$)')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem$)')", false}, {"seriesByTag('name=~^cpu|mem')", 0, "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem)')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem)')", false}, {"seriesByTag('name=~^cpu|mem$')", 0, "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem$)')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem$)')", false}, {"seriesByTag('name=rps', 'key=~value')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=%' AND match(x, '^key=.*value'), Tags))", "", false}, // test for issue #244 {"seriesByTag('name=rps', 'key=~^value')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=value%' AND match(x, '^key=value'), Tags))", "", false}, {"seriesByTag('name=rps', 'key=~^value.*')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=value%' AND match(x, '^key=value.*'), Tags))", "", false}, {"seriesByTag('name=rps', 'key=~^valu[a-e]')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=valu%' AND match(x, '^key=valu[a-e]'), Tags))", "", false}, {"seriesByTag('name=rps', 'key=~^value$')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x='key=value', Tags))", "", false}, {"seriesByTag('name=rps', 'key=~hello.world')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=%' AND match(x, '^key=.*hello.world'), Tags))", "", false}, {`seriesByTag('cpu=cpu-total','host=~Vladimirs-MacBook-Pro\.local')`, 0, `(Tag1='cpu=cpu-total') AND (arrayExists((x) -> x LIKE 'host=%' AND match(x, '^host=.*Vladimirs-MacBook-Pro\\.local'), Tags))`, "", false}, // grafana multi-value variable produce this {"seriesByTag('name=value','what=*')", 0, "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))", "", false}, // If All masked to value with * {"seriesByTag('name=value','what=*x')", 0, "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%x', Tags))", "", false}, // If All masked to value with * {"seriesByTag('name=value','what!=*x')", 0, "(Tag1='__name__=value') AND (NOT arrayExists((x) -> x LIKE 'what=%x', Tags))", "", false}, // If All masked to value with * {"seriesByTag('name={avg,max}')", 0, "Tag1 IN ('__name__=avg','__name__=max')", "", false}, {"seriesByTag('name=m{in}')", 0, "Tag1='__name__=min'", "", false}, {"seriesByTag('name=m{in,ax}')", 0, "Tag1 IN ('__name__=min','__name__=max')", "", false}, {"seriesByTag('name=m{in,ax')", 0, "", "", true}, {"seriesByTag('name=value','what={avg,max}')", 0, "(Tag1='__name__=value') AND ((has(Tags, 'what=avg')) OR (has(Tags, 'what=max')))", "", false}, {"seriesByTag('name=value','what!={avg,max}')", 0, "(Tag1='__name__=value') AND (NOT arrayExists((x) -> x IN ('what=avg','what=max'), Tags))", "", false}, // grafana workaround for multi-value variables default, masked with * {"seriesByTag('name=value','what=~*')", 0, "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))", "", false}, // If All masked to value with * // empty tag value during autocompletion {"seriesByTag('name=value','what=~')", 0, "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))", "", false}, // If All masked to value with * // testcases for useCarbonBehaviour=false {"seriesByTag('what=')", 0, "Tag1='what='", "", false}, {"seriesByTag('name=value','what=')", 0, "(Tag1='__name__=value') AND (has(Tags, 'what='))", "", false}, // testcases for dontMatchMissingTags=false {"seriesByTag('key!=value')", 0, "NOT arrayExists((x) -> x='key=value', Tags)", "", false}, {"seriesByTag('dc!=de', 'cpu=cpu-total')", 0, "(Tag1='cpu=cpu-total') AND (NOT arrayExists((x) -> x='dc=de', Tags))", "", false}, {"seriesByTag('dc!=de', 'cpu!=cpu-total')", 0, "(NOT arrayExists((x) -> x='dc=de', Tags)) AND (NOT arrayExists((x) -> x='cpu=cpu-total', Tags))", "", false}, {"seriesByTag('dc!=~de|us', 'cpu=cpu-total')", 0, "(Tag1='cpu=cpu-total') AND (NOT arrayExists((x) -> x LIKE 'dc=%' AND match(x, '^dc=.*(de|us)'), Tags))", "", false}, } for i, test := range table { t.Run(test.query+"#"+strconv.Itoa(i), func(t *testing.T) { testName := fmt.Sprintf("query: %#v", test.query) config := config.New() config.ClickHouse.TagsMinInQuery = test.minTags terms, err := ParseSeriesByTag(test.query, config) sort.Sort(TaggedTermList(terms)) if test.isErr { if err != nil { return } } require.NoError(err, testName+", err") var w, pw *where.Where if err == nil { w, pw, err = TaggedWhere(terms, false, false) } if test.isErr { require.Error(err, testName+", err") return } else { assert.NoError(err, testName+", err") } assert.Equal(test.where, w.String(), testName+", where") assert.Equal(test.prewhere, pw.String(), testName+", prewhere") }) } } func TestTaggedWhere_UseCarbonBehaviourFlag(t *testing.T) { assert := assert.New(t) require := require.New(t) table := []struct { query string minTags int where string prewhere string isErr bool }{ // test for issue #195 {"seriesByTag()", 0, "", "", true}, {"seriesByTag('')", 0, "", "", true}, // incomplete {"seriesByTag('key=value)", 0, "", "", true}, // missing quote {"seriesByTag(key=value)", 0, "", "", true}, // info about _tag "directory" {"seriesByTag('key=value')", 0, "Tag1='key=value'", "", false}, {"seriesByTag('key=value')", 1, "Tag1='key=value'", "", false}, {"seriesByTag('key=value')", 2, "", "", true}, // test case for wildcarded name, must be not first check {"seriesByTag('name=*', 'key=value')", 0, "(Tag1='key=value') AND (arrayExists((x) -> x LIKE '__name__=%', Tags))", "", false}, {"seriesByTag('name=*', 'key=value')", 1, "(Tag1='key=value') AND (arrayExists((x) -> x LIKE '__name__=%', Tags))", "", false}, {"seriesByTag('name=*', 'key=value')", 2, "", "", true}, {"seriesByTag('name=*', 'key=value*')", 0, "(Tag1 LIKE '__name__=%') AND (arrayExists((x) -> x LIKE 'key=value%', Tags))", "", false}, {"seriesByTag('name=rps')", 0, "Tag1='__name__=rps'", "", false}, {"seriesByTag('name=~cpu.usage')", 0, "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu.usage')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu.usage')", false}, {"seriesByTag('name=~cpu.usage')", 1, "", "", true}, {"seriesByTag('name=~cpu|mem')", 0, "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem)')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem)')", false}, {"seriesByTag('name=~cpu|mem$')", 0, "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem$)')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem$)')", false}, {"seriesByTag('name=~^cpu|mem')", 0, "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem)')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem)')", false}, {"seriesByTag('name=~^cpu|mem$')", 0, "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem$)')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem$)')", false}, {"seriesByTag('name=rps', 'key=~value')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=%' AND match(x, '^key=.*value'), Tags))", "", false}, // test for issue #244 {"seriesByTag('name=rps', 'key=~^value')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=value%' AND match(x, '^key=value'), Tags))", "", false}, {"seriesByTag('name=rps', 'key=~^value.*')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=value%' AND match(x, '^key=value.*'), Tags))", "", false}, {"seriesByTag('name=rps', 'key=~^valu[a-e]')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=valu%' AND match(x, '^key=valu[a-e]'), Tags))", "", false}, {"seriesByTag('name=rps', 'key=~^value$')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x='key=value', Tags))", "", false}, {"seriesByTag('name=rps', 'key=~hello.world')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=%' AND match(x, '^key=.*hello.world'), Tags))", "", false}, {`seriesByTag('cpu=cpu-total','host=~Vladimirs-MacBook-Pro\.local')`, 0, `(Tag1='cpu=cpu-total') AND (arrayExists((x) -> x LIKE 'host=%' AND match(x, '^host=.*Vladimirs-MacBook-Pro\\.local'), Tags))`, "", false}, // grafana multi-value variable produce this {"seriesByTag('name=value','what=*')", 0, "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))", "", false}, // If All masked to value with * {"seriesByTag('name=value','what=*x')", 0, "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%x', Tags))", "", false}, // If All masked to value with * {"seriesByTag('name=value','what!=*x')", 0, "(Tag1='__name__=value') AND (NOT arrayExists((x) -> x LIKE 'what=%x', Tags))", "", false}, // If All masked to value with * {"seriesByTag('name={avg,max}')", 0, "Tag1 IN ('__name__=avg','__name__=max')", "", false}, {"seriesByTag('name=m{in}')", 0, "Tag1='__name__=min'", "", false}, {"seriesByTag('name=m{in,ax}')", 0, "Tag1 IN ('__name__=min','__name__=max')", "", false}, {"seriesByTag('name=m{in,ax')", 0, "", "", true}, {"seriesByTag('name=value','what={avg,max}')", 0, "(Tag1='__name__=value') AND ((has(Tags, 'what=avg')) OR (has(Tags, 'what=max')))", "", false}, {"seriesByTag('name=value','what!={avg,max}')", 0, "(Tag1='__name__=value') AND (NOT arrayExists((x) -> x IN ('what=avg','what=max'), Tags))", "", false}, // grafana workaround for multi-value variables default, masked with * {"seriesByTag('name=value','what=~*')", 0, "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))", "", false}, // If All masked to value with * // empty tag value during autocompletion {"seriesByTag('name=value','what=~')", 0, "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))", "", false}, // If All masked to value with * // testcases for useCarbonBehaviour=true {"seriesByTag('what=')", 0, "NOT arrayExists((x) -> x LIKE 'what=%', Tags)", "", false}, {"seriesByTag('name=value','what=')", 0, "(Tag1='__name__=value') AND (NOT arrayExists((x) -> x LIKE 'what=%', Tags))", "", false}, // testcases for dontMatchMissingTags=false {"seriesByTag('key!=value')", 0, "NOT arrayExists((x) -> x='key=value', Tags)", "", false}, {"seriesByTag('dc!=de', 'cpu=cpu-total')", 0, "(Tag1='cpu=cpu-total') AND (NOT arrayExists((x) -> x='dc=de', Tags))", "", false}, {"seriesByTag('dc!=de', 'cpu!=cpu-total')", 0, "(NOT arrayExists((x) -> x='dc=de', Tags)) AND (NOT arrayExists((x) -> x='cpu=cpu-total', Tags))", "", false}, {"seriesByTag('dc!=~de|us', 'cpu=cpu-total')", 0, "(Tag1='cpu=cpu-total') AND (NOT arrayExists((x) -> x LIKE 'dc=%' AND match(x, '^dc=.*(de|us)'), Tags))", "", false}, } for i, test := range table { t.Run(test.query+"#"+strconv.Itoa(i), func(t *testing.T) { testName := fmt.Sprintf("query: %#v", test.query) config := config.New() config.ClickHouse.TagsMinInQuery = test.minTags terms, err := ParseSeriesByTag(test.query, config) sort.Sort(TaggedTermList(terms)) if test.isErr { if err != nil { return } } require.NoError(err, testName+", err") var w, pw *where.Where if err == nil { w, pw, err = TaggedWhere(terms, true, false) } if test.isErr { require.Error(err, testName+", err") return } else { assert.NoError(err, testName+", err") } assert.Equal(test.where, w.String(), testName+", where") assert.Equal(test.prewhere, pw.String(), testName+", prewhere") }) } } func TestTaggedWhere_DontMatchMissingTagsFlag(t *testing.T) { assert := assert.New(t) require := require.New(t) table := []struct { query string minTags int where string prewhere string isErr bool }{ // test for issue #195 {"seriesByTag()", 0, "", "", true}, {"seriesByTag('')", 0, "", "", true}, // incomplete {"seriesByTag('key=value)", 0, "", "", true}, // missing quote {"seriesByTag(key=value)", 0, "", "", true}, // info about _tag "directory" {"seriesByTag('key=value')", 0, "Tag1='key=value'", "", false}, {"seriesByTag('key=value')", 1, "Tag1='key=value'", "", false}, {"seriesByTag('key=value')", 2, "", "", true}, // test case for wildcarded name, must be not first check {"seriesByTag('name=*', 'key=value')", 0, "(Tag1='key=value') AND (arrayExists((x) -> x LIKE '__name__=%', Tags))", "", false}, {"seriesByTag('name=*', 'key=value')", 1, "(Tag1='key=value') AND (arrayExists((x) -> x LIKE '__name__=%', Tags))", "", false}, {"seriesByTag('name=*', 'key=value')", 2, "", "", true}, {"seriesByTag('name=*', 'key=value*')", 0, "(Tag1 LIKE '__name__=%') AND (arrayExists((x) -> x LIKE 'key=value%', Tags))", "", false}, {"seriesByTag('name=rps')", 0, "Tag1='__name__=rps'", "", false}, {"seriesByTag('name=~cpu.usage')", 0, "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu.usage')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu.usage')", false}, {"seriesByTag('name=~cpu.usage')", 1, "", "", true}, {"seriesByTag('name=~cpu|mem')", 0, "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem)')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem)')", false}, {"seriesByTag('name=~cpu|mem$')", 0, "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem$)')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem$)')", false}, {"seriesByTag('name=~^cpu|mem')", 0, "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem)')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem)')", false}, {"seriesByTag('name=~^cpu|mem$')", 0, "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem$)')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem$)')", false}, {"seriesByTag('name=rps', 'key=~value')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=%' AND match(x, '^key=.*value'), Tags))", "", false}, // test for issue #244 {"seriesByTag('name=rps', 'key=~^value')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=value%' AND match(x, '^key=value'), Tags))", "", false}, {"seriesByTag('name=rps', 'key=~^value.*')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=value%' AND match(x, '^key=value.*'), Tags))", "", false}, {"seriesByTag('name=rps', 'key=~^valu[a-e]')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=valu%' AND match(x, '^key=valu[a-e]'), Tags))", "", false}, {"seriesByTag('name=rps', 'key=~^value$')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x='key=value', Tags))", "", false}, {"seriesByTag('name=rps', 'key=~hello.world')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=%' AND match(x, '^key=.*hello.world'), Tags))", "", false}, {`seriesByTag('cpu=cpu-total','host=~Vladimirs-MacBook-Pro\.local')`, 0, `(Tag1='cpu=cpu-total') AND (arrayExists((x) -> x LIKE 'host=%' AND match(x, '^host=.*Vladimirs-MacBook-Pro\\.local'), Tags))`, "", false}, // grafana multi-value variable produce this {"seriesByTag('name=value','what=*')", 0, "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))", "", false}, // If All masked to value with * {"seriesByTag('name=value','what=*x')", 0, "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%x', Tags))", "", false}, // If All masked to value with * // {"seriesByTag('name=value','what!=*x')", 0, "(Tag1='__name__=value') AND (NOT arrayExists((x) -> x LIKE 'what=%x', Tags))", "", false}, // If All masked to value with * {"seriesByTag('name=value','what!=*x')", 0, "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags) AND NOT arrayExists((x) -> x LIKE 'what=%x', Tags))", "", false}, {"seriesByTag('name={avg,max}')", 0, "Tag1 IN ('__name__=avg','__name__=max')", "", false}, {"seriesByTag('name=m{in}')", 0, "Tag1='__name__=min'", "", false}, {"seriesByTag('name=m{in,ax}')", 0, "Tag1 IN ('__name__=min','__name__=max')", "", false}, {"seriesByTag('name=m{in,ax')", 0, "", "", true}, {"seriesByTag('name=value','what={avg,max}')", 0, "(Tag1='__name__=value') AND ((has(Tags, 'what=avg')) OR (has(Tags, 'what=max')))", "", false}, // {"seriesByTag('name=value','what!={avg,max}')", 0, "(Tag1='__name__=value') AND (NOT arrayExists((x) -> x IN ('what=avg','what=max'), Tags))", "", false}, {"seriesByTag('name=value','what!={avg,max}')", 0, "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags) AND NOT arrayExists((x) -> x IN ('what=avg','what=max'), Tags))", "", false}, // grafana workaround for multi-value variables default, masked with * {"seriesByTag('name=value','what=~*')", 0, "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))", "", false}, // If All masked to value with * // empty tag value during autocompletion {"seriesByTag('name=value','what=~')", 0, "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))", "", false}, // If All masked to value with * // testcases for useCarbonBehaviour=false {"seriesByTag('what=')", 0, "Tag1='what='", "", false}, {"seriesByTag('name=value','what=')", 0, "(Tag1='__name__=value') AND (has(Tags, 'what='))", "", false}, // testcases for dontMatchMissingTags=true {"seriesByTag('key!=value')", 0, "Tag1 LIKE 'key=%' AND NOT arrayExists((x) -> x='key=value', Tags)", "", false}, {"seriesByTag('dc!=de', 'cpu=cpu-total')", 0, "(Tag1='cpu=cpu-total') AND (arrayExists((x) -> x LIKE 'dc=%', Tags) AND NOT arrayExists((x) -> x='dc=de', Tags))", "", false}, {"seriesByTag('dc!=de', 'cpu!=cpu-total')", 0, "(Tag1 LIKE 'dc=%' AND NOT arrayExists((x) -> x='dc=de', Tags)) AND (arrayExists((x) -> x LIKE 'cpu=%', Tags) AND NOT arrayExists((x) -> x='cpu=cpu-total', Tags))", "", false}, {"seriesByTag('dc!=~de|us', 'cpu=cpu-total')", 0, "(Tag1='cpu=cpu-total') AND (arrayExists((x) -> x LIKE 'dc=%', Tags) AND NOT arrayExists((x) -> x LIKE 'dc=%' AND match(x, '^dc=.*(de|us)'), Tags))", "", false}, } for i, test := range table { t.Run(test.query+"#"+strconv.Itoa(i), func(t *testing.T) { testName := fmt.Sprintf("query: %#v", test.query) config := config.New() config.ClickHouse.TagsMinInQuery = test.minTags terms, err := ParseSeriesByTag(test.query, config) sort.Sort(TaggedTermList(terms)) if test.isErr { if err != nil { return } } require.NoError(err, testName+", err") var w, pw *where.Where if err == nil { w, pw, err = TaggedWhere(terms, false, true) } if test.isErr { require.Error(err, testName+", err") return } else { assert.NoError(err, testName+", err") } assert.Equal(test.where, w.String(), testName+", where") assert.Equal(test.prewhere, pw.String(), testName+", prewhere") }) } } func TestTaggedWhere_BothFeatureFlags(t *testing.T) { assert := assert.New(t) require := require.New(t) table := []struct { query string minTags int where string prewhere string isErr bool }{ // test for issue #195 {"seriesByTag()", 0, "", "", true}, {"seriesByTag('')", 0, "", "", true}, // incomplete {"seriesByTag('key=value)", 0, "", "", true}, // missing quote {"seriesByTag(key=value)", 0, "", "", true}, // info about _tag "directory" {"seriesByTag('key=value')", 0, "Tag1='key=value'", "", false}, {"seriesByTag('key=value')", 1, "Tag1='key=value'", "", false}, {"seriesByTag('key=value')", 2, "", "", true}, // test case for wildcarded name, must be not first check {"seriesByTag('name=*', 'key=value')", 0, "(Tag1='key=value') AND (arrayExists((x) -> x LIKE '__name__=%', Tags))", "", false}, {"seriesByTag('name=*', 'key=value')", 1, "(Tag1='key=value') AND (arrayExists((x) -> x LIKE '__name__=%', Tags))", "", false}, {"seriesByTag('name=*', 'key=value')", 2, "", "", true}, {"seriesByTag('name=*', 'key=value*')", 0, "(Tag1 LIKE '__name__=%') AND (arrayExists((x) -> x LIKE 'key=value%', Tags))", "", false}, {"seriesByTag('name=rps')", 0, "Tag1='__name__=rps'", "", false}, {"seriesByTag('name=~cpu.usage')", 0, "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu.usage')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu.usage')", false}, {"seriesByTag('name=~cpu.usage')", 1, "", "", true}, {"seriesByTag('name=~cpu|mem')", 0, "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem)')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem)')", false}, {"seriesByTag('name=~cpu|mem$')", 0, "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem$)')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*(cpu|mem$)')", false}, {"seriesByTag('name=~^cpu|mem')", 0, "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem)')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem)')", false}, {"seriesByTag('name=~^cpu|mem$')", 0, "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem$)')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=(cpu|mem$)')", false}, {"seriesByTag('name=rps', 'key=~value')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=%' AND match(x, '^key=.*value'), Tags))", "", false}, // test for issue #244 {"seriesByTag('name=rps', 'key=~^value')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=value%' AND match(x, '^key=value'), Tags))", "", false}, {"seriesByTag('name=rps', 'key=~^value.*')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=value%' AND match(x, '^key=value.*'), Tags))", "", false}, {"seriesByTag('name=rps', 'key=~^valu[a-e]')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=valu%' AND match(x, '^key=valu[a-e]'), Tags))", "", false}, {"seriesByTag('name=rps', 'key=~^value$')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x='key=value', Tags))", "", false}, {"seriesByTag('name=rps', 'key=~hello.world')", 0, "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=%' AND match(x, '^key=.*hello.world'), Tags))", "", false}, {`seriesByTag('cpu=cpu-total','host=~Vladimirs-MacBook-Pro\.local')`, 0, `(Tag1='cpu=cpu-total') AND (arrayExists((x) -> x LIKE 'host=%' AND match(x, '^host=.*Vladimirs-MacBook-Pro\\.local'), Tags))`, "", false}, // grafana multi-value variable produce this {"seriesByTag('name=value','what=*')", 0, "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))", "", false}, // If All masked to value with * {"seriesByTag('name=value','what=*x')", 0, "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%x', Tags))", "", false}, // If All masked to value with * // {"seriesByTag('name=value','what!=*x')", 0, "(Tag1='__name__=value') AND (NOT arrayExists((x) -> x LIKE 'what=%x', Tags))", "", false}, // If All masked to value with * {"seriesByTag('name=value','what!=*x')", 0, "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags) AND NOT arrayExists((x) -> x LIKE 'what=%x', Tags))", "", false}, {"seriesByTag('name={avg,max}')", 0, "Tag1 IN ('__name__=avg','__name__=max')", "", false}, {"seriesByTag('name=m{in}')", 0, "Tag1='__name__=min'", "", false}, {"seriesByTag('name=m{in,ax}')", 0, "Tag1 IN ('__name__=min','__name__=max')", "", false}, {"seriesByTag('name=m{in,ax')", 0, "", "", true}, {"seriesByTag('name=value','what={avg,max}')", 0, "(Tag1='__name__=value') AND ((has(Tags, 'what=avg')) OR (has(Tags, 'what=max')))", "", false}, // {"seriesByTag('name=value','what!={avg,max}')", 0, "(Tag1='__name__=value') AND (NOT arrayExists((x) -> x IN ('what=avg','what=max'), Tags))", "", false}, {"seriesByTag('name=value','what!={avg,max}')", 0, "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags) AND NOT arrayExists((x) -> x IN ('what=avg','what=max'), Tags))", "", false}, // grafana workaround for multi-value variables default, masked with * {"seriesByTag('name=value','what=~*')", 0, "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))", "", false}, // If All masked to value with * // empty tag value during autocompletion {"seriesByTag('name=value','what=~')", 0, "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))", "", false}, // If All masked to value with * // testcases for useCarbonBehaviour=true {"seriesByTag('what=')", 0, "NOT arrayExists((x) -> x LIKE 'what=%', Tags)", "", false}, {"seriesByTag('name=value','what=')", 0, "(Tag1='__name__=value') AND (NOT arrayExists((x) -> x LIKE 'what=%', Tags))", "", false}, // testcases for dontMatchMissingTags=true {"seriesByTag('key!=value')", 0, "Tag1 LIKE 'key=%' AND NOT arrayExists((x) -> x='key=value', Tags)", "", false}, {"seriesByTag('dc!=de', 'cpu=cpu-total')", 0, "(Tag1='cpu=cpu-total') AND (arrayExists((x) -> x LIKE 'dc=%', Tags) AND NOT arrayExists((x) -> x='dc=de', Tags))", "", false}, {"seriesByTag('dc!=de', 'cpu!=cpu-total')", 0, "(Tag1 LIKE 'dc=%' AND NOT arrayExists((x) -> x='dc=de', Tags)) AND (arrayExists((x) -> x LIKE 'cpu=%', Tags) AND NOT arrayExists((x) -> x='cpu=cpu-total', Tags))", "", false}, {"seriesByTag('dc!=~de|us', 'cpu=cpu-total')", 0, "(Tag1='cpu=cpu-total') AND (arrayExists((x) -> x LIKE 'dc=%', Tags) AND NOT arrayExists((x) -> x LIKE 'dc=%' AND match(x, '^dc=.*(de|us)'), Tags))", "", false}, } for i, test := range table { t.Run(test.query+"#"+strconv.Itoa(i), func(t *testing.T) { testName := fmt.Sprintf("query: %#v", test.query) config := config.New() config.ClickHouse.TagsMinInQuery = test.minTags terms, err := ParseSeriesByTag(test.query, config) sort.Sort(TaggedTermList(terms)) if test.isErr { if err != nil { return } } require.NoError(err, testName+", err") var w, pw *where.Where if err == nil { w, pw, err = TaggedWhere(terms, true, true) } if test.isErr { require.Error(err, testName+", err") return } else { assert.NoError(err, testName+", err") } assert.Equal(test.where, w.String(), testName+", where") assert.Equal(test.prewhere, pw.String(), testName+", prewhere") }) } } func TestParseSeriesByTag(t *testing.T) { assert := assert.New(t) ok := func(query string, expected []TaggedTerm) { config := config.New() p, err := ParseSeriesByTag(query, config) assert.NoError(err) assert.Equal(len(expected), len(p)) length := len(expected) if length < len(p) { length = len(p) } for i := 0; i < length; i++ { if i >= len(p) { t.Errorf("%s\n- [%d]=%+v", query, i, expected[i]) } else if i >= len(expected) { t.Errorf("%s\n+ [%d]=%+v", query, i, p[i]) } else if p[i] != expected[i] { t.Errorf("%s\n- [%d]=%+v\n+ [%d]=%+v", query, i, expected[i], i, p[i]) } } } ok(`seriesByTag('key=value')`, []TaggedTerm{ {Op: TaggedTermEq, Key: "key", Value: "value"}, }) ok(`seriesByTag('name=rps')`, []TaggedTerm{ {Op: TaggedTermEq, Key: "__name__", Value: "rps"}, }) ok(`seriesByTag('name=~cpu.usage')`, []TaggedTerm{ {Op: TaggedTermMatch, Key: "__name__", Value: "cpu.usage"}, }) ok(`seriesByTag('name!=cpu.usage')`, []TaggedTerm{ {Op: TaggedTermNe, Key: "__name__", Value: "cpu.usage"}, }) ok(`seriesByTag('name!=~cpu.usage')`, []TaggedTerm{ {Op: TaggedTermNotMatch, Key: "__name__", Value: "cpu.usage"}, }) ok(`seriesByTag('cpu=cpu-total','host=~Vladimirs-MacBook-Pro\.local')`, []TaggedTerm{ {Op: TaggedTermEq, Key: "cpu", Value: "cpu-total"}, {Op: TaggedTermMatch, Key: "host", Value: `Vladimirs-MacBook-Pro\.local`}, }) } func newInt(i int) *int { p := new(int) *p = i return p } func TestParseSeriesByTagWithCosts(t *testing.T) { assert := assert.New(t) taggedCosts := map[string]*config.Costs{ "environment": {Cost: newInt(100)}, "dc": {Cost: newInt(60)}, "project": {Cost: newInt(50)}, "__name__": {Cost: newInt(0), ValuesCost: map[string]int{"high_cost": 70}}, "key": {ValuesCost: map[string]int{"value2": 70, "value3": -1, "val*4": -1, "^val.*4$": -1}}, } ok := func(query string, expected []TaggedTerm) { config := config.New() config.ClickHouse.TaggedCosts = taggedCosts terms, err := ParseSeriesByTag(query, config) SetCosts(terms, config.ClickHouse.TaggedCosts) SortTaggedTermsByCost(terms) assert.NoError(err) length := len(expected) if length < len(terms) { length = len(terms) } for i := 0; i < length; i++ { if i >= len(terms) { t.Errorf("%s\n- [%d]=%+v", query, i, expected[i]) } else if i >= len(expected) { t.Errorf("%s\n+ [%d]=%+v", query, i, terms[i]) } else if terms[i] != expected[i] { t.Errorf("%s\n- [%d]=%+v\n+ [%d]=%+v", query, i, expected[i], i, terms[i]) } } } ok(`seriesByTag('environment=production', 'dc=west', 'key=value')`, []TaggedTerm{ {Op: TaggedTermEq, Key: "key", Value: "value"}, {Op: TaggedTermEq, Key: "dc", Value: "west", Cost: 60, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "environment", Value: "production", Cost: 100, NonDefaultCost: true}, }) // Check for values cost (key=value2) ok(`seriesByTag('environment=production', 'dc=west', 'key=value2')`, []TaggedTerm{ {Op: TaggedTermEq, Key: "dc", Value: "west", Cost: 60, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "key", Value: "value2", Cost: 70, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "environment", Value: "production", Cost: 100, NonDefaultCost: true}, }) // Check for __name_ preference ok(`seriesByTag('environment=production', 'dc=west', 'key=value', 'name=cpu.load_avg')`, []TaggedTerm{ {Op: TaggedTermEq, Key: "__name__", Value: "cpu.load_avg", Cost: 0, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "key", Value: "value"}, {Op: TaggedTermEq, Key: "dc", Value: "west", Cost: 60, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "environment", Value: "production", Cost: 100, NonDefaultCost: true}, }) // Check for __name_ preference overrided ok(`seriesByTag('environment=production', 'dc=west', 'name=cpu.load_avg', 'key=value3')`, []TaggedTerm{ {Op: TaggedTermEq, Key: "key", Value: "value3", Cost: -1, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "__name__", Value: "cpu.load_avg", Cost: 0, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "dc", Value: "west", Cost: 60, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "environment", Value: "production", Cost: 100, NonDefaultCost: true}, }) // wildcard (dc=west*) ok(`seriesByTag('environment=production', 'dc=west*', 'name=cpu.load_avg', 'key=value3')`, []TaggedTerm{ {Op: TaggedTermEq, Key: "key", Value: "value3", Cost: -1, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "__name__", Value: "cpu.load_avg", Cost: 0, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "environment", Value: "production", Cost: 100, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "dc", Value: "west*", HasWildcard: true}, }) // wildcard cost -1 ok(`seriesByTag('dc=west*', 'environment=production', 'name=cpu.load_avg', 'key=val*4')`, []TaggedTerm{ {Op: TaggedTermEq, Key: "key", Value: "val*4", Cost: -1, HasWildcard: true, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "__name__", Value: "cpu.load_avg", Cost: 0, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "environment", Value: "production", Cost: 100, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "dc", Value: "west*", HasWildcard: true}, }) // match cost -1 - not as wildcard ok(`seriesByTag('dc=~west.*', 'environment=production', 'name=cpu.load_avg', 'key=~^val.*4$')`, []TaggedTerm{ {Op: TaggedTermMatch, Key: "key", Value: "^val.*4$", Cost: -1, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "__name__", Value: "cpu.load_avg", Cost: 0, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "environment", Value: "production", Cost: 100, NonDefaultCost: true}, {Op: TaggedTermMatch, Key: "dc", Value: "west.*"}, }) // match cost -1 - and no cost ok(`seriesByTag('dc=~west.*', 'environment=production', 'Name=cpu.load_avg', 'key=~^val.*4$')`, []TaggedTerm{ {Op: TaggedTermMatch, Key: "key", Value: "^val.*4$", Cost: -1, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "Name", Value: "cpu.load_avg"}, {Op: TaggedTermEq, Key: "environment", Value: "production", Cost: 100, NonDefaultCost: true}, {Op: TaggedTermMatch, Key: "dc", Value: "west.*"}, }) // reduce cost for __name__ ok(`seriesByTag('dc=~west.*', 'environment=production', 'name=high_cost', 'key=~^val.*4$', 'key2=~^val.*4$', 'key3=val.*4')`, []TaggedTerm{ {Op: TaggedTermMatch, Key: "key", Value: "^val.*4$", Cost: -1, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "__name__", Value: "high_cost", Cost: 70, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "environment", Value: "production", Cost: 100, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "key3", Value: "val.*4", HasWildcard: true}, {Op: TaggedTermMatch, Key: "dc", Value: "west.*"}, {Op: TaggedTermMatch, Key: "key2", Value: "^val.*4$"}, }) } func BenchmarkParseSeriesByTag(b *testing.B) { benchmarks := []string{ "seriesByTag('key=value')", "seriesByTag('name=*', 'key=value')", "seriesByTag('name=value', '')", } for _, bm := range benchmarks { b.Run(bm, func(b *testing.B) { for i := 0; i < b.N; i++ { _, _ = ParseSeriesByTag(bm, nil) } }) } } func TestParseSeriesByTagWithCostsFromCountTable(t *testing.T) { assert := assert.New(t) var from, until int64 // 2022-11-11 00:01:00 +05:00 && 2022-11-11 00:01:10 +05:00 from, until = 1668106860, 1668106870 taggedCosts := map[string]*config.Costs{ "environment": {Cost: newInt(100)}, "dc": {Cost: newInt(60)}, "project": {Cost: newInt(50)}, "__name__": {Cost: newInt(0), ValuesCost: map[string]int{"high_cost": 70}}, "key": {ValuesCost: map[string]int{"value2": 70, "value3": -1, "val*4": -1, "^val.*4$": -1}}, } ok := func( testName, query, sql string, response *chtest.TestResponse, expected []TaggedTerm, expectedErr error, useTagCostsFromConfig bool, ) { srv := chtest.NewTestServer() defer srv.Close() cfg, _ := config.DefaultConfig() cfg.ClickHouse.URL = srv.URL if useTagCostsFromConfig { cfg.ClickHouse.TaggedCosts = taggedCosts } srv.AddResponce(sql, response) opts := clickhouse.Options{ Timeout: cfg.ClickHouse.IndexTimeout, ConnectTimeout: cfg.ClickHouse.ConnectTimeout, TLSConfig: cfg.ClickHouse.TLSConfig, CheckRequestProgress: cfg.FeatureFlags.LogQueryProgress, ProgressSendingInterval: cfg.ClickHouse.ProgressSendingInterval, } taggedFinder := NewTagged( cfg.ClickHouse.URL, cfg.ClickHouse.TaggedTable, "tag1_count_table", // non-empty string, tagged finder will query clickhouse for costs true, cfg.FeatureFlags.UseCarbonBehavior, cfg.FeatureFlags.DontMatchMissingTags, false, opts, cfg.ClickHouse.TaggedCosts, ) terms, err := taggedFinder.PrepareTaggedTerms(context.Background(), cfg, query, from, until) if expectedErr != nil { assert.Equal(expectedErr, err, testName+", err") return } assert.NoError(err) length := len(expected) if length < len(terms) { length = len(terms) } for i := 0; i < length; i++ { if i >= len(terms) { t.Errorf("%s\n- [%d]=%+v", testName, i, expected[i]) } else if i >= len(expected) { t.Errorf("%s\n+ [%d]=%+v", testName, i, terms[i]) } else if terms[i] != expected[i] { t.Errorf("%s\n- [%d]=%+v\n+ [%d]=%+v", testName, i, expected[i], i, terms[i]) } } } ok( `3 TaggedTermEq, database contains all of them`, `seriesByTag('environment=production', 'dc=west', 'key=value')`, `SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+ `(((Tag1='environment=production') OR (Tag1='dc=west')) OR (Tag1='key=value')) `+ `AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+ `GROUP BY Tag1 FORMAT TabSeparatedRaw`, &chtest.TestResponse{ Body: []byte("environment=production\t100\ndc=west\t10\nkey=value\t1\n"), }, []TaggedTerm{ {Op: TaggedTermEq, Key: "key", Value: "value", Cost: 1, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "dc", Value: "west", Cost: 10, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "environment", Value: "production", Cost: 100, NonDefaultCost: true}, }, nil, false, ) ok( `3 TaggedTermEq, database doesn't contain 1 of them`, `seriesByTag('environment=production', 'dc=west', 'key=value')`, `SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+ `(((Tag1='environment=production') OR (Tag1='dc=west')) OR (Tag1='key=value')) `+ `AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+ `GROUP BY Tag1 FORMAT TabSeparatedRaw`, &chtest.TestResponse{ Body: []byte("environment=production\t100\nkey=value\t1\n"), }, []TaggedTerm{ {Op: TaggedTermEq, Key: "environment", Value: "production", Cost: 0, NonDefaultCost: false}, {Op: TaggedTermEq, Key: "dc", Value: "west", Cost: 0, NonDefaultCost: false}, {Op: TaggedTermEq, Key: "key", Value: "value", Cost: 0, NonDefaultCost: false}, }, nil, false, ) ok( `3 TaggedTermEq, database doesn't contain any of them`, `seriesByTag('environment=production', 'dc=west', 'key=value')`, `SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+ `(((Tag1='environment=production') OR (Tag1='dc=west')) OR (Tag1='key=value')) `+ `AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+ `GROUP BY Tag1 FORMAT TabSeparatedRaw`, &chtest.TestResponse{ Body: []byte(""), }, []TaggedTerm{ {Op: TaggedTermEq, Key: "environment", Value: "production", Cost: 0, NonDefaultCost: false}, {Op: TaggedTermEq, Key: "dc", Value: "west", Cost: 0, NonDefaultCost: false}, {Op: TaggedTermEq, Key: "key", Value: "value", Cost: 0, NonDefaultCost: false}, }, nil, false, ) ok( `3 TaggedTermEq, one of them has a wildcard`, `seriesByTag('environment=production', 'dc=*', 'key=value')`, `SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+ `((Tag1='environment=production') OR (Tag1='key=value')) `+ `AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+ `GROUP BY Tag1 FORMAT TabSeparatedRaw`, &chtest.TestResponse{ Body: []byte("environment=production\t100\nkey=value\t2\n"), }, []TaggedTerm{ {Op: TaggedTermEq, Key: "key", Value: "value", Cost: 2, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "environment", Value: "production", Cost: 100, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "dc", Value: "*", Cost: 0, NonDefaultCost: false, HasWildcard: true}, }, nil, false, ) ok( `2 TaggedTermEq, 2 TaggedTermMatch`, `seriesByTag('environment=production', 'dc=west', 'status=~^o.*', 'key=~val.*')`, `SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+ `((Tag1='environment=production') OR (Tag1='dc=west')) `+ `AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+ `GROUP BY Tag1 FORMAT TabSeparatedRaw`, &chtest.TestResponse{ Body: []byte("environment=production\t100\ndc=west\t10\n"), }, []TaggedTerm{ {Op: TaggedTermEq, Key: "dc", Value: "west", Cost: 10, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "environment", Value: "production", Cost: 100, NonDefaultCost: true}, {Op: TaggedTermMatch, Key: "status", Value: "^o.*", Cost: 0}, {Op: TaggedTermMatch, Key: "key", Value: "val.*", Cost: 0}, }, nil, false, ) ok( `2 TaggedTermEq, 2 TaggedTermNe`, `seriesByTag('environment=production', 'dc=west', 'status!=on', 'key!=value')`, `SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+ `((Tag1='environment=production') OR (Tag1='dc=west')) `+ `AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+ `GROUP BY Tag1 FORMAT TabSeparatedRaw`, &chtest.TestResponse{ Body: []byte("environment=production\t100\ndc=west\t10\n"), }, []TaggedTerm{ {Op: TaggedTermEq, Key: "dc", Value: "west", Cost: 10, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "environment", Value: "production", Cost: 100, NonDefaultCost: true}, {Op: TaggedTermNe, Key: "status", Value: "on", Cost: 0}, {Op: TaggedTermNe, Key: "key", Value: "value", Cost: 0}, }, nil, false, ) ok( `2 TaggedTermEq, 2 TaggedTermNotMatch`, `seriesByTag('environment=production', 'dc=west', 'status!=~^o.*', 'key!=~val.*')`, `SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+ `((Tag1='environment=production') OR (Tag1='dc=west')) `+ `AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+ `GROUP BY Tag1 FORMAT TabSeparatedRaw`, &chtest.TestResponse{ Body: []byte("environment=production\t100\ndc=west\t10\n"), }, []TaggedTerm{ {Op: TaggedTermEq, Key: "dc", Value: "west", Cost: 10, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "environment", Value: "production", Cost: 100, NonDefaultCost: true}, {Op: TaggedTermNotMatch, Key: "status", Value: "^o.*", Cost: 0}, {Op: TaggedTermNotMatch, Key: "key", Value: "val.*", Cost: 0}, }, nil, false, ) ok( `3 TaggedTermMatch`, `seriesByTag('environment=~prod', 'dc=~west', 'key=~^val')`, ``, nil, []TaggedTerm{ {Op: TaggedTermMatch, Key: "environment", Value: "prod", Cost: 0}, {Op: TaggedTermMatch, Key: "dc", Value: "west", Cost: 0}, {Op: TaggedTermMatch, Key: "key", Value: "^val", Cost: 0}, }, nil, false, ) ok( `3 TaggedTermEq, 2 have same tag`, `seriesByTag('environment=production', 'dc=west', 'dc=east')`, `SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+ `(((Tag1='environment=production') OR (Tag1='dc=west')) OR (Tag1='dc=east')) `+ `AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+ `GROUP BY Tag1 FORMAT TabSeparatedRaw`, &chtest.TestResponse{ Body: []byte("environment=production\t100\ndc=west\t10\ndc=east\t5\n"), }, []TaggedTerm{ {Op: TaggedTermEq, Key: "dc", Value: "east", Cost: 5, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "dc", Value: "west", Cost: 10, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "environment", Value: "production", Cost: 100, NonDefaultCost: true}, }, nil, false, ) ok( `3 TaggedTermEq, 1 of them has __name__ key`, `seriesByTag('name=load.avg', 'environment=production', 'dc=west')`, `SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+ `(((Tag1='__name__=load.avg') OR (Tag1='environment=production')) OR (Tag1='dc=west')) `+ `AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+ `GROUP BY Tag1 FORMAT TabSeparatedRaw`, &chtest.TestResponse{ Body: []byte("environment=production\t100\ndc=west\t10\n__name__=load.avg\t10000\n"), }, []TaggedTerm{ {Op: TaggedTermEq, Key: "dc", Value: "west", Cost: 10, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "environment", Value: "production", Cost: 100, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "__name__", Value: "load.avg", Cost: 10000, NonDefaultCost: true}, }, nil, false, ) ok( `3 TaggedTermEq, 1 of them has __name__ key, 1 does not exist in count table`, `seriesByTag('environment=production', 'dc=west', 'name=load.avg')`, `SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+ `(((Tag1='environment=production') OR (Tag1='dc=west')) OR (Tag1='__name__=load.avg')) `+ `AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+ `GROUP BY Tag1 FORMAT TabSeparatedRaw`, &chtest.TestResponse{ Body: []byte("environment=production\t100\n__name__=load.avg\t10000\n"), }, []TaggedTerm{ {Op: TaggedTermEq, Key: "__name__", Value: "load.avg", Cost: 0, NonDefaultCost: false}, {Op: TaggedTermEq, Key: "environment", Value: "production", Cost: 0, NonDefaultCost: false}, {Op: TaggedTermEq, Key: "dc", Value: "west", Cost: 0, NonDefaultCost: false}, }, nil, false, ) ok( `Clickhouse returned broken response`, `seriesByTag('environment=production', 'dc=west', 'key=value')`, `SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+ `(((Tag1='environment=production') OR (Tag1='dc=west')) OR (Tag1='key=value')) `+ `AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+ `GROUP BY Tag1 FORMAT TabSeparatedRaw`, &chtest.TestResponse{ Body: []byte("broken_response"), }, nil, fmt.Errorf("failed to parse result from clickhouse while querying for tag costs: no tag count"), false, ) ok( `3 TaggedTermEq, 1 of them has __name__ key, 1 does not exist in count table, fallback to config costs`, `seriesByTag('name=high_cost', 'environment=production', 'dc=west')`, `SELECT Tag1, sum(Count) as cnt FROM tag1_count_table WHERE `+ `(((Tag1='__name__=high_cost') OR (Tag1='environment=production')) OR (Tag1='dc=west')) `+ `AND (Date >= '`+date.FromTimestampToDaysFormat(from)+`' AND Date <= '`+date.FromTimestampToDaysFormat(until)+`') `+ `GROUP BY Tag1 FORMAT TabSeparatedRaw`, &chtest.TestResponse{ Body: []byte("environment=production\t100\n__name__=load.avg\t10000\n"), }, []TaggedTerm{ {Op: TaggedTermEq, Key: "dc", Value: "west", Cost: 60, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "__name__", Value: "high_cost", Cost: 70, NonDefaultCost: true}, {Op: TaggedTermEq, Key: "environment", Value: "production", Cost: 100, NonDefaultCost: true}, }, nil, true, ) } func TestTaggedFinder_whereFilter(t *testing.T) { tests := []struct { name string query string from int64 until int64 dailyEnabled bool useCarbonBehavior bool dontMatchMissingTags bool taggedCosts map[string]*config.Costs want string wantPre string }{ { name: "nodaily", query: "seriesByTag('name=metric')", from: 1668106860, // 2022-11-11 00:01:00 +05:00 until: 1668106870, // 2022-11-11 00:01:10 +05:00 dailyEnabled: false, want: "(Tag1='__name__=metric') AND (Date >='" + date.FromTimestampToDaysFormat(1668106860) + "')", wantPre: "", }, { name: "midnight at utc (direct)", query: "seriesByTag('name=metric')", from: 1668124800, // 2022-11-11 00:00:00 UTC until: 1668124810, // 2022-11-11 00:00:10 UTC dailyEnabled: true, want: "(Tag1='__name__=metric') AND (Date >='" + date.FromTimestampToDaysFormat(1668124800) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(1668124810) + "')", wantPre: "", }, { name: "", query: "seriesByTag('emptyval=', 'what=value')", from: 1668124800, // 2022-11-11 00:00:00 UTC until: 1668124810, // 2022-11-11 00:00:10 UTC dailyEnabled: true, useCarbonBehavior: true, want: "((Tag1='what=value') AND (NOT arrayExists((x) -> x LIKE 'emptyval=%', Tags))) AND (Date >='" + date.FromTimestampToDaysFormat(1668124800) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(1668124810) + "')", wantPre: "", }, } for _, tt := range tests { t.Run(tt.name+" "+time.Unix(tt.from, 0).Format(time.RFC3339), func(t *testing.T) { config := config.New() config.ClickHouse.TaggedCosts = tt.taggedCosts config.FeatureFlags.UseCarbonBehavior = tt.useCarbonBehavior config.FeatureFlags.DontMatchMissingTags = tt.dontMatchMissingTags f := NewTagged( "http://localhost:8123/", "graphite_tags", "", tt.dailyEnabled, tt.useCarbonBehavior, tt.dontMatchMissingTags, false, clickhouse.Options{}, tt.taggedCosts, ) terms, err := f.PrepareTaggedTerms(context.Background(), config, tt.query, tt.from, tt.until) if err != nil { t.Fatal(err) } got, gotDate, err := f.whereFilter(terms, tt.from, tt.until) if err != nil { t.Fatal(err) } if got.String() != tt.want { t.Errorf("TaggedFinder.whereFilter()[0] = %v, want %v", got, tt.want) } if gotDate.String() != tt.wantPre { t.Errorf("TaggedFinder.whereFilter()[1] = %v, want %v", gotDate, tt.wantPre) } }) } } func TestTaggedFinder_Abs(t *testing.T) { tests := []struct { name string v []byte cached bool want []byte }{ { name: "cached", v: []byte("test_metric;colon=:;forward=/;hash=#;host=127.0.0.1;minus=-;percent=%;plus=+;underscore=_"), cached: true, want: []byte("test_metric;colon=:;forward=/;hash=#;host=127.0.0.1;minus=-;percent=%;plus=+;underscore=_"), }, { name: "escaped", v: []byte(url.QueryEscape("instance:cpu_utilization?ratio_avg") + "?" + url.QueryEscape("dc") + "=" + url.QueryEscape("qwe+1") + "&" + url.QueryEscape("fqdn") + "=" + url.QueryEscape("asd&a") + "&" + url.QueryEscape("instance") + "=" + url.QueryEscape("10.33.10.10:9100") + "&" + url.QueryEscape("job") + "=" + url.QueryEscape("node&a")), want: []byte("instance:cpu_utilization?ratio_avg;dc=qwe+1;fqdn=asd&a;instance=10.33.10.10:9100;job=node&a"), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var tf *TaggedFinder if tt.cached { tf = NewCachedTags(nil) } else { tf = NewTagged("http:/127.0.0.1:8123", "graphite_tags", "", true, false, false, false, clickhouse.Options{}, nil) } if got := string(tf.Abs(tt.v)); got != string(tt.want) { t.Errorf("TaggedDecode() =\n%q\nwant\n%q", got, string(tt.want)) } }) } } ================================================ FILE: finder/tags_count_querier.go ================================================ package finder import ( "bytes" "context" "fmt" "strconv" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/helper/date" "github.com/lomik/graphite-clickhouse/metrics" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/pkg/where" "github.com/msaf1980/go-stringutils" ) type TagCountQuerier struct { url string table string opts clickhouse.Options useCarbonBehavior bool dontMatchMissingTags bool dailyEnabled bool body []byte stats []metrics.FinderStat } func NewTagCountQuerier(url, table string, opts clickhouse.Options, useCarbonBehavior, dontMatchMissingTags, dailyEnabled bool) *TagCountQuerier { return &TagCountQuerier{ url: url, table: table, opts: opts, useCarbonBehavior: useCarbonBehavior, dontMatchMissingTags: dontMatchMissingTags, dailyEnabled: dailyEnabled, } } func (tcq *TagCountQuerier) GetCostsFromCountTable(ctx context.Context, terms []TaggedTerm, from int64, until int64) (map[string]*config.Costs, error) { if len(terms) < 2 { return nil, nil } w := where.New() eqTermCount := 0 for i := 0; i < len(terms); i++ { if terms[i].Op == TaggedTermEq && !terms[i].HasWildcard && terms[i].Value != "" { sqlTerm, err := TaggedTermWhere1(&terms[i], tcq.useCarbonBehavior, tcq.dontMatchMissingTags) if err != nil { return nil, err } w.Or(sqlTerm) eqTermCount++ } } if w.SQL() == "" { return nil, nil } if tcq.dailyEnabled { w.Andf( "Date >= '%s' AND Date <= '%s'", date.FromTimestampToDaysFormat(from), date.UntilTimestampToDaysFormat(until), ) } else { w.Andf( "Date >= '%s'", date.FromTimestampToDaysFormat(from), ) } sql := fmt.Sprintf("SELECT Tag1, sum(Count) as cnt FROM %s %s GROUP BY Tag1 FORMAT TabSeparatedRaw", tcq.table, w.SQL()) var err error tcq.stats = append(tcq.stats, metrics.FinderStat{}) stat := &tcq.stats[len(tcq.stats)-1] stat.Table = tcq.table tcq.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query(scope.WithTable(ctx, tcq.table), tcq.url, sql, tcq.opts, nil) if err != nil { return nil, err } rows := tcq.List() // create cost var to validate CH response without writing to t.taggedCosts var costs map[string]*config.Costs costs, err = chResultToCosts(rows) if err != nil { return nil, err } // The metric does not exist if the response has less rows // than there were tags with '=' op in the initial request // This is due to each tag-value pair of a metric being written // exactly one time as Tag1 if len(rows) < eqTermCount { tcq.body = []byte{} return nil, nil } return costs, nil } func chResultToCosts(body [][]byte) (map[string]*config.Costs, error) { costs := make(map[string]*config.Costs, 0) for i := 0; i < len(body); i++ { s := stringutils.UnsafeString(body[i]) tag, val, count, err := parseTag1CountRow(s) if err != nil { return nil, fmt.Errorf("failed to parse result from clickhouse while querying for tag costs: %s", err.Error()) } if costs[tag] == nil { costs[tag] = &config.Costs{Cost: nil, ValuesCost: make(map[string]int, 0)} } costs[tag].ValuesCost[val] = count } return costs, nil } func parseTag1CountRow(s string) (string, string, int, error) { var ( tag1, count, tag, val string cnt, n int err error ) if tag1, count, n = stringutils.Split2(s, "\t"); n != 2 { return "", "", 0, fmt.Errorf("no tag count") } if tag, val, n = stringutils.Split2(tag1, "="); n != 2 { return "", "", 0, fmt.Errorf("no '=' in Tag1") } if cnt, err = strconv.Atoi(count); err != nil { return "", "", 0, fmt.Errorf("can't convert count to int") } return tag, val, cnt, nil } func (t *TagCountQuerier) List() [][]byte { if t.body == nil { return [][]byte{} } rows := bytes.Split(t.body, []byte{'\n'}) skip := 0 for i := 0; i < len(rows); i++ { if len(rows[i]) == 0 { skip++ continue } if skip > 0 { rows[i-skip] = rows[i] } } rows = rows[:len(rows)-skip] return rows } func (tcq *TagCountQuerier) Stats() []metrics.FinderStat { return tcq.stats } ================================================ FILE: finder/unescape.go ================================================ package finder import "strings" func ishex(c byte) bool { switch { case '0' <= c && c <= '9': return true case 'a' <= c && c <= 'f': return true case 'A' <= c && c <= 'F': return true } return false } func unhex(c byte) byte { switch { case '0' <= c && c <= '9': return c - '0' case 'a' <= c && c <= 'f': return c - 'a' + 10 case 'A' <= c && c <= 'F': return c - 'A' + 10 } return 0 } func isPercentEscape(s string, i int) bool { return i+2 < len(s) && ishex(s[i+1]) && ishex(s[i+2]) } // unescape unescapes a string. func unescape(s string) string { first := strings.IndexByte(s, '%') if first == -1 { return s } var t strings.Builder t.Grow(len(s)) t.WriteString(s[:first]) LOOP: for i := first; i < len(s); i++ { switch s[i] { case '%': if len(s) < i+3 { t.WriteString(s[i:]) break LOOP } if !isPercentEscape(s, i) { t.WriteString(s[i : i+3]) } else { t.WriteByte(unhex(s[i+1])<<4 | unhex(s[i+2])) } i += 2 default: t.WriteByte(s[i]) } } return t.String() } ================================================ FILE: go.mod ================================================ module github.com/lomik/graphite-clickhouse go 1.23.1 require ( github.com/BurntSushi/toml v0.3.1 github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 github.com/cactus/go-statsd-client/v5 v5.1.0 github.com/go-graphite/carbonapi v0.16.1 github.com/go-graphite/protocol v1.0.0 github.com/gogo/protobuf v1.3.2 github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc github.com/json-iterator/go v1.1.12 github.com/klauspost/compress v1.17.9 github.com/lomik/carbon-clickhouse v0.11.8 github.com/lomik/graphite-pickle v0.0.0-20171221213606-614e8df42119 github.com/lomik/og-rek v0.0.0-20170411191824-628eefeb8d80 github.com/lomik/prometheus-ui-static v0.2.54-1.1 github.com/lomik/zapwriter v0.0.0-20210624082824-c1161d1eb463 github.com/msaf1980/go-expirecache v0.0.2 github.com/msaf1980/go-metrics v0.0.14 github.com/msaf1980/go-stringutils v0.1.6 github.com/msaf1980/go-syncutils v0.0.3 github.com/msaf1980/go-timeutils v0.0.4 github.com/pelletier/go-toml v1.9.5 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.20.3 github.com/prometheus/client_model v0.6.1 github.com/prometheus/common/assets v0.2.0 github.com/prometheus/prometheus v0.0.0-20240827104400-e6cfa720fbe6 github.com/stretchr/testify v1.9.0 go.uber.org/zap v1.24.0 golang.org/x/sync v0.12.0 ) require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 // indirect github.com/ansel1/merry v1.6.2 // indirect github.com/ansel1/merry/v2 v2.0.2 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aws/aws-sdk-go v1.55.5 // indirect github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dennwc/varint v1.0.0 // indirect github.com/digitalocean/godo v1.122.0 // indirect github.com/docker/docker v27.2.0+incompatible // indirect github.com/edsrzf/mmap-go v1.1.0 // indirect github.com/envoyproxy/go-control-plane v0.13.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/analysis v0.22.2 // indirect github.com/go-openapi/errors v0.22.0 // indirect github.com/go-openapi/jsonpointer v0.20.2 // indirect github.com/go-openapi/jsonreference v0.20.4 // indirect github.com/go-openapi/loads v0.21.5 // indirect github.com/go-openapi/spec v0.20.14 // indirect github.com/go-openapi/strfmt v0.23.0 // indirect github.com/go-openapi/swag v0.22.9 // indirect github.com/go-openapi/validate v0.23.0 // indirect github.com/go-zookeeper/zk v1.0.4 // indirect github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gophercloud/gophercloud v1.14.0 // indirect github.com/hashicorp/consul/api v1.29.4 // indirect github.com/hetznercloud/hcloud-go/v2 v2.13.1 // indirect github.com/ionos-cloud/sdk-go/v6 v6.2.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/linode/linodego v1.40.0 // indirect github.com/lomik/stop v0.0.0-20161127103810-188e98d969bd // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mdlayher/socket v0.4.1 // indirect github.com/mdlayher/vsock v1.2.1 // indirect github.com/miekg/dns v1.1.62 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/alertmanager v0.27.0 // indirect github.com/prometheus/common v0.59.1 // indirect github.com/prometheus/common/sigv4 v0.1.0 // indirect github.com/prometheus/exporter-toolkit v0.12.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30 // indirect github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c // indirect github.com/stretchr/objx v0.5.2 // indirect go.mongodb.org/mongo-driver v1.14.0 // indirect go.opentelemetry.io/collector/pdata v1.14.1 // indirect go.opentelemetry.io/collector/semconv v0.108.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect go.opentelemetry.io/otel v1.29.0 // indirect go.opentelemetry.io/otel/metric v1.29.0 // indirect go.opentelemetry.io/otel/trace v1.29.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/goleak v1.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.36.0 // indirect golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect golang.org/x/sys v0.31.0 // indirect golang.org/x/text v0.23.0 // indirect golang.org/x/time v0.6.0 // indirect golang.org/x/tools v0.24.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/grpc v1.66.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apimachinery v0.31.0 // indirect k8s.io/client-go v0.31.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect ) replace ( k8s.io/klog => github.com/simonpasquier/klog-gokit v0.3.0 k8s.io/klog/v2 => github.com/simonpasquier/klog-gokit/v3 v3.5.0 ) // Exclude linodego v1.0.0 as it is no longer published on github. exclude github.com/linode/linodego v1.0.0 // Exclude grpc v1.30.0 because of breaking changes. See #7621. exclude ( github.com/grpc-ecosystem/grpc-gateway v1.14.7 google.golang.org/api v0.30.0 ) ================================================ FILE: go.sum ================================================ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 h1:LkHbJbgF3YyvC53aqYGR+wWQDn2Rdp9AQdGndf9QvY4= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0/go.mod h1:QyiQdW4f4/BIfB8ZutZ2s+28RAgfa/pT+zS++ZHyM1I= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0 h1:bXwSugBiSbgtz7rOtbfGf+woewp4f06orW9OP5BjHLA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0/go.mod h1:Y/HgrePTmGy9HjdSGTqZNa+apUpTVIEVKXJyARP2lrk= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Code-Hex/go-generics-cache v1.5.1 h1:6vhZGc5M7Y/YD8cIUcY8kcuQLB4cHR7U+0KMqAA0KcU= github.com/Code-Hex/go-generics-cache v1.5.1/go.mod h1:qxcC9kRVrct9rHeiYpFWSoW1vxyillCVzX13KZG8dl4= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Shopify/sarama v1.29.0/go.mod h1:2QpgD79wpdAESqNQMxNc0KYMkycd4slxGdV3TWSVqrU= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 h1:t3eaIm0rUkzbrIewtiFmMK5RXHej2XnoXNhxVsAYUfg= github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= github.com/ansel1/merry v1.5.0/go.mod h1:wUy/yW0JX0ix9GYvUbciq+bi3jW/vlKPlbpI7qdZpOw= github.com/ansel1/merry v1.5.1/go.mod h1:wUy/yW0JX0ix9GYvUbciq+bi3jW/vlKPlbpI7qdZpOw= github.com/ansel1/merry v1.6.2 h1:0xr40haRrfVzmOH/JVOu7KOKGEI1c/7q5EmgTEbn+Ng= github.com/ansel1/merry v1.6.2/go.mod h1:pAcMW+2uxIgpzEON021vMtFsrymREY6faJWiiz1QGVQ= github.com/ansel1/merry/v2 v2.0.1/go.mod h1:dD5OhpiPrVkvgseRYd+xgYlx7s6ytU3v9BTTJlDA7FM= github.com/ansel1/merry/v2 v2.0.2 h1:xPHMhTp2iOkGCN1q+vqetL1Ww692cFuiASHjUMMjAiY= github.com/ansel1/merry/v2 v2.0.2/go.mod h1:dD5OhpiPrVkvgseRYd+xgYlx7s6ytU3v9BTTJlDA7FM= github.com/ansel1/vespucci/v4 v4.1.1/go.mod h1:zzdrO4IgBfgcGMbGTk/qNGL8JPslmW3nPpcBHKReFYY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps= github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 h1:N7oVaKyGp8bttX0bfZGmcGkjz7DLQXhAn3DNd3T0ous= github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874/go.mod h1:r5xuitiExdLAJ09PR7vBVENGvp4ZuTBeWTGtxuX3K+c= github.com/cactus/go-statsd-client/v5 v5.1.0 h1:sbbdfIl9PgisjEoXzvXI1lwUKWElngsjJKaZeC021P4= github.com/cactus/go-statsd-client/v5 v5.1.0/go.mod h1:COEvJ1E+/E2L4q6QE5CkjWPi4eeDw9maJBMIuMPBZbY= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnThWgvH2wg8376yUJmPhEH4H3kw= github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE= github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= github.com/digitalocean/godo v1.122.0 h1:ziytLQi8QKtDp2K1A+YrYl2dWLHLh2uaMzWvcz9HkKg= github.com/digitalocean/godo v1.122.0/go.mod h1:WQVH83OHUy6gC4gXpEVQKtxTd4L5oCp+5OialidkPLY= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/docker v27.2.0+incompatible h1:Rk9nIVdfH3+Vz4cyI/uhbINhEZ/oLmc+CBXmH6fbNk4= github.com/docker/docker v27.2.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/eclipse/paho.mqtt.golang v1.3.5/go.mod h1:eTzb4gxwwyWpqBUHGQZ4ABAV7+Jgm1PklsYT/eo8Hcc= github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.13.0 h1:HzkeUz1Knt+3bK+8LG1bxOO/jzWZmdxpwC51i202les= github.com/envoyproxy/go-control-plane v0.13.0/go.mod h1:GRaKG3dwvFoTg4nj7aXdZnvMg4d7nvT/wl9WgVXn3Q8= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM= github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4= github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb h1:IT4JYU7k4ikYg1SCxNI1/Tieq/NFvh6dzLdgi7eu0tM= github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb/go.mod h1:bH6Xx7IW64qjjJq8M2u4dxNaBiDfKK+z/3eGDpXEQhc= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-graphite/carbonapi v0.16.1 h1:pbFmR0CqQH9tWYAehOXKZjfbaJRKVx0yJBs4dGFtSdg= github.com/go-graphite/carbonapi v0.16.1/go.mod h1:vefIVMyWf471SFMpHS4CSbySjGaMPtNxmMTnTlU2tSc= github.com/go-graphite/protocol v1.0.0 h1:Fqb0mkVVtfMrn6vw6Ntm3raf3gVVZCOVdZu4JosW5qE= github.com/go-graphite/protocol v1.0.0/go.mod h1:eonkg/0UGhJUYu+PshOg1NzWSUcXskr/yHeQXJHJr8Y= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/analysis v0.22.2 h1:ZBmNoP2h5omLKr/srIC9bfqrUGzT6g6gNv03HE9Vpj0= github.com/go-openapi/analysis v0.22.2/go.mod h1:pDF4UbZsQTo/oNuRfAWWd4dAh4yuYf//LYorPTjrpvo= github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w= github.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE= github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU= github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4= github.com/go-openapi/loads v0.21.5 h1:jDzF4dSoHw6ZFADCGltDb2lE4F6De7aWSpe+IcsRzT0= github.com/go-openapi/loads v0.21.5/go.mod h1:PxTsnFBoBe+z89riT+wYt3prmSBP6GDAQh2l9H1Flz8= github.com/go-openapi/spec v0.20.14 h1:7CBlRnw+mtjFGlPDRZmAMnq35cRzI91xj03HVyUi/Do= github.com/go-openapi/spec v0.20.14/go.mod h1:8EOhTpBoFiask8rrgwbLC3zmJfz4zsCUueRuPM6GNkw= github.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c= github.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4= github.com/go-openapi/swag v0.22.9 h1:XX2DssF+mQKM2DHsbgZK74y/zj4mo9I99+89xUmuZCE= github.com/go-openapi/swag v0.22.9/go.mod h1:3/OXnFfnMAwBD099SwYRk7GD3xOrr1iL7d/XNLXVVwE= github.com/go-openapi/validate v0.23.0 h1:2l7PJLzCis4YUGEoW6eoQw3WhyM65WSIcjX6SQnlfDw= github.com/go-openapi/validate v0.23.0/go.mod h1:EeiAZ5bmpSIOJV1WLfyYF9qp/B1ZgSaEpHTJHtN5cbE= github.com/go-resty/resty/v2 v2.13.1 h1:x+LHXBI2nMB1vqndymf26quycC4aggYJ7DECYbiz03g= github.com/go-resty/resty/v2 v2.13.1/go.mod h1:GznXlLxkq6Nh4sU59rPmUw3VtgpO3aS96ORAI6Q7d+0= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-zookeeper/zk v1.0.4 h1:DPzxraQx7OrPyXq2phlGlNSIyWEsAox0RJmjTseMV6I= github.com/go-zookeeper/zk v1.0.4/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gophercloud/gophercloud v1.14.0 h1:Bt9zQDhPrbd4qX7EILGmy+i7GP35cc+AAL2+wIJpUE8= github.com/gophercloud/gophercloud v1.14.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrRoT6yV5+wkrOpcszoIsO4+4ds248= github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.29.4 h1:P6slzxDLBOxUSj3fWo2o65VuKtbtOXFi7TSSgtXutuE= github.com/hashicorp/consul/api v1.29.4/go.mod h1:HUlfw+l2Zy68ceJavv2zAyArl2fqhGWnMycyt56sBgg= github.com/hashicorp/cronexpr v1.1.2 h1:wG/ZYIKT+RT3QkOdgYc+xsKWVRgnxJ1OJtjjy84fJ9A= github.com/hashicorp/cronexpr v1.1.2/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4= github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/nomad/api v0.0.0-20240717122358-3d93bd3778f3 h1:fgVfQ4AC1avVOnu2cfms8VAiD8lUq3vWI8mTocOXN/w= github.com/hashicorp/nomad/api v0.0.0-20240717122358-3d93bd3778f3/go.mod h1:svtxn6QnrQ69P23VvIWMR34tg3vmwLz4UdUzm1dSCgE= github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= github.com/hetznercloud/hcloud-go/v2 v2.13.1 h1:jq0GP4QaYE5d8xR/Zw17s9qoaESRJMXfGmtD1a/qckQ= github.com/hetznercloud/hcloud-go/v2 v2.13.1/go.mod h1:dhix40Br3fDiBhwaSG/zgaYOFFddpfBm/6R1Zz0IiF0= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/ionos-cloud/sdk-go/v6 v6.2.1 h1:mxxN+frNVmbFrmmFfXnBC3g2USYJrl6mc1LW2iNYbFY= github.com/ionos-cloud/sdk-go/v6 v6.2.1/go.mod h1:SXrO9OGyWjd2rZhAhEpdYN6VUAODzzqRdqA9BCviQtI= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJis0QP7YMxobob6zhzq6Yre00= github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/linode/linodego v1.40.0 h1:7ESY0PwK94hoggoCtIroT1Xk6b1flrFBNZ6KwqbTqlI= github.com/linode/linodego v1.40.0/go.mod h1:NsUw4l8QrLdIofRg1NYFBbW5ZERnmbZykVBszPZLORM= github.com/lomik/carbon-clickhouse v0.11.8 h1:Kgix46vequoJ60Qd4+3k2ocEh0hLBrdOO7df/G3D1aY= github.com/lomik/carbon-clickhouse v0.11.8/go.mod h1:c9H5G+2CwVqOcxxUc2WOL6i+rlOTmbd9G+AqIeLttgs= github.com/lomik/graphite-pickle v0.0.0-20171221213606-614e8df42119 h1:9kRJjaYdyzqGcGMeWeVif1vkToJvqzPEe5Vqx4IDXBg= github.com/lomik/graphite-pickle v0.0.0-20171221213606-614e8df42119/go.mod h1:C0xsTshsU0n/LkhSbjZx2UkLuWSa3uFmq9D35Ch4rNE= github.com/lomik/og-rek v0.0.0-20170411191824-628eefeb8d80 h1:KVyDGUXjVOdHQt24wIgY4ZdGFXHtQHLWw0L/MAK3Kb0= github.com/lomik/og-rek v0.0.0-20170411191824-628eefeb8d80/go.mod h1:T7SQVaLtK7mcQIEVzveZVJzsDQpAtzTs2YoezrIBdvI= github.com/lomik/prometheus-ui-static v0.2.54-1.1 h1:ilVQwP1nKxW85y3BD41mttIInQ/3pCNrTJvqI3znP3A= github.com/lomik/prometheus-ui-static v0.2.54-1.1/go.mod h1:aYbUFvTr1G39aW1WMI4Bp1nKhI9u/DsBcLaR5nKz0KE= github.com/lomik/stop v0.0.0-20161127103810-188e98d969bd h1:hUNpVzZOYNANa5s8XMBEt8IBj3m1GWcUN6ewUXPVA6c= github.com/lomik/stop v0.0.0-20161127103810-188e98d969bd/go.mod h1:3pLqdYIrxHYk+VsfIlrTcBD9J34YkGq8iN9yzJuhrP0= github.com/lomik/zapwriter v0.0.0-20210624082824-c1161d1eb463 h1:SN/0TEkyYpp8tit79JPUnecebCGZsXiYYPxN8i3I6Rk= github.com/lomik/zapwriter v0.0.0-20210624082824-c1161d1eb463/go.mod h1:rWIJAUD2hPOAyOzc3jBShAhN4CAZeLAyzUA/n8tE8ak= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA= github.com/mdlayher/vsock v1.2.1 h1:pC1mTJTvjo1r9n9fbm7S1j04rCgCzhCOS5DY0zqHlnQ= github.com/mdlayher/vsock v1.2.1/go.mod h1:NRfCibel++DgeMD8z/hP+PPTjlNJsdPOmxcnENvE+SE= github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ= github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/msaf1980/go-expirecache v0.0.2 h1:lkxQMd/cXnz/WTS5IO1HC399dxR9DrqNAhaET7gPKLE= github.com/msaf1980/go-expirecache v0.0.2/go.mod h1:AVemStNEitwcK0IDFtGBQ9GZJesybwaTe8mG1pCCajM= github.com/msaf1980/go-metrics v0.0.14 h1:gD0kCG5MDbon33Nkz49yW6kz3yu0DHzDN0SxjGTWlTA= github.com/msaf1980/go-metrics v0.0.14/go.mod h1:8VcR8MdyvIJpcVLOVFKbhb27+60tXy0M+zq7Ag8a6Pw= github.com/msaf1980/go-stringutils v0.1.2/go.mod h1:AxmV/6JuQUAtZJg5XmYATB5ZwCWgtpruVHY03dswRf8= github.com/msaf1980/go-stringutils v0.1.6 h1:qri8o+4XLJCJYemHcvJY6xJhrGTmllUoPwayKEj4NSg= github.com/msaf1980/go-stringutils v0.1.6/go.mod h1:xpicaTIpLAVzL0gUQkciB1zjypDGKsOCI25cKQbRQYA= github.com/msaf1980/go-syncutils v0.0.3 h1:bd6+yTSB8/CmpG7M6j1gq5sJMyPqecjJcBf19s2Y6u4= github.com/msaf1980/go-syncutils v0.0.3/go.mod h1:zoZwQNkDATcfKq5lQPK6dmJT7Z01COxw/vd8bcJyC9w= github.com/msaf1980/go-timeutils v0.0.4 h1:qdWcThz2gMTb73d3uDjwfXNsIzhpIjMlBQCnqc4pa6M= github.com/msaf1980/go-timeutils v0.0.4/go.mod h1:r252j2O/ZLuwNMp/rlSYhbQdxg6glZ3MzgvskE/ItGY= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/ovh/go-ovh v1.6.0 h1:ixLOwxQdzYDx296sXcgS35TOPEahJkpjMGtzPadCjQI= github.com/ovh/go-ovh v1.6.0/go.mod h1:cTVDnl94z4tl8pP1uZ/8jlVxntjSIf09bNcQ5TJSC7c= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/alertmanager v0.27.0 h1:V6nTa2J5V4s8TG4C4HtrBP/WNSebCCTYGGv4qecA/+I= github.com/prometheus/alertmanager v0.27.0/go.mod h1:8Ia/R3urPmbzJ8OsdvmZvIprDwvwmYCmUbwBL+jlPOE= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.20.3 h1:oPksm4K8B+Vt35tUhw6GbSNSgVlVSBH0qELP/7u83l4= github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.59.1 h1:LXb1quJHWm1P6wq/U824uxYi4Sg0oGvNeUm1z5dJoX0= github.com/prometheus/common v0.59.1/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0= github.com/prometheus/common/assets v0.2.0 h1:0P5OrzoHrYBOSM1OigWL3mY8ZvV2N4zIE/5AahrSrfM= github.com/prometheus/common/assets v0.2.0/go.mod h1:D17UVUE12bHbim7HzwUvtqm6gwBEaDQ0F+hIGbFbccI= github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4= github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI= github.com/prometheus/exporter-toolkit v0.12.0 h1:DkE5RcEZR3lQA2QD5JLVQIf41dFKNsVMXFhgqcif7fo= github.com/prometheus/exporter-toolkit v0.12.0/go.mod h1:fQH0KtTn0yrrS0S82kqppRjDDiwMfIQUwT+RBRRhwUc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/prometheus/prometheus v0.0.0-20240827104400-e6cfa720fbe6 h1:T1gXvCkxUmoQYSlrM+eXRrLfzHAVDyONlWVXtfWugoo= github.com/prometheus/prometheus v0.0.0-20240827104400-e6cfa720fbe6/go.mod h1:xlLByHhk2g3ycakQGrMaU8K7OySZx98BzeCR99991NY= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30 h1:yoKAVkEVwAqbGbR8n87rHQ1dulL25rKloGadb3vm770= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30/go.mod h1:sH0u6fq6x4R5M7WxkoQFY/o7UaiItec0o1LinLCJNq8= github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c h1:aqg5Vm5dwtvL+YgDpBcK1ITf3o96N/K7/wsRXQnUTEs= github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c/go.mod h1:owqhoLW1qZoYLZzLnBw+QkPP9WZnjlSWihhxAJC1+/M= github.com/simonpasquier/klog-gokit/v3 v3.5.0 h1:ewnk+ickph0hkQFgdI4pffKIbruAxxWcg0Fe/vQmLOM= github.com/simonpasquier/klog-gokit/v3 v3.5.0/go.mod h1:S9flvRzzpaYLYtXI2w8jf9R/IU/Cy14NrbvDUevNP1E= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs= github.com/vultr/govultr/v2 v2.17.2/go.mod h1:ZFOKGWmgjytfyjeyAdhQlSWwTjh2ig+X49cAp50dzXI= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xdg/scram v1.0.3/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xhhuango/json v1.19.0/go.mod h1:ynZo8WeuBMtTh7LMR1ljdu/8QxceUVbYEcAsPJ7iUb8= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80= go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opentelemetry.io/collector/pdata v1.14.1 h1:wXZjtQA7Vy5HFqco+yA95ENyMQU5heBB1IxMHQf6mUk= go.opentelemetry.io/collector/pdata v1.14.1/go.mod h1:z1dTjwwtcoXxZx2/nkHysjxMeaxe9pEmYTEr4SMNIx8= go.opentelemetry.io/collector/semconv v0.108.1 h1:Txk9tauUnamZaxS5vlf1O0uZ4VD6nioRBR0nX8L/fU4= go.opentelemetry.io/collector/semconv v0.108.1/go.mod h1:zCJ5njhWpejR+A40kiEoeFm1xq1uzyZwMnRNX6/D82A= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA= golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210427231257-85d9c07bbe3a/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20220208230804-65c12eb4c068/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed h1:3RgNmBoI9MZhsj3QxC+AP/qQhNwpCLOvYDYYsFrhFt0= google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo= k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE= k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc= k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8= k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= ================================================ FILE: graphite-clickhouse.go ================================================ package main import ( "context" "encoding/json" "flag" "fmt" "io" "log" "math/rand" "net/http" _ "net/http/pprof" "os" "os/signal" "runtime" "runtime/debug" "strings" "sync" "syscall" "time" "github.com/lomik/zapwriter" "go.uber.org/zap" "github.com/lomik/graphite-clickhouse/autocomplete" "github.com/lomik/graphite-clickhouse/capabilities" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/find" "github.com/lomik/graphite-clickhouse/healthcheck" "github.com/lomik/graphite-clickhouse/helper/rollup" "github.com/lomik/graphite-clickhouse/index" "github.com/lomik/graphite-clickhouse/logs" "github.com/lomik/graphite-clickhouse/metrics" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/prometheus" "github.com/lomik/graphite-clickhouse/render" "github.com/lomik/graphite-clickhouse/sd" "github.com/lomik/graphite-clickhouse/tagger" ) // Version of graphite-clickhouse const Version = "0.14.0" func init() { scope.Version = Version } type LogResponseWriter struct { http.ResponseWriter status int cached bool } func (w *LogResponseWriter) WriteHeader(status int) { w.status = status w.ResponseWriter.WriteHeader(status) } func (w *LogResponseWriter) Status() int { if w.status == 0 { return http.StatusOK } return w.status } func WrapResponseWriter(w http.ResponseWriter) *LogResponseWriter { if wrapped, ok := w.(*LogResponseWriter); ok { return wrapped } return &LogResponseWriter{ResponseWriter: w} } type App struct { config *config.Config } func (app *App) Handler(handler http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { writer := WrapResponseWriter(w) r = scope.HttpRequest(r) w.Header().Add("X-Gch-Request-ID", scope.RequestID(r.Context())) handler.ServeHTTP(writer, r) }) } var ( BuildVersion = "(development build)" srv *http.Server ) func sdList(name string, args []string) { descr := "List registered nodes in SD" flagName := "sd-list" flagSet := flag.NewFlagSet(descr, flag.ExitOnError) help := flagSet.Bool("help", false, "Print help") configFile := flagSet.String("config", "/etc/graphite-clickhouse/graphite-clickhouse.conf", "Filename of config") exactConfig := flagSet.Bool("exact-config", false, "Ensure that all config params are contained in the target struct.") flagSet.Usage = func() { fmt.Fprintf(os.Stderr, "Usage of %s %s:\n", name, flagName) flagSet.PrintDefaults() } flagSet.Parse(args) if *help || flagSet.NArg() > 0 { flagSet.Usage() return } cfg, _, err := config.ReadConfig(*configFile, *exactConfig) if err != nil { log.Fatal(err) } if cfg.Common.SD != "" && cfg.NeedLoadAvgColect() { var s sd.SD logger := zapwriter.Default() if s, err = sd.New(&cfg.Common, "", logger); err != nil { fmt.Fprintf(os.Stderr, "service discovery type %q can be registered", cfg.Common.SDType.String()) os.Exit(1) } if nodes, err := s.Nodes(); err == nil { for _, node := range nodes { fmt.Printf("%s/%s: %s (%s)\n", s.Namespace(), node.Key, node.Value, time.Unix(node.Flags, 0).UTC().Format(time.RFC3339Nano)) } } else { fmt.Fprintln(os.Stderr, err.Error()) os.Exit(1) } } } func sdDelete(name string, args []string) { descr := "Delete registered nodes for local hostname in SD" flagName := "sd-delete" flagSet := flag.NewFlagSet(descr, flag.ExitOnError) help := flagSet.Bool("help", false, "Print help") configFile := flagSet.String("config", "/etc/graphite-clickhouse/graphite-clickhouse.conf", "Filename of config") exactConfig := flagSet.Bool("exact-config", false, "Ensure that all config params are contained in the target struct.") flagSet.Usage = func() { fmt.Fprintf(os.Stderr, "Usage of %s %s:\n", name, flagName) flagSet.PrintDefaults() } flagSet.Parse(args) if *help || flagSet.NArg() > 0 { flagSet.Usage() return } cfg, _, err := config.ReadConfig(*configFile, *exactConfig) if err != nil { log.Fatal(err) } if cfg.Common.SD != "" && cfg.NeedLoadAvgColect() { var s sd.SD logger := zapwriter.Default() if s, err = sd.New(&cfg.Common, "", logger); err != nil { fmt.Fprintf(os.Stderr, "service discovery type %q can be registered", cfg.Common.SDType.String()) os.Exit(1) } hostname, _ := os.Hostname() hostname, _, _ = strings.Cut(hostname, ".") if err = s.Clear("", ""); err != nil { fmt.Fprintln(os.Stderr, err.Error()) os.Exit(1) } } } func sdEvict(name string, args []string) { descr := "Delete registered nodes for hostnames in SD" flagName := "sd-evict" flagSet := flag.NewFlagSet(descr, flag.ExitOnError) help := flagSet.Bool("help", false, "Print help") configFile := flagSet.String("config", "/etc/graphite-clickhouse/graphite-clickhouse.conf", "Filename of config") exactConfig := flagSet.Bool("exact-config", false, "Ensure that all config params are contained in the target struct.") flagSet.Usage = func() { fmt.Fprintf(os.Stderr, "Usage of %s %s:\n", name, flagName) flagSet.PrintDefaults() fmt.Fprintf(os.Stderr, " HOST []string\n List of hostnames\n") } flagSet.Parse(args) if *help { flagSet.Usage() return } cfg, _, err := config.ReadConfig(*configFile, *exactConfig) if err != nil { log.Fatal(err) } if cfg.Common.SD != "" && cfg.NeedLoadAvgColect() { for _, host := range flagSet.Args() { var s sd.SD logger := zapwriter.Default() if s, err = sd.New(&cfg.Common, host, logger); err != nil { fmt.Fprintf(os.Stderr, "service discovery type %q can be registered", cfg.Common.SDType.String()) os.Exit(1) } err = s.Clear("", "") } } } func sdExpired(name string, args []string) { descr := "List expired registered nodes in SD" flagName := "sd-expired" flagSet := flag.NewFlagSet(descr, flag.ExitOnError) help := flagSet.Bool("help", false, "Print help") configFile := flagSet.String("config", "/etc/graphite-clickhouse/graphite-clickhouse.conf", "Filename of config") exactConfig := flagSet.Bool("exact-config", false, "Ensure that all config params are contained in the target struct.") flagSet.Usage = func() { fmt.Fprintf(os.Stderr, "Usage of %s %s:\n", name, flagName) flagSet.PrintDefaults() } flagSet.Parse(args) if *help || flagSet.NArg() > 0 { flagSet.Usage() return } cfg, _, err := config.ReadConfig(*configFile, *exactConfig) if err != nil { log.Fatal(err) } if cfg.Common.SD != "" && cfg.NeedLoadAvgColect() { var s sd.SD logger := zapwriter.Default() if s, err = sd.New(&cfg.Common, "", logger); err != nil { fmt.Fprintf(os.Stderr, "service discovery type %q can be registered", cfg.Common.SDType.String()) os.Exit(1) } if err = sd.Cleanup(&cfg.Common, s, true); err != nil { fmt.Fprintln(os.Stderr, err.Error()) os.Exit(1) } } } func sdClean(name string, args []string) { descr := "Cleanup expired registered nodes in SD" flagName := "sd-clean" flagSet := flag.NewFlagSet(descr, flag.ExitOnError) help := flagSet.Bool("help", false, "Print help") configFile := flagSet.String("config", "/etc/graphite-clickhouse/graphite-clickhouse.conf", "Filename of config") exactConfig := flagSet.Bool("exact-config", false, "Ensure that all config params are contained in the target struct.") flagSet.Usage = func() { fmt.Fprintf(os.Stderr, "Usage of %s %s:\n", name, flagName) flagSet.PrintDefaults() } flagSet.Parse(args) if *help || flagSet.NArg() > 0 { flagSet.Usage() return } cfg, _, err := config.ReadConfig(*configFile, *exactConfig) if err != nil { log.Fatal(err) } if cfg.Common.SD != "" && cfg.NeedLoadAvgColect() { var s sd.SD logger := zapwriter.Default() if s, err = sd.New(&cfg.Common, "", logger); err != nil { fmt.Fprintf(os.Stderr, "service discovery type %q can be registered", cfg.Common.SDType.String()) os.Exit(1) } if err = sd.Cleanup(&cfg.Common, s, false); err != nil { fmt.Fprintln(os.Stderr, err.Error()) os.Exit(1) } } } func printMatchedRollupRules(metric string, age uint32, rollupRules *rollup.Rules) { // check metric rollup rules prec, aggr, aggrPattern, retentionPattern := rollupRules.Lookup(metric, age, true) fmt.Printf(" metric %q, age %d -> precision=%d, aggr=%s\n", metric, age, prec, aggr.Name()) if aggrPattern != nil { fmt.Printf(" aggr pattern: type=%s, regexp=%q, function=%s", aggrPattern.RuleType.String(), aggrPattern.Regexp, aggrPattern.Function) if len(aggrPattern.Retention) > 0 { fmt.Print(", retentions:\n") for i := range aggrPattern.Retention { fmt.Printf(" [age: %d, precision: %d]\n", aggrPattern.Retention[i].Age, aggrPattern.Retention[i].Precision) } } else { fmt.Print("\n") } } if retentionPattern != nil { fmt.Printf(" retention pattern: type=%s, regexp=%q, function=%s, retentions:\n", retentionPattern.RuleType.String(), retentionPattern.Regexp, retentionPattern.Function) for i := range retentionPattern.Retention { fmt.Printf(" [age: %d, precision: %d]\n", retentionPattern.Retention[i].Age, retentionPattern.Retention[i].Precision) } } } func checkRollupMatch(name string, args []string) { descr := "Match metric against rollup rules" flagName := "match" flagSet := flag.NewFlagSet(descr, flag.ExitOnError) help := flagSet.Bool("help", false, "Print help") rollupFile := flagSet.String("rollup", "", "Filename of rollup rules file") configFile := flagSet.String("config", "", "Filename of config") exactConfig := flagSet.Bool("exact-config", false, "Ensure that all config params are contained in the target struct.") table := flagSet.String("table", "", "Table in config for lookup rules") age := flagSet.Uint64("age", 0, "Age") flagSet.Usage = func() { fmt.Fprintf(os.Stderr, "Usage of %s %s:\n", name, flagName) flagSet.PrintDefaults() fmt.Fprintf(os.Stderr, " METRIC []string\n List of metric names\n") } flagSet.Parse(args) if *help { flagSet.Usage() return } if *rollupFile == "" && *configFile == "" { fmt.Fprint(os.Stderr, "set rollup and/or config file\n") os.Exit(1) } if *rollupFile != "" { fmt.Printf("rollup file %q\n", *rollupFile) if rollup, err := rollup.NewXMLFile(*rollupFile, 0, ""); err == nil { for _, metric := range flagSet.Args() { printMatchedRollupRules(metric, uint32(*age), rollup.Rules()) } } else { log.Fatal(err) } } if *configFile != "" { cfg, _, err := config.ReadConfig(*configFile, *exactConfig) if err != nil { log.Fatal(err) } ec := 0 for i := range cfg.DataTable { var rulesTable string if *table == "" || *table == cfg.DataTable[i].Table { if cfg.DataTable[i].RollupConf == "auto" || cfg.DataTable[i].RollupConf == "" { rulesTable = cfg.DataTable[i].Table if cfg.DataTable[i].RollupAutoTable != "" { rulesTable = cfg.DataTable[i].RollupAutoTable } fmt.Printf("table %q, rollup rules table %q in Clickhouse\n", cfg.DataTable[i].Table, rulesTable) } else { fmt.Printf("rollup file %q\n", cfg.DataTable[i].RollupConf) } rules := cfg.DataTable[i].Rollup.Rules() if rules == nil { if cfg.DataTable[i].RollupConf == "auto" || cfg.DataTable[i].RollupConf == "" { rules, err = rollup.RemoteLoad(cfg.ClickHouse.URL, cfg.ClickHouse.TLSConfig, rulesTable) if err != nil { ec = 1 fmt.Fprintf(os.Stderr, "%v\n", err) } } } if rules != nil { for _, metric := range flagSet.Args() { printMatchedRollupRules(metric, uint32(*age), rules) } } } } os.Exit(ec) } } func main() { rand.Seed(time.Now().UnixNano()) var err error /* CONFIG start */ configFile := flag.String("config", "/etc/graphite-clickhouse/graphite-clickhouse.conf", "Filename of config") printDefaultConfig := flag.Bool("config-print-default", false, "Print default config") checkConfig := flag.Bool("check-config", false, "Check config and exit") exactConfig := flag.Bool("exact-config", false, "Ensure that all config params are contained in the target struct.") buildTags := flag.Bool("tags", false, "Build tags table") pprof := flag.String( "pprof", "", "Additional pprof listen addr for non-server modes (tagger, etc..), overrides pprof-listen from common ", ) printVersion := flag.Bool("version", false, "Print version") verbose := flag.Bool("verbose", false, "Verbose (print config on startup)") flag.Usage = func() { fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0]) flag.PrintDefaults() fmt.Fprintf(os.Stderr, "\n\nAdditional commands:\n") fmt.Fprintf(os.Stderr, " sd-list List registered nodes in SD\n") fmt.Fprintf(os.Stderr, " sd-delete Delete registered nodes for local hostname in SD\n") fmt.Fprintf(os.Stderr, " sd-evict Delete registered nodes for hostnames in SD\n") fmt.Fprintf(os.Stderr, " sd-clean Cleanup expired registered nodes in SD\n") fmt.Fprintf(os.Stderr, " sd-expired List expired registered nodes in SD\n") fmt.Fprintf(os.Stderr, " match Match metric against rollup rules\n") } if len(os.Args) > 1 { switch os.Args[1] { case "sd-list", "-sd-list": sdList(os.Args[0], os.Args[2:]) return case "sd-delete", "-sd-delete": sdDelete(os.Args[0], os.Args[2:]) return case "sd-evict", "-sd-evict": sdEvict(os.Args[0], os.Args[2:]) return case "sd-clean", "-sd-clean": sdClean(os.Args[0], os.Args[2:]) return case "sd-expired", "-sd-expired": sdExpired(os.Args[0], os.Args[2:]) return case "match", "-match": checkRollupMatch(os.Args[0], os.Args[2:]) return } } flag.Parse() if *printVersion { fmt.Print(Version) return } if *printDefaultConfig { if err = config.PrintDefaultConfig(); err != nil { log.Fatal(err) } return } cfg, warns, err := config.ReadConfig(*configFile, *exactConfig) if err != nil { log.Fatal(err) } // config parsed successfully. Exit in check-only mode if *checkConfig { return } if err = zapwriter.ApplyConfig(cfg.Logging); err != nil { log.Fatal(err) } localManager, err := zapwriter.NewManager(cfg.Logging) if err != nil { log.Fatal(err) } logger := localManager.Logger("start") if len(warns) > 0 { zapwriter.Logger("config").Warn("warnings", warns...) } if *verbose { logger.Info("starting graphite-clickhouse", zap.String("build_version", BuildVersion), zap.Any("config", cfg), ) } else { logger.Info("starting graphite-clickhouse", zap.String("build_version", BuildVersion), ) } runtime.GOMAXPROCS(cfg.Common.MaxCPU) if cfg.Common.MemoryReturnInterval > 0 { go func() { t := time.NewTicker(cfg.Common.MemoryReturnInterval) for { <-t.C debug.FreeOSMemory() } }() } /* CONFIG end */ if pprof != nil && *pprof != "" || cfg.Common.PprofListen != "" { listen := cfg.Common.PprofListen if *pprof != "" { listen = *pprof } go func() { log.Fatal(http.ListenAndServe(listen, nil)) }() } /* CONSOLE COMMANDS start */ if *buildTags { if err := tagger.Make(cfg); err != nil { log.Fatal(err) } return } /* CONSOLE COMMANDS end */ app := App{config: cfg} mux := http.NewServeMux() mux.Handle("/_internal/capabilities/", app.Handler(capabilities.NewHandler(cfg))) mux.Handle("/metrics/find/", app.Handler(find.NewHandler(cfg))) mux.Handle("/metrics/index.json", app.Handler(index.NewHandler(cfg))) mux.Handle("/render/", app.Handler(render.NewHandler(cfg))) mux.Handle("/tags/autoComplete/tags", app.Handler(autocomplete.NewTags(cfg))) mux.Handle("/tags/autoComplete/values", app.Handler(autocomplete.NewValues(cfg))) mux.HandleFunc("/alive", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) io.WriteString(w, "Graphite-clickhouse is alive.\n") }) mux.Handle("/health", app.Handler(healthcheck.NewHandler(cfg))) mux.HandleFunc("/debug/config", func(w http.ResponseWriter, r *http.Request) { status := http.StatusOK start := time.Now() accessLogger := scope.LoggerWithHeaders(r.Context(), r, app.config.Common.HeadersToLog) defer func() { d := time.Since(start) logs.AccessLog(accessLogger, app.config, r, status, d, time.Duration(0), false, false) }() b, err := json.MarshalIndent(cfg, "", " ") if err != nil { status = http.StatusInternalServerError http.Error(w, err.Error(), status) return } w.Write(b) }) if cfg.Prometheus.Listen != "" { if err := prometheus.Run(cfg); err != nil { log.Fatal(err) } } if metrics.Graphite != nil { metrics.Graphite.Start(nil) } var exitWait sync.WaitGroup srv = &http.Server{ Addr: cfg.Common.Listen, Handler: mux, } exitWait.Add(1) go func() { defer exitWait.Done() if err := srv.ListenAndServe(); err != http.ErrServerClosed { // unexpected error. port in use? log.Fatalf("ListenAndServe(): %v", err) } }() if cfg.Common.SD != "" && cfg.NeedLoadAvgColect() { go func() { time.Sleep(time.Millisecond * 100) sdLogger := localManager.Logger("service discovery") sd.Register(&cfg.Common, sdLogger) }() } go func() { stop := make(chan os.Signal, 1) signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) <-stop logger.Info("stoping graphite-clickhouse") if cfg.Common.SD != "" { // unregister SD sd.Stop() time.Sleep(10 * time.Second) } // initiating the shutdown ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) srv.Shutdown(ctx) cancel() }() exitWait.Wait() logger.Info("stop graphite-clickhouse") } ================================================ FILE: healthcheck/healthcheck.go ================================================ package healthcheck import ( "fmt" "io" "net/http" "net/url" "strings" "sync/atomic" "time" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/msaf1980/go-stringutils" "go.uber.org/zap" ) // Handler serves /render requests type Handler struct { config *config.Config last int64 failed int32 } // NewHandler generates new *Handler func NewHandler(config *config.Config) *Handler { h := &Handler{ config: config, failed: 1, } return h } func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { var ( query string failed int32 ) if h.config.ClickHouse.IndexTable != "" { // non-existing name with wrong level query = "SELECT Path FROM " + h.config.ClickHouse.IndexTable + " WHERE ((Level=20002) AND (Path IN ('NonExistient','NonExistient.'))) AND (Date='1970-02-12') GROUP BY Path FORMAT TabSeparatedRaw" } else if h.config.ClickHouse.TaggedTable != "" { // non-existing partition query = "SELECT Path FROM " + h.config.ClickHouse.TaggedTable + " WHERE (Tag1='__name__=NonExistient') AND (Date='1970-02-12') GROUP BY Path FORMAT TabSeparatedRaw" } if query != "" { failed = 1 now := time.Now().Unix() for { last := atomic.LoadInt64(&h.last) if now-last < 10 { failed = atomic.LoadInt32(&h.failed) break } // one query in 10 seconds for prevent overloading if !atomic.CompareAndSwapInt64(&h.last, last, now) { continue } logger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("healthcheck") client := http.Client{ Timeout: 2 * time.Second, } var u string if pos := strings.Index(h.config.ClickHouse.URL, "/?"); pos > 0 { u = h.config.ClickHouse.URL[:pos+2] + "query=" + url.QueryEscape(query) } else { u = h.config.ClickHouse.URL + "/?query=" + url.QueryEscape(query) } req, _ := http.NewRequest(http.MethodGet, u, nil) resp, err := client.Do(req) if err != nil { logger.Error("healthcheck error", zap.Error(err), ) } if resp.Body != nil { if body, err := io.ReadAll(resp.Body); err == nil { if resp.StatusCode == http.StatusOK { failed = 0 } else { failed = 1 logger.Error("healthcheck error", zap.String("error", stringutils.UnsafeString(body)), ) } } else { failed = 1 logger.Error("healthcheck error", zap.Error(err), ) } resp.Body.Close() } else { failed = 1 logger.Error("healthcheck error", zap.Error(err), ) } atomic.StoreInt32(&h.failed, failed) break } } if failed > 0 { http.Error(w, "Storage healthcheck failed", http.StatusServiceUnavailable) } else { w.WriteHeader(http.StatusOK) fmt.Fprintf(w, "Graphite-clickhouse is alive.\n") } } ================================================ FILE: helper/RowBinary/encode.go ================================================ package RowBinary import ( "encoding/binary" "io" "math" "time" ) const NullUint32 = ^uint32(0) func DateToUint16(t time.Time) uint16 { return uint16(time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC).Unix() / 86400) } type Encoder struct { wrapped io.Writer buffer []byte } func NewEncoder(w io.Writer) *Encoder { return &Encoder{ wrapped: w, buffer: make([]byte, 256), } } func (w *Encoder) Date(value time.Time) error { return w.Uint16(DateToUint16(value)) } func (w *Encoder) Uint8(value uint8) error { _, err := w.wrapped.Write([]byte{value}) return err } func (w *Encoder) Uint16(value uint16) error { binary.LittleEndian.PutUint16(w.buffer, value) _, err := w.wrapped.Write(w.buffer[:2]) return err } func (w *Encoder) Uint32(value uint32) error { binary.LittleEndian.PutUint32(w.buffer, value) _, err := w.wrapped.Write(w.buffer[:4]) return err } func (w *Encoder) NullableUint32(value uint32) error { if value == NullUint32 { _, err := w.wrapped.Write([]byte{1}) return err } _, err := w.wrapped.Write([]byte{0}) if err != nil { return err } return w.Uint32(value) } func (w *Encoder) Uint64(value uint64) error { binary.LittleEndian.PutUint64(w.buffer, value) _, err := w.wrapped.Write(w.buffer[:8]) return err } func (w *Encoder) Float64(value float64) error { return w.Uint64(math.Float64bits(value)) } func (w *Encoder) NullableFloat64(value float64) error { if math.IsNaN(value) { _, err := w.wrapped.Write([]byte{1}) return err } _, err := w.wrapped.Write([]byte{0}) if err != nil { return err } return w.Float64(value) } func (w *Encoder) Bytes(value []byte) error { n := binary.PutUvarint(w.buffer, uint64(len(value))) _, err := w.wrapped.Write(w.buffer[:n]) if err != nil { return err } _, err = w.wrapped.Write(value) return err } func (w *Encoder) String(value string) error { return w.Bytes([]byte(value)) } func (w *Encoder) StringList(value []string) error { n := binary.PutUvarint(w.buffer, uint64(len(value))) _, err := w.wrapped.Write(w.buffer[:n]) if err != nil { return err } for i := 0; i < len(value); i++ { err = w.String(value[i]) if err != nil { return err } } return nil } func (w *Encoder) Uint32List(value []uint32) error { n := binary.PutUvarint(w.buffer, uint64(len(value))) _, err := w.wrapped.Write(w.buffer[:n]) if err != nil { return err } for i := 0; i < len(value); i++ { err = w.Uint32(value[i]) if err != nil { return err } } return nil } func (w *Encoder) NullableUint32List(value []uint32) error { n := binary.PutUvarint(w.buffer, uint64(len(value))) _, err := w.wrapped.Write(w.buffer[:n]) if err != nil { return err } for i := 0; i < len(value); i++ { err = w.NullableUint32(value[i]) if err != nil { return err } } return nil } func (w *Encoder) Float64List(value []float64) error { n := binary.PutUvarint(w.buffer, uint64(len(value))) _, err := w.wrapped.Write(w.buffer[:n]) if err != nil { return err } for i := 0; i < len(value); i++ { err = w.Float64(value[i]) if err != nil { return err } } return nil } func (w *Encoder) NullableFloat64List(value []float64) error { n := binary.PutUvarint(w.buffer, uint64(len(value))) _, err := w.wrapped.Write(w.buffer[:n]) if err != nil { return err } for i := 0; i < len(value); i++ { err = w.NullableFloat64(value[i]) if err != nil { return err } } return nil } ================================================ FILE: helper/clickhouse/clickhouse.go ================================================ package clickhouse import ( "context" "crypto/tls" "encoding/binary" "encoding/json" "errors" "fmt" "html" "io" "math/rand" "net" "net/http" "net/url" "sort" "strconv" "strings" "time" "github.com/lomik/graphite-clickhouse/helper/errs" httpHelper "github.com/lomik/graphite-clickhouse/helper/http" "github.com/lomik/graphite-clickhouse/limiter" "github.com/lomik/graphite-clickhouse/pkg/scope" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) type ErrWithDescr struct { err string data string } type ContentEncoding string const ( ContentEncodingNone ContentEncoding = "none" ContentEncodingGzip ContentEncoding = "gzip" ContentEncodingZstd ContentEncoding = "zstd" ) const ( ClickHouseProgressHeader string = "X-Clickhouse-Progress" ClickHouseSummaryHeader string = "X-Clickhouse-Summary" ) func NewErrWithDescr(err string, data string) error { return &ErrWithDescr{err, data} } func (e *ErrWithDescr) Error() string { return e.err + ": " + e.data } func (e *ErrWithDescr) PrependDescription(test string) { e.data = test + e.data } var ErrInvalidTimeRange = errors.New("Invalid or empty time range") var ErrUvarintRead = errors.New("ReadUvarint: Malformed array") var ErrUvarintOverflow = errors.New("ReadUvarint: varint overflows a 64-bit integer") var ErrClickHouseResponse = errors.New("Malformed response from clickhouse") func extractClickhouseError(e string) (int, string) { if strings.HasPrefix(e, "clickhouse response status 500: Code:") || strings.HasPrefix(e, "Malformed response from clickhouse") { if start := strings.Index(e, ": Limit for "); start != -1 { e := e[start+8:] if end := strings.Index(e, " (version "); end != -1 { e = e[0:end] } return http.StatusForbidden, "Storage read limit " + e } else if start := strings.Index(e, ": Memory limit "); start != -1 { return http.StatusForbidden, "Storage read limit for memory" } else if strings.HasPrefix(e, "clickhouse response status 500: Code: 170,") { // distributed table configuration error // clickhouse response status 500: Code: 170, e.displayText() = DB::Exception: Requested cluster 'cluster' not found return http.StatusServiceUnavailable, "Storage configuration error" } } if strings.HasPrefix(e, "clickhouse response status 404: Code: 60. DB::Exception: Table default.") { return http.StatusServiceUnavailable, "Storage default tables damaged" } if strings.HasPrefix(e, "clickhouse response status 500: Code: 427") || strings.HasPrefix(e, "clickhouse response status 400: Code: 427.") { return http.StatusBadRequest, "Incorrect regex syntax" } return http.StatusServiceUnavailable, "Storage unavailable" } func HandleError(w http.ResponseWriter, err error) (status int, queueFail bool) { status = http.StatusOK errStr := err.Error() if err == ErrInvalidTimeRange { status = http.StatusBadRequest http.Error(w, errStr, status) return } if err == limiter.ErrTimeout || err == limiter.ErrOverflow { queueFail = true status = http.StatusServiceUnavailable http.Error(w, err.Error(), status) return } if _, ok := err.(*ErrWithDescr); ok { status, errStr = extractClickhouseError(errStr) http.Error(w, errStr, status) return } netErr, ok := err.(net.Error) if ok { if netErr.Timeout() { status = http.StatusGatewayTimeout http.Error(w, "Storage read timeout", status) } else if strings.HasSuffix(errStr, "connect: no route to host") || strings.HasPrefix(errStr, "dial tcp: lookup ") { // DNS lookup status = http.StatusServiceUnavailable http.Error(w, "Storage route error", status) } else if strings.HasSuffix(errStr, "connect: connection refused") || strings.HasSuffix(errStr, ": connection reset by peer") { status = http.StatusServiceUnavailable http.Error(w, "Storage connect error", status) } else { status = http.StatusServiceUnavailable http.Error(w, "Storage network error", status) } return } errCode, ok := err.(errs.ErrorWithCode) if ok { if (errCode.Code > 500 && errCode.Code < 512) || errCode.Code == http.StatusBadRequest || errCode.Code == http.StatusForbidden { status = errCode.Code http.Error(w, html.EscapeString(errStr), status) } else { status = http.StatusInternalServerError http.Error(w, html.EscapeString(errStr), status) } return } if errors.Is(err, context.Canceled) { status = http.StatusGatewayTimeout http.Error(w, "Storage read context canceled", status) } else { //logger.Debug("query", zap.Error(err)) status = http.StatusInternalServerError http.Error(w, html.EscapeString(errStr), status) } return } type Options struct { TLSConfig *tls.Config Timeout time.Duration ConnectTimeout time.Duration ProgressSendingInterval time.Duration CheckRequestProgress bool } type LoggedReader struct { reader io.ReadCloser logger *zap.Logger start time.Time finished bool queryID string read_rows int64 read_bytes int64 } func (r *LoggedReader) Read(p []byte) (int, error) { n, err := r.reader.Read(p) if err != nil && !r.finished { r.finished = true r.logger.Info("query", zap.String("query_id", r.queryID), zap.Duration("time", time.Since(r.start))) } return n, err } func (r *LoggedReader) Close() error { err := r.reader.Close() if !r.finished { r.finished = true r.logger.Info("query", zap.String("query_id", r.queryID), zap.Duration("time", time.Since(r.start))) } return err } func (r *LoggedReader) ChReadRows() int64 { return r.read_rows } func (r *LoggedReader) ChReadBytes() int64 { return r.read_bytes } type queryStats struct { readRows int64 readBytes int64 loggerFields []zapcore.Field rawHeader string } func formatSQL(q string) string { s := strings.Split(q, "\n") for i := 0; i < len(s); i++ { s[i] = strings.TrimSpace(s[i]) } return strings.Join(s, " ") } func Query(ctx context.Context, dsn string, query string, opts Options, extData *ExternalData) ([]byte, int64, int64, error) { return Post(ctx, dsn, query, nil, opts, extData) } func Post(ctx context.Context, dsn string, query string, postBody io.Reader, opts Options, extData *ExternalData) ([]byte, int64, int64, error) { return do(ctx, dsn, query, postBody, ContentEncodingNone, opts, extData) } // Deprecated: use PostWithEncoding instead func PostGzip(ctx context.Context, dsn string, query string, postBody io.Reader, opts Options, extData *ExternalData) ([]byte, int64, int64, error) { return do(ctx, dsn, query, postBody, ContentEncodingGzip, opts, extData) } func PostWithEncoding(ctx context.Context, dsn string, query string, postBody io.Reader, encoding ContentEncoding, opts Options, extData *ExternalData) ([]byte, int64, int64, error) { return do(ctx, dsn, query, postBody, encoding, opts, extData) } func Reader(ctx context.Context, dsn string, query string, opts Options, extData *ExternalData) (*LoggedReader, error) { return reader(ctx, dsn, query, nil, ContentEncodingNone, opts, extData) } func reader(ctx context.Context, dsn string, query string, postBody io.Reader, encoding ContentEncoding, opts Options, extData *ExternalData) (bodyReader *LoggedReader, err error) { if postBody != nil && extData != nil { err = fmt.Errorf("postBody and extData could not be passed in one request") return } var chQueryID string start := time.Now() requestID := scope.RequestID(ctx) queryForLogger := query if len(queryForLogger) > 500 { queryForLogger = queryForLogger[:395] + "<...>" + queryForLogger[len(queryForLogger)-100:] } logger := scope.Logger(ctx).With(zap.String("query", formatSQL(queryForLogger))) defer func() { // fmt.Println(time.Since(start), formatSQL(queryForLogger)) if err != nil { logger.Error("query", zap.Error(err), zap.Duration("time", time.Since(start))) } }() p, err := url.Parse(dsn) if err != nil { return } var b [8]byte binary.LittleEndian.PutUint64(b[:], rand.Uint64()) queryID := fmt.Sprintf("%x", b) q := p.Query() q.Set("query_id", fmt.Sprintf("%s::%s", requestID, queryID)) // Get X-Clickhouse-Summary header // TODO: remove when https://github.com/ClickHouse/ClickHouse/issues/16207 is done q.Set("send_progress_in_http_headers", "1") q.Set("http_headers_progress_interval_ms", strconv.FormatInt(opts.ProgressSendingInterval.Milliseconds(), 10)) p.RawQuery = q.Encode() var contentHeader string if postBody != nil { q := p.Query() q.Set("query", query) p.RawQuery = q.Encode() } else if extData != nil { q := p.Query() q.Set("query", query) p.RawQuery = q.Encode() postBody, contentHeader, err = extData.buildBody(ctx, p) if err != nil { return } } else { postBody = strings.NewReader(query) } url := p.String() req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, postBody) if err != nil { return } req.Header.Add("User-Agent", scope.ClickhouseUserAgent(ctx)) if contentHeader != "" { req.Header.Add("Content-Type", contentHeader) } switch encoding { case ContentEncodingNone: // no encoding case ContentEncodingGzip: req.Header.Add("Content-Encoding", "gzip") case ContentEncodingZstd: req.Header.Add("Content-Encoding", "zstd") default: return nil, fmt.Errorf("unknown encoding: %s", encoding) } var resp *http.Response if opts.CheckRequestProgress { resp, err = sendRequestWithProgressCheck(req, &opts) } else { resp, err = sendRequestViaDefaultClient(req, &opts) } if err != nil { if opts.CheckRequestProgress && resp != nil { stats, parse_err := getQueryStats(resp, ClickHouseProgressHeader) if parse_err != nil { logger.Warn("query", zap.Error(err), zap.String("clickhouse-progress", stats.rawHeader)) } logger = logger.With(stats.loggerFields...) } return } // chproxy overwrite our query id. So read it again chQueryID = resp.Header.Get("X-ClickHouse-Query-Id") stats, err := getQueryStats(resp, ClickHouseSummaryHeader) if err != nil { summaryHeader := resp.Header.Get(ClickHouseSummaryHeader) logger.Warn("query", zap.Error(err), zap.String("clickhouse-summary", summaryHeader)) err = nil } read_rows, read_bytes, fields := stats.readRows, stats.readBytes, stats.loggerFields if len(fields) > 0 { sort.Slice(fields, func(i, j int) bool { return fields[i].Key < fields[j].Key }) logger = logger.With(fields...) } // check for return 5xx error, may be 502 code if clickhouse accesed via reverse proxy if resp.StatusCode > http.StatusInternalServerError && resp.StatusCode < 512 { body, _ := io.ReadAll(resp.Body) resp.Body.Close() err = errs.NewErrorWithCode(string(body), resp.StatusCode) return } else if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(resp.Body) resp.Body.Close() err = NewErrWithDescr("clickhouse response status "+strconv.Itoa(resp.StatusCode), string(body)) return } bodyReader = &LoggedReader{ reader: resp.Body, logger: logger, start: start, queryID: chQueryID, read_rows: read_rows, read_bytes: read_bytes, } return } func getQueryStats(resp *http.Response, statsHeaderName string) (queryStats, error) { read_rows := int64(-1) read_bytes := int64(-1) if resp == nil { return queryStats{ readRows: read_rows, readBytes: read_bytes, loggerFields: []zapcore.Field{}, }, nil } statsHeader := "" statsHeaders := resp.Header.Values(statsHeaderName) if len(statsHeaders) > 0 { statsHeader = statsHeaders[len(statsHeaders)-1] } else { return queryStats{ readRows: read_rows, readBytes: read_bytes, loggerFields: []zapcore.Field{}, }, nil } stats := make(map[string]string) err := json.Unmarshal([]byte(statsHeader), &stats) if err != nil { return queryStats{ readRows: read_rows, readBytes: read_bytes, loggerFields: []zapcore.Field{}, rawHeader: statsHeader, }, err } // TODO: use in carbon metrics sender when it will be implemented fields := make([]zapcore.Field, 0, len(stats)) for k, v := range stats { fields = append(fields, zap.String(k, v)) switch k { case "read_rows": read_rows, _ = strconv.ParseInt(v, 10, 64) case "read_bytes": read_bytes, _ = strconv.ParseInt(v, 10, 64) } } sort.Slice(fields, func(i int, j int) bool { return fields[i].Key < fields[j].Key }) return queryStats{ readRows: read_rows, readBytes: read_bytes, loggerFields: fields, rawHeader: statsHeader, }, nil } func sendRequestViaDefaultClient(request *http.Request, opts *Options) (*http.Response, error) { client := &http.Client{ Timeout: opts.Timeout, Transport: &http.Transport{ DialContext: (&net.Dialer{ Timeout: opts.ConnectTimeout, }).DialContext, TLSClientConfig: opts.TLSConfig, DisableKeepAlives: true, }, } return client.Do(request) } func sendRequestWithProgressCheck(request *http.Request, opts *Options) (*http.Response, error) { transport := &http.Transport{ DialContext: (&net.Dialer{ Timeout: opts.ConnectTimeout, }).DialContext, TLSClientConfig: opts.TLSConfig, DisableKeepAlives: true, } return httpHelper.DoHTTPOverTCP(request.Context(), transport, request, opts.Timeout) } func do(ctx context.Context, dsn string, query string, postBody io.Reader, encoding ContentEncoding, opts Options, extData *ExternalData) ([]byte, int64, int64, error) { bodyReader, err := reader(ctx, dsn, query, postBody, encoding, opts, extData) if err != nil { return nil, 0, 0, err } body, err := io.ReadAll(bodyReader) bodyReader.Close() if err != nil { return nil, bodyReader.ChReadRows(), bodyReader.ChReadBytes(), err } return body, bodyReader.ChReadRows(), bodyReader.ChReadBytes(), nil } func ReadUvarint(array []byte) (uint64, int, error) { var x uint64 var s uint l := len(array) - 1 for i := 0; ; i++ { if i > l { return x, i + 1, ErrUvarintRead } if array[i] < 0x80 { if i > 9 || i == 9 && array[i] > 1 { return x, i + 1, ErrUvarintOverflow } return x | uint64(array[i])< match(x, '^t=.**a*') UInt8 : 1': while executing 'FUNCTION arrayExists(__lambda_11 :: 7, Tags :: 3) -> arrayExists(lambda(tuple(x), and(like(x, 't=%'), match(x, '^t=.**a*'))), Tags) UInt8 : 6' (version 21.3.20.1 (official build))`, wantStatus: http.StatusBadRequest, wantMessage: "Incorrect regex syntax", }, { errStr: `clickhouse response status 500: Code: 427. DB::Exception: OptimizedRegularExpression: cannot compile re2: ^t=.**a*, error: bad repetition operator: **. Look at https://github.com/google/re2/wiki/Syntax for reference. Please note that if you specify regex as an SQL string literal, the slashes have to be additionally escaped. For example, to match an opening brace, write '\(' -- the first slash is for SQL and the second one is for regex: while executing 'FUNCTION and(like(x, 't=%') :: 3, match(x, '^t=.**a*') :: 1) -> and(like(x, 't=%'), match(x, '^t=.**a*')) UInt8 : 2': while executing 'FUNCTION and(and(greaterOrEquals(Date, '2024-10-28'), lessOrEquals(Date, '2024-10-28')) :: 0, and(equals(Tag1, '__name__=request_success_total.counter'), arrayExists(lambda(tuple(x), equals(x, 'app=test')), Tags), arrayExists(lambda(tuple(x), equals(x, 'project=Test')), Tags), arrayExists(lambda(tuple(x), equals(x, 'environment=TEST')), Tags), arrayExists(lambda(tuple(x), and(like(x, 't=%'), match(x, '^t=.**a*'))), Tags)) :: 3) -> and(and(greaterOrEquals(Date, '2024-10-28'), lessOrEquals(Date, '2024-10-28')), and(equals(Tag1, '__name__=request_success_total.counter'), arrayExists(lambda(tuple(x), equals(x, 'app=test')), Tags), arrayExists(lambda(tuple(x), equals(x, 'project=Test')), Tags), arrayExists(lambda(tuple(x), equals(x, 'environment=TEST')), Tags), arrayExists(lambda(tuple(x), and(like(x, 't=%'), match(x, '^t=.**a*'))), Tags))) UInt8 : 6'. (CANNOT_COMPILE_REGEXP) (version 22.8.21.38 (official build))`, wantStatus: http.StatusBadRequest, wantMessage: "Incorrect regex syntax", }, { errStr: "Other error", wantStatus: http.StatusServiceUnavailable, wantMessage: "Storage unavailable", }, } for _, tt := range tests { t.Run(tt.errStr, func(t *testing.T) { gotStatus, gotMessage := extractClickhouseError(tt.errStr) assert.Equal(t, tt.wantStatus, gotStatus) assert.Equal(t, tt.wantMessage, gotMessage) }) } } ================================================ FILE: helper/clickhouse/external-data.go ================================================ package clickhouse import ( "bytes" "context" "fmt" "mime/multipart" "net/url" "os" "path" "strings" "github.com/lomik/graphite-clickhouse/pkg/scope" "go.uber.org/zap" ) // ExternalTable is a structure to use ClickHouse feature that creates a temporary table for a query type ExternalTable struct { // Table name Name string Columns []Column // ClickHouse input/output format Format string Data []byte } // Column is a pair of Name and Type for temporary table structure type Column struct { Name string // ClickHouse data type Type string } func (c *Column) String() string { return c.Name + " " + c.Type } // ExternalData is a type to use ClickHouse external data feature. You could use it to pass multiple // temporary tables for a query. type ExternalData struct { Tables []ExternalTable debug *extDataDebug } type extDataDebug struct { dir string perm os.FileMode } // NewExternalData returns the `*ExternalData` object for `tables` func NewExternalData(tables ...ExternalTable) *ExternalData { return &ExternalData{Tables: tables, debug: nil} } // SetDebug sets the directory and file permission for an external table data dump. Works only if // both `debugDir` and `perm` are set func (e *ExternalData) SetDebug(debugDir string, perm os.FileMode) { if debugDir == "" || perm == 0 { e.debug = nil } e.debug = &extDataDebug{debugDir, perm} } // buildBody returns multiform body, content type header and error func (e *ExternalData) buildBody(ctx context.Context, u *url.URL) (*bytes.Buffer, string, error) { body := new(bytes.Buffer) header := "" writer := multipart.NewWriter(body) for _, t := range e.Tables { part, err := writer.CreateFormFile(t.Name, t.Name) if err != nil { return nil, header, err } // Send each table in separated form _, err = part.Write(t.Data) if err != nil { return nil, header, err } // Set name_format and name_structure for the table q := u.Query() if t.Format != "" { q.Set(t.Name+"_format", t.Format) } structure := make([]string, 0, len(t.Columns)) for _, c := range t.Columns { structure = append(structure, c.String()) } q.Set(t.Name+"_structure", strings.Join(structure, ",")) u.RawQuery = q.Encode() } err := writer.Close() if err != nil { return nil, header, err } header = writer.FormDataContentType() du := *u // Do not lock the execution by debugging process go e.debugDump(ctx, du) return body, header, nil } func (e *ExternalData) debugDump(ctx context.Context, u url.URL) { if e.debug == nil || !scope.Debug(ctx, "External-Data") { // Do not dump if the settings are not set return } requestID := scope.RequestID(ctx) logger := scope.Logger(ctx) command := "curl " for _, t := range e.Tables { filename := path.Join(e.debug.dir, fmt.Sprintf("ext-%v:%v.%v", t.Name, requestID, t.Format)) err := os.WriteFile(filename, t.Data, e.debug.perm) if err != nil { logger.Warn("external-data", zap.Error(err)) // The debug command couldn't be built w/o all external tables return } command += fmt.Sprintf("-F '%v=@%v;' ", t.Name, filename) } // Change query_id to not interfere with the original one q := u.Query() q["query_id"] = []string{fmt.Sprintf("%v:debug", requestID)} u.RawQuery = q.Encode() command += "'" + u.Redacted() + "'" logger.Info("external-data", zap.String("debug command", command)) } ================================================ FILE: helper/clickhouse/external-data_test.go ================================================ package clickhouse import ( "context" "fmt" "math/rand" "net/url" "os" "path/filepath" "strings" "testing" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/stretchr/testify/assert" ) func getTestCases() (tables []ExternalTable) { tables = []ExternalTable{ { Name: "test1", Columns: []Column{ { Name: "aString", Type: "String", }, { Name: "anInt", Type: "Int32", }, }, Format: "TSV", Data: []byte(`f 3`), }, { Name: "test2", Columns: []Column{ { Name: "aFloat", Type: "Float32", }, { Name: "aDate", Type: "Date", }, }, Format: "TSKV", Data: []byte(`aFloat=13.13 aDate=2013-12-13`), }, } return } func TestColumnString(t *testing.T) { tables := getTestCases() for _, table := range tables { for _, c := range table.Columns { assert.Equal(t, c.Name+" "+c.Type, c.String(), "Column.String doesn't work") } } } func TestNewExternalData(t *testing.T) { tables := getTestCases() for _, tt := range [][]ExternalTable{tables, tables[0:1], tables[1:]} { extData := NewExternalData(tt...) assert.ElementsMatch(t, extData.Tables, tt, "tables don't match ExternalData") } } func TestBuildBody(t *testing.T) { tables := getTestCases() for _, tt := range [][]ExternalTable{tables, tables[0:1], tables[1:]} { extData := NewExternalData(tt...) u := &url.URL{} body, header, err := extData.buildBody(context.Background(), u) assert.NoError(t, err, "body is not built") assert.Regexp(t, "^multipart/form-data; boundary=[A-Fa-f0-9]+$", header, "header does not match") contentID := strings.TrimPrefix(header, "multipart/form-data; boundary=") var b string vals := make(url.Values) for _, table := range tt { b += "--" + contentID b += "\r\nContent-Disposition: form-data; name=\"" + table.Name + "\"; filename=\"" + table.Name + "\"\r\n" b += "Content-Type: application/octet-stream\r\n\r\n" + string(table.Data) + "\r\n" vals[table.Name+"_format"] = []string{table.Format} vals[table.Name+"_structure"] = make([]string, 0) for _, c := range table.Columns { vals[table.Name+"_structure"] = append(vals[table.Name+"_structure"], c.String()) } } b += "--" + contentID + "--\r\n" assert.Equal(t, b, body.String(), "built body and expected body don't match") } } func TestDebugDump(t *testing.T) { extData := NewExternalData(getTestCases()...) dir, err := os.MkdirTemp(".", "external-data") if err != nil { t.Fatalf("unable to create directory %s: %v", dir, err) } defer os.RemoveAll(dir) reqID := fmt.Sprintf("%x", rand.Uint32()) ctx := scope.WithRequestID(context.Background(), reqID) ctx = scope.WithDebug(ctx, "External-Data") extData.SetDebug(dir, 0640) u := url.URL{} extData.debugDump(ctx, u) for _, table := range extData.Tables { dumpFile := filepath.Join(dir, fmt.Sprintf("ext-%v:%v.%v", table.Name, reqID, table.Format)) assert.FileExists(t, dumpFile) data, err := os.ReadFile(dumpFile) assert.NoError(t, err, "unable to read dump file: %w", err) assert.Equal(t, table.Data, data, "data in the file and source are different") } } ================================================ FILE: helper/client/datetime.go ================================================ package client import ( "time" "github.com/lomik/graphite-clickhouse/helper/datetime" ) func MetricsTimestampTruncate(metrics []Metric, precision time.Duration) { if precision == 0 { return } for i := range metrics { metrics[i].StartTime = datetime.TimestampTruncate(metrics[i].StartTime, precision) metrics[i].StopTime = datetime.TimestampTruncate(metrics[i].StopTime, precision) metrics[i].RequestStartTime = datetime.TimestampTruncate(metrics[i].RequestStartTime, precision) metrics[i].RequestStopTime = datetime.TimestampTruncate(metrics[i].RequestStopTime, precision) } } ================================================ FILE: helper/client/errros.go ================================================ package client import "strconv" type HttpError struct { statusCode int message string } func NewHttpError(statusCode int, message string) *HttpError { return &HttpError{ statusCode: statusCode, message: message, } } func (e *HttpError) Error() string { return strconv.Itoa(e.statusCode) + ": " + e.message } ================================================ FILE: helper/client/find.go ================================================ package client import ( "bytes" "fmt" "io" "net/http" "net/url" protov2 "github.com/go-graphite/protocol/carbonapi_v2_pb" protov3 "github.com/go-graphite/protocol/carbonapi_v3_pb" pickle "github.com/lomik/og-rek" ) type FindMatch struct { Path string `toml:"path"` IsLeaf bool `toml:"is_leaf"` } // MetricsFind do /metrics/find/ request // Valid formats are carbonapi_v3_pb. protobuf, pickle func MetricsFind(client *http.Client, address string, format FormatType, query string, from, until int64) (string, []FindMatch, http.Header, error) { if format == FormatDefault { format = FormatPb_v3 } rUrl := "/metrics/find/" queryParams := fmt.Sprintf("%s?format=%s, from=%d, until=%d, query %s", rUrl, format.String(), from, until, query) var fromStr, untilStr string u, err := url.Parse(address + rUrl) if err != nil { return queryParams, nil, nil, err } v := url.Values{ "format": []string{format.String()}, } var reader io.Reader switch format { case FormatPb_v3: var body []byte r := protov3.MultiGlobRequest{ Metrics: []string{query}, StartTime: from, StopTime: until, } body, err = r.Marshal() if err != nil { return query, nil, nil, err } if body != nil { reader = bytes.NewReader(body) } case FormatProtobuf, FormatPickle: v["query"] = []string{query} if from > 0 { v["from"] = []string{fromStr} } if until > 0 { v["until"] = []string{untilStr} } default: return queryParams, nil, nil, ErrUnsupportedFormat } u.RawQuery = v.Encode() req, err := http.NewRequest(http.MethodGet, u.String(), reader) if err != nil { return queryParams, nil, nil, err } resp, err := client.Do(req) if err != nil { return queryParams, nil, nil, err } defer resp.Body.Close() b, err := io.ReadAll(resp.Body) if err != nil { return queryParams, nil, nil, err } if resp.StatusCode == http.StatusNotFound { return queryParams, nil, resp.Header, nil } else if resp.StatusCode != http.StatusOK { return queryParams, nil, resp.Header, NewHttpError(resp.StatusCode, string(b)) } var globs []FindMatch switch format { case FormatProtobuf: var globsv2 protov2.GlobResponse if err = globsv2.Unmarshal(b); err != nil { return queryParams, nil, resp.Header, err } for _, m := range globsv2.Matches { globs = append(globs, FindMatch{Path: m.Path, IsLeaf: m.IsLeaf}) } case FormatPb_v3: var globsv3 protov3.MultiGlobResponse if err = globsv3.Unmarshal(b); err != nil { return queryParams, nil, resp.Header, err } for _, m := range globsv3.Metrics { for _, v := range m.Matches { globs = append(globs, FindMatch{Path: v.Path, IsLeaf: v.IsLeaf}) } } case FormatPickle: reader := bytes.NewReader(b) decoder := pickle.NewDecoder(reader) p, err := decoder.Decode() if err != nil { return queryParams, nil, resp.Header, err } for _, v := range p.([]interface{}) { m := v.(map[interface{}]interface{}) path := m["metric_path"].(string) isLeaf := m["isLeaf"].(bool) globs = append(globs, FindMatch{Path: path, IsLeaf: isLeaf}) } default: return queryParams, nil, resp.Header, ErrUnsupportedFormat } return queryParams, globs, resp.Header, nil } ================================================ FILE: helper/client/render.go ================================================ package client import ( "bytes" "encoding/json" "errors" "fmt" "io" "math" "net/http" "net/url" "strconv" "strings" protov2 "github.com/go-graphite/protocol/carbonapi_v2_pb" protov3 "github.com/go-graphite/protocol/carbonapi_v3_pb" pickle "github.com/lomik/og-rek" ) var ( ErrInvalidFrom = errors.New("invalid from") ErrInvalidUntil = errors.New("invalid until") ) type Metric struct { Name string `toml:"name"` PathExpression string `toml:"path"` ConsolidationFunc string `toml:"consolidation"` StartTime int64 `toml:"start"` StopTime int64 `toml:"stop"` StepTime int64 `toml:"step"` XFilesFactor float32 `toml:"xfiles"` HighPrecisionTimestamps bool `toml:"precision"` Values []float64 `toml:"values"` AppliedFunctions []string `toml:"applied_functions"` RequestStartTime int64 `toml:"req_start"` RequestStopTime int64 `toml:"req_stop"` } // Render do /metrics/find/ request // Valid formats are carbonapi_v3_pb. protobuf, pickle, json func Render(client *http.Client, address string, format FormatType, targets []string, filteringFunctions []*protov3.FilteringFunction, maxDataPoints, from, until int64) (string, []Metric, http.Header, error) { rUrl := "/render/" if format == FormatDefault { format = FormatPb_v3 } queryParams := fmt.Sprintf("%s?format=%s, from=%d, until=%d, targets [%s]", rUrl, format.String(), from, until, strings.Join(targets, ",")) if len(targets) == 0 { return queryParams, nil, nil, nil } if from <= 0 { return queryParams, nil, nil, ErrInvalidFrom } if until <= 0 { return queryParams, nil, nil, ErrInvalidUntil } fromStr := strconv.FormatInt(from, 10) untilStr := strconv.FormatInt(until, 10) maxDataPointsStr := strconv.FormatInt(maxDataPoints, 10) u, err := url.Parse(address + rUrl) if err != nil { return queryParams, nil, nil, err } var v url.Values var reader io.Reader switch format { case FormatPb_v3: v = url.Values{ "format": []string{format.String()}, } u.RawQuery = v.Encode() var body []byte r := protov3.MultiFetchRequest{ Metrics: make([]protov3.FetchRequest, len(targets)), } for i, target := range targets { r.Metrics[i] = protov3.FetchRequest{ Name: target, StartTime: from, StopTime: until, PathExpression: target, FilterFunctions: filteringFunctions, MaxDataPoints: maxDataPoints, } } body, err = r.Marshal() if err != nil { return queryParams, nil, nil, err } if body != nil { reader = bytes.NewReader(body) } case FormatPb_v2, FormatProtobuf, FormatPickle, FormatJSON: v := url.Values{ "format": []string{format.String()}, "from": []string{fromStr}, "until": []string{untilStr}, "target": targets, "maxDataPoints": []string{maxDataPointsStr}, } u.RawQuery = v.Encode() default: return queryParams, nil, nil, ErrUnsupportedFormat } req, err := http.NewRequest(http.MethodGet, u.String(), reader) if err != nil { return queryParams, nil, nil, err } resp, err := client.Do(req) if err != nil { return queryParams, nil, nil, err } defer resp.Body.Close() b, err := io.ReadAll(resp.Body) if err != nil { return queryParams, nil, nil, err } if resp.StatusCode == http.StatusNotFound { return queryParams, nil, resp.Header, nil } else if resp.StatusCode != http.StatusOK { return queryParams, nil, resp.Header, NewHttpError(resp.StatusCode, string(b)) } metrics, err := Decode(b, format) if err != nil { return queryParams, nil, resp.Header, err } return queryParams, metrics, resp.Header, nil } // Decode converts data in the give format to a Metric func Decode(b []byte, format FormatType) ([]Metric, error) { var ( metrics []Metric err error ) switch format { case FormatPb_v3: var r protov3.MultiFetchResponse err = r.Unmarshal(b) if err != nil { return nil, err } metrics = make([]Metric, 0, len(r.Metrics)) for _, m := range r.Metrics { metrics = append(metrics, Metric{ Name: m.Name, PathExpression: m.PathExpression, ConsolidationFunc: m.ConsolidationFunc, StartTime: m.StartTime, StopTime: m.StopTime, StepTime: m.StepTime, XFilesFactor: m.XFilesFactor, HighPrecisionTimestamps: m.HighPrecisionTimestamps, Values: m.Values, AppliedFunctions: m.AppliedFunctions, RequestStartTime: m.RequestStartTime, RequestStopTime: m.StopTime, }) } case FormatPb_v2, FormatProtobuf: var r protov2.MultiFetchResponse err = r.Unmarshal(b) if err != nil { return nil, err } metrics = make([]Metric, 0, len(r.Metrics)) for _, m := range r.Metrics { for i, a := range m.IsAbsent { if a { m.Values[i] = math.NaN() } } metrics = append(metrics, Metric{ Name: m.Name, StartTime: int64(m.StartTime), StopTime: int64(m.StopTime), StepTime: int64(m.StepTime), Values: m.Values, }) } case FormatPickle: reader := bytes.NewReader(b) decoder := pickle.NewDecoder(reader) p, err := decoder.Decode() if err != nil { return nil, err } for _, v := range p.([]interface{}) { m := v.(map[interface{}]interface{}) vals := m["values"].([]interface{}) values := make([]float64, len(vals)) for i, vv := range vals { if _, isNaN := vv.(pickle.None); isNaN { values[i] = math.NaN() } else { values[i] = vv.(float64) } } metrics = append(metrics, Metric{ Name: m["name"].(string), PathExpression: m["pathExpression"].(string), StartTime: m["start"].(int64), StopTime: m["end"].(int64), StepTime: m["step"].(int64), Values: values, }) } case FormatJSON: var r jsonResponse err = json.Unmarshal(b, &r) if err != nil { return nil, err } metrics = make([]Metric, 0, len(r.Metrics)) for _, m := range r.Metrics { values := make([]float64, len(m.Values)) for i, v := range m.Values { if v == nil { values[i] = math.NaN() } else { values[i] = *v } } metrics = append(metrics, Metric{ Name: m.Name, PathExpression: m.PathExpression, StartTime: m.StartTime, StopTime: m.StopTime, StepTime: m.StepTime, Values: values, }) } default: return nil, ErrUnsupportedFormat } return metrics, nil } // jsonResponse is a simple struct to decode JSON responses for testing purposes type jsonResponse struct { Metrics []jsonMetric `json:"metrics"` } type jsonMetric struct { Name string `json:"name"` PathExpression string `json:"pathExpression"` Values []*float64 `json:"values"` StartTime int64 `json:"startTime"` StopTime int64 `json:"stopTime"` StepTime int64 `json:"stepTime"` } ================================================ FILE: helper/client/requests.go ================================================ package client import protov3 "github.com/go-graphite/protocol/carbonapi_v3_pb" type MultiGlobRequestV3 struct { protov3.MultiGlobRequest } func (r *MultiGlobRequestV3) Marshal() ([]byte, error) { return r.MultiGlobRequest.Marshal() } func (r *MultiGlobRequestV3) LogInfo() interface{} { return r.MultiGlobRequest } ================================================ FILE: helper/client/tags.go ================================================ package client import ( "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "strconv" "strings" "github.com/msaf1980/go-stringutils" ) // TagsNames do /tags/autoComplete/tags request with query like [tagPrefix];tag1=value1;tag2=~value* // Valid formats are json func TagsNames(client *http.Client, address string, format FormatType, query string, limit uint64, from, until int64) (string, []string, http.Header, error) { rTags := "/tags/autoComplete/tags" if format == FormatDefault { format = FormatJSON } var queryParams string switch format { case FormatJSON: break default: queryParams = fmt.Sprintf("%s?format=%s, from=%d, until=%d, limits=%d, query %s", rTags, format.String(), from, until, limit, query) return queryParams, nil, nil, ErrUnsupportedFormat } u, err := url.Parse(address + rTags) if err != nil { return queryParams, nil, nil, err } var tagPrefix string var exprs []string if query != "" && query != "<>" { args := strings.Split(query, ";") if len(args) < 1 { return queryParams, nil, nil, ErrInvalidQuery } exprs = make([]string, 0, len(args)) for i, arg := range args { delim := strings.IndexRune(arg, '=') if i == 0 && delim == -1 { tagPrefix = arg } else if delim <= 0 { return queryParams, nil, nil, errors.New("invalid expr: " + arg) } else { exprs = append(exprs, arg) } } } v := make([]string, 0, 2+len(exprs)) var rawQuery stringutils.Builder rawQuery.Grow(128) v = append(v, "format="+format.String()) rawQuery.WriteString("format=") rawQuery.WriteString(url.QueryEscape(format.String())) if tagPrefix != "" { v = append(v, "tagPrefix="+tagPrefix) rawQuery.WriteString("&tagPrefix=") rawQuery.WriteString(url.QueryEscape(tagPrefix)) } for _, expr := range exprs { v = append(v, "expr="+expr) rawQuery.WriteString("&expr=") rawQuery.WriteString(url.QueryEscape(expr)) } if from > 0 { fromStr := strconv.FormatInt(from, 10) v = append(v, "from="+fromStr) rawQuery.WriteString("&from=") rawQuery.WriteString(fromStr) } if until > 0 { untilStr := strconv.FormatInt(until, 10) v = append(v, "until="+untilStr) rawQuery.WriteString("&until=") rawQuery.WriteString(untilStr) } if limit > 0 { limitStr := strconv.FormatUint(limit, 10) v = append(v, "limit="+limitStr) rawQuery.WriteString("&limit=") rawQuery.WriteString(limitStr) } queryParams = fmt.Sprintf("%s %q", rTags, v) u.RawQuery = rawQuery.String() req, err := http.NewRequest(http.MethodGet, u.String(), nil) if err != nil { return queryParams, nil, nil, err } resp, err := client.Do(req) if err != nil { return queryParams, nil, nil, err } defer resp.Body.Close() b, err := io.ReadAll(resp.Body) if err != nil { return queryParams, nil, nil, err } if resp.StatusCode == http.StatusNotFound { return u.RawQuery, nil, resp.Header, nil } else if resp.StatusCode != http.StatusOK { return queryParams, nil, resp.Header, NewHttpError(resp.StatusCode, string(b)) } var values []string err = json.Unmarshal(b, &values) if err != nil { return queryParams, nil, resp.Header, errors.New(err.Error() + ": " + string(b)) } return queryParams, values, resp.Header, nil } // TagsValues do /tags/autoComplete/values request with query like searchTag[=valuePrefix];tag1=value1;tag2=~value* // Valid formats are json func TagsValues(client *http.Client, address string, format FormatType, query string, limit uint64, from, until int64) (string, []string, http.Header, error) { rTags := "/tags/autoComplete/values" if format == FormatDefault { format = FormatJSON } var queryParams string switch format { case FormatJSON: break default: queryParams = fmt.Sprintf("%s?format=%s, from=%d, until=%d, limits=%d, query %s", rTags, format.String(), from, until, limit, query) return queryParams, nil, nil, ErrUnsupportedFormat } u, err := url.Parse(address + rTags) if err != nil { return queryParams, nil, nil, err } var ( tag string valuePrefix string exprs []string ) if query != "" && query != "<>" { args := strings.Split(query, ";") if len(args) < 2 { return queryParams, nil, nil, ErrInvalidQuery } vals := strings.Split(args[0], "=") tag = vals[0] if len(vals) > 2 { return queryParams, nil, nil, errors.New("invalid tag: " + args[0]) } else if len(vals) == 2 { valuePrefix = vals[1] } exprs = make([]string, 0, len(args)-1) for i := 1; i < len(args); i++ { expr := args[i] if strings.IndexRune(expr, '=') <= 0 { return queryParams, nil, nil, errors.New("invalid expr: " + expr) } exprs = append(exprs, expr) } } v := make([]string, 0, 2+len(exprs)) var rawQuery stringutils.Builder rawQuery.Grow(128) v = append(v, "format="+format.String()) rawQuery.WriteString("format=") rawQuery.WriteString(url.QueryEscape(format.String())) if tag != "" { v = append(v, "tag="+tag) rawQuery.WriteString("&tag=") rawQuery.WriteString(url.QueryEscape(tag)) } if valuePrefix != "" { v = append(v, "valuePrefix="+valuePrefix) rawQuery.WriteString("&valuePrefix=") rawQuery.WriteString(url.QueryEscape(valuePrefix)) } for _, expr := range exprs { v = append(v, "expr="+expr) rawQuery.WriteString("&expr=") rawQuery.WriteString(url.QueryEscape(expr)) } if from > 0 { fromStr := strconv.FormatInt(from, 10) v = append(v, "from="+fromStr) rawQuery.WriteString("&from=") rawQuery.WriteString(fromStr) } if until > 0 { untilStr := strconv.FormatInt(until, 10) v = append(v, "until="+untilStr) rawQuery.WriteString("&until=") rawQuery.WriteString(untilStr) } if limit > 0 { limitStr := strconv.FormatUint(limit, 10) v = append(v, "limit="+limitStr) rawQuery.WriteString("&limit=") rawQuery.WriteString(limitStr) } queryParams = fmt.Sprintf("%s %q", rTags, v) u.RawQuery = rawQuery.String() req, err := http.NewRequest(http.MethodGet, u.String(), nil) if err != nil { return queryParams, nil, nil, err } resp, err := client.Do(req) if err != nil { return u.RawQuery, nil, nil, err } defer resp.Body.Close() b, err := io.ReadAll(resp.Body) if err != nil { return queryParams, nil, nil, err } if resp.StatusCode == http.StatusNotFound { return queryParams, nil, resp.Header, nil } else if resp.StatusCode != http.StatusOK { return queryParams, nil, resp.Header, NewHttpError(resp.StatusCode, string(b)) } var values []string err = json.Unmarshal(b, &values) if err != nil { return queryParams, nil, resp.Header, errors.New(err.Error() + ": " + string(b)) } return queryParams, values, resp.Header, nil } ================================================ FILE: helper/client/types.go ================================================ package client import ( "errors" "fmt" ) type FormatType int const ( FormatDefault FormatType = iota FormatJSON FormatProtobuf FormatPb_v2 // alias for FormatProtobuf FormatPb_v3 FormatPickle ) var formatStrings []string = []string{"default", "json", "protobuf", "carbonapi_v2_pb", "carbonapi_v3_pb", "pickle"} func (a *FormatType) String() string { return formatStrings[*a] } func FormatTypes() []string { return formatStrings } func (a *FormatType) Set(value string) error { switch value { case "json": *a = FormatJSON case "protobuf": *a = FormatProtobuf case "carbonapi_v2_pb": *a = FormatPb_v2 case "carbonapi_v3_pb": *a = FormatPb_v3 case "pickle": *a = FormatPickle default: return fmt.Errorf("invalid format type %s", value) } return nil } func (a *FormatType) UnmarshalText(text []byte) error { return a.Set(string(text)) } var ( ErrUnsupportedFormat = errors.New("unsupported format") ErrInvalidQuery = errors.New("invalid query") //ErrEmptyQuery = errors.New("missing query") ) ================================================ FILE: helper/date/date.go ================================================ package date import "time" var FromTimestampToDaysFormat func(int64) string var FromTimeToDaysFormat func(time.Time) string var UntilTimestampToDaysFormat func(int64) string var UntilTimeToDaysFormat func(time.Time) string // SetDefault() is for broken SlowTimestampToDays in carbon-clickhouse func SetDefault() { FromTimestampToDaysFormat = DefaultTimestampToDaysFormat FromTimeToDaysFormat = DefaultTimeToDaysFormat UntilTimestampToDaysFormat = DefaultTimestampToDaysFormat UntilTimeToDaysFormat = DefaultTimeToDaysFormat } // SetUTC() is for UTCTimestampToDays in carbon-clickhouse (see https://github.com/go-graphite/carbon-clickhouse/pull/114) func SetUTC() { FromTimestampToDaysFormat = UTCTimestampToDaysFormat FromTimeToDaysFormat = UTCTimeToDaysFormat UntilTimestampToDaysFormat = UTCTimestampToDaysFormat UntilTimeToDaysFormat = UTCTimeToDaysFormat } // SetBoth() is for mixed SlowTimestampToDays/UTCTimestampToDays (before rebuild tables complete) func SetBoth() { FromTimestampToDaysFormat = MinTimestampToDaysFormat FromTimeToDaysFormat = MinTimeToDaysFormat UntilTimestampToDaysFormat = MaxTimestampToDaysFormat UntilTimeToDaysFormat = MaxTimeToDaysFormat } func init() { SetDefault() } // from carbon-clickhouse, port of SlowTimestampToDays, broken symmetic, not always UTC func DefaultTimestampToDaysFormat(ts int64) string { t := time.Unix(ts, 0) return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC).Format("2006-01-02") } // from carbon-clickhouse, port of SlowTimestampToDays, broken symmetic, not always UTC func DefaultTimeToDaysFormat(t time.Time) string { return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC).Format("2006-01-02") } func UTCTimestampToDaysFormat(timestamp int64) string { return time.Unix(timestamp, 0).UTC().Format("2006-01-02") } func UTCTimeToDaysFormat(t time.Time) string { return t.UTC().Format("2006-01-02") } func defaultDate(t time.Time) time.Time { return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC) } func minLocalAndUTC(t time.Time) time.Time { tu := defaultDate(t.UTC()) td := defaultDate(t) if tu.Unix() < td.Unix() { return tu } else { return td } } // MinTimestampToDaysFormat return formatted minimum (local, UTC) date func MinTimestampToDaysFormat(ts int64) string { t := minLocalAndUTC(time.Unix(ts, 0)) return t.Format("2006-01-02") } // MinTimeToDaysFormat return formatted minimum (local, UTC) date func MinTimeToDaysFormat(t time.Time) string { t = minLocalAndUTC(t) return t.Format("2006-01-02") } func maxLocalAndUTC(t time.Time) time.Time { tu := defaultDate(t.UTC()) td := defaultDate(t) if tu.Unix() > td.Unix() { return tu } else { return td } } // MaxTimestampToDaysFormat return formatted maximum (local, UTC) date func MaxTimestampToDaysFormat(ts int64) string { t := maxLocalAndUTC(time.Unix(ts, 0)) return t.Format("2006-01-02") } // MaxTimeToDaysFormat return formatted maximum (local, UTC) date func MaxTimeToDaysFormat(t time.Time) string { t = maxLocalAndUTC(t) return t.Format("2006-01-02") } ================================================ FILE: helper/date/date_test.go ================================================ package date import ( "os" "strconv" "testing" "time" ) var verbose bool func isVerbose() bool { for _, arg := range os.Args { if arg == "-test.v=true" { return true } } return false } func init() { verbose = isVerbose() } // TimestampDaysFormat is broken symmetic with carbon-clickhouse of SlowTimestampToDays, not always UTC // $ TZ=Etc/GMT-5 go test -v -timeout 30s -run ^TestTimestampDaysFormat$ github.com/lomik/graphite-clickhouse/helper/date // === RUN TestTimestampDaysFormat // === RUN TestTimestampDaysFormat/1668106870_2022-11-11T00:01:10+05:00_2022-11-10T19:01:10Z_[0] // // date_test.go:62: Warning (TimestampDaysFormat broken) TimestampDaysFormat(1668106870) = 2022-11-11, want UTC 2022-11-10 // // --- FAIL: TestTimestampDaysFormat (0.00s) // // --- FAIL: TestTimestampDaysFormat/1668106870_2022-11-11T00:01:10+05:00_2022-11-10T19:01:10Z_[0] (0.00s) // // FAIL // FAIL github.com/lomik/graphite-clickhouse/helper/date 0.001s // // $ TZ=Etc/GMT+5 go test -v -timeout 30s -run ^TestTimestampDaysFormat$ github.com/lomik/graphite-clickhouse/helper/date // === RUN TestTimestampDaysFormat // === RUN TestTimestampDaysFormat/1668124800_2022-11-10T19:00:00-05:00_2022-11-11T00:00:00Z_[1] // // date_test.go:62: Warning (TimestampDaysFormat broken) TimestampDaysFormat(1668124800) = 2022-11-10, want UTC 2022-11-11 // // === RUN TestTimestampDaysFormat/1668142799_2022-11-10T23:59:59-05:00_2022-11-11T04:59:59Z_[2] // // date_test.go:62: Warning (TimestampDaysFormat broken) TimestampDaysFormat(1668142799) = 2022-11-10, want UTC 2022-11-11 // // === RUN TestTimestampDaysFormat/1650776160_2022-04-23T23:56:00-05:00_2022-04-24T04:56:00Z_[3] // // date_test.go:62: Warning (TimestampDaysFormat broken) TimestampDaysFormat(1650776160) = 2022-04-23, want UTC 2022-04-24 // // --- FAIL: TestTimestampDaysFormat (0.00s) func TestDefaultTimestampToDaysFormat(t *testing.T) { tests := []struct { ts int64 want string }{ { ts: 1668106870, // 2022-11-11 00:01:10 +05:00 ; 2022-11-10 19:01:10 UTC // select toDate(1650776160,'UTC') // 2022-11-10 want: time.Unix(1668106870, 0).Format("2006-01-02"), }, { ts: 1668124800, // 2022-11-11 00:00:00 UTC want: time.Unix(1668124800, 0).Format("2006-01-02"), }, { ts: 1668142799, // 2022-11-10 23:59:59 -05:00; 2022-11-11 04:59:59 UTC want: time.Unix(1668142799, 0).Format("2006-01-02"), }, { ts: 1650776160, // graphite-clickhouse issue #184, graphite-clickhouse in UTC, clickhouse in PDT(UTC-7) // 2022-04-24 4:56:00 // select toDate(1650776160,'UTC') // 2022-04-24 // select toDate(1650776160,'Etc/GMT+7') // 2022-04-23 want: time.Unix(1650776160, 0).Format("2006-01-02"), }, } for i, tt := range tests { t.Run(strconv.FormatInt(tt.ts, 10)+" "+time.Unix(tt.ts, 0).Format(time.RFC3339)+" "+time.Unix(tt.ts, 0).UTC().Format(time.RFC3339)+" ["+strconv.Itoa(i)+"]", func(t *testing.T) { if got := DefaultTimestampToDaysFormat(tt.ts); got != tt.want { t.Errorf("DefaultTimestampDaysFormat(%d) = %s, want %s", tt.ts, got, tt.want) } else if gotUTC := UTCTimestampToDaysFormat(tt.ts); got != gotUTC { // Run to see a warning // go test -v -timeout 30s -run ^TestTimestampDaysFormat$ github.com/lomik/graphite-clickhouse/helper/date if verbose { t.Errorf("Warning (DefaultTimestampDaysFormat broken) DefaultTimestampDaysFormat(%d) = %s, want UTC %s", tt.ts, got, gotUTC) } else { t.Logf("Warning (DefaultTimestampDaysFormat broken) DefaultTimestampDaysFormat(%d) = %s, want UTC %s", tt.ts, got, gotUTC) } } }) } } func TestDefaultTimeToDaysFormat(t *testing.T) { tests := []struct { ts int64 want string }{ { ts: 1668106870, // 2022-11-11 00:01:10 +05:00 ; 2022-11-10 19:01:10 UTC // select toDate(1650776160,'UTC') // 2022-11-10 want: time.Unix(1668106870, 0).Format("2006-01-02"), }, { ts: 1668124800, // 2022-11-11 00:00:00 UTC want: time.Unix(1668124800, 0).Format("2006-01-02"), }, { ts: 1668142799, // 2022-11-10 23:59:59 -05:00; 2022-11-11 04:59:59 UTC want: time.Unix(1668142799, 0).Format("2006-01-02"), }, { ts: 1650776160, // graphite-clickhouse issue #184, graphite-clickhouse in UTC, clickhouse in PDT(UTC-7) // 2022-04-24 4:56:00 // select toDate(1650776160,'UTC') // 2022-04-24 // select toDate(1650776160,'Etc/GMT+7') // 2022-04-23 want: time.Unix(1650776160, 0).Format("2006-01-02"), }, } for i, tt := range tests { t.Run(strconv.FormatInt(tt.ts, 10)+" "+time.Unix(tt.ts, 0).UTC().Format(time.RFC3339)+" ["+strconv.Itoa(i)+"]", func(t *testing.T) { if got := DefaultTimeToDaysFormat(time.Unix(tt.ts, 0)); got != tt.want { t.Errorf("DefaultTimeDaysFormat() = %s, want %s", got, tt.want) } }) } } func TestUTCTimestampToDaysFormat(t *testing.T) { tests := []struct { ts int64 want string }{ { ts: 1668106870, // 2022-11-11 00:01:10 +05:00 ; 2022-11-10 19:01:10 UTC // select toDate(1650776160,'UTC') // 2022-11-10 want: "2022-11-10", }, { ts: 1668124800, // 2022-11-11 00:00:00 UTC want: "2022-11-11", }, { ts: 1668142799, // 2022-11-10 23:59:59 -05:00; 2022-11-11 04:59:59 UTC want: "2022-11-11", }, { ts: 1650776160, // graphite-clickhouse issue #184, graphite-clickhouse in UTC, clickhouse in PDT(UTC-7) // 2022-04-24 4:56:00 // select toDate(1650776160,'UTC') // 2022-04-24 // select toDate(1650776160,'Etc/GMT+7') // 2022-04-23 want: "2022-04-24", }, } for i, tt := range tests { t.Run(strconv.FormatInt(tt.ts, 10)+" "+time.Unix(tt.ts, 0).Format(time.RFC3339)+" "+time.Unix(tt.ts, 0).UTC().Format(time.RFC3339)+" ["+strconv.Itoa(i)+"]", func(t *testing.T) { if got := UTCTimestampToDaysFormat(tt.ts); got != tt.want { t.Errorf("UTCTimestampDaysFormat(%d) = %s, want %s", tt.ts, got, tt.want) } }) } } func TestUTCTimeToDaysFormat(t *testing.T) { tests := []struct { ts int64 want string }{ { ts: 1668106870, // 2022-11-11 00:01:10 +05:00 ; 2022-11-10 19:01:10 UTC // select toDate(1650776160,'UTC') // 2022-11-10 want: "2022-11-10", }, { ts: 1668124800, // 2022-11-11 00:00:00 UTC want: "2022-11-11", }, { ts: 1668142799, // 2022-11-10 23:59:59 -05:00; 2022-11-11 04:59:59 UTC want: "2022-11-11", }, { ts: 1650776160, // graphite-clickhouse issue #184, graphite-clickhouse in UTC, clickhouse in PDT(UTC-7) // 2022-04-24 4:56:00 // select toDate(1650776160,'UTC') // 2022-04-24 // select toDate(1650776160,'Etc/GMT+7') // 2022-04-23 want: "2022-04-24", }, } for i, tt := range tests { t.Run(strconv.FormatInt(tt.ts, 10)+" "+time.Unix(tt.ts, 0).UTC().Format(time.RFC3339)+" ["+strconv.Itoa(i)+"]", func(t *testing.T) { if got := UTCTimeToDaysFormat(time.Unix(tt.ts, 0)); got != tt.want { t.Errorf("UTCTimeDaysFormat() = %s, want %s", got, tt.want) } }) } } func TestMinMaxTimestampToDaysFormat(t *testing.T) { tests := []struct { ts int64 }{ { ts: 1668106870, // 2022-11-11 00:01:10 +05:00 ; 2022-11-10 19:01:10 UTC // select toDate(1650776160,'UTC') // 2022-11-10 }, { ts: 1668124800, // 2022-11-11 00:00:00 UTC }, { ts: 1668142799, // 2022-11-10 23:59:59 -05:00; 2022-11-11 04:59:59 UTC }, { ts: 1650776160, // graphite-clickhouse issue #184, graphite-clickhouse in UTC, clickhouse in PDT(UTC-7) // 2022-04-24 4:56:00 // select toDate(1650776160,'UTC') // 2022-04-24 // select toDate(1650776160,'Etc/GMT+7') // 2022-04-23 }, } for i, tt := range tests { t.Run(strconv.FormatInt(tt.ts, 10)+" "+time.Unix(tt.ts, 0).UTC().Format(time.RFC3339)+" ["+strconv.Itoa(i)+"]", func(t *testing.T) { gotMin := MinTimestampToDaysFormat(tt.ts) timeMin, _ := time.Parse("2006-01-02", gotMin) gotMax := MaxTimestampToDaysFormat(tt.ts) timeMax, _ := time.Parse("2006-01-02", gotMax) got := DefaultTimestampToDaysFormat(tt.ts) tm, _ := time.Parse("2006-01-02", got) if timeMin.UnixNano() > timeMax.UnixNano() || tm.UnixNano() > timeMax.UnixNano() || tm.UnixNano() < timeMin.UnixNano() { t.Errorf("MinTimeDaysFormat() = %s > MaxTimeDaysFormat() = %s, DefaultTimeDaysFormat() = %s", gotMin, gotMax, got) } else { t.Logf("MinTimeDaysFormat() = %s, MaxTimeDaysFormat() = %s, DefaultTimeDaysFormat() = %s", gotMin, gotMax, got) } }) } } ================================================ FILE: helper/datetime/datetime.go ================================================ package datetime import ( "errors" "strconv" "strings" "time" "github.com/go-graphite/carbonapi/pkg/parser" ) var ErrBadTime = errors.New("bad time") // parseTime parses a time and returns hours and minutes func parseTime(s string) (hour, minute int, err error) { switch s { case "midnight": return 0, 0, nil case "noon": return 12, 0, nil case "teatime": return 16, 0, nil } parts := strings.Split(s, ":") if len(parts) != 2 { return 0, 0, ErrBadTime } hour, err = strconv.Atoi(parts[0]) if err != nil { return 0, 0, ErrBadTime } minute, err = strconv.Atoi(parts[1]) if err != nil { return 0, 0, ErrBadTime } return hour, minute, nil } var TimeFormats = []string{"20060102", "01/02/06"} // DateParamToEpoch turns a passed string parameter into a unix epoch func DateParamToEpoch(s string, tz *time.Location, now time.Time, truncate time.Duration) int64 { if s == "" { // return the default if nothing was passed return 0 } // relative timestamp if s[0] == '-' { offset, err := parser.IntervalString(s, -1) if err != nil { return 0 } return now.Add(time.Duration(offset) * time.Second).Unix() } else if s[0] == '+' { offset, err := parser.IntervalString(s, 1) if err != nil { return 0 } return now.Add(time.Duration(offset) * time.Second).Unix() } switch s { case "now": return now.Unix() case "rnow": return TimeTruncate(now, truncate).Unix() case "midnight", "noon", "teatime": yy, mm, dd := now.Date() hh, min, _ := parseTime(s) // error ignored, we know it's valid dt := time.Date(yy, mm, dd, hh, min, 0, 0, tz) return dt.Unix() } sint, err := strconv.Atoi(s) // need to check that len(s) != 8 to avoid turning 20060102 into seconds if err == nil && len(s) != 8 { return int64(sint) // We got a timestamp so returning it } s = strings.Replace(s, "_", " ", 1) // Go can't parse _ in date strings var ts, ds string split := strings.Fields(s) var t time.Time switch { case len(split) == 1: delim := strings.IndexAny(s, "+-") if delim == -1 { ds = s } else { ds = s[:delim] switch ds { case "now", "today": t = now case "rnow", "rtoday": t = TimeTruncate(now, truncate) // nothing case "midnight", "noon", "teatime": yy, mm, dd := now.Date() hh, min, _ := parseTime(s) // error ignored, we know it's valid t = time.Date(yy, mm, dd, hh, min, 0, 0, tz) case "yesterday": t = now.AddDate(0, 0, -1) case "tomorrow": t = now.AddDate(0, 0, 1) default: return 0 } s = s[delim:] for len(s) > 0 { delim := strings.IndexAny(s[1:], "+-") if delim == -1 { ts = s s = s[:0] } else { ts = s[:delim+1] s = s[delim+1:] } offset, err := parser.IntervalString(ts, 1) if err != nil { offset64, err := strconv.ParseInt(ts, 10, 32) if err != nil { return 0 } offset = int32(offset64) } t = t.Add(time.Duration(offset) * time.Second) } return t.Unix() } case len(split) == 2: ts, ds = split[0], split[1] case len(split) > 2: return 0 } dateStringSwitch: switch ds { case "now", "today": t = now case "rnow", "rtoday": t = TimeTruncate(now, truncate) case "midnight", "noon", "teatime": yy, mm, dd := now.Date() hh, min, _ := parseTime(s) // error ignored, we know it's valid t = time.Date(yy, mm, dd, hh, min, 0, 0, tz) case "yesterday": t = now.AddDate(0, 0, -1) case "ryesterday": t = TimeTruncate(now, truncate).AddDate(0, 0, -1) case "tomorrow": t = now.AddDate(0, 0, 1) case "rtomorrow": t = TimeTruncate(now, truncate).AddDate(0, 0, 1) default: for _, format := range TimeFormats { t, err = time.ParseInLocation(format, ds, tz) if err == nil { break dateStringSwitch } } return 0 } var hour, minute int if ts != "" { // defaults to hour=0, minute=0 on error, which is midnight, which is fine for now hour, minute, _ = parseTime(ts) } yy, mm, dd := t.Date() t = time.Date(yy, mm, dd, hour, minute, 0, 0, tz) return t.Unix() } func Timezone(qtz string) (*time.Location, error) { if qtz == "" { qtz = "Local" } return time.LoadLocation(qtz) } func TimestampTruncate(ts int64, truncate time.Duration) int64 { if ts == 0 || truncate == 0 { return ts } tm := time.Unix(ts, 0).UTC() return tm.Truncate(truncate).UTC().Unix() } func TimeTruncate(tm time.Time, truncate time.Duration) time.Time { if truncate == 0 { return tm } return tm.Truncate(truncate) } ================================================ FILE: helper/datetime/datetime_test.go ================================================ package datetime import ( "testing" "time" ) func TestDateParamToEpoch(t *testing.T) { timeZone := time.Local //16 Aug 1994 15:30 now := time.Date(1994, time.August, 16, 15, 30, 0, 100, timeZone) const shortForm = "15:04:05 2006-Jan-02" var tests = []struct { input string output string }{ {"midnight", "00:00:00 1994-Aug-16"}, {"noon", "12:00:00 1994-Aug-16"}, {"teatime", "16:00:00 1994-Aug-16"}, {"tomorrow", "00:00:00 1994-Aug-17"}, {"noon 08/12/94", "12:00:00 1994-Aug-12"}, {"midnight 20060812", "00:00:00 2006-Aug-12"}, {"noon tomorrow", "12:00:00 1994-Aug-17"}, {"17:04 19940812", "17:04:00 1994-Aug-12"}, {"-1day", "15:30:00 1994-Aug-15"}, {"19940812", "00:00:00 1994-Aug-12"}, {"midnight-10", "23:59:50 1994-Aug-15"}, {"midnight-1s", "23:59:59 1994-Aug-15"}, {"midnight-1day", "00:00:00 1994-Aug-15"}, {"midnight-1day+1s", "00:00:01 1994-Aug-15"}, } for _, tt := range tests { var ( want int64 wantTime string ) if tt.output != "" { ts, err := time.ParseInLocation(shortForm, tt.output, timeZone) if err != nil { t.Fatalf("error parsing time: %q: %v", tt.output, err) } want = ts.Unix() wantTime = ts.Format(time.RFC3339Nano) } got := DateParamToEpoch(tt.input, timeZone, now, 0) if got != want { gotTime := time.Unix(got, 0).Format(time.RFC3339Nano) t.Errorf("dateParamToEpoch(%q, local)=\n%v (%s)\nwant\n%v (%s)", tt.input, got, gotTime, want, wantTime) } } } func TestDateParamToEpochTruncate(t *testing.T) { timeZone := time.Local //16 Aug 1994 15:30 now := time.Date(1994, time.August, 16, 15, 30, 0, 100, timeZone) const shortForm = "15:04:05 2006-Jan-02" var tests = []struct { input string output string }{ {"midnight", "00:00:00 1994-Aug-16"}, {"noon", "12:00:00 1994-Aug-16"}, {"teatime", "16:00:00 1994-Aug-16"}, {"tomorrow", "00:00:00 1994-Aug-17"}, {"noon 08/12/94", "12:00:00 1994-Aug-12"}, {"midnight 20060812", "00:00:00 2006-Aug-12"}, {"noon tomorrow", "12:00:00 1994-Aug-17"}, {"17:04 19940812", "17:04:00 1994-Aug-12"}, {"-1day", "15:30:00 1994-Aug-15"}, {"19940812", "00:00:00 1994-Aug-12"}, {"midnight-10", "23:59:50 1994-Aug-15"}, {"midnight-1s", "23:59:59 1994-Aug-15"}, {"midnight-1day", "00:00:00 1994-Aug-15"}, // truncate {"now-1", "15:29:59 1994-Aug-16"}, {"now-45s", "15:29:15 1994-Aug-16"}, } for _, tt := range tests { var ( want int64 wantTime string ) if tt.output != "" { ts, err := time.ParseInLocation(shortForm, tt.output, timeZone) if err != nil { t.Fatalf("error parsing time: %q: %v", tt.output, err) } want = ts.Unix() wantTime = ts.Format(time.RFC3339Nano) } got := DateParamToEpoch(tt.input, timeZone, now, 10*time.Second) if got != want { gotTime := time.Unix(got, 0).Format(time.RFC3339Nano) t.Errorf("dateParamToEpoch(%q, local)=\n%v (%s)\nwant\n%v (%s)", tt.input, got, gotTime, want, wantTime) } } } ================================================ FILE: helper/errs/errors.go ================================================ package errs import "fmt" type ErrorWithCode struct { err string Code int // error code } func NewErrorWithCode(err string, code int) error { return ErrorWithCode{err, code} } func NewErrorfWithCode(code int, f string, args ...interface{}) error { return ErrorWithCode{fmt.Sprintf(f, args...), code} } func (e ErrorWithCode) Error() string { return e.err } ================================================ FILE: helper/headers/headers.go ================================================ package headers import "net/http" func GetHeaders(header *http.Header, keys []string) map[string]string { if len(keys) > 0 { headers := make(map[string]string) for _, key := range keys { value := header.Get(key) if len(value) > 0 { headers[key] = value } } return headers } return nil } ================================================ FILE: helper/http/live-http-client.go ================================================ package http import ( "bufio" "bytes" "context" "io" "net" "net/http" "time" ) const TCPNetwork string = "tcp" func DoHTTPOverTCP(ctx context.Context, transport *http.Transport, req *http.Request, timeout time.Duration) (*http.Response, error) { conn, err := transport.DialContext(ctx, TCPNetwork, req.URL.Host) if err != nil { return nil, err } err = conn.SetDeadline(time.Now().Add(timeout)) if err != nil { return nil, err } err = req.Write(conn) if err != nil { return nil, err } var backup_buf bytes.Buffer reader := bufio.NewReader(io.TeeReader(conn, &backup_buf)) for { line, err := reader.ReadString('\n') if err != nil { if netErr, ok := err.(net.Error); ok && netErr.Timeout() { fake_body_delimer := bytes.NewBuffer([]byte{'\r', '\n', '\r', '\n'}) resp, err := http.ReadResponse(bufio.NewReader(io.MultiReader(&backup_buf, fake_body_delimer)), nil) if err != nil { return nil, err } return resp, netErr } return nil, err } if line == "\r\n" { break } } full_resp_stream := io.MultiReader(&backup_buf, conn) resp, err := http.ReadResponse(bufio.NewReader(full_resp_stream), nil) return resp, err } ================================================ FILE: helper/pickle/pickle.go ================================================ package pickle import ( "encoding/binary" "io" "math" ) var EmptyList = []byte{0x28, 0x6c, 0x70, 0x30, 0xa, 0x2e} // Pickle encoder type Writer struct { w io.Writer } func NewWriter(w io.Writer) *Writer { return &Writer{w: w} } func (p *Writer) Mark() { p.w.Write([]byte{'('}) } func (p *Writer) Stop() { p.w.Write([]byte{'.'}) } func (p *Writer) Append() { p.w.Write([]byte{'a'}) } func (p *Writer) SetItem() { p.w.Write([]byte{'s'}) } func (p *Writer) List() { p.w.Write([]byte{'(', 'l'}) } func (p *Writer) Dict() { p.w.Write([]byte{'(', 'd'}) } func (p *Writer) TupleEnd() { p.w.Write([]byte{'t'}) } func (p *Writer) Bytes(byt []byte) { l := len(byt) if l < 256 { p.w.Write([]byte{'U', byte(l)}) } else { var b [5]byte b[0] = 'T' binary.LittleEndian.PutUint32(b[1:5], uint32(l)) p.w.Write(b[:]) } p.w.Write(byt) } func (p *Writer) String(v string) { p.Bytes([]byte(v)) } func (p *Writer) Uint32(v uint32) { p.w.Write([]byte{'J'}) var b [4]byte binary.LittleEndian.PutUint32(b[:], v) p.w.Write(b[:]) } func (p *Writer) AppendFloat64(v float64) { u := math.Float64bits(v) var b [10]byte b[0] = 'G' b[9] = 'a' binary.BigEndian.PutUint64(b[1:10], u) p.w.Write(b[:]) } func (p *Writer) AppendNulls(count int) { for i := 0; i < count; i++ { p.w.Write([]byte{'N', 'a'}) } } func (p *Writer) Bool(b bool) { if b { p.w.Write([]byte{'I', '0', '1', '\n'}) } else { p.w.Write([]byte{'I', '0', '0', '\n'}) } } ================================================ FILE: helper/point/func.go ================================================ package point import ( "fmt" "math" ) // CleanUp removes points with empty metric // for run after Deduplicate, Merge, etc for result cleanup func CleanUp(points []Point) []Point { l := len(points) squashed := 0 for i := 0; i < l; i++ { if points[i].MetricID == 0 || math.IsNaN(points[i].Value) { squashed++ continue } if squashed > 0 { points[i-squashed] = points[i] } } return points[:l-squashed] } // Uniq removes points with equal metric and time func Uniq(points []Point) []Point { l := len(points) var i, n int // i - current position of iterator // n - position on first record with current key (metric + time) for i = 1; i < l; i++ { if points[i].MetricID != points[n].MetricID || points[i].Time != points[n].Time { n = i continue } if points[i].Timestamp > points[n].Timestamp { points[n] = points[i] } points[i].MetricID = 0 // mark for remove } return CleanUp(points) } // FillNulls accepts an ordered []Point for one metric and returns a generator that will return all points for specific // interval. Generator returns EmptyPoint when it's finished func FillNulls(points []Point, from, until, step uint32) (start, stop, count uint32, getter GetValueOrNaN) { start = from - (from % step) if start < from { start += step } stop = until - (until % step) + step count = (stop - start) / step last := start - step currentPoint := 0 var metricID uint32 if len(points) > 0 { metricID = points[0].MetricID } getter = func() (float64, error) { if stop <= last { return 0, ErrTimeGreaterStop } for i := currentPoint; i < len(points); i++ { point := points[i] if metricID != point.MetricID { return 0, fmt.Errorf("the point MetricID %d differs from other %d: %w", point.MetricID, metricID, ErrWrongMetricID) } if point.Time < start { // Points begin before request's start currentPoint++ continue } if point.Time <= last { // This is definitely an error. Possible reason is unsorted points return 0, fmt.Errorf("the time is less or equal to previous %d < %d: %w", point.Time, last, ErrPointsUnsorted) } if stop <= point.Time { break } if last+step < point.Time { // There are nulls in slice last += step return math.NaN(), nil } last = point.Time currentPoint = i + 1 return point.Value, nil } if last+step < stop { last += step return math.NaN(), nil } return 0, ErrTimeGreaterStop } return } ================================================ FILE: helper/point/func_test.go ================================================ package point import ( "errors" "math" "testing" "github.com/stretchr/testify/assert" ) var nan = math.NaN() func TestUniq(t *testing.T) { tests := [][2][]Point{ { { // in Point{MetricID: 1, Time: 1478025152, Timestamp: 1, Value: 1}, Point{MetricID: 1, Time: 1478025152, Timestamp: 2, Value: 2}, Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1}, }, { // out Point{MetricID: 1, Time: 1478025152, Timestamp: 2, Value: 2}, Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1}, }, }, { { // in Point{MetricID: 1, Time: 1478025152, Timestamp: 3, Value: 1}, Point{MetricID: 1, Time: 1478025152, Timestamp: 2, Value: 2}, Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1}, }, { // out Point{MetricID: 1, Time: 1478025152, Timestamp: 3, Value: 1}, Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1}, }, }, { { // in Point{MetricID: 1, Time: 1478025152, Timestamp: 3, Value: nan}, Point{MetricID: 1, Time: 1478025152, Timestamp: 2, Value: 2}, Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1}, }, { // out Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1}, }, }, } for _, test := range tests { result := Uniq(test[0]) assert.Equal(t, test[1], result) } } func TestCleanUp(t *testing.T) { tests := [][2][]Point{ { { // in Point{MetricID: 1, Time: 1478025152, Timestamp: 1, Value: 1}, Point{MetricID: 0, Time: 1478025152, Timestamp: 2, Value: 2}, Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1}, }, { // out Point{MetricID: 1, Time: 1478025152, Timestamp: 1, Value: 1}, Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1}, }, }, { { // in Point{MetricID: 0, Time: 1478025152, Timestamp: 3, Value: 1}, Point{MetricID: 0, Time: 1478025152, Timestamp: 2, Value: 2}, Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1}, }, { // out Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: 1}, }, }, { { // in Point{MetricID: 0, Time: 1478025152, Timestamp: 3, Value: 1}, Point{MetricID: 0, Time: 1478025152, Timestamp: 2, Value: 2}, Point{MetricID: 0, Time: 1478025155, Timestamp: 1, Value: 1}, }, { // out }, }, { { // in Point{MetricID: 1, Time: 1478025152, Timestamp: 3, Value: nan}, Point{MetricID: 1, Time: 1478025152, Timestamp: 2, Value: 2}, Point{MetricID: 1, Time: 1478025155, Timestamp: 1, Value: nan}, }, { // out Point{MetricID: 1, Time: 1478025152, Timestamp: 2, Value: 2}, }, }, } for _, test := range tests { result := CleanUp(test[0]) assert.Equal(t, test[1], result) } } func TestFillNulls(t *testing.T) { type in struct { points []Point from uint32 until uint32 step uint32 } type expected struct { values []float64 start uint32 stop uint32 count uint32 err error } tests := []struct { name string in expected expected }{ { name: "shorter with NaNs", in: in{ []Point{ {1, 1, 0, 0}, {1, 12, 2, 0}, {1, 2, 4, 0}, {1, 4, 8, 0}, }, 1, 13, 2, }, expected: expected{[]float64{12, 2, nan, 4, nan, nan}, 2, 14, 6, nil}, }, { name: "longer than time interval, but wrong step", in: in{ []Point{ {1, 1, 0, 0}, {1, 12, 2, 0}, {1, 2, 4, 0}, {1, 3, 6, 0}, {1, 4, 8, 0}, }, 2, 7, 1, }, expected: expected{[]float64{12, nan, 2, nan, 3, nan}, 2, 8, 6, nil}, }, { name: "wrong metric ID", in: in{ []Point{ {1, 1, 0, 0}, {1, 12, 2, 0}, {2, 12, 4, 0}, }, 1, 13, 2, }, expected: expected{[]float64{12}, 2, 14, 6, ErrWrongMetricID}, }, { name: "unsorted points cause error", in: in{ []Point{ {1, 12, 4, 0}, {1, 2, 2, 0}, {1, 1, 6, 0}, }, 1, 13, 2, }, expected: expected{[]float64{nan, 12}, 2, 14, 6, ErrPointsUnsorted}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { result := expected{} var ( getter GetValueOrNaN value float64 ) result.start, result.stop, result.count, getter = FillNulls(test.points, test.from, test.until, test.step) result.values = make([]float64, 0, result.count) for { value, result.err = getter() if result.err != nil { break } result.values = append(result.values, value) } if !errors.Is(result.err, ErrTimeGreaterStop) { assert.ErrorIs(t, result.err, test.expected.err) } result.err = nil test.expected.err = nil // Comparing values requires work around NaNs assert.Equal(t, len(result.values), len(test.expected.values), "the length of expected and got values are different") for i := range result.values { if math.IsNaN(test.expected.values[i]) { assert.True(t, math.IsNaN(result.values[i])) continue } assert.Equal(t, test.expected.values[i], result.values[i]) } result.values = []float64{} test.expected.values = []float64{} assert.Equal(t, test.expected, result) }) } } ================================================ FILE: helper/point/point.go ================================================ package point import "fmt" type Point struct { MetricID uint32 Value float64 Time uint32 Timestamp uint32 // keep max if metric and time equal on two points } // GetValueOrNaN returns Value for the next point or NaN if the value is omited. ErrTimeGreaterStop shows the normal ending. Any else error is considered as real error type GetValueOrNaN func() (float64, error) // ErrTimeGreaterStop shows the correct over for GetValueOrNaN var ErrTimeGreaterStop = fmt.Errorf("the points for time interval are rover") // ErrWrongMetricID shows the Point.MetricID is wrong somehow var ErrWrongMetricID = fmt.Errorf("the point MetricID is wrong") // ErrPointsUnsorted returns for unsorted []Point or Points var ErrPointsUnsorted = fmt.Errorf("the points are unsorted") ================================================ FILE: helper/point/points.go ================================================ package point import ( "fmt" "sort" ) // Points is a structure that stores points and additional information about them, e.g. steps, aggregating functions and names. type Points struct { list []Point idMap map[string]uint32 metrics []string steps []uint32 aggs []*string uniqAgg []string } // NextMetric returns the list of points for one metric name type NextMetric func() []Point // NewPoints return new empty Points func NewPoints() *Points { return &Points{ list: make([]Point, 0), idMap: make(map[string]uint32), metrics: make([]string, 0), } } // AppendPoint creates a Point with given values and appends it to list func (pp *Points) AppendPoint(metricID uint32, value float64, time uint32, version uint32) { pp.list = append(pp.list, Point{ MetricID: metricID, Value: value, Time: time, Timestamp: version, }) } // MetricID checks if metric name already exists and returns the ID for it. If not, it creates it first. func (pp *Points) MetricID(metricName string) uint32 { id := pp.idMap[metricName] if id == 0 { pp.metrics = append(pp.metrics, metricName) id = uint32(len(pp.metrics)) pp.idMap[metricName] = id } return id } // MetricIDBytes checks if metric name already exists and returns the ID for it. If not, it creates it first. func (pp *Points) MetricIDBytes(metricNameBytes []byte) uint32 { // @TODO: optimize? return pp.MetricID(string(metricNameBytes)) } // MetricName returns name for metric with given metricID or empty string when ID does not exist func (pp *Points) MetricName(metricID uint32) string { i := int(metricID) if i < 1 || len(pp.metrics) < i { return "" } return pp.metrics[i-1] } // List returns list of points func (pp *Points) List() []Point { return pp.list } // ReplaceList replaces list of points func (pp *Points) ReplaceList(list []Point) { pp.list = list } // GetStep returns uint32 step for given metric id. func (pp *Points) GetStep(id uint32) (uint32, error) { i := int(id) if i < 1 || len(pp.steps) < i { return 0, fmt.Errorf("wrong id %d for given steps %d: %w", i, len(pp.steps), ErrWrongMetricID) } return pp.steps[i-1], nil } // SetSteps accepts map of metric name as keys and step as values and sets slice of uint32 steps for existing metrics in Data.Points func (pp *Points) SetSteps(steps map[uint32][]string) { if len(steps) == 0 { return } pp.steps = make([]uint32, len(pp.metrics)) for step, mm := range steps { for _, m := range mm { if id, ok := pp.idMap[m]; ok { pp.steps[id-1] = step } } } } // GetAggregation returns string function for given metric id. func (pp *Points) GetAggregation(id uint32) (string, error) { i := int(id) if i < 1 || len(pp.aggs) < i { return "", fmt.Errorf("wrong id %d for given functions %d: %w", i, len(pp.aggs), ErrWrongMetricID) } return *pp.aggs[i-1], nil } // SetAggregations accepts map of metric name as keys and function as values and sets slice of functions for existing metrics in Data.Points func (pp *Points) SetAggregations(functions map[string][]string) { pp.aggs = make([]*string, len(pp.metrics)) pp.uniqAgg = make([]string, 0, len(functions)) for f := range functions { pp.uniqAgg = append(pp.uniqAgg, f) } for i, f := range pp.uniqAgg { for _, m := range functions[f] { if id, ok := pp.idMap[m]; ok { pp.aggs[id-1] = &pp.uniqAgg[i] } } } } func (pp *Points) Len() int { return len(pp.list) } func (pp *Points) Less(i, j int) bool { if pp.list[i].MetricID == pp.list[j].MetricID { return pp.list[i].Time < pp.list[j].Time } return pp.list[i].MetricID < pp.list[j].MetricID } func (pp *Points) Swap(i, j int) { pp.list[i], pp.list[j] = pp.list[j], pp.list[i] } // Sort sorts the points list by ID, Time func (pp *Points) Sort() { sort.Sort(pp) } // Uniq cleans up the points func (pp *Points) Uniq() { pp.list = Uniq(pp.list) } // GroupByMetric returns NextMetric function, that by each call returns points for one next metric. // It should be called only on sorted and cleaned Points. func (pp *Points) GroupByMetric() NextMetric { var i, n int l := pp.Len() // i - current position of iterator // n - position of the first record with current metric return func() []Point { if n == l { return []Point{} } for i = n; i < l; i++ { if pp.list[i].MetricID != pp.list[n].MetricID { points := pp.list[n:i] n = i return points } } points := pp.list[n:i] n = i return points } } ================================================ FILE: helper/rollup/aggr.go ================================================ package rollup import ( "github.com/lomik/graphite-clickhouse/helper/point" ) var AggrMap = map[string]*Aggr{ "avg": {"avg", AggrAvg}, "max": {"max", AggrMax}, "min": {"min", AggrMin}, "sum": {"sum", AggrSum}, "any": {"any", AggrAny}, "anyLast": {"anyLast", AggrAnyLast}, } type Aggr struct { name string f func(points []point.Point) (r float64) } func (ag *Aggr) Name() string { if ag == nil { return "" } return ag.name } func (ag *Aggr) String() string { if ag == nil { return "" } return ag.name } func (ag *Aggr) Do(points []point.Point) (r float64) { if ag == nil || ag.f == nil { return 0 } return ag.f(points) } func AggrSum(points []point.Point) (r float64) { for _, p := range points { r += p.Value } return } func AggrMax(points []point.Point) (r float64) { if len(points) > 0 { r = points[0].Value } for _, p := range points { if p.Value > r { r = p.Value } } return } func AggrMin(points []point.Point) (r float64) { if len(points) > 0 { r = points[0].Value } for _, p := range points { if p.Value < r { r = p.Value } } return } func AggrAvg(points []point.Point) (r float64) { if len(points) == 0 { return } r = AggrSum(points) / float64(len(points)) return } func AggrAny(points []point.Point) (r float64) { if len(points) > 0 { r = points[0].Value } return } func AggrAnyLast(points []point.Point) (r float64) { if len(points) > 0 { r = points[len(points)-1].Value } return } ================================================ FILE: helper/rollup/compact.go ================================================ package rollup import ( "fmt" "strconv" "strings" ) /* compact form of rollup rules for tests regexp;function;age:precision,age:precision,... */ func parseCompact(body string) (*Rules, error) { lines := strings.Split(body, "\n") patterns := make([]Pattern, 0) for _, line := range lines { if strings.TrimSpace(line) == "" { continue } p2 := strings.LastIndexByte(line, ';') if p2 < 0 { return nil, fmt.Errorf("can't parse line: %#v", line) } p1 := strings.LastIndexByte(line[:p2], ';') if p1 < 0 { return nil, fmt.Errorf("can't parse line: %#v", line) } regexp := strings.TrimSpace(line[:p1]) function := strings.TrimSpace(line[p1+1 : p2]) retention := make([]Retention, 0) if strings.TrimSpace(line[p2+1:]) != "" { arr := strings.Split(line[p2+1:], ",") for _, r := range arr { p := strings.Split(r, ":") if len(p) != 2 { return nil, fmt.Errorf("can't parse line: %#v", line) } age, err := strconv.ParseUint(strings.TrimSpace(p[0]), 10, 32) if err != nil { return nil, err } precision, err := strconv.ParseUint(strings.TrimSpace(p[1]), 10, 32) if err != nil { return nil, err } retention = append(retention, Retention{Age: uint32(age), Precision: uint32(precision)}) } } patterns = append(patterns, Pattern{Regexp: regexp, Function: function, Retention: retention}) } return (&Rules{Pattern: patterns}).compile() } ================================================ FILE: helper/rollup/compact_test.go ================================================ package rollup import ( "testing" "github.com/stretchr/testify/assert" ) func TestParseCompact(t *testing.T) { config := ` click_cost;any;0:3600,86400:60 ;max;0:60,3600:300,86400:3600` expected, _ := (&Rules{ Pattern: []Pattern{ {Regexp: "click_cost", Function: "any", Retention: []Retention{ {Age: 0, Precision: 3600}, {Age: 86400, Precision: 60}, }}, {Regexp: "", Function: "max", Retention: []Retention{ {Age: 0, Precision: 60}, {Age: 3600, Precision: 300}, {Age: 86400, Precision: 3600}, }}, }, }).compile() assert := assert.New(t) r, err := parseCompact(config) assert.NoError(err) assert.Equal(expected, r) } ================================================ FILE: helper/rollup/remote.go ================================================ package rollup import ( "context" "crypto/tls" "encoding/json" "strconv" "strings" "time" "github.com/lomik/zapwriter" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/pkg/scope" ) type rollupRulesResponseRecord struct { RuleType RuleType `json:"rule_type"` Regexp string `json:"regexp"` Function string `json:"function"` Age string `json:"age"` Precision string `json:"precision"` IsDefault int `json:"is_default"` } type rollupRulesResponse struct { Data []rollupRulesResponseRecord `json:"data"` } func parseJson(body []byte) (*Rules, error) { resp := &rollupRulesResponse{} err := json.Unmarshal(body, resp) if err != nil { return nil, err } r := &Rules{ Pattern: make([]Pattern, 0), } makeRetention := func(d *rollupRulesResponseRecord) (Retention, error) { age, err := strconv.ParseInt(d.Age, 10, 32) if err != nil { return Retention{}, err } prec, err := strconv.ParseInt(d.Precision, 10, 32) if err != nil { return Retention{}, err } return Retention{Age: uint32(age), Precision: uint32(prec)}, nil } last := func() *Pattern { if len(r.Pattern) == 0 { return nil } return &r.Pattern[len(r.Pattern)-1] } defaultFunction := "" defaultRetention := make([]Retention, 0) // var last *Pattern for _, d := range resp.Data { if d.IsDefault == 1 { if d.Function != "" { defaultFunction = d.Function } if d.Age != "" && d.Precision != "" && d.Precision != "0" { rt, err := makeRetention(&d) if err != nil { return nil, err } defaultRetention = append(defaultRetention, rt) } } else { if last() == nil || last().Regexp != d.Regexp || last().Function != d.Function { r.Pattern = append(r.Pattern, Pattern{ RuleType: d.RuleType, Retention: make([]Retention, 0), Regexp: d.Regexp, Function: d.Function, }) } if d.Age != "" && d.Precision != "" && d.Precision != "0" { rt, err := makeRetention(&d) if err != nil { return nil, err } last().Retention = append(last().Retention, rt) } } } if defaultFunction != "" || len(defaultRetention) != 0 { r.Pattern = append(r.Pattern, Pattern{ Regexp: "", Function: defaultFunction, Retention: defaultRetention, }) } return r.compile() } var timeoutRulesLoad = 10 * time.Second func RemoteLoad(addr string, tlsConf *tls.Config, table string) (*Rules, error) { var db string arr := strings.SplitN(table, ".", 2) if len(arr) == 1 { db = "default" } else { db, table = arr[0], arr[1] } query := `SELECT rule_type, regexp, function, age, precision, is_default FROM system.graphite_retentions ARRAY JOIN Tables AS table WHERE (table.database = '` + db + `') AND (table.table = '` + table + `') ORDER BY is_default ASC, priority ASC, regexp ASC, age ASC FORMAT JSON` body, _, _, err := clickhouse.Query( scope.New(context.Background()).WithLogger(zapwriter.Logger("rollup")).WithTable("system.graphite_retentions"), addr, query, clickhouse.Options{ Timeout: timeoutRulesLoad, ConnectTimeout: timeoutRulesLoad, TLSConfig: tlsConf, CheckRequestProgress: false, ProgressSendingInterval: 10 * time.Second, }, nil, ) if err != nil && strings.Contains(err.Error(), " Missing columns: 'rule_type' ") { // for old version query = `SELECT regexp, function, age, precision, is_default FROM system.graphite_retentions ARRAY JOIN Tables AS table WHERE (table.database = '` + db + `') AND (table.table = '` + table + `') ORDER BY is_default ASC, priority ASC, regexp ASC, age ASC FORMAT JSON` body, _, _, err = clickhouse.Query( scope.New(context.Background()).WithLogger(zapwriter.Logger("rollup")).WithTable("system.graphite_retentions"), addr, query, clickhouse.Options{ Timeout: timeoutRulesLoad, ConnectTimeout: timeoutRulesLoad, TLSConfig: tlsConf, CheckRequestProgress: false, ProgressSendingInterval: 10 * time.Second, }, nil, ) } if err != nil { return nil, err } r, err := parseJson(body) if r != nil { r.Updated = time.Now().Unix() } return r, err } ================================================ FILE: helper/rollup/remote_test.go ================================================ package rollup import ( "encoding/json" "regexp" "testing" "github.com/stretchr/testify/assert" ) func assertJsonEqual(t *testing.T, expected string, actual string) { var e, a interface{} assert := assert.New(t) assert.NoError(json.Unmarshal([]byte(expected), &e)) assert.NoError(json.Unmarshal([]byte(actual), &a)) assert.Equal(e, a) } func TestParseJson(t *testing.T) { response := `{ "meta": [ { "name": "regexp", "type": "String" }, { "name": "function", "type": "String" }, { "name": "age", "type": "UInt64" }, { "name": "precision", "type": "UInt64" }, { "name": "is_default", "type": "UInt8" } ], "data": [ { "regexp": "^hourly", "function": "", "age": "0", "precision": "3600", "is_default": 0 }, { "regexp": "^hourly", "function": "", "age": "3600", "precision": "13600", "is_default": 0 }, { "regexp": "^live", "function": "", "age": "0", "precision": "1", "is_default": 0 }, { "regexp": "total$", "function": "sum", "age": "0", "precision": "0", "is_default": 0 }, { "regexp": "min$", "function": "min", "age": "0", "precision": "0", "is_default": 0 }, { "regexp": "max$", "function": "max", "age": "0", "precision": "0", "is_default": 0 }, { "regexp": "", "function": "max", "age": "0", "precision": "60", "is_default": 1 } ], "rows": 7, "statistics": { "elapsed": 0.00053715, "rows_read": 7, "bytes_read": 1158 } }` compact := ` ^hourly;;0:3600,3600:13600 ^live;;0:1 total$;sum; min$;min; max$;max; ;max;0:60 ` assert := assert.New(t) expected, err := parseCompact(compact) assert.NoError(err) r, err := parseJson([]byte(response)) assert.NotNil(r) assert.NoError(err) assert.Equal(expected, r) } func TestParseJsonTyped(t *testing.T) { response := `{ "meta": [ { "name": "rule_type", "type": "String" }, { "name": "regexp", "type": "String" }, { "name": "function", "type": "String" }, { "name": "age", "type": "UInt64" }, { "name": "precision", "type": "UInt64" }, { "name": "is_default", "type": "UInt8" } ], "data": [ { "rule_type": "all", "regexp": "^hourly", "function": "", "age": "0", "precision": "3600", "is_default": 0 }, { "rule_type": "all", "regexp": "^hourly", "function": "", "age": "3600", "precision": "13600", "is_default": 0 }, { "rule_type": "all", "regexp": "^live", "function": "", "age": "0", "precision": "1", "is_default": 0 }, { "rule_type": "plain", "regexp": "total$", "function": "sum", "age": "0", "precision": "0", "is_default": 0 }, { "rule_type": "plain", "regexp": "min$", "function": "min", "age": "0", "precision": "0", "is_default": 0 }, { "rule_type": "plain", "regexp": "max$", "function": "max", "age": "0", "precision": "0", "is_default": 0 }, { "rule_type": "tagged", "regexp": "^tag_name\\?", "function": "min" }, { "rule_type": "tag_list", "regexp": "fake3;tag=Fake3", "function": "sum" }, { "rule_type": "all", "regexp": "", "function": "max", "age": "0", "precision": "60", "is_default": 1 } ], "rows": 7, "statistics": { "elapsed": 0.00053715, "rows_read": 7, "bytes_read": 1158 } }` expected := &Rules{ Separated: true, Pattern: []Pattern{ { Regexp: "^hourly", Retention: []Retention{ {Age: 0, Precision: 3600}, {Age: 3600, Precision: 13600}, }, re: regexp.MustCompile("^hourly"), }, { Regexp: "^live", Retention: []Retention{ {Age: 0, Precision: 1}, }, re: regexp.MustCompile("^live"), }, { RuleType: RulePlain, Regexp: "total$", Function: "sum", re: regexp.MustCompile("total$"), aggr: AggrMap["sum"], }, { RuleType: RulePlain, Regexp: "min$", Function: "min", re: regexp.MustCompile("min$"), aggr: AggrMap["min"], }, { RuleType: RulePlain, Regexp: "max$", Function: "max", re: regexp.MustCompile("max$"), aggr: AggrMap["max"], }, { RuleType: RuleTagged, Regexp: `^tag_name\?`, Function: "min", re: regexp.MustCompile(`^tag_name\?`), aggr: AggrMap["min"], }, { RuleType: RuleTagged, Regexp: `^fake3\?(.*&)?tag=Fake3(&.*)?$`, Function: "sum", re: regexp.MustCompile(`^fake3\?(.*&)?tag=Fake3(&.*)?$`), aggr: AggrMap["sum"], }, { Regexp: ".*", Function: "max", Retention: []Retention{ {Age: 0, Precision: 60}, }, aggr: AggrMap["max"], }, }, PatternPlain: []Pattern{ { Regexp: "^hourly", Retention: []Retention{ {Age: 0, Precision: 3600}, {Age: 3600, Precision: 13600}, }, re: regexp.MustCompile("^hourly"), }, { Regexp: "^live", Retention: []Retention{ {Age: 0, Precision: 1}, }, re: regexp.MustCompile("^live"), }, { RuleType: RulePlain, Regexp: "total$", Function: "sum", re: regexp.MustCompile("total$"), aggr: AggrMap["sum"], }, { RuleType: RulePlain, Regexp: "min$", Function: "min", re: regexp.MustCompile("min$"), aggr: AggrMap["min"], }, { RuleType: RulePlain, Regexp: "max$", Function: "max", re: regexp.MustCompile("max$"), aggr: AggrMap["max"], }, { Regexp: ".*", Function: "max", Retention: []Retention{ {Age: 0, Precision: 60}, }, aggr: AggrMap["max"], }, }, PatternTagged: []Pattern{ { Regexp: "^hourly", Retention: []Retention{ {Age: 0, Precision: 3600}, {Age: 3600, Precision: 13600}, }, re: regexp.MustCompile("^hourly"), }, { Regexp: "^live", Retention: []Retention{ {Age: 0, Precision: 1}, }, re: regexp.MustCompile("^live"), }, { RuleType: RuleTagged, Regexp: `^tag_name\?`, Function: "min", re: regexp.MustCompile(`^tag_name\?`), aggr: AggrMap["min"], }, { RuleType: RuleTagged, Regexp: `^fake3\?(.*&)?tag=Fake3(&.*)?$`, Function: "sum", re: regexp.MustCompile(`^fake3\?(.*&)?tag=Fake3(&.*)?$`), aggr: AggrMap["sum"], }, { Regexp: ".*", Function: "max", Retention: []Retention{ {Age: 0, Precision: 60}, }, aggr: AggrMap["max"], }, }, } assert := assert.New(t) r, err := parseJson([]byte(response)) assert.NotNil(r) assert.NoError(err) assert.Equal(expected, r) } ================================================ FILE: helper/rollup/rollup.go ================================================ package rollup import ( "crypto/tls" "encoding/json" "fmt" "os" "sync" "time" "github.com/lomik/zapwriter" "go.uber.org/zap" ) type Rollup struct { mu sync.RWMutex rules *Rules tlsConfig *tls.Config addr string table string defaultPrecision uint32 defaultFunction string interval time.Duration } func NewAuto(addr string, tlsConfig *tls.Config, table string, interval time.Duration, defaultPrecision uint32, defaultFunction string) (*Rollup, error) { r := &Rollup{ addr: addr, tlsConfig: tlsConfig, table: table, interval: interval, defaultPrecision: defaultPrecision, defaultFunction: defaultFunction, } go r.updateWorker() return r, nil } func NewXMLFile(filename string, defaultPrecision uint32, defaultFunction string) (*Rollup, error) { rollupConfBody, err := os.ReadFile(filename) if err != nil { return nil, err } rules, err := parseXML(rollupConfBody) if err != nil { return nil, err } rules, err = rules.prepare(defaultPrecision, defaultFunction) if err != nil { return nil, err } return (&Rollup{ rules: rules, defaultPrecision: defaultPrecision, defaultFunction: defaultFunction, }), nil } func NewDefault(defaultPrecision uint32, defaultFunction string) (*Rollup, error) { rules := &Rules{Pattern: []Pattern{}} rules, err := rules.prepare(defaultPrecision, defaultFunction) if err != nil { return nil, err } return (&Rollup{ rules: rules, defaultPrecision: defaultPrecision, defaultFunction: defaultFunction, }), nil } func (r *Rollup) Rules() *Rules { r.mu.RLock() rules := r.rules r.mu.RUnlock() return rules } func (r *Rollup) update() error { rules, err := RemoteLoad(r.addr, r.tlsConfig, r.table) if err != nil { zapwriter.Logger("rollup").Error(fmt.Sprintf("rollup rules update failed for table %#v", r.table), zap.Error(err)) return err } rules, err = rules.prepare(r.defaultPrecision, r.defaultFunction) if err != nil { zapwriter.Logger("rollup").Error(fmt.Sprintf("rollup rules update failed for table %#v", r.table), zap.Error(err)) return err } r.mu.Lock() r.rules = rules r.mu.Unlock() return nil } func (r *Rollup) updateWorker() { for { r.update() // If we still have no rules - try every second to fetch them if r.rules == nil { time.Sleep(1 * time.Second) } else if r.interval != 0 { time.Sleep(r.interval) } else { break } } } func (r *Rollup) MarshalJSON() ([]byte, error) { return json.Marshal(r.Rules()) } ================================================ FILE: helper/rollup/rules.go ================================================ package rollup import ( "encoding/xml" "fmt" "regexp" "sort" "strings" "time" "github.com/lomik/graphite-clickhouse/pkg/dry" "github.com/lomik/graphite-clickhouse/helper/point" ) type Retention struct { Age uint32 `json:"age"` Precision uint32 `json:"precision"` } type RuleType uint8 const ( RuleAll RuleType = iota RulePlain RuleTagged RuleTagList ) var timeNow = time.Now var ruleTypeStrings []string = []string{"all", "plain", "tagged", "tag_list"} func (r *RuleType) String() string { return ruleTypeStrings[*r] } func (r *RuleType) Set(value string) error { switch strings.ToLower(value) { case "all": *r = RuleAll case "plain": *r = RulePlain case "tagged": *r = RuleTagged case "tag_list": *r = RuleTagList default: return fmt.Errorf("invalid rule type %s", value) } return nil } func (r *RuleType) UnmarshalJSON(data []byte) error { s := string(data) if strings.HasPrefix(s, `"`) && strings.HasSuffix(s, `"`) { s = s[1 : len(s)-1] } return r.Set(s) } func (r *RuleType) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { var s string if err := d.DecodeElement(&s, &start); err != nil { return err } return r.Set(s) } func splitTags(tagsStr string) (tags []string) { vals := strings.Split(tagsStr, ";") tags = make([]string, 0, len(vals)) // remove empthy elements for _, v := range vals { if v != "" { tags = append(tags, v) } } return } func buildTaggedRegex(regexpStr string) string { // see buildTaggedRegex in https://github.com/ClickHouse/ClickHouse/blob/780a1b2abea918d3205d149db7689a31fdff2f70/src/Processors/Merges/Algorithms/Graphite.cpp#L241 // // * tags list in format (for name or any value can use regexp, alphabet sorting not needed) // * spaces are not stiped and used as tag and value part // * name must be first (if used) // * // * tag1=value1; tag2=VALUE2_REGEX;tag3=value3 // * or // * name;tag1=value1;tag2=VALUE2_REGEX;tag3=value3 // * or for one tag // * tag1=value1 // * // * Resulting regex against metric like // * name?tag1=value1&tag2=value2 // * // * So, // * // * name // * produce // * name\? // * // * tag2=val2 // * produce // * [\?&]tag2=val2(&.*)?$ // * // * nam.* ; tag1=val1 ; tag2=val2 // * produce // * nam.*\?(.*&)?tag1=val1&(.*&)?tag2=val2(&.*)?$ tags := splitTags(regexpStr) if strings.Contains(tags[0], "=") { regexpStr = "[\\?&]" } else { if len(tags) == 1 { // only name return "^" + tags[0] + "\\?" } // start with name value regexpStr = "^" + tags[0] + "\\?(.*&)?" tags = tags[1:] } sort.Strings(tags) // sorted tag keys regexpStr = regexpStr + strings.Join(tags, "&(.*&)?") + "(&.*)?$" // close regex return regexpStr } type Pattern struct { RuleType RuleType `json:"rule_type"` Regexp string `json:"regexp"` Function string `json:"function"` Retention []Retention `json:"retention"` aggr *Aggr re *regexp.Regexp } type Rules struct { Pattern []Pattern `json:"pattern"` Updated int64 `json:"updated"` Separated bool `json:"-"` PatternPlain []Pattern `json:"-"` PatternTagged []Pattern `json:"-"` } // NewMockRulles creates mock rollup for tests func NewMockRules(pattern []Pattern, defaultPrecision uint32, defaultFunction string) (*Rules, error) { rules, err := (&Rules{Pattern: pattern}).compile() if err != nil { return nil, err } rules, err = rules.prepare(defaultPrecision, defaultFunction) if err != nil { return nil, err } return rules, nil } // should never be used in real conditions var superDefaultFunction = AggrMap["avg"] const superDefaultPrecision = uint32(60) func (p *Pattern) compile() error { if p.RuleType == RuleTagList { // convert to tagged rule type p.RuleType = RuleTagged p.Regexp = buildTaggedRegex(p.Regexp) } var err error if p.Regexp != "" && p.Regexp != ".*" { p.re, err = regexp.Compile(p.Regexp) if err != nil { return err } } else { p.Regexp = ".*" p.re = nil } if p.Function != "" { var exists bool p.aggr, exists = AggrMap[p.Function] if !exists { return fmt.Errorf("unknown function %#v", p.Function) } } if len(p.Retention) > 0 { // reverse sort by age sort.Slice(p.Retention, func(i, j int) bool { return p.Retention[i].Age < p.Retention[j].Age }) } else { p.Retention = nil } return nil } func (r *Rules) compile() (*Rules, error) { if r.Pattern == nil { r.Pattern = make([]Pattern, 0) } r.PatternPlain = make([]Pattern, 0) r.PatternTagged = make([]Pattern, 0) r.Separated = false for i := range r.Pattern { if err := r.Pattern[i].compile(); err != nil { return r, err } if !r.Separated && r.Pattern[i].RuleType != RuleAll { r.Separated = true } } if r.Separated { for i := range r.Pattern { switch r.Pattern[i].RuleType { case RulePlain: r.PatternPlain = append(r.PatternPlain, r.Pattern[i]) case RuleTagged: r.PatternTagged = append(r.PatternTagged, r.Pattern[i]) default: r.PatternPlain = append(r.PatternPlain, r.Pattern[i]) r.PatternTagged = append(r.PatternTagged, r.Pattern[i]) } } } return r, nil } func (r *Rules) prepare(defaultPrecision uint32, defaultFunction string) (*Rules, error) { defaultAggr := AggrMap[defaultFunction] if defaultFunction != "" && defaultAggr == nil { return r, fmt.Errorf("unknown function %#v", defaultFunction) } return r.withDefault(defaultPrecision, defaultAggr).withSuperDefault().setUpdated(), nil } func (r *Rules) withDefault(defaultPrecision uint32, defaultFunction *Aggr) *Rules { patterns := make([]Pattern, len(r.Pattern)+1) copy(patterns, r.Pattern) var retention []Retention if defaultPrecision != 0 { retention = []Retention{{Age: 0, Precision: defaultPrecision}} } patterns = append(patterns, Pattern{ Regexp: ".*", Function: defaultFunction.Name(), Retention: retention, }) n, _ := (&Rules{Pattern: patterns, Updated: r.Updated}).compile() return n } func (r *Rules) setUpdated() *Rules { r.Updated = timeNow().Unix() return r } func (r *Rules) withSuperDefault() *Rules { return r.withDefault(superDefaultPrecision, superDefaultFunction) } // Lookup returns precision and aggregate function for metric name and age func (r *Rules) Lookup(metric string, age uint32, verbose bool) (precision uint32, ag *Aggr, aggrPattern, retentionPattern *Pattern) { if r.Separated { if strings.Contains(metric, "?") { return lookup(metric, age, r.PatternTagged, verbose) } return lookup(metric, age, r.PatternPlain, verbose) } return lookup(metric, age, r.Pattern, verbose) } // Lookup returns precision and aggregate function for metric name and age func lookup(metric string, age uint32, patterns []Pattern, verbose bool) (precision uint32, ag *Aggr, aggrPattern, retentionPattern *Pattern) { precisionFound := false for n, p := range patterns { // pattern hasn't interested data if (ag != nil || p.aggr == nil) && (precisionFound || len(p.Retention) == 0) { continue } // metric not matched regexp if p.re != nil && !p.re.MatchString(metric) { continue } if ag == nil && p.aggr != nil { if verbose { aggrPattern = &patterns[n] } ag = p.aggr } if !precisionFound && len(p.Retention) > 0 { for i, r := range p.Retention { if age < r.Age { if i > 0 { precision = p.Retention[i-1].Precision precisionFound = true if verbose { retentionPattern = &patterns[n] } } break } if i == len(p.Retention)-1 { precision = r.Precision precisionFound = true if verbose { retentionPattern = &patterns[n] } break } } } // all found if ag != nil && precisionFound { return } } if ag == nil { ag = superDefaultFunction } if !precisionFound { precision = superDefaultPrecision } return } // LookupBytes returns precision and aggregate function for metric name and age func (r *Rules) LookupBytes(metric []byte, age uint32, verbose bool) (precision uint32, ag *Aggr, aggrPattern, retentionPattern *Pattern) { return r.Lookup(dry.UnsafeString(metric), age, verbose) } func doMetricPrecision(points []point.Point, precision uint32, aggr *Aggr) []point.Point { l := len(points) var i, n int // i - current position of iterator // n - position of the first record with time rounded to precision if l == 0 { return points } // set first point time t := points[0].Time t = t - (t % precision) points[0].Time = t for i = 1; i < l; i++ { t = points[i].Time t = t - (t % precision) points[i].Time = t if points[n].Time == t { points[i].MetricID = 0 } else { if i > n+1 { points[n].Value = aggr.Do(points[n:i]) } n = i } } if i > n+1 { points[n].Value = aggr.Do(points[n:i]) } return point.CleanUp(points) } // RollupMetricAge rolling up list of points of ONE metric sorted by key "time" // returns (new points slice, precision) func (r *Rules) RollupMetricAge(metricName string, age uint32, points []point.Point) ([]point.Point, uint32, error) { l := len(points) if l == 0 { return points, 1, nil } precision, ag, _, _ := r.Lookup(metricName, age, false) points = doMetricPrecision(points, precision, ag) return points, precision, nil } // RollupMetric rolling up list of points of ONE metric sorted by key "time" // returns (new points slice, precision) func (r *Rules) RollupMetric(metricName string, from uint32, points []point.Point) ([]point.Point, uint32, error) { now := uint32(timeNow().Unix()) age := uint32(0) if now > from { age = now - from } return r.RollupMetricAge(metricName, age, points) } // RollupPoints groups sorted Points by metric name and apply rollup one by one. // If the `step` parameter is 0, it will be got from the current *Rules, otherwise it will be used directly. func (r *Rules) RollupPoints(pp *point.Points, from int64, step int64) error { if from < 0 || step < 0 { return fmt.Errorf("from and step must be >= 0: %v, %v", from, step) } now := timeNow().Unix() age := int64(0) if now > from { age = now - from } var i, n int // i - current position of iterator // n - position of the first record with current metric l := pp.Len() if l == 0 { return nil } oldPoints := pp.List() newPoints := make([]point.Point, 0, pp.Len()) rollup := func(p []point.Point) ([]point.Point, error) { metricName := pp.MetricName(p[0].MetricID) var err error if step == 0 { p, _, err = r.RollupMetricAge(metricName, uint32(age), p) } else { _, agg, _, _ := r.Lookup(metricName, uint32(from), false) p = doMetricPrecision(p, uint32(step), agg) } for i := range p { p[i].MetricID = p[0].MetricID } return p, err } for i = 1; i < l; i++ { if oldPoints[i].MetricID != oldPoints[n].MetricID { points, err := rollup(oldPoints[n:i]) if err != nil { return err } newPoints = append(newPoints, points...) n = i continue } } points, err := rollup(oldPoints[n:i]) if err != nil { return err } newPoints = append(newPoints, points...) pp.ReplaceList(newPoints) return nil } ================================================ FILE: helper/rollup/rules_test.go ================================================ package rollup import ( "fmt" "regexp" "strconv" "testing" "time" "github.com/lomik/graphite-clickhouse/helper/point" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestMetricPrecision(t *testing.T) { tests := [][2][]point.Point{ { { // in {MetricID: 1, Time: 1478025152, Value: 3}, {MetricID: 1, Time: 1478025154, Value: 2}, {MetricID: 1, Time: 1478025255, Value: 1}, }, { // out {MetricID: 1, Time: 1478025120, Value: 5}, {MetricID: 1, Time: 1478025240, Value: 1}, }, }, } for _, test := range tests { result := doMetricPrecision(test[0], 60, AggrMap["sum"]) assert.Equal(t, test[1], result) } } func Test_buildTaggedRegex(t *testing.T) { tests := []struct { tagsStr string want string match string nomatch string }{ { tagsStr: `cpu\.loadavg;project=DB.*;env=st.*`, want: `^cpu\.loadavg\?(.*&)?env=st.*&(.*&)?project=DB.*(&.*)?$`, match: `cpu.loadavg?env=staging&project=DBAAS`, nomatch: `cpu.loadavg?env=staging&project=D`, }, { tagsStr: `project=DB.*;env=staging;`, want: `[\?&]env=staging&(.*&)?project=DB.*(&.*)?$`, match: `cpu.loadavg?env=staging&project=DBPG`, nomatch: `cpu.loadavg?env=stagingN&project=DBAAS`, }, { tagsStr: "env=staging;", want: `[\?&]env=staging(&.*)?$`, match: `cpu.loadavg?env=staging&project=DPG`, nomatch: `cpu.loadavg?env=stagingN`, }, { tagsStr: " env = staging ;", // spaces are allowed, want: `[\?&] env = staging (&.*)?$`, match: `cpu.loadavg? env = staging &project=DPG`, nomatch: `cpu.loadavg?env=stagingN`, }, { tagsStr: "name;", want: `^name\?`, match: `name?env=staging&project=DPG`, nomatch: `nameN?env=stagingN`, }, { tagsStr: "name", want: `^name\?`, match: `name?env=staging&project=DPG`, nomatch: `nameN?env=stagingN`, }, } for _, tt := range tests { t.Run(tt.tagsStr, func(t *testing.T) { if got := buildTaggedRegex(tt.tagsStr); got != tt.want { t.Errorf("buildTaggedRegex(%q) = %v, want %v", tt.tagsStr, got, tt.want) } else { re := regexp.MustCompile(got) if tt.match != "" && !re.Match([]byte(tt.match)) { t.Errorf("match(%q, %q) must be true", tt.tagsStr, tt.match) } if tt.nomatch != "" && re.Match([]byte(tt.nomatch)) { t.Errorf("match(%q, %q) must be false", tt.tagsStr, tt.match) } } }) } } func TestLookup(t *testing.T) { config := ` ^hourly;;3600:60,86400:3600 ^live;;0:1 total$;sum; min$;min; max$;max; ;avg; ;;60:10 ;;0:42` table := [][4]string{ {"hello.world", "0", "avg", "42"}, {"hourly.rps", "0", "avg", "42"}, {"hourly.rps_total", "0", "sum", "42"}, {"live.rps_total", "0", "sum", "1"}, {"hourly.rps_min", "0", "min", "42"}, {"hourly.rps_min", "1", "min", "42"}, {"hourly.rps_min", "59", "min", "42"}, {"hourly.rps_min", "60", "min", "10"}, {"hourly.rps_min", "61", "min", "10"}, {"hourly.rps_min", "3599", "min", "10"}, {"hourly.rps_min", "3600", "min", "60"}, {"hourly.rps_min", "3601", "min", "60"}, {"hourly.rps_min", "86399", "min", "60"}, {"hourly.rps_min", "86400", "min", "3600"}, {"hourly.rps_min", "86401", "min", "3600"}, } r, err := parseCompact(config) require.NoError(t, err) for _, c := range table { t.Run(fmt.Sprintf("%#v", c[:]), func(t *testing.T) { assert := assert.New(t) age, err := strconv.Atoi(c[1]) assert.NoError(err) precision, ag, _, _ := r.Lookup(c[0], uint32(age), false) assert.Equal(c[2], ag.String()) assert.Equal(c[3], fmt.Sprintf("%d", precision)) }) } } func TestLookupTyped(t *testing.T) { config := ` ^hourly 3600 60 86400 3600 ^live 0 1 tag_list fake3;tag3=Fake3 0 1 tag_list tag5=Fake5;tag3=Fake3 0 90 tag_list fake_name 0 20 plain total$ sum plain min$ min plain max$ max tagged total? sum tagged min\? min tagged max\? max tagged ^hourly sum avg 0 42 60 10 ` table := [][4]string{ {"hello.world", "0", "avg", "42"}, {"hourly.rps", "0", "avg", "42"}, {"hourly.rps?tag=value", "0", "sum", "42"}, {"hourly.rps", "0", "avg", "42"}, {"hourly.rps_total", "0", "sum", "42"}, {"live.rps_total", "0", "sum", "1"}, {"hourly.rps_min", "0", "min", "42"}, {"hourly.rps_min?tag=value", "0", "min", "42"}, {"hourly.rps_min", "1", "min", "42"}, {"hourly.rps_min", "59", "min", "42"}, {"hourly.rps_min?tag=value", "59", "min", "42"}, {"hourly.rps_min", "60", "min", "10"}, {"hourly.rps_min", "61", "min", "10"}, {"hourly.rps_min", "3599", "min", "10"}, {"hourly.rps_min", "3600", "min", "60"}, {"hourly.rps_min", "3601", "min", "60"}, {"hourly.rps_min", "86399", "min", "60"}, {"hourly.rps_min", "86400", "min", "3600"}, {"hourly.rps_min", "86401", "min", "3600"}, {"fake3?tag3=Fake3", "0", "avg", "1"}, {"fake3?tag1=Fake1&tag3=Fake3", "0", "avg", "1"}, {"fake3?tag1=Fake1&tag3=Fake3&tag4=Fake4", "0", "avg", "1"}, {"fake3?tag3=Fake", "0", "avg", "42"}, {"fake3?tag1=Fake1&tag3=Fake", "0", "avg", "42"}, {"fake3?tag1=Fake1&tag3=Fake&tag4=Fake4", "0", "avg", "42"}, {"fake?tag3=Fake3", "0", "avg", "42"}, {"fake_name?tag3=Fake3", "0", "avg", "20"}, {"fake5?tag1=Fake1&tag3=Fake3&tag4=Fake4&tag5=Fake5", "0", "avg", "90"}, {"fake5?tag3=Fake3&tag4=Fake4&tag5=Fake5&tag6=Fake6", "0", "avg", "90"}, {"fake5?tag4=Fake4&tag5=Fake5&tag6=Fake6", "0", "avg", "42"}, } r, err := parseXML([]byte(config)) require.NoError(t, err) for _, c := range table { t.Run(fmt.Sprintf("%#v", c[:]), func(t *testing.T) { assert := assert.New(t) age, err := strconv.Atoi(c[1]) assert.NoError(err) precision, ag, _, _ := r.Lookup(c[0], uint32(age), false) assert.Equal(c[2], ag.String()) assert.Equal(c[3], fmt.Sprintf("%d", precision)) }) } } func TestRules_RollupPoints(t *testing.T) { config := ` ^10sec;;0:10,3600:60 ;max;0:20` r, err := parseCompact(config) require.NoError(t, err) timeNow = func() time.Time { return time.Unix(10010, 0) } newPoints := func() *point.Points { pp := point.NewPoints() id10Sec := pp.MetricID("10sec") pp.AppendPoint(id10Sec, 1.0, 10, 0) pp.AppendPoint(id10Sec, 2.0, 20, 0) pp.AppendPoint(id10Sec, 3.0, 30, 0) pp.AppendPoint(id10Sec, 6.0, 60, 0) pp.AppendPoint(id10Sec, 7.0, 70, 0) idDefault := pp.MetricID("default") pp.AppendPoint(idDefault, 2.0, 20, 0) pp.AppendPoint(idDefault, 4.0, 40, 0) pp.AppendPoint(idDefault, 6.0, 60, 0) pp.AppendPoint(idDefault, 8.0, 80, 0) return pp } pointsTo60SecNoDefault := func() *point.Points { pp := point.NewPoints() id10Sec := pp.MetricID("10sec") pp.AppendPoint(id10Sec, 3.0, 0, 0) pp.AppendPoint(id10Sec, 7.0, 60, 0) idDefault := pp.MetricID("default") pp.AppendPoint(idDefault, 2.0, 20, 0) pp.AppendPoint(idDefault, 4.0, 40, 0) pp.AppendPoint(idDefault, 6.0, 60, 0) pp.AppendPoint(idDefault, 8.0, 80, 0) return pp } pointsTo60Sec := func() *point.Points { pp := point.NewPoints() id10Sec := pp.MetricID("10sec") pp.AppendPoint(id10Sec, 3.0, 0, 0) pp.AppendPoint(id10Sec, 7.0, 60, 0) idDefault := pp.MetricID("default") pp.AppendPoint(idDefault, 4.0, 0, 0) pp.AppendPoint(idDefault, 8.0, 60, 0) return pp } tests := []struct { name string pp *point.Points from int64 step int64 want *point.Points wantErr bool }{ { name: "without step and no rollup", pp: newPoints(), from: int64(10000), step: int64(0), want: newPoints(), }, { name: "without step", pp: newPoints(), from: int64(10), step: int64(0), want: pointsTo60SecNoDefault(), }, { name: "with step 10", pp: newPoints(), from: int64(10000), step: int64(10), want: newPoints(), }, { name: "with step 60", pp: newPoints(), from: int64(10), step: int64(60), want: pointsTo60Sec(), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if err := r.RollupPoints(tt.pp, tt.from, tt.step); (err != nil) != tt.wantErr { t.Errorf("Rules.RollupPoints() error = %v, wantErr %v", err, tt.wantErr) } else if err == nil { assert.Equal(t, tt.want, tt.pp) } }) } } var benchConfig = ` ^hourly 3600 60 86400 3600 ^live 0 1 \.fake1\..*\.Fake1\. 3600 60 86400 3600 3600 60 86400 3600 \.fake2\..*\.Fake2\. 3600 60 86400 3600 3600 60 86400 3600 \.fake3\..*\.Fake3\. 3600 60 86400 3600 3600 60 86400 3600 \.fake4\..*\.Fake4\. 3600 60 86400 3600 3600 60 86400 3600 total$ sum min$ min max$ max total? sum min\? min max\? max ^hourly sum avg 0 42 60 10 ` var benchConfigSeparated = ` plain ^hourly 3600 60 86400 3600 plain ^live 0 1 plain \.fake1\..*\.Fake1\. 3600 60 86400 3600 tagged 3600 60 86400 3600 plain \.fake2\..*\.Fake2\. 3600 60 86400 3600 tagged 3600 60 86400 3600 plain \.fake3\..*\.Fake3\. 3600 60 86400 3600 tagged 3600 60 86400 3600 plain \.fake4\..*\.Fake4\. 3600 60 86400 3600 tagged 3600 60 86400 3600 plain total$ sum plain min$ min plain max$ max tagged total? sum tagged min\? min tagged max\? max tagged ^hourly sum avg 0 42 60 10 ` func BenchmarkLookupSum(b *testing.B) { r, err := parseXML([]byte(benchConfig)) require.NoError(b, err) b.ResetTimer() for i := 0; i < b.N; i++ { precision, ag, _, _ := r.Lookup("test.sum", 1, false) _ = precision _ = ag } } func BenchmarkLookupSumSeparated(b *testing.B) { r, err := parseXML([]byte(benchConfigSeparated)) require.NoError(b, err) b.ResetTimer() for i := 0; i < b.N; i++ { precision, ag, _, _ := r.Lookup("test.sum", 1, false) _ = precision _ = ag } } func BenchmarkLookupSumTagged(b *testing.B) { r, err := parseXML([]byte(benchConfig)) require.NoError(b, err) b.ResetTimer() for i := 0; i < b.N; i++ { precision, ag, _, _ := r.Lookup("sum?env=test&tag=Fake5", 1, false) _ = precision _ = ag } } func BenchmarkLookupSumTaggedSeparated(b *testing.B) { r, err := parseXML([]byte(benchConfigSeparated)) require.NoError(b, err) b.ResetTimer() for i := 0; i < b.N; i++ { precision, ag, _, _ := r.Lookup("sum?env=test&tag=Fake5", 1, false) _ = precision _ = ag } } func BenchmarkLookupMax(b *testing.B) { r, err := parseXML([]byte(benchConfig)) require.NoError(b, err) b.ResetTimer() for i := 0; i < b.N; i++ { precision, ag, _, _ := r.Lookup("test.max", 1, false) _ = precision _ = ag } } func BenchmarkLookupMaxSeparated(b *testing.B) { r, err := parseXML([]byte(benchConfigSeparated)) require.NoError(b, err) b.ResetTimer() for i := 0; i < b.N; i++ { precision, ag, _, _ := r.Lookup("test.max", 1, false) _ = precision _ = ag } } func BenchmarkLookupMaxTagged(b *testing.B) { r, err := parseXML([]byte(benchConfig)) require.NoError(b, err) b.ResetTimer() for i := 0; i < b.N; i++ { precision, ag, _, _ := r.Lookup("max?env=test&tag=Fake5", 1, false) _ = precision _ = ag } } func BenchmarkLookupMaxTaggedSeparated(b *testing.B) { r, err := parseXML([]byte(benchConfigSeparated)) require.NoError(b, err) b.ResetTimer() for i := 0; i < b.N; i++ { precision, ag, _, _ := r.Lookup("max?env=test&tag=Fake5", 1, false) _ = precision _ = ag } } func BenchmarkLookupDefault(b *testing.B) { r, err := parseXML([]byte(benchConfig)) require.NoError(b, err) b.ResetTimer() for i := 0; i < b.N; i++ { precision, ag, _, _ := r.Lookup("test.p95", 1, false) _ = precision _ = ag } } func BenchmarkLookupDefaultSeparated(b *testing.B) { r, err := parseXML([]byte(benchConfigSeparated)) require.NoError(b, err) b.ResetTimer() for i := 0; i < b.N; i++ { precision, ag, _, _ := r.Lookup("test.p95", 1, false) _ = precision _ = ag } } func BenchmarkLookupDefaultTagged(b *testing.B) { r, err := parseXML([]byte(benchConfig)) require.NoError(b, err) b.ResetTimer() for i := 0; i < b.N; i++ { precision, ag, _, _ := r.Lookup("p95?env=test&tag=Fake5", 1, false) _ = precision _ = ag } } func BenchmarkLookupDefaultTaggedSeparated(b *testing.B) { r, err := parseXML([]byte(benchConfigSeparated)) require.NoError(b, err) b.ResetTimer() for i := 0; i < b.N; i++ { precision, ag, _, _ := r.Lookup("p95?env=test&tag=Fake5", 1, false) _ = precision _ = ag } } ================================================ FILE: helper/rollup/xml.go ================================================ package rollup import ( "encoding/xml" ) /* click_cost any 0 3600 86400 60 max 0 60 3600 300 86400 3600 */ type ClickhouseRollupXML struct { Rules RulesXML `xml:"graphite_rollup"` } type RetentionXML struct { Age uint32 `xml:"age"` Precision uint32 `xml:"precision"` } type PatternXML struct { RuleType RuleType `xml:"rule_type"` Regexp string `xml:"regexp"` Function string `xml:"function"` Retention []*RetentionXML `xml:"retention"` } type RulesXML struct { Pattern []*PatternXML `xml:"pattern"` Default *PatternXML `xml:"default"` } func (r *RetentionXML) retention() Retention { return Retention{Age: r.Age, Precision: r.Precision} } func (p *PatternXML) pattern() Pattern { result := Pattern{ RuleType: p.RuleType, Regexp: p.Regexp, Function: p.Function, Retention: make([]Retention, 0, len(p.Retention)), } for _, r := range p.Retention { result.Retention = append(result.Retention, r.retention()) } return result } func parseXML(body []byte) (*Rules, error) { r := &RulesXML{} err := xml.Unmarshal(body, r) if err != nil { return nil, err } // Maybe we've got Clickhouse's graphite.xml? if r.Default == nil && r.Pattern == nil { y := &ClickhouseRollupXML{} err = xml.Unmarshal(body, y) if err != nil { return nil, err } r = &y.Rules } patterns := make([]Pattern, 0, uint64(len(r.Pattern))+4) for _, p := range r.Pattern { patterns = append(patterns, p.pattern()) } if r.Default != nil { patterns = append(patterns, r.Default.pattern()) } return (&Rules{Pattern: patterns}).compile() } ================================================ FILE: helper/rollup/xml_test.go ================================================ package rollup import ( "fmt" "regexp" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestParseXML(t *testing.T) { config := ` click_cost any 0 3600 86400 60 without_function 0 3600 86400 60 without_retention min max 0 60 3600 300 86400 3600 ` compact := ` click_cost;any;0:3600,86400:60 without_function;;0:3600,86400:60 without_retention;min; ;max;0:60,3600:300,86400:3600 ` expected, _ := (&Rules{ Pattern: []Pattern{ {Regexp: "click_cost", Function: "any", Retention: []Retention{ {Age: 86400, Precision: 60}, {Age: 0, Precision: 3600}, }}, {Regexp: "without_function", Function: "", Retention: []Retention{ {Age: 0, Precision: 3600}, {Age: 86400, Precision: 60}, }}, {Regexp: "without_retention", Function: "min", Retention: nil}, {Regexp: "", Function: "max", Retention: []Retention{ {Age: 0, Precision: 60}, {Age: 3600, Precision: 300}, {Age: 86400, Precision: 3600}, }}, }, }).compile() t.Run("default", func(t *testing.T) { assert := assert.New(t) r, err := parseXML([]byte(config)) assert.NoError(err) assert.Equal(expected, r) // check sorting assert.Equal(uint32(0), r.Pattern[0].Retention[0].Age) assert.Equal(uint32(3600), r.Pattern[0].Retention[0].Precision) }) t.Run("inside yandex tag", func(t *testing.T) { assert := assert.New(t) r, err := parseXML([]byte(fmt.Sprintf("%s", config))) assert.NoError(err) assert.Equal(expected, r) }) t.Run("compare with compact", func(t *testing.T) { assert := assert.New(t) expectedCompact, err := parseCompact(compact) assert.NoError(err) r, err := parseXML([]byte(fmt.Sprintf("%s", config))) assert.NoError(err) assert.Equal(expectedCompact, r) }) } func TestParseXMLTyped(t *testing.T) { config := ` all> click_cost any 0 3600 86400 60 without_function 0 3600 86400 60 plain without_retention min tagged ^((.*)|.)sum\? sum tag_list fake3;tag=Fake3 min tagged min max 0 60 3600 300 86400 3600 ` expected := &Rules{ Separated: true, Pattern: []Pattern{ { Regexp: "click_cost", Function: "any", Retention: []Retention{ {Age: 0, Precision: 3600}, {Age: 86400, Precision: 60}, }, aggr: AggrMap["any"], re: regexp.MustCompile("click_cost"), }, { Regexp: "without_function", Function: "", Retention: []Retention{ {Age: 0, Precision: 3600}, {Age: 86400, Precision: 60}, }, re: regexp.MustCompile("without_function"), }, { Regexp: "without_retention", RuleType: RulePlain, Function: "min", Retention: nil, aggr: AggrMap["min"], re: regexp.MustCompile("without_retention"), }, { Regexp: `^((.*)|.)sum\?`, RuleType: RuleTagged, Function: "sum", Retention: nil, aggr: AggrMap["sum"], re: regexp.MustCompile(`^((.*)|.)sum\?`), }, { Regexp: `^fake3\?(.*&)?tag=Fake3(&.*)?$`, RuleType: RuleTagged, Function: "min", Retention: nil, aggr: AggrMap["min"], re: regexp.MustCompile(`^fake3\?(.*&)?tag=Fake3(&.*)?$`), }, { Regexp: `^fake4\\?(.*&)?tag4=Fake4(&.*)?$`, RuleType: RuleTagged, Function: "min", Retention: nil, aggr: AggrMap["min"], re: regexp.MustCompile(`^fake4\\?(.*&)?tag4=Fake4(&.*)?$`), }, { Regexp: ".*", Function: "max", Retention: []Retention{ {Age: 0, Precision: 60}, {Age: 3600, Precision: 300}, {Age: 86400, Precision: 3600}, }, aggr: AggrMap["max"], }, }, PatternPlain: []Pattern{ { Regexp: "click_cost", Function: "any", Retention: []Retention{ {Age: 0, Precision: 3600}, {Age: 86400, Precision: 60}, }, aggr: AggrMap["any"], re: regexp.MustCompile("click_cost"), }, { Regexp: "without_function", Function: "", Retention: []Retention{ {Age: 0, Precision: 3600}, {Age: 86400, Precision: 60}, }, re: regexp.MustCompile("without_function"), }, { Regexp: "without_retention", RuleType: RulePlain, Function: "min", Retention: nil, aggr: AggrMap["min"], re: regexp.MustCompile("without_retention"), }, { Regexp: ".*", Function: "max", Retention: []Retention{ {Age: 0, Precision: 60}, {Age: 3600, Precision: 300}, {Age: 86400, Precision: 3600}, }, aggr: AggrMap["max"], }, }, PatternTagged: []Pattern{ { Regexp: "click_cost", Function: "any", Retention: []Retention{ {Age: 0, Precision: 3600}, {Age: 86400, Precision: 60}, }, aggr: AggrMap["any"], re: regexp.MustCompile("click_cost"), }, { Regexp: "without_function", Function: "", Retention: []Retention{ {Age: 0, Precision: 3600}, {Age: 86400, Precision: 60}, }, re: regexp.MustCompile("without_function"), }, { Regexp: `^((.*)|.)sum\?`, RuleType: RuleTagged, Function: "sum", Retention: nil, aggr: AggrMap["sum"], re: regexp.MustCompile(`^((.*)|.)sum\?`), }, { Regexp: `^fake3\?(.*&)?tag=Fake3(&.*)?$`, RuleType: RuleTagged, Function: "min", Retention: nil, aggr: AggrMap["min"], re: regexp.MustCompile(`^fake3\?(.*&)?tag=Fake3(&.*)?$`), }, { Regexp: `^fake4\\?(.*&)?tag4=Fake4(&.*)?$`, RuleType: RuleTagged, Function: "min", Retention: nil, aggr: AggrMap["min"], re: regexp.MustCompile(`^fake4\\?(.*&)?tag4=Fake4(&.*)?$`), }, { Regexp: ".*", Function: "max", Retention: []Retention{ {Age: 0, Precision: 60}, {Age: 3600, Precision: 300}, {Age: 86400, Precision: 3600}, }, aggr: AggrMap["max"], }, }, } t.Run("default", func(t *testing.T) { assert := assert.New(t) r, err := parseXML([]byte(config)) require.NoError(t, err) assert.Equal(expected, r) // check sorting assert.Equal(uint32(0), r.Pattern[0].Retention[0].Age) assert.Equal(uint32(3600), r.Pattern[0].Retention[0].Precision) }) t.Run("inside yandex tag", func(t *testing.T) { assert := assert.New(t) r, err := parseXML([]byte(fmt.Sprintf("%s", config))) assert.NoError(err) assert.Equal(expected, r) }) } ================================================ FILE: helper/tests/clickhouse/server.go ================================================ package clickhouse import ( "io" "net/http" "net/http/httptest" "sync" "sync/atomic" ) type TestResponse struct { Headers map[string]string Body []byte Code int } type TestHandler struct { sync.RWMutex responceMap map[string]*TestResponse queries uint64 } type TestServer struct { *httptest.Server handler *TestHandler } func (h *TestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { body, _ := io.ReadAll(r.Body) req := string(body) h.RLock() resp, ok := h.responceMap[req] h.RUnlock() atomic.AddUint64(&h.queries, 1) if ok { for k, v := range resp.Headers { w.Header().Set(k, v) } if resp.Code == 0 || resp.Code == http.StatusOK { w.Write(resp.Body) } else { http.Error(w, string(resp.Body), http.StatusInternalServerError) } } else { http.Error(w, "Query not added: "+req, http.StatusInternalServerError) } } func NewTestServer() *TestServer { h := &TestHandler{responceMap: make(map[string]*TestResponse)} srv := httptest.NewServer(h) return &TestServer{Server: srv, handler: h} } func (s *TestServer) AddResponce(request string, response *TestResponse) { s.handler.Lock() s.handler.responceMap[request] = response s.handler.Unlock() } func (s *TestServer) Queries() uint64 { return s.handler.queries } ================================================ FILE: helper/tests/compare/compare.go ================================================ package compare import "math" const eps = 0.0000000001 func NearlyEqualSlice(a, b []float64) bool { if len(a) != len(b) { return false } for i, v := range a { // "same" if math.IsNaN(a[i]) && math.IsNaN(b[i]) { continue } if math.IsNaN(a[i]) || math.IsNaN(b[i]) { // unexpected NaN return false } // "close enough" if math.Abs(v-b[i]) > eps { return false } } return true } func NearlyEqual(a, b float64) bool { if math.IsNaN(a) && math.IsNaN(b) { return true } if math.IsNaN(a) || math.IsNaN(b) { // unexpected NaN return false } if math.Abs(a-b) > eps { return false } return true } func Max(a, b int) int { if a >= b { return a } return b } ================================================ FILE: helper/tests/compare/expand/expand.go ================================================ package expand import ( "go/token" "go/types" "strconv" "strings" ) func ExpandTimestamp(fs *token.FileSet, s string, replace map[string]string) (int64, error) { if s == "" { return 0, nil } for k, v := range replace { s = strings.ReplaceAll(s, k, v) } if tv, err := types.Eval(fs, nil, token.NoPos, s); err == nil { return strconv.ParseInt(tv.Value.String(), 10, 32) } else { return 0, err } } ================================================ FILE: helper/utils/utils.go ================================================ package utils import "time" // TimestampTruncate truncate timestamp with duration func TimestampTruncate(ts int64, duration time.Duration) int64 { tm := time.Unix(ts, 0).UTC() return tm.Truncate(duration).UTC().Unix() } ================================================ FILE: helper/utils/utils_test.go ================================================ package utils import ( "fmt" "testing" "time" ) func TestTimestampTruncate(t *testing.T) { // reverse sorted tests := []struct { ts int64 duration time.Duration want int64 }{ { ts: 1628876563, duration: 2 * time.Second, want: 1628876562, }, { ts: 1628876563, duration: 10 * time.Second, want: 1628876560, }, { ts: 1628876563, duration: time.Minute, want: 1628876520, }, { ts: 1628876563, duration: time.Hour, want: 1628874000, }, { ts: 1628876563, duration: 24 * time.Hour, want: 1628812800, }, } for i, tt := range tests { t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { if got := TimestampTruncate(tt.ts, tt.duration); got != tt.want { t.Errorf("timestampTruncate(%d, %d) = %v, want %v", tt.ts, tt.duration, got, tt.want) } }) } } ================================================ FILE: index/handler.go ================================================ package index import ( "net/http" "time" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/logs" "github.com/lomik/graphite-clickhouse/pkg/scope" ) type Handler struct { config *config.Config } func NewHandler(config *config.Config) *Handler { return &Handler{ config: config, } } func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { accessLogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("http") logger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("index") r = r.WithContext(scope.WithLogger(r.Context(), logger)) status := http.StatusOK start := time.Now() defer func() { d := time.Since(start) logs.AccessLog(accessLogger, h.config, r, status, d, time.Duration(0), false, false) }() i, err := New(h.config, r.Context()) if err != nil { status = http.StatusBadRequest http.Error(w, err.Error(), status) return } i.WriteJSON(w) i.Close() } ================================================ FILE: index/index.go ================================================ package index import ( "bufio" "bytes" "context" "encoding/json" "fmt" "io" "net/http" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/finder" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/pkg/scope" ) type Index struct { config *config.Config rowsReader io.ReadCloser } func New(config *config.Config, ctx context.Context) (*Index, error) { var reader io.ReadCloser var err error opts := clickhouse.Options{ TLSConfig: config.ClickHouse.TLSConfig, ConnectTimeout: config.ClickHouse.ConnectTimeout, CheckRequestProgress: config.FeatureFlags.LogQueryProgress, ProgressSendingInterval: config.ClickHouse.ProgressSendingInterval, } if config.ClickHouse.IndexTable != "" { opts.Timeout = config.ClickHouse.IndexTimeout reader, err = clickhouse.Reader( scope.WithTable(ctx, config.ClickHouse.IndexTable), config.ClickHouse.URL, fmt.Sprintf( "SELECT Path FROM %s WHERE Date = '%s' AND Level >= %d AND Level < %d GROUP BY Path", config.ClickHouse.IndexTable, finder.DefaultTreeDate, finder.TreeLevelOffset, finder.ReverseTreeLevelOffset, ), opts, nil, ) } else { opts.Timeout = config.ClickHouse.TreeTimeout reader, err = clickhouse.Reader( scope.WithTable(ctx, config.ClickHouse.TreeTable), config.ClickHouse.URL, fmt.Sprintf("SELECT Path FROM %s GROUP BY Path", config.ClickHouse.TreeTable), opts, nil, ) } if err != nil { return nil, err } return &Index{ config: config, rowsReader: reader, }, nil } func (i *Index) Close() error { return i.rowsReader.Close() } func (i *Index) WriteJSON(w http.ResponseWriter) error { _, err := w.Write([]byte("[")) if err != nil { return err } s := bufio.NewScanner(i.rowsReader) idx := 0 for s.Scan() { b := s.Bytes() if len(b) == 0 { continue } if b[len(b)-1] == '.' { continue } json_b, err := json.Marshal(string(b)) if err != nil { return err } jsonParts := [][]byte{ nil, json_b, } if idx != 0 { jsonParts[0] = []byte{','} } jsonified := bytes.Join(jsonParts, []byte("")) _, err = w.Write(jsonified) if err != nil { return err } idx++ } if err := s.Err(); err != nil { return err } _, err = w.Write([]byte("]")) return err } ================================================ FILE: index/index_test.go ================================================ package index import ( "bytes" "encoding/json" "io" "net/http/httptest" "strings" "testing" ) func TestWriteJSONEmptyRows(t *testing.T) { rows := []string{ "", "testing.leaf", "", "testing.leaf.node", "", } metrics, err := writeRows(rows) if err != nil { t.Fatalf("Error during transform or unmarshal: %s", err) } if len(metrics) != 2 { t.Fatalf("Wrong metrics slice length = %d: %s", len(metrics), metrics) } if metrics[0] != "testing.leaf" || metrics[1] != "testing.leaf.node" { t.Fatalf("Wrong metrics contents: %s", metrics) } } func TestWriteJSONNonleafRows(t *testing.T) { rows := []string{ "testing.leaf", "testing.nonleaf.", "testing.leaf.node", "testing.\"broken\".node", } metrics, err := writeRows(rows) if err != nil { t.Fatalf("Error during transform or unmarshal: %s", err) } if len(metrics) != 3 { t.Fatalf("Wrong metrics slice length = %d: %s", len(metrics), metrics) } if metrics[0] != "testing.leaf" || metrics[1] != "testing.leaf.node" || metrics[2] != "testing.\"broken\".node" { t.Fatalf("Wrong metrics contents: %s", metrics) } } func TestWriteJSONEmptyIndex(t *testing.T) { rows := []string{} metrics, err := writeRows(rows) if err != nil { t.Fatalf("Error during transform or unmarshal: %s", err) } if len(metrics) != 0 { t.Fatalf("Wrong metrics slice length = %d: %s", len(metrics), metrics) } } func indexForBytes(b []byte) *Index { buffer := bytes.NewBuffer(b) return &Index{ config: nil, rowsReader: io.NopCloser(buffer), } } func writeRows(rows []string) ([]string, error) { rowsBytes := []byte(strings.Join(rows, string('\n'))) index := indexForBytes(rowsBytes) mockResponse := httptest.NewRecorder() err := index.WriteJSON(mockResponse) if err != nil { return nil, err } var metrics []string err = json.Unmarshal(mockResponse.Body.Bytes(), &metrics) if err != nil { return nil, err } return metrics, nil } ================================================ FILE: issues/daytime/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: issues/daytime/graphite-clickhouse-internal-aggr.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = true tagged-table = "graphite_tags" tagged-autocomplete-days = 1 date-format = "both" [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: issues/daytime/graphite-clickhouse.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = false tagged-table = "graphite_tags" tagged-autocomplete-days = 1 date-format = "both" [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: issues/daytime/test.toml ================================================ [test] precision = "10s" [[test.clickhouse]] version = "24.2" dir = "tests/clickhouse/rollup" [test.carbon_clickhouse] #version = "v0.11.1" image = "msaf1980/carbon-clickhouse" version = "tz" template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse-internal-aggr.conf.tpl" [[test.input]] name = "test.plain1" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 2.0, time = "rnow"}] [[test.input]] name = "test.plain2" points = [{value = 2.0, time = "rnow-30"}, {value = 1.0, time = "rnow-20"}, {value = 1.5, time = "rnow-10"}, {value = 2.5, time = "rnow"}] [[test.input]] name = "test2.plain" points = [{value = 1.0, time = "rnow-30"}, {value = 2.0, time = "rnow-20"}, {value = 2.5, time = "rnow-10"}, {value = 3.5, time = "rnow"}] [[test.input]] name = "metric1;tag1=value1;tag2=value21;tag3=value3" points = [{value = 2.0, time = "rnow-30"}, {value = 2.5, time = "rnow-20"}, {value = 2.0, time = "rnow-10"}, {value = 3.0, time = "rnow"}] [[test.input]] name = "metric1;tag2=value22;tag4=value4" points = [{value = 1.0, time = "rnow-30"}, {value = 2.0, time = "rnow-20"}, {value = 0.0, time = "rnow-10"}, {value = 1.0, time = "rnow"}] [[test.input]] name = "metric1;tag1=value1;tag2=value23;tag3=value3" points = [{value = 0.5, time = "rnow-30"}, {value = 1.5, time = "rnow-20"}, {value = 4.0, time = "rnow-10"}, {value = 3.0, time = "rnow"}] [[test.input]] name = "metric2;tag2=value21;tag4=value4" points = [{value = 2.0, time = "rnow-30"}, {value = 1.0, time = "rnow-20"}, {value = 0.0, time = "rnow-10"}, {value = 1.0, time = "rnow"}] ###################################### # Check metrics find [[test.find_checks]] formats = [ "pickle", "protobuf", "carbonapi_v3_pb" ] query = "test" result = [ { path = "test", is_leaf = false } ] [[test.find_checks]] formats = [ "pickle", "protobuf", "carbonapi_v3_pb" ] query = "test.pl*" result = [ { path = "test.plain1", is_leaf = true }, { path = "test.plain2", is_leaf = true } ] # End - Check metrics find ###################################### # Check tags autocomplete [[test.tags_checks]] query = "tag1;tag2=value21" result = [ "value1" ] [[test.tags_checks]] query = "name;tag2=value21;tag1=~value" result = [ "metric1", ] # End - Check tags autocomplete ########################################################################## # Plain metrics (carbonapi_v3_pb) # test.plain1 # test.plain2 # test2.plain [[test.render_checks]] from = "rnow-10" until = "rnow" targets = [ "test.plain*", "test{1,2}.plain" ] [[test.render_checks.result]] name = "test.plain1" path = "test.plain*" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, 2.0] [[test.render_checks.result]] name = "test.plain2" path = "test.plain*" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.5, 2.5] [[test.render_checks.result]] name = "test2.plain" path = "test{1,2}.plain" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [2.5, 3.5] # End - Plain metrics (carbonapi_v3_pb) ########################################################################## # Plain metrics (carbonapi_v2_pb) [[test.render_checks]] formats = [ "protobuf", "carbonapi_v2_pb" ] from = "rnow-10" until = "rnow+1" targets = [ "test.plain*", "test{1,2}.plain" ] [[test.render_checks.result]] name = "test.plain1" start = "rnow-10" stop = "rnow+10" step = 10 values = [1.0, 2.0] [[test.render_checks.result]] name = "test.plain2" start = "rnow-10" stop = "rnow+10" step = 10 values = [1.5, 2.5] [[test.render_checks.result]] name = "test2.plain" start = "rnow-10" stop = "rnow+10" step = 10 values = [2.5, 3.5] # End - Plain metrics (carbonapi_v2_pb) ########################################################################## # Plain metrics (pickle) [[test.render_checks]] formats = [ "pickle" ] from = "rnow-10" until = "rnow+1" targets = [ "test.plain*", "test{1,2}.plain" ] [[test.render_checks.result]] name = "test.plain1" path = "test.plain*" start = "rnow-10" stop = "rnow+10" step = 10 values = [1.0, 2.0] [[test.render_checks.result]] name = "test.plain2" path = "test.plain*" start = "rnow-10" stop = "rnow+10" step = 10 values = [1.5, 2.5] [[test.render_checks.result]] name = "test2.plain" path = "test{1,2}.plain" start = "rnow-10" stop = "rnow+10" step = 10 values = [2.5, 3.5] # End - Plain metrics (pickle) ########################################################################## # Taged metrics (carbonapi_v3_pb) # metric1;tag1=value1;tag2=value21;tag3=value3 # metric1;tag2=value22;tag4=value4 # metric1;tag1=value1;tag2=value23;tag3=value3 # metric2;tag2=value21;tag4=value4 [[test.render_checks]] from = "rnow-10" until = "rnow+1" targets = [ "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')", "seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')" ] [[test.render_checks.result]] name = "metric1;tag1=value1;tag2=value21;tag3=value3" path = "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [2.0, 3.0] [[test.render_checks.result]] name = "metric1;tag1=value1;tag2=value23;tag3=value3" path = "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [4.0, 3.0] [[test.render_checks.result]] name = "metric2;tag2=value21;tag4=value4" path = "seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [0.0, 1.0] # End - Tagged metrics (carbonapi_v3_pb) ########################################################################## # Tagged metrics (carbonapi_v2_pb) [[test.render_checks]] formats = [ "protobuf", "carbonapi_v2_pb" ] from = "rnow-10" until = "rnow+1" targets = [ "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')", "seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')" ] [[test.render_checks.result]] name = "metric1;tag1=value1;tag2=value21;tag3=value3" start = "rnow-10" stop = "rnow+10" step = 10 values = [2.0, 3.0] [[test.render_checks.result]] name = "metric1;tag1=value1;tag2=value23;tag3=value3" start = "rnow-10" stop = "rnow+10" step = 10 values = [4.0, 3.0] [[test.render_checks.result]] name = "metric2;tag2=value21;tag4=value4" start = "rnow-10" stop = "rnow+10" step = 10 values = [0.0, 1.0] # End - Tagged metrics (carbonapi_v2_pb) ########################################################################## # Tagged metrics (pickle) [[test.render_checks]] formats = [ "pickle" ] from = "rnow-10" until = "rnow+1" targets = [ "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')", "seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')" ] [[test.render_checks.result]] name = "metric1;tag1=value1;tag2=value21;tag3=value3" path = "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')" start = "rnow-10" stop = "rnow+10" step = 10 values = [2.0, 3.0] [[test.render_checks.result]] name = "metric1;tag1=value1;tag2=value23;tag3=value3" path = "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')" start = "rnow-10" stop = "rnow+10" step = 10 values = [4.0, 3.0] [[test.render_checks.result]] name = "metric2;tag2=value21;tag4=value4" path = "seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')" start = "rnow-10" stop = "rnow+10" step = 10 values = [0.0, 1.0] # End - Tagged metrics (pickle) ########################################################################## # Midnight # points for check https://github.com/go-graphite/graphite-clickhouse/issues/184 [[test.input]] name = "test.midnight" points = [{value = 3.0, time = "midnight+60s"}] [[test.input]] name = "now;scope=midnight" points = [{value = 4.0, time = "midnight+60s"}] [[test.find_checks]] name = "Midnight (direct)" query = "test.midnight*" result = [{ path = "test.midnight", is_leaf = true }] [[test.find_checks]] name = "Midnight" query = "test.midnight" from = "midnight+60s" until = "midnight+70s" result = [{ path = "test.midnight", is_leaf = true }] [[test.find_checks]] name = "Midnight (reverse)" query = "*test.midnight" result = [{ path = "test.midnight", is_leaf = true }] [[test.find_checks]] name = "Midnight" query = "test.midnight" from = "midnight+60s" until = "midnight+70s" result = [{ path = "test.midnight", is_leaf = true }] [[test.tags_checks]] name = "Midnight" query = "name;scope=midnight" result = [ "now", ] [[test.render_checks]] name = "Midnight (direct)" formats = [ "protobuf" ] from = "midnight+60s" until = "midnight+70s" targets = [ "test.midnight*", ] [[test.render_checks.result]] name = "test.midnight" start = "midnight+60s" stop = "midnight+80s" step = 10 values = [3.0, nan] [[test.render_checks]] name = "Midnight (reverse)" formats = [ "protobuf" ] from = "midnight+60s" until = "midnight+70s" targets = [ "*test.midnight", ] dump_if_empty = [ "SELECT Date, Path FROM graphite_index WHERE ((Level=2) AND (Path LIKE 'test.midnight%')) GROUP BY Date, Path" ] [[test.render_checks.result]] name = "test.midnight" start = "midnight+60s" stop = "midnight+80s" step = 10 values = [3.0, nan] [[test.render_checks]] name = "Midnight" formats = [ "protobuf" ] from = "midnight+60s" until = "midnight+70s" targets = [ "seriesByTag('name=now', 'scope=midnight')", ] [[test.render_checks.result]] name = "now;scope=midnight" start = "midnight+60s" stop = "midnight+80s" step = 10 values = [4.0, nan] # End - Midnight ########################################################################## # Day end # points for check https://github.com/go-graphite/graphite-clickhouse/issues/184 [[test.input]] name = "test.23h" points = [{value = 3.0, time = "midnight+1380m"}] [[test.input]] name = "now;scope=23h" points = [{value = 4.0, time = "midnight+1380m"}] [[test.find_checks]] name = "Day end" query = "test.23h" from = "midnight+1380m" until = "midnight+1381m" result = [{ path = "test.23h", is_leaf = true }] [[test.find_checks]] name = "Day end" query = "test.23h" from = "midnight+1380m" until = "midnight+1381m" result = [{ path = "test.23h", is_leaf = true }] [[test.tags_checks]] name = "Day end" query = "name;scope=23h" result = [ "now", ] dump_if_empty = [ "SELECT Date, Tags FROM graphite_tags WHERE ((Tag1='scope=23h') AND (arrayJoin(Tags) LIKE '__name__=%')) GROUP BY Date, Tags ORDER BY Date, Tags" ] [[test.render_checks]] name = "Day end" formats = [ "protobuf" ] from = "midnight+1380m" until = "midnight+1380m+10s" targets = [ "test.23h", ] dump_if_empty = [ "SELECT Date, Path FROM graphite_index WHERE ((Level=2) AND (Path IN ('test.23h','test.23h.'))) GROUP BY Date, Path" ] [[test.render_checks.result]] name = "test.23h" start = "midnight+1380m" stop = "midnight+1380m+20s" step = 10 values = [3.0, nan] [[test.render_checks]] name = "Day end" formats = [ "protobuf" ] from = "midnight+1380m" until = "midnight+1380m+10s" targets = [ "seriesByTag('name=now', 'scope=23h')", ] [[test.render_checks.result]] name = "now;scope=23h" start = "midnight+1380m" stop = "midnight+1380m+20s" step = 10 values = [4.0, nan] # End - Day end ########################################################################## ================================================ FILE: limiter/alimiter.go ================================================ package limiter import ( "context" "time" "github.com/lomik/graphite-clickhouse/load_avg" "github.com/lomik/graphite-clickhouse/metrics" ) var ( ctxMain, Stop = context.WithCancel(context.Background()) checkDelay = time.Second * 60 ) // calc reserved slots count based on load average (for protect overload) func getWeighted(n, max int) int { if n <= 0 { return 0 } loadAvg := load_avg.Load() if loadAvg < 0.6 { return 0 } l := int(float64(n) * loadAvg) if l >= max { if max <= 1 { return 1 } return max - 1 } return l } // ALimiter provide limiter amount of requests/concurrently executing requests (adaptive with load avg) type ALimiter struct { limiter limiter concurrentLimiter limiter concurrent int n int m metrics.WaitMetric } // NewServerLimiter creates a limiter for specific servers list. func NewALimiter(capacity, concurrent, n int, enableMetrics bool, scope, sub string) ServerLimiter { if capacity <= 0 && concurrent <= 0 { return NoopLimiter{} } if n >= concurrent { n = concurrent - 1 } if n <= 0 { return NewWLimiter(capacity, concurrent, enableMetrics, scope, sub) } a := &ALimiter{ m: metrics.NewWaitMetric(enableMetrics, scope, sub), concurrent: concurrent, n: n, } a.concurrentLimiter.ch = make(chan struct{}, concurrent) a.concurrentLimiter.cap = concurrent go a.balance() return a } func (sl *ALimiter) balance() int { var last int for { start := time.Now() n := getWeighted(sl.n, sl.concurrent) if n > last { for i := 0; i < n-last; i++ { if sl.concurrentLimiter.enter(ctxMain, "balance") != nil { break } } last = n } else if n < last { for i := 0; i < last-n; i++ { sl.concurrentLimiter.leave(ctxMain, "balance") } last = n } delay := time.Since(start) if delay < checkDelay { time.Sleep(checkDelay - delay) } } } func (sl *ALimiter) Capacity() int { return sl.limiter.capacity() } func (sl *ALimiter) Enter(ctx context.Context, s string) (err error) { if sl.limiter.cap > 0 { if err = sl.limiter.tryEnter(ctx, s); err != nil { sl.m.WaitErrors.Add(1) return } } if sl.concurrentLimiter.cap > 0 { if sl.concurrentLimiter.enter(ctx, s) != nil { if sl.limiter.cap > 0 { sl.limiter.leave(ctx, s) } sl.m.WaitErrors.Add(1) err = ErrTimeout } } sl.m.Requests.Add(1) return } // TryEnter claims one of free slots without blocking. func (sl *ALimiter) TryEnter(ctx context.Context, s string) (err error) { if sl.limiter.cap > 0 { if err = sl.limiter.tryEnter(ctx, s); err != nil { sl.m.WaitErrors.Add(1) return } } if sl.concurrentLimiter.cap > 0 { if sl.concurrentLimiter.tryEnter(ctx, s) != nil { if sl.limiter.cap > 0 { sl.limiter.leave(ctx, s) } sl.m.WaitErrors.Add(1) err = ErrTimeout } } sl.m.Requests.Add(1) return } // Frees a slot in limiter func (sl *ALimiter) Leave(ctx context.Context, s string) { if sl.limiter.cap > 0 { sl.limiter.leave(ctx, s) } sl.concurrentLimiter.leave(ctx, s) } // SendDuration send StatsD duration iming func (sl *ALimiter) SendDuration(queueMs int64) { if sl.m.WaitTimeName != "" { metrics.Gstatsd.Timing(sl.m.WaitTimeName, queueMs, 1.0) } } // Unregiter unregister graphite metric func (sl *ALimiter) Unregiter() { sl.m.Unregister() } // Enabled return enabled flag, if false - it's a noop limiter and can be safely skiped func (sl *ALimiter) Enabled() bool { return true } ================================================ FILE: limiter/alimiter_test.go ================================================ package limiter import ( "context" "fmt" "strconv" "sync" "testing" "time" "github.com/lomik/graphite-clickhouse/load_avg" "github.com/stretchr/testify/require" ) func Test_getWeighted(t *testing.T) { tests := []struct { loadAvg float64 n int max int want int }{ {loadAvg: 0, max: 100, n: 100, want: 0}, {loadAvg: 0.2, max: 100, n: 100, want: 0}, {loadAvg: 0.7, max: 100, n: 100, want: 70}, {loadAvg: 0.8, max: 100, n: 100, want: 80}, {loadAvg: 0.999, max: 100, n: 100, want: 99}, {loadAvg: 0.999, max: 100, n: 1, want: 0}, {loadAvg: 1, max: 1, n: 100, want: 1}, {loadAvg: 1, max: 100, n: 100, want: 99}, {loadAvg: 1, max: 101, n: 100, want: 100}, {loadAvg: 1, max: 200, n: 100, want: 100}, {loadAvg: 2, max: 100, n: 200, want: 99}, {loadAvg: 2, max: 200, n: 200, want: 199}, {loadAvg: 2, max: 300, n: 200, want: 299}, {loadAvg: 2, max: 400, n: 200, want: 399}, {loadAvg: 2, max: 401, n: 200, want: 400}, {loadAvg: 2, max: 402, n: 200, want: 400}, } for n, tt := range tests { t.Run(strconv.Itoa(n), func(t *testing.T) { load_avg.Store(tt.loadAvg) if got := getWeighted(tt.n, tt.max); got != tt.want { t.Errorf("load avg = %f getWeighted(%d, %d) = %v, want %v", tt.loadAvg, tt.n, tt.max, got, tt.want) } }) } } func TestNewALimiter(t *testing.T) { capacity := 14 concurrent := 12 n := 10 checkDelay = time.Millisecond * 10 limiter := NewALimiter(capacity, concurrent, n, false, "", "") // inital - load not collected load_avg.Store(0) var i int ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*100) for i = 0; i < concurrent; i++ { require.NoError(t, limiter.Enter(ctx, "render"), "try to lock with load_avg = 0 [%d]", i) } require.Error(t, limiter.Enter(ctx, "render")) for i = 0; i < concurrent; i++ { limiter.Leave(ctx, "render") } cancel() // load_avg 0.5 load_avg.Store(0.5) k := getWeighted(n, concurrent) require.Equal(t, 0, k) // load_avg 0.6 load_avg.Store(0.6) k = getWeighted(n, concurrent) require.Equal(t, n*6/10, k) time.Sleep(checkDelay * 2) ctx, cancel = context.WithTimeout(context.Background(), time.Millisecond*100) for i = 0; i < concurrent-k; i++ { require.NoError(t, limiter.Enter(ctx, "render"), "try to lock with load_avg = 0.5 [%d]", i) } require.Error(t, limiter.Enter(ctx, "render")) for i = 0; i < concurrent-k; i++ { limiter.Leave(ctx, "render") } cancel() // // load_avg 1 load_avg.Store(1) k = getWeighted(n, concurrent) require.Equal(t, n, k) time.Sleep(checkDelay * 2) ctx, cancel = context.WithTimeout(context.Background(), time.Millisecond*10) for i = 0; i < concurrent-n; i++ { require.NoError(t, limiter.Enter(ctx, "render"), "try to lock with load_avg = 1 [%d]", i) } require.Error(t, limiter.Enter(ctx, "render")) for i = 0; i < concurrent-n; i++ { limiter.Leave(ctx, "render") } cancel() } type testLimiter struct { l int c int n int concurrencyLevel int } func Benchmark_Limiter_Parallel(b *testing.B) { tests := []testLimiter{ // WLimiter {l: 2000, c: 10, concurrencyLevel: 1}, {l: 2000, c: 10, concurrencyLevel: 10}, {l: 2000, c: 10, concurrencyLevel: 20}, {l: 2000, c: 10, concurrencyLevel: 50}, {l: 2000, c: 10, concurrencyLevel: 100}, {l: 2000, c: 10, concurrencyLevel: 1000}, // ALimiter {l: 2000, c: 10, n: 50, concurrencyLevel: 1}, {l: 2000, c: 10, n: 50, concurrencyLevel: 10}, {l: 2000, c: 10, n: 50, concurrencyLevel: 20}, {l: 2000, c: 10, n: 50, concurrencyLevel: 50}, {l: 2000, c: 10, n: 50, concurrencyLevel: 100}, {l: 2000, c: 10, n: 50, concurrencyLevel: 1000}, } load_avg.Store(0.5) for _, tt := range tests { b.Run(fmt.Sprintf("L%d_C%d_N%d_CONCURRENCY%d", tt.l, tt.c, tt.n, tt.concurrencyLevel), func(b *testing.B) { var ( err error ) limiter := NewALimiter(tt.l, tt.c, tt.n, false, "", "") wgStart := sync.WaitGroup{} wg := sync.WaitGroup{} wgStart.Add(tt.concurrencyLevel) ctx := context.Background() b.ResetTimer() for i := 0; i < tt.concurrencyLevel; i++ { wg.Add(1) go func() { wgStart.Done() wgStart.Wait() // Test routine for n := 0; n < b.N; n++ { errW := limiter.Enter(ctx, "render") if errW == nil { limiter.Leave(ctx, "render") } else { err = errW break } } // End test routine wg.Done() }() } wg.Wait() b.StopTimer() if err != nil { b.Fatal(b, err) } }) } } ================================================ FILE: limiter/interface.go ================================================ package limiter import ( "context" "errors" ) var ( ErrTimeout = errors.New("timeout exceeded") ErrOverflow = errors.New("storage maximum queries exceeded") ) type ServerLimiter interface { Capacity() int Enabled() bool TryEnter(ctx context.Context, s string) error Enter(ctx context.Context, s string) error Leave(ctx context.Context, s string) SendDuration(queueMs int64) Unregiter() } ================================================ FILE: limiter/limiter.go ================================================ package limiter import ( "context" "github.com/lomik/graphite-clickhouse/metrics" ) type limiter struct { ch chan struct{} cap int } // Limiter provides interface to limit amount of requests type Limiter struct { limiter limiter metrics metrics.WaitMetric } // NewServerLimiter creates a limiter for specific servers list. func NewLimiter(capacity int, enableMetrics bool, scope, sub string) ServerLimiter { if capacity <= 0 { return NoopLimiter{} } return &Limiter{ limiter: limiter{ ch: make(chan struct{}, capacity), cap: capacity, }, metrics: metrics.NewWaitMetric(enableMetrics, scope, sub), } } func (sl *Limiter) Capacity() int { return sl.limiter.capacity() } // Enter claims one of free slots or blocks until there is one. func (sl *Limiter) Enter(ctx context.Context, s string) (err error) { if err = sl.limiter.enter(ctx, s); err != nil { sl.metrics.WaitErrors.Add(1) } sl.metrics.Requests.Add(1) return } // TryEnter claims one of free slots without blocking. func (sl *Limiter) TryEnter(ctx context.Context, s string) (err error) { if err = sl.limiter.tryEnter(ctx, s); err != nil { sl.metrics.WaitErrors.Add(1) } sl.metrics.Requests.Add(1) return } // Frees a slot in limiter func (sl *Limiter) Leave(ctx context.Context, s string) { sl.limiter.leave(ctx, s) } // SendDuration send StatsD duration iming func (sl *Limiter) SendDuration(queueMs int64) { if sl.metrics.WaitTimeName != "" { metrics.Gstatsd.Timing(sl.metrics.WaitTimeName, queueMs, 1.0) } } // Unregiter unregister graphite metric func (sl *Limiter) Unregiter() { sl.metrics.Unregister() } // Enabled return enabled flag, if false - it's a noop limiter and can be safely skiped func (sl *Limiter) Enabled() bool { return true } func (sl *limiter) capacity() int { return sl.cap } // Enter claims one of free slots or blocks until there is one. func (sl *limiter) enter(ctx context.Context, s string) error { select { case sl.ch <- struct{}{}: return nil case <-ctx.Done(): return ErrTimeout } } // TryEnter claims one of free slots without blocking. func (sl *limiter) tryEnter(ctx context.Context, s string) error { select { case sl.ch <- struct{}{}: return nil default: return ErrOverflow } } // Frees a slot in limiter func (sl *limiter) leave(ctx context.Context, s string) { <-sl.ch } ================================================ FILE: limiter/noop.go ================================================ package limiter import ( "context" ) // ServerLimiter provides interface to limit amount of requests type NoopLimiter struct { } func (l NoopLimiter) Capacity() int { return 0 } // Enter claims one of free slots or blocks until there is one. func (l NoopLimiter) Enter(ctx context.Context, s string) error { return nil } // TryEnter claims one of free slots without blocking func (l NoopLimiter) TryEnter(ctx context.Context, s string) error { return nil } // Frees a slot in limiter func (l NoopLimiter) Leave(ctx context.Context, s string) { } // SendDuration send StatsD duration iming func (l NoopLimiter) SendDuration(queueMs int64) { } // Unregiter unregister graphite metric func (l NoopLimiter) Unregiter() { } // Enabled return enabled flag, if false - it's a noop limiter and can be safely skiped func (l NoopLimiter) Enabled() bool { return false } ================================================ FILE: limiter/wlimiter.go ================================================ package limiter import ( "context" "github.com/lomik/graphite-clickhouse/metrics" ) // WLimiter provide limiter amount of requests/concurrently executing requests type WLimiter struct { limiter limiter concurrentLimiter limiter metrics metrics.WaitMetric } // NewServerLimiter creates a limiter for specific servers list. func NewWLimiter(capacity, concurrent int, enableMetrics bool, scope, sub string) ServerLimiter { if capacity <= 0 && concurrent <= 0 { return NoopLimiter{} } if concurrent <= 0 { return NewLimiter(capacity, enableMetrics, scope, sub) } w := &WLimiter{ metrics: metrics.NewWaitMetric(enableMetrics, scope, sub), } if capacity > 0 { w.limiter.ch = make(chan struct{}, capacity) w.limiter.cap = capacity } if concurrent > 0 { w.concurrentLimiter.ch = make(chan struct{}, concurrent) w.concurrentLimiter.cap = concurrent } return w } func (sl *WLimiter) Capacity() int { return sl.limiter.capacity() } func (sl *WLimiter) Enter(ctx context.Context, s string) (err error) { if sl.limiter.cap > 0 { if err = sl.limiter.tryEnter(ctx, s); err != nil { sl.metrics.WaitErrors.Add(1) return } } if sl.concurrentLimiter.cap > 0 { if sl.concurrentLimiter.enter(ctx, s) != nil { if sl.limiter.cap > 0 { sl.limiter.leave(ctx, s) } sl.metrics.WaitErrors.Add(1) err = ErrTimeout } } sl.metrics.Requests.Add(1) return } // TryEnter claims one of free slots without blocking. func (sl *WLimiter) TryEnter(ctx context.Context, s string) (err error) { if sl.limiter.cap > 0 { if err = sl.limiter.tryEnter(ctx, s); err != nil { sl.metrics.WaitErrors.Add(1) return } } if sl.concurrentLimiter.cap > 0 { if sl.concurrentLimiter.tryEnter(ctx, s) != nil { if sl.limiter.cap > 0 { sl.limiter.leave(ctx, s) } sl.metrics.WaitErrors.Add(1) err = ErrTimeout } } sl.metrics.Requests.Add(1) return } // Frees a slot in limiter func (sl *WLimiter) Leave(ctx context.Context, s string) { if sl.limiter.cap > 0 { sl.limiter.leave(ctx, s) } sl.concurrentLimiter.leave(ctx, s) } // SendDuration send StatsD duration iming func (sl *WLimiter) SendDuration(queueMs int64) { if sl.metrics.WaitTimeName != "" { metrics.Gstatsd.Timing(sl.metrics.WaitTimeName, queueMs, 1.0) } } // Unregiter unregister graphite metric func (sl *WLimiter) Unregiter() { sl.metrics.Unregister() } // Enabled return enabled flag, if false - it's a noop limiter and can be safely skiped func (sl *WLimiter) Enabled() bool { return true } ================================================ FILE: load_avg/load_avg.go ================================================ package load_avg import ( "math" "github.com/msaf1980/go-syncutils/atomic" ) var loadAvgStore atomic.Float64 func Load() float64 { return loadAvgStore.Load() } func Store(f float64) { loadAvgStore.Store(f) } func Weight(weight int, degraged, degragedLoadAvg, normalizedLoadAvg float64) int64 { if weight <= 0 || degraged <= 1 || normalizedLoadAvg >= 2.0 { return 1 } if normalizedLoadAvg > degragedLoadAvg { normalizedLoadAvg *= degraged } normalizedLoadAvg = math.Round(10*normalizedLoadAvg) / 10 if normalizedLoadAvg == 0 { return 2 * int64(weight) } normalizedLoadAvg = math.Log10(normalizedLoadAvg) w := int64(weight) - int64(float64(weight)*normalizedLoadAvg) if w <= 0 { return 1 } return w } ================================================ FILE: load_avg/load_avg_default.go ================================================ //go:build !linux // +build !linux package load_avg func Normalized() (float64, error) { return 0, nil } func CpuCount() (uint64, error) { return 0, nil } ================================================ FILE: load_avg/load_avg_linux.go ================================================ //go:build linux // +build linux package load_avg import ( "os" "strings" "syscall" "github.com/msaf1980/go-stringutils" ) func Normalized() (float64, error) { var info syscall.Sysinfo_t err := syscall.Sysinfo(&info) if err != nil { return 0, err } cpus, err := CpuCount() if err != nil { return 0, err } const si_load_shift = 16 load := float64(info.Loads[0]) / float64(1< 0 { requestMetric.RangeS = c.FindRangeS requestMetric.RangeNames = c.FindRangeNames requestMetric.RangeMetrics = make([]ReqMetric, len(c.FindRangeS)) for i := range c.FindRangeS { requestMetric.RangeMetrics[i].RequestsH = metrics.NewVSumHistogram(c.BucketsWidth, c.BucketsLabels).SetNameTotal("") requestMetric.RangeMetrics[i].Errors = metrics.NewCounter() requestMetric.RangeMetrics[i].MetricsCountName = scope + "." + requestMetric.RangeNames[i] + ".metrics" requestMetric.RangeMetrics[i].PointsCountName = scope + "." + requestMetric.RangeNames[i] + ".points" metrics.Register(scope+"."+c.FindRangeNames[i]+".requests", requestMetric.RangeMetrics[i].RequestsH) metrics.Register(scope+"."+c.FindRangeNames[i]+".errors", requestMetric.RangeMetrics[i].Errors) if c.ExtendedStat { requestMetric.RangeMetrics[i].Requests200 = metrics.NewCounter() requestMetric.RangeMetrics[i].Requests400 = metrics.NewCounter() requestMetric.RangeMetrics[i].Requests403 = metrics.NewCounter() requestMetric.RangeMetrics[i].Requests404 = metrics.NewCounter() requestMetric.RangeMetrics[i].Requests500 = metrics.NewCounter() requestMetric.RangeMetrics[i].Requests503 = metrics.NewCounter() requestMetric.RangeMetrics[i].Requests504 = metrics.NewCounter() requestMetric.RangeMetrics[i].Requests5xx = metrics.NewCounter() requestMetric.RangeMetrics[i].Requests4xx = metrics.NewCounter() metrics.Register(scope+"."+c.FindRangeNames[i]+".requests_status_code.200", requestMetric.RangeMetrics[i].Requests200) metrics.Register(scope+"."+c.FindRangeNames[i]+".requests_status_code.400", requestMetric.RangeMetrics[i].Requests400) metrics.Register(scope+"."+c.FindRangeNames[i]+".requests_status_code.403", requestMetric.RangeMetrics[i].Requests403) metrics.Register(scope+"."+c.FindRangeNames[i]+".requests_status_code.404", requestMetric.RangeMetrics[i].Requests404) metrics.Register(scope+"."+c.FindRangeNames[i]+".requests_status_code.4xx", requestMetric.RangeMetrics[i].Requests4xx) metrics.Register(scope+"."+c.FindRangeNames[i]+".requests_status_code.500", requestMetric.RangeMetrics[i].Requests500) metrics.Register(scope+"."+c.FindRangeNames[i]+".requests_status_code.503", requestMetric.RangeMetrics[i].Requests503) metrics.Register(scope+"."+c.FindRangeNames[i]+".requests_status_code.504", requestMetric.RangeMetrics[i].Requests504) metrics.Register(scope+"."+c.FindRangeNames[i]+".requests_status_code.5xx", requestMetric.RangeMetrics[i].Requests5xx) } } } } else { requestMetric.RequestsH = metrics.NilHistogram{} } return requestMetric } func initRenderMetrics(scope string, c *Config) *RenderMetrics { requestMetric := &RenderMetrics{ RenderMetric: RenderMetric{ ReqMetric: ReqMetric{ Errors: metrics.NewCounter(), MetricsCountName: scope + ".all.metrics", PointsCountName: scope + ".all.points", }, }, } if c == nil || Graphite == nil || !c.ExtendedStat { requestMetric.Requests200 = metrics.NilCounter{} requestMetric.Requests400 = metrics.NilCounter{} requestMetric.Requests403 = metrics.NilCounter{} requestMetric.Requests404 = metrics.NilCounter{} requestMetric.Requests500 = metrics.NilCounter{} requestMetric.Requests503 = metrics.NilCounter{} requestMetric.Requests504 = metrics.NilCounter{} requestMetric.Requests5xx = metrics.NilCounter{} requestMetric.Requests4xx = metrics.NilCounter{} } else { requestMetric.Requests200 = metrics.NewCounter() requestMetric.Requests400 = metrics.NewCounter() requestMetric.Requests403 = metrics.NewCounter() requestMetric.Requests404 = metrics.NewCounter() requestMetric.Requests500 = metrics.NewCounter() requestMetric.Requests503 = metrics.NewCounter() requestMetric.Requests504 = metrics.NewCounter() requestMetric.Requests5xx = metrics.NewCounter() requestMetric.Requests4xx = metrics.NewCounter() } if c != nil && Graphite != nil { requestMetric.RequestsH = metrics.NewVSumHistogram(c.BucketsWidth, c.BucketsLabels).SetNameTotal("") requestMetric.FinderH = metrics.NewVSumHistogram(c.BucketsWidth, c.BucketsLabels).SetNameTotal("") metrics.Register(scope+".all.requests", requestMetric.RequestsH) metrics.Register(scope+".all.requests_finder", requestMetric.FinderH) metrics.Register(scope+".all.errors", requestMetric.Errors) if c.ExtendedStat { metrics.Register(scope+".all.requests_status_code.200", requestMetric.Requests200) metrics.Register(scope+".all.requests_status_code.400", requestMetric.Requests400) metrics.Register(scope+".all.requests_status_code.403", requestMetric.Requests403) metrics.Register(scope+".all.requests_status_code.404", requestMetric.Requests404) metrics.Register(scope+".all.requests_status_code.4xx", requestMetric.Requests4xx) metrics.Register(scope+".all.requests_status_code.500", requestMetric.Requests500) metrics.Register(scope+".all.requests_status_code.503", requestMetric.Requests503) metrics.Register(scope+".all.requests_status_code.504", requestMetric.Requests504) metrics.Register(scope+".all.requests_status_code.5xx", requestMetric.Requests5xx) } if len(c.RangeS) > 0 { requestMetric.RangeS = c.RangeS requestMetric.RangeNames = c.RangeNames requestMetric.RangeMetrics = make([]RenderMetric, len(c.RangeS)) for i := range c.RangeS { requestMetric.RangeMetrics[i].RequestsH = metrics.NewVSumHistogram(c.BucketsWidth, c.BucketsLabels).SetNameTotal("") requestMetric.RangeMetrics[i].FinderH = metrics.NewVSumHistogram(c.BucketsWidth, c.BucketsLabels).SetNameTotal("") requestMetric.RangeMetrics[i].Errors = metrics.NewCounter() requestMetric.RangeMetrics[i].MetricsCountName = scope + "." + requestMetric.RangeNames[i] + ".metrics" requestMetric.RangeMetrics[i].PointsCountName = scope + "." + requestMetric.RangeNames[i] + ".points" metrics.Register(scope+"."+c.RangeNames[i]+".requests", requestMetric.RangeMetrics[i].RequestsH) metrics.Register(scope+"."+c.RangeNames[i]+".requests_finder", requestMetric.RangeMetrics[i].FinderH) metrics.Register(scope+"."+c.RangeNames[i]+".errors", requestMetric.RangeMetrics[i].Errors) if c.ExtendedStat { requestMetric.RangeMetrics[i].Requests200 = metrics.NewCounter() requestMetric.RangeMetrics[i].Requests400 = metrics.NewCounter() requestMetric.RangeMetrics[i].Requests403 = metrics.NewCounter() requestMetric.RangeMetrics[i].Requests404 = metrics.NewCounter() requestMetric.RangeMetrics[i].Requests500 = metrics.NewCounter() requestMetric.RangeMetrics[i].Requests503 = metrics.NewCounter() requestMetric.RangeMetrics[i].Requests504 = metrics.NewCounter() requestMetric.RangeMetrics[i].Requests5xx = metrics.NewCounter() requestMetric.RangeMetrics[i].Requests4xx = metrics.NewCounter() metrics.Register(scope+"."+c.RangeNames[i]+".requests_status_code.200", requestMetric.RangeMetrics[i].Requests200) metrics.Register(scope+"."+c.RangeNames[i]+".requests_status_code.400", requestMetric.RangeMetrics[i].Requests400) metrics.Register(scope+"."+c.RangeNames[i]+".requests_status_code.403", requestMetric.RangeMetrics[i].Requests403) metrics.Register(scope+"."+c.RangeNames[i]+".requests_status_code.404", requestMetric.RangeMetrics[i].Requests404) metrics.Register(scope+"."+c.RangeNames[i]+".requests_status_code.4xx", requestMetric.RangeMetrics[i].Requests4xx) metrics.Register(scope+"."+c.RangeNames[i]+".requests_status_code.500", requestMetric.RangeMetrics[i].Requests500) metrics.Register(scope+"."+c.RangeNames[i]+".requests_status_code.503", requestMetric.RangeMetrics[i].Requests503) metrics.Register(scope+"."+c.RangeNames[i]+".requests_status_code.504", requestMetric.RangeMetrics[i].Requests504) metrics.Register(scope+"."+c.RangeNames[i]+".requests_status_code.5xx", requestMetric.RangeMetrics[i].Requests5xx) } } } } else { requestMetric.RequestsH = metrics.NilHistogram{} requestMetric.FinderH = metrics.NilHistogram{} } return requestMetric } func SendFindMetrics(r *FindMetrics, statusCode int, durationMs, untilFromS int64, extended bool, metricsCount int64) { fromPos := -1 if len(r.RangeS) > 0 { fromPos = metrics.SearchInt64Le(r.RangeS, untilFromS) } r.RequestsH.Add(durationMs) if fromPos >= 0 { r.RangeMetrics[fromPos].RequestsH.Add(durationMs) } switch statusCode { case 200: if extended { r.Requests200.Add(1) Gstatsd.Timing(r.MetricsCountName, metricsCount, 1.0) if fromPos >= 0 { r.RangeMetrics[fromPos].Requests200.Add(1) Gstatsd.Timing(r.RangeMetrics[fromPos].MetricsCountName, metricsCount, 1.0) } } case 400: r.Errors.Add(1) if extended { r.Requests400.Add(1) if fromPos >= 0 { r.RangeMetrics[fromPos].Requests400.Add(1) r.RangeMetrics[fromPos].Errors.Add(1) } } case 403: r.Errors.Add(1) if extended { r.Requests403.Add(1) if fromPos >= 0 { r.RangeMetrics[fromPos].Requests403.Add(1) r.RangeMetrics[fromPos].Errors.Add(1) } } case 404: if extended { r.Requests404.Add(1) Gstatsd.Timing(r.MetricsCountName, metricsCount, 1.0) if fromPos >= 0 { r.RangeMetrics[fromPos].Requests404.Add(1) Gstatsd.Timing(r.RangeMetrics[fromPos].MetricsCountName, metricsCount, 1.0) } } case 500: r.Errors.Add(1) if extended { r.Requests500.Add(1) if fromPos >= 0 { r.RangeMetrics[fromPos].Requests500.Add(1) r.RangeMetrics[fromPos].Errors.Add(1) } } case 503: r.Errors.Add(1) if extended { r.Requests503.Add(1) if fromPos >= 0 { r.RangeMetrics[fromPos].Requests503.Add(1) r.RangeMetrics[fromPos].Errors.Add(1) } } case 504: r.Errors.Add(1) if extended { r.Requests504.Add(1) if fromPos >= 0 { r.RangeMetrics[fromPos].Requests504.Add(1) r.RangeMetrics[fromPos].Errors.Add(1) } } default: if extended { if statusCode > 500 { r.Requests5xx.Add(1) if fromPos >= 0 { r.RangeMetrics[fromPos].Requests5xx.Add(1) r.RangeMetrics[fromPos].Errors.Add(1) } } else { r.Requests4xx.Add(1) if fromPos >= 0 { r.RangeMetrics[fromPos].Requests4xx.Add(1) r.RangeMetrics[fromPos].Errors.Add(1) } } } r.Errors.Add(1) } } func SendRenderMetrics(r *RenderMetrics, statusCode int, start, fetch, end time.Time, untilFromS int64, extended bool, metricsCount, points int64) { fromPos := -1 if len(r.RangeS) > 0 { fromPos = metrics.SearchInt64Le(r.RangeS, untilFromS) } startMs := start.UnixMilli() endMs := end.UnixMilli() var ( durFinderMs int64 durFetchMs int64 ) durMs := endMs - startMs if fetch.IsZero() { durFinderMs = durMs } else { fetchMs := fetch.UnixMilli() durFinderMs = fetchMs - startMs durFetchMs = endMs - fetchMs } r.RequestsH.Add(durMs) r.FinderH.Add(durFinderMs) if fromPos >= 0 { r.RangeMetrics[fromPos].RequestsH.Add(durMs) r.RangeMetrics[fromPos].FinderH.Add(durFinderMs) } switch statusCode { case 200: if extended { r.Requests200.Add(1) Gstatsd.Timing(r.MetricsCountName, metricsCount, 1.0) Gstatsd.Timing(r.PointsCountName, points, 1.0) if fromPos >= 0 { r.RangeMetrics[fromPos].Requests200.Add(1) Gstatsd.Timing(r.RangeMetrics[fromPos].MetricsCountName, metricsCount, 1.0) if durFetchMs > 0 { Gstatsd.Timing(r.RangeMetrics[fromPos].PointsCountName, points, 1.0) } r.RangeMetrics[fromPos].FinderH.Add(durFinderMs) } } case 400: r.Errors.Add(1) if extended { r.Requests400.Add(1) if fromPos >= 0 { r.RangeMetrics[fromPos].Requests400.Add(1) r.RangeMetrics[fromPos].Errors.Add(1) } } case 403: r.Errors.Add(1) if extended { r.Requests403.Add(1) if fromPos >= 0 { r.RangeMetrics[fromPos].Requests403.Add(1) r.RangeMetrics[fromPos].Errors.Add(1) } } case 404: if extended { r.Requests404.Add(1) Gstatsd.Timing(r.MetricsCountName, metricsCount, 1.0) if fromPos >= 0 { r.RangeMetrics[fromPos].Requests404.Add(1) Gstatsd.Timing(r.RangeMetrics[fromPos].MetricsCountName, metricsCount, 1.0) Gstatsd.Timing(r.RangeMetrics[fromPos].PointsCountName, points, 1.0) if durFetchMs > 0 { Gstatsd.Timing(r.RangeMetrics[fromPos].PointsCountName, points, 1.0) } r.RangeMetrics[fromPos].FinderH.Add(durFinderMs) } } case 500: r.Errors.Add(1) if extended { r.Requests500.Add(1) if fromPos >= 0 { r.RangeMetrics[fromPos].Requests500.Add(1) r.RangeMetrics[fromPos].Errors.Add(1) } } case 503: r.Errors.Add(1) if extended { r.Requests503.Add(1) if fromPos >= 0 { r.RangeMetrics[fromPos].Requests503.Add(1) r.RangeMetrics[fromPos].Errors.Add(1) } } case 504: r.Errors.Add(1) if extended { r.Requests504.Add(1) if fromPos >= 0 { r.RangeMetrics[fromPos].Requests504.Add(1) r.RangeMetrics[fromPos].Errors.Add(1) } } default: if extended { if statusCode > 500 { r.Requests5xx.Add(1) if fromPos >= 0 { r.RangeMetrics[fromPos].Requests5xx.Add(1) r.RangeMetrics[fromPos].Errors.Add(1) } } else { r.Requests4xx.Add(1) if fromPos >= 0 { r.RangeMetrics[fromPos].Requests4xx.Add(1) r.RangeMetrics[fromPos].Errors.Add(1) } } } r.Errors.Add(1) } } type rangeName struct { name string v int64 } func InitMetrics(c *Config, findWaitQueue, tagsWaitQueue bool) { if c != nil && Graphite != nil { metrics.RegisterRuntimeMemStats(nil) go metrics.CaptureRuntimeMemStats(c.MetricInterval) if len(c.BucketsWidth) == 0 { c.BucketsWidth = []int64{200, 500, 1000, 2000, 3000, 5000, 7000, 10000, 15000, 20000, 25000, 30000, 40000, 50000, 60000} } labels := make([]string, len(c.BucketsWidth)+1) for i := 0; i <= len(c.BucketsWidth); i++ { if i >= len(c.BucketsLabels) || c.BucketsLabels[i] == "" { if i < len(c.BucketsWidth) { labels[i] = fmt.Sprintf("_to_%dms", c.BucketsWidth[i]) } else { labels[i] = "_to_inf" } } else { labels[i] = c.BucketsLabels[i] } } c.BucketsLabels = labels if len(c.Ranges) > 0 { // c.RangeS = make([]int64, 0, len(c.Range)+1) untilFrom := make([]rangeName, 0, len(c.Ranges)+1) for name, v := range c.Ranges { if v <= 0 { untilFrom = append(untilFrom, rangeName{name: name, v: math.MaxInt64}) } else { untilFrom = append(untilFrom, rangeName{name: name, v: int64(v.Seconds())}) } } sort.Slice(untilFrom, func(i, j int) bool { return untilFrom[i].v < untilFrom[j].v }) if untilFrom[len(untilFrom)-1].v != math.MaxInt64 { untilFrom = append(untilFrom, rangeName{name: "history", v: math.MaxInt64}) } c.RangeS = make([]int64, len(untilFrom)) c.RangeNames = make([]string, len(untilFrom)) for i := range untilFrom { c.RangeNames[i] = untilFrom[i].name c.RangeS[i] = untilFrom[i].v } } if len(c.FindRanges) > 0 { // c.RangeS = make([]int64, 0, len(c.Range)+1) untilFrom := make([]rangeName, 0, len(c.Ranges)+1) for name, v := range c.FindRanges { if v <= 0 { untilFrom = append(untilFrom, rangeName{name: name, v: math.MaxInt64}) } else { untilFrom = append(untilFrom, rangeName{name: name, v: int64(v.Seconds())}) } } sort.Slice(untilFrom, func(i, j int) bool { return untilFrom[i].v < untilFrom[j].v }) if untilFrom[len(untilFrom)-1].v != math.MaxInt64 { untilFrom = append(untilFrom, rangeName{name: "history", v: math.MaxInt64}) } c.FindRangeS = make([]int64, len(untilFrom)) c.FindRangeNames = make([]string, len(untilFrom)) for i := range untilFrom { c.FindRangeNames[i] = untilFrom[i].name c.FindRangeS[i] = untilFrom[i].v } } } initFindCacheMetrics(c) FindRequestMetric = initFindMetrics("find", c, findWaitQueue) TagsRequestMetric = initFindMetrics("tags", c, tagsWaitQueue) RenderRequestMetric = initRenderMetrics("render", c) } func DisableMetrics() { metrics.UseNilMetrics = true InitMetrics(nil, false, false) } func UnregisterAll() { metrics.DefaultRegistry.UnregisterAll() } ================================================ FILE: metrics/metrics_test.go ================================================ package metrics import ( "math" "strconv" "testing" "time" "github.com/msaf1980/go-metrics" "github.com/msaf1980/go-metrics/graphite" "github.com/stretchr/testify/assert" ) func max(a, b int) int { if a > b { return a } return b } func compareInterface(t *testing.T, name string, i interface{}, notNil bool) { m := metrics.Get(name) if notNil { assert.Truef(t, i == m, name+"\nwant\n%+v\ngot\n%+v", i, m) } else { assert.Nilf(t, m, name) } } func TestInitMetrics(t *testing.T) { tests := []struct { name string c Config findWaitQueue bool tagsWaitQueue bool want Config wantFindCountName string wantFindRangesMetricsCountNames []string wantRenderMetricsCountName string wantRenderRangesMetricsCountNames []string wantRenderPointsCountName string wantRenderRangesPointsCountNames []string }{ { name: "labels (all)", c: Config{ MetricEndpoint: "127.0.0.1:2003", MetricInterval: 10 * time.Second, MetricTimeout: time.Second, MetricPrefix: "graphite", BucketsWidth: []int64{200, 500, 1000, 2000, 3000}, BucketsLabels: []string{ "_to_200ms", "_to_500ms", "_to_1000ms", "_to_2000ms", "_to_3000ms", "_to_last", }, }, want: Config{ MetricEndpoint: "127.0.0.1:2003", MetricInterval: 10 * time.Second, MetricTimeout: time.Second, MetricPrefix: "graphite", BucketsWidth: []int64{200, 500, 1000, 2000, 3000}, BucketsLabels: []string{ "_to_200ms", "_to_500ms", "_to_1000ms", "_to_2000ms", "_to_3000ms", "_to_last", }, }, wantFindCountName: "find.all.metrics", wantRenderMetricsCountName: "render.all.metrics", wantRenderPointsCountName: "render.all.points", }, { name: "labels (part)", c: Config{ MetricEndpoint: "127.0.0.1:2003", MetricInterval: 10 * time.Second, MetricTimeout: time.Second, MetricPrefix: "graphite", BucketsWidth: []int64{200, 500, 1000, 2000, 3000}, BucketsLabels: []string{ "_to_200ms", "_to_500ms", "_to_1000ms", "_to_2000ms", "_to_3000ms", }, }, want: Config{ MetricEndpoint: "127.0.0.1:2003", MetricInterval: 10 * time.Second, MetricTimeout: time.Second, MetricPrefix: "graphite", BucketsWidth: []int64{200, 500, 1000, 2000, 3000}, BucketsLabels: []string{ "_to_200ms", "_to_500ms", "_to_1000ms", "_to_2000ms", "_to_3000ms", "_to_inf", }, }, wantFindCountName: "find.all.metrics", wantRenderMetricsCountName: "render.all.metrics", wantRenderPointsCountName: "render.all.points", }, { name: "labels (default)", c: Config{ MetricEndpoint: "127.0.0.1:2003", MetricInterval: 10 * time.Second, MetricTimeout: time.Second, MetricPrefix: "graphite", Ranges: map[string]time.Duration{ "1h": time.Hour, "3d": 72 * time.Hour, "7d": 168 * time.Hour, "30d": 720 * time.Hour, "90d": 2160 * time.Hour, }, }, want: Config{ MetricEndpoint: "127.0.0.1:2003", MetricInterval: 10 * time.Second, MetricTimeout: time.Second, MetricPrefix: "graphite", BucketsWidth: []int64{200, 500, 1000, 2000, 3000, 5000, 7000, 10000, 15000, 20000, 25000, 30000, 40000, 50000, 60000}, BucketsLabels: []string{ "_to_200ms", "_to_500ms", "_to_1000ms", "_to_2000ms", "_to_3000ms", "_to_5000ms", "_to_7000ms", "_to_10000ms", "_to_15000ms", "_to_20000ms", "_to_25000ms", "_to_30000ms", "_to_40000ms", "_to_50000ms", "_to_60000ms", "_to_inf", }, // until-from = { "1h" = "1h", "3d" = "72h", "7d" = "168h", "30d" = "720h", "90d" = "2160h" } Ranges: map[string]time.Duration{ "1h": time.Hour, "3d": 72 * time.Hour, "7d": 168 * time.Hour, "30d": 720 * time.Hour, "90d": 2160 * time.Hour, }, RangeNames: []string{"1h", "3d", "7d", "30d", "90d", "history"}, RangeS: []int64{3600, 259200, 604800, 2592000, 7776000, math.MaxInt64}, }, wantFindCountName: "find.all.metrics", wantRenderMetricsCountName: "render.all.metrics", wantRenderRangesMetricsCountNames: []string{ "render.1h.metrics", "render.3d.metrics", "render.7d.metrics", "render.30d.metrics", "render.90d.metrics", "render.history.metrics", }, wantRenderPointsCountName: "render.all.points", wantRenderRangesPointsCountNames: []string{ "render.1h.points", "render.3d.points", "render.7d.points", "render.30d.points", "render.90d.points", "render.history.points", }, }, { name: "ranges", c: Config{ MetricEndpoint: "127.0.0.1:2003", MetricInterval: 10 * time.Second, MetricTimeout: time.Second, MetricPrefix: "graphite", Ranges: map[string]time.Duration{ "1h": time.Hour, "3d": 72 * time.Hour, "7d": 168 * time.Hour, "30d": 720 * time.Hour, "90d": 2160 * time.Hour, "last": 0, }, }, want: Config{ MetricEndpoint: "127.0.0.1:2003", MetricInterval: 10 * time.Second, MetricTimeout: time.Second, MetricPrefix: "graphite", BucketsWidth: []int64{200, 500, 1000, 2000, 3000, 5000, 7000, 10000, 15000, 20000, 25000, 30000, 40000, 50000, 60000}, BucketsLabels: []string{ "_to_200ms", "_to_500ms", "_to_1000ms", "_to_2000ms", "_to_3000ms", "_to_5000ms", "_to_7000ms", "_to_10000ms", "_to_15000ms", "_to_20000ms", "_to_25000ms", "_to_30000ms", "_to_40000ms", "_to_50000ms", "_to_60000ms", "_to_inf", }, // until-from = { "1h" = "1h", "3d" = "72h", "7d" = "168h", "30d" = "720h", "90d" = "2160h" } Ranges: map[string]time.Duration{ "1h": time.Hour, "3d": 72 * time.Hour, "7d": 168 * time.Hour, "30d": 720 * time.Hour, "90d": 2160 * time.Hour, "last": 0, }, RangeNames: []string{"1h", "3d", "7d", "30d", "90d", "last"}, RangeS: []int64{3600, 259200, 604800, 2592000, 7776000, math.MaxInt64}, }, wantFindCountName: "find.all.metrics", wantRenderMetricsCountName: "render.all.metrics", wantRenderRangesMetricsCountNames: []string{ "render.1h.metrics", "render.3d.metrics", "render.7d.metrics", "render.30d.metrics", "render.90d.metrics", "render.last.metrics", }, wantRenderPointsCountName: "render.all.points", wantRenderRangesPointsCountNames: []string{ "render.1h.points", "render.3d.points", "render.7d.points", "render.30d.points", "render.90d.points", "render.last.points", }, }, { name: "all", c: Config{ MetricEndpoint: "127.0.0.1:2003", Statsd: "127.0.0.1:8125", MetricInterval: 10 * time.Second, MetricTimeout: time.Second, MetricPrefix: "graphite", ExtendedStat: true, Ranges: map[string]time.Duration{ "1h": time.Hour, "3d": 72 * time.Hour, "7d": 168 * time.Hour, "30d": 720 * time.Hour, "90d": 2160 * time.Hour, "last": 0, }, }, want: Config{ MetricEndpoint: "127.0.0.1:2003", Statsd: "127.0.0.1:8125", MetricInterval: 10 * time.Second, MetricTimeout: time.Second, MetricPrefix: "graphite", ExtendedStat: true, BucketsWidth: []int64{200, 500, 1000, 2000, 3000, 5000, 7000, 10000, 15000, 20000, 25000, 30000, 40000, 50000, 60000}, BucketsLabels: []string{ "_to_200ms", "_to_500ms", "_to_1000ms", "_to_2000ms", "_to_3000ms", "_to_5000ms", "_to_7000ms", "_to_10000ms", "_to_15000ms", "_to_20000ms", "_to_25000ms", "_to_30000ms", "_to_40000ms", "_to_50000ms", "_to_60000ms", "_to_inf", }, // until-from = { "1h" = "1h", "3d" = "72h", "7d" = "168h", "30d" = "720h", "90d" = "2160h" } Ranges: map[string]time.Duration{ "1h": time.Hour, "3d": 72 * time.Hour, "7d": 168 * time.Hour, "30d": 720 * time.Hour, "90d": 2160 * time.Hour, "last": 0, }, RangeNames: []string{"1h", "3d", "7d", "30d", "90d", "last"}, RangeS: []int64{3600, 259200, 604800, 2592000, 7776000, math.MaxInt64}, }, wantFindCountName: "find.all.metrics", wantRenderMetricsCountName: "render.all.metrics", wantRenderRangesMetricsCountNames: []string{ "render.1h.metrics", "render.3d.metrics", "render.7d.metrics", "render.30d.metrics", "render.90d.metrics", "render.last.metrics", }, wantRenderPointsCountName: "render.all.points", wantRenderRangesPointsCountNames: []string{ "render.1h.points", "render.3d.points", "render.7d.points", "render.30d.points", "render.90d.points", "render.last.points", }, }, { name: "all (with find-ranges)", c: Config{ MetricEndpoint: "127.0.0.1:2003", Statsd: "127.0.0.1:8125", MetricInterval: 10 * time.Second, MetricTimeout: time.Second, MetricPrefix: "graphite", ExtendedStat: true, Ranges: map[string]time.Duration{ "1h": time.Hour, "3d": 72 * time.Hour, "7d": 168 * time.Hour, "30d": 720 * time.Hour, "90d": 2160 * time.Hour, "last": 0, }, FindRanges: map[string]time.Duration{ "1h": time.Hour, "3d": 72 * time.Hour, "7d": 168 * time.Hour, "30d": 720 * time.Hour, "last": 0, }, FindRangeNames: []string{"1h", "3d", "7d", "30d", "last"}, FindRangeS: []int64{3600, 259200, 604800, 2592000, math.MaxInt64}, }, want: Config{ MetricEndpoint: "127.0.0.1:2003", Statsd: "127.0.0.1:8125", MetricInterval: 10 * time.Second, MetricTimeout: time.Second, MetricPrefix: "graphite", ExtendedStat: true, BucketsWidth: []int64{200, 500, 1000, 2000, 3000, 5000, 7000, 10000, 15000, 20000, 25000, 30000, 40000, 50000, 60000}, BucketsLabels: []string{ "_to_200ms", "_to_500ms", "_to_1000ms", "_to_2000ms", "_to_3000ms", "_to_5000ms", "_to_7000ms", "_to_10000ms", "_to_15000ms", "_to_20000ms", "_to_25000ms", "_to_30000ms", "_to_40000ms", "_to_50000ms", "_to_60000ms", "_to_inf", }, // until-from = { "1h" = "1h", "3d" = "72h", "7d" = "168h", "30d" = "720h", "90d" = "2160h" } Ranges: map[string]time.Duration{ "1h": time.Hour, "3d": 72 * time.Hour, "7d": 168 * time.Hour, "30d": 720 * time.Hour, "90d": 2160 * time.Hour, "last": 0, }, RangeNames: []string{"1h", "3d", "7d", "30d", "90d", "last"}, RangeS: []int64{3600, 259200, 604800, 2592000, 7776000, math.MaxInt64}, FindRanges: map[string]time.Duration{ "1h": time.Hour, "3d": 72 * time.Hour, "7d": 168 * time.Hour, "30d": 720 * time.Hour, "last": 0, }, FindRangeNames: []string{"1h", "3d", "7d", "30d", "last"}, FindRangeS: []int64{3600, 259200, 604800, 2592000, math.MaxInt64}, }, wantFindCountName: "find.all.metrics", wantFindRangesMetricsCountNames: []string{ "find.1h.metrics", "find.3d.metrics", "find.7d.metrics", "find.30d.metrics", "find.last.metrics", }, wantRenderMetricsCountName: "render.all.metrics", wantRenderRangesMetricsCountNames: []string{ "render.1h.metrics", "render.3d.metrics", "render.7d.metrics", "render.30d.metrics", "render.90d.metrics", "render.last.metrics", }, wantRenderPointsCountName: "render.all.points", wantRenderRangesPointsCountNames: []string{ "render.1h.points", "render.3d.points", "render.7d.points", "render.30d.points", "render.90d.points", "render.last.points", }, }, } for n, tt := range tests { t.Run(tt.name+"#"+strconv.Itoa(n), func(t *testing.T) { FindRequestMetric = nil TagsRequestMetric = nil RenderRequestMetric = nil UnregisterAll() c := tt.c Graphite = &graphite.Graphite{} InitMetrics(&c, tt.findWaitQueue, tt.tagsWaitQueue) Graphite = nil assert.Equal(t, tt.want, c) // FindRequestH compareInterface(t, "find.all.requests", FindRequestMetric.RequestsH, true) // FindRequestCount compareInterface(t, "find.all.requests_status_code.200", FindRequestMetric.Requests200, c.ExtendedStat) compareInterface(t, "find.all.requests_status_code.400", FindRequestMetric.Requests400, c.ExtendedStat) compareInterface(t, "find.all.requests_status_code.403", FindRequestMetric.Requests403, c.ExtendedStat) compareInterface(t, "find.all.requests_status_code.404", FindRequestMetric.Requests404, c.ExtendedStat) compareInterface(t, "find.all.requests_status_code.4xx", FindRequestMetric.Requests4xx, c.ExtendedStat) compareInterface(t, "find.all.requests_status_code.500", FindRequestMetric.Requests500, c.ExtendedStat) compareInterface(t, "find.all.requests_status_code.503", FindRequestMetric.Requests503, c.ExtendedStat) compareInterface(t, "find.all.requests_status_code.504", FindRequestMetric.Requests504, c.ExtendedStat) compareInterface(t, "find.all.requests_status_code.5xx", FindRequestMetric.Requests5xx, c.ExtendedStat) //FindRequestMetric assert.Equal(t, tt.wantFindCountName, FindRequestMetric.MetricsCountName) for i := 0; i < max(len(c.FindRangeS), len(tt.wantFindRangesMetricsCountNames)); i++ { if i < len(c.FindRangeNames) { // FindRequestH compareInterface(t, "find."+c.FindRangeNames[i]+".requests", FindRequestMetric.RangeMetrics[i].RequestsH, true) // FindRequestCount compareInterface(t, "find."+c.FindRangeNames[i]+".requests_status_code.200", FindRequestMetric.RangeMetrics[i].Requests200, c.ExtendedStat) compareInterface(t, "find."+c.FindRangeNames[i]+".requests_status_code.400", FindRequestMetric.RangeMetrics[i].Requests400, c.ExtendedStat) compareInterface(t, "find."+c.FindRangeNames[i]+".requests_status_code.403", FindRequestMetric.RangeMetrics[i].Requests403, c.ExtendedStat) compareInterface(t, "find."+c.FindRangeNames[i]+".requests_status_code.404", FindRequestMetric.RangeMetrics[i].Requests404, c.ExtendedStat) compareInterface(t, "find."+c.FindRangeNames[i]+".requests_status_code.4xx", FindRequestMetric.RangeMetrics[i].Requests4xx, c.ExtendedStat) compareInterface(t, "find."+c.FindRangeNames[i]+".requests_status_code.500", FindRequestMetric.RangeMetrics[i].Requests500, c.ExtendedStat) compareInterface(t, "find."+c.FindRangeNames[i]+".requests_status_code.503", FindRequestMetric.RangeMetrics[i].Requests503, c.ExtendedStat) compareInterface(t, "find."+c.FindRangeNames[i]+".requests_status_code.504", FindRequestMetric.RangeMetrics[i].Requests504, c.ExtendedStat) compareInterface(t, "find."+c.FindRangeNames[i]+".requests_status_code.5xx", FindRequestMetric.RangeMetrics[i].Requests5xx, c.ExtendedStat) } var want, got string if i < len(tt.wantFindRangesMetricsCountNames) { want = tt.wantFindRangesMetricsCountNames[i] } if i < len(FindRequestMetric.RangeMetrics) { got = FindRequestMetric.RangeMetrics[i].MetricsCountName } assert.Equal(t, want, got) } assert.Equal(t, tt.want.FindRangeS, FindRequestMetric.RangeS) assert.Equal(t, tt.want.FindRangeNames, FindRequestMetric.RangeNames) assert.Equalf(t, len(tt.want.FindRangeS), len(FindRequestMetric.RangeMetrics), "FindRequestMetric.RangeMetrics") // RenderRequestH compareInterface(t, "render.all.requests", RenderRequestMetric.RequestsH, true) compareInterface(t, "render.all.requests_finder", RenderRequestMetric.FinderH, true) // RenderRequestCount compareInterface(t, "render.all.requests_status_code.200", RenderRequestMetric.Requests200, c.ExtendedStat) compareInterface(t, "render.all.requests_status_code.400", RenderRequestMetric.Requests400, c.ExtendedStat) compareInterface(t, "render.all.requests_status_code.403", RenderRequestMetric.Requests403, c.ExtendedStat) compareInterface(t, "render.all.requests_status_code.404", RenderRequestMetric.Requests404, c.ExtendedStat) compareInterface(t, "render.all.requests_status_code.4xx", RenderRequestMetric.Requests4xx, c.ExtendedStat) compareInterface(t, "render.all.requests_status_code.500", RenderRequestMetric.Requests500, c.ExtendedStat) compareInterface(t, "render.all.requests_status_code.503", RenderRequestMetric.Requests503, c.ExtendedStat) compareInterface(t, "render.all.requests_status_code.504", RenderRequestMetric.Requests504, c.ExtendedStat) compareInterface(t, "render.all.requests_status_code.5xx", RenderRequestMetric.Requests5xx, c.ExtendedStat) // RenderRequestMetric assert.Equal(t, tt.wantRenderMetricsCountName, RenderRequestMetric.MetricsCountName) assert.Equal(t, tt.wantRenderPointsCountName, RenderRequestMetric.PointsCountName) for i := 0; i < max(len(c.RangeS), len(tt.wantRenderRangesMetricsCountNames)); i++ { var want, got string if i < len(c.RangeNames) { // FindRequestH compareInterface(t, "render."+c.RangeNames[i]+".requests", RenderRequestMetric.RangeMetrics[i].RequestsH, true) compareInterface(t, "render."+c.RangeNames[i]+".requests_finder", RenderRequestMetric.RangeMetrics[i].FinderH, true) // FindRequestCount compareInterface(t, "render."+c.RangeNames[i]+".requests_status_code.200", RenderRequestMetric.RangeMetrics[i].Requests200, c.ExtendedStat) compareInterface(t, "render."+c.RangeNames[i]+".requests_status_code.400", RenderRequestMetric.RangeMetrics[i].Requests400, c.ExtendedStat) compareInterface(t, "render."+c.RangeNames[i]+".requests_status_code.403", RenderRequestMetric.RangeMetrics[i].Requests403, c.ExtendedStat) compareInterface(t, "render."+c.RangeNames[i]+".requests_status_code.404", RenderRequestMetric.RangeMetrics[i].Requests404, c.ExtendedStat) compareInterface(t, "render."+c.RangeNames[i]+".requests_status_code.4xx", RenderRequestMetric.RangeMetrics[i].Requests4xx, c.ExtendedStat) compareInterface(t, "render."+c.RangeNames[i]+".requests_status_code.500", RenderRequestMetric.RangeMetrics[i].Requests500, c.ExtendedStat) compareInterface(t, "render."+c.RangeNames[i]+".requests_status_code.503", RenderRequestMetric.RangeMetrics[i].Requests503, c.ExtendedStat) compareInterface(t, "render."+c.RangeNames[i]+".requests_status_code.504", RenderRequestMetric.RangeMetrics[i].Requests504, c.ExtendedStat) compareInterface(t, "render."+c.RangeNames[i]+".requests_status_code.5xx", RenderRequestMetric.RangeMetrics[i].Requests5xx, c.ExtendedStat) } if i < len(tt.wantRenderRangesMetricsCountNames) { want = tt.wantRenderRangesMetricsCountNames[i] } if i < len(RenderRequestMetric.RangeMetrics) { got = RenderRequestMetric.RangeMetrics[i].MetricsCountName } assert.Equalf(t, want, got, strconv.Itoa(i)) if i < len(tt.wantRenderRangesPointsCountNames) { want = tt.wantRenderRangesPointsCountNames[i] } if i < len(tt.wantRenderRangesPointsCountNames) { got = RenderRequestMetric.RangeMetrics[i].PointsCountName } assert.Equalf(t, want, got, strconv.Itoa(i)) } assert.Equal(t, tt.want.RangeS, RenderRequestMetric.RangeS) assert.Equal(t, tt.want.RangeNames, RenderRequestMetric.RangeNames) assert.Equalf(t, len(tt.want.RangeS), len(RenderRequestMetric.RangeMetrics), "RenderRequestMetric.RangeMetrics") // cleanup global vars FindRequestMetric = nil RenderRequestMetric = nil UnregisterAll() }) } } ================================================ FILE: metrics/query_metrics.go ================================================ package metrics import "github.com/msaf1980/go-metrics" type QueryMetric struct { RequestsH metrics.Histogram Errors metrics.Counter ReadRowsName string ReadBytesName string ChReadRowsName string ChReadBytesName string } type QueryMetrics struct { QueryMetric RangeNames []string RangeS []int64 RangeMetrics []QueryMetric } type FinderStat struct { ReadBytes int64 ChReadRows int64 ChReadBytes int64 Table string } var ( QMetrics map[string]*QueryMetrics = make(map[string]*QueryMetrics) AutocompleteQMetric *QueryMetrics FindQMetric *QueryMetrics ) func InitQueryMetrics(table string, c *Config) *QueryMetrics { if table == "" { table = "default" } if q, exist := QMetrics[table]; exist { return q } queryMetric := &QueryMetrics{ QueryMetric: QueryMetric{ Errors: metrics.NewCounter(), ReadRowsName: "query." + table + ".all.read_rows", ReadBytesName: "query." + table + ".all.read_bytes", ChReadRowsName: "query." + table + ".all.ch_read_rows", ChReadBytesName: "query." + table + ".all.ch_read_bytes", }, } if c != nil && Graphite != nil { queryMetric.RequestsH = metrics.NewVSumHistogram(c.BucketsWidth, c.BucketsLabels).SetNameTotal("") metrics.Register("query."+table+".all.requests", queryMetric.RequestsH) metrics.Register("query."+table+".all.errors", queryMetric.Errors) if len(c.RangeS) > 0 { queryMetric.RangeS = c.RangeS queryMetric.RangeNames = c.RangeNames queryMetric.RangeMetrics = make([]QueryMetric, len(c.RangeS)) for i := range c.RangeS { queryMetric.RangeMetrics[i].RequestsH = metrics.NewVSumHistogram(c.BucketsWidth, c.BucketsLabels).SetNameTotal("") metrics.Register("query."+table+"."+queryMetric.RangeNames[i]+".requests", queryMetric.RangeMetrics[i].RequestsH) queryMetric.RangeMetrics[i].Errors = metrics.NewCounter() metrics.Register("query."+table+"."+queryMetric.RangeNames[i]+".errors", queryMetric.RangeMetrics[i].Errors) queryMetric.RangeMetrics[i].ReadRowsName = "query." + table + "." + queryMetric.RangeNames[i] + ".read_rows" queryMetric.RangeMetrics[i].ReadBytesName = "query." + table + "." + queryMetric.RangeNames[i] + ".read_bytes" queryMetric.RangeMetrics[i].ChReadRowsName = "query." + table + "." + queryMetric.RangeNames[i] + ".ch_read_rows" queryMetric.RangeMetrics[i].ChReadBytesName = "query." + table + "." + queryMetric.RangeNames[i] + ".ch_read_bytes" } } } else { queryMetric.RequestsH = metrics.NilHistogram{} } QMetrics[table] = queryMetric return queryMetric } func SendQueryRead(r *QueryMetrics, from, until, durationMs, read_rows, read_bytes, ch_read_rows, ch_read_bytes int64, err bool) { r.RequestsH.Add(durationMs) if ch_read_rows > 0 { Gstatsd.Timing(r.ChReadBytesName, ch_read_bytes, 1.0) Gstatsd.Timing(r.ChReadRowsName, ch_read_rows, 1.0) } if err { r.Errors.Add(1) } else { Gstatsd.Timing(r.ReadBytesName, read_bytes, 1.0) Gstatsd.Timing(r.ReadRowsName, read_rows, 1.0) } if len(r.RangeS) > 0 { fromPos := metrics.SearchInt64Le(r.RangeS, until-from) r.RangeMetrics[fromPos].RequestsH.Add(durationMs) if ch_read_rows > 0 { Gstatsd.Timing(r.RangeMetrics[fromPos].ChReadBytesName, ch_read_bytes, 1.0) Gstatsd.Timing(r.RangeMetrics[fromPos].ChReadRowsName, ch_read_rows, 1.0) } if err { r.RangeMetrics[fromPos].Errors.Add(1) } else { Gstatsd.Timing(r.RangeMetrics[fromPos].ReadBytesName, read_bytes, 1.0) Gstatsd.Timing(r.RangeMetrics[fromPos].ReadRowsName, read_rows, 1.0) } } } func SendQueryReadChecked(r *QueryMetrics, from, until, durationMs, read_rows, read_bytes, ch_read_rows, ch_read_bytes int64, err bool) { if r != nil { SendQueryRead(r, from, until, durationMs, read_rows, read_bytes, ch_read_rows, ch_read_bytes, err) } } func SendQueryReadByTable(from, until, durationMs, read_rows int64, stats []FinderStat, err bool) { for _, stat := range stats { if r, ok := QMetrics[stat.Table]; ok { SendQueryRead(r, from, until, durationMs, read_rows, stat.ReadBytes, stat.ChReadRows, stat.ChReadBytes, err) } } } ================================================ FILE: metrics/statsd.go ================================================ package metrics import ( "time" "github.com/cactus/go-statsd-client/v5/statsd" ) // NullSender is disabled sender (if stat need to be disabled) type NullSender struct{} func (NullSender) Inc(string, int64, float32, ...statsd.Tag) error { return nil } func (NullSender) Dec(string, int64, float32, ...statsd.Tag) error { return nil } func (NullSender) Gauge(string, int64, float32, ...statsd.Tag) error { return nil } func (NullSender) GaugeDelta(string, int64, float32, ...statsd.Tag) error { return nil } func (NullSender) Timing(string, int64, float32, ...statsd.Tag) error { return nil } func (NullSender) TimingDuration(string, time.Duration, float32, ...statsd.Tag) error { return nil } func (NullSender) Set(string, string, float32, ...statsd.Tag) error { return nil } func (NullSender) SetInt(string, int64, float32, ...statsd.Tag) error { return nil } func (NullSender) Raw(string, string, float32, ...statsd.Tag) error { return nil } func (NullSender) NewSubStatter(string) statsd.SubStatter { return NullSender{} } func (NullSender) SetPrefix(string) {} func (NullSender) SetSamplerFunc(statsd.SamplerFunc) {} func (NullSender) Close() error { return nil } var Gstatsd statsd.Statter = NullSender{} ================================================ FILE: nfpm.yaml ================================================ --- name: ${NAME} description: ${DESCRIPTION} # Common packages config arch: "${ARCH}" # amd64, arm64 platform: "linux" version: "${VERSION_STRING}" maintainer: &m "Roman Lomonosov " vendor: *m homepage: "https://github.com/go-graphite/${NAME}" license: "MIT" section: "admin" priority: "optional" contents: - src: deploy/root/usr/ dst: /usr expand: true - src: deploy/root/etc/logrotate.d/${NAME} dst: /etc/logrotate.d/${NAME} type: config|noreplace expand: true - src: out/root/etc/${NAME}/${NAME}.conf dst: /etc/${NAME}/${NAME}.conf type: config|noreplace expand: true - src: "out/${NAME}-linux-${ARCH}" dst: /usr/bin/${NAME} expand: true # docs - src: LICENSE dst: /usr/share/doc/${NAME}/LICENSE expand: true ================================================ FILE: packages.sh ================================================ #!/bin/sh -e cd "$( dirname "$0" )" ROOT=$PWD docker run -i -e "DEVEL=${DEVEL:-0}" --rm -v "$ROOT:/root/go/src/github.com/lomik/graphite-clickhouse" golang bash -e << 'EOF' cd /root/ export TZ=Europe/Moscow ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.40.0 cd /root/go/src/github.com/lomik/graphite-clickhouse # go reads the VCS state git config --global --add safe.directory "$PWD" make nfpm-deb nfpm-rpm chmod -R a+w *.deb *.rpm out/ EOF ================================================ FILE: pkg/alias/map.go ================================================ package alias import ( "bytes" "sync" "github.com/lomik/graphite-clickhouse/finder" "github.com/lomik/graphite-clickhouse/pkg/reverse" ) // Value of Map type Value struct { Target string DisplayName string } // Map from real metric name to display name and target type Map struct { data map[string][]Value lock sync.RWMutex } // New returns new Map func New() *Map { return &Map{ data: make(map[string][]Value), lock: sync.RWMutex{}, } } // Merge data from finder.Result into aliases map func (m *Map) Merge(r finder.Result, useCache bool) { m.MergeTarget(r, "", useCache) } // MergeTarget data from finder.Result into aliases map func (m *Map) MergeTarget(r finder.Result, target string, saveCache bool) []byte { var buf bytes.Buffer series := r.Series() buf.Grow(len(series) * 24) for i := 0; i < len(series); i++ { if saveCache { buf.Write(series[i]) buf.WriteByte('\n') } key := string(series[i]) if len(key) == 0 { continue } abs := string(r.Abs(series[i])) m.lock.Lock() if x, ok := m.data[key]; ok { m.data[key] = append(x, Value{Target: target, DisplayName: abs}) } else { m.data[key] = []Value{{Target: target, DisplayName: abs}} } m.lock.Unlock() } if saveCache { return buf.Bytes() } else { return nil } } // Len returns count of keys func (m *Map) Len() int { m.lock.RLock() defer m.lock.RUnlock() return len(m.data) } // Size returns count of values func (m *Map) Size() int { s := 0 m.lock.RLock() defer m.lock.RUnlock() for _, v := range m.data { s += len(v) } return s } // Series returns keys of aliases map func (m *Map) Series(isReverse bool) []string { series := make([]string, 0, m.Len()) for k := range m.data { if isReverse { series = append(series, reverse.String(k)) } else { series = append(series, k) } } return series } // DisplayNames returns DisplayName from all Values func (m *Map) DisplayNames() []string { dn := make([]string, 0, m.Size()) for _, v := range m.data { for _, a := range v { dn = append(dn, a.DisplayName) } } return dn } // Get returns aliases for metric func (m *Map) Get(metric string) []Value { return m.data[metric] } ================================================ FILE: pkg/alias/map_tagged_test.go ================================================ package alias import ( "sort" "sync" "testing" "github.com/lomik/graphite-clickhouse/finder" "github.com/stretchr/testify/assert" ) var taggedResult *finder.MockFinder = finder.NewMockTagged([][]byte{ []byte("cpu.loadavg?env=test&host=host1"), []byte("cpu.loadavg?env=production&host=dc-host2"), []byte("cpu.loadavg?env=staging&host=stg-host3"), }) var taggedTarget string = "seriesByTag('name=cpu.loadavg)" func createAMTagged() *Map { am := New() am.MergeTarget(taggedResult, taggedTarget, false) return am } func TestCreationTagged(t *testing.T) { am := createAMTagged() for _, m := range taggedResult.List() { metric := string(m) v, ok := am.data[metric] assert.True(t, ok, "metric %m is not found in Map", metric) assert.Equal(t, taggedTarget, v[0].Target) // convert cpu.loadavg?env=test&host=host1 to cpu.loadavg;env=test;host=host1 assert.Equal(t, string(finder.TaggedDecode(m)), v[0].DisplayName) } } func TestAsyncMergeTagged(t *testing.T) { testEnvResult := [][]byte{ []byte("cpu.loadavg?env=test&host=host1"), []byte("cpu.loadavg?env=test&host=host2"), } targetTest := "seriesByTag('name=cpu.loadavg', 'env=test')" prodEnvResult := [][]byte{ []byte("cpu.loadavg?env=production&host=dc-host3"), []byte("cpu.loadavg?env=production&host=dc-host4"), } targetProd := "seriesByTag('name=cpu.loadavg', 'env=prod')" am := New() wg := sync.WaitGroup{} wg.Add(2) go func() { am.MergeTarget(finder.NewMockTagged(testEnvResult), targetTest, false) wg.Done() }() go func() { am.MergeTarget(finder.NewMockTagged(prodEnvResult), targetProd, false) wg.Done() }() resultAM := &Map{ data: map[string][]Value{ "cpu.loadavg?env=test&host=host1": { {Target: "seriesByTag('name=cpu.loadavg', 'env=test')", DisplayName: "cpu.loadavg;env=test;host=host1"}, }, "cpu.loadavg?env=test&host=host2": { {Target: "seriesByTag('name=cpu.loadavg', 'env=test')", DisplayName: "cpu.loadavg;env=test;host=host2"}, }, "cpu.loadavg?env=production&host=dc-host3": { {Target: "seriesByTag('name=cpu.loadavg', 'env=prod')", DisplayName: "cpu.loadavg;env=production;host=dc-host3"}, }, "cpu.loadavg?env=production&host=dc-host4": { {Target: "seriesByTag('name=cpu.loadavg', 'env=prod')", DisplayName: "cpu.loadavg;env=production;host=dc-host4"}, }, }, } wg.Wait() if !assert.Equal(t, resultAM.Len(), am.Len()) { t.FailNow() } for i := range am.data { var dv Values = am.data[i] sort.Sort(&dv) am.data[i] = dv } assert.Equal(t, resultAM, am) } func Benchmark_MergeTargetTagged(b *testing.B) { result := [][]byte{ []byte("cpu.loadavg?env=test&host=host1"), []byte("cpu.loadavg?env=production&host=dc-host2"), } for i := 0; i < b.N; i++ { am := createAM() am.MergeTarget(finder.NewMockTagged(result), taggedTarget, false) _ = am } } ================================================ FILE: pkg/alias/map_test.go ================================================ package alias import ( "sort" "sync" "testing" "github.com/lomik/graphite-clickhouse/finder" "github.com/stretchr/testify/assert" ) type Values []Value func (v *Values) Len() int { return len(*v) } func (v *Values) Less(i, j int) bool { return (*v)[i].Target < (*v)[j].Target } func (v *Values) Swap(i, j int) { vp := *v vp[i], vp[j] = vp[j], vp[i] } var finderResult *finder.MockFinder = finder.NewMockFinder([][]byte{ []byte("5_sec.name.max"), []byte("1_min.name.avg"), []byte("5_min.name.min"), []byte("10_min.name.any"), // defaults will be used }) var findTarget string = "*.name.*" func createAM() *Map { am := New() am.MergeTarget(finderResult, findTarget, false) return am } func TestCreation(t *testing.T) { am := createAM() for _, m := range finderResult.List() { metric := string(m) v, ok := am.data[metric] assert.True(t, ok, "metric %m is not found in Map", metric) assert.Equal(t, findTarget, v[0].Target) assert.Equal(t, metric, v[0].DisplayName) } } func TestAsyncMerge(t *testing.T) { am := New() target2 := "5*.name.*" wg := sync.WaitGroup{} wg.Add(2) go func() { am.MergeTarget(finderResult, findTarget, false) wg.Done() }() go func() { result := [][]byte{ []byte("5_sec.name.max"), []byte("5_min.name.avg"), []byte("5_min.name.min"), } am.MergeTarget(finder.NewMockFinder(result), target2, false) wg.Done() }() resultAM := &Map{ data: map[string][]Value{ "5_sec.name.max": { {Target: "*.name.*", DisplayName: "5_sec.name.max"}, {Target: "5*.name.*", DisplayName: "5_sec.name.max"}, }, "1_min.name.avg": { {Target: "*.name.*", DisplayName: "1_min.name.avg"}, }, "5_min.name.min": { {Target: "*.name.*", DisplayName: "5_min.name.min"}, {Target: "5*.name.*", DisplayName: "5_min.name.min"}, }, "10_min.name.any": { {Target: "*.name.*", DisplayName: "10_min.name.any"}, }, "5_min.name.avg": { {Target: "5*.name.*", DisplayName: "5_min.name.avg"}, }, }, } wg.Wait() if !assert.Equal(t, resultAM.Len(), am.Len()) { t.FailNow() } for i := range am.data { var dv Values = am.data[i] sort.Sort(&dv) am.data[i] = dv } assert.Equal(t, resultAM, am) } func TestLen(t *testing.T) { am := createAM() assert.Equal(t, 4, am.Len()) result := [][]byte{ []byte("5_sec.name.any"), []byte("5_min.name.min"), // it's repeated } am.MergeTarget(finder.NewMockFinder(result), findTarget, false) assert.Equal(t, 5, am.Len()) } func TestSize(t *testing.T) { am := createAM() assert.Equal(t, 4, am.Size()) result := [][]byte{ []byte("5_sec.name.any"), []byte("5_min.name.min"), // it's repeated, but it increases Size } am.MergeTarget(finder.NewMockFinder(result), findTarget, false) assert.Equal(t, 6, am.Size()) } func TestDisplayNames(t *testing.T) { am := createAM() sortedDisplayNames := am.DisplayNames() sort.Strings(sortedDisplayNames) expectedSeries := finderResult.Strings() sort.Strings(expectedSeries) assert.Equal(t, expectedSeries, sortedDisplayNames) anotherFinderResult := finder.NewMockFinder([][]byte{ []byte("5_sec.name.any"), []byte("5_min.name.min"), // it's repeated, but it increases Size }) am.MergeTarget(anotherFinderResult, findTarget, false) sortedDisplayNames = am.DisplayNames() sort.Strings(sortedDisplayNames) expectedSeries = append(expectedSeries, anotherFinderResult.Strings()...) sort.Strings(expectedSeries) assert.Equal(t, expectedSeries, sortedDisplayNames) } func TestGet(t *testing.T) { am := createAM() assert.Equal(t, []Value{{Target: "*.name.*", DisplayName: "5_sec.name.max"}}, am.Get("5_sec.name.max")) } func Benchmark_MergeTargetFinder(b *testing.B) { result := [][]byte{ []byte("5_sec.name.any"), []byte("5_min.name.min"), } for i := 0; i < b.N; i++ { am := createAM() am.MergeTarget(finder.NewMockFinder(result), findTarget, false) _ = am } } ================================================ FILE: pkg/dry/math.go ================================================ package dry // Max returns the larger of x or y. func Max(x, y int64) int64 { if x > y { return x } return y } // Min returns the lower of x or y. func Min(x, y int64) int64 { if x < y { return x } return y } // Ceil returns integer greater or equal to x and denominator d division. // Works only with x >= 0 and d > 0. It returns 0 with other values. func Ceil(x, d int64) int64 { if x <= 0 || d <= 0 { return int64(0) } return (x + d - 1) / d } // CeilToMultiplier returns the integer greater or equal to x and multiplier m product. // Works only with x >= 0 and m > 0. It returns 0 with other values. func CeilToMultiplier(x, m int64) int64 { return Ceil(x, m) * m } // FloorToMultiplier returns the integer less or equal to x and multiplier m product. // Works only with x >= 0 and m > 0. It returns 0 with other values. func FloorToMultiplier(x, m int64) int64 { if x <= 0 || m <= 0 { return int64(0) } return x / m * m } // GCD returns the absolute greatest common divisor calculated via Euclidean algorithm func GCD(a, b int64) int64 { if b < 0 { b = -b } var t int64 for b != 0 { t = b b = a % b a = t } return a } // LCM returns the absolute least common multiple of 2 integers via GDB func LCM(a, b int64) int64 { if a*b < 0 { return -a / GCD(a, b) * b } return a / GCD(a, b) * b } ================================================ FILE: pkg/dry/math_test.go ================================================ package dry import ( "testing" "github.com/stretchr/testify/assert" ) func TestMax(t *testing.T) { assert := assert.New(t) assert.Equal(int64(1), Max(1, -1)) assert.Equal(int64(2), Max(1, 2)) assert.Equal(int64(3), Max(3, 3)) } func TestMin(t *testing.T) { assert := assert.New(t) assert.Equal(int64(-1), Min(1, -1)) assert.Equal(int64(1), Min(1, 2)) assert.Equal(int64(3), Min(3, 3)) } func TestCeil(t *testing.T) { assert := assert.New(t) assert.Equal(int64(0), Ceil(0, -1)) assert.Equal(int64(3), Ceil(5, 2)) assert.Equal(int64(1), Ceil(5, 5)) // if quotient is integer we should get quotient without +1 assert.Equal(int64(2), Ceil(100001, 100000)) // if quotient is any fraction bigger than integer then we get +1 } func TestCeilToMultiplier(t *testing.T) { assert := assert.New(t) assert.Equal(int64(0), CeilToMultiplier(0, -1)) assert.Equal(int64(0), CeilToMultiplier(1, 0)) assert.Equal(int64(0), CeilToMultiplier(1, -1)) assert.Equal(int64(2), CeilToMultiplier(1, 2)) assert.Equal(int64(6), CeilToMultiplier(4, 3)) assert.Equal(int64(6), CeilToMultiplier(6, 3)) } func TestFloorToMultiplier(t *testing.T) { assert := assert.New(t) assert.Equal(int64(0), FloorToMultiplier(0, -1)) assert.Equal(int64(0), FloorToMultiplier(1, 0)) assert.Equal(int64(0), FloorToMultiplier(1, -1)) assert.Equal(int64(0), FloorToMultiplier(1, 2)) assert.Equal(int64(3), FloorToMultiplier(4, 3)) assert.Equal(int64(6), FloorToMultiplier(6, 3)) } func TestGCD(t *testing.T) { assert := assert.New(t) assert.Equal(int64(1), GCD(1, -1)) assert.Equal(int64(1), GCD(-1, 1)) assert.Equal(int64(1), GCD(-1, -1)) assert.Equal(int64(1), GCD(1, 2)) assert.Equal(int64(1), GCD(4, 3)) assert.Equal(int64(3), GCD(6, 3)) } func TestLCM(t *testing.T) { assert := assert.New(t) assert.Equal(int64(1), LCM(1, -1)) assert.Equal(int64(1), LCM(-1, 1)) assert.Equal(int64(1), LCM(-1, -1)) assert.Equal(int64(2), LCM(1, 2)) assert.Equal(int64(6), LCM(6, 3)) assert.Equal(int64(12), LCM(4, 3)) } ================================================ FILE: pkg/dry/strings.go ================================================ package dry // RemoveEmptyStrings removes empty strings from list and returns truncated slice func RemoveEmptyStrings(stringList []string) []string { rm := 0 for i := 0; i < len(stringList); i++ { if stringList[i] == "" { rm++ continue } if rm > 0 { stringList[i-rm] = stringList[i] } } return stringList[:len(stringList)-rm] } ================================================ FILE: pkg/dry/strings_test.go ================================================ package dry import ( "testing" "github.com/stretchr/testify/assert" ) func TestRemoveEmptyStrings(t *testing.T) { assert := assert.New(t) assert.Equal([]string{"lorem", " ", "ipsum"}, RemoveEmptyStrings([]string{"", "", "lorem", "", " ", "ipsum", ""}), ) } ================================================ FILE: pkg/dry/unsafe.go ================================================ package dry import ( "reflect" "unsafe" ) // UnsafeString returns string object from byte slice without copying func UnsafeString(b []byte) string { return *(*string)(unsafe.Pointer(&b)) } // UnsafeStringBytes returns the string bytes func UnsafeStringBytes(s *string) []byte { return *(*[]byte)(unsafe.Pointer((*reflect.SliceHeader)(unsafe.Pointer(s)))) } ================================================ FILE: pkg/dry/unsafe_test.go ================================================ package dry import ( "testing" "github.com/stretchr/testify/assert" ) func TestUnsafeString(t *testing.T) { assert := assert.New(t) assert.Equal("hello", UnsafeString([]byte{'h', 'e', 'l', 'l', 'o'})) assert.Equal("h", UnsafeString([]byte{'h'})) assert.Equal("", UnsafeString([]byte{})) assert.Equal("", UnsafeString(nil)) } ================================================ FILE: pkg/reverse/reverse.go ================================================ package reverse import ( "bytes" "strings" ) func String(path string) string { // don't reverse tagged path if strings.IndexByte(path, '?') >= 0 { return path } a := strings.Split(path, ".") l := len(a) for i := 0; i < l/2; i++ { a[i], a[l-i-1] = a[l-i-1], a[i] } return strings.Join(a, ".") } func reverse(m []byte) { i := 0 j := len(m) - 1 for i < j { m[i], m[j] = m[j], m[i] i++ j-- } } func Inplace(path []byte) { if bytes.IndexByte(path, '?') >= 0 { return } reverse(path) var a, b int l := len(path) for b = 0; b < l; b++ { if path[b] == '.' { reverse(path[a:b]) a = b + 1 } } reverse(path[a:b]) } func Bytes(path []byte) []byte { // @TODO: test // don't reverse tagged path if bytes.IndexByte(path, '?') >= 0 { return path } r := make([]byte, len(path)) copy(r, path) Inplace(r) return r } ================================================ FILE: pkg/reverse/reverse_test.go ================================================ package reverse import ( "testing" "github.com/stretchr/testify/assert" ) func TestReverse(t *testing.T) { assert := assert.New(t) table := map[string]string{ "carbon.agents.carbon-clickhouse.graphite1.tcp.metricsReceived": "metricsReceived.tcp.graphite1.carbon-clickhouse.agents.carbon", "": "", ".": ".", "carbon..xx": "xx..carbon", ".hello..world.": ".world..hello.", "metric_name?label=value": "metric_name?label=value", } for k, expected := range table { assert.Equal(expected, String(k)) p := k assert.Equal([]byte(expected), Bytes([]byte(k))) // check k is unchanged assert.Equal(p, k) // inplace b := make([]byte, len(k)) copy(b, k) Inplace(b) assert.Equal(expected, string(b)) } } ================================================ FILE: pkg/scope/context.go ================================================ package scope import ( "context" "go.uber.org/zap" ) // Context wrapper for context.Context with chain constructor type Context struct { context.Context } // New ... func New(ctx context.Context) *Context { return &Context{ctx} } // With ... func (c *Context) With(key string, value interface{}) *Context { return New(With(c.Context, key, value)) } // WithRequestID ... func (c *Context) WithRequestID(requestID string) *Context { return New(WithRequestID(c.Context, requestID)) } // WithLogger ... func (c *Context) WithLogger(logger *zap.Logger) *Context { return New(WithLogger(c.Context, logger)) } // WithTable ... func (c *Context) WithTable(table string) *Context { return New(WithTable(c.Context, table)) } ================================================ FILE: pkg/scope/http_request.go ================================================ package scope import ( "context" "encoding/binary" "fmt" "math/rand" "net" "net/http" "regexp" "strings" ) var ( requestIdRegexp = regexp.MustCompile("^[a-zA-Z0-9_.-]+$") passHeaders = []string{ "X-Dashboard-Id", "X-Grafana-Org-Id", "X-Panel-Id", "X-Forwarded-For", } ) func HttpRequest(r *http.Request) *http.Request { requestID := r.Header.Get("X-Request-Id") if requestID == "" || !requestIdRegexp.MatchString(requestID) { var b [16]byte binary.LittleEndian.PutUint64(b[:], rand.Uint64()) binary.LittleEndian.PutUint64(b[8:], rand.Uint64()) requestID = fmt.Sprintf("%x", b) } ctx := r.Context() ctx = WithRequestID(ctx, requestID) // Process all X-Gch-Debug-* headers debugPrefix := "X-Gch-Debug-" for name, values := range r.Header { if strings.HasPrefix(name, debugPrefix) && len(values) != 0 && values[0] != "" { ctx = WithDebug(ctx, strings.TrimPrefix(name, debugPrefix)) } } // Append the server IP to X-Forwarded-For if exists, else ignore if xff := r.Header.Get("X-Forwarded-For"); xff != "" { clientIP, _, _ := net.SplitHostPort(r.RemoteAddr) r.Header.Set("X-Forwarded-For", fmt.Sprintf("%s, %s", xff, clientIP)) } for _, h := range passHeaders { hv := r.Header.Get(h) if hv != "" { ctx = With(ctx, h, hv) } } return r.WithContext(ctx) } func Grafana(ctx context.Context) string { o, d, p := String(ctx, "X-Grafana-Org-Id"), String(ctx, "X-Dashboard-Id"), String(ctx, "X-Panel-Id") if o != "" || d != "" || p != "" { return fmt.Sprintf("Org:%s; Dashboard:%s; Panel:%s", o, d, p) } return "" } ================================================ FILE: pkg/scope/key.go ================================================ package scope import ( "context" "fmt" ) // key is type for context.Value keys type scopeKey string // With returns a copy of parent in which the value associated with key is val. func With(ctx context.Context, key string, value interface{}) context.Context { return context.WithValue(ctx, scopeKey(key), value) } // String returns the string value associated with this context for key func String(ctx context.Context, key string) string { if value, ok := ctx.Value(scopeKey(key)).(string); ok { return value } return "" } // Bool returns the true if particular key of the context is set func Bool(ctx context.Context, key string) bool { if _, ok := ctx.Value(scopeKey(key)).(bool); ok { return true } return false } // WithRequestID ... func WithRequestID(ctx context.Context, requestID string) context.Context { return With(ctx, "requestID", requestID) } // RequestID ... func RequestID(ctx context.Context) string { return String(ctx, "requestID") } // WithTable ... func WithTable(ctx context.Context, table string) context.Context { return With(ctx, "table", table) } // Table ... func Table(ctx context.Context) string { return String(ctx, "table") } // WithDebug returns the context with debug-name func WithDebug(ctx context.Context, name string) context.Context { return With(ctx, "debug-"+name, true) } // Debug returns true if debug-name should be enabled func Debug(ctx context.Context, name string) bool { return Bool(ctx, "debug-"+name) } // ClickhouseUserAgent ... func ClickhouseUserAgent(ctx context.Context) string { grafana := Grafana(ctx) if grafana != "" { return fmt.Sprintf("Graphite-Clickhouse/%s (table:%s) Grafana(%s)", Version, Table(ctx), grafana) } return fmt.Sprintf("Graphite-Clickhouse/%s (table:%s)", Version, Table(ctx)) } ================================================ FILE: pkg/scope/logger.go ================================================ package scope import ( "context" "net/http" "github.com/lomik/graphite-clickhouse/helper/headers" "github.com/lomik/zapwriter" "go.uber.org/zap" ) var ( CarbonapiUUIDName = "carbonapi_uuid" RequestHeadersName = "request_headers" ) // Logger returns zap.Logger instance func Logger(ctx context.Context) *zap.Logger { logger := ctx.Value(scopeKey("logger")) var zapLogger *zap.Logger if logger != nil { if zl, ok := logger.(*zap.Logger); ok { zapLogger = zl return zapLogger } } if zapLogger == nil { zapLogger = zapwriter.Default() } requestId := RequestID(ctx) if requestId != "" { zapLogger = zapLogger.With(zap.String("request_id", requestId)) } return zapLogger } // Logger returns zap.Logger instance func LoggerWithHeaders(ctx context.Context, r *http.Request, headersToLog []string) *zap.Logger { logger := ctx.Value(scopeKey("logger")) var zapLogger *zap.Logger if logger != nil { if zl, ok := logger.(*zap.Logger); ok { zapLogger = zl return zapLogger } } if zapLogger == nil { zapLogger = zapwriter.Default() } requestId := RequestID(ctx) if requestId != "" { zapLogger = zapLogger.With(zap.String("request_id", requestId)) } carbonapiUUID := r.Header.Get("X-Ctx-Carbonapi-Uuid") if carbonapiUUID != "" { zapLogger = zapLogger.With(zap.String("carbonapi_uuid", carbonapiUUID)) } requestHeaders := headers.GetHeaders(&r.Header, headersToLog) if len(requestHeaders) > 0 { zapLogger = zapLogger.With(zap.Any("request_headers", requestHeaders)) } return zapLogger } // WithLogger ... func WithLogger(ctx context.Context, logger *zap.Logger) context.Context { return With(ctx, "logger", logger) } ================================================ FILE: pkg/scope/version.go ================================================ package scope var Version string ================================================ FILE: pkg/where/match.go ================================================ package where import ( "fmt" "strings" ) var opEq = "=" // ClearGlob cleanup grafana globs like {name} func ClearGlob(query string) string { p := 0 s := strings.IndexAny(query, "{[") if s == -1 { return query } found := false var builder strings.Builder for { var e int if query[s] == '{' { e = strings.IndexAny(query[s:], "}.") if e == -1 || query[s+e] == '.' { // { not closed, glob with error break } e += s + 1 delim := strings.IndexRune(query[s+1:e], ',') if delim == -1 { if !found { builder.Grow(len(query) - 2) found = true } builder.WriteString(query[p:s]) builder.WriteString(query[s+1 : e-1]) p = e } } else { e = strings.IndexAny(query[s+1:], "].") if e == -1 || query[s+e] == '.' { // [ not closed, glob with error break } else { symbols := 0 for _, c := range query[s+1 : s+e+1] { _ = c // for loop over runes symbols++ if symbols == 2 { break } } if symbols <= 1 { if !found { builder.Grow(len(query) - 2) found = true } builder.WriteString(query[p:s]) builder.WriteString(query[s+1 : s+e+1]) p = e + s + 2 } } e += s + 2 } if e >= len(query) { break } s = strings.IndexAny(query[e:], "{[") if s == -1 { break } s += e } if found { if p < len(query) { builder.WriteString(query[p:]) } return builder.String() } return query } func HasUnmatchedBrackets(query string) bool { matchingBracket := map[rune]rune{ '}': '{', ']': '[', } stack := make([]rune, 0) nodeHasUnmatched := func(query string) bool { for _, c := range query { if c == '{' || c == '[' { stack = append(stack, c) } if c == '}' || c == ']' { if len(stack) == 0 || stack[len(stack)-1] != matchingBracket[c] { return true } stack = stack[:len(stack)-1] } } return len(stack) != 0 } for _, node := range strings.Split(query, ".") { if nodeHasUnmatched(node) { return true } } return false } func glob(field string, query string, optionalDotAtEnd bool) string { if query == "*" { return "" } query = ClearGlob(query) if !HasWildcard(query) { if optionalDotAtEnd { return In(field, []string{query, query + "."}) } else { return Eq(field, query) } } w := New() // before any wildcard symbol simplePrefix := query[:strings.IndexAny(query, "[]{}*?")] if len(simplePrefix) > 0 { w.And(HasPrefix(field, simplePrefix)) } // prefix search like "metric.name.xx*" if len(simplePrefix) == len(query)-1 && query[len(query)-1] == '*' { return HasPrefix(field, simplePrefix) } // Q() replaces \ with \\, so using \. does not work here. // work around with [.] postfix := `$` if optionalDotAtEnd { postfix = `[.]?$` } if simplePrefix == "" { return fmt.Sprintf("match(%s, %s)", field, quote(`^`+GlobToRegexp(query)+postfix)) } return fmt.Sprintf("%s AND match(%s, %s)", HasPrefix(field, simplePrefix), field, quote(`^`+GlobToRegexp(query)+postfix), ) } // Glob ... func Glob(field string, query string) string { return glob(field, query, false) } // TreeGlob ... func TreeGlob(field string, query string) string { return glob(field, query, true) } func ConcatMatchKV(key, value string) string { startLine := value[0] == '^' endLine := value[len(value)-1] == '$' if startLine && endLine { return key + opEq + value[1:] } else if startLine { return key + opEq + value[1:] + "\\\\%" } return key + opEq + "\\\\%" + value } func Match(field string, key, value string) string { if len(value) == 0 || value == "*" { return Like(field, key+"=%") } expr := ConcatMatchKV(key, value) simplePrefix := NonRegexpPrefix(expr) if len(simplePrefix) == len(expr) { return Eq(field, expr) } else if len(simplePrefix) == len(expr)-1 && expr[len(expr)-1] == '$' { return Eq(field, simplePrefix) } if simplePrefix == "" { return fmt.Sprintf("match(%s, %s)", field, quoteRegex(key, value)) } return fmt.Sprintf("%s AND match(%s, %s)", HasPrefix(field, simplePrefix), field, quoteRegex(key, value), ) } ================================================ FILE: pkg/where/match_test.go ================================================ package where import "testing" func Test_ClearGlob(t *testing.T) { tests := []struct { query string want string }{ {"a.{a,b}.te{s}t.b", "a.{a,b}.test.b"}, {"a.{a,b}.te{s,t}*.b", "a.{a,b}.te{s,t}*.b"}, {"a.{a,b}.test*.b", "a.{a,b}.test*.b"}, {"a.[b].te{s}t.b", "a.b.test.b"}, {"a.[ab].te{s,t}*.b", "a.[ab].te{s,t}*.b"}, {"a.{a,b.}.te{s,t}*.b", "a.{a,b.}.te{s,t}*.b"}, // some broken {"О.[б].те{s}t.b", "О.б.теst.b"}, // utf-8 string {"О.[].те{}t.b", "О..теt.b"}, // utf-8 string with empthy blocks } for _, tt := range tests { t.Run(tt.query, func(t *testing.T) { if got := ClearGlob(tt.query); got != tt.want { t.Errorf("ClearGlob() = %v, want %v", got, tt.want) } }) } } func Test_HasUnmatchedBrackets(t *testing.T) { tests := []struct { query string want bool }{ {"a.{a,b.te{s}t.b", true}, {"a.{a,b}.te{s}t.b", false}, {"a.{a,b}.te{s,t}}*.b", true}, {"a.{a,b}.test*.b", false}, {"a.a,b}.test*.b", true}, {"a.{a,b.test*.b}", true}, {"a.[a,b.test*.b]", true}, {"a.[a,b].test*.b", false}, {"a.[b].te{s}t.b", false}, {"a.{[cd],[ef]}.b", false}, {"a.[ab].te{s,t}*.b", false}, {"a.{a,b.}.te{s,t}*.b", true}, // dots are not escaped inside curly brackets {"О.[б].те{s}t.b", false}, // utf-8 string {"О.[б.теs}t.b", true}, {"О.[].те{}t.b", false}, // utf-8 string with empthy blocks } for _, tt := range tests { t.Run(tt.query, func(t *testing.T) { if got := HasUnmatchedBrackets(tt.query); got != tt.want { t.Errorf("HasUnmatchedBrackets() = %v, want %v", got, tt.want) } }) } } func TestGlob(t *testing.T) { field := "test" tests := []struct { query string want string }{ {"a.{a,b}.te{s}t.b", "test LIKE 'a.%' AND match(test, '^a[.](a|b)[.]test[.]b$')"}, {"a.{a,b}.te{s,t}*.b", "test LIKE 'a.%' AND match(test, '^a[.](a|b)[.]te(s|t)([^.]*?)[.]b$')"}, {"a.{a,b}.test*.b", "test LIKE 'a.%' AND match(test, '^a[.](a|b)[.]test([^.]*?)[.]b$')"}, {"a.[b].te{s}t.b", "test='a.b.test.b'"}, {"a.[ab].te{s,t}*.b", "test LIKE 'a.%' AND match(test, '^a[.][ab][.]te(s|t)([^.]*?)[.]b$')"}, } for _, tt := range tests { t.Run(tt.query, func(t *testing.T) { if got := Glob(field, tt.query); got != tt.want { t.Errorf("Glob() = %v, want %v", got, tt.want) } }) } } ================================================ FILE: pkg/where/where.go ================================================ package where import ( "fmt" "net/http" "regexp" "strings" "unsafe" "github.com/lomik/graphite-clickhouse/helper/date" "github.com/lomik/graphite-clickhouse/helper/errs" ) func unsafeString(b []byte) string { return *(*string)(unsafe.Pointer(&b)) } // workaraund for Grafana multi-value variables, expand S{a,b,c}E to [SaE,SbE,ScE] func GlobExpandSimple(value, prefix string, result *[]string) error { if len(value) == 0 { // we at the end of glob *result = append(*result, prefix) return nil } start := strings.IndexAny(value, "{}") if start == -1 { *result = append(*result, prefix+value) } else { end := strings.Index(value[start:], "}") if end <= 1 { return errs.NewErrorWithCode("malformed glob: "+value, http.StatusBadRequest) } if end == -1 || strings.IndexAny(value[start+1:start+end], "{}") != -1 { return errs.NewErrorWithCode("malformed glob: "+value, http.StatusBadRequest) } if start > 0 { prefix = prefix + value[0:start] } g := value[start+1 : start+end] values := strings.Split(g, ",") var postfix string if end+start-1 < len(value) { postfix = value[start+end+1:] } for _, v := range values { if err := GlobExpandSimple(postfix, prefix+v, result); err != nil { return err } } } return nil } func GlobToRegexp(g string) string { s := g s = strings.ReplaceAll(s, ".", "[.]") s = strings.ReplaceAll(s, "$", "[$]") s = strings.ReplaceAll(s, "{", "(") s = strings.ReplaceAll(s, "}", ")") s = strings.ReplaceAll(s, "?", "[^.]") s = strings.ReplaceAll(s, ",", "|") s = strings.ReplaceAll(s, "*", "([^.]*?)") return s } func HasWildcard(target string) bool { return strings.IndexAny(target, "[]{}*?") > -1 } func IndexLastWildcard(target string) int { return strings.LastIndexAny(target, "[]{}*?") } func IndexWildcard(target string) int { return strings.IndexAny(target, "[]{}*?") } func MaxWildcardDistance(query string) int { if !HasWildcard(query) { return -1 } w := IndexWildcard(query) firstWildcardNode := strings.Count(query[:w], ".") w = IndexLastWildcard(query) lastWildcardNode := strings.Count(query[w:], ".") return max(firstWildcardNode, lastWildcardNode) } func NonRegexpPrefix(expr string) string { s := regexp.QuoteMeta(expr) for i := 0; i < len(expr); i++ { if expr[i] != s[i] || expr[i] == '\\' { if len(expr) > i+1 && expr[i] == '|' { eq := strings.LastIndexAny(expr[:i], "=~") if eq > 0 { return expr[:eq+1] } } return expr[:i] } } return expr } func escape(s string) string { s = strings.ReplaceAll(s, `\`, `\\`) s = strings.ReplaceAll(s, `'`, `\'`) return s } func escapeRegex(s string) string { s = escape(s) if strings.Contains(s, "|") { s = "(" + s + ")" } return s } func likeEscape(s string) string { s = strings.ReplaceAll(s, `_`, `\_`) s = strings.ReplaceAll(s, `%`, `\%`) s = strings.ReplaceAll(s, `\`, `\\`) s = strings.ReplaceAll(s, `'`, `\'`) return s } func quote(value interface{}) string { switch v := value.(type) { case int: return fmt.Sprintf("%#v", v) case uint32: return fmt.Sprintf("%#v", v) case string: return fmt.Sprintf("'%s'", escape(v)) case []byte: return fmt.Sprintf("'%s'", escape(unsafeString(v))) default: panic("not implemented") } } func quoteRegex(key, value string) string { startLine := value[0] == '^' if startLine { return fmt.Sprintf("'^%s%s%s'", key, opEq, escapeRegex(value[1:])) } return fmt.Sprintf("'^%s%s.*%s'", key, opEq, escapeRegex(value)) } func Like(field, s string) string { return fmt.Sprintf("%s LIKE '%s'", field, s) } func Eq(field, value interface{}) string { return fmt.Sprintf("%s=%s", field, quote(value)) } func HasPrefix(field, prefix string) string { return fmt.Sprintf("%s LIKE '%s%%'", field, likeEscape(prefix)) } func HasPrefixAndNotEq(field, prefix string) string { return fmt.Sprintf("%s LIKE '%s_%%'", field, likeEscape(prefix)) } func HasPrefixBytes(field, prefix []byte) string { return fmt.Sprintf("%s LIKE '%s%%'", field, likeEscape(unsafeString(prefix))) } func ArrayHas(field, element string) string { return fmt.Sprintf("has(%s, %s)", field, quote(element)) } func In(field string, list []string) string { if len(list) == 1 { return Eq(field, list[0]) } var buf strings.Builder buf.WriteString(field) buf.WriteString(" IN (") for i, v := range list { if i > 0 { buf.WriteByte(',') } buf.WriteString(quote(v)) } buf.WriteByte(')') return buf.String() } func InTable(field string, table string) string { return fmt.Sprintf("%s in %s", field, table) } func DateBetween(field string, from int64, until int64) string { return fmt.Sprintf( "%s >= '%s' AND %s <= '%s'", field, date.FromTimestampToDaysFormat(from), field, date.UntilTimestampToDaysFormat(until), ) } func TimestampBetween(field string, from int64, until int64) string { return fmt.Sprintf("%s >= %d AND %s <= %d", field, from, field, until) } type Where struct { where string } func New() *Where { return &Where{} } func (w *Where) And(exp string) { if exp == "" { return } if w.where != "" { w.where = fmt.Sprintf("(%s) AND (%s)", w.where, exp) } else { w.where = exp } } func (w *Where) Or(exp string) { if exp == "" { return } if w.where != "" { w.where = fmt.Sprintf("(%s) OR (%s)", w.where, exp) } else { w.where = exp } } func (w *Where) Andf(format string, obj ...interface{}) { w.And(fmt.Sprintf(format, obj...)) } func (w *Where) String() string { return w.where } func (w *Where) SQL() string { if w.where == "" { return "" } return "WHERE " + w.where } func (w *Where) PreWhereSQL() string { if w.where == "" { return "" } return "PREWHERE " + w.where } ================================================ FILE: pkg/where/where_test.go ================================================ package where import ( "fmt" "testing" "github.com/stretchr/testify/assert" ) func TestGlobExpandSimple(t *testing.T) { tests := []struct { value string want []string wantErr bool }{ {"{a,bc,d}", []string{"a", "bc", "d"}, false}, {"S{a,bc,d}", []string{"Sa", "Sbc", "Sd"}, false}, {"{a,bc,d}E", []string{"aE", "bcE", "dE"}, false}, {"S{a,bc,d}E", []string{"SaE", "SbcE", "SdE"}, false}, {"S{a,bc,d}E{f,h}", []string{"SaEf", "SaEh", "SbcEf", "SbcEh", "SdEf", "SdEh"}, false}, {"test{a,b}", []string{"testa", "testb"}, false}, {"S{a,bc,d}}E{f,h}", nil, true}, //error {"S{{a,bc,d}E{f,h}", nil, true}, //error } for _, tt := range tests { t.Run(tt.value, func(t *testing.T) { var got []string err := GlobExpandSimple(tt.value, "", &got) if tt.wantErr { assert.Error(t, err, "Expand() not returns error for %v", tt.value) } else { assert.NoErrorf(t, err, "Expand() returns error %v for %v", err, tt.value) } assert.Equal(t, tt.want, got, "Expand() result") }) } } func TestGlobToRegexp(t *testing.T) { table := []struct { glob string regexp string }{ {`test.*.foo`, `test[.]([^.]*?)[.]foo`}, {`test.{foo,bar}`, `test[.](foo|bar)`}, {`test?.foo`, `test[^.][.]foo`}, {`test?.$foo`, `test[^.][.][$]foo`}, } for _, test := range table { testName := fmt.Sprintf("glob: %#v", test.glob) regexp := GlobToRegexp(test.glob) assert.Equal(t, test.regexp, regexp, testName) } } func TestNonRegexpPrefix(t *testing.T) { table := []struct { expr string prefix string }{ {`test[.]([^.]*?)[.]foo`, `test`}, {`__name__=cpu.load`, `__name__=cpu`}, {`__name__=~(cpu|mem)`, `__name__=~`}, {`__name__=~cpu|mem`, `__name__=~`}, {`__name__=~^host`, `__name__=~`}, } for _, test := range table { testName := fmt.Sprintf("expr: %#v", test.expr) prefix := NonRegexpPrefix(test.expr) assert.Equal(t, test.prefix, prefix, testName) } } func TestMaxWildcardDistance(t *testing.T) { table := []struct { glob string dist int }{ {`a.b.c.d.e`, -1}, {`test.*.foo.bar`, 2}, {`test.foo.*.*.bar.count`, 2}, {`test.foo.bar.*.bar.foo.test`, 3}, {`test.foo.bar.foobar.*.middle.*.foobar.bar.foo.test`, 4}, {`*.test.foo.bar.*`, 0}, } for _, test := range table { testName := fmt.Sprintf("glob: %#v", test.glob) dist := MaxWildcardDistance(test.glob) assert.Equal(t, test.dist, dist, testName) } } ================================================ FILE: prometheus/.gitignore ================================================ tmp ================================================ FILE: prometheus/empty_iterator.go ================================================ //go:build !noprom // +build !noprom package prometheus import ( "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/tsdb/chunkenc" ) // Iterator is a simple iterator that can only get the next value. // Iterator iterates over the samples of a time series, in timestamp-increasing order. type emptyIterator struct{} var emptyIteratorValue chunkenc.Iterator = &emptyIterator{} // Next advances the iterator by one and returns the type of the value // at the new position (or ValNone if the iterator is exhausted). func (it *emptyIterator) Next() chunkenc.ValueType { return chunkenc.ValNone } // Seek advances the iterator forward to the first sample with a // timestamp equal or greater than t. If the current sample found by a // previous `Next` or `Seek` operation already has this property, Seek // has no effect. If a sample has been found, Seek returns the type of // its value. Otherwise, it returns ValNone, after with the iterator is // exhausted. func (it *emptyIterator) Seek(t int64) chunkenc.ValueType { return chunkenc.ValNone } // At returns the current timestamp/value pair if the value is a float. // Before the iterator has advanced, the behaviour is unspecified. func (it *emptyIterator) At() (int64, float64) { return 0, 0 } // AtHistogram returns the current timestamp/value pair if the value is // a histogram with integer counts. Before the iterator has advanced, // the behaviour is unspecified. func (it *emptyIterator) AtHistogram(histogram *histogram.Histogram) (int64, *histogram.Histogram) { return 0, nil } // AtFloatHistogram returns the current timestamp/value pair if the // value is a histogram with floating-point counts. It also works if the // value is a histogram with integer counts, in which case a // FloatHistogram copy of the histogram is returned. Before the iterator // has advanced, the behaviour is unspecified. func (it *emptyIterator) AtFloatHistogram(histogram *histogram.FloatHistogram) (int64, *histogram.FloatHistogram) { return 0, nil } // AtT returns the current timestamp. // Before the iterator has advanced, the behaviour is unspecified. func (it *emptyIterator) AtT() int64 { return 0 } // Err returns the current error. It should be used only after the // iterator is exhausted, i.e. `Next` or `Seek` have returned ValNone. func (it *emptyIterator) Err() error { return nil } ================================================ FILE: prometheus/exemplar.go ================================================ //go:build !noprom // +build !noprom package prometheus import ( "context" "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/storage" ) type nopExemplarQueryable struct { } type nopExemplarQuerier struct { } var _ storage.ExemplarQueryable = &nopExemplarQueryable{} var _ storage.ExemplarQuerier = &nopExemplarQuerier{} func (e *nopExemplarQueryable) ExemplarQuerier(ctx context.Context) (storage.ExemplarQuerier, error) { return &nopExemplarQuerier{}, nil } func (e *nopExemplarQuerier) Select(start, end int64, matchers ...[]*labels.Matcher) ([]exemplar.QueryResult, error) { return []exemplar.QueryResult{}, nil } ================================================ FILE: prometheus/gatherer.go ================================================ //go:build !noprom // +build !noprom package prometheus import ( "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" ) type nopGatherer struct{} var _ prometheus.Gatherer = &nopGatherer{} func (*nopGatherer) Gather() ([]*dto.MetricFamily, error) { return []*dto.MetricFamily{}, nil } ================================================ FILE: prometheus/labels.go ================================================ //go:build !noprom // +build !noprom package prometheus import ( "net/url" "sort" "strings" "github.com/prometheus/prometheus/model/labels" ) func urlParse(rawurl string) (*url.URL, error) { p := strings.IndexByte(rawurl, '?') if p < 0 { return url.Parse(rawurl) } m, err := url.Parse(rawurl[p:]) if m != nil { m.Path = rawurl[:p] } return m, err } func Labels(path string) labels.Labels { u, err := urlParse(path) if err != nil { return labels.Labels{labels.Label{Name: "__name__", Value: path}} } q := u.Query() lb := make(labels.Labels, len(q)+1) lb[0].Name = "__name__" lb[0].Value = u.Path i := 0 for k, v := range q { i++ lb[i].Name = k lb[i].Value = v[0] } if len(lb) > 1 { sort.Slice(lb, func(i, j int) bool { return lb[i].Name < lb[j].Name }) } return lb } ================================================ FILE: prometheus/labels_test.go ================================================ package prometheus import ( "testing" "github.com/stretchr/testify/assert" ) func TestLabels(t *testing.T) { assert := assert.New(t) table := [][2]string{ { "cpu_usage_system?cpu=cpu5&host=telegraf-b9468c8b5-g47xt&instance=telegraf.default%3A9273&job=telegraf", `{__name__="cpu_usage_system", cpu="cpu5", host="telegraf-b9468c8b5-g47xt", instance="telegraf.default:9273", job="telegraf"}`, }, { "cpu_usage_system", `{__name__="cpu_usage_system"}`, }, { ":metric:?instance=localhost", `{__name__=":metric:", instance="localhost"}`, }, } for _, c := range table { assert.Equal(c[1], Labels(c[0]).String()) } } ================================================ FILE: prometheus/local_storage.go ================================================ //go:build !noprom // +build !noprom package prometheus import ( "context" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/index" "github.com/prometheus/prometheus/web" ) var _ web.LocalStorage = &storageImpl{} func (s *storageImpl) CleanTombstones() error { return nil } func (s *storageImpl) Delete(ctx context.Context, mint, maxt int64, ms ...*labels.Matcher) error { return nil } func (s *storageImpl) Snapshot(dir string, withHead bool) error { return nil } func (s *storageImpl) Stats(statsByLabelName string, limit int) (*tsdb.Stats, error) { return &tsdb.Stats{ IndexPostingStats: &index.PostingsStats{}, }, nil } func (s *storageImpl) WALReplayStatus() (tsdb.WALReplayStatus, error) { return tsdb.WALReplayStatus{}, nil } ================================================ FILE: prometheus/logger.go ================================================ //go:build !noprom // +build !noprom package prometheus import ( "go.uber.org/zap" ) type errorLevel interface { String() string } type logger struct { z *zap.Logger } func (l *logger) Log(keyvals ...interface{}) error { lg := l.z var msg string var level errorLevel for i := 1; i < len(keyvals); i += 2 { keyObj := keyvals[i-1] keyStr, ok := keyObj.(string) if !ok { l.z.Error("can't handle log, wrong key", zap.Any("keyvals", keyvals)) return nil } if keyStr == "level" { level, ok = keyvals[i].(errorLevel) if !ok { l.z.Error("can't handle log, wrong level", zap.Any("keyvals", keyvals)) return nil } continue } if keyStr == "msg" { msg, ok = keyvals[i].(string) if !ok { l.z.Error("can't handle log, wrong msg", zap.Any("keyvals", keyvals)) return nil } continue } lg = lg.With(zap.Any(keyStr, keyvals[i])) } switch level.String() { case "debug": lg.Debug(msg) case "info": lg.Info(msg) case "warn": lg.Warn(msg) case "error": lg.Error(msg) default: l.z.Error("can't handle log, unknown level", zap.Any("keyvals", keyvals)) return nil } return nil } ================================================ FILE: prometheus/matcher.go ================================================ //go:build !noprom // +build !noprom package prometheus import ( "fmt" "sort" "github.com/lomik/graphite-clickhouse/finder" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/prompb" ) var prompbMatchMap = map[prompb.LabelMatcher_Type]finder.TaggedTermOp{ prompb.LabelMatcher_EQ: finder.TaggedTermEq, prompb.LabelMatcher_RE: finder.TaggedTermMatch, prompb.LabelMatcher_NEQ: finder.TaggedTermNe, prompb.LabelMatcher_NRE: finder.TaggedTermNotMatch, } var promqlMatchMap = map[labels.MatchType]finder.TaggedTermOp{ labels.MatchEqual: finder.TaggedTermEq, labels.MatchNotEqual: finder.TaggedTermNe, labels.MatchRegexp: finder.TaggedTermMatch, labels.MatchNotRegexp: finder.TaggedTermNotMatch, } func makeTaggedFromPromPB(matchers []*prompb.LabelMatcher) ([]finder.TaggedTerm, error) { terms := make([]finder.TaggedTerm, 0, len(matchers)) for i := 0; i < len(matchers); i++ { if matchers[i] == nil { continue } op, ok := prompbMatchMap[matchers[i].Type] if !ok { return nil, fmt.Errorf("unknown matcher type %#v", matchers[i].GetType()) } terms = append(terms, finder.TaggedTerm{ Key: matchers[i].Name, Value: matchers[i].Value, Op: op, }) } sort.Sort(finder.TaggedTermList(terms)) return terms, nil } func makeTaggedFromPromQL(matchers []*labels.Matcher) ([]finder.TaggedTerm, error) { terms := make([]finder.TaggedTerm, 0, len(matchers)) for i := 0; i < len(matchers); i++ { if matchers[i] == nil { continue } op, ok := promqlMatchMap[matchers[i].Type] if !ok { return nil, fmt.Errorf("unknown matcher type %#v", matchers[i].Type) } terms = append(terms, finder.TaggedTerm{ Key: matchers[i].Name, Value: matchers[i].Value, Op: op, }) } sort.Sort(finder.TaggedTermList(terms)) return terms, nil } // func wherePromPB(matchers []*prompb.LabelMatcher) (string, error) { // if len(matchers) == 0 { // return "", nil // } // terms := make([]finder.TaggedTerm, 0, len(matchers)) // for i := 0; i < len(matchers); i++ { // if matchers[i] == nil { // continue // } // op, ok := prompbMatchMap[matchers[i].Type] // if !ok { // return "", fmt.Errorf("unknown matcher type %#v", matchers[i].GetType()) // } // terms = append(terms, finder.TaggedTerm{ // Key: matchers[i].Name, // Value: matchers[i].Value, // Op: op, // }) // } // sort.Sort(finder.TaggedTermList(terms)) // w := where.New() // w.And(finder.TaggedTermWhere1(&terms[0])) // for i := 1; i < len(terms); i++ { // w.And(finder.TaggedTermWhereN(&terms[i])) // } // return w.String(), nil // } // func wherePromQL(matchers []*labels.Matcher) (string, error) { // if len(matchers) == 0 { // return "", nil // } // terms := make([]finder.TaggedTerm, 0, len(matchers)) // for i := 0; i < len(matchers); i++ { // if matchers[i] == nil { // continue // } // op, ok := promqlMatchMap[matchers[i].Type] // if !ok { // return "", fmt.Errorf("unknown matcher type %#v", matchers[i].Type) // } // terms = append(terms, finder.TaggedTerm{ // Key: matchers[i].Name, // Value: matchers[i].Value, // Op: op, // }) // } // sort.Sort(finder.TaggedTermList(terms)) // w := where.New() // w.And(finder.TaggedTermWhere1(&terms[0])) // for i := 1; i < len(terms); i++ { // w.And(finder.TaggedTermWhereN(&terms[i])) // } // return w.String(), nil // } ================================================ FILE: prometheus/metrics_set.go ================================================ //go:build !noprom // +build !noprom package prometheus import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/util/annotations" ) // SeriesSet contains a set of series. type metricsSet struct { metrics []string current int } type metric struct { name string } var _ storage.SeriesSet = &metricsSet{} func (ms *metricsSet) At() storage.Series { return &metric{name: ms.metrics[ms.current]} } // Iterator returns a new iterator of the data of the series. func (s *metric) Iterator(iterator chunkenc.Iterator) chunkenc.Iterator { return emptyIteratorValue } func (s *metric) Labels() labels.Labels { return Labels(s.name) } // Err returns the current error. func (ms *metricsSet) Err() error { return nil } func (ms *metricsSet) Next() bool { if ms.current < 0 { ms.current = 0 } else { ms.current++ } return ms.current < len(ms.metrics) } func newMetricsSet(metrics []string) storage.SeriesSet { return &metricsSet{metrics: metrics, current: -1} } // Warnings ... func (s *metricsSet) Warnings() annotations.Annotations { return nil } ================================================ FILE: prometheus/querier.go ================================================ //go:build !noprom // +build !noprom package prometheus import ( "context" "fmt" "strings" "time" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/util/annotations" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/pkg/where" "github.com/prometheus/prometheus/model/labels" ) // Querier provides reading access to time series data. type Querier struct { config *config.Config mint int64 maxt int64 } // Close releases the resources of the Querier. func (q *Querier) Close() error { return nil } // LabelValues returns all potential values for a label name. func (q *Querier) LabelValues(ctx context.Context, label string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { // @TODO: support matchers w := where.New() w.And(where.HasPrefix("Tag1", label+"=")) fromDate := timeNow().AddDate(0, 0, -q.config.ClickHouse.TaggedAutocompleDays) w.Andf("Date >= '%s'", fromDate.Format("2006-01-02")) sql := fmt.Sprintf("SELECT splitByChar('=', Tag1)[2] as value FROM %s %s GROUP BY value ORDER BY value", q.config.ClickHouse.TaggedTable, w.SQL(), ) body, _, _, err := clickhouse.Query( scope.WithTable(ctx, q.config.ClickHouse.TaggedTable), q.config.ClickHouse.URL, sql, clickhouse.Options{ TLSConfig: q.config.ClickHouse.TLSConfig, Timeout: q.config.ClickHouse.IndexTimeout, ConnectTimeout: q.config.ClickHouse.ConnectTimeout, CheckRequestProgress: q.config.FeatureFlags.LogQueryProgress, ProgressSendingInterval: q.config.ClickHouse.ProgressSendingInterval, }, nil, ) if err != nil { return nil, nil, err } rows := strings.Split(string(body), "\n") if len(rows) > 0 && rows[len(rows)-1] == "" { rows = rows[:len(rows)-1] } return rows, nil, nil } // LabelNames returns all the unique label names present in the block in sorted order. func (q *Querier) LabelNames(ctx context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { // @TODO support matchers w := where.New() fromDate := time.Now().AddDate(0, 0, -q.config.ClickHouse.TaggedAutocompleDays).UTC() w.Andf("Date >= '%s'", fromDate.Format("2006-01-02")) sql := fmt.Sprintf("SELECT splitByChar('=', Tag1)[1] as value FROM %s %s GROUP BY value ORDER BY value", q.config.ClickHouse.TaggedTable, w.SQL(), ) body, _, _, err := clickhouse.Query( scope.WithTable(ctx, q.config.ClickHouse.TaggedTable), q.config.ClickHouse.URL, sql, clickhouse.Options{ Timeout: q.config.ClickHouse.IndexTimeout, ConnectTimeout: q.config.ClickHouse.ConnectTimeout, TLSConfig: q.config.ClickHouse.TLSConfig, CheckRequestProgress: q.config.FeatureFlags.LogQueryProgress, ProgressSendingInterval: q.config.ClickHouse.ProgressSendingInterval, }, nil, ) if err != nil { return nil, nil, err } rows := strings.Split(string(body), "\n") if len(rows) > 0 && rows[len(rows)-1] == "" { rows = rows[:len(rows)-1] } return rows, nil, nil } ================================================ FILE: prometheus/querier_select.go ================================================ //go:build !noprom // +build !noprom package prometheus import ( "context" "time" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/finder" "github.com/lomik/graphite-clickhouse/limiter" "github.com/lomik/graphite-clickhouse/pkg/alias" "github.com/lomik/graphite-clickhouse/render/data" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/storage" ) // override in unit tests for stable results var timeNow = time.Now func (q *Querier) lookup(ctx context.Context, from, until int64, qlimiter limiter.ServerLimiter, queueDuration *time.Duration, labelsMatcher ...*labels.Matcher) (*alias.Map, error) { terms, err := makeTaggedFromPromQL(labelsMatcher) if err != nil { return nil, err } var ( limitCtx context.Context cancel context.CancelFunc ) if qlimiter.Enabled() { limitCtx, cancel = context.WithTimeout(ctx, q.config.ClickHouse.IndexTimeout) defer cancel() start := time.Now() err = qlimiter.Enter(limitCtx, "render") *queueDuration += time.Since(start) if err != nil { // status = http.StatusServiceUnavailable // queueFail = true // http.Error(w, err.Error(), status) return nil, err } defer qlimiter.Leave(limitCtx, "render") } // TODO: implement use stat for Prometheus queries fndResult, err := finder.FindTagged(ctx, q.config, terms, from, until) if err != nil { return nil, err } am := alias.New() am.Merge(fndResult, false) return am, nil } func (q *Querier) timeRange(hints *storage.SelectHints) (int64, int64) { var from, until time.Time // ClickHouse supported range of values by the Date type: [1970-01-01, 2149-06-06] if hints != nil && hints.Start > 0 && hints.Start < 5662310400000 { from = time.Unix(hints.Start/1000, (hints.Start%1000)*1000000) } if hints != nil && hints.End > 0 && hints.End < 5662310400000 { until = time.Unix(hints.End/1000, (hints.End%1000)*1000000) } if until.IsZero() { if q.maxt > 0 && q.maxt < 5662310400000 { until = time.Unix(q.maxt/1000, (q.maxt%1000)*1000000) } else { until = timeNow() } } if from.IsZero() { if q.mint > 0 && q.mint < 5662310400000 { from = time.Unix(q.mint/1000, (q.mint%1000)*1000000) } else { from = until.AddDate(0, 0, -q.config.ClickHouse.TaggedAutocompleDays) } } return from.Unix(), until.Unix() } // Select returns a set of series that matches the given label matchers. func (q *Querier) Select(ctx context.Context, sortSeries bool, hints *storage.SelectHints, labelsMatcher ...*labels.Matcher) storage.SeriesSet { var ( queueDuration time.Duration ) from, until := q.timeRange(hints) qlimiter := data.GetQueryLimiterFrom("", q.config, from, until) am, err := q.lookup(ctx, from, until, qlimiter, &queueDuration, labelsMatcher...) if err != nil { return nil //, nil, err @TODO } if am.Len() == 0 { return emptySeriesSet() } if hints != nil && hints.Func == "series" { // /api/v1/series?match[]=... return newMetricsSet(am.DisplayNames()) //, nil, nil } var step int64 = 60000 if hints.Step != 0 { step = hints.Step } maxDataPoints := 1000 * (until - from) / step multiTarget := data.MultiTarget{ data.TimeFrame{ From: from, Until: until, MaxDataPoints: maxDataPoints, }: data.NewTargets([]string{}, am), } reply, err := multiTarget.Fetch(ctx, q.config, config.ContextPrometheus, qlimiter, &queueDuration) if err != nil { return nil // , nil, err @TODO } if len(reply) == 0 { return emptySeriesSet() //, nil, nil } ss, err := makeSeriesSet(reply[0].Data, step) if err != nil { return nil // , nil, err @TODO } return ss //, nil, nil } ================================================ FILE: prometheus/querier_select_test.go ================================================ //go:build !noprom // +build !noprom package prometheus import ( "testing" "time" "github.com/lomik/graphite-clickhouse/config" "github.com/prometheus/prometheus/storage" "github.com/stretchr/testify/require" ) func TestQuerier_timeRange(t *testing.T) { timeNow = func() time.Time { // 2022-11-29 09:30:47 UTC return time.Unix(1669714247, 0) } cfg := &config.Config{ ClickHouse: config.ClickHouse{ TaggedAutocompleDays: 4, }, } tests := []struct { name string mint int64 maxt int64 hints *storage.SelectHints wantFrom int64 wantUntil int64 }{ { name: "default from/until", wantFrom: 1669368647, // timeNow() - config.Clickhouse.TaggedAutocompleDays wantUntil: 1669714247, // timeNow() result }, { name: "start/end in SelectHints", hints: &storage.SelectHints{ Start: 1669453200000, End: 1669626000000, }, wantFrom: 1669453200, wantUntil: 1669626000, }, { name: "start/end in SelectHints overflow", hints: &storage.SelectHints{ // ClickHouse supported range of values by the Date type: [1970-01-01, 2149-06-06] Start: 5662310400001, End: 5662310400100, }, wantFrom: 1669368647, // timeNow() - config.Clickhouse.TaggedAutocompleDays wantUntil: 1669714247, // timeNow() result }, { name: "no start/end in SelectHints", hints: &storage.SelectHints{}, mint: 1669194000000, maxt: 1669280400000, wantFrom: 1669194000, wantUntil: 1669280400, }, { name: "no start/end in SelectHints, mint/maxt overflow", hints: &storage.SelectHints{}, // ClickHouse supported range of values by the Date type: [1970-01-01, 2149-06-06] mint: 5662310400001, maxt: 5662310400100, wantFrom: 1669368647, // timeNow() - config.Clickhouse.TaggedAutocompleDays wantUntil: 1669714247, // timeNow() result }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := newStorage(cfg) // Querier returns a new Querier on the storage. sq, err := s.Querier(tt.mint, tt.maxt) require.NoError(t, err) q := sq.(*Querier) gotFrom, gotUntil := q.timeRange(tt.hints) if gotFrom != tt.wantFrom { t.Errorf("Querier.timeRange().from got = %v, want %v", gotFrom, tt.wantFrom) } if gotUntil != tt.wantUntil { t.Errorf("Querier.timeRange().until got = %v, want %v", gotUntil, tt.wantUntil) } }) } } ================================================ FILE: prometheus/run.go ================================================ //go:build !noprom // +build !noprom package prometheus import ( "context" "log" "net/http" "time" "github.com/grafana/regexp" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/zapwriter" "github.com/prometheus/client_golang/prometheus" promConfig "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/notifier" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/rules" "github.com/prometheus/prometheus/scrape" "github.com/prometheus/prometheus/web" "github.com/prometheus/prometheus/web/ui" uiStatic "github.com/lomik/prometheus-ui-static" "github.com/prometheus/common/assets" ) func Run(config *config.Config) error { // use precompiled static from github.com/lomik/prometheus-ui-static ui.Assets = http.FS(assets.New(uiStatic.EmbedFS)) zapLogger := &logger{ z: zapwriter.Logger("prometheus"), } storage := newStorage(config) corsOrigin, err := regexp.Compile("^$") if err != nil { return err } queryEngine := promql.NewEngine(promql.EngineOpts{ Logger: zapLogger, Timeout: time.Minute, MaxSamples: 50000000, LookbackDelta: config.Prometheus.LookbackDelta, }) scrapeManager, err := scrape.NewManager(&scrape.Options{}, zapLogger, storage, prometheus.DefaultRegisterer) if err != nil { return err } rulesManager := rules.NewManager(&rules.ManagerOptions{ Logger: zapLogger, Appendable: storage, Queryable: storage, }) notifierManager := notifier.NewManager(¬ifier.Options{}, zapLogger) promHandler := web.New(zapLogger, &web.Options{ ListenAddress: config.Prometheus.Listen, MaxConnections: 500, Storage: storage, ExemplarStorage: &nopExemplarQueryable{}, ExternalURL: config.Prometheus.ExternalURL, RoutePrefix: "/", QueryEngine: queryEngine, ScrapeManager: scrapeManager, RuleManager: rulesManager, Flags: make(map[string]string), LocalStorage: storage, Gatherer: &nopGatherer{}, Notifier: notifierManager, CORSOrigin: corsOrigin, PageTitle: config.Prometheus.PageTitle, LookbackDelta: config.Prometheus.LookbackDelta, RemoteReadConcurrencyLimit: config.Prometheus.RemoteReadConcurrencyLimit, }) promHandler.ApplyConfig(&promConfig.Config{}) promHandler.SetReady(true) go func() { log.Fatal(promHandler.Run(context.Background(), nil, "")) }() return nil } ================================================ FILE: prometheus/run_dummy.go ================================================ //go:build noprom // +build noprom package prometheus import ( "github.com/lomik/graphite-clickhouse/config" ) func Run(config *config.Config) error { return nil } ================================================ FILE: prometheus/series_set.go ================================================ //go:build !noprom // +build !noprom package prometheus import ( "log" "math" "github.com/prometheus/prometheus/util/annotations" "github.com/lomik/graphite-clickhouse/helper/point" "github.com/lomik/graphite-clickhouse/render/data" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" ) // SeriesIterator iterates over the data of a time series. type seriesIterator struct { metricName string points []point.Point current int step int64 } // Series represents a single time series. type series struct { metricName string points []point.Point step int64 } // SeriesSet contains a set of series. type seriesSet struct { series []series current int } var _ storage.SeriesSet = &seriesSet{} func makeSeriesSet(data *data.Data, step int64) (storage.SeriesSet, error) { ss := &seriesSet{series: make([]series, 0), current: -1} if data == nil { return ss, nil } if data.Len() == 0 { return ss, nil } nextMetric := data.GroupByMetric() for { points := nextMetric() if len(points) == 0 { break } metricName := data.MetricName(points[0].MetricID) for _, v := range data.AM.Get(metricName) { ss.series = append(ss.series, series{metricName: v.DisplayName, points: points, step: step}) } } return ss, nil } func emptySeriesSet() storage.SeriesSet { return &seriesSet{series: make([]series, 0), current: -1} } // func (sit *seriesIterator) logger() *zap.Logger { // return zap.L() //.With(zap.String("metric", sit.metricName)) // } // Seek advances the iterator forward to the value at or after // the given timestamp. func (sit *seriesIterator) Seek(t int64) chunkenc.ValueType { tt := uint32(t / 1000) if t%1000 != 0 { tt++ } for ; sit.current < len(sit.points); sit.current++ { if sit.points[sit.current].Time >= tt { // sit.logger().Debug("seriesIterator.Seek", zap.Int64("t", t), zap.Bool("ret", true)) return chunkenc.ValFloat } } // sit.logger().Debug("seriesIterator.Seek", zap.Int64("t", t), zap.Bool("ret", false)) return chunkenc.ValNone } // At returns the current timestamp/value pair. func (sit *seriesIterator) At() (t int64, v float64) { index := sit.current if index == len(sit.points) { return int64(sit.points[len(sit.points)-1].Time)*1000 + sit.step, math.NaN() } if index < 0 || index >= len(sit.points) { index = 0 } p := sit.points[index] // sit.logger().Debug("seriesIterator.At", zap.Int64("t", int64(p.Time)*1000), zap.Float64("v", p.Value)) return int64(p.Time) * 1000, p.Value } // AtHistogram returns the current timestamp/value pair if the value is // a histogram with integer counts. Before the iterator has advanced, // the behaviour is unspecified. func (sit *seriesIterator) AtHistogram(histogram *histogram.Histogram) (int64, *histogram.Histogram) { log.Fatal("seriesIterator.AtHistogram not implemented") return 0, nil // @TODO } // AtFloatHistogram returns the current timestamp/value pair if the // value is a histogram with floating-point counts. It also works if the // value is a histogram with integer counts, in which case a // FloatHistogram copy of the histogram is returned. Before the iterator // has advanced, the behaviour is unspecified. func (sit *seriesIterator) AtFloatHistogram(histogram *histogram.FloatHistogram) (int64, *histogram.FloatHistogram) { log.Fatal("seriesIterator.AtFloatHistogram not implemented") return 0, nil // @TODO } // AtT returns the current timestamp. // Before the iterator has advanced, the behaviour is unspecified. func (sit *seriesIterator) AtT() int64 { t, _ := sit.At() return t } // Next advances the iterator by one. func (sit *seriesIterator) Next() chunkenc.ValueType { if sit.current < len(sit.points) { if sit.step == 0 && sit.current == len(sit.points)-1 { return chunkenc.ValNone } sit.current++ // sit.logger().Debug("seriesIterator.Next", zap.Bool("ret", true)) return chunkenc.ValFloat } // sit.logger().Debug("seriesIterator.Next", zap.Bool("ret", false)) return chunkenc.ValNone } // Err returns the current error. func (sit *seriesIterator) Err() error { return nil } // Err returns the current error. func (ss *seriesSet) Err() error { return nil } func (ss *seriesSet) At() storage.Series { if ss == nil || ss.current < 0 || ss.current >= len(ss.series) { // zap.L().Debug("seriesSet.At", zap.String("metricName", "nil")) return nil } s := &ss.series[ss.current] // zap.L().Debug("seriesSet.At", zap.String("metricName", s.name())) return s } func (ss *seriesSet) Next() bool { if ss == nil || ss.current+1 >= len(ss.series) { // zap.L().Debug("seriesSet.Next", zap.Bool("ret", false)) return false } ss.current++ // zap.L().Debug("seriesSet.Next", zap.Bool("ret", true)) return true } // Warnings ... func (s *seriesSet) Warnings() annotations.Annotations { return nil } // Iterator returns a new iterator of the data of the series. func (s *series) Iterator(iterator chunkenc.Iterator) chunkenc.Iterator { return &seriesIterator{metricName: s.metricName, points: s.points, current: -1, step: s.step} } func (s *series) name() string { return s.metricName } func (s *series) Labels() labels.Labels { return Labels(s.name()) } ================================================ FILE: prometheus/storage.go ================================================ //go:build !noprom // +build !noprom package prometheus import ( "context" "github.com/lomik/graphite-clickhouse/config" "github.com/prometheus/prometheus/storage" ) type storageImpl struct { config *config.Config } var _ storage.Storage = &storageImpl{} func newStorage(config *config.Config) *storageImpl { return &storageImpl{config: config} } // Querier returns a new Querier on the storage. func (s *storageImpl) Querier(mint, maxt int64) (storage.Querier, error) { return &Querier{ config: s.config, mint: mint, maxt: maxt, }, nil } // ChunkQuerier ... func (s *storageImpl) ChunkQuerier(mint, maxt int64) (storage.ChunkQuerier, error) { return nil, nil } // Appender ... func (s *storageImpl) Appender(ctx context.Context) storage.Appender { return nil } // StartTime ... func (s *storageImpl) StartTime() (int64, error) { return 0, nil } // Close ... func (s *storageImpl) Close() error { return nil } ================================================ FILE: render/data/carbonlink.go ================================================ package data import ( "context" "time" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/point" "github.com/lomik/graphite-clickhouse/pkg/scope" "go.uber.org/zap" graphitePickle "github.com/lomik/graphite-pickle" ) type carbonlinkFetcher interface { CacheQueryMulti(context.Context, []string) (map[string][]graphitePickle.DataPoint, error) } // carbonlink to get data from carbonlink server globally type carbonlinkClient struct { carbonlinkFetcher totalTimeout time.Duration } var carbonlink *carbonlinkClient = nil // setCarbonlinkClient setup the client once. Does nothing if Config.Carbonlink.Server is not set func setCarbonlinkClient(config *config.Carbonlink) { if carbonlink != nil { return } if config.Server == "" { return } carbonlink = &carbonlinkClient{ graphitePickle.NewCarbonlinkClient( config.Server, config.Retries, config.Threads, config.ConnectTimeout, config.QueryTimeout, ), config.TotalTimeout, } return } // queryCarbonlink returns callable result fetcher func queryCarbonlink(parentCtx context.Context, carbonlink *carbonlinkClient, metrics []string) func() *point.Points { logger := scope.Logger(parentCtx) if carbonlink == nil { return func() *point.Points { return nil } } carbonlinkResponseChan := make(chan *point.Points, 1) fetchResult := func() *point.Points { result := <-carbonlinkResponseChan return result } go func() { ctx, cancel := context.WithTimeout(parentCtx, carbonlink.totalTimeout) defer cancel() res, err := carbonlink.CacheQueryMulti(ctx, metrics) if err != nil { logger.Info("carbonlink failed", zap.Error(err)) } result := point.NewPoints() if res != nil && len(res) > 0 { tm := uint32(time.Now().Unix()) for metric, points := range res { metricID := result.MetricID(metric) for _, p := range points { result.AppendPoint(metricID, p.Value, uint32(p.Timestamp), tm) } } } carbonlinkResponseChan <- result }() return fetchResult } ================================================ FILE: render/data/carbonlink_test.go ================================================ package data import ( "context" "sort" "testing" "time" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/point" graphitePickle "github.com/lomik/graphite-pickle" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) type carbonlinkMocked struct { mock.Mock } func (c *carbonlinkMocked) CacheQueryMulti(ctx context.Context, metrics []string) (map[string][]graphitePickle.DataPoint, error) { args := c.Called(ctx, metrics) return args.Get(0).(map[string][]graphitePickle.DataPoint), args.Error(1) } func TestSetCarbonlingClient(t *testing.T) { assert.Nil(t, carbonlink, "client is set in the begining of tests") cfg := config.New() cfg.Carbonlink.Server = "localhost:0" setCarbonlinkClient(&cfg.Carbonlink) assert.NotNil(t, carbonlink, "client is not set aftert setCarbonlinkClient") assert.IsType(t, &graphitePickle.CarbonlinkClient{}, carbonlink.carbonlinkFetcher, "") carbonlink = nil } func TestQueryCarbonlink(t *testing.T) { carbonlink = nil res := make(map[string][]graphitePickle.DataPoint) metrics := []string{"metric1", "metric2"} dataPoints := []graphitePickle.DataPoint{ { Timestamp: 1500000000, Value: 13, }, { Timestamp: 1500000060, Value: 14, }, } for _, m := range metrics { res[m] = dataPoints } testGrCarbonlinkClient := new(carbonlinkMocked) testGrCarbonlinkClient.On("CacheQueryMulti", mock.AnythingOfType("*context.timerCtx"), metrics).Return(res, nil) carbonlink = &carbonlinkClient{testGrCarbonlinkClient, time.Duration(0)} now := uint32(time.Now().Unix()) points := queryCarbonlink(context.Background(), carbonlink, metrics)() // Result points.metrics are not ordered pMetrics := []string{points.MetricName(1), points.MetricName(2)} i := 0 for _, m := range pMetrics { for _, dp := range dataPoints { // There is a tiny chance that point will have greated Timestamp than now. Here we test it's at most the next second assert.GreaterOrEqual(t, uint32(1), (points.List()[i].Timestamp - now), "difference between now and point.Timestamp is greater than 1") expectedPoint := point.Point{MetricID: points.MetricID(m), Value: dp.Value, Time: uint32(dp.Timestamp), Timestamp: points.List()[i].Timestamp} assert.Equal(t, expectedPoint, points.List()[i], "point is not correct") i++ } } sort.Strings(pMetrics) assert.Equal(t, metrics, pMetrics, "sorted points.metrics is not the same as in request") carbonlink = nil emptyPoints := queryCarbonlink(context.Background(), carbonlink, metrics)() assert.Nil(t, emptyPoints, "points are not nil") } ================================================ FILE: render/data/ch_response.go ================================================ package data import ( "errors" "math" v2pb "github.com/go-graphite/protocol/carbonapi_v2_pb" v3pb "github.com/go-graphite/protocol/carbonapi_v3_pb" "github.com/lomik/graphite-clickhouse/helper/point" ) // CHResponse contains the parsed Data and From/Until timestamps type CHResponse struct { Data *Data From int64 Until int64 // if true, return points for all metrics, replacing empty results with list of NaN AppendOutEmptySeries bool AppliedFunctions map[string][]string } // CHResponses is a slice of CHResponse type CHResponses []CHResponse // EmptyResponse returns an CHResponses with one element containing emptyData for the following encoding func EmptyResponse() CHResponses { return CHResponses{{Data: emptyData}} } // ToMultiFetchResponseV2 returns protobuf v2pb.MultiFetchResponse message for given CHResponse func (c *CHResponse) ToMultiFetchResponseV2() (*v2pb.MultiFetchResponse, error) { mfr := &v2pb.MultiFetchResponse{Metrics: make([]v2pb.FetchResponse, 0)} data := c.Data addResponse := func(name string, step uint32, points []point.Point) error { from, until := uint32(c.From), uint32(c.Until) start, stop, count, getValue := point.FillNulls(points, from, until, step) values := make([]float64, 0, count) isAbsent := make([]bool, 0, count) for { value, err := getValue() if err != nil { if errors.Is(err, point.ErrTimeGreaterStop) { break } // if err is not point.ErrTimeGreaterStop, the points are corrupted return err } if math.IsNaN(value) { values = append(values, 0) isAbsent = append(isAbsent, true) } else { values = append(values, value) isAbsent = append(isAbsent, false) } } for _, a := range data.AM.Get(name) { fr := v2pb.FetchResponse{ Name: a.DisplayName, StartTime: int32(start), StopTime: int32(stop), StepTime: int32(step), Values: values, IsAbsent: isAbsent, } mfr.Metrics = append(mfr.Metrics, fr) } return nil } // process metrics with points writtenMetrics := make(map[string]struct{}) nextMetric := data.GroupByMetric() for { points := nextMetric() if len(points) == 0 { break } id := points[0].MetricID name := data.MetricName(id) writtenMetrics[name] = struct{}{} step, err := data.GetStep(id) if err != nil { return nil, err } if err := addResponse(name, step, points); err != nil { return nil, err } } // process metrics with no points if c.AppendOutEmptySeries && len(writtenMetrics) < data.AM.Len() && data.CommonStep > 0 { for _, metricName := range data.AM.Series(false) { if _, done := writtenMetrics[metricName]; !done { err := addResponse(metricName, uint32(data.CommonStep), []point.Point{}) if err != nil { return nil, err } } } } return mfr, nil } // ToMultiFetchResponseV2 returns protobuf v2pb.MultiFetchResponse message for given CHResponses func (cc *CHResponses) ToMultiFetchResponseV2() (*v2pb.MultiFetchResponse, error) { mfr := &v2pb.MultiFetchResponse{Metrics: make([]v2pb.FetchResponse, 0)} for _, c := range *cc { m, err := c.ToMultiFetchResponseV2() if err != nil { return nil, err } mfr.Metrics = append(mfr.Metrics, m.Metrics...) } return mfr, nil } // ToMultiFetchResponseV3 returns protobuf v3pb.MultiFetchResponse message for given CHResponse func (c *CHResponse) ToMultiFetchResponseV3() (*v3pb.MultiFetchResponse, error) { mfr := &v3pb.MultiFetchResponse{Metrics: make([]v3pb.FetchResponse, 0)} data := c.Data addResponse := func(name, function string, step uint32, points []point.Point) error { from, until := uint32(c.From), uint32(c.Until) start, stop, count, getValue := point.FillNulls(points, from, until, step) values := make([]float64, 0, count) for { value, err := getValue() if err != nil { if errors.Is(err, point.ErrTimeGreaterStop) { break } // if err is not point.ErrTimeGreaterStop, the points are corrupted return err } values = append(values, value) } for _, a := range data.AM.Get(name) { fr := v3pb.FetchResponse{ Name: a.DisplayName, PathExpression: a.Target, ConsolidationFunc: function, StartTime: int64(start), StopTime: int64(stop), StepTime: int64(step), XFilesFactor: 0, HighPrecisionTimestamps: false, Values: values, AppliedFunctions: c.AppliedFunctions[a.Target], RequestStartTime: c.From, RequestStopTime: c.Until, } mfr.Metrics = append(mfr.Metrics, fr) } return nil } // process metrics with points writtenMetrics := make(map[string]struct{}) nextMetric := data.GroupByMetric() for { points := nextMetric() if len(points) == 0 { break } id := points[0].MetricID name := data.MetricName(id) writtenMetrics[name] = struct{}{} consolidationFunc, err := data.GetAggregation(id) if err != nil { return nil, err } step, err := data.GetStep(id) if err != nil { return nil, err } if err := addResponse(name, consolidationFunc, step, points); err != nil { return nil, err } } // process metrics with no points if c.AppendOutEmptySeries && len(writtenMetrics) < data.AM.Len() && data.CommonStep > 0 { for _, metricName := range data.AM.Series(false) { if _, done := writtenMetrics[metricName]; !done { err := addResponse(metricName, "any", uint32(data.CommonStep), []point.Point{}) if err != nil { return nil, err } } } } return mfr, nil } // ToMultiFetchResponseV3 returns protobuf v3pb.MultiFetchResponse message for given CHResponses func (cc *CHResponses) ToMultiFetchResponseV3() (*v3pb.MultiFetchResponse, error) { mfr := &v3pb.MultiFetchResponse{Metrics: make([]v3pb.FetchResponse, 0)} for _, c := range *cc { m, err := c.ToMultiFetchResponseV3() if err != nil { return nil, err } mfr.Metrics = append(mfr.Metrics, m.Metrics...) } return mfr, nil } ================================================ FILE: render/data/common_step.go ================================================ package data import ( "context" "sync" "time" "github.com/lomik/graphite-clickhouse/pkg/dry" ) // This is used to calculate lowest common multiplier of metrics for ClickHouse internal aggregation // Collect amount of targets; // Wait until all targets will send the step, and calculate the LCM on the fly // Return the calculated LCM() type commonStep struct { result int64 wg sync.WaitGroup lock sync.RWMutex } func (c *commonStep) addTargets(delta int) { c.wg.Add(delta) } func (c *commonStep) doneTarget() { c.wg.Done() } func (c *commonStep) calculateUnsafe(a, b int64) int64 { if a == 0 || b == 0 { return dry.Max(a, b) } return dry.LCM(a, b) } func (c *commonStep) calculate(value int64) { c.lock.Lock() c.result = c.calculateUnsafe(c.result, value) c.lock.Unlock() c.doneTarget() } func (c *commonStep) getResult() int64 { ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) defer cancel() ch := make(chan int64) go func(ch chan int64) { c.wg.Wait() c.lock.RLock() defer c.lock.RUnlock() ch <- c.result }(ch) select { case r := <-ch: return r case <-ctx.Done(): // -1 is a definitely wrong value, it will break following ClickHouse query // This possible, when one of the queries in request already returned error return -1 } } ================================================ FILE: render/data/common_step_test.go ================================================ package data import ( "sync" "testing" "github.com/stretchr/testify/assert" ) type wrapper struct { *commonStep calcCounter int cLock sync.RWMutex } func (w *wrapper) calc(step int64) { w.cLock.Lock() w.calcCounter++ w.calculate(step) w.cLock.Unlock() } func newWrapper() *wrapper { c := &commonStep{ result: 0, wg: sync.WaitGroup{}, lock: sync.RWMutex{}, } return &wrapper{ commonStep: c, cLock: sync.RWMutex{}, } } func TestCommonStepWorker(t *testing.T) { w := newWrapper() w.addTargets(4) go func() { lastStep := int64(0) for i := 0; i < 20000; i++ { w.calculateUnsafe(lastStep, 0) } w.calc(0) assert.Equal(t, int64(120), w.commonStep.getResult()) }() go func() { lastStep := int64(0) for i := 0; i < 30000; i++ { w.calculateUnsafe(lastStep, 6) } w.calc(6) assert.Equal(t, int64(120), w.commonStep.getResult()) }() go func() { lastStep := int64(0) for i := 0; i < 40000; i++ { w.calculateUnsafe(lastStep, 8) } w.calc(8) assert.Equal(t, int64(120), w.commonStep.getResult()) }() go func() { lastStep := int64(0) for i := 0; i < 50000; i++ { w.calculateUnsafe(lastStep, 10) } w.calc(10) assert.Equal(t, int64(120), w.commonStep.getResult()) }() assert.Equal(t, int64(120), w.commonStep.getResult()) assert.Equal(t, 4, w.calcCounter) } ================================================ FILE: render/data/data.go ================================================ package data import ( "bufio" "context" "encoding/binary" "errors" "fmt" "io" "math" "sync" "time" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/helper/point" "github.com/lomik/graphite-clickhouse/pkg/alias" "github.com/lomik/graphite-clickhouse/pkg/reverse" ) var errClickHouseResponse = errors.New("Malformed response from clickhouse") // ReadUvarint reads unsigned int with variable length var ReadUvarint = clickhouse.ReadUvarint // Data stores parsed response from ClickHouse server type Data struct { *point.Points AM *alias.Map CommonStep int64 } var emptyData *Data = &Data{Points: point.NewPoints(), AM: alias.New()} func contextIsValid(ctx context.Context) error { select { case <-ctx.Done(): return ctx.Err() default: return nil } } // GetStep returns the commonStep for all points or, if unset, step for metric ID id func (d *Data) GetStep(id uint32) (uint32, error) { if 0 < d.CommonStep { return uint32(d.CommonStep), nil } return d.Points.GetStep(id) } // GetAggregation returns the generic whisper compatible name for an aggregation of metric with ID id func (d *Data) GetAggregation(id uint32) (string, error) { function, err := d.Points.GetAggregation(id) if err != nil { return function, err } switch function { case "any": return "first", nil case "anyLast": return "last", nil default: return function, nil } } // data wraps Data and adds asynchronous processing of data // data.wait() should be used with the same context as prepareData and parseResponse to check the error type data struct { *Data length int // readed bytes count spent time.Duration // time spent on parsing b chan io.ReadCloser e chan error mut sync.RWMutex wg sync.WaitGroup } // prepareData returns new data with asynchronous processing points from carbonlinkClient func prepareData(ctx context.Context, targets int, fetcher func() *point.Points) *data { data := &data{ Data: &Data{Points: point.NewPoints()}, b: make(chan io.ReadCloser, 1), e: make(chan error, targets), mut: sync.RWMutex{}, wg: sync.WaitGroup{}, } data.wg.Add(1) extraPoints := make(chan *point.Points, 1) go func() { // add extraPoints. With NameToID defer func() { data.wg.Done() close(extraPoints) }() // First check is context is already done if err := contextIsValid(ctx); err != nil { data.e <- fmt.Errorf("prepareData failed: %w", err) return } select { case extraPoints <- fetcher(): p := <-extraPoints if p != nil { data.mut.Lock() defer data.mut.Unlock() extraList := p.List() for i := 0; i < len(extraList); i++ { data.Points.AppendPoint( data.Points.MetricID(p.MetricName(extraList[i].MetricID)), extraList[i].Value, extraList[i].Time, extraList[i].Timestamp, ) } } return case <-ctx.Done(): data.e <- fmt.Errorf("prepareData failed: %w", ctx.Err()) return } }() return data } // setSteps sets commonStep for aggregated requests and per-metric step for non-aggregated func (d *data) setSteps(cond *conditions) { if cond.aggregated { d.CommonStep = cond.step return } d.Points.SetSteps(cond.steps) } // Error handler for data splitting functions func splitErrorHandler(data *[]byte, atEOF bool, tokenLen int, err error) (int, []byte, error) { if err == clickhouse.ErrUvarintRead { if atEOF { return 0, nil, clickhouse.NewErrWithDescr(errClickHouseResponse.Error(), string(*data)) } // signal for read more return 0, nil, nil } else if err != nil || (len(*data) < tokenLen && atEOF) { return 0, nil, clickhouse.NewErrWithDescr(errClickHouseResponse.Error(), string(*data)) } // signal for read more return 0, nil, nil } // dataSplitAggregated is a split function for bufio.Scanner for read row binary response for queries of aggregated data func dataSplitAggregated(data []byte, atEOF bool) (advance int, token []byte, err error) { if len(data) == 0 && atEOF { // stop return 0, nil, nil } nameLen, readBytes, err := ReadUvarint(data) tokenLen := readBytes + int(nameLen) if err != nil || len(data) < tokenLen { return splitErrorHandler(&data, atEOF, tokenLen, err) } timeLen, readBytes, err := ReadUvarint(data[tokenLen:]) tokenLen += readBytes + int(timeLen)*4 if err != nil || len(data) < tokenLen { return splitErrorHandler(&data, atEOF, tokenLen, err) } valueLen, readBytes, err := ReadUvarint(data[tokenLen:]) tokenLen += readBytes + int(valueLen)*8 if err != nil || len(data) < tokenLen { return splitErrorHandler(&data, atEOF, tokenLen, err) } if timeLen != valueLen { return 0, nil, clickhouse.NewErrWithDescr(errClickHouseResponse.Error()+": Different amount of Times and Values", string(data)) } return tokenLen, data[:tokenLen], nil } // dataSplitUnaggregated is a split function for bufio.Scanner for read row binary response for queries of unaggregated data func dataSplitUnaggregated(data []byte, atEOF bool) (advance int, token []byte, err error) { if len(data) == 0 && atEOF { // stop return 0, nil, nil } nameLen, readBytes, err := ReadUvarint(data) tokenLen := readBytes + int(nameLen) if err != nil || len(data) < tokenLen { return splitErrorHandler(&data, atEOF, tokenLen, err) } timeLen, readBytes, err := ReadUvarint(data[tokenLen:]) tokenLen += readBytes + int(timeLen)*4 if err != nil || len(data) < tokenLen { return splitErrorHandler(&data, atEOF, tokenLen, err) } valueLen, readBytes, err := ReadUvarint(data[tokenLen:]) tokenLen += readBytes + int(valueLen)*8 if err != nil || len(data) < tokenLen { return splitErrorHandler(&data, atEOF, tokenLen, err) } timestampLen, readBytes, err := ReadUvarint(data[tokenLen:]) tokenLen += readBytes + int(timestampLen)*4 if err != nil || len(data) < tokenLen { return splitErrorHandler(&data, atEOF, tokenLen, err) } if timeLen != valueLen || timeLen != timestampLen { return 0, nil, clickhouse.NewErrWithDescr(errClickHouseResponse.Error()+": Different amount of Values, Times and Timestamps", string(data)) } return tokenLen, data[:tokenLen], nil } // readResponse reads the ClickHouse body into *Data and merges with extraPoints. // Expected, that on error the context will be cancelled on the upper level. func (d *data) parseResponse(ctx context.Context, bodyReader io.ReadCloser, cond *conditions) error { pp := d.Points dataSplit := dataSplitUnaggregated if cond.aggregated { dataSplit = dataSplitAggregated } // Prevent starting parser if context is done if err := contextIsValid(ctx); err != nil { return ctx.Err() } // Then wait if there is an active parser working select { case d.b <- bodyReader: case <-ctx.Done(): return fmt.Errorf("parseResponse failed: %w", ctx.Err()) } var metricID uint32 d.mut.Lock() defer func() { d.mut.Unlock() <-d.b }() // Are we still good to go? if err := contextIsValid(ctx); err != nil { return fmt.Errorf("parseResponse failed: %w", ctx.Err()) } start := time.Now() scanner := bufio.NewScanner(bodyReader) scanner.Buffer(make([]byte, 1048576), 67108864) scanner.Split(dataSplit) var rowStart []byte for scanner.Scan() { rowStart = scanner.Bytes() d.length += len(rowStart) nameLen, readBytes, err := ReadUvarint(rowStart) if err != nil { return errClickHouseResponse } row := rowStart[readBytes:] name := row[:int(nameLen)] row = row[int(nameLen):] if cond.isReverse { metricID = pp.MetricIDBytes(reverse.Bytes(name)) } else { metricID = pp.MetricIDBytes(name) } arrayLen, readBytes, err := ReadUvarint(row) if err != nil { return errClickHouseResponse } times := make([]uint32, 0, arrayLen) values := make([]float64, 0, arrayLen) row = row[readBytes:] for i := uint64(0); i < arrayLen; i++ { times = append(times, binary.LittleEndian.Uint32(row[:4])) row = row[4:] } row = row[readBytes:] for i := uint64(0); i < arrayLen; i++ { values = append(values, math.Float64frombits(binary.LittleEndian.Uint64(row[:8]))) row = row[8:] } timestamps := times if !cond.aggregated { timestamps = make([]uint32, 0, arrayLen) row = row[readBytes:] for i := uint64(0); i < arrayLen; i++ { timestamps = append(timestamps, binary.LittleEndian.Uint32(row[:4])) row = row[4:] } } for i := range times { pp.AppendPoint(metricID, values[i], times[i], timestamps[i]) } } d.spent += time.Since(start) err := scanner.Err() if err != nil { dataErr, ok := err.(*clickhouse.ErrWithDescr) if ok { // format full error string, sometimes parse not failed at start orf error string dataErr.PrependDescription(string(rowStart)) } bodyReader.Close() return err } return nil } func (d *data) wait(ctx context.Context) error { // First check is context is already done if err := contextIsValid(ctx); err != nil { return fmt.Errorf("prepareData failed: %w", err) } // if anything is already in error channel select { case err := <-d.e: return err default: } // watch if workers are done parsersDone := make(chan struct{}, 1) go func() { d.wg.Wait() parsersDone <- struct{}{} }() // and watch for all channels select { case <-ctx.Done(): return ctx.Err() case err := <-d.e: return err case <-parsersDone: return nil } } ================================================ FILE: render/data/data_parse_test.go ================================================ package data import ( "bytes" "context" "fmt" "io" "testing" "time" "github.com/lomik/graphite-clickhouse/helper/RowBinary" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/helper/point" "github.com/lomik/graphite-clickhouse/pkg/reverse" "github.com/stretchr/testify/assert" ) type pointValues struct { Values []float64 Times []uint32 Timestamps []uint32 } type testPoint struct { Metric string PointValues *pointValues } func makeAggregatedBody(points []testPoint) []byte { buf := new(bytes.Buffer) w := RowBinary.NewEncoder(buf) for i := 0; i < len(points); i++ { w.String(points[i].Metric) w.Uint32List(points[i].PointValues.Times) w.Float64List(points[i].PointValues.Values) } return buf.Bytes() } func makeUnaggregatedBody(points []testPoint) []byte { buf := new(bytes.Buffer) w := RowBinary.NewEncoder(buf) for i := 0; i < len(points); i++ { w.String(points[i].Metric) w.Uint32List(points[i].PointValues.Times) w.Float64List(points[i].PointValues.Values) w.Uint32List(points[i].PointValues.Timestamps) } return buf.Bytes() } func testCarbonlinkReaderNil() *point.Points { return nil } func TestUnaggregatedDataParse(t *testing.T) { ctx := context.Background() cond := &conditions{Targets: &Targets{isReverse: false}, aggregated: false} t.Run("empty response", func(t *testing.T) { body := []byte{} r := io.NopCloser(bytes.NewReader(body)) d := prepareData(ctx, 1, testCarbonlinkReaderNil) err := d.parseResponse(ctx, r, cond) assert.NoError(t, err) werr := d.wait(ctx) assert.NoError(t, werr) assert.Empty(t, d.Points.List()) }) table := [][]testPoint{ { {"hello.world", &pointValues{[]float64{42.1}, []uint32{1520056686}, []uint32{1520056706}}}, }, { {"hello.world", &pointValues{[]float64{42.1}, []uint32{1520056686}, []uint32{1520056706}}}, {"foobar", &pointValues{[]float64{42.2}, []uint32{1520056687}, []uint32{1520056707}}}, }, { {"samelen1", &pointValues{[]float64{42.1}, []uint32{1520056686}, []uint32{1520056706}}}, {"samelen2", &pointValues{[]float64{42.2}, []uint32{1520056687}, []uint32{1520056707}}}, }, { {"long.metric.with.points.key1", &pointValues{[]float64{42.1, 42.2}, []uint32{1520056686, 1520056687}, []uint32{1520056706, 1520056687}}}, {"long.metric.with.points.key2", &pointValues{[]float64{42.2}, []uint32{1520056687}, []uint32{1520056707}}}, }, } for i := 0; i < len(table); i++ { t.Run(fmt.Sprintf("ok #%d", i), func(t *testing.T) { body := makeUnaggregatedBody(table[i]) r := io.NopCloser(bytes.NewReader(body)) d := prepareData(ctx, 1, testCarbonlinkReaderNil) err := d.parseResponse(ctx, r, cond) assert.NoError(t, err) werr := d.wait(ctx) assert.NoError(t, werr) // point number p := 0 for j := 0; j < len(table[i]); j++ { for m := 0; m < len(table[i][j].PointValues.Times); m++ { assert.Equal(t, table[i][j].Metric, d.Points.MetricName(d.Points.List()[p].MetricID)) assert.Equal(t, table[i][j].PointValues.Times[m], d.Points.List()[p].Time) assert.Equal(t, table[i][j].PointValues.Values[m], d.Points.List()[p].Value) assert.Equal(t, table[i][j].PointValues.Timestamps[m], d.Points.List()[p].Timestamp) p++ } } }) } for i := 0; i < len(table); i++ { t.Run(fmt.Sprintf("reversed #%d", i), func(t *testing.T) { cond := &conditions{Targets: &Targets{isReverse: true}, aggregated: false} body := makeUnaggregatedBody(table[i]) r := io.NopCloser(bytes.NewReader(body)) d := prepareData(ctx, 1, testCarbonlinkReaderNil) err := d.parseResponse(ctx, r, cond) assert.NoError(t, err) werr := d.wait(ctx) assert.NoError(t, werr) // point number p := 0 for j := 0; j < len(table[i]); j++ { for m := 0; m < len(table[i][j].PointValues.Times); m++ { assert.Equal(t, table[i][j].Metric, reverse.String(d.Points.MetricName(d.Points.List()[p].MetricID))) assert.Equal(t, table[i][j].PointValues.Times[m], d.Points.List()[p].Time) assert.Equal(t, table[i][j].PointValues.Values[m], d.Points.List()[p].Value) assert.Equal(t, table[i][j].PointValues.Timestamps[m], d.Points.List()[p].Timestamp) p++ } } }) } t.Run("malformed ClickHouse body", func(t *testing.T) { body := makeUnaggregatedBody([]testPoint{ { Metric: "hello.world", PointValues: &pointValues{ Values: []float64{42.1}, Times: []uint32{1520056686}, Timestamps: []uint32{1520056706, 1520056707}, }, }, }) r := io.NopCloser(bytes.NewReader(body)) d := prepareData(ctx, 1, testCarbonlinkReaderNil) err := d.parseResponse(ctx, r, cond) assert.Error(t, err) }) t.Run("incomplete response", func(t *testing.T) { points := []testPoint{ { Metric: "hello.world", PointValues: &pointValues{ Values: []float64{42.1}, Times: []uint32{1520056686}, Timestamps: []uint32{1520056706}, }, }, { Metric: "bye-bye.sky", PointValues: &pointValues{ Values: []float64{42.42}, Times: []uint32{1520056686}, Timestamps: []uint32{1520056706}, }, }, } body := makeUnaggregatedBody(points) firstMetricLength := len(makeUnaggregatedBody(points[:1])) for i := 1; i < len(body)-1; i++ { if i == firstMetricLength { // length of the first metric continue } r := io.NopCloser(bytes.NewReader(body[:i])) d := prepareData(ctx, 1, testCarbonlinkReaderNil) err := d.parseResponse(ctx, r, cond) assert.Error(t, err) assert.True(t, (d.length == 0 || d.length == firstMetricLength), "length of read data is wrong") } }) } func TestAggregatedDataParse(t *testing.T) { ctx := context.Background() cond := &conditions{Targets: &Targets{isReverse: false}, aggregated: true} t.Run("empty response", func(t *testing.T) { body := []byte{} d := prepareData(ctx, 1, testCarbonlinkReaderNil) r := io.NopCloser(bytes.NewReader(body)) err := d.parseResponse(ctx, r, cond) assert.NoError(t, err) assert.Empty(t, d.Points.List()) }) t.Run("incomplete response", func(t *testing.T) { points := []testPoint{ { Metric: "hello.world", PointValues: &pointValues{ Values: []float64{42.1}, Times: []uint32{1520056686}, }, }, { Metric: "bye-bye.sky", PointValues: &pointValues{ Values: []float64{42.1}, Times: []uint32{1520056686}, }, }, } body := makeAggregatedBody(points) firstMetricLength := len(makeAggregatedBody(points[:1])) for i := 1; i < len(body)-1; i++ { if i == firstMetricLength { // length of the first metric continue } r := io.NopCloser(bytes.NewReader(body[:i])) d := prepareData(ctx, 1, testCarbonlinkReaderNil) err := d.parseResponse(ctx, r, cond) assert.Error(t, err) assert.True(t, (d.length == 0 || d.length == firstMetricLength), "length of read data is wrong") } }) t.Run("malformed ClickHouse body", func(t *testing.T) { points := []testPoint{ { // different length of -Resample arrays Metric: "different.arrays.in.body", PointValues: &pointValues{ Values: []float64{42.1}, Times: []uint32{1520056706, 1520056707}, }, }, } body := makeAggregatedBody(points) r := io.NopCloser(bytes.NewReader(body)) d := prepareData(ctx, 1, testCarbonlinkReaderNil) err := d.parseResponse(ctx, r, cond) assert.Error(t, err) }) t.Run("normal work", func(t *testing.T) { points := []testPoint{ { Metric: "hello.world", PointValues: &pointValues{ Values: []float64{42.1}, Times: []uint32{1520056686}, }, }, { Metric: "null.in.the.middle", PointValues: &pointValues{ Values: []float64{42.1, 43}, Times: []uint32{1520056686, 1520056690}, }, }, } body := makeAggregatedBody(points) r := io.NopCloser(bytes.NewReader(body)) d := prepareData(ctx, 1, testCarbonlinkReaderNil) err := d.parseResponse(ctx, r, cond) result := []point.Point{ {MetricID: 1, Value: 42.1, Time: 1520056686, Timestamp: 1520056686}, {MetricID: 2, Value: 42.1, Time: 1520056686, Timestamp: 1520056686}, {MetricID: 2, Value: 43, Time: 1520056690, Timestamp: 1520056690}, } assert.NoError(t, err) assert.Equal(t, result, d.Points.List()) }) t.Run("reversed", func(t *testing.T) { points := []testPoint{ { Metric: "hello.world", PointValues: &pointValues{ Values: []float64{42.1}, Times: []uint32{1520056686}, }, }, { Metric: "null.in.the.middle", PointValues: &pointValues{ Values: []float64{42.1, 43}, Times: []uint32{1520056686, 1520056690}, }, }, } body := makeAggregatedBody(points) r := io.NopCloser(bytes.NewReader(body)) cond := &conditions{Targets: &Targets{isReverse: true}, aggregated: true} d := prepareData(ctx, 1, testCarbonlinkReaderNil) err := d.parseResponse(ctx, r, cond) assert.NoError(t, err) assert.Equal(t, "world.hello", d.Points.MetricName(1)) assert.Equal(t, "middle.the.in.null", d.Points.MetricName(2)) }) } func TestPrepareDataParse(t *testing.T) { ctx := context.Background() t.Run("empty datapoints", func(t *testing.T) { data := prepareData(ctx, 1, testCarbonlinkReaderNil) err := data.wait(ctx) assert.NoError(t, err) assert.Equal(t, &Data{Points: point.NewPoints()}, data.Data) }) t.Run("cancelled context", func(t *testing.T) { ctx, cancel := context.WithCancel(ctx) cancel() data := prepareData(ctx, 1, testCarbonlinkReaderNil) err := data.wait(ctx) assert.ErrorIs(t, err, context.Canceled) assert.Equal(t, &Data{Points: point.NewPoints()}, data.Data) }) t.Run("data contains points", func(t *testing.T) { //points := []point.Point{{1, 42.1, 1520056686, 1520056686}} extraPoints := point.NewPoints() extraPoints.MetricID("some.metric1") extraPoints.MetricID("some.metric2") extraPoints.AppendPoint(1, 1, 3, 3) extraPoints.AppendPoint(2, 1, 3, 3) reader := func() *point.Points { time.Sleep(1 * time.Millisecond) return extraPoints } d := prepareData(ctx, 1, reader) err := d.wait(ctx) assert.NoError(t, err) assert.Equal( t, []point.Point{ {MetricID: 1, Value: 1, Time: 3, Timestamp: 3}, {MetricID: 2, Value: 1, Time: 3, Timestamp: 3}, }, d.Points.List(), ) assert.Equal(t, "some.metric1", d.Points.MetricName(1)) assert.Equal(t, "some.metric2", d.Points.MetricName(2)) }) } func TestAsyncDataParse(t *testing.T) { ctx := context.Background() cond := &conditions{Targets: &Targets{isReverse: false}, aggregated: false} // normal work is tested in other places t.Run("context deadline exceeded", func(t *testing.T) { extraPoints := point.NewPoints() extraPoints.MetricID("some.metric1") extraPoints.MetricID("some.metric2") extraPoints.AppendPoint(1, 1, 3, 3) extraPoints.AppendPoint(2, 1, 3, 3) reader := func() *point.Points { return extraPoints } ctx, cancel := context.WithTimeout(ctx, -1*time.Nanosecond) defer cancel() d := prepareData(ctx, 1, reader) assert.Len(t, d.Points.List(), 0, "timeout should prevent points parsing") body := makeUnaggregatedBody([]testPoint{ { Metric: "hello.world", PointValues: &pointValues{ Values: []float64{42.1}, Times: []uint32{1520056686}, Timestamps: []uint32{1520056706}, }, }, }) r := io.NopCloser(bytes.NewReader(body)) err := d.parseResponse(ctx, r, cond) assert.ErrorIs(t, err, context.DeadlineExceeded, "parseResponse shouldn't return error on a done context") assert.Len(t, d.Points.List(), 0, "timeout should prevent points parsing") err = d.wait(ctx) assert.ErrorIs(t, err, context.DeadlineExceeded, "data.wait returns 'context dedline exceeded'") }) t.Run("context deadline faster than carbonlink reader", func(t *testing.T) { extraPoints := point.NewPoints() extraPoints.MetricID("some.metric1") extraPoints.MetricID("some.metric2") extraPoints.AppendPoint(1, 1, 3, 3) extraPoints.AppendPoint(2, 1, 3, 3) reader := func() *point.Points { time.Sleep(1 * time.Second) return extraPoints } ctx, cancel := context.WithTimeout(ctx, 50*time.Nanosecond) defer cancel() d := prepareData(ctx, 1, reader) err := d.wait(ctx) assert.Len(t, d.Points.List(), 0, "timeout should prevent points parsing") assert.ErrorIs(t, err, context.DeadlineExceeded, "data.wait returns 'context dedline exceeded'") }) t.Run("cancel context before different steps", func(t *testing.T) { ctx, cancel := context.WithCancel(ctx) body := []byte{} // works fine d := prepareData(ctx, 1, testCarbonlinkReaderNil) r := io.NopCloser(bytes.NewReader(body)) err := d.parseResponse(ctx, r, cond) assert.NoError(t, err) err = d.wait(ctx) assert.NoError(t, err) cancel() // fails after context is cancelled err = d.wait(ctx) assert.ErrorIs(t, err, context.Canceled) r = io.NopCloser(bytes.NewReader(body)) err = d.parseResponse(ctx, r, cond) assert.ErrorIs(t, err, context.Canceled) }) t.Run("wait fails on errors", func(t *testing.T) { d := prepareData(ctx, 1, testCarbonlinkReaderNil) d.e <- clickhouse.ErrClickHouseResponse err := d.wait(ctx) assert.ErrorIs(t, err, clickhouse.ErrClickHouseResponse, "err %v is not expected", err) }) } ================================================ FILE: render/data/multi_target.go ================================================ package data import ( "context" "fmt" "net/http" "sync" "time" v3pb "github.com/go-graphite/protocol/carbonapi_v3_pb" "go.uber.org/zap" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/errs" "github.com/lomik/graphite-clickhouse/limiter" "github.com/lomik/graphite-clickhouse/pkg/alias" "github.com/lomik/graphite-clickhouse/pkg/scope" ) // TimeFrame contains information about fetch request time conditions type TimeFrame struct { From int64 Until int64 MaxDataPoints int64 } // MultiTarget is a map of TimeFrame keys and targets slice of strings values type MultiTarget map[TimeFrame]*Targets func MFRToMultiTarget(v3Request *v3pb.MultiFetchRequest) MultiTarget { multiTarget := make(MultiTarget) if len(v3Request.Metrics) > 0 { for _, m := range v3Request.Metrics { tf := TimeFrame{ From: m.StartTime, Until: m.StopTime, MaxDataPoints: m.MaxDataPoints, } if _, ok := multiTarget[tf]; ok { target := multiTarget[tf] target.Append(m.PathExpression) } else { multiTarget[tf] = NewTargetsOne(m.PathExpression, len(v3Request.Metrics), alias.New()) } if len(m.FilterFunctions) > 0 { multiTarget[tf].SetFilteringFunctions(m.PathExpression, m.FilterFunctions) } } } return multiTarget } func (m *MultiTarget) checkMetricsLimitExceeded(num int) error { if num <= 0 { // zero or negative means unlimited return nil } for _, t := range *m { if num < t.AM.Len() { return errs.NewErrorWithCode(fmt.Sprintf("metrics limit exceeded: %d < %d", num, t.AM.Len()), http.StatusForbidden) } } return nil } func getDataTimeout(cfg *config.Config, m *MultiTarget) time.Duration { dataTimeout := cfg.ClickHouse.DataTimeout if len(cfg.ClickHouse.QueryParams) > 1 { var maxDuration time.Duration for tf := range *m { duration := time.Second * time.Duration(tf.Until-tf.From) if duration >= maxDuration { maxDuration = duration } } n := config.GetQueryParam(cfg.ClickHouse.QueryParams, maxDuration) return cfg.ClickHouse.QueryParams[n].DataTimeout } return dataTimeout } func GetQueryLimiter(username string, cfg *config.Config, m *MultiTarget) (string, limiter.ServerLimiter) { n := 0 if username != "" && len(cfg.ClickHouse.UserLimits) > 0 { if u, ok := cfg.ClickHouse.UserLimits[username]; ok { return username, u.Limiter } } if len(cfg.ClickHouse.QueryParams) > 1 { var maxDuration time.Duration for tf := range *m { duration := time.Second * time.Duration(tf.Until-tf.From) if duration >= maxDuration { maxDuration = duration } } n = config.GetQueryParam(cfg.ClickHouse.QueryParams, maxDuration) } return "", cfg.ClickHouse.QueryParams[n].Limiter } func GetQueryLimiterFrom(username string, cfg *config.Config, from, until int64) limiter.ServerLimiter { n := 0 if username != "" && len(cfg.ClickHouse.UserLimits) > 0 { if u, ok := cfg.ClickHouse.UserLimits[username]; ok { return u.Limiter } } if len(cfg.ClickHouse.QueryParams) > 1 { n = config.GetQueryParam(cfg.ClickHouse.QueryParams, time.Second*time.Duration(until-from)) } return cfg.ClickHouse.QueryParams[n].Limiter } func GetQueryParam(username string, cfg *config.Config, m *MultiTarget) (*config.QueryParam, int) { n := 0 if len(cfg.ClickHouse.QueryParams) > 1 { var maxDuration time.Duration for tf := range *m { duration := time.Second * time.Duration(tf.Until-tf.From) if duration >= maxDuration { maxDuration = duration } } n = config.GetQueryParam(cfg.ClickHouse.QueryParams, maxDuration) } return &cfg.ClickHouse.QueryParams[n], n } // Fetch fetches the parsed ClickHouse data returns CHResponses func (m *MultiTarget) Fetch(ctx context.Context, cfg *config.Config, chContext string, qlimiter limiter.ServerLimiter, queueDuration *time.Duration) (CHResponses, error) { var ( lock sync.RWMutex wg sync.WaitGroup entered int ) logger := scope.Logger(ctx) setCarbonlinkClient(&cfg.Carbonlink) err := m.checkMetricsLimitExceeded(cfg.Common.MaxMetricsPerTarget) if err != nil { logger.Error("data fetch", zap.Error(err)) return nil, err } dataTimeout := getDataTimeout(cfg, m) ctxTimeout, cancel := context.WithTimeout(ctx, dataTimeout) defer func() { for i := 0; i < entered; i++ { qlimiter.Leave(ctxTimeout, "render") } cancel() }() errors := make([]error, 0, len(*m)) query := newQuery(cfg, len(*m)) for tf, targets := range *m { tf, targets := tf, targets cond := &conditions{TimeFrame: &tf, Targets: targets, aggregated: cfg.ClickHouse.InternalAggregation, appendEmptySeries: cfg.Common.AppendEmptySeries, } if cond.MaxDataPoints <= 0 || int64(cfg.ClickHouse.MaxDataPoints) < cond.MaxDataPoints { cond.MaxDataPoints = int64(cfg.ClickHouse.MaxDataPoints) } err := cond.selectDataTable(cfg, cond.TimeFrame, chContext) if err != nil { lock.Lock() errors = append(errors, err) lock.Unlock() logger.Error("data tables is not specified", zap.Error(err)) return EmptyResponse(), err } if qlimiter.Enabled() { start := time.Now() err = qlimiter.Enter(ctxTimeout, "render") *queueDuration += time.Since(start) if err != nil { // status = http.StatusServiceUnavailable // queueFail = true // http.Error(w, err.Error(), status) lock.Lock() errors = append(errors, err) lock.Unlock() break } entered++ } wg.Add(1) go func(cond *conditions) { defer wg.Done() err := query.getDataPoints(ctxTimeout, cond) if err != nil { lock.Lock() errors = append(errors, err) lock.Unlock() return } }(cond) } wg.Wait() for len(errors) != 0 { return EmptyResponse(), errors[0] } return query.CHResponses, nil } ================================================ FILE: render/data/multi_target_test.go ================================================ package data import ( "testing" "time" "github.com/lomik/graphite-clickhouse/config" ) func Test_getDataTimeout(t *testing.T) { tests := []struct { name string cfg *config.Config m *MultiTarget want time.Duration }{ { name: "one DataTimeout", cfg: &config.Config{ ClickHouse: config.ClickHouse{ DataTimeout: time.Second, QueryParams: []config.QueryParam{ { // default params Duration: 0, DataTimeout: time.Second, }, }, }, }, m: &MultiTarget{ TimeFrame{ From: 1647198000, Until: 1647234000, }: &Targets{}, }, want: time.Second, }, { name: "default DataTimeout", cfg: &config.Config{ ClickHouse: config.ClickHouse{ DataTimeout: time.Second, QueryParams: []config.QueryParam{ { // default params Duration: 0, DataTimeout: time.Second, }, { Duration: time.Hour, DataTimeout: time.Minute, }, }, }, }, m: &MultiTarget{ TimeFrame{ // 1 hour - 1s From: 1647198000, Until: 1647201600 - 1, }: &Targets{}, }, want: time.Second, }, { name: "1m DataTimeout (1 param), select 1h duration", cfg: &config.Config{ ClickHouse: config.ClickHouse{ DataTimeout: time.Second * 10, QueryParams: []config.QueryParam{ { // default params Duration: 0, DataTimeout: time.Second, }, { Duration: time.Hour, DataTimeout: time.Minute, }, }, }, }, m: &MultiTarget{ TimeFrame{ // 1 hour From: 1647198000, Until: 1647201600, }: &Targets{}, }, want: time.Minute, }, { name: "1m DataTimeout (2 param), select 1h duration", cfg: &config.Config{ ClickHouse: config.ClickHouse{ DataTimeout: time.Second, QueryParams: []config.QueryParam{ { // default params Duration: 0, DataTimeout: time.Second, }, { Duration: time.Hour, DataTimeout: time.Minute, }, { Duration: time.Hour * 2, DataTimeout: 10 * time.Minute, }, }, }, }, m: &MultiTarget{ TimeFrame{ // 1 hour From: 1647198000, Until: 1647201600, }: &Targets{}, }, want: time.Minute, }, { name: "10m DataTimeout (2 param), select 2h1s duration", cfg: &config.Config{ ClickHouse: config.ClickHouse{ DataTimeout: time.Second, QueryParams: []config.QueryParam{ { // default params Duration: 0, DataTimeout: time.Second, }, { Duration: time.Hour, DataTimeout: time.Minute, }, { Duration: time.Hour * 2, DataTimeout: 10 * time.Minute, }, }, }, }, m: &MultiTarget{ TimeFrame{ // 2 hour 1s From: 1647198000, Until: 1647205201, }: &Targets{}, }, want: 10 * time.Minute, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := getDataTimeout(tt.cfg, tt.m); got != tt.want { t.Errorf("getDataTimeout() = %v, want %v", got, tt.want) } }) } } ================================================ FILE: render/data/query.go ================================================ package data import ( "context" "crypto/tls" "errors" "fmt" "net/http" "os" "strings" "sync" "sync/atomic" "time" "go.uber.org/zap" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/helper/errs" "github.com/lomik/graphite-clickhouse/helper/rollup" "github.com/lomik/graphite-clickhouse/metrics" "github.com/lomik/graphite-clickhouse/pkg/dry" "github.com/lomik/graphite-clickhouse/pkg/reverse" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/pkg/where" ) // from, until, step, function, table, prewhere, where // arrayFilter(x->isNotNull(x)) - do not pass nulls to client // -Resample - group time and values by time intervals and apply aggregation function // -OrNull - if there aren't points in an interval, null will be returned // intDiv(Time, x)*x - round Time down to step multiplier // TODO: support custom aggregating functions const queryAggregated = `WITH anyResample(%[1]d, %[2]d, %[3]d)(toUInt32(intDiv(Time, %[3]d)*%[3]d), Time) AS mask SELECT Path, arrayFilter(m->m!=0, mask) AS times, arrayFilter((v,m)->m!=0, %[4]sResample(%[1]d, %[2]d, %[3]d)(Value, Time), mask) AS values FROM %[5]s %[6]s %[7]s GROUP BY Path FORMAT RowBinary` // table, prewhere, where const queryUnaggregated = `SELECT Path, groupArray(Time), groupArray(Value), groupArray(Timestamp) FROM %s %s %s GROUP BY Path FORMAT RowBinary` // name of external-data table with metrics paths const extTableName = "metrics_list" type query struct { CHResponses cStep *commonStep chTLSConfig *tls.Config chQueryParams []config.QueryParam chConnectTimeout time.Duration chProgressSendingInterval time.Duration debugDir string debugExtDataPerm os.FileMode featureFlags *config.FeatureFlags lock sync.RWMutex } type conditions struct { *TimeFrame *Targets // aggregated shows is it request with ClickHouse aggregation or not aggregated bool // step is used in requests for proper until/from calculation. It's max(steps) for non-aggregated // requests and LCM(steps) for aggregated requests step int64 // from is aligned to step from int64 // until is aligned to step until int64 // metricUnreversed grouped by step steps map[uint32][]string // prewhere contains PREWHERE condition prewhere string // where contains WHERE condition where string // show list of NaN values instead of empty results appendEmptySeries bool // metricUnreversed grouped by aggregating function aggregations map[string][]string // External-data bodies grouped by aggregatig function. For non-aggregated requests "" used as a key extDataBodies map[string]*strings.Builder metricsRequested []string metricsUnreverse []string metricsLookup []string appliedFunctions map[string][]string } func newQuery(cfg *config.Config, targets int) *query { var cStep *commonStep = nil if cfg.ClickHouse.InternalAggregation { cStep = &commonStep{ result: 0, wg: sync.WaitGroup{}, lock: sync.RWMutex{}, } cStep.addTargets(targets) } query := &query{ CHResponses: make([]CHResponse, 0, targets), cStep: cStep, chQueryParams: cfg.ClickHouse.QueryParams, chConnectTimeout: cfg.ClickHouse.ConnectTimeout, chProgressSendingInterval: cfg.ClickHouse.ProgressSendingInterval, chTLSConfig: cfg.ClickHouse.TLSConfig, debugDir: cfg.Debug.Directory, debugExtDataPerm: cfg.Debug.ExternalDataPerm, featureFlags: &cfg.FeatureFlags, lock: sync.RWMutex{}, } return query } func (q *query) appendReply(chr CHResponse) { q.lock.Lock() q.CHResponses = append(q.CHResponses, chr) q.lock.Unlock() } func (q *query) getParam(from, until int64) (string, time.Duration) { duration := time.Second * time.Duration(until-from) n := config.GetQueryParam(q.chQueryParams, duration) return q.chQueryParams[n].URL, q.chQueryParams[n].DataTimeout } func (q *query) getDataPoints(ctx context.Context, cond *conditions) error { logger := scope.Logger(ctx) var err error cond.prepareMetricsLists() if len(cond.metricsRequested) == 0 { q.cStep.doneTarget() return nil } // carbonlink request carbonlinkResponseRead := queryCarbonlink(ctx, carbonlink, cond.metricsUnreverse) err = cond.prepareLookup() if err != nil { logger.Error("prepare_lookup", zap.Error(err)) return errs.NewErrorWithCode(err.Error(), http.StatusBadRequest) } cond.setStep(q.cStep) if cond.step < 1 { return ErrSetStepTimeout } cond.setFromUntil() cond.setPrewhere() cond.setWhere() queryContext, queryCancel := context.WithCancel(ctx) defer queryCancel() data := prepareData(queryContext, len(cond.extDataBodies), carbonlinkResponseRead) var ch_read_bytes, ch_read_rows int64 for agg, extTableBody := range cond.extDataBodies { extData := q.metricsListExtData(extTableBody) query := cond.generateQuery(agg) data.wg.Add(1) go func() { defer data.wg.Done() chURL, chDataTimeout := q.getParam(cond.from, cond.until) body, err := clickhouse.Reader( scope.WithTable(ctx, cond.pointsTable), chURL, query, clickhouse.Options{ Timeout: chDataTimeout, ConnectTimeout: q.chConnectTimeout, TLSConfig: q.chTLSConfig, CheckRequestProgress: q.featureFlags.LogQueryProgress, ProgressSendingInterval: q.chProgressSendingInterval, }, extData, ) if err == nil { atomic.AddInt64(&ch_read_bytes, body.ChReadBytes()) atomic.AddInt64(&ch_read_rows, body.ChReadRows()) err = data.parseResponse(queryContext, body, cond) if err != nil { logger.Error("reader", zap.Error(err)) data.e <- err queryCancel() } } else { logger.Error("reader", zap.Error(err)) data.e <- err queryCancel() } }() } err = data.wait(queryContext) metrics.SendQueryRead(cond.queryMetrics, cond.from, cond.until, data.spent.Milliseconds(), int64(data.Points.Len()), int64(data.length), ch_read_rows, ch_read_bytes, err != nil) if err != nil { logger.Error( "data_parser", zap.Error(err), zap.Int("read_bytes", data.length), zap.String("runtime", data.spent.String()), zap.Duration("runtime_ns", data.spent), ) return err } logger.Info( "data_parse", zap.Int("read_bytes", data.length), zap.Int("read_points", data.Points.Len()), zap.String("runtime", data.spent.String()), zap.Duration("runtime_ns", data.spent), ) data.setSteps(cond) data.Points.SetAggregations(cond.aggregations) // ClickHouse returns sorted and uniq values, when internal aggregation is used // But if carbonlink is used, we still need to sort, filter and rollup points if !cond.aggregated || carbonlink != nil { sortStart := time.Now() data.Points.Sort() d := time.Since(sortStart) logger.Debug("sort", zap.String("runtime", d.String()), zap.Duration("runtime_ns", d)) data.Points.Uniq() rollupStart := time.Now() err = cond.rollupRules.RollupPoints(data.Points, cond.From, data.CommonStep) if err != nil { logger.Error("rollup failed", zap.Error(err)) return err } rollupTime := time.Since(rollupStart) logger.Debug( "rollup", zap.String("runtime", rollupTime.String()), zap.Duration("runtime_ns", rollupTime), ) } data.AM = cond.AM q.appendReply(CHResponse{ Data: data.Data, From: cond.From, Until: cond.Until, AppendOutEmptySeries: cond.appendEmptySeries, AppliedFunctions: cond.appliedFunctions, }) return nil } func (q *query) metricsListExtData(body *strings.Builder) *clickhouse.ExternalData { extTable := clickhouse.ExternalTable{ Name: extTableName, Columns: []clickhouse.Column{{ Name: "Path", Type: "String", }}, Format: "TSV", Data: []byte(body.String()), } extData := clickhouse.NewExternalData(extTable) extData.SetDebug(q.debugDir, q.debugExtDataPerm) return extData } func (c *conditions) prepareMetricsLists() { c.metricsUnreverse = c.AM.Series(false) c.metricsRequested = c.metricsUnreverse if c.isReverse { c.metricsRequested = make([]string, len(c.metricsRequested)) for i := range c.metricsRequested { c.metricsRequested[i] = reverse.String(c.metricsUnreverse[i]) } } c.metricsLookup = c.metricsRequested if c.rollupUseReverted { c.metricsLookup = c.metricsUnreverse } } func (c *conditions) prepareLookup() error { age := uint32(dry.Max(0, time.Now().Unix()-c.From)) c.aggregations = make(map[string][]string) c.appliedFunctions = make(map[string][]string) c.extDataBodies = make(map[string]*strings.Builder) c.steps = make(map[uint32][]string) aggName := "" for i := range c.metricsRequested { step, agg, _, _ := c.rollupRules.Lookup(c.metricsLookup[i], age, false) // Override agregation with an argument of consolidateBy function. // consolidateBy with its argument is passed through FilteringFunctions field of carbonapi_v3_pb protocol. // Currently it just finds the first target matching the metric // to avoid making multiple request for every type of aggregation for a given metric. for _, alias := range c.AM.Get(c.metricsUnreverse[i]) { requestedAgg, err := c.GetRequestedAggregation(alias.Target) if err != nil { return fmt.Errorf("failed to choose appropriate aggregation for '%s': %s", alias.Target, err.Error()) } if requestedAgg != "" { agg = rollup.AggrMap[requestedAgg] c.appliedFunctions[alias.Target] = []string{graphiteConsolidationFunction} break } } if _, ok := c.steps[step]; !ok { c.steps[step] = make([]string, 0) } // Fill up metric names for steps only for non-aggregated requests. // Aggregated use commonStep if !c.aggregated { c.steps[step] = append(c.steps[step], c.metricsUnreverse[i]) } // Fill up metric names for aggregations if mm, ok := c.aggregations[agg.Name()]; ok { c.aggregations[agg.Name()] = append(mm, c.metricsUnreverse[i]) } else { c.aggregations[agg.Name()] = []string{c.metricsUnreverse[i]} } // Build external-data bodies. For non-aggregated requests there is only one request if c.aggregated { aggName = agg.Name() } if mm, ok := c.extDataBodies[aggName]; ok { mm.WriteString(c.metricsRequested[i] + "\n") } else { var mm strings.Builder c.extDataBodies[aggName] = &mm mm.WriteString(c.metricsRequested[i] + "\n") } } return nil } var ErrSetStepTimeout = errors.New("unexpected error, setStep timeout") func (c *conditions) setStep(cStep *commonStep) { step := int64(0) if !c.aggregated { // Use max(steps) for s := range c.steps { step = dry.Max(step, int64(s)) } c.step = step return } // Use LCM(steps) // XXX: This could cause problems, when MutliFetchRequest uses different MaxDataPoints, // but currently (2021-04-22) it's not possible for s := range c.steps { step = cStep.calculateUnsafe(step, int64(s)) } cStep.calculate(step) rStep := cStep.getResult() if rStep == -1 { c.step = -1 return } step = dry.Max(rStep, dry.Ceil(c.Until-c.From, c.MaxDataPoints)) c.step = dry.CeilToMultiplier(step, rStep) return } func (c *conditions) setFromUntil() { c.from = dry.CeilToMultiplier(c.From, c.step) c.until = dry.FloorToMultiplier(c.Until, c.step) + c.step - 1 } func (c *conditions) setPrewhere() { pw := where.New() pw.And(where.DateBetween("Date", c.from, c.until)) c.prewhere = pw.PreWhereSQL() } func (c *conditions) setWhere() { wr := where.New() wr.And(where.InTable("Path", extTableName)) wr.And(where.TimestampBetween("Time", c.from, c.until)) c.where = wr.SQL() } func (c *conditions) generateQuery(agg string) string { if c.aggregated { return c.generateQueryaAggregated(agg) } return c.generateQueryUnaggregated() } func (c *conditions) generateQueryaAggregated(agg string) string { return fmt.Sprintf( queryAggregated, c.from, c.until, c.step, agg, c.pointsTable, c.prewhere, c.where, ) } func (c *conditions) generateQueryUnaggregated() string { return fmt.Sprintf(queryUnaggregated, c.pointsTable, c.prewhere, c.where) } ================================================ FILE: render/data/query_test.go ================================================ package data import ( "fmt" "sort" "strings" "sync" "testing" "time" v3pb "github.com/go-graphite/protocol/carbonapi_v3_pb" "github.com/lomik/graphite-clickhouse/finder" "github.com/lomik/graphite-clickhouse/helper/date" "github.com/lomik/graphite-clickhouse/helper/rollup" "github.com/lomik/graphite-clickhouse/pkg/alias" "github.com/lomik/graphite-clickhouse/pkg/reverse" "github.com/stretchr/testify/assert" ) func genPattern(regexp, function string, retention []rollup.Retention) rollup.Pattern { return rollup.Pattern{Regexp: regexp, Function: function, Retention: retention} } var finderResult *finder.MockFinder = finder.NewMockFinder([][]byte{ []byte("5_sec.name.max"), []byte("1_min.name.avg"), []byte("5_min.name.min"), []byte("10_min.name.any"), // defaults will be used in rollup.Rules }) func newAM() *alias.Map { am := alias.New() am.MergeTarget(finderResult, "*.name.*", false) return am } func newRules(reversed bool) *rollup.Rules { fiveSec := []rollup.Retention{{Age: 0, Precision: 5}, {Age: 3600, Precision: 60}} oneMin := []rollup.Retention{{Age: 0, Precision: 60}, {Age: 3600, Precision: 300}} fiveMin := []rollup.Retention{{Age: 0, Precision: 300}, {Age: 3600, Precision: 1200}} emptyRet := make([]rollup.Retention, 0) var pattern []rollup.Pattern if reversed { pattern = []rollup.Pattern{ genPattern("[.]5_sec$", "", fiveSec), genPattern("[.]1_min$", "", oneMin), genPattern("[.]5_min$", "", fiveMin), genPattern("^max[.]", "max", emptyRet), genPattern("^min[.]", "min", emptyRet), genPattern("^avg[.]", "avg", emptyRet), } } else { pattern = []rollup.Pattern{ genPattern("^5_sec[.]", "", fiveSec), genPattern("^1_min[.]", "", oneMin), genPattern("^5_min[.]", "", fiveMin), genPattern("[.]max$", "max", emptyRet), genPattern("[.]min$", "min", emptyRet), genPattern("[.]avg$", "avg", emptyRet), } } rules, _ := rollup.NewMockRules(pattern, 30, "avg") return rules } func ageToTimestamp(age int64) int64 { return time.Now().Unix() - age } // fromAge and untilAge are relative age of timeframe func newCondition(fromAge, untilAge, maxDataPoints int64) *conditions { tf := TimeFrame{ageToTimestamp(fromAge), ageToTimestamp(untilAge), maxDataPoints} tt := NewTargets([]string{"*.name.*"}, newAM()) tt.pointsTable = "graphite.data" tt.rollupRules = newRules(false) return &conditions{TimeFrame: &tf, Targets: tt} } func extTableString(et map[string]*strings.Builder) map[string]string { ett := make(map[string]string) for a := range et { ett[a] = et[a].String() } return ett } func TestPrepareMetricsLists(t *testing.T) { t.Run("unreversed request", func(t *testing.T) { cond := newCondition(0, 0, 60) cond.isReverse = false cond.rollupUseReverted = false cond.prepareMetricsLists() expectedSeries := finderResult.Strings() sort.Strings(expectedSeries) sort.Strings(cond.metricsLookup) sort.Strings(cond.metricsRequested) sort.Strings(cond.metricsUnreverse) assert.Equal(t, expectedSeries, cond.metricsUnreverse) assert.Equal(t, cond.metricsRequested, cond.metricsUnreverse) assert.Equal(t, cond.metricsLookup, cond.metricsRequested) // nothing should change in case of cond.isReverse == false cond.rollupUseReverted = true cond.prepareMetricsLists() sort.Strings(cond.metricsLookup) sort.Strings(cond.metricsRequested) sort.Strings(cond.metricsUnreverse) assert.Equal(t, expectedSeries, cond.metricsUnreverse) assert.Equal(t, cond.metricsRequested, cond.metricsUnreverse) assert.Equal(t, cond.metricsLookup, cond.metricsRequested) }) t.Run("reversed request", func(t *testing.T) { cond := newCondition(0, 0, 60) cond.isReverse = true cond.rollupUseReverted = false cond.prepareMetricsLists() for i := range cond.metricsRequested { assert.Equal(t, cond.metricsRequested[i], reverse.String(cond.metricsUnreverse[i])) } expectedSeries := finderResult.Strings() sort.Strings(expectedSeries) expectedSeriesReversed := make([]string, len(expectedSeries)) for i := range expectedSeries { expectedSeriesReversed[i] = reverse.String(expectedSeries[i]) } sort.Strings(expectedSeriesReversed) sort.Strings(cond.metricsLookup) sort.Strings(cond.metricsRequested) sort.Strings(cond.metricsUnreverse) assert.Equal(t, expectedSeries, cond.metricsUnreverse) assert.Equal(t, expectedSeriesReversed, cond.metricsRequested) assert.Equal(t, cond.metricsLookup, cond.metricsRequested) cond.rollupUseReverted = true cond.prepareMetricsLists() for i := range cond.metricsRequested { assert.Equal(t, cond.metricsRequested[i], reverse.String(cond.metricsUnreverse[i])) } sort.Strings(cond.metricsLookup) sort.Strings(cond.metricsRequested) sort.Strings(cond.metricsUnreverse) assert.Equal(t, expectedSeries, cond.metricsUnreverse) assert.Equal(t, expectedSeriesReversed, cond.metricsRequested) assert.Equal(t, cond.metricsLookup, cond.metricsUnreverse) }) } func TestPrepareLookup(t *testing.T) { // cases: // - aggregater / non-aggregated // - proper/inproper lookup rules for reversed table // testing: // - c.aggregations // - c.extDataBodies: only for c.isReverse=false // - c.steps t.Run("aggregated non-reverse query", func(t *testing.T) { cond := newCondition(5400, 1800, 5) cond.aggregated = true cond.isReverse = false cond.prepareMetricsLists() sort.Strings(cond.metricsLookup) sort.Strings(cond.metricsRequested) sort.Strings(cond.metricsUnreverse) cond.prepareLookup() aggregations := map[string][]string{ "avg": {"10_min.name.any", "1_min.name.avg"}, "max": {"5_sec.name.max"}, "min": {"5_min.name.min"}, } assert.Equal(t, aggregations, cond.aggregations) // Steps saves only values, not the metrics list steps := map[uint32][]string{ 30: {}, 60: {}, 300: {}, 1200: {}, } assert.Equal(t, steps, cond.steps) bodies := make(map[string]string) for a, m := range aggregations { bodies[a] = strings.Join(m, "\n") + "\n" } assert.Equal(t, bodies, extTableString(cond.extDataBodies)) cond.From = ageToTimestamp(1800) cond.Until = ageToTimestamp(0) cond.prepareLookup() steps = map[uint32][]string{ 30: {}, 60: {}, 300: {}, 5: {}, } assert.Equal(t, steps, cond.steps) assert.Equal(t, aggregations, cond.aggregations) assert.Equal(t, bodies, extTableString(cond.extDataBodies)) }) t.Run("non-aggregated non-reverse query", func(t *testing.T) { cond := newCondition(5400, 1800, 5) cond.aggregated = false cond.isReverse = false cond.prepareMetricsLists() sort.Strings(cond.metricsLookup) sort.Strings(cond.metricsRequested) sort.Strings(cond.metricsUnreverse) cond.prepareLookup() aggregations := map[string][]string{ "avg": {"10_min.name.any", "1_min.name.avg"}, "max": {"5_sec.name.max"}, "min": {"5_min.name.min"}, } assert.Equal(t, aggregations, cond.aggregations) // Steps saves only values, not the metrics list steps := map[uint32][]string{ 30: {"10_min.name.any"}, 60: {"5_sec.name.max"}, 300: {"1_min.name.avg"}, 1200: {"5_min.name.min"}, } assert.Equal(t, steps, cond.steps) bodies := map[string]string{"": "10_min.name.any\n1_min.name.avg\n5_min.name.min\n5_sec.name.max\n"} assert.Equal(t, bodies, extTableString(cond.extDataBodies)) cond.From = ageToTimestamp(1800) cond.Until = ageToTimestamp(0) cond.prepareLookup() steps = map[uint32][]string{ 5: {"5_sec.name.max"}, 30: {"10_min.name.any"}, 60: {"1_min.name.avg"}, 300: {"5_min.name.min"}, } assert.Equal(t, steps, cond.steps) assert.Equal(t, aggregations, cond.aggregations) assert.Equal(t, bodies, extTableString(cond.extDataBodies)) }) t.Run("reverse query with improper rules", func(t *testing.T) { cond := newCondition(5400, 1800, 5) cond.aggregated = false cond.isReverse = true cond.prepareMetricsLists() sort.Strings(cond.metricsUnreverse) sort.Strings(cond.metricsRequested) cond.prepareLookup() aggregations := map[string][]string{ "avg": {"10_min.name.any", "1_min.name.avg", "5_min.name.min", "5_sec.name.max"}, } assert.Equal(t, aggregations, cond.aggregations) // Steps saves only values, not the metrics list steps := map[uint32][]string{ 30: {"10_min.name.any", "1_min.name.avg", "5_min.name.min", "5_sec.name.max"}, } assert.Equal(t, steps, cond.steps) bodies := map[string]string{"": "any.name.10_min\navg.name.1_min\nmax.name.5_sec\nmin.name.5_min\n"} assert.Equal(t, bodies, extTableString(cond.extDataBodies)) cond.From = ageToTimestamp(1800) cond.Until = ageToTimestamp(0) cond.prepareLookup() assert.Equal(t, steps, cond.steps) assert.Equal(t, aggregations, cond.aggregations) assert.Equal(t, bodies, extTableString(cond.extDataBodies)) }) t.Run("reverse query with proper rules", func(t *testing.T) { cond := newCondition(5400, 1800, 5) cond.rollupRules = newRules(true) cond.aggregated = false cond.isReverse = true cond.prepareMetricsLists() cond.prepareLookup() for a := range cond.aggregations { sort.Strings(cond.aggregations[a]) } aggregations := map[string][]string{ "avg": {"10_min.name.any", "1_min.name.avg"}, "max": {"5_sec.name.max"}, "min": {"5_min.name.min"}, } assert.Equal(t, aggregations, cond.aggregations) // Steps saves only values, not the metrics list steps := map[uint32][]string{ 30: {"10_min.name.any"}, 60: {"5_sec.name.max"}, 300: {"1_min.name.avg"}, 1200: {"5_min.name.min"}, } assert.Equal(t, steps, cond.steps) cond.From = ageToTimestamp(1800) cond.Until = ageToTimestamp(0) cond.prepareLookup() steps = map[uint32][]string{ 5: {"5_sec.name.max"}, 30: {"10_min.name.any"}, 60: {"1_min.name.avg"}, 300: {"5_min.name.min"}, } for a := range cond.aggregations { sort.Strings(cond.aggregations[a]) } assert.Equal(t, steps, cond.steps) assert.Equal(t, aggregations, cond.aggregations) }) t.Run("non-reverse query with override aggregation", func(t *testing.T) { cond := newCondition(5400, 1800, 5) cond.aggregated = true cond.isReverse = false cond.prepareMetricsLists() sort.Strings(cond.metricsLookup) sort.Strings(cond.metricsRequested) sort.Strings(cond.metricsUnreverse) var aggregations map[string][]string for _, aggrStr := range []string{"avg", "min", "max", "sum"} { cond.SetFilteringFunctions( "*.name.*", []*v3pb.FilteringFunction{{Name: "consolidateBy", Arguments: []string{aggrStr}}}, ) cond.prepareLookup() aggregations = map[string][]string{ aggrStr: {"10_min.name.any", "1_min.name.avg", "5_min.name.min", "5_sec.name.max"}, } assert.Equal(t, aggregations, cond.aggregations) } // Steps saves only values, not the metrics list steps := map[uint32][]string{ 30: {}, 60: {}, 300: {}, 1200: {}, } assert.Equal(t, steps, cond.steps) bodies := make(map[string]string) for a, m := range aggregations { bodies[a] = strings.Join(m, "\n") + "\n" } assert.Equal(t, bodies, extTableString(cond.extDataBodies)) cond.From = ageToTimestamp(1800) cond.Until = ageToTimestamp(0) cond.prepareLookup() steps = map[uint32][]string{ 30: {}, 60: {}, 300: {}, 5: {}, } assert.Equal(t, steps, cond.steps) assert.Equal(t, aggregations, cond.aggregations) assert.Equal(t, bodies, extTableString(cond.extDataBodies)) }) } func TestSetStep(t *testing.T) { t.Run("unaggregated max", func(t *testing.T) { cond := newCondition(1800, 0, 1) cond.prepareMetricsLists() cond.prepareLookup() cond.setStep(nil) var step int64 = 300 assert.Equal(t, step, cond.step) cond.From = ageToTimestamp(5400) cond.prepareLookup() cond.setStep(nil) step = 1200 assert.Equal(t, step, cond.step) }) t.Run("aggregated common step", func(t *testing.T) { cStep := &commonStep{ result: 0, wg: sync.WaitGroup{}, lock: sync.RWMutex{}, } cond := newCondition(1800, 0, 2) cond.aggregated = true cond.prepareMetricsLists() cond.prepareLookup() cStep.addTargets(1) cond.setStep(cStep) var step int64 = 1800 / 2 assert.Equal(t, step, cond.step) cStep.addTargets(1) cStep.result = 0 cond.From = ageToTimestamp(1200) cond.Until = ageToTimestamp(700) cond.MaxDataPoints = 5 cond.setStep(cStep) step = 300 assert.Equal(t, step, cond.step) cStep.addTargets(1) cStep.result = 0 cond.MaxDataPoints = 10 cond.steps = map[uint32][]string{1: {}, 5: {}, 3: {}, 4: {}} cond.setStep(cStep) step = 60 assert.Equal(t, step, cond.step) cStep.addTargets(1) cStep.result = 0 cond.MaxDataPoints = 7 cond.steps = map[uint32][]string{1: {}, 5: {}, 8: {}, 4: {}} cond.setStep(cStep) step = 80 assert.Equal(t, step, cond.step) cStep.addTargets(1) cond.MaxDataPoints = 6 cond.setStep(cStep) step = 120 assert.Equal(t, step, cond.step) }) } func TestSetFromUntil(t *testing.T) { type in struct { from int64 until int64 step int64 } type out struct { from int64 until int64 } tests := []struct { in in out out }{ {in: in{4, 9, 2}, out: out{4, 9}}, {in: in{4, 19, 3}, out: out{6, 20}}, {in: in{4, 29, 5}, out: out{5, 29}}, {in: in{7, 108, 7}, out: out{7, 111}}, {in: in{7, 108, 13}, out: out{13, 116}}, } for tn, test := range tests { t.Run(fmt.Sprintf("setFromUntil %d", tn), func(t *testing.T) { cond := &conditions{ TimeFrame: &TimeFrame{From: test.in.from, Until: test.in.until}, step: test.in.step, } cond.setFromUntil() result := out{cond.from, cond.until} assert.Equal(t, test.out, result) }) } } // prewhere, where and both generators are checked here func TestGenerateQuery(t *testing.T) { table := "graphite.table" type in struct { from int64 until int64 step int64 agg string } tests := []struct { in in aggregated string unaggregated string }{ { in: in{1668124800, 1668325322, 1, "avg"}, aggregated: ("WITH anyResample(1668124800, 1668325322, 1)(toUInt32(intDiv(Time, 1)*1), Time) AS mask\n" + "SELECT Path,\n arrayFilter(m->m!=0, mask) AS times,\n" + " arrayFilter((v,m)->m!=0, avgResample(1668124800, 1668325322, 1)(Value, Time), mask) AS values\n" + "FROM graphite.table\n" + "PREWHERE Date >= '" + date.FromTimestampToDaysFormat(1668124800) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(1668325322) + "'\n" + "WHERE (Path in metrics_list) AND (Time >= 1668124800 AND Time <= 1668325322)\n" + "GROUP BY Path\n" + "FORMAT RowBinary"), unaggregated: ("SELECT Path, groupArray(Time), groupArray(Value), groupArray(Timestamp)\n" + "FROM graphite.table\n" + "PREWHERE Date >= '" + date.FromTimestampToDaysFormat(1668124800) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(1668325322) + "'\n" + "WHERE (Path in metrics_list) AND (Time >= 1668124800 AND Time <= 1668325322)\n" + "GROUP BY Path\n" + "FORMAT RowBinary"), }, { in: in{11111, 33333, 11111, "min"}, aggregated: ("WITH anyResample(11111, 33333, 11111)(toUInt32(intDiv(Time, 11111)*11111), Time) AS mask\n" + "SELECT Path,\n arrayFilter(m->m!=0, mask) AS times,\n" + " arrayFilter((v,m)->m!=0, minResample(11111, 33333, 11111)(Value, Time), mask) AS values\n" + "FROM graphite.table\n" + "PREWHERE Date >= '" + date.FromTimestampToDaysFormat(11111) + "' AND Date <= '" + date.FromTimestampToDaysFormat(33333) + "'\n" + "WHERE (Path in metrics_list) AND (Time >= 11111 AND Time <= 33333)\n" + "GROUP BY Path\n" + "FORMAT RowBinary"), unaggregated: ("SELECT Path, groupArray(Time), groupArray(Value), groupArray(Timestamp)\n" + "FROM graphite.table\n" + "PREWHERE Date >= '" + date.FromTimestampToDaysFormat(11111) + "' AND Date <= '" + date.UntilTimestampToDaysFormat(33333) + "'\n" + "WHERE (Path in metrics_list) AND (Time >= 11111 AND Time <= 33333)\n" + "GROUP BY Path\n" + "FORMAT RowBinary"), }, } for tn, test := range tests { t.Run(fmt.Sprintf("generate query %d", tn), func(t *testing.T) { cond := &conditions{ Targets: &Targets{}, from: test.in.from, until: test.in.until, step: test.in.step, } cond.pointsTable = table cond.setPrewhere() cond.setWhere() unaggQuery := cond.generateQuery(test.in.agg) assert.Equal(t, test.unaggregated, unaggQuery) cond.aggregated = true aggQuery := cond.generateQuery(test.in.agg) assert.Equal(t, test.aggregated, aggQuery) }) } } ================================================ FILE: render/data/targets.go ================================================ package data import ( "fmt" "time" v3pb "github.com/go-graphite/protocol/carbonapi_v3_pb" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/rollup" "github.com/lomik/graphite-clickhouse/metrics" "github.com/lomik/graphite-clickhouse/pkg/alias" ) const graphiteConsolidationFunction = "consolidateBy" type FilteringFunctionsByTarget map[string][]*v3pb.FilteringFunction type Cache struct { Cached bool TS int64 // cached timestamp Timeout int32 TimeoutStr string Key string // cache key M *metrics.CacheMetric } // Targets represents requested metrics type Targets struct { // List contains queried metrics, e.g. [metric.{name1,name2}, metric.name[3-9]] List []string Cache []Cache Cached bool // all is cached // AM stores found expanded metrics AM *alias.Map filteringFunctionsByTarget FilteringFunctionsByTarget pointsTable string isReverse bool rollupRules *rollup.Rules rollupUseReverted bool queryMetrics *metrics.QueryMetrics } func NewTargets(list []string, am *alias.Map) *Targets { targets := &Targets{ List: list, Cache: make([]Cache, len(list)), AM: am, filteringFunctionsByTarget: make(FilteringFunctionsByTarget), } return targets } func NewTargetsOne(target string, capacity int, am *alias.Map) *Targets { list := make([]string, 1, capacity) list[0] = target targets := &Targets{ List: list, Cache: make([]Cache, len(list)), AM: am, filteringFunctionsByTarget: make(FilteringFunctionsByTarget), } return targets } func (tt *Targets) Append(target string) { tt.List = append(tt.List, target) tt.Cache = append(tt.Cache, Cache{}) } func (tt *Targets) SetFilteringFunctions(target string, filteringFunctions []*v3pb.FilteringFunction) { tt.filteringFunctionsByTarget[target] = filteringFunctions } func (tt *Targets) selectDataTable(cfg *config.Config, tf *TimeFrame, context string) error { now := time.Now().Unix() TableLoop: for i := 0; i < len(cfg.DataTable); i++ { t := &cfg.DataTable[i] if !t.ContextMap[context] { continue TableLoop } if t.MaxInterval != 0 && (tf.Until-tf.From) > int64(t.MaxInterval.Seconds()) { continue TableLoop } if t.MinInterval != 0 && (tf.Until-tf.From) < int64(t.MinInterval.Seconds()) { continue TableLoop } if t.MaxAge != 0 && tf.From < now-int64(t.MaxAge.Seconds()) { continue TableLoop } if t.MinAge != 0 && tf.Until > now-int64(t.MinAge.Seconds()) { continue TableLoop } if t.TargetMatchAllRegexp != nil { for j := 0; j < len(tt.List); j++ { if !t.TargetMatchAllRegexp.MatchString(tt.List[j]) { continue TableLoop } } } if t.TargetMatchAnyRegexp != nil { matched := false TargetsLoop: for j := 0; j < len(tt.List); j++ { if t.TargetMatchAnyRegexp.MatchString(tt.List[j]) { matched = true break TargetsLoop } } if !matched { continue TableLoop } } tt.pointsTable = t.Table tt.isReverse = t.Reverse tt.rollupUseReverted = t.RollupUseReverted tt.rollupRules = t.Rollup.Rules() tt.queryMetrics = t.QueryMetrics return nil } return fmt.Errorf("data tables is not specified for %v", tt.List[0]) } func (tt *Targets) GetRequestedAggregation(target string) (string, error) { if ffs, ok := tt.filteringFunctionsByTarget[target]; !ok { return "", nil } else { for _, filteringFunc := range ffs { ffName := filteringFunc.GetName() ffArgs := filteringFunc.GetArguments() if ffName != graphiteConsolidationFunction { continue } if len(ffArgs) < 1 { return "", fmt.Errorf("no argumets were provided to consolidateBy function") } switch ffArgs[0] { // 'last' in graphite == clickhouse aggregate function 'anyLast' case "last": return "anyLast", nil // 'first' in graphite == clickhouse aggregate function 'any' case "first": return "any", nil // Graphite standard supports both average and avg. // It is the only aggregation that has two aliases. // https://graphite.readthedocs.io/en/latest/functions.html#graphite.render.functions.consolidateBy case "average": return "avg", nil // avg, sum, max, min have the same name in clickhouse case "avg", "sum", "max", "min": return ffArgs[0], nil default: return "", fmt.Errorf( "unknown \"%s\" argument function (allowed argumets are: 'avg', 'average', 'sum', 'max', 'min', 'last', 'first'): recieved %s", graphiteConsolidationFunction, ffArgs[0], ) } } } return "", nil } ================================================ FILE: render/data/targets_test.go ================================================ package data import ( "fmt" "testing" "time" "github.com/lomik/graphite-clickhouse/config" "github.com/stretchr/testify/assert" ) func TestSelectDataTableTime(t *testing.T) { cfg := config.New() cfg.DataTable = []config.DataTable{ { Table: "first_day", MaxAge: 24 * time.Hour, }, { Table: "second_day", MinAge: 24 * time.Hour, MaxAge: 48 * time.Hour, }, { Table: "two_days_min_interval", MaxAge: 48 * time.Hour, MinInterval: 2 * time.Hour, }, { Table: "two_days_min_max_interval", MaxAge: 48 * time.Hour, MinInterval: 30 * time.Minute, MaxInterval: 1 * time.Hour, }, { Table: "two_days_max_interval", MaxAge: 48 * time.Hour, MaxInterval: 2 * time.Hour, }, { Table: "three_days", MaxAge: 72 * time.Hour, }, { Table: "unlimited", }, } err := cfg.ProcessDataTables() assert.NoError(t, err) tg := NewTargets([]string{"metric"}, nil) tests := []struct { *TimeFrame config.DataTable err error }{ { &TimeFrame{ageToTimestamp(3600*24 - 1), ageToTimestamp(1800), 1}, cfg.DataTable[0], nil, }, { &TimeFrame{ageToTimestamp(3600*48 - 1), ageToTimestamp(24*3600 + 1), 1}, cfg.DataTable[1], nil, }, { &TimeFrame{ageToTimestamp(3600 * 26), ageToTimestamp(3600 * 23), 1}, cfg.DataTable[2], nil, }, { &TimeFrame{ageToTimestamp(3600*24 + 1600), ageToTimestamp(3600*24 - 1600), 1}, cfg.DataTable[3], nil, }, { &TimeFrame{ageToTimestamp(3600*24 + 2000), ageToTimestamp(3600*24 - 2000), 1}, cfg.DataTable[4], nil, }, { &TimeFrame{ageToTimestamp(3600*72 - 1), ageToTimestamp(3600*11 - 1), 1}, cfg.DataTable[5], nil, }, { &TimeFrame{ageToTimestamp(3600 * 100), ageToTimestamp(3600*11 - 1), 1}, cfg.DataTable[6], nil, }, } for i, test := range tests { t.Run(fmt.Sprintf("%d-%s", i+1, test.DataTable.Table), func(t *testing.T) { err := tg.selectDataTable(cfg, test.TimeFrame, config.ContextGraphite) assert.Equal(t, test.err, err) assert.Equal(t, test.DataTable.Table, tg.pointsTable) }) } } func TestSelectDataTableMatch(t *testing.T) { cfg := config.New() cfg.DataTable = []config.DataTable{ { Table: "all", TargetMatchAll: "^all.*avg", }, { Table: "any", TargetMatchAny: "^any.*avg", }, { Table: "unlimited", }, } err := cfg.ProcessDataTables() assert.NoError(t, err) tf := &TimeFrame{ageToTimestamp(3600*24 - 1), ageToTimestamp(1800), 1} tests := []struct { *Targets config.DataTable err error }{ { NewTargets([]string{"allinclucive.in.avg", "all.metrics.for.avg"}, nil), cfg.DataTable[0], nil, }, { NewTargets([]string{"allinclucive.in.avg", "any.metrics.for.avg"}, nil), cfg.DataTable[1], nil, }, { NewTargets([]string{"allinclucive.in.avg", "some.metrics.for.avg"}, nil), cfg.DataTable[2], nil, }, } for i, test := range tests { t.Run(fmt.Sprintf("%d-%s", i+1, test.DataTable.Table), func(t *testing.T) { err := test.Targets.selectDataTable(cfg, tf, config.ContextGraphite) assert.Equal(t, test.err, err) assert.Equal(t, test.DataTable.Table, test.Targets.pointsTable) }) } } ================================================ FILE: render/handler.go ================================================ package render import ( "context" "fmt" "net/http" "strings" "sync" "time" "go.uber.org/zap" "github.com/go-graphite/carbonapi/pkg/parser" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/finder" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/helper/utils" "github.com/lomik/graphite-clickhouse/limiter" "github.com/lomik/graphite-clickhouse/logs" "github.com/lomik/graphite-clickhouse/metrics" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/render/data" "github.com/lomik/graphite-clickhouse/render/reply" ) // Handler serves /render requests type Handler struct { config *config.Config } // NewHandler generates new *Handler func NewHandler(config *config.Config) *Handler { h := &Handler{ config: config, } return h } func targetKey(from, until int64, target, ttl string) string { return time.Unix(from, 0).Format("2006-01-02") + ";" + time.Unix(until, 0).Format("2006-01-02") + ";" + target + ";ttl=" + ttl } func getCacheTimeout(now time.Time, from, until int64, cacheConfig *config.CacheConfig) (int32, string, *metrics.CacheMetric) { if cacheConfig.ShortDuration == 0 { return cacheConfig.DefaultTimeoutSec, cacheConfig.DefaultTimeoutStr, metrics.DefaultCacheMetrics } duration := time.Second * time.Duration(until-from) if duration > cacheConfig.ShortDuration || now.Unix()-until > cacheConfig.ShortUntilOffsetSec { return cacheConfig.DefaultTimeoutSec, cacheConfig.DefaultTimeoutStr, metrics.DefaultCacheMetrics } // short cache ttl return cacheConfig.ShortTimeoutSec, cacheConfig.ShortTimeoutStr, metrics.ShortCacheMetrics } // try to fetch cached finder queries func (h *Handler) finderCached(ts time.Time, fetchRequests data.MultiTarget, logger *zap.Logger, metricsLen *int) (cachedFind int, maxCacheTimeoutStr string, err error) { var lock sync.RWMutex var maxCacheTimeout int32 errors := make([]error, 0, len(fetchRequests)) var wg sync.WaitGroup for tf, targets := range fetchRequests { for i, expr := range targets.List { wg.Add(1) go func(tf data.TimeFrame, target string, targets *data.Targets, n int) { defer wg.Done() targets.Cache[n].Timeout, targets.Cache[n].TimeoutStr, targets.Cache[n].M = getCacheTimeout(ts, tf.From, tf.Until, &h.config.Common.FindCacheConfig) if targets.Cache[n].Timeout > 0 { if maxCacheTimeout < targets.Cache[n].Timeout { maxCacheTimeout = targets.Cache[n].Timeout maxCacheTimeoutStr = targets.Cache[n].TimeoutStr } targets.Cache[n].TS = utils.TimestampTruncate(ts.Unix(), time.Duration(targets.Cache[n].Timeout)*time.Second) targets.Cache[n].Key = targetKey(tf.From, tf.Until, target, targets.Cache[n].TimeoutStr) body, err := h.config.Common.FindCache.Get(targets.Cache[n].Key) if err == nil { if len(body) > 0 { targets.Cache[n].M.CacheHits.Add(1) var f finder.Finder if strings.HasPrefix(target, "seriesByTag(") { f = finder.NewCachedTags(body) } else { f = finder.NewCachedIndex(body) } targets.AM.MergeTarget(f.(finder.Result), target, false) lock.Lock() amLen := targets.AM.Len() *metricsLen += amLen lock.Unlock() targets.Cache[n].Cached = true logger.Info("finder", zap.String("get_cache", targets.Cache[n].Key), zap.Time("timestamp_cached", time.Unix(targets.Cache[n].TS, 0)), zap.Int("metrics", amLen), zap.Bool("find_cached", true), zap.String("ttl", targets.Cache[n].TimeoutStr), zap.Int64("from", tf.From), zap.Int64("until", tf.Until)) } return } } }(tf, expr, targets, i) } } wg.Wait() if len(errors) != 0 { err = errors[0] return } for _, targets := range fetchRequests { var cached int for _, c := range targets.Cache { if c.Cached { cached++ } } cachedFind += cached if cached == len(targets.Cache) { targets.Cached = true } } return } // try to fetch finder queries func (h *Handler) finder(fetchRequests data.MultiTarget, ctx context.Context, logger *zap.Logger, qlimiter limiter.ServerLimiter, metricsLen *int, queueDuration *time.Duration, useCache bool) (maxDuration int64, err error) { var ( wg sync.WaitGroup lock sync.RWMutex entered int limitCtx context.Context cancel context.CancelFunc ) if qlimiter.Enabled() { // no reason wait longer than index-timeout limitCtx, cancel = context.WithTimeout(ctx, h.config.ClickHouse.IndexTimeout) defer func() { for i := 0; i < entered; i++ { qlimiter.Leave(limitCtx, "render") } defer cancel() }() } errors := make([]error, 0, len(fetchRequests)) for tf, targets := range fetchRequests { for i, expr := range targets.List { d := tf.Until - tf.From if maxDuration < d { maxDuration = d } if targets.Cache[i].Cached { continue } if qlimiter.Enabled() { start := time.Now() err = qlimiter.Enter(limitCtx, "render") *queueDuration += time.Since(start) if err != nil { lock.Lock() errors = append(errors, err) lock.Unlock() break } entered++ } wg.Add(1) go func(tf data.TimeFrame, target string, targets *data.Targets, n int) { defer wg.Done() var fndResult finder.Result var err error fStart := time.Now() fndResult, err = finder.Find(h.config, ctx, target, tf.From, tf.Until) d := time.Since(fStart).Milliseconds() if err != nil { metrics.SendQueryReadByTable(tf.From, tf.Until, d, 0, fndResult.Stats(), true) logger.Error("find", zap.Error(err)) lock.Lock() errors = append(errors, err) lock.Unlock() return } body := targets.AM.MergeTarget(fndResult, target, useCache) cacheTimeout := targets.Cache[n].Timeout if useCache && cacheTimeout > 0 { cacheTimeoutStr := targets.Cache[n].TimeoutStr key := targets.Cache[n].Key targets.Cache[n].M.CacheMisses.Add(1) h.config.Common.FindCache.Set(key, body, cacheTimeout) logger.Info("finder", zap.String("set_cache", key), zap.Time("timestamp_cached", time.Unix(targets.Cache[n].TS, 0)), zap.Int("metrics", targets.AM.Len()), zap.Bool("find_cached", false), zap.String("ttl", cacheTimeoutStr), zap.Int64("from", tf.From), zap.Int64("until", tf.Until)) } lock.Lock() rows := targets.AM.Len() lock.Unlock() *metricsLen += rows metrics.SendQueryReadByTable(tf.From, tf.Until, d, int64(rows), fndResult.Stats(), false) }(tf, expr, targets, i) } } wg.Wait() if len(errors) != 0 { err = errors[0] } return } func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { var ( maxDuration int64 targetsLen int metricsLen int pointsCount int64 fetchStart time.Time cachedFind bool queueFail bool queueDuration time.Duration err error fetchRequests data.MultiTarget luser string ) start := time.Now() status := http.StatusOK accessLogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("http") logger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("render") r = r.WithContext(scope.WithLogger(r.Context(), logger)) username := r.Header.Get("X-Forwarded-User") var qlimiter limiter.ServerLimiter = limiter.NoopLimiter{} defer func() { if rec := recover(); rec != nil { status = http.StatusInternalServerError logger.Error("panic during eval:", zap.String("requestID", scope.String(r.Context(), "requestID")), zap.Any("reason", rec), zap.Stack("stack"), ) answer := fmt.Sprintf("%v\nStack trace: %v", rec, zap.Stack("").String) http.Error(w, answer, status) } end := time.Now() logs.AccessLog(accessLogger, h.config, r, status, end.Sub(start), queueDuration, cachedFind, queueFail) qlimiter.SendDuration(queueDuration.Milliseconds()) metrics.SendRenderMetrics(metrics.RenderRequestMetric, status, start, fetchStart, end, maxDuration, h.config.Metrics.ExtendedStat, int64(metricsLen), pointsCount) }() r.ParseMultipartForm(1024 * 1024) formatter, err := reply.GetFormatter(r) if err != nil { status = http.StatusBadRequest logger.Error("formatter", zap.Error(err)) http.Error(w, fmt.Sprintf("Failed to parse request: %v", err.Error()), status) return } fetchRequests, err = formatter.ParseRequest(r) if err != nil { status = http.StatusBadRequest http.Error(w, fmt.Sprintf("Failed to parse request: %v", err.Error()), status) return } for tf, targets := range fetchRequests { if tf.From >= tf.Until { // wrong duration if err != nil { status, _ = clickhouse.HandleError(w, clickhouse.ErrInvalidTimeRange) return } } targetsLen += len(targets.List) } luser, qlimiter = data.GetQueryLimiter(username, h.config, &fetchRequests) logger.Debug("use user limiter", zap.String("username", username), zap.String("luser", luser)) var maxCacheTimeoutStr string useCache := h.config.Common.FindCache != nil && !parser.TruthyBool(r.FormValue("noCache")) if useCache { var cached int cached, maxCacheTimeoutStr, err = h.finderCached(start, fetchRequests, logger, &metricsLen) if err != nil { status, _ = clickhouse.HandleError(w, err) return } if cached > 0 { if cached == targetsLen && metricsLen == 0 { // all from cache and no metric status = http.StatusNotFound formatter.Reply(w, r, data.EmptyResponse()) return } cachedFind = true } } maxDuration, err = h.finder(fetchRequests, r.Context(), logger, qlimiter, &metricsLen, &queueDuration, useCache) if err != nil { status, queueFail = clickhouse.HandleError(w, err) return } logger.Info("finder", zap.Int("metrics", metricsLen), zap.Bool("find_cached", cachedFind)) if cachedFind { w.Header().Set("X-Cached-Find", maxCacheTimeoutStr) } if metricsLen == 0 { status = http.StatusNotFound formatter.Reply(w, r, data.EmptyResponse()) return } fetchStart = time.Now() reply, err := fetchRequests.Fetch(r.Context(), h.config, config.ContextGraphite, qlimiter, &queueDuration) if err != nil { status, queueFail = clickhouse.HandleError(w, err) return } if len(reply) == 0 { status = http.StatusNotFound formatter.Reply(w, r, reply) return } for i := range reply { pointsCount += int64(reply[i].Data.Len()) } rStart := time.Now() formatter.Reply(w, r, reply) d := time.Since(rStart) logger.Debug("reply", zap.String("runtime", d.String()), zap.Duration("runtime_ns", d)) } ================================================ FILE: render/handler_test.go ================================================ package render import ( "fmt" "testing" "time" "github.com/lomik/graphite-clickhouse/config" ) func Test_getCacheTimeout(t *testing.T) { cacheConfig := config.CacheConfig{ ShortTimeoutSec: 60, ShortTimeoutStr: "60", DefaultTimeoutSec: 300, DefaultTimeoutStr: "300", ShortDuration: 3 * time.Hour, ShortUntilOffsetSec: 120, } now := int64(1636985018) tests := []struct { name string now time.Time from int64 until int64 want int32 wantStr string }{ { name: "short: from = now - 600, until = now - 120", now: time.Unix(now, 0), from: now - 600, until: now - 120, want: 60, wantStr: "60", }, { name: "short: from = now - 10800", now: time.Unix(now, 0), from: now - 10800, until: now, want: 60, wantStr: "60", }, { name: "short: from = now - 10810, until = now - 120", now: time.Unix(now, 0), from: now - 10800, until: now - 120, want: 60, wantStr: "60", }, { name: "short: from = now - 10800, until now - 121", now: time.Unix(now, 0), from: now - 10800, until: now - 121, want: 300, wantStr: "300", }, { name: "default: from = now - 10801", now: time.Unix(now, 0), from: now - 10801, until: now, want: 300, wantStr: "300", }, { name: "short: from = now - 122, until = now - 121", now: time.Unix(now, 0), from: now - 122, until: now - 121, want: 300, wantStr: "300", }, } for i, tt := range tests { t.Run(fmt.Sprintf("[%d] %s", i, tt.name), func(t *testing.T) { got, gotStr, _ := getCacheTimeout(tt.now, tt.from, tt.until, &cacheConfig) if got != tt.want { t.Errorf("getCacheTimeout() = %v, want %v", got, tt.want) } if gotStr != tt.wantStr { t.Errorf("getCacheTimeout() = %q, want %q", gotStr, tt.wantStr) } }) } } ================================================ FILE: render/reply/formatter.go ================================================ package reply import ( "fmt" "math" "net/http" "strconv" "github.com/lomik/graphite-clickhouse/pkg/alias" "github.com/lomik/graphite-clickhouse/pkg/dry" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/render/data" "go.uber.org/zap" ) // Formatter implements request parser and response generator type Formatter interface { // Parse request ParseRequest(r *http.Request) (data.MultiTarget, error) // Generate reply payload Reply(http.ResponseWriter, *http.Request, data.CHResponses) } // GetFormatter returns a proper interface for render format func GetFormatter(r *http.Request) (Formatter, error) { format := r.FormValue("format") switch format { case "carbonapi_v3_pb": return &V3PB{}, nil case "pickle": return &Pickle{}, nil case "protobuf": return &V2PB{}, nil case "carbonapi_v2_pb": return &V2PB{}, nil } err := fmt.Errorf("format %v is not supported, supported formats: carbonapi_v3_pb, pickle, protobuf (aka carbonapi_v2_pb)", format) if !scope.Debug(r.Context(), "Output") { return nil, err } switch format { case "json": return &JSON{}, nil } err = fmt.Errorf("%w\n(formats available for output debug: json)", err) return nil, err } func parseRequestForms(r *http.Request) (data.MultiTarget, error) { fromTimestamp, err := strconv.ParseInt(r.FormValue("from"), 10, 32) if err != nil { return nil, fmt.Errorf("cannot parse from") } untilTimestamp, err := strconv.ParseInt(r.FormValue("until"), 10, 32) if err != nil { return nil, fmt.Errorf("cannot parse until") } maxDataPoints, err := strconv.ParseInt(r.FormValue("maxDataPoints"), 10, 32) if err != nil { maxDataPoints = int64(math.MaxInt64) } targets := dry.RemoveEmptyStrings(r.Form["target"]) tf := data.TimeFrame{ From: fromTimestamp, Until: untilTimestamp, MaxDataPoints: maxDataPoints, } multiTarget := make(data.MultiTarget) multiTarget[tf] = data.NewTargets(targets, alias.New()) if len(targets) > 0 { logger := scope.Logger(r.Context()).Named("form_parser") for _, t := range targets { logger.Info( "target", zap.Int64("from", tf.From), zap.Int64("until", tf.Until), zap.Int64("maxDataPoints", tf.MaxDataPoints), zap.String("target", t), ) } } return multiTarget, nil } ================================================ FILE: render/reply/formatter_test.go ================================================ package reply import ( "context" "fmt" "io" "math" "net/http" "net/http/httptest" "sort" "testing" "github.com/stretchr/testify/require" "github.com/lomik/graphite-clickhouse/finder" "github.com/lomik/graphite-clickhouse/helper/client" "github.com/lomik/graphite-clickhouse/helper/point" "github.com/lomik/graphite-clickhouse/pkg/alias" "github.com/lomik/graphite-clickhouse/render/data" ) var results = []client.Metric{ { Name: "test.metric1", PathExpression: "test.*", StartTime: 1688990040, StepTime: 60, StopTime: 1688990520, Values: func() []float64 { temp := emptyValues(8) temp[2] = 3 return temp }(), }, { Name: "test.metric2", PathExpression: "test.*", StartTime: 1688990040, StepTime: 60, StopTime: 1688990520, Values: emptyValues(8), }, { Name: "test.metric3", PathExpression: "test.*", StartTime: 1688990040, StepTime: 60, StopTime: 1688990520, Values: emptyValues(8), }, } func TestFormatterReply(t *testing.T) { formatters := []struct { impl Formatter name string format client.FormatType }{ {&V3PB{}, "v3pb", client.FormatPb_v3}, {&V2PB{}, "v2pb", client.FormatPb_v2}, {&JSON{}, "json", client.FormatJSON}, {&Pickle{}, "pickle", client.FormatPickle}, } tests := []struct { name string input data.CHResponses // result when CHResponse.AppendOutEmptySeries is false expectedWithoutEmpty []client.Metric // result when CHResponse.AppendOutEmptySeries is true expectedWithEmpty []client.Metric }{ { name: "no index found", input: data.EmptyResponse(), expectedWithoutEmpty: []client.Metric{}, expectedWithEmpty: []client.Metric{}, }, { name: "three metrics; test.metric1 with points and other with NaN", input: prepareCHResponses(1688990000, 1688990460, [][]byte{[]byte("test.metric1"), []byte("test.metric2"), []byte("test.metric3")}, map[string][]point.Point{ "test.metric1": {{Value: 3, Time: 1688990160, Timestamp: 1688990204}}, }, ), expectedWithoutEmpty: results[:1], expectedWithEmpty: results, }, { name: "three metrics, no points in all", input: prepareCHResponses(1688990000, 1688990460, [][]byte{[]byte("test.metric1"), []byte("test.metric2"), []byte("test.metric3")}, map[string][]point.Point{}, ), expectedWithoutEmpty: []client.Metric{}, expectedWithEmpty: append([]client.Metric{ { Name: results[0].Name, PathExpression: results[0].PathExpression, StartTime: results[0].StartTime, StopTime: results[0].StopTime, StepTime: results[0].StepTime, Values: emptyValues(8), }, }, results[1:]...), }, } for _, formatter := range formatters { t.Run(fmt.Sprintf("format=%s", formatter.name), func(t *testing.T) { for _, tt := range tests { // case 0: test for AppendOutEmptySeries = false // case 1: test for AppendOutEmptySeries = true for i := 0; i < 2; i++ { var expected []client.Metric var testName string switch i { case 0: expected = tt.expectedWithoutEmpty testName = fmt.Sprintf("NoAppend: %s", tt.name) for j := range tt.input { tt.input[j].AppendOutEmptySeries = false } case 1: expected = tt.expectedWithEmpty testName = fmt.Sprintf("WithAppend: %s", tt.name) for j := range tt.input { tt.input[j].AppendOutEmptySeries = true } } t.Run(testName, func(t *testing.T) { ctx := context.Background() // if tt.protobufDebug { // ctx = scope.WithDebug(ctx, "Protobuf") // } w := httptest.NewRecorder() r, err := http.NewRequestWithContext(ctx, "", "", nil) if err != nil { require.NoErrorf(t, err, "failed to create request") } formatter.impl.Reply(w, r, tt.input) response := w.Result() defer response.Body.Close() // then require.Equal(t, http.StatusOK, response.StatusCode) data, err := io.ReadAll(response.Body) require.NoError(t, err) got, err := client.Decode(data, formatter.format) require.NoError(t, err) if !equalMetrics(expected, got) { t.Errorf("metrics not equal: expected:\n%#v\ngot:\n%#v\n", expected, got) } }) } } }) } } // prepareCHResponses prepares CHResponses for tests. func prepareCHResponses(from, until int64, indices [][]byte, points map[string][]point.Point) data.CHResponses { // alias idx := finder.NewMockFinder(indices) m := alias.New() m.MergeTarget(idx, "test.*", false) // points pts := point.NewPoints() stringIndex := make([]string, 0, len(indices)) for _, each := range indices { stringIndex = append(stringIndex, string(each)) } for k, v := range points { id := pts.MetricID(k) for _, eachPoint := range v { pts.AppendPoint(id, eachPoint.Value, eachPoint.Time, eachPoint.Timestamp) } } pts.SetAggregations(map[string][]string{ "avg": stringIndex, }) sort.Sort(pts) return data.CHResponses{{ Data: &data.Data{ Points: pts, AM: m, CommonStep: 60, }, From: from, Until: until, }} } // emptyValues prefill slice of `size` with math.NaN func emptyValues(size int) []float64 { arr := make([]float64, 0, size) for i := 0; i < size; i++ { arr = append(arr, math.NaN()) } return arr } // equalMetrics returns true if two slices of client.Metric are equal. // This function only compares important fields of client.Metric. func equalMetrics(m1, m2 []client.Metric) bool { if len(m1) != len(m2) { return false } sort.Slice(m1, func(i, j int) bool { return m1[i].Name < m1[j].Name }) sort.Slice(m2, func(i, j int) bool { return m2[i].Name < m2[j].Name }) for i := 0; i < len(m1); i++ { // compare props if m1[i].Name != m2[i].Name || m1[i].StartTime != m2[i].StartTime || m1[i].StopTime != m2[i].StopTime || m1[i].StepTime != m2[i].StepTime { return false } // compare values if len(m1[i].Values) != len(m2[i].Values) { return false } for j := 0; j < len(m1[i].Values); j++ { a, b := m1[i].Values[j], m2[i].Values[j] if math.IsNaN(a) && math.IsNaN(b) { continue } if a != b { return false } } } return true } ================================================ FILE: render/reply/json.go ================================================ package reply import ( "bytes" "encoding/json" "errors" "fmt" "math" "net/http" v3pb "github.com/go-graphite/protocol/carbonapi_v3_pb" "go.uber.org/zap" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/render/data" ) // JSON is an implementation of carbonapi_v3_pb MultiGlobRequest and MultiFetchResponse interconnection. It accepts the // normal forms parser of `Content-Type: application/json` POST requests with JSON representation of MultiGlobRequest. type JSON struct{} func marshalJSON(mfr *v3pb.MultiFetchResponse) []byte { buf := bytes.Buffer{} buf.WriteString(`{"metrics":[`) for _, m := range mfr.Metrics { buf.WriteRune('{') if m.Name != "" { buf.WriteString(fmt.Sprintf(`"name":%q,`, m.Name)) } if m.PathExpression != "" { buf.WriteString(fmt.Sprintf(`"pathExpression":%q,`, m.PathExpression)) } if m.ConsolidationFunc != "" { buf.WriteString(fmt.Sprintf(`"consolidationFunc":%q,`, m.ConsolidationFunc)) } buf.WriteString(fmt.Sprintf(`"startTime":%d,`, m.StartTime)) buf.WriteString(fmt.Sprintf(`"stopTime":%d,`, m.StopTime)) buf.WriteString(fmt.Sprintf(`"stepTime":%d,`, m.StepTime)) buf.WriteString(fmt.Sprintf(`"xFilesFactor":%f,`, m.XFilesFactor)) if m.HighPrecisionTimestamps { buf.WriteString(`"highPrecisionTimestamp":true,`) } if len(m.Values) != 0 { buf.WriteString(`"values":[`) for _, v := range m.Values { if math.IsNaN(v) || math.IsInf(v, 0) { buf.WriteString("null,") continue } buf.WriteString(fmt.Sprintf("%f,", v)) } buf.Truncate(buf.Len() - 1) buf.WriteString("],") } buf.WriteString(fmt.Sprintf(`"requestStartTime":%d,`, m.RequestStartTime)) buf.WriteString(fmt.Sprintf(`"requestStopTime":%d,`, m.RequestStopTime)) buf.Truncate(buf.Len() - 1) buf.WriteString("},") } if len(mfr.Metrics) != 0 { buf.Truncate(buf.Len() - 1) } buf.WriteString("]}") return buf.Bytes() } func parseJSONBody(r *http.Request) (data.MultiTarget, error) { logger := scope.Logger(r.Context()).Named("json_parser") var pv3Request v3pb.MultiFetchRequest err := json.NewDecoder(r.Body).Decode(&pv3Request) if err != nil { return nil, err } fetchRequests := data.MFRToMultiTarget(&pv3Request) if len(pv3Request.Metrics) > 0 { for _, m := range pv3Request.Metrics { logger.Info( "json_target", zap.Int64("from", m.StartTime), zap.Int64("until", m.StopTime), zap.Int64("maxDataPoints", m.MaxDataPoints), zap.String("target", m.PathExpression), ) } } return fetchRequests, nil } // ParseRequest first tries to get body for application/json and convert it to carbonapi_v3_pb.MultiFetchRequest. As a fail-over it // parses request forms. func (*JSON) ParseRequest(r *http.Request) (data.MultiTarget, error) { if !scope.Debug(r.Context(), "Output") { return nil, errors.New("json format is only enabled for debugging purposes, pass 'X-Gch-Debug-Output: true' header") } fetchRequests, err := parseJSONBody(r) if err == nil { return fetchRequests, err } return parseRequestForms(r) } // Reply response to request with JSON representation of carbonapi_v3_pb.MultiFetchResponse. func (*JSON) Reply(w http.ResponseWriter, r *http.Request, multiData data.CHResponses) { mfr, err := multiData.ToMultiFetchResponseV3() if err != nil { http.Error(w, fmt.Sprintf("failed to convert response to v3pb.MultiFetchResponse: %v", err), http.StatusInternalServerError) return } response := marshalJSON(mfr) w.Write(response) } ================================================ FILE: render/reply/pickle.go ================================================ package reply import ( "bufio" "errors" "fmt" "math" "net/http" "time" graphitePickle "github.com/lomik/graphite-pickle" "go.uber.org/zap" "github.com/lomik/graphite-clickhouse/helper/point" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/render/data" ) // Pickle is a formatter for python object serialization format. type Pickle struct{} // ParseRequest parses target/from/until/maxDataPoints URL forms values func (*Pickle) ParseRequest(r *http.Request) (data.MultiTarget, error) { return parseRequestForms(r) } // Reply serializes ClickHouse response to pickle format func (*Pickle) Reply(w http.ResponseWriter, r *http.Request, multiData data.CHResponses) { var pickleTime time.Duration // Pickle format always contain single request/response data := multiData[0].Data from := uint32(multiData[0].From) until := uint32(multiData[0].Until) logger := scope.Logger(r.Context()) defer func() { logger.Debug("pickle", zap.String("runtime", pickleTime.String()), zap.Duration("runtime_ns", pickleTime), ) }() if data.AM.Len() == 0 { w.Write(graphitePickle.EmptyList) return } writer := bufio.NewWriterSize(w, 1024*1024) p := graphitePickle.NewWriter(writer) defer writer.Flush() p.List() writeAlias := func(name string, pathExpression string, points []point.Point, step uint32) { pickleStart := time.Now() p.Dict() p.String("name") p.String(name) p.SetItem() p.String("pathExpression") p.String(pathExpression) p.SetItem() p.String("step") p.Uint32(step) p.SetItem() start, end, _, getValue := point.FillNulls(points, from, until, step) p.String("values") p.List() for { value, err := getValue() if err != nil { if errors.Is(err, point.ErrTimeGreaterStop) { break } // if err is not point.ErrTimeGreaterStop, the points are corrupted return } if !math.IsNaN(value) { p.AppendFloat64(value) continue } p.AppendNulls(1) } p.SetItem() p.String("start") p.Uint32(start) p.SetItem() p.String("end") p.Uint32(end) p.SetItem() p.Append() pickleTime += time.Since(pickleStart) } // write points and mark as written in writeMap writeMetric := func(points []point.Point, writeMap map[string]struct{}) error { metricName := data.MetricName(points[0].MetricID) writeMap[metricName] = struct{}{} step, err := data.GetStep(points[0].MetricID) if err != nil { logger.Error("fail to get step", zap.Error(err)) http.Error(w, fmt.Sprintf("failed to get step for metric: %v", data.MetricName(points[0].MetricID)), http.StatusInternalServerError) return err } for _, a := range data.AM.Get(metricName) { writeAlias(a.DisplayName, a.Target, points, step) } return nil } nextMetric := data.GroupByMetric() writtenMetrics := make(map[string]struct{}) // fill metrics with points for { points := nextMetric() if len(points) == 0 { break } if err := writeMetric(points, writtenMetrics); err != nil { return } } // fill metrics without points with NaN if multiData[0].AppendOutEmptySeries && len(writtenMetrics) != data.AM.Len() && data.CommonStep > 0 { for _, metricName := range data.AM.Series(false) { if _, done := writtenMetrics[metricName]; !done { for _, a := range data.AM.Get(metricName) { writeAlias(a.DisplayName, a.Target, []point.Point{}, uint32(data.CommonStep)) } } } } p.Stop() } ================================================ FILE: render/reply/protobuf.go ================================================ package reply import ( "bufio" "bytes" "fmt" "io" "math" "net/http" "go.uber.org/zap" "github.com/lomik/graphite-clickhouse/helper/point" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/render/data" ) var pbVarints []byte const ( repeated = 2 flt32 = 5 protobufMaxVarintBytes = 10 // maximum length of a varint ) type pb interface { initBuffer() writeBody(writer *bufio.Writer, target, name, function string, from, until, step uint32, points []point.Point) } func replyProtobuf(p pb, w http.ResponseWriter, r *http.Request, multiData data.CHResponses) { logger := scope.Logger(r.Context()) // var multiResponse carbonzipperpb.MultiFetchResponse writer := bufio.NewWriterSize(w, 1024*1024) defer writer.Flush() p.initBuffer() totalWritten := 0 for _, d := range multiData { data := d.Data from := uint32(d.From) until := uint32(d.Until) totalWritten++ nextMetric := data.GroupByMetric() writtenMetrics := make(map[string]struct{}) // fill metrics with points for { points := nextMetric() if len(points) == 0 { break } metricName := data.MetricName(points[0].MetricID) writtenMetrics[metricName] = struct{}{} step, err := data.GetStep(points[0].MetricID) if err != nil { logger.Error("fail to get step", zap.Error(err)) http.Error(w, fmt.Sprintf("failed to get step for metric: %v", data.MetricName(points[0].MetricID)), http.StatusInternalServerError) return } function, err := data.GetAggregation(points[0].MetricID) if err != nil { logger.Error("fail to get function", zap.Error(err)) http.Error(w, fmt.Sprintf("failed to get function for metric: %v", data.MetricName(points[0].MetricID)), http.StatusInternalServerError) return } for _, a := range data.AM.Get(metricName) { p.writeBody(writer, a.Target, a.DisplayName, function, from, until, step, points) } } // fill metrics without points with NaN if d.AppendOutEmptySeries && len(writtenMetrics) < data.AM.Len() && data.CommonStep > 0 { for _, metricName := range data.AM.Series(false) { if _, done := writtenMetrics[metricName]; !done { for _, a := range data.AM.Get(metricName) { p.writeBody(writer, a.Target, a.DisplayName, "any", from, until, uint32(data.CommonStep), []point.Point{}) } } } } } if totalWritten == 0 { w.WriteHeader(http.StatusNotFound) return } } func init() { // precalculate varints buf := bytes.NewBuffer(nil) for i := uint64(0); i < 16384; i++ { buf.Write(VarintEncode(i)) } pbVarints = buf.Bytes() } func VarintEncode(x uint64) []byte { var buf [protobufMaxVarintBytes]byte var n int for n = 0; x > 127; n++ { buf[n] = 0x80 | uint8(x&0x7F) x >>= 7 } buf[n] = uint8(x) n++ return buf[0:n] } func VarintWrite(w io.Writer, x uint64) { // for ResponseWriter. ignore write result if x < 128 { w.Write(pbVarints[x : x+1]) } else if x < 16384 { w.Write(pbVarints[x*2-128 : x*2-126]) } else { w.Write(VarintEncode(x)) } } func VarintLen(x uint64) uint64 { if x < 128 { return 1 } if x < 16384 { return 2 } j := uint64(2) for i := uint64(16384); i <= x; i *= 128 { j++ } return j } func WriteByteN(w *bufio.Writer, value byte, n int) { // @TODO: optimize for i := 0; i < n; i++ { w.WriteByte(value) } } func Fixed64Encode(x uint64) []byte { return []byte{ uint8(x), uint8(x >> 8), uint8(x >> 16), uint8(x >> 24), uint8(x >> 32), uint8(x >> 40), uint8(x >> 48), uint8(x >> 56), } } func Fixed32Encode(x uint32) []byte { return []byte{ uint8(x), uint8(x >> 8), uint8(x >> 16), uint8(x >> 24), } } func ProtobufWriteSingle(w io.Writer, value float32) { w.Write(Fixed32Encode(math.Float32bits(value))) } func ProtobufWriteDouble(w io.Writer, value float64) { w.Write(Fixed64Encode(math.Float64bits(value))) } func ProtobufWriteDoubleN(w io.Writer, value float64, n int) { b := Fixed64Encode(math.Float64bits(value)) for i := 0; i < n; i++ { w.Write(b) } } ================================================ FILE: render/reply/protobuf_test.go ================================================ package reply import ( "encoding/binary" "testing" ) func TestVarintLen(t *testing.T) { buf := make([]byte, binary.MaxVarintLen64) for i := uint64(0); i < 1000000; i++ { n := binary.PutUvarint(buf, i) if VarintLen(i) != uint64(n) { t.FailNow() } } } ================================================ FILE: render/reply/v2_pb.go ================================================ package reply import ( "bufio" "bytes" "errors" "fmt" "math" "net/http" "github.com/lomik/graphite-clickhouse/helper/point" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/render/data" ) // V2PB is a formatter for carbonapi_v2_pb type V2PB struct { b1 *bytes.Buffer b2 *bytes.Buffer } // ParseRequest parses target/from/until/maxDataPoints URL forms values func (*V2PB) ParseRequest(r *http.Request) (data.MultiTarget, error) { return parseRequestForms(r) } // Reply serializes ClickHouse response to carbonapi_v2_pb.MultiFetchResponse format func (v *V2PB) Reply(w http.ResponseWriter, r *http.Request, multiData data.CHResponses) { if scope.Debug(r.Context(), "Protobuf") { v.replyDebug(w, r, multiData) } replyProtobuf(v, w, r, multiData) } func (v *V2PB) initBuffer() { v.b1 = new(bytes.Buffer) v.b2 = new(bytes.Buffer) } func (v *V2PB) replyDebug(w http.ResponseWriter, r *http.Request, multiData data.CHResponses) { mfr, err := multiData.ToMultiFetchResponseV2() if err != nil { http.Error(w, fmt.Sprintf("failed to convert response to v2pb.MultiFetchResponse: %v", err), http.StatusInternalServerError) } response, err := mfr.Marshal() if err != nil { http.Error(w, fmt.Sprintf("failed to marshal v2pb.MultiFetchResponse: %v", err), http.StatusInternalServerError) } w.Write(response) } func (v *V2PB) writeBody(writer *bufio.Writer, target, name, function string, from, until, step uint32, points []point.Point) { start, stop, count, getValue := point.FillNulls(points, from, until, step) v.b1.Reset() v.b2.Reset() // name VarintWrite(v.b1, (1<<3)+repeated) // tag VarintWrite(v.b1, uint64(len(name))) v.b1.WriteString(name) // start VarintWrite(v.b1, 2<<3) VarintWrite(v.b1, uint64(start)) // stop VarintWrite(v.b1, 3<<3) VarintWrite(v.b1, uint64(stop)) // step VarintWrite(v.b1, 4<<3) VarintWrite(v.b1, uint64(step)) // start write to output // Write values VarintWrite(v.b1, (5<<3)+repeated) VarintWrite(v.b1, uint64(8*count)) // Write isAbsent VarintWrite(v.b2, (6<<3)+repeated) VarintWrite(v.b2, uint64(count)) for { value, err := getValue() if err != nil { if errors.Is(err, point.ErrTimeGreaterStop) { break } // if err is not point.ErrTimeGreaterStop, the points are corrupted return } if !math.IsNaN(value) { ProtobufWriteDouble(v.b1, value) v.b2.WriteByte(0) continue } ProtobufWriteDouble(v.b1, 0) v.b2.WriteByte(1) } // repeated FetchResponse metrics = 1; // write tag and len VarintWrite(writer, (1<<3)+repeated) VarintWrite(writer, uint64(v.b1.Len())+uint64(v.b2.Len())) writer.Write(v.b1.Bytes()) writer.Write(v.b2.Bytes()) } ================================================ FILE: render/reply/v2_pb_test.go ================================================ package reply import ( "bufio" "bytes" "math" "reflect" "testing" v2pb "github.com/go-graphite/protocol/carbonapi_v2_pb" "github.com/lomik/graphite-clickhouse/helper/point" ) type testV2PB struct { name string target string function string response v2pb.MultiFetchResponse from uint32 until uint32 step uint32 points []point.Point } func TestV2PBWriteBody(t *testing.T) { tests := []testV2PB{ { name: "singlePoint", function: "avg", from: 4, until: 13, step: 5, target: "*", points: []point.Point{ { MetricID: 0, Value: 1.0, Time: 5, Timestamp: 5, }, }, response: v2pb.MultiFetchResponse{ Metrics: []v2pb.FetchResponse{ { Name: "singlePoint", StartTime: 5, StopTime: 10, StepTime: 5, Values: []float64{1.0}, IsAbsent: []bool{false}, }, }, }, }, { name: "multiPoint", function: "max", from: 1, until: 5, step: 1, target: "multiPoint", points: []point.Point{ { MetricID: 0, Value: 1.0, Time: 2, Timestamp: 2, }, { MetricID: 0, Value: math.NaN(), Time: 3, Timestamp: 3, }, { MetricID: 0, Value: 3.0, Time: 4, Timestamp: 4, }, }, response: v2pb.MultiFetchResponse{ Metrics: []v2pb.FetchResponse{ { Name: "multiPoint", StartTime: 1, StopTime: 6, StepTime: 1, Values: []float64{math.NaN(), 1.0, math.NaN(), 3.0, math.NaN()}, IsAbsent: []bool{true, false, true, false, true}, }, }, }, }, } for _, tt := range tests { testName := tt.name t.Run(testName, func(t *testing.T) { correctResp, _ := tt.response.Marshal() b := bytes.Buffer{} w := bufio.NewWriter(&b) v := &V2PB{} v.initBuffer() v.writeBody(w, tt.target, tt.name, tt.function, tt.from, tt.until, tt.step, tt.points) w.Flush() var resp v2pb.MultiFetchResponse data := b.Bytes() if bytes.Compare(data, correctResp) != 0 { t.Logf("different byte response.\ngot:\n%v\n\nexpected:\n%v", data, correctResp) } err := resp.Unmarshal(data) if err != nil { t.Fatalf("failed to unmarshal reply, got '%v'", err) } if len(resp.Metrics) != len(tt.response.Metrics) { t.Fatalf("incorrect amount of metrics, expected %v, got %v", len(resp.Metrics), len(tt.response.Metrics)) } for i := range resp.Metrics { if resp.Metrics[i].Name != tt.response.Metrics[i].Name { if !reflect.DeepEqual(resp.Metrics[i], tt.response.Metrics[i]) { t.Fatalf("replies are not same.\ngot:\n%+v\n\nexpected:\n%+v", resp.Metrics[i], tt.response.Metrics[i]) } } } }) } } ================================================ FILE: render/reply/v3_pb.go ================================================ package reply import ( "bufio" "bytes" "encoding/json" "errors" "fmt" "io" "net/http" v3pb "github.com/go-graphite/protocol/carbonapi_v3_pb" "github.com/lomik/graphite-clickhouse/helper/point" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/render/data" "go.uber.org/zap" ) // V3PB is a formatter for carbonapi_v3_pb type V3PB struct { b *bytes.Buffer } // ParseRequest reads the requests parameters from carbonapi_v3_pb.MultiFetchRequest func (*V3PB) ParseRequest(r *http.Request) (data.MultiTarget, error) { logger := scope.Logger(r.Context()).Named("pb3parser") body, err := io.ReadAll(r.Body) if err != nil { logger.Error("failed to read request", zap.Error(err)) return nil, fmt.Errorf("failed to read request body: %w", err) } var pv3Request v3pb.MultiFetchRequest if err := pv3Request.Unmarshal(body); err != nil { logger.Error("failed to unmarshal request", zap.Error(err)) return nil, fmt.Errorf("failed to unmarshal request: %w", err) } multiTarget := data.MFRToMultiTarget(&pv3Request) if len(pv3Request.Metrics) > 0 { for _, m := range pv3Request.Metrics { logger.Info( "pb3_target", zap.Int64("from", m.StartTime), zap.Int64("until", m.StopTime), zap.Int64("maxDataPoints", m.MaxDataPoints), zap.String("target", m.PathExpression), ) } } if scope.Debug(r.Context(), "Output") { request, err := json.Marshal(pv3Request) if err == nil { logger.Info("v3pb_request", zap.ByteString("json", request)) } } return multiTarget, nil } // Reply serializes ClickHouse response to carbonapi_v3_pb.MultiFetchResponse format func (v *V3PB) Reply(w http.ResponseWriter, r *http.Request, multiData data.CHResponses) { if scope.Debug(r.Context(), "Protobuf") { v.replyDebug(w, r, multiData) } replyProtobuf(v, w, r, multiData) } func (v *V3PB) initBuffer() { v.b = new(bytes.Buffer) } func (v *V3PB) replyDebug(w http.ResponseWriter, r *http.Request, multiData data.CHResponses) { mfr, err := multiData.ToMultiFetchResponseV3() if err != nil { http.Error(w, fmt.Sprintf("failed to convert response to v3pb.MultiFetchResponse: %v", err), http.StatusInternalServerError) } response, err := mfr.Marshal() if err != nil { http.Error(w, fmt.Sprintf("failed to marshal v3pb.MultiFetchResponse: %v", err), http.StatusInternalServerError) } w.Write(response) } func (v *V3PB) writeBody(writer *bufio.Writer, target, name, function string, from, until, step uint32, points []point.Point) { start, stop, count, getValue := point.FillNulls(points, from, until, step) v.b.Reset() // First chunk // name VarintWrite(v.b, (1<<3)+repeated) // tag VarintWrite(v.b, uint64(len(name))) v.b.WriteString(name) // pathExpression VarintWrite(v.b, (2<<3)+repeated) // tag VarintWrite(v.b, uint64(len(target))) v.b.WriteString(target) consolidationFunc := function // consolidationFunc VarintWrite(v.b, (3<<3)+repeated) // tag VarintWrite(v.b, uint64(len(consolidationFunc))) v.b.WriteString(consolidationFunc) // start VarintWrite(v.b, 4<<3) // tag VarintWrite(v.b, uint64(start)) // stop VarintWrite(v.b, 5<<3) // tag VarintWrite(v.b, uint64(stop)) // step VarintWrite(v.b, 6<<3) // tag VarintWrite(v.b, uint64(step)) // xFilesFactor VarintWrite(v.b, (7<<3)+flt32) // tag ProtobufWriteSingle(v.b, 0.0) // highPrecisionTimestamps VarintWrite(v.b, 8<<3) // tag v.b.WriteByte('\x00') // False // Values header VarintWrite(v.b, (9<<3)+repeated) // tag VarintWrite(v.b, uint64(8*count)) for { value, err := getValue() if err != nil { if errors.Is(err, point.ErrTimeGreaterStop) { break } // if err is not point.ErrTimeGreaterStop, the points are corrupted return } ProtobufWriteDouble(v.b, value) } // rest fields, that goes after values // Fields with default values are skipped, so this should be uncommented if support for appliedFunctions will be // implemented // appliedFunctions //VarintWrite(mb2, (10<<3)+Repeated) // tag //VarintWrite(mb2, VarintLen(0)) // currently not supported // requestStartTime VarintWrite(v.b, 11<<3) VarintWrite(v.b, uint64(from)) // requestStopTime VarintWrite(v.b, 12<<3) VarintWrite(v.b, uint64(until)) // start write to output // repeated FetchResponse metrics = 1; // write tag and len VarintWrite(writer, (1<<3)+2) VarintWrite(writer, uint64(v.b.Len())) writer.Write(v.b.Bytes()) } ================================================ FILE: render/reply/v3_pb_test.go ================================================ package reply import ( "bufio" "bytes" "math" "reflect" "testing" v3pb "github.com/go-graphite/protocol/carbonapi_v3_pb" "github.com/lomik/graphite-clickhouse/helper/point" ) type testV3PB struct { name string target string function string response v3pb.MultiFetchResponse from uint32 until uint32 step uint32 points []point.Point } func TestV3PBWriteBody(t *testing.T) { tests := []testV3PB{ { name: "singlePoint", function: "avg", from: 4, until: 13, step: 5, target: "*", points: []point.Point{ { MetricID: 0, Value: 1.0, Time: 5, Timestamp: 5, }, }, response: v3pb.MultiFetchResponse{ Metrics: []v3pb.FetchResponse{ { Name: "singlePoint", PathExpression: "*", ConsolidationFunc: "avg", XFilesFactor: 0, HighPrecisionTimestamps: false, StartTime: 5, StopTime: 10, Values: []float64{1.0}, AppliedFunctions: []string{}, RequestStartTime: 4, RequestStopTime: 13, }, }, }, }, { name: "multiPoint", function: "max", from: 1, until: 5, step: 1, target: "multiPoint", points: []point.Point{ { MetricID: 0, Value: 1.0, Time: 2, Timestamp: 2, }, { MetricID: 0, Value: math.NaN(), Time: 3, Timestamp: 3, }, { MetricID: 0, Value: 3.0, Time: 4, Timestamp: 4, }, }, response: v3pb.MultiFetchResponse{ Metrics: []v3pb.FetchResponse{ { Name: "multiPoint", PathExpression: "multiPoint", ConsolidationFunc: "max", XFilesFactor: 0, HighPrecisionTimestamps: false, StartTime: 1, StopTime: 6, Values: []float64{math.NaN(), 1.0, math.NaN(), 3.0, math.NaN()}, AppliedFunctions: []string{}, RequestStartTime: 1, RequestStopTime: 6, }, }, }, }, } for _, tt := range tests { testName := tt.name t.Run(testName, func(t *testing.T) { correctResp, _ := tt.response.Marshal() b := bytes.Buffer{} w := bufio.NewWriter(&b) v := &V3PB{} v.initBuffer() v.writeBody(w, tt.target, tt.name, tt.function, tt.from, tt.until, tt.step, tt.points) w.Flush() var resp v3pb.MultiFetchResponse data := b.Bytes() if bytes.Compare(data, correctResp) != 0 { t.Logf("different byte response.\ngot:\n%v\n\nexpected:\n%v", data, correctResp) } err := resp.Unmarshal(data) if err != nil { t.Fatalf("failed to unmarshal reply, got '%v'", err) } if len(resp.Metrics) != len(tt.response.Metrics) { t.Fatalf( "incorrect amount of metrics, expected %v, got %v", len(resp.Metrics), len(tt.response.Metrics), ) } for i := range resp.Metrics { if resp.Metrics[i].Name != tt.response.Metrics[i].Name { if !reflect.DeepEqual(resp.Metrics[i], tt.response.Metrics[i]) { t.Fatalf( "replies are not same.\ngot:\n%+v\n\nexpected:\n%+v", resp.Metrics[i], tt.response.Metrics[i], ) } } } }) } } ================================================ FILE: sd/nginx/nginx.go ================================================ package nginx import ( "encoding/base64" "errors" "strconv" "strings" "time" "github.com/lomik/graphite-clickhouse/sd/utils" jsoniter "github.com/json-iterator/go" "github.com/msaf1980/go-stringutils" "go.uber.org/zap" ) type ErrInvalidKey struct { key string val string } func (e ErrInvalidKey) Error() string { return "list key '" + e.key + "' is invalid: '" + e.val + "'" } var ( json = jsoniter.ConfigCompatibleWithStandardLibrary ErrNoKey = errors.New("list key no found") timeNow = time.Now ) func splitNode(node string) (dc, host, listen string, ok bool) { var v string dc, v, ok = strings.Cut(node, "/") if !ok { return } host, v, ok = strings.Cut(v, "/") if !ok { return } listen, _, ok = strings.Cut(v, "/") ok = !ok return } // Nginx register node for https://github.com/weibocom/nginx-upsync-module (with consul) type Nginx struct { weight int64 hostname string namespace string body []byte backupBody []byte url stringutils.Builder pos int // truncate offset for base address nsEnd string logger *zap.Logger } func New(url, namespace, hostname string, logger *zap.Logger) *Nginx { if namespace == "" { namespace = "graphite" } sd := &Nginx{ logger: logger, body: make([]byte, 128), backupBody: []byte(`{"backup":1,"max_fails":0}`), nsEnd: "upstreams/" + namespace + "/", hostname: hostname, namespace: namespace, } sd.setWeight(1) sd.url.WriteString(url) sd.url.WriteByte('/') if namespace != "" { sd.url.WriteString(namespace) sd.url.WriteByte('/') } sd.pos = sd.url.Len() return sd } func (sd *Nginx) setWeight(weight int64) { if weight <= 0 { weight = 1 } if sd.weight != weight { sd.weight = weight sd.body = sd.body[:0] sd.body = append(sd.body, `{"weight":`...) sd.body = strconv.AppendInt(sd.body, weight, 10) sd.body = append(sd.body, `,"max_fails":0}`...) } } func (sd *Nginx) Namespace() string { return sd.namespace } func (sd *Nginx) List() (nodes []string, err error) { sd.url.Truncate(sd.pos) sd.url.WriteString("?recurse") var data []byte data, err = utils.HttpGet(sd.url.String()) if err != nil { return } var iNodes []interface{} if err = json.Unmarshal(data, &iNodes); err != nil { return nil, err } nodes = make([]string, 0, len(iNodes)) for _, i := range iNodes { if jNode, ok := i.(map[string]interface{}); ok { if i, ok := jNode["Key"]; ok { if s, ok := i.(string); ok { if strings.HasPrefix(s, sd.nsEnd) { s = s[len(sd.nsEnd):] _, host, _, ok := splitNode(s) if ok && host == sd.hostname { nodes = append(nodes, s) } } else { return nil, ErrInvalidKey{key: sd.nsEnd, val: s} } } else { return nil, ErrNoKey } } } else { return nil, ErrNoKey } } return } func (sd *Nginx) ListMap() (nodes map[string]string, err error) { sd.url.Truncate(sd.pos) sd.url.WriteString("?recurse") var data []byte data, err = utils.HttpGet(sd.url.String()) if err != nil { return } var iNodes []interface{} if err = json.Unmarshal(data, &iNodes); err != nil { return nil, err } nodes = make(map[string]string) for _, i := range iNodes { if jNode, ok := i.(map[string]interface{}); ok { if i, ok := jNode["Key"]; ok { if s, ok := i.(string); ok { if strings.HasPrefix(s, sd.nsEnd) { s = s[len(sd.nsEnd):] _, host, _, ok := splitNode(s) if ok && host == sd.hostname { if i, ok := jNode["Value"]; ok { if v, ok := i.(string); ok { d, err := base64.StdEncoding.DecodeString(v) if err != nil { return nil, err } nodes[s] = stringutils.UnsafeString(d) } else { nodes[s] = "" } } else { nodes[s] = "" } } } else { return nil, ErrInvalidKey{key: sd.nsEnd, val: s} } } else { return nil, ErrNoKey } } } else { return nil, ErrNoKey } } return } func (sd *Nginx) Nodes() (nodes []utils.KV, err error) { sd.url.Truncate(sd.pos) sd.url.WriteString("?recurse") var data []byte data, err = utils.HttpGet(sd.url.String()) if err != nil { return } var iNodes []interface{} if err = json.Unmarshal(data, &iNodes); err != nil { return nil, err } nodes = make([]utils.KV, 0, 3) for _, i := range iNodes { if jNode, ok := i.(map[string]interface{}); ok { if i, ok := jNode["Key"]; ok { if s, ok := i.(string); ok { if strings.HasPrefix(s, sd.nsEnd) { s = s[len(sd.nsEnd):] kv := utils.KV{Key: s} if i, ok := jNode["Value"]; ok { if v, ok := i.(string); ok { d, err := base64.StdEncoding.DecodeString(v) if err != nil { return nil, err } kv.Value = stringutils.UnsafeString(d) } } if i, ok := jNode["Flags"]; ok { switch v := i.(type) { case float64: kv.Flags = int64(v) case int: kv.Flags = int64(v) case int64: kv.Flags = v } } nodes = append(nodes, kv) } else { return nil, ErrInvalidKey{key: sd.nsEnd, val: s} } } else { return nil, ErrNoKey } } } else { return nil, ErrNoKey } } return } func (sd *Nginx) update(ip, port string, dc []string) (err error) { if len(dc) == 0 { sd.url.Truncate(sd.pos) sd.url.WriteString("_/") sd.url.WriteString(sd.hostname) sd.url.WriteByte('/') if ip != "" { sd.url.WriteString(ip) } sd.url.WriteString(port) // add custom query flags sd.url.WriteByte('?') sd.url.WriteString("flags=") sd.url.WriteInt(timeNow().Unix(), 10) if err = utils.HttpPut(sd.url.String(), sd.body); err != nil { sd.logger.Error("put", zap.String("address", sd.url.String()[sd.pos:]), zap.Error(err)) return } } else { flags := make([]byte, 0, 32) flags = append(flags, "?flags="...) flags = strconv.AppendInt(flags, timeNow().Unix(), 10) for i := 0; i < len(dc); i++ { // cfg.Common.SDDc sd.url.Truncate(sd.pos) sd.url.WriteString(dc[i]) sd.url.WriteByte('/') sd.url.WriteString(sd.hostname) sd.url.WriteByte('/') n := sd.url.Len() if ip != "" { sd.url.WriteString(ip) } sd.url.WriteString(port) // add custom query flags sd.url.Write(flags) if i == 0 { if nErr := utils.HttpPut(sd.url.String(), sd.body); nErr != nil { sd.logger.Error( "put", zap.String("address", sd.url.String()[n:]), zap.String("dc", dc[i]), zap.Error(nErr), ) err = nErr } } else { if nErr := utils.HttpPut(sd.url.String(), sd.backupBody); nErr != nil { sd.logger.Error( "put", zap.String("address", sd.url.String()[n:]), zap.String("dc", dc[i]), zap.Error(nErr), ) err = nErr } } } } return } func (sd *Nginx) Update(ip, port string, dc []string, weight int64) error { sd.setWeight(weight) return sd.update(ip, port, dc) } func (sd *Nginx) DeleteNode(node string) (err error) { sd.url.Truncate(sd.pos) sd.url.WriteString(node) if err = utils.HttpDelete(sd.url.String()); err != nil { sd.logger.Error("delete", zap.String("address", sd.url.String()[sd.pos:]), zap.Error(err)) } return } func (sd *Nginx) Delete(ip, port string, dc []string) (err error) { if len(dc) == 0 { sd.url.Truncate(sd.pos) sd.url.WriteString("_/") sd.url.WriteString(sd.hostname) sd.url.WriteByte('/') if ip != "" { sd.url.WriteString(ip) } sd.url.WriteString(port) if err = utils.HttpDelete(sd.url.String()); err != nil { sd.logger.Error("delete", zap.String("address", sd.url.String()[sd.pos:]), zap.Error(err)) } } else { for i := 0; i < len(dc); i++ { // cfg.Common.SDDc sd.url.Truncate(sd.pos) sd.url.WriteString(dc[i]) sd.url.WriteByte('/') sd.url.WriteString(sd.hostname) sd.url.WriteByte('/') n := sd.url.Len() if ip != "" { sd.url.WriteString(ip) } sd.url.WriteString(port) if nErr := utils.HttpDelete(sd.url.String()); nErr != nil { sd.logger.Error( "delete", zap.String("address", sd.url.String()[n:]), zap.String("dc", dc[i]), zap.Error(nErr), ) err = nErr } } } return } func (sd *Nginx) Clear(preserveIP, preservePort string) (err error) { var nodes []string nodes, err = sd.List() if err != nil { sd.logger.Error( "list", zap.String("address", sd.url.String()[sd.pos:]), zap.Error(err), ) return } if len(nodes) == 0 { return } preserveListen := preserveIP + preservePort sd.url.WriteByte('/') for _, node := range nodes { sd.url.Truncate(sd.pos) _, host, listen, _ := splitNode(node) if host == sd.hostname && listen != preserveListen { sd.url.WriteString(node) if nErr := utils.HttpDelete(sd.url.String()); nErr != nil { sd.logger.Error( "delete", zap.String("address", sd.url.String()), zap.Error(nErr), ) err = nErr } } } return } ================================================ FILE: sd/nginx/nginx_test.go ================================================ //go:build test_sd // +build test_sd package nginx import ( "sort" "testing" "time" "github.com/lomik/graphite-clickhouse/sd/utils" "github.com/lomik/zapwriter" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) var ( hostname1 = "test_host1" ip1 = "192.168.0.1" hostname2 = "test_host2" ip2 = "192.168.1.25" port = ":9090" dc1 = []string{"dc1", "dc2", "dc3"} dc2 = []string{"dc2", "dc1", "dc3"} hostname3 = "test_host3" nilStringSlice []string ) func TestNginx(t *testing.T) { timeNow = func() time.Time { return time.Unix(1682408721, 0) } logger := zapwriter.Default() sd1 := New("http://127.0.0.1:8500/v1/kv/upstreams", "graphite", hostname1, logger) sd2 := New("http://127.0.0.1:8500/v1/kv/upstreams", "", hostname2, logger) err := sd1.Clear("", "") require.True(t, err == nil || err == utils.ErrNotFound, err) err = sd2.Clear("", "") require.True(t, err == nil || err == utils.ErrNotFound, err) nodes, err := sd1.List() require.True(t, err == nil || err == utils.ErrNotFound, err) assert.Equal(t, nilStringSlice, nodes) nodes, err = sd2.List() require.True(t, err == nil || err == utils.ErrNotFound, err) assert.Equal(t, nilStringSlice, nodes) // register new require.NoError(t, sd1.Update(ip1, port, nil, 10)) nodes, err = sd1.List() require.NoError(t, err) sort.Strings(nodes) assert.Equal( t, []string{ "_/test_host1/192.168.0.1:9090", }, nodes, ) nodesMap, err := sd1.ListMap() require.NoError(t, err) assert.Equal( t, map[string]string{ "_/test_host1/192.168.0.1:9090": `{"weight":10,"max_fails":0}`, }, nodesMap, ) // register new require.NoError(t, sd2.Update(ip2, port, nil, 21)) nodes, err = sd2.List() sort.Strings(nodes) require.NoError(t, err) assert.Equal( t, []string{ "_/test_host2/192.168.1.25:9090", }, nodes, ) nodesMap, err = sd2.ListMap() require.NoError(t, err) assert.Equal( t, map[string]string{ "_/test_host2/192.168.1.25:9090": `{"weight":21,"max_fails":0}`, }, nodesMap, ) // update require.NoError(t, sd2.Update(ip2, port, nil, 25)) nodes, err = sd2.List() sort.Strings(nodes) require.NoError(t, err) assert.Equal( t, []string{ "_/test_host2/192.168.1.25:9090", }, nodes, ) nodesMap, err = sd2.ListMap() require.NoError(t, err) assert.Equal( t, map[string]string{ "_/test_host2/192.168.1.25:9090": `{"weight":25,"max_fails":0}`, }, nodesMap, ) // delete require.NoError(t, sd2.Delete(ip2, port, nil)) nodes, err = sd2.List() sort.Strings(nodes) require.NoError(t, err) assert.Equal(t, []string{}, nodes) nodesMap, err = sd1.ListMap() require.NoError(t, err) assert.Equal( t, map[string]string{ "_/test_host1/192.168.0.1:9090": `{"weight":10,"max_fails":0}`, }, nodesMap, ) // cleanup require.NoError(t, sd2.Update(ip2, port, nil, 25)) require.NoError(t, sd2.Update(ip1, port, nil, 25)) nodesMap, err = sd2.ListMap() require.NoError(t, err) assert.Equal( t, map[string]string{ "_/test_host2/192.168.1.25:9090": `{"weight":25,"max_fails":0}`, "_/test_host2/192.168.0.1:9090": `{"weight":25,"max_fails":0}`, }, nodesMap, ) nodesV, err := sd2.Nodes() require.NoError(t, err) assert.Equal( t, []utils.KV{ {Key: "_/test_host1/192.168.0.1:9090", Value: `{"weight":10,"max_fails":0}`, Flags: 1682408721}, {Key: "_/test_host2/192.168.0.1:9090", Value: `{"weight":25,"max_fails":0}`, Flags: 1682408721}, {Key: "_/test_host2/192.168.1.25:9090", Value: `{"weight":25,"max_fails":0}`, Flags: 1682408721}, }, nodesV, ) require.NoError(t, sd2.Clear(ip2, port)) nodesMap, err = sd2.ListMap() require.NoError(t, err) assert.Equal( t, map[string]string{ "_/test_host2/192.168.1.25:9090": `{"weight":25,"max_fails":0}`, }, nodesMap, ) // clear all require.NoError(t, sd1.Clear("", "")) nodes, err = sd1.List() require.NoError(t, err) assert.Equal(t, []string{}, nodes) require.NoError(t, sd2.Clear("", "")) nodes, err = sd2.List() require.True(t, err == nil || err == utils.ErrNotFound, err) assert.Equal(t, nilStringSlice, nodes) } func TestNginxDC(t *testing.T) { timeNow = func() time.Time { return time.Unix(1682408721, 0) } logger := zapwriter.Default() sd1 := New("http://127.0.0.1:8500/v1/kv/upstreams", "", hostname1, logger) sd2 := New("http://127.0.0.1:8500/v1/kv/upstreams", "graphite", hostname2, logger) err := sd1.Clear("", "") require.True(t, err == nil || err == utils.ErrNotFound, err) err = sd2.Clear("", "") require.True(t, err == nil || err == utils.ErrNotFound, err) nodes, err := sd1.List() require.True(t, err == nil || err == utils.ErrNotFound, err) assert.Equal(t, nilStringSlice, nodes) nodes, err = sd2.List() require.True(t, err == nil || err == utils.ErrNotFound, err) assert.Equal(t, nilStringSlice, nodes) // register new require.NoError(t, sd1.Update(ip1, port, dc1, 10)) nodes, err = sd1.List() require.NoError(t, err) sort.Strings(nodes) assert.Equal( t, []string{ "dc1/test_host1/192.168.0.1:9090", "dc2/test_host1/192.168.0.1:9090", "dc3/test_host1/192.168.0.1:9090", }, nodes, ) nodesMap, err := sd1.ListMap() require.NoError(t, err) assert.Equal( t, map[string]string{ "dc1/test_host1/192.168.0.1:9090": `{"weight":10,"max_fails":0}`, "dc2/test_host1/192.168.0.1:9090": `{"backup":1,"max_fails":0}`, "dc3/test_host1/192.168.0.1:9090": `{"backup":1,"max_fails":0}`, }, nodesMap, ) // register new require.NoError(t, sd2.Update(ip2, port, dc2, 21)) nodes, err = sd2.List() sort.Strings(nodes) require.NoError(t, err) assert.Equal( t, []string{ "dc1/test_host2/192.168.1.25:9090", "dc2/test_host2/192.168.1.25:9090", "dc3/test_host2/192.168.1.25:9090", }, nodes, ) nodesMap, err = sd2.ListMap() require.NoError(t, err) assert.Equal( t, map[string]string{ "dc2/test_host2/192.168.1.25:9090": `{"weight":21,"max_fails":0}`, "dc1/test_host2/192.168.1.25:9090": `{"backup":1,"max_fails":0}`, "dc3/test_host2/192.168.1.25:9090": `{"backup":1,"max_fails":0}`, }, nodesMap, ) // update require.NoError(t, sd2.Update(ip2, port, dc2, 25)) nodes, err = sd2.List() sort.Strings(nodes) require.NoError(t, err) assert.Equal( t, []string{ "dc1/test_host2/192.168.1.25:9090", "dc2/test_host2/192.168.1.25:9090", "dc3/test_host2/192.168.1.25:9090", }, nodes, ) nodesMap, err = sd2.ListMap() require.NoError(t, err) assert.Equal( t, map[string]string{ "dc2/test_host2/192.168.1.25:9090": `{"weight":25,"max_fails":0}`, "dc1/test_host2/192.168.1.25:9090": `{"backup":1,"max_fails":0}`, "dc3/test_host2/192.168.1.25:9090": `{"backup":1,"max_fails":0}`, }, nodesMap, ) // delete require.NoError(t, sd2.Delete(ip2, port, dc2)) nodes, err = sd2.List() sort.Strings(nodes) require.NoError(t, err) assert.Equal(t, []string{}, nodes) nodesMap, err = sd1.ListMap() require.NoError(t, err) assert.Equal( t, map[string]string{ "dc1/test_host1/192.168.0.1:9090": `{"weight":10,"max_fails":0}`, "dc2/test_host1/192.168.0.1:9090": `{"backup":1,"max_fails":0}`, "dc3/test_host1/192.168.0.1:9090": `{"backup":1,"max_fails":0}`, }, nodesMap, ) // cleanup require.NoError(t, sd2.Update(ip2, port, dc2, 25)) require.NoError(t, sd2.Update(ip1, port, dc2, 25)) nodesMap, err = sd2.ListMap() require.NoError(t, err) assert.Equal( t, map[string]string{ "dc2/test_host2/192.168.1.25:9090": `{"weight":25,"max_fails":0}`, "dc1/test_host2/192.168.1.25:9090": `{"backup":1,"max_fails":0}`, "dc3/test_host2/192.168.1.25:9090": `{"backup":1,"max_fails":0}`, "dc2/test_host2/192.168.0.1:9090": `{"weight":25,"max_fails":0}`, "dc1/test_host2/192.168.0.1:9090": `{"backup":1,"max_fails":0}`, "dc3/test_host2/192.168.0.1:9090": `{"backup":1,"max_fails":0}`, }, nodesMap, ) nodesV, err := sd2.Nodes() require.NoError(t, err) assert.Equal( t, []utils.KV{ {Key: "dc1/test_host1/192.168.0.1:9090", Value: `{"weight":10,"max_fails":0}`, Flags: 1682408721}, {Key: "dc1/test_host2/192.168.0.1:9090", Value: `{"backup":1,"max_fails":0}`, Flags: 1682408721}, {Key: "dc1/test_host2/192.168.1.25:9090", Value: `{"backup":1,"max_fails":0}`, Flags: 1682408721}, {Key: "dc2/test_host1/192.168.0.1:9090", Value: `{"backup":1,"max_fails":0}`, Flags: 1682408721}, {Key: "dc2/test_host2/192.168.0.1:9090", Value: `{"weight":25,"max_fails":0}`, Flags: 1682408721}, {Key: "dc2/test_host2/192.168.1.25:9090", Value: `{"weight":25,"max_fails":0}`, Flags: 1682408721}, {Key: "dc3/test_host1/192.168.0.1:9090", Value: `{"backup":1,"max_fails":0}`, Flags: 1682408721}, {Key: "dc3/test_host2/192.168.0.1:9090", Value: `{"backup":1,"max_fails":0}`, Flags: 1682408721}, {Key: "dc3/test_host2/192.168.1.25:9090", Value: `{"backup":1,"max_fails":0}`, Flags: 1682408721}, }, nodesV, ) require.NoError(t, sd2.Clear(ip2, port)) nodesMap, err = sd2.ListMap() require.NoError(t, err) assert.Equal( t, map[string]string{ "dc2/test_host2/192.168.1.25:9090": `{"weight":25,"max_fails":0}`, "dc1/test_host2/192.168.1.25:9090": `{"backup":1,"max_fails":0}`, "dc3/test_host2/192.168.1.25:9090": `{"backup":1,"max_fails":0}`, }, nodesMap, ) // clear all require.NoError(t, sd1.Clear("", "")) nodes, err = sd1.List() require.NoError(t, err) assert.Equal(t, []string{}, nodes) require.NoError(t, sd2.Clear("", "")) nodes, err = sd2.List() assert.Equal(t, nilStringSlice, nodes) assert.Equal(t, nilStringSlice, nodes) } ================================================ FILE: sd/nginx/tests/nginx_cleanup_test.go ================================================ //go:build test_sd // +build test_sd package nginx_test import ( "testing" "time" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/sd" "github.com/lomik/graphite-clickhouse/sd/utils" "github.com/lomik/zapwriter" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) var ( hostname1 = "test_host1" ip1 = "192.168.0.1" hostname2 = "test_host2" ip2 = "192.168.1.25" port = ":9090" dc1 = []string{"dc1", "dc2", "dc3"} dc2 = []string{"dc2", "dc1", "dc3"} hostname3 = "test_host3" nilStringSlice []string ) func cleanup(nodes []utils.KV, start, end int64) { for i := range nodes { if nodes[i].Flags >= start && nodes[i].Flags <= end { nodes[i].Flags = start } } } func TestNginxExpire(t *testing.T) { logger := zapwriter.Default() cfg := &config.Common{ SDType: config.SDNginx, SD: "http://127.0.0.1:8500/v1/kv/upstreams", SDNamespace: "graphite", //default SDExpire: time.Second * 5, } sd1, _ := sd.New(cfg, hostname1, logger) sd2, _ := sd.New(cfg, hostname2, logger) err := sd1.Clear("", "") require.True(t, err == nil || err == utils.ErrNotFound, err) err = sd2.Clear("", "") require.True(t, err == nil || err == utils.ErrNotFound, err) nodes, err := sd1.List() require.True(t, err == nil || err == utils.ErrNotFound, err) assert.Equal(t, nilStringSlice, nodes) nodes, err = sd2.List() require.True(t, err == nil || err == utils.ErrNotFound, err) assert.Equal(t, nilStringSlice, nodes) // check cleanup expired start := time.Now().Unix() require.NoError(t, sd1.Update(ip1, port, nil, 10)) time.Sleep(cfg.SDExpire + time.Second) require.NoError(t, sd2.Update(ip2, port, nil, 10)) nodesV, err := sd1.Nodes() end := time.Now().Unix() require.NoError(t, err) // reset timestamp for compare cleanup(nodesV, start, end) assert.Equal( t, []utils.KV{ {Key: "_/test_host1/192.168.0.1:9090", Value: "{\"weight\":10,\"max_fails\":0}", Flags: start}, {Key: "_/test_host2/192.168.1.25:9090", Value: "{\"weight\":10,\"max_fails\":0}", Flags: start}, }, nodesV, "start = %d, end = %d", start, end, ) sd.Cleanup(cfg, sd1, false) nodesV, err = sd1.Nodes() require.NoError(t, err) // reset timestamp for compare cleanup(nodesV, start, end) assert.Equal( t, []utils.KV{ {Key: "_/test_host2/192.168.1.25:9090", Value: "{\"weight\":10,\"max_fails\":0}", Flags: start}, }, nodesV, "start = %d, end = %d", start, end, ) } func TestNginxExpireDC(t *testing.T) { logger := zapwriter.Default() cfg1 := &config.Common{ SDType: config.SDNginx, SD: "http://127.0.0.1:8500/v1/kv/upstreams", SDNamespace: "graphite", //default SDDc: dc1, SDExpire: time.Second * 5, } sd1, _ := sd.New(cfg1, hostname1, logger) cfg2 := &config.Common{ SDType: config.SDNginx, SD: "http://127.0.0.1:8500/v1/kv/upstreams", SDNamespace: "", //default SDDc: dc2, SDExpire: time.Second * 5, } sd2, _ := sd.New(cfg2, hostname2, logger) err := sd1.Clear("", "") require.True(t, err == nil || err == utils.ErrNotFound, err) err = sd2.Clear("", "") require.True(t, err == nil || err == utils.ErrNotFound, err) nodes, err := sd1.List() require.True(t, err == nil || err == utils.ErrNotFound, err) assert.Equal(t, nilStringSlice, nodes) nodes, err = sd2.List() require.True(t, err == nil || err == utils.ErrNotFound, err) assert.Equal(t, nilStringSlice, nodes) // check cleanup expired start := time.Now().Unix() require.NoError(t, sd1.Update(ip1, port, dc1, 10)) time.Sleep(cfg1.SDExpire + time.Second) require.NoError(t, sd2.Update(ip2, port, dc2, 10)) nodesV, err := sd1.Nodes() end := time.Now().Unix() require.NoError(t, err) // reset timestamp for compare cleanup(nodesV, start, end) assert.Equal( t, []utils.KV{ {Key: "dc1/test_host1/192.168.0.1:9090", Value: "{\"weight\":10,\"max_fails\":0}", Flags: start}, {Key: "dc1/test_host2/192.168.1.25:9090", Value: "{\"backup\":1,\"max_fails\":0}", Flags: start}, {Key: "dc2/test_host1/192.168.0.1:9090", Value: "{\"backup\":1,\"max_fails\":0}", Flags: start}, {Key: "dc2/test_host2/192.168.1.25:9090", Value: "{\"weight\":10,\"max_fails\":0}", Flags: start}, {Key: "dc3/test_host1/192.168.0.1:9090", Value: "{\"backup\":1,\"max_fails\":0}", Flags: start}, {Key: "dc3/test_host2/192.168.1.25:9090", Value: "{\"backup\":1,\"max_fails\":0}", Flags: start}, }, nodesV, "start = %d, end = %d", start, end, ) sd.Cleanup(cfg1, sd1, false) nodesV, err = sd1.Nodes() require.NoError(t, err) // reset timestamp for compare cleanup(nodesV, start, end) assert.Equal( t, []utils.KV{ {Key: "dc1/test_host2/192.168.1.25:9090", Value: "{\"backup\":1,\"max_fails\":0}", Flags: start}, {Key: "dc2/test_host2/192.168.1.25:9090", Value: "{\"weight\":10,\"max_fails\":0}", Flags: start}, {Key: "dc3/test_host2/192.168.1.25:9090", Value: "{\"backup\":1,\"max_fails\":0}", Flags: start}, }, nodesV, "start = %d, end = %d", start, end, ) } ================================================ FILE: sd/register.go ================================================ package sd import ( "errors" "fmt" "os" "strings" "time" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/load_avg" "github.com/lomik/graphite-clickhouse/sd/nginx" "github.com/lomik/graphite-clickhouse/sd/utils" "go.uber.org/zap" ) var ( // ctxMain, Stop = context.WithCancel(context.Background()) stop chan struct{} = make(chan struct{}, 1) delay = time.Second * 10 hostname string ) type SD interface { // Update update node record Update(listenIP, listenPort string, dc []string, weight int64) error // Delete delete node record (with ip/port/dcs) Delete(ip, port string, dcs []string) error // Delete delete node record DeleteNode(node string) (err error) // Clear clear node record (all except with current listen IP/port) Clear(listenIP, listenPort string) error // Nodes return all registered nodes (for all hostnames in namespace) Nodes() (nodes []utils.KV, err error) // List return all registered nodes for hostname List() (nodes []string, err error) // Namespace return namespace Namespace() string } func New(cfg *config.Common, hostname string, logger *zap.Logger) (SD, error) { switch cfg.SDType { case config.SDNginx: sd := nginx.New(cfg.SD, cfg.SDNamespace, hostname, logger) return sd, nil default: return nil, errors.New("serive discovery type not registered") } } func Register(cfg *config.Common, logger *zap.Logger) { var ( listenIP string prevIP string registerFirst bool sd SD err error load float64 w int64 ) if cfg.SD != "" { if strings.HasPrefix(cfg.Listen, ":") { registerFirst = true listenIP = utils.GetLocalIP() prevIP = listenIP } hostname, _ = os.Hostname() hostname, _, _ = strings.Cut(hostname, ".") sd, err = New(cfg, hostname, logger) if err != nil { panic("serive discovery type not registered") } load, err = load_avg.Normalized() if err == nil { load_avg.Store(load) } logger.Info("init sd", zap.String("hostname", hostname), ) w = load_avg.Weight(cfg.BaseWeight, cfg.DegragedMultiply, cfg.DegragedLoad, load) sd.Update(listenIP, cfg.Listen, cfg.SDDc, w) sd.Clear(listenIP, cfg.Listen) } LOOP: for { load, err = load_avg.Normalized() if err == nil { load_avg.Store(load) } if sd != nil { w = load_avg.Weight(cfg.BaseWeight, cfg.DegragedMultiply, cfg.DegragedLoad, load) if registerFirst { // if listen on all ip, try to register with first ip listenIP = utils.GetLocalIP() } sd.Update(listenIP, cfg.Listen, cfg.SDDc, w) if prevIP != listenIP { sd.Delete(prevIP, cfg.Listen, cfg.SDDc) prevIP = listenIP } } t := time.After(delay) select { case <-t: continue case <-stop: break LOOP } } if sd != nil { if err := sd.Clear("", ""); err == nil { logger.Info("cleanup sd", zap.String("hostname", hostname), ) } else { logger.Warn("cleanup sd", zap.String("hostname", hostname), zap.Error(err), ) } } } func Stop() { stop <- struct{}{} } func Cleanup(cfg *config.Common, sd SD, checkOnly bool) error { if cfg.SD != "" && cfg.SDExpire > 0 { ts := time.Now().Unix() - int64(cfg.SDExpire.Seconds()) if nodes, err := sd.Nodes(); err == nil { for _, node := range nodes { if node.Flags > 0 { if ts > node.Flags { if checkOnly { fmt.Printf("%s: %s (%s), expired\n", node.Key, node.Value, time.Unix(node.Flags, 0).UTC().Format(time.RFC3339Nano)) } else { if err = sd.DeleteNode(node.Key); err != nil { return err } fmt.Printf("%s: %s (%s), deleted\n", node.Key, node.Value, time.Unix(node.Flags, 0).UTC().Format(time.RFC3339Nano)) } } } else { fmt.Printf("%s: %s (%s)\n", node.Key, node.Value, time.Unix(node.Flags, 0).UTC().Format(time.RFC3339Nano)) } } } else { return err } } return nil } ================================================ FILE: sd/utils/utils.go ================================================ package utils import ( "bytes" "errors" "io" "net" "net/http" "time" "github.com/lomik/graphite-clickhouse/helper/errs" ) var ErrNotFound = errors.New("entry not found") type KV struct { Key string Value string Flags int64 } func HttpGet(url string) ([]byte, error) { client := &http.Client{Timeout: 2 * time.Second} resp, err := client.Get(url) if err != nil { return nil, err } data, err := io.ReadAll(resp.Body) resp.Body.Close() if resp.StatusCode == http.StatusNotFound { return nil, ErrNotFound } if resp.StatusCode != http.StatusOK { return nil, errs.NewErrorWithCode(string(data), resp.StatusCode) } return data, err } func HttpPut(url string, body []byte) error { req, err := http.NewRequest(http.MethodPut, url, bytes.NewBuffer(body)) if err != nil { return err } req.Header.Set("Content-Type", "application/json") client := &http.Client{Timeout: 2 * time.Second} resp, err := client.Do(req) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode == http.StatusNotFound { return ErrNotFound } if resp.StatusCode != http.StatusOK { data, _ := io.ReadAll(resp.Body) return errs.NewErrorWithCode(string(data), resp.StatusCode) } return nil } func HttpDelete(url string) error { req, err := http.NewRequest(http.MethodDelete, url, nil) if err != nil { return err } client := &http.Client{Timeout: 2 * time.Second} resp, err := client.Do(req) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode == http.StatusNotFound { return ErrNotFound } if resp.StatusCode != http.StatusOK { data, _ := io.ReadAll(resp.Body) return errs.NewErrorWithCode(string(data), resp.StatusCode) } return nil } // GetLocalIP returns the non loopback local IP of the host func GetLocalIP() string { addrs, err := net.InterfaceAddrs() if err != nil { return "" } for _, address := range addrs { // check the address type and if it is not a loopback the display it if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { if ipnet.IP.To4() != nil { return ipnet.IP.String() } } } return "" } ================================================ FILE: tagger/metric.go ================================================ package tagger import ( "bytes" "encoding/json" "github.com/lomik/graphite-clickhouse/pkg/dry" ) type Metric struct { Path []byte Level int ParentIndex int Tags *Set } func (m *Metric) ParentPath() []byte { if len(m.Path) == 0 { return nil } index := bytes.LastIndexByte(m.Path[:len(m.Path)-1], '.') if index < 0 { return nil } return m.Path[:index+1] } func (m *Metric) IsLeaf() uint8 { if len(m.Path) > 0 && m.Path[len(m.Path)-1] == '.' { return 0 } return 1 } func (m *Metric) MarshalJSON() ([]byte, error) { return json.Marshal(map[string]interface{}{ "Path": dry.UnsafeString(m.Path), "Level": m.Level, "Tags": m.Tags, "IsLeaf": m.IsLeaf(), }) } ================================================ FILE: tagger/rule.go ================================================ package tagger import ( "bytes" "os" "path/filepath" "regexp" "github.com/BurntSushi/toml" ) type Rule struct { Single string `toml:"tag"` List []string `toml:"tags"` re *regexp.Regexp `toml:"-"` Equal string `toml:"equal"` HasPrefix string `toml:"has-prefix"` HasSuffix string `toml:"has-suffix"` Contains string `toml:"contains"` Regexp string `toml:"regexp"` BytesEqual []byte `toml:"-"` BytesHasPrefix []byte `toml:"-"` BytesHasSuffix []byte `toml:"-"` BytesContains []byte `toml:"-"` Tags *Set `toml:"-"` } type Rules struct { Rule []Rule `toml:"rule"` prefix *Tree `toml:"-"` suffix *Tree `toml:"-"` // @TODO contains *Tree `toml:"-"` other []*Rule `toml:"-"` } func ParseFile(filename string) (*Rules, error) { c, err := os.ReadFile(filename) if err != nil { return nil, err } return Parse(string(c)) } func ParseGlob(glob string) (*Rules, error) { content := []byte{} files, err := filepath.Glob(glob) if err != nil { return nil, err } for i := 0; i < len(files); i++ { c, err := os.ReadFile(files[i]) if err != nil { return nil, err } content = append(append(content, '\n'), c...) } return Parse(string(content)) } func Parse(content string) (*Rules, error) { rules := &Rules{ prefix: &Tree{}, suffix: &Tree{}, contains: &Tree{}, other: make([]*Rule, 0), } if _, err := toml.Decode(content, rules); err != nil { return nil, err } var err error for i := 0; i < len(rules.Rule); i++ { rule := &rules.Rule[i] rule.Tags = EmptySet if rule.Single != "" { rule.Tags = rule.Tags.Add(rule.Single) } if rule.List != nil { rule.Tags = rule.Tags.Add(rule.List...) } // compile and check regexp rule.re, err = regexp.Compile(rule.Regexp) if err != nil { return nil, err } if rule.Equal != "" { rule.BytesEqual = []byte(rule.Equal) } if rule.Contains != "" { rule.BytesContains = []byte(rule.Contains) } if rule.HasPrefix != "" { rule.BytesHasPrefix = []byte(rule.HasPrefix) } if rule.HasSuffix != "" { rule.BytesHasSuffix = []byte(rule.HasSuffix) } if rule.BytesHasPrefix != nil { rules.prefix.Add(rule.BytesHasPrefix, rule) } else if rule.BytesEqual != nil { rules.prefix.Add(rule.BytesEqual, rule) } else if rule.BytesContains != nil { rules.contains.Add(rule.BytesContains, rule) } else if rule.BytesHasSuffix != nil { rules.suffix.AddSuffix(rule.BytesHasSuffix, rule) } else { rules.other = append(rules.other, rule) } } return rules, nil } func (r *Rules) Match(m *Metric) { r.matchPrefix(m) r.matchSuffix(m) r.matchContains(m) r.matchOther(m) } func matchByPrefix(path []byte, tree *Tree, m *Metric) { x := tree i := 0 for { if i >= len(path) { break } x = x.Next[path[i]] if x == nil { break } if x.Rules != nil { for _, rule := range x.Rules { rule.Match(m) } } i++ } } func matchBySuffix(path []byte, tree *Tree, m *Metric) { x := tree i := len(path) - 1 for { if i <= 0 { break } x = x.Next[path[i]] if x == nil { break } if x.Rules != nil { for _, rule := range x.Rules { rule.Match(m) } } i-- } } func (r *Rules) matchPrefix(m *Metric) { matchByPrefix(m.Path, r.prefix, m) } func (r *Rules) matchSuffix(m *Metric) { matchBySuffix(m.Path, r.suffix, m) } func (r *Rules) matchContains(m *Metric) { for i := 0; i < len(m.Path); i++ { matchByPrefix(m.Path[i:], r.contains, m) } } func (r *Rules) matchOther(m *Metric) { for _, rule := range r.other { rule.Match(m) } } func (r *Rule) Match(m *Metric) { if r.BytesEqual != nil && !bytes.Equal(m.Path, r.BytesEqual) { return } if r.BytesHasPrefix != nil && !bytes.HasPrefix(m.Path, r.BytesHasPrefix) { return } if r.BytesHasSuffix != nil && !bytes.HasSuffix(m.Path, r.BytesHasSuffix) { return } if r.BytesContains != nil && !bytes.Contains(m.Path, r.BytesContains) { return } if r.re != nil && !r.re.Match(m.Path) { return } m.Tags = m.Tags.Merge(r.Tags) } ================================================ FILE: tagger/rule_test.go ================================================ package tagger import ( "fmt" "sort" "testing" "github.com/stretchr/testify/assert" ) var RulesConf = ` [[rule]] tag = "prefix" has-prefix = "prefix" [[rule]] tag = "suffix" has-suffix = "suffix" [[rule]] tag = "contains" contains = "contains" [[rule]] tag = "equal" equal = "equal" [[rule]] tag = "regexp" regexp = "reg[e]xp" ` func TestRules(t *testing.T) { assert := assert.New(t) rules, err := Parse(RulesConf) assert.NoError(err) table := []struct { path string method string // "" for all, "prefix", "suffix", "contains" for use only specified tree expectedTags []string }{ {"prefix.metric", "", []string{"prefix"}}, {"prefix.metric", "prefix", []string{"prefix"}}, {"prefix.metric", "suffix", nil}, {"prefix.metric", "contains", nil}, {"prefix.metric", "other", nil}, {"metric.suffix", "", []string{"suffix"}}, {"metric.suffix", "prefix", nil}, {"metric.suffix", "suffix", []string{"suffix"}}, {"metric.suffix", "contains", nil}, {"metric.suffix", "other", nil}, {"hello.contains.world", "", []string{"contains"}}, {"hello.contains.world", "prefix", nil}, {"hello.contains.world", "suffix", nil}, {"hello.contains.world", "contains", []string{"contains"}}, {"hello.contains.world", "other", nil}, {"hello.regexp.world", "", []string{"regexp"}}, {"hello.regexp.world", "prefix", nil}, {"hello.regexp.world", "suffix", nil}, {"hello.regexp.world", "contains", nil}, {"hello.regexp.world", "other", []string{"regexp"}}, {"prefix.suffix", "", []string{"prefix", "suffix"}}, } for i := 0; i < len(table); i++ { t := table[i] m := Metric{Path: []byte(t.path), Tags: EmptySet} switch t.method { case "": rules.Match(&m) case "prefix": rules.matchPrefix(&m) case "suffix": rules.matchSuffix(&m) case "contains": rules.matchContains(&m) case "other": rules.matchOther(&m) } expected := t.expectedTags if expected == nil { expected = []string{} } sort.Strings(expected) tags := m.Tags.List() sort.Strings(tags) assert.Equal(expected, tags, fmt.Sprintf("path: %s, method: %s", t.path, t.method)) } } ================================================ FILE: tagger/set.go ================================================ package tagger import ( "encoding/json" ) // set with copy-on-write type Set struct { data map[string]bool list []string json []byte } var EmptySet = &Set{ data: make(map[string]bool), list: make([]string, 0), } func (s *Set) Add(tag ...string) *Set { var newList []string for _, t := range tag { if !s.data[t] { if newList == nil { newList = make([]string, len(s.list)+1) copy(newList, s.list) newList[len(newList)-1] = t } else { newList = append(newList, t) } } } // no new tags if newList == nil { return s } // new tag n := &Set{ data: make(map[string]bool), list: newList, } for _, t := range n.list { n.data[t] = true } return n } func (s *Set) Merge(other *Set) *Set { return s.Add(other.list...) } func (s *Set) Len() int { return len(s.list) } func (s *Set) List() []string { return s.list } func (s *Set) MarshalJSON() ([]byte, error) { if s.json != nil { return s.json, nil } var err error s.json, err = json.Marshal(s.list) if err != nil { return nil, err } return s.json, nil } ================================================ FILE: tagger/tagger.go ================================================ package tagger import ( "bytes" "compress/gzip" "context" "fmt" "io" "os" "runtime" "sort" "time" "go.uber.org/zap" "go.uber.org/zap/zapcore" "golang.org/x/sync/errgroup" "github.com/lomik/zapwriter" "github.com/klauspost/compress/zstd" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/helper/RowBinary" "github.com/lomik/graphite-clickhouse/helper/clickhouse" "github.com/lomik/graphite-clickhouse/pkg/scope" ) const SelectChunksCount = 10 type nopCloser struct { io.Writer } func (nopCloser) Close() error { return nil } func countMetrics(body []byte) (int, error) { var namelen uint64 bodyLen := len(body) var count, offset, readBytes int var err error for { if offset >= bodyLen { if offset == bodyLen { return count, nil } return 0, clickhouse.ErrClickHouseResponse } namelen, readBytes, err = clickhouse.ReadUvarint(body[offset:]) if err != nil { return 0, err } offset += readBytes + int(namelen) count++ } } func pathLevel(path []byte) int { if len(path) == 0 { return 0 } if path[len(path)-1] == '.' { return bytes.Count(path, []byte{'.'}) } return bytes.Count(path, []byte{'.'}) + 1 } func Make(cfg *config.Config) error { var start time.Time var block string logger := zapwriter.Logger("tagger") chOpts := clickhouse.Options{ TLSConfig: cfg.ClickHouse.TLSConfig, Timeout: cfg.ClickHouse.IndexTimeout, ConnectTimeout: cfg.ClickHouse.ConnectTimeout, CheckRequestProgress: cfg.FeatureFlags.LogQueryProgress, ProgressSendingInterval: cfg.ClickHouse.ProgressSendingInterval, } begin := func(b string, fields ...zapcore.Field) { block = b start = time.Now() logger.Info(fmt.Sprintf("begin %s", block), fields...) } end := func() { var m runtime.MemStats runtime.ReadMemStats(&m) d := time.Since(start) logger.Info(fmt.Sprintf("end %s", block), zap.Duration("time", d), zap.Uint64("mem_rss_mb", (m.Sys-m.HeapReleased)/1048576), ) } version := uint32(time.Now().Unix()) if cfg.Tags.Version != 0 { version = cfg.Tags.Version } logger.Info("start", zap.Uint32("version", version)) // Parse rules begin("parse rules") rules, err := ParseGlob(cfg.Tags.Rules) if err != nil { return err } date, err := time.ParseInLocation("2006-01-02", cfg.Tags.Date, time.Local) if err != nil { return err } end() selectChunksCount := SelectChunksCount if cfg.Tags.SelectChunksCount != 0 { selectChunksCount = cfg.Tags.SelectChunksCount } // Read clickhouse begin("read metrics", zap.Int("chunks_count", selectChunksCount)) var bodies [][]byte if cfg.Tags.InputFile != "" { body, err := os.ReadFile(cfg.Tags.InputFile) if err != nil { return err } bodies = [][]byte{body} } else { bodies = make([][]byte, selectChunksCount) extraWhere := "" if cfg.Tags.ExtraWhere != "" { extraWhere = fmt.Sprintf("AND (%s)", cfg.Tags.ExtraWhere) } for i := 0; i < selectChunksCount; i++ { bodies[i], _, _, err = clickhouse.Query( scope.New(context.Background()).WithLogger(logger).WithTable(cfg.ClickHouse.IndexTable), cfg.ClickHouse.URL, fmt.Sprintf( "SELECT Path FROM %s WHERE cityHash64(Path) %% %d = %d %s AND Level > 20000 AND Level < 30000 AND Date = '1970-02-12' GROUP BY Path FORMAT RowBinary", cfg.ClickHouse.IndexTable, selectChunksCount, i, extraWhere, ), chOpts, nil, ) if err != nil { return err } } } end() begin("parse metrics") var count int for i := 0; i < len(bodies); i++ { c, err := countMetrics(bodies[i]) if err != nil { return err } count += c } metricList := make([]Metric, count) index := 0 var maxLevel int for i := 0; i < len(bodies); i++ { body := bodies[i] var namelen uint64 bodyLen := len(body) var offset, readBytes int for ; ; index++ { if offset >= bodyLen { if offset == bodyLen { break } return clickhouse.ErrClickHouseResponse } namelen, readBytes, err = clickhouse.ReadUvarint(body[offset:]) if err != nil { return err } metricList[index].Path = body[offset+readBytes : offset+readBytes+int(namelen)] metricList[index].Level = pathLevel(metricList[index].Path) if metricList[index].Level > maxLevel { maxLevel = metricList[index].Level } offset += readBytes + int(namelen) } } end() begin("sort") start = time.Now() sort.Slice(metricList, func(i, j int) bool { return bytes.Compare(metricList[i].Path, metricList[j].Path) < 0 }) end() begin("make map") levelMap := make([]int, maxLevel+1) for index := 0; index < len(metricList); index++ { m := &metricList[index] levelMap[m.Level] = index if m.Level > 0 { parentIndex := levelMap[m.Level-1] if bytes.Equal(m.ParentPath(), metricList[parentIndex].Path) { m.ParentIndex = parentIndex } else { m.ParentIndex = -1 } } } end() begin("match", zap.Int("metrics_count", len(metricList))) for index := 0; index < count; index++ { m := &metricList[index] if m.ParentIndex < 0 { m.Tags = EmptySet } else { m.Tags = metricList[m.ParentIndex].Tags } rules.Match(m) } end() // copy from childs to parents begin("copy tags from childs to parents") for index := 0; index < count; index++ { m := &metricList[index] for p := m.ParentIndex; p >= 0; p = metricList[p].ParentIndex { metricList[p].Tags = metricList[p].Tags.Merge(m.Tags) } } end() begin("remove metrics without tags", zap.Int("metrics_count", len(metricList))) i := 0 for _, m := range metricList { if m.Tags == nil || m.Tags.Len() == 0 { continue } metricList[i] = m i++ } metricList = metricList[:i] end() if len(metricList) == 0 { logger.Info("nothing to do", zap.Int("metrics_count", len(metricList))) return nil } begin("cut metrics into parts", zap.Int("metrics_count", len(metricList))) metricListParts, tagsCount := cutMetricsIntoParts(metricList, cfg.Tags.Threads) threads := len(metricListParts) end() begin("marshal RowBinary", zap.String("compression", string(cfg.Tags.Compression)), zap.Int("tags_count", tagsCount), zap.Int("threads", threads), zap.Int("max_cpu", cfg.Common.MaxCPU)) binaryParts := make([]*bytes.Buffer, threads) eg := new(errgroup.Group) eg.SetLimit(cfg.Common.MaxCPU) for i := 0; i < threads; i++ { binaryParts[i] = new(bytes.Buffer) wc, err := wrapWithCompressor(cfg, binaryParts[i]) if err != nil { return err } metricList := metricListParts[i] eg.Go(func() error { return encodeMetricsToRowBinary(metricList, date, version, wc) }) } err = eg.Wait() if err != nil { return err } emptyRecord := new(bytes.Buffer) wc, err := wrapWithCompressor(cfg, emptyRecord) if err != nil { return err } err = encodeEmptyMetricToRowBinary(date, version, wc) if err != nil { return err } end() if cfg.Tags.OutputFile != "" { begin(fmt.Sprintf("write to %#v", cfg.Tags.OutputFile)) f, err := os.Create(cfg.Tags.OutputFile) if err != nil { return err } for i := 0; i < threads; i++ { // just concatenate the parts because zstd and gzip allow it _, err = binaryParts[i].WriteTo(f) if err != nil { return err } } _, err = emptyRecord.WriteTo(f) if err != nil { return err } err = f.Close() if err != nil { return err } end() } else { begin("upload to clickhouse", zap.Int("threads", threads)) upload := func(outBuf *bytes.Buffer) error { _, _, _, err := clickhouse.PostWithEncoding( scope.New(context.Background()).WithLogger(logger).WithTable(cfg.ClickHouse.TagTable), cfg.ClickHouse.URL, fmt.Sprintf("INSERT INTO %s (Date,Version,Level,Path,IsLeaf,Tags,Tag1) FORMAT RowBinary", cfg.ClickHouse.TagTable), outBuf, cfg.Tags.Compression, chOpts, nil, ) return err } eg := new(errgroup.Group) for i := 0; i < threads; i++ { outBuf := binaryParts[i] eg.Go(func() error { return upload(outBuf) }) } err = eg.Wait() if err != nil { return err } err = upload(emptyRecord) if err != nil { return err } end() } return nil } func cutMetricsIntoParts(metricList []Metric, threads int) ([][]Metric, int) { tagsCount := 0 for _, m := range metricList { tagsCount += m.Tags.Len() } if threads < 2 { return [][]Metric{metricList}, tagsCount } parts := make([][]Metric, 0, threads) i := 0 partSize := (tagsCount-1)/threads + 1 // round up for cases like 99/50 cnt := 0 for j, m := range metricList { // assert m.Tags != nil && m.Tags.Len() != 0 cnt += m.Tags.Len() if cnt >= partSize { parts = append(parts, metricList[i:j+1]) i = j + 1 cnt = 0 } } if i < len(metricList) { parts = append(parts, metricList[i:]) } return parts, tagsCount } func wrapWithCompressor(cfg *config.Config, writer io.Writer) (io.WriteCloser, error) { var wc io.WriteCloser var err error switch cfg.Tags.Compression { case clickhouse.ContentEncodingNone: wc = nopCloser{writer} case clickhouse.ContentEncodingGzip: wc = gzip.NewWriter(writer) case clickhouse.ContentEncodingZstd: wc, err = zstd.NewWriter(writer, zstd.WithEncoderConcurrency(1)) if err != nil { return nil, err } default: return nil, fmt.Errorf("unknown compression: %s", cfg.Tags.Compression) } return wc, nil } func encodeMetricsToRowBinary(metricList []Metric, date time.Time, version uint32, wc io.WriteCloser) error { encoder := RowBinary.NewEncoder(wc) days := RowBinary.DateToUint16(date) metricBuffer := new(bytes.Buffer) metricEncoder := RowBinary.NewEncoder(metricBuffer) for i := 0; i < len(metricList); i++ { m := &metricList[i] if m.Tags == nil || m.Tags.Len() == 0 { continue } metricBuffer.Reset() // Date err := metricEncoder.Uint16(days) if err != nil { return err } // Version err = metricEncoder.Uint32(version) if err != nil { return err } // Level err = metricEncoder.Uint32(uint32(m.Level)) if err != nil { return err } // Path err = metricEncoder.Bytes(m.Path) if err != nil { return err } // IsLeaf err = metricEncoder.Uint8(m.IsLeaf()) if err != nil { return err } // Tags err = metricEncoder.StringList(m.Tags.List()) if err != nil { return err } for _, tag := range m.Tags.List() { _, err = wc.Write(metricBuffer.Bytes()) if err != nil { return err } // Tag1 err = encoder.String(tag) if err != nil { return err } } } wc.Close() return nil } // Empty record With Level=0, Path=0 and Without Tags // It is needed to filter current tags func encodeEmptyMetricToRowBinary(date time.Time, version uint32, wc io.WriteCloser) error { encoder := RowBinary.NewEncoder(wc) days := RowBinary.DateToUint16(date) // Date err := encoder.Uint16(days) if err != nil { return err } // Version err = encoder.Uint32(version) if err != nil { return err } // Level=0 err = encoder.Uint32(0) if err != nil { return err } // Path="" err = encoder.Bytes([]byte{}) if err != nil { return err } // IsLeaf=0 err = encoder.Uint8(0) if err != nil { return err } // Tags=[] err = encoder.StringList([]string{}) if err != nil { return err } // Tag1="" err = encoder.String("") if err != nil { return err } wc.Close() return nil } ================================================ FILE: tagger/tagger_test.go ================================================ package tagger import ( "encoding/json" "fmt" "math/rand" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestCutMetricsIntoParts(t *testing.T) { require := assert.New(t) metricList1 := []Metric{ {Tags: new(Set).Add("tag1", "tag2")}, {Tags: new(Set).Add("tag3", "tag4", "tag5")}, {Tags: new(Set).Add("tag6")}, } metricList2 := []Metric{ {Tags: new(Set).Add("tag1")}, {Tags: new(Set).Add("tag2")}, {Tags: new(Set).Add("tag3", "tag4", "tag5", "tag6")}, } metricList3 := []Metric{ {Tags: new(Set).Add("tag1")}, {Tags: new(Set).Add("tag2")}, {Tags: new(Set).Add("tag3")}, {Tags: new(Set).Add("tag4")}, {Tags: new(Set).Add("tag5", "tag6", "tag7", "tag8", "tag9")}, } metricList4 := []Metric{ {Tags: new(Set).Add("tag1")}, {Tags: new(Set).Add("tag2")}, {Tags: new(Set).Add("tag3")}, {Tags: new(Set).Add("tag4")}, {Tags: new(Set).Add("tag5")}, {Tags: new(Set).Add("tag6")}, } metricList5 := []Metric{ {Tags: new(Set).Add("tag1", "tag2", "tag3", "tag4")}, {Tags: new(Set).Add("tag5")}, {Tags: new(Set).Add("tag6")}, {Tags: new(Set).Add("tag7")}, {Tags: new(Set).Add("tag8")}, {Tags: new(Set).Add("tag9")}, } metricList6 := []Metric{ {Tags: new(Set).Add("tag0", "tag1")}, {Tags: new(Set).Add("tag0")}, {Tags: new(Set).Add("tag0")}, {Tags: new(Set).Add("tag0", "tag1")}, {Tags: new(Set).Add("tag0", "tag1")}, {Tags: new(Set).Add("tag0", "tag1")}, {Tags: new(Set).Add("tag0", "tag1")}, {Tags: new(Set).Add("tag0", "tag1")}, } testCases := []struct { name string metricList []Metric threads int want [][]Metric }{ {"case 0.0", []Metric{}, 0, [][]Metric{{}}}, {"case 1.1", metricList1, 0, [][]Metric{ { {Tags: new(Set).Add("tag1", "tag2")}, {Tags: new(Set).Add("tag3", "tag4", "tag5")}, {Tags: new(Set).Add("tag6")}, }, }}, {"case 1.2", metricList1, 1, [][]Metric{ { {Tags: new(Set).Add("tag1", "tag2")}, {Tags: new(Set).Add("tag3", "tag4", "tag5")}, {Tags: new(Set).Add("tag6")}, }, }}, {"case 1.3", metricList1, 2, [][]Metric{ { {Tags: new(Set).Add("tag1", "tag2")}, {Tags: new(Set).Add("tag3", "tag4", "tag5")}, }, { {Tags: new(Set).Add("tag6")}, }, }}, {"case 1.4", metricList1, 3, [][]Metric{ { {Tags: new(Set).Add("tag1", "tag2")}, }, { {Tags: new(Set).Add("tag3", "tag4", "tag5")}, }, { {Tags: new(Set).Add("tag6")}, }, }}, {"case 2.1", metricList2, 2, [][]Metric{ { {Tags: new(Set).Add("tag1")}, {Tags: new(Set).Add("tag2")}, {Tags: new(Set).Add("tag3", "tag4", "tag5", "tag6")}, }, }}, {"case 2.2", metricList2, 3, [][]Metric{ { {Tags: new(Set).Add("tag1")}, {Tags: new(Set).Add("tag2")}, }, { {Tags: new(Set).Add("tag3", "tag4", "tag5", "tag6")}, }, }}, {"case 3.1", metricList3, 2, [][]Metric{ { {Tags: new(Set).Add("tag1")}, {Tags: new(Set).Add("tag2")}, {Tags: new(Set).Add("tag3")}, {Tags: new(Set).Add("tag4")}, {Tags: new(Set).Add("tag5", "tag6", "tag7", "tag8", "tag9")}, }, }}, {"case 3.2", metricList3, 3, [][]Metric{ { {Tags: new(Set).Add("tag1")}, {Tags: new(Set).Add("tag2")}, {Tags: new(Set).Add("tag3")}, }, { {Tags: new(Set).Add("tag4")}, {Tags: new(Set).Add("tag5", "tag6", "tag7", "tag8", "tag9")}, }, }}, {"case 4.1", metricList4, 2, [][]Metric{ { {Tags: new(Set).Add("tag1")}, {Tags: new(Set).Add("tag2")}, {Tags: new(Set).Add("tag3")}, }, { {Tags: new(Set).Add("tag4")}, {Tags: new(Set).Add("tag5")}, {Tags: new(Set).Add("tag6")}, }, }}, {"case 4.2", metricList4, 3, [][]Metric{ { {Tags: new(Set).Add("tag1")}, {Tags: new(Set).Add("tag2")}, }, { {Tags: new(Set).Add("tag3")}, {Tags: new(Set).Add("tag4")}, }, { {Tags: new(Set).Add("tag5")}, {Tags: new(Set).Add("tag6")}, }, }}, {"case 4.3", metricList4, 4, [][]Metric{ { {Tags: new(Set).Add("tag1")}, {Tags: new(Set).Add("tag2")}, }, { {Tags: new(Set).Add("tag3")}, {Tags: new(Set).Add("tag4")}, }, { {Tags: new(Set).Add("tag5")}, {Tags: new(Set).Add("tag6")}, }, }}, {"case 5.1", metricList5, 2, [][]Metric{ { {Tags: new(Set).Add("tag1", "tag2", "tag3", "tag4")}, {Tags: new(Set).Add("tag5")}, }, { {Tags: new(Set).Add("tag6")}, {Tags: new(Set).Add("tag7")}, {Tags: new(Set).Add("tag8")}, {Tags: new(Set).Add("tag9")}, }, }}, {"case 5.2", metricList5, 3, [][]Metric{ { {Tags: new(Set).Add("tag1", "tag2", "tag3", "tag4")}, }, { {Tags: new(Set).Add("tag5")}, {Tags: new(Set).Add("tag6")}, {Tags: new(Set).Add("tag7")}, }, { {Tags: new(Set).Add("tag8")}, {Tags: new(Set).Add("tag9")}, }, }}, {"case 5.3", metricList5, 4, [][]Metric{ { {Tags: new(Set).Add("tag1", "tag2", "tag3", "tag4")}, }, { {Tags: new(Set).Add("tag5")}, {Tags: new(Set).Add("tag6")}, {Tags: new(Set).Add("tag7")}, }, { {Tags: new(Set).Add("tag8")}, {Tags: new(Set).Add("tag9")}, }, }}, {"case 5.4", metricList5, 5, [][]Metric{ { {Tags: new(Set).Add("tag1", "tag2", "tag3", "tag4")}, }, { {Tags: new(Set).Add("tag5")}, {Tags: new(Set).Add("tag6")}, }, { {Tags: new(Set).Add("tag7")}, {Tags: new(Set).Add("tag8")}, }, { {Tags: new(Set).Add("tag9")}, }, }}, {"case 6.1", metricList6, 5, [][]Metric{ { {Tags: new(Set).Add("tag0", "tag1")}, {Tags: new(Set).Add("tag0")}, }, { {Tags: new(Set).Add("tag0")}, {Tags: new(Set).Add("tag0", "tag1")}, }, { {Tags: new(Set).Add("tag0", "tag1")}, {Tags: new(Set).Add("tag0", "tag1")}, }, { {Tags: new(Set).Add("tag0", "tag1")}, {Tags: new(Set).Add("tag0", "tag1")}, }, }}, {"case 6.2", metricList6, 7, [][]Metric{ { {Tags: new(Set).Add("tag0", "tag1")}, }, { {Tags: new(Set).Add("tag0")}, {Tags: new(Set).Add("tag0")}, }, { {Tags: new(Set).Add("tag0", "tag1")}, }, { {Tags: new(Set).Add("tag0", "tag1")}, }, { {Tags: new(Set).Add("tag0", "tag1")}, }, { {Tags: new(Set).Add("tag0", "tag1")}, }, { {Tags: new(Set).Add("tag0", "tag1")}, }, }}, } for _, tc := range testCases { // if !strings.HasPrefix(tc.name, "case 6.") { // continue // } t.Run(tc.name, func(t *testing.T) { got, _ := cutMetricsIntoParts(tc.metricList, tc.threads) require.Equal(len(tc.want), len(got), "unexpected number of parts") require.Equal(tc.want, got) }) } } func TestCutMetricsIntoPartsRandom(t *testing.T) { require := require.New(t) rand.Seed(time.Now().UnixNano()) for n := 0; n < 1000; n++ { metricList := make([]Metric, rand.Intn(100)) tagsMax := rand.Intn(100) + 1 tagsCnt := 0 for i := range metricList { tags := make([]string, rand.Intn(tagsMax)+1) tagsCnt += len(tags) for j := range tags { tags[j] = fmt.Sprintf("tag%d", j) } metricList[i].Tags = new(Set).Add(tags...) } threads := rand.Intn(110) parts, _ := cutMetricsIntoParts(metricList, threads) if threads == 0 { threads = 1 } if len(parts) > threads { v, _ := json.MarshalIndent(parts, "", " ") fmt.Println(string(v)) } require.LessOrEqual(len(parts), threads, fmt.Sprint(tagsCnt, len(metricList), len(parts), threads)) if len(metricList) > 0 { require.LessOrEqual(len(parts), len(metricList), fmt.Sprint(tagsCnt, len(metricList), len(parts), threads)) } i := 0 for _, p := range parts { for _, m := range p { require.Equal(metricList[i], m) i++ } } require.Equal(len(metricList), i) } } ================================================ FILE: tagger/tree.go ================================================ package tagger type Tree struct { Next [256]*Tree Rules []*Rule } func (t *Tree) Add(prefix []byte, rule *Rule) { x := t for i := 0; i < len(prefix); i++ { if x.Next[prefix[i]] == nil { x.Next[prefix[i]] = &Tree{} } x = x.Next[prefix[i]] } if x.Rules == nil { x.Rules = make([]*Rule, 0) } x.Rules = append(x.Rules, rule) } func (t *Tree) AddSuffix(suffix []byte, rule *Rule) { x := t for i := len(suffix) - 1; i >= 0; i-- { if x.Next[suffix[i]] == nil { x.Next[suffix[i]] = &Tree{} } x = x.Next[suffix[i]] } if x.Rules == nil { x.Rules = make([]*Rule, 0) } x.Rules = append(x.Rules, rule) } ================================================ FILE: tests/agg_internal/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: tests/agg_internal/graphite-clickhouse-internal-aggr.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = true tagged-table = "graphite_tags" tagged-autocomplete-days = 1 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/agg_internal/test.toml ================================================ [test] precision = "10s" [[test.clickhouse]] version = "21.3" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "22.8" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "24.2" dir = "tests/clickhouse/rollup" [test.carbon_clickhouse] template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse-internal-aggr.conf.tpl" [[test.input]] name = "test.avg" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 2.0, time = "rnow"}] [[test.input]] name = "test.sum" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 2.0, time = "rnow"}] [[test.input]] name = "test.min" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 2.0, time = "rnow"}] [[test.input]] name = "test.max" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 2.0, time = "rnow", delay = "5s"}] # Test aggregations [[test.input]] name = "test.avg" points = [{value = 0.0, time = "rnow-1"}, {value = 4.0, time = "rnow+1"}] # Test aggregations [[test.input]] name = "test.sum" points = [{value = 0.0, time = "rnow-1"}, {value = 4.0, time = "rnow+1"}] # Test aggregations [[test.input]] name = "test.min" points = [{value = 0.0, time = "rnow-1"}, {value = 4.0, time = "rnow+1"}] # Test aggregations [[test.input]] name = "test.max" points = [{value = 0.0, time = "rnow-1"}, {value = 4.0, time = "rnow+1"}] ########################################################################## # Aggregated, Deduplication not work with internal aggregation ########################################################################## [[test.render_checks]] name = "Test rollup" from = "rnow-10" until = "rnow+10" targets = [ "test.{avg,min,max,sum}" ] [[test.render_checks.result]] name = "test.avg" path = "test.{avg,min,max,sum}" consolidation = "avg" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [0.5, 3.0, nan] [[test.render_checks.result]] name = "test.sum" path = "test.{avg,min,max,sum}" consolidation = "sum" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [1.0, 6.0, nan] [[test.render_checks.result]] name = "test.min" path = "test.{avg,min,max,sum}" consolidation = "min" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [0.0, 2.0, nan] [[test.render_checks.result]] name = "test.max" path = "test.{avg,min,max,sum}" consolidation = "max" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [1.0, 4.0, nan] # End - Test rollup ########################################################################## ================================================ FILE: tests/agg_latest/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: tests/agg_latest/graphite-clickhouse.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = false tagged-table = "graphite_tags" tagged-autocomplete-days = 1 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/agg_latest/test.toml ================================================ [test] precision = "10s" #[[test.clickhouse]] #version = "20.3" #dir = "tests/clickhouse/rollup" #[[test.clickhouse]] #version = "20.8" #dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "21.3" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "22.8" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "24.2" dir = "tests/clickhouse/rollup" [test.carbon_clickhouse] template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse.conf.tpl" [[test.input]] name = "test.avg" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 2.0, time = "rnow"}] [[test.input]] name = "test.sum" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 2.0, time = "rnow"}] [[test.input]] name = "test.min" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 2.0, time = "rnow"}] [[test.input]] name = "test.max" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 2.0, time = "rnow", delay = "5s"}] # Test aggregations [[test.input]] name = "test.avg" points = [{value = 0.0, time = "rnow-1"}, {value = 4.0, time = "rnow+1"}] # Test aggregations [[test.input]] name = "test.sum" points = [{value = 0.0, time = "rnow-1"}, {value = 4.0, time = "rnow+1"}] # Test aggregations [[test.input]] name = "test.min" points = [{value = 0.0, time = "rnow-1"}, {value = 4.0, time = "rnow+1"}] # Test aggregations [[test.input]] name = "test.max" points = [{value = 0.0, time = "rnow-1"}, {value = 4.0, time = "rnow+1"}] ########################################################################## # Not aggregated, Deduplication work with version >= 21.3 ########################################################################## [[test.render_checks]] name = "Test rollup" from = "rnow-10" until = "rnow+10" targets = [ "test.{avg,min,max,sum}" ] [[test.render_checks.result]] name = "test.avg" path = "test.{avg,min,max,sum}" consolidation = "avg" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [0.0, 4.0, nan] [[test.render_checks.result]] name = "test.sum" path = "test.{avg,min,max,sum}" consolidation = "sum" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [0.0, 4.0, nan] [[test.render_checks.result]] name = "test.min" path = "test.{avg,min,max,sum}" consolidation = "min" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [0.0, 4.0, nan] [[test.render_checks.result]] name = "test.max" path = "test.{avg,min,max,sum}" consolidation = "max" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [0.0, 4.0, nan] # End - Test rollup ########################################################################## ================================================ FILE: tests/agg_merge/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: tests/agg_merge/graphite-clickhouse-internal-aggr.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = true tagged-table = "graphite_tags" tagged-autocomplete-days = 1 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/agg_merge/graphite-clickhouse.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = false tagged-table = "graphite_tags" tagged-autocomplete-days = 1 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/agg_merge/test.toml ================================================ [test] precision = "10s" [[test.clickhouse]] version = "21.3" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "22.8" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "24.2" dir = "tests/clickhouse/rollup" [test.carbon_clickhouse] template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse-internal-aggr.conf.tpl" [[test.input]] name = "test.avg" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 0.0, time = "rnow-1"}, {value = 2.0, time = "rnow"}, {value = 4.0, time = "rnow+1"}] [[test.input]] name = "test.sum" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 0.0, time = "rnow-1"}, {value = 2.0, time = "rnow"}, {value = 4.0, time = "rnow+1"}] [[test.input]] name = "test.min" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 0.0, time = "rnow-1"}, {value = 2.0, time = "rnow"}, {value = 4.0, time = "rnow+1"}] [[test.input]] name = "test.max" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 0.0, time = "rnow-1"}, {value = 2.0, time = "rnow"}, {value = 4.0, time = "rnow+1"}] ########################################################################## # Aggregated at merge ########################################################################## [[test.render_checks]] optimize = [ "graphite", "graphite_reverse" ] from = "rnow-10" until = "rnow+10" targets = [ "test.{avg,min,max,sum}" ] [[test.render_checks.result]] name = "test.avg" path = "test.{avg,min,max,sum}" consolidation = "avg" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [0.5, 3.0, nan] [[test.render_checks.result]] name = "test.sum" path = "test.{avg,min,max,sum}" consolidation = "sum" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [1.0, 6.0, nan] [[test.render_checks.result]] name = "test.min" path = "test.{avg,min,max,sum}" consolidation = "min" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [0.0, 2.0, nan] [[test.render_checks.result]] name = "test.max" path = "test.{avg,min,max,sum}" consolidation = "max" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [1.0, 4.0, nan] # End - Test rollup ########################################################################## ================================================ FILE: tests/agg_oneblock/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: tests/agg_oneblock/graphite-clickhouse-internal-aggr.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = true tagged-table = "graphite_tags" tagged-autocomplete-days = 1 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/agg_oneblock/graphite-clickhouse.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = false tagged-table = "graphite_tags" tagged-autocomplete-days = 1 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/agg_oneblock/test.toml ================================================ [test] precision = "10s" [[test.clickhouse]] version = "21.3" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "22.8" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "24.2" dir = "tests/clickhouse/rollup" [test.carbon_clickhouse] template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse-internal-aggr.conf.tpl" [[test.input]] name = "test.avg" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 0.0, time = "rnow-1"}, {value = 2.0, time = "rnow"}, {value = 4.0, time = "rnow+1"}] [[test.input]] name = "test.sum" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 0.0, time = "rnow-1"}, {value = 2.0, time = "rnow"}, {value = 4.0, time = "rnow+1"}] [[test.input]] name = "test.min" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 0.0, time = "rnow-1"}, {value = 2.0, time = "rnow"}, {value = 4.0, time = "rnow+1"}] [[test.input]] name = "test.max" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 0.0, time = "rnow-1"}, {value = 2.0, time = "rnow"}, {value = 4.0, time = "rnow+1"}] ########################################################################## # Aggregated, Deduplication not work at one block ########################################################################## [[test.render_checks]] from = "rnow-10" until = "rnow+10" targets = [ "test.{avg,min,max,sum}" ] dump_if_empty = [ "SELECT Date, Path FROM graphite_index WHERE ((Level=2) AND (Path LIKE 'test.%' AND match(Path, '^test[.](avg|min|max|sum)[.]?$'))) GROUP BY Date, Path" ] [[test.render_checks.result]] name = "test.avg" path = "test.{avg,min,max,sum}" consolidation = "avg" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [0.5, 3.0, nan] [[test.render_checks.result]] name = "test.sum" path = "test.{avg,min,max,sum}" consolidation = "sum" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [1.0, 6.0, nan] [[test.render_checks.result]] name = "test.min" path = "test.{avg,min,max,sum}" consolidation = "min" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [0.0, 2.0, nan] [[test.render_checks.result]] name = "test.max" path = "test.{avg,min,max,sum}" consolidation = "max" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [1.0, 4.0, nan] # End - Test rollup ########################################################################## ================================================ FILE: tests/clickhouse/rollup/config.xml ================================================ debug /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.err.log 2000M 20 8123 9000 9009 test-clickhouse-s1 0.0.0.0 1073741824 1073741824 /var/lib/clickhouse/ /var/lib/clickhouse/tmp/ users.xml default default system query_log
7500 system part_log
7500 ================================================ FILE: tests/clickhouse/rollup/init.sql ================================================ CREATE TABLE IF NOT EXISTS default.graphite_reverse ( Path String, Value Float64, Time UInt32, Date Date, Timestamp UInt32 ) ENGINE = GraphiteMergeTree('graphite_rollup') PARTITION BY Date ORDER BY (Path, Time); CREATE TABLE IF NOT EXISTS default.graphite ( Path String, Value Float64, Time UInt32, Date Date, Timestamp UInt32 ) ENGINE = GraphiteMergeTree('graphite_rollup') PARTITION BY Date ORDER BY (Path, Time); CREATE TABLE IF NOT EXISTS default.graphite_index ( Date Date, Level UInt32, Path String, Version UInt32 ) ENGINE = ReplacingMergeTree(Version) PARTITION BY Date ORDER BY (Level, Path, Date); CREATE TABLE IF NOT EXISTS default.graphite_tags ( Date Date, Tag1 String, Path String, Tags Array(String), Version UInt32 ) ENGINE = ReplacingMergeTree(Version) PARTITION BY Date ORDER BY (Tag1, Path, Date); CREATE TABLE IF NOT EXISTS default.tag1_count_per_day ( Date Date, Tag1 String, Count UInt64 ) ENGINE = SummingMergeTree ORDER BY (Date, Tag1); CREATE MATERIALIZED VIEW IF NOT EXISTS default.tag1_count_per_day_mv TO default.tag1_count_per_day AS SELECT Date AS Date, Tag1 AS Tag1, count(*) AS Count FROM default.graphite_tags GROUP BY (Date, Tag1); ================================================ FILE: tests/clickhouse/rollup/rollup.xml ================================================ avg 0 10 \.sum$ sum \.sum\? sum \.min$ min \.min\? min \.max$ max \.max\? max ================================================ FILE: tests/clickhouse/rollup/users.xml ================================================ 1 0 40000000000 1 1048576000 1000000 1000000 720000 6000000000000 40 16 25571520 1073741824 random 1000 1 ::/0 default default ::1 127.0.0.1 readonly default 3600 0 0 0 0 0 ================================================ FILE: tests/clickhouse/rollup_tls/config.xml ================================================ debug /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.err.log 2000M 20 8123 9000 8443 9440 none false /etc/clickhouse-server/rootCA.crt /etc/clickhouse-server/server.crt /etc/clickhouse-server/server.key true true /etc/clickhouse-server/rootCA.crt 9009 test-clickhouse-s1 0.0.0.0 1073741824 1073741824 /var/lib/clickhouse/ /var/lib/clickhouse/tmp/ users.xml default default system query_log
7500 system part_log
7500 ================================================ FILE: tests/clickhouse/rollup_tls/init.sql ================================================ CREATE TABLE IF NOT EXISTS default.graphite_reverse ( Path String, Value Float64, Time UInt32, Date Date, Timestamp UInt32 ) ENGINE = GraphiteMergeTree('graphite_rollup') PARTITION BY toYYYYMM(Date) ORDER BY (Path, Time); CREATE TABLE IF NOT EXISTS default.graphite ( Path String, Value Float64, Time UInt32, Date Date, Timestamp UInt32 ) ENGINE = GraphiteMergeTree('graphite_rollup') PARTITION BY toYYYYMM(Date) ORDER BY (Path, Time); CREATE TABLE IF NOT EXISTS default.graphite_index ( Date Date, Level UInt32, Path String, Version UInt32 ) ENGINE = ReplacingMergeTree(Version) PARTITION BY toYYYYMM(Date) ORDER BY (Level, Path, Date); CREATE TABLE IF NOT EXISTS default.graphite_tags ( Date Date, Tag1 String, Path String, Tags Array(String), Version UInt32 ) ENGINE = ReplacingMergeTree(Version) PARTITION BY toYYYYMM(Date) ORDER BY (Tag1, Path, Date); CREATE TABLE IF NOT EXISTS default.tag1_count_per_day ( Date Date, Tag1 String, Count UInt64 ) ENGINE = SummingMergeTree ORDER BY (Date, Tag1); CREATE MATERIALIZED VIEW IF NOT EXISTS default.tag1_count_per_day_mv TO default.tag1_count_per_day AS SELECT Date AS Date, Tag1 AS Tag1, count(*) AS Count FROM default.graphite_tags GROUP BY (Date, Tag1); ================================================ FILE: tests/clickhouse/rollup_tls/rollup.xml ================================================ avg 0 10 \.sum$ sum \.sum\? sum \.min$ min \.min\? min \.max$ max \.max\? max ================================================ FILE: tests/clickhouse/rollup_tls/rootCA.crt ================================================ -----BEGIN CERTIFICATE----- MIIDHTCCAgWgAwIBAgIURx5itXwLHeiQES1LzCHF7F8RNEkwDQYJKoZIhvcNAQEL BQAwHjEcMBoGA1UEAwwTbG9yZHZpcmRleC5sb2NhbCBDQTAeFw0yNDA4MDkxMjMy MzJaFw0zNDA4MDcxMjMyMzJaMB4xHDAaBgNVBAMME2xvcmR2aXJkZXgubG9jYWwg Q0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDuiK4tBYzNtROmhuXD 80HsVVk2/+/TXV85Aey7oo2gxxJJ09iARnjJadNrbBUdoL42XtmBCkYY+pXYUWPD hvals2AbXiAePg7DlAHJfpaQTzHlsPvAUMjqbD6cFaQ7DfNQHcz2emmFhcRYzlQM h0Ob3v2yhogG7PuKaiTLTKYcHnRKfEIobQEIq16ABaaCFKzR6tpvrUJFYtkJ8EUz jhrSg67qy7yiHiMmGQVq526X2oZYhMbSGjiPkaMZHdFkxZgJF5iQhANG9djvcopO jdFfsJYM9rVxAjwO/P3fq5dpuQxWLLo6ZmholsixPZs1s8paEnonSDtyoNLsykwD 2mFdAgMBAAGjUzBRMB0GA1UdDgQWBBS6BlL90Mo/+aHonqIqaewM8CyxnTAfBgNV HSMEGDAWgBS6BlL90Mo/+aHonqIqaewM8CyxnTAPBgNVHRMBAf8EBTADAQH/MA0G CSqGSIb3DQEBCwUAA4IBAQAIwTN3II6HdPfMsLvYoOmzcvUE9Y6QndI20eLqp3p8 6KnU+lgLdSkLjc9BKwLh/Jhuy4H3u1nHpW8Jkgy/8irG2uaUvgKlutfApFQshAo7 /k9xdH36ER0LF/bW5hQ535H76OaE+eaexx2zU50kPVuntal577d8HBfrKVI41KU8 CVdqYTwEqHwjSyRhmmRqLi7Yo+i0o0hRwH39LxYXY2rup/V6uRyLXSIDUZ9VeqVt K8XDAbLV1s4kzR/OdpYcJuTWX9gFUlNHpGDkOSy9ggc5zxKaHlwGGZsvVSb4f+VF C89ABPZs+26EvExIih+civiC1XWIghP8RsiNyBOK3TOf -----END CERTIFICATE----- ================================================ FILE: tests/clickhouse/rollup_tls/server.crt ================================================ -----BEGIN CERTIFICATE----- MIIDYDCCAkigAwIBAgIUElpBaTpcXsRXXP5YNhdKwu+lmE0wDQYJKoZIhvcNAQEL BQAwHjEcMBoGA1UEAwwTbG9yZHZpcmRleC5sb2NhbCBDQTAeFw0yNDA4MDkxMjM2 NTJaFw0zNDA4MDcxMjM2NTJaMFYxCzAJBgNVBAYTAlJVMRIwEAYDVQQIDAlUYXRh cnN0YW4xDjAMBgNVBAcMBUthemFuMQ8wDQYDVQQKDAZLb250dXIxEjAQBgNVBAMM CWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKdkGsVL lK0ecS7+pEzEFpKmKOrSMKGCfOkqIVNO4f21njvg3rOx+j/1G8+D1eFHJJkotsx/ HfJg2sgMosltIlR19f1CzV6ewQLYN7fw4d8aMq1B1lnzzvfUjjygdxB353RiaCHI eQ1xkTPPmdZMEgaYwto2nrNrCOTb/kZig6pQeQ1YLV4c1daiI9L7OJhwKIb9yqT6 gT+jXrrRZWE5o0sSKBw16h+iFXy/niO+2+VLuAHXturTg8m0U+NagexJZzkM4wt1 9pXAlODxu6y0en3lU2ngfGVV22HGSYsyKjBWAzA4HM3wQ6D6DrEa2W2ezV1MQfrS FYGVO8DBziaV41cCAwEAAaNeMFwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAAB MB0GA1UdDgQWBBRTabpN+c8bOE2B5o214IliRqUIqjAfBgNVHSMEGDAWgBS6BlL9 0Mo/+aHonqIqaewM8CyxnTANBgkqhkiG9w0BAQsFAAOCAQEAJal0TGS91yRK4ATZ sifjon3w7Q47WAbhXNFasuFdEdEexcWmc+gzhYW+snnVUlHT9y1J675i/Le6ry7y /pkzzdSoyx7CVHlU81gZLCts1lzufDl/cE5vDG4Sjnx1SepumUy9IrXhCaAaH19s EiywBsZ1uPC3XqAlaXLUYQglmtzXzeOMDXVRz4n3+SujkZ+DD2UMmTvWe9P1D8Ss gMkg8iMvNtm90MjVFgddLf9QjHYEJjNvzaRdQXvsnCOBwR/kyKimaZxV34QCC7cl QMhc6SzEmGqz+NflrzJyAIOMkDxotyqZX6v5kYFwEMhvBa6tXi+mmxx4lQ6VElT9 pB9GKA== -----END CERTIFICATE----- ================================================ FILE: tests/clickhouse/rollup_tls/server.key ================================================ -----BEGIN PRIVATE KEY----- MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCnZBrFS5StHnEu /qRMxBaSpijq0jChgnzpKiFTTuH9tZ474N6zsfo/9RvPg9XhRySZKLbMfx3yYNrI DKLJbSJUdfX9Qs1ensEC2De38OHfGjKtQdZZ88731I48oHcQd+d0YmghyHkNcZEz z5nWTBIGmMLaNp6zawjk2/5GYoOqUHkNWC1eHNXWoiPS+ziYcCiG/cqk+oE/o166 0WVhOaNLEigcNeofohV8v54jvtvlS7gB17bq04PJtFPjWoHsSWc5DOMLdfaVwJTg 8bustHp95VNp4HxlVdthxkmLMiowVgMwOBzN8EOg+g6xGtltns1dTEH60hWBlTvA wc4mleNXAgMBAAECggEAMrhgbDvUlwhMX2MFQcWA2XrDlzONTMMPOk9rvaR/UbMA eUBP+r8JBuwsOxrFafd2nXn6ucgiuNikMk2x3brV1iXQHadqNyt/bG87ot64cjOr +1ehrav0oJ+lYbV1nmXWmitfRi1KkMpCpyJWiNqP87PCBwDZ4Z+jGEWYrJcZMjem gAFoaUw0hrLc7FJe8sogC9j3gyIfjVq7k+epPlW6VsRW7h+aZowq2Bbik2VYz1Py aIdpaZwf8Jrhn7Qo0V39OVEr/VVLKFzNlyLpp+XeXmXT8sinmWNtTWKUB2rzL9bR oa+OeRTJIyzJXwpIBKte4TIKhtnmWANEWjuGf16m0QKBgQDFjMFwIYt6+8LEmL3Y xfpI9Hgy/PGLA/Y49ZR52OmEpXvQO12SQsDxMLIMXECSnjIIew+aJgly9HR3i+uK eFjLjKBABx5xmdfzLT05YcTASRWoj4tZrLrLM4Vytcs4xCSbiWbYFsgbbls38Abs PbmcD7oU/n/F+GrGJCHRKEtNRQKBgQDY6v8nvMDS46WLou2VzAA25mGOUlY5jzW1 WR0WxU2cLZwl+2upLj5UYRHXQtCOrVIGaMXEUdQgf+w3rvGnet14LeBByFQBs3wP TnluBEwG/ByZfjOwqAOULfIHJq75HyCZ5XR5H8tIm4hf8rb3BiJ1fe2bJSgJbst4 TLmOPljx6wKBgQCHOP3//zY2jLaZU+Q/yeS0o4LThAjim2ejPZbQgQX3Yj8KHljC kSb48dguVcdtlROycmoPnhHBuksuuXwVYKOHUU8wBK92G1SShFjwOlgvNte4delx DKcgCLhD+OSOitR0Eu1u5Mk83aFa/NYAR5ARn0JEtKBJpu2Pi5QKU4aX8QKBgQCo ufnozfBq2bouKHiHmVvdWEwv6Sm6sgOD4SI4URZyUiPwg2WV/itrdOnst8MECBsS czLJ5yCKexahpYnAzVgxn/WdFZcKj7MDMPZRNjRxBm+0kS7hzX6jJy3olBVsH+M6 8fksMifsfVaR03iwIuxw2ZgVosxGshDArWV0GFkVKwKBgFnn1YGbKQM/9EnmO79d WrNsj3P9uzngeoi9eYQbzU8ow0qBkFch36iY9tVwxwpG4k8Job9q3LBBRtbdu0eE /snY1cMd9PVuh3HaBg/hFgA2cMnk3CG+7+wLmkonRSpdIhuQjQXbB7xMmCFImypK RO4iqN/2YDPT1z5UGJEDTkBH -----END PRIVATE KEY----- ================================================ FILE: tests/clickhouse/rollup_tls/users.xml ================================================ 1 0 40000000000 1 1048576000 1000000 1000000 720000 6000000000000 40 16 25571520 1073741824 random 1000 1 ::/0 default default ::1 127.0.0.1 readonly default 3600 0 0 0 0 0 ================================================ FILE: tests/consolidateBy/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: tests/consolidateBy/graphite-clickhouse.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [feature-flags] use-carbon-behaviour = false dont-match-missing-tags = true [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = true tagged-table = "graphite_tags" tagged-autocomplete-days = 1 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/consolidateBy/test.toml ================================================ [test] precision = "10s" [[test.clickhouse]] version = "21.3" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "22.8" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "24.2" dir = "tests/clickhouse/rollup" [test.carbon_clickhouse] template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse.conf.tpl" ####################################################################################### [[test.input]] name = "request_success_total.counter;app=test;project=Test;environment=TEST" points = [{value = 3.0, time = "1000"}, {value = 0.0, time = "1010"}, {value = 1.0, time = "1020"}, {value = 2.0, time = "1030"}] [[test.input]] name = "request_success_total.counter;app=test;project=Test;environment=TEST;t=q" points = [{value = 3.0, time = "1000"}, {value = 0.0, time = "1010"}, {value = 1.0, time = "1020"}, {value = 2.0, time = "1030"}] [[test.input]] name = "test;env=prod" points = [{value = 3.0, time = "1000"}, {value = 0.0, time = "1010"}, {value = 1.0, time = "1020"}, {value = 2.0, time = "1030"}] [[test.input]] name = "test;env=dr" points = [{value = 3.0, time = "1000"}, {value = 0.0, time = "1010"}, {value = 1.0, time = "1020"}, {value = 2.0, time = "1030"}] # consolidateBy('max') [[test.render_checks]] from = "1000" until = "1030" max_data_points = 2 timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')", ] filtering_functions = [ "consolidateBy('max')" ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "max" start = "1000" stop = "1040" step = 20 req_start = "1000" req_stop = "1040" values = [3.0, 2.0] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "max" start = "1000" stop = "1040" step = 20 req_start = "1000" req_stop = "1040" values = [3.0, 2.0] # consolidateBy('min') [[test.render_checks]] from = "1000" until = "1030" max_data_points = 2 timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')", ] filtering_functions = [ "consolidateBy('min')" ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "min" start = "1000" stop = "1040" step = 20 req_start = "1000" req_stop = "1040" values = [0.0, 1.0] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "min" start = "1000" stop = "1040" step = 20 req_start = "1000" req_stop = "1040" values = [0.0, 1.0] # consolidateBy('sum') [[test.render_checks]] from = "1000" until = "1030" max_data_points = 2 timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')", ] filtering_functions = [ "consolidateBy('sum')" ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "sum" start = "1000" stop = "1040" step = 20 req_start = "1000" req_stop = "1040" values = [3.0, 3.0] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "sum" start = "1000" stop = "1040" step = 20 req_start = "1000" req_stop = "1040" values = [3.0, 3.0] # consolidateBy('avg') [[test.render_checks]] from = "1000" until = "1030" max_data_points = 2 timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')", ] filtering_functions = [ "consolidateBy('avg')" ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "avg" start = "1000" stop = "1040" step = 20 req_start = "1000" req_stop = "1040" values = [1.5, 1.5] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "avg" start = "1000" stop = "1040" step = 20 req_start = "1000" req_stop = "1040" values = [1.5, 1.5] # consolidateBy('average') [[test.render_checks]] from = "1000" until = "1030" max_data_points = 2 timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')", ] filtering_functions = [ "consolidateBy('average')" ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "avg" start = "1000" stop = "1040" step = 20 req_start = "1000" req_stop = "1040" values = [1.5, 1.5] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "avg" start = "1000" stop = "1040" step = 20 req_start = "1000" req_stop = "1040" values = [1.5, 1.5] # consolidateBy('last') [[test.render_checks]] from = "1000" until = "1030" max_data_points = 2 timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')", ] filtering_functions = [ "consolidateBy('last')" ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "last" start = "1000" stop = "1040" step = 20 req_start = "1000" req_stop = "1040" values = [0.0, 2.0] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "last" start = "1000" stop = "1040" step = 20 req_start = "1000" req_stop = "1040" values = [0.0, 2.0] # consolidateBy('first') [[test.render_checks]] from = "1000" until = "1030" max_data_points = 2 timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')", ] filtering_functions = [ "consolidateBy('first')" ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "first" start = "1000" stop = "1040" step = 20 req_start = "1000" req_stop = "1040" values = [3.0, 1.0] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "first" start = "1000" stop = "1040" step = 20 req_start = "1000" req_stop = "1040" values = [3.0, 1.0] # consolidateBy('invalid') [[test.render_checks]] from = "1000" until = "1030" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')", ] filtering_functions = [ "consolidateBy('invalid')" ] error_regexp = "^400: failed to choose appropriate aggregation" ================================================ FILE: tests/consul.sh ================================================ #!/usr/bin/env bash if [ "$1" != "" ]; then wget -q https://releases.hashicorp.com/consul/${1}/consul_${1}_linux_amd64.zip || exit 1 unzip consul_${1}_linux_amd64.zip || exit 1 fi ./consul agent -server -bootstrap -data-dir=/tmp/consul -bind=127.0.0.1 ================================================ FILE: tests/emptyseries_append/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: tests/emptyseries_append/graphite-clickhouse.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] append-empty-series = true [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = true tagged-table = "graphite_tags" tagged-autocomplete-days = 1 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/emptyseries_append/test.toml ================================================ [test] precision = "10s" [[test.clickhouse]] version = "21.3" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "22.8" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "24.2" dir = "tests/clickhouse/rollup" [test.carbon_clickhouse] template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse.conf.tpl" [[test.input]] name = "test.avg" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow+21"}, {value = 2.0, time = "rnow+30"}] [[test.input]] name = "test.sum" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 2.0, time = "rnow"}] [[test.input]] name = "test.min" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 2.0, time = "rnow"}] [[test.input]] name = "test.max" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 2.0, time = "rnow", delay = "5s"}] # Test aggregations [[test.input]] name = "test.avg" points = [{value = 0.0, time = "rnow-30"}, {value = 4.0, time = "rnow+30"}] # Test aggregations [[test.input]] name = "test.sum" points = [{value = 0.0, time = "rnow-1"}, {value = 4.0, time = "rnow+1"}] # Test aggregations [[test.input]] name = "test.min" points = [{value = 0.0, time = "rnow-1"}, {value = 4.0, time = "rnow+1"}] # Test aggregations [[test.input]] name = "test.max" points = [{value = 0.0, time = "rnow-1"}, {value = 4.0, time = "rnow+1"}] ########################################################################## # Aggregated, Deduplication not work with internal aggregation ########################################################################## [[test.render_checks]] name = "Test rollup" from = "rnow-10" until = "rnow+10" targets = [ "test.{avg,min,max,sum}" ] [[test.render_checks.result]] name = "test.avg" path = "test.{avg,min,max,sum}" consolidation = "any" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [nan, nan, nan] [[test.render_checks.result]] name = "test.sum" path = "test.{avg,min,max,sum}" consolidation = "sum" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [1.0, 6.0, nan] [[test.render_checks.result]] name = "test.min" path = "test.{avg,min,max,sum}" consolidation = "min" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [0.0, 2.0, nan] [[test.render_checks.result]] name = "test.max" path = "test.{avg,min,max,sum}" consolidation = "max" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [1.0, 4.0, nan] # End - Test rollup ########################################################################## ================================================ FILE: tests/emptyseries_noappend/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: tests/emptyseries_noappend/graphite-clickhouse.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] append-empty-series = false [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = true tagged-table = "graphite_tags" tagged-autocomplete-days = 1 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/emptyseries_noappend/test.toml ================================================ [test] precision = "10s" [[test.clickhouse]] version = "21.3" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "22.8" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "24.2" dir = "tests/clickhouse/rollup" [test.carbon_clickhouse] template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse.conf.tpl" [[test.input]] name = "test.avg" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow+21"}, {value = 2.0, time = "rnow+30"}] [[test.input]] name = "test.sum" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 2.0, time = "rnow"}] [[test.input]] name = "test.min" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 2.0, time = "rnow"}] [[test.input]] name = "test.max" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 2.0, time = "rnow", delay = "5s"}] # Test aggregations [[test.input]] name = "test.avg" points = [{value = 0.0, time = "rnow-30"}, {value = 4.0, time = "rnow+30"}] # Test aggregations [[test.input]] name = "test.sum" points = [{value = 0.0, time = "rnow-1"}, {value = 4.0, time = "rnow+1"}] # Test aggregations [[test.input]] name = "test.min" points = [{value = 0.0, time = "rnow-1"}, {value = 4.0, time = "rnow+1"}] # Test aggregations [[test.input]] name = "test.max" points = [{value = 0.0, time = "rnow-1"}, {value = 4.0, time = "rnow+1"}] ########################################################################## # Aggregated, Deduplication not work with internal aggregation ########################################################################## [[test.render_checks]] name = "Test rollup" from = "rnow-10" until = "rnow+10" targets = [ "test.{avg,min,max,sum}" ] [[test.render_checks.result]] name = "test.sum" path = "test.{avg,min,max,sum}" consolidation = "sum" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [1.0, 6.0, nan] [[test.render_checks.result]] name = "test.min" path = "test.{avg,min,max,sum}" consolidation = "min" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [0.0, 2.0, nan] [[test.render_checks.result]] name = "test.max" path = "test.{avg,min,max,sum}" consolidation = "max" start = "rnow-10" stop = "rnow+20" step = 10 req_start = "rnow-10" req_stop = "rnow+20" values = [1.0, 4.0, nan] # End - Test rollup ########################################################################## ================================================ FILE: tests/error_handling/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: tests/error_handling/graphite-clickhouse.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [clickhouse] url = "{{ .PROXY_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "1s" query-params = [ { duration = "1h", url = "{{ .PROXY_URL }}/?max_rows_to_read=1&max_result_bytes=1&readonly=2&log_queries=1", data-timeout = "5s" }, { duration = "7h", url = "{{ .PROXY_URL }}/?max_memory_usage=1&max_memory_usage_for_user=1&readonly=2&log_queries=1", data-timeout = "5s" } ] index-table = "graphite_index" index-use-daily = true index-timeout = "1s" internal-aggregation = false tagged-table = "graphite_tags" tagged-autocomplete-days = 1 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/error_handling/test.toml ================================================ [test] precision = "10s" [[test.clickhouse]] version = "24.2" dir = "tests/clickhouse/rollup" [test.carbon_clickhouse] template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse.conf.tpl" [[test.input]] name = "test.plain1" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 2.0, time = "rnow"}] [[test.input]] name = "test.plain2" points = [{value = 2.0, time = "rnow-30"}, {value = 1.0, time = "rnow-20"}, {value = 1.5, time = "rnow-10"}, {value = 2.5, time = "rnow"}] [[test.input]] name = "test2.plain" points = [{value = 1.0, time = "rnow-30"}, {value = 2.0, time = "rnow-20"}, {value = 2.5, time = "rnow-10"}, {value = 3.5, time = "rnow"}] [[test.input]] name = "metric1;tag1=value1;tag2=value21;tag3=value3" points = [{value = 2.0, time = "rnow-30"}, {value = 2.5, time = "rnow-20"}, {value = 2.0, time = "rnow-10"}, {value = 3.0, time = "rnow"}] [[test.input]] name = "metric1;tag2=value22;tag4=value4" points = [{value = 1.0, time = "rnow-30"}, {value = 2.0, time = "rnow-20"}, {value = 0.0, time = "rnow-10"}, {value = 1.0, time = "rnow"}] [[test.input]] name = "metric1;tag1=value1;tag2=value23;tag3=value3" points = [{value = 0.5, time = "rnow-30"}, {value = 1.5, time = "rnow-20"}, {value = 4.0, time = "rnow-10"}, {value = 3.0, time = "rnow"}] [[test.input]] name = "metric2;tag2=value21;tag4=value4" points = [{value = 2.0, time = "rnow-30"}, {value = 1.0, time = "rnow-20"}, {value = 0.0, time = "rnow-10"}, {value = 1.0, time = "rnow"}] [[test.find_checks]] query = "test" result = [ { path = "test", is_leaf = false } ] # Check index-timeout [[test.find_checks]] query = "test" timeout = "2s" proxy_delay = "1500ms" error_regexp = "^504: Storage read timeout" [[test.tags_checks]] query = "tag1;tag2=value21" result = [ "value1" ] # Check index-timeout [[test.tags_checks]] query = "tag1;tag2=value21" timeout = "2s" proxy_delay = "1500ms" error_regexp = "^504: Storage read timeout" [[test.input]] name = "test.long" points = [ {value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 2.0, time = "rnow"}, {value = 3.0, time = "rnow-3600"}, {value = 3.0, time = "rnow-3590"}, {value = 3.0, time = "rnow-3580"}, {value = 3.0, time = "rnow-3570"}, {value = 3.0, time = "rnow-3560"}, {value = 3.0, time = "rnow-3550"}, {value = 3.0, time = "rnow-3540"}, {value = 3.0, time = "rnow-3530"}, {value = 3.0, time = "rnow-3520"}, {value = 3.0, time = "rnow-3510"}, {value = 3.0, time = "rnow-3500"}, {value = 3.0, time = "rnow-3490"}, {value = 3.0, time = "rnow-14200"}, {value = 3.0, time = "rnow-14400"}, {value = 3.0, time = "rnow-86200"}, {value = 3.0, time = "rnow-86400"}, {value = 3.0, time = "rnow-172600"}, {value = 3.0, time = "rnow-172800"}, {value = 2.0, time = "rnow-345200"}, {value = 2.0, time = "rnow-345210"}, {value = 2.0, time = "rnow-345220"}, {value = 2.0, time = "rnow-345230"}, {value = 2.0, time = "rnow-345240"}, {value = 2.0, time = "rnow-345250"}, {value = 2.0, time = "rnow-345260"}, {value = 2.0, time = "rnow-345270"}, {value = 2.0, time = "rnow-345280"}, {value = 2.0, time = "rnow-345290"}, {value = 2.0, time = "rnow-345300"}, {value = 2.0, time = "rnow-345310"}, {value = 2.0, time = "rnow-345320"}, {value = 2.0, time = "rnow-345330"}, {value = 2.0, time = "rnow-345340"}, {value = 2.0, time = "rnow-345350"}, {value = 2.0, time = "rnow-345360"}, {value = 2.0, time = "rnow-345370"}, {value = 2.0, time = "rnow-345380"}, {value = 2.0, time = "rnow-345390"}, {value = 2.0, time = "rnow-345400"}, {value = 2.0, time = "rnow-345410"}, {value = 2.0, time = "rnow-345420"}, {value = 2.0, time = "rnow-345430"}, {value = 2.0, time = "rnow-345440"}, {value = 2.0, time = "rnow-345450"}, {value = 2.0, time = "rnow-345460"}, {value = 2.0, time = "rnow-345470"}, {value = 2.0, time = "rnow-345480"}, {value = 2.0, time = "rnow-345490"}, {value = 2.0, time = "rnow-345500"}, {value = 2.0, time = "rnow-345510"}, {value = 2.0, time = "rnow-345520"}, {value = 2.0, time = "rnow-345530"}, {value = 2.0, time = "rnow-345540"}, {value = 2.0, time = "rnow-345550"}, {value = 2.0, time = "rnow-345560"}, {value = 2.0, time = "rnow-345570"}, {value = 2.0, time = "rnow-345580"}, {value = 2.0, time = "rnow-345590"}, {value = 2.0, time = "rnow-345600"}, {value = 2.0, time = "rnow-431800"}, {value = 2.0, time = "rnow-431810"}, {value = 2.0, time = "rnow-431820"}, {value = 2.0, time = "rnow-431830"}, {value = 2.0, time = "rnow-431840"}, {value = 2.0, time = "rnow-431850"}, {value = 2.0, time = "rnow-431860"}, {value = 2.0, time = "rnow-431870"}, {value = 2.0, time = "rnow-431880"}, {value = 2.0, time = "rnow-431890"}, {value = 2.0, time = "rnow-431900"}, {value = 2.0, time = "rnow-431910"}, {value = 2.0, time = "rnow-431920"}, {value = 2.0, time = "rnow-431930"}, {value = 2.0, time = "rnow-431940"}, {value = 2.0, time = "rnow-431950"}, {value = 2.0, time = "rnow-431960"}, {value = 2.0, time = "rnow-431970"}, {value = 2.0, time = "rnow-431980"}, {value = 2.0, time = "rnow-431990"}, {value = 2.0, time = "rnow-432000"}, {value = 2.0, time = "rnow-432010"}, {value = 2.0, time = "rnow-432020"}, {value = 2.0, time = "rnow-432030"}, {value = 2.0, time = "rnow-432040"}, {value = 2.0, time = "rnow-432050"}, {value = 2.0, time = "rnow-432060"}, {value = 2.0, time = "rnow-432070"}, {value = 2.0, time = "rnow-432080"}, {value = 2.0, time = "rnow-432090"}, {value = 2.0, time = "rnow-432100"}, {value = 2.0, time = "rnow-432110"}, {value = 2.0, time = "rnow-432120"}, {value = 2.0, time = "rnow-432130"}, {value = 2.0, time = "rnow-432140"}, {value = 2.0, time = "rnow-432150"}, {value = 2.0, time = "rnow-432160"}, {value = 2.0, time = "rnow-432170"}, {value = 2.0, time = "rnow-432180"}, {value = 2.0, time = "rnow-432190"}, {value = 2.0, time = "rnow-432200"}, {value = 2.0, time = "rnow-518000"}, {value = 2.0, time = "rnow-518010"}, {value = 2.0, time = "rnow-518020"}, {value = 2.0, time = "rnow-518030"}, {value = 2.0, time = "rnow-518040"}, {value = 2.0, time = "rnow-518050"}, {value = 2.0, time = "rnow-518060"}, {value = 2.0, time = "rnow-518070"}, {value = 2.0, time = "rnow-518080"}, {value = 2.0, time = "rnow-518090"}, {value = 2.0, time = "rnow-518100"}, {value = 2.0, time = "rnow-518110"}, {value = 2.0, time = "rnow-518120"}, {value = 2.0, time = "rnow-518130"}, {value = 2.0, time = "rnow-518140"}, {value = 2.0, time = "rnow-518150"}, {value = 2.0, time = "rnow-518160"}, {value = 2.0, time = "rnow-518170"}, {value = 2.0, time = "rnow-518180"}, {value = 2.0, time = "rnow-518190"}, {value = 2.0, time = "rnow-518200"}, {value = 2.0, time = "rnow-518210"}, {value = 2.0, time = "rnow-518220"}, {value = 2.0, time = "rnow-518230"}, {value = 2.0, time = "rnow-518240"}, {value = 2.0, time = "rnow-518250"}, {value = 2.0, time = "rnow-518260"}, {value = 2.0, time = "rnow-518270"}, {value = 2.0, time = "rnow-518280"}, {value = 2.0, time = "rnow-518290"}, {value = 2.0, time = "rnow-518300"}, {value = 2.0, time = "rnow-518310"}, {value = 2.0, time = "rnow-518320"}, {value = 2.0, time = "rnow-518330"}, {value = 2.0, time = "rnow-518340"}, {value = 2.0, time = "rnow-518350"}, {value = 2.0, time = "rnow-518360"}, {value = 2.0, time = "rnow-518370"}, {value = 2.0, time = "rnow-518380"}, {value = 2.0, time = "rnow-518390"}, {value = 2.0, time = "rnow-518400"}, {value = 3.0, time = "rnow-604400"}, {value = 3.0, time = "rnow-604410"}, {value = 3.0, time = "rnow-604420"}, {value = 3.0, time = "rnow-604430"}, {value = 3.0, time = "rnow-604440"}, {value = 3.0, time = "rnow-604450"}, {value = 3.0, time = "rnow-604460"}, {value = 3.0, time = "rnow-604470"}, {value = 3.0, time = "rnow-604480"}, {value = 3.0, time = "rnow-604490"}, {value = 3.0, time = "rnow-604500"}, {value = 3.0, time = "rnow-604510"}, {value = 3.0, time = "rnow-604520"}, {value = 3.0, time = "rnow-604530"}, {value = 3.0, time = "rnow-604540"}, {value = 3.0, time = "rnow-604550"}, {value = 3.0, time = "rnow-604560"}, {value = 3.0, time = "rnow-604570"}, {value = 3.0, time = "rnow-604580"}, {value = 3.0, time = "rnow-604590"}, {value = 3.0, time = "rnow-604600"}, {value = 3.0, time = "rnow-604610"}, {value = 3.0, time = "rnow-604620"}, {value = 3.0, time = "rnow-604630"}, {value = 3.0, time = "rnow-604640"}, {value = 3.0, time = "rnow-604650"}, {value = 3.0, time = "rnow-604660"}, {value = 3.0, time = "rnow-604670"}, {value = 3.0, time = "rnow-604680"}, {value = 3.0, time = "rnow-604690"}, {value = 3.0, time = "rnow-604700"}, {value = 3.0, time = "rnow-604710"}, {value = 3.0, time = "rnow-604720"}, {value = 3.0, time = "rnow-604730"}, {value = 3.0, time = "rnow-604740"}, {value = 3.0, time = "rnow-604750"}, {value = 3.0, time = "rnow-604760"}, {value = 3.0, time = "rnow-604770"}, {value = 3.0, time = "rnow-604780"}, {value = 3.0, time = "rnow-604790"}, {value = 3.0, time = "rnow-604800"}, {value = 2.0, time = "rnow-690800"}, {value = 2.0, time = "rnow-690810"}, {value = 2.0, time = "rnow-690820"}, {value = 2.0, time = "rnow-690830"}, {value = 2.0, time = "rnow-690840"}, {value = 2.0, time = "rnow-690850"}, {value = 2.0, time = "rnow-690860"}, {value = 2.0, time = "rnow-690870"}, {value = 2.0, time = "rnow-690880"}, {value = 2.0, time = "rnow-690890"}, {value = 2.0, time = "rnow-690900"}, {value = 2.0, time = "rnow-690910"}, {value = 2.0, time = "rnow-690920"}, {value = 2.0, time = "rnow-690930"}, {value = 2.0, time = "rnow-690940"}, {value = 2.0, time = "rnow-690950"}, {value = 2.0, time = "rnow-690960"}, {value = 2.0, time = "rnow-690970"}, {value = 2.0, time = "rnow-690980"}, {value = 2.0, time = "rnow-690990"}, {value = 2.0, time = "rnow-691000"}, {value = 2.0, time = "rnow-691010"}, {value = 2.0, time = "rnow-691020"}, {value = 2.0, time = "rnow-691030"}, {value = 2.0, time = "rnow-691040"}, {value = 2.0, time = "rnow-691050"}, {value = 2.0, time = "rnow-691060"}, {value = 2.0, time = "rnow-691070"}, {value = 2.0, time = "rnow-691080"}, {value = 2.0, time = "rnow-691090"}, {value = 2.0, time = "rnow-691100"}, {value = 2.0, time = "rnow-691110"}, {value = 2.0, time = "rnow-691120"}, {value = 2.0, time = "rnow-691130"}, {value = 2.0, time = "rnow-691140"}, {value = 2.0, time = "rnow-691150"}, {value = 2.0, time = "rnow-691160"}, {value = 2.0, time = "rnow-691170"}, {value = 2.0, time = "rnow-691180"}, {value = 2.0, time = "rnow-691190"}, {value = 2.0, time = "rnow-691200"}, {value = 2.0, time = "rnow-777200"}, {value = 2.0, time = "rnow-777210"}, {value = 2.0, time = "rnow-777220"}, {value = 2.0, time = "rnow-777230"}, {value = 2.0, time = "rnow-777240"}, {value = 2.0, time = "rnow-777250"}, {value = 2.0, time = "rnow-777260"}, {value = 2.0, time = "rnow-777270"}, {value = 2.0, time = "rnow-777280"}, {value = 2.0, time = "rnow-777290"}, {value = 2.0, time = "rnow-777300"}, {value = 2.0, time = "rnow-777310"}, {value = 2.0, time = "rnow-777320"}, {value = 2.0, time = "rnow-777330"}, {value = 2.0, time = "rnow-777340"}, {value = 2.0, time = "rnow-777350"}, {value = 2.0, time = "rnow-777360"}, {value = 2.0, time = "rnow-777370"}, {value = 2.0, time = "rnow-777380"}, {value = 2.0, time = "rnow-777390"}, {value = 2.0, time = "rnow-777400"}, {value = 2.0, time = "rnow-777410"}, {value = 2.0, time = "rnow-777420"}, {value = 2.0, time = "rnow-777430"}, {value = 2.0, time = "rnow-777440"}, {value = 2.0, time = "rnow-777450"}, {value = 2.0, time = "rnow-777460"}, {value = 2.0, time = "rnow-777470"}, {value = 2.0, time = "rnow-777480"}, {value = 2.0, time = "rnow-777490"}, {value = 2.0, time = "rnow-777500"}, {value = 2.0, time = "rnow-777510"}, {value = 2.0, time = "rnow-777520"}, {value = 2.0, time = "rnow-777530"}, {value = 2.0, time = "rnow-777540"}, {value = 2.0, time = "rnow-777550"}, {value = 2.0, time = "rnow-777560"}, {value = 2.0, time = "rnow-777570"}, {value = 2.0, time = "rnow-777580"}, {value = 2.0, time = "rnow-777590"}, {value = 2.0, time = "rnow-777600"}, {value = 3.0, time = "rnow-863800"}, {value = 3.0, time = "rnow-864000"} ] [[test.render_checks]] from = "rnow-10" until = "rnow+1" targets = [ "test.long" ] [[test.render_checks.result]] name = "test.long" path = "test.long" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, 2.0] # Check addional queryparam (storage read limit) [[test.render_checks]] from = "rnow-21600" until = "rnow" targets = [ "test.long" ] timeout = "5s" error_regexp = "^403: Storage read limit for rows" # Check data-timeout on addional queryparam [[test.render_checks]] from = "rnow-14200" until = "rnow" targets = [ "test.long" ] timeout = "2s" proxy_delay = "1500ms" error_regexp = "^504: Storage read timeout" # Check addional queryparam (storage read limit) [[test.render_checks]] from = "rnow-864000" until = "rnow" targets = [ "test.long" ] timeout = "40s" error_regexp = "^403: Storage read limit for memory" ================================================ FILE: tests/feature_flags_both_true/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: tests/feature_flags_both_true/graphite-clickhouse.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [feature-flags] use-carbon-behaviour = true dont-match-missing-tags = true [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = true tagged-table = "graphite_tags" tagged-autocomplete-days = 1 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/feature_flags_both_true/test.toml ================================================ [test] precision = "10s" [[test.clickhouse]] version = "21.3" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "22.8" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "24.2" dir = "tests/clickhouse/rollup" [test.carbon_clickhouse] template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse.conf.tpl" ####################################################################################### [[test.input]] name = "request_success_total.counter;app=test;project=Test;environment=TEST" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "request_success_total.counter;app=test;project=Test;environment=TEST;t=q" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "request_success_total.counter;app=test;project=Test;environment=TEST;t=qac" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "request_success_total.counter;app=test;project=Test;environment=TEST;t=cqa" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "test;env=prod" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "test;env=dr" points = [{value = 1.0, time = "rnow-10"}] ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ## seriesByTag('t=') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('t=')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('t=')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=dr" path = "seriesByTag('t=')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=prod" path = "seriesByTag('t=')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~')", ] ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ### seriesByTag('dc!=ru') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('dc!=ru')", ] ## seriesByTag('name=request_success_total.counter', 'dc!=ru') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'dc!=ru')", ] ## seriesByTag('dc!=~ru') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('dc!=~ru')", ] ## seriesByTag('name=request_success_total.counter','dc!=~ru') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter','dc!=~ru')", ] ### seriesByTag('t!=~qac') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('t!=~qac')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('t!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('t!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ## seriesByTag('name=request_success_total.counter', 't!=~qac') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 't!=~qac')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 't!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 't!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')", ] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')", ] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~q*') ### [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ## seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=test') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=test')", ] [[test.render_checks.result]] name = "test;env=dr" path = "seriesByTag('name=test')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=prod" path = "seriesByTag('name=test')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=test','env!=~stage|env') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=test','env!=~stage|env')", ] [[test.render_checks.result]] name = "test;env=dr" path = "seriesByTag('name=test','env!=~stage|env')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=prod" path = "seriesByTag('name=test','env!=~stage|env')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*a*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*a*')", ] error_regexp = "^400: Incorrect regex syntax" # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # # End - Test no flags # ######################################################################### ================================================ FILE: tests/feature_flags_dont_match_missing_tags/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: tests/feature_flags_dont_match_missing_tags/graphite-clickhouse.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [feature-flags] dont-match-missing-tags = true [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = true tagged-table = "graphite_tags" tagged-autocomplete-days = 1 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/feature_flags_dont_match_missing_tags/test.toml ================================================ [test] precision = "10s" [[test.clickhouse]] version = "21.3" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "22.8" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "24.2" dir = "tests/clickhouse/rollup" [test.carbon_clickhouse] template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse.conf.tpl" ####################################################################################### [[test.input]] name = "request_success_total.counter;app=test;project=Test;environment=TEST" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "request_success_total.counter;app=test;project=Test;environment=TEST;t=q" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "request_success_total.counter;app=test;project=Test;environment=TEST;t=qac" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "request_success_total.counter;app=test;project=Test;environment=TEST;t=cqa" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "test;env=prod" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "test;env=dr" points = [{value = 1.0, time = "rnow-10"}] ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=')", ] ## seriesByTag('t=') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('t=')", ] # seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~')", ] ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ### seriesByTag('dc!=ru') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('dc!=ru')", ] ## seriesByTag('name=request_success_total.counter', 'dc!=ru') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'dc!=ru')", ] ## seriesByTag('dc!=~ru') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('dc!=~ru')", ] ## seriesByTag('name=request_success_total.counter','dc!=~ru') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter','dc!=~ru')", ] ### seriesByTag('t!=~qac') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('t!=~qac')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('t!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('t!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ## seriesByTag('name=request_success_total.counter', 't!=~qac') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 't!=~qac')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 't!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 't!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')", ] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')", ] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~q*') ### [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ## seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=test') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=test')", ] [[test.render_checks.result]] name = "test;env=dr" path = "seriesByTag('name=test')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=prod" path = "seriesByTag('name=test')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=test','env!=~stage|env') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=test','env!=~stage|env')", ] [[test.render_checks.result]] name = "test;env=dr" path = "seriesByTag('name=test','env!=~stage|env')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=prod" path = "seriesByTag('name=test','env!=~stage|env')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*a*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*a*')", ] error_regexp = "^400: Incorrect regex syntax" # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # # End - Test no flags # ######################################################################### ================================================ FILE: tests/feature_flags_false/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: tests/feature_flags_false/graphite-clickhouse.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = true tagged-table = "graphite_tags" tagged-autocomplete-days = 1 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/feature_flags_false/test.toml ================================================ [test] precision = "10s" [[test.clickhouse]] version = "21.3" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "22.8" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "24.2" dir = "tests/clickhouse/rollup" [test.carbon_clickhouse] template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse.conf.tpl" ####################################################################################### [[test.input]] name = "request_success_total.counter;app=test;project=Test;environment=TEST" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "request_success_total.counter;app=test;project=Test;environment=TEST;t=q" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "request_success_total.counter;app=test;project=Test;environment=TEST;t=qac" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "request_success_total.counter;app=test;project=Test;environment=TEST;t=cqa" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "test;env=prod" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "test;env=dr" points = [{value = 1.0, time = "rnow-10"}] ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=')", ] ## seriesByTag('t=') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('t=')", ] # seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ### seriesByTag('dc!=ru') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('dc!=ru')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('dc!=ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('dc!=ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('dc!=ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('dc!=ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=dr" path = "seriesByTag('dc!=ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=prod" path = "seriesByTag('dc!=ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ## seriesByTag('name=request_success_total.counter', 'dc!=ru') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'dc!=ru')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 'dc!=ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'dc!=ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'dc!=ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'dc!=ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ## seriesByTag('dc!=~ru') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('dc!=~ru')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('dc!=~ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('dc!=~ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('dc!=~ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('dc!=~ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=dr" path = "seriesByTag('dc!=~ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=prod" path = "seriesByTag('dc!=~ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ## seriesByTag('name=request_success_total.counter','dc!=~ru') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter','dc!=~ru')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter','dc!=~ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter','dc!=~ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter','dc!=~ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter','dc!=~ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ### seriesByTag('t!=~qac') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('t!=~qac')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('t!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('t!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('t!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=dr" path = "seriesByTag('t!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=prod" path = "seriesByTag('t!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ## seriesByTag('name=request_success_total.counter', 't!=~qac') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 't!=~qac')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 't!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 't!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 't!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ## seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=test') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=test')", ] [[test.render_checks.result]] name = "test;env=dr" path = "seriesByTag('name=test')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=prod" path = "seriesByTag('name=test')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=test','env!=~stage|env') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=test','env!=~stage|env')", ] [[test.render_checks.result]] name = "test;env=dr" path = "seriesByTag('name=test','env!=~stage|env')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=prod" path = "seriesByTag('name=test','env!=~stage|env')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*a*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*a*')", ] error_regexp = "^400: Incorrect regex syntax" # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # # End - Test no flags # ######################################################################### ================================================ FILE: tests/feature_flags_use_carbon_behaviour/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: tests/feature_flags_use_carbon_behaviour/graphite-clickhouse.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [feature-flags] use-carbon-behaviour = true [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = true tagged-table = "graphite_tags" tagged-autocomplete-days = 1 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/feature_flags_use_carbon_behaviour/test.toml ================================================ [test] precision = "10s" [[test.clickhouse]] version = "21.3" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "22.8" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "24.2" dir = "tests/clickhouse/rollup" [test.carbon_clickhouse] template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse.conf.tpl" ####################################################################################### [[test.input]] name = "request_success_total.counter;app=test;project=Test;environment=TEST" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "request_success_total.counter;app=test;project=Test;environment=TEST;t=q" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "request_success_total.counter;app=test;project=Test;environment=TEST;t=qac" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "request_success_total.counter;app=test;project=Test;environment=TEST;t=cqa" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "test;env=prod" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "test;env=dr" points = [{value = 1.0, time = "rnow-10"}] ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ## seriesByTag('t=') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('t=')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('t=')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=dr" path = "seriesByTag('t=')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=prod" path = "seriesByTag('t=')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ### seriesByTag('dc!=ru') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('dc!=ru')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('dc!=ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('dc!=ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('dc!=ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('dc!=ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=dr" path = "seriesByTag('dc!=ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=prod" path = "seriesByTag('dc!=ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ## seriesByTag('name=request_success_total.counter', 'dc!=ru') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'dc!=ru')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 'dc!=ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'dc!=ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'dc!=ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'dc!=ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ## seriesByTag('dc!=~ru') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('dc!=~ru')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('dc!=~ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('dc!=~ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('dc!=~ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('dc!=~ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=dr" path = "seriesByTag('dc!=~ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=prod" path = "seriesByTag('dc!=~ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ## seriesByTag('name=request_success_total.counter','dc!=~ru') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter','dc!=~ru')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter','dc!=~ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter','dc!=~ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter','dc!=~ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter','dc!=~ru')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ### seriesByTag('t!=~qac') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('t!=~qac')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('t!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('t!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('t!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=dr" path = "seriesByTag('t!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=prod" path = "seriesByTag('t!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] ## seriesByTag('name=request_success_total.counter', 't!=~qac') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 't!=~qac')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 't!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 't!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 't!=~qac')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 'logger!=default')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't!=~cq.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=q*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ## seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*c')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*') [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~c.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~q$')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~a')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*a$')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~.*c$')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~^q.*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=test') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=test')", ] [[test.render_checks.result]] name = "test;env=dr" path = "seriesByTag('name=test')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=prod" path = "seriesByTag('name=test')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=test','env!=~stage|env') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=test','env!=~stage|env')", ] [[test.render_checks.result]] name = "test;env=dr" path = "seriesByTag('name=test','env!=~stage|env')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=prod" path = "seriesByTag('name=test','env!=~stage|env')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*a*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*a*')", ] error_regexp = "^400: Incorrect regex syntax" # ### seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*') ### [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')", ] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=cqa" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=q" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "request_success_total.counter;app=test;environment=TEST;project=Test;t=qac" path = "seriesByTag('name=request_success_total.counter', 'app=test', 'project=Test', 'environment=TEST', 't=~*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # # End - Test no flags # ######################################################################### ================================================ FILE: tests/find_cache/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: tests/find_cache/graphite-clickhouse-cached.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [common.find-cache] type = "mem" size-mb = 1 default-timeout = 300 short-timeout = 60 short-duration = "240s" find-timeout = 120 [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = false tagged-table = "graphite_tags" tagged-autocomplete-days = 1 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/find_cache/graphite-clickhouse-internal-aggr-cached.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [common.find-cache] type = "mem" size-mb = 1 default-timeout = 300 short-timeout = 60 short-duration = "240s" find-timeout = 120 [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = true tagged-table = "graphite_tags" tagged-autocomplete-days = 1 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/find_cache/test.toml ================================================ [test] precision = "10s" [[test.clickhouse]] version = "24.2" dir = "tests/clickhouse/rollup" delay = "10s" [test.carbon_clickhouse] template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse-cached.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse-internal-aggr-cached.conf.tpl" ########################################################################## [[test.input]] name = "test.cache" points = [{value = 1.0, time = "midnight-270s"}, {value = 3.0, time = "now"}] [[test.input]] name = "cache;scope=test" points = [{value = 2.0, time = "midnight-270s"}, {value = 4.0, time = "now"}] ########################################################################## [[test.find_checks]] query = "test" cache_ttl = 120 result = [{ path = "test", is_leaf = false }] [[test.find_checks]] query = "test.cache" cache_ttl = 120 result = [{ path = "test.cache", is_leaf = true }] ########################################################################## [[test.tags_checks]] query = "name;scope=test" cache_ttl = 120 result = [ "cache", ] ########################################################################## # Short cache TTL [[test.render_checks]] from = "rnow" until = "rnow+10" cache_ttl = 60 targets = [ "test.cache" ] [[test.render_checks.result]] name = "test.cache" path = "test.cache" consolidation = "avg" start = "rnow" stop = "rnow+20" step = 10 req_start = "rnow" req_stop = "rnow+20" values = [3.0, nan] # Already in find cache [[test.render_checks]] from = "rnow" until = "rnow+20" in_cache = true cache_ttl = 60 targets = [ "test.cache" ] [[test.render_checks.result]] name = "test.cache" path = "test.cache" consolidation = "avg" start = "rnow" stop = "rnow+30" step = 10 req_start = "rnow" req_stop = "rnow+30" values = [3.0, nan, nan] ########################################################################## # Short cache TTL [[test.render_checks]] from = "rnow" until = "rnow+10" cache_ttl = 60 targets = [ "seriesByTag('scope=test')" ] [[test.render_checks.result]] name = "cache;scope=test" path = "seriesByTag('scope=test')" consolidation = "avg" start = "rnow" stop = "rnow+20" step = 10 req_start = "rnow" req_stop = "rnow+20" values = [4.0, nan] # Already in find cache [[test.render_checks]] from = "rnow" until = "rnow+20" in_cache = true cache_ttl = 60 targets = [ "seriesByTag('scope=test')" ] [[test.render_checks.result]] name = "cache;scope=test" path = "seriesByTag('scope=test')" consolidation = "avg" start = "rnow" stop = "rnow+30" step = 10 req_start = "rnow" req_stop = "rnow+30" values = [4.0, nan, nan] ########################################################################## # Default cache TTL [[test.render_checks]] from = "midnight-270s" until = "midnight-20s" cache_ttl = 300 targets = [ "test.cache" ] [[test.render_checks.result]] name = "test.cache" path = "test.cache" consolidation = "avg" start = "midnight-270s" stop = "midnight-10s" step = 10 req_start = "midnight-270s" req_stop = "midnight-10s" values = [1.0, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan] # Already in find cache [[test.render_checks]] from = "midnight-270s" until = "midnight-10s" in_cache = true cache_ttl = 300 targets = [ "test.cache" ] [[test.render_checks.result]] name = "test.cache" path = "test.cache" consolidation = "avg" start = "midnight-270s" stop = "midnight" step = 10 req_start = "midnight-270s" req_stop = "midnight" values = [1.0, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan] # Fetch points from 2 days, not in cache [[test.render_checks]] from = "midnight-270s" until = "midnight" cache_ttl = 300 targets = [ "test.cache" ] [[test.render_checks.result]] name = "test.cache" path = "test.cache" consolidation = "avg" start = "midnight-270s" stop = "midnight+10s" step = 10 req_start = "midnight-270s" req_stop = "midnight+10s" values = [1.0, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan] ########################################################################## ================================================ FILE: tests/limitera/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: tests/limitera/graphite-clickhouse-internal-aggr-cached.conf.tpl ================================================ # Adaptive limiter with throttle queries and limit max queries [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [common.find-cache] type = "mem" size-mb = 1 default-timeout = 300 short-timeout = 60 short-duration = "240s" find-timeout = 120 [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = true tagged-table = "graphite_tags" tagged-autocomplete-days = 1 render-max-concurrent = 6 render-adaptive-queries = 2 find-max-concurrent = 4 find-adaptive-queries = 2 tags-max-concurrent = 4 tags-adaptive-queries = 2 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/limitera/test.toml ================================================ [test] precision = "10s" [[test.clickhouse]] version = "24.2" dir = "tests/clickhouse/rollup" delay = "10s" [test.carbon_clickhouse] template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse-internal-aggr-cached.conf.tpl" ########################################################################## [[test.input]] name = "test.cache" points = [{value = 1.0, time = "midnight-270s"}, {value = 3.0, time = "now"}] [[test.input]] name = "cache;scope=test" points = [{value = 2.0, time = "midnight-270s"}, {value = 4.0, time = "now"}] ########################################################################## [[test.find_checks]] query = "test" result = [{ path = "test", is_leaf = false }] [[test.find_checks]] query = "test.cache" result = [{ path = "test.cache", is_leaf = true }] ########################################################################## [[test.tags_checks]] query = "name;scope=test" result = [ "cache", ] ########################################################################## [[test.render_checks]] from = "rnow" until = "rnow+10" targets = [ "test.cache" ] [[test.render_checks.result]] name = "test.cache" path = "test.cache" consolidation = "avg" start = "rnow" stop = "rnow+20" step = 10 req_start = "rnow" req_stop = "rnow+20" values = [3.0, nan] ########################################################################## ================================================ FILE: tests/limitermax/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: tests/limitermax/graphite-clickhouse-internal-aggr-cached.conf.tpl ================================================ # Limiter with limit max connections [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [common.find-cache] type = "mem" size-mb = 1 default-timeout = 300 short-timeout = 60 short-duration = "240s" find-timeout = 120 [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = true tagged-table = "graphite_tags" tagged-autocomplete-days = 1 render-max-queries = 100 find-max-queries = 50 tags-max-queries = 50 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/limitermax/test.toml ================================================ [test] precision = "10s" [[test.clickhouse]] version = "24.2" dir = "tests/clickhouse/rollup" delay = "10s" [test.carbon_clickhouse] template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse-internal-aggr-cached.conf.tpl" ########################################################################## [[test.input]] name = "test.cache" points = [{value = 1.0, time = "midnight-270s"}, {value = 3.0, time = "now"}] [[test.input]] name = "cache;scope=test" points = [{value = 2.0, time = "midnight-270s"}, {value = 4.0, time = "now"}] ########################################################################## [[test.find_checks]] query = "test" result = [{ path = "test", is_leaf = false }] [[test.find_checks]] query = "test.cache" result = [{ path = "test.cache", is_leaf = true }] ########################################################################## [[test.tags_checks]] query = "name;scope=test" result = [ "cache", ] ########################################################################## [[test.render_checks]] from = "rnow" until = "rnow+10" targets = [ "test.cache" ] [[test.render_checks.result]] name = "test.cache" path = "test.cache" consolidation = "avg" start = "rnow" stop = "rnow+20" step = 10 req_start = "rnow" req_stop = "rnow+20" values = [3.0, nan] ########################################################################## ================================================ FILE: tests/limiterw/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: tests/limiterw/graphite-clickhouse-internal-aggr-cached.conf.tpl ================================================ # Limiter with throttle queries [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [common.find-cache] type = "mem" size-mb = 1 default-timeout = 300 short-timeout = 60 short-duration = "240s" find-timeout = 120 [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = true tagged-table = "graphite_tags" tagged-autocomplete-days = 1 render-max-concurrent = 6 find-max-concurrent = 4 tags-max-concurrent = 4 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/limiterw/test.toml ================================================ [test] precision = "10s" [[test.clickhouse]] version = "24.2" dir = "tests/clickhouse/rollup" delay = "10s" [test.carbon_clickhouse] template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse-internal-aggr-cached.conf.tpl" ########################################################################## [[test.input]] name = "test.cache" points = [{value = 1.0, time = "midnight-270s"}, {value = 3.0, time = "now"}] [[test.input]] name = "cache;scope=test" points = [{value = 2.0, time = "midnight-270s"}, {value = 4.0, time = "now"}] ########################################################################## [[test.find_checks]] query = "test" result = [{ path = "test", is_leaf = false }] [[test.find_checks]] query = "test.cache" result = [{ path = "test.cache", is_leaf = true }] ########################################################################## [[test.tags_checks]] query = "name;scope=test" result = [ "cache", ] ########################################################################## [[test.render_checks]] from = "rnow" until = "rnow+10" targets = [ "test.cache" ] [[test.render_checks.result]] name = "test.cache" path = "test.cache" consolidation = "avg" start = "rnow" stop = "rnow+20" step = 10 req_start = "rnow" req_stop = "rnow+20" values = [3.0, nan] ########################################################################## ================================================ FILE: tests/limiterwn/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: tests/limiterwn/graphite-clickhouse-internal-aggr-cached.conf.tpl ================================================ # Limiter with throttle queries and limit max queries [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [common.find-cache] type = "mem" size-mb = 1 default-timeout = 300 short-timeout = 60 short-duration = "240s" find-timeout = 120 [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = true tagged-table = "graphite_tags" tagged-autocomplete-days = 1 render-max-queries = 100 render-max-concurrent = 6 find-max-queries = 50 find-max-concurrent = 4 tags-max-queries = 50 tags-max-concurrent = 4 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/limiterwn/test.toml ================================================ [test] precision = "10s" [[test.clickhouse]] version = "24.2" dir = "tests/clickhouse/rollup" delay = "10s" [test.carbon_clickhouse] template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse-internal-aggr-cached.conf.tpl" ########################################################################## [[test.input]] name = "test.cache" points = [{value = 1.0, time = "midnight-270s"}, {value = 3.0, time = "now"}] [[test.input]] name = "cache;scope=test" points = [{value = 2.0, time = "midnight-270s"}, {value = 4.0, time = "now"}] ########################################################################## [[test.find_checks]] query = "test" result = [{ path = "test", is_leaf = false }] [[test.find_checks]] query = "test.cache" result = [{ path = "test.cache", is_leaf = true }] ########################################################################## [[test.tags_checks]] query = "name;scope=test" result = [ "cache", ] ########################################################################## [[test.render_checks]] from = "rnow" until = "rnow+10" targets = [ "test.cache" ] [[test.render_checks.result]] name = "test.cache" path = "test.cache" consolidation = "avg" start = "rnow" stop = "rnow+20" step = 10 req_start = "rnow" req_stop = "rnow+20" values = [3.0, nan] ########################################################################## ================================================ FILE: tests/one_table/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: tests/one_table/graphite-clickhouse-internal-aggr.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = true tagged-table = "graphite_tags" tagged-autocomplete-days = 1 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/one_table/graphite-clickhouse.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = false tagged-table = "graphite_tags" tagged-autocomplete-days = 1 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/one_table/test.toml ================================================ [test] precision = "10s" [[test.clickhouse]] version = "21.3" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "22.8" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "24.2" dir = "tests/clickhouse/rollup" [test.carbon_clickhouse] template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse-internal-aggr.conf.tpl" [[test.input]] name = "test.plain1" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 2.0, time = "rnow"}] [[test.input]] name = "test.plain2" points = [{value = 2.0, time = "rnow-30"}, {value = 1.0, time = "rnow-20"}, {value = 1.5, time = "rnow-10"}, {value = 2.5, time = "rnow"}] [[test.input]] name = "test2.plain" points = [{value = 1.0, time = "rnow-30"}, {value = 2.0, time = "rnow-20"}, {value = 2.5, time = "rnow-10"}, {value = 3.5, time = "rnow"}] [[test.input]] name = "metric1;tag1=value1;tag2=value21;tag3=value3" points = [{value = 2.0, time = "rnow-30"}, {value = 2.5, time = "rnow-20"}, {value = 2.0, time = "rnow-10"}, {value = 3.0, time = "rnow"}] [[test.input]] name = "metric1;tag2=value22;tag4=value4" points = [{value = 1.0, time = "rnow-30"}, {value = 2.0, time = "rnow-20"}, {value = 0.0, time = "rnow-10"}, {value = 1.0, time = "rnow"}] [[test.input]] name = "metric1;tag1=value1;tag2=value23;tag3=value3" points = [{value = 0.5, time = "rnow-30"}, {value = 1.5, time = "rnow-20"}, {value = 4.0, time = "rnow-10"}, {value = 3.0, time = "rnow"}] [[test.input]] name = "metric2;tag2=value21;tag4=value4" points = [{value = 2.0, time = "rnow-30"}, {value = 1.0, time = "rnow-20"}, {value = 0.0, time = "rnow-10"}, {value = 1.0, time = "rnow"}] [[test.input]] name = "test_metric;minus=-;plus=+;percent=%;underscore=_;colon=:;hash=#;forward=/;host=127.0.0.1" points = [{value = 2.1, time = "rnow-30"}, {value = 0.1, time = "rnow-20"}, {value = 0.2, time = "rnow-10"}, {value = 1.5, time = "rnow"}] ###################################### # Check metrics find [[test.find_checks]] formats = [ "pickle", "protobuf", "carbonapi_v3_pb" ] query = "test" result = [ { path = "test", is_leaf = false } ] [[test.find_checks]] formats = [ "pickle", "protobuf", "carbonapi_v3_pb" ] query = "test.pl*" result = [ { path = "test.plain1", is_leaf = true }, { path = "test.plain2", is_leaf = true } ] # End - Check metrics find ###################################### # Check tags autocomplete [[test.tags_checks]] query = "tag1;tag2=value21" result = [ "value1" ] [[test.tags_checks]] query = "name;tag2=value21;tag1=~value" result = [ "metric1", ] [[test.tags_checks]] query = "colon;percent=%" result = [ ":", ] # End - Check tags autocomplete ########################################################################## # Plain metrics (carbonapi_v3_pb) # test.plain1 # test.plain2 # test2.plain [[test.render_checks]] from = "rnow-10" until = "rnow" targets = [ "test.plain*", "test{1,2}.plain" ] [[test.render_checks.result]] name = "test.plain1" path = "test.plain*" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, 2.0] [[test.render_checks.result]] name = "test.plain2" path = "test.plain*" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.5, 2.5] [[test.render_checks.result]] name = "test2.plain" path = "test{1,2}.plain" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [2.5, 3.5] # End - Plain metrics (carbonapi_v3_pb) ########################################################################## # Plain metrics (carbonapi_v2_pb) [[test.render_checks]] formats = [ "protobuf", "carbonapi_v2_pb" ] from = "rnow-10" until = "rnow+1" targets = [ "test.plain*", "test{1,2}.plain" ] [[test.render_checks.result]] name = "test.plain1" start = "rnow-10" stop = "rnow+10" step = 10 values = [1.0, 2.0] [[test.render_checks.result]] name = "test.plain2" start = "rnow-10" stop = "rnow+10" step = 10 values = [1.5, 2.5] [[test.render_checks.result]] name = "test2.plain" start = "rnow-10" stop = "rnow+10" step = 10 values = [2.5, 3.5] # End - Plain metrics (carbonapi_v2_pb) ########################################################################## # Plain metrics (pickle) [[test.render_checks]] formats = [ "pickle" ] from = "rnow-10" until = "rnow+1" targets = [ "test.plain*", "test{1,2}.plain" ] [[test.render_checks.result]] name = "test.plain1" path = "test.plain*" start = "rnow-10" stop = "rnow+10" step = 10 values = [1.0, 2.0] [[test.render_checks.result]] name = "test.plain2" path = "test.plain*" start = "rnow-10" stop = "rnow+10" step = 10 values = [1.5, 2.5] [[test.render_checks.result]] name = "test2.plain" path = "test{1,2}.plain" start = "rnow-10" stop = "rnow+10" step = 10 values = [2.5, 3.5] # End - Plain metrics (pickle) ########################################################################## # Taged metrics (carbonapi_v3_pb) # metric1;tag1=value1;tag2=value21;tag3=value3 # metric1;tag2=value22;tag4=value4 # metric1;tag1=value1;tag2=value23;tag3=value3 # metric2;tag2=value21;tag4=value4 [[test.render_checks]] from = "rnow-10" until = "rnow+1" targets = [ "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')", "seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')" ] [[test.render_checks.result]] name = "metric1;tag1=value1;tag2=value21;tag3=value3" path = "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [2.0, 3.0] [[test.render_checks.result]] name = "metric1;tag1=value1;tag2=value23;tag3=value3" path = "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [4.0, 3.0] [[test.render_checks.result]] name = "metric2;tag2=value21;tag4=value4" path = "seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [0.0, 1.0] # End - Tagged metrics (carbonapi_v3_pb) ########################################################################## # Tagged metrics (carbonapi_v2_pb) [[test.render_checks]] formats = [ "protobuf", "carbonapi_v2_pb" ] from = "rnow-10" until = "rnow+1" targets = [ "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')", "seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')" ] [[test.render_checks.result]] name = "metric1;tag1=value1;tag2=value21;tag3=value3" start = "rnow-10" stop = "rnow+10" step = 10 values = [2.0, 3.0] [[test.render_checks.result]] name = "metric1;tag1=value1;tag2=value23;tag3=value3" start = "rnow-10" stop = "rnow+10" step = 10 values = [4.0, 3.0] [[test.render_checks.result]] name = "metric2;tag2=value21;tag4=value4" start = "rnow-10" stop = "rnow+10" step = 10 values = [0.0, 1.0] # End - Tagged metrics (carbonapi_v2_pb) ########################################################################## # Tagged metrics (pickle) [[test.render_checks]] formats = [ "pickle" ] from = "rnow-10" until = "rnow+1" targets = [ "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')", "seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')" ] [[test.render_checks.result]] name = "metric1;tag1=value1;tag2=value21;tag3=value3" path = "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')" start = "rnow-10" stop = "rnow+10" step = 10 values = [2.0, 3.0] [[test.render_checks.result]] name = "metric1;tag1=value1;tag2=value23;tag3=value3" path = "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')" start = "rnow-10" stop = "rnow+10" step = 10 values = [4.0, 3.0] [[test.render_checks.result]] name = "metric2;tag2=value21;tag4=value4" path = "seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')" start = "rnow-10" stop = "rnow+10" step = 10 values = [0.0, 1.0] # End - Tagged metrics (pickle) ########################################################################## # Unescape [[test.render_checks]] formats = [ "protobuf", "carbonapi_v2_pb" ] from = "rnow-10" until = "rnow+1" targets = [ "seriesByTag('percent=%')", ] [[test.render_checks.result]] name = "test_metric;colon=:;forward=/;hash=#;host=127.0.0.1;minus=-;percent=%;plus=+;underscore=_" start = "rnow-10" stop = "rnow+10" step = 10 values = [0.2, 1.5] # End - Tagged metrics (pickle) ########################################################################## # Midnight # points for check https://github.com/go-graphite/graphite-clickhouse/issues/184 [[test.input]] name = "test.midnight" points = [{value = 3.0, time = "midnight+60s"}] [[test.input]] name = "now;scope=midnight" points = [{value = 4.0, time = "midnight+60s"}] [[test.find_checks]] name = "Midnight (direct)" query = "test.midnight*" result = [{ path = "test.midnight", is_leaf = true }] [[test.find_checks]] name = "Midnight" query = "test.midnight" from = "midnight+60s" until = "midnight+70s" result = [{ path = "test.midnight", is_leaf = true }] [[test.find_checks]] name = "Midnight (reverse)" query = "*test.midnight" result = [{ path = "test.midnight", is_leaf = true }] [[test.find_checks]] name = "Midnight" query = "test.midnight" from = "midnight+60s" until = "midnight+70s" result = [{ path = "test.midnight", is_leaf = true }] [[test.tags_checks]] name = "Midnight" query = "name;scope=midnight" result = [ "now", ] [[test.render_checks]] name = "Midnight (direct)" formats = [ "protobuf" ] from = "midnight+60s" until = "midnight+70s" targets = [ "test.midnight*", ] [[test.render_checks.result]] name = "test.midnight" start = "midnight+60s" stop = "midnight+80s" step = 10 values = [3.0, nan] [[test.render_checks]] name = "Midnight (reverse)" formats = [ "protobuf" ] from = "midnight+60s" until = "midnight+70s" targets = [ "*test.midnight", ] [[test.render_checks.result]] name = "test.midnight" start = "midnight+60s" stop = "midnight+80s" step = 10 values = [3.0, nan] [[test.render_checks]] name = "Midnight" formats = [ "protobuf" ] from = "midnight+60s" until = "midnight+70s" targets = [ "seriesByTag('name=now', 'scope=midnight')", ] [[test.render_checks.result]] name = "now;scope=midnight" start = "midnight+60s" stop = "midnight+80s" step = 10 values = [4.0, nan] # End - Midnight ########################################################################## # Day end # points for check https://github.com/go-graphite/graphite-clickhouse/issues/184 [[test.input]] name = "test.23h" points = [{value = 3.0, time = "midnight+1380m"}] [[test.input]] name = "now;scope=23h" points = [{value = 4.0, time = "midnight+1380m"}] [[test.find_checks]] name = "Day end" query = "test.23h" from = "midnight+1380m" until = "midnight+1381m" result = [{ path = "test.23h", is_leaf = true }] [[test.find_checks]] name = "Day end" query = "test.23h" from = "midnight+1380m" until = "midnight+1381m" result = [{ path = "test.23h", is_leaf = true }] [[test.tags_checks]] name = "Day end" query = "name;scope=23h" result = [ "now", ] [[test.render_checks]] name = "Day end" formats = [ "protobuf" ] from = "midnight+1380m" until = "midnight+1380m+10s" targets = [ "test.23h", ] [[test.render_checks.result]] name = "test.23h" start = "midnight+1380m" stop = "midnight+1380m+20s" step = 10 values = [3.0, nan] [[test.render_checks]] name = "Day end" formats = [ "protobuf" ] from = "midnight+1380m" until = "midnight+1380m+10s" targets = [ "seriesByTag('name=now', 'scope=23h')", ] [[test.render_checks.result]] name = "now;scope=23h" start = "midnight+1380m" stop = "midnight+1380m+20s" step = 10 values = [4.0, nan] # End - Day end ########################################################################## ================================================ FILE: tests/tags_min_in_query/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: tests/tags_min_in_query/graphite-clickhouse.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [feature-flags] use-carbon-behaviour = true [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = true tagged-table = "graphite_tags" tagged-autocomplete-days = 1 tags-min-in-query = 1 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/tags_min_in_query/test.toml ================================================ [test] precision = "10s" [[test.clickhouse]] version = "21.3" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "22.8" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "24.2" dir = "tests/clickhouse/rollup" [test.carbon_clickhouse] template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse.conf.tpl" ####################################################################################### [[test.input]] name = "test;env=prod" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "test;env=dev" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "test;env=stage" points = [{value = 1.0, time = "rnow-10"}] [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('name=test')", ] [[test.render_checks.result]] name = "test;env=prod" path = "seriesByTag('name=test')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=dev" path = "seriesByTag('name=test')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "test;env=stage" path = "seriesByTag('name=test')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('env=dev')", ] [[test.render_checks.result]] name = "test;env=dev" path = "seriesByTag('env=dev')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] # due to 'use-carbon-behaviour = true' [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('env=')", ] error_regexp = "^403: seriesByTag argument has too much wildcard and regex terms" [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('env!=prod')", ] error_regexp = "^403: seriesByTag argument has too much wildcard and regex terms" [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('env!=')", ] error_regexp = "^403: seriesByTag argument has too much wildcard and regex terms" [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('env=~')", ] error_regexp = "^403: seriesByTag argument has too much wildcard and regex terms" [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('env=~pr')", ] error_regexp = "^403: seriesByTag argument has too much wildcard and regex terms" [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('env!=~')", ] error_regexp = "^403: seriesByTag argument has too much wildcard and regex terms" [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "seriesByTag('env!=~pr')", ] error_regexp = "^403: seriesByTag argument has too much wildcard and regex terms" ================================================ FILE: tests/tls/ca.crt ================================================ -----BEGIN CERTIFICATE----- MIIDHTCCAgWgAwIBAgIURx5itXwLHeiQES1LzCHF7F8RNEkwDQYJKoZIhvcNAQEL BQAwHjEcMBoGA1UEAwwTbG9yZHZpcmRleC5sb2NhbCBDQTAeFw0yNDA4MDkxMjMy MzJaFw0zNDA4MDcxMjMyMzJaMB4xHDAaBgNVBAMME2xvcmR2aXJkZXgubG9jYWwg Q0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDuiK4tBYzNtROmhuXD 80HsVVk2/+/TXV85Aey7oo2gxxJJ09iARnjJadNrbBUdoL42XtmBCkYY+pXYUWPD hvals2AbXiAePg7DlAHJfpaQTzHlsPvAUMjqbD6cFaQ7DfNQHcz2emmFhcRYzlQM h0Ob3v2yhogG7PuKaiTLTKYcHnRKfEIobQEIq16ABaaCFKzR6tpvrUJFYtkJ8EUz jhrSg67qy7yiHiMmGQVq526X2oZYhMbSGjiPkaMZHdFkxZgJF5iQhANG9djvcopO jdFfsJYM9rVxAjwO/P3fq5dpuQxWLLo6ZmholsixPZs1s8paEnonSDtyoNLsykwD 2mFdAgMBAAGjUzBRMB0GA1UdDgQWBBS6BlL90Mo/+aHonqIqaewM8CyxnTAfBgNV HSMEGDAWgBS6BlL90Mo/+aHonqIqaewM8CyxnTAPBgNVHRMBAf8EBTADAQH/MA0G CSqGSIb3DQEBCwUAA4IBAQAIwTN3II6HdPfMsLvYoOmzcvUE9Y6QndI20eLqp3p8 6KnU+lgLdSkLjc9BKwLh/Jhuy4H3u1nHpW8Jkgy/8irG2uaUvgKlutfApFQshAo7 /k9xdH36ER0LF/bW5hQ535H76OaE+eaexx2zU50kPVuntal577d8HBfrKVI41KU8 CVdqYTwEqHwjSyRhmmRqLi7Yo+i0o0hRwH39LxYXY2rup/V6uRyLXSIDUZ9VeqVt K8XDAbLV1s4kzR/OdpYcJuTWX9gFUlNHpGDkOSy9ggc5zxKaHlwGGZsvVSb4f+VF C89ABPZs+26EvExIih+civiC1XWIghP8RsiNyBOK3TOf -----END CERTIFICATE----- ================================================ FILE: tests/tls/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: tests/tls/client.crt ================================================ -----BEGIN CERTIFICATE----- MIIDXTCCAkWgAwIBAgIUH+CPx0invXJZGZk7WQ0TOl2duV4wDQYJKoZIhvcNAQEL BQAwHjEcMBoGA1UEAwwTbG9yZHZpcmRleC5sb2NhbCBDQTAeFw0yNDA4MDkxMjM4 NTdaFw0zNDA4MDcxMjM4NTdaMFMxCzAJBgNVBAYTAlJVMRIwEAYDVQQIDAlUYXRh cnN0YW4xDjAMBgNVBAcMBUthemFuMQ8wDQYDVQQKDAZLb250dXIxDzANBgNVBAMM BmNsaWVudDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAN4ksMPzkoon SBbAIutgLjpEJOoEVb5iHbBzkAn9c9EDwkHVrUGFlx69QwkBncoomV09WW3dCMlf FX8ClHZ5/vEpJAxQVHYTyeNpzRE+gtDuun0NN+TPgTv3Q/wBrBds/4xl1UxtuwQW QkrZtREi71SYcdkuWnMv4OiA6EZnhJUBuPW6oV0Sa12PeEcmQJuliHGGDd72l50d ZsYJi/WhVgmJ5FlUkED8cVxKDbXhk3rGkXpkU/eyfEh12sNr0nX6BpPNCts3puM9 8lkzJ2luSkfwtp46s/pQwgs0aADVd37WaV1DbNGT2iqSnnPlVHR+DoGfb8+c3AGP zKzuvMNH2xMCAwEAAaNeMFwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMB0G A1UdDgQWBBRK5htFqO/QAQp6O2yvQOjwIFaXDDAfBgNVHSMEGDAWgBS6BlL90Mo/ +aHonqIqaewM8CyxnTANBgkqhkiG9w0BAQsFAAOCAQEABl6sVpN1O/fRF1RFKfvc pYpzFdqQpH2lva26Ove3PMMn3gYD3fgH3JKt1JHJ8mejJ/fJDReM1hD5MtR8buuF P/UHEg0cJ47ljLFHjnjJX4IobuxAVRkkt+1mx7/HLQoJjPEyzDwuKazz1XcXQd4c 4F3oa/nmo7/Nzf7NnnSEvNkwv3Anc18qAnwxCaONR0mkEWfJ0sZlcnxS1FlVEVtG kSymZJa6VsRMqgRDsrTyaOF0WcYuL7+onlywc2+A7fjbPzlFhTL83/yiZA+IDcrV OC81JN67uh4aK3nXlCHBDU1jFdr9u0jCGwo+wmWfKea7r6KG/J0a2IEokfIdSnmh wQ== -----END CERTIFICATE----- ================================================ FILE: tests/tls/client.key ================================================ -----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDeJLDD85KKJ0gW wCLrYC46RCTqBFW+Yh2wc5AJ/XPRA8JB1a1BhZcevUMJAZ3KKJldPVlt3QjJXxV/ ApR2ef7xKSQMUFR2E8njac0RPoLQ7rp9DTfkz4E790P8AawXbP+MZdVMbbsEFkJK 2bURIu9UmHHZLlpzL+DogOhGZ4SVAbj1uqFdEmtdj3hHJkCbpYhxhg3e9pedHWbG CYv1oVYJieRZVJBA/HFcSg214ZN6xpF6ZFP3snxIddrDa9J1+gaTzQrbN6bjPfJZ MydpbkpH8LaeOrP6UMILNGgA1Xd+1mldQ2zRk9oqkp5z5VR0fg6Bn2/PnNwBj8ys 7rzDR9sTAgMBAAECggEACA7kqqGx9c5UUHRKeqdT20vlhZJVWev35RK2wuYCxtjl ZGX6kZ869XkqxATe/cUDQIfyhTMOF9/vHlsFmlaf54z1KyKELdRXRSdCsmwbarYo 5ahjwqpppwyNLB2+FGDL1Ff3/icXhZ/Dv9tYih/uS+9LvJ5wgYUsb21dqlAkVb4C SK03xmOQ/osaDUZpVj1E6uhiNcs3hc1z5nTLZXFeGjVdtJUePXoAzO6saJOcbcsy 4iSzmCcT/WJ1T3crlXU6v+v7gc1L1/7uAq5yDTVHtwlxoU+SqcmbKKlUECH86cTs xT16UtU71SlmQtsnwDYavdb017vhB4+6JKOAFcyTwQKBgQDq8rje8qnacw1Nz1xD JS+p0R4dkE8uD8usQaTtajpd0lGci28zcl8pr/fPpVwlF8VSePDrMjaf8tqs5Dq/ KRri3NEnXLhoKsGOn9PiGDwchDG57Lya4OnJ9dDa/FWUzCvxyttBpQZwocaMTEBU C0nD2G+SxVMdEjzhXKfQFVn7eQKBgQDyDD/hU8tFT28Bge7lRYG73JGZfr4ZeBu0 EOGeu/402fOECngVK1b39VDOTD9me3QJKbKKRtjiUJq+0oFLXyl9nUbBIV8xhBF+ r9jNd3W0aClzR4u9oxCTnyvodpElWChBWnTZu1EcCASz8KUm0IY+dnbu12I3K4uX ti8n+xpb6wKBgC2zgRp9AWUotBHKoBu/hAH4V29QvtYq5GdhbX9xBmFxo8ZbqQnM 2Y32WLHfbIkakpt0Qwi8/7slNjwjOPouOLigU17gvk4k4vmnRUPZivfRDwsnbZiC 33cVhcbTBqKnBHVIDFY8j4AhN8namzi96V9bHnjiQUSKY6VCrLHhNVuhAoGBAO1Q I0WKAW8oLV7eBNrXZhZJcJt9D2crQoYuUvdtvBQXaNEZ7pha0L71z08kpLiW67Kc Jke6pKRngQD8pPXADI7zJ87tKEcFBJ4gTMFOkaHaymETUagRe4ww8DzQGwjxQS6q QIzFQgXouqutkk7W/fe58GvF0q7iy89oOR3K7RIXAoGAVPk+MFC5cjyYc+srSoGg K66BhwVyhsjF+7n2qptSFa8OXTtIVV/TBnpeW0l2lD1EGs4RLNz7wmgqa7eU6co3 tzFJqhGQPm0785QfSk3aOSS3OGzR3TqkDUt8LLK8rXIoUuFgNyRKnsp4ncSN75zu FL/drPzHeFzuHozRy8DxxBE= -----END PRIVATE KEY----- ================================================ FILE: tests/tls/graphite-clickhouse.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] append-empty-series = false [clickhouse] url = "{{ .CLICKHOUSE_TLS_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = true tagged-table = "graphite_tags" tagged-autocomplete-days = 1 [clickhouse.tls] ca-cert = ["{{- .TEST_DIR -}}/ca.crt"] server-name = "localhost" [[clickhouse.tls.certificates]] key = "{{- .TEST_DIR -}}/client.key" cert = "{{- .TEST_DIR -}}/client.crt" [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/tls/test.toml ================================================ [test] precision = "10s" [[test.clickhouse]] version = "21.3" tls = true dir = "tests/clickhouse/rollup_tls" [[test.clickhouse]] version = "22.8" tls = true dir = "tests/clickhouse/rollup_tls" [[test.clickhouse]] version = "24.2" tls = true dir = "tests/clickhouse/rollup_tls" [test.carbon_clickhouse] template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse.conf.tpl" [[test.input]] name = "test.plain1" points = [{value = 3.0, time = "rnow-30"}, {value = 0.0, time = "rnow-20"}, {value = 1.0, time = "rnow-10"}, {value = 2.0, time = "rnow"}] [[test.input]] name = "test.plain2" points = [{value = 2.0, time = "rnow-30"}, {value = 1.0, time = "rnow-20"}, {value = 1.5, time = "rnow-10"}, {value = 2.5, time = "rnow"}] [[test.input]] name = "test2.plain" points = [{value = 1.0, time = "rnow-30"}, {value = 2.0, time = "rnow-20"}, {value = 2.5, time = "rnow-10"}, {value = 3.5, time = "rnow"}] [[test.input]] name = "metric1;tag1=value1;tag2=value21;tag3=value3" points = [{value = 2.0, time = "rnow-30"}, {value = 2.5, time = "rnow-20"}, {value = 2.0, time = "rnow-10"}, {value = 3.0, time = "rnow"}] [[test.input]] name = "metric1;tag2=value22;tag4=value4" points = [{value = 1.0, time = "rnow-30"}, {value = 2.0, time = "rnow-20"}, {value = 0.0, time = "rnow-10"}, {value = 1.0, time = "rnow"}] [[test.input]] name = "metric1;tag1=value1;tag2=value23;tag3=value3" points = [{value = 0.5, time = "rnow-30"}, {value = 1.5, time = "rnow-20"}, {value = 4.0, time = "rnow-10"}, {value = 3.0, time = "rnow"}] [[test.input]] name = "metric2;tag2=value21;tag4=value4" points = [{value = 2.0, time = "rnow-30"}, {value = 1.0, time = "rnow-20"}, {value = 0.0, time = "rnow-10"}, {value = 1.0, time = "rnow"}] [[test.input]] name = "test_metric;minus=-;plus=+;percent=%;underscore=_;colon=:;hash=#;forward=/;host=127.0.0.1" points = [{value = 2.1, time = "rnow-30"}, {value = 0.1, time = "rnow-20"}, {value = 0.2, time = "rnow-10"}, {value = 1.5, time = "rnow"}] ###################################### # Check metrics find [[test.find_checks]] formats = [ "pickle", "protobuf", "carbonapi_v3_pb" ] query = "test" result = [ { path = "test", is_leaf = false } ] [[test.find_checks]] formats = [ "pickle", "protobuf", "carbonapi_v3_pb" ] query = "test.pl*" result = [ { path = "test.plain1", is_leaf = true }, { path = "test.plain2", is_leaf = true } ] # End - Check metrics find ###################################### # Check tags autocomplete [[test.tags_checks]] query = "tag1;tag2=value21" result = [ "value1" ] [[test.tags_checks]] query = "name;tag2=value21;tag1=~value" result = [ "metric1", ] [[test.tags_checks]] query = "colon;percent=%" result = [ ":", ] # End - Check tags autocomplete ########################################################################## # Plain metrics (carbonapi_v3_pb) # test.plain1 # test.plain2 # test2.plain [[test.render_checks]] from = "rnow-10" until = "rnow" targets = [ "test.plain*", "test{1,2}.plain" ] [[test.render_checks.result]] name = "test.plain1" path = "test.plain*" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, 2.0] [[test.render_checks.result]] name = "test.plain2" path = "test.plain*" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.5, 2.5] [[test.render_checks.result]] name = "test2.plain" path = "test{1,2}.plain" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [2.5, 3.5] # End - Plain metrics (carbonapi_v3_pb) ########################################################################## # Plain metrics (carbonapi_v2_pb) [[test.render_checks]] formats = [ "protobuf", "carbonapi_v2_pb" ] from = "rnow-10" until = "rnow+1" targets = [ "test.plain*", "test{1,2}.plain" ] [[test.render_checks.result]] name = "test.plain1" start = "rnow-10" stop = "rnow+10" step = 10 values = [1.0, 2.0] [[test.render_checks.result]] name = "test.plain2" start = "rnow-10" stop = "rnow+10" step = 10 values = [1.5, 2.5] [[test.render_checks.result]] name = "test2.plain" start = "rnow-10" stop = "rnow+10" step = 10 values = [2.5, 3.5] # End - Plain metrics (carbonapi_v2_pb) ########################################################################## # Plain metrics (pickle) [[test.render_checks]] formats = [ "pickle" ] from = "rnow-10" until = "rnow+1" targets = [ "test.plain*", "test{1,2}.plain" ] [[test.render_checks.result]] name = "test.plain1" path = "test.plain*" start = "rnow-10" stop = "rnow+10" step = 10 values = [1.0, 2.0] [[test.render_checks.result]] name = "test.plain2" path = "test.plain*" start = "rnow-10" stop = "rnow+10" step = 10 values = [1.5, 2.5] [[test.render_checks.result]] name = "test2.plain" path = "test{1,2}.plain" start = "rnow-10" stop = "rnow+10" step = 10 values = [2.5, 3.5] # End - Plain metrics (pickle) ########################################################################## # Taged metrics (carbonapi_v3_pb) # metric1;tag1=value1;tag2=value21;tag3=value3 # metric1;tag2=value22;tag4=value4 # metric1;tag1=value1;tag2=value23;tag3=value3 # metric2;tag2=value21;tag4=value4 [[test.render_checks]] from = "rnow-10" until = "rnow+1" targets = [ "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')", "seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')" ] [[test.render_checks.result]] name = "metric1;tag1=value1;tag2=value21;tag3=value3" path = "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [2.0, 3.0] [[test.render_checks.result]] name = "metric1;tag1=value1;tag2=value23;tag3=value3" path = "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [4.0, 3.0] [[test.render_checks.result]] name = "metric2;tag2=value21;tag4=value4" path = "seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [0.0, 1.0] # End - Tagged metrics (carbonapi_v3_pb) ########################################################################## # Tagged metrics (carbonapi_v2_pb) [[test.render_checks]] formats = [ "protobuf", "carbonapi_v2_pb" ] from = "rnow-10" until = "rnow+1" targets = [ "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')", "seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')" ] [[test.render_checks.result]] name = "metric1;tag1=value1;tag2=value21;tag3=value3" start = "rnow-10" stop = "rnow+10" step = 10 values = [2.0, 3.0] [[test.render_checks.result]] name = "metric1;tag1=value1;tag2=value23;tag3=value3" start = "rnow-10" stop = "rnow+10" step = 10 values = [4.0, 3.0] [[test.render_checks.result]] name = "metric2;tag2=value21;tag4=value4" start = "rnow-10" stop = "rnow+10" step = 10 values = [0.0, 1.0] # End - Tagged metrics (carbonapi_v2_pb) ########################################################################## # Tagged metrics (pickle) [[test.render_checks]] formats = [ "pickle" ] from = "rnow-10" until = "rnow+1" targets = [ "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')", "seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')" ] [[test.render_checks.result]] name = "metric1;tag1=value1;tag2=value21;tag3=value3" path = "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')" start = "rnow-10" stop = "rnow+10" step = 10 values = [2.0, 3.0] [[test.render_checks.result]] name = "metric1;tag1=value1;tag2=value23;tag3=value3" path = "seriesByTag('name=metric1', 'tag2=~value', 'tag3=value*')" start = "rnow-10" stop = "rnow+10" step = 10 values = [4.0, 3.0] [[test.render_checks.result]] name = "metric2;tag2=value21;tag4=value4" path = "seriesByTag('name=metric2', 'tag2=~value', 'tag4=value4')" start = "rnow-10" stop = "rnow+10" step = 10 values = [0.0, 1.0] # End - Tagged metrics (pickle) ########################################################################## # Unescape [[test.render_checks]] formats = [ "protobuf", "carbonapi_v2_pb" ] from = "rnow-10" until = "rnow+1" targets = [ "seriesByTag('percent=%')", ] [[test.render_checks.result]] name = "test_metric;colon=:;forward=/;hash=#;host=127.0.0.1;minus=-;percent=%;plus=+;underscore=_" start = "rnow-10" stop = "rnow+10" step = 10 values = [0.2, 1.5] # End - Tagged metrics (pickle) ########################################################################## # Midnight # points for check https://github.com/go-graphite/graphite-clickhouse/issues/184 [[test.input]] name = "test.midnight" points = [{value = 3.0, time = "midnight+60s"}] [[test.input]] name = "now;scope=midnight" points = [{value = 4.0, time = "midnight+60s"}] [[test.find_checks]] name = "Midnight (direct)" query = "test.midnight*" result = [{ path = "test.midnight", is_leaf = true }] [[test.find_checks]] name = "Midnight" query = "test.midnight" from = "midnight+60s" until = "midnight+70s" result = [{ path = "test.midnight", is_leaf = true }] [[test.find_checks]] name = "Midnight (reverse)" query = "*test.midnight" result = [{ path = "test.midnight", is_leaf = true }] [[test.find_checks]] name = "Midnight" query = "test.midnight" from = "midnight+60s" until = "midnight+70s" result = [{ path = "test.midnight", is_leaf = true }] [[test.tags_checks]] name = "Midnight" query = "name;scope=midnight" result = [ "now", ] [[test.render_checks]] name = "Midnight (direct)" formats = [ "protobuf" ] from = "midnight+60s" until = "midnight+70s" targets = [ "test.midnight*", ] [[test.render_checks.result]] name = "test.midnight" start = "midnight+60s" stop = "midnight+80s" step = 10 values = [3.0, nan] [[test.render_checks]] name = "Midnight (reverse)" formats = [ "protobuf" ] from = "midnight+60s" until = "midnight+70s" targets = [ "*test.midnight", ] [[test.render_checks.result]] name = "test.midnight" start = "midnight+60s" stop = "midnight+80s" step = 10 values = [3.0, nan] [[test.render_checks]] name = "Midnight" formats = [ "protobuf" ] from = "midnight+60s" until = "midnight+70s" targets = [ "seriesByTag('name=now', 'scope=midnight')", ] [[test.render_checks.result]] name = "now;scope=midnight" start = "midnight+60s" stop = "midnight+80s" step = 10 values = [4.0, nan] # End - Midnight ########################################################################## # Day end # points for check https://github.com/go-graphite/graphite-clickhouse/issues/184 [[test.input]] name = "test.23h" points = [{value = 3.0, time = "midnight+1380m"}] [[test.input]] name = "now;scope=23h" points = [{value = 4.0, time = "midnight+1380m"}] [[test.find_checks]] name = "Day end" query = "test.23h" from = "midnight+1380m" until = "midnight+1381m" result = [{ path = "test.23h", is_leaf = true }] [[test.find_checks]] name = "Day end" query = "test.23h" from = "midnight+1380m" until = "midnight+1381m" result = [{ path = "test.23h", is_leaf = true }] [[test.tags_checks]] name = "Day end" query = "name;scope=23h" result = [ "now", ] [[test.render_checks]] name = "Day end" formats = [ "protobuf" ] from = "midnight+1380m" until = "midnight+1380m+10s" targets = [ "test.23h", ] [[test.render_checks.result]] name = "test.23h" start = "midnight+1380m" stop = "midnight+1380m+20s" step = 10 values = [3.0, nan] [[test.render_checks]] name = "Day end" formats = [ "protobuf" ] from = "midnight+1380m" until = "midnight+1380m+10s" targets = [ "seriesByTag('name=now', 'scope=23h')", ] [[test.render_checks.result]] name = "now;scope=23h" start = "midnight+1380m" stop = "midnight+1380m+20s" step = 10 values = [4.0, nan] # End - Day end ########################################################################## ================================================ FILE: tests/wildcard_min_distance/carbon-clickhouse.conf.tpl ================================================ [common] [data] path = "/etc/carbon-clickhouse/data" chunk-interval = "1s" chunk-auto-interval = "" [upload.graphite_index] type = "index" table = "graphite_index" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_tags] type = "tagged" table = "graphite_tags" threads = 3 url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" cache-ttl = "1h" [upload.graphite_reverse] type = "points-reverse" table = "graphite_reverse" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [upload.graphite] type = "points" table = "graphite" url = "{{ .CLICKHOUSE_URL }}/" timeout = "2m30s" zero-timestamp = false [tcp] listen = ":2003" enabled = true drop-future = "0s" drop-past = "0s" [logging] file = "/etc/carbon-clickhouse/carbon-clickhouse.log" level = "debug" ================================================ FILE: tests/wildcard_min_distance/graphite-clickhouse.conf.tpl ================================================ [common] listen = "{{ .GCH_ADDR }}" max-cpu = 0 max-metrics-in-render-answer = 10000 max-metrics-per-target = 10000 headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ] [clickhouse] url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1" data-timeout = "30s" wildcard-min-distance = 1 index-table = "graphite_index" index-use-daily = true index-timeout = "1m" internal-aggregation = true tagged-table = "graphite_tags" tagged-autocomplete-days = 1 [[data-table]] # # clickhouse table name table = "graphite" # # points in table are stored with reverse path reverse = false rollup-conf = "auto" [[logging]] logger = "" file = "{{ .GCH_DIR }}/graphite-clickhouse.log" level = "info" encoding = "json" encoding-time = "iso8601" encoding-duration = "seconds" ================================================ FILE: tests/wildcard_min_distance/test.toml ================================================ [test] precision = "10s" [[test.clickhouse]] version = "21.3" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "22.8" dir = "tests/clickhouse/rollup" [[test.clickhouse]] version = "24.2" dir = "tests/clickhouse/rollup" [test.carbon_clickhouse] template = "carbon-clickhouse.conf.tpl" [[test.graphite_clickhouse]] template = "graphite-clickhouse.conf.tpl" [[test.input]] name = "team_one.prod.test.metric_one" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "team_two.stage.test.metric_one" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "team_one.dev.test.metric_two" points = [{value = 1.0, time = "rnow-10"}] [[test.input]] name = "team_one.dev.nontest.metric_one" points = [{value = 1.0, time = "rnow-10"}] [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "team_one.prod.test.metric_one", ] [[test.render_checks.result]] name = "team_one.prod.test.metric_one" path = "team_one.prod.test.metric_one" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "*.dev.test.metric_two", ] [[test.render_checks.result]] name = "team_one.dev.test.metric_two" path = "*.dev.test.metric_two" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "*.*.test.metric_one", ] [[test.render_checks.result]] name = "team_one.prod.test.metric_one" path = "*.*.test.metric_one" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "team_two.stage.test.metric_one" path = "*.*.test.metric_one" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "team_two.stage.test.*", ] [[test.render_checks.result]] name = "team_two.stage.test.metric_one" path = "team_two.stage.test.*" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "team_one.*.test.*", ] [[test.render_checks.result]] name = "team_one.prod.test.metric_one" path = "team_one.*.test.*" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks.result]] name = "team_one.dev.test.metric_two" path = "team_one.*.test.*" consolidation = "avg" start = "rnow-10" stop = "rnow+10" step = 10 req_start = "rnow-10" req_stop = "rnow+10" values = [1.0, nan] [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "*.prod.test.*", ] error_regexp = "^400: query has wildcards way too early at the start and at the end of it" [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "*.*.test.*", ] error_regexp = "^400: query has wildcards way too early at the start and at the end of it" [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "*.*.*.*", ] error_regexp = "^400: query has wildcards way too early at the start and at the end of it" [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "*.*", ] error_regexp = "^400: query has wildcards way too early at the start and at the end of it" [[test.render_checks]] from = "rnow-10" until = "rnow+1" timeout = "1h" targets = [ "*", ]