Showing preview only (1,442K chars total). Download the full file or copy to clipboard to get everything.
Repository: lomik/graphite-clickhouse
Branch: master
Commit: b816c69a5c9a
Files: 293
Total size: 1.3 MB
Directory structure:
gitextract_n919tixz/
├── .gitattributes
├── .github/
│ └── workflows/
│ ├── codeql.yml
│ ├── docker.yml
│ ├── lint.yml
│ ├── release.yml
│ ├── tests-sd.yml
│ └── tests.yml
├── .gitignore
├── .golangci.yml
├── Dockerfile
├── LICENSE
├── Makefile
├── README.md
├── autocomplete/
│ ├── autocomplete.go
│ └── autocomplete_test.go
├── cache/
│ └── cache.go
├── capabilities/
│ └── handler.go
├── cmd/
│ ├── e2e-test/
│ │ ├── carbon-clickhouse.go
│ │ ├── checks.go
│ │ ├── clickhouse.go
│ │ ├── container.go
│ │ ├── e2etesting.go
│ │ ├── errors.go
│ │ ├── graphite-clickhouse.go
│ │ ├── main.go
│ │ ├── rproxy.go
│ │ └── utils.go
│ └── graphite-clickhouse-client/
│ └── main.go
├── config/
│ ├── .gitignore
│ ├── config.go
│ ├── config_test.go
│ ├── json.go
│ └── json_test.go
├── deploy/
│ ├── doc/
│ │ ├── .gitignore
│ │ └── config.md
│ └── root/
│ └── usr/
│ └── lib/
│ └── systemd/
│ └── system/
│ └── graphite-clickhouse.service
├── doc/
│ ├── aggregation.md
│ ├── config.md
│ ├── debugging.md
│ ├── graphite_clickhouse.gliffy
│ ├── index-table.md
│ └── release.md
├── find/
│ ├── find.go
│ ├── handler.go
│ ├── handler_json_test.go
│ └── handler_test.go
├── finder/
│ ├── base.go
│ ├── blacklist.go
│ ├── date.go
│ ├── date_reverse.go
│ ├── date_reverse_test.go
│ ├── finder.go
│ ├── index.go
│ ├── index_test.go
│ ├── mock.go
│ ├── plain_from_tagged.go
│ ├── plain_from_tagged_test.go
│ ├── prefix.go
│ ├── prefix_test.go
│ ├── reverse.go
│ ├── reverse_test.go
│ ├── split.go
│ ├── split_test.go
│ ├── tag.go
│ ├── tag_test.go
│ ├── tagged.go
│ ├── tagged_test.go
│ ├── tags_count_querier.go
│ └── unescape.go
├── go.mod
├── go.sum
├── graphite-clickhouse.go
├── healthcheck/
│ └── healthcheck.go
├── helper/
│ ├── RowBinary/
│ │ └── encode.go
│ ├── clickhouse/
│ │ ├── clickhouse.go
│ │ ├── clickhouse_test.go
│ │ ├── external-data.go
│ │ └── external-data_test.go
│ ├── client/
│ │ ├── datetime.go
│ │ ├── errros.go
│ │ ├── find.go
│ │ ├── render.go
│ │ ├── requests.go
│ │ ├── tags.go
│ │ └── types.go
│ ├── date/
│ │ ├── date.go
│ │ └── date_test.go
│ ├── datetime/
│ │ ├── datetime.go
│ │ └── datetime_test.go
│ ├── errs/
│ │ └── errors.go
│ ├── headers/
│ │ └── headers.go
│ ├── http/
│ │ └── live-http-client.go
│ ├── pickle/
│ │ └── pickle.go
│ ├── point/
│ │ ├── func.go
│ │ ├── func_test.go
│ │ ├── point.go
│ │ └── points.go
│ ├── rollup/
│ │ ├── aggr.go
│ │ ├── compact.go
│ │ ├── compact_test.go
│ │ ├── remote.go
│ │ ├── remote_test.go
│ │ ├── rollup.go
│ │ ├── rules.go
│ │ ├── rules_test.go
│ │ ├── xml.go
│ │ └── xml_test.go
│ ├── tests/
│ │ ├── clickhouse/
│ │ │ └── server.go
│ │ └── compare/
│ │ ├── compare.go
│ │ └── expand/
│ │ └── expand.go
│ └── utils/
│ ├── utils.go
│ └── utils_test.go
├── index/
│ ├── handler.go
│ ├── index.go
│ └── index_test.go
├── issues/
│ └── daytime/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse-internal-aggr.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── limiter/
│ ├── alimiter.go
│ ├── alimiter_test.go
│ ├── interface.go
│ ├── limiter.go
│ ├── noop.go
│ └── wlimiter.go
├── load_avg/
│ ├── load_avg.go
│ ├── load_avg_default.go
│ ├── load_avg_linux.go
│ └── load_avg_test.go
├── logs/
│ └── logger.go
├── metrics/
│ ├── limiter_metrics.go
│ ├── metrics.go
│ ├── metrics_test.go
│ ├── query_metrics.go
│ └── statsd.go
├── nfpm.yaml
├── packages.sh
├── pkg/
│ ├── alias/
│ │ ├── map.go
│ │ ├── map_tagged_test.go
│ │ └── map_test.go
│ ├── dry/
│ │ ├── math.go
│ │ ├── math_test.go
│ │ ├── strings.go
│ │ ├── strings_test.go
│ │ ├── unsafe.go
│ │ └── unsafe_test.go
│ ├── reverse/
│ │ ├── reverse.go
│ │ └── reverse_test.go
│ ├── scope/
│ │ ├── context.go
│ │ ├── http_request.go
│ │ ├── key.go
│ │ ├── logger.go
│ │ └── version.go
│ └── where/
│ ├── match.go
│ ├── match_test.go
│ ├── where.go
│ └── where_test.go
├── prometheus/
│ ├── .gitignore
│ ├── empty_iterator.go
│ ├── exemplar.go
│ ├── gatherer.go
│ ├── labels.go
│ ├── labels_test.go
│ ├── local_storage.go
│ ├── logger.go
│ ├── matcher.go
│ ├── metrics_set.go
│ ├── querier.go
│ ├── querier_select.go
│ ├── querier_select_test.go
│ ├── run.go
│ ├── run_dummy.go
│ ├── series_set.go
│ └── storage.go
├── render/
│ ├── data/
│ │ ├── carbonlink.go
│ │ ├── carbonlink_test.go
│ │ ├── ch_response.go
│ │ ├── common_step.go
│ │ ├── common_step_test.go
│ │ ├── data.go
│ │ ├── data_parse_test.go
│ │ ├── multi_target.go
│ │ ├── multi_target_test.go
│ │ ├── query.go
│ │ ├── query_test.go
│ │ ├── targets.go
│ │ └── targets_test.go
│ ├── handler.go
│ ├── handler_test.go
│ └── reply/
│ ├── formatter.go
│ ├── formatter_test.go
│ ├── json.go
│ ├── pickle.go
│ ├── protobuf.go
│ ├── protobuf_test.go
│ ├── v2_pb.go
│ ├── v2_pb_test.go
│ ├── v3_pb.go
│ └── v3_pb_test.go
├── sd/
│ ├── nginx/
│ │ ├── nginx.go
│ │ ├── nginx_test.go
│ │ └── tests/
│ │ └── nginx_cleanup_test.go
│ ├── register.go
│ └── utils/
│ └── utils.go
├── tagger/
│ ├── metric.go
│ ├── rule.go
│ ├── rule_test.go
│ ├── set.go
│ ├── tagger.go
│ ├── tagger_test.go
│ └── tree.go
└── tests/
├── agg_internal/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse-internal-aggr.conf.tpl
│ └── test.toml
├── agg_latest/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── agg_merge/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse-internal-aggr.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── agg_oneblock/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse-internal-aggr.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── clickhouse/
│ ├── rollup/
│ │ ├── config.xml
│ │ ├── init.sql
│ │ ├── rollup.xml
│ │ └── users.xml
│ └── rollup_tls/
│ ├── config.xml
│ ├── init.sql
│ ├── rollup.xml
│ ├── rootCA.crt
│ ├── server.crt
│ ├── server.key
│ └── users.xml
├── consolidateBy/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── consul.sh
├── emptyseries_append/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── emptyseries_noappend/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── error_handling/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── feature_flags_both_true/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── feature_flags_dont_match_missing_tags/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── feature_flags_false/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── feature_flags_use_carbon_behaviour/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── find_cache/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse-cached.conf.tpl
│ ├── graphite-clickhouse-internal-aggr-cached.conf.tpl
│ └── test.toml
├── limitera/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse-internal-aggr-cached.conf.tpl
│ └── test.toml
├── limitermax/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse-internal-aggr-cached.conf.tpl
│ └── test.toml
├── limiterw/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse-internal-aggr-cached.conf.tpl
│ └── test.toml
├── limiterwn/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse-internal-aggr-cached.conf.tpl
│ └── test.toml
├── one_table/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse-internal-aggr.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── tags_min_in_query/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── tls/
│ ├── ca.crt
│ ├── carbon-clickhouse.conf.tpl
│ ├── client.crt
│ ├── client.key
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
└── wildcard_min_distance/
├── carbon-clickhouse.conf.tpl
├── graphite-clickhouse.conf.tpl
└── test.toml
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitattributes
================================================
================================================
FILE: .github/workflows/codeql.yml
================================================
# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
name: "CodeQL"
on:
push:
branches: [master]
pull_request:
# The branches below must be a subset of the branches above
branches: [master]
jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
# Override automatic language detection by changing the below list
# Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
language: ['go']
# Learn more...
# https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
# We must fetch at least the immediate parents so that if this is
# a pull request then we can checkout the head.
fetch-depth: 2
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
# queries: ./path/to/local/query, your-org/your-repo/queries@main
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v3
# ℹ️ Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
# and modify them (or add more) to build your code if your project
# uses a compiled language
#- run: |
# make bootstrap
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
================================================
FILE: .github/workflows/docker.yml
================================================
name: Docker images
on:
push:
branches: [ master ]
tags: [ 'v*' ]
pull_request:
branches: [ master ]
jobs:
docker:
name: Build image
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Docker meta
id: meta
uses: docker/metadata-action@v3
with:
images: ghcr.io/${{ github.repository }}
# create latest tag for branch events
flavor: |
latest=${{ github.event_name == 'push' && github.ref_type == 'branch' }}
tags: |
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}.{{minor}}.{{patch}}
- name: Login to DockerHub
if: github.event_name != 'pull_request'
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push
id: docker_build
uses: docker/build-push-action@v2
with:
# push for non-pr events
push: ${{ github.event_name != 'pull_request' }}
context: .
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
================================================
FILE: .github/workflows/lint.yml
================================================
name: Lint
on:
pull_request:
jobs:
golangci:
name: lint
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v4
with:
go-version-file: go.mod
- name: Run linter
uses: golangci/golangci-lint-action@v7
with:
version: v2.0.2
================================================
FILE: .github/workflows/release.yml
================================================
name: Upload Packages to new release
on:
release:
types:
- published
jobs:
build:
name: Build
runs-on: ubuntu-latest
env:
BINARY: ${{ github.event.repository.name }}
CGO_ENABLED: 0
outputs:
matrix: ${{ steps.build.outputs.matrix }}
steps:
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: ^1
- uses: actions/checkout@v4
name: Checkout
- name: Test
run: make test
env:
CGO_ENABLED: 1
- name: Build packages
id: build
run: |
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.40.0
make nfpm-deb nfpm-rpm
make sum-files
ARTIFACTS=
# Upload all deb and rpm packages
for package in *deb *rpm; do ARTIFACTS=${ARTIFACTS}\"$package\",\ ; done
echo ::set-output name=matrix::{\"file\": [${ARTIFACTS} \"sha256sum\", \"md5sum\"]}
- name: Check version
id: check_version
run: |
./out/${BINARY}-linux-amd64 -version
[ v$(./out/${BINARY}-linux-amd64 -version) = ${{ github.event.release.tag_name }} ]
- name: Artifact
id: artifact
uses: actions/upload-artifact@v4
with:
name: packages
retention-days: 1
path: |
*.deb
*.rpm
sha256sum
md5sum
- name: Push packages to the stable repo
run: make packagecloud-stable
env:
PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
upload:
needs: build
runs-on: ubuntu-latest
strategy:
matrix: ${{fromJson(needs.build.outputs.matrix)}}
steps:
- name: Download artifact
uses: actions/download-artifact@v4.1.7
with:
name: packages
- name: Upload ${{ matrix.file }}
id: upload
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ github.event.release.upload_url }}
asset_path: ${{ matrix.file }}
asset_name: ${{ matrix.file }}
asset_content_type: application/octet-stream
================================================
FILE: .github/workflows/tests-sd.yml
================================================
name: Tests register in SD
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
tests:
env:
CGO_ENABLED: 0
name: Test register in SD
runs-on: ubuntu-latest
strategy:
matrix:
go:
- ^1
steps:
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go }}
- name: Check out code
uses: actions/checkout@v4
- name: Start consul
run: |
./tests/consul.sh 1.15.2 > /tmp/consul.log &
sleep 30
shell: bash
- name: Test
run: go test ./sd/nginx -tags=test_sd -v
================================================
FILE: .github/workflows/tests.yml
================================================
name: Tests
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
tests:
env:
CGO_ENABLED: 0
name: Test code
runs-on: ubuntu-latest
strategy:
matrix:
go:
- ^1.20
- ^1.21
- ^1
steps:
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go }}
- name: Check out code
uses: actions/checkout@v4
- name: Checkout to the latest tag
run: |
# Fetch all tags
git fetch --depth=1 --tags
# Get the latest tag
VERS=$(git tag -l | sort -Vr | head -n1)
# Fetch everything to the latest tag
git fetch --shallow-since=$(git log $VERS -1 --format=%at)
if: ${{ github.event_name == 'push' }} # only when built from master
- name: Build project
run: make
- name: Validate default configs
run: |
./graphite-clickhouse -config-print-default > /tmp/graphite-clickhouse.conf
./graphite-clickhouse -config /tmp/graphite-clickhouse.conf -check-config
- name: Check documentation consistency
run: |
make config
git diff --exit-code
- name: Test
run: make test
env:
CGO_ENABLED: 1
- name: Test (with GMT-5)
run: |
go clean -testcache
TZ=Etc/GMT-5 make test
env:
CGO_ENABLED: 1
- name: Test (with GMT+5)
run: |
go clean -testcache
TZ=Etc/GMT+5 make test
env:
CGO_ENABLED: 1
- name: Integration tests
run: |
make e2e-test
./e2e-test -config tests -abort -rmi
# TODO (msaf1980): find a way to set TZ in carbon-clickhouse docker (or run locally)
# run with clickhouse.date-format = "both"
# - name: Integration tests (with Etc/GMT-5)
# run: |
# make e2e-test
# sudo timedatectl set-timezone Etc/GMT-5
# ./e2e-test -config issues/daytime
# - name: Integration tests (with Etc/GMT+5)
# run: |
# make e2e-test
# sudo timedatectl set-timezone Etc/GMT+5
# ./e2e-test -config issues/daytime
- name: Check packaging
run: |
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.40.0
make DEVEL=1 nfpm-deb nfpm-rpm
make sum-files
- name: Artifact
id: artifact
uses: actions/upload-artifact@v4
with:
name: packages-${{ matrix.go }}
path: |
*.deb
*.rpm
sha256sum
md5sum
- name: Push packages to the autobuilds repo
if: ${{ github.event_name == 'push' && matrix.go == '^1' }} # only when built from master with latest go
run: make DEVEL=1 packagecloud-autobuilds
env:
PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
================================================
FILE: .gitignore
================================================
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
graphite-clickhouse
.vscode
/out/
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof
================================================
FILE: .golangci.yml
================================================
version: "2"
linters:
default: none
enable:
- asasalint
- asciicheck
- bidichk
- bodyclose
# - contextcheck
- decorder
# - dogsled
- durationcheck
# - errcheck
# - errorlint
# - fatcontext
- ginkgolinter
- gocheckcompilerdirectives
- gochecksumtype
# - goconst
# - gocyclo
# - godot
- goheader
- govet
- grouper
# - ineffassign
- loggercheck
# - makezero
# - misspell
# - mnd
# - nilerr
# - noctx
# - nosprintfhostport
- prealloc
# - predeclared
- promlinter
- protogetter
- reassign
# - revive
- rowserrcheck
- sloglint
- spancheck
- sqlclosecheck
# - staticcheck
# - testifylint
- tparallel
- unconvert
# - unparam
# - unused
- usestdlibvars
# - wastedassign
- whitespace
- wsl
settings:
gocyclo:
min-complexity: 15
govet:
settings:
printf:
funcs:
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
- (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
unparam:
check-exported: false
exclusions:
generated: lax
presets:
- comments
- common-false-positives
- legacy
- std-error-handling
rules:
- linters:
- errcheck
- contextcheck
- goconst
- mnd
path: _test\.go
- linters:
- godot
path: notifier/registrator.go
paths:
- third_party$
- builtin$
- examples$
formatters:
enable:
- gofmt
# - gofumpt
- goimports
exclusions:
generated: lax
paths:
- third_party$
- builtin$
- examples$
================================================
FILE: Dockerfile
================================================
FROM golang:alpine as builder
WORKDIR /go/src/github.com/lomik/graphite-clickhouse
COPY . .
ENV GOPATH=/go
RUN apk add git --no-cache
RUN go build -ldflags '-extldflags "-static"' github.com/lomik/graphite-clickhouse
FROM alpine:latest
RUN apk --no-cache add ca-certificates
WORKDIR /
COPY --from=builder /go/src/github.com/lomik/graphite-clickhouse/graphite-clickhouse /usr/bin/graphite-clickhouse
CMD ["graphite-clickhouse"]
================================================
FILE: LICENSE
================================================
MIT License
Copyright (c) 2016 Roman Lomonosov
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: Makefile
================================================
NAME:=graphite-clickhouse
DESCRIPTION:="Graphite cluster backend with ClickHouse support"
MODULE:=github.com/lomik/graphite-clickhouse
GO ?= go
export GO111MODULE := on
TEMPDIR:=$(shell mktemp -d)
DEVEL ?= 0
ifeq ($(DEVEL), 0)
VERSION:=$(shell sh -c 'grep "const Version" $(NAME).go | cut -d\" -f2')
else
VERSION:=$(shell sh -c 'git describe --always --tags | sed -e "s/^v//i"')
endif
SRCS:=$(shell find . -name '*.go')
all: $(NAME)
.PHONY: clean
clean:
rm -f $(NAME) $(NAME)-client
rm -rf out
rm -f *deb *rpm
rm -f sha256sum md5sum
$(NAME): $(SRCS)
$(GO) build -ldflags '-X main.BuildVersion=$(VERSION)' $(MODULE)
debug: $(SRCS)
$(GO) build -ldflags '-X main.BuildVersion=$(VERSION)' -gcflags=all='-N -l' $(MODULE)
deploy/doc/graphite-clickhouse.conf: $(NAME)
./$(NAME) -config-print-default > $@
doc/config.md: deploy/doc/graphite-clickhouse.conf deploy/doc/config.md
@echo 'Generating $@...'
@printf '[//]: # (This file is built out of deploy/doc/config.md, please do not edit it manually) \n' > $@
@printf '[//]: # (To rebuild it run `make config`)\n\n' >> $@
@cat deploy/doc/config.md >> $@
@printf '\n```toml\n' >> $@
@cat deploy/doc/graphite-clickhouse.conf >> $@
@printf '```\n' >> $@
config: doc/config.md
# run after prometheus upgrade
prometheus/ui:
vendor_prometheus_ui.sh
test:
$(GO) test -race ./...
e2e-test: $(NAME)
$(GO) build $(MODULE)/cmd/e2e-test
client: $(NAME)
$(GO) build $(MODULE)/cmd/graphite-clickhouse-client
gox-build: out/$(NAME)-linux-amd64 out/$(NAME)-linux-arm64 out/root/etc/$(NAME)/$(NAME).conf
ARCH = amd64 arm64
out/$(NAME)-linux-%: out $(SRCS)
GOOS=linux GOARCH=$* $(GO) build -ldflags '-X main.BuildVersion=$(VERSION)' -o $@ $(MODULE)
out:
mkdir -p out
out/root/etc/$(NAME)/$(NAME).conf: $(NAME)
mkdir -p "$(shell dirname $@)"
./$(NAME) -config-print-default > $@
nfpm-deb: gox-build
$(MAKE) nfpm-build-deb ARCH=amd64
$(MAKE) nfpm-build-deb ARCH=arm64
nfpm-rpm: gox-build
$(MAKE) nfpm-build-rpm ARCH=amd64
$(MAKE) nfpm-build-rpm ARCH=arm64
nfpm-build-%: nfpm.yaml
NAME=$(NAME) DESCRIPTION=$(DESCRIPTION) ARCH=$(ARCH) VERSION_STRING=$(VERSION) nfpm package --packager $*
.ONESHELL:
RPM_VERSION:=$(subst -,_,$(VERSION))
packagecloud-push-rpm: $(wildcard $(NAME)-$(RPM_VERSION)-1.*.rpm)
for pkg in $^; do
package_cloud push $(REPO)/el/7 $${pkg} || true
package_cloud push $(REPO)/el/8 $${pkg} || true
package_cloud push $(REPO)/el/9 $${pkg} || true
done
.ONESHELL:
packagecloud-push-deb: $(wildcard $(NAME)_$(VERSION)_*.deb)
for pkg in $^; do
package_cloud push $(REPO)/ubuntu/xenial $${pkg} || true
package_cloud push $(REPO)/ubuntu/bionic $${pkg} || true
package_cloud push $(REPO)/ubuntu/focal $${pkg} || true
package_cloud push $(REPO)/debian/stretch $${pkg} || true
package_cloud push $(REPO)/debian/buster $${pkg} || true
package_cloud push $(REPO)/debian/bullseye $${pkg} || true
done
packagecloud-push:
@$(MAKE) packagecloud-push-rpm
@$(MAKE) packagecloud-push-deb
packagecloud-autobuilds:
$(MAKE) packagecloud-push REPO=go-graphite/autobuilds
packagecloud-stable:
$(MAKE) packagecloud-push REPO=go-graphite/stable
sum-files: | sha256sum md5sum
md5sum:
md5sum $(wildcard $(NAME)_$(VERSION)*.deb) $(wildcard $(NAME)-$(VERSION)*.rpm) > md5sum
sha256sum:
sha256sum $(wildcard $(NAME)_$(VERSION)*.deb) $(wildcard $(NAME)-$(VERSION)*.rpm) > sha256sum
.PHONY: lint
lint:
golangci-lint run
================================================
FILE: README.md
================================================
[](https://packagecloud.io/go-graphite/stable)
[](https://packagecloud.io/go-graphite/stable)
# graphite-clickhouse
Graphite cluster backend with ClickHouse support
## Work scheme
Gray components are optional or alternative
## TL;DR
[Preconfigured docker-compose](https://github.com/lomik/graphite-clickhouse-tldr)
### Docker
Docker images are available on [packages](https://github.com/lomik/graphite-clickhouse/pkgs/container/graphite-clickhouse) page.
## Compatibility
- [x] [graphite-web 1.1.0](https://github.com/graphite-project/graphite-web)
- [x] [graphite-web 0.9.15](https://github.com/graphite-project/graphite-web/tree/0.9.15)
- [x] [graphite-web 1.0.0](https://github.com/graphite-project/graphite-web)
- [x] [carbonapi 0.14.1+](https://github.com/go-graphite/carbonapi)
- [x] [carbonzipper](https://github.com/go-graphite/carbonzipper) (DEPRECATED, is part of carbonapi currently)
## Build
Required golang 1.18+
```sh
# build binary
git clone https://github.com/lomik/graphite-clickhouse.git
cd graphite-clickhouse
make
```
## Installation
1. Setup [Yandex ClickHouse](https://github.com/yandex/ClickHouse) and [carbon-clickhouse](https://github.com/lomik/carbon-clickhouse)
2. Setup and configure `graphite-clickhouse`
3. Add graphite-clickhouse `host:port` to graphite-web [CLUSTER_SERVERS](http://graphite.readthedocs.io/en/latest/config-local-settings.html#cluster-configuration)
## Configuration
See [configuration documentation](./doc/config.md).
### Special headers processing
Some HTTP headers are processed specially by the service
#### Request headers
*Grafana headers*: `X-Dashboard-Id`, `X-Grafana-Org-Id`, and `X-Panel-Id` are logged and passed further to the ClickHouse.
*Debug headers* (see [debugging.md](./doc/debugging.md) for details):
- `X-Gch-Debug-External-Data` - when this header is set to anything and every of `directory`, `directory-perm`, and `external-data-perm` parameters in `[debug]` is set and valid, service will save the dump of external data tables in the directory for debug output.
- `X-Gch-Debug-Output` - header to enable special processing for `format=carbonapi_v3_pb` and `format=json` render output.
- `X-Gch-Debug-Protobuf` - header enables the original marshallers for `protobuf` and `carbonapi_v3_pb` to check the binary data integrity.
#### Response headers
- `X-Gch-Request-Id` - the current request ID.
- `X-Cached-Find` - Flag for find cache hit.
## Run on same host with old graphite-web 0.9.x
By default graphite-web won't connect to CLUSTER_SERVER on localhost. Cheat:
```python
class ForceLocal(str):
def split(self, *args, **kwargs):
return ["8.8.8.8", "8080"]
CLUSTER_SERVERS = [ForceLocal("127.0.0.1:9090")]
```
================================================
FILE: autocomplete/autocomplete.go
================================================
package autocomplete
import (
"context"
"encoding/json"
"fmt"
"net/http"
"sort"
"strconv"
"strings"
"time"
"github.com/go-graphite/carbonapi/pkg/parser"
"github.com/msaf1980/go-stringutils"
"go.uber.org/zap"
"github.com/lomik/graphite-clickhouse/config"
"github.com/lomik/graphite-clickhouse/finder"
"github.com/lomik/graphite-clickhouse/helper/clickhouse"
"github.com/lomik/graphite-clickhouse/helper/date"
"github.com/lomik/graphite-clickhouse/helper/utils"
"github.com/lomik/graphite-clickhouse/logs"
"github.com/lomik/graphite-clickhouse/metrics"
"github.com/lomik/graphite-clickhouse/pkg/scope"
"github.com/lomik/graphite-clickhouse/pkg/where"
)
// override in unit tests for stable results
var timeNow = time.Now
type Handler struct {
config *config.Config
isValues bool
}
func NewTags(config *config.Config) *Handler {
h := &Handler{
config: config,
}
return h
}
func NewValues(config *config.Config) *Handler {
h := &Handler{
config: config,
isValues: true,
}
return h
}
func dateString(autocompleteDays int, tm time.Time) (string, string) {
fromDate := date.FromTimeToDaysFormat(tm.AddDate(0, 0, -autocompleteDays))
untilDate := date.UntilTimeToDaysFormat(tm)
return fromDate, untilDate
}
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Don't process, if the tagged table is not set
if h.config.ClickHouse.TaggedTable == "" {
w.Write([]byte{'[', ']'})
return
}
if h.isValues {
h.ServeValues(w, r)
} else {
h.ServeTags(w, r)
}
}
func getTagCountQuerier(config *config.Config, opts clickhouse.Options) *finder.TagCountQuerier {
var tcq *finder.TagCountQuerier = nil
if config.ClickHouse.TagsCountTable != "" {
tcq = finder.NewTagCountQuerier(
config.ClickHouse.URL,
config.ClickHouse.TagsCountTable,
opts,
config.FeatureFlags.UseCarbonBehavior,
config.FeatureFlags.DontMatchMissingTags,
config.ClickHouse.TaggedUseDaily,
)
}
return tcq
}
func (h *Handler) requestExpr(r *http.Request, tcq *finder.TagCountQuerier, from, until int64) (*where.Where, *where.Where, map[string]bool, error) {
formExpr := r.Form["expr"]
expr := make([]string, 0, len(formExpr))
for i := 0; i < len(formExpr); i++ {
if formExpr[i] != "" {
expr = append(expr, formExpr[i])
}
}
usedTags := make(map[string]bool)
wr := where.New()
pw := where.New()
if len(expr) == 0 {
return wr, pw, usedTags, nil
}
terms, err := finder.ParseTaggedConditions(expr, h.config, true)
if err != nil {
return wr, pw, usedTags, err
}
if tcq != nil {
tagValuesCosts, err := tcq.GetCostsFromCountTable(r.Context(), terms, from, until)
if err != nil {
return wr, pw, usedTags, err
}
if tagValuesCosts != nil {
finder.SetCosts(terms, tagValuesCosts)
} else if len(h.config.ClickHouse.TaggedCosts) != 0 {
finder.SetCosts(terms, h.config.ClickHouse.TaggedCosts)
}
}
finder.SortTaggedTermsByCost(terms)
wr, pw, err = finder.TaggedWhere(terms, h.config.FeatureFlags.UseCarbonBehavior, h.config.FeatureFlags.DontMatchMissingTags)
if err != nil {
return wr, pw, usedTags, err
}
for i := 0; i < len(expr); i++ {
a := strings.Split(expr[i], "=")
usedTags[a[0]] = true
}
return wr, pw, usedTags, nil
}
func taggedKey(typ string, truncateSec int32, fromDate, untilDate string, tag string, exprs []string, tagPrefix string, limit int) (string, string) {
ts := utils.TimestampTruncate(timeNow().Unix(), time.Duration(truncateSec)*time.Second)
var sb stringutils.Builder
sb.Grow(128)
sb.WriteString(typ)
sb.WriteString(fromDate)
sb.WriteByte(';')
sb.WriteString(untilDate)
sb.WriteString(";limit=")
sb.WriteInt(int64(limit), 10)
tagStart := sb.Len()
if tagPrefix != "" {
sb.WriteString(";tagPrefix=")
sb.WriteString(tagPrefix)
}
if tag != "" {
sb.WriteString(";tag=")
sb.WriteString(tag)
}
for _, expr := range exprs {
sb.WriteString(";expr='")
sb.WriteString(strings.Replace(expr, " = ", "=", 1))
sb.WriteByte('\'')
}
exprEnd := sb.Len()
sb.WriteString(";ts=")
sb.WriteString(strconv.FormatInt(ts, 10))
s := sb.String()
return s, s[tagStart:exprEnd]
}
func taggedValuesKey(typ string, truncateSec int32, fromDate, untilDate string, tag string, exprs []string, valuePrefix string, limit int) (string, string) {
ts := utils.TimestampTruncate(timeNow().Unix(), time.Duration(truncateSec)*time.Second)
var sb stringutils.Builder
sb.Grow(128)
sb.WriteString(typ)
sb.WriteString(fromDate)
sb.WriteByte(';')
sb.WriteString(untilDate)
sb.WriteString(";limit=")
sb.WriteInt(int64(limit), 10)
tagStart := sb.Len()
if valuePrefix != "" {
sb.WriteString(";valuePrefix=")
sb.WriteString(valuePrefix)
}
if tag != "" {
sb.WriteString(";tag=")
sb.WriteString(tag)
}
for _, expr := range exprs {
sb.WriteString(";expr='")
sb.WriteString(strings.Replace(expr, " = ", "=", 1))
sb.WriteByte('\'')
}
exprEnd := sb.Len()
sb.WriteString(";ts=")
sb.WriteString(strconv.FormatInt(ts, 10))
s := sb.String()
return s, s[tagStart:exprEnd]
}
// func taggedTagsQuery(exprs []string, tagPrefix string, limit int) []string {
// query := make([]string, 0, 3+len(exprs))
// if tagPrefix != "" {
// query = append(query, "tagPrefix="+tagPrefix)
// }
// for _, expr := range exprs {
// query = append(query, "expr='"+expr+"'")
// }
// query = append(query, "limit="+strconv.Itoa(limit))
// return query
// }
func (h *Handler) ServeTags(w http.ResponseWriter, r *http.Request) {
start := timeNow()
status := http.StatusOK
accessLogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("http")
logger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("autocomplete")
r = r.WithContext(scope.WithLogger(r.Context(), logger))
var (
err error
chReadRows int64
chReadBytes int64
metricsCount int64
readBytes int64
queueFail bool
queueDuration time.Duration
findCache bool
opts clickhouse.Options
)
username := r.Header.Get("X-Forwarded-User")
limiter := h.config.GetUserTagsLimiter(username)
defer func() {
if rec := recover(); rec != nil {
status = http.StatusInternalServerError
logger.Error("panic during eval:",
zap.String("requestID", scope.String(r.Context(), "requestID")),
zap.Any("reason", rec),
zap.Stack("stack"),
)
answer := fmt.Sprintf("%v\nStack trace: %v", rec, zap.Stack("").String)
http.Error(w, answer, status)
}
d := time.Since(start)
dMS := d.Milliseconds()
logs.AccessLog(accessLogger, h.config, r, status, d, queueDuration, findCache, queueFail)
limiter.SendDuration(queueDuration.Milliseconds())
metrics.SendFindMetrics(metrics.TagsRequestMetric, status, dMS, 0, h.config.Metrics.ExtendedStat, metricsCount)
if !findCache && chReadRows > 0 && chReadBytes > 0 {
errored := status != http.StatusOK && status != http.StatusNotFound
metrics.SendQueryRead(metrics.AutocompleteQMetric, 0, 0, dMS, metricsCount, readBytes, chReadRows, chReadBytes, errored)
}
}()
r.ParseMultipartForm(1024 * 1024)
tagPrefix := r.FormValue("tagPrefix")
limitStr := r.FormValue("limit")
limit := 10000
var body []byte
if limitStr != "" {
limit, err = strconv.Atoi(limitStr)
if err == finder.ErrCostlySeriesByTag {
status = http.StatusForbidden
http.Error(w, err.Error(), status)
return
} else if err != nil {
status = http.StatusBadRequest
http.Error(w, err.Error(), status)
return
}
}
fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, start)
var key string
exprs := r.Form["expr"]
// params := taggedTagsQuery(exprs, tagPrefix, limit)
useCache := h.config.Common.FindCache != nil && h.config.Common.FindCacheConfig.FindTimeoutSec > 0 && !parser.TruthyBool(r.FormValue("noCache"))
if useCache {
key, _ = taggedKey("tags;", h.config.Common.FindCacheConfig.FindTimeoutSec, fromDate, untilDate, "", exprs, tagPrefix, limit)
body, err = h.config.Common.FindCache.Get(key)
if err == nil {
if metrics.FinderCacheMetrics != nil {
metrics.FinderCacheMetrics.CacheHits.Add(1)
}
findCache = true
w.Header().Set("X-Cached-Find", strconv.Itoa(int(h.config.Common.FindCacheConfig.FindTimeoutSec)))
}
}
opts = clickhouse.Options{
TLSConfig: h.config.ClickHouse.TLSConfig,
Timeout: h.config.ClickHouse.IndexTimeout,
ConnectTimeout: h.config.ClickHouse.ConnectTimeout,
CheckRequestProgress: h.config.FeatureFlags.LogQueryProgress,
ProgressSendingInterval: h.config.ClickHouse.ProgressSendingInterval,
}
wr, pw, usedTags, err := h.requestExpr(
r,
getTagCountQuerier(h.config, opts),
start.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Unix(),
start.Unix(),
)
if err != nil {
status = http.StatusBadRequest
http.Error(w, err.Error(), status)
return
}
if !findCache {
var valueSQL string
if len(usedTags) == 0 {
valueSQL = "splitByChar('=', Tag1)[1] AS value"
if tagPrefix != "" {
wr.And(where.HasPrefix("Tag1", tagPrefix))
}
} else {
valueSQL = "splitByChar('=', arrayJoin(Tags))[1] AS value"
if tagPrefix != "" {
wr.And(where.HasPrefix("arrayJoin(Tags)", tagPrefix))
}
}
queryLimit := limit + len(usedTags)
wr.Andf("Date >= '%s' AND Date <= '%s'", fromDate, untilDate)
sql := fmt.Sprintf("SELECT %s FROM %s %s %s GROUP BY value ORDER BY value LIMIT %d",
valueSQL,
h.config.ClickHouse.TaggedTable,
pw.PreWhereSQL(),
wr.SQL(),
queryLimit,
)
var (
entered bool
ctx context.Context
cancel context.CancelFunc
)
if limiter.Enabled() {
ctx, cancel = context.WithTimeout(context.Background(), h.config.ClickHouse.IndexTimeout)
defer cancel()
err = limiter.Enter(ctx, "tags")
queueDuration = time.Since(start)
if err != nil {
status = http.StatusServiceUnavailable
queueFail = true
logger.Error(err.Error())
http.Error(w, err.Error(), status)
return
}
queueDuration = time.Since(start)
entered = true
defer func() {
if entered {
limiter.Leave(ctx, "tags")
entered = false
}
}()
}
body, chReadRows, chReadBytes, err = clickhouse.Query(
scope.WithTable(r.Context(), h.config.ClickHouse.TaggedTable),
h.config.ClickHouse.URL,
sql,
opts,
nil,
)
if entered {
// release early as possible
limiter.Leave(ctx, "tags")
entered = false
}
if err != nil {
status, _ = clickhouse.HandleError(w, err)
return
}
readBytes = int64(len(body))
if useCache {
if metrics.FinderCacheMetrics != nil {
metrics.FinderCacheMetrics.CacheMisses.Add(1)
}
h.config.Common.FindCache.Set(key, body, h.config.Common.FindCacheConfig.FindTimeoutSec)
}
}
rows := strings.Split(stringutils.UnsafeString(body), "\n")
tags := make([]string, 0, uint64(len(rows))+1) // +1 - reserve for "name" tag
hasName := false
for i := 0; i < len(rows); i++ {
if rows[i] == "" {
continue
}
if rows[i] == "__name__" {
rows[i] = "name"
}
if usedTags[rows[i]] {
continue
}
tags = append(tags, rows[i])
if rows[i] == "name" {
hasName = true
}
}
if !hasName && !usedTags["name"] && (tagPrefix == "" || strings.HasPrefix("name", tagPrefix)) {
tags = append(tags, "name")
}
sort.Strings(tags)
if len(tags) > limit {
tags = tags[:limit]
}
if useCache {
if findCache {
logger.Info("finder", zap.String("get_cache", key),
zap.Int("metrics", len(rows)), zap.Bool("find_cached", true),
zap.Int32("ttl", h.config.Common.FindCacheConfig.FindTimeoutSec))
} else {
logger.Info("finder", zap.String("set_cache", key),
zap.Int("metrics", len(rows)), zap.Bool("find_cached", false),
zap.Int32("ttl", h.config.Common.FindCacheConfig.FindTimeoutSec))
}
}
b, err := json.Marshal(tags)
if err != nil {
status = http.StatusInternalServerError
http.Error(w, err.Error(), status)
return
}
metricsCount = int64(len(tags))
w.Write(b)
}
// func taggedValuesQuery(tag string, exprs []string, valuePrefix string, limit int) []string {
// query := make([]string, 0, 3+len(exprs))
// if tag != "" {
// query = append(query, "tag="+tag)
// }
// if valuePrefix != "" {
// query = append(query, "valuePrefix="+valuePrefix)
// }
// for _, expr := range exprs {
// query = append(query, "expr='"+expr+"'")
// }
// query = append(query, "limit="+strconv.Itoa(limit))
// return query
// }
func (h *Handler) ServeValues(w http.ResponseWriter, r *http.Request) {
start := timeNow()
status := http.StatusOK
accessLogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("http")
logger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("autocomplete")
r = r.WithContext(scope.WithLogger(r.Context(), logger))
var (
err error
body []byte
chReadRows int64
chReadBytes int64
metricsCount int64
queueFail bool
queueDuration time.Duration
findCache bool
opts clickhouse.Options
)
username := r.Header.Get("X-Forwarded-User")
limiter := h.config.GetUserTagsLimiter(username)
defer func() {
if rec := recover(); rec != nil {
status = http.StatusInternalServerError
logger.Error("panic during eval:",
zap.String("requestID", scope.String(r.Context(), "requestID")),
zap.Any("reason", rec),
zap.Stack("stack"),
)
answer := fmt.Sprintf("%v\nStack trace: %v", rec, zap.Stack("").String)
http.Error(w, answer, status)
}
d := time.Since(start)
dMS := d.Milliseconds()
logs.AccessLog(accessLogger, h.config, r, status, d, queueDuration, findCache, queueFail)
limiter.SendDuration(queueDuration.Milliseconds())
metrics.SendFindMetrics(metrics.TagsRequestMetric, status, dMS, 0, h.config.Metrics.ExtendedStat, metricsCount)
if !findCache && chReadRows > 0 && chReadBytes > 0 {
errored := status != http.StatusOK && status != http.StatusNotFound
metrics.SendQueryRead(metrics.AutocompleteQMetric, 0, 0, dMS, metricsCount, int64(len(body)), chReadRows, chReadBytes, errored)
}
}()
r.ParseMultipartForm(1024 * 1024)
tag := r.FormValue("tag")
if tag == "name" {
tag = "__name__"
}
valuePrefix := r.FormValue("valuePrefix")
limitStr := r.FormValue("limit")
limit := 10000
if limitStr != "" {
limit, err = strconv.Atoi(limitStr)
if err != nil {
status = http.StatusBadRequest
http.Error(w, err.Error(), status)
return
}
}
fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, start)
var key string
exprs := r.Form["expr"]
// params := taggedValuesQuery(tag, exprs, valuePrefix, limit)
// taggedKey(tag, , "valuePrefix="+valuePrefix, limit)
useCache := h.config.Common.FindCache != nil && h.config.Common.FindCacheConfig.FindTimeoutSec > 0 && !parser.TruthyBool(r.FormValue("noCache"))
if useCache {
// logger = logger.With(zap.String("use_cache", "true"))
key, _ = taggedValuesKey("values;", h.config.Common.FindCacheConfig.FindTimeoutSec, fromDate, untilDate, tag, exprs, valuePrefix, limit)
body, err = h.config.Common.FindCache.Get(key)
if err == nil {
if metrics.FinderCacheMetrics != nil {
metrics.FinderCacheMetrics.CacheHits.Add(1)
}
findCache = true
w.Header().Set("X-Cached-Find", strconv.Itoa(int(h.config.Common.FindCacheConfig.FindTimeoutSec)))
}
}
opts = clickhouse.Options{
TLSConfig: h.config.ClickHouse.TLSConfig,
Timeout: h.config.ClickHouse.IndexTimeout,
ConnectTimeout: h.config.ClickHouse.ConnectTimeout,
CheckRequestProgress: h.config.FeatureFlags.LogQueryProgress,
ProgressSendingInterval: h.config.ClickHouse.ProgressSendingInterval,
}
if !findCache {
wr, pw, usedTags, err := h.requestExpr(
r,
getTagCountQuerier(h.config, opts),
start.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Unix(),
start.Unix(),
)
if err == finder.ErrCostlySeriesByTag {
status = http.StatusForbidden
http.Error(w, err.Error(), status)
return
} else if err != nil {
status = http.StatusBadRequest
http.Error(w, err.Error(), status)
return
}
var valueSQL string
if len(usedTags) == 0 {
valueSQL = fmt.Sprintf("substr(Tag1, %d) AS value", len(tag)+2)
wr.And(where.HasPrefix("Tag1", tag+"="+valuePrefix))
} else {
prefixSelector := where.HasPrefix("x", tag+"="+valuePrefix)
valueSQL = fmt.Sprintf("substr(arrayFilter(x -> %s, Tags)[1], %d) AS value", prefixSelector, len(tag)+2)
wr.And("arrayExists(x -> " + prefixSelector + ", Tags)")
}
wr.Andf("Date >= '%s' AND Date <= '%s'", fromDate, untilDate)
sql := fmt.Sprintf("SELECT %s FROM %s %s %s GROUP BY value ORDER BY value LIMIT %d",
valueSQL,
h.config.ClickHouse.TaggedTable,
pw.PreWhereSQL(),
wr.SQL(),
limit,
)
var (
entered bool
ctx context.Context
cancel context.CancelFunc
)
if limiter.Enabled() {
ctx, cancel = context.WithTimeout(context.Background(), h.config.ClickHouse.IndexTimeout)
defer cancel()
err = limiter.Enter(ctx, "tags")
queueDuration = time.Since(start)
if err != nil {
status = http.StatusServiceUnavailable
queueFail = true
logger.Error(err.Error())
http.Error(w, err.Error(), status)
return
}
queueDuration = time.Since(start)
entered = true
defer func() {
if entered {
limiter.Leave(ctx, "tags")
entered = false
}
}()
}
body, chReadRows, chReadBytes, err = clickhouse.Query(
scope.WithTable(r.Context(), h.config.ClickHouse.TaggedTable),
h.config.ClickHouse.URL,
sql,
opts,
nil,
)
if entered {
// release early as possible
limiter.Leave(ctx, "tags")
entered = false
}
if err != nil {
status, _ = clickhouse.HandleError(w, err)
return
}
if useCache {
if metrics.FinderCacheMetrics != nil {
metrics.FinderCacheMetrics.CacheMisses.Add(1)
}
h.config.Common.FindCache.Set(key, body, h.config.Common.FindCacheConfig.FindTimeoutSec)
}
}
var rows []string
if len(body) > 0 {
rows = strings.Split(stringutils.UnsafeString(body), "\n")
if len(rows) > 0 && rows[len(rows)-1] == "" {
rows = rows[:len(rows)-1]
}
metricsCount = int64(len(rows))
}
if useCache {
if findCache {
logger.Info("finder", zap.String("get_cache", key),
zap.Int("metrics", len(rows)), zap.Bool("find_cached", true),
zap.Int32("ttl", h.config.Common.FindCacheConfig.FindTimeoutSec))
} else {
logger.Info("finder", zap.String("set_cache", key),
zap.Int("metrics", len(rows)), zap.Bool("find_cached", false),
zap.Int32("ttl", h.config.Common.FindCacheConfig.FindTimeoutSec))
}
}
b, err := json.Marshal(rows)
if err != nil {
status = http.StatusInternalServerError
http.Error(w, err.Error(), status)
return
}
w.Write(b)
}
================================================
FILE: autocomplete/autocomplete_test.go
================================================
package autocomplete
import (
"io"
"net/http"
"net/http/httptest"
"strconv"
"testing"
"time"
"github.com/lomik/graphite-clickhouse/config"
"github.com/lomik/graphite-clickhouse/helper/date"
chtest "github.com/lomik/graphite-clickhouse/helper/tests/clickhouse"
"github.com/lomik/graphite-clickhouse/metrics"
"github.com/stretchr/testify/assert"
)
func NewRequest(method, url string, body io.Reader) *http.Request {
r, _ := http.NewRequest(method, url, body)
return r
}
type testStruct struct {
request *http.Request
wantCode int
want string
wantContent string
}
func testResponce(t *testing.T, step int, h *Handler, tt *testStruct, wantCachedFind string) {
w := httptest.NewRecorder()
h.ServeHTTP(w, tt.request)
s := w.Body.String()
assert.Equalf(t, tt.wantCode, w.Code, "code mismatch step %d\n,%s", step, s)
if w.Code == http.StatusOK {
if tt.wantContent != "" {
contentType := w.Header().Get("Content-Type")
assert.Equalf(t, tt.wantContent, contentType, "content type mismatch, step %d", step)
}
cachedFindHeader := w.Header().Get("X-Cached-Find")
assert.Equalf(t, cachedFindHeader, wantCachedFind, "cached find '%s' mismatch, want be %v, step %d", cachedFindHeader, wantCachedFind, step)
assert.Equalf(t, tt.want, s, "Step %d", step)
}
}
func TestHandler_ServeTags(t *testing.T) {
timeNow = func() time.Time {
return time.Unix(1669714247, 0)
}
metrics.DisableMetrics()
srv := chtest.NewTestServer()
defer srv.Close()
cfg, _ := config.DefaultConfig()
cfg.ClickHouse.URL = srv.URL
cfg.ClickHouse.TaggedTable = "graphite_tagged"
h := NewTags(cfg)
now := timeNow()
fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)
// Test 1: Get all tags without filters
srv.AddResponce(
"SELECT splitByChar('=', Tag1)[1] AS value FROM graphite_tagged WHERE "+
"Date >= '"+fromDate+"' AND Date <= '"+untilDate+"' GROUP BY value ORDER BY value LIMIT 10000",
&chtest.TestResponse{
Body: []byte("__name__\nenvironment\nproject\nhost\n"),
})
// Test 2: Get tags with prefix filter
srv.AddResponce(
"SELECT splitByChar('=', Tag1)[1] AS value FROM graphite_tagged WHERE "+
"(Tag1 LIKE 'pr%') AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000",
&chtest.TestResponse{
Body: []byte("project\n"),
})
// Test 3: Get tags with expr filters
srv.AddResponce(
"SELECT splitByChar('=', arrayJoin(Tags))[1] AS value FROM graphite_tagged WHERE "+
"(Tag1='environment=production') AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10001",
&chtest.TestResponse{
Body: []byte("__name__\nhost\nproject\n"),
})
// Test 4: Get tags with multiple expr filters
srv.AddResponce(
"SELECT splitByChar('=', arrayJoin(Tags))[1] AS value FROM graphite_tagged WHERE "+
"((Tag1='environment=production') AND (has(Tags, 'project=web'))) AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10002",
&chtest.TestResponse{
Body: []byte("__name__\nhost\n"),
})
// Test 5: Get tags with prefix and expr filters
srv.AddResponce(
"SELECT splitByChar('=', arrayJoin(Tags))[1] AS value FROM graphite_tagged WHERE "+
"((Tag1='environment=production') AND (arrayJoin(Tags) LIKE 'h%')) AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10001",
&chtest.TestResponse{
Body: []byte("host\n"),
})
tests := []testStruct{
{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/tags", nil),
wantCode: http.StatusOK,
want: `["environment","host","name","project"]`,
},
{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/tags?tagPrefix=pr", nil),
wantCode: http.StatusOK,
want: `["project"]`,
},
{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/tags?expr=environment%3Dproduction", nil),
wantCode: http.StatusOK,
want: `["host","name","project"]`,
},
{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/tags?expr=environment%3Dproduction&expr=project%3Dweb", nil),
wantCode: http.StatusOK,
want: `["host","name"]`,
},
{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/tags?expr=environment%3Dproduction&tagPrefix=h", nil),
wantCode: http.StatusOK,
want: `["host"]`,
},
}
for i, tt := range tests {
t.Run("Test#"+strconv.Itoa(i), func(t *testing.T) {
testResponce(t, i, h, &tt, "")
})
}
}
func TestHandler_ServeTagsWithCache(t *testing.T) {
timeNow = func() time.Time {
return time.Unix(1669714247, 0)
}
metrics.DisableMetrics()
srv := chtest.NewTestServer()
defer srv.Close()
cfg, _ := config.DefaultConfig()
cfg.ClickHouse.URL = srv.URL
cfg.ClickHouse.TaggedTable = "graphite_tagged"
// Enable cache
cfg.Common.FindCacheConfig = config.CacheConfig{
Type: "mem",
Size: 8192,
FindTimeoutSec: 1,
}
var err error
cfg.Common.FindCache, err = config.CreateCache("autocomplete", &cfg.Common.FindCacheConfig)
if err != nil {
t.Fatalf("Failed to create find cache: %v", err)
}
h := NewTags(cfg)
now := timeNow()
fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)
srv.AddResponce(
"SELECT splitByChar('=', Tag1)[1] AS value FROM graphite_tagged WHERE "+
"Date >= '"+fromDate+"' AND Date <= '"+untilDate+"' GROUP BY value ORDER BY value LIMIT 10000",
&chtest.TestResponse{
Body: []byte("__name__\nenvironment\nproject\nhost\n"),
})
test := testStruct{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/tags", nil),
wantCode: http.StatusOK,
want: `["environment","host","name","project"]`,
}
// First request - should hit the database
testResponce(t, 0, h, &test, "")
assert.Equal(t, uint64(1), srv.Queries())
// Second request - should hit the cache
testResponce(t, 1, h, &test, "1")
assert.Equal(t, uint64(1), srv.Queries()) // No new queries
// Wait for cache expiration
time.Sleep(time.Second * 2)
// Third request - should hit the database again
testResponce(t, 2, h, &test, "")
assert.Equal(t, uint64(2), srv.Queries())
}
func TestHandler_ServeValues(t *testing.T) {
timeNow = func() time.Time {
return time.Unix(1669714247, 0)
}
metrics.DisableMetrics()
srv := chtest.NewTestServer()
defer srv.Close()
cfg, _ := config.DefaultConfig()
cfg.ClickHouse.URL = srv.URL
cfg.ClickHouse.TaggedTable = "graphite_tagged"
h := NewValues(cfg)
now := timeNow()
until := strconv.FormatInt(now.Unix(), 10)
from := strconv.FormatInt(now.Add(-time.Minute).Unix(), 10)
fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)
srv.AddResponce(
"SELECT substr(arrayFilter(x -> x LIKE 'host=%', Tags)[1], 6) AS value FROM graphite_tagged WHERE (((Tag1='environment=production') AND (has(Tags, 'project=web'))) AND (arrayExists(x -> x LIKE 'host=%', Tags))) AND "+
"(Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000",
&chtest.TestResponse{
Body: []byte("host1\nhost2\ndc-host2\ndc-host3\n"),
})
tests := []testStruct{
{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?"+
"expr=environment%3Dproduction"+"&"+"expr=project%3Dweb"+"&"+"tag=host"+
"&limit=10000&from="+from+"&until="+until, nil),
wantCode: http.StatusOK,
want: "[\"host1\",\"host2\",\"dc-host2\",\"dc-host3\"]",
wantContent: "text/plain; charset=utf-8",
},
}
var queries uint64
for i, tt := range tests {
t.Run(tt.request.URL.RawQuery+"#"+strconv.Itoa(i), func(t *testing.T) {
for i := 0; i < 2; i++ {
testResponce(t, i, h, &tt, "")
}
assert.Equal(t, uint64(2), srv.Queries()-queries)
queries = srv.Queries()
})
}
}
func TestHandler_ServeValuesWithValuePrefix(t *testing.T) {
timeNow = func() time.Time {
return time.Unix(1669714247, 0)
}
metrics.DisableMetrics()
srv := chtest.NewTestServer()
defer srv.Close()
cfg, _ := config.DefaultConfig()
cfg.ClickHouse.URL = srv.URL
cfg.ClickHouse.TaggedTable = "graphite_tagged"
h := NewValues(cfg)
now := timeNow()
fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)
// Test with valuePrefix
srv.AddResponce(
"SELECT substr(Tag1, 6) AS value FROM graphite_tagged WHERE "+
"(Tag1 LIKE 'host=dc-%') AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 100",
&chtest.TestResponse{
Body: []byte("dc-host1\ndc-host2\ndc-host3\n"),
})
test := testStruct{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=host&valuePrefix=dc-&limit=100", nil),
wantCode: http.StatusOK,
want: "[\"dc-host1\",\"dc-host2\",\"dc-host3\"]",
}
testResponce(t, 0, h, &test, "")
}
func TestHandler_ServeValuesNameTag(t *testing.T) {
timeNow = func() time.Time {
return time.Unix(1669714247, 0)
}
metrics.DisableMetrics()
srv := chtest.NewTestServer()
defer srv.Close()
cfg, _ := config.DefaultConfig()
cfg.ClickHouse.URL = srv.URL
cfg.ClickHouse.TaggedTable = "graphite_tagged"
h := NewValues(cfg)
now := timeNow()
fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)
// Test with name tag (which should be converted to __name__)
srv.AddResponce(
`SELECT substr(Tag1, 10) AS value FROM graphite_tagged WHERE `+
`(Tag1 LIKE '\\_\\_name\\_\\_=metric.%') AND (Date >= '`+fromDate+`' AND Date <= '`+untilDate+`') GROUP BY value ORDER BY value LIMIT 10000`,
&chtest.TestResponse{
Body: []byte("metric.cpu.usage\nmetric.memory.used\nmetric.disk.io\n"),
})
test := testStruct{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=name&valuePrefix=metric.", nil),
wantCode: http.StatusOK,
want: "[\"metric.cpu.usage\",\"metric.memory.used\",\"metric.disk.io\"]",
}
testResponce(t, 0, h, &test, "")
}
func TestHandler_ServeValuesWithCache(t *testing.T) {
timeNow = func() time.Time {
return time.Unix(1669714247, 0)
}
metrics.DisableMetrics()
srv := chtest.NewTestServer()
defer srv.Close()
cfg, _ := config.DefaultConfig()
cfg.ClickHouse.URL = srv.URL
cfg.ClickHouse.TaggedTable = "graphite_tagged"
// Enable cache
cfg.Common.FindCacheConfig = config.CacheConfig{
Type: "mem",
Size: 8192,
FindTimeoutSec: 1,
}
var err error
cfg.Common.FindCache, err = config.CreateCache("autocomplete", &cfg.Common.FindCacheConfig)
if err != nil {
t.Fatalf("Failed to create find cache: %v", err)
}
h := NewValues(cfg)
now := timeNow()
fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)
srv.AddResponce(
"SELECT substr(Tag1, 6) AS value FROM graphite_tagged WHERE "+
"(Tag1 LIKE 'host=%') AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000",
&chtest.TestResponse{
Body: []byte("host1\nhost2\nhost3\n"),
})
test := testStruct{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=host", nil),
wantCode: http.StatusOK,
want: "[\"host1\",\"host2\",\"host3\"]",
}
// First request - should hit the database
testResponce(t, 0, h, &test, "")
assert.Equal(t, uint64(1), srv.Queries())
// Second request - should hit the cache
testResponce(t, 1, h, &test, "1")
assert.Equal(t, uint64(1), srv.Queries()) // No new queries
// Wait for cache expiration
time.Sleep(time.Second * 2)
// Third request - should hit the database again
testResponce(t, 2, h, &test, "")
assert.Equal(t, uint64(2), srv.Queries())
}
func TestHandler_ServeValuesWithCacheAndExpr(t *testing.T) {
timeNow = func() time.Time {
return time.Unix(1669714247, 0)
}
metrics.DisableMetrics()
srv := chtest.NewTestServer()
defer srv.Close()
cfg, _ := config.DefaultConfig()
cfg.ClickHouse.URL = srv.URL
cfg.ClickHouse.TaggedTable = "graphite_tagged"
// Enable cache
cfg.Common.FindCacheConfig = config.CacheConfig{
Type: "mem",
Size: 8192,
FindTimeoutSec: 1,
}
var err error
cfg.Common.FindCache, err = config.CreateCache("autocomplete", &cfg.Common.FindCacheConfig)
if err != nil {
t.Fatalf("Failed to create find cache: %v", err)
}
h := NewValues(cfg)
now := timeNow()
fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)
srv.AddResponce(
"SELECT substr(arrayFilter(x -> x LIKE 'host=%', Tags)[1], 6) AS value FROM graphite_tagged WHERE "+
"((Tag1='environment=production') AND (arrayExists(x -> x LIKE 'host=%', Tags))) AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000",
&chtest.TestResponse{
Body: []byte("prod-host1\nprod-host2\n"),
})
test := testStruct{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=host&expr=environment%3Dproduction", nil),
wantCode: http.StatusOK,
want: "[\"prod-host1\",\"prod-host2\"]",
}
// First request - should hit the database
testResponce(t, 0, h, &test, "")
assert.Equal(t, uint64(1), srv.Queries())
// Second request - should hit the cache
testResponce(t, 1, h, &test, "1")
assert.Equal(t, uint64(1), srv.Queries()) // No new queries
// Test with different valuePrefix - should not hit cache
test2 := testStruct{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=host&expr=environment%3Dproduction&valuePrefix=prod-host1", nil),
wantCode: http.StatusOK,
want: "[\"prod-host1\"]",
}
srv.AddResponce(
"SELECT substr(arrayFilter(x -> x LIKE 'host=prod-host1%', Tags)[1], 6) AS value FROM graphite_tagged WHERE "+
"((Tag1='environment=production') AND (arrayExists(x -> x LIKE 'host=prod-host1%', Tags))) AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000",
&chtest.TestResponse{
Body: []byte("prod-host1\n"),
})
// Should hit the database because valuePrefix is different
testResponce(t, 2, h, &test2, "")
assert.Equal(t, uint64(2), srv.Queries())
}
func TestHandler_ServeValuesWithInvalidLimit(t *testing.T) {
timeNow = func() time.Time {
return time.Unix(1669714247, 0)
}
metrics.DisableMetrics()
srv := chtest.NewTestServer()
defer srv.Close()
cfg, _ := config.DefaultConfig()
cfg.ClickHouse.URL = srv.URL
cfg.ClickHouse.TaggedTable = "graphite_tagged"
h := NewValues(cfg)
test := testStruct{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=host&limit=invalid", nil),
wantCode: http.StatusBadRequest,
want: "", // Error response
}
testResponce(t, 0, h, &test, "")
}
func TestHandler_ServeValuesWithMultipleExpr(t *testing.T) {
timeNow = func() time.Time {
return time.Unix(1669714247, 0)
}
metrics.DisableMetrics()
srv := chtest.NewTestServer()
defer srv.Close()
cfg, _ := config.DefaultConfig()
cfg.ClickHouse.URL = srv.URL
cfg.ClickHouse.TaggedTable = "graphite_tagged"
h := NewValues(cfg)
now := timeNow()
fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)
// Test with multiple expressions and valuePrefix
srv.AddResponce(
"SELECT substr(arrayFilter(x -> x LIKE 'host=dc-%', Tags)[1], 6) AS value FROM graphite_tagged WHERE "+
"(((Tag1='environment=production') AND (has(Tags, 'project=web'))) AND (arrayExists(x -> x LIKE 'host=dc-%', Tags))) AND "+
"(Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000",
&chtest.TestResponse{
Body: []byte("dc-host1\ndc-host2\n"),
})
test := testStruct{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=host&expr=environment%3Dproduction&expr=project%3Dweb&valuePrefix=dc-", nil),
wantCode: http.StatusOK,
want: "[\"dc-host1\",\"dc-host2\"]",
}
testResponce(t, 0, h, &test, "")
}
func TestHandler_ServeValuesNoCache(t *testing.T) {
timeNow = func() time.Time {
return time.Unix(1669714247, 0)
}
metrics.DisableMetrics()
srv := chtest.NewTestServer()
defer srv.Close()
cfg, _ := config.DefaultConfig()
cfg.ClickHouse.URL = srv.URL
cfg.ClickHouse.TaggedTable = "graphite_tagged"
// Enable cache
cfg.Common.FindCacheConfig = config.CacheConfig{
Type: "mem",
Size: 8192,
FindTimeoutSec: 60,
}
var err error
cfg.Common.FindCache, err = config.CreateCache("autocomplete", &cfg.Common.FindCacheConfig)
if err != nil {
t.Fatalf("Failed to create find cache: %v", err)
}
h := NewValues(cfg)
now := timeNow()
fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)
srv.AddResponce(
"SELECT substr(Tag1, 6) AS value FROM graphite_tagged WHERE "+
"(Tag1 LIKE 'host=%') AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000",
&chtest.TestResponse{
Body: []byte("host1\nhost2\n"),
})
test := testStruct{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=host&noCache=true", nil),
wantCode: http.StatusOK,
want: "[\"host1\",\"host2\"]",
}
// First request with noCache=true - should always hit the database
testResponce(t, 0, h, &test, "")
assert.Equal(t, uint64(1), srv.Queries())
// Second request with noCache=true - should hit the database again
testResponce(t, 1, h, &test, "")
assert.Equal(t, uint64(2), srv.Queries()) // Should increase
}
func TestHandler_ServeValuesEmptyResult(t *testing.T) {
timeNow = func() time.Time {
return time.Unix(1669714247, 0)
}
metrics.DisableMetrics()
srv := chtest.NewTestServer()
defer srv.Close()
cfg, _ := config.DefaultConfig()
cfg.ClickHouse.URL = srv.URL
cfg.ClickHouse.TaggedTable = "graphite_tagged"
h := NewValues(cfg)
now := timeNow()
fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)
// Test empty result
srv.AddResponce(
"SELECT substr(Tag1, 13) AS value FROM graphite_tagged WHERE "+
"(Tag1 LIKE 'nonexistent=%') AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000",
&chtest.TestResponse{
Body: []byte(""), // Empty response
})
test := testStruct{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=nonexistent", nil),
wantCode: http.StatusOK,
want: "null", // Empty array
}
testResponce(t, 0, h, &test, "")
}
func TestHandler_ServeTagsWithCostOptimization(t *testing.T) {
timeNow = func() time.Time {
return time.Unix(1669714247, 0)
}
metrics.DisableMetrics()
srv := chtest.NewTestServer()
defer srv.Close()
cfg, _ := config.DefaultConfig()
cfg.ClickHouse.URL = srv.URL
cfg.ClickHouse.TaggedTable = "graphite_tagged"
cfg.ClickHouse.TagsCountTable = "tag1_count_per_day"
h := NewTags(cfg)
now := timeNow()
fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)
from := now.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Unix()
until := now.Unix()
// Test case: Tags query with multiple expressions should use cost optimization
// First response: tags count query to get costs
srv.AddResponce(
"SELECT Tag1, sum(Count) as cnt FROM tag1_count_per_day WHERE "+
"((Tag1='environment=production') OR (Tag1='project=web')) AND "+
"(Date >= '"+date.FromTimestampToDaysFormat(from)+"' AND Date <= '"+date.UntilTimestampToDaysFormat(until)+"') GROUP BY Tag1 FORMAT TabSeparatedRaw",
&chtest.TestResponse{
Body: []byte("environment=production\t10000\nproject=web\t500\n"),
})
// Second response: main tags query (should be ordered based on costs)
srv.AddResponce(
"SELECT splitByChar('=', arrayJoin(Tags))[1] AS value FROM graphite_tagged WHERE "+
"((Tag1='project=web') AND (has(Tags, 'environment=production'))) AND "+ // Lower cost term first
"(Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10002",
&chtest.TestResponse{
Body: []byte("__name__\nhost\nregion\n"),
})
test := testStruct{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/tags?expr=environment%3Dproduction&expr=project%3Dweb", nil),
wantCode: http.StatusOK,
want: `["host","name","region"]`,
}
testResponce(t, 0, h, &test, "")
assert.Equal(t, uint64(2), srv.Queries()) // Should have 2 queries: cost query + main query
}
func TestHandler_ServeValuesWithCostOptimization(t *testing.T) {
timeNow = func() time.Time {
return time.Unix(1669714247, 0)
}
metrics.DisableMetrics()
srv := chtest.NewTestServer()
defer srv.Close()
cfg, _ := config.DefaultConfig()
cfg.ClickHouse.URL = srv.URL
cfg.ClickHouse.TaggedTable = "graphite_tagged"
cfg.ClickHouse.TagsCountTable = "tag1_count_per_day"
h := NewValues(cfg)
now := timeNow()
fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)
from := now.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Unix()
until := now.Unix()
// First response: tags count query for cost optimization
srv.AddResponce(
"SELECT Tag1, sum(Count) as cnt FROM tag1_count_per_day WHERE "+
"((Tag1='environment=production') OR (Tag1='datacenter=us-east')) AND "+
"(Date >= '"+date.FromTimestampToDaysFormat(from)+"' AND Date <= '"+date.UntilTimestampToDaysFormat(until)+"') GROUP BY Tag1 FORMAT TabSeparatedRaw",
&chtest.TestResponse{
Body: []byte("environment=production\t5000\ndatacenter=us-east\t100\n"),
})
// Second response: values query (should use optimized order)
srv.AddResponce(
"SELECT substr(arrayFilter(x -> x LIKE 'host=%', Tags)[1], 6) AS value FROM graphite_tagged WHERE "+
"(((Tag1='datacenter=us-east') AND (has(Tags, 'environment=production'))) AND "+ // Lower cost first
"(arrayExists(x -> x LIKE 'host=%', Tags))) AND "+
"(Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000",
&chtest.TestResponse{
Body: []byte("host1\nhost2\nhost3\n"),
})
test := testStruct{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=host&expr=environment%3Dproduction&expr=datacenter%3Dus-east", nil),
wantCode: http.StatusOK,
want: "[\"host1\",\"host2\",\"host3\"]",
}
testResponce(t, 0, h, &test, "")
assert.Equal(t, uint64(2), srv.Queries())
}
func TestHandler_ServeTagsWithWildcardExpressions(t *testing.T) {
timeNow = func() time.Time {
return time.Unix(1669714247, 0)
}
metrics.DisableMetrics()
srv := chtest.NewTestServer()
defer srv.Close()
cfg, _ := config.DefaultConfig()
cfg.ClickHouse.URL = srv.URL
cfg.ClickHouse.TaggedTable = "graphite_tagged"
cfg.ClickHouse.TagsCountTable = "tag1_count_per_day"
h := NewTags(cfg)
now := timeNow()
fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)
// Test with wildcard expressions (should not query tags count table)
srv.AddResponce(
"SELECT splitByChar('=', arrayJoin(Tags))[1] AS value FROM graphite_tagged WHERE "+
"(Tag1 LIKE 'environment=prod%') AND (Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10001",
&chtest.TestResponse{
Body: []byte("__name__\nhost\nproject\n"),
})
test := testStruct{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/tags?expr=environment%3Dprod*", nil),
wantCode: http.StatusOK,
want: `["host","name","project"]`,
}
testResponce(t, 0, h, &test, "")
assert.Equal(t, uint64(1), srv.Queries()) // Only 1 query since wildcards skip cost optimization
}
func TestHandler_ServeValuesWithNoEqualityTerms(t *testing.T) {
timeNow = func() time.Time {
return time.Unix(1669714247, 0)
}
metrics.DisableMetrics()
srv := chtest.NewTestServer()
defer srv.Close()
cfg, _ := config.DefaultConfig()
cfg.ClickHouse.URL = srv.URL
cfg.ClickHouse.TaggedTable = "graphite_tagged"
cfg.ClickHouse.TagsCountTable = "tag1_count_per_day"
h := NewValues(cfg)
now := timeNow()
fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)
// Test with != operator (should not use tags count table)
srv.AddResponce(
"SELECT substr(arrayFilter(x -> x LIKE 'host=%', Tags)[1], 6) AS value FROM graphite_tagged WHERE "+
"((NOT arrayExists((x) -> x='environment=development', Tags)) AND (arrayExists(x -> x LIKE 'host=%', Tags))) AND "+
"(Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000",
&chtest.TestResponse{
Body: []byte("host1\nhost2\n"),
})
test := testStruct{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=host&expr=environment%21%3Ddevelopment", nil),
wantCode: http.StatusOK,
want: "[\"host1\",\"host2\"]",
}
testResponce(t, 0, h, &test, "")
assert.Equal(t, uint64(1), srv.Queries()) // Only 1 query since no equality terms
}
func TestHandler_ServeTagsWithHighCostTags(t *testing.T) {
timeNow = func() time.Time {
return time.Unix(1669714247, 0)
}
metrics.DisableMetrics()
srv := chtest.NewTestServer()
defer srv.Close()
cfg, _ := config.DefaultConfig()
cfg.ClickHouse.URL = srv.URL
cfg.ClickHouse.TaggedTable = "graphite_tagged"
cfg.ClickHouse.TagsCountTable = "tag1_count_per_day"
h := NewTags(cfg)
now := timeNow()
fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)
from := now.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Unix()
until := now.Unix()
// Test with high cardinality tags - tags should be reordered based on cost
srv.AddResponce(
"SELECT Tag1, sum(Count) as cnt FROM tag1_count_per_day WHERE "+
"(((Tag1='__name__=high.cost.metric') OR (Tag1='environment=production')) OR (Tag1='dc=west')) AND "+
"(Date >= '"+date.FromTimestampToDaysFormat(from)+"' AND Date <= '"+date.UntilTimestampToDaysFormat(until)+"') GROUP BY Tag1 FORMAT TabSeparatedRaw",
&chtest.TestResponse{
Body: []byte("__name__=high.cost.metric\t1000000\nenvironment=production\t10000\ndc=west\t50\n"),
})
// Query should use lowest cost tag first (dc=west)
srv.AddResponce(
"SELECT splitByChar('=', arrayJoin(Tags))[1] AS value FROM graphite_tagged WHERE "+
"(((Tag1='dc=west') AND (has(Tags, 'environment=production'))) AND (has(Tags, '__name__=high.cost.metric'))) AND "+
"(Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10003",
&chtest.TestResponse{
Body: []byte("host\nproject\nregion\n"),
})
test := testStruct{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/tags?expr=name%3Dhigh.cost.metric&expr=environment%3Dproduction&expr=dc%3Dwest", nil),
wantCode: http.StatusOK,
want: `["host","project","region"]`,
}
testResponce(t, 0, h, &test, "")
assert.Equal(t, uint64(2), srv.Queries())
}
func TestHandler_ServeValuesWithMixedOperators(t *testing.T) {
timeNow = func() time.Time {
return time.Unix(1669714247, 0)
}
metrics.DisableMetrics()
srv := chtest.NewTestServer()
defer srv.Close()
cfg, _ := config.DefaultConfig()
cfg.ClickHouse.URL = srv.URL
cfg.ClickHouse.TaggedTable = "graphite_tagged"
cfg.ClickHouse.TagsCountTable = "tag1_count_per_day"
h := NewValues(cfg)
now := timeNow()
fromDate, untilDate := dateString(h.config.ClickHouse.TaggedAutocompleDays, now)
from := now.AddDate(0, 0, -h.config.ClickHouse.TaggedAutocompleDays).Unix()
until := now.Unix()
// Test with mixed operators (only equality operators should be in cost query)
srv.AddResponce(
"SELECT Tag1, sum(Count) as cnt FROM tag1_count_per_day WHERE "+
"((Tag1='environment=production') OR (Tag1='project=api')) AND "+
"(Date >= '"+date.FromTimestampToDaysFormat(from)+"' AND Date <= '"+date.UntilTimestampToDaysFormat(until)+"') GROUP BY Tag1 FORMAT TabSeparatedRaw",
&chtest.TestResponse{
Body: []byte("environment=production\t8000\nproject=api\t200\n"),
})
// Main query should include != operator but order by cost
srv.AddResponce(
"SELECT substr(arrayFilter(x -> x LIKE 'host=%', Tags)[1], 6) AS value FROM graphite_tagged WHERE "+
"((((Tag1='project=api') AND (has(Tags, 'environment=production'))) AND (NOT arrayExists((x) -> x='dc=east', Tags))) AND "+
"(arrayExists(x -> x LIKE 'host=%', Tags))) AND "+
"(Date >= '"+fromDate+"' AND Date <= '"+untilDate+"') GROUP BY value ORDER BY value LIMIT 10000",
&chtest.TestResponse{
Body: []byte("host1\nhost2\n"),
})
test := testStruct{
request: NewRequest("GET", srv.URL+"/tags/autoComplete/values?tag=host&expr=environment%3Dproduction&expr=project%3Dapi&expr=dc%21%3Deast", nil),
wantCode: http.StatusOK,
want: "[\"host1\",\"host2\"]",
}
testResponce(t, 0, h, &test, "")
assert.Equal(t, uint64(2), srv.Queries()) // Cost query only for equality terms
}
================================================
FILE: cache/cache.go
================================================
package cache
import (
"crypto/sha256"
"encoding/hex"
"errors"
"sync/atomic"
"time"
"github.com/bradfitz/gomemcache/memcache"
"github.com/msaf1980/go-expirecache"
)
var (
ErrTimeout = errors.New("cache: timeout")
ErrNotFound = errors.New("cache: not found")
)
type BytesCache interface {
Get(k string) ([]byte, error)
Set(k string, v []byte, expire int32)
}
func NewExpireCache(maxsize uint64) BytesCache {
ec := expirecache.New[string, []byte](maxsize)
go ec.ApproximateCleaner(10 * time.Second)
return &ExpireCache{ec: ec}
}
type ExpireCache struct {
ec *expirecache.Cache[string, []byte]
}
func (ec ExpireCache) Get(k string) ([]byte, error) {
v, ok := ec.ec.Get(k)
if !ok {
return nil, ErrNotFound
}
return v, nil
}
func (ec ExpireCache) Set(k string, v []byte, expire int32) {
ec.ec.Set(k, v, uint64(len(v)), expire)
}
func NewMemcached(prefix string, servers ...string) BytesCache {
return &MemcachedCache{prefix: prefix, client: memcache.New(servers...)}
}
type MemcachedCache struct {
prefix string
client *memcache.Client
timeouts uint64
}
func (m *MemcachedCache) Get(k string) ([]byte, error) {
key := sha256.Sum256([]byte(k))
hk := hex.EncodeToString(key[:])
done := make(chan bool, 1)
var err error
var item *memcache.Item
go func() {
item, err = m.client.Get(m.prefix + hk)
done <- true
}()
timeout := time.After(50 * time.Millisecond)
select {
case <-timeout:
atomic.AddUint64(&m.timeouts, 1)
return nil, ErrTimeout
case <-done:
}
if err != nil {
// translate to internal cache miss error
if errors.Is(err, memcache.ErrCacheMiss) {
err = ErrNotFound
}
return nil, err
}
return item.Value, nil
}
func (m *MemcachedCache) Set(k string, v []byte, expire int32) {
key := sha256.Sum256([]byte(k))
hk := hex.EncodeToString(key[:])
go func() {
_ = m.client.Set(&memcache.Item{Key: m.prefix + hk, Value: v, Expiration: expire})
}()
}
func (m *MemcachedCache) Timeouts() uint64 {
return atomic.LoadUint64(&m.timeouts)
}
================================================
FILE: capabilities/handler.go
================================================
package capabilities
import (
"encoding/json"
"io"
"net/http"
"os"
"time"
v3pb "github.com/go-graphite/protocol/carbonapi_v3_pb"
"github.com/lomik/graphite-clickhouse/config"
"github.com/lomik/graphite-clickhouse/logs"
"github.com/lomik/graphite-clickhouse/pkg/scope"
)
type Handler struct {
config *config.Config
}
func NewHandler(config *config.Config) *Handler {
return &Handler{
config: config,
}
}
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
accessLogger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("http")
logger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("capabilities")
r = r.WithContext(scope.WithLogger(r.Context(), logger))
r.ParseMultipartForm(1024 * 1024)
format := r.FormValue("format")
accepts := r.Header["Accept"]
for _, accept := range accepts {
if accept == "application/x-carbonapi-v3-pb" {
format = "carbonapi_v3_pb"
break
}
}
status := http.StatusOK
start := time.Now()
defer func() {
d := time.Since(start)
logs.AccessLog(accessLogger, h.config, r, status, d, time.Duration(0), false, false)
}()
if format == "carbonapi_v3_pb" || format == "json" {
body, err := io.ReadAll(r.Body)
if err != nil {
status = http.StatusBadRequest
http.Error(w, "Bad request (malformed body)", status)
}
var pv3Request v3pb.CapabilityRequest
err = pv3Request.Unmarshal(body)
if err != nil {
status = http.StatusBadRequest
http.Error(w, "Bad request (malformed body)", status)
}
hostname, err := os.Hostname()
if err != nil {
hostname = "(unknown)"
}
pvResponse := v3pb.CapabilityResponse{
SupportedProtocols: []string{"carbonapi_v3_pb", "carbonapi_v2_pb", "graphite-web-pickle"},
Name: hostname,
HighPrecisionTimestamps: false,
SupportFilteringFunctions: false,
LikeSplittedRequests: false,
SupportStreaming: false,
}
var data []byte
contentType := ""
switch format {
case "json":
contentType = "application/json"
data, err = json.Marshal(pvResponse)
if err != nil {
status = http.StatusInternalServerError
http.Error(w, err.Error(), status)
return
}
case "carbonapi_v3_pb":
contentType = "application/x-carbonapi-v3-pb"
data, err = pvResponse.Marshal()
if err != nil {
status = http.StatusBadRequest
http.Error(w, "Bad request (unsupported format)", status)
return
}
}
w.Header().Set("Content-Type", contentType)
w.Write(data)
} else {
status = http.StatusBadRequest
http.Error(w, "Bad request (unsupported format)", status)
}
}
================================================
FILE: cmd/e2e-test/carbon-clickhouse.go
================================================
package main
import (
"fmt"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"text/template"
)
var CchContainerName = "carbon-clickhouse-gch-test"
type CarbonClickhouse struct {
Version string `toml:"version"`
DockerImage string `toml:"image"`
Template string `toml:"template"` // carbon-clickhouse config template
TZ string `toml:"tz"` // override timezone
address string `toml:"-"`
container string `toml:"-"`
storeDir string `toml:"-"`
}
func (c *CarbonClickhouse) Start(testDir, clickhouseURL string) (string, error) {
if len(c.Version) == 0 {
c.Version = "latest"
}
if len(c.DockerImage) == 0 {
c.DockerImage = "ghcr.io/go-graphite/carbon-clickhouse"
}
var err error
c.address, err = getFreeTCPPort("")
if err != nil {
return "", err
}
c.container = CchContainerName
c.storeDir, err = os.MkdirTemp("", "carbon-clickhouse")
if err != nil {
return "", err
}
c.address, err = getFreeTCPPort("")
if err != nil {
c.Cleanup()
return "", err
}
name := filepath.Base(c.Template)
tpl := path.Join(testDir, c.Template)
tmpl, err := template.New(name).ParseFiles(tpl)
if err != nil {
c.Cleanup()
return "", err
}
param := struct {
CLICKHOUSE_URL string
CCH_ADDR string
}{
CLICKHOUSE_URL: clickhouseURL,
CCH_ADDR: c.address,
}
configFile := path.Join(c.storeDir, "carbon-clickhouse.conf")
f, err := os.OpenFile(configFile, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
c.Cleanup()
return "", err
}
err = tmpl.ExecuteTemplate(f, name, param)
if err != nil {
c.Cleanup()
return "", err
}
tz := os.Getenv("TZ")
cchStart := []string{"run", "-d",
"--name", c.container,
"-p", c.address + ":2003",
"-v", c.storeDir + ":/etc/carbon-clickhouse",
// TZ, need to be same as graphite-clickhouse for prevent bugs, ike issue #184
"-v", "/etc/timezone:/etc/timezone:ro",
"-v", "/etc/localtime:/etc/localtime:ro",
"-e", "TZ=" + tz,
"--network", DockerNetwork,
}
cchStart = append(cchStart, c.DockerImage+":"+c.Version)
cmd := exec.Command(DockerBinary, cchStart...)
out, err := cmd.CombinedOutput()
if err == nil {
dateLocal, _ := exec.Command("date").Output()
dateLocalStr := strings.TrimRight(string(dateLocal), "\n")
_, dateOnCCH := containerExec(c.container, []string{"date"})
fmt.Printf("date local %s, on carbon-clickhouse %s\n", dateLocalStr, dateOnCCH)
}
return string(out), err
}
func (c *CarbonClickhouse) Stop(delete bool) (string, error) {
if len(c.container) == 0 {
return "", nil
}
chStop := []string{"stop", c.container}
cmd := exec.Command(DockerBinary, chStop...)
out, err := cmd.CombinedOutput()
if err == nil && delete {
return c.Delete()
}
return string(out), err
}
func (c *CarbonClickhouse) Delete() (string, error) {
if len(c.container) == 0 {
return "", nil
}
chDel := []string{"rm", c.container}
cmd := exec.Command(DockerBinary, chDel...)
out, err := cmd.CombinedOutput()
if err == nil {
c.container = ""
}
c.Cleanup()
return string(out), err
}
func (c *CarbonClickhouse) Cleanup() {
if c.storeDir != "" {
os.RemoveAll(c.storeDir)
c.storeDir = ""
}
}
func (c *CarbonClickhouse) Address() string {
return c.address
}
func (c *CarbonClickhouse) Container() string {
return c.container
}
================================================
FILE: cmd/e2e-test/checks.go
================================================
package main
import (
"bufio"
"fmt"
"net/http"
"os"
"reflect"
"sort"
"strconv"
"strings"
"time"
"github.com/go-graphite/protocol/carbonapi_v3_pb"
"github.com/lomik/graphite-clickhouse/helper/client"
"github.com/lomik/graphite-clickhouse/helper/datetime"
"github.com/lomik/graphite-clickhouse/helper/tests/compare"
)
func isFindCached(header http.Header) (string, bool) {
if header == nil {
return "", false
}
v, exist := header["X-Cached-Find"]
if len(v) == 0 {
return "", false
}
return v[0], exist
}
func requestId(header http.Header) string {
if header == nil {
return ""
}
v, exist := header["X-Gch-Request-Id"]
if exist && len(v) > 0 {
return v[0]
}
return ""
}
func compareFindMatch(errors *[]string, name, url string, actual, expected []client.FindMatch, findCached bool, cacheTTL int, header http.Header) {
var cacheTTLStr string
if findCached {
cacheTTLStr = strconv.Itoa(cacheTTL)
}
id := requestId(header)
if header != nil {
v, actualFindCached := isFindCached(header)
if actualFindCached != findCached || cacheTTLStr != v {
*errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s: X-Cached-Find want '%s', got '%s'", name, id, url, cacheTTLStr, v))
}
}
maxLen := compare.Max(len(expected), len(actual))
for i := 0; i < maxLen; i++ {
if i > len(actual)-1 {
*errors = append(*errors, fmt.Sprintf("- TRY[%s] %s %s [%d] = %+v", name, id, url, i, expected[i]))
} else if i > len(expected)-1 {
*errors = append(*errors, fmt.Sprintf("+ TRY[%s] %s %s [%d] = %+v", name, id, url, i, actual[i]))
} else if expected[i] != actual[i] {
*errors = append(*errors, fmt.Sprintf("- TRY[%s] %s %s [%d] = %+v", name, id, url, i, expected[i]))
*errors = append(*errors, fmt.Sprintf("+ TRY[%s] %s %s [%d] = %+v", name, id, url, i, actual[i]))
}
}
}
func verifyMetricsFind(ch *Clickhouse, gch *GraphiteClickhouse, check *MetricsFindCheck) []string {
var errors []string
httpClient := http.Client{
Timeout: check.Timeout,
}
address := gch.URL()
for _, format := range check.Formats {
name := ""
if url, result, respHeader, err := client.MetricsFind(&httpClient, address, format, check.Query, check.from, check.until); err == nil {
id := requestId(respHeader)
if check.ErrorRegexp != "" {
errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s'", "", id, url, check.ErrorRegexp))
}
compareFindMatch(&errors, name, url, result, check.Result, check.InCache, check.CacheTTL, respHeader)
if len(result) == 0 && len(check.Result) > 0 {
gch.Grep(id)
if len(check.DumpIfEmpty) > 0 {
for _, q := range check.DumpIfEmpty {
if out, err := ch.Query(q); err == nil {
fmt.Fprintf(os.Stderr, "%s\n%s", q, out)
} else {
fmt.Fprintf(os.Stderr, "%s: %s\n", err.Error(), q)
}
}
}
}
if check.CacheTTL > 0 && check.ErrorRegexp == "" {
// second query must be find-cached
name = "cache"
if url, result, respHeader, err = client.MetricsFind(&httpClient, address, format, check.Query, check.from, check.until); err == nil {
compareFindMatch(&errors, name, url, result, check.Result, true, check.CacheTTL, respHeader)
} else {
errStr := strings.TrimRight(err.Error(), "\n")
errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: %s", name, requestId(respHeader), url, errStr))
}
}
} else {
errStr := strings.TrimRight(err.Error(), "\n")
if check.errorRegexp == nil || !check.errorRegexp.MatchString(errStr) {
errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s', got '%s'", "", requestId(respHeader), url, check.ErrorRegexp, errStr))
} else {
fmt.Printf("EXPECTED ERROR, SUCCESS %s : %s\n", url, errStr)
}
}
}
return errors
}
func compareTags(errors *[]string, name, url string, actual, expected []string, findCached bool, cacheTTL int, header http.Header) {
var cacheTTLStr string
if findCached {
cacheTTLStr = strconv.Itoa(cacheTTL)
}
id := requestId(header)
if header != nil {
v, actualFindCached := isFindCached(header)
if actualFindCached != findCached || cacheTTLStr != v {
*errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s: X-Cached-Find want '%s', got '%s'", name, id, url, cacheTTLStr, v))
}
}
maxLen := compare.Max(len(expected), len(actual))
for i := 0; i < maxLen; i++ {
if i > len(actual)-1 {
*errors = append(*errors, fmt.Sprintf("- TRY[%s] %s %s [%d] = %+v", name, id, url, i, expected[i]))
} else if i > len(expected)-1 {
*errors = append(*errors, fmt.Sprintf("+ TRY[%s] %s %s [%d] = %+v", name, id, url, i, actual[i]))
} else if expected[i] != actual[i] {
*errors = append(*errors, fmt.Sprintf("- TRY[%s] %s %s [%d] = %+v", name, id, url, i, expected[i]))
*errors = append(*errors, fmt.Sprintf("+ TRY[%s] %s %s [%d] = %+v", name, id, url, i, actual[i]))
}
}
}
func verifyTags(ch *Clickhouse, gch *GraphiteClickhouse, check *TagsCheck) []string {
var errors []string
httpClient := http.Client{
Timeout: check.Timeout,
}
address := gch.URL()
for _, format := range check.Formats {
var (
result []string
err error
url string
respHeader http.Header
)
name := ""
if check.Names {
url, result, respHeader, err = client.TagsNames(&httpClient, address, format, check.Query, check.Limits, check.from, check.until)
} else {
url, result, respHeader, err = client.TagsValues(&httpClient, address, format, check.Query, check.Limits, check.from, check.until)
}
if err == nil {
id := requestId(respHeader)
if check.ErrorRegexp != "" {
errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s'", "", id, url, check.ErrorRegexp))
}
compareTags(&errors, name, url, result, check.Result, check.InCache, check.CacheTTL, respHeader)
if len(result) == 0 && len(check.Result) > 0 {
gch.Grep(id)
if len(check.DumpIfEmpty) > 0 {
for _, q := range check.DumpIfEmpty {
if out, err := ch.Query(q); err == nil {
fmt.Fprintf(os.Stderr, "%s\n%s", q, out)
} else {
fmt.Fprintf(os.Stderr, "%s: %s\n", err.Error(), q)
}
}
}
}
if check.CacheTTL > 0 && check.ErrorRegexp == "" {
// second query must be find-cached
name = "cache"
if check.Names {
url, result, respHeader, err = client.TagsNames(&httpClient, address, format, check.Query, check.Limits, check.from, check.until)
} else {
url, result, respHeader, err = client.TagsValues(&httpClient, address, format, check.Query, check.Limits, check.from, check.until)
}
if err == nil {
compareTags(&errors, name, url, result, check.Result, true, check.CacheTTL, respHeader)
} else {
errStr := strings.TrimRight(err.Error(), "\n")
errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: %s", name, requestId(respHeader), url, errStr))
}
}
} else {
errStr := strings.TrimRight(err.Error(), "\n")
if check.errorRegexp == nil || !check.errorRegexp.MatchString(errStr) {
errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s', got '%s'", "", requestId(respHeader), url, check.ErrorRegexp, errStr))
} else {
fmt.Printf("EXPECTED ERROR, SUCCESS %s : %s\n", url, errStr)
}
}
}
return errors
}
func compareRender(errors *[]string, name, url string, actual, expected []client.Metric, findCached bool, header http.Header, cacheTTL int) {
var cacheTTLStr string
if findCached {
cacheTTLStr = strconv.Itoa(cacheTTL)
}
sort.Slice(actual, func(i, j int) bool {
return actual[i].Name < actual[j].Name
})
id := requestId(header)
if header != nil {
v, actualFindCached := isFindCached(header)
if actualFindCached != findCached || cacheTTLStr != v {
*errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s: X-Cached-Find want '%s', got '%s'", name, id, url, cacheTTLStr, v))
}
}
maxLen := compare.Max(len(expected), len(actual))
for i := 0; i < maxLen; i++ {
if i > len(actual)-1 {
*errors = append(*errors, fmt.Sprintf("- TRY[%s] %s %s [%d] = %+v", name, id, url, i, expected[i]))
} else if i > len(expected)-1 {
*errors = append(*errors, fmt.Sprintf("+ TRY[%s] %s %s [%d] = %+v", name, id, url, i, actual[i]))
} else if actual[i].Name != expected[i].Name {
*errors = append(*errors, fmt.Sprintf("- TRY[%s] %s %s [%d] = %+v", name, id, url, i, expected[i]))
*errors = append(*errors, fmt.Sprintf("+ TRY[%s] %s %s [%d] = %+v", name, id, url, i, actual[i]))
} else {
if actual[i].PathExpression != expected[i].PathExpression {
*errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].PathExpression, got '%s', want '%s'", name, id, url, actual[i].Name, i, actual[i].PathExpression, expected[i].PathExpression))
}
if actual[i].ConsolidationFunc != expected[i].ConsolidationFunc {
*errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].ConsolidationFunc, got '%s', want '%s'", name, id, url, actual[i].Name, i, actual[i].ConsolidationFunc, expected[i].ConsolidationFunc))
}
if actual[i].ConsolidationFunc != expected[i].ConsolidationFunc {
*errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].ConsolidationFunc, got '%s', want '%s'", name, id, url, actual[i].Name, i, actual[i].ConsolidationFunc, expected[i].ConsolidationFunc))
}
if actual[i].StartTime != expected[i].StartTime {
*errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].StartTime, got %d, want %d", name, id, url, actual[i].Name, i, actual[i].StartTime, expected[i].StartTime))
}
if actual[i].StopTime != expected[i].StopTime {
*errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].StopTime, got %d, want %d", name, id, url, actual[i].Name, i, actual[i].StopTime, expected[i].StopTime))
}
if actual[i].StepTime != expected[i].StepTime {
*errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].StepTime, got %d, want %d", name, id, url, actual[i].Name, i, actual[i].StepTime, expected[i].StepTime))
}
if actual[i].RequestStartTime != expected[i].RequestStartTime {
*errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].RequestStartTime, got %d, want %d", name, id, url, actual[i].Name, i, actual[i].RequestStartTime, expected[i].RequestStartTime))
}
if actual[i].RequestStopTime != expected[i].RequestStopTime {
*errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].RequestStopTime, got %d, want %d", name, id, url, actual[i].Name, i, actual[i].RequestStopTime, expected[i].RequestStopTime))
}
if actual[i].HighPrecisionTimestamps != expected[i].HighPrecisionTimestamps {
*errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].HighPrecisionTimestamps, got %v, want %v", name, id, url, actual[i].Name, i, actual[i].HighPrecisionTimestamps, expected[i].HighPrecisionTimestamps))
}
if !reflect.DeepEqual(actual[i].AppliedFunctions, expected[i].AppliedFunctions) {
*errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].AppliedFunctions, got '%s', want '%s'", name, id, url, actual[i].Name, i, actual[i].AppliedFunctions, expected[i].AppliedFunctions))
}
if !compare.NearlyEqual(float64(actual[i].XFilesFactor), float64(expected[i].XFilesFactor)) {
*errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].XFilesFactor, got %g, want %g", name, id, url, actual[i].Name, i, actual[i].XFilesFactor, expected[i].XFilesFactor))
}
if !compare.NearlyEqualSlice(actual[i].Values, expected[i].Values) {
*errors = append(*errors, fmt.Sprintf("TRY[%s] %s %s '%s': mismatch [%d].Values, got %g, want %g", name, id, url, actual[i].Name, i, actual[i].Values, expected[i].Values))
}
}
}
}
func parseFilteringFunctions(strFilteringFuncs []string) ([]*carbonapi_v3_pb.FilteringFunction, error) {
res := make([]*carbonapi_v3_pb.FilteringFunction, 0, len(strFilteringFuncs))
for _, strFF := range strFilteringFuncs {
strFFSplit := strings.Split(strFF, "(")
if len(strFFSplit) != 2 {
return nil, fmt.Errorf("could not parse filtering function: %s", strFF)
}
name := strFFSplit[0]
args := strings.Split(strFFSplit[1], ",")
for i := range args {
args[i] = strings.TrimSpace(args[i])
args[i] = strings.Trim(args[i], ")'")
}
res = append(res, &carbonapi_v3_pb.FilteringFunction{Name: name, Arguments: args})
}
return res, nil
}
func verifyRender(ch *Clickhouse, gch *GraphiteClickhouse, check *RenderCheck, defaultPreision time.Duration) []string {
var errors []string
httpClient := http.Client{
Timeout: check.Timeout,
}
address := gch.URL()
from := datetime.TimestampTruncate(check.from, defaultPreision)
until := datetime.TimestampTruncate(check.until, defaultPreision)
for _, format := range check.Formats {
var filteringFunctions []*carbonapi_v3_pb.FilteringFunction
if format == client.FormatPb_v3 {
var err error
filteringFunctions, err = parseFilteringFunctions(check.FilteringFunctions)
if err != nil {
errors = append(errors, err.Error())
continue
}
}
if url, result, respHeader, err := client.Render(&httpClient, address, format, check.Targets, filteringFunctions, check.MaxDataPoints, from, until); err == nil {
id := requestId(respHeader)
name := ""
if check.ErrorRegexp != "" {
errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s'", "", id, url, check.ErrorRegexp))
}
compareRender(&errors, name, url, result, check.result, check.InCache, respHeader, check.CacheTTL)
if len(result) == 0 && len(check.result) > 0 {
gch.Grep(id)
if len(check.DumpIfEmpty) > 0 {
for _, q := range check.DumpIfEmpty {
if out, err := ch.Query(q); err == nil {
fmt.Fprintf(os.Stderr, "%s\n%s", q, out)
} else {
fmt.Fprintf(os.Stderr, "%s: %s\n", err.Error(), q)
}
}
}
}
if check.CacheTTL > 0 && check.ErrorRegexp == "" {
// second query must be find-cached
name = "cache"
if url, result, respHeader, err = client.Render(&httpClient, address, format, check.Targets, filteringFunctions, check.MaxDataPoints, from, until); err == nil {
compareRender(&errors, name, url, result, check.result, true, respHeader, check.CacheTTL)
} else {
errStr := strings.TrimRight(err.Error(), "\n")
errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: %s", name, requestId(respHeader), url, errStr))
}
}
} else {
errStr := strings.TrimRight(err.Error(), "\n")
if check.errorRegexp == nil || !check.errorRegexp.MatchString(errStr) {
errors = append(errors, fmt.Sprintf("TRY[%s] %s %s: want error with '%s', got '%s'", "", requestId(respHeader), url, check.ErrorRegexp, errStr))
} else {
fmt.Printf("EXPECTED ERROR, SUCCESS %s : %s\n", url, errStr)
}
}
}
return errors
}
func debug(test *TestSchema, ch *Clickhouse, gch *GraphiteClickhouse) {
for {
cmd := gch.Cmd()
fmt.Println(cmd)
fmt.Printf("graphite-clickhouse URL: %s , clickhouse URL: %s , proxy URL: %s (delay %v)\n",
gch.URL(), ch.URL(), test.Proxy.URL(), test.Proxy.GetDelay())
fmt.Printf("graphite-clickhouse log: %s , clickhouse container: %s\n",
gch.storeDir+"/graphite-clickhouse.log", ch.container)
fmt.Println("Some queries was failed, press y for continue after debug test, k for kill graphite-clickhouse:")
in := bufio.NewScanner(os.Stdin)
in.Scan()
s := in.Text()
if s == "y" || s == "Y" {
break
} else if s == "k" || s == "K" {
gch.Stop(false)
}
}
}
================================================
FILE: cmd/e2e-test/clickhouse.go
================================================
package main
import (
"bytes"
"errors"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"strconv"
"strings"
"github.com/msaf1980/go-stringutils"
)
var (
ClickhouseContainerName = "clickhouse-server-gch-test"
ClickhouseOldImage = "yandex/clickhouse-server"
ClickhouseDefaultImage = "clickhouse/clickhouse-server"
)
type Clickhouse struct {
Version string `toml:"version"`
Dir string `toml:"dir"`
TLSEnabled bool `toml:"tls"`
DockerImage string `toml:"image"`
TZ string `toml:"tz"` // override timezone
httpAddress string `toml:"-"`
httpsAddress string `toml:"-"`
url string `toml:"-"`
tlsurl string `toml:"-"`
container string `toml:"-"`
}
func (c *Clickhouse) CheckConfig(rootDir string) error {
if c.Version == "" {
c.Version = "latest"
}
if len(c.Dir) == 0 {
return ErrNoSetDir
}
if !strings.HasPrefix(c.Dir, "/") {
c.Dir = rootDir + "/" + c.Dir
}
if c.DockerImage == "" {
if c.Version == "latest" {
c.DockerImage = ClickhouseDefaultImage
} else {
splitV := strings.Split(c.Version, ".")
majorV, err := strconv.Atoi(splitV[0])
if err != nil {
c.DockerImage = ClickhouseDefaultImage
} else if majorV >= 21 {
c.DockerImage = ClickhouseDefaultImage
} else {
c.DockerImage = ClickhouseOldImage
}
}
}
return nil
}
func (c *Clickhouse) Key() string {
return c.DockerImage + ":" + c.Version + " " + c.Dir + " TZ " + c.TZ
}
func (c *Clickhouse) Start() (string, error) {
var err error
c.httpAddress, err = getFreeTCPPort("")
if err != nil {
return "", err
}
port := strings.Split(c.httpAddress, ":")[1]
c.url = "http://" + c.httpAddress
c.container = ClickhouseContainerName
// tz, _ := localTZLocationName()
chStart := []string{"run", "-d",
"--name", c.container,
"--ulimit", "nofile=262144:262144",
"-p", port + ":8123",
// "-e", "TZ=" + tz, // workaround for TZ=":/etc/localtime"
"-v", c.Dir + "/config.xml:/etc/clickhouse-server/config.xml",
"-v", c.Dir + "/users.xml:/etc/clickhouse-server/users.xml",
"-v", c.Dir + "/rollup.xml:/etc/clickhouse-server/config.d/rollup.xml",
"-v", c.Dir + "/init.sql:/docker-entrypoint-initdb.d/init.sql",
"--network", DockerNetwork,
}
if c.TLSEnabled {
c.httpsAddress, err = getFreeTCPPort("")
if err != nil {
return "", err
}
port = strings.Split(c.httpsAddress, ":")[1]
c.tlsurl = "https://" + c.httpsAddress
chStart = append(chStart,
"-v", c.Dir+"/server.crt:/etc/clickhouse-server/server.crt",
"-v", c.Dir+"/server.key:/etc/clickhouse-server/server.key",
"-v", c.Dir+"/rootCA.crt:/etc/clickhouse-server/rootCA.crt",
"-p", port+":8443",
)
}
if c.TZ != "" {
chStart = append(chStart, "-e", "TZ="+c.TZ)
}
chStart = append(chStart, c.DockerImage+":"+c.Version)
cmd := exec.Command(DockerBinary, chStart...)
out, err := cmd.CombinedOutput()
return string(out), err
}
func (c *Clickhouse) Stop(delete bool) (string, error) {
if len(c.container) == 0 {
return "", nil
}
chStop := []string{"stop", c.container}
cmd := exec.Command(DockerBinary, chStop...)
out, err := cmd.CombinedOutput()
if err == nil && delete {
return c.Delete()
}
return string(out), err
}
func (c *Clickhouse) Delete() (string, error) {
if len(c.container) == 0 {
return "", nil
}
chDel := []string{"rm", c.container}
cmd := exec.Command(DockerBinary, chDel...)
out, err := cmd.CombinedOutput()
if err == nil {
c.container = ""
}
return string(out), err
}
func (c *Clickhouse) URL() string {
return c.url
}
func (c *Clickhouse) TLSURL() string {
return c.tlsurl
}
func (c *Clickhouse) Container() string {
return c.container
}
func (c *Clickhouse) Exec(sql string) (bool, string) {
return containerExec(c.container, []string{"sh", "-c", "clickhouse-client -q '" + sql + "'"})
}
func (c *Clickhouse) Query(sql string) (string, error) {
reader := strings.NewReader(sql)
request, err := http.NewRequest(http.MethodPost, c.URL(), reader)
if err != nil {
return "", err
}
resp, err := http.DefaultClient.Do(request)
if err != nil {
return "", err
}
defer resp.Body.Close()
msg, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
if resp.StatusCode != http.StatusOK {
return "", errors.New(resp.Status + ": " + string(bytes.TrimRight(msg, "\n")))
}
return string(msg), nil
}
func (c *Clickhouse) Alive() bool {
if len(c.container) == 0 {
return false
}
req, err := http.DefaultClient.Get(c.URL())
if err != nil {
return false
}
defer req.Body.Close()
return req.StatusCode == http.StatusOK
}
func (c *Clickhouse) CopyLog(destDir string, tail uint64) error {
if len(c.container) == 0 {
return nil
}
dest := destDir + "/clickhouse-server.log"
chArgs := []string{"cp", c.container + ":/var/log/clickhouse-server/clickhouse-server.log", dest}
cmd := exec.Command(DockerBinary, chArgs...)
out, err := cmd.CombinedOutput()
if err != nil {
return errors.New(err.Error() + ": " + string(bytes.TrimRight(out, "\n")))
}
if tail > 0 {
out, _ := exec.Command("tail", "-"+strconv.FormatUint(tail, 10), dest).Output()
fmt.Fprintf(os.Stderr, "CLICKHOUSE-SERVER.LOG %s", stringutils.UnsafeString(out))
}
return nil
}
func (c *Clickhouse) CopyErrLog(destDir string, tail uint64) error {
if len(c.container) == 0 {
return nil
}
dest := destDir + "/clickhouse-server.err.log"
chArgs := []string{"cp", c.container + ":/var/log/clickhouse-server/clickhouse-server.err.log", dest}
cmd := exec.Command(DockerBinary, chArgs...)
out, err := cmd.CombinedOutput()
if err != nil {
return errors.New(err.Error() + ": " + string(bytes.TrimRight(out, "\n")))
}
if tail > 0 {
out, _ := exec.Command("tail", "-"+strconv.FormatUint(tail, 10), dest).Output()
fmt.Fprintf(os.Stderr, "CLICKHOUSE-SERVER.ERR %s", stringutils.UnsafeString(out))
}
return nil
}
================================================
FILE: cmd/e2e-test/container.go
================================================
package main
import (
"os/exec"
"strings"
)
var (
DockerBinary string
DockerNetwork string = "graphite-ch-test"
)
func imageDelete(image, version string) (bool, string) {
if len(DockerBinary) == 0 {
panic("docker not set")
}
chArgs := []string{"rmi", image + ":" + version}
cmd := exec.Command(DockerBinary, chArgs...)
out, err := cmd.CombinedOutput()
s := strings.Trim(string(out), "\n")
if err == nil {
return true, s
}
return false, err.Error() + ": " + s
}
func containerExist(name string) (bool, string) {
if len(DockerBinary) == 0 {
panic("docker not set")
}
chInspect := []string{"inspect", "--format", "'{{.Name}}'", name}
cmd := exec.Command(DockerBinary, chInspect...)
out, err := cmd.CombinedOutput()
s := strings.Trim(string(out), "\n")
if err == nil {
return true, s
}
return false, err.Error() + ": " + s
}
func containerRemove(name string) (bool, string) {
if len(DockerBinary) == 0 {
panic("docker not set")
}
chInspect := []string{"rm", "-f", name}
cmd := exec.Command(DockerBinary, chInspect...)
out, err := cmd.CombinedOutput()
s := strings.Trim(string(out), "\n")
if err == nil {
return true, s
}
return false, err.Error() + ": " + s
}
func containerExec(name string, args []string) (bool, string) {
if len(DockerBinary) == 0 {
panic("docker not set")
}
dCmd := []string{"exec", name}
dCmd = append(dCmd, args...)
cmd := exec.Command(DockerBinary, dCmd...)
out, err := cmd.CombinedOutput()
s := strings.Trim(string(out), "\n")
if err == nil {
return true, s
}
return false, err.Error() + ": " + s
}
================================================
FILE: cmd/e2e-test/e2etesting.go
================================================
package main
import (
"bufio"
"fmt"
"net"
"os"
"path"
"regexp"
"sort"
"strconv"
"strings"
"time"
"go.uber.org/zap"
"github.com/lomik/graphite-clickhouse/helper/client"
"github.com/lomik/graphite-clickhouse/helper/datetime"
"github.com/pelletier/go-toml"
)
var (
preSQL = []string{
"TRUNCATE TABLE IF EXISTS graphite_reverse",
"TRUNCATE TABLE IF EXISTS graphite",
"TRUNCATE TABLE IF EXISTS graphite_index",
"TRUNCATE TABLE IF EXISTS graphite_tags",
}
)
type Point struct {
Value float64 `toml:"value"`
Time string `toml:"time"`
Delay time.Duration `toml:"delay"`
time int64 `toml:"-"`
}
type InputMetric struct {
Name string `toml:"name"`
Points []Point `toml:"points"`
Round time.Duration `toml:"round"`
}
type Metric struct {
Name string `toml:"name"`
PathExpression string `toml:"path"`
ConsolidationFunc string `toml:"consolidation"`
StartTime string `toml:"start"`
StopTime string `toml:"stop"`
StepTime int64 `toml:"step"`
XFilesFactor float32 `toml:"xfiles"`
HighPrecisionTimestamps bool `toml:"high_precision"`
Values []float64 `toml:"values"`
AppliedFunctions []string `toml:"applied_functions"`
RequestStartTime string `toml:"req_start"`
RequestStopTime string `toml:"req_stop"`
}
type RenderCheck struct {
Name string `toml:"name"`
Formats []client.FormatType `toml:"formats"`
From string `toml:"from"`
Until string `toml:"until"`
Targets []string `toml:"targets"`
MaxDataPoints int64 `toml:"max_data_points"`
FilteringFunctions []string `toml:"filtering_functions"`
Timeout time.Duration `toml:"timeout"`
DumpIfEmpty []string `toml:"dump_if_empty"`
Optimize []string `toml:"optimize"` // optimize tables before run tests
InCache bool `toml:"in_cache"` // already in cache
CacheTTL int `toml:"cache_ttl"`
ProxyDelay time.Duration `toml:"proxy_delay"`
ProxyBreakWithCode int `toml:"proxy_break_with_code"`
Result []Metric `toml:"result"`
ErrorRegexp string `toml:"error_regexp"`
from int64 `toml:"-"`
until int64 `toml:"-"`
errorRegexp *regexp.Regexp `toml:"-"`
result []client.Metric `toml:"-"`
}
type MetricsFindCheck struct {
Name string `toml:"name"`
Formats []client.FormatType `toml:"formats"`
From string `toml:"from"`
Until string `toml:"until"`
Query string `toml:"query"`
Timeout time.Duration `toml:"timeout"`
DumpIfEmpty []string `toml:"dump_if_empty"`
InCache bool `toml:"in_cache"` // already in cache
CacheTTL int `toml:"cache_ttl"`
ProxyDelay time.Duration `toml:"proxy_delay"`
ProxyBreakWithCode int `toml:"proxy_break_with_code"`
Result []client.FindMatch `toml:"result"`
ErrorRegexp string `toml:"error_regexp"`
from int64 `toml:"-"`
until int64 `toml:"-"`
errorRegexp *regexp.Regexp `toml:"-"`
}
type TagsCheck struct {
Name string `toml:"name"`
Names bool `toml:"names"` // TagNames or TagValues
Formats []client.FormatType `toml:"formats"`
From string `toml:"from"`
Until string `toml:"until"`
Query string `toml:"query"`
Limits uint64 `toml:"limits"`
Timeout time.Duration `toml:"timeout"`
DumpIfEmpty []string `toml:"dump_if_empty"`
InCache bool `toml:"in_cache"` // already in cache
CacheTTL int `toml:"cache_ttl"`
ProxyDelay time.Duration `toml:"proxy_delay"`
ProxyBreakWithCode int `toml:"proxy_break_with_code"`
Result []string `toml:"result"`
ErrorRegexp string `toml:"error_regexp"`
from int64 `toml:"-"`
until int64 `toml:"-"`
errorRegexp *regexp.Regexp `toml:"-"`
}
type TestSchema struct {
Input []InputMetric `toml:"input"` // carbon-clickhouse input
Clickhouse []Clickhouse `toml:"clickhouse"`
Proxy HttpReverseProxy `toml:"clickhouse_proxy"`
Cch CarbonClickhouse `toml:"carbon_clickhouse"`
Gch []GraphiteClickhouse `toml:"graphite_clickhouse"`
FindChecks []*MetricsFindCheck `toml:"find_checks"`
TagsChecks []*TagsCheck `toml:"tags_checks"`
RenderChecks []*RenderCheck `toml:"render_checks"`
Precision time.Duration `toml:"precision"`
dir string `toml:"-"`
name string `toml:"-"` // test alias (from config name)
chVersions map[string]bool `toml:"-"`
// input map[string][]Point `toml:"-"`
}
func (schema *TestSchema) HasTLSSettings() bool {
return strings.Contains(schema.dir, "tls")
}
func getFreeTCPPort(name string) (string, error) {
if len(name) == 0 {
name = "127.0.0.1:0"
} else if !strings.Contains(name, ":") {
name = name + ":0"
}
addr, err := net.ResolveTCPAddr("tcp", name)
if err != nil {
return name, err
}
l, err := net.ListenTCP("tcp", addr)
if err != nil {
return name, err
}
defer l.Close()
return l.Addr().String(), nil
}
func sendPlain(network, address string, metrics []InputMetric) error {
if conn, err := net.DialTimeout(network, address, time.Second); err != nil {
return err
} else {
bw := bufio.NewWriter(conn)
for _, m := range metrics {
conn.SetDeadline(time.Now().Add(time.Second))
for _, point := range m.Points {
if _, err = fmt.Fprintf(bw, "%s %f %d\n", m.Name, point.Value, point.time); err != nil {
conn.Close()
return err
}
if point.Delay > 0 {
if err = bw.Flush(); err != nil {
conn.Close()
return err
}
time.Sleep(point.Delay)
}
}
}
if err = bw.Flush(); err != nil {
conn.Close()
return err
}
return conn.Close()
}
}
func verifyGraphiteClickhouse(test *TestSchema, gch *GraphiteClickhouse, clickhouse *Clickhouse, testDir, clickhouseDir string, verbose, breakOnError bool, logger *zap.Logger) (testSuccess bool, verifyCount, verifyFailed int) {
testSuccess = true
err := gch.Start(testDir, clickhouse.URL(), test.Proxy.URL(), clickhouse.TLSURL())
if err != nil {
logger.Error("starting graphite-clickhouse",
zap.String("config", test.name),
zap.String("clickhouse version", clickhouse.Version),
zap.String("clickhouse config", clickhouseDir),
zap.String("graphite-clickhouse config", gch.ConfigTpl),
zap.Error(err),
)
testSuccess = false
return
}
for i := 100; i < 1000; i += 200 {
time.Sleep(time.Duration(i) * time.Millisecond)
if gch.Alive() {
break
}
}
// start tests
for n, check := range test.FindChecks {
verifyCount++
test.Proxy.SetDelay(check.ProxyDelay)
test.Proxy.SetBreakStatusCode(check.ProxyBreakWithCode)
if len(check.Formats) == 0 {
check.Formats = []client.FormatType{client.FormatPb_v3}
}
if errs := verifyMetricsFind(clickhouse, gch, check); len(errs) > 0 {
verifyFailed++
for _, e := range errs {
fmt.Fprintln(os.Stderr, e)
}
logger.Error("verify metrics find",
zap.String("config", test.name),
zap.String("clickhouse version", clickhouse.Version),
zap.String("clickhouse config", clickhouseDir),
zap.String("graphite-clickhouse config", gch.ConfigTpl),
zap.String("query", check.Query),
zap.String("from_raw", check.From),
zap.String("until_raw", check.Until),
zap.Int64("from", check.from),
zap.Int64("until", check.until),
zap.String("name", check.Name+"["+strconv.Itoa(n)+"]"),
)
if breakOnError {
debug(test, clickhouse, gch)
}
} else if verbose {
logger.Info("verify metrics find",
zap.String("config", test.name),
zap.String("clickhouse version", clickhouse.Version),
zap.String("clickhouse config", clickhouseDir),
zap.String("graphite-clickhouse config", gch.ConfigTpl),
zap.String("query", check.Query),
zap.String("from_raw", check.From),
zap.String("until_raw", check.Until),
zap.Int64("from", check.from),
zap.Int64("until", check.until),
zap.String("name", check.Name+"["+strconv.Itoa(n)+"]"),
)
}
}
for n, check := range test.TagsChecks {
verifyCount++
test.Proxy.SetDelay(check.ProxyDelay)
test.Proxy.SetBreakStatusCode(check.ProxyBreakWithCode)
if len(check.Formats) == 0 {
check.Formats = []client.FormatType{client.FormatJSON}
}
if errs := verifyTags(clickhouse, gch, check); len(errs) > 0 {
verifyFailed++
for _, e := range errs {
fmt.Fprintln(os.Stderr, e)
}
logger.Error("verify tags",
zap.String("config", test.name),
zap.String("clickhouse version", clickhouse.Version),
zap.String("clickhouse config", clickhouseDir),
zap.String("graphite-clickhouse config", gch.ConfigTpl),
zap.Bool("name", check.Names),
zap.String("query", check.Query),
zap.String("from_raw", check.From),
zap.String("until_raw", check.Until),
zap.Int64("from", check.from),
zap.Int64("until", check.until),
zap.String("name", check.Name+"["+strconv.Itoa(n)+"]"),
)
if breakOnError {
debug(test, clickhouse, gch)
}
} else if verbose {
logger.Info("verify tags",
zap.String("config", test.name),
zap.String("clickhouse version", clickhouse.Version),
zap.String("clickhouse config", clickhouseDir),
zap.String("graphite-clickhouse config", gch.ConfigTpl),
zap.Bool("name", check.Names),
zap.String("query", check.Query),
zap.String("from_raw", check.From),
zap.String("until_raw", check.Until),
zap.Int64("from", check.from),
zap.Int64("until", check.until),
zap.String("name", check.Name+"["+strconv.Itoa(n)+"]"),
)
}
}
for n, check := range test.RenderChecks {
verifyCount++
test.Proxy.SetDelay(check.ProxyDelay)
test.Proxy.SetBreakStatusCode(check.ProxyBreakWithCode)
if len(check.Formats) == 0 {
check.Formats = []client.FormatType{client.FormatPb_v3}
}
if len(check.Optimize) > 0 {
for _, table := range check.Optimize {
if success, out := clickhouse.Exec("OPTIMIZE TABLE " + table + " FINAL"); !success {
logger.Error("optimize table",
zap.String("config", test.name),
zap.String("clickhouse version", clickhouse.Version),
zap.String("clickhouse config", clickhouseDir),
zap.String("graphite-clickhouse config", gch.ConfigTpl),
zap.Strings("targets", check.Targets),
zap.Strings("filtering_functions", check.FilteringFunctions),
zap.String("from_raw", check.From),
zap.String("until_raw", check.Until),
zap.Int64("from", check.from),
zap.Int64("until", check.until),
zap.String("name", check.Name+"["+strconv.Itoa(n)+"]"),
zap.String("table", table),
zap.String("out", out),
)
time.Sleep(5 * time.Second)
}
}
}
if errs := verifyRender(clickhouse, gch, check, test.Precision); len(errs) > 0 {
verifyFailed++
for _, e := range errs {
fmt.Fprintln(os.Stderr, e)
}
logger.Error("verify render",
zap.String("config", test.name),
zap.String("clickhouse version", clickhouse.Version),
zap.String("clickhouse config", clickhouseDir),
zap.String("graphite-clickhouse config", gch.ConfigTpl),
zap.Strings("targets", check.Targets),
zap.Strings("filtering_functions", check.FilteringFunctions),
zap.String("from_raw", check.From),
zap.String("until_raw", check.Until),
zap.Int64("from", check.from),
zap.Int64("until", check.until),
zap.String("name", check.Name+"["+strconv.Itoa(n)+"]"),
)
if breakOnError {
debug(test, clickhouse, gch)
}
} else if verbose {
logger.Info("verify render",
zap.String("config", test.name),
zap.String("clickhouse version", clickhouse.Version),
zap.String("clickhouse config", clickhouseDir),
zap.String("graphite-clickhouse config", gch.ConfigTpl),
zap.Strings("targets", check.Targets),
zap.Strings("filtering_functions", check.FilteringFunctions),
zap.String("from_raw", check.From),
zap.String("until_raw", check.Until),
zap.Int64("from", check.from),
zap.Int64("until", check.until),
zap.String("name", check.Name+"["+strconv.Itoa(n)+"]"),
)
}
}
if verifyFailed > 0 {
testSuccess = false
logger.Error("verify",
zap.String("config", test.name),
zap.String("clickhouse version", clickhouse.Version),
zap.String("clickhouse config", clickhouseDir),
zap.String("graphite-clickhouse config", gch.ConfigTpl),
zap.Int64("count", int64(verifyCount)),
zap.Int64("failed", int64(verifyFailed)),
)
}
err = gch.Stop(true)
if err != nil {
logger.Error("stoping graphite-clickhouse",
zap.String("config", test.name),
zap.String("gch", gch.ConfigTpl),
zap.String("clickhouse version", clickhouse.Version),
zap.String("clickhouse config", clickhouseDir),
zap.Error(err),
)
testSuccess = false
}
return
}
func testGraphiteClickhouse(test *TestSchema, clickhouse *Clickhouse, testDir, rootDir string, verbose, breakOnError bool, logger *zap.Logger) (testSuccess bool, verifyCount, verifyFailed int) {
testSuccess = true
for _, sql := range preSQL {
if success, out := clickhouse.Exec(sql); !success {
logger.Error("pre-execute",
zap.String("config", test.name),
zap.Any("clickhouse version", clickhouse.Version),
zap.String("clickhouse config", clickhouse.Dir),
zap.String("sql", sql),
zap.String("out", out),
)
return
}
}
if err := test.Proxy.Start(clickhouse.URL()); err != nil {
logger.Error("starting clickhouse proxy",
zap.String("config", test.name),
zap.Any("clickhouse version", clickhouse.Version),
zap.String("clickhouse config", clickhouse.Dir),
zap.Error(err),
)
return
}
out, err := test.Cch.Start(testDir, "http://"+clickhouse.Container()+":8123")
if err != nil {
logger.Error("starting carbon-clickhouse",
zap.String("config", test.name),
zap.String("clickhouse version", clickhouse.Version),
zap.String("clickhouse config", clickhouse.Dir),
zap.Error(err),
zap.String("out", out),
)
testSuccess = false
}
if testSuccess {
logger.Info("starting e2e test",
zap.String("config", test.name),
zap.String("clickhouse version", clickhouse.Version),
zap.String("clickhouse config", clickhouse.Dir),
)
time.Sleep(200 * time.Millisecond)
// Populate test data
err = sendPlain("tcp", test.Cch.address, test.Input)
if err != nil {
logger.Error("send plain to carbon-clickhouse",
zap.String("config", test.name),
zap.String("clickhouse version", clickhouse.Version),
zap.String("clickhouse config", clickhouse.Dir),
zap.Error(err),
)
testSuccess = false
}
if testSuccess {
time.Sleep(2 * time.Second)
}
if testSuccess {
for _, gch := range test.Gch {
stepSuccess, vCount, vFailed := verifyGraphiteClickhouse(test, &gch, clickhouse, testDir, clickhouse.Dir, verbose, breakOnError, logger)
verifyCount += vCount
verifyFailed += vFailed
if !stepSuccess {
testSuccess = false
}
}
}
}
out, err = test.Cch.Stop(true)
if err != nil {
logger.Error("stoping carbon-clickhouse",
zap.String("config", test.name),
zap.String("clickhouse version", clickhouse.Version),
zap.String("clickhouse config", clickhouse.Dir),
zap.Error(err),
zap.String("out", out),
)
testSuccess = false
}
test.Proxy.Stop()
if testSuccess {
logger.Info("end e2e test",
zap.String("config", test.name),
zap.String("status", "success"),
zap.String("clickhouse version", clickhouse.Version),
zap.String("clickhouse config", clickhouse.Dir),
)
} else {
logger.Error("end e2e test",
zap.String("config", test.name),
zap.String("status", "failed"),
zap.String("clickhouse version", clickhouse.Version),
zap.String("clickhouse config", clickhouse.Dir),
)
}
return
}
func runTest(cfg *MainConfig, clickhouse *Clickhouse, rootDir string, now time.Time, verbose, breakOnError bool, logger *zap.Logger) (failed, total, verifyCount, verifyFailed int) {
var isRunning bool
total++
if exist, out := containerExist(CchContainerName); exist {
logger.Error("carbon-clickhouse already exist",
zap.String("container", CchContainerName),
zap.String("out", out),
)
isRunning = true
}
if isRunning {
failed++
return
}
success, vCount, vFailed := testGraphiteClickhouse(cfg.Test, clickhouse, cfg.Test.dir, rootDir, verbose, breakOnError, logger)
if !success {
failed++
}
verifyCount += vCount
verifyFailed += vFailed
return
}
func clickhouseStart(clickhouse *Clickhouse, logger *zap.Logger) bool {
out, err := clickhouse.Start()
if err != nil {
logger.Error("starting clickhouse",
zap.Any("clickhouse version", clickhouse.Version),
zap.String("clickhouse config", clickhouse.Dir),
zap.Error(err),
zap.String("out", out),
)
clickhouse.Stop(true)
return false
}
return true
}
func clickhouseStop(clickhouse *Clickhouse, logger *zap.Logger) (result bool) {
result = true
if !clickhouse.Alive() {
clickhouse.CopyLog(os.TempDir(), 10)
result = false
}
out, err := clickhouse.Stop(true)
if err != nil {
logger.Error("stoping clickhouse",
zap.String("clickhouse version", clickhouse.Version),
zap.String("clickhouse config", clickhouse.Dir),
zap.Error(err),
zap.String("out", out),
)
result = false
}
return result
}
func initTest(cfg *MainConfig, rootDir string, now time.Time, verbose, breakOnError bool, logger *zap.Logger) bool {
tz, err := datetime.Timezone("")
if err != nil {
fmt.Printf("can't get timezone: %s\n", err.Error())
os.Exit(1)
}
// prepare
for n, m := range cfg.Test.Input {
for i := range m.Points {
m.Points[i].time = datetime.DateParamToEpoch(m.Points[i].Time, tz, now, cfg.Test.Precision)
if m.Points[i].time == 0 {
err = ErrTimestampInvalid
}
if err != nil {
logger.Error("failed to read config",
zap.String("config", cfg.Test.name),
zap.Error(err),
zap.String("input", m.Name),
zap.Int("metric", n),
zap.Int("point", i),
zap.String("time", m.Points[i].Time),
)
return false
}
}
}
for n, find := range cfg.Test.FindChecks {
if find.Timeout == 0 {
find.Timeout = 10 * time.Second
}
find.from = datetime.DateParamToEpoch(find.From, tz, now, cfg.Test.Precision)
if find.from == 0 && find.From != "" {
err = ErrTimestampInvalid
}
if err != nil {
logger.Error("failed to read config",
zap.String("config", cfg.Test.name),
zap.Error(err),
zap.String("query", find.Query),
zap.String("from", find.From),
zap.Int("step", n),
)
return false
}
find.until = datetime.DateParamToEpoch(find.Until, tz, now, cfg.Test.Precision)
if find.until == 0 && find.Until != "" {
err = ErrTimestampInvalid
}
if err != nil {
logger.Error("failed to read config",
zap.String("config", cfg.Test.name),
zap.Error(err),
zap.String("query", find.Query),
zap.String("until", find.Until),
zap.Int("step", n),
)
return false
}
if find.ErrorRegexp != "" {
find.errorRegexp = regexp.MustCompile(find.ErrorRegexp)
}
}
for n, tags := range cfg.Test.TagsChecks {
if tags.Timeout == 0 {
tags.Timeout = 10 * time.Second
}
tags.from = datetime.DateParamToEpoch(tags.From, tz, now, cfg.Test.Precision)
if tags.from == 0 && tags.From != "" {
err = ErrTimestampInvalid
}
if err != nil {
logger.Error("failed to read config",
zap.String("config", cfg.Test.name),
zap.Error(err),
zap.String("query", tags.Query),
zap.String("from", tags.From),
zap.Int("find", n),
)
return false
}
tags.until = datetime.DateParamToEpoch(tags.Until, tz, now, cfg.Test.Precision)
if tags.until == 0 && tags.Until != "" {
err = ErrTimestampInvalid
}
if err != nil {
logger.Error("failed to read config",
zap.String("config", cfg.Test.name),
zap.Error(err),
zap.String("query", tags.Query),
zap.String("until", tags.Until),
zap.Int("tags", n),
zap.Bool("names", tags.Names),
)
return false
}
if tags.ErrorRegexp != "" {
tags.errorRegexp = regexp.MustCompile(tags.ErrorRegexp)
}
}
for n, r := range cfg.Test.RenderChecks {
if r.Timeout == 0 {
r.Timeout = 10 * time.Second
}
r.from = datetime.DateParamToEpoch(r.From, tz, now, cfg.Test.Precision)
if r.from == 0 && r.From != "" {
err = ErrTimestampInvalid
}
if err != nil {
logger.Error("failed to read config",
zap.String("config", cfg.Test.name),
zap.Error(err),
zap.Strings("targets", r.Targets),
zap.String("from", r.From),
zap.Int("render", n),
)
return false
}
r.until = datetime.DateParamToEpoch(r.Until, tz, now, cfg.Test.Precision)
if r.until == 0 && r.Until != "" {
err = ErrTimestampInvalid
}
if err != nil {
logger.Error("failed to read config",
zap.String("config", cfg.Test.name),
zap.Error(err),
zap.Strings("targets", r.Targets),
zap.String("until", r.Until),
zap.Int("render", n),
)
return false
}
if r.ErrorRegexp != "" {
r.errorRegexp = regexp.MustCompile(r.ErrorRegexp)
}
sort.Slice(r.Result, func(i, j int) bool {
return r.Result[i].Name < r.Result[j].Name
})
r.result = make([]client.Metric, len(r.Result))
for i, result := range r.Result {
r.result[i].StartTime = datetime.DateParamToEpoch(result.StartTime, tz, now, cfg.Test.Precision)
if r.result[i].StartTime == 0 && result.StartTime != "" {
err = ErrTimestampInvalid
}
if err != nil {
logger.Error("failed to read config",
zap.String("config", cfg.Test.name),
zap.Error(err),
zap.Strings("targets", r.Targets),
zap.Int("render", n),
zap.String("metric", result.Name),
zap.String("start", result.StartTime),
)
return false
}
r.result[i].StopTime = datetime.DateParamToEpoch(result.StopTime, tz, now, cfg.Test.Precision)
if r.result[i].StopTime == 0 && result.StopTime != "" {
err = ErrTimestampInvalid
}
if err != nil {
logger.Error("failed to read config",
zap.String("config", cfg.Test.name),
zap.Error(err),
zap.Strings("targets", r.Targets),
zap.Int("render", n),
zap.String("metric", result.Name),
zap.String("stop", result.StopTime),
)
return false
}
r.result[i].RequestStartTime = datetime.DateParamToEpoch(result.RequestStartTime, tz, now, cfg.Test.Precision)
if r.result[i].RequestStartTime == 0 && result.RequestStartTime != "" {
err = ErrTimestampInvalid
}
if err != nil {
logger.Error("failed to read config",
zap.String("config", cfg.Test.name),
zap.Error(err),
zap.Strings("targets", r.Targets),
zap.Int("render", n),
zap.String("metric", result.Name),
zap.String("req_start", result.RequestStartTime),
)
return false
}
r.result[i].RequestStopTime = datetime.DateParamToEpoch(result.RequestStopTime, tz, now, cfg.Test.Precision)
if r.result[i].RequestStopTime == 0 && result.RequestStopTime != "" {
err = ErrTimestampInvalid
}
if err != nil {
logger.Error("failed to read config",
zap.String("config", cfg.Test.name),
zap.Error(err),
zap.Strings("targets", r.Targets),
zap.Int("render", n),
zap.String("metric", result.Name),
zap.String("req_stop", result.RequestStopTime),
)
return false
}
r.result[i].StepTime = result.StepTime
r.result[i].Name = result.Name
r.result[i].PathExpression = result.PathExpression
r.result[i].ConsolidationFunc = result.ConsolidationFunc
r.result[i].XFilesFactor = result.XFilesFactor
r.result[i].HighPrecisionTimestamps = result.HighPrecisionTimestamps
r.result[i].AppliedFunctions = result.AppliedFunctions
r.result[i].Values = result.Values
}
}
return true
}
func loadConfig(config string, rootDir string) (*MainConfig, error) {
d, err := os.ReadFile(config)
if err != nil {
return nil, err
}
confShort := strings.ReplaceAll(config, rootDir+"/", "")
var cfg = &MainConfig{}
if err := toml.Unmarshal(d, cfg); err != nil {
return nil, err
}
cfg.Test.name = confShort
cfg.Test.dir = path.Dir(config)
if cfg.Test == nil {
return nil, ErrNoTest
}
cfg.Test.chVersions = make(map[string]bool)
for i := range cfg.Test.Clickhouse {
if err := cfg.Test.Clickhouse[i].CheckConfig(rootDir); err == nil {
cfg.Test.chVersions[cfg.Test.Clickhouse[i].Key()] = true
} else {
return nil, fmt.Errorf("[%d] %s", i, err.Error())
}
}
return cfg, nil
}
================================================
FILE: cmd/e2e-test/errors.go
================================================
package main
import "errors"
var (
ErrTimestampInvalid = errors.New("invalid timestamp")
ErrNoTest = errors.New("no test section")
ErrNoSetDir = errors.New("dir not set")
)
================================================
FILE: cmd/e2e-test/graphite-clickhouse.go
================================================
package main
import (
"errors"
"fmt"
"net/http"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"syscall"
"text/template"
"github.com/msaf1980/go-stringutils"
"github.com/lomik/graphite-clickhouse/helper/client"
)
type GraphiteClickhouse struct {
Binary string `toml:"binary"`
ConfigTpl string `toml:"template"`
TestDir string `toml:"-"`
TZ string `toml:"tz"` // override timezone
storeDir string `toml:"-"`
configFile string `toml:"-"`
address string `toml:"-"`
cmd *exec.Cmd `toml:"-"`
}
func (c *GraphiteClickhouse) Start(testDir, chURL, chProxyURL, chTLSURL string) error {
if c.cmd != nil {
return errors.New("carbon-clickhouse already started")
}
if len(c.Binary) == 0 {
c.Binary = "./graphite-clickhouse"
}
if len(c.ConfigTpl) == 0 {
return errors.New("graphite-clickhouse config template not set")
}
var err error
c.storeDir, err = os.MkdirTemp("", "graphite-clickhouse")
if err != nil {
return err
}
c.address, err = getFreeTCPPort("")
if err != nil {
c.Cleanup()
return err
}
c.TestDir, err = filepath.Abs(testDir)
if err != nil {
return err
}
name := filepath.Base(c.ConfigTpl)
tmpl, err := template.New(name).ParseFiles(path.Join(testDir, c.ConfigTpl))
if err != nil {
c.Cleanup()
return err
}
param := struct {
CLICKHOUSE_URL string
CLICKHOUSE_TLS_URL string
PROXY_URL string
GCH_ADDR string
GCH_DIR string
TEST_DIR string
}{
CLICKHOUSE_URL: chURL,
CLICKHOUSE_TLS_URL: chTLSURL,
PROXY_URL: chProxyURL,
GCH_ADDR: c.address,
GCH_DIR: c.storeDir,
TEST_DIR: c.TestDir,
}
c.configFile = path.Join(c.storeDir, "graphite-clickhouse.conf")
f, err := os.OpenFile(c.configFile, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
c.Cleanup()
return err
}
err = tmpl.ExecuteTemplate(f, name, param)
if err != nil {
c.Cleanup()
return err
}
c.cmd = exec.Command(c.Binary, "-config", c.configFile)
c.cmd.Stdout = os.Stdout
c.cmd.Stderr = os.Stderr
if c.TZ != "" {
c.cmd.Env = append(c.cmd.Env, "TZ="+c.TZ)
}
err = c.cmd.Start()
if err != nil {
c.Cleanup()
return err
}
return nil
}
func (c *GraphiteClickhouse) Alive() bool {
if c.cmd == nil {
return false
}
_, _, _, err := client.MetricsFind(http.DefaultClient, "http://"+c.address+"/alive", client.FormatDefault, "NonExistentTarget", 0, 0)
return err == nil
}
func (c *GraphiteClickhouse) Stop(cleanup bool) error {
if cleanup {
defer c.Cleanup()
}
if c.cmd == nil {
return nil
}
var err error
if err = c.cmd.Process.Kill(); err == nil {
if err = c.cmd.Wait(); err != nil {
if exitErr, ok := err.(*exec.ExitError); ok {
if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
ec := status.ExitStatus()
if ec == 0 || ec == -1 {
return nil
}
}
}
}
}
return err
}
func (c *GraphiteClickhouse) Cleanup() {
if len(c.storeDir) > 0 {
os.RemoveAll(c.storeDir)
c.storeDir = ""
c.cmd = nil
}
}
func (c *GraphiteClickhouse) URL() string {
return "http://" + c.address
}
func (c *GraphiteClickhouse) Cmd() string {
return strings.Join(c.cmd.Args, " ")
}
func (c *GraphiteClickhouse) Grep(s string) {
out, _ := exec.Command("grep", "-F", s, c.storeDir+"/graphite-clickhouse.log").Output()
fmt.Fprintf(os.Stderr, "GREP %s", stringutils.UnsafeString(out))
}
================================================
FILE: cmd/e2e-test/main.go
================================================
package main
import (
"flag"
"log"
"os"
"path"
"runtime"
"time"
"go.uber.org/zap"
)
type MainConfig struct {
Test *TestSchema `toml:"test"`
}
func IsDir(filename string) (bool, error) {
info, err := os.Stat(filename)
if os.IsNotExist(err) {
return false, nil
} else if err != nil {
return false, err
}
return info.IsDir(), nil
}
func expandDir(dirname string, paths *[]string) error {
files, err := os.ReadDir(dirname)
if err != nil {
return err
}
for _, file := range files {
if file.IsDir() {
if err = expandDir(path.Join(dirname, file.Name()), paths); err != nil {
return err
}
} else {
ext := path.Ext(file.Name())
if ext == ".toml" {
*paths = append(*paths, path.Join(dirname, file.Name()))
}
}
}
return nil
}
func expandFilename(filename string, paths *[]string) error {
if len(filename) == 0 {
return nil
}
isDir, err := IsDir(filename)
if err == nil {
if isDir {
if err = expandDir(filename, paths); err != nil {
return err
}
} else {
*paths = append(*paths, filename)
}
}
return err
}
func main() {
_, filename, _, _ := runtime.Caller(0)
rootDir := path.Dir(path.Dir(path.Dir(filename))) // carbon-clickhouse repositiry root dir
config := flag.String("config", "", "toml configuration file or dir where toml files is searched (recursieve)")
verbose := flag.Bool("verbose", false, "verbose")
breakOnError := flag.Bool("break", false, "break and wait user response if request failed")
abortOnError := flag.Bool("abort", false, "abort tests if test failed")
cleanup := flag.Bool("cleanup", false, "delete containers if exists before start")
rmi := flag.Bool("rmi", false, "delete images after test end (for low space usage))")
flag.Parse()
logger, err := zap.NewProduction()
if err != nil {
log.Fatal(err)
}
DockerBinary = os.Getenv("DOCKER_E2E")
if DockerBinary == "" {
DockerBinary = "docker"
}
if *cleanup {
if exist, _ := containerExist(CchContainerName); exist {
if ok, out := containerRemove(CchContainerName); !ok {
logger.Fatal("failed to cleanup",
zap.String("container", CchContainerName),
zap.String("error", out),
)
}
}
if exist, _ := containerExist(ClickhouseContainerName); exist {
if ok, out := containerRemove(ClickhouseContainerName); !ok {
logger.Fatal("failed to cleanup",
zap.String("container", ClickhouseContainerName),
zap.String("error", out),
)
}
}
if *config == "" {
return
}
}
var allConfigs []string
err = expandFilename(*config, &allConfigs)
if err != nil {
logger.Fatal(
"config",
zap.Error(err),
)
}
if len(allConfigs) == 0 {
logger.Fatal("config should be non-null")
}
chVersions := make(map[string]Clickhouse)
configs := make([]*MainConfig, 0, len(allConfigs))
for _, config := range allConfigs {
cfg, err := loadConfig(config, rootDir)
if err == nil {
configs = append(configs, cfg)
for _, ch := range cfg.Test.Clickhouse {
chVersions[ch.Key()] = ch
}
now := time.Now()
if !initTest(cfg, rootDir, now, *verbose, *breakOnError, logger) {
os.Exit(1)
}
} else {
logger.Error("failed to read config",
zap.String("config", config),
zap.Error(err),
zap.Any("decode", cfg),
)
}
}
failed := 0
total := 0
verifyCount := 0
verifyFailed := 0
_, err = cmdExec(DockerBinary, "network", "inspect", DockerNetwork)
if err != nil {
out, err := cmdExec(DockerBinary, "network", "create", DockerNetwork)
if err != nil {
logger.Error("failed to create network",
zap.Error(err),
zap.String("out", out),
)
os.Exit(1)
}
}
for chVersion := range chVersions {
ch := chVersions[chVersion]
if exist, out := containerExist(ClickhouseContainerName); exist {
logger.Error("clickhouse already exist",
zap.String("container", ClickhouseContainerName),
zap.String("out", out),
)
os.Exit(1)
}
logger.Info("clickhouse",
zap.Any("clickhouse image", ch.DockerImage),
zap.Any("clickhouse version", ch.Version),
zap.String("clickhouse config", ch.Dir),
zap.String("tz", ch.TZ),
)
if clickhouseStart(&ch, logger) {
time.Sleep(100 * time.Millisecond)
for i := 200; i < 3000; i += 200 {
if ch.Alive() {
break
}
time.Sleep(time.Duration(i) * time.Millisecond)
}
if !ch.Alive() {
logger.Error("starting clickhouse",
zap.Any("clickhouse version", ch.Version),
zap.String("clickhouse config", ch.Dir),
zap.String("error", "clickhouse is down"),
)
failed++
total++
verifyCount++
verifyFailed++
} else {
for _, config := range configs {
if config.Test.chVersions[chVersion] {
now := time.Now()
if initTest(config, rootDir, now, *verbose, *breakOnError, logger) {
testFailed, testTotal, vCount, vFailed := runTest(config, &ch, rootDir, now, *verbose, *breakOnError, logger)
failed += testFailed
total += testTotal
verifyCount += vCount
verifyFailed += vFailed
} else {
failed++
total++
verifyCount++
verifyFailed++
}
}
}
}
if !clickhouseStop(&ch, logger) {
failed++
verifyFailed++
}
} else {
failed++
total++
verifyCount++
verifyFailed++
}
if *rmi {
if success, out := imageDelete(ch.DockerImage, ch.Version); !success {
logger.Error("docker remove image",
zap.Any("clickhouse version", ch.Version),
zap.String("clickhouse config", ch.Dir),
zap.String("out", out),
)
}
}
if *abortOnError && failed > 0 {
break
}
}
if failed > 0 {
logger.Error("tests ended",
zap.String("status", "failed"),
zap.Int("test_count", total),
zap.Int("test_failed", failed),
zap.Int("checks", verifyCount),
zap.Int("failed", verifyFailed),
zap.Int("configs", len(allConfigs)),
)
os.Exit(1)
} else {
logger.Info("tests ended",
zap.String("status", "success"),
zap.Int("test_count", total),
zap.Int("test_failed", failed),
zap.Int("checks", verifyCount),
zap.Int("failed", verifyFailed),
zap.Int("configs", len(allConfigs)),
)
}
}
================================================
FILE: cmd/e2e-test/rproxy.go
================================================
package main
import (
"errors"
"net/http"
"net/http/httptest"
"net/http/httputil"
"net/url"
"sync"
"sync/atomic"
"time"
"github.com/lomik/graphite-clickhouse/pkg/dry"
)
type AtomicDuration struct {
val int64
}
func (d *AtomicDuration) Store(duration time.Duration) {
atomic.StoreInt64(&d.val, duration.Nanoseconds())
}
func (d *AtomicDuration) Load() time.Duration {
return time.Duration(atomic.LoadInt64(&d.val))
}
func (d *AtomicDuration) MarshalText() ([]byte, error) {
s := d.Load().String()
return dry.UnsafeStringBytes(&s), nil
}
func (d *AtomicDuration) UnmarshalText(b []byte) error {
val, err := time.ParseDuration(dry.UnsafeString(b))
if err != nil {
return err
}
d.Store(val)
return nil
}
type HttpReverseProxy struct {
Delay AtomicDuration `toml:"delay"`
BreakWithStatusCode int64 `toml:"break_with_status_code"`
srv *httptest.Server
remote *url.URL
wg sync.WaitGroup
}
func (p *HttpReverseProxy) Start(remoteURL string) (err error) {
if p.srv != nil {
err = errors.New("reverse proxy already started")
return
}
if p.BreakWithStatusCode < 0 {
p.BreakWithStatusCode = 0
}
if p.remote, err = url.Parse(remoteURL); err != nil {
err = errors.New("reverse proxy already started")
return
}
p.srv = httptest.NewUnstartedServer(p)
p.wg.Add(1)
go func() {
defer p.wg.Done()
p.srv.Start()
}()
return
}
func (p *HttpReverseProxy) Stop() {
if p.srv == nil {
return
}
p.srv.CloseClientConnections()
p.srv.Close()
p.wg.Wait()
p.srv = nil
}
func (p *HttpReverseProxy) URL() string {
return p.srv.URL
}
func (p *HttpReverseProxy) SetDelay(delay time.Duration) {
p.Delay.Store(delay)
}
func (p *HttpReverseProxy) GetDelay() time.Duration {
return p.Delay.Load()
}
func (p *HttpReverseProxy) SetBreakStatusCode(statusCode int) {
atomic.StoreInt64(&p.BreakWithStatusCode, int64(statusCode))
}
func (p *HttpReverseProxy) GetBreakStatusCode() int {
return int(atomic.LoadInt64(&p.BreakWithStatusCode))
}
func (p *HttpReverseProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
r.Host = p.remote.Host
delay := p.GetDelay()
if delay != 0 {
time.Sleep(delay)
}
breakWithStatusCode := p.GetBreakStatusCode()
if breakWithStatusCode != 0 {
http.Error(w, "", breakWithStatusCode)
return
}
proxy := httputil.NewSingleHostReverseProxy(p.remote)
proxy.ServeHTTP(w, r)
}
================================================
FILE: cmd/e2e-test/utils.go
================================================
package main
import "os/exec"
func cmdExec(programm string, args ...string) (string, error) {
cmd := exec.Command(programm, args...)
out, err := cmd.CombinedOutput()
return string(out), err
}
================================================
FILE: cmd/graphite-clickhouse-client/main.go
================================================
package main
import (
"flag"
"fmt"
"net/http"
"os"
"strconv"
"strings"
"time"
"github.com/go-graphite/protocol/carbonapi_v3_pb"
"github.com/lomik/graphite-clickhouse/helper/client"
"github.com/lomik/graphite-clickhouse/helper/datetime"
)
type StringSlice []string
func (u *StringSlice) Set(value string) error {
*u = append(*u, value)
return nil
}
func (u *StringSlice) String() string {
return "[ " + strings.Join(*u, ", ") + " ]"
}
func (u *StringSlice) Type() string {
return "[]string"
}
func main() {
address := flag.String("address", "http://127.0.0.1:9090", "Address of graphite-clickhouse server")
fromStr := flag.String("from", "0", "from")
untilStr := flag.String("until", "", "until")
maxDataPointsStr := flag.String("maxDataPoints", "1048576", "Maximum amount of datapoints in response")
metricsFind := flag.String("find", "", "Query for /metrics/find/ , valid formats are carbonapi_v3_pb. protobuf, pickle")
tagsValues := flag.String("tags_values", "", "Query for /tags/autoComplete/values (with query like 'searchTag[=valuePrefix];tag1=value1;tag2=~value*' or '<>' for empty)")
tagsNames := flag.String("tags_names", "", "Query for /tags/autoComplete/tags (with query like '[tagPrefix];tag1=value1;tag2=~value*[' or '<>' for empty)")
limit := flag.Uint64("limit", 0, "limit for some queries (tags_values, tags_values)")
timeout := flag.Duration("timeout", time.Minute, "request timeout")
var targets StringSlice
flag.Var(&targets, "target", "Target for /render")
format := client.FormatDefault
flag.Var(&format, "format", fmt.Sprintf("Response format %v", client.FormatTypes()))
flag.Parse()
ec := 0
tz, err := datetime.Timezone("")
if err != nil {
fmt.Printf("can't get timezone: %s\n", err.Error())
os.Exit(1)
}
now := time.Now()
from := datetime.DateParamToEpoch(*fromStr, tz, now, 0)
if from == 0 && len(targets) > 0 {
fmt.Printf("invalid from: %s\n", *fromStr)
os.Exit(1)
}
var until int64
if *untilStr == "" && len(targets) > 0 {
*untilStr = "now"
}
until = datetime.DateParamToEpoch(*untilStr, tz, now, 0)
if until == 0 && len(targets) > 0 {
fmt.Printf("invalid until: %s\n", *untilStr)
os.Exit(1)
}
maxDataPoints, err := strconv.ParseInt(*maxDataPointsStr, 10, 64)
if err != nil {
fmt.Printf("invalid maxDataPoints: %s\n", *maxDataPointsStr)
os.Exit(1)
}
httpClient := http.Client{
Timeout: *timeout,
}
if *metricsFind != "" {
formatFind := format
if formatFind == client.FormatDefault {
formatFind = client.FormatPb_v3
}
queryRaw, r, respHeader, err := client.MetricsFind(&httpClient, *address, formatFind, *metricsFind, from, until)
if respHeader != nil {
fmt.Printf("Responce header: %+v\n", respHeader)
}
fmt.Print("'")
fmt.Print(queryRaw)
fmt.Print("' = ")
if err == nil {
if len(r) > 0 {
fmt.Println("[")
for i, m := range r {
fmt.Printf(" { Path: '%s', IsLeaf: %v }", m.Path, m.IsLeaf)
if i < len(r)-1 {
fmt.Println(",")
} else {
fmt.Println("")
}
}
fmt.Println("]")
} else {
fmt.Println("[]")
}
} else {
ec = 1
fmt.Printf("'%s'\n", strings.TrimRight(err.Error(), "\n"))
}
}
if *tagsValues != "" {
formatTags := format
if formatTags == client.FormatDefault {
formatTags = client.FormatJSON
}
queryRaw, r, respHeader, err := client.TagsValues(&httpClient, *address, formatTags, *tagsValues, *limit, from, until)
if respHeader != nil {
fmt.Printf("Responce header: %+v\n", respHeader)
}
fmt.Print("'")
fmt.Print(queryRaw)
fmt.Print("' = ")
if err == nil {
if len(r) > 0 {
fmt.Println("[")
for i, v := range r {
fmt.Printf(" { Value: '%s' }", v)
if i < len(r)-1 {
fmt.Println(",")
} else {
fmt.Println("")
}
}
fmt.Println("]")
} else {
fmt.Println("[]")
}
} else {
ec = 1
fmt.Printf("'%s'\n", strings.TrimRight(err.Error(), "\n"))
}
}
if *tagsNames != "" {
formatTags := format
if formatTags == client.FormatDefault {
formatTags = client.FormatJSON
}
queryRaw, r, respHeader, err := client.TagsNames(&httpClient, *address, formatTags, *tagsNames, *limit, from, until)
if respHeader != nil {
fmt.Printf("Responce header: %+v\n", respHeader)
}
fmt.Print("'")
fmt.Print(queryRaw)
fmt.Print("' = ")
if err == nil {
if len(r) > 0 {
fmt.Println("[")
for i, v := range r {
fmt.Printf(" { Tag: '%s' }", v)
if i < len(r)-1 {
fmt.Println(",")
} else {
fmt.Println("")
}
}
fmt.Println("]")
} else {
fmt.Println("[]")
}
} else {
ec = 1
fmt.Printf("'%s'\n", strings.TrimRight(err.Error(), "\n"))
}
}
if len(targets) > 0 {
formatRender := format
if formatRender == client.FormatDefault {
formatRender = client.FormatPb_v3
}
queryRaw, r, respHeader, err := client.Render(&httpClient, *address, formatRender, targets, []*carbonapi_v3_pb.FilteringFunction{}, maxDataPoints, from, until)
if respHeader != nil {
fmt.Printf("Responce header: %+v\n", respHeader)
}
fmt.Print("'")
fmt.Print(queryRaw)
fmt.Print("' = ")
if err == nil {
if len(r) > 0 {
fmt.Println("[")
for i, m := range r {
fmt.Println(" {")
fmt.Printf(" Name: '%s', PathExpression: '%v',\n", m.Name, m.PathExpression)
fmt.Printf(" ConsolidationFunc: %s, XFilesFactor: %f, AppliedFunctions: %s,\n", m.ConsolidationFunc, m.XFilesFactor, m.AppliedFunctions)
fmt.Printf(" Start: %d, Stop: %d, Step: %d, RequestStart: %d, RequestStop: %d,\n", m.StartTime, m.StopTime, m.StepTime, m.RequestStartTime, m.RequestStopTime)
fmt.Printf(" Values: %+v\n", m.Values)
if i == len(r) {
fmt.Println(" }")
} else {
fmt.Println(" },")
}
}
fmt.Println("]")
} else {
fmt.Println("[]")
}
} else {
ec = 1
fmt.Printf("'%s'\n", strings.TrimRight(err.Error(), "\n"))
}
}
os.Exit(ec)
}
================================================
FILE: config/.gitignore
================================================
tests_tmp/
================================================
FILE: config/config.go
================================================
package config
import (
"bytes"
"crypto/tls"
"fmt"
"net"
"net/url"
"os"
"reflect"
"regexp"
"sort"
"strconv"
"strings"
"time"
"github.com/cactus/go-statsd-client/v5/statsd"
"github.com/lomik/carbon-clickhouse/helper/config"
"github.com/msaf1980/go-metrics/graphite"
"github.com/msaf1980/go-timeutils/duration"
toml "github.com/pelletier/go-toml"
"github.com/pkg/errors"
"go.uber.org/zap"
"github.com/lomik/zapwriter"
"github.com/lomik/graphite-clickhouse/cache"
"github.com/lomik/graphite-clickhouse/helper/clickhouse"
"github.com/lomik/graphite-clickhouse/helper/date"
"github.com/lomik/graphite-clickhouse/helper/rollup"
"github.com/lomik/graphite-clickhouse/limiter"
"github.com/lomik/graphite-clickhouse/metrics"
)
type SDType uint8
const (
SDNone SDType = iota
SDNginx // https://github.com/weibocom/nginx-upsync-module
)
var sdTypeStrings map[SDType]string = map[SDType]string{SDNone: "", SDNginx: "nginx"}
func (a *SDType) Set(value string) error {
switch value {
case "nginx":
*a = SDNginx
case "", "0":
*a = SDNone
default:
return fmt.Errorf("invalid sd type %q", value)
}
return nil
}
func (a *SDType) UnmarshalText(data []byte) error {
return a.Set(string(data))
}
func (a *SDType) MarshalText() ([]byte, error) {
return []byte(a.String()), nil
}
func (a *SDType) UnmarshalJSON(data []byte) error {
return a.Set(string(data))
}
func (a *SDType) MarshalJSON() ([]byte, error) {
return []byte(a.String()), nil
}
func (a *SDType) String() string {
return sdTypeStrings[*a]
}
func (a *SDType) Type() string {
return "service_discovery_type"
}
// Cache config
type CacheConfig struct {
Type string `toml:"type" json:"type" comment:"cache type"`
Size int `toml:"size-mb" json:"size-mb" comment:"cache size"`
MemcachedServers []string `toml:"memcached-servers" json:"memcached-servers" comment:"memcached servers"`
DefaultTimeoutSec int32 `toml:"default-timeout" json:"default-timeout" comment:"default cache ttl"`
DefaultTimeoutStr string `toml:"-" json:"-"`
ShortTimeoutSec int32 `toml:"short-timeout" json:"short-timeout" comment:"short-time cache ttl"`
ShortTimeoutStr string `toml:"-" json:"-"`
FindTimeoutSec int32 `toml:"find-timeout" json:"find-timeout" comment:"finder/tags autocompleter cache ttl"`
ShortDuration time.Duration `toml:"short-duration" json:"short-duration" comment:"maximum diration, used with short_timeout"`
ShortUntilOffsetSec int64 `toml:"short-offset" json:"short-offset" comment:"offset beetween now and until for select short cache timeout"`
}
// Common config
type Common struct {
Listen string `toml:"listen" json:"listen" comment:"general listener"`
PprofListen string `toml:"pprof-listen" json:"pprof-listen" comment:"listener to serve /debug/pprof requests. '-pprof' argument overrides it"`
MaxCPU int `toml:"max-cpu" json:"max-cpu"`
MaxMetricsInFindAnswer int `toml:"max-metrics-in-find-answer" json:"max-metrics-in-find-answer" comment:"limit number of results from find query, 0=unlimited"`
MaxMetricsPerTarget int `toml:"max-metrics-per-target" json:"max-metrics-per-target" comment:"limit numbers of queried metrics per target in /render requests, 0 or negative = unlimited"`
AppendEmptySeries bool `toml:"append-empty-series" json:"append-empty-series" comment:"if true, always return points for all metrics, replacing empty results with list of NaN"`
TargetBlacklist []string `toml:"target-blacklist" json:"target-blacklist" comment:"daemon returns empty response if query matches any of regular expressions" commented:"true"`
Blacklist []*regexp.Regexp `toml:"-" json:"-"` // compiled TargetBlacklist
MemoryReturnInterval time.Duration `toml:"memory-return-interval" json:"memory-return-interval" comment:"daemon will return the freed memory to the OS when it>0"`
HeadersToLog []string `toml:"headers-to-log" json:"headers-to-log" comment:"additional request headers to log"`
BaseWeight int `toml:"base_weight" json:"base_weight" comment:"service discovery base weight (on idle)"`
DegragedMultiply float64 `toml:"degraged-multiply" json:"degraged-multiply" comment:"service discovery degraded load avg multiplier (if normalized load avg > degraged_load_avg) (default 4.0)"`
DegragedLoad float64 `toml:"degraged-load-avg" json:"degraged-load-avg" comment:"service discovery normilized load avg degraded point (default 1.0)"`
SDType SDType `toml:"service-discovery-type" json:"service-discovery-type" comment:"service discovery type"`
SD string `toml:"service-discovery" json:"service-discovery" comment:"service discovery address (consul)"`
SDNamespace string `toml:"service-discovery-ns" json:"service-discovery-ns" comment:"service discovery namespace (graphite by default)"`
SDDc []string `toml:"service-discovery-ds" json:"service-discovery-ds" comment:"service discovery datacenters (first - is primary, in other register as backup)"`
SDExpire time.Duration `toml:"service-discovery-expire" json:"service-discovery-expire" comment:"service discovery expire duration for cleanup (minimum is 24h, if enabled)"`
FindCacheConfig CacheConfig `toml:"find-cache" json:"find-cache" comment:"find/tags cache config"`
FindCache cache.BytesCache `toml:"-" json:"-"`
}
// FeatureFlags contains feature flags that significantly change how gch responds to some requests
type FeatureFlags struct {
UseCarbonBehavior bool `toml:"use-carbon-behaviour" json:"use-carbon-behaviour" comment:"if true, prefers carbon's behaviour on how tags are treated"`
DontMatchMissingTags bool `toml:"dont-match-missing-tags" json:"dont-match-missing-tags" comment:"if true, seriesByTag terms containing '!=' or '!=~' operators will not match metrics that don't have the tag at all"`
LogQueryProgress bool `toml:"log-query-progress" json:"log-query-progress" comment:"if true, gch will log affected rows count by clickhouse query"`
}
// IndexReverseRule contains rules to use direct or reversed request to index table
type IndexReverseRule struct {
Suffix string `toml:"suffix,omitempty" json:"suffix" comment:"rule is used when the target suffix is matched"`
Prefix string `toml:"prefix,omitempty" json:"prefix" comment:"rule is used when the target prefix is matched"`
RegexStr string `toml:"regex,omitempty" json:"regex" comment:"rule is used when the target regex is matched"`
Regex *regexp.Regexp `toml:"-" json:"-"`
Reverse string `toml:"reverse" json:"reverse" comment:"same as index-reverse"`
}
type Costs struct {
Cost *int `toml:"cost" json:"cost" comment:"default cost (for wildcarded equalence or matched with regex, or if no value cost set)"`
ValuesCost map[string]int `toml:"values-cost" json:"values-cost" comment:"cost with some value (for equalence without wildcards) (additional tuning, usually not needed)"`
}
// IndexReverses is a slise of ptrs to IndexReverseRule
type IndexReverses []*IndexReverseRule
const (
IndexAuto = iota
IndexDirect = iota
IndexReversed = iota
)
// IndexReverse maps setting name to value
var IndexReverse = map[string]uint8{
"direct": IndexDirect,
"auto": IndexAuto,
"reversed": IndexReversed,
}
// IndexReverseNames contains valid names for index-reverse setting
var IndexReverseNames = []string{"auto", "direct", "reversed"}
type UserLimits struct {
MaxQueries int `toml:"max-queries" json:"max-queries" comment:"Max queries to fetch data"`
ConcurrentQueries int `toml:"concurrent-queries" json:"concurrent-queries" comment:"Concurrent queries to fetch data"`
AdaptiveQueries int `toml:"adaptive-queries" json:"adaptive-queries" comment:"Adaptive queries (based on load average) for increase/decrease concurrent queries"`
Limiter limiter.ServerLimiter `toml:"-" json:"-"`
}
type QueryParam struct {
Duration time.Duration `toml:"duration" json:"duration" comment:"minimal duration (beetween from/until) for select query params"`
URL string `toml:"url" json:"url" comment:"url for queries with durations greater or equal than"`
DataTimeout time.Duration `toml:"data-timeout" json:"data-timeout" comment:"total timeout to fetch data"`
MaxQueries int `toml:"max-queries" json:"max-queries" comment:"Max queries to fetch data"`
ConcurrentQueries int `toml:"concurrent-queries" json:"concurrent-queries" comment:"Concurrent queries to fetch data"`
AdaptiveQueries int `toml:"adaptive-queries" json:"adaptive-queries" comment:"Adaptive queries (based on load average) for increase/decrease concurrent queries"`
Limiter limiter.ServerLimiter `toml:"-" json:"-"`
}
func binarySearchQueryParamLe(a []QueryParam, duration time.Duration, start, end int) int {
length := end - start
if length <= 0 {
return -1 // not found
} else if length == 1 {
if a[start].Duration > duration {
return -1
}
return start
}
var result int
mid := start + length/2
if a[mid].Duration > duration {
result = binarySearchQueryParamLe(a, duration, start, mid)
} else {
if result = binarySearchQueryParamLe(a, duration, mid+1, end); result == -1 {
result = mid
}
}
return result
}
// ClickHouse config
type ClickHouse struct {
URL string `toml:"url" json:"url" comment:"default url, see https://clickhouse.tech/docs/en/interfaces/http. Can be overwritten with query-params"`
DataTimeout time.Duration `toml:"data-timeout" json:"data-timeout" comment:"default total timeout to fetch data, can be overwritten with query-params"`
QueryParams []QueryParam `toml:"query-params" json:"query-params" comment:"customized query params (url, data timeout, limiters) for durations greater or equal"`
ProgressSendingInterval time.Duration `toml:"progress-sending-interval" json:"progress-sending-interval" comment:"time interval for ch query progress sending, it's equal to http_headers_progress_interval_ms header"`
RenderMaxQueries int `toml:"render-max-queries" json:"render-max-queries" comment:"Max queries to render queiries"`
RenderConcurrentQueries int `toml:"render-concurrent-queries" json:"render-concurrent-queries" comment:"Concurrent queries to render queiries"`
RenderAdaptiveQueries int `toml:"render-adaptive-queries" json:"render-adaptive-queries" comment:"Render adaptive queries (based on load average) for increase/decrease concurrent queries"`
FindMaxQueries int `toml:"find-max-queries" json:"find-max-queries" comment:"Max queries for find queries"`
FindConcurrentQueries int `toml:"find-concurrent-queries" json:"find-concurrent-queries" comment:"Find concurrent queries for find queries"`
FindAdaptiveQueries int `toml:"find-adaptive-queries" json:"find-adaptive-queries" comment:"Find adaptive queries (based on load average) for increase/decrease concurrent queries"`
FindLimiter limiter.ServerLimiter `toml:"-" json:"-"`
TagsMaxQueries int `toml:"tags-max-queries" json:"tags-max-queries" comment:"Max queries for tags queries"`
TagsConcurrentQueries int `toml:"tags-concurrent-queries" json:"tags-concurrent-queries" comment:"Concurrent queries for tags queries"`
TagsAdaptiveQueries int `toml:"tags-adaptive-queries" json:"tags-adaptive-queries" comment:"Tags adaptive queries (based on load average) for increase/decrease concurrent queries"`
TagsLimiter limiter.ServerLimiter `toml:"-" json:"-"`
WildcardMinDistance int `toml:"wildcard-min-distance" json:"wildcard-min-distance" comment:"If a wildcard appears both at the start and the end of a plain query at a distance (in terms of nodes) less than wildcard-min-distance, then it will be discarded. This parameter can be used to discard expensive queries."`
TrySplitQuery bool `toml:"try-split-query" json:"try-split-query" comment:"Plain queries like '{first,second}.custom.metric.*' are also a subject to wildcard-min-distance restriction. But can be split into 2 queries: 'first.custom.metric.*', 'second.custom.metric.*'. Note that: only one list will be split; if there are wildcard in query before (after) list then reverse (direct) notation will be preferred; if there are wildcards before and after list, then query will not be split"`
MaxNodeToSplitIndex int `toml:"max-node-to-split-index" json:"max-node-to-split-index" comment:"Used only if try-split-query is true. Query that contains list will be split if its (list) node index is less or equal to max-node-to-split-index. By default is 0. It is recommended to have this value set to 2 or 3 and increase it very carefully, because 3 or 4 plain nodes without wildcards have good selectivity"`
TagsMinInQuery int `toml:"tags-min-in-query" json:"tags-min-in-query" comment:"Minimum tags in seriesByTag query"`
TagsMinInAutocomplete int `toml:"tags-min-in-autocomplete" json:"tags-min-in-autocomplete" comment:"Minimum tags in autocomplete query"`
UserLimits map[string]UserLimits `toml:"user-limits" json:"user-limits" comment:"customized query limiter for some users" commented:"true"`
DateFormat string `toml:"date-format" json:"date-format" comment:"Date format (default, utc, both)"`
IndexTable string `toml:"index-table" json:"index-table" comment:"see doc/index-table.md"`
IndexUseDaily bool `toml:"index-use-daily" json:"index-use-daily"`
IndexReverse string `toml:"index-reverse" json:"index-reverse" comment:"see doc/config.md"`
IndexReverses IndexReverses `toml:"index-reverses" json:"index-reverses" comment:"see doc/config.md" commented:"true"`
IndexTimeout time.Duration `toml:"index-timeout" json:"index-timeout" comment:"total timeout to fetch series list from index"`
TaggedTable string `toml:"tagged-table" json:"tagged-table" comment:"'tagged' table from carbon-clickhouse, required for seriesByTag"`
TagsCountTable string `toml:"tags-count-table" json:"tags-count-table" comment:"Table that contains the total amounts of each tag-value pair. It is used to avoid usage of high cardinality tag-value pairs when querying TaggedTable. If left empty, basic sorting will be used. See more detailed description in doc/config.md"`
TaggedAutocompleDays int `toml:"tagged-autocomplete-days" json:"tagged-autocomplete-days" comment:"or how long the daemon will query tags during autocomplete"`
TaggedUseDaily bool `toml:"tagged-use-daily" json:"tagged-use-daily" comment:"whether to use date filter when searching for the metrics in the tagged-table"`
TaggedCosts map[string]*Costs `toml:"tagged-costs" json:"tagged-costs" comment:"costs for tags (for tune which tag will be used as primary), by default is 0, increase for costly (with poor selectivity) tags" commented:"true"`
TreeTable string `toml:"tree-table" json:"tree-table" comment:"old index table, DEPRECATED, see description in doc/config.md" commented:"true"`
ReverseTreeTable string `toml:"reverse-tree-table" json:"reverse-tree-table" commented:"true"`
DateTreeTable string `toml:"date-tree-table" json:"date-tree-table" commented:"true"`
DateTreeTableVersion int `toml:"date-tree-table-version" json:"date-tree-table-version" commented:"true"`
TreeTimeout time.Duration `toml:"tree-timeout" json:"tree-timeout" commented:"true"`
TagTable string `toml:"tag-table" json:"tag-table" comment:"is not recommended to use, https://github.com/lomik/graphite-clickhouse/wiki/TagsRU" commented:"true"`
ExtraPrefix string `toml:"extra-prefix" json:"extra-prefix" comment:"add extra prefix (directory in graphite) for all metrics, w/o trailing dot"`
ConnectTimeout time.Duration `toml:"connect-timeout" json:"connect-timeout" comment:"TCP connection timeout"`
// TODO: remove in v0.14
DataTableLegacy string `toml:"data-table" json:"data-table" comment:"will be removed in 0.14" commented:"true"`
// TODO: remove in v0.14
RollupConfLegacy string `toml:"rollup-conf" json:"-" commented:"true"`
MaxDataPoints int `toml:"max-data-points" json:"max-data-points" comment:"max points per metric when internal-aggregation=true"`
// InternalAggregation controls if ClickHouse itself or graphite-clickhouse aggregates points to proper retention
InternalAggregation bool `toml:"internal-aggregation" json:"internal-aggregation" comment:"ClickHouse-side aggregation, see doc/aggregation.md"`
TLSParams config.TLS `toml:"tls" json:"tls" comment:"mTLS HTTPS configuration for connecting to clickhouse server" commented:"true"`
TLSConfig *tls.Config `toml:"-" json:"-"`
}
func clickhouseURLValidate(chURL string) (*url.URL, error) {
u, err := url.Parse(chURL)
if err != nil {
return nil, fmt.Errorf("error %q in url %q", err.Error(), chURL)
} else if u.Scheme != "http" && u.Scheme != "https" {
return nil, fmt.Errorf("scheme not supported in url %q", chURL)
} else if strings.Contains(u.RawQuery, " ") {
return nil, fmt.Errorf("space not allowed in url %q", chURL)
}
return u, nil
}
// Tags config
type Tags struct {
Rules string `toml:"rules" json:"rules"`
Date string `toml:"date" json:"date"`
ExtraWhere string `toml:"extra-where" json:"extra-where"`
InputFile string `toml:"input-file" json:"input-file"`
OutputFile string `toml:"output-file" json:"output-file"`
Threads int `toml:"threads" json:"threads" comment:"number of threads for uploading tags to clickhouse (1 by default)"`
Compression clickhouse.ContentEncoding `toml:"compression" json:"compression" comment:"compression method for tags before sending them to clickhouse (i.e. content encoding): gzip (default), none, zstd"`
Version uint32 `toml:"version" json:"version" comment:"fixed tags version for testing purposes (by default the current timestamp is used for each upload)"`
SelectChunksCount int `toml:"select-chunks-count" json:"select-chunks-count" comment:"number of chunks for selecting metrics from clickhouse (10 by default)"`
}
// Carbonlink configuration
type Carbonlink struct {
Server string `toml:"server" json:"server"`
Threads int `toml:"threads-per-request" json:"threads-per-request"`
Retries int `toml:"-" json:"-"`
ConnectTimeout time.Duration `toml:"connect-timeout" json:"connect-timeout"`
QueryTimeout time.Duration `toml:"query-timeout" json:"query-timeout"`
TotalTimeout time.Duration `toml:"total-timeout" json:"total-timeout" comment:"timeout for querying and parsing response"`
}
// Prometheus configuration
type Prometheus struct {
Listen string `toml:"listen" json:"listen" comment:"listen addr for prometheus ui and api"`
ExternalURLRaw string `toml:"external-url" json:"external-url" comment:"allows to set URL for redirect manually"`
ExternalURL *url.URL `toml:"-" json:"-"`
PageTitle string `toml:"page-title" json:"page-title"`
LookbackDelta time.Duration `toml:"lookback-delta" json:"lookback-delta"`
RemoteReadConcurrencyLimit int `toml:"remote-read-concurrency-limit" json:"remote-read-concurrency-limit" comment:"concurrently handled remote read requests"`
}
const (
// ContextGraphite for data tables
ContextGraphite = "graphite"
// ContextPrometheus for data tables
ContextPrometheus = "prometheus"
)
var knownDataTableContext = map[string]bool{
ContextGraphite: true,
ContextPrometheus: true,
}
// DataTable configs
type DataTable struct {
Table string `toml:"table" json:"table" comment:"data table from carbon-clickhouse"`
Reverse bool `toml:"reverse" json:"reverse" comment:"if it stores direct or reversed metrics"`
MaxAge time.Duration `toml:"max-age" json:"max-age" comment:"maximum age stored in the table"`
MinAge time.Duration `toml:"min-age" json:"min-age" comment:"minimum age stored in the table"`
MaxInterval time.Duration `toml:"max-interval" json:"max-interval" comment:"maximum until-from interval allowed for the table"`
MinInterval time.Duration `toml:"min-interval" json:"min-interval" comment:"minimum until-from interval allowed for the table"`
TargetMatchAny string `toml:"target-match-any" json:"target-match-any" comment:"table allowed only if any metrics in target matches regexp"`
TargetMatchAll string `toml:"target-match-all" json:"target-match-all" comment:"table allowed only if all metrics in target matches regexp"`
TargetMatchAnyRegexp *regexp.Regexp `toml:"-" json:"-"`
TargetMatchAllRegexp *regexp.Regexp `toml:"-" json:"-"`
RollupConf string `toml:"rollup-conf" json:"-" comment:"custom rollup.xml file for table, 'auto' and 'none' are allowed as well"`
RollupAutoTable string `toml:"rollup-auto-table" json:"rollup-auto-table" comment:"custom table for 'rollup-conf=auto', useful for Distributed or MatView"`
RollupAutoInterval *time.Duration `toml:"rollup-auto-interval" json:"rollup-auto-interval" comment:"rollup update interval for 'rollup-conf=auto'"`
RollupDefaultPrecision uint32 `toml:"rollup-default-precision" json:"rollup-default-precision" comment:"is used when none of rules match"`
RollupDefaultFunction string `toml:"rollup-default-function" json:"rollup-default-function" comment:"is used when none of rules match"`
RollupUseReverted bool `toml:"rollup-use-reverted" json:"rollup-use-reverted" comment:"should be set to true if you don't have reverted regexps in rollup-conf for reversed tables"`
Context []string `toml:"context" json:"context" comment:"valid values are 'graphite' of 'prometheus'"`
ContextMap map[string]bool `toml:"-" json:"-"`
Rollup *rollup.Rollup `toml:"-" json:"rollup-conf"`
QueryMetrics *metrics.QueryMetrics `toml:"-" json:"-"`
}
// Debug config
type Debug struct {
Directory string `toml:"directory" json:"directory" comment:"the directory for additional debug output"`
DirectoryPerm os.FileMode `toml:"directory-perm" json:"directory-perm" comment:"permissions for directory, octal value is set as 0o755"`
// If ExternalDataPerm > 0 and X-Gch-Debug-Ext-Data HTTP header is set, the external data used in the query
// will be saved in the DebugDir directory
ExternalDataPerm os.FileMode `toml:"external-data-perm" json:"external-data-perm" comment:"permissions for directory, octal value is set as 0o640"`
}
// Config is the daemon configuration
type Config struct {
Common Common `toml:"common" json:"common"`
FeatureFlags FeatureFlags `toml:"feature-flags" json:"feature-flags"`
Metrics metrics.Config `toml:"metrics" json:"metrics"`
ClickHouse ClickHouse `toml:"clickhouse" json:"clickhouse"`
DataTable []DataTable `toml:"data-table" json:"data-table" comment:"data tables, see doc/config.md for additional info"`
Tags Tags `toml:"tags" json:"tags" comment:"is not recommended to use, https://github.com/lomik/graphite-clickhouse/wiki/TagsRU" commented:"true"`
Carbonlink Carbonlink `toml:"carbonlink" json:"carbonlink"`
Prometheus Prometheus `toml:"prometheus" json:"prometheus"`
Debug Debug `toml:"debug" json:"debug" comment:"see doc/debugging.md"`
Logging []zapwriter.Config `toml:"logging" json:"logging"`
}
// New returns *Config with default values
func New() *Config {
cfg := &Config{
Common: Common{
Listen: ":9090",
PprofListen: "",
// MetricPrefix: "carbon.graphite-clickhouse.{host}",
// MetricInterval: time.Minute,
// MetricEndpoint: MetricEndpointLocal,
MaxCPU: 1,
MaxMetricsInFindAnswer: 0,
MaxMetricsPerTarget: 15000, // This is arbitrary value to protect CH from overload
MemoryReturnInterval: 0,
FindCacheConfig: CacheConfig{
Type: "null",
DefaultTimeoutSec: 0,
ShortTimeoutSec: 0,
FindTimeoutSec: 0,
},
DegragedMultiply: 4.0,
DegragedLoad: 1.0,
},
ClickHouse: ClickHouse{
URL: "http://localhost:8123?cancel_http_readonly_queries_on_client_close=1",
DataTimeout: time.Minute,
ProgressSendingInterval: 10 * time.Second,
IndexTable: "graphite_index",
IndexUseDaily: true,
TaggedUseDaily: true,
IndexReverse: "auto",
IndexReverses: IndexReverses{},
IndexTimeout: time.Minute,
TaggedTable: "graphite_tagged",
TaggedAutocompleDays: 7,
ExtraPrefix: "",
ConnectTimeout: time.Second,
DataTableLegacy: "",
RollupConfLegacy: "auto",
MaxDataPoints: 1048576,
InternalAggregation: true,
FindLimiter: limiter.NoopLimiter{},
TagsLimiter: limiter.NoopLimiter{},
},
Tags: Tags{
Threads: 1,
Compression: "gzip",
},
Carbonlink: Carbonlink{
Threads: 10,
Retries: 2,
ConnectTimeout: 50 * time.Millisecond,
QueryTimeout: 50 * time.Millisecond,
TotalTimeout: 500 * time.Millisecond,
},
Prometheus: Prometheus{
ExternalURLRaw: "",
PageTitle: "Prometheus Time Series Collection and Processing Server",
Listen: ":9092",
LookbackDelta: 5 * time.Minute,
RemoteReadConcurrencyLimit: 10,
},
Debug: Debug{
Directory: "",
DirectoryPerm: 0755,
ExternalDataPerm: 0,
},
Logging: nil,
}
return cfg
}
// Compile checks if IndexReverseRule are valid in the IndexReverses and compiles regexps if set
func (ir IndexReverses) Compile() error {
var err error
for i, n := range ir {
if len(n.RegexStr) > 0 {
if n.Regex, err = regexp.Compile(n.RegexStr); err != nil {
return err
}
} else if len(n.Prefix) == 0 && len(n.Suffix) == 0 {
return fmt.Errorf("empthy index-use-reverses[%d] rule", i)
}
if _, ok := IndexReverse[n.Reverse]; !ok {
return fmt.Errorf("%s is not valid value for index-reverses.reverse", n.Reverse)
}
}
return nil
}
func newLoggingConfig() zapwriter.Config {
cfg := zapwriter.NewConfig()
cfg.File = "/var/log/graphite-clickhouse/graphite-clickhouse.log"
return cfg
}
func DefaultConfig() (*Config, error) {
cfg := New()
if cfg.Logging == nil {
cfg.Logging = make([]zapwriter.Config, 0)
}
if len(cfg.Logging) == 0 {
cfg.Logging = append(cfg.Logging, newLoggingConfig())
}
if len(cfg.DataTable) == 0 {
interval := time.Minute
cfg.DataTable = []DataTable{
{
Table: "graphite_data",
RollupConf: "auto",
RollupAutoInterval: &interval,
},
}
}
if len(cfg.ClickHouse.IndexReverses) == 0 {
cfg.ClickHouse.IndexReverses = IndexReverses{
&IndexReverseRule{Suffix: "suffix", Reverse: "auto"},
&IndexReverseRule{Prefix: "prefix", Reverse: "direct"},
&IndexReverseRule{RegexStr: "regex", Reverse: "reversed"},
}
err := cfg.ClickHouse.IndexReverses.Compile()
if err != nil {
return nil, err
}
}
return cfg, nil
}
// PrintDefaultConfig prints the default config with some additions to be useful
func PrintDefaultConfig() error {
buf := new(bytes.Buffer)
cfg, err := DefaultConfig()
if err != nil {
return err
}
encoder := toml.NewEncoder(buf).Indentation(" ").Order(toml.OrderPreserve).CompactComments(true)
if err := encoder.Encode(cfg); err != nil {
return err
}
out := strings.Replace(buf.String(), "\n", "", 1)
fmt.Print(out)
return nil
}
// ReadConfig reads the content of the file with given name and process it to the *Config
func ReadConfig(filename string, exactConfig bool) (*Config, []zap.Field, error) {
var err error
var body []byte
if filename != "" {
body, err = os.ReadFile(filename)
if err != nil {
return nil, nil, err
}
}
return Unmarshal(body, exactConfig)
}
// Unmarshal process the body to *Config
func Unmarshal(body []byte, exactConfig bool) (cfg *Config, warns []zap.Field, err error) {
deprecations := make(map[string]error)
cfg = New()
if len(body) != 0 {
// TODO: remove in v0.14
if bytes.Index(body, []byte("\n[logging]\n")) != -1 || bytes.Index(body, []byte("[logging]")) == 0 {
deprecations["logging"] = fmt.Errorf("single [logging] value became multivalue [[logging]]; please, adjust your config")
body = bytes.ReplaceAll(body, []byte("\n[logging]\n"), []byte("\n[[logging]]\n"))
if bytes.Index(body, []byte("[logging]")) == 0 {
body = bytes.Replace(body, []byte("[logging]"), []byte("[[logging]]"), 1)
}
}
decoder := toml.NewDecoder(bytes.NewReader(body))
decoder.Strict(exactConfig)
err := decoder.Decode(cfg)
if err != nil {
return nil, nil, err
}
}
if cfg.Logging == nil {
cfg.Logging = make([]zapwriter.Config, 0)
}
if cfg.ClickHouse.RenderConcurrentQueries > cfg.ClickHouse.RenderMaxQueries && cfg.ClickHouse.RenderMaxQueries > 0 {
cfg.ClickHouse.RenderConcurrentQueries = 0
}
chURL, err := clickhouseURLValidate(cfg.ClickHouse.URL)
if err != nil {
return nil, nil, err
}
if !reflect.DeepEqual(cfg.ClickHouse.TLSParams, config.TLS{}) {
tlsConfig, warnings, err := config.ParseClientTLSConfig(&cfg.ClickHouse.TLSParams)
if err != nil {
return nil, nil, err
}
if chURL.Scheme == "https" {
cfg.ClickHouse.TLSConfig = tlsConfig
} else {
warnings = append(warnings, "TLS configurations is ignored because scheme is not HTTPS")
}
warns = append(warns, zap.Strings("tls-config", warnings))
}
for i := range cfg.ClickHouse.QueryParams {
if cfg.ClickHouse.QueryParams[i].ConcurrentQueries > cfg.ClickHouse.QueryParams[i].MaxQueries && cfg.ClickHouse.QueryParams[i].MaxQueries > 0 {
cfg.ClickHouse.QueryParams[i].ConcurrentQueries = 0
}
if cfg.ClickHouse.QueryParams[i].Duration == 0 {
return nil, nil, fmt.Errorf("query duration param not set for: %+v", cfg.ClickHouse.QueryParams[i])
}
if cfg.ClickHouse.QueryParams[i].DataTimeout == 0 {
cfg.ClickHouse.QueryParams[i].DataTimeout = cfg.ClickHouse.DataTimeout
}
if cfg.ClickHouse.QueryParams[i].URL == "" {
// reuse default url
cfg.ClickHouse.QueryParams[i].URL = cfg.ClickHouse.URL
}
if _, err = clickhouseURLValidate(cfg.ClickHouse.QueryParams[i].URL); err != nil {
return nil, nil, err
}
}
cfg.ClickHouse.QueryParams = append(
[]QueryParam{{
URL: cfg.ClickHouse.URL, DataTimeout: cfg.ClickHouse.DataTimeout,
MaxQueries: cfg.ClickHouse.RenderMaxQueries, ConcurrentQueries: cfg.ClickHouse.RenderConcurrentQueries,
AdaptiveQueries: cfg.ClickHouse.RenderAdaptiveQueries,
}},
cfg.ClickHouse.QueryParams...,
)
sort.SliceStable(cfg.ClickHouse.QueryParams, func(i, j int) bool {
return cfg.ClickHouse.QueryParams[i].Duration < cfg.ClickHouse.QueryParams[j].Duration
})
if len(cfg.Logging) == 0 {
cfg.Logging = append(cfg.Logging, newLoggingConfig())
}
if err = zapwriter.CheckConfig(cfg.Logging, nil); err != nil {
return nil, nil, err
}
// Check if debug directory exists or could be created
if cfg.Debug.Directory != "" {
info, err := os.Stat(cfg.Debug.Directory)
if os.IsNotExist(err) {
err := os.MkdirAll(cfg.Debug.Directory, os.ModeDir|cfg.Debug.DirectoryPerm)
if err != nil {
return nil, nil, err
}
} else if !info.IsDir() {
return nil, nil, fmt.Errorf("the file for external data debug dumps exists and is not a directory: %v", cfg.Debug.Directory)
}
}
if _, ok := IndexReverse[cfg.ClickHouse.IndexReverse]; !ok {
return nil, nil, fmt.Errorf("%s is not valid value for index-reverse", cfg.ClickHouse.IndexReverse)
}
err = cfg.ClickHouse.IndexReverses.Compile()
if err != nil {
return nil, nil, err
}
if cfg.Common.FindCache, err = CreateCache("index", &cfg.Common.FindCacheConfig); err == nil {
if cfg.Common.FindCacheConfig.Type != "null" {
warns = append(warns, zap.Any("enable find cache", zap.String("type", cfg.Common.FindCacheConfig.Type)))
}
} else {
return nil, nil, err
}
l := len(cfg.Common.TargetBlacklist)
if l > 0 {
cfg.Common.Blacklist = make([]*regexp.Regexp, l)
for i := 0; i < l; i++ {
r, err := regexp.Compile(cfg.Common.TargetBlacklist[i])
if err != nil {
return nil, nil, err
}
cfg.Common.Blacklist[i] = r
}
}
err = cfg.ProcessDataTables()
if err != nil {
return nil, nil, err
}
// compute prometheus external url
rawURL := cfg.Prometheus.ExternalURLRaw
if rawURL == "" {
hostname, err := os.Hostname()
if err != nil {
return nil, nil, err
}
_, port, err := net.SplitHostPort(cfg.Common.Listen)
if err != nil {
return nil, nil, err
}
rawURL = fmt.Sprintf("http://%s:%s/", hostname, port)
}
cfg.Prometheus.ExternalURL, err = url.Parse(rawURL)
if err != nil {
return nil, nil, err
}
cfg.Prometheus.ExternalURL.Path = strings.TrimRight(cfg.Prometheus.ExternalURL.Path, "/")
checkDeprecations(cfg, deprecations)
if len(deprecations) != 0 {
deprecationList := make([]error, len(deprecations))
for name, message := range deprecations {
deprecationList = append(deprecationList, errors.Wrap(message, name))
}
warns = append(warns, zap.Errors("config deprecations", deprecationList))
}
switch strings.ToLower(cfg.ClickHouse.DateFormat) {
case "utc":
date.SetUTC()
case "both":
date.SetBoth()
default:
if cfg.ClickHouse.DateFormat != "" && cfg.ClickHouse.DateFormat != "default" {
return nil, nil, fmt.Errorf("unsupported date-format: %s", cfg.ClickHouse.DateFormat)
}
}
if cfg.ClickHouse.FindConcurrentQueries > cfg.ClickHouse.FindMaxQueries && cfg.ClickHouse.FindMaxQueries > 0 {
cfg.ClickHouse.FindConcurrentQueries = 0
}
if cfg.ClickHouse.TagsConcurrentQueries > cfg.ClickHouse.TagsMaxQueries && cfg.ClickHouse.TagsMaxQueries > 0 {
cfg.ClickHouse.TagsConcurrentQueries = 0
}
metricsEnabled := cfg.setupGraphiteMetrics()
cfg.ClickHouse.FindLimiter = limiter.NewALimiter(
cfg.ClickHouse.FindMaxQueries, cfg.ClickHouse.FindConcurrentQueries, cfg.ClickHouse.FindAdaptiveQueries,
metricsEnabled, "find", "all",
)
cfg.ClickHouse.TagsLimiter = limiter.NewALimiter(
cfg.ClickHouse.TagsMaxQueries, cfg.ClickHouse.TagsConcurrentQueries, cfg.ClickHouse.TagsAdaptiveQueries,
metricsEnabled, "tags", "all",
)
for i := range cfg.ClickHouse.QueryParams {
cfg.ClickHouse.QueryParams[i].Limiter = limiter.NewALimiter(
cfg.ClickHouse.QueryParams[i].MaxQueries, cfg.ClickHouse.QueryParams[i].ConcurrentQueries,
cfg.ClickHouse.QueryParams[i].AdaptiveQueries,
metricsEnabled, "render", duration.String(cfg.ClickHouse.QueryParams[i].Duration),
)
}
for u, q := range cfg.ClickHouse.UserLimits {
q.Limiter = limiter.NewALimiter(
q.MaxQueries, q.ConcurrentQueries, q.AdaptiveQueries, metricsEnabled, u, "all",
)
cfg.ClickHouse.UserLimits[u] = q
}
return cfg, warns, nil
}
// NeedLoadAvgColect check if load avg collect is neeeded
func (c *Config) NeedLoadAvgColect() bool {
if c.Common.SD != "" {
if c.Common.DegragedMultiply <= 0 {
c.Common.DegragedMultiply = 4.0
}
if c.Common.DegragedLoad <= 0 {
c.Common.DegragedLoad = 1.0
}
if c.Common.BaseWeight <= 0 {
c.Common.BaseWeight = 100
}
if c.Common.SDNamespace == "" {
c.Common.SDNamespace = "graphite"
}
if c.Common.SDExpire < 24*time.Hour {
c.Common.SDExpire = 24 * time.Hour
}
return true
}
if c.ClickHouse.RenderAdaptiveQueries > 0 {
return true
}
if c.ClickHouse.FindAdaptiveQueries > 0 {
return true
}
if c.ClickHouse.TagsAdaptiveQueries > 0 {
return true
}
for _, u := range c.ClickHouse.UserLimits {
if u.AdaptiveQueries > 0 {
return true
}
}
return false
}
// ProcessDataTables checks if legacy `data`-table config is used, compiles regexps for `target-match-any` and `target-match-all`
// parameters, sets the rollup configuration and proper context.
func (c *Config) ProcessDataTables() (err error) {
if c.ClickHouse.DataTableLegacy != "" {
c.DataTable = append(c.DataTable, DataTable{
Table: c.ClickHouse.DataTableLegacy,
RollupConf: c.ClickHouse.RollupConfLegacy,
})
}
for i := 0; i < len(c.DataTable); i++ {
if c.DataTable[i].TargetMatchAny != "" {
r, err := regexp.Compile(c.DataTable[i].TargetMatchAny)
if err != nil {
return err
}
c.DataTable[i].TargetMatchAnyRegexp = r
}
if c.DataTable[i].TargetMatchAll != "" {
r, err := regexp.Compile(c.DataTable[i].TargetMatchAll)
if err != nil {
return err
}
c.DataTable[i].TargetMatchAllRegexp = r
}
rdp := c.DataTable[i].RollupDefaultPrecision
rdf := c.DataTable[i].RollupDefaultFunction
if c.DataTable[i].RollupConf == "auto" || c.DataTable[i].RollupConf == "" {
table := c.DataTable[i].Table
interval := time.Minute
if c.DataTable[i].RollupAutoTable != "" {
table = c.DataTable[i].RollupAutoTable
}
if c.DataTable[i].RollupAutoInterval != nil {
interval = *c.DataTable[i].RollupAutoInterval
}
c.DataTable[i].Rollup, err = rollup.NewAuto(
c.ClickHouse.URL,
c.ClickHouse.TLSConfig,
table,
interval,
rdp,
rdf,
)
} else if c.DataTable[i].RollupConf == "none" {
c.DataTable[i].Rollup, err = rollup.NewDefault(rdp, rdf)
} else {
c.DataTable[i].Rollup, err = rollup.NewXMLFile(c.DataTable[i].RollupConf, rdp, rdf)
}
if err != nil {
return err
}
if len(c.DataTable[i].Context) == 0 {
c.DataTable[i].ContextMap = knownDataTableContext
} else {
c.DataTable[i].ContextMap = make(map[string]bool)
for _, ctx := range c.DataTable[i].Context {
if !knownDataTableContext[ctx] {
return fmt.Errorf("unknown context %#v", ctx)
}
c.DataTable[i].ContextMap[ctx] = true
}
}
}
return nil
}
func checkDeprecations(cfg *Config, d map[string]error) {
if cfg.ClickHouse.DataTableLegacy != "" {
d["data-table"] = fmt.Errorf("data-table parameter in [clickhouse] is deprecated; use [[data-table]]")
}
}
func CreateCache(cacheName string, cacheConfig *CacheConfig) (cache.BytesCache, error) {
if cacheConfig.DefaultTimeoutSec <= 0 && cacheConfig.ShortTimeoutSec <= 0 && cacheConfig.FindTimeoutSec <= 0 {
return nil, nil
}
if cacheConfig.DefaultTimeoutSec < cacheConfig.ShortTimeoutSec {
cacheConfig.DefaultTimeoutSec = cacheConfig.ShortTimeoutSec
}
if cacheConfig.ShortTimeoutSec < 0 || cacheConfig.DefaultTimeoutSec == cacheConfig.ShortTimeoutSec {
// broken value or short timeout not need due to equal
cacheConfig.ShortTimeoutSec = 0
}
if cacheConfig.DefaultTimeoutSec < cacheConfig.ShortTimeoutSec {
cacheConfig.DefaultTimeoutSec = cacheConfig.ShortTimeoutSec
}
if cacheConfig.ShortDuration == 0 {
cacheConfig.ShortDuration = 3 * time.Hour
}
if cacheConfig.ShortUntilOffsetSec == 0 {
cacheConfig.ShortUntilOffsetSec = 120
}
cacheConfig.DefaultTimeoutStr = strconv.Itoa(int(cacheConfig.DefaultTimeoutSec))
cacheConfig.ShortTimeoutStr = strconv.Itoa(int(cacheConfig.ShortTimeoutSec))
switch cacheConfig.Type {
case "memcache":
if len(cacheConfig.MemcachedServers) == 0 {
return nil, fmt.Errorf(cacheName + ": memcache cache requested but no memcache servers provided")
}
return cache.NewMemcached("gch-"+cacheName, cacheConfig.MemcachedServers...), nil
case "mem":
return cache.NewExpireCache(uint64(cacheConfig.Size * 1024 * 1024)), nil
case "null":
// defaults
return nil, nil
default:
return nil, fmt.Errorf(
"%s: unknown cache type '%s', known_cache_types 'null', 'mem', 'memcache'",
cacheName,
cacheConfig.Type,
)
}
}
func (c *Config) setupGraphiteMetrics() bool {
if c.Metrics.MetricEndpoint == "" {
metrics.DisableMetrics()
} else {
if c.Metrics.MetricInterval == 0 {
c.Metrics.MetricInterval = 60 * time.Second
}
if c.Metrics.MetricTimeout == 0 {
c.Metrics.MetricTimeout = time.Second
}
hostname, _ := os.Hostname()
fqdn := strings.ReplaceAll(hostname, ".", "_")
hostname = strings.Split(hostname, ".")[0]
c.Metrics.MetricPrefix = strings.ReplaceAll(c.Metrics.MetricPrefix, "{prefix}", c.Metrics.MetricPrefix)
c.Metrics.MetricPrefix = strings.ReplaceAll(c.Metrics.MetricPrefix, "{fqdn}", fqdn)
c.Metrics.MetricPrefix = strings.ReplaceAll(c.Metrics.MetricPrefix, "{host}", hostname)
// register our metrics with graphite
metrics.Graphite = graphite.New(c.Metrics.MetricInterval, c.Metrics.MetricPrefix, c.Metrics.MetricEndpoint, c.Metrics.MetricTimeout)
if c.Metrics.Statsd != "" && c.Metrics.ExtendedStat {
var err error
config := &statsd.ClientConfig{
Address: c.Metrics.Statsd,
Prefix: c.Metrics.MetricPrefix,
ResInterval: 5 * time.Minute,
UseBuffered: true,
FlushInterval: 300 * time.Millisecond,
}
metrics.Gstatsd, err = statsd.NewClientWithConfig(config)
if err != nil {
metrics.Gstatsd = metrics.NullSender{}
fmt.Fprintf(os.Stderr, "statsd init: %v\n", err)
}
}
metrics.InitMetrics(&c.Metrics, c.ClickHouse.FindMaxQueries > 0, c.ClickHouse.TagsMaxQueries > 0)
}
metrics.AutocompleteQMetric = metrics.InitQueryMetrics("tags", &c.Metrics)
metrics.FindQMetric = metrics.InitQueryMetrics("find", &c.Metrics)
for i := 0; i < len(c.DataTable); i++ {
c.DataTable[i].QueryMetrics = metrics.InitQueryMetrics(c.DataTable[i].Table, &c.Metrics)
}
if c.ClickHouse.IndexTable != "" {
metrics.InitQueryMetrics(c.ClickHouse.IndexTable, &c.Metrics)
}
if c.ClickHouse.TaggedTable != "" {
metrics.InitQueryMetrics(c.ClickHouse.TaggedTable, &c.Metrics)
}
if c.ClickHouse.TagsCountTable != "" {
metrics.InitQueryMetrics(c.ClickHouse.TagsCountTable, &c.Metrics)
}
return metrics.Graphite != nil
}
func (c *Config) GetUserFindLimiter(username string) limiter.ServerLimiter {
if username != "" && len(c.ClickHouse.UserLimits) > 0 {
if q, ok := c.ClickHouse.UserLimits[username]; ok {
return q.Limiter
}
}
return c.ClickHouse.FindLimiter
}
func (c *Config) GetUserTagsLimiter(username string) limiter.ServerLimiter {
if username != "" && len(c.ClickHouse.UserLimits) > 0 {
if q, ok := c.ClickHouse.UserLimits[username]; ok {
return q.Limiter
}
}
return c.ClickHouse.TagsLimiter
}
// search on sorted slice
func GetQueryParam(a []QueryParam, duration time.Duration) int {
if indx := binarySearchQueryParamLe(a, duration, 0, len(a)); indx == -1 {
return 0
} else {
return indx
}
}
================================================
FILE: config/config_test.go
================================================
package config
import (
"fmt"
"io/fs"
"math"
"net/url"
"os"
"regexp"
"regexp/syntax"
"syscall"
"testing"
"time"
"github.com/lomik/zapwriter"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/lomik/graphite-clickhouse/limiter"
"github.com/lomik/graphite-clickhouse/metrics"
)
func TestProcessDataTables(t *testing.T) {
type in struct {
table DataTable
tableLegacy string
}
type out struct {
tables []DataTable
err error
}
type ctx map[string]bool
regexpCompileWrapper := func(re string) *regexp.Regexp {
r, _ := regexp.Compile(re)
return r
}
tests := []struct {
name string
in in
out out
}{
{
name: "legacy table only",
in: in{
tableLegacy: "graphite.data",
},
out: out{
[]DataTable{
{
Table: "graphite.data",
RollupConf: "auto",
ContextMap: ctx{"graphite": true, "prometheus": true},
},
},
nil,
},
},
{
name: "legacy and normal tables",
in: in{
table: DataTable{Table: "graphite.new_data"},
tableLegacy: "graphite.data",
},
out: out{
[]DataTable{
{
Table: "graphite.new_data",
ContextMap: ctx{"graphite": true, "prometheus": true},
},
{
Table: "graphite.data",
RollupConf: "auto",
ContextMap: ctx{"graphite": true, "prometheus": true},
},
},
nil,
},
},
{
name: "fail to compile TargetMatchAll",
in: in{
table: DataTable{Table: "graphite.data", TargetMatchAll: "[2223"},
},
out: out{
[]DataTable{{Table: "graphite.data", TargetMatchAll: "[2223"}},
&syntax.Error{Code: syntax.ErrMissingBracket, Expr: "[2223"},
},
},
{
name: "fail to compile TargetMatchAny",
in: in{
table: DataTable{Table: "graphite.data", TargetMatchAny: "[2223"},
},
out: out{
[]DataTable{{Table: "graphite.data", TargetMatchAny: "[2223"}},
&syntax.Error{Code: syntax.ErrMissingBracket, Expr: "[2223"},
},
},
{
name: "fail to compile TargetMatchAny",
in: in{
table: DataTable{Table: "graphite.data", TargetMatchAny: "[2223"},
},
out: out{
[]DataTable{{Table: "graphite.data", TargetMatchAny: "[2223"}},
&syntax.Error{Code: syntax.ErrMissingBracket, Expr: "[2223"},
},
},
{
name: "fail to read xml rollup",
in: in{
table: DataTable{Table: "graphite.data", RollupConf: "/some/file/that/does/not/hopefully/exists/on/the/disk"},
},
out: out{
[]DataTable{{Table: "graphite.data", RollupConf: "/some/file/that/does/not/hopefully/exists/on/the/disk"}},
&fs.PathError{Op: "open", Path: "/some/file/that/does/not/hopefully/exists/on/the/disk", Err: syscall.ENOENT},
},
},
{
name: "unknown context",
in: in{
table: DataTable{Table: "graphite.data", Context: []string{"unexpected"}},
},
out: out{
[]DataTable{
{
Table: "graphite.data",
Context: []string{"unexpected"},
ContextMap: ctx{},
},
},
fmt.Errorf("unknown context \"unexpected\""),
},
},
{
name: "check all works",
in: in{
table: DataTable{
Table: "graphite.data",
Reverse: true,
TargetMatchAll: "^.*[asdf][.].*",
TargetMatchAny: "^.*{a|s|d|f}[.].*",
RollupConf: "none",
RollupDefaultFunction: "any",
RollupDefaultPrecision: 61,
RollupUseReverted: true,
Context: []string{"prometheus"},
},
tableLegacy: "table",
},
out: out{
[]DataTable{
{
Table: "graphite.data",
Reverse: true,
TargetMatchAll: "^.*[asdf][.].*",
TargetMatchAny: "^.*{a|s|d|f}[.].*",
TargetMatchAllRegexp: regexpCompileWrapper("^.*[asdf][.].*"),
TargetMatchAnyRegexp: regexpCompileWrapper("^.*{a|s|d|f}[.].*"),
RollupConf: "none",
RollupDefaultFunction: "any",
RollupDefaultPrecision: 61,
RollupUseReverted: true,
Context: []string{"prometheus"},
ContextMap: ctx{"prometheus": true},
},
{
Table: "table",
RollupConf: "auto",
ContextMap: ctx{"graphite": true, "prometheus": true},
},
},
nil,
},
},
{
name: "unknown context",
in: in{
table: DataTable{Table: "graphite.data", Context: []string{"unexpected"}},
},
out: out{
[]DataTable{
{
Table: "graphite.data",
Context: []string{"unexpected"},
ContextMap: ctx{},
},
},
fmt.Errorf("unknown context \"unexpected\""),
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := New()
if test.in.table.Table != "" {
cfg.DataTable = []DataTable{test.in.table}
}
if test.in.tableLegacy != "" {
cfg.ClickHouse.DataTableLegacy = test.in.tableLegacy
}
err := cfg.ProcessDataTables()
if err != nil {
assert.Equal(t, test.out.err, err)
return
}
assert.Equal(t, len(test.out.tables), len(cfg.DataTable))
// it's difficult to check rollup.Rollup because Rules.updated field
// We explicitly don't check it here
for i := range cfg.DataTable {
test.out.tables[i].Rollup = nil
cfg.DataTable[i].Rollup = nil
}
assert.Equal(t, test.out.tables, cfg.DataTable)
})
}
}
func TestKnownDataTableContext(t *testing.T) {
assert.Equal(t, map[string]bool{ContextGraphite: true, ContextPrometheus: true}, knownDataTableContext)
}
func TestReadConfig(t *testing.T) {
body := []byte(
`[common]
listen = "[::1]:9090"
pprof-listen = "127.0.0.1:9091"
max-cpu = 15
max-metrics-in-find-answer = 13
max-metrics-per-target = 16
target-blacklist = ['^blacklisted']
memory-return-interval = "12s150ms"
[clickhouse]
url = "http://somehost:8123"
index-table = "graphite_index"
index-use-daily = false
index-reverse = "direct"
index-reverses = [
{suffix = "suf", prefix = "pref", reverse = "direct"},
{regex = "^reg$", reverse = "reversed"},
]
tagged-table = "graphite_tags"
tagged-autocomplete-days = 5
tagged-use-daily = false
tree-table = "tree"
reverse-tree-table = "reversed_tree"
date-tree-table = "data_tree"
date-tree-table-version = 2
tag-table = "tag_table"
extra-prefix = "tum.pu-dum"
data-table = "data"
rollup-conf = "none"
max-data-points = 8000
internal-aggregation = true
data-timeout = "64s"
index-timeout = "4s"
tree-timeout = "5s"
connect-timeout = "2s"
# DataTable is tested in TestProcessDataTables
# [[data-table]]
# table = "another_data"
# rollup-conf = "auto"
# rollup-conf-table = "another_table"
[tags]
rules = "filename"
date = "2012-12-12"
extra-where = "AND case"
input-file = "input"
output-file = "output"
threads = 5
compression = "zstd"
version = 42
select-chunks-count = 15
[carbonlink]
server = "server:3333"
threads-per-request = 5
connect-timeout = "250ms"
query-timeout = "350ms"
total-timeout = "800ms"
[prometheus]
listen = ":9092"
external-url = "https://server:3456/uri"
page-title = "Prometheus Time Series"
lookback-delta = "5m"
[debug]
directory = "tests_tmp"
directory-perm = 0o755
external-data-perm = 0o640
[[logging]]
logger = "debugger"
file = "stdout"
level = "debug"
encoding = "console"
encoding-time = "iso8601"
encoding-duration = "string"
sample-tick = "5ms"
sample-initial = 1
sample-thereafter = 2
[[logging]]
logger = "logger"
file = "tests_tmp/logger.txt"
level = "info"
encoding = "json"
encoding-time = "epoch"
encoding-duration = "seconds"
sample-tick = "50ms"
sample-initial = 10
sample-thereafter = 12
`,
)
config, _, err := Unmarshal(body, false)
expected := New()
require.NoError(t, err)
// Common
expected.Common = Common{
Listen: "[::1]:9090",
PprofListen: "127.0.0.1:9091",
MaxCPU: 15,
MaxMetricsInFindAnswer: 13,
MaxMetricsPerTarget: 16,
TargetBlacklist: []string{"^blacklisted"},
Blacklist: make([]*regexp.Regexp, 1),
MemoryReturnInterval: 12150000000,
FindCacheConfig: CacheConfig{
Type: "null",
DefaultTimeoutSec: 0,
ShortTimeoutSec: 0,
},
DegragedMultiply: 4.0,
DegragedLoad: 1.0,
}
expe
gitextract_n919tixz/
├── .gitattributes
├── .github/
│ └── workflows/
│ ├── codeql.yml
│ ├── docker.yml
│ ├── lint.yml
│ ├── release.yml
│ ├── tests-sd.yml
│ └── tests.yml
├── .gitignore
├── .golangci.yml
├── Dockerfile
├── LICENSE
├── Makefile
├── README.md
├── autocomplete/
│ ├── autocomplete.go
│ └── autocomplete_test.go
├── cache/
│ └── cache.go
├── capabilities/
│ └── handler.go
├── cmd/
│ ├── e2e-test/
│ │ ├── carbon-clickhouse.go
│ │ ├── checks.go
│ │ ├── clickhouse.go
│ │ ├── container.go
│ │ ├── e2etesting.go
│ │ ├── errors.go
│ │ ├── graphite-clickhouse.go
│ │ ├── main.go
│ │ ├── rproxy.go
│ │ └── utils.go
│ └── graphite-clickhouse-client/
│ └── main.go
├── config/
│ ├── .gitignore
│ ├── config.go
│ ├── config_test.go
│ ├── json.go
│ └── json_test.go
├── deploy/
│ ├── doc/
│ │ ├── .gitignore
│ │ └── config.md
│ └── root/
│ └── usr/
│ └── lib/
│ └── systemd/
│ └── system/
│ └── graphite-clickhouse.service
├── doc/
│ ├── aggregation.md
│ ├── config.md
│ ├── debugging.md
│ ├── graphite_clickhouse.gliffy
│ ├── index-table.md
│ └── release.md
├── find/
│ ├── find.go
│ ├── handler.go
│ ├── handler_json_test.go
│ └── handler_test.go
├── finder/
│ ├── base.go
│ ├── blacklist.go
│ ├── date.go
│ ├── date_reverse.go
│ ├── date_reverse_test.go
│ ├── finder.go
│ ├── index.go
│ ├── index_test.go
│ ├── mock.go
│ ├── plain_from_tagged.go
│ ├── plain_from_tagged_test.go
│ ├── prefix.go
│ ├── prefix_test.go
│ ├── reverse.go
│ ├── reverse_test.go
│ ├── split.go
│ ├── split_test.go
│ ├── tag.go
│ ├── tag_test.go
│ ├── tagged.go
│ ├── tagged_test.go
│ ├── tags_count_querier.go
│ └── unescape.go
├── go.mod
├── go.sum
├── graphite-clickhouse.go
├── healthcheck/
│ └── healthcheck.go
├── helper/
│ ├── RowBinary/
│ │ └── encode.go
│ ├── clickhouse/
│ │ ├── clickhouse.go
│ │ ├── clickhouse_test.go
│ │ ├── external-data.go
│ │ └── external-data_test.go
│ ├── client/
│ │ ├── datetime.go
│ │ ├── errros.go
│ │ ├── find.go
│ │ ├── render.go
│ │ ├── requests.go
│ │ ├── tags.go
│ │ └── types.go
│ ├── date/
│ │ ├── date.go
│ │ └── date_test.go
│ ├── datetime/
│ │ ├── datetime.go
│ │ └── datetime_test.go
│ ├── errs/
│ │ └── errors.go
│ ├── headers/
│ │ └── headers.go
│ ├── http/
│ │ └── live-http-client.go
│ ├── pickle/
│ │ └── pickle.go
│ ├── point/
│ │ ├── func.go
│ │ ├── func_test.go
│ │ ├── point.go
│ │ └── points.go
│ ├── rollup/
│ │ ├── aggr.go
│ │ ├── compact.go
│ │ ├── compact_test.go
│ │ ├── remote.go
│ │ ├── remote_test.go
│ │ ├── rollup.go
│ │ ├── rules.go
│ │ ├── rules_test.go
│ │ ├── xml.go
│ │ └── xml_test.go
│ ├── tests/
│ │ ├── clickhouse/
│ │ │ └── server.go
│ │ └── compare/
│ │ ├── compare.go
│ │ └── expand/
│ │ └── expand.go
│ └── utils/
│ ├── utils.go
│ └── utils_test.go
├── index/
│ ├── handler.go
│ ├── index.go
│ └── index_test.go
├── issues/
│ └── daytime/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse-internal-aggr.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── limiter/
│ ├── alimiter.go
│ ├── alimiter_test.go
│ ├── interface.go
│ ├── limiter.go
│ ├── noop.go
│ └── wlimiter.go
├── load_avg/
│ ├── load_avg.go
│ ├── load_avg_default.go
│ ├── load_avg_linux.go
│ └── load_avg_test.go
├── logs/
│ └── logger.go
├── metrics/
│ ├── limiter_metrics.go
│ ├── metrics.go
│ ├── metrics_test.go
│ ├── query_metrics.go
│ └── statsd.go
├── nfpm.yaml
├── packages.sh
├── pkg/
│ ├── alias/
│ │ ├── map.go
│ │ ├── map_tagged_test.go
│ │ └── map_test.go
│ ├── dry/
│ │ ├── math.go
│ │ ├── math_test.go
│ │ ├── strings.go
│ │ ├── strings_test.go
│ │ ├── unsafe.go
│ │ └── unsafe_test.go
│ ├── reverse/
│ │ ├── reverse.go
│ │ └── reverse_test.go
│ ├── scope/
│ │ ├── context.go
│ │ ├── http_request.go
│ │ ├── key.go
│ │ ├── logger.go
│ │ └── version.go
│ └── where/
│ ├── match.go
│ ├── match_test.go
│ ├── where.go
│ └── where_test.go
├── prometheus/
│ ├── .gitignore
│ ├── empty_iterator.go
│ ├── exemplar.go
│ ├── gatherer.go
│ ├── labels.go
│ ├── labels_test.go
│ ├── local_storage.go
│ ├── logger.go
│ ├── matcher.go
│ ├── metrics_set.go
│ ├── querier.go
│ ├── querier_select.go
│ ├── querier_select_test.go
│ ├── run.go
│ ├── run_dummy.go
│ ├── series_set.go
│ └── storage.go
├── render/
│ ├── data/
│ │ ├── carbonlink.go
│ │ ├── carbonlink_test.go
│ │ ├── ch_response.go
│ │ ├── common_step.go
│ │ ├── common_step_test.go
│ │ ├── data.go
│ │ ├── data_parse_test.go
│ │ ├── multi_target.go
│ │ ├── multi_target_test.go
│ │ ├── query.go
│ │ ├── query_test.go
│ │ ├── targets.go
│ │ └── targets_test.go
│ ├── handler.go
│ ├── handler_test.go
│ └── reply/
│ ├── formatter.go
│ ├── formatter_test.go
│ ├── json.go
│ ├── pickle.go
│ ├── protobuf.go
│ ├── protobuf_test.go
│ ├── v2_pb.go
│ ├── v2_pb_test.go
│ ├── v3_pb.go
│ └── v3_pb_test.go
├── sd/
│ ├── nginx/
│ │ ├── nginx.go
│ │ ├── nginx_test.go
│ │ └── tests/
│ │ └── nginx_cleanup_test.go
│ ├── register.go
│ └── utils/
│ └── utils.go
├── tagger/
│ ├── metric.go
│ ├── rule.go
│ ├── rule_test.go
│ ├── set.go
│ ├── tagger.go
│ ├── tagger_test.go
│ └── tree.go
└── tests/
├── agg_internal/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse-internal-aggr.conf.tpl
│ └── test.toml
├── agg_latest/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── agg_merge/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse-internal-aggr.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── agg_oneblock/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse-internal-aggr.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── clickhouse/
│ ├── rollup/
│ │ ├── config.xml
│ │ ├── init.sql
│ │ ├── rollup.xml
│ │ └── users.xml
│ └── rollup_tls/
│ ├── config.xml
│ ├── init.sql
│ ├── rollup.xml
│ ├── rootCA.crt
│ ├── server.crt
│ ├── server.key
│ └── users.xml
├── consolidateBy/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── consul.sh
├── emptyseries_append/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── emptyseries_noappend/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── error_handling/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── feature_flags_both_true/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── feature_flags_dont_match_missing_tags/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── feature_flags_false/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── feature_flags_use_carbon_behaviour/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── find_cache/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse-cached.conf.tpl
│ ├── graphite-clickhouse-internal-aggr-cached.conf.tpl
│ └── test.toml
├── limitera/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse-internal-aggr-cached.conf.tpl
│ └── test.toml
├── limitermax/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse-internal-aggr-cached.conf.tpl
│ └── test.toml
├── limiterw/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse-internal-aggr-cached.conf.tpl
│ └── test.toml
├── limiterwn/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse-internal-aggr-cached.conf.tpl
│ └── test.toml
├── one_table/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse-internal-aggr.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── tags_min_in_query/
│ ├── carbon-clickhouse.conf.tpl
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
├── tls/
│ ├── ca.crt
│ ├── carbon-clickhouse.conf.tpl
│ ├── client.crt
│ ├── client.key
│ ├── graphite-clickhouse.conf.tpl
│ └── test.toml
└── wildcard_min_distance/
├── carbon-clickhouse.conf.tpl
├── graphite-clickhouse.conf.tpl
└── test.toml
SYMBOL INDEX (1236 symbols across 178 files)
FILE: autocomplete/autocomplete.go
type Handler (line 31) | type Handler struct
method ServeHTTP (line 60) | func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
method requestExpr (line 90) | func (h *Handler) requestExpr(r *http.Request, tcq *finder.TagCountQue...
method ServeTags (line 232) | func (h *Handler) ServeTags(w http.ResponseWriter, r *http.Request) {
method ServeValues (line 517) | func (h *Handler) ServeValues(w http.ResponseWriter, r *http.Request) {
function NewTags (line 36) | func NewTags(config *config.Config) *Handler {
function NewValues (line 44) | func NewValues(config *config.Config) *Handler {
function dateString (line 53) | func dateString(autocompleteDays int, tm time.Time) (string, string) {
function getTagCountQuerier (line 74) | func getTagCountQuerier(config *config.Config, opts clickhouse.Options) ...
function taggedKey (line 142) | func taggedKey(typ string, truncateSec int32, fromDate, untilDate string...
function taggedValuesKey (line 181) | func taggedValuesKey(typ string, truncateSec int32, fromDate, untilDate ...
FILE: autocomplete/autocomplete_test.go
function NewRequest (line 18) | func NewRequest(method, url string, body io.Reader) *http.Request {
type testStruct (line 24) | type testStruct struct
function testResponce (line 31) | func testResponce(t *testing.T, step int, h *Handler, tt *testStruct, wa...
function TestHandler_ServeTags (line 53) | func TestHandler_ServeTags(t *testing.T) {
function TestHandler_ServeTagsWithCache (line 147) | func TestHandler_ServeTagsWithCache(t *testing.T) {
function TestHandler_ServeValues (line 209) | func TestHandler_ServeValues(t *testing.T) {
function TestHandler_ServeValuesWithValuePrefix (line 262) | func TestHandler_ServeValuesWithValuePrefix(t *testing.T) {
function TestHandler_ServeValuesNameTag (line 298) | func TestHandler_ServeValuesNameTag(t *testing.T) {
function TestHandler_ServeValuesWithCache (line 334) | func TestHandler_ServeValuesWithCache(t *testing.T) {
function TestHandler_ServeValuesWithCacheAndExpr (line 396) | func TestHandler_ServeValuesWithCacheAndExpr(t *testing.T) {
function TestHandler_ServeValuesWithInvalidLimit (line 469) | func TestHandler_ServeValuesWithInvalidLimit(t *testing.T) {
function TestHandler_ServeValuesWithMultipleExpr (line 494) | func TestHandler_ServeValuesWithMultipleExpr(t *testing.T) {
function TestHandler_ServeValuesNoCache (line 531) | func TestHandler_ServeValuesNoCache(t *testing.T) {
function TestHandler_ServeValuesEmptyResult (line 586) | func TestHandler_ServeValuesEmptyResult(t *testing.T) {
function TestHandler_ServeTagsWithCostOptimization (line 622) | func TestHandler_ServeTagsWithCostOptimization(t *testing.T) {
function TestHandler_ServeValuesWithCostOptimization (line 673) | func TestHandler_ServeValuesWithCostOptimization(t *testing.T) {
function TestHandler_ServeTagsWithWildcardExpressions (line 724) | func TestHandler_ServeTagsWithWildcardExpressions(t *testing.T) {
function TestHandler_ServeValuesWithNoEqualityTerms (line 762) | func TestHandler_ServeValuesWithNoEqualityTerms(t *testing.T) {
function TestHandler_ServeTagsWithHighCostTags (line 801) | func TestHandler_ServeTagsWithHighCostTags(t *testing.T) {
function TestHandler_ServeValuesWithMixedOperators (line 851) | func TestHandler_ServeValuesWithMixedOperators(t *testing.T) {
FILE: cache/cache.go
type BytesCache (line 20) | type BytesCache interface
function NewExpireCache (line 25) | func NewExpireCache(maxsize uint64) BytesCache {
type ExpireCache (line 32) | type ExpireCache struct
method Get (line 36) | func (ec ExpireCache) Get(k string) ([]byte, error) {
method Set (line 46) | func (ec ExpireCache) Set(k string, v []byte, expire int32) {
function NewMemcached (line 50) | func NewMemcached(prefix string, servers ...string) BytesCache {
type MemcachedCache (line 54) | type MemcachedCache struct
method Get (line 60) | func (m *MemcachedCache) Get(k string) ([]byte, error) {
method Set (line 95) | func (m *MemcachedCache) Set(k string, v []byte, expire int32) {
method Timeouts (line 104) | func (m *MemcachedCache) Timeouts() uint64 {
FILE: capabilities/handler.go
type Handler (line 16) | type Handler struct
method ServeHTTP (line 26) | func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
function NewHandler (line 20) | func NewHandler(config *config.Config) *Handler {
FILE: cmd/e2e-test/carbon-clickhouse.go
type CarbonClickhouse (line 15) | type CarbonClickhouse struct
method Start (line 29) | func (c *CarbonClickhouse) Start(testDir, clickhouseURL string) (strin...
method Stop (line 117) | func (c *CarbonClickhouse) Stop(delete bool) (string, error) {
method Delete (line 134) | func (c *CarbonClickhouse) Delete() (string, error) {
method Cleanup (line 153) | func (c *CarbonClickhouse) Cleanup() {
method Address (line 160) | func (c *CarbonClickhouse) Address() string {
method Container (line 164) | func (c *CarbonClickhouse) Container() string {
FILE: cmd/e2e-test/checks.go
function isFindCached (line 20) | func isFindCached(header http.Header) (string, bool) {
function requestId (line 33) | func requestId(header http.Header) string {
function compareFindMatch (line 46) | func compareFindMatch(errors *[]string, name, url string, actual, expect...
function verifyMetricsFind (line 74) | func verifyMetricsFind(ch *Clickhouse, gch *GraphiteClickhouse, check *M...
function compareTags (line 130) | func compareTags(errors *[]string, name, url string, actual, expected []...
function verifyTags (line 158) | func verifyTags(ch *Clickhouse, gch *GraphiteClickhouse, check *TagsChec...
function compareRender (line 234) | func compareRender(errors *[]string, name, url string, actual, expected ...
function parseFilteringFunctions (line 314) | func parseFilteringFunctions(strFilteringFuncs []string) ([]*carbonapi_v...
function verifyRender (line 337) | func verifyRender(ch *Clickhouse, gch *GraphiteClickhouse, check *Render...
function debug (line 407) | func debug(test *TestSchema, ch *Clickhouse, gch *GraphiteClickhouse) {
FILE: cmd/e2e-test/clickhouse.go
type Clickhouse (line 23) | type Clickhouse struct
method CheckConfig (line 39) | func (c *Clickhouse) CheckConfig(rootDir string) error {
method Key (line 72) | func (c *Clickhouse) Key() string {
method Start (line 76) | func (c *Clickhouse) Start() (string, error) {
method Stop (line 131) | func (c *Clickhouse) Stop(delete bool) (string, error) {
method Delete (line 148) | func (c *Clickhouse) Delete() (string, error) {
method URL (line 165) | func (c *Clickhouse) URL() string {
method TLSURL (line 169) | func (c *Clickhouse) TLSURL() string {
method Container (line 173) | func (c *Clickhouse) Container() string {
method Exec (line 177) | func (c *Clickhouse) Exec(sql string) (bool, string) {
method Query (line 181) | func (c *Clickhouse) Query(sql string) (string, error) {
method Alive (line 207) | func (c *Clickhouse) Alive() bool {
method CopyLog (line 222) | func (c *Clickhouse) CopyLog(destDir string, tail uint64) error {
method CopyErrLog (line 246) | func (c *Clickhouse) CopyErrLog(destDir string, tail uint64) error {
FILE: cmd/e2e-test/container.go
function imageDelete (line 13) | func imageDelete(image, version string) (bool, string) {
function containerExist (line 31) | func containerExist(name string) (bool, string) {
function containerRemove (line 49) | func containerRemove(name string) (bool, string) {
function containerExec (line 67) | func containerExec(name string, args []string) (bool, string) {
FILE: cmd/e2e-test/e2etesting.go
type Point (line 32) | type Point struct
type InputMetric (line 40) | type InputMetric struct
type Metric (line 46) | type Metric struct
type RenderCheck (line 61) | type RenderCheck struct
type MetricsFindCheck (line 89) | type MetricsFindCheck struct
type TagsCheck (line 113) | type TagsCheck struct
type TestSchema (line 139) | type TestSchema struct
method HasTLSSettings (line 158) | func (schema *TestSchema) HasTLSSettings() bool {
function getFreeTCPPort (line 162) | func getFreeTCPPort(name string) (string, error) {
function sendPlain (line 184) | func sendPlain(network, address string, metrics []InputMetric) error {
function verifyGraphiteClickhouse (line 219) | func verifyGraphiteClickhouse(test *TestSchema, gch *GraphiteClickhouse,...
function testGraphiteClickhouse (line 449) | func testGraphiteClickhouse(test *TestSchema, clickhouse *Clickhouse, te...
function runTest (line 561) | func runTest(cfg *MainConfig, clickhouse *Clickhouse, rootDir string, no...
function clickhouseStart (line 591) | func clickhouseStart(clickhouse *Clickhouse, logger *zap.Logger) bool {
function clickhouseStop (line 608) | func clickhouseStop(clickhouse *Clickhouse, logger *zap.Logger) (result ...
function initTest (line 632) | func initTest(cfg *MainConfig, rootDir string, now time.Time, verbose, b...
function loadConfig (line 886) | func loadConfig(config string, rootDir string) (*MainConfig, error) {
FILE: cmd/e2e-test/graphite-clickhouse.go
type GraphiteClickhouse (line 20) | type GraphiteClickhouse struct
method Start (line 33) | func (c *GraphiteClickhouse) Start(testDir, chURL, chProxyURL, chTLSUR...
method Alive (line 119) | func (c *GraphiteClickhouse) Alive() bool {
method Stop (line 129) | func (c *GraphiteClickhouse) Stop(cleanup bool) error {
method Cleanup (line 155) | func (c *GraphiteClickhouse) Cleanup() {
method URL (line 163) | func (c *GraphiteClickhouse) URL() string {
method Cmd (line 167) | func (c *GraphiteClickhouse) Cmd() string {
method Grep (line 171) | func (c *GraphiteClickhouse) Grep(s string) {
FILE: cmd/e2e-test/main.go
type MainConfig (line 14) | type MainConfig struct
function IsDir (line 18) | func IsDir(filename string) (bool, error) {
function expandDir (line 29) | func expandDir(dirname string, paths *[]string) error {
function expandFilename (line 51) | func expandFilename(filename string, paths *[]string) error {
function main (line 70) | func main() {
FILE: cmd/e2e-test/rproxy.go
type AtomicDuration (line 16) | type AtomicDuration struct
method Store (line 20) | func (d *AtomicDuration) Store(duration time.Duration) {
method Load (line 24) | func (d *AtomicDuration) Load() time.Duration {
method MarshalText (line 28) | func (d *AtomicDuration) MarshalText() ([]byte, error) {
method UnmarshalText (line 33) | func (d *AtomicDuration) UnmarshalText(b []byte) error {
type HttpReverseProxy (line 44) | type HttpReverseProxy struct
method Start (line 53) | func (p *HttpReverseProxy) Start(remoteURL string) (err error) {
method Stop (line 81) | func (p *HttpReverseProxy) Stop() {
method URL (line 92) | func (p *HttpReverseProxy) URL() string {
method SetDelay (line 96) | func (p *HttpReverseProxy) SetDelay(delay time.Duration) {
method GetDelay (line 100) | func (p *HttpReverseProxy) GetDelay() time.Duration {
method SetBreakStatusCode (line 104) | func (p *HttpReverseProxy) SetBreakStatusCode(statusCode int) {
method GetBreakStatusCode (line 108) | func (p *HttpReverseProxy) GetBreakStatusCode() int {
method ServeHTTP (line 112) | func (p *HttpReverseProxy) ServeHTTP(w http.ResponseWriter, r *http.Re...
FILE: cmd/e2e-test/utils.go
function cmdExec (line 5) | func cmdExec(programm string, args ...string) (string, error) {
FILE: cmd/graphite-clickhouse-client/main.go
type StringSlice (line 17) | type StringSlice
method Set (line 19) | func (u *StringSlice) Set(value string) error {
method String (line 24) | func (u *StringSlice) String() string {
method Type (line 28) | func (u *StringSlice) Type() string {
function main (line 32) | func main() {
FILE: config/config.go
type SDType (line 35) | type SDType
method Set (line 44) | func (a *SDType) Set(value string) error {
method UnmarshalText (line 57) | func (a *SDType) UnmarshalText(data []byte) error {
method MarshalText (line 61) | func (a *SDType) MarshalText() ([]byte, error) {
method UnmarshalJSON (line 65) | func (a *SDType) UnmarshalJSON(data []byte) error {
method MarshalJSON (line 69) | func (a *SDType) MarshalJSON() ([]byte, error) {
method String (line 73) | func (a *SDType) String() string {
method Type (line 77) | func (a *SDType) Type() string {
constant SDNone (line 38) | SDNone SDType = iota
constant SDNginx (line 39) | SDNginx
type CacheConfig (line 82) | type CacheConfig struct
type Common (line 96) | type Common struct
type FeatureFlags (line 123) | type FeatureFlags struct
type IndexReverseRule (line 130) | type IndexReverseRule struct
type Costs (line 138) | type Costs struct
type IndexReverses (line 144) | type IndexReverses
method Compile (line 446) | func (ir IndexReverses) Compile() error {
constant IndexAuto (line 147) | IndexAuto = iota
constant IndexDirect (line 148) | IndexDirect = iota
constant IndexReversed (line 149) | IndexReversed = iota
type UserLimits (line 162) | type UserLimits struct
type QueryParam (line 170) | type QueryParam struct
function binarySearchQueryParamLe (line 182) | func binarySearchQueryParamLe(a []QueryParam, duration time.Duration, st...
type ClickHouse (line 209) | type ClickHouse struct
function clickhouseURLValidate (line 268) | func clickhouseURLValidate(chURL string) (*url.URL, error) {
type Tags (line 282) | type Tags struct
type Carbonlink (line 295) | type Carbonlink struct
type Prometheus (line 305) | type Prometheus struct
constant ContextGraphite (line 316) | ContextGraphite = "graphite"
constant ContextPrometheus (line 318) | ContextPrometheus = "prometheus"
type DataTable (line 327) | type DataTable struct
type Debug (line 351) | type Debug struct
type Config (line 360) | type Config struct
method NeedLoadAvgColect (line 778) | func (c *Config) NeedLoadAvgColect() bool {
method ProcessDataTables (line 826) | func (c *Config) ProcessDataTables() (err error) {
method setupGraphiteMetrics (line 959) | func (c *Config) setupGraphiteMetrics() bool {
method GetUserFindLimiter (line 1026) | func (c *Config) GetUserFindLimiter(username string) limiter.ServerLim...
method GetUserTagsLimiter (line 1036) | func (c *Config) GetUserTagsLimiter(username string) limiter.ServerLim...
function New (line 374) | func New() *Config {
function newLoggingConfig (line 466) | func newLoggingConfig() zapwriter.Config {
function DefaultConfig (line 473) | func DefaultConfig() (*Config, error) {
function PrintDefaultConfig (line 512) | func PrintDefaultConfig() error {
function ReadConfig (line 534) | func ReadConfig(filename string, exactConfig bool) (*Config, []zap.Field...
function Unmarshal (line 549) | func Unmarshal(body []byte, exactConfig bool) (cfg *Config, warns []zap....
function checkDeprecations (line 903) | func checkDeprecations(cfg *Config, d map[string]error) {
function CreateCache (line 909) | func CreateCache(cacheName string, cacheConfig *CacheConfig) (cache.Byte...
function GetQueryParam (line 1047) | func GetQueryParam(a []QueryParam, duration time.Duration) int {
FILE: config/config_test.go
function TestProcessDataTables (line 23) | func TestProcessDataTables(t *testing.T) {
function TestKnownDataTableContext (line 228) | func TestKnownDataTableContext(t *testing.T) {
function TestReadConfig (line 232) | func TestReadConfig(t *testing.T) {
function TestReadConfigGraphiteWithLimiter (line 444) | func TestReadConfigGraphiteWithLimiter(t *testing.T) {
function TestReadConfigGraphiteWithALimiter (line 761) | func TestReadConfigGraphiteWithALimiter(t *testing.T) {
function TestGetQueryParamBroken (line 1088) | func TestGetQueryParamBroken(t *testing.T) {
function TestGetQueryParam (line 1120) | func TestGetQueryParam(t *testing.T) {
function TestClickHouse_Validate (line 1297) | func TestClickHouse_Validate(t *testing.T) {
FILE: config/json.go
method MarshalJSON (line 8) | func (c *ClickHouse) MarshalJSON() ([]byte, error) {
FILE: config/json_test.go
function TestClickhouseUrlPassword (line 10) | func TestClickhouseUrlPassword(t *testing.T) {
FILE: find/find.go
type Find (line 17) | type Find struct
method isResultsLimitExceeded (line 45) | func (f *Find) isResultsLimitExceeded(numResults int) bool {
method WritePickle (line 50) | func (f *Find) WritePickle(w io.Writer) error {
method WriteProtobuf (line 94) | func (f *Find) WriteProtobuf(w io.Writer) error {
method WriteProtobufV3 (line 144) | func (f *Find) WriteProtobufV3(w io.Writer) error {
method WriteJSON (line 200) | func (f *Find) WriteJSON(w io.Writer) error {
function NewCached (line 24) | func NewCached(config *config.Config, body []byte) *Find {
function New (line 31) | func New(config *config.Config, ctx context.Context, query string) (*Fin...
FILE: find/handler.go
type Handler (line 22) | type Handler struct
method ServeHTTP (line 34) | func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
method Reply (line 228) | func (h *Handler) Reply(w http.ResponseWriter, r *http.Request, f *Fin...
function NewHandler (line 27) | func NewHandler(config *config.Config) *Handler {
FILE: find/handler_json_test.go
function NewRequest (line 17) | func NewRequest(method, url string, body io.Reader) *http.Request {
type testStruct (line 23) | type testStruct struct
function testResponce (line 30) | func testResponce(t *testing.T, step int, h *Handler, tt *testStruct, wa...
function TestHandler_ServeValuesJSON (line 52) | func TestHandler_ServeValuesJSON(t *testing.T) {
function TestHandler_ServeValuesCachedJSON (line 104) | func TestHandler_ServeValuesCachedJSON(t *testing.T) {
FILE: find/handler_test.go
type clickhouseMock (line 12) | type clickhouseMock struct
method ServeHTTP (line 16) | func (m *clickhouseMock) ServeHTTP(w http.ResponseWriter, r *http.Requ...
function TestFind (line 24) | func TestFind(t *testing.T) {
FILE: finder/base.go
type BaseFinder (line 19) | type BaseFinder struct
method where (line 36) | func (b *BaseFinder) where(query string) *where.Where {
method Execute (line 46) | func (b *BaseFinder) Execute(ctx context.Context, config *config.Confi...
method makeList (line 66) | func (b *BaseFinder) makeList(onlySeries bool) [][]byte {
method List (line 96) | func (b *BaseFinder) List() [][]byte {
method Series (line 100) | func (b *BaseFinder) Series() [][]byte {
method Abs (line 104) | func (b *BaseFinder) Abs(v []byte) []byte {
method Bytes (line 108) | func (b *BaseFinder) Bytes() ([]byte, error) {
method Stats (line 112) | func (b *BaseFinder) Stats() []metrics.FinderStat {
function NewBase (line 27) | func NewBase(url string, table string, opts clickhouse.Options) Finder {
FILE: finder/blacklist.go
type BlacklistFinder (line 11) | type BlacklistFinder struct
method Execute (line 24) | func (p *BlacklistFinder) Execute(ctx context.Context, config *config....
method List (line 35) | func (p *BlacklistFinder) List() [][]byte {
method Series (line 44) | func (p *BlacklistFinder) Series() [][]byte {
method Abs (line 52) | func (p *BlacklistFinder) Abs(v []byte) []byte {
method Bytes (line 56) | func (p *BlacklistFinder) Bytes() ([]byte, error) {
method Stats (line 60) | func (p *BlacklistFinder) Stats() []metrics.FinderStat {
function WrapBlacklist (line 17) | func WrapBlacklist(f Finder, blacklist []*regexp.Regexp) *BlacklistFinder {
FILE: finder/date.go
type DateFinder (line 15) | type DateFinder struct
method Execute (line 34) | func (b *DateFinder) Execute(ctx context.Context, config *config.Confi...
function NewDateFinder (line 20) | func NewDateFinder(url string, table string, tableVersion int, opts clic...
FILE: finder/date_reverse.go
type DateFinderV3 (line 15) | type DateFinderV3 struct
method whereFilter (line 30) | func (f *DateFinderV3) whereFilter(query string, from int64, until int...
method Execute (line 43) | func (f *DateFinderV3) Execute(ctx context.Context, config *config.Con...
method List (line 63) | func (f *DateFinderV3) List() [][]byte {
method Series (line 72) | func (f *DateFinderV3) Series() [][]byte {
function NewDateFinderV3 (line 20) | func NewDateFinderV3(url string, table string, opts clickhouse.Options) ...
FILE: finder/date_reverse_test.go
function TestDateFinderV3_whereFilter (line 11) | func TestDateFinderV3_whereFilter(t *testing.T) {
FILE: finder/finder.go
type Result (line 13) | type Result interface
type Finder (line 20) | type Finder interface
function newPlainFinder (line 25) | func newPlainFinder(ctx context.Context, config *config.Config, query st...
function Find (line 107) | func Find(config *config.Config, ctx context.Context, query string, from...
function Leaf (line 116) | func Leaf(value []byte) ([]byte, bool) {
function FindTagged (line 124) | func FindTagged(ctx context.Context, config *config.Config, terms []Tagg...
FILE: finder/index.go
constant ReverseLevelOffset (line 19) | ReverseLevelOffset = 10000
constant TreeLevelOffset (line 20) | TreeLevelOffset = 20000
constant ReverseTreeLevelOffset (line 21) | ReverseTreeLevelOffset = 30000
constant DefaultTreeDate (line 23) | DefaultTreeDate = "1970-02-12"
constant queryAuto (line 26) | queryAuto = config.IndexAuto
constant queryDirect (line 27) | queryDirect = config.IndexDirect
constant queryReversed (line 28) | queryReversed = config.IndexReversed
type IndexFinder (line 31) | type IndexFinder struct
method where (line 69) | func (idx *IndexFinder) where(query string, levelOffset int) *where.Wh...
method checkReverses (line 80) | func (idx *IndexFinder) checkReverses(query string) uint8 {
method useReverse (line 100) | func (idx *IndexFinder) useReverse(query string) bool {
method whereFilter (line 164) | func (idx *IndexFinder) whereFilter(query string, from int64, until in...
method Execute (line 199) | func (idx *IndexFinder) Execute(ctx context.Context, config *config.Co...
method Abs (line 228) | func (idx *IndexFinder) Abs(v []byte) []byte {
method bodySplit (line 263) | func (idx *IndexFinder) bodySplit() {
method makeList (line 286) | func (idx *IndexFinder) makeList(onlySeries bool) [][]byte {
method List (line 290) | func (idx *IndexFinder) List() [][]byte {
method Series (line 294) | func (idx *IndexFinder) Series() [][]byte {
method Bytes (line 298) | func (idx *IndexFinder) Bytes() ([]byte, error) {
method Stats (line 302) | func (idx *IndexFinder) Stats() []metrics.FinderStat {
function NewCachedIndex (line 46) | func NewCachedIndex(body []byte) Finder {
function NewIndex (line 56) | func NewIndex(url string, table string, dailyEnabled bool, reverse strin...
function useDaily (line 132) | func useDaily(dailyEnabled bool, from, until int64) bool {
function calculateIndexLevelOffset (line 136) | func calculateIndexLevelOffset(useDaily, reverse bool) int {
function addDatesToWhere (line 152) | func addDatesToWhere(w *where.Where, useDaily bool, from, until int64) {
function validatePlainQuery (line 180) | func validatePlainQuery(query string, wildcardMinDistance int) error {
function splitIndexBody (line 232) | func splitIndexBody(body []byte, useReverse, useCache bool) ([]byte, [][...
function makeList (line 272) | func makeList(rows [][]byte, onlySeries bool) [][]byte {
FILE: finder/index_test.go
function Test_useReverse (line 14) | func Test_useReverse(t *testing.T) {
function Test_useReverseWithSetConfig (line 35) | func Test_useReverseWithSetConfig(t *testing.T) {
function Test_checkReverses (line 63) | func Test_checkReverses(t *testing.T) {
function Benchmark_useReverseDepth (line 96) | func Benchmark_useReverseDepth(b *testing.B) {
function Benchmark_useReverseDepthPrefixSuffix (line 111) | func Benchmark_useReverseDepthPrefixSuffix(b *testing.B) {
function Benchmark_useReverseDepthRegex (line 126) | func Benchmark_useReverseDepthRegex(b *testing.B) {
function TestIndexFinder_whereFilter (line 141) | func TestIndexFinder_whereFilter(t *testing.T) {
FILE: finder/mock.go
type MockFinder (line 13) | type MockFinder struct
method Execute (line 33) | func (m *MockFinder) Execute(ctx context.Context, config *config.Confi...
method List (line 39) | func (m *MockFinder) List() [][]byte {
method Series (line 44) | func (m *MockFinder) Series() [][]byte {
method Abs (line 49) | func (m *MockFinder) Abs(v []byte) []byte {
method Bytes (line 53) | func (m *MockFinder) Bytes() ([]byte, error) {
method Strings (line 58) | func (m *MockFinder) Strings() []string {
method Stats (line 63) | func (m *MockFinder) Stats() []metrics.FinderStat {
function NewMockFinder (line 19) | func NewMockFinder(result [][]byte) *MockFinder {
function NewMockTagged (line 26) | func NewMockTagged(result [][]byte) *MockFinder {
FILE: finder/plain_from_tagged.go
type plainFromTaggedFinder (line 16) | type plainFromTaggedFinder struct
method Target (line 67) | func (f *plainFromTaggedFinder) Target() string {
method Execute (line 71) | func (f *plainFromTaggedFinder) Execute(ctx context.Context, config *c...
method Series (line 76) | func (f *plainFromTaggedFinder) Series() [][]byte {
method Abs (line 85) | func (f *plainFromTaggedFinder) Abs(value []byte) []byte {
method List (line 126) | func (f *plainFromTaggedFinder) List() [][]byte {
method Bytes (line 130) | func (f *plainFromTaggedFinder) Bytes() ([]byte, error) {
method Stats (line 134) | func (f *plainFromTaggedFinder) Stats() []metrics.FinderStat {
function makePlainFromTagged (line 23) | func makePlainFromTagged(matchers []TaggedTerm) *plainFromTaggedFinder {
type taggedLabel (line 80) | type taggedLabel struct
FILE: finder/plain_from_tagged_test.go
function TestPlainFromTaggedFinderAbs (line 9) | func TestPlainFromTaggedFinderAbs(t *testing.T) {
FILE: finder/prefix.go
type PrefixMatchResult (line 13) | type PrefixMatchResult
constant PrefixNotMatched (line 16) | PrefixNotMatched PrefixMatchResult = iota
constant PrefixMatched (line 17) | PrefixMatched
constant PrefixPartialMathed (line 18) | PrefixPartialMathed
type PrefixFinder (line 21) | type PrefixFinder struct
method Execute (line 46) | func (p *PrefixFinder) Execute(ctx context.Context, config *config.Con...
method List (line 83) | func (p *PrefixFinder) List() [][]byte {
method Series (line 103) | func (p *PrefixFinder) Series() [][]byte {
method Abs (line 115) | func (p *PrefixFinder) Abs(value []byte) []byte {
method Bytes (line 119) | func (p *PrefixFinder) Bytes() ([]byte, error) {
method Stats (line 123) | func (p *PrefixFinder) Stats() []metrics.FinderStat {
function bytesConcat (line 29) | func bytesConcat(s1 []byte, s2 []byte) []byte {
function WrapPrefix (line 37) | func WrapPrefix(f Finder, prefix string) *PrefixFinder {
FILE: finder/prefix_test.go
function TestPrefixFinderExecute (line 12) | func TestPrefixFinderExecute(t *testing.T) {
function TestPrefixFinderAbs (line 54) | func TestPrefixFinderAbs(t *testing.T) {
function TestPrefixFinderList (line 63) | func TestPrefixFinderList(t *testing.T) {
FILE: finder/reverse.go
type ReverseFinder (line 14) | type ReverseFinder struct
method Execute (line 54) | func (r *ReverseFinder) Execute(ctx context.Context, config *config.Co...
method List (line 69) | func (r *ReverseFinder) List() [][]byte {
method Series (line 82) | func (r *ReverseFinder) Series() [][]byte {
method Abs (line 95) | func (r *ReverseFinder) Abs(v []byte) []byte {
method Bytes (line 99) | func (f *ReverseFinder) Bytes() ([]byte, error) {
method Stats (line 103) | func (f *ReverseFinder) Stats() []metrics.FinderStat {
function ReverseString (line 22) | func ReverseString(target string) string {
function ReverseBytes (line 33) | func ReverseBytes(target []byte) []byte {
function WrapReverse (line 45) | func WrapReverse(f Finder, url string, table string, opts clickhouse.Opt...
FILE: finder/reverse_test.go
function TestReverse (line 9) | func TestReverse(t *testing.T) {
FILE: finder/split.go
type indexFinderParams (line 17) | type indexFinderParams struct
type SplitIndexFinder (line 29) | type SplitIndexFinder struct
method Execute (line 73) | func (splitFinder *SplitIndexFinder) Execute(
method whereFilter (line 241) | func (splitFinder *SplitIndexFinder) whereFilter(queries []string, fro...
method List (line 295) | func (splitFinder *SplitIndexFinder) List() [][]byte {
method Series (line 304) | func (splitFinder *SplitIndexFinder) Series() [][]byte {
method Abs (line 314) | func (splitFinder *SplitIndexFinder) Abs(v []byte) []byte {
method Bytes (line 324) | func (splitFinder *SplitIndexFinder) Bytes() ([]byte, error) {
method Stats (line 332) | func (splitFinder *SplitIndexFinder) Stats() []metrics.FinderStat {
function WrapSplitIndex (line 43) | func WrapSplitIndex(
function splitQuery (line 130) | func splitQuery(query string, maxNodeToSplitIdx int) ([]string, error) {
function splitPartOfQuery (line 226) | func splitPartOfQuery(prefix, queryPart, suffix string) ([]string, error) {
FILE: finder/split_test.go
function Test_splitQuery (line 16) | func Test_splitQuery(t *testing.T) {
function TestSplitIndexFinder_whereFilter (line 203) | func TestSplitIndexFinder_whereFilter(t *testing.T) {
FILE: finder/tag.go
type TagState (line 16) | type TagState
constant TagRoot (line 19) | TagRoot TagState = iota
constant TagSkip (line 20) | TagSkip
constant TagInfoRoot (line 21) | TagInfoRoot
constant TagList (line 22) | TagList
constant TagListSeriesRoot (line 23) | TagListSeriesRoot
constant TagListSeries (line 24) | TagListSeries
constant TagListParam (line 25) | TagListParam
type TagQ (line 28) | type TagQ struct
method String (line 33) | func (q TagQ) String() string {
method Where (line 49) | func (q *TagQ) Where(field string) string {
type TagFinder (line 65) | type TagFinder struct
method tagListSQL (line 92) | func (t *TagFinder) tagListSQL() (string, error) {
method seriesSQL (line 123) | func (t *TagFinder) seriesSQL() (string, error) {
method MakeSQL (line 149) | func (t *TagFinder) MakeSQL(query string) (string, error) {
method Execute (line 214) | func (t *TagFinder) Execute(ctx context.Context, config *config.Config...
method List (line 249) | func (t *TagFinder) List() [][]byte {
method Series (line 305) | func (t *TagFinder) Series() [][]byte {
method Abs (line 338) | func (t *TagFinder) Abs(v []byte) []byte {
method Bytes (line 346) | func (t *TagFinder) Bytes() ([]byte, error) {
method Stats (line 350) | func (t *TagFinder) Stats() []metrics.FinderStat {
function WrapTag (line 81) | func WrapTag(f Finder, url string, table string, opts clickhouse.Options...
FILE: finder/tag_test.go
function TestTagsMakeSQL (line 16) | func TestTagsMakeSQL(t *testing.T) {
function _TestTags (line 59) | func _TestTags(t *testing.T) {
FILE: finder/tagged.go
type TaggedTermOp (line 28) | type TaggedTermOp
constant TaggedTermEq (line 31) | TaggedTermEq TaggedTermOp = 1
constant TaggedTermMatch (line 32) | TaggedTermMatch TaggedTermOp = 2
constant TaggedTermNe (line 33) | TaggedTermNe TaggedTermOp = 3
constant TaggedTermNotMatch (line 34) | TaggedTermNotMatch TaggedTermOp = 4
type TaggedTerm (line 37) | type TaggedTerm struct
method concat (line 121) | func (term *TaggedTerm) concat() string {
method concatMask (line 125) | func (term *TaggedTerm) concatMask() string {
type TaggedTermList (line 49) | type TaggedTermList
method Len (line 51) | func (s TaggedTermList) Len() int {
method Swap (line 54) | func (s TaggedTermList) Swap(i, j int) {
method Less (line 57) | func (s TaggedTermList) Less(i, j int) bool {
type TaggedFinder (line 78) | type TaggedFinder struct
method Execute (line 473) | func (t *TaggedFinder) Execute(ctx context.Context, config *config.Con...
method whereFilter (line 482) | func (t *TaggedFinder) whereFilter(terms []TaggedTerm, from int64, unt...
method ExecutePrepared (line 504) | func (t *TaggedFinder) ExecutePrepared(ctx context.Context, terms []Ta...
method List (line 522) | func (t *TaggedFinder) List() [][]byte {
method Series (line 547) | func (t *TaggedFinder) Series() [][]byte {
method Abs (line 598) | func (t *TaggedFinder) Abs(v []byte) []byte {
method Bytes (line 606) | func (t *TaggedFinder) Bytes() ([]byte, error) {
method Stats (line 610) | func (t *TaggedFinder) Stats() []metrics.FinderStat {
method PrepareTaggedTerms (line 614) | func (t *TaggedFinder) PrepareTaggedTerms(ctx context.Context, cfg *co...
function NewTagged (line 93) | func NewTagged(url string, table, tag1CountTable string, dailyEnabled, u...
function TaggedTermWhere1 (line 130) | func TaggedTermWhere1(term *TaggedTerm, useCarbonBehaviour, dontMatchMis...
function TaggedTermWhereN (line 205) | func TaggedTermWhereN(term *TaggedTerm, useCarbonBehaviour, dontMatchMis...
function setCost (line 284) | func setCost(term *TaggedTerm, costs *config.Costs) {
function ParseTaggedConditions (line 302) | func ParseTaggedConditions(conditions []string, config *config.Config, a...
function parseString (line 368) | func parseString(s string) (string, string, error) {
function seriesByTagArgs (line 389) | func seriesByTagArgs(query string) ([]string, error) {
function ParseSeriesByTag (line 427) | func ParseSeriesByTag(query string, config *config.Config) ([]TaggedTerm...
function TaggedWhere (line 440) | func TaggedWhere(terms []TaggedTerm, useCarbonBehaviour, dontMatchMissin...
function NewCachedTags (line 467) | func NewCachedTags(body []byte) *TaggedFinder {
function tagsParse (line 551) | func tagsParse(path string) (string, []string, error) {
function TaggedDecode (line 565) | func TaggedDecode(v []byte) []byte {
function SortTaggedTermsByCost (line 639) | func SortTaggedTermsByCost(terms []TaggedTerm) {
function SetCosts (line 674) | func SetCosts(terms []TaggedTerm, costs map[string]*config.Costs) {
FILE: finder/tagged_test.go
function TestTaggedWhere (line 21) | func TestTaggedWhere(t *testing.T) {
function TestTaggedWhere_UseCarbonBehaviourFlag (line 122) | func TestTaggedWhere_UseCarbonBehaviourFlag(t *testing.T) {
function TestTaggedWhere_DontMatchMissingTagsFlag (line 223) | func TestTaggedWhere_DontMatchMissingTagsFlag(t *testing.T) {
function TestTaggedWhere_BothFeatureFlags (line 326) | func TestTaggedWhere_BothFeatureFlags(t *testing.T) {
function TestParseSeriesByTag (line 429) | func TestParseSeriesByTag(t *testing.T) {
function newInt (line 480) | func newInt(i int) *int {
function TestParseSeriesByTagWithCosts (line 487) | func TestParseSeriesByTagWithCosts(t *testing.T) {
function BenchmarkParseSeriesByTag (line 594) | func BenchmarkParseSeriesByTag(b *testing.B) {
function TestParseSeriesByTagWithCostsFromCountTable (line 609) | func TestParseSeriesByTagWithCostsFromCountTable(t *testing.T) {
function TestTaggedFinder_whereFilter (line 929) | func TestTaggedFinder_whereFilter(t *testing.T) {
function TestTaggedFinder_Abs (line 1014) | func TestTaggedFinder_Abs(t *testing.T) {
FILE: finder/tags_count_querier.go
type TagCountQuerier (line 18) | type TagCountQuerier struct
method GetCostsFromCountTable (line 40) | func (tcq *TagCountQuerier) GetCostsFromCountTable(ctx context.Context...
method List (line 156) | func (t *TagCountQuerier) List() [][]byte {
method Stats (line 181) | func (tcq *TagCountQuerier) Stats() []metrics.FinderStat {
function NewTagCountQuerier (line 29) | func NewTagCountQuerier(url, table string, opts clickhouse.Options, useC...
function chResultToCosts (line 113) | func chResultToCosts(body [][]byte) (map[string]*config.Costs, error) {
function parseTag1CountRow (line 134) | func parseTag1CountRow(s string) (string, string, int, error) {
FILE: finder/unescape.go
function ishex (line 5) | func ishex(c byte) bool {
function unhex (line 18) | func unhex(c byte) byte {
function isPercentEscape (line 31) | func isPercentEscape(s string, i int) bool {
function unescape (line 36) | func unescape(s string) string {
FILE: graphite-clickhouse.go
constant Version (line 42) | Version = "0.14.0"
function init (line 44) | func init() {
type LogResponseWriter (line 48) | type LogResponseWriter struct
method WriteHeader (line 54) | func (w *LogResponseWriter) WriteHeader(status int) {
method Status (line 59) | func (w *LogResponseWriter) Status() int {
function WrapResponseWriter (line 67) | func WrapResponseWriter(w http.ResponseWriter) *LogResponseWriter {
type App (line 75) | type App struct
method Handler (line 79) | func (app *App) Handler(handler http.Handler) http.Handler {
function sdList (line 96) | func sdList(name string, args []string) {
function sdDelete (line 139) | func sdDelete(name string, args []string) {
function sdEvict (line 181) | func sdEvict(name string, args []string) {
function sdExpired (line 220) | func sdExpired(name string, args []string) {
function sdClean (line 259) | func sdClean(name string, args []string) {
function printMatchedRollupRules (line 298) | func printMatchedRollupRules(metric string, age uint32, rollupRules *rol...
function checkRollupMatch (line 326) | func checkRollupMatch(name string, args []string) {
function main (line 415) | func main() {
FILE: healthcheck/healthcheck.go
type Handler (line 19) | type Handler struct
method ServeHTTP (line 35) | func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
function NewHandler (line 26) | func NewHandler(config *config.Config) *Handler {
FILE: helper/RowBinary/encode.go
constant NullUint32 (line 10) | NullUint32 = ^uint32(0)
function DateToUint16 (line 12) | func DateToUint16(t time.Time) uint16 {
type Encoder (line 16) | type Encoder struct
method Date (line 28) | func (w *Encoder) Date(value time.Time) error {
method Uint8 (line 32) | func (w *Encoder) Uint8(value uint8) error {
method Uint16 (line 37) | func (w *Encoder) Uint16(value uint16) error {
method Uint32 (line 44) | func (w *Encoder) Uint32(value uint32) error {
method NullableUint32 (line 51) | func (w *Encoder) NullableUint32(value uint32) error {
method Uint64 (line 65) | func (w *Encoder) Uint64(value uint64) error {
method Float64 (line 72) | func (w *Encoder) Float64(value float64) error {
method NullableFloat64 (line 76) | func (w *Encoder) NullableFloat64(value float64) error {
method Bytes (line 90) | func (w *Encoder) Bytes(value []byte) error {
method String (line 103) | func (w *Encoder) String(value string) error {
method StringList (line 107) | func (w *Encoder) StringList(value []string) error {
method Uint32List (line 125) | func (w *Encoder) Uint32List(value []uint32) error {
method NullableUint32List (line 143) | func (w *Encoder) NullableUint32List(value []uint32) error {
method Float64List (line 161) | func (w *Encoder) Float64List(value []float64) error {
method NullableFloat64List (line 179) | func (w *Encoder) NullableFloat64List(value []float64) error {
function NewEncoder (line 21) | func NewEncoder(w io.Writer) *Encoder {
FILE: helper/clickhouse/clickhouse.go
type ErrWithDescr (line 30) | type ErrWithDescr struct
method Error (line 52) | func (e *ErrWithDescr) Error() string {
method PrependDescription (line 56) | func (e *ErrWithDescr) PrependDescription(test string) {
type ContentEncoding (line 35) | type ContentEncoding
constant ContentEncodingNone (line 38) | ContentEncodingNone ContentEncoding = "none"
constant ContentEncodingGzip (line 39) | ContentEncodingGzip ContentEncoding = "gzip"
constant ContentEncodingZstd (line 40) | ContentEncodingZstd ContentEncoding = "zstd"
constant ClickHouseProgressHeader (line 44) | ClickHouseProgressHeader string = "X-Clickhouse-Progress"
constant ClickHouseSummaryHeader (line 45) | ClickHouseSummaryHeader string = "X-Clickhouse-Summary"
function NewErrWithDescr (line 48) | func NewErrWithDescr(err string, data string) error {
function extractClickhouseError (line 65) | func extractClickhouseError(e string) (int, string) {
function HandleError (line 94) | func HandleError(w http.ResponseWriter, err error) (status int, queueFai...
type Options (line 167) | type Options struct
type LoggedReader (line 175) | type LoggedReader struct
method Read (line 185) | func (r *LoggedReader) Read(p []byte) (int, error) {
method Close (line 195) | func (r *LoggedReader) Close() error {
method ChReadRows (line 206) | func (r *LoggedReader) ChReadRows() int64 {
method ChReadBytes (line 210) | func (r *LoggedReader) ChReadBytes() int64 {
type queryStats (line 214) | type queryStats struct
function formatSQL (line 221) | func formatSQL(q string) string {
function Query (line 230) | func Query(ctx context.Context, dsn string, query string, opts Options, ...
function Post (line 234) | func Post(ctx context.Context, dsn string, query string, postBody io.Rea...
function PostGzip (line 239) | func PostGzip(ctx context.Context, dsn string, query string, postBody io...
function PostWithEncoding (line 243) | func PostWithEncoding(ctx context.Context, dsn string, query string, pos...
function Reader (line 247) | func Reader(ctx context.Context, dsn string, query string, opts Options,...
function reader (line 251) | func reader(ctx context.Context, dsn string, query string, postBody io.R...
function getQueryStats (line 408) | func getQueryStats(resp *http.Response, statsHeaderName string) (querySt...
function sendRequestViaDefaultClient (line 470) | func sendRequestViaDefaultClient(request *http.Request, opts *Options) (...
function sendRequestWithProgressCheck (line 485) | func sendRequestWithProgressCheck(request *http.Request, opts *Options) ...
function do (line 497) | func do(ctx context.Context, dsn string, query string, postBody io.Reade...
function ReadUvarint (line 513) | func ReadUvarint(array []byte) (uint64, int, error) {
FILE: helper/clickhouse/clickhouse_test.go
function Test_extractClickhouseError (line 10) | func Test_extractClickhouseError(t *testing.T) {
FILE: helper/clickhouse/external-data.go
type ExternalTable (line 18) | type ExternalTable struct
type Column (line 28) | type Column struct
method String (line 34) | func (c *Column) String() string {
type ExternalData (line 40) | type ExternalData struct
method SetDebug (line 57) | func (e *ExternalData) SetDebug(debugDir string, perm os.FileMode) {
method buildBody (line 66) | func (e *ExternalData) buildBody(ctx context.Context, u *url.URL) (*by...
method debugDump (line 111) | func (e *ExternalData) debugDump(ctx context.Context, u url.URL) {
type extDataDebug (line 45) | type extDataDebug struct
function NewExternalData (line 51) | func NewExternalData(tables ...ExternalTable) *ExternalData {
FILE: helper/clickhouse/external-data_test.go
function getTestCases (line 17) | func getTestCases() (tables []ExternalTable) {
function TestColumnString (line 54) | func TestColumnString(t *testing.T) {
function TestNewExternalData (line 63) | func TestNewExternalData(t *testing.T) {
function TestBuildBody (line 71) | func TestBuildBody(t *testing.T) {
function TestDebugDump (line 102) | func TestDebugDump(t *testing.T) {
FILE: helper/client/datetime.go
function MetricsTimestampTruncate (line 9) | func MetricsTimestampTruncate(metrics []Metric, precision time.Duration) {
FILE: helper/client/errros.go
type HttpError (line 5) | type HttpError struct
method Error (line 17) | func (e *HttpError) Error() string {
function NewHttpError (line 10) | func NewHttpError(statusCode int, message string) *HttpError {
FILE: helper/client/find.go
type FindMatch (line 16) | type FindMatch struct
function MetricsFind (line 23) | func MetricsFind(client *http.Client, address string, format FormatType,...
FILE: helper/client/render.go
type Metric (line 25) | type Metric struct
function Render (line 42) | func Render(client *http.Client, address string, format FormatType, targ...
function Decode (line 151) | func Decode(b []byte, format FormatType) ([]Metric, error) {
type jsonResponse (line 277) | type jsonResponse struct
type jsonMetric (line 281) | type jsonMetric struct
FILE: helper/client/requests.go
type MultiGlobRequestV3 (line 5) | type MultiGlobRequestV3 struct
method Marshal (line 9) | func (r *MultiGlobRequestV3) Marshal() ([]byte, error) {
method LogInfo (line 13) | func (r *MultiGlobRequestV3) LogInfo() interface{} {
FILE: helper/client/tags.go
function TagsNames (line 18) | func TagsNames(client *http.Client, address string, format FormatType, q...
function TagsValues (line 152) | func TagsValues(client *http.Client, address string, format FormatType, ...
FILE: helper/client/types.go
type FormatType (line 8) | type FormatType
method String (line 21) | func (a *FormatType) String() string {
method Set (line 29) | func (a *FormatType) Set(value string) error {
method UnmarshalText (line 48) | func (a *FormatType) UnmarshalText(text []byte) error {
constant FormatDefault (line 11) | FormatDefault FormatType = iota
constant FormatJSON (line 12) | FormatJSON
constant FormatProtobuf (line 13) | FormatProtobuf
constant FormatPb_v2 (line 14) | FormatPb_v2
constant FormatPb_v3 (line 15) | FormatPb_v3
constant FormatPickle (line 16) | FormatPickle
function FormatTypes (line 25) | func FormatTypes() []string {
FILE: helper/date/date.go
function SetDefault (line 11) | func SetDefault() {
function SetUTC (line 19) | func SetUTC() {
function SetBoth (line 27) | func SetBoth() {
function init (line 34) | func init() {
function DefaultTimestampToDaysFormat (line 39) | func DefaultTimestampToDaysFormat(ts int64) string {
function DefaultTimeToDaysFormat (line 45) | func DefaultTimeToDaysFormat(t time.Time) string {
function UTCTimestampToDaysFormat (line 49) | func UTCTimestampToDaysFormat(timestamp int64) string {
function UTCTimeToDaysFormat (line 53) | func UTCTimeToDaysFormat(t time.Time) string {
function defaultDate (line 57) | func defaultDate(t time.Time) time.Time {
function minLocalAndUTC (line 61) | func minLocalAndUTC(t time.Time) time.Time {
function MinTimestampToDaysFormat (line 73) | func MinTimestampToDaysFormat(ts int64) string {
function MinTimeToDaysFormat (line 79) | func MinTimeToDaysFormat(t time.Time) string {
function maxLocalAndUTC (line 84) | func maxLocalAndUTC(t time.Time) time.Time {
function MaxTimestampToDaysFormat (line 96) | func MaxTimestampToDaysFormat(ts int64) string {
function MaxTimeToDaysFormat (line 102) | func MaxTimeToDaysFormat(t time.Time) string {
FILE: helper/date/date_test.go
function isVerbose (line 12) | func isVerbose() bool {
function init (line 22) | func init() {
function TestDefaultTimestampToDaysFormat (line 55) | func TestDefaultTimestampToDaysFormat(t *testing.T) {
function TestDefaultTimeToDaysFormat (line 101) | func TestDefaultTimeToDaysFormat(t *testing.T) {
function TestUTCTimestampToDaysFormat (line 139) | func TestUTCTimestampToDaysFormat(t *testing.T) {
function TestUTCTimeToDaysFormat (line 177) | func TestUTCTimeToDaysFormat(t *testing.T) {
function TestMinMaxTimestampToDaysFormat (line 215) | func TestMinMaxTimestampToDaysFormat(t *testing.T) {
FILE: helper/datetime/datetime.go
function parseTime (line 15) | func parseTime(s string) (hour, minute int, err error) {
function DateParamToEpoch (line 47) | func DateParamToEpoch(s string, tz *time.Location, now time.Time, trunca...
function Timezone (line 195) | func Timezone(qtz string) (*time.Location, error) {
function TimestampTruncate (line 203) | func TimestampTruncate(ts int64, truncate time.Duration) int64 {
function TimeTruncate (line 213) | func TimeTruncate(tm time.Time, truncate time.Duration) time.Time {
FILE: helper/datetime/datetime_test.go
function TestDateParamToEpoch (line 8) | func TestDateParamToEpoch(t *testing.T) {
function TestDateParamToEpochTruncate (line 62) | func TestDateParamToEpochTruncate(t *testing.T) {
FILE: helper/errs/errors.go
type ErrorWithCode (line 5) | type ErrorWithCode struct
method Error (line 18) | func (e ErrorWithCode) Error() string { return e.err }
function NewErrorWithCode (line 10) | func NewErrorWithCode(err string, code int) error {
function NewErrorfWithCode (line 14) | func NewErrorfWithCode(code int, f string, args ...interface{}) error {
FILE: helper/headers/headers.go
function GetHeaders (line 5) | func GetHeaders(header *http.Header, keys []string) map[string]string {
FILE: helper/http/live-http-client.go
constant TCPNetwork (line 13) | TCPNetwork string = "tcp"
function DoHTTPOverTCP (line 15) | func DoHTTPOverTCP(ctx context.Context, transport *http.Transport, req *...
FILE: helper/pickle/pickle.go
type Writer (line 12) | type Writer struct
method Mark (line 20) | func (p *Writer) Mark() {
method Stop (line 24) | func (p *Writer) Stop() {
method Append (line 28) | func (p *Writer) Append() {
method SetItem (line 32) | func (p *Writer) SetItem() {
method List (line 36) | func (p *Writer) List() {
method Dict (line 40) | func (p *Writer) Dict() {
method TupleEnd (line 44) | func (p *Writer) TupleEnd() {
method Bytes (line 48) | func (p *Writer) Bytes(byt []byte) {
method String (line 63) | func (p *Writer) String(v string) {
method Uint32 (line 67) | func (p *Writer) Uint32(v uint32) {
method AppendFloat64 (line 76) | func (p *Writer) AppendFloat64(v float64) {
method AppendNulls (line 88) | func (p *Writer) AppendNulls(count int) {
method Bool (line 94) | func (p *Writer) Bool(b bool) {
function NewWriter (line 16) | func NewWriter(w io.Writer) *Writer {
FILE: helper/point/func.go
function CleanUp (line 10) | func CleanUp(points []Point) []Point {
function Uniq (line 29) | func Uniq(points []Point) []Point {
function FillNulls (line 55) | func FillNulls(points []Point, from, until, step uint32) (start, stop, c...
FILE: helper/point/func_test.go
function TestUniq (line 14) | func TestUniq(t *testing.T) {
function TestCleanUp (line 56) | func TestCleanUp(t *testing.T) {
function TestFillNulls (line 106) | func TestFillNulls(t *testing.T) {
FILE: helper/point/point.go
type Point (line 5) | type Point struct
type GetValueOrNaN (line 13) | type GetValueOrNaN
FILE: helper/point/points.go
type Points (line 9) | type Points struct
method AppendPoint (line 31) | func (pp *Points) AppendPoint(metricID uint32, value float64, time uin...
method MetricID (line 41) | func (pp *Points) MetricID(metricName string) uint32 {
method MetricIDBytes (line 53) | func (pp *Points) MetricIDBytes(metricNameBytes []byte) uint32 {
method MetricName (line 59) | func (pp *Points) MetricName(metricID uint32) string {
method List (line 69) | func (pp *Points) List() []Point {
method ReplaceList (line 74) | func (pp *Points) ReplaceList(list []Point) {
method GetStep (line 79) | func (pp *Points) GetStep(id uint32) (uint32, error) {
method SetSteps (line 89) | func (pp *Points) SetSteps(steps map[uint32][]string) {
method GetAggregation (line 106) | func (pp *Points) GetAggregation(id uint32) (string, error) {
method SetAggregations (line 116) | func (pp *Points) SetAggregations(functions map[string][]string) {
method Len (line 133) | func (pp *Points) Len() int {
method Less (line 137) | func (pp *Points) Less(i, j int) bool {
method Swap (line 145) | func (pp *Points) Swap(i, j int) {
method Sort (line 150) | func (pp *Points) Sort() {
method Uniq (line 155) | func (pp *Points) Uniq() {
method GroupByMetric (line 161) | func (pp *Points) GroupByMetric() NextMetric {
type NextMetric (line 19) | type NextMetric
function NewPoints (line 22) | func NewPoints() *Points {
FILE: helper/rollup/aggr.go
type Aggr (line 16) | type Aggr struct
method Name (line 21) | func (ag *Aggr) Name() string {
method String (line 29) | func (ag *Aggr) String() string {
method Do (line 37) | func (ag *Aggr) Do(points []point.Point) (r float64) {
function AggrSum (line 45) | func AggrSum(points []point.Point) (r float64) {
function AggrMax (line 53) | func AggrMax(points []point.Point) (r float64) {
function AggrMin (line 67) | func AggrMin(points []point.Point) (r float64) {
function AggrAvg (line 81) | func AggrAvg(points []point.Point) (r float64) {
function AggrAny (line 91) | func AggrAny(points []point.Point) (r float64) {
function AggrAnyLast (line 99) | func AggrAnyLast(points []point.Point) (r float64) {
FILE: helper/rollup/compact.go
function parseCompact (line 15) | func parseCompact(body string) (*Rules, error) {
FILE: helper/rollup/compact_test.go
function TestParseCompact (line 9) | func TestParseCompact(t *testing.T) {
FILE: helper/rollup/remote.go
type rollupRulesResponseRecord (line 17) | type rollupRulesResponseRecord struct
type rollupRulesResponse (line 25) | type rollupRulesResponse struct
function parseJson (line 29) | func parseJson(body []byte) (*Rules, error) {
function RemoteLoad (line 115) | func RemoteLoad(addr string, tlsConf *tls.Config, table string) (*Rules,...
FILE: helper/rollup/remote_test.go
function assertJsonEqual (line 11) | func assertJsonEqual(t *testing.T, expected string, actual string) {
function TestParseJson (line 21) | func TestParseJson(t *testing.T) {
function TestParseJsonTyped (line 129) | func TestParseJsonTyped(t *testing.T) {
FILE: helper/rollup/rollup.go
type Rollup (line 15) | type Rollup struct
method Rules (line 79) | func (r *Rollup) Rules() *Rules {
method update (line 87) | func (r *Rollup) update() error {
method updateWorker (line 107) | func (r *Rollup) updateWorker() {
method MarshalJSON (line 122) | func (r *Rollup) MarshalJSON() ([]byte, error) {
function NewAuto (line 26) | func NewAuto(addr string, tlsConfig *tls.Config, table string, interval ...
function NewXMLFile (line 41) | func NewXMLFile(filename string, defaultPrecision uint32, defaultFunctio...
function NewDefault (line 64) | func NewDefault(defaultPrecision uint32, defaultFunction string) (*Rollu...
FILE: helper/rollup/rules.go
type Retention (line 16) | type Retention struct
type RuleType (line 21) | type RuleType
method String (line 34) | func (r *RuleType) String() string {
method Set (line 38) | func (r *RuleType) Set(value string) error {
method UnmarshalJSON (line 55) | func (r *RuleType) UnmarshalJSON(data []byte) error {
method UnmarshalXML (line 64) | func (r *RuleType) UnmarshalXML(d *xml.Decoder, start xml.StartElement...
constant RuleAll (line 24) | RuleAll RuleType = iota
constant RulePlain (line 25) | RulePlain
constant RuleTagged (line 26) | RuleTagged
constant RuleTagList (line 27) | RuleTagList
function splitTags (line 73) | func splitTags(tagsStr string) (tags []string) {
function buildTaggedRegex (line 86) | func buildTaggedRegex(regexpStr string) string {
type Pattern (line 137) | type Pattern struct
method compile (line 174) | func (p *Pattern) compile() error {
type Rules (line 146) | type Rules struct
method compile (line 211) | func (r *Rules) compile() (*Rules, error) {
method prepare (line 247) | func (r *Rules) prepare(defaultPrecision uint32, defaultFunction strin...
method withDefault (line 256) | func (r *Rules) withDefault(defaultPrecision uint32, defaultFunction *...
method setUpdated (line 275) | func (r *Rules) setUpdated() *Rules {
method withSuperDefault (line 280) | func (r *Rules) withSuperDefault() *Rules {
method Lookup (line 285) | func (r *Rules) Lookup(metric string, age uint32, verbose bool) (preci...
method LookupBytes (line 366) | func (r *Rules) LookupBytes(metric []byte, age uint32, verbose bool) (...
method RollupMetricAge (line 411) | func (r *Rules) RollupMetricAge(metricName string, age uint32, points ...
method RollupMetric (line 425) | func (r *Rules) RollupMetric(metricName string, from uint32, points []...
method RollupPoints (line 438) | func (r *Rules) RollupPoints(pp *point.Points, from int64, step int64)...
function NewMockRules (line 155) | func NewMockRules(pattern []Pattern, defaultPrecision uint32, defaultFun...
constant superDefaultPrecision (line 172) | superDefaultPrecision = uint32(60)
function lookup (line 298) | func lookup(metric string, age uint32, patterns []Pattern, verbose bool)...
function doMetricPrecision (line 370) | func doMetricPrecision(points []point.Point, precision uint32, aggr *Agg...
FILE: helper/rollup/rules_test.go
function TestMetricPrecision (line 15) | func TestMetricPrecision(t *testing.T) {
function Test_buildTaggedRegex (line 36) | func Test_buildTaggedRegex(t *testing.T) {
function TestLookup (line 95) | func TestLookup(t *testing.T) {
function TestLookupTyped (line 140) | func TestLookupTyped(t *testing.T) {
function TestRules_RollupPoints (line 283) | func TestRules_RollupPoints(t *testing.T) {
function BenchmarkLookupSum (line 705) | func BenchmarkLookupSum(b *testing.B) {
function BenchmarkLookupSumSeparated (line 718) | func BenchmarkLookupSumSeparated(b *testing.B) {
function BenchmarkLookupSumTagged (line 731) | func BenchmarkLookupSumTagged(b *testing.B) {
function BenchmarkLookupSumTaggedSeparated (line 744) | func BenchmarkLookupSumTaggedSeparated(b *testing.B) {
function BenchmarkLookupMax (line 757) | func BenchmarkLookupMax(b *testing.B) {
function BenchmarkLookupMaxSeparated (line 770) | func BenchmarkLookupMaxSeparated(b *testing.B) {
function BenchmarkLookupMaxTagged (line 783) | func BenchmarkLookupMaxTagged(b *testing.B) {
function BenchmarkLookupMaxTaggedSeparated (line 796) | func BenchmarkLookupMaxTaggedSeparated(b *testing.B) {
function BenchmarkLookupDefault (line 809) | func BenchmarkLookupDefault(b *testing.B) {
function BenchmarkLookupDefaultSeparated (line 822) | func BenchmarkLookupDefaultSeparated(b *testing.B) {
function BenchmarkLookupDefaultTagged (line 835) | func BenchmarkLookupDefaultTagged(b *testing.B) {
function BenchmarkLookupDefaultTaggedSeparated (line 848) | func BenchmarkLookupDefaultTaggedSeparated(b *testing.B) {
FILE: helper/rollup/xml.go
type ClickhouseRollupXML (line 39) | type ClickhouseRollupXML struct
type RetentionXML (line 43) | type RetentionXML struct
method retention (line 60) | func (r *RetentionXML) retention() Retention {
type PatternXML (line 48) | type PatternXML struct
method pattern (line 64) | func (p *PatternXML) pattern() Pattern {
type RulesXML (line 55) | type RulesXML struct
function parseXML (line 79) | func parseXML(body []byte) (*Rules, error) {
FILE: helper/rollup/xml_test.go
function TestParseXML (line 12) | func TestParseXML(t *testing.T) {
function TestParseXMLTyped (line 115) | func TestParseXMLTyped(t *testing.T) {
FILE: helper/tests/clickhouse/server.go
type TestResponse (line 11) | type TestResponse struct
type TestHandler (line 17) | type TestHandler struct
method ServeHTTP (line 28) | func (h *TestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
type TestServer (line 23) | type TestServer struct
method AddResponce (line 62) | func (s *TestServer) AddResponce(request string, response *TestRespons...
method Queries (line 68) | func (s *TestServer) Queries() uint64 {
function NewTestServer (line 54) | func NewTestServer() *TestServer {
FILE: helper/tests/compare/compare.go
constant eps (line 5) | eps = 0.0000000001
function NearlyEqualSlice (line 7) | func NearlyEqualSlice(a, b []float64) bool {
function NearlyEqual (line 31) | func NearlyEqual(a, b float64) bool {
function Max (line 48) | func Max(a, b int) int {
FILE: helper/tests/compare/expand/expand.go
function ExpandTimestamp (line 10) | func ExpandTimestamp(fs *token.FileSet, s string, replace map[string]str...
FILE: helper/utils/utils.go
function TimestampTruncate (line 6) | func TimestampTruncate(ts int64, duration time.Duration) int64 {
FILE: helper/utils/utils_test.go
function TestTimestampTruncate (line 9) | func TestTimestampTruncate(t *testing.T) {
FILE: index/handler.go
type Handler (line 12) | type Handler struct
method ServeHTTP (line 22) | func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
function NewHandler (line 16) | func NewHandler(config *config.Config) *Handler {
FILE: index/index.go
type Index (line 18) | type Index struct
method Close (line 67) | func (i *Index) Close() error {
method WriteJSON (line 71) | func (i *Index) WriteJSON(w http.ResponseWriter) error {
function New (line 23) | func New(config *config.Config, ctx context.Context) (*Index, error) {
FILE: index/index_test.go
function TestWriteJSONEmptyRows (line 12) | func TestWriteJSONEmptyRows(t *testing.T) {
function TestWriteJSONNonleafRows (line 35) | func TestWriteJSONNonleafRows(t *testing.T) {
function TestWriteJSONEmptyIndex (line 57) | func TestWriteJSONEmptyIndex(t *testing.T) {
function indexForBytes (line 70) | func indexForBytes(b []byte) *Index {
function writeRows (line 79) | func writeRows(rows []string) ([]string, error) {
FILE: limiter/alimiter.go
function getWeighted (line 17) | func getWeighted(n, max int) int {
type ALimiter (line 40) | type ALimiter struct
method balance (line 74) | func (sl *ALimiter) balance() int {
method Capacity (line 104) | func (sl *ALimiter) Capacity() int {
method Enter (line 108) | func (sl *ALimiter) Enter(ctx context.Context, s string) (err error) {
method TryEnter (line 134) | func (sl *ALimiter) TryEnter(ctx context.Context, s string) (err error) {
method Leave (line 160) | func (sl *ALimiter) Leave(ctx context.Context, s string) {
method SendDuration (line 169) | func (sl *ALimiter) SendDuration(queueMs int64) {
method Unregiter (line 176) | func (sl *ALimiter) Unregiter() {
method Enabled (line 181) | func (sl *ALimiter) Enabled() bool {
function NewALimiter (line 50) | func NewALimiter(capacity, concurrent, n int, enableMetrics bool, scope,...
FILE: limiter/alimiter_test.go
function Test_getWeighted (line 15) | func Test_getWeighted(t *testing.T) {
function TestNewALimiter (line 50) | func TestNewALimiter(t *testing.T) {
type testLimiter (line 125) | type testLimiter struct
function Benchmark_Limiter_Parallel (line 132) | func Benchmark_Limiter_Parallel(b *testing.B) {
FILE: limiter/interface.go
type ServerLimiter (line 13) | type ServerLimiter interface
FILE: limiter/limiter.go
type limiter (line 9) | type limiter struct
method capacity (line 83) | func (sl *limiter) capacity() int {
method enter (line 88) | func (sl *limiter) enter(ctx context.Context, s string) error {
method tryEnter (line 98) | func (sl *limiter) tryEnter(ctx context.Context, s string) error {
method leave (line 108) | func (sl *limiter) leave(ctx context.Context, s string) {
type Limiter (line 15) | type Limiter struct
method Capacity (line 35) | func (sl *Limiter) Capacity() int {
method Enter (line 40) | func (sl *Limiter) Enter(ctx context.Context, s string) (err error) {
method TryEnter (line 51) | func (sl *Limiter) TryEnter(ctx context.Context, s string) (err error) {
method Leave (line 62) | func (sl *Limiter) Leave(ctx context.Context, s string) {
method SendDuration (line 67) | func (sl *Limiter) SendDuration(queueMs int64) {
method Unregiter (line 74) | func (sl *Limiter) Unregiter() {
method Enabled (line 79) | func (sl *Limiter) Enabled() bool {
function NewLimiter (line 21) | func NewLimiter(capacity int, enableMetrics bool, scope, sub string) Ser...
FILE: limiter/noop.go
type NoopLimiter (line 8) | type NoopLimiter struct
method Capacity (line 11) | func (l NoopLimiter) Capacity() int {
method Enter (line 16) | func (l NoopLimiter) Enter(ctx context.Context, s string) error {
method TryEnter (line 21) | func (l NoopLimiter) TryEnter(ctx context.Context, s string) error {
method Leave (line 26) | func (l NoopLimiter) Leave(ctx context.Context, s string) {
method SendDuration (line 30) | func (l NoopLimiter) SendDuration(queueMs int64) {
method Unregiter (line 34) | func (l NoopLimiter) Unregiter() {
method Enabled (line 38) | func (l NoopLimiter) Enabled() bool {
FILE: limiter/wlimiter.go
type WLimiter (line 10) | type WLimiter struct
method Capacity (line 42) | func (sl *WLimiter) Capacity() int {
method Enter (line 46) | func (sl *WLimiter) Enter(ctx context.Context, s string) (err error) {
method TryEnter (line 72) | func (sl *WLimiter) TryEnter(ctx context.Context, s string) (err error) {
method Leave (line 98) | func (sl *WLimiter) Leave(ctx context.Context, s string) {
method SendDuration (line 107) | func (sl *WLimiter) SendDuration(queueMs int64) {
method Unregiter (line 114) | func (sl *WLimiter) Unregiter() {
method Enabled (line 119) | func (sl *WLimiter) Enabled() bool {
function NewWLimiter (line 17) | func NewWLimiter(capacity, concurrent int, enableMetrics bool, scope, su...
FILE: load_avg/load_avg.go
function Load (line 11) | func Load() float64 {
function Store (line 15) | func Store(f float64) {
function Weight (line 19) | func Weight(weight int, degraged, degragedLoadAvg, normalizedLoadAvg flo...
FILE: load_avg/load_avg_default.go
function Normalized (line 6) | func Normalized() (float64, error) {
function CpuCount (line 10) | func CpuCount() (uint64, error) {
FILE: load_avg/load_avg_linux.go
function Normalized (line 14) | func Normalized() (float64, error) {
function CpuCount (line 33) | func CpuCount() (uint64, error) {
FILE: load_avg/load_avg_test.go
function TestWeight (line 8) | func TestWeight(t *testing.T) {
FILE: logs/logger.go
function AccessLog (line 14) | func AccessLog(logger *zap.Logger, config *config.Config, r *http.Reques...
FILE: metrics/metrics.go
type Config (line 15) | type Config struct
type CacheMetric (line 33) | type CacheMetric struct
type ReqMetric (line 44) | type ReqMetric struct
type WaitMetric (line 60) | type WaitMetric struct
method Unregister (line 91) | func (w *WaitMetric) Unregister() {
function NewWaitMetric (line 68) | func NewWaitMetric(enable bool, scope, sub string) WaitMetric {
function NewDisabledWaitMetric (line 98) | func NewDisabledWaitMetric() *WaitMetric {
type FindMetrics (line 104) | type FindMetrics struct
type RenderMetric (line 111) | type RenderMetric struct
type RenderMetrics (line 116) | type RenderMetrics struct
function initFindCacheMetrics (line 127) | func initFindCacheMetrics(c *Config) {
function initFindMetrics (line 152) | func initFindMetrics(scope string, c *Config, waitQueue bool) *FindMetri...
function initRenderMetrics (line 242) | func initRenderMetrics(scope string, c *Config) *RenderMetrics {
function SendFindMetrics (line 339) | func SendFindMetrics(r *FindMetrics, statusCode int, durationMs, untilFr...
function SendRenderMetrics (line 450) | func SendRenderMetrics(r *RenderMetrics, statusCode int, start, fetch, e...
type rangeName (line 594) | type rangeName struct
function InitMetrics (line 599) | func InitMetrics(c *Config, findWaitQueue, tagsWaitQueue bool) {
function DisableMetrics (line 689) | func DisableMetrics() {
function UnregisterAll (line 695) | func UnregisterAll() {
FILE: metrics/metrics_test.go
function max (line 14) | func max(a, b int) int {
function compareInterface (line 22) | func compareInterface(t *testing.T, name string, i interface{}, notNil b...
function TestInitMetrics (line 31) | func TestInitMetrics(t *testing.T) {
FILE: metrics/query_metrics.go
type QueryMetric (line 5) | type QueryMetric struct
type QueryMetrics (line 14) | type QueryMetrics struct
type FinderStat (line 21) | type FinderStat struct
function InitQueryMetrics (line 34) | func InitQueryMetrics(table string, c *Config) *QueryMetrics {
function SendQueryRead (line 83) | func SendQueryRead(r *QueryMetrics, from, until, durationMs, read_rows, ...
function SendQueryReadChecked (line 116) | func SendQueryReadChecked(r *QueryMetrics, from, until, durationMs, read...
function SendQueryReadByTable (line 122) | func SendQueryReadByTable(from, until, durationMs, read_rows int64, stat...
FILE: metrics/statsd.go
type NullSender (line 10) | type NullSender struct
method Inc (line 12) | func (NullSender) Inc(string, int64, float32, ...statsd.Tag) error ...
method Dec (line 13) | func (NullSender) Dec(string, int64, float32, ...statsd.Tag) error ...
method Gauge (line 14) | func (NullSender) Gauge(string, int64, float32, ...statsd.Tag) error ...
method GaugeDelta (line 15) | func (NullSender) GaugeDelta(string, int64, float32, ...statsd.Tag) er...
method Timing (line 16) | func (NullSender) Timing(string, int64, float32, ...statsd.Tag) error ...
method TimingDuration (line 17) | func (NullSender) TimingDuration(string, time.Duration, float32, ...st...
method Set (line 18) | func (NullSender) Set(string, string, float32, ...statsd.Tag) error ...
method SetInt (line 19) | func (NullSender) SetInt(string, int64, float32, ...statsd.Tag) error ...
method Raw (line 20) | func (NullSender) Raw(string, string, float32, ...statsd.Tag) error ...
method NewSubStatter (line 21) | func (NullSender) NewSubStatter(string) statsd.SubStatter ...
method SetPrefix (line 22) | func (NullSender) SetPrefix(string) ...
method SetSamplerFunc (line 23) | func (NullSender) SetSamplerFunc(statsd.SamplerFunc) ...
method Close (line 24) | func (NullSender) Close() error ...
FILE: pkg/alias/map.go
type Value (line 12) | type Value struct
type Map (line 18) | type Map struct
method Merge (line 32) | func (m *Map) Merge(r finder.Result, useCache bool) {
method MergeTarget (line 37) | func (m *Map) MergeTarget(r finder.Result, target string, saveCache bo...
method Len (line 73) | func (m *Map) Len() int {
method Size (line 81) | func (m *Map) Size() int {
method Series (line 95) | func (m *Map) Series(isReverse bool) []string {
method DisplayNames (line 110) | func (m *Map) DisplayNames() []string {
method Get (line 123) | func (m *Map) Get(metric string) []Value {
function New (line 24) | func New() *Map {
FILE: pkg/alias/map_tagged_test.go
function createAMTagged (line 20) | func createAMTagged() *Map {
function TestCreationTagged (line 27) | func TestCreationTagged(t *testing.T) {
function TestAsyncMergeTagged (line 40) | func TestAsyncMergeTagged(t *testing.T) {
function Benchmark_MergeTargetTagged (line 99) | func Benchmark_MergeTargetTagged(b *testing.B) {
FILE: pkg/alias/map_test.go
type Values (line 12) | type Values
method Len (line 14) | func (v *Values) Len() int {
method Less (line 18) | func (v *Values) Less(i, j int) bool {
method Swap (line 22) | func (v *Values) Swap(i, j int) {
function createAM (line 36) | func createAM() *Map {
function TestCreation (line 43) | func TestCreation(t *testing.T) {
function TestAsyncMerge (line 55) | func TestAsyncMerge(t *testing.T) {
function TestLen (line 113) | func TestLen(t *testing.T) {
function TestSize (line 125) | func TestSize(t *testing.T) {
function TestDisplayNames (line 137) | func TestDisplayNames(t *testing.T) {
function TestGet (line 157) | func TestGet(t *testing.T) {
function Benchmark_MergeTargetFinder (line 162) | func Benchmark_MergeTargetFinder(b *testing.B) {
FILE: pkg/dry/math.go
function Max (line 4) | func Max(x, y int64) int64 {
function Min (line 13) | func Min(x, y int64) int64 {
function Ceil (line 23) | func Ceil(x, d int64) int64 {
function CeilToMultiplier (line 33) | func CeilToMultiplier(x, m int64) int64 {
function FloorToMultiplier (line 39) | func FloorToMultiplier(x, m int64) int64 {
function GCD (line 48) | func GCD(a, b int64) int64 {
function LCM (line 64) | func LCM(a, b int64) int64 {
FILE: pkg/dry/math_test.go
function TestMax (line 9) | func TestMax(t *testing.T) {
function TestMin (line 17) | func TestMin(t *testing.T) {
function TestCeil (line 25) | func TestCeil(t *testing.T) {
function TestCeilToMultiplier (line 34) | func TestCeilToMultiplier(t *testing.T) {
function TestFloorToMultiplier (line 45) | func TestFloorToMultiplier(t *testing.T) {
function TestGCD (line 56) | func TestGCD(t *testing.T) {
function TestLCM (line 67) | func TestLCM(t *testing.T) {
FILE: pkg/dry/strings.go
function RemoveEmptyStrings (line 4) | func RemoveEmptyStrings(stringList []string) []string {
FILE: pkg/dry/strings_test.go
function TestRemoveEmptyStrings (line 9) | func TestRemoveEmptyStrings(t *testing.T) {
FILE: pkg/dry/unsafe.go
function UnsafeString (line 9) | func UnsafeString(b []byte) string {
function UnsafeStringBytes (line 14) | func UnsafeStringBytes(s *string) []byte {
FILE: pkg/dry/unsafe_test.go
function TestUnsafeString (line 9) | func TestUnsafeString(t *testing.T) {
FILE: pkg/reverse/reverse.go
function String (line 8) | func String(path string) string {
function reverse (line 24) | func reverse(m []byte) {
function Inplace (line 35) | func Inplace(path []byte) {
function Bytes (line 55) | func Bytes(path []byte) []byte {
FILE: pkg/reverse/reverse_test.go
function TestReverse (line 9) | func TestReverse(t *testing.T) {
FILE: pkg/scope/context.go
type Context (line 10) | type Context struct
method With (line 20) | func (c *Context) With(key string, value interface{}) *Context {
method WithRequestID (line 25) | func (c *Context) WithRequestID(requestID string) *Context {
method WithLogger (line 30) | func (c *Context) WithLogger(logger *zap.Logger) *Context {
method WithTable (line 35) | func (c *Context) WithTable(table string) *Context {
function New (line 15) | func New(ctx context.Context) *Context {
FILE: pkg/scope/http_request.go
function HttpRequest (line 24) | func HttpRequest(r *http.Request) *http.Request {
function Grafana (line 61) | func Grafana(ctx context.Context) string {
FILE: pkg/scope/key.go
type scopeKey (line 9) | type scopeKey
function With (line 12) | func With(ctx context.Context, key string, value interface{}) context.Co...
function String (line 17) | func String(ctx context.Context, key string) string {
function Bool (line 26) | func Bool(ctx context.Context, key string) bool {
function WithRequestID (line 35) | func WithRequestID(ctx context.Context, requestID string) context.Context {
function RequestID (line 40) | func RequestID(ctx context.Context) string {
function WithTable (line 45) | func WithTable(ctx context.Context, table string) context.Context {
function Table (line 50) | func Table(ctx context.Context) string {
function WithDebug (line 55) | func WithDebug(ctx context.Context, name string) context.Context {
function Debug (line 60) | func Debug(ctx context.Context, name string) bool {
function ClickhouseUserAgent (line 65) | func ClickhouseUserAgent(ctx context.Context) string {
FILE: pkg/scope/logger.go
function Logger (line 18) | func Logger(ctx context.Context) *zap.Logger {
function LoggerWithHeaders (line 43) | func LoggerWithHeaders(ctx context.Context, r *http.Request, headersToLo...
function WithLogger (line 78) | func WithLogger(ctx context.Context, logger *zap.Logger) context.Context {
FILE: pkg/where/match.go
function ClearGlob (line 11) | func ClearGlob(query string) string {
function HasUnmatchedBrackets (line 102) | func HasUnmatchedBrackets(query string) bool {
function glob (line 136) | func glob(field string, query string, optionalDotAtEnd bool) string {
function Glob (line 183) | func Glob(field string, query string) string {
function TreeGlob (line 188) | func TreeGlob(field string, query string) string {
function ConcatMatchKV (line 192) | func ConcatMatchKV(key, value string) string {
function Match (line 205) | func Match(field string, key, value string) string {
FILE: pkg/where/match_test.go
function Test_ClearGlob (line 5) | func Test_ClearGlob(t *testing.T) {
function Test_HasUnmatchedBrackets (line 28) | func Test_HasUnmatchedBrackets(t *testing.T) {
function TestGlob (line 58) | func TestGlob(t *testing.T) {
FILE: pkg/where/where.go
function unsafeString (line 14) | func unsafeString(b []byte) string {
function GlobExpandSimple (line 19) | func GlobExpandSimple(value, prefix string, result *[]string) error {
function GlobToRegexp (line 61) | func GlobToRegexp(g string) string {
function HasWildcard (line 74) | func HasWildcard(target string) bool {
function IndexLastWildcard (line 78) | func IndexLastWildcard(target string) int {
function IndexWildcard (line 82) | func IndexWildcard(target string) int {
function MaxWildcardDistance (line 86) | func MaxWildcardDistance(query string) int {
function NonRegexpPrefix (line 99) | func NonRegexpPrefix(expr string) string {
function escape (line 117) | func escape(s string) string {
function escapeRegex (line 124) | func escapeRegex(s string) string {
function likeEscape (line 133) | func likeEscape(s string) string {
function quote (line 142) | func quote(value interface{}) string {
function quoteRegex (line 157) | func quoteRegex(key, value string) string {
function Like (line 166) | func Like(field, s string) string {
function Eq (line 170) | func Eq(field, value interface{}) string {
function HasPrefix (line 174) | func HasPrefix(field, prefix string) string {
function HasPrefixAndNotEq (line 178) | func HasPrefixAndNotEq(field, prefix string) string {
function HasPrefixBytes (line 182) | func HasPrefixBytes(field, prefix []byte) string {
function ArrayHas (line 186) | func ArrayHas(field, element string) string {
function In (line 190) | func In(field string, list []string) string {
function InTable (line 213) | func InTable(field string, table string) string {
function DateBetween (line 217) | func DateBetween(field string, from int64, until int64) string {
function TimestampBetween (line 224) | func TimestampBetween(field string, from int64, until int64) string {
type Where (line 228) | type Where struct
method And (line 236) | func (w *Where) And(exp string) {
method Or (line 248) | func (w *Where) Or(exp string) {
method Andf (line 260) | func (w *Where) Andf(format string, obj ...interface{}) {
method String (line 264) | func (w *Where) String() string {
method SQL (line 268) | func (w *Where) SQL() string {
method PreWhereSQL (line 276) | func (w *Where) PreWhereSQL() string {
function New (line 232) | func New() *Where {
FILE: pkg/where/where_test.go
function TestGlobExpandSimple (line 10) | func TestGlobExpandSimple(t *testing.T) {
function TestGlobToRegexp (line 41) | func TestGlobToRegexp(t *testing.T) {
function TestNonRegexpPrefix (line 59) | func TestNonRegexpPrefix(t *testing.T) {
function TestMaxWildcardDistance (line 78) | func TestMaxWildcardDistance(t *testing.T) {
FILE: prometheus/empty_iterator.go
type emptyIterator (line 13) | type emptyIterator struct
method Next (line 19) | func (it *emptyIterator) Next() chunkenc.ValueType { return chunkenc.V...
method Seek (line 27) | func (it *emptyIterator) Seek(t int64) chunkenc.ValueType { return chu...
method At (line 31) | func (it *emptyIterator) At() (int64, float64) { return 0, 0 }
method AtHistogram (line 36) | func (it *emptyIterator) AtHistogram(histogram *histogram.Histogram) (...
method AtFloatHistogram (line 45) | func (it *emptyIterator) AtFloatHistogram(histogram *histogram.FloatHi...
method AtT (line 51) | func (it *emptyIterator) AtT() int64 { return 0 }
method Err (line 55) | func (it *emptyIterator) Err() error { return nil }
FILE: prometheus/exemplar.go
type nopExemplarQueryable (line 14) | type nopExemplarQueryable struct
method ExemplarQuerier (line 23) | func (e *nopExemplarQueryable) ExemplarQuerier(ctx context.Context) (s...
type nopExemplarQuerier (line 17) | type nopExemplarQuerier struct
method Select (line 27) | func (e *nopExemplarQuerier) Select(start, end int64, matchers ...[]*l...
FILE: prometheus/gatherer.go
type nopGatherer (line 11) | type nopGatherer struct
method Gather (line 15) | func (*nopGatherer) Gather() ([]*dto.MetricFamily, error) {
FILE: prometheus/labels.go
function urlParse (line 14) | func urlParse(rawurl string) (*url.URL, error) {
function Labels (line 28) | func Labels(path string) labels.Labels {
FILE: prometheus/labels_test.go
function TestLabels (line 9) | func TestLabels(t *testing.T) {
FILE: prometheus/local_storage.go
method CleanTombstones (line 17) | func (s *storageImpl) CleanTombstones() error {
method Delete (line 21) | func (s *storageImpl) Delete(ctx context.Context, mint, maxt int64, ms ....
method Snapshot (line 25) | func (s *storageImpl) Snapshot(dir string, withHead bool) error {
method Stats (line 29) | func (s *storageImpl) Stats(statsByLabelName string, limit int) (*tsdb.S...
method WALReplayStatus (line 35) | func (s *storageImpl) WALReplayStatus() (tsdb.WALReplayStatus, error) {
FILE: prometheus/logger.go
type errorLevel (line 10) | type errorLevel interface
type logger (line 14) | type logger struct
method Log (line 18) | func (l *logger) Log(keyvals ...interface{}) error {
FILE: prometheus/matcher.go
function makeTaggedFromPromPB (line 29) | func makeTaggedFromPromPB(matchers []*prompb.LabelMatcher) ([]finder.Tag...
function makeTaggedFromPromQL (line 54) | func makeTaggedFromPromQL(matchers []*labels.Matcher) ([]finder.TaggedTe...
FILE: prometheus/metrics_set.go
type metricsSet (line 14) | type metricsSet struct
method At (line 25) | func (ms *metricsSet) At() storage.Series {
method Err (line 39) | func (ms *metricsSet) Err() error { return nil }
method Next (line 41) | func (ms *metricsSet) Next() bool {
method Warnings (line 56) | func (s *metricsSet) Warnings() annotations.Annotations { return nil }
type metric (line 19) | type metric struct
method Iterator (line 30) | func (s *metric) Iterator(iterator chunkenc.Iterator) chunkenc.Iterator {
method Labels (line 34) | func (s *metric) Labels() labels.Labels {
function newMetricsSet (line 51) | func newMetricsSet(metrics []string) storage.SeriesSet {
FILE: prometheus/querier.go
type Querier (line 23) | type Querier struct
method Close (line 30) | func (q *Querier) Close() error {
method LabelValues (line 35) | func (q *Querier) LabelValues(ctx context.Context, label string, hints...
method LabelNames (line 74) | func (q *Querier) LabelNames(ctx context.Context, hints *storage.Label...
FILE: prometheus/querier_select.go
method lookup (line 22) | func (q *Querier) lookup(ctx context.Context, from, until int64, qlimite...
method timeRange (line 63) | func (q *Querier) timeRange(hints *storage.SelectHints) (int64, int64) {
method Select (line 95) | func (q *Querier) Select(ctx context.Context, sortSeries bool, hints *st...
FILE: prometheus/querier_select_test.go
function TestQuerier_timeRange (line 15) | func TestQuerier_timeRange(t *testing.T) {
FILE: prometheus/run.go
function Run (line 28) | func Run(config *config.Config) error {
FILE: prometheus/run_dummy.go
function Run (line 10) | func Run(config *config.Config) error {
FILE: prometheus/series_set.go
type seriesIterator (line 22) | type seriesIterator struct
method Seek (line 81) | func (sit *seriesIterator) Seek(t int64) chunkenc.ValueType {
method At (line 99) | func (sit *seriesIterator) At() (t int64, v float64) {
method AtHistogram (line 117) | func (sit *seriesIterator) AtHistogram(histogram *histogram.Histogram)...
method AtFloatHistogram (line 127) | func (sit *seriesIterator) AtFloatHistogram(histogram *histogram.Float...
method AtT (line 134) | func (sit *seriesIterator) AtT() int64 {
method Next (line 140) | func (sit *seriesIterator) Next() chunkenc.ValueType {
method Err (line 155) | func (sit *seriesIterator) Err() error { return nil }
type series (line 30) | type series struct
method Iterator (line 188) | func (s *series) Iterator(iterator chunkenc.Iterator) chunkenc.Iterator {
method name (line 192) | func (s *series) name() string {
method Labels (line 196) | func (s *series) Labels() labels.Labels {
type seriesSet (line 37) | type seriesSet struct
method Err (line 158) | func (ss *seriesSet) Err() error { return nil }
method At (line 160) | func (ss *seriesSet) At() storage.Series {
method Next (line 171) | func (ss *seriesSet) Next() bool {
method Warnings (line 183) | func (s *seriesSet) Warnings() annotations.Annotations {
function makeSeriesSet (line 44) | func makeSeriesSet(data *data.Data, step int64) (storage.SeriesSet, erro...
function emptySeriesSet (line 71) | func emptySeriesSet() storage.SeriesSet {
FILE: prometheus/storage.go
type storageImpl (line 13) | type storageImpl struct
method Querier (line 24) | func (s *storageImpl) Querier(mint, maxt int64) (storage.Querier, erro...
method ChunkQuerier (line 33) | func (s *storageImpl) ChunkQuerier(mint, maxt int64) (storage.ChunkQue...
method Appender (line 38) | func (s *storageImpl) Appender(ctx context.Context) storage.Appender {
method StartTime (line 43) | func (s *storageImpl) StartTime() (int64, error) {
method Close (line 48) | func (s *storageImpl) Close() error {
function newStorage (line 19) | func newStorage(config *config.Config) *storageImpl {
FILE: render/data/carbonlink.go
type carbonlinkFetcher (line 15) | type carbonlinkFetcher interface
type carbonlinkClient (line 20) | type carbonlinkClient struct
function setCarbonlinkClient (line 28) | func setCarbonlinkClient(config *config.Carbonlink) {
function queryCarbonlink (line 52) | func queryCarbonlink(parentCtx context.Context, carbonlink *carbonlinkCl...
FILE: render/data/carbonlink_test.go
type carbonlinkMocked (line 16) | type carbonlinkMocked struct
method CacheQueryMulti (line 20) | func (c *carbonlinkMocked) CacheQueryMulti(ctx context.Context, metric...
function TestSetCarbonlingClient (line 25) | func TestSetCarbonlingClient(t *testing.T) {
function TestQueryCarbonlink (line 36) | func TestQueryCarbonlink(t *testing.T) {
FILE: render/data/ch_response.go
type CHResponse (line 14) | type CHResponse struct
method ToMultiFetchResponseV2 (line 30) | func (c *CHResponse) ToMultiFetchResponseV2() (*v2pb.MultiFetchRespons...
method ToMultiFetchResponseV3 (line 129) | func (c *CHResponse) ToMultiFetchResponseV3() (*v3pb.MultiFetchRespons...
type CHResponses (line 24) | type CHResponses
method ToMultiFetchResponseV2 (line 113) | func (cc *CHResponses) ToMultiFetchResponseV2() (*v2pb.MultiFetchRespo...
method ToMultiFetchResponseV3 (line 215) | func (cc *CHResponses) ToMultiFetchResponseV3() (*v3pb.MultiFetchRespo...
function EmptyResponse (line 27) | func EmptyResponse() CHResponses { return CHResponses{{Data: emptyData}} }
FILE: render/data/common_step.go
type commonStep (line 15) | type commonStep struct
method addTargets (line 21) | func (c *commonStep) addTargets(delta int) {
method doneTarget (line 25) | func (c *commonStep) doneTarget() {
method calculateUnsafe (line 29) | func (c *commonStep) calculateUnsafe(a, b int64) int64 {
method calculate (line 37) | func (c *commonStep) calculate(value int64) {
method getResult (line 44) | func (c *commonStep) getResult() int64 {
FILE: render/data/common_step_test.go
type wrapper (line 10) | type wrapper struct
method calc (line 16) | func (w *wrapper) calc(step int64) {
function newWrapper (line 23) | func newWrapper() *wrapper {
function TestCommonStepWorker (line 36) | func TestCommonStepWorker(t *testing.T) {
FILE: render/data/data.go
type Data (line 26) | type Data struct
method GetStep (line 44) | func (d *Data) GetStep(id uint32) (uint32, error) {
method GetAggregation (line 53) | func (d *Data) GetAggregation(id uint32) (string, error) {
function contextIsValid (line 34) | func contextIsValid(ctx context.Context) error {
type data (line 71) | type data struct
method setSteps (line 136) | func (d *data) setSteps(cond *conditions) {
method parseResponse (line 239) | func (d *data) parseResponse(ctx context.Context, bodyReader io.ReadCl...
method wait (line 353) | func (d *data) wait(ctx context.Context) error {
function prepareData (line 82) | func prepareData(ctx context.Context, targets int, fetcher func() *point...
function splitErrorHandler (line 146) | func splitErrorHandler(data *[]byte, atEOF bool, tokenLen int, err error...
function dataSplitAggregated (line 161) | func dataSplitAggregated(data []byte, atEOF bool) (advance int, token []...
function dataSplitUnaggregated (line 196) | func dataSplitUnaggregated(data []byte, atEOF bool) (advance int, token ...
FILE: render/data/data_parse_test.go
type pointValues (line 18) | type pointValues struct
type testPoint (line 24) | type testPoint struct
function makeAggregatedBody (line 29) | func makeAggregatedBody(points []testPoint) []byte {
function makeUnaggregatedBody (line 42) | func makeUnaggregatedBody(points []testPoint) []byte {
function testCarbonlinkReaderNil (line 56) | func testCarbonlinkReaderNil() *point.Points {
function TestUnaggregatedDataParse (line 60) | func TestUnaggregatedDataParse(t *testing.T) {
function TestAggregatedDataParse (line 208) | func TestAggregatedDataParse(t *testing.T) {
function TestPrepareDataParse (line 337) | func TestPrepareDataParse(t *testing.T) {
function TestAsyncDataParse (line 384) | func TestAsyncDataParse(t *testing.T) {
FILE: render/data/multi_target.go
type TimeFrame (line 21) | type TimeFrame struct
type MultiTarget (line 28) | type MultiTarget
method checkMetricsLimitExceeded (line 56) | func (m *MultiTarget) checkMetricsLimitExceeded(num int) error {
method Fetch (line 153) | func (m *MultiTarget) Fetch(ctx context.Context, cfg *config.Config, c...
function MFRToMultiTarget (line 30) | func MFRToMultiTarget(v3Request *v3pb.MultiFetchRequest) MultiTarget {
function getDataTimeout (line 71) | func getDataTimeout(cfg *config.Config, m *MultiTarget) time.Duration {
function GetQueryLimiter (line 92) | func GetQueryLimiter(username string, cfg *config.Config, m *MultiTarget...
function GetQueryLimiterFrom (line 117) | func GetQueryLimiterFrom(username string, cfg *config.Config, from, unti...
function GetQueryParam (line 133) | func GetQueryParam(username string, cfg *config.Config, m *MultiTarget) ...
FILE: render/data/multi_target_test.go
function Test_getDataTimeout (line 10) | func Test_getDataTimeout(t *testing.T) {
FILE: render/data/query.go
constant queryAggregated (line 34) | queryAggregated = `WITH anyResample(%[1]d, %[2]d, %[3]d)(toUInt32(intDiv...
constant queryUnaggregated (line 45) | queryUnaggregated = `SELECT Path, groupArray(Time), groupArray(Value), g...
constant extTableName (line 53) | extTableName = "metrics_list"
type query (line 55) | type query struct
method appendReply (line 127) | func (q *query) appendReply(chr CHResponse) {
method getParam (line 133) | func (q *query) getParam(from, until int64) (string, time.Duration) {
method getDataPoints (line 141) | func (q *query) getDataPoints(ctx context.Context, cond *conditions) e...
method metricsListExtData (line 284) | func (q *query) metricsListExtData(body *strings.Builder) *clickhouse....
type conditions (line 69) | type conditions struct
method prepareMetricsLists (line 301) | func (c *conditions) prepareMetricsLists() {
method prepareLookup (line 318) | func (c *conditions) prepareLookup() error {
method setStep (line 382) | func (c *conditions) setStep(cStep *commonStep) {
method setFromUntil (line 417) | func (c *conditions) setFromUntil() {
method setPrewhere (line 422) | func (c *conditions) setPrewhere() {
method setWhere (line 428) | func (c *conditions) setWhere() {
method generateQuery (line 435) | func (c *conditions) generateQuery(agg string) string {
method generateQueryaAggregated (line 443) | func (c *conditions) generateQueryaAggregated(agg string) string {
method generateQueryUnaggregated (line 451) | func (c *conditions) generateQueryUnaggregated() string {
function newQuery (line 99) | func newQuery(cfg *config.Config, targets int) *query {
FILE: render/data/query_test.go
function genPattern (line 20) | func genPattern(regexp, function string, retention []rollup.Retention) r...
function newAM (line 31) | func newAM() *alias.Map {
function newRules (line 38) | func newRules(reversed bool) *rollup.Rules {
function ageToTimestamp (line 71) | func ageToTimestamp(age int64) int64 {
function newCondition (line 76) | func newCondition(fromAge, untilAge, maxDataPoints int64) *conditions {
function extTableString (line 85) | func extTableString(et map[string]*strings.Builder) map[string]string {
function TestPrepareMetricsLists (line 94) | func TestPrepareMetricsLists(t *testing.T) {
function TestPrepareLookup (line 163) | func TestPrepareLookup(t *testing.T) {
function TestSetStep (line 394) | func TestSetStep(t *testing.T) {
function TestSetFromUntil (line 470) | func TestSetFromUntil(t *testing.T) {
function TestGenerateQuery (line 507) | func TestGenerateQuery(t *testing.T) {
FILE: render/data/targets.go
constant graphiteConsolidationFunction (line 14) | graphiteConsolidationFunction = "consolidateBy"
type FilteringFunctionsByTarget (line 16) | type FilteringFunctionsByTarget
type Cache (line 17) | type Cache struct
type Targets (line 27) | type Targets struct
method Append (line 66) | func (tt *Targets) Append(target string) {
method SetFilteringFunctions (line 71) | func (tt *Targets) SetFilteringFunctions(target string, filteringFunct...
method selectDataTable (line 75) | func (tt *Targets) selectDataTable(cfg *config.Config, tf *TimeFrame, ...
method GetRequestedAggregation (line 134) | func (tt *Targets) GetRequestedAggregation(target string) (string, err...
function NewTargets (line 42) | func NewTargets(list []string, am *alias.Map) *Targets {
function NewTargetsOne (line 53) | func NewTargetsOne(target string, capacity int, am *alias.Map) *Targets {
FILE: render/data/targets_test.go
function TestSelectDataTableTime (line 12) | func TestSelectDataTableTime(t *testing.T) {
function TestSelectDataTableMatch (line 104) | func TestSelectDataTableMatch(t *testing.T) {
FILE: render/handler.go
type Handler (line 28) | type Handler struct
method finderCached (line 59) | func (h *Handler) finderCached(ts time.Time, fetchRequests data.MultiT...
method finder (line 145) | func (h *Handler) finder(fetchRequests data.MultiTarget, ctx context.C...
method ServeHTTP (line 251) | func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
function NewHandler (line 33) | func NewHandler(config *config.Config) *Handler {
function targetKey (line 41) | func targetKey(from, until int64, target, ttl string) string {
function getCacheTimeout (line 45) | func getCacheTimeout(now time.Time, from, until int64, cacheConfig *conf...
FILE: render/handler_test.go
function Test_getCacheTimeout (line 11) | func Test_getCacheTimeout(t *testing.T) {
FILE: render/reply/formatter.go
type Formatter (line 17) | type Formatter interface
function GetFormatter (line 25) | func GetFormatter(r *http.Request) (Formatter, error) {
function parseRequestForms (line 53) | func parseRequestForms(r *http.Request) (data.MultiTarget, error) {
FILE: render/reply/formatter_test.go
function TestFormatterReply (line 53) | func TestFormatterReply(t *testing.T) {
function prepareCHResponses (line 172) | func prepareCHResponses(from, until int64, indices [][]byte, points map[...
function emptyValues (line 210) | func emptyValues(size int) []float64 {
function equalMetrics (line 221) | func equalMetrics(m1, m2 []client.Metric) bool {
FILE: render/reply/json.go
type JSON (line 20) | type JSON struct
method ParseRequest (line 110) | func (*JSON) ParseRequest(r *http.Request) (data.MultiTarget, error) {
method Reply (line 124) | func (*JSON) Reply(w http.ResponseWriter, r *http.Request, multiData d...
function marshalJSON (line 22) | func marshalJSON(mfr *v3pb.MultiFetchResponse) []byte {
function parseJSONBody (line 81) | func parseJSONBody(r *http.Request) (data.MultiTarget, error) {
FILE: render/reply/pickle.go
type Pickle (line 20) | type Pickle struct
method ParseRequest (line 23) | func (*Pickle) ParseRequest(r *http.Request) (data.MultiTarget, error) {
method Reply (line 28) | func (*Pickle) Reply(w http.ResponseWriter, r *http.Request, multiData...
FILE: render/reply/protobuf.go
constant repeated (line 21) | repeated = 2
constant flt32 (line 22) | flt32 = 5
constant protobufMaxVarintBytes (line 23) | protobufMaxVarintBytes = 10
type pb (line 26) | type pb interface
function replyProtobuf (line 31) | func replyProtobuf(p pb, w http.ResponseWriter, r *http.Request, multiDa...
function init (line 101) | func init() {
function VarintEncode (line 112) | func VarintEncode(x uint64) []byte {
function VarintWrite (line 128) | func VarintWrite(w io.Writer, x uint64) {
function VarintLen (line 139) | func VarintLen(x uint64) uint64 {
function WriteByteN (line 156) | func WriteByteN(w *bufio.Writer, value byte, n int) {
function Fixed64Encode (line 163) | func Fixed64Encode(x uint64) []byte {
function Fixed32Encode (line 176) | func Fixed32Encode(x uint32) []byte {
function ProtobufWriteSingle (line 185) | func ProtobufWriteSingle(w io.Writer, value float32) {
function ProtobufWriteDouble (line 189) | func ProtobufWriteDouble(w io.Writer, value float64) {
function ProtobufWriteDoubleN (line 193) | func ProtobufWriteDoubleN(w io.Writer, value float64, n int) {
FILE: render/reply/protobuf_test.go
function TestVarintLen (line 8) | func TestVarintLen(t *testing.T) {
FILE: render/reply/v2_pb.go
type V2PB (line 17) | type V2PB struct
method ParseRequest (line 23) | func (*V2PB) ParseRequest(r *http.Request) (data.MultiTarget, error) {
method Reply (line 28) | func (v *V2PB) Reply(w http.ResponseWriter, r *http.Request, multiData...
method initBuffer (line 36) | func (v *V2PB) initBuffer() {
method replyDebug (line 41) | func (v *V2PB) replyDebug(w http.ResponseWriter, r *http.Request, mult...
method writeBody (line 55) | func (v *V2PB) writeBody(writer *bufio.Writer, target, name, function ...
FILE: render/reply/v2_pb_test.go
type testV2PB (line 14) | type testV2PB struct
function TestV2PBWriteBody (line 25) | func TestV2PBWriteBody(t *testing.T) {
FILE: render/reply/v3_pb.go
type V3PB (line 20) | type V3PB struct
method ParseRequest (line 25) | func (*V3PB) ParseRequest(r *http.Request) (data.MultiTarget, error) {
method Reply (line 65) | func (v *V3PB) Reply(w http.ResponseWriter, r *http.Request, multiData...
method initBuffer (line 73) | func (v *V3PB) initBuffer() {
method replyDebug (line 77) | func (v *V3PB) replyDebug(w http.ResponseWriter, r *http.Request, mult...
method writeBody (line 91) | func (v *V3PB) writeBody(writer *bufio.Writer, target, name, function ...
FILE: render/reply/v3_pb_test.go
type testV3PB (line 15) | type testV3PB struct
function TestV3PBWriteBody (line 26) | func TestV3PBWriteBody(t *testing.T) {
FILE: sd/nginx/nginx.go
type ErrInvalidKey (line 17) | type ErrInvalidKey struct
method Error (line 22) | func (e ErrInvalidKey) Error() string {
function splitNode (line 32) | func splitNode(node string) (dc, host, listen string, ok bool) {
type Nginx (line 52) | type Nginx struct
method setWeight (line 93) | func (sd *Nginx) setWeight(weight int64) {
method Namespace (line 107) | func (sd *Nginx) Namespace() string {
method List (line 111) | func (sd *Nginx) List() (nodes []string, err error) {
method ListMap (line 155) | func (sd *Nginx) ListMap() (nodes map[string]string, err error) {
method Nodes (line 212) | func (sd *Nginx) Nodes() (nodes []utils.KV, err error) {
method update (line 276) | func (sd *Nginx) update(ip, port string, dc []string) (err error) {
method Update (line 344) | func (sd *Nginx) Update(ip, port string, dc []string, weight int64) er...
method DeleteNode (line 350) | func (sd *Nginx) DeleteNode(node string) (err error) {
method Delete (line 361) | func (sd *Nginx) Delete(ip, port string, dc []string) (err error) {
method Clear (line 406) | func (sd *Nginx) Clear(preserveIP, preservePort string) (err error) {
function New (line 65) | func New(url, namespace, hostname string, logger *zap.Logger) *Nginx {
FILE: sd/nginx/nginx_test.go
function TestNginx (line 30) | func TestNginx(t *testing.T) {
function TestNginxDC (line 166) | func TestNginxDC(t *testing.T) {
FILE: sd/nginx/tests/nginx_cleanup_test.go
function cleanup (line 31) | func cleanup(nodes []utils.KV, start, end int64) {
function TestNginxExpire (line 39) | func TestNginxExpire(t *testing.T) {
function TestNginxExpireDC (line 99) | func TestNginxExpireDC(t *testing.T) {
FILE: sd/register.go
type SD (line 24) | type SD interface
function New (line 41) | func New(cfg *config.Common, hostname string, logger *zap.Logger) (SD, e...
function Register (line 51) | func Register(cfg *config.Common, logger *zap.Logger) {
function Stop (line 134) | func Stop() {
function Cleanup (line 138) | func Cleanup(cfg *config.Common, sd SD, checkOnly bool) error {
FILE: sd/utils/utils.go
type KV (line 16) | type KV struct
function HttpGet (line 22) | func HttpGet(url string) ([]byte, error) {
function HttpPut (line 44) | func HttpPut(url string, body []byte) error {
function HttpDelete (line 73) | func HttpDelete(url string) error {
function GetLocalIP (line 101) | func GetLocalIP() string {
FILE: tagger/metric.go
type Metric (line 10) | type Metric struct
method ParentPath (line 17) | func (m *Metric) ParentPath() []byte {
method IsLeaf (line 30) | func (m *Metric) IsLeaf() uint8 {
method MarshalJSON (line 38) | func (m *Metric) MarshalJSON() ([]byte, error) {
FILE: tagger/rule.go
type Rule (line 12) | type Rule struct
method Match (line 204) | func (r *Rule) Match(m *Metric) {
type Rules (line 28) | type Rules struct
method Match (line 129) | func (r *Rules) Match(m *Metric) {
method matchPrefix (line 184) | func (r *Rules) matchPrefix(m *Metric) {
method matchSuffix (line 188) | func (r *Rules) matchSuffix(m *Metric) {
method matchContains (line 192) | func (r *Rules) matchContains(m *Metric) {
method matchOther (line 198) | func (r *Rules) matchOther(m *Metric) {
function ParseFile (line 36) | func ParseFile(filename string) (*Rules, error) {
function ParseGlob (line 45) | func ParseGlob(glob string) (*Rules, error) {
function Parse (line 65) | func Parse(content string) (*Rules, error) {
function matchByPrefix (line 136) | func matchByPrefix(path []byte, tree *Tree, m *Metric) {
function matchBySuffix (line 160) | func matchBySuffix(path []byte, tree *Tree, m *Metric) {
FILE: tagger/rule_test.go
function TestRules (line 33) | func TestRules(t *testing.T) {
FILE: tagger/set.go
type Set (line 8) | type Set struct
method Add (line 19) | func (s *Set) Add(tag ...string) *Set {
method Merge (line 52) | func (s *Set) Merge(other *Set) *Set {
method Len (line 56) | func (s *Set) Len() int {
method List (line 60) | func (s *Set) List() []string {
method MarshalJSON (line 64) | func (s *Set) MarshalJSON() ([]byte, error) {
FILE: tagger/tagger.go
constant SelectChunksCount (line 27) | SelectChunksCount = 10
type nopCloser (line 29) | type nopCloser struct
method Close (line 33) | func (nopCloser) Close() error { return nil }
function countMetrics (line 35) | func countMetrics(body []byte) (int, error) {
function pathLevel (line 63) | func pathLevel(path []byte) int {
function Make (line 75) | func Make(cfg *config.Config) error {
function cutMetricsIntoParts (line 430) | func cutMetricsIntoParts(metricList []Metric, threads int) ([][]Metric, ...
function wrapWithCompressor (line 462) | func wrapWithCompressor(cfg *config.Config, writer io.Writer) (io.WriteC...
function encodeMetricsToRowBinary (line 484) | func encodeMetricsToRowBinary(metricList []Metric, date time.Time, versi...
function encodeEmptyMetricToRowBinary (line 551) | func encodeEmptyMetricToRowBinary(date time.Time, version uint32, wc io....
FILE: tagger/tagger_test.go
function TestCutMetricsIntoParts (line 14) | func TestCutMetricsIntoParts(t *testing.T) {
function TestCutMetricsIntoPartsRandom (line 294) | func TestCutMetricsIntoPartsRandom(t *testing.T) {
FILE: tagger/tree.go
type Tree (line 3) | type Tree struct
method Add (line 8) | func (t *Tree) Add(prefix []byte, rule *Rule) {
method AddSuffix (line 26) | func (t *Tree) AddSuffix(suffix []byte, rule *Rule) {
FILE: tests/clickhouse/rollup/init.sql
type default (line 1) | CREATE TABLE IF NOT EXISTS default.graphite_reverse (
type default (line 11) | CREATE TABLE IF NOT EXISTS default.graphite (
type default (line 21) | CREATE TABLE IF NOT EXISTS default.graphite_index (
type default (line 30) | CREATE TABLE IF NOT EXISTS default.graphite_tags (
type default (line 40) | CREATE TABLE IF NOT EXISTS default.tag1_count_per_day
FILE: tests/clickhouse/rollup_tls/init.sql
type default (line 1) | CREATE TABLE IF NOT EXISTS default.graphite_reverse (
type default (line 11) | CREATE TABLE IF NOT EXISTS default.graphite (
type default (line 21) | CREATE TABLE IF NOT EXISTS default.graphite_index (
type default (line 30) | CREATE TABLE IF NOT EXISTS default.graphite_tags (
type default (line 40) | CREATE TABLE IF NOT EXISTS default.tag1_count_per_day
Condensed preview — 293 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (1,542K chars).
[
{
"path": ".gitattributes",
"chars": 0,
"preview": ""
},
{
"path": ".github/workflows/codeql.yml",
"chars": 2316,
"preview": "# For most projects, this workflow file will not need changing; you simply need\n# to commit it to your repository.\n#\n# Y"
},
{
"path": ".github/workflows/docker.yml",
"chars": 1371,
"preview": "name: Docker images\n\non:\n push:\n branches: [ master ]\n tags: [ 'v*' ]\n pull_request:\n branches: [ master ]\n\nj"
},
{
"path": ".github/workflows/lint.yml",
"chars": 339,
"preview": "name: Lint\non:\n pull_request:\n\njobs:\n golangci:\n name: lint\n runs-on: ubuntu-22.04\n steps:\n - uses: acti"
},
{
"path": ".github/workflows/release.yml",
"chars": 2111,
"preview": "name: Upload Packages to new release\n\non:\n release:\n types:\n - published\n\njobs:\n build:\n name: Build\n ru"
},
{
"path": ".github/workflows/tests-sd.yml",
"chars": 634,
"preview": "name: Tests register in SD\n\non:\n push:\n branches: [ master ]\n pull_request:\n branches: [ master ]\n\njobs:\n\n test"
},
{
"path": ".github/workflows/tests.yml",
"chars": 2825,
"preview": "name: Tests\n\non:\n push:\n branches: [ master ]\n pull_request:\n branches: [ master ]\n\njobs:\n\n tests:\n env:\n "
},
{
"path": ".gitignore",
"chars": 300,
"preview": "# Compiled Object files, Static and Dynamic libs (Shared Objects)\n*.o\n*.a\n*.so\ngraphite-clickhouse\n.vscode\n/out/\n\n# Fold"
},
{
"path": ".golangci.yml",
"chars": 1874,
"preview": "version: \"2\"\nlinters:\n default: none\n enable:\n - asasalint\n - asciicheck\n - bidichk\n - bodyclose\n# - co"
},
{
"path": "Dockerfile",
"chars": 436,
"preview": "FROM golang:alpine as builder\n\nWORKDIR /go/src/github.com/lomik/graphite-clickhouse\nCOPY . .\n\nENV GOPATH=/go\n\nRUN apk ad"
},
{
"path": "LICENSE",
"chars": 1072,
"preview": "MIT License\n\nCopyright (c) 2016 Roman Lomonosov\n\nPermission is hereby granted, free of charge, to any person obtaining a"
},
{
"path": "Makefile",
"chars": 3424,
"preview": "NAME:=graphite-clickhouse\nDESCRIPTION:=\"Graphite cluster backend with ClickHouse support\"\nMODULE:=github.com/lomik/graph"
},
{
"path": "README.md",
"chars": 2882,
"preview": "[](https://packagecloud.io/go-graphite/stable)\n[![rpm"
},
{
"path": "autocomplete/autocomplete.go",
"chars": 18883,
"preview": "package autocomplete\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"g"
},
{
"path": "autocomplete/autocomplete_test.go",
"chars": 28501,
"preview": "package autocomplete\n\nimport (\n\t\"io\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/"
},
{
"path": "cache/cache.go",
"chars": 2024,
"preview": "package cache\n\nimport (\n\t\"crypto/sha256\"\n\t\"encoding/hex\"\n\t\"errors\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/bradfitz/gomemca"
},
{
"path": "capabilities/handler.go",
"chars": 2666,
"preview": "package capabilities\n\nimport (\n\t\"encoding/json\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\t\"time\"\n\n\tv3pb \"github.com/go-graphite/protocol/"
},
{
"path": "cmd/e2e-test/carbon-clickhouse.go",
"chars": 3286,
"preview": "package main\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"text/template\"\n)\n\nvar CchContainerN"
},
{
"path": "cmd/e2e-test/checks.go",
"chars": 15551,
"preview": "package main\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"os\"\n\t\"reflect\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/"
},
{
"path": "cmd/e2e-test/clickhouse.go",
"chars": 5861,
"preview": "package main\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/"
},
{
"path": "cmd/e2e-test/container.go",
"chars": 1599,
"preview": "package main\n\nimport (\n\t\"os/exec\"\n\t\"strings\"\n)\n\nvar (\n\tDockerBinary string\n\tDockerNetwork string = \"graphite-ch-test\"\n)"
},
{
"path": "cmd/e2e-test/e2etesting.go",
"chars": 24834,
"preview": "package main\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"path\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"go.uber.o"
},
{
"path": "cmd/e2e-test/errors.go",
"chars": 196,
"preview": "package main\n\nimport \"errors\"\n\nvar (\n\tErrTimestampInvalid = errors.New(\"invalid timestamp\")\n\tErrNoTest = error"
},
{
"path": "cmd/e2e-test/graphite-clickhouse.go",
"chars": 3425,
"preview": "package main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"syscall\"\n\t\"te"
},
{
"path": "cmd/e2e-test/main.go",
"chars": 6146,
"preview": "package main\n\nimport (\n\t\"flag\"\n\t\"log\"\n\t\"os\"\n\t\"path\"\n\t\"runtime\"\n\t\"time\"\n\n\t\"go.uber.org/zap\"\n)\n\ntype MainConfig struct {\n\t"
},
{
"path": "cmd/e2e-test/rproxy.go",
"chars": 2396,
"preview": "package main\n\nimport (\n\t\"errors\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"net/http/httputil\"\n\t\"net/url\"\n\t\"sync\"\n\t\"sync/atomic\""
},
{
"path": "cmd/e2e-test/utils.go",
"chars": 198,
"preview": "package main\n\nimport \"os/exec\"\n\nfunc cmdExec(programm string, args ...string) (string, error) {\n\tcmd := exec.Command(pro"
},
{
"path": "cmd/graphite-clickhouse-client/main.go",
"chars": 5988,
"preview": "package main\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"os\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/go-graphite/protocol"
},
{
"path": "config/.gitignore",
"chars": 11,
"preview": "tests_tmp/\n"
},
{
"path": "config/config.go",
"chars": 46638,
"preview": "package config\n\nimport (\n\t\"bytes\"\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"net\"\n\t\"net/url\"\n\t\"os\"\n\t\"reflect\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strconv\"\n\t"
},
{
"path": "config/config_test.go",
"chars": 35233,
"preview": "package config\n\nimport (\n\t\"fmt\"\n\t\"io/fs\"\n\t\"math\"\n\t\"net/url\"\n\t\"os\"\n\t\"regexp\"\n\t\"regexp/syntax\"\n\t\"syscall\"\n\t\"testing\"\n\t\"tim"
},
{
"path": "config/json.go",
"chars": 423,
"preview": "package config\n\nimport (\n\t\"encoding/json\"\n\t\"net/url\"\n)\n\nfunc (c *ClickHouse) MarshalJSON() ([]byte, error) {\n\ttype Click"
},
{
"path": "config/json_test.go",
"chars": 536,
"preview": "package config\n\nimport (\n\t\"encoding/json\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestClickhouseUrlPas"
},
{
"path": "deploy/doc/.gitignore",
"chars": 66,
"preview": "# autogenerated config for documentation\ngraphite-clickhouse.conf\n"
},
{
"path": "deploy/doc/config.md",
"chars": 11247,
"preview": "# Configuration\n\n## Common `[common]`\n\n### Finder cache\n\nSpecify what storage to use for finder cache. This cache store"
},
{
"path": "deploy/root/usr/lib/systemd/system/graphite-clickhouse.service",
"chars": 374,
"preview": "[Unit]\nDescription=Graphite cluster backend with ClickHouse support\nDocumentation=https://github.com/lomik/graphite-clic"
},
{
"path": "doc/aggregation.md",
"chars": 7052,
"preview": "# Enable ClickHouse aggregation\n\nThe feature was added in [v0.12.0](https://github.com/lomik/graphite-clickhouse/release"
},
{
"path": "doc/config.md",
"chars": 22156,
"preview": "[//]: # (This file is built out of deploy/doc/config.md, please do not edit it manually) \n[//]: # (To rebuild it run `m"
},
{
"path": "doc/debugging.md",
"chars": 5605,
"preview": "# Debug graphite-clickhouse\n## General config\nThe `debug` section contains common parameters:\n\n```toml\n[debug]\ndirectory"
},
{
"path": "doc/graphite_clickhouse.gliffy",
"chars": 61656,
"preview": "{\"contentType\":\"application/gliffy+json\",\"version\":\"1.3\",\"stage\":{\"background\":\"#FFFFFF\",\"width\":700,\"height\":710,\"nodeI"
},
{
"path": "doc/index-table.md",
"chars": 3291,
"preview": "# Index table\nThe `index` type table is used to look up metrics that [match the query](https://graphite.readthedocs.io/e"
},
{
"path": "doc/release.md",
"chars": 66,
"preview": "# New release\n\n- Update `const Version` in graphite-clickhouse.go\n"
},
{
"path": "find/find.go",
"chars": 4042,
"preview": "package find\n\nimport (\n\t\"context\"\n\t\"io\"\n\n\t\"github.com/gogo/protobuf/proto\"\n\t\"github.com/msaf1980/go-stringutils\"\n\n\tv2pb "
},
{
"path": "find/handler.go",
"chars": 6476,
"preview": "package find\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/go-graphite/carbonapi/pkg/pa"
},
{
"path": "find/handler_json_test.go",
"chars": 5208,
"preview": "package find\n\nimport (\n\t\"io\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite"
},
{
"path": "find/handler_test.go",
"chars": 1443,
"preview": "package find\n\nimport (\n\t\"io\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"testing\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\""
},
{
"path": "finder/base.go",
"chars": 2356,
"preview": "package finder\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\""
},
{
"path": "finder/blacklist.go",
"chars": 1201,
"preview": "package finder\n\nimport (\n\t\"context\"\n\t\"regexp\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphit"
},
{
"path": "finder/date.go",
"chars": 1762,
"preview": "package finder\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/gr"
},
{
"path": "finder/date_reverse.go",
"chars": 1878,
"preview": "package finder\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-c"
},
{
"path": "finder/date_reverse_test.go",
"chars": 1723,
"preview": "package finder\n\nimport (\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/lomi"
},
{
"path": "finder/finder.go",
"chars": 4474,
"preview": "package finder\n\nimport (\n\t\"context\"\n\t\"strings\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/clickhouse\"\n\t\"github.com/l"
},
{
"path": "finder/index.go",
"chars": 7069,
"preview": "package finder\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"strings\"\n\n\t\"github.com/lomik/graphite-clickhouse/confi"
},
{
"path": "finder/index_test.go",
"chars": 5781,
"preview": "package finder\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/gr"
},
{
"path": "finder/mock.go",
"chars": 1477,
"preview": "package finder\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"strings\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lom"
},
{
"path": "finder/plain_from_tagged.go",
"chars": 2746,
"preview": "package finder\n\nimport (\n\t\"context\"\n\t\"net/url\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/lomik/graphite-clickhouse/con"
},
{
"path": "finder/plain_from_tagged_test.go",
"chars": 733,
"preview": "package finder\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestPlainFromTaggedFinderAbs(t *testi"
},
{
"path": "finder/prefix.go",
"chars": 2606,
"preview": "package finder\n\nimport (\n\t\"context\"\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lo"
},
{
"path": "finder/prefix_test.go",
"chars": 2739,
"preview": "package finder\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/stret"
},
{
"path": "finder/reverse.go",
"chars": 2201,
"preview": "package finder\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"strings\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lom"
},
{
"path": "finder/reverse_test.go",
"chars": 436,
"preview": "package finder\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestReverse(t *testing.T) {\n\tassert :"
},
{
"path": "finder/split.go",
"chars": 9012,
"preview": "package finder\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"strings\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"gith"
},
{
"path": "finder/split_test.go",
"chars": 12383,
"preview": "package finder\n\nimport (\n\t\"fmt\"\n\t\"net/http\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github."
},
{
"path": "finder/tag.go",
"chars": 7084,
"preview": "package finder\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github."
},
{
"path": "finder/tag_test.go",
"chars": 3668,
"preview": "package finder\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n\n\t\"github.com/lomik"
},
{
"path": "finder/tagged.go",
"chars": 18042,
"preview": "package finder\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"sort\"\n\t\"strings\"\n\n\t\"github.com/lomik/graphite-clickhou"
},
{
"path": "finder/tagged_test.go",
"chars": 52535,
"preview": "package finder\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net/url\"\n\t\"sort\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite"
},
{
"path": "finder/tags_count_querier.go",
"chars": 4342,
"preview": "package finder\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"strconv\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github."
},
{
"path": "finder/unescape.go",
"chars": 1053,
"preview": "package finder\n\nimport \"strings\"\n\nfunc ishex(c byte) bool {\n\tswitch {\n\tcase '0' <= c && c <= '9':\n\t\treturn true\n\tcase 'a"
},
{
"path": "go.mod",
"chars": 7078,
"preview": "module github.com/lomik/graphite-clickhouse\n\ngo 1.23.1\n\nrequire (\n\tgithub.com/BurntSushi/toml v0.3.1\n\tgithub.com/bradfit"
},
{
"path": "go.sum",
"chars": 87284,
"preview": "cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=\ncloud.google.com/go v0.34.0/go.mod h1"
},
{
"path": "graphite-clickhouse.go",
"chars": 17517,
"preview": "package main\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"flag\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"math/rand\"\n\t\"net/http\"\n\t_ \"net/http/ppr"
},
{
"path": "healthcheck/healthcheck.go",
"chars": 2829,
"preview": "package healthcheck\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strings\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/lomik/"
},
{
"path": "helper/RowBinary/encode.go",
"chars": 3464,
"preview": "package RowBinary\n\nimport (\n\t\"encoding/binary\"\n\t\"io\"\n\t\"math\"\n\t\"time\"\n)\n\nconst NullUint32 = ^uint32(0)\n\nfunc DateToUint16"
},
{
"path": "helper/clickhouse/clickhouse.go",
"chars": 14122,
"preview": "package clickhouse\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"encoding/binary\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"html\"\n\t\"io\""
},
{
"path": "helper/clickhouse/clickhouse_test.go",
"chars": 5183,
"preview": "package clickhouse\n\nimport (\n\t\"net/http\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc Test_extractClickhous"
},
{
"path": "helper/clickhouse/external-data.go",
"chars": 3403,
"preview": "package clickhouse\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"mime/multipart\"\n\t\"net/url\"\n\t\"os\"\n\t\"path\"\n\t\"strings\"\n\n\t\"github."
},
{
"path": "helper/clickhouse/external-data_test.go",
"chars": 3169,
"preview": "package clickhouse\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"net/url\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"g"
},
{
"path": "helper/client/datetime.go",
"chars": 597,
"preview": "package client\n\nimport (\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/datetime\"\n)\n\nfunc MetricsTimestampTrunca"
},
{
"path": "helper/client/errros.go",
"chars": 327,
"preview": "package client\n\nimport \"strconv\"\n\ntype HttpError struct {\n\tstatusCode int\n\tmessage string\n}\n\nfunc NewHttpError(status"
},
{
"path": "helper/client/find.go",
"chars": 3222,
"preview": "package client\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\n\tprotov2 \"github.com/go-graphite/protocol/carbona"
},
{
"path": "helper/client/render.go",
"chars": 7111,
"preview": "package client\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strconv\"\n\t\"st"
},
{
"path": "helper/client/requests.go",
"chars": 323,
"preview": "package client\n\nimport protov3 \"github.com/go-graphite/protocol/carbonapi_v3_pb\"\n\ntype MultiGlobRequestV3 struct {\n\tprot"
},
{
"path": "helper/client/tags.go",
"chars": 6674,
"preview": "package client\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github."
},
{
"path": "helper/client/types.go",
"chars": 1040,
"preview": "package client\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n)\n\ntype FormatType int\n\nconst (\n\tFormatDefault FormatType = iota\n\tFormatJSON\n\t"
},
{
"path": "helper/date/date.go",
"chars": 3086,
"preview": "package date\n\nimport \"time\"\n\nvar FromTimestampToDaysFormat func(int64) string\nvar FromTimeToDaysFormat func(time.Time) s"
},
{
"path": "helper/date/date_test.go",
"chars": 8643,
"preview": "package date\n\nimport (\n\t\"os\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n)\n\nvar verbose bool\n\nfunc isVerbose() bool {\n\tfor _, arg := r"
},
{
"path": "helper/datetime/datetime.go",
"chars": 4510,
"preview": "package datetime\n\nimport (\n\t\"errors\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/go-graphite/carbonapi/pkg/parser\"\n)\n\nva"
},
{
"path": "helper/datetime/datetime_test.go",
"chars": 2968,
"preview": "package datetime\n\nimport (\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestDateParamToEpoch(t *testing.T) {\n\ttimeZone := time.Local\n\t//16"
},
{
"path": "helper/errs/errors.go",
"chars": 367,
"preview": "package errs\n\nimport \"fmt\"\n\ntype ErrorWithCode struct {\n\terr string\n\tCode int // error code\n}\n\nfunc NewErrorWithCode(er"
},
{
"path": "helper/headers/headers.go",
"chars": 316,
"preview": "package headers\n\nimport \"net/http\"\n\nfunc GetHeaders(header *http.Header, keys []string) map[string]string {\n\tif len(keys"
},
{
"path": "helper/http/live-http-client.go",
"chars": 1196,
"preview": "package http\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"time\"\n)\n\nconst TCPNetwork string = \"tcp\"\n"
},
{
"path": "helper/pickle/pickle.go",
"chars": 1474,
"preview": "package pickle\n\nimport (\n\t\"encoding/binary\"\n\t\"io\"\n\t\"math\"\n)\n\nvar EmptyList = []byte{0x28, 0x6c, 0x70, 0x30, 0xa, 0x2e}\n\n"
},
{
"path": "helper/point/func.go",
"chars": 2496,
"preview": "package point\n\nimport (\n\t\"fmt\"\n\t\"math\"\n)\n\n// CleanUp removes points with empty metric\n// for run after Deduplicate, Merg"
},
{
"path": "helper/point/func_test.go",
"chars": 5156,
"preview": "package point\n\nimport (\n\t\"errors\"\n\t\"math\"\n\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nvar nan = math.NaN()\n\nfu"
},
{
"path": "helper/point/point.go",
"chars": 780,
"preview": "package point\n\nimport \"fmt\"\n\ntype Point struct {\n\tMetricID uint32\n\tValue float64\n\tTime uint32\n\tTimestamp uint3"
},
{
"path": "helper/point/points.go",
"chars": 4488,
"preview": "package point\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n)\n\n// Points is a structure that stores points and additional information about t"
},
{
"path": "helper/rollup/aggr.go",
"chars": 1512,
"preview": "package rollup\n\nimport (\n\t\"github.com/lomik/graphite-clickhouse/helper/point\"\n)\n\nvar AggrMap = map[string]*Aggr{\n\t\"avg\":"
},
{
"path": "helper/rollup/compact.go",
"chars": 1440,
"preview": "package rollup\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n)\n\n/*\ncompact form of rollup rules for tests\n\nregexp;function;age:"
},
{
"path": "helper/rollup/compact_test.go",
"chars": 667,
"preview": "package rollup\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestParseCompact(t *testing.T) {\n\tcon"
},
{
"path": "helper/rollup/remote.go",
"chars": 4335,
"preview": "package rollup\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/lomik/zap"
},
{
"path": "helper/rollup/remote_test.go",
"chars": 6724,
"preview": "package rollup\n\nimport (\n\t\"encoding/json\"\n\t\"regexp\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc assertJson"
},
{
"path": "helper/rollup/rollup.go",
"chars": 2602,
"preview": "package rollup\n\nimport (\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"os\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/lomik/zapwriter\"\n\t\"go"
},
{
"path": "helper/rollup/rules.go",
"chars": 11148,
"preview": "package rollup\n\nimport (\n\t\"encoding/xml\"\n\t\"fmt\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickh"
},
{
"path": "helper/rollup/rules_test.go",
"chars": 19897,
"preview": "package rollup\n\nimport (\n\t\"fmt\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/p"
},
{
"path": "helper/rollup/xml.go",
"chars": 2132,
"preview": "package rollup\n\nimport (\n\t\"encoding/xml\"\n)\n\n/*\n<graphite_rollup>\n \t<pattern>\n \t\t<regexp>click_cost</regexp>\n \t\t<function"
},
{
"path": "helper/rollup/xml_test.go",
"chars": 7939,
"preview": "package rollup\n\nimport (\n\t\"fmt\"\n\t\"regexp\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testif"
},
{
"path": "helper/tests/clickhouse/server.go",
"chars": 1293,
"preview": "package clickhouse\n\nimport (\n\t\"io\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"sync\"\n\t\"sync/atomic\"\n)\n\ntype TestResponse struct {"
},
{
"path": "helper/tests/compare/compare.go",
"chars": 711,
"preview": "package compare\n\nimport \"math\"\n\nconst eps = 0.0000000001\n\nfunc NearlyEqualSlice(a, b []float64) bool {\n\tif len(a) != len"
},
{
"path": "helper/tests/compare/expand/expand.go",
"chars": 418,
"preview": "package expand\n\nimport (\n\t\"go/token\"\n\t\"go/types\"\n\t\"strconv\"\n\t\"strings\"\n)\n\nfunc ExpandTimestamp(fs *token.FileSet, s stri"
},
{
"path": "helper/utils/utils.go",
"chars": 224,
"preview": "package utils\n\nimport \"time\"\n\n// TimestampTruncate truncate timestamp with duration\nfunc TimestampTruncate(ts int64, dur"
},
{
"path": "helper/utils/utils_test.go",
"chars": 891,
"preview": "package utils\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestTimestampTruncate(t *testing.T) {\n\t// reverse sorted\n\ttest"
},
{
"path": "index/handler.go",
"chars": 1005,
"preview": "package index\n\nimport (\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-"
},
{
"path": "index/index.go",
"chars": 2364,
"preview": "package index\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\n\t\"github.com/lomik/graph"
},
{
"path": "index/index_test.go",
"chars": 1946,
"preview": "package index\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"io\"\n\t\"net/http/httptest\"\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestWriteJSON"
},
{
"path": "issues/daytime/carbon-clickhouse.conf.tpl",
"chars": 806,
"preview": "[common]\n\n[data]\npath = \"/etc/carbon-clickhouse/data\"\nchunk-interval = \"1s\"\nchunk-auto-interval = \"\"\n\n[upload.graphite_i"
},
{
"path": "issues/daytime/graphite-clickhouse-internal-aggr.conf.tpl",
"chars": 802,
"preview": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nhead"
},
{
"path": "issues/daytime/graphite-clickhouse.conf.tpl",
"chars": 803,
"preview": "[common]\nlisten = \"{{ .GCH_ADDR }}\"\nmax-cpu = 0\nmax-metrics-in-render-answer = 10000\nmax-metrics-per-target = 10000\nhead"
},
{
"path": "issues/daytime/test.toml",
"chars": 11677,
"preview": "[test]\nprecision = \"10s\"\n\n[[test.clickhouse]]\nversion = \"24.2\"\ndir = \"tests/clickhouse/rollup\"\n\n[test.carbon_clickhouse]"
},
{
"path": "limiter/alimiter.go",
"chars": 3500,
"preview": "package limiter\n\nimport (\n\t\"context\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/load_avg\"\n\t\"github.com/lomik/graphi"
},
{
"path": "limiter/alimiter_test.go",
"chars": 4561,
"preview": "package limiter\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"strconv\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhous"
},
{
"path": "limiter/interface.go",
"chars": 403,
"preview": "package limiter\n\nimport (\n\t\"context\"\n\t\"errors\"\n)\n\nvar (\n\tErrTimeout = errors.New(\"timeout exceeded\")\n\tErrOverflow = err"
},
{
"path": "limiter/limiter.go",
"chars": 2352,
"preview": "package limiter\n\nimport (\n\t\"context\"\n\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n)\n\ntype limiter struct {\n\tch chan"
},
{
"path": "limiter/noop.go",
"chars": 874,
"preview": "package limiter\n\nimport (\n\t\"context\"\n)\n\n// ServerLimiter provides interface to limit amount of requests\ntype NoopLimiter"
},
{
"path": "limiter/wlimiter.go",
"chars": 2557,
"preview": "package limiter\n\nimport (\n\t\"context\"\n\n\t\"github.com/lomik/graphite-clickhouse/metrics\"\n)\n\n// WLimiter provide limiter amo"
},
{
"path": "load_avg/load_avg.go",
"chars": 734,
"preview": "package load_avg\n\nimport (\n\t\"math\"\n\n\t\"github.com/msaf1980/go-syncutils/atomic\"\n)\n\nvar loadAvgStore atomic.Float64\n\nfunc "
},
{
"path": "load_avg/load_avg_default.go",
"chars": 160,
"preview": "//go:build !linux\n// +build !linux\n\npackage load_avg\n\nfunc Normalized() (float64, error) {\n\treturn 0, nil\n}\n\nfunc CpuCou"
},
{
"path": "load_avg/load_avg_linux.go",
"chars": 669,
"preview": "//go:build linux\n// +build linux\n\npackage load_avg\n\nimport (\n\t\"os\"\n\t\"strings\"\n\t\"syscall\"\n\n\t\"github.com/msaf1980/go-strin"
},
{
"path": "load_avg/load_avg_test.go",
"chars": 3828,
"preview": "package load_avg\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n)\n\nfunc TestWeight(t *testing.T) {\n\ttests := []struct {\n\t\tweight i"
},
{
"path": "logs/logger.go",
"chars": 1067,
"preview": "package logs\n\nimport (\n\t\"net\"\n\t\"net/http\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.co"
},
{
"path": "metrics/limiter_metrics.go",
"chars": 16,
"preview": "package metrics\n"
},
{
"path": "metrics/metrics.go",
"chars": 23159,
"preview": "package metrics\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"sort\"\n\t\"time\"\n\n\t\"github.com/msaf1980/go-metrics\"\n\t\"github.com/msaf1980/go-met"
},
{
"path": "metrics/metrics_test.go",
"chars": 19800,
"preview": "package metrics\n\nimport (\n\t\"math\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/msaf1980/go-metrics\"\n\t\"github.com/msaf1980"
},
{
"path": "metrics/query_metrics.go",
"chars": 4145,
"preview": "package metrics\n\nimport \"github.com/msaf1980/go-metrics\"\n\ntype QueryMetric struct {\n\tRequestsH metrics.Histogram\n\t"
},
{
"path": "metrics/statsd.go",
"chars": 1517,
"preview": "package metrics\n\nimport (\n\t\"time\"\n\n\t\"github.com/cactus/go-statsd-client/v5/statsd\"\n)\n\n// NullSender is disabled sender "
},
{
"path": "nfpm.yaml",
"chars": 801,
"preview": "---\nname: ${NAME}\ndescription: ${DESCRIPTION}\n\n# Common packages config\narch: \"${ARCH}\" # amd64, arm64\nplatform: \"linux"
},
{
"path": "packages.sh",
"chars": 566,
"preview": "#!/bin/sh -e\n\ncd \"$( dirname \"$0\" )\"\nROOT=$PWD\n\ndocker run -i -e \"DEVEL=${DEVEL:-0}\" --rm -v \"$ROOT:/root/go/src/github"
},
{
"path": "pkg/alias/map.go",
"chars": 2205,
"preview": "package alias\n\nimport (\n\t\"bytes\"\n\t\"sync\"\n\n\t\"github.com/lomik/graphite-clickhouse/finder\"\n\t\"github.com/lomik/graphite-cli"
},
{
"path": "pkg/alias/map_tagged_test.go",
"chars": 2849,
"preview": "package alias\n\nimport (\n\t\"sort\"\n\t\"sync\"\n\t\"testing\"\n\n\t\"github.com/lomik/graphite-clickhouse/finder\"\n\t\"github.com/stretchr"
},
{
"path": "pkg/alias/map_test.go",
"chars": 3904,
"preview": "package alias\n\nimport (\n\t\"sort\"\n\t\"sync\"\n\t\"testing\"\n\n\t\"github.com/lomik/graphite-clickhouse/finder\"\n\t\"github.com/stretchr"
},
{
"path": "pkg/dry/math.go",
"chars": 1346,
"preview": "package dry\n\n// Max returns the larger of x or y.\nfunc Max(x, y int64) int64 {\n\tif x > y {\n\t\treturn x\n\t}\n\n\treturn y\n}\n\n/"
},
{
"path": "pkg/dry/math_test.go",
"chars": 2027,
"preview": "package dry\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestMax(t *testing.T) {\n\tassert := asser"
},
{
"path": "pkg/dry/strings.go",
"chars": 353,
"preview": "package dry\n\n// RemoveEmptyStrings removes empty strings from list and returns truncated slice\nfunc RemoveEmptyStrings(s"
},
{
"path": "pkg/dry/strings_test.go",
"chars": 268,
"preview": "package dry\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestRemoveEmptyStrings(t *testing.T) {\n\t"
},
{
"path": "pkg/dry/unsafe.go",
"chars": 364,
"preview": "package dry\n\nimport (\n\t\"reflect\"\n\t\"unsafe\"\n)\n\n// UnsafeString returns string object from byte slice without copying\nfunc"
},
{
"path": "pkg/dry/unsafe_test.go",
"chars": 335,
"preview": "package dry\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestUnsafeString(t *testing.T) {\n\tassert"
},
{
"path": "pkg/reverse/reverse.go",
"chars": 866,
"preview": "package reverse\n\nimport (\n\t\"bytes\"\n\t\"strings\"\n)\n\nfunc String(path string) string {\n\t// don't reverse tagged path\n\tif str"
},
{
"path": "pkg/reverse/reverse_test.go",
"chars": 800,
"preview": "package reverse\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestReverse(t *testing.T) {\n\tassert "
},
{
"path": "pkg/scope/context.go",
"chars": 731,
"preview": "package scope\n\nimport (\n\t\"context\"\n\n\t\"go.uber.org/zap\"\n)\n\n// Context wrapper for context.Context with chain constructor\n"
},
{
"path": "pkg/scope/http_request.go",
"chars": 1587,
"preview": "package scope\n\nimport (\n\t\"context\"\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"net\"\n\t\"net/http\"\n\t\"regexp\"\n\t\"strings\"\n)\n\nvar"
},
{
"path": "pkg/scope/key.go",
"chars": 1797,
"preview": "package scope\n\nimport (\n\t\"context\"\n\t\"fmt\"\n)\n\n// key is type for context.Value keys\ntype scopeKey string\n\n// With returns"
},
{
"path": "pkg/scope/logger.go",
"chars": 1686,
"preview": "package scope\n\nimport (\n\t\"context\"\n\t\"net/http\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper/headers\"\n\t\"github.com/lomi"
},
{
"path": "pkg/scope/version.go",
"chars": 34,
"preview": "package scope\n\nvar Version string\n"
},
{
"path": "pkg/where/match.go",
"chars": 4146,
"preview": "package where\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\nvar opEq = \"=\"\n\n// ClearGlob cleanup grafana globs like {name}\nfunc ClearGl"
},
{
"path": "pkg/where/match_test.go",
"chars": 2314,
"preview": "package where\n\nimport \"testing\"\n\nfunc Test_ClearGlob(t *testing.T) {\n\ttests := []struct {\n\t\tquery string\n\t\twant string\n"
},
{
"path": "pkg/where/where.go",
"chars": 5726,
"preview": "package where\n\nimport (\n\t\"fmt\"\n\t\"net/http\"\n\t\"regexp\"\n\t\"strings\"\n\t\"unsafe\"\n\n\t\"github.com/lomik/graphite-clickhouse/helper"
},
{
"path": "pkg/where/where_test.go",
"chars": 2406,
"preview": "package where\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestGlobExpandSimple(t *testing"
},
{
"path": "prometheus/.gitignore",
"chars": 4,
"preview": "tmp\n"
},
{
"path": "prometheus/empty_iterator.go",
"chars": 2383,
"preview": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"github.com/prometheus/prometheus/model/histogram\"\n\t"
},
{
"path": "prometheus/exemplar.go",
"chars": 713,
"preview": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"context\"\n\n\t\"github.com/prometheus/prometheus/model/"
},
{
"path": "prometheus/gatherer.go",
"chars": 333,
"preview": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"github.com/prometheus/client_golang/prometheus\"\n\tdt"
},
{
"path": "prometheus/labels.go",
"chars": 815,
"preview": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"net/url\"\n\t\"sort\"\n\t\"strings\"\n\n\t\"github.com/prometheu"
},
{
"path": "prometheus/labels_test.go",
"chars": 652,
"preview": "package prometheus\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestLabels(t *testing.T) {\n\tasser"
},
{
"path": "prometheus/local_storage.go",
"chars": 832,
"preview": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"context\"\n\n\t\"github.com/prometheus/prometheus/model/"
},
{
"path": "prometheus/logger.go",
"chars": 1162,
"preview": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"go.uber.org/zap\"\n)\n\ntype errorLevel interface {\n\tSt"
},
{
"path": "prometheus/matcher.go",
"chars": 3436,
"preview": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\n\t\"github.com/lomik/graphite-clickhous"
},
{
"path": "prometheus/metrics_set.go",
"chars": 1194,
"preview": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"github.com/prometheus/prometheus/model/labels\"\n\t\"gi"
},
{
"path": "prometheus/querier.go",
"chars": 3248,
"preview": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/pr"
},
{
"path": "prometheus/querier_select.go",
"chars": 3680,
"preview": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"context\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-click"
},
{
"path": "prometheus/querier_select_test.go",
"chars": 2417,
"preview": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-click"
},
{
"path": "prometheus/run.go",
"chars": 2603,
"preview": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"context\"\n\t\"log\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/g"
},
{
"path": "prometheus/run_dummy.go",
"chars": 169,
"preview": "//go:build noprom\n// +build noprom\n\npackage prometheus\n\nimport (\n\t\"github.com/lomik/graphite-clickhouse/config\"\n)\n\nfunc "
},
{
"path": "prometheus/series_set.go",
"chars": 5343,
"preview": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"log\"\n\t\"math\"\n\n\t\"github.com/prometheus/prometheus/ut"
},
{
"path": "prometheus/storage.go",
"chars": 942,
"preview": "//go:build !noprom\n// +build !noprom\n\npackage prometheus\n\nimport (\n\t\"context\"\n\n\t\"github.com/lomik/graphite-clickhouse/co"
},
{
"path": "render/data/carbonlink.go",
"chars": 2045,
"preview": "package data\n\nimport (\n\t\"context\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/lomik/graphite-cl"
},
{
"path": "render/data/carbonlink_test.go",
"chars": 2625,
"preview": "package data\n\nimport (\n\t\"context\"\n\t\"sort\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.co"
},
{
"path": "render/data/ch_response.go",
"chars": 6039,
"preview": "package data\n\nimport (\n\t\"errors\"\n\t\"math\"\n\n\tv2pb \"github.com/go-graphite/protocol/carbonapi_v2_pb\"\n\tv3pb \"github.com/go-g"
},
{
"path": "render/data/common_step.go",
"chars": 1314,
"preview": "package data\n\nimport (\n\t\"context\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/pkg/dry\"\n)\n\n// This is used to"
},
{
"path": "render/data/common_step_test.go",
"chars": 1354,
"preview": "package data\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\ntype wrapper struct {\n\t*commonStep\n\t"
},
{
"path": "render/data/data.go",
"chars": 9391,
"preview": "package data\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"encoding/binary\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.c"
},
{
"path": "render/data/data_parse_test.go",
"chars": 13304,
"preview": "package data\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/helpe"
},
{
"path": "render/data/multi_target.go",
"chars": 5843,
"preview": "package data\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"sync\"\n\t\"time\"\n\n\tv3pb \"github.com/go-graphite/protocol/carbonapi_v"
},
{
"path": "render/data/multi_target_test.go",
"chars": 3141,
"preview": "package data\n\nimport (\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n)\n\nfunc Test_getDataTimeout(t "
},
{
"path": "render/data/query.go",
"chars": 12276,
"preview": "package data\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"os\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"t"
},
{
"path": "render/data/query_test.go",
"chars": 16559,
"preview": "package data\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\tv3pb \"github.com/go-graphite/protocol/carb"
},
{
"path": "render/data/targets.go",
"chars": 4775,
"preview": "package data\n\nimport (\n\t\"fmt\"\n\t\"time\"\n\n\tv3pb \"github.com/go-graphite/protocol/carbonapi_v3_pb\"\n\t\"github.com/lomik/graphi"
},
{
"path": "render/data/targets_test.go",
"chars": 3226,
"preview": "package data\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n\t\"github.com/stretchr/t"
},
{
"path": "render/handler.go",
"chars": 10667,
"preview": "package render\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"go.uber.org/zap\"\n\n\t\"github.com/go-g"
},
{
"path": "render/handler_test.go",
"chars": 1935,
"preview": "package render\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhouse/config\"\n)\n\nfunc Test_getCache"
},
{
"path": "render/reply/formatter.go",
"chars": 2314,
"preview": "package reply\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"net/http\"\n\t\"strconv\"\n\n\t\"github.com/lomik/graphite-clickhouse/pkg/alias\"\n\t\"githu"
},
{
"path": "render/reply/formatter_test.go",
"chars": 6373,
"preview": "package reply\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"sort\"\n\t\"testing\"\n\n\t\"github.co"
},
{
"path": "render/reply/json.go",
"chars": 3541,
"preview": "package reply\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"net/http\"\n\n\tv3pb \"github.com/go-graphite/pr"
},
{
"path": "render/reply/pickle.go",
"chars": 3460,
"preview": "package reply\n\nimport (\n\t\"bufio\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"net/http\"\n\t\"time\"\n\n\tgraphitePickle \"github.com/lomik/graphit"
},
{
"path": "render/reply/protobuf.go",
"chars": 4072,
"preview": "package reply\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"net/http\"\n\n\t\"go.uber.org/zap\"\n\n\t\"github.com/lomik/graph"
},
{
"path": "render/reply/protobuf_test.go",
"chars": 267,
"preview": "package reply\n\nimport (\n\t\"encoding/binary\"\n\t\"testing\"\n)\n\nfunc TestVarintLen(t *testing.T) {\n\tbuf := make([]byte, binary."
},
{
"path": "render/reply/v2_pb.go",
"chars": 2712,
"preview": "package reply\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"net/http\"\n\n\t\"github.com/lomik/graphite-clickhouse/h"
},
{
"path": "render/reply/v2_pb_test.go",
"chars": 2813,
"preview": "package reply\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"math\"\n\t\"reflect\"\n\t\"testing\"\n\n\tv2pb \"github.com/go-graphite/protocol/carbonap"
},
{
"path": "render/reply/v3_pb.go",
"chars": 4435,
"preview": "package reply\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\n\tv3pb \"github.com/go-grap"
},
{
"path": "render/reply/v3_pb_test.go",
"chars": 3373,
"preview": "package reply\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"math\"\n\t\"reflect\"\n\t\"testing\"\n\n\tv3pb \"github.com/go-graphite/protocol/carbonap"
},
{
"path": "sd/nginx/nginx.go",
"chars": 8965,
"preview": "package nginx\n\nimport (\n\t\"encoding/base64\"\n\t\"errors\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/lomik/graphite-clickhou"
}
]
// ... and 93 more files (download for full content)
About this extraction
This page contains the full source code of the lomik/graphite-clickhouse GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 293 files (1.3 MB), approximately 455.3k tokens, and a symbol index with 1236 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.