Showing preview only (305K chars total). Download the full file or copy to clipboard to get everything.
Repository: PostHog/HouseWatch
Branch: main
Commit: 9b126c0d588c
Files: 136
Total size: 272.5 KB
Directory structure:
gitextract_43962rdt/
├── .github/
│ └── workflows/
│ ├── docker-api.yaml
│ ├── docker-frontend.yaml
│ └── release-chart.yaml
├── .gitignore
├── .pre-commit-config.yaml
├── Dockerfile
├── LICENSE
├── README.md
├── bin/
│ ├── celery
│ ├── docker
│ ├── migrate
│ ├── serve
│ └── start
├── charts/
│ └── housewatch/
│ ├── .helmignore
│ ├── Chart.yaml
│ ├── templates/
│ │ ├── _helpers.tpl
│ │ ├── deployment-nginx.yaml
│ │ ├── deployment-web.yaml
│ │ ├── deployment-worker.yaml
│ │ ├── nginx-configmap.yaml
│ │ ├── redis.yaml
│ │ └── service.yaml
│ └── values.yaml
├── docker/
│ ├── Caddyfile
│ └── clickhouse-server/
│ └── config.d/
│ └── config.xml
├── docker-compose.dev.yml
├── docker-compose.yml
├── frontend/
│ ├── .gitignore
│ ├── .node-version
│ ├── .prettierrc
│ ├── Dockerfile
│ ├── Dockerfile.dev
│ ├── README.md
│ ├── index.html
│ ├── package.json
│ ├── public/
│ │ ├── fonts/
│ │ │ └── OFL.txt
│ │ ├── index.html
│ │ ├── manifest.json
│ │ └── robots.txt
│ ├── src/
│ │ ├── App.tsx
│ │ ├── Layout.tsx
│ │ ├── index.css
│ │ ├── index.tsx
│ │ ├── pages/
│ │ │ ├── AITools/
│ │ │ │ ├── AIToolsPage.tsx
│ │ │ │ └── NaturalLanguageQueryEditor.tsx
│ │ │ ├── Backups/
│ │ │ │ ├── Backups.tsx
│ │ │ │ └── ScheduledBackups.tsx
│ │ │ ├── Clusters/
│ │ │ │ └── Clusters.tsx
│ │ │ ├── DiskUsage/
│ │ │ │ └── DiskUsage.tsx
│ │ │ ├── Errors/
│ │ │ │ └── Errors.tsx
│ │ │ ├── Logs/
│ │ │ │ └── Logs.tsx
│ │ │ ├── Operations/
│ │ │ │ └── Operations.tsx
│ │ │ ├── Overview/
│ │ │ │ ├── Overview.tsx
│ │ │ │ └── tips.ts
│ │ │ ├── QueryEditor/
│ │ │ │ ├── Benchmark.tsx
│ │ │ │ ├── QueryEditor.tsx
│ │ │ │ ├── QueryEditorPage.tsx
│ │ │ │ ├── SavedQueries.tsx
│ │ │ │ └── SavedQuery.tsx
│ │ │ ├── RunningQueries/
│ │ │ │ └── RunningQueries.tsx
│ │ │ ├── SchemaStats/
│ │ │ │ ├── SchemaStats.tsx
│ │ │ │ └── SchemaTable.tsx
│ │ │ └── SlowQueries/
│ │ │ ├── ExampleQueriesTab.tsx
│ │ │ ├── ExplainTab.tsx
│ │ │ ├── MetricsTab.tsx
│ │ │ ├── NormalizedQueryTab.tsx
│ │ │ ├── QueryDetail.tsx
│ │ │ └── SlowQueries.tsx
│ │ ├── react-app-env.d.ts
│ │ └── utils/
│ │ ├── dateUtils.ts
│ │ └── usePollingEffect.tsx
│ ├── tsconfig.json
│ └── vite.config.ts
├── housewatch/
│ ├── __init__.py
│ ├── admin/
│ │ └── __init__.py
│ ├── admin.py
│ ├── ai/
│ │ └── templates.py
│ ├── api/
│ │ ├── __init__.py
│ │ ├── analyze.py
│ │ ├── async_migration.py
│ │ ├── backups.py
│ │ ├── cluster.py
│ │ ├── instance.py
│ │ └── saved_queries.py
│ ├── apps.py
│ ├── asgi.py
│ ├── async_migrations/
│ │ ├── __init__.py
│ │ ├── async_migration_utils.py
│ │ └── runner.py
│ ├── celery.py
│ ├── clickhouse/
│ │ ├── __init__.py
│ │ ├── backups.py
│ │ ├── client.py
│ │ ├── clusters.py
│ │ ├── queries/
│ │ │ ├── __init__.py
│ │ │ └── sql.py
│ │ └── table.py
│ ├── gunicorn.conf.py
│ ├── migrations/
│ │ ├── 0001_initial.py
│ │ ├── 0002_asyncmigration_asyncmigration_unique name.py
│ │ ├── 0003_asyncmigration_operations_and_more.py
│ │ ├── 0004_asyncmigration_last_error.py
│ │ ├── 0005_asyncmigration_finished_at.py
│ │ ├── 0006_savedquery.py
│ │ ├── 0007_scheduledbackup_scheduledbackuprun.py
│ │ ├── 0008_remove_scheduledbackup_aws_endpoint_url_and_more.py
│ │ ├── 0009_scheduledbackup_cluster_alter_scheduledbackup_id.py
│ │ ├── 0010_scheduledbackup_incremental_schedule_and_more.py
│ │ ├── 0011_scheduledbackup_is_sharded_and_more.py
│ │ ├── 0012_preferredreplica.py
│ │ └── __init__.py
│ ├── models/
│ │ ├── __init__.py
│ │ ├── async_migration.py
│ │ ├── backup.py
│ │ ├── instance.py
│ │ ├── preferred_replica.py
│ │ └── saved_queries.py
│ ├── settings/
│ │ ├── __init__.py
│ │ └── utils.py
│ ├── tasks/
│ │ └── __init__.py
│ ├── tests/
│ │ └── test_backup_table_fixture.sql
│ ├── urls.py
│ ├── utils/
│ │ ├── __init__.py
│ │ └── encrypted_fields/
│ │ ├── fields.py
│ │ └── hkdf.py
│ ├── views.py
│ └── wsgi.py
├── manage.py
├── mypy.ini
├── pyproject.toml
├── pytest.ini
├── requirements-dev.in
├── requirements-dev.txt
├── requirements.in
├── requirements.txt
└── runtime.txt
================================================
FILE CONTENTS
================================================
================================================
FILE: .github/workflows/docker-api.yaml
================================================
name: API Docker build
on:
push:
jobs:
docker:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
-
uses: actions/checkout@v3
-
name: Set up QEMU
uses: docker/setup-qemu-action@v2
with:
platforms: arm64
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
-
name: Docker image metadata
id: meta
uses: docker/metadata-action@v4
with:
images: ghcr.io/${{ github.repository }}/api
tags: |
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
type=raw,value=main,enable=${{ github.ref == 'refs/heads/main' }}
type=ref,event=pr
type=sha
-
name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
-
name: Build and push
uses: docker/build-push-action@v4
with:
context: .
platforms: linux/amd64,linux/arm64
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
================================================
FILE: .github/workflows/docker-frontend.yaml
================================================
name: Frontend Docker build
on:
push:
jobs:
docker:
runs-on: ubuntu-latest
defaults:
run:
working-directory: "frontend"
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
with:
platforms: arm64
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Docker image metadata
id: meta
uses: docker/metadata-action@v4
with:
images: ghcr.io/${{ github.repository }}/frontend
tags: |
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
type=raw,value=main,enable=${{ github.ref == 'refs/heads/main' }}
type=ref,event=pr
type=sha
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- uses: pnpm/action-setup@v2
with:
version: 8.6.12
- name: Install Node.js
uses: actions/setup-node@v4
with:
node-version-file: frontend/.node-version
cache: "pnpm"
cache-dependency-path: frontend/pnpm-lock.yaml
- name: Install dependencies
run: pnpm i --frozen-lockfile
- name: Build
run: pnpm build
- name: Build and push
uses: docker/build-push-action@v4
with:
context: frontend/
platforms: linux/amd64,linux/arm64
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
================================================
FILE: .github/workflows/release-chart.yaml
================================================
name: Release Chart
on:
pull_request:
paths:
- charts/**
- .github/workflows/release-chart.yaml
push:
branches:
- main
paths:
- charts/**
- .github/workflows/release-chart.yaml
jobs:
release:
name: Release chart to repo
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install Helm
uses: azure/setup-helm@v1
with:
version: v3.12.2
- name: Add -<sha> to version in charts/housewatch/Chart.yaml and update Chart.lock
if: github.ref != 'refs/heads/main'
run: |
sed -i 's/^version: \(.*\)$/version: \1-${{ github.sha }}/g' charts/housewatch/Chart.yaml
- name: Configure Git
run: |
git config user.name "Max Hedgehog"
git config user.email "127861667+max-hedgehog[bot]@users.noreply.github.com"
git fetch origin gh-pages --depth=1
- name: Package
run: |
helm dependency update charts/housewatch/
mkdir -p .cr-release-packages
cd .cr-release-packages
helm package ../charts/housewatch
cd -
set -x
- name: Run chart-releaser
uses: helm/chart-releaser-action@4b85f2c82c80ff4284ff8520f47bbe69dd89b0aa
if: github.ref == 'refs/heads/main' && github.repository == 'PostHog/HouseWatch'
env:
CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
with:
skip_existing: true
skip_packaging: true
================================================
FILE: .gitignore
================================================
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/#use-with-ide
.pdm.toml
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
env/
__pycache__/
node_modules
housewatch.sqlite3
.DS_Store
yarn.lock
charts/housewatch/charts/
.pnpm/
================================================
FILE: .pre-commit-config.yaml
================================================
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.3.0
hooks:
- id: check-yaml
- id: end-of-file-fixer
- id: trailing-whitespace
- repo: https://github.com/astral-sh/ruff-pre-commit
# Ruff version.
rev: v0.0.275
hooks:
- id: ruff
args: [--fix, --exit-non-zero-on-fix]
- repo: https://github.com/psf/black
rev: 22.10.0
hooks:
- id: black
================================================
FILE: Dockerfile
================================================
FROM python:3.10
ENV PYTHONUNBUFFERED 1
WORKDIR /code
COPY requirements.txt ./
RUN apt-get update && \
apt-get install -y --no-install-recommends \
"build-essential" \
"git" \
"libpq-dev" \
"libxmlsec1" \
"libxmlsec1-dev" \
"libffi-dev" \
"pkg-config" \
&& \
rm -rf /var/lib/apt/lists/* && \
pip install -r requirements.txt --compile --no-cache-dir
USER root
COPY manage.py manage.py
COPY housewatch housewatch/
COPY bin bin/
RUN DEBUG=1 python manage.py collectstatic --noinput
================================================
FILE: LICENSE
================================================
MIT License
Copyright (c) 2023 PostHog
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: README.md
================================================
<p align="center">
<img src="./banner-light.png">
</p>
<p align="center">
<img src="./overview.png">
</p>
## 📈 Open source tool for monitoring and managing ClickHouse clusters
- Get an overview of cluster load and performance
- Drill down into your queries and understand the load they put on your cluster
- Search through logs and errors
- Monitor and kill running queries with the click of a button
- Get stats on your disk usage per node, and understand how much disk space tables, columns, and parts take up
- Run your own queries straight from the interface to further dig into performance and cluster issues
- Setup operations to run in the background with automatic rollbacks for failures
## 💻 Deploy
To deploy HouseWatch, clone this repo and then run the following, substituting the environment variables for the relevant values of one of your ClickHouse instances:
```bash
SITE_ADDRESS=<SITE_ADDRESS> \
CLICKHOUSE_HOST=localhost \
CLICKHOUSE_CLUSTER=mycluster \
CLICKHOUSE_USER=default \
CLICKHOUSE_PASSWORD=xxxxxxxxxxx \
docker compose -f docker-compose.yml up
```
`SITE_ADDRESS` here is the address that the UI will be running on. It can be a domain name or simply a port like `:80`.
After running the above, the UI will be running on the address you specified. This will be something like http://localhost if you used `:80` for your `SITE_ADDRESS` above. I would think twice about exposing this to the internet, as it is not currently secured in any way.
<details>
<summary>Read more</summary>
<br />
The following are the supported environment variables for configuring your HouseWatch deployment:
- `CLICKHOUSE_HOST`: Required - hostname of the instance to connect to.
- `CLICKHOUSE_USER`: Required - username to access ClickHouse. Can be a read-only user, but in that case not all features will work.
- `CLICKHOUSE_PASSWORD`: Required - password for the specified user.
- `CLICKHOUSE_DATABASE`: Optional - database to connect to by default.
- `CLICKHOUSE_CLUSTER`: Optional - cluster name, to analyze data from the whole cluster.
- `CLICKHOUSE_SECURE`: Optional - see [clickhouse-driver docs](https://clickhouse-driver.readthedocs.io/en/latest/index.html) for more information
- `CLICKHOUSE_VERIFY`: Optional - see [clickhouse-driver docs](https://clickhouse-driver.readthedocs.io/en/latest/index.html) for more information
- `CLICKHOUSE_CA`: Optional - see [clickhouse-driver docs](https://clickhouse-driver.readthedocs.io/en/latest/index.html) for more information
- `OPENAI_API_KEY`: Optional - enables the experimental "AI Tools" page, which currently features a natural language query editor
- `OPENAI_MODEL`: Optional - a valid OpenAI model (e.g. `gpt-3.5-turbo`, `gpt-4`) that you have access to with the key above to be used for the AI features
</details>
## 🏡 Running locally
To run HouseWatch locally along with a local ClickHouse instance, execute:
```bash
docker compose -f docker-compose.dev.yml up -d
```
then go to http://localhost:8080
## 💡 Motivation
At PostHog we manage a few large ClickHouse clusters and found ourselves in need of a tool to monitor and manage these more easily.
ClickHouse is fantastic at introspection, providing a lot of metadata about the system in its system tables so that it can be easily queried. However, knowing exactly how to query and parse the available information can be a difficult task. Over the years at PostHog, we've developed great intuition for how to debug ClickHouse issues using ClickHouse, and HouseWatch is the compilation of this knowledge into a tool.
Beyond monitoring, we also built internal systems and processes for managing the clusters that spanned various platforms. We would use Grafana to look at metrics, SSH into nodes for running operations and using specialized tooling, query via Metabase to dig deeper into the data in the system tables and create dashboards, and then a combination of tools baked into the PostHog product for further debugging and streamlined operations such as our [async migrations](https://posthog.com/blog/async-migrations) tool, and internal views for listing queries and analyzing their performance.
As a result, we felt it was appropriate to have these tools live in one place. Ultimately, our vision for HouseWatch is that it can both serve the purpose of a pganalyze for the ClickHouse ecosystem, while also including tooling for taking action on insights derived from the analysis.
## 🏗️ Status of the project
HouseWatch is in its early days and we have a lot more features in mind that we'd like to build into it going forward. The code could also use some cleaning up :) As of right now, it is considered Beta software and you should exercise caution when using it in production.
One potential approach is to connect HouseWatch to ClickHouse using a read-only user. In this case, the cluster management features will not work (e.g. operations, query editor), but the analysis toolset will function normally.
HouseWatch was created and is maintained by [PostHog](https://posthog.com) and [yakkomajuri](https://github.com/yakkomajuri).
## ℹ️ Contributing
Contributions are certainly welcome! However, if you'd like to build a new feature, please open up an issue first.
## ⭐ Features
<h2 align="center">Query performance</h3>
<div style="display: flex">
<img src="./slow-queries.png" width="48%">
<img src="./normalized-query.png" width="48%">
</div>
<div style="display: flex">
<img src="./query-stats.png" width="48%">
<img src="./explain.png" width="48%">
</div>
<br />
<h2 align="center">Schema stats</h3>
<div style="display: flex">
<img src="./schema.png" width="48%">
<img src="./schema-drilldown.png" width="48%">
</div>
<br />
<h2 align="center">Query benchmarking</h3>
<div style="display: flex">
<img src="./benchmark1.png" width="48%">
<img src="./benchmark2.png" width="48%">
</div>
<br />
<h2 align="center">Logs</h3>
<p align="center">
<img src="./logs.png" align="center">
</p>
<br />
<h2 align="center">Query editor</h3>
<p align="center">
<img src="./query-editor.png">
</p>
<br />
<h2 align="center">Disk usage</h3>
<p align="center">
<img src="./disk-usage.png">
</p>
<br />
<h2 align="center">Errors</h3>
<p align="center">
<img src="./errors.png">
</p>
<br />
<h2 align="center">Operations</h3>
<p align="center">
<img src="./operations.png">
</p>
## 🗒️ To-do list
A public list of things we intend to do with HouseWatch in the near future.
<details>
<summary>See list</summary>
<br />
<b>Features</b>
- [ ] System issues tab
- [ ] EXPLAIN visualizer
- [ ] Multiple instance support
- [ ] Stats on page cache hit percentage
- [ ] Make operations resilient to Celery going down (as we do in PostHog with async migrations)
- [ ] Read-only mode
- [ ] Button to force refresh running queries list
- [ ] Logs pagination
- [ ] Allow copying example queries
- [ ] Configurable time ranges
- [ ] Whole cluster schema stats
- [ ] More operation controls: view, delete, edit, re-run, display errors
<b>Developer experience</b>
- [ ] Configure instance from UI
- [ ] Publish a Docker image
- [ ] Development docker-compose.yml with baked in ClickHouse
<b>Cleanup</b>
- [ ] Extract README images out of repo
- [ ] Make banner subtitle work on dark mode
- [ ] Fetch data independently on the query analyzer
- [ ] Breakpoint for logs search
- [ ] Run Django "production server"
- [ ] Write tests :)
- [ ] Query editor pipe all errors to client
- [ ] Abstraction to load data from API as JSON
</details>
================================================
FILE: bin/celery
================================================
#!/bin/bash
set -e
celery -A housewatch worker -B
================================================
FILE: bin/docker
================================================
#!/bin/bash
set -e
./bin/migrate
./bin/serve
================================================
FILE: bin/migrate
================================================
#!/bin/bash
set -e
python manage.py migrate
================================================
FILE: bin/serve
================================================
#!/bin/bash
exec gunicorn housewatch.wsgi -c housewatch/gunicorn.conf.py \
--worker-tmp-dir /dev/shm
================================================
FILE: bin/start
================================================
#!/bin/bash
set -e
export DEBUG=1
./bin/celery & python manage.py runserver
================================================
FILE: charts/housewatch/.helmignore
================================================
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
================================================
FILE: charts/housewatch/Chart.yaml
================================================
apiVersion: v2
name: housewatch
description: Open source tool for monitoring and managing ClickHouse clusters
type: application
version: 0.1.9
appVersion: "0.1.2"
dependencies:
- name: postgresql
version: "12.10.0"
repository: "oci://registry-1.docker.io/bitnamicharts"
- name: rabbitmq
version: "12.1.3"
repository: "oci://registry-1.docker.io/bitnamicharts"
================================================
FILE: charts/housewatch/templates/_helpers.tpl
================================================
{{/*
Expand the name of the chart.
*/}}
{{- define "housewatch.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "housewatch.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "housewatch.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "housewatch.labels" -}}
helm.sh/chart: {{ include "housewatch.chart" . }}
{{ include "housewatch.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "housewatch.selectorLabels" -}}
app.kubernetes.io/name: {{ include "housewatch.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "housewatch.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "housewatch.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}
================================================
FILE: charts/housewatch/templates/deployment-nginx.yaml
================================================
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "housewatch.fullname" . }}-nginx
labels:
{{- include "housewatch.labels" . | nindent 4 }}
spec:
selector:
matchLabels:
{{- include "housewatch.selectorLabels" . | nindent 6 }}
app.kubernetes.io/service: nginx
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "housewatch.selectorLabels" . | nindent 8 }}
app.kubernetes.io/service: nginx
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
volumes:
- name: frontend
emptyDir: {}
- name: nginx-config
configMap:
name: {{ include "housewatch.fullname" . }}-nginx
initContainers:
- name: frontend-copy
image: "{{ .Values.image.frontendRepository }}:{{ .Values.image.tag }}"
command: [sh, -cex]
args:
- cp -r /frontend/build/* /http/
volumeMounts:
- mountPath: /http
name: frontend
containers:
- name: nginx
image: "{{ .Values.nginx.image.repository }}:{{ .Values.nginx.image.tag }}"
ports:
- name: http
containerPort: 80
protocol: TCP
volumeMounts:
- mountPath: /http
name: frontend
- mountPath: /etc/nginx/nginx.conf
name: nginx-config
subPath: nginx.conf
resources:
{{- toYaml .Values.nginx.resources | nindent 12 }}
================================================
FILE: charts/housewatch/templates/deployment-web.yaml
================================================
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "housewatch.fullname" . }}
labels:
{{- include "housewatch.labels" . | nindent 4 }}
spec:
selector:
matchLabels:
{{- include "housewatch.selectorLabels" . | nindent 6 }}
app.kubernetes.io/service: web
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "housewatch.selectorLabels" . | nindent 8 }}
app.kubernetes.io/service: web
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: web
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
command: ["bash", "-c"]
args:
- |
python manage.py migrate
python manage.py runserver 0.0.0.0:8000
ports:
- name: api
containerPort: 8000
protocol: TCP
env:
- name: REDIS_URL
value: redis://{{ include "housewatch.fullname" . }}-redis:6379
- name: CLICKHOUSE_HOST
value: "{{ .Values.clickhouse.host }}"
- name: CLICKHOUSE_DATABASE
value: "{{ .Values.clickhouse.database }}"
- name: CLICKHOUSE_USER
value: "{{ .Values.clickhouse.user }}"
- name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: "{{ .Values.clickhouse.secretName }}"
key: "{{ .Values.clickhouse.secretPasswordKey }}"
- name: CLICKHOUSE_CLUSTER
value: {{ .Values.clickhouse.cluster }}
- name: CLICKHOUSE_SECURE
value: "{{ .Values.clickhouse.secure }}"
- name: CLICKHOUSE_VERIFY
value: "{{ .Values.clickhouse.verify }}"
- name: CLICKHOUSE_CA
value: "{{ .Values.clickhouse.ca }}"
- name: DATABASE_URL
value: "{{ . | tpl .Values.database_url }}"
- name: RABBITMQ_URL
value: "amqp://{{ .Values.rabbitmq.auth.username }}:{{ .Values.rabbitmq.auth.password }}@{{ .Release.Name }}-rabbitmq:5672/"
livenessProbe:
httpGet:
path: /
port: api
readinessProbe:
httpGet:
path: /
port: api
resources:
{{- toYaml .Values.web.resources | nindent 12 }}
================================================
FILE: charts/housewatch/templates/deployment-worker.yaml
================================================
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "housewatch.fullname" . }}-worker
labels:
{{- include "housewatch.labels" . | nindent 4 }}
spec:
selector:
matchLabels:
{{- include "housewatch.selectorLabels" . | nindent 6 }}
app.kubernetes.io/service: worker
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "housewatch.selectorLabels" . | nindent 8 }}
app.kubernetes.io/service: worker
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: worker
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
command: ["./bin/celery"]
env:
- name: REDIS_URL
value: redis://{{ include "housewatch.fullname" . }}-redis:6379
- name: CLICKHOUSE_HOST
value: "{{ .Values.clickhouse.host }}"
- name: CLICKHOUSE_DATABASE
value: "{{ .Values.clickhouse.database }}"
- name: CLICKHOUSE_USER
value: "{{ .Values.clickhouse.user }}"
- name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: "{{ .Values.clickhouse.secretName }}"
key: "{{ .Values.clickhouse.secretPasswordKey }}"
- name: CLICKHOUSE_CLUSTER
value: "{{ .Values.clickhouse.cluster }}"
- name: CLICKHOUSE_SECURE
value: "{{ .Values.clickhouse.secure }}"
- name: CLICKHOUSE_VERIFY
value: "{{ .Values.clickhouse.verify }}"
- name: CLICKHOUSE_CA
value: "{{ .Values.clickhouse.ca }}"
- name: DATABASE_URL
value: "{{ . | tpl .Values.database_url }}"
- name: RABBITMQ_URL
value: "amqp://{{ .Values.rabbitmq.auth.username }}:{{ .Values.rabbitmq.auth.password }}@{{ .Release.Name }}-rabbitmq:5672/"
resources:
{{- toYaml .Values.worker.resources | nindent 12 }}
================================================
FILE: charts/housewatch/templates/nginx-configmap.yaml
================================================
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "housewatch.fullname" . }}-nginx
data:
nginx.conf: |
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
sendfile on;
keepalive_timeout 65;
server {
listen 80;
location / {
root /http;
try_files $uri $uri/ /index.html =404;
}
location /api {
proxy_pass http://{{ include "housewatch.fullname" . }}-api:8000;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
}
location /admin {
proxy_pass http://{{ include "housewatch.fullname" . }}-api:8000;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
}
location /healthz {
proxy_pass http://{{ include "housewatch.fullname" . }}-api:8000;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
}
}
}
================================================
FILE: charts/housewatch/templates/redis.yaml
================================================
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "housewatch.fullname" . }}-redis
labels:
{{- include "housewatch.labels" . | nindent 4 }}
spec:
replicas: 1
selector:
matchLabels:
{{- include "housewatch.selectorLabels" . | nindent 6 }}
app.kubernetes.io/service: redis
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "housewatch.selectorLabels" . | nindent 8 }}
app.kubernetes.io/service: redis
spec:
containers:
- name: redis
image: "{{ .Values.redis.image.repository }}:{{ .Values.redis.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
ports:
- name: redis
containerPort: 6379
protocol: TCP
resources:
{{- toYaml .Values.redis.resources | nindent 12 }}
================================================
FILE: charts/housewatch/templates/service.yaml
================================================
apiVersion: v1
kind: Service
metadata:
name: {{ include "housewatch.fullname" . }}
annotations:
{{- toYaml .Values.service.annotations | nindent 4 }}
labels:
{{- include "housewatch.labels" . | nindent 4 }}
spec:
type: ClusterIP
ports:
- port: 80
targetPort: http
protocol: TCP
name: http
selector:
{{- include "housewatch.selectorLabels" . | nindent 4 }}
app.kubernetes.io/service: nginx
---
apiVersion: v1
kind: Service
metadata:
name: {{ include "housewatch.fullname" . }}-api
labels:
{{- include "housewatch.labels" . | nindent 4 }}
spec:
type: ClusterIP
ports:
- port: 8000
targetPort: api
protocol: TCP
name: api
selector:
{{- include "housewatch.selectorLabels" . | nindent 4 }}
app.kubernetes.io/service: web
---
apiVersion: v1
kind: Service
metadata:
name: {{ include "housewatch.fullname" . }}-redis
labels:
{{- include "housewatch.labels" . | nindent 4 }}
spec:
type: ClusterIP
ports:
- port: 6379
targetPort: 6379
protocol: TCP
name: redis
selector:
{{- include "housewatch.selectorLabels" . | nindent 4 }}
app.kubernetes.io/service: redis
================================================
FILE: charts/housewatch/values.yaml
================================================
image:
repository: ghcr.io/posthog/housewatch/api
frontendRepository: ghcr.io/posthog/housewatch/frontend
tag: main
nginx:
image:
repository: nginx
tag: stable
clickhouse:
user: default
host: clickhouse
database: default
secure: "false"
verify: "false"
ca: ""
service:
annotations: {}
web:
resources:
requests:
cpu: 100m
memory: 500Mi
limits:
memory: 500Mi
frontend:
resources:
requests:
cpu: 500m
memory: 2Gi
limits:
memory: 2Gi
worker:
resources:
requests:
cpu: 100m
memory: 1500Mi
limits:
memory: 1500Mi
database_url: postgres://housewatch:housewatch@{{ include "postgresql.primary.fullname" .Subcharts.postgresql }}:5432/housewatch
postgresql:
auth:
database: housewatch
username: housewatch
password: housewatch
rabbitmq:
auth:
username: housewatch
password: housewatch
erlangCookie: housewatch
redis:
image:
repository: redis
tag: 6.2.7-alpine
resources:
requests:
cpu: 100m
memory: 1Gi
limits:
memory: 1Gi
================================================
FILE: docker/Caddyfile
================================================
{
debug
}
{$SITE_ADDRESS} {
reverse_proxy web:3000
reverse_proxy /api/* app:8000
reverse_proxy /logout app:8000
reverse_proxy /admin/ app:8000
}
================================================
FILE: docker/clickhouse-server/config.d/config.xml
================================================
<clickhouse>
<!-- Listen wildcard address to allow accepting connections from other containers and host
network. -->
<listen_host>::</listen_host>
<listen_host>0.0.0.0</listen_host>
<listen_try>1</listen_try>
<!--
<logger>
<console>1</console>
</logger>
-->
<remote_servers>
<housewatch>
<shard>
<replica>
<host>clickhouse</host>
<port>9000</port>
</replica>
</shard>
</housewatch>
</remote_servers>
</clickhouse>
================================================
FILE: docker-compose.dev.yml
================================================
version: "3"
services:
app:
build: .
environment: &django_env
DEBUG: 1
REDIS_URL: redis://redis:6379
RABBITMQ_URL: amqp://posthog:posthog@rabbitmq:5672
DATABASE_URL: postgres://housewatch:housewatch@db:5432/housewatch
CLICKHOUSE_HOST: clickhouse
CLICKHOUSE_DATABASE: default
CLICKHOUSE_USER: default
CLICKHOUSE_PASSWORD: ""
CLICKHOUSE_CLUSTER: housewatch
CLICKHOUSE_SECURE: false
CLICKHOUSE_VERIFY: false
CLICKHOUSE_CA: ""
AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION
command:
- bash
- -c
- |
python manage.py runserver 0.0.0.0:8000
ports:
- "8000:8000"
volumes:
- .:/code
depends_on:
migrations:
condition: service_completed_successfully
clickhouse:
condition: service_started
db:
condition: service_healthy
redis:
condition: service_started
rabbitmq:
condition: service_started
migrations:
build: .
environment: *django_env
command: python manage.py migrate
depends_on:
db:
condition: service_healthy
volumes:
- .:/code
web:
build:
context: ./frontend
dockerfile: Dockerfile.dev
caddy:
image: caddy:2.6.1
restart: unless-stopped
ports:
- "8080:8080"
- "443:443"
environment:
SITE_ADDRESS: ":8080"
volumes:
- ./docker/Caddyfile:/etc/caddy/Caddyfile
depends_on:
- web
- app
db:
image: postgres:16-alpine
restart: on-failure
environment:
POSTGRES_USER: housewatch
POSTGRES_DB: housewatch
POSTGRES_PASSWORD: housewatch
healthcheck:
test: ["CMD-SHELL", "pg_isready -U housewatch"]
interval: 5s
timeout: 5s
ports:
- "5432:5432"
redis:
image: redis:6.2.7-alpine
restart: on-failure
command: redis-server --maxmemory-policy allkeys-lru --maxmemory 200mb
worker:
build: .
environment:
<<: *django_env
command:
- ./bin/celery
volumes:
- .:/code
depends_on:
- clickhouse
- db
- redis
- rabbitmq
clickhouse:
image: ${CLICKHOUSE_SERVER_IMAGE:-clickhouse/clickhouse-server:23.12.5}
restart: on-failure
depends_on:
- zookeeper
volumes:
- ./docker/clickhouse-server/config.d:/etc/clickhouse-server/config.d
ports:
- "8123:8123"
zookeeper:
image: zookeeper:3.7.0
restart: on-failure
rabbitmq:
image: rabbitmq:3.12.2-management-alpine
ports:
- "15672:15672" # Web management UI
- "5672:5672" # Default RabbitMQ broker port
environment:
RABBITMQ_DEFAULT_USER: posthog
RABBITMQ_DEFAULT_PASS: posthog
================================================
FILE: docker-compose.yml
================================================
version: "3"
services:
app:
build: .
environment: &django_env
DATABASE_URL: postgres://housewatch:housewatch@db:5432/housewatch
RABBITMQ_URL: amqp://posthog:posthog@rabbitmq:5672
REDIS_URL: redis://redis:6379
CLICKHOUSE_HOST: $CLICKHOUSE_HOST
CLICKHOUSE_DATABASE: $CLICKHOUSE_DATABASE
CLICKHOUSE_USER: $CLICKHOUSE_USER
CLICKHOUSE_PASSWORD: $CLICKHOUSE_PASSWORD
CLICKHOUSE_CLUSTER: $CLICKHOUSE_CLUSTER
CLICKHOUSE_SECURE: $CLICKHOUSE_SECURE
CLICKHOUSE_VERIFY: $CLICKHOUSE_VERIFY
CLICKHOUSE_CA: $CLICKHOUSE_CA
AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION
command:
- bash
- -c
- |
python manage.py migrate
python manage.py runserver 0.0.0.0:8000
volumes:
- .:/code
ports:
- "8000:8000"
depends_on:
- db
- redis
- rabbitmq
- clickhouse
web:
build: ./frontend
ports:
- "3000:3000"
worker:
build: .
environment:
<<: *django_env
command:
- ./bin/celery
volumes:
- .:/code
depends_on:
- db
- redis
- rabbitmq
- clickhouse
redis:
image: redis:6.2.7-alpine
restart: on-failure
ports:
- "6388:6379"
command: redis-server --maxmemory-policy allkeys-lru --maxmemory 200mb
db:
image: postgres:14-alpine
restart: on-failure
environment:
POSTGRES_USER: housewatch
POSTGRES_DB: housewatch
POSTGRES_PASSWORD: housewatch
caddy:
image: caddy:2.6.1
restart: unless-stopped
ports:
- "80:80"
- "443:443"
environment:
SITE_ADDRESS: $SITE_ADDRESS
volumes:
- ./docker/Caddyfile:/etc/caddy/Caddyfile
depends_on:
- web
- app
rabbitmq:
image: rabbitmq:3.12.2-management-alpine
ports:
- "15672:15672" # Web management UI
- "5672:5672" # Default RabbitMQ broker port
environment:
RABBITMQ_DEFAULT_USER: posthog
RABBITMQ_DEFAULT_PASS: posthog
================================================
FILE: frontend/.gitignore
================================================
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
# dependencies
/node_modules
/.pnp
.pnp.js
# testing
/coverage
# production
/build
# misc
.DS_Store
.env.local
.env.development.local
.env.test.local
.env.production.local
npm-debug.log*
yarn-debug.log*
yarn-error.log*
================================================
FILE: frontend/.node-version
================================================
20.4.0
================================================
FILE: frontend/.prettierrc
================================================
{
"trailingComma": "es5",
"tabWidth": 4,
"semi": false,
"singleQuote": true,
"printWidth": 120
}
================================================
FILE: frontend/Dockerfile
================================================
FROM alpine:latest
WORKDIR /frontend
COPY build/ build/
CMD ["echo", "Serve the files from /frontend/build, don't run this container directly"]
================================================
FILE: frontend/Dockerfile.dev
================================================
FROM node:20.4.0-alpine
ENV PNPM_HOME="/pnpm"
ENV PATH="$PNPM_HOME:$PATH"
RUN corepack enable
WORKDIR /frontend
COPY . .
RUN pnpm i
CMD ["pnpm", "vite", "--port", "3000", "--host"]
================================================
FILE: frontend/README.md
================================================
# Getting Started with Create React App
This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app).
## Available Scripts
In the project directory, you can run:
### `npm start`
Runs the app in the development mode.\
Open [http://localhost:3000](http://localhost:3000) to view it in the browser.
The page will reload if you make edits.\
You will also see any lint errors in the console.
### `npm test`
Launches the test runner in the interactive watch mode.\
See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information.
### `npm run build`
Builds the app for production to the `build` folder.\
It correctly bundles React in production mode and optimizes the build for the best performance.
The build is minified and the filenames include the hashes.\
Your app is ready to be deployed!
See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information.
### `npm run eject`
**Note: this is a one-way operation. Once you `eject`, you can’t go back!**
If you aren’t satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project.
Instead, it will copy all the configuration files and the transitive dependencies (webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you’re on your own.
You don’t have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn’t feel obligated to use this feature. However we understand that this tool wouldn’t be useful if you couldn’t customize it when you are ready for it.
## Learn More
You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started).
To learn React, check out the [React documentation](https://reactjs.org/).
================================================
FILE: frontend/index.html
================================================
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<link rel="icon" />
<link rel="icon" type="image/x-icon" href="/favicons/favicon.ico" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="theme-color" content="#000000" />
<meta name="description" content="Web site created using create-react-app" />
<title>HouseWatch</title>
</head>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>
<div id="root"></div>
<script type="module" src="/src/index.tsx"></script>
</body>
</html>
================================================
FILE: frontend/package.json
================================================
{
"name": "housewatch",
"version": "1.0.0-beta",
"dependencies": {
"@ant-design/charts": "^1.4.2",
"@ant-design/icons": "^5.2.0",
"@ant-design/plots": "^2.1.12",
"antd": "^5.13.2",
"prismjs": "^1.29.0",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-router-dom": "5.2.0",
"react-scripts": "^2.1.3",
"react-simple-code-editor": "^0.13.1",
"sql-formatter-plus": "^1.3.6",
"swr": "^2.2.4",
"typescript": "^4.9.5",
"uuid": "^9.0.0"
},
"devDependencies": {
"@types/jest": "^27.5.2",
"@types/lodash": "^4.14.192",
"@types/node": "^16.18.22",
"@types/react": "^18.2.21",
"@types/react-dom": "^18.2.7",
"@types/react-router-dom": "5.3.3",
"@types/uuid": "^9.0.1",
"@vitejs/plugin-react": "^4.2.1",
"vite": "^5.0.12"
},
"pnpm": {
"overrides": {
"node-forge": "1.3.2"
}
},
"scripts": {
"dev": "vite",
"build": "tsc && vite build",
"lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
"preview": "vite preview"
},
"eslintConfig": {
"extends": [
"react-app",
"react-app/jest"
]
},
"proxy": "http://localhost:8000",
"browserslist": {
"production": [
">0.2%",
"not dead",
"not op_mini all"
],
"development": [
"last 1 chrome version",
"last 1 firefox version",
"last 1 safari version"
]
},
"packageManager": "pnpm@9.15.9+sha512.68046141893c66fad01c079231128e9afb89ef87e2691d69e4d40eee228988295fd4682181bae55b58418c3a253bde65a505ec7c5f9403ece5cc3cd37dcf2531"
}
================================================
FILE: frontend/public/fonts/OFL.txt
================================================
Copyright (c) 2015 Indian Type Foundry (info@indiantypefoundry.com)
This Font Software is licensed under the SIL Open Font License, Version 1.1.
This license is copied below, and is also available with a FAQ at:
http://scripts.sil.org/OFL
-----------------------------------------------------------
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
-----------------------------------------------------------
PREAMBLE
The goals of the Open Font License (OFL) are to stimulate worldwide
development of collaborative font projects, to support the font creation
efforts of academic and linguistic communities, and to provide a free and
open framework in which fonts may be shared and improved in partnership
with others.
The OFL allows the licensed fonts to be used, studied, modified and
redistributed freely as long as they are not sold by themselves. The
fonts, including any derivative works, can be bundled, embedded,
redistributed and/or sold with any software provided that any reserved
names are not used by derivative works. The fonts and derivatives,
however, cannot be released under any other type of license. The
requirement for fonts to remain under this license does not apply
to any document created using the fonts or their derivatives.
DEFINITIONS
"Font Software" refers to the set of files released by the Copyright
Holder(s) under this license and clearly marked as such. This may
include source files, build scripts and documentation.
"Reserved Font Name" refers to any names specified as such after the
copyright statement(s).
"Original Version" refers to the collection of Font Software components as
distributed by the Copyright Holder(s).
"Modified Version" refers to any derivative made by adding to, deleting,
or substituting -- in part or in whole -- any of the components of the
Original Version, by changing formats or by porting the Font Software to a
new environment.
"Author" refers to any designer, engineer, programmer, technical
writer or other person who contributed to the Font Software.
PERMISSION & CONDITIONS
Permission is hereby granted, free of charge, to any person obtaining
a copy of the Font Software, to use, study, copy, merge, embed, modify,
redistribute, and sell modified and unmodified copies of the Font
Software, subject to the following conditions:
1) Neither the Font Software nor any of its individual components,
in Original or Modified Versions, may be sold by itself.
2) Original or Modified Versions of the Font Software may be bundled,
redistributed and/or sold with any software, provided that each copy
contains the above copyright notice and this license. These can be
included either as stand-alone text files, human-readable headers or
in the appropriate machine-readable metadata fields within text or
binary files as long as those fields can be easily viewed by the user.
3) No Modified Version of the Font Software may use the Reserved Font
Name(s) unless explicit written permission is granted by the corresponding
Copyright Holder. This restriction only applies to the primary font name as
presented to the users.
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
Software shall not be used to promote, endorse or advertise any
Modified Version, except to acknowledge the contribution(s) of the
Copyright Holder(s) and the Author(s) or with their explicit written
permission.
5) The Font Software, modified or unmodified, in part or in whole,
must be distributed entirely under this license, and must not be
distributed under any other license. The requirement for fonts to
remain under this license does not apply to any document created
using the Font Software.
TERMINATION
This license becomes null and void if any of the above conditions are
not met.
DISCLAIMER
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
OTHER DEALINGS IN THE FONT SOFTWARE.
================================================
FILE: frontend/public/index.html
================================================
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<link rel="icon" />
<link rel="icon" type="image/x-icon" href="/favicons/favicon.ico" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="theme-color" content="#000000" />
<meta name="description" content="Web site created using create-react-app" />
<link rel="apple-touch-icon" href="%PUBLIC_URL%/logo192.png" />
<link rel="manifest" href="%PUBLIC_URL%/manifest.json" />
<title>HouseWatch</title>
</head>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>
<div id="root"></div>
</body>
</html>
================================================
FILE: frontend/public/manifest.json
================================================
{
"short_name": "HouseWatch",
"name": "HouseWatch",
"icons": [],
"start_url": ".",
"display": "standalone",
"theme_color": "#000000",
"background_color": "#ffffff"
}
================================================
FILE: frontend/public/robots.txt
================================================
# https://www.robotstxt.org/robotstxt.html
User-agent: *
Disallow:
================================================
FILE: frontend/src/App.tsx
================================================
import React from 'react'
import { BrowserRouter as Router } from 'react-router-dom'
import Layout from './Layout'
function App() {
return (
<div className="App">
<Router>
<Layout />
</Router>
</div>
)
}
export default App
================================================
FILE: frontend/src/Layout.tsx
================================================
// @ts-nocheck
import React, { useEffect, useState } from 'react'
import { DiskUsage } from './pages/DiskUsage/DiskUsage'
import SlowQueries from './pages/SlowQueries/SlowQueries'
import Schema from './pages/SchemaStats/SchemaStats'
import QueryDetail from './pages/SlowQueries/QueryDetail'
import SchemaTable from './pages/SchemaStats/SchemaTable'
import Overview from './pages/Overview/Overview'
import Clusters from './pages/Clusters/Clusters'
import Backups from './pages/Backups/Backups'
import ScheduledBackups from './pages/Backups/ScheduledBackups'
import Errors from './pages/Errors/Errors'
import { Switch, Route, useHistory } from 'react-router-dom'
import { Operations } from './pages/Operations/Operations'
import RunningQueries from './pages/RunningQueries/RunningQueries'
import Logs from './pages/Logs/Logs'
import {
ApartmentOutlined,
CloudServerOutlined,
CodeOutlined,
DashboardOutlined,
HddOutlined,
HomeOutlined,
WarningOutlined,
ClockCircleOutlined,
GithubFilled,
BarsOutlined,
FormOutlined,
ToolOutlined,
SaveOutlined,
} from '@ant-design/icons'
import { ConfigProvider, MenuProps } from 'antd'
import { Layout, Menu } from 'antd'
import QueryEditorPage from './pages/QueryEditor/QueryEditorPage'
import AIToolsPage from './pages/AITools/AIToolsPage'
const { Header, Content, Footer, Sider } = Layout
type MenuItem = Required<MenuProps>['items'][number]
const items: MenuItem[] = [
{ key: '', icon: <HomeOutlined />, label: 'Overview' },
{ key: 'clusters', label: 'Clusters', icon: <CloudServerOutlined /> },
{
key: 'backup',
label: 'Backup',
icon: <SaveOutlined />,
children: [
{ key: 'backups', label: 'Adhoc Backups' },
{ key: 'scheduled_backups', label: 'Scheduled Backups' },
],
},
{ key: 'query_performance', label: 'Query performance', icon: <ClockCircleOutlined /> },
{ key: 'running_queries', label: 'Running queries', icon: <DashboardOutlined /> },
{ key: 'schema', label: 'Schema stats', icon: <HddOutlined /> },
{ key: 'disk_usage', label: 'Disk usage', icon: <ApartmentOutlined /> },
{ key: 'logs', label: 'Logs', icon: <BarsOutlined /> },
{ key: 'errors', label: 'Errors', icon: <WarningOutlined /> },
{ key: 'query_editor', label: 'Query editor', icon: <FormOutlined /> },
{ key: 'operations', label: 'Operations', icon: <CodeOutlined /> },
{ key: 'ai_tools', label: 'AI Tools', icon: <ToolOutlined /> },
]
export default function AppLayout(): JSX.Element {
const [hostname, setHostname] = useState('')
const fetchHostname = async () => {
const response = await fetch(`/api/analyze/hostname`)
const responseJson = await response.json()
setHostname(responseJson.hostname)
}
useEffect(() => {
fetchHostname()
}, [])
const history = useHistory()
const openPage = history.location.pathname.split('/')[1]
return (
<ConfigProvider theme={{ token: { colorPrimary: '#ffb200', colorPrimaryBg: 'black' } }}>
<Layout style={{ minHeight: '100vh' }}>
<Sider className="sidebar">
<div className="clickable" onClick={() => history.push('')}>
<h1
style={{ fontSize: 20, color: '#ffb200', textAlign: 'center', fontFamily: 'Hind Siliguri' }}
>
HouseWatch
</h1>
</div>
<Menu
defaultSelectedKeys={[openPage]}
theme="dark"
mode="inline"
items={items}
onClick={(info) => history.push(`/${info.key}`)}
/>
</Sider>
<Layout>
<Header
style={{
background: 'rgb(231 231 231)',
borderBottom: '1px solid #c7c7c7',
display: 'inline-block',
}}
>
<p style={{ textAlign: 'center', margin: 0 }}>
<b>{hostname}</b>
</p>
</Header>
<Content style={{ margin: 'auto', display: 'block', width: '85%', marginTop: 20 }}>
<Switch>
<Route exact path="/" component={Overview}></Route>
<Route exact path="/clusters" component={Clusters}></Route>
<Route exact path="/backups" component={Backups}></Route>
<Route exact path="/scheduled_backups" component={ScheduledBackups}></Route>
<Route exact path="/disk_usage">
<DiskUsage />
</Route>
<Route exact path="/query_performance" component={SlowQueries}></Route>
<Route exact path="/schema" component={Schema}></Route>
<Route exact path="/schema/:table" component={SchemaTable}></Route>
<Route exact path="/query_performance/:query_hash" component={QueryDetail}></Route>
<Route exact path="/operations" component={Operations}></Route>
<Route exact path="/running_queries" component={RunningQueries}></Route>
<Route exact path="/logs" component={Logs}></Route>
<Route exact path="/errors" component={Errors}></Route>
<Route exact path="/query_editor" component={QueryEditorPage}></Route>
<Route exact path="/query_editor/:tab" component={QueryEditorPage}></Route>
<Route exact path="/query_editor/:tab/:id" component={QueryEditorPage}></Route>
<Route exact path="/ai_tools" component={AIToolsPage}></Route>
</Switch>
</Content>
<Footer style={{ textAlign: 'center' }}>
<p style={{ lineHeight: 2 }}>
Created by{' '}
<a href="https://posthog.com" target="_blank" rel="noopener noreferrer">
PostHog
</a>
</p>
<a
href="https://github.com/PostHog/HouseWatch"
target="_blank"
rel="noopener noreferrer"
style={{ color: 'black' }}
>
<GithubFilled />
</a>
</Footer>
</Layout>
</Layout>
</ConfigProvider>
)
}
================================================
FILE: frontend/src/index.css
================================================
@font-face {
font-family: 'Hind Siliguri';
src: url('fonts/HindSiliguri-Medium.ttf');
}
html {
overflow: hidden;
height: 100%;
}
body {
margin: 0;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 'Ubuntu', 'Cantarell', 'Fira Sans',
'Droid Sans', 'Helvetica Neue', sans-serif;
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
color: #262626;
font-size: 0.875rem;
line-height: 1.57;
font-weight: 400;
overflow: auto;
height: 100%;
}
.ant-menu-item-selected {
color: black !important;
}
h1 {
font-size: 24px;
}
h1,
h2,
h3,
h4,
h5,
h6 {
font-family: sans-serif;
}
strong,
b {
font-weight: 600;
}
code {
font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', monospace;
}
.clickable:hover,
.clickable div:hover,
.code-editor:hover textarea {
cursor: pointer !important;
}
.code-editor textarea {
line-height: 2 !important;
font-weight: 100 !important;
}
.code-editor:hover {
background-color: #ededed;
}
.ant-tabs-tab-active .ant-tabs-tab-btn,
.ant-tabs-tab-btn:hover {
color: #1677ff !important;
}
.ant-tabs-ink-bar {
background: #1677ff !important;
}
.run-async-migration-btn:hover {
border-color: #64a0f4 !important;
background: #13a5ff !important;
}
input,
textarea,
button,
select,
.ant-select-selector {
box-shadow: none !important;
}
.sidebar .ant-layout-sider-children {
position: fixed;
}
.ant-select-item-option-selected:not(.ant-select-item-option-disabled) {
background-color: initial !important;
}
================================================
FILE: frontend/src/index.tsx
================================================
import React from 'react'
import ReactDOM from 'react-dom/client'
import './index.css'
import App from './App'
const root = ReactDOM.createRoot(document.getElementById('root') as HTMLElement)
root.render(<App />)
================================================
FILE: frontend/src/pages/AITools/AIToolsPage.tsx
================================================
import React, { useEffect, useState } from 'react'
import { Tabs } from 'antd'
import { useHistory } from 'react-router-dom'
import NaturalLanguageQueryEditor from './NaturalLanguageQueryEditor'
export default function AIToolsPage() {
const history = useHistory()
const [error, setError] = useState<string | null>(null)
const loadData = async () => {
const res = await fetch('/api/analyze/ai_tools_available')
const resJson = await res.json()
if ('error' in resJson) {
setError(resJson['error'])
}
}
useEffect(() => {
loadData()
}, [])
return (
<>
<h1>AI Tools (Alpha)</h1>
{error ? (
<p style={{ marginTop: 50 }}>{error}</p>
) : (
<>
<Tabs
items={[
{
key: 'natural_language',
label: `Natural language query editor`,
children: <NaturalLanguageQueryEditor />,
},
]}
defaultActiveKey="natural_language"
onChange={(tab) => history.push(`/query_editor/${tab}`)}
/>
</>
)}
</>
)
}
================================================
FILE: frontend/src/pages/AITools/NaturalLanguageQueryEditor.tsx
================================================
import React, { useEffect, useState } from 'react'
import { Select, Checkbox, Button, Table, ConfigProvider } from 'antd'
import TextArea from 'antd/es/input/TextArea'
// @ts-ignore
import { highlight, languages } from 'prismjs/components/prism-core' // @ts-ignore
import 'prismjs/components/prism-sql'
import 'prismjs/themes/prism.css'
import Editor from 'react-simple-code-editor'
// @ts-ignore
import { format } from 'sql-formatter-plus'
export interface TableData {
table: string
database: string
}
export default function NaturalLanguageQueryEditor() {
const [query, setQuery] = useState('')
const [tables, setTables] = useState<TableData[] | null>(null)
const [tablesToQuery, setTablesToQuery] = useState([])
const [readonly, setReadonly] = useState(true)
const [data, setData] = useState([{}])
const [sql, setSql] = useState<string | null>(null)
const [loading, setLoading] = useState(false)
const [error, setError] = useState('')
const columns = data.length > 0 ? Object.keys(data[0]).map((column) => ({ title: column, dataIndex: column })) : []
const runQuery = async () => {
setLoading(true)
setSql(null)
const res = await fetch('/api/analyze/natural_language_query', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
query: query,
tables_to_query: tablesToQuery,
readonly: readonly || false,
}),
})
const resJson = await res.json()
if (resJson.error) {
setError(resJson.error)
setData([])
} else {
setData(resJson.result)
}
setSql(resJson.sql)
setLoading(false)
}
const loadTableData = async () => {
const res = await fetch('/api/analyze/tables')
const resJson = await res.json()
setTables(resJson)
}
useEffect(() => {
loadTableData()
}, [])
const selectOptions = (tables || []).map((t) => ({
value: [t.database, t.table].join('>>>>>'),
label: [t.database, t.table].join('.'),
}))
return (
<>
<div id="nl-form">
<div>
<p>Select the tables you'd like to query:</p>
<Select
placeholder="system.query_log"
optionFilterProp="children"
options={selectOptions}
style={{ width: 600 }}
onChange={(value) => {
setTablesToQuery(value)
}}
mode="multiple"
showSearch={false}
/>
</div>
<br />
<div>
<Checkbox checked={readonly} onChange={(e) => setReadonly(e.target.value)}>
Read-only
</Checkbox>
</div>
<br />
<div>
<p>Describe what you'd like to query (the more specific the better):</p>
<TextArea id="nl-query-textarea" onChange={(e) => setQuery(e.target.value)} placeholder='give me the 10 slowest queries over the last hour and their memory usage in gb' />
</div>
<br />
<Button
type="primary"
style={{ width: '100%', boxShadow: 'none' }}
onClick={runQuery}
disabled={loading || tablesToQuery.length < 1 || !query}
>
Run
</Button>
</div>
<br />
{sql ? (
<details>
<summary style={{ color: '#1677ff', cursor: 'pointer' }}>Show SQL</summary>
<br />
<Editor
value={format(sql)}
onValueChange={() => {}}
highlight={(code) => highlight(code, languages.sql)}
padding={10}
style={{
fontFamily: '"Fira code", "Fira Mono", monospace',
fontSize: 16,
border: '1px solid rgb(216, 216, 216)',
borderRadius: 4,
boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)',
marginBottom: 5,
}}
disabled
className="code-editor"
/>
</details>
) : null}
<br />
<ConfigProvider renderEmpty={() => <p style={{ color: '#c40000', fontFamily: 'monospace' }}>{error}</p>}>
<Table columns={columns} dataSource={data} scroll={{ x: 400 }} loading={loading} />
</ConfigProvider>
</>
)
}
================================================
FILE: frontend/src/pages/Backups/Backups.tsx
================================================
import React, { useEffect, useState } from 'react'
import { usePollingEffect } from '../../utils/usePollingEffect'
import { ColumnType } from 'antd/es/table'
import { Table, Button, Form, Input, Checkbox, Modal, Tag, Col, Progress, Row, Tooltip, notification } from 'antd'
import useSWR, { mutate } from 'swr'
interface BackupRow {
id: string
name: string
status: string
error: string
start_time: string
end_time: string
num_files: number
total_size: number
num_entries: number
uncompressed_size: number
compressed_size: number
files_read: number
bytes_read: number
}
interface Backups {
backups: BackupRow[]
}
type FieldType = {
database?: string
table?: string
bucket?: string
path?: string
is_sharded?: boolean
aws_access_key_id?: string
aws_secret_access_key?: string
}
export default function Backups() {
const [open, setOpen] = useState(false)
const [confirmLoading, setConfirmLoading] = useState(false)
const [form] = Form.useForm() // Hook to get form API
const handleSubmit = async () => {
try {
// Validate and get form values
const values = await form.validateFields()
setConfirmLoading(true)
const res = await fetch(`/api/backups`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(values),
})
setOpen(false)
setConfirmLoading(false)
mutate('/api/backups')
return await res.json()
} catch (error) {
notification.error({
message: 'Creating backup failed',
})
}
}
const showModal = () => {
setOpen(true)
}
const handleCancel = () => {
console.log('Clicked cancel button')
setOpen(false)
}
const loadData = async (url: string) => {
try {
const res = await fetch(url)
const resJson = await res.json()
const backups = { backups: resJson }
return backups
} catch (err) {
notification.error({ message: 'Failed to load data' })
}
}
const { data, error, isLoading } = useSWR('/api/backups', loadData)
const columns: ColumnType<BackupRow>[] = [
{ title: 'UUID', dataIndex: 'id' },
{ title: 'Name', dataIndex: 'name' },
{
title: 'Status',
dataIndex: 'status',
render: (_, { status }) => {
var color = 'volcano'
switch (status) {
case 'CREATING_BACKUP' || 'RESTORING':
color = 'black'
break
case 'BACKUP_CREATED' || 'RESTORED':
color = 'green'
break
case 'BACKUP_FAILED' || 'RESTORE_FAILED':
color = 'volcano'
break
}
return (
<Tag color={color} key={status}>
{status.toUpperCase()}
</Tag>
)
},
},
{ title: 'Error', dataIndex: 'error' },
{ title: 'Start', dataIndex: 'start_time' },
{ title: 'End', dataIndex: 'end_time' },
{ title: 'Size', dataIndex: 'total_size' },
]
usePollingEffect(
async () => {
mutate('/api/backups')
},
[],
{ interval: 5000 }
)
return isLoading ? (
<div>loading...</div>
) : error ? (
<div>error</div>
) : (
<div>
<h1 style={{ textAlign: 'left' }}>Backups</h1>
<Button onClick={showModal}>Create Backup</Button>
<Modal
title="Create Backup"
open={open}
onOk={handleSubmit}
confirmLoading={confirmLoading}
onCancel={handleCancel}
>
<Form
name="basic"
form={form}
labelCol={{ span: 8 }}
wrapperCol={{ span: 16 }}
style={{ maxWidth: 600 }}
initialValues={{ remember: true }}
autoComplete="on"
>
<Form.Item<FieldType>
label="Database"
name="database"
initialValue="default"
rules={[{ required: true, message: 'Please select a database to back up from' }]}
>
<Input />
</Form.Item>
<Form.Item<FieldType>
label="Table"
name="table"
initialValue="test_backup"
rules={[{ required: true, message: 'Please select a table to back up' }]}
>
<Input />
</Form.Item>
<Form.Item<FieldType>
label="Is Sharded"
name="is_sharded"
initialValue="false"
valuePropName="checked"
rules={[{ required: true, message: 'Is this table sharded?' }]}
>
<Checkbox defaultChecked={false} />
</Form.Item>
<Form.Item<FieldType>
label="S3 Bucket"
name="bucket"
initialValue="posthog-clickhouse"
rules={[{ required: true, message: 'What S3 bucket to backup into' }]}
>
<Input />
</Form.Item>
<Form.Item<FieldType>
label="S3 Path"
name="path"
initialValue="testing/test_backup/7"
rules={[{ required: true, message: 'What is the path in the bucket to backup to' }]}
>
<Input />
</Form.Item>
<Form.Item<FieldType>
label="AWS Access Key ID"
name="aws_access_key_id"
initialValue="AKIAIOSFODNN7EXAMPLE"
rules={[{ required: false, message: 'AWS Access Key ID to use for access to the S3 bucket' }]}
>
<Input />
</Form.Item>
<Form.Item<FieldType>
label="AWS Secret Access Key"
name="aws_secret_access_key"
initialValue="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
rules={[{ required: false, message: 'AWS Secret Access Key used to access S3 bucket' }]}
>
<Input />
</Form.Item>
</Form>
</Modal>
<Table columns={columns} dataSource={data!.backups} loading={isLoading} />
</div>
)
}
================================================
FILE: frontend/src/pages/Backups/ScheduledBackups.tsx
================================================
import React, { useEffect, useState } from 'react'
import { usePollingEffect } from '../../utils/usePollingEffect'
import { ColumnType } from 'antd/es/table'
import {
Switch,
Select,
Table,
Button,
Form,
Input,
Checkbox,
Modal,
Tag,
Col,
Progress,
Row,
Tooltip,
notification,
} from 'antd'
import DeleteOutlined from '@ant-design/icons/DeleteOutlined'
import EditOutlined from '@ant-design/icons/EditOutlined'
import { Clusters } from '../Clusters/Clusters'
import useSWR, { mutate } from 'swr'
interface ScheduleRow {
id: string
created_at: string
enabled: boolean
last_run_time: string
cluster: string
schedule: string
incremental_schedule: string
table: string
is_sharded: boolean
database: string
bucket: string
path: string
last_run: string
}
interface Backups {
backups: ScheduleRow[]
}
type FieldType = {
cluster?: string
schedule?: string
incremental_schedule?: string
database?: string
table?: string
is_sharded?: boolean
bucket?: string
path?: string
aws_access_key_id?: string
aws_secret_access_key?: string
}
export default function ScheduledBackups() {
const [open, setOpen] = useState(false)
const [confirmLoading, setConfirmLoading] = useState(false)
const [form] = Form.useForm() // Hook to get form API
const [editingRow, setEditingRow] = useState<ScheduleRow | null>(null) // <-- New state to hold the editing row data
const handleSubmit = async () => {
try {
const method = editingRow ? 'PATCH' : 'POST'
const url = editingRow ? `/api/scheduled_backups/${editingRow.id}` : '/api/scheduled_backups'
const values = await form.validateFields()
setConfirmLoading(true)
const res = await fetch(url, {
method: method,
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(values),
})
setOpen(false)
setConfirmLoading(false)
setEditingRow(null)
mutate('/api/scheduled_backups')
return await res.json()
} catch (error) {
notification.error({
message: 'Creating backup failed',
})
}
}
const handleCancel = () => {
console.log('Clicked cancel button')
setOpen(false)
form.resetFields()
setEditingRow(null)
}
const showModal = (rowData: ScheduleRow | null = null) => {
setEditingRow(rowData)
if (rowData) {
form.setFieldsValue(rowData)
} else {
form.resetFields()
}
setOpen(true)
}
const handleEdit = (rowData: ScheduleRow) => {
showModal(rowData)
}
const fetchBackups = async (url: string) => {
try {
const res = await fetch(url)
const resJson = await res.json()
const backups = { backups: resJson.results }
return backups
} catch (err) {
notification.error({ message: 'Failed to load data' })
}
}
const fetchClusters = async (url: string) => {
try {
const res = await fetch(url)
const resJson = await res.json()
const clusters = { clusters: resJson }
return clusters
} catch (err) {
notification.error({ message: 'Failed to load data' })
}
}
const { data: backups, error: backupsError, isLoading: backupsIsLoading } = useSWR(
'/api/scheduled_backups',
fetchBackups
)
const { data: clusters, error: clustersError, isLoading: clustersIsLoading } = useSWR(
'/api/clusters',
fetchClusters
)
const columns: ColumnType<ScheduleRow>[] = [
{
title: 'Enabled',
dataIndex: 'enabled',
render: (_, sched) => {
const toggleEnabled = async () => {
try {
const res = await fetch(`/api/scheduled_backups/${sched.id}`, {
method: 'PATCH',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ enabled: !sched.enabled }),
})
mutate('/api/scheduled_backups')
return await res.json()
} catch (error) {
notification.error({
message: 'Failed to toggle backup',
})
}
}
return <Switch defaultChecked={sched.enabled} onChange={toggleEnabled} />
},
},
{ title: 'Cluster', dataIndex: 'cluster' },
{ title: 'Schedule', dataIndex: 'schedule' },
{ title: 'Incremental Schedule', dataIndex: 'incremental_schedule' },
{ title: 'Last Run Time', dataIndex: 'last_run_time' },
{ title: 'Database', dataIndex: 'database' },
{ title: 'Table', dataIndex: 'table' },
{ title: 'Is Sharded', dataIndex: 'is_sharded', render: (_, sched) => (sched.is_sharded ? 'Yes' : 'No') },
{ title: 'S3 Location', dataIndex: 'bucket', render: (_, sched) => 's3://' + sched.bucket + '/' + sched.path },
{
title: 'Actions',
dataIndex: 'id',
render: (id: string, rowData: ScheduleRow) => {
const deleteBackup = async () => {
try {
const res = await fetch(`/api/scheduled_backups/${id}`, {
method: 'DELETE',
})
mutate('/api/scheduled_backups')
return await res.text()
} catch (error) {
notification.error({
message: 'Failed to delete backup',
})
}
}
return (
<>
<EditOutlined onClick={() => handleEdit(rowData)} />
<DeleteOutlined onClick={() => deleteBackup()} style={{ marginLeft: '15px' }} />
</>
)
},
},
]
usePollingEffect(
async () => {
mutate('/api/scheduled_backups')
},
[],
{ interval: 5000 }
)
const default_form_values = {
schedule: '0 0 * * *',
incremental_schedule: '0 0 * * *',
database: 'default',
table: 'test_backup',
bucket: 'posthog-clickhouse',
path: 'testing/test_backup/7',
aws_access_key_id: 'AKIAIOSFODNN7EXAMPLE',
aws_secret_access_key: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
}
return backupsIsLoading || clustersIsLoading ? (
<div>loading...</div>
) : backupsError || clustersError ? (
<div>error</div>
) : (
<div>
<h1 style={{ textAlign: 'left' }}>Scheduled Backups</h1>
<p style={{ textAlign: 'left' }}>
It's a bit of a hack, but if you want to backup a database just omit the table when you create the
scheduled backup.
</p>
<Button onClick={() => showModal()}>Create Backup</Button>
<Modal
title={editingRow ? 'Edit Backup' : 'Create Backup'}
open={open}
onOk={handleSubmit}
confirmLoading={confirmLoading}
onCancel={handleCancel}
>
<Form
name="basic"
form={form}
labelCol={{ span: 8 }}
wrapperCol={{ span: 16 }}
style={{ maxWidth: 600 }}
initialValues={editingRow ? editingRow : default_form_values}
autoComplete="on"
>
<Form.Item name="id" hidden={true}>
<Input />
</Form.Item>
<Form.Item name="cluster" label="Cluster">
<Select>
{clusters!.clusters.map((cluster: any) => (
<Select.Option value={cluster.cluster}>{cluster.cluster}</Select.Option>
))}
</Select>
</Form.Item>
<Form.Item<FieldType>
label="Schedule"
name="schedule"
rules={[{ required: true, message: 'Please provide a cron schedule for the backup' }]}
>
<Input />
</Form.Item>
<Form.Item<FieldType>
label="Incremental Schedule"
name="incremental_schedule"
rules={[
{ required: true, message: 'Please provide a cron schedule for the incremental backup' },
]}
>
<Input />
</Form.Item>
<Form.Item<FieldType>
label="Database"
name="database"
rules={[{ required: true, message: 'Please select a database to back up from' }]}
>
<Input />
</Form.Item>
<Form.Item<FieldType>
label="Table"
name="table"
rules={[{ required: false, message: 'Please select a table to back up' }]}
>
<Input />
</Form.Item>
<Form.Item<FieldType>
label="Is Sharded"
name="is_sharded"
initialValue="false"
valuePropName="checked"
rules={[{ required: true, message: 'Is this table sharded?' }]}
>
<Checkbox defaultChecked={false} />
</Form.Item>
<Form.Item<FieldType>
label="S3 Bucket"
name="bucket"
rules={[{ required: true, message: 'What S3 bucket to backup into' }]}
>
<Input />
</Form.Item>
<Form.Item<FieldType>
label="S3 Path"
name="path"
rules={[{ required: true, message: 'What is the path in the bucket to backup to' }]}
>
<Input />
</Form.Item>
<Form.Item<FieldType>
label="AWS Access Key ID"
name="aws_access_key_id"
rules={[{ required: true, message: 'AWS Access Key ID to use for access to the S3 bucket' }]}
>
<Input />
</Form.Item>
<Form.Item<FieldType>
label="AWS Secret Access Key"
name="aws_secret_access_key"
rules={[{ required: true, message: 'AWS Secret Access Key used to access S3 bucket' }]}
>
<Input />
</Form.Item>
</Form>
</Modal>
<Table columns={columns} dataSource={backups!.backups} loading={backupsIsLoading} />
</div>
)
}
================================================
FILE: frontend/src/pages/Clusters/Clusters.tsx
================================================
import React, { useEffect, useState } from 'react'
import { ColumnType } from 'antd/es/table'
import { Table, Col, Row, Tooltip, notification } from 'antd'
import useSWR from 'swr'
interface ClusterNode {
cluster: string
shard_num: number
shard_weight: number
replica_num: number
host_name: string
host_address: string
port: number
is_local: boolean
user: string
default_database: string
errors_count: number
slowdowns_count: number
estimated_recovery_time: number
}
interface Cluster {
cluster: string
nodes: ClusterNode[]
}
export interface Clusters {
clusters: Cluster[]
}
export default function Clusters() {
const loadData = async (url: string) => {
try {
const res = await fetch(url)
const resJson = await res.json()
const clusters = { clusters: resJson }
return clusters
} catch (err) {
notification.error({ message: 'Failed to load data' })
}
}
const { data, error, isLoading } = useSWR('/api/clusters', loadData)
const columns: ColumnType<ClusterNode>[] = [
{ title: 'Cluster', dataIndex: 'cluster' },
{ title: 'Shard Number', dataIndex: 'shard_num' },
{ title: 'Shard Weight', dataIndex: 'shard_weight' },
{ title: 'Replica Number', dataIndex: 'replica_num' },
{ title: 'Host Name', dataIndex: 'host_name' },
{ title: 'Host Address', dataIndex: 'host_address' },
{ title: 'Port', dataIndex: 'port' },
{ title: 'Is Local', dataIndex: 'is_local' },
{ title: 'User', dataIndex: 'user' },
{ title: 'Default Database', dataIndex: 'default_database' },
{ title: 'Errors Count', dataIndex: 'errors_count' },
{ title: 'Slowdowns Count', dataIndex: 'slowdowns_count' },
{ title: 'Recovery Time', dataIndex: 'estimated_recovery_time' },
]
return isLoading ? (
<div>loading...</div>
) : error ? (
<div>error</div>
) : (
<div>
<h1 style={{ textAlign: 'left' }}>Clusters</h1>
<p>These are the clusters that are configured in the connected ClickHouse instance</p>
<div>
<ul>
{data!.clusters.map((cluster: any) => (
<>
<h1 key={cluster.cluster}>{cluster.cluster}</h1>
<Table columns={columns} dataSource={cluster.nodes} loading={isLoading} />
</>
))}
</ul>
</div>
</div>
)
}
================================================
FILE: frontend/src/pages/DiskUsage/DiskUsage.tsx
================================================
import React, { useEffect, useState } from 'react'
import { Pie } from '@ant-design/plots'
import { Card, Spin, Row, Col, notification } from 'antd'
import useSWR from 'swr'
interface NodeData {
node: string
space_used: number
free_space: number
}
export function DiskUsage(): JSX.Element {
const loadData = async (url: string) => {
try {
const res = await fetch(url)
const resJson = await res.json()
return resJson
} catch {
notification.error({ message: 'Failed to load data' })
}
}
const { data, error, isLoading } = useSWR('/api/analyze/cluster_overview', loadData)
const rows = []
if (!isLoading) {
for (let i = 0; i < data.length; i += 2) {
rows.push(data.slice(i, i + 2))
}
}
return isLoading ? (
<div>loading...</div>
) : error ? (
<div>error</div>
) : (
<div style={{ textAlign: 'left' }}>
<h1>Disk usage</h1>
<br />
<div style={{ display: 'block' }}>
{data.length === 0 ? (
<Spin />
) : (
<>
{rows.map((row, i) => (
<Row key={`disk-usage-row-${i}`} gutter={8} style={{ marginBottom: 8 }}>
<Col span={12}>
<Card>
<h2 style={{ textAlign: 'center' }}>{row[0].node}</h2>
<Pie
data={[
{
type: 'Used disk space',
value: row[0].space_used,
},
{
type: 'Free disk space',
value: row[0].free_space,
},
]}
appendPadding={10}
angleField="value"
colorField="type"
radius={0.9}
label={{
type: 'inner',
offset: '-30%',
content: ({ percent }: { percent: number }) =>
`${(percent * 100).toFixed(0)}%`,
style: {
fontSize: 14,
textAlign: 'center',
},
}}
interactions={[
{
type: 'element-active',
},
]}
style={{
display: 'block',
}}
color={['#FFB816', '#175FFF']}
tooltip={{
formatter: (v: any) => {
return {
name: v.type,
value: `${(v.value / 1000000000).toFixed(2)}GB`,
}
},
}}
/>
</Card>
</Col>
<Col span={12}>
{row[1] ? (
<Card>
<h2 style={{ textAlign: 'center' }}>{row[1].node}</h2>
<Pie
data={[
{
type: 'Used disk space',
value: row[1].space_used,
},
{
type: 'Free disk space',
value: row[1].free_space,
},
]}
appendPadding={10}
angleField="value"
colorField="type"
radius={0.9}
label={{
type: 'inner',
offset: '-30%',
content: ({ percent }: { percent: number }) =>
`${(percent * 100).toFixed(0)}%`,
style: {
fontSize: 14,
textAlign: 'center',
},
}}
interactions={[
{
type: 'element-active',
},
]}
style={{
display: 'block',
}}
color={['#FFB816', '#175FFF']}
tooltip={{
formatter: (v: any) => {
return {
name: v.type,
value: `${(v.value / 1000000000).toFixed(2)}GB`,
}
},
}}
/>
</Card>
) : null}
</Col>
</Row>
))}
</>
)}
</div>
</div>
)
}
================================================
FILE: frontend/src/pages/Errors/Errors.tsx
================================================
import React, { useEffect, useState } from 'react'
import { Table, notification } from 'antd'
import { ColumnsType } from 'antd/es/table'
import { isoTimestampToHumanReadable } from '../../utils/dateUtils'
import useSWR from 'swr'
interface ErrorData {
name: string
count: number
max_last_error_time: string
}
export default function CollapsibleTable() {
const slowQueriesColumns: ColumnsType<ErrorData> = [
{
title: 'Error',
dataIndex: 'name',
render: (_, item) => (
<p style={{}}>
<b>{item.name}</b>
</p>
),
},
{ title: 'Occurrences', dataIndex: 'count', render: (_, item) => <>{item.count}</> },
{
title: 'Most recent occurence',
dataIndex: 'max_last_error_time',
render: (_, item) => isoTimestampToHumanReadable(item.max_last_error_time),
},
]
const loadData = async (url: string) => {
try {
const res = await fetch(url)
const resJson = await res.json()
const slowQueriesData = resJson.map((error: ErrorData, idx: number) => ({ key: idx, ...error }))
return slowQueriesData
} catch {
notification.error({ message: 'Failed to load data' })
}
}
const { data, error, isLoading, mutate } = useSWR('/api/analyze/errors', loadData)
return isLoading ? (
<div>loading...</div>
) : error ? (
<div>error</div>
) : (
<div>
<h1 style={{ textAlign: 'left' }}>Errors</h1>
<br />
<div>
<Table columns={slowQueriesColumns} dataSource={data} size="small" loading={isLoading} />
</div>
</div>
)
}
================================================
FILE: frontend/src/pages/Logs/Logs.tsx
================================================
import { Table, Typography, Input, Card, ConfigProvider, Empty } from 'antd'
import React, { useEffect, useState } from 'react'
import { Column } from '@ant-design/charts'
const { Paragraph } = Typography
export default function Logs() {
// very simplistic error handling where both requests set this property
// mostly because the only error we expect is that the table doesn't exist
const [error, setError] = useState('')
const [logs, setLogs] = useState([])
const [loadingLogsFrequency, setLoadingLogsFrequency] = useState(false)
const [loadingLogs, setLoadingLogs] = useState(false)
const [logsFrequency, setLogsFrequency] = useState([])
const [logMessageFilter, setLogMessageFilter] = useState('')
const columns = [
{ title: 'Time', dataIndex: 'event_time' },
{ title: 'Level', dataIndex: 'level' },
{ title: 'Host', dataIndex: 'hostname' },
{
title: 'Message',
dataIndex: 'message',
key: 'message',
render: (_: any, item: any) => {
return (
<Paragraph
style={{ maxWidth: '100%', fontFamily: 'monospace' }}
ellipsis={{
rows: 2,
expandable: true,
}}
>
{item.message}
</Paragraph>
)
},
},
]
const url = '/api/analyze/logs'
const fetchLogs = async (messageIlike = '') => {
setLoadingLogs(true)
const res = await fetch(url, {
method: 'POST',
body: JSON.stringify({ message_ilike: messageIlike }),
headers: {
'Content-Type': 'application/json',
},
})
const resJson = await res.json()
if (resJson.error) {
setError(resJson.error)
} else {
setLogs(resJson)
}
setLoadingLogs(false)
}
const fetchLogsFrequency = async (messageIlike = '') => {
setLoadingLogsFrequency(true)
const res = await fetch('/api/analyze/logs_frequency', {
method: 'POST',
body: JSON.stringify({ message_ilike: messageIlike }),
headers: {
'Content-Type': 'application/json',
},
})
const resJson = await res.json()
if (resJson.error) {
setError(resJson.error)
} else {
setLogsFrequency(resJson)
}
setLoadingLogsFrequency(false)
}
useEffect(() => {
fetchLogs(logMessageFilter)
fetchLogsFrequency(logMessageFilter)
}, [logMessageFilter])
return (
<>
<h1 style={{ textAlign: 'left' }}>Logs</h1>
<Input
style={{ boxShadow: 'none' }}
onChange={e => setLogMessageFilter(e.target.value)}
value={logMessageFilter}
/>
<br />
<br />
<Card style={{ boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)' }}>
<Column
xField="hour"
yField="total"
color="#ffb200"
style={{ height: 150 }}
data={logsFrequency}
loading={loadingLogsFrequency}
/>
</Card>
<br />
<ConfigProvider
renderEmpty={() => (
<Empty
description={
error === 'text_log table does not exist' ? (
<>
Your ClickHouse instance does not have the <code>text_log</code> table. See{' '}
<a
href="https://clickhouse.com/docs/en/operations/system-tables/text_log"
target="_blank"
rel="noreferrer noopener"
>
these docs
</a>{' '}
on how to configure it.
</>
) : (
''
)
}
/>
)}
>
<Table columns={columns} dataSource={logs} loading={loadingLogs} />
</ConfigProvider>
</>
)
}
================================================
FILE: frontend/src/pages/Operations/Operations.tsx
================================================
// @ts-nocheck
import React, { useEffect, useState } from 'react'
import { useHistory } from 'react-router-dom'
import Editor from 'react-simple-code-editor'
// @ts-ignore
import { highlight, languages } from 'prismjs/components/prism-core'
import 'prismjs/components/prism-sql'
import 'prismjs/themes/prism.css'
import { Button, Input, Progress, Table, Tabs, notification } from 'antd'
import TextArea from 'antd/es/input/TextArea'
import { ColumnType } from 'antd/es/table'
import { isoTimestampToHumanReadable } from '../../utils/dateUtils'
import useSWR from 'swr'
const OPERATION_STATUS_TO_HUMAN = {
0: 'Not started',
1: 'Running',
2: 'Completed successfully',
3: 'Errored',
4: 'Rolled back',
5: 'Starting',
6: 'Failed at startup',
}
const statusSortOrder = [5, 1, 0, 4, 6, 3, 2]
const OPERATION_STATUS_TO_FONT_COLOR = {
0: 'black',
1: 'black',
2: 'green',
3: 'red',
4: 'orange',
5: 'black',
6: 'red',
}
interface AsyncMigrationData {
id: number
name: string
description: string
status: number
progress: number
started_at: string
finished_at: string
}
export function OperationControls({
status,
progress,
id,
triggerOperation,
}: {
status: number
progress: number
id: number
triggerOperation: () => Promise<void>
}): JSX.Element {
return (
<div style={{ width: 100 }}>
{[0, 4, 6].includes(status) ? (
<Button
className="run-async-migration-btn"
style={{ color: 'white', background: '#1677ff' }}
onClick={() => triggerOperation(id)}
>
Run
</Button>
) : status === 3 ? (
<Button danger>Rollback</Button>
) : (
<Progress percent={progress} />
)}
</div>
)
}
export function OperationsList(): JSX.Element {
const fetchAndUpdateOperationsIfNeeded = async (url: string) => {
const response = await fetch(url)
const responseJson = await response.json()
const results = responseJson.results
if (JSON.stringify(results) !== JSON.stringify(operations)) {
return results
}
}
const triggerOperation = async id => {
await fetch(`/api/async_migrations/${id}/trigger`, { method: 'POST' })
await fetchAndUpdateOperationsIfNeeded()
}
const { data: operations, error, isLoading, mutate } = useSWR(
'/api/async_migrations',
fetchAndUpdateOperationsIfNeeded
)
useEffect(() => {
const intervalId = setInterval(() => {
mutate('/api/async_migrations')
}, 5000)
return () => {
try {
clearInterval(intervalId)
} catch {}
}
}, [])
const columns: ColumnType<AsyncMigrationData>[] = [
{
title: 'Name',
render: (_, migration) => migration.name,
},
{
title: 'Description',
render: (_, migration) => migration.description,
},
{
title: 'Status',
render: (_, migration) => (
<span style={{ color: OPERATION_STATUS_TO_FONT_COLOR[migration.status] }}>
{OPERATION_STATUS_TO_HUMAN[migration.status]}
</span>
),
sorter: (a, b) => statusSortOrder.indexOf(a.status) - statusSortOrder.indexOf(b.status),
defaultSortOrder: 'ascend',
},
{
title: 'Started at',
render: (_, migration) => (migration.started_at ? isoTimestampToHumanReadable(migration.started_at) : ''),
},
{
title: 'Finished at',
render: (_, migration) => (migration.finished_at ? isoTimestampToHumanReadable(migration.finished_at) : ''),
},
{
title: '',
render: (_, migration) => (
<OperationControls
status={migration.status}
progress={migration.progress}
id={migration.id}
triggerOperation={triggerOperation}
/>
),
},
]
return isLoading ? (
<div>loading...</div>
) : error ? (
<div>error</div>
) : (
<Table columns={columns} dataSource={operations} />
)
}
export function CreateNewOperation(): JSX.Element {
const history = useHistory()
const [operationOperationsCount, setOperationOperationsCount] = useState(1)
const [code, setCode] = useState({})
const createOperation = async () => {
const form = document.getElementById('create-migration-form')
const formData = new FormData(form)
const operations: string[] = []
const rollbackOperations: string[] = []
const operationData = {
name: '',
description: '',
operations: operations,
rollbackOperations: rollbackOperations,
}
for (const [key, value] of formData.entries()) {
if (key.includes('operation')) {
operations.push(value)
continue
}
if (key.includes('rollback')) {
rollbackOperations.push(value)
continue
}
operationData[key] = value
}
const res = await fetch('/api/async_migrations', {
method: 'POST',
body: JSON.stringify(operationData),
headers: {
'Content-Type': 'application/json',
},
})
if (String(res.status)[0] === '2') {
history.go(0)
} else {
notification.error({
message: 'Error creating operation! Check if you do not have an operation with the same name already.',
})
}
}
return (
<div>
<form style={{ textAlign: 'left', marginLeft: 20, overflowY: 'auto' }} id="create-migration-form">
<h3>Details</h3>
<Input id="create-migration-form-name" name="name" placeholder="Name" style={{ width: 400 }} />
<br />
<br />
<TextArea
id="create-migration-form-description"
name="description"
placeholder="Description"
style={{ width: 800 }}
rows={3}
/>
<br />
<br />
<h3>Operations</h3>
{[...Array(operationOperationsCount)].map((_, i) => (
<span key={i}>
<h4>#{i + 1}</h4>
<Editor
id={`create-migration-form-operation-${i + 1}`}
name={`operation-${i + 1}`}
value={
code[`operation-${i + 1}`] ||
`CREATE TABLE test_table ( foo String ) Engine=MergeTree() ORDER BY foo`
}
onValueChange={value => setCode({ ...code, [`operation-${i + 1}`]: value })}
highlight={code => highlight(code, languages.sql)}
padding={10}
style={{
fontFamily: '"Fira code", "Fira Mono", monospace',
fontSize: 14,
width: 800,
minHeight: 200,
border: '1px solid #d9d9d9',
borderRadius: 4,
background: 'white',
boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)',
}}
rows={5}
/>
<br />
<br />
<Editor
id={`create-migration-form-rollback-${i + 1}`}
name={`rollback-${i + 1}`}
value={code[`rollback-${i + 1}`] || `DROP TABLE IF EXISTS test_table`}
onValueChange={value => setCode({ ...code, [`rollback-${i + 1}`]: value })}
highlight={code => highlight(code, languages.sql)}
padding={10}
style={{
fontFamily: '"Fira code", "Fira Mono", monospace',
fontSize: 14,
width: 800,
minHeight: 200,
border: '1px solid #d9d9d9',
borderRadius: 4,
background: 'white',
boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)',
}}
rows={5}
/>
<br />
<br />
</span>
))}
{operationOperationsCount > 1 ? (
<>
<Button onClick={() => setOperationOperationsCount(operationOperationsCount - 1)} danger>
-
</Button>{' '}
</>
) : null}
<Button
onClick={() => setOperationOperationsCount(operationOperationsCount + 1)}
style={{ color: 'rgb(22 166 255)', borderColor: 'rgb(22 166 255)' }}
>
+
</Button>
</form>
<div style={{ textAlign: 'center' }}>
<Button style={{ color: 'white', background: '#1677ff' }} variant="contained" onClick={createOperation}>
Save
</Button>
</div>
</div>
)
}
export function Operations(): JSX.Element {
return (
<div style={{ display: 'block', margin: 'auto' }}>
<h1 style={{ textAlign: 'left' }}>Operations (Alpha)</h1>
<p>
Create long-running operations to run in the background in your ClickHouse cluster. Useful for large
data migrations, specify SQL commands to run in order with corresponding rollbacks, such that if the
operation fails, you rollback to a safe state.
</p>
<p>
<b>Please exercise caution!</b> This functionality is still in Alpha.
</p>
<Tabs
items={[
{
key: 'list',
label: `Operations`,
children: <OperationsList />,
},
{
key: 'create',
label: `Create new operation`,
children: <CreateNewOperation />,
},
]}
defaultActiveKey="list"
/>
<br />
</div>
)
}
================================================
FILE: frontend/src/pages/Overview/Overview.tsx
================================================
import React, { useEffect, useState } from 'react'
import { Line } from '@ant-design/charts'
import { Card, Col, Row, Tooltip, notification } from 'antd'
import InfoCircleOutlined from '@ant-design/icons/InfoCircleOutlined'
import { clickhouseTips } from './tips'
import useSWR from 'swr'
interface MetricData {
day_start: string
total: number
}
interface QueryGraphsData {
execution_count: MetricData[]
memory_usage: MetricData[]
read_bytes: MetricData[]
cpu: MetricData[]
}
export default function Overview() {
const loadData = async (url: string) => {
try {
const res = await fetch(url)
const resJson = await res.json()
const execution_count = resJson.execution_count
const memory_usage = resJson.memory_usage
const read_bytes = resJson.read_bytes
const cpu = resJson.cpu
return { execution_count, memory_usage, read_bytes, cpu }
} catch (err) {
notification.error({ message: 'Failed to load data' })
}
}
const { data, error, isLoading } = useSWR('/api/analyze/query_graphs', loadData)
const now = new Date()
const dayOfTheYear = Math.floor(
(now.getTime() - new Date(now.getFullYear(), 0, 0).getTime()) / (1000 * 60 * 60 * 24)
)
console.log(data, error, isLoading)
return isLoading ? (
<div>loading</div>
) : error ? (
<div>error</div>
) : (
<div>
<h1 style={{ textAlign: 'left' }}>Overview</h1>
<Card title="💡 ClickHouse tip of the day">{clickhouseTips[dayOfTheYear % clickhouseTips.length]}</Card>
<br />
<Row gutter={8} style={{ paddingBottom: 8 }}>
<Col span={12}>
<Card style={{ boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)' }} title="Number of queries">
<Line
data={data!.execution_count.map((dataPoint: any) => ({
...dataPoint,
day_start: dataPoint.day_start.split('T')[0],
}))}
xField={'day_start'}
yField={'total'}
xAxis={{ tickCount: 5 }}
style={{ padding: 20, height: 300 }}
color="#ffb200"
loading={isLoading}
/>
</Card>
</Col>
<Col span={12}>
<Card style={{ boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)' }} title="Data read (GB)">
<Line
data={data!.read_bytes.map((dataPoint: any) => ({
day_start: dataPoint.day_start.split('T')[0],
total: dataPoint.total / 1000000000,
}))}
xField={'day_start'}
yField={'total'}
xAxis={{ tickCount: 5 }}
style={{ padding: 20, height: 300 }}
color="#ffb200"
loading={isLoading}
/>
</Card>
</Col>
</Row>
<Row gutter={8}>
<Col span={12}>
<Card style={{ boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)' }} title="Memory usage (GB)">
<Line
data={data!.memory_usage.map((dataPoint: any) => ({
day_start: dataPoint.day_start.split('T')[0],
total: dataPoint.total / 1000000000,
}))}
xField={'day_start'}
yField={'total'}
style={{ padding: 20, height: 300 }}
color="#ffb200"
loading={isLoading}
/>
</Card>
</Col>
<Col span={12}>
<Card
style={{ boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)' }}
title={
<>
CPU usage (seconds){' '}
<Tooltip
title={`Calculated from OSCPUVirtualTimeMicroseconds metric from ClickHouse query log's ProfileEvents.`}
>
<InfoCircleOutlined />
</Tooltip>
</>
}
>
<Line
data={data!.cpu.map((dataPoint: any) => ({
day_start: dataPoint.day_start.split('T')[0],
total: dataPoint.total,
}))}
xField={'day_start'}
yField={'total'}
style={{ padding: 20, height: 300 }}
color="#ffb200"
loading={isLoading}
/>
</Card>
</Col>
</Row>
<br />
</div>
)
}
================================================
FILE: frontend/src/pages/Overview/tips.ts
================================================
export const clickhouseTips = [
`Consider benchmarking different join algorithms if your queries contain expensive joins. You may find that algorithms other than the default perform significantly better for your workloads.`,
`If you store JSON data in a VARCHAR column, consider materializing frequently acessed properties using materialized columns for much faster queries.`,
`You can use the log_comment setting to add metadata to queries that will show up on the query log, including on distributed queries. For instance, you can add a stringified JSON object as a comment to tag queries for analysis.`,
`Dictionaries can be an effective tool in large data migrations or backfills.`,
`Make sure you push as many of your query filters down to the innermost subquery for better performance. Unlike other databases, ClickHouse does not have a query planner, so you want to minimize the amount of data fetched from other shards.`,
`If a column stores values with low cardinality (e.g. country codes), use the LowCardinality data type to improve performance and reduce storage usage. A low cardinality VARCHAR would be defined as LowCardinality(VARCHAR) in the table creation query.`,
`quantile is not an exact function but rather a sampled approximation. Use quantileExactExclusive for exact results.`,
`ClickHouse is great at introspection, and its system tables contain a lot of metadata about the server. Learning what information is available where can be a great tool in debugging issues and mapping out areas of improvement. A lot of HouseWatch features are effectively wrappers over ClickHouse system tables.`,
`Killing a mutation with KILL MUTATION does not kill ongoing merges triggered by the mutation. If you absolutely need to stop ongoing merges as well, you should use SYSTEM STOP MERGES. However, you should not keep merges off for too long, as you may end up with too many parts unmerged, which is problematic for ClickHouse.`,
`Set mutations_sync=2 on a mutation to wait for all replicas to complete the mutation.`,
`ClickHouse does not support changing table engines in place, requiring you thus to create a new table and move data to it. However, rather than using INSERT to move the data over, you can use ATTACH PARTITION for near-instant operations instead, provided the tables contain the same "structure" i.e. same columns/ORDER BY/PARTITION BY.`,
`Consider benchmarking different compression algorithms for large columns for more efficient queries and storage usage.`,
]
================================================
FILE: frontend/src/pages/QueryEditor/Benchmark.tsx
================================================
import { Button, Row, Col, Card, Divider, Spin } from 'antd'
import React, { useState } from 'react'
// @ts-ignore
import { highlight, languages } from 'prismjs/components/prism-core'
import 'prismjs/components/prism-sql'
import 'prismjs/themes/prism.css'
import Editor from 'react-simple-code-editor'
import { Column } from '@ant-design/charts'
import useSWR from 'swr'
export interface BenchmarkingData {
benchmarking_result: {
query_version: string
cpu: number
read_bytes: number
memory_usage: number
duration_ms: number
network_receive_bytes: number
read_bytes_from_other_shards: number
}[]
}
const DEFAULT_QUERY1 = `SELECT number FROM system.errors errors
JOIN (
SELECT * FROM system.numbers LIMIT 1000
) numbers
ON numbers.number = toUInt64(errors.code)
SETTINGS join_algorithm = 'default'
`
const DEFAULT_QUERY2 = `SELECT number FROM system.errors errors
JOIN (
SELECT * FROM system.numbers LIMIT 1000
) numbers
ON numbers.number = toUInt64(errors.code)
SETTINGS join_algorithm = 'parallel_hash'
`
export default function QueryBenchmarking() {
const [query1, setQuery1] = useState(DEFAULT_QUERY1)
const [query2, setQuery2] = useState(DEFAULT_QUERY2)
const [runningBenchmark, setRunningBenchmark] = useState(false)
const [error, setError] = useState<{ error_location: string; error: string } | null>(null)
const [data, setData] = useState<BenchmarkingData | null>(null)
const runBenchmark = async () => {
setRunningBenchmark(true)
try {
setData(null)
setError(null)
const res = await fetch('/api/analyze/benchmark', {
method: 'POST',
body: JSON.stringify({ query1, query2 }),
headers: {
'Content-Type': 'application/json',
},
})
const resJson = await res.json()
if (resJson.error) {
setError(resJson)
} else {
setData(resJson)
}
} catch (error) {
setError({ error: String(error), error_location: '' })
}
setRunningBenchmark(false)
}
return (
<>
<p>
A simple benchmarking tool for analyzing how one query performs against another. Useful for testing
different approaches to writing the same query when optimizing for performance. Note that this tool only
runs each query once, and page cache is not cleared (this requires manual action on the node itself), so
results are best taken as an indication of direction than a full-on benchmark.
</p>
<Divider />
<Row gutter={4}>
<Col span={12}>
<p style={{ textAlign: 'center' }}>
<b>Control</b>
</p>
<Editor
value={query1}
onValueChange={code => setQuery1(code)}
highlight={code => highlight(code, languages.sql)}
padding={10}
style={{
fontFamily: '"Fira code", "Fira Mono", monospace',
fontSize: 16,
minHeight: 350,
border: '1px solid rgb(216, 216, 216)',
borderRadius: 4,
boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)',
marginBottom: 5,
}}
/>
</Col>
<Col span={12}>
<p style={{ textAlign: 'center' }}>
<b>Test</b>
</p>
<Editor
value={query2}
onValueChange={code => setQuery2(code)}
highlight={code => highlight(code, languages.sql)}
padding={10}
style={{
fontFamily: '"Fira code", "Fira Mono", monospace',
fontSize: 16,
minHeight: 350,
border: '1px solid rgb(216, 216, 216)',
borderRadius: 4,
boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)',
marginBottom: 5,
}}
/>
</Col>
</Row>
<Button
type="primary"
style={{ width: '100%', boxShadow: 'none' }}
onClick={runBenchmark}
disabled={runningBenchmark}
>
Benchmark
</Button>
<br />
<br />
{error ? (
<>
<Card
style={{ textAlign: 'center' }}
title={
<>
{error.error_location === 'benchmark'
? 'Error loading benchmark results'
: `Error on ${error.error_location} query`}
</>
}
>
<code style={{ color: '#c40000' }}>{error.error}</code>
</Card>
</>
) : data && data.benchmarking_result ? (
<>
<Row gutter={8} style={{ marginBottom: 4, height: 350 }}>
<Col span={12}>
<Card
style={{ boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)', height: 350 }}
title="Duration (ms)"
>
<Column
data={data.benchmarking_result || []}
xField="query_version"
yField="duration_ms"
height={250}
color={({ query_version }) => (query_version === 'Control' ? '#6495F9' : '#F7DA46')}
/>
</Card>
</Col>
<Col span={12}>
<Card
style={{ boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)', height: 350 }}
title="Read bytes"
>
<Column
data={data.benchmarking_result || []}
xField="query_version"
yField="read_bytes"
height={250}
color={({ query_version }) => (query_version === 'Control' ? '#6495F9' : '#F7DA46')}
/>
</Card>
</Col>
</Row>
<Row gutter={8} style={{ marginBottom: 4, height: 350 }}>
<Col span={12}>
<Card
style={{ boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)', height: 350 }}
title="CPU usage"
>
<Column
data={data.benchmarking_result || []}
xField="query_version"
yField="cpu"
height={250}
color={({ query_version }) => (query_version === 'Control' ? '#6495F9' : '#F7DA46')}
/>
</Card>
</Col>
<Col span={12}>
<Card
style={{ boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)', height: 350 }}
title="Memory usage"
>
<Column
data={data.benchmarking_result || []}
xField="query_version"
yField="memory_usage"
height={250}
color={({ query_version }) => (query_version === 'Control' ? '#6495F9' : '#F7DA46')}
/>
</Card>
</Col>
</Row>
<Row gutter={8} style={{ marginBottom: 4, height: 350 }}>
<Col span={12}>
<Card
style={{ boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)', height: 350 }}
title="Bytes received from network"
>
<Column
data={data.benchmarking_result || []}
xField="query_version"
yField="network_receive_bytes"
height={250}
color={({ query_version }) => (query_version === 'Control' ? '#6495F9' : '#F7DA46')}
/>
</Card>
</Col>
<Col span={12}>
<Card
style={{ boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)', height: 350 }}
title="Read bytes from other shards"
>
<Column
data={data.benchmarking_result || []}
xField="query_version"
yField="read_bytes_from_other_shards"
height={250}
color={({ query_version }) => (query_version === 'Control' ? '#6495F9' : '#F7DA46')}
/>
</Card>
</Col>
</Row>
</>
) : runningBenchmark ? (
<div style={{ margin: 0, textAlign: 'center' }}>
<Spin />
</div>
) : null}
</>
)
}
================================================
FILE: frontend/src/pages/QueryEditor/QueryEditor.tsx
================================================
import { Table, Button, ConfigProvider, Row, Col, Tooltip, Modal, Input, notification } from 'antd'
import React, { useState } from 'react'
// @ts-ignore
import { highlight, languages } from 'prismjs/components/prism-core'
import 'prismjs/components/prism-sql'
import 'prismjs/themes/prism.css'
import Editor from 'react-simple-code-editor'
import { v4 as uuidv4 } from 'uuid'
import SaveOutlined from '@ant-design/icons/SaveOutlined'
function CreateSavedQueryModal({
modalOpen = false,
setModalOpen,
saveQuery,
}: {
modalOpen: boolean
setModalOpen: (open: boolean) => void
saveQuery: (name: string) => Promise<void>
}) {
const [queryName, setQueryName] = useState<string>('')
return (
<>
<Modal
title="Query name"
open={modalOpen}
onOk={() => saveQuery(queryName)}
onCancel={() => setModalOpen(false)}
>
<Input value={queryName} onChange={e => setQueryName(e.target.value)} />
</Modal>
</>
)
}
export default function QueryEditor() {
const [sql, setSql] = useState(
'SELECT type, query, query_duration_ms, formatReadableSize(memory_usage)\nFROM system.query_log\nWHERE type > 1 AND is_initial_query\nORDER BY event_time DESC\nLIMIT 10'
)
const [error, setError] = useState('')
const [data, setData] = useState([{}])
const [runningQueryId, setRunningQueryId] = useState<null | string>(null)
const [modalOpen, setModalOpen] = useState(false)
const columns = data.length > 0 ? Object.keys(data[0]).map(column => ({ title: column, dataIndex: column })) : []
const saveQuery = async (queryName: string) => {
try {
const res = await fetch('/api/saved_queries', {
method: 'POST',
body: JSON.stringify({ name: queryName, query: sql }),
headers: {
'Content-Type': 'application/json',
},
})
if (String(res.status)[0] !== '2') {
throw new Error()
}
setModalOpen(false)
notification.success({ message: 'Query saved successfully' })
} catch (error) {
notification.error({ message: `Couldn't save query` })
}
}
const query = async (sql = '') => {
const queryId = uuidv4()
setRunningQueryId(queryId)
try {
setData([])
const res = await fetch('/api/analyze/query', {
method: 'POST',
body: JSON.stringify({ sql, query_id: queryId }),
headers: {
'Content-Type': 'application/json',
},
})
const resJson = await res.json()
if (resJson.error) {
setError(resJson.error)
} else {
setData(resJson.result)
}
} catch (error) {
setError(String(error))
}
setRunningQueryId(null)
}
const cancelRunningQuery = async () => {
if (runningQueryId) {
await fetch(`http://localhost:8000/api/analyze/${runningQueryId}/kill_query`, {
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
},
body: new URLSearchParams({
query_id: runningQueryId,
}),
})
setRunningQueryId(null)
setData([{}])
}
}
return (
<>
<Row>
<Col span={23}>
{' '}
<p>
<i>Note that HouseWatch does not add limits to queries automatically.</i>
</p>
</Col>
<Col span={1}>
{data && Object.keys(data[0] || {}).length > 0 ? (
<Tooltip title="Save query">
<Button style={{ background: 'transparent' }} onClick={() => setModalOpen(true)}>
<SaveOutlined />
</Button>
</Tooltip>
) : null}
</Col>
</Row>
<Editor
value={sql}
onValueChange={code => setSql(code)}
highlight={code => highlight(code, languages.sql)}
padding={10}
style={{
fontFamily: '"Fira code", "Fira Mono", monospace',
fontSize: 16,
minHeight: 200,
border: '1px solid rgb(216, 216, 216)',
borderRadius: 4,
boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)',
marginBottom: 5,
}}
/>
<Button
type="primary"
style={{ width: '100%', boxShadow: 'none' }}
onClick={() => (runningQueryId ? cancelRunningQuery() : query(sql))}
>
{runningQueryId ? 'Cancel' : 'Run'}
</Button>
<br />
<br />
<ConfigProvider renderEmpty={() => <p style={{ color: '#c40000', fontFamily: 'monospace' }}>{error}</p>}>
<Table columns={columns} dataSource={data} loading={!error && data.length < 1} scroll={{ x: 400 }} />
</ConfigProvider>
<CreateSavedQueryModal setModalOpen={setModalOpen} saveQuery={saveQuery} modalOpen={modalOpen} />
</>
)
}
================================================
FILE: frontend/src/pages/QueryEditor/QueryEditorPage.tsx
================================================
import React from 'react'
import SavedQueries from './SavedQueries'
import QueryEditor from './QueryEditor'
import { Tabs } from 'antd'
import QueryBenchmarking from './Benchmark'
import { useHistory } from 'react-router-dom'
export default function QueryEditorPage({ match }: { match: { params: { tab: string; id: string } } }) {
const history = useHistory()
let defaultActiveTab = 'run'
if (['run', 'saved_queries', 'benchmark'].includes(match.params.tab)) {
defaultActiveTab = match.params.tab
} else {
history.push('/query_editor/run')
}
return (
<>
<h1 style={{ textAlign: 'left' }}>Query editor</h1>
<Tabs
items={[
{
key: 'run',
label: `Run query`,
children: <QueryEditor />,
},
{
key: 'saved_queries',
label: `Saved queries`,
children: <SavedQueries match={match} />,
},
{
key: 'benchmark',
label: `Query benchmarking`,
children: <QueryBenchmarking />,
},
]}
defaultActiveKey={defaultActiveTab}
onChange={(tab) => history.push(`/query_editor/${tab}`)}
/>
</>
)
}
================================================
FILE: frontend/src/pages/QueryEditor/SavedQueries.tsx
================================================
import { Table, Button, Row, Col, Tooltip } from 'antd'
import React, { useEffect, useState } from 'react'
import { ColumnType } from 'antd/es/table'
import SavedQuery from './SavedQuery'
import ReloadOutlined from '@ant-design/icons/ReloadOutlined'
import { useHistory } from 'react-router-dom'
import { isoTimestampToHumanReadable } from '../../utils/dateUtils'
export interface SavedQueryData {
id: number
name: string
query: string
}
export default function SavedQueries({ match }: { match: { params: { id: string } } }) {
const [savedQueries, setSavedQueries] = useState([])
const [activeQuery, setActiveQuery] = useState<SavedQueryData | null>(null)
const history = useHistory()
const loadData = async () => {
const res = await fetch('/api/saved_queries')
const resJson = await res.json()
setSavedQueries(resJson.results)
if (match && match.params && match.params.id) {
setActiveQuery(resJson.results.find((q: SavedQueryData) => q.id === Number(match.params.id)) || null)
}
}
useEffect(() => {
loadData()
}, [])
const columns: ColumnType<{ name: string; id: number; query: string; created_at: string }>[] = [
{
title: 'Name',
dataIndex: 'name',
render: (_, item) => (
<span
style={{ color: '#1677ff', cursor: 'pointer' }}
onClick={() => {
setActiveQuery(item)
history.push(`/query_editor/saved_queries/${item.id}`)
}}
>
{item.name}
</span>
),
},
{
title: 'Created at',
render: (_, item) => (item.created_at ? isoTimestampToHumanReadable(item.created_at) : ''),
},
]
return (
<>
{activeQuery ? (
<>
<a
onClick={() => {
setActiveQuery(null)
history.push(`/query_editor/saved_queries`)
}}
style={{ float: 'right' }}
>
← Return to saved queries list
</a>
<SavedQuery {...activeQuery} />
</>
) : (
<>
<Row style={{ marginBottom: 2 }}>
<Col span={23}></Col>
<Col span={1}>
<Tooltip title="Refresh list">
<Button style={{ background: 'transparent' }} onClick={loadData}>
<ReloadOutlined />
</Button>
</Tooltip>
</Col>
</Row>
<Table columns={columns} dataSource={savedQueries} />
</>
)}
</>
)
}
================================================
FILE: frontend/src/pages/QueryEditor/SavedQuery.tsx
================================================
import { Table, ConfigProvider } from 'antd'
import React, { useEffect, useState } from 'react'
// @ts-ignore
import { highlight, languages } from 'prismjs/components/prism-core'
import 'prismjs/components/prism-sql'
import 'prismjs/themes/prism.css'
import Editor from 'react-simple-code-editor'
import { SavedQueryData } from './SavedQueries'
export default function SavedQuery({ id, query, name }: SavedQueryData) {
const [error, setError] = useState('')
const [data, setData] = useState([{}])
const columns = data.length > 0 ? Object.keys(data[0]).map((column) => ({ title: column, dataIndex: column })) : []
const loadData = async () => {
try {
setData([])
setError('')
const res = await fetch('/api/analyze/query', {
method: 'POST',
body: JSON.stringify({ sql: query }),
headers: {
'Content-Type': 'application/json',
},
})
const resJson = await res.json()
if (resJson.error) {
setError(resJson.error)
} else {
setData(resJson.result)
}
} catch (error) {
setError(String(error))
}
}
useEffect(() => {
loadData()
}, [])
return (
<>
<h2 style={{ textAlign: 'left' }}>{name}</h2>
<Editor
value={query}
onValueChange={() => {}}
highlight={(code) => highlight(code, languages.sql)}
padding={10}
style={{
fontFamily: '"Fira code", "Fira Mono", monospace',
fontSize: 16,
minHeight: 200,
border: '1px solid rgb(216, 216, 216)',
borderRadius: 4,
boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)',
marginBottom: 5,
}}
disabled
/>
<br />
<br />
<ConfigProvider renderEmpty={() => <p style={{ color: '#c40000', fontFamily: 'monospace' }}>{error}</p>}>
<Table columns={columns} dataSource={data} loading={!error && data.length < 1} scroll={{ x: 400 }} />
</ConfigProvider>
</>
)
}
================================================
FILE: frontend/src/pages/RunningQueries/RunningQueries.tsx
================================================
import { Table, Button, notification, Typography, Tooltip, Spin } from 'antd'
import { usePollingEffect } from '../../utils/usePollingEffect'
import React, { useState } from 'react'
import { ColumnType } from 'antd/es/table'
const { Paragraph } = Typography
interface RunningQueryData {
query: string
read_rows: number
read_rows_readable: string
query_id: string
total_rows_approx: number
total_rows_approx_readable: string
elapsed: number
memory_usage: string
}
function KillQueryButton({ queryId }: any) {
const [isLoading, setIsLoading] = useState(false)
const [isKilled, setIsKilled] = useState(false)
const killQuery = async () => {
setIsLoading(true)
try {
const res = await fetch(`/api/analyze/${queryId}/kill_query`, {
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
},
body: new URLSearchParams({
query_id: queryId,
}),
})
setIsKilled(true)
setIsLoading(false)
return await res.json()
} catch (err) {
setIsLoading(false)
notification.error({
message: 'Killing query failed',
})
}
}
return (
<>
{isKilled ? (
<Button disabled>Query killed</Button>
) : (
<Button danger onClick={killQuery} loading={isLoading}>
Kill query
</Button>
)}
</>
)
}
export default function RunningQueries() {
const [runningQueries, setRunningQueries] = useState([])
const [loadingRunningQueries, setLoadingRunningQueries] = useState(false)
const columns: ColumnType<RunningQueryData>[] = [
{
title: 'Query',
dataIndex: 'normalized_query',
key: 'query',
render: (_: any, item) => {
let index = 0
return (
<Paragraph
style={{ maxWidth: '100%', fontFamily: 'monospace' }}
ellipsis={{
rows: 2,
expandable: true,
}}
>
{item.query.replace(/(\?)/g, () => {
index = index + 1
return '$' + index
})}
</Paragraph>
)
},
},
{ title: 'User', dataIndex: 'user' },
{ title: 'Elapsed time', dataIndex: 'elapsed' },
{
title: 'Rows read',
dataIndex: 'read_rows',
render: (_: any, item) => (
<Tooltip title={`~${item.read_rows}/${item.total_rows_approx}`}>
~{item.read_rows_readable}/{item.total_rows_approx_readable}
</Tooltip>
),
},
{ title: 'Memory Usage', dataIndex: 'memory_usage' },
{
title: 'Actions',
render: (_: any, item) => <KillQueryButton queryId={item.query_id} />,
},
]
usePollingEffect(
async () => {
setLoadingRunningQueries(true)
const res = await fetch('/api/analyze/running_queries')
const resJson = await res.json()
setRunningQueries(resJson)
setLoadingRunningQueries(false)
},
[],
{ interval: 5000 }
)
return (
<>
<h1 style={{ textAlign: 'left' }}>Running queries {loadingRunningQueries ? <Spin /> : null}</h1>
<br />
<Table
columns={columns}
dataSource={runningQueries}
loading={runningQueries.length == 0 && loadingRunningQueries}
/>
</>
)
}
================================================
FILE: frontend/src/pages/SchemaStats/SchemaStats.tsx
================================================
// @ts-nocheck
import React, { useEffect, useState } from 'react'
import { Treemap } from '@ant-design/charts'
import { Spin, Table } from 'antd'
import { useHistory } from 'react-router-dom'
export default function Schema() {
const history = useHistory()
const testSchemaData = {
name: 'root',
children: [],
}
const [schema, setSchema] = useState([])
const defaultConfig: React.ComponentProps<typeof Treemap> = {
data: testSchemaData,
colorField: 'name',
style: { cursor: 'pointer' },
label: {
style: {
fill: 'black',
fontSize: 14,
fontWeight: 600,
},
},
drilldown: {
enabled: true,
breadCrumb: {
rootText: 'Start over',
position: 'top-left',
},
},
tooltip: {
formatter: (v) => {
const root = v.path[v.path.length - 1]
return {
name: v.name,
value: `${(v.value / 1000000).toFixed(2)}mb (${((v.value / root.value) * 100).toFixed(2)}%)`,
}
},
},
}
const [config, setConfig] = useState(defaultConfig)
const loadData = async () => {
try {
const res = await fetch('/api/analyze/tables')
const resJson = await res.json()
const filteredRes = resJson.filter((r: { total_bytes: number }) => r.total_bytes > 0)
const filteredResUrls = filteredRes
.map((fr: { name: string }) => `/api/analyze/${fr.name}/schema`)
.slice(0, 1)
const nestedRes = await Promise.all(
filteredResUrls.map((_url: string) => fetch(_url).then((res2) => res2.json()))
)
const configDataChildren = filteredRes.map((table: { name: string; total_bytes: number }) => ({
value: table.total_bytes,
...table,
}))
const configDataChildrenWithDrilldown = configDataChildren.map((child) => {
if (nestedRes[0][0].table == child.name) {
const nestedChildren = nestedRes[0].map((nR) => ({
name: nR.column,
category: nR.table,
value: nR.compressed,
}))
return { ...child, children: nestedChildren }
}
return child
})
const newConfigData = { ...config.data, children: configDataChildrenWithDrilldown }
setConfig({ ...config, data: newConfigData })
setSchema(filteredRes)
} catch {
notification.error({ message: 'Failed to load data' })
return
}
}
useEffect(() => {
loadData()
}, [])
return (
<div>
<h1 style={{ textAlign: 'left' }}>Schema stats</h1>
<h2>Largest tables</h2>
<p>
Click on the rectangles to get further information about parts and columns for the table. Note that this
only covers data stored on the connected node, not the whole cluster.
</p>
<div style={{ marginBottom: 50 }}>
{config.data.children.length < 1 ? (
<Spin />
) : (
<Treemap
{...config}
onEvent={(node, event) => {
if (event.type === 'element:click') {
history.push(`/schema/${event.data.data.name}`)
}
}}
rectStyle={{ cursor: 'pointer ' }}
/>
)}
</div>
<div>
<h2 style={{ textAlign: 'left' }}>All tables</h2>
<Table
dataSource={schema}
onRow={(table, rowIndex) => {
return {
onClick: (event) => {
history.push(`/schema/${table.name}`)
},
}
}}
rowClassName={() => 'cursor-pointer'}
columns={[
{ dataIndex: 'name', title: 'Name' },
{ dataIndex: 'readable_bytes', title: 'Size', sorter: (a, b) => a.total_bytes - b.total_bytes },
{
dataIndex: 'total_rows',
title: 'Rows',
defaultSortOrder: 'descend',
sorter: (a, b) => a.total_rows - b.total_rows,
},
{ dataIndex: 'engine', title: 'Engine' },
{ dataIndex: 'partition_key', title: 'Partition Key' },
]}
loading={config.data.children.length < 1}
/>
</div>
</div>
)
}
================================================
FILE: frontend/src/pages/SchemaStats/SchemaTable.tsx
================================================
// @ts-nocheck
import React, { useEffect } from 'react'
import { usePollingEffect } from '../../utils/usePollingEffect'
import { Treemap } from '@ant-design/charts'
import { Table, Tabs, TabsProps, notification } from 'antd'
import { useHistory } from 'react-router-dom'
function TableTreeMap({ schema, dataIndex }) {
const config = {
data: {
name: 'root',
children: schema.map((table) => ({ name: table[dataIndex], value: table.compressed, ...table })),
},
colorField: 'name',
style: { cursor: 'pointer' },
label: {
style: {
fill: 'black',
fontSize: 14,
fontWeight: 600,
},
},
drilldown: {
enabled: true,
breadCrumb: {
rootText: 'Start over',
},
},
tooltip: {
formatter: (v) => {
const root = v.path[v.path.length - 1]
return {
name: v.name,
value: `${(v.value / 1000000).toFixed(2)}mb (percentage: ${((v.value / root.value) * 100).toFixed(
2
)}%)`,
}
},
},
}
return (
<div>
<Treemap {...config} />
</div>
)
}
export function ColumnsData({ table }: { table: string }): JSX.Element {
const [schema, setSchema] = React.useState([])
const url = `/api/analyze/${table}/schema`
useEffect
usePollingEffect(
async () =>
setSchema(
await fetch(url)
.then((response) => {
return response.json()
})
.catch((err) => {
return []
})
),
[],
{ interval: 3000 } // optional
)
const schemaCols = [
{ dataIndex: 'column', title: 'Name' },
{ dataIndex: 'type', title: 'type' },
{ dataIndex: 'compressed_readable', title: 'Compressed' },
{ dataIndex: 'uncompressed', title: 'Uncompressed' },
]
return (
<>
{schema && <TableTreeMap schema={schema} dataIndex="column" />}
<div style={{ marginTop: 50 }}>
<Table dataSource={schema.map((d) => ({ id: d.column, ...d }))} columns={schemaCols} />
</div>
</>
)
}
export function PartsData({ table }: { table: string }): JSX.Element {
const [partData, setPartData] = React.useState([])
const loadData = async () => {
try {
const res = await fetch(`/api/analyze/${table}/parts`)
const resJson = await res.json()
setPartData(resJson)
} catch {
notification.error({ message: 'Failed to load data' })
}
}
useEffect(() => {
loadData()
}, [])
const schemaCols = [
{ dataIndex: 'part', title: 'Name' },
{ dataIndex: 'compressed_readable', title: 'Compressed' },
{ dataIndex: 'uncompressed', title: 'Uncompressed' },
]
return (
<>
{partData && <TableTreeMap schema={partData} dataIndex="part" />}
<div style={{ marginTop: 50 }}>
<Table dataSource={partData.map((d) => ({ id: d.part, ...d }))} columns={schemaCols} size="middle" />
</div>
</>
)
}
export default function CollapsibleTable({ match }) {
const history = useHistory()
const items: TabsProps['items'] = [
{
key: 'columns',
label: `Columns`,
children: <ColumnsData table={match.params.table} />,
},
{
key: 'parts',
label: `Parts`,
children: <PartsData table={match.params.table} />,
},
]
return (
<div>
<a onClick={() => history.push(`/schema/`)}>← Return to tables list</a>
<h1>Table: {match.params.table}</h1>
<Tabs defaultActiveKey="columns" items={items} />
</div>
)
}
================================================
FILE: frontend/src/pages/SlowQueries/ExampleQueriesTab.tsx
================================================
import React, { useEffect, useState } from 'react'
// @ts-ignore
import { highlight, languages } from 'prismjs/components/prism-core' // @ts-ignore
import 'prismjs/components/prism-sql'
import 'prismjs/components/prism-yaml'
import 'prismjs/themes/prism.css'
import Editor from 'react-simple-code-editor'
// @ts-ignore
import { Table, notification } from 'antd'
import { NoDataSpinner, QueryDetailData } from './QueryDetail'
export default function ExampleQueriesTab({ query_hash }: { query_hash: string }) {
const [data, setData] = useState<{ example_queries: QueryDetailData['example_queries'] } | null>(null)
const loadData = async () => {
try {
const res = await fetch(`/api/analyze/${query_hash}/query_examples`)
const resJson = await res.json()
setData(resJson)
} catch {
notification.error({ message: 'Failed to load data' })
}
}
useEffect(() => {
loadData()
}, [])
return data ? (
<Table
columns={[
{
title: 'Query',
dataIndex: 'query',
render: (_, item) => (
<Editor
value={item.query}
onValueChange={() => {}}
highlight={(code) => highlight(code, languages.sql)}
padding={10}
style={{
fontFamily: '"Fira code", "Fira Mono", monospace',
}}
disabled
/>
),
},
]}
dataSource={data.example_queries}
/>
) : (
NoDataSpinner
)
}
================================================
FILE: frontend/src/pages/SlowQueries/ExplainTab.tsx
================================================
import React, { useEffect, useState } from 'react'
// @ts-ignore
import { highlight, languages } from 'prismjs/components/prism-core' // @ts-ignore
import 'prismjs/components/prism-sql'
import 'prismjs/components/prism-yaml'
import 'prismjs/themes/prism.css'
import Editor from 'react-simple-code-editor'
// @ts-ignore
import { NoDataSpinner, QueryDetailData, copyToClipboard } from './QueryDetail'
import { notification } from 'antd'
export default function ExplainTab({ query_hash }: { query_hash: string }) {
const [data, setData] = useState<{ explain: QueryDetailData['explain'] } | null>(null)
const loadData = async () => {
try {
const res = await fetch(`/api/analyze/${query_hash}/query_explain`)
const resJson = await res.json()
setData(resJson)
} catch {
notification.error({ message: 'Failed to load data' })
}
}
useEffect(() => {
loadData()
}, [])
return data ? (
<div onClick={() => copyToClipboard((data.explain || [{ explain: '' }]).map((row) => row.explain).join('\n'))}>
<Editor
value={(data.explain || [{ explain: '' }]).map((row) => row.explain).join('\n')}
onValueChange={() => {}}
highlight={(code) => highlight(code, languages.yaml)}
padding={10}
style={{
fontFamily: '"Fira code", "Fira Mono", monospace',
fontSize: 12,
border: '1px solid rgb(216, 216, 216)',
borderRadius: 4,
boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)',
marginBottom: 5,
}}
disabled
className="code-editor"
/>
</div>
) : (
NoDataSpinner
)
}
================================================
FILE: frontend/src/pages/SlowQueries/MetricsTab.tsx
================================================
import React, { useEffect, useState } from 'react'
import { Line } from '@ant-design/plots'
// @ts-ignore
import { Card, Col, Row, Tooltip, notification } from 'antd'
import InfoCircleOutlined from '@ant-design/icons/InfoCircleOutlined'
import { NoDataSpinner, QueryDetailData } from './QueryDetail'
export default function MetricsTab({ query_hash }: { query_hash: string }) {
const [data, setData] = useState<Omit<QueryDetailData, 'explain' | 'normalized_query' | 'example_queries'> | null>(
null
)
const loadData = async () => {
try {
const res = await fetch(`/api/analyze/${query_hash}/query_metrics`)
const resJson = await res.json()
setData(resJson)
} catch {
notification.error({ message: 'Failed to load data' })
}
}
useEffect(() => {
loadData()
}, [])
return data ? (
<>
<br />
<Row gutter={8} style={{ paddingBottom: 8 }}>
<Col span={12}>
<Card style={{ boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)' }} title="Number of queries">
<Line
data={data.execution_count.map(dataPoint => ({
...dataPoint,
day_start: dataPoint.day_start.split('T')[0],
}))}
xField={'day_start'}
yField={'total'}
xAxis={{ tickCount: 5 }}
style={{ padding: 20, height: 300 }}
color="#ffb200"
/>
</Card>
</Col>
<Col span={12}>
<Card style={{ boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)' }} title="Data read (GB)">
<Line
data={data.read_bytes.map(dataPoint => ({
day_start: dataPoint.day_start.split('T')[0],
total: dataPoint.total / 1000000000,
}))}
xField={'day_start'}
yField={'total'}
xAxis={{ tickCount: 5 }}
style={{ padding: 20, height: 300 }}
color="#ffb200"
/>
</Card>
</Col>
</Row>
<Row gutter={8}>
<Col span={12}>
<Card style={{ boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)' }} title="Memory usage (GB)">
<Line
data={data.memory_usage.map(dataPoint => ({
day_start: dataPoint.day_start.split('T')[0],
total: dataPoint.total / 1000000000,
}))}
xField={'day_start'}
yField={'total'}
style={{ padding: 20, height: 300 }}
color="#ffb200"
/>
</Card>
</Col>
<Col span={12}>
<Card
style={{ boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)' }}
title={
<>
CPU usage (seconds){' '}
<Tooltip
title={`Calculated from OSCPUVirtualTimeMicroseconds metric from ClickHouse query log's ProfileEvents.`}
>
<span>
<InfoCircleOutlined />
</span>
</Tooltip>
</>
}
>
<Line
data={data.cpu.map(dataPoint => ({
day_start: dataPoint.day_start.split('T')[0],
total: dataPoint.total,
}))}
xField={'day_start'}
yField={'total'}
style={{ padding: 20, height: 300 }}
color="#ffb200"
/>
</Card>
</Col>
</Row>
</>
) : (
NoDataSpinner
)
}
================================================
FILE: frontend/src/pages/SlowQueries/NormalizedQueryTab.tsx
================================================
import React, { useEffect, useState } from 'react'
// @ts-ignore
import { highlight, languages } from 'prismjs/components/prism-core' // @ts-ignore
import 'prismjs/components/prism-sql'
import 'prismjs/components/prism-yaml'
import 'prismjs/themes/prism.css'
import Editor from 'react-simple-code-editor'
// @ts-ignore
import { format } from 'sql-formatter-plus'
import { NoDataSpinner, copyToClipboard } from './QueryDetail'
import { notification } from 'antd'
export default function NormalizedQueryTab({ query_hash }: { query_hash: string }) {
const [data, setData] = useState<{ query: string } | null>(null)
const loadData = async () => {
try {
const res = await fetch(`/api/analyze/${query_hash}/query_normalized`)
const resJson = await res.json()
setData(resJson)
} catch {
notification.error({ message: 'Failed to load data' })
}
}
useEffect(() => {
loadData()
}, [])
let index = 0
return data ? (
<div onClick={() => copyToClipboard(data.query)}>
<Editor
value={format(
data.query.replace(/(\?)/g, () => {
index = index + 1
return '$' + index
})
)}
onValueChange={() => {}}
highlight={(code) => highlight(code, languages.sql)}
padding={10}
style={{
fontFamily: '"Fira code", "Fira Mono", monospace',
fontSize: 16,
border: '1px solid rgb(216, 216, 216)',
borderRadius: 4,
boxShadow: '2px 2px 2px 2px rgb(217 208 208 / 20%)',
marginBottom: 5,
}}
disabled
className="code-editor"
/>
</div>
) : (
NoDataSpinner
)
}
================================================
FILE: frontend/src/pages/SlowQueries/QueryDetail.tsx
================================================
import React, { useEffect, useState } from 'react'
// @ts-ignore
import { Spin, Tabs, TabsProps, notification } from 'antd'
import { useHistory } from 'react-router-dom'
import NormalizedQueryTab from './NormalizedQueryTab'
import MetricsTab from './MetricsTab'
import ExplainTab from './ExplainTab'
import ExampleQueriesTab from './ExampleQueriesTab'
interface MetricData {
day_start: string
total: number
}
export interface QueryDetailData {
query: string
explain: {
explain: string
}[]
example_queries: {
query: string
}[]
execution_count: MetricData[]
memory_usage: MetricData[]
read_bytes: MetricData[]
cpu: MetricData[]
}
export const NoDataSpinner = (
<div style={{ height: 500 }}>
<Spin size="large" style={{ margin: 'auto', display: 'block', marginTop: 50 }} />
</div>
)
export const copyToClipboard = (value: string) => {
notification.info({
message: 'Copied to clipboard!',
placement: 'bottomRight',
duration: 1.5,
style: { fontSize: 10 },
})
navigator.clipboard.writeText(value)
}
export default function QueryDetail({ match }: { match: { params: { query_hash: string } } }) {
const history = useHistory()
const items: TabsProps['items'] = [
{
key: 'query',
label: `Query`,
children: <NormalizedQueryTab query_hash={match.params.query_hash} />,
},
{
key: 'metrics',
label: `Metrics`,
children: <MetricsTab query_hash={match.params.query_hash} />,
},
{
key: 'explain',
label: `Explain`,
children: <ExplainTab query_hash={match.params.query_hash} />,
},
{
key: 'examples',
label: `Example queries`,
children: <ExampleQueriesTab query_hash={match.params.query_hash} />,
},
]
return (
<>
<a onClick={() => history.push(`/query_performance/`)}>← Return to queries list</a>
<h1>Query analyzer</h1>
<Tabs items={items} defaultActiveKey="query" />
<br />
<br />
</>
)
}
================================================
FILE: frontend/src/pages/SlowQueries/SlowQueries.tsx
================================================
import React, { useEffect, useState } from 'react'
import { Select, Table, Typography, notification } from 'antd'
import { useHistory } from 'react-router-dom'
import { ColumnType } from 'antd/es/table'
const { Paragraph } = Typography
interface SlowQueryData {
normalized_query: string
normalized_query_hash: string
avg_duration: number
calls_per_minute: number
percentage_iops: number
percentage_runtime: number
read_bytes: number
total_read_bytes: number
}
export default function CollapsibleTable() {
const [loadingSlowQueries, setLoadingSlowQueries] = useState(false)
const [slowQueries, setSlowQueries] = useState<SlowQueryData[]>([])
const [timeRange, setTimeRange] = useState('-1w')
const history = useHistory()
const slowQueriesColumns: ColumnType<SlowQueryData>[] = [
{
title: 'Query',
dataIndex: 'normalized_query',
key: 'query',
render: (_, item) => {
let index = 0
return (
<Paragraph
className="clickable"
style={{ maxWidth: '100%', fontFamily: 'monospace' }}
ellipsis={{
rows: 2,
expandable: false,
}}
>
{item.normalized_query.replace(/SELECT.*FROM/g, 'SELECT ... FROM').replace(/(\?)/g, () => {
index = index + 1
return '$' + index
})}
</Paragraph>
)
},
},
{
title: 'Avg time (ms)',
dataIndex: 'avg_duration',
defaultSortOrder: 'descend',
render: (_, item) => <>{item.avg_duration.toFixed(0)}ms</>,
sorter: (a, b) => a.avg_duration - b.avg_duration,
},
{
title: 'Calls / min',
dataIndex: 'calls_per_minute',
render: (_, item) => <>{item.calls_per_minute.toFixed(3)}</>,
sorter: (a, b) => a.calls_per_minute - b.calls_per_minute,
},
{ title: '% of all iops', render: (_, item) => <>{item.percentage_iops.toFixed(1)}%</> },
{
title: '% of runtime',
render: (_, item) => <>{item.percentage_runtime.toFixed(1)}%</>,
dataIndex: 'percentage_runtime',
sorter: (a, b) => a.percentage_runtime - b.percentage_runtime,
},
{
title: 'Total iops',
dataIndex: 'total_read_bytes',
sorter: (a, b) => a.total_read_bytes - b.total_read_bytes,
},
]
const loadData = async (timeRange = '-1w') => {
setSlowQueries([])
setLoadingSlowQueries(true)
try {
const res = await fetch(`/api/analyze/slow_queries?time_range=${timeRange}`)
const resJson = await res.json()
const slowQueriesData = resJson.map((error: SlowQueryData, idx: number) => ({ key: idx, ...error }))
setSlowQueries(slowQueriesData)
setLoadingSlowQueries(false)
} catch {
notification.error({ message: 'Failed to load data' })
}
}
useEffect(() => {
loadData()
}, [])
return (
<div>
<h1 style={{ textAlign: 'left' }}>Query performance</h1>
<p>Click on queries to display more details.</p>
<div>
<Select
placeholder="system.query_log"
optionFilterProp="children"
options={[
{ label: 'Last week', value: '-1w' },
{ label: 'Last two weeks', value: '-2w' },
{ label: 'Last month', value: '-1m' },
{ label: 'Last three months', value: '-3m' }
]}
style={{ width: 200, float: 'right', marginBottom: 4 }}
onChange={(value) => {
setTimeRange(value)
loadData(value)
}}
showSearch={false}
value={timeRange}
/>
<Table
columns={slowQueriesColumns}
onRow={(query, _) => {
return {
onClick: () => {
history.push(`/query_performance/${query.normalized_query_hash}`)
},
}
}}
rowClassName={() => 'cursor-pointer'}
dataSource={slowQueries}
loading={loadingSlowQueries}
size="small"
/>
</div>
</div>
)
}
================================================
FILE: frontend/src/react-app-env.d.ts
================================================
/// <reference types="react-scripts" />
================================================
FILE: frontend/src/utils/dateUtils.ts
================================================
const monthNames = ["January", "February", "March", "April", "May", "June",
"July", "August", "September", "October", "November", "December"
]
export function isoTimestampToHumanReadable(isoDate: string): string {
const date = new Date(isoDate)
// no need to display the year if it's this year
const year = new Date().getFullYear() === date.getFullYear() ? '' : `, ${date.getFullYear()}`
// Prepare the date format
const formattedDate = monthNames[date.getMonth()] + ' '
+ date.getDate()
+ year + ' '
+ ('0' + date.getHours()).slice(-2) + ':'
+ ('0' + date.getMinutes()).slice(-2)
return formattedDate
}
================================================
FILE: frontend/src/utils/usePollingEffect.tsx
================================================
import { useEffect, useRef } from 'react'
export function usePollingEffect(
asyncCallback: any,
dependencies = [],
{
interval = 3000, // 3 seconds,
onCleanUp = () => {},
} = {}
) {
const timeoutIdRef = useRef<number | null>(null)
useEffect(() => {
let _stopped = false
;(async function pollingCallback() {
try {
await asyncCallback()
} finally {
// Set timeout after it finished, unless stopped
timeoutIdRef.current = !_stopped && window.setTimeout(pollingCallback, interval)
}
})()
// Clean up if dependencies change
return () => {
_stopped = true // prevent racing conditions
if (timeoutIdRef.current) {
clearTimeout(timeoutIdRef.current)
}
onCleanUp()
}
}, [...dependencies, interval])
}
================================================
FILE: frontend/tsconfig.json
================================================
{
"compilerOptions": {
"target": "es5",
"lib": ["dom", "dom.iterable", "esnext"],
"allowJs": true,
"skipLibCheck": true,
"esModuleInterop": true,
"allowSyntheticDefaultImports": true,
"strict": true,
"forceConsistentCasingInFileNames": true,
"noFallthroughCasesInSwitch": true,
"module": "esnext",
"moduleResolution": "node",
"resolveJsonModule": true,
"isolatedModules": true,
"noEmit": true,
"jsx": "preserve"
},
"include": ["src"]
}
================================================
FILE: frontend/vite.config.ts
================================================
import { defineConfig } from 'vite'
import react from '@vitejs/plugin-react'
// https://vitejs.dev/config/
export default defineConfig({
plugins: [react()],
server: {
proxy: {
"/api": {
target: "http://127.0.0.1:8000",
secure: false,
ws: true,
},
"/admin/": {
target: "http://127.0.0.1:8000",
secure: false,
ws: true,
},
"/logout": {
target: "http://127.0.0.1:8000",
secure: false,
ws: true,
},
},
},
base: "/",
build: {
outDir: "./build"
}
})
================================================
FILE: housewatch/__init__.py
================================================
================================================
FILE: housewatch/admin/__init__.py
================================================
from django.contrib import admin
from housewatch.models.preferred_replica import PreferredReplica
admin.site.register(PreferredReplica)
================================================
FILE: housewatch/admin.py
================================================
from django.utils.html import format_html
def html_link(url, text, new_tab=False):
if new_tab:
return format_html('<a href="{}" target="_blank">{}</a>', url, text)
return format_html('<a href="{}">{}</a>', url, text)
def error_span(text):
return format_html('<span style="color: red">{}</span>', text)
================================================
FILE: housewatch/ai/templates.py
================================================
NATURAL_LANGUAGE_QUERY_SYSTEM_PROMPT = """
You are a program that turns natural language queries into valid ClickHouse SQL. You do not have conversational abilities.
Given a prompt, you will reply with the appropriate SQL.
The prompts you will receive will ALWAYS come following this structure:
```
# Tables to query
## <table1 name>
<table1 schema>
## <table2 name>
<table2 schema>
# Query
<user's natural language query>
```
Your responses must ALWAYS be plain JSON with the following structure:
```json
{
"sql": "<generated ClickHouse SQL for prompt>",
"error": "<an error message if you cannot generate the SQL, defaults to null>"
}
```
Example prompt:
```
# Tables to query
## users
CREATE TABLE users (uid Int16, created_at DateTime64) ENGINE=Memory
## user_metadata
CREATE TABLE user_metadata (uid Int16, metadata String) ENGINE=Memory
# Query
give me the ID and metadata of users created in the last hour
```
Example response:
```json
{
"sql": "SELECT users.uid, user_metadata.metadata FROM users JOIN user_metadata ON users.uid = user_metadata.uid WHERE created_at > now() - INTERVAL 1 HOUR",
"error": null
}
```
Rules:
- You must only query valid columns from the tables specified under `tables_to_query`. However, you do not always need to query all the provided tables. If more than one table is provided, consider how the user may want a JOIN between some or all of the tables, but not always.
- Do not include any characters such as `\t` and `\n` in the SQL
"""
NATURAL_LANGUAGE_QUERY_USER_PROMPT = """
# Tables to query
%(tables_to_query)s
# Query
%(query)s
"""
TABLE_PROMPT = """
# {database}.{table}
{create_table_query}
"""
================================================
FILE: housewatch/api/__init__.py
================================================
================================================
FILE: housewatch/api/analyze.py
================================================
from rest_framework.viewsets import GenericViewSet
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.decorators import action
from django.conf import settings
from housewatch.clickhouse.client import run_query, existing_system_tables
from housewatch.clickhouse.queries.sql import (
SLOW_QUERIES_SQL,
SCHEMA_SQL,
QUERY_EXECUTION_COUNT_SQL,
QUERY_LOAD_SQL,
ERRORS_SQL,
QUERY_MEMORY_USAGE_SQL,
QUERY_READ_BYTES_SQL,
RUNNING_QUERIES_SQL,
KILL_QUERY_SQL,
PARTS_SQL,
NODE_STORAGE_SQL,
GET_QUERY_BY_NORMALIZED_HASH_SQL,
QUERY_CPU_USAGE_SQL,
LOGS_SQL,
LOGS_FREQUENCY_SQL,
EXPLAIN_QUERY,
BENCHMARKING_SQL,
AVAILABLE_TABLES_SQL,
TABLE_SCHEMAS_SQL,
)
from uuid import uuid4
import json
from time import sleep
import os
import openai
from housewatch.ai.templates import (
NATURAL_LANGUAGE_QUERY_SYSTEM_PROMPT,
NATURAL_LANGUAGE_QUERY_USER_PROMPT,
TABLE_PROMPT,
)
openai.api_key = os.getenv("OPENAI_API_KEY")
OPENAI_MODEL = os.getenv("OPENAI_MODEL", "gpt-3.5-turbo")
DEFAULT_DAYS = 7
TIME_RANGE_TO_CLICKHOUSE_INTERVAL = {
"-1w": "INTERVAL 1 WEEK",
"-2w": "INTERVAL 2 WEEK",
"-1m": "INTERVAL 1 MONTH",
"-3m": "INTERVAL 3 MONTH",
}
class AnalyzeViewset(GenericViewSet):
def list(self, request: Request) -> Response:
pass
@action(detail=False, methods=["GET"])
def slow_queries(self, request: Request):
ch_interval = TIME_RANGE_TO_CLICKHOUSE_INTERVAL[request.GET.get("time_range", "-1w")]
params = {"limit": 100, "date_from": f"now() - {ch_interval}"}
query_result = run_query(SLOW_QUERIES_SQL, params)
return Response(query_result)
@action(detail=True, methods=["GET"])
def query_normalized(self, request: Request, pk: str):
query_details = run_query(GET_QUERY_BY_NORMALIZED_HASH_SQL, {"normalized_query_hash": pk})
normalized_query = query_details[0]["normalized_query"]
return Response(
{
"query": normalized_query,
}
)
@action(detail=True, methods=["GET"])
def query_metrics(self, request: Request, pk: str):
days = request.GET.get("days", DEFAULT_DAYS)
conditions = "AND event_time > now() - INTERVAL 1 WEEK AND toString(normalized_query_hash) = '{}'".format(pk)
execution_count = run_query(QUERY_EXECUTION_COUNT_SQL, {"days": days, "conditions": conditions})
memory_usage = run_query(QUERY_MEMORY_USAGE_SQL, {"days": days, "conditions": conditions})
read_bytes = run_query(QUERY_READ_BYTES_SQL, {"days": days, "conditions": conditions})
cpu = run_query(QUERY_CPU_USAGE_SQL, {"days": days, "conditions": conditions})
return Response(
{"execution_count": execution_count, "memory_usage": memory_usage, "read_bytes": read_bytes, "cpu": cpu}
)
@action(detail=True, methods=["GET"])
def query_explain(self, request: Request, pk: str):
query_details = run_query(GET_QUERY_BY_NORMALIZED_HASH_SQL, {"normalized_query_hash": pk})
example_queries = query_details[0]["example_queries"]
explain = run_query(EXPLAIN_QUERY, {"query": example_queries[0]})
return Response(
{
"explain": explain,
}
)
@action(detail=True, methods=["GET"])
def query_examples(self, request: Request, pk: str):
query_details = run_query(GET_QUERY_BY_NORMALIZED_HASH_SQL, {"normalized_query_hash": pk})
example_queries = query_details[0]["example_queries"]
return Response(
{
"example_queries": [{"query": q} for q in example_queries],
}
)
@action(detail=False, methods=["GET"])
def query_graphs(self, request: Request):
days = request.GET.get("days", DEFAULT_DAYS)
execution_count = run_query(QUERY_EXECUTION_COUNT_SQL, {"days": days, "conditions": ""})
memory_usage = run_query(QUERY_MEMORY_USAGE_SQL, {"days": days, "conditions": ""})
read_bytes = run_query(QUERY_READ_BYTES_SQL, {"days": days, "conditions": ""})
cpu = run_query(QUERY_CPU_USAGE_SQL, {"days": days, "conditions": ""})
return Response(
{"execution_count": execution_count, "memory_usage": memory_usage, "read_bytes": read_bytes, "cpu": cpu}
)
@action(detail=False, methods=["POST"])
def logs(self, request: Request):
if "text_log" not in existing_system_tables:
return Response(status=418, data={"error": "text_log table does not exist"})
query_result = run_query(
LOGS_SQL, {"message": f"%{request.data['message_ilike']}%" if request.data["message_ilike"] else "%"}
)
return Response(query_result)
@action(detail=False, methods=["POST"])
def logs_frequency(self, request: Request):
if "text_log" not in existing_system_tables:
return Response(status=418, data={"error": "text_log table does not exist"})
query_result = run_query(
LOGS_FREQUENCY_SQL,
{"message": f"%{request.data['message_ilike']}%" if request.data["message_ilike"] else "%"},
)
return Response(query_result)
@action(detail=False, methods=["POST"])
def query(self, request: Request):
query_id = request.data["query_id"] if "query_id" in request.data else None
try:
query_result = run_query(request.data["sql"], query_id=query_id, use_cache=False, substitute_params=False)
except Exception as e:
return Response(status=418, data={"error": str(e)})
return Response({"result": query_result})
@action(detail=False, methods=["GET"])
def hostname(self, request: Request):
return Response({"hostname": settings.CLICKHOUSE_HOST})
@action(detail=True, methods=["GET"])
def schema(self, request: Request, pk: str):
query_result = run_query(SCHEMA_SQL, {"table": pk})
return Response(query_result)
@action(detail=True, methods=["GET"])
def parts(self, request: Request, pk: str):
query_result = run_query(PARTS_SQL, {"table": pk})
return Response(query_result)
@action(detail=False, methods=["GET"])
def query_load(self, request: Request):
params = {
"column_alias": "average_query_duration",
"math_func": "avg",
"load_metric": "query_duration_ms",
"date_to": "now()",
"date_from": "now() - INTERVAL 2 WEEK",
}
query_result = run_query(QUERY_LOAD_SQL, params)
return Response(query_result)
@action(detail=False, methods=["GET"])
def errors(self, request: Request):
params = {"date_from": "now() - INTERVAL 2 WEEK"}
query_result = run_query(ERRORS_SQL, params)
return Response(query_result)
@action(detail=False, methods=["GET"])
def running_queries(self, request: Request):
query_result = run_query(RUNNING_QUERIES_SQL, use_cache=False)
return Response(query_result)
@action(detail=True, methods=["POST"])
def kill_query(self, request: Request, pk: str):
query_result = run_query(KILL_QUERY_SQL, {"query_id": request.data["query_id"]}, use_cache=False)
return Response(query_result)
@action(detail=False, methods=["GET"])
def cluster_overview(self, request: Request):
storage_query_result = run_query(NODE_STORAGE_SQL, {})
full_result = []
for i in range(len(storage_query_result)):
node_result = {**storage_query_result[i]}
full_result.append(node_result)
return Response(full_result)
@action(detail=False, methods=["POST"])
def benchmark(self, request: Request):
query1_tag = f"benchmarking_q1_{str(uuid4())}"
query2_tag = f"benchmarking_q2_{str(uuid4())}"
error_location = None
# we use min_bytes_to_use_direct_io to try to not use the page cache
# docs: https://clickhouse.com/docs/en/operations/settings/settings#settings-min-bytes-to-use-direct-io
# it's unclear how well this works so this needs digging (see https://github.com/ClickHouse/ClickHouse/issues/36301)
try:
error_location = "Control"
query1_result = run_query(
request.data["query1"],
settings={"log_comment": query1_tag, "min_bytes_to_use_direct_io": 1},
use_cache=False,
substitute_params=False,
)
error_location = "Test"
query2_result = run_query(
request.data["query2"],
settings={"log_comment": query2_tag, "min_bytes_to_use_direct_io": 1},
use_cache=False,
substitute_params=False,
)
error_location = "benchmark"
is_result_equal = json.dumps(query1_result, default=str) == json.dumps(query2_result, default=str)
# make sure the query log populates
run_query("SYSTEM FLUSH LOGS")
benchmarking_result = []
i = 0
while len(benchmarking_result) == 0 and i < 10:
benchmarking_result = run_query(
BENCHMARKING_SQL, params={"query1_tag": query1_tag, "query2_tag": query2_tag}, use_cache=False
)
i += 1
sleep(0.5)
return Response({"is_result_equal": is_result_equal, "benchmarking_result": benchmarking_result})
except Exception as e:
return Response(status=418, data={"error": str(e), "error_location": error_location})
@action(detail=False, methods=["GET"])
def ai_tools_available(self, request: Request):
openai_api_key = os.getenv("OPENAI_API_KEY")
if not openai_api_key:
return Response(
status=400,
data={
"error": "OPENAI_API_KEY not set. To use the AI toolset you must pass in an OpenAI API key via the OPENAI_API_KEY environment variable."
},
)
return Response({"status": "ok"})
@action(detail=False, methods=["GET"])
def tables(self, request: Request):
query_result = run_query(AVAILABLE_TABLES_SQL, use_cache=False)
return Response(query_result)
@action(detail=False, methods=["POST"])
def natural_language_query(self, request: Request):
table_schema_sql_conditions = []
for full_table_name in request.data["tables_to_query"]:
database, table = full_table_name.split(">>>>>")
condition = f"(database = '{database}' AND table = '{table}')"
table_schema_sql_conditions.append(condition)
table_schemas = run_query(TABLE_SCHEMAS_SQL, {"conditions": " OR ".join(table_schema_sql_conditions)})
user_prompt_tables = ""
for row in table_schemas:
user_prompt_tables += TABLE_PROMPT.format(
database=row["database"], table=row["table"], create_table_query=row["create_table_query"]
)
final_user_prompt = NATURAL_LANGUAGE_QUERY_USER_PROMPT % {
"tables_to_query": user_prompt_tables,
"query": request.data["query"],
}
try:
completion = openai.ChatCompletion.create(
model=OPENAI_MODEL,
messages=[
{"role": "system", "content": NATURAL_LANGUAGE_QUERY_SYSTEM_PROMPT},
{"role": "user", "content": final_user_prompt},
],
)
except Exception as e:
return Response(status=418, data={"error": str(e), "sql": None})
response_json = json.loads(completion.choices[0].message["content"])
sql = response_json["sql"]
error = response_json["error"]
if error:
return Respon
gitextract_43962rdt/ ├── .github/ │ └── workflows/ │ ├── docker-api.yaml │ ├── docker-frontend.yaml │ └── release-chart.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── Dockerfile ├── LICENSE ├── README.md ├── bin/ │ ├── celery │ ├── docker │ ├── migrate │ ├── serve │ └── start ├── charts/ │ └── housewatch/ │ ├── .helmignore │ ├── Chart.yaml │ ├── templates/ │ │ ├── _helpers.tpl │ │ ├── deployment-nginx.yaml │ │ ├── deployment-web.yaml │ │ ├── deployment-worker.yaml │ │ ├── nginx-configmap.yaml │ │ ├── redis.yaml │ │ └── service.yaml │ └── values.yaml ├── docker/ │ ├── Caddyfile │ └── clickhouse-server/ │ └── config.d/ │ └── config.xml ├── docker-compose.dev.yml ├── docker-compose.yml ├── frontend/ │ ├── .gitignore │ ├── .node-version │ ├── .prettierrc │ ├── Dockerfile │ ├── Dockerfile.dev │ ├── README.md │ ├── index.html │ ├── package.json │ ├── public/ │ │ ├── fonts/ │ │ │ └── OFL.txt │ │ ├── index.html │ │ ├── manifest.json │ │ └── robots.txt │ ├── src/ │ │ ├── App.tsx │ │ ├── Layout.tsx │ │ ├── index.css │ │ ├── index.tsx │ │ ├── pages/ │ │ │ ├── AITools/ │ │ │ │ ├── AIToolsPage.tsx │ │ │ │ └── NaturalLanguageQueryEditor.tsx │ │ │ ├── Backups/ │ │ │ │ ├── Backups.tsx │ │ │ │ └── ScheduledBackups.tsx │ │ │ ├── Clusters/ │ │ │ │ └── Clusters.tsx │ │ │ ├── DiskUsage/ │ │ │ │ └── DiskUsage.tsx │ │ │ ├── Errors/ │ │ │ │ └── Errors.tsx │ │ │ ├── Logs/ │ │ │ │ └── Logs.tsx │ │ │ ├── Operations/ │ │ │ │ └── Operations.tsx │ │ │ ├── Overview/ │ │ │ │ ├── Overview.tsx │ │ │ │ └── tips.ts │ │ │ ├── QueryEditor/ │ │ │ │ ├── Benchmark.tsx │ │ │ │ ├── QueryEditor.tsx │ │ │ │ ├── QueryEditorPage.tsx │ │ │ │ ├── SavedQueries.tsx │ │ │ │ └── SavedQuery.tsx │ │ │ ├── RunningQueries/ │ │ │ │ └── RunningQueries.tsx │ │ │ ├── SchemaStats/ │ │ │ │ ├── SchemaStats.tsx │ │ │ │ └── SchemaTable.tsx │ │ │ └── SlowQueries/ │ │ │ ├── ExampleQueriesTab.tsx │ │ │ ├── ExplainTab.tsx │ │ │ ├── MetricsTab.tsx │ │ │ ├── NormalizedQueryTab.tsx │ │ │ ├── QueryDetail.tsx │ │ │ └── SlowQueries.tsx │ │ ├── react-app-env.d.ts │ │ └── utils/ │ │ ├── dateUtils.ts │ │ └── usePollingEffect.tsx │ ├── tsconfig.json │ └── vite.config.ts ├── housewatch/ │ ├── __init__.py │ ├── admin/ │ │ └── __init__.py │ ├── admin.py │ ├── ai/ │ │ └── templates.py │ ├── api/ │ │ ├── __init__.py │ │ ├── analyze.py │ │ ├── async_migration.py │ │ ├── backups.py │ │ ├── cluster.py │ │ ├── instance.py │ │ └── saved_queries.py │ ├── apps.py │ ├── asgi.py │ ├── async_migrations/ │ │ ├── __init__.py │ │ ├── async_migration_utils.py │ │ └── runner.py │ ├── celery.py │ ├── clickhouse/ │ │ ├── __init__.py │ │ ├── backups.py │ │ ├── client.py │ │ ├── clusters.py │ │ ├── queries/ │ │ │ ├── __init__.py │ │ │ └── sql.py │ │ └── table.py │ ├── gunicorn.conf.py │ ├── migrations/ │ │ ├── 0001_initial.py │ │ ├── 0002_asyncmigration_asyncmigration_unique name.py │ │ ├── 0003_asyncmigration_operations_and_more.py │ │ ├── 0004_asyncmigration_last_error.py │ │ ├── 0005_asyncmigration_finished_at.py │ │ ├── 0006_savedquery.py │ │ ├── 0007_scheduledbackup_scheduledbackuprun.py │ │ ├── 0008_remove_scheduledbackup_aws_endpoint_url_and_more.py │ │ ├── 0009_scheduledbackup_cluster_alter_scheduledbackup_id.py │ │ ├── 0010_scheduledbackup_incremental_schedule_and_more.py │ │ ├── 0011_scheduledbackup_is_sharded_and_more.py │ │ ├── 0012_preferredreplica.py │ │ └── __init__.py │ ├── models/ │ │ ├── __init__.py │ │ ├── async_migration.py │ │ ├── backup.py │ │ ├── instance.py │ │ ├── preferred_replica.py │ │ └── saved_queries.py │ ├── settings/ │ │ ├── __init__.py │ │ └── utils.py │ ├── tasks/ │ │ └── __init__.py │ ├── tests/ │ │ └── test_backup_table_fixture.sql │ ├── urls.py │ ├── utils/ │ │ ├── __init__.py │ │ └── encrypted_fields/ │ │ ├── fields.py │ │ └── hkdf.py │ ├── views.py │ └── wsgi.py ├── manage.py ├── mypy.ini ├── pyproject.toml ├── pytest.ini ├── requirements-dev.in ├── requirements-dev.txt ├── requirements.in ├── requirements.txt └── runtime.txt
SYMBOL INDEX (205 symbols across 70 files)
FILE: frontend/src/App.tsx
function App (line 5) | function App() {
FILE: frontend/src/Layout.tsx
type MenuItem (line 40) | type MenuItem = Required<MenuProps>['items'][number]
function AppLayout (line 65) | function AppLayout(): JSX.Element {
FILE: frontend/src/pages/AITools/AIToolsPage.tsx
function AIToolsPage (line 6) | function AIToolsPage() {
FILE: frontend/src/pages/AITools/NaturalLanguageQueryEditor.tsx
type TableData (line 12) | interface TableData {
function NaturalLanguageQueryEditor (line 17) | function NaturalLanguageQueryEditor() {
FILE: frontend/src/pages/Backups/Backups.tsx
type BackupRow (line 7) | interface BackupRow {
type Backups (line 23) | interface Backups {
type FieldType (line 27) | type FieldType = {
function Backups (line 37) | function Backups() {
FILE: frontend/src/pages/Backups/ScheduledBackups.tsx
type ScheduleRow (line 25) | interface ScheduleRow {
type Backups (line 41) | interface Backups {
type FieldType (line 45) | type FieldType = {
function ScheduledBackups (line 58) | function ScheduledBackups() {
FILE: frontend/src/pages/Clusters/Clusters.tsx
type ClusterNode (line 6) | interface ClusterNode {
type Cluster (line 22) | interface Cluster {
type Clusters (line 27) | interface Clusters {
function Clusters (line 31) | function Clusters() {
FILE: frontend/src/pages/DiskUsage/DiskUsage.tsx
type NodeData (line 7) | interface NodeData {
function DiskUsage (line 13) | function DiskUsage(): JSX.Element {
FILE: frontend/src/pages/Errors/Errors.tsx
type ErrorData (line 8) | interface ErrorData {
function CollapsibleTable (line 14) | function CollapsibleTable() {
FILE: frontend/src/pages/Logs/Logs.tsx
function Logs (line 7) | function Logs() {
FILE: frontend/src/pages/Operations/Operations.tsx
constant OPERATION_STATUS_TO_HUMAN (line 16) | const OPERATION_STATUS_TO_HUMAN = {
constant OPERATION_STATUS_TO_FONT_COLOR (line 28) | const OPERATION_STATUS_TO_FONT_COLOR = {
type AsyncMigrationData (line 38) | interface AsyncMigrationData {
function OperationControls (line 48) | function OperationControls({
function OperationsList (line 78) | function OperationsList(): JSX.Element {
function CreateNewOperation (line 158) | function CreateNewOperation(): JSX.Element {
function Operations (line 299) | function Operations(): JSX.Element {
FILE: frontend/src/pages/Overview/Overview.tsx
type MetricData (line 8) | interface MetricData {
type QueryGraphsData (line 13) | interface QueryGraphsData {
function Overview (line 20) | function Overview() {
FILE: frontend/src/pages/QueryEditor/Benchmark.tsx
type BenchmarkingData (line 11) | interface BenchmarkingData {
constant DEFAULT_QUERY1 (line 23) | const DEFAULT_QUERY1 = `SELECT number FROM system.errors errors
constant DEFAULT_QUERY2 (line 31) | const DEFAULT_QUERY2 = `SELECT number FROM system.errors errors
function QueryBenchmarking (line 39) | function QueryBenchmarking() {
FILE: frontend/src/pages/QueryEditor/QueryEditor.tsx
function CreateSavedQueryModal (line 11) | function CreateSavedQueryModal({
function QueryEditor (line 36) | function QueryEditor() {
FILE: frontend/src/pages/QueryEditor/QueryEditorPage.tsx
function QueryEditorPage (line 8) | function QueryEditorPage({ match }: { match: { params: { tab: string; id...
FILE: frontend/src/pages/QueryEditor/SavedQueries.tsx
type SavedQueryData (line 9) | interface SavedQueryData {
function SavedQueries (line 15) | function SavedQueries({ match }: { match: { params: { id: string } } }) {
FILE: frontend/src/pages/QueryEditor/SavedQuery.tsx
function SavedQuery (line 10) | function SavedQuery({ id, query, name }: SavedQueryData) {
FILE: frontend/src/pages/RunningQueries/RunningQueries.tsx
type RunningQueryData (line 8) | interface RunningQueryData {
function KillQueryButton (line 19) | function KillQueryButton({ queryId }: any) {
function RunningQueries (line 58) | function RunningQueries() {
FILE: frontend/src/pages/SchemaStats/SchemaStats.tsx
function Schema (line 8) | function Schema() {
FILE: frontend/src/pages/SchemaStats/SchemaTable.tsx
function TableTreeMap (line 9) | function TableTreeMap({ schema, dataIndex }) {
function ColumnsData (line 50) | function ColumnsData({ table }: { table: string }): JSX.Element {
function PartsData (line 89) | function PartsData({ table }: { table: string }): JSX.Element {
function CollapsibleTable (line 122) | function CollapsibleTable({ match }) {
FILE: frontend/src/pages/SlowQueries/ExampleQueriesTab.tsx
function ExampleQueriesTab (line 12) | function ExampleQueriesTab({ query_hash }: { query_hash: string }) {
FILE: frontend/src/pages/SlowQueries/ExplainTab.tsx
function ExplainTab (line 12) | function ExplainTab({ query_hash }: { query_hash: string }) {
FILE: frontend/src/pages/SlowQueries/MetricsTab.tsx
function MetricsTab (line 8) | function MetricsTab({ query_hash }: { query_hash: string }) {
FILE: frontend/src/pages/SlowQueries/NormalizedQueryTab.tsx
function NormalizedQueryTab (line 13) | function NormalizedQueryTab({ query_hash }: { query_hash: string }) {
FILE: frontend/src/pages/SlowQueries/QueryDetail.tsx
type MetricData (line 10) | interface MetricData {
type QueryDetailData (line 15) | interface QueryDetailData {
function QueryDetail (line 45) | function QueryDetail({ match }: { match: { params: { query_hash: string ...
FILE: frontend/src/pages/SlowQueries/SlowQueries.tsx
type SlowQueryData (line 7) | interface SlowQueryData {
function CollapsibleTable (line 18) | function CollapsibleTable() {
FILE: frontend/src/utils/dateUtils.ts
function isoTimestampToHumanReadable (line 6) | function isoTimestampToHumanReadable(isoDate: string): string {
FILE: frontend/src/utils/usePollingEffect.tsx
function usePollingEffect (line 3) | function usePollingEffect(
FILE: housewatch/admin.py
function html_link (line 4) | def html_link(url, text, new_tab=False):
function error_span (line 10) | def error_span(text):
FILE: housewatch/api/analyze.py
class AnalyzeViewset (line 54) | class AnalyzeViewset(GenericViewSet):
method list (line 55) | def list(self, request: Request) -> Response:
method slow_queries (line 59) | def slow_queries(self, request: Request):
method query_normalized (line 66) | def query_normalized(self, request: Request, pk: str):
method query_metrics (line 77) | def query_metrics(self, request: Request, pk: str):
method query_explain (line 90) | def query_explain(self, request: Request, pk: str):
method query_examples (line 102) | def query_examples(self, request: Request, pk: str):
method query_graphs (line 113) | def query_graphs(self, request: Request):
method logs (line 124) | def logs(self, request: Request):
method logs_frequency (line 133) | def logs_frequency(self, request: Request):
method query (line 143) | def query(self, request: Request):
method hostname (line 152) | def hostname(self, request: Request):
method schema (line 156) | def schema(self, request: Request, pk: str):
method parts (line 161) | def parts(self, request: Request, pk: str):
method query_load (line 166) | def query_load(self, request: Request):
method errors (line 180) | def errors(self, request: Request):
method running_queries (line 188) | def running_queries(self, request: Request):
method kill_query (line 194) | def kill_query(self, request: Request, pk: str):
method cluster_overview (line 199) | def cluster_overview(self, request: Request):
method benchmark (line 210) | def benchmark(self, request: Request):
method ai_tools_available (line 253) | def ai_tools_available(self, request: Request):
method tables (line 265) | def tables(self, request: Request):
method natural_language_query (line 270) | def natural_language_query(self, request: Request):
FILE: housewatch/api/async_migration.py
class AsyncMigrationSerializer (line 12) | class AsyncMigrationSerializer(serializers.ModelSerializer):
class Meta (line 13) | class Meta:
method create (line 42) | def create(self, validated_data):
class AsyncMigrationsViewset (line 49) | class AsyncMigrationsViewset(viewsets.ModelViewSet):
method trigger (line 54) | def trigger(self, request, **kwargs):
FILE: housewatch/api/backups.py
class BackupViewset (line 14) | class BackupViewset(GenericViewSet):
method list (line 15) | def list(self, request: Request) -> Response:
method retrieve (line 19) | def retrieve(self, request: Request, pk: str) -> Response:
method restore (line 24) | def restore(self, request: Request, pk: str) -> Response:
method create (line 28) | def create(self, request: Request) -> Response:
class ScheduledBackupSerializer (line 40) | class ScheduledBackupSerializer(serializers.ModelSerializer):
class Meta (line 41) | class Meta:
method validate (line 46) | def validate(self, data):
class ScheduledBackupViewset (line 54) | class ScheduledBackupViewset(ModelViewSet):
method run (line 59) | def run(self, request: Request, pk: str) -> Response:
FILE: housewatch/api/cluster.py
class ClusterViewset (line 12) | class ClusterViewset(GenericViewSet):
method list (line 13) | def list(self, request: Request) -> Response:
method retrieve (line 16) | def retrieve(self, request: Request, pk: str) -> Response:
FILE: housewatch/api/instance.py
class InstanceSerializer (line 14) | class InstanceSerializer(ModelSerializer):
class Meta (line 15) | class Meta:
class InstanceViewset (line 20) | class InstanceViewset(ModelViewSet):
FILE: housewatch/api/saved_queries.py
class SavedQuerySerializer (line 9) | class SavedQuerySerializer(serializers.ModelSerializer):
class Meta (line 10) | class Meta:
class SavedQueryViewset (line 16) | class SavedQueryViewset(viewsets.ModelViewSet):
FILE: housewatch/apps.py
class HouseWatchConfig (line 4) | class HouseWatchConfig(AppConfig):
FILE: housewatch/async_migrations/async_migration_utils.py
function execute_op (line 20) | def execute_op(sql: str, args=None, *, query_id: str, timeout_seconds: i...
function mark_async_migration_as_running (line 31) | def mark_async_migration_as_running(migration: AsyncMigration) -> bool:
function halt_starting_migration (line 46) | def halt_starting_migration(migration: AsyncMigration) -> bool:
function update_async_migration (line 57) | def update_async_migration(
function process_error (line 100) | def process_error(
function trigger_migration (line 125) | def trigger_migration(migration: AsyncMigration, fresh_start: bool = True):
function complete_migration (line 161) | def complete_migration(migration: AsyncMigration, email: bool = True):
FILE: housewatch/async_migrations/runner.py
function start_async_migration (line 27) | def start_async_migration(migration: AsyncMigration, ignore_posthog_vers...
function run_async_migration_operations (line 57) | def run_async_migration_operations(migration: AsyncMigration) -> bool:
function run_async_migration_next_op (line 64) | def run_async_migration_next_op(migration: AsyncMigration):
function update_migration_progress (line 130) | def update_migration_progress(migration: AsyncMigration):
function attempt_migration_rollback (line 145) | def attempt_migration_rollback(migration: AsyncMigration):
FILE: housewatch/celery.py
function setup_periodic_tasks (line 30) | def setup_periodic_tasks(sender: Celery, **kwargs):
function run_backup (line 35) | def run_backup(backup_id: str, incremental: bool = False):
function schedule_backups (line 44) | def schedule_backups():
function run_async_migration (line 87) | def run_async_migration(migration_name: str):
FILE: housewatch/clickhouse/backups.py
function execute_backup (line 19) | def execute_backup(
function get_backups (line 77) | def get_backups(cluster=None):
function get_backup (line 86) | def get_backup(backup, cluster=None):
function create_table_backup (line 95) | def create_table_backup(
function create_database_backup (line 122) | def create_database_backup(database, bucket, path, aws_key=None, aws_sec...
function run_backup (line 146) | def run_backup(backup_id, incremental=False):
function restore_backup (line 198) | def restore_backup(backup):
FILE: housewatch/clickhouse/client.py
function run_query_on_shards (line 26) | def run_query_on_shards(
function run_query (line 64) | def run_query(
FILE: housewatch/clickhouse/clusters.py
function get_clusters (line 8) | def get_clusters():
function get_cluster (line 20) | def get_cluster(cluster):
function get_shards (line 25) | def get_shards(cluster):
function get_node_per_shard (line 33) | def get_node_per_shard(cluster):
FILE: housewatch/clickhouse/table.py
function is_replicated_table (line 4) | def is_replicated_table(database, table):
function table_engine_full (line 9) | def table_engine_full(database, table):
function parse_engine (line 14) | def parse_engine(engine_full):
function is_sharded_table (line 20) | def is_sharded_table(database, table):
FILE: housewatch/gunicorn.conf.py
class CustomGunicornLogger (line 6) | class CustomGunicornLogger(glogging.Logger):
method setup (line 7) | def setup(self, cfg):
class HealthCheckFilter (line 15) | class HealthCheckFilter(logging.Filter):
method filter (line 16) | def filter(self, record):
FILE: housewatch/migrations/0001_initial.py
class Migration (line 7) | class Migration(migrations.Migration):
FILE: housewatch/migrations/0002_asyncmigration_asyncmigration_unique name.py
class Migration (line 6) | class Migration(migrations.Migration):
FILE: housewatch/migrations/0003_asyncmigration_operations_and_more.py
class Migration (line 6) | class Migration(migrations.Migration):
FILE: housewatch/migrations/0004_asyncmigration_last_error.py
class Migration (line 6) | class Migration(migrations.Migration):
FILE: housewatch/migrations/0005_asyncmigration_finished_at.py
class Migration (line 6) | class Migration(migrations.Migration):
FILE: housewatch/migrations/0006_savedquery.py
class Migration (line 6) | class Migration(migrations.Migration):
FILE: housewatch/migrations/0007_scheduledbackup_scheduledbackuprun.py
class Migration (line 8) | class Migration(migrations.Migration):
FILE: housewatch/migrations/0008_remove_scheduledbackup_aws_endpoint_url_and_more.py
class Migration (line 8) | class Migration(migrations.Migration):
FILE: housewatch/migrations/0009_scheduledbackup_cluster_alter_scheduledbackup_id.py
class Migration (line 7) | class Migration(migrations.Migration):
FILE: housewatch/migrations/0010_scheduledbackup_incremental_schedule_and_more.py
class Migration (line 6) | class Migration(migrations.Migration):
FILE: housewatch/migrations/0011_scheduledbackup_is_sharded_and_more.py
class Migration (line 6) | class Migration(migrations.Migration):
FILE: housewatch/migrations/0012_preferredreplica.py
class Migration (line 7) | class Migration(migrations.Migration):
FILE: housewatch/models/async_migration.py
class MigrationStatus (line 4) | class MigrationStatus:
class AsyncMigration (line 15) | class AsyncMigration(models.Model):
class Meta (line 16) | class Meta:
FILE: housewatch/models/backup.py
class ScheduledBackup (line 8) | class ScheduledBackup(models.Model):
method cron_schedule (line 31) | def cron_schedule(self):
method minute (line 34) | def minute(self):
method hour (line 37) | def hour(self):
method day_of_week (line 40) | def day_of_week(self):
method day_of_month (line 43) | def day_of_month(self):
method month_of_year (line 46) | def month_of_year(self):
method is_database_backup (line 49) | def is_database_backup(self):
method is_table_backup (line 52) | def is_table_backup(self):
method save (line 55) | def save(self, *args, **kwargs):
class ScheduledBackupRun (line 63) | class ScheduledBackupRun(models.Model):
FILE: housewatch/models/instance.py
class Instance (line 5) | class Instance(models.Model):
FILE: housewatch/models/preferred_replica.py
class PreferredReplica (line 5) | class PreferredReplica(models.Model):
FILE: housewatch/models/saved_queries.py
class SavedQuery (line 4) | class SavedQuery(models.Model):
FILE: housewatch/settings/__init__.py
function get_from_env (line 51) | def get_from_env(key: str, default: Any = None, *, optional: bool = Fals...
FILE: housewatch/settings/utils.py
function get_from_env (line 11) | def get_from_env(key: str, default: Any = None, *, optional: bool = Fals...
function get_list (line 25) | def get_list(text: str) -> List[str]:
FILE: housewatch/tests/test_backup_table_fixture.sql
type test_backup (line 1) | CREATE TABLE test_backup (
FILE: housewatch/urls.py
class DefaultRouterPlusPlus (line 14) | class DefaultRouterPlusPlus(ExtendedDefaultRouter):
method __init__ (line 17) | def __init__(self, *args, **kwargs):
FILE: housewatch/utils/__init__.py
function str_to_bool (line 4) | def str_to_bool(value: Any) -> bool:
FILE: housewatch/utils/encrypted_fields/fields.py
class EncryptedField (line 22) | class EncryptedField(models.Field):
method __init__ (line 27) | def __init__(self, *args, **kwargs):
method keys (line 37) | def keys(self):
method fernet_keys (line 44) | def fernet_keys(self):
method fernet (line 50) | def fernet(self):
method get_internal_type (line 55) | def get_internal_type(self):
method get_db_prep_save (line 58) | def get_db_prep_save(self, value, connection):
method from_db_value (line 64) | def from_db_value(self, value, expression, connection, *args):
method validators (line 70) | def validators(self):
function get_prep_lookup (line 81) | def get_prep_lookup(self):
class EncryptedTextField (line 94) | class EncryptedTextField(EncryptedField, models.TextField):
class EncryptedCharField (line 98) | class EncryptedCharField(EncryptedField, models.CharField):
class EncryptedEmailField (line 102) | class EncryptedEmailField(EncryptedField, models.EmailField):
class EncryptedIntegerField (line 106) | class EncryptedIntegerField(EncryptedField, models.IntegerField):
class EncryptedDateField (line 110) | class EncryptedDateField(EncryptedField, models.DateField):
class EncryptedDateTimeField (line 114) | class EncryptedDateTimeField(EncryptedField, models.DateTimeField):
FILE: housewatch/utils/encrypted_fields/hkdf.py
function derive_fernet_key (line 14) | def derive_fernet_key(input_key):
FILE: housewatch/views.py
function homepage (line 9) | def homepage(request):
function healthz (line 13) | def healthz(request):
FILE: manage.py
function main (line 7) | def main():
Condensed preview — 136 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (299K chars).
[
{
"path": ".github/workflows/docker-api.yaml",
"chars": 1342,
"preview": "name: API Docker build\n\non:\n push:\n\njobs:\n docker:\n runs-on: ubuntu-latest\n\n permissions:\n contents: read\n "
},
{
"path": ".github/workflows/docker-frontend.yaml",
"chars": 1777,
"preview": "name: Frontend Docker build\n\non:\n push:\n\njobs:\n docker:\n runs-on: ubuntu-latest\n defaults:\n run:\n wo"
},
{
"path": ".github/workflows/release-chart.yaml",
"chars": 1569,
"preview": "name: Release Chart\n\non:\n pull_request:\n paths:\n - charts/**\n - .github/workflows/release-chart.yaml\n pus"
},
{
"path": ".gitignore",
"chars": 3183,
"preview": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packagi"
},
{
"path": ".pre-commit-config.yaml",
"chars": 428,
"preview": "repos:\n - repo: https://github.com/pre-commit/pre-commit-hooks\n rev: v2.3.0\n hooks:\n - id: check-yaml\n "
},
{
"path": "Dockerfile",
"chars": 531,
"preview": "FROM python:3.10\n\nENV PYTHONUNBUFFERED 1\n\nWORKDIR /code\n\nCOPY requirements.txt ./\n\nRUN apt-get update && \\\n apt-get i"
},
{
"path": "LICENSE",
"chars": 1064,
"preview": "MIT License\n\nCopyright (c) 2023 PostHog\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof"
},
{
"path": "README.md",
"chars": 7519,
"preview": "<p align=\"center\">\n <img src=\"./banner-light.png\">\n</p>\n\n\n<p align=\"center\">\n <img src=\"./overview.png\">\n</p>\n\n\n## 📈 O"
},
{
"path": "bin/celery",
"chars": 51,
"preview": "#!/bin/bash\nset -e\n\ncelery -A housewatch worker -B\n"
},
{
"path": "bin/docker",
"chars": 46,
"preview": "#!/bin/bash\nset -e\n\n./bin/migrate\n./bin/serve\n"
},
{
"path": "bin/migrate",
"chars": 45,
"preview": "#!/bin/bash\nset -e\n\npython manage.py migrate\n"
},
{
"path": "bin/serve",
"chars": 105,
"preview": "#!/bin/bash\nexec gunicorn housewatch.wsgi -c housewatch/gunicorn.conf.py \\\n --worker-tmp-dir /dev/shm\n"
},
{
"path": "bin/start",
"chars": 79,
"preview": "#!/bin/bash\n\nset -e\n\nexport DEBUG=1\n\n./bin/celery & python manage.py runserver\n"
},
{
"path": "charts/housewatch/.helmignore",
"chars": 349,
"preview": "# Patterns to ignore when building packages.\n# This supports shell glob matching, relative path matching, and\n# negation"
},
{
"path": "charts/housewatch/Chart.yaml",
"chars": 380,
"preview": "apiVersion: v2\nname: housewatch\ndescription: Open source tool for monitoring and managing ClickHouse clusters\ntype: appl"
},
{
"path": "charts/housewatch/templates/_helpers.tpl",
"chars": 1812,
"preview": "{{/*\nExpand the name of the chart.\n*/}}\n{{- define \"housewatch.name\" -}}\n{{- default .Chart.Name .Values.nameOverride | "
},
{
"path": "charts/housewatch/templates/deployment-nginx.yaml",
"chars": 1634,
"preview": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: {{ include \"housewatch.fullname\" . }}-nginx\n labels:\n {{- inc"
},
{
"path": "charts/housewatch/templates/deployment-web.yaml",
"chars": 2518,
"preview": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: {{ include \"housewatch.fullname\" . }}\n labels:\n {{- include \""
},
{
"path": "charts/housewatch/templates/deployment-worker.yaml",
"chars": 2130,
"preview": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: {{ include \"housewatch.fullname\" . }}-worker\n labels:\n {{- in"
},
{
"path": "charts/housewatch/templates/nginx-configmap.yaml",
"chars": 1347,
"preview": "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: {{ include \"housewatch.fullname\" . }}-nginx\ndata:\n nginx.conf: |\n e"
},
{
"path": "charts/housewatch/templates/redis.yaml",
"chars": 955,
"preview": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: {{ include \"housewatch.fullname\" . }}-redis\n labels:\n {{- inc"
},
{
"path": "charts/housewatch/templates/service.yaml",
"chars": 1190,
"preview": "apiVersion: v1\nkind: Service\nmetadata:\n name: {{ include \"housewatch.fullname\" . }}\n annotations:\n {{- toYaml .Valu"
},
{
"path": "charts/housewatch/values.yaml",
"chars": 1103,
"preview": "image:\n repository: ghcr.io/posthog/housewatch/api\n frontendRepository: ghcr.io/posthog/housewatch/frontend\n tag: mai"
},
{
"path": "docker/Caddyfile",
"chars": 166,
"preview": "{\n debug\n}\n\n{$SITE_ADDRESS} {\n reverse_proxy web:3000\n reverse_proxy /api/* app:8000\n reverse_proxy /logout "
},
{
"path": "docker/clickhouse-server/config.d/config.xml",
"chars": 590,
"preview": "<clickhouse>\n <!-- Listen wildcard address to allow accepting connections from other containers and host\n network."
},
{
"path": "docker-compose.dev.yml",
"chars": 2848,
"preview": "version: \"3\"\n\nservices:\n app:\n build: .\n environment: &django_env\n DEBUG: 1\n REDIS_URL: redis://redis:6"
},
{
"path": "docker-compose.yml",
"chars": 2113,
"preview": "version: \"3\"\n\nservices:\n app:\n build: .\n environment: &django_env\n DATABASE_URL: postgres://housewatch:house"
},
{
"path": "frontend/.gitignore",
"chars": 310,
"preview": "# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.\n\n# dependencies\n/node_modules\n/.pn"
},
{
"path": "frontend/.node-version",
"chars": 7,
"preview": "20.4.0\n"
},
{
"path": "frontend/.prettierrc",
"chars": 117,
"preview": "{\n \"trailingComma\": \"es5\",\n \"tabWidth\": 4,\n \"semi\": false,\n \"singleQuote\": true,\n \"printWidth\": 120\n}\n"
},
{
"path": "frontend/Dockerfile",
"chars": 146,
"preview": "FROM alpine:latest\n\nWORKDIR /frontend\n\nCOPY build/ build/\n\nCMD [\"echo\", \"Serve the files from /frontend/build, don't run"
},
{
"path": "frontend/Dockerfile.dev",
"chars": 187,
"preview": "FROM node:20.4.0-alpine\n\nENV PNPM_HOME=\"/pnpm\"\nENV PATH=\"$PNPM_HOME:$PATH\"\n\nRUN corepack enable\n\nWORKDIR /frontend\n\nCOPY"
},
{
"path": "frontend/README.md",
"chars": 2103,
"preview": "# Getting Started with Create React App\n\nThis project was bootstrapped with [Create React App](https://github.com/facebo"
},
{
"path": "frontend/index.html",
"chars": 645,
"preview": "<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\" />\n <link rel=\"icon\" />\n <link r"
},
{
"path": "frontend/package.json",
"chars": 1834,
"preview": "{\n \"name\": \"housewatch\",\n \"version\": \"1.0.0-beta\",\n \"dependencies\": {\n \"@ant-design/charts\": \"^1.4.2\",\n "
},
{
"path": "frontend/public/fonts/OFL.txt",
"chars": 4463,
"preview": "Copyright (c) 2015 Indian Type Foundry (info@indiantypefoundry.com)\r\n\r\nThis Font Software is licensed under the SIL Open"
},
{
"path": "frontend/public/index.html",
"chars": 722,
"preview": "<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\" />\n <link rel=\"icon\" />\n <link r"
},
{
"path": "frontend/public/manifest.json",
"chars": 194,
"preview": "{\n \"short_name\": \"HouseWatch\",\n \"name\": \"HouseWatch\",\n \"icons\": [],\n \"start_url\": \".\",\n \"display\": \"stand"
},
{
"path": "frontend/public/robots.txt",
"chars": 67,
"preview": "# https://www.robotstxt.org/robotstxt.html\nUser-agent: *\nDisallow:\n"
},
{
"path": "frontend/src/App.tsx",
"chars": 289,
"preview": "import React from 'react'\nimport { BrowserRouter as Router } from 'react-router-dom'\nimport Layout from './Layout'\n\nfunc"
},
{
"path": "frontend/src/Layout.tsx",
"chars": 7021,
"preview": "// @ts-nocheck\nimport React, { useEffect, useState } from 'react'\nimport { DiskUsage } from './pages/DiskUsage/DiskUsage"
},
{
"path": "frontend/src/index.css",
"chars": 1620,
"preview": "@font-face {\n font-family: 'Hind Siliguri';\n src: url('fonts/HindSiliguri-Medium.ttf');\n}\n\nhtml {\n overflow: hi"
},
{
"path": "frontend/src/index.tsx",
"chars": 214,
"preview": "import React from 'react'\nimport ReactDOM from 'react-dom/client'\nimport './index.css'\nimport App from './App'\n\nconst ro"
},
{
"path": "frontend/src/pages/AITools/AIToolsPage.tsx",
"chars": 1364,
"preview": "import React, { useEffect, useState } from 'react'\nimport { Tabs } from 'antd'\nimport { useHistory } from 'react-router-"
},
{
"path": "frontend/src/pages/AITools/NaturalLanguageQueryEditor.tsx",
"chars": 5050,
"preview": "import React, { useEffect, useState } from 'react'\nimport { Select, Checkbox, Button, Table, ConfigProvider } from 'antd"
},
{
"path": "frontend/src/pages/Backups/Backups.tsx",
"chars": 7304,
"preview": "import React, { useEffect, useState } from 'react'\nimport { usePollingEffect } from '../../utils/usePollingEffect'\nimpor"
},
{
"path": "frontend/src/pages/Backups/ScheduledBackups.tsx",
"chars": 11897,
"preview": "import React, { useEffect, useState } from 'react'\nimport { usePollingEffect } from '../../utils/usePollingEffect'\nimpor"
},
{
"path": "frontend/src/pages/Clusters/Clusters.tsx",
"chars": 2621,
"preview": "import React, { useEffect, useState } from 'react'\nimport { ColumnType } from 'antd/es/table'\nimport { Table, Col, Row, "
},
{
"path": "frontend/src/pages/DiskUsage/DiskUsage.tsx",
"chars": 7581,
"preview": "import React, { useEffect, useState } from 'react'\nimport { Pie } from '@ant-design/plots'\nimport { Card, Spin, Row, Col"
},
{
"path": "frontend/src/pages/Errors/Errors.tsx",
"chars": 1794,
"preview": "import React, { useEffect, useState } from 'react'\nimport { Table, notification } from 'antd'\nimport { ColumnsType } fro"
},
{
"path": "frontend/src/pages/Logs/Logs.tsx",
"chars": 4632,
"preview": "import { Table, Typography, Input, Card, ConfigProvider, Empty } from 'antd'\nimport React, { useEffect, useState } from "
},
{
"path": "frontend/src/pages/Operations/Operations.tsx",
"chars": 11292,
"preview": "// @ts-nocheck\nimport React, { useEffect, useState } from 'react'\nimport { useHistory } from 'react-router-dom'\nimport E"
},
{
"path": "frontend/src/pages/Overview/Overview.tsx",
"chars": 5483,
"preview": "import React, { useEffect, useState } from 'react'\nimport { Line } from '@ant-design/charts'\nimport { Card, Col, Row, To"
},
{
"path": "frontend/src/pages/Overview/tips.ts",
"chars": 2543,
"preview": "export const clickhouseTips = [\n `Consider benchmarking different join algorithms if your queries contain expensive j"
},
{
"path": "frontend/src/pages/QueryEditor/Benchmark.tsx",
"chars": 10770,
"preview": "import { Button, Row, Col, Card, Divider, Spin } from 'antd'\nimport React, { useState } from 'react'\n// @ts-ignore\nimpor"
},
{
"path": "frontend/src/pages/QueryEditor/QueryEditor.tsx",
"chars": 5625,
"preview": "import { Table, Button, ConfigProvider, Row, Col, Tooltip, Modal, Input, notification } from 'antd'\nimport React, { useS"
},
{
"path": "frontend/src/pages/QueryEditor/QueryEditorPage.tsx",
"chars": 1471,
"preview": "import React from 'react'\nimport SavedQueries from './SavedQueries'\nimport QueryEditor from './QueryEditor'\nimport { Tab"
},
{
"path": "frontend/src/pages/QueryEditor/SavedQueries.tsx",
"chars": 3031,
"preview": "import { Table, Button, Row, Col, Tooltip } from 'antd'\nimport React, { useEffect, useState } from 'react'\nimport { Colu"
},
{
"path": "frontend/src/pages/QueryEditor/SavedQuery.tsx",
"chars": 2342,
"preview": "import { Table, ConfigProvider } from 'antd'\nimport React, { useEffect, useState } from 'react'\n// @ts-ignore\nimport { h"
},
{
"path": "frontend/src/pages/RunningQueries/RunningQueries.tsx",
"chars": 3943,
"preview": "import { Table, Button, notification, Typography, Tooltip, Spin } from 'antd'\nimport { usePollingEffect } from '../../ut"
},
{
"path": "frontend/src/pages/SchemaStats/SchemaStats.tsx",
"chars": 5130,
"preview": "// @ts-nocheck\nimport React, { useEffect, useState } from 'react'\nimport { Treemap } from '@ant-design/charts'\nimport { "
},
{
"path": "frontend/src/pages/SchemaStats/SchemaTable.tsx",
"chars": 4103,
"preview": "// @ts-nocheck\nimport React, { useEffect } from 'react'\nimport { usePollingEffect } from '../../utils/usePollingEffect'\n"
},
{
"path": "frontend/src/pages/SlowQueries/ExampleQueriesTab.tsx",
"chars": 1795,
"preview": "import React, { useEffect, useState } from 'react'\n// @ts-ignore\nimport { highlight, languages } from 'prismjs/component"
},
{
"path": "frontend/src/pages/SlowQueries/ExplainTab.tsx",
"chars": 1851,
"preview": "import React, { useEffect, useState } from 'react'\n// @ts-ignore\nimport { highlight, languages } from 'prismjs/component"
},
{
"path": "frontend/src/pages/SlowQueries/MetricsTab.tsx",
"chars": 4616,
"preview": "import React, { useEffect, useState } from 'react'\nimport { Line } from '@ant-design/plots'\n// @ts-ignore\nimport { Card,"
},
{
"path": "frontend/src/pages/SlowQueries/NormalizedQueryTab.tsx",
"chars": 1939,
"preview": "import React, { useEffect, useState } from 'react'\n// @ts-ignore\nimport { highlight, languages } from 'prismjs/component"
},
{
"path": "frontend/src/pages/SlowQueries/QueryDetail.tsx",
"chars": 2206,
"preview": "import React, { useEffect, useState } from 'react'\n// @ts-ignore\nimport { Spin, Tabs, TabsProps, notification } from 'an"
},
{
"path": "frontend/src/pages/SlowQueries/SlowQueries.tsx",
"chars": 4877,
"preview": "import React, { useEffect, useState } from 'react'\nimport { Select, Table, Typography, notification } from 'antd'\nimport"
},
{
"path": "frontend/src/react-app-env.d.ts",
"chars": 40,
"preview": "/// <reference types=\"react-scripts\" />\n"
},
{
"path": "frontend/src/utils/dateUtils.ts",
"chars": 665,
"preview": "\nconst monthNames = [\"January\", \"February\", \"March\", \"April\", \"May\", \"June\",\n\"July\", \"August\", \"September\", \"October\", \""
},
{
"path": "frontend/src/utils/usePollingEffect.tsx",
"chars": 932,
"preview": "import { useEffect, useRef } from 'react'\n\nexport function usePollingEffect(\n asyncCallback: any,\n dependencies = "
},
{
"path": "frontend/tsconfig.json",
"chars": 568,
"preview": "{\n \"compilerOptions\": {\n \"target\": \"es5\",\n \"lib\": [\"dom\", \"dom.iterable\", \"esnext\"],\n \"allowJs\":"
},
{
"path": "frontend/vite.config.ts",
"chars": 709,
"preview": "import { defineConfig } from 'vite'\nimport react from '@vitejs/plugin-react'\n\n// https://vitejs.dev/config/\nexport defau"
},
{
"path": "housewatch/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "housewatch/admin/__init__.py",
"chars": 139,
"preview": "from django.contrib import admin\n\nfrom housewatch.models.preferred_replica import PreferredReplica\n\n\nadmin.site.register"
},
{
"path": "housewatch/admin.py",
"chars": 326,
"preview": "from django.utils.html import format_html\n\n\ndef html_link(url, text, new_tab=False):\n if new_tab:\n return form"
},
{
"path": "housewatch/ai/templates.py",
"chars": 1676,
"preview": "NATURAL_LANGUAGE_QUERY_SYSTEM_PROMPT = \"\"\"\nYou are a program that turns natural language queries into valid ClickHouse S"
},
{
"path": "housewatch/api/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "housewatch/api/analyze.py",
"chars": 12380,
"preview": "from rest_framework.viewsets import GenericViewSet\nfrom rest_framework.request import Request\nfrom rest_framework.respon"
},
{
"path": "housewatch/api/async_migration.py",
"chars": 4178,
"preview": "import structlog\nfrom rest_framework import serializers, viewsets\nfrom rest_framework.decorators import action\nfrom hous"
},
{
"path": "housewatch/api/backups.py",
"chars": 2340,
"preview": "import structlog\nfrom croniter import croniter\nfrom rest_framework.decorators import action\nfrom rest_framework.request "
},
{
"path": "housewatch/api/cluster.py",
"chars": 539,
"preview": "import structlog\nfrom rest_framework.decorators import action\nfrom rest_framework.request import Request\nfrom rest_frame"
},
{
"path": "housewatch/api/instance.py",
"chars": 649,
"preview": "import structlog\nfrom rest_framework.decorators import action\nfrom rest_framework.request import Request\nfrom rest_frame"
},
{
"path": "housewatch/api/saved_queries.py",
"chars": 519,
"preview": "import structlog\nfrom rest_framework import serializers, viewsets\nfrom housewatch.models.saved_queries import SavedQuery"
},
{
"path": "housewatch/apps.py",
"chars": 127,
"preview": "from django.apps import AppConfig\n\n\nclass HouseWatchConfig(AppConfig):\n name = \"housewatch\"\n verbose_name = \"House"
},
{
"path": "housewatch/asgi.py",
"chars": 394,
"preview": "\"\"\"\nASGI config for billing project.\n\nIt exposes the ASGI callable as a module-level variable named ``application``.\n\nFo"
},
{
"path": "housewatch/async_migrations/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "housewatch/async_migrations/async_migration_utils.py",
"chars": 6148,
"preview": "from datetime import datetime\nfrom typing import Optional\n\nimport structlog\nfrom django.db import transaction\nfrom djang"
},
{
"path": "housewatch/async_migrations/runner.py",
"chars": 5983,
"preview": "import structlog\nfrom sentry_sdk.api import capture_exception\n\n\nfrom housewatch.async_migrations.async_migration_utils i"
},
{
"path": "housewatch/celery.py",
"chars": 3690,
"preview": "import os\nfrom datetime import datetime\n\nimport structlog\nfrom croniter import croniter\nfrom celery import Celery\nfrom d"
},
{
"path": "housewatch/clickhouse/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "housewatch/clickhouse/backups.py",
"chars": 7226,
"preview": "import structlog\nfrom collections import defaultdict\nfrom datetime import datetime\nfrom typing import Dict, Optional\nfro"
},
{
"path": "housewatch/clickhouse/client.py",
"chars": 3236,
"preview": "import os\nfrom typing import Dict, Optional\nfrom clickhouse_pool import ChPool\nfrom clickhouse_driver import Client\nfrom"
},
{
"path": "housewatch/clickhouse/clusters.py",
"chars": 1977,
"preview": "import random\nfrom collections import defaultdict\nfrom housewatch.clickhouse.client import run_query\n\nfrom housewatch.mo"
},
{
"path": "housewatch/clickhouse/queries/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "housewatch/clickhouse/queries/sql.py",
"chars": 8319,
"preview": "import os\n\nch_cluster = os.getenv(\"CLICKHOUSE_CLUSTER\", None)\n\nQUERY_LOG_SYSTEM_TABLE = f\"clusterAllReplicas('{ch_cluste"
},
{
"path": "housewatch/clickhouse/table.py",
"chars": 823,
"preview": "from housewatch.clickhouse.client import run_query\n\n\ndef is_replicated_table(database, table):\n QUERY = \"\"\"SELECT eng"
},
{
"path": "housewatch/gunicorn.conf.py",
"chars": 526,
"preview": "import logging\n\nfrom gunicorn import glogging\n\n\nclass CustomGunicornLogger(glogging.Logger):\n def setup(self, cfg):\n "
},
{
"path": "housewatch/migrations/0001_initial.py",
"chars": 778,
"preview": "# Generated by Django 4.1.1 on 2023-03-29 01:06\n\nfrom django.db import migrations, models\nimport django.utils.timezone\n\n"
},
{
"path": "housewatch/migrations/0002_asyncmigration_asyncmigration_unique name.py",
"chars": 1237,
"preview": "# Generated by Django 4.1.1 on 2023-03-29 16:26\n\nfrom django.db import migrations, models\n\n\nclass Migration(migrations.M"
},
{
"path": "housewatch/migrations/0003_asyncmigration_operations_and_more.py",
"chars": 635,
"preview": "# Generated by Django 4.1.1 on 2023-03-29 17:18\n\nfrom django.db import migrations, models\n\n\nclass Migration(migrations.M"
},
{
"path": "housewatch/migrations/0004_asyncmigration_last_error.py",
"chars": 451,
"preview": "# Generated by Django 4.1.1 on 2023-03-29 17:47\n\nfrom django.db import migrations, models\n\n\nclass Migration(migrations.M"
},
{
"path": "housewatch/migrations/0005_asyncmigration_finished_at.py",
"chars": 419,
"preview": "# Generated by Django 4.1.1 on 2023-03-29 20:53\n\nfrom django.db import migrations, models\n\n\nclass Migration(migrations.M"
},
{
"path": "housewatch/migrations/0006_savedquery.py",
"chars": 625,
"preview": "# Generated by Django 4.1.1 on 2023-05-25 17:44\n\nfrom django.db import migrations, models\n\n\nclass Migration(migrations.M"
},
{
"path": "housewatch/migrations/0007_scheduledbackup_scheduledbackuprun.py",
"chars": 1917,
"preview": "# Generated by Django 4.1.1 on 2023-08-17 01:06\n\nfrom django.db import migrations, models\nimport django.db.models.deleti"
},
{
"path": "housewatch/migrations/0008_remove_scheduledbackup_aws_endpoint_url_and_more.py",
"chars": 1613,
"preview": "# Generated by Django 4.1.1 on 2023-08-19 00:06\n\nfrom django.db import migrations, models\nimport django.db.models.deleti"
},
{
"path": "housewatch/migrations/0009_scheduledbackup_cluster_alter_scheduledbackup_id.py",
"chars": 665,
"preview": "# Generated by Django 4.1.1 on 2023-08-24 01:33\n\nfrom django.db import migrations, models\nimport uuid\n\n\nclass Migration("
},
{
"path": "housewatch/migrations/0010_scheduledbackup_incremental_schedule_and_more.py",
"chars": 1167,
"preview": "# Generated by Django 4.1.1 on 2023-09-12 02:19\n\nfrom django.db import migrations, models\n\n\nclass Migration(migrations.M"
},
{
"path": "housewatch/migrations/0011_scheduledbackup_is_sharded_and_more.py",
"chars": 615,
"preview": "# Generated by Django 4.1.1 on 2024-01-31 06:36\n\nfrom django.db import migrations, models\n\n\nclass Migration(migrations.M"
},
{
"path": "housewatch/migrations/0012_preferredreplica.py",
"chars": 693,
"preview": "# Generated by Django 4.1.1 on 2024-05-29 16:34\n\nfrom django.db import migrations, models\nimport uuid\n\n\nclass Migration("
},
{
"path": "housewatch/migrations/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "housewatch/models/__init__.py",
"chars": 111,
"preview": "from .instance import Instance\nfrom .backup import ScheduledBackup, ScheduledBackupRun\n\n__all__ = [\"Instance\"]\n"
},
{
"path": "housewatch/models/async_migration.py",
"chars": 1818,
"preview": "from django.db import models\n\n\nclass MigrationStatus:\n NotStarted = 0\n Running = 1\n CompletedSuccessfully = 2\n "
},
{
"path": "housewatch/models/backup.py",
"chars": 3319,
"preview": "import uuid\nfrom croniter import croniter\nfrom django.db import models\n\nfrom housewatch.utils.encrypted_fields.fields im"
},
{
"path": "housewatch/models/instance.py",
"chars": 425,
"preview": "from django.db import models\nfrom django.utils import timezone\n\n\nclass Instance(models.Model):\n created_at: models.Da"
},
{
"path": "housewatch/models/preferred_replica.py",
"chars": 388,
"preview": "import uuid\nfrom django.db import models\n\n\nclass PreferredReplica(models.Model):\n id: models.UUIDField = models.UUIDF"
},
{
"path": "housewatch/models/saved_queries.py",
"chars": 332,
"preview": "from django.db import models\n\n\nclass SavedQuery(models.Model):\n id: models.BigAutoField = models.BigAutoField(primary"
},
{
"path": "housewatch/settings/__init__.py",
"chars": 8764,
"preview": "\"\"\"\nDjango settings for housewatch project.\n\nGenerated by 'django-admin startproject' using Django 4.1.1.\n\nFor more info"
},
{
"path": "housewatch/settings/utils.py",
"chars": 833,
"preview": "import os\nfrom typing import Any, Callable, List, Optional\n\nfrom django.core.exceptions import ImproperlyConfigured\n\nfro"
},
{
"path": "housewatch/tasks/__init__.py",
"chars": 142,
"preview": "# Make tasks ready for celery autoimport\n\nfrom . import customer, report, usage\n\n__all__ = [\"customer\", \"usage\", \"report"
},
{
"path": "housewatch/tests/test_backup_table_fixture.sql",
"chars": 275,
"preview": "CREATE TABLE test_backup (\n id UUID DEFAULT generateUUIDv4(),\n name String,\n timestamp DateTime DEFAULT now()\n)"
},
{
"path": "housewatch/urls.py",
"chars": 1577,
"preview": "from django.contrib import admin\nfrom django.contrib.auth import views as auth_views\nfrom django.urls import path\nfrom r"
},
{
"path": "housewatch/utils/__init__.py",
"chars": 322,
"preview": "from typing import Any\n\n\ndef str_to_bool(value: Any) -> bool:\n \"\"\"Return whether the provided string (or any value re"
},
{
"path": "housewatch/utils/encrypted_fields/fields.py",
"chars": 3766,
"preview": "from cryptography.fernet import Fernet, MultiFernet\nfrom django.conf import settings\nfrom django.core.exceptions import "
},
{
"path": "housewatch/utils/encrypted_fields/hkdf.py",
"chars": 719,
"preview": "import base64\n\nfrom cryptography.hazmat.primitives import hashes\nfrom cryptography.hazmat.primitives.kdf.hkdf import HKD"
},
{
"path": "housewatch/views.py",
"chars": 302,
"preview": "import structlog\nfrom django.contrib.auth.decorators import login_required\nfrom django.http import JsonResponse\n\nlogger "
},
{
"path": "housewatch/wsgi.py",
"chars": 394,
"preview": "\"\"\"\nWSGI config for billing project.\n\nIt exposes the WSGI callable as a module-level variable named ``application``.\n\nFo"
},
{
"path": "manage.py",
"chars": 666,
"preview": "#!/usr/bin/env python\n\"\"\"Django's command-line utility for administrative tasks.\"\"\"\nimport os\nimport sys\n\n\ndef main():\n "
},
{
"path": "mypy.ini",
"chars": 341,
"preview": "[mypy]\nmypy_path = ./\nplugins =\n mypy_django_plugin.main,\n mypy_drf_plugin.main\nstrict_optional = True\nno_implicit"
},
{
"path": "pyproject.toml",
"chars": 685,
"preview": "[tool.black]\nline-length = 120\ntarget-version = ['py38']\nexclude = '''\n/(\n \\.git\n | \\.mypy_cache\n | \\.venv\n | \\.en"
},
{
"path": "pytest.ini",
"chars": 119,
"preview": "[pytest]\nenv =\n DEBUG=1\n TEST=1\nDJANGO_SETTINGS_MODULE = housewatch.settings\naddopts = -p no:warnings --reuse-db\n"
},
{
"path": "requirements-dev.in",
"chars": 63,
"preview": "black==23.3.0\nruff==0.0.275\npip-tools==7.3.0\npre-commit==3.3.3\n"
},
{
"path": "requirements-dev.txt",
"chars": 1030,
"preview": "#\n# This file is autogenerated by pip-compile with Python 3.11\n# by the following command:\n#\n# pip-compile requiremen"
},
{
"path": "requirements.in",
"chars": 1039,
"preview": "aiohttp==3.8.4\naiosignal==1.3.1\namqp==5.1.1\nasgiref==3.6.0\nasync-timeout==4.0.2\nattrs==22.2.0\nbilliard==3.6.4.0\ncelery=="
},
{
"path": "requirements.txt",
"chars": 4580,
"preview": "#\n# This file is autogenerated by pip-compile with Python 3.11\n# by the following command:\n#\n# pip-compile requiremen"
},
{
"path": "runtime.txt",
"chars": 14,
"preview": "python-3.8.10\n"
}
]
About this extraction
This page contains the full source code of the PostHog/HouseWatch GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 136 files (272.5 KB), approximately 66.1k tokens, and a symbol index with 205 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.