Repository: simonw/django-sql-dashboard
Branch: main
Commit: cbfba6c706d8
Files: 69
Total size: 161.5 KB
Directory structure:
gitextract_krhmgle2/
├── .dockerignore
├── .github/
│ └── workflows/
│ ├── publish.yml
│ └── test.yml
├── .gitignore
├── .readthedocs.yaml
├── Dockerfile
├── LICENSE
├── README.md
├── conftest.py
├── django_sql_dashboard/
│ ├── __init__.py
│ ├── admin.py
│ ├── apps.py
│ ├── migrations/
│ │ ├── 0001_initial.py
│ │ ├── 0002_dashboard_permissions.py
│ │ ├── 0003_update_metadata.py
│ │ ├── 0004_add_description_help_text.py
│ │ └── __init__.py
│ ├── models.py
│ ├── templates/
│ │ └── django_sql_dashboard/
│ │ ├── _css.html
│ │ ├── _script.html
│ │ ├── base.html
│ │ ├── dashboard.html
│ │ ├── saved_dashboard.html
│ │ └── widgets/
│ │ ├── _base_widget.html
│ │ ├── bar_label-bar_quantity.html
│ │ ├── big_number-label.html
│ │ ├── completed_count-total_count.html
│ │ ├── default.html
│ │ ├── error.html
│ │ ├── html.html
│ │ ├── markdown.html
│ │ └── wordcloud_count-wordcloud_word.html
│ ├── templatetags/
│ │ ├── __init__.py
│ │ └── django_sql_dashboard.py
│ ├── urls.py
│ ├── utils.py
│ └── views.py
├── docker-compose.yml
├── docs/
│ ├── .gitignore
│ ├── Makefile
│ ├── conf.py
│ ├── contributing.md
│ ├── index.md
│ ├── requirements.txt
│ ├── saved-dashboards.md
│ ├── security.md
│ ├── setup.md
│ ├── sql.md
│ └── widgets.md
├── pyproject.toml
├── pytest_plugins/
│ ├── __init__.py
│ └── pytest_use_postgresql.py
└── test_project/
├── config/
│ ├── __init__.py
│ ├── asgi.py
│ ├── settings.py
│ ├── settings_interactive.py
│ ├── urls.py
│ └── wsgi.py
├── extra_models/
│ └── models.py
├── manage.py
├── test_dashboard.py
├── test_dashboard_permissions.py
├── test_docs.py
├── test_export.py
├── test_parameters.py
├── test_save_dashboard.py
├── test_utils.py
├── test_widgets.py
└── wait-for-postgres.sh
================================================
FILE CONTENTS
================================================
================================================
FILE: .dockerignore
================================================
.venv
.vscode
__pycache__/
*.py[cod]
*$py.class
venv
.eggs
.pytest_cache
*.egg-info
.DS_Store
================================================
FILE: .github/workflows/publish.yml
================================================
name: Publish Python Package
on:
release:
types: [created]
jobs:
test:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
postgresql-version: [14, 15, 16, 17, 18]
steps:
- uses: actions/checkout@v6
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
- uses: actions/cache@v5
name: Configure pip caching
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install PostgreSQL
env:
POSTGRESQL_VERSION: ${{ matrix.postgresql-version }}
run: |
sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
sudo apt-get update
sudo apt-get -y install "postgresql-$POSTGRESQL_VERSION"
- name: Install dependencies
run: |
pip install -e . --group dev
- name: Run tests
env:
POSTGRESQL_VERSION: ${{ matrix.postgresql-version }}
run: |
export POSTGRESQL_PATH="/usr/lib/postgresql/$POSTGRESQL_VERSION/bin/postgres"
export INITDB_PATH="/usr/lib/postgresql/$POSTGRESQL_VERSION/bin/initdb"
export PYTHONPATH="pytest_plugins:test_project"
pytest
deploy:
runs-on: ubuntu-latest
needs: [test]
steps:
- uses: actions/checkout@v6
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: "3.12"
- uses: actions/cache@v5
name: Configure pip caching
with:
path: ~/.cache/pip
key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-publish-pip-
- name: Install dependencies
run: |
pip install setuptools wheel twine
- name: Publish
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
run: |
python setup.py sdist bdist_wheel
twine upload dist/*
================================================
FILE: .github/workflows/test.yml
================================================
name: Test
on: [push, pull_request]
jobs:
test:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
postgresql-version: [14, 15, 16, 17, 18]
steps:
- uses: actions/checkout@v6
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
- uses: actions/cache@v5
name: Configure pip caching
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install PostgreSQL
env:
POSTGRESQL_VERSION: ${{ matrix.postgresql-version }}
run: |
sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
sudo apt-get update
sudo apt-get -y install "postgresql-$POSTGRESQL_VERSION"
- name: Install dependencies
run: |
pip install -e . --group dev
- name: Run tests
env:
POSTGRESQL_VERSION: ${{ matrix.postgresql-version }}
run: |
export POSTGRESQL_PATH="/usr/lib/postgresql/$POSTGRESQL_VERSION/bin/postgres"
export INITDB_PATH="/usr/lib/postgresql/$POSTGRESQL_VERSION/bin/initdb"
export PYTHONPATH="pytest_plugins:test_project"
pytest
- name: Check formatting
run: black . --check
================================================
FILE: .gitignore
================================================
.venv
.vscode
__pycache__/
*.py[cod]
*$py.class
venv
.eggs
.pytest_cache
*.egg-info
.DS_Store
================================================
FILE: .readthedocs.yaml
================================================
version: 2
build:
os: ubuntu-22.04
tools:
python: "3.11"
sphinx:
configuration: docs/conf.py
formats:
- pdf
- epub
python:
install:
- requirements: docs/requirements.txt
================================================
FILE: Dockerfile
================================================
FROM python:3.9
WORKDIR /app
# Set up the minimum structure needed to install
# django_sql_dashboard's dependencies and the package itself
# in development mode.
COPY setup.py README.md .
RUN mkdir django_sql_dashboard && pip install -e '.[test]'
# We need to have postgres installed in this container
# because the automated test suite actually spins up
# (and shuts down) a database inside the container.
RUN apt-get update && apt-get install -y \
postgresql postgresql-contrib \
&& rm -rf /var/lib/apt/lists/*
# Install dependencies needed for editing documentation.
COPY docs/requirements.txt .
RUN pip install -r requirements.txt
ARG GID=1000
ARG UID=1000
# Set up a non-root user. Aside from being best practice,
# we also need to do this because the test suite refuses to
# run as the root user.
RUN groupadd -g ${GID} appuser && useradd -r -u ${UID} -g appuser appuser
USER appuser
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: README.md
================================================
# django-sql-dashboard
[](https://pypi.org/project/django-sql-dashboard/)
[](https://github.com/simonw/django-sql-dashboard/releases)
[](https://github.com/simonw/django-sql-dashboard/actions?query=workflow%3ATest)
[](http://django-sql-dashboard.datasette.io/en/latest/?badge=latest)
[](https://github.com/simonw/django-sql-dashboard/blob/main/LICENSE)
Django SQL Dashboard provides an authenticated interface for executing read-only SQL queries directly against your PostgreSQL database, bringing a useful subset of [Datasette](https://datasette.io/) to Django.
Applications include ad-hoc analysis and debugging, plus the creation of reporting dashboards that can be shared with team members or published online.
See my blog for [more about this project](https://simonwillison.net/2021/May/10/django-sql-dashboard/), including [a video demo](https://www.youtube.com/watch?v=ausrmMZkPEY).
Features include:
- Safely run read-only one or more SQL queries against your database and view the results in your browser
- Bookmark queries and share those links with other members of your team
- Create [saved dashboards](https://django-sql-dashboard.datasette.io/en/latest/saved-dashboards.html) from your queries, with full control over who can view and edit them
- [Named parameters](https://django-sql-dashboard.datasette.io/en/latest/sql.html#sql-parameters) such as `select * from entries where id = %(id)s` will be turned into form fields, allowing quick creation of interactive dashboards
- Produce [bar charts](https://django-sql-dashboard.datasette.io/en/latest/widgets.html#bar-label-bar-quantity), [progress bars](https://django-sql-dashboard.datasette.io/en/latest/widgets.html#total-count-completed-count) and more from SQL queries, with the ability to easily create new [custom dashboard widgets](https://django-sql-dashboard.datasette.io/en/latest/widgets.html#custom-widgets) using the Django template system
- Write SQL queries that safely construct and render [markdown](https://django-sql-dashboard.datasette.io/en/latest/widgets.html#markdown) and [HTML](https://django-sql-dashboard.datasette.io/en/latest/widgets.html#html)
- Export the full results of a SQL query as a downloadable CSV or TSV file, using a combination of Django's [streaming HTTP response](https://docs.djangoproject.com/en/3.2/ref/request-response/#django.http.StreamingHttpResponse) mechanism and PostgreSQL [server-side cursors](https://www.psycopg.org/docs/usage.html#server-side-cursors) to efficiently stream large amounts of data without running out of resources
- Copy and paste the results of SQL queries directly into tools such as Google Sheets or Excel
- Uses Django's authentication system, so dashboard accounts can be granted using Django's Admin tools
## Documentation
Full documentation is at [django-sql-dashboard.datasette.io](https://django-sql-dashboard.datasette.io/)
## Screenshot
## Alternatives
- [django-sql-explorer](https://github.com/groveco/django-sql-explorer) provides a related set of functionality that also works against database backends other than PostgreSQL
================================================
FILE: conftest.py
================================================
import pytest
from django.contrib.auth.models import Permission
from django_sql_dashboard.models import Dashboard
def pytest_collection_modifyitems(items):
"""Add django_db marker with databases to tests that need database access."""
for item in items:
fixturenames = getattr(item, "fixturenames", ())
# Tests using client fixtures or dashboard_db need both databases
if any(f in fixturenames for f in ("admin_client", "client", "dashboard_db")):
item.add_marker(pytest.mark.django_db(databases=["default", "dashboard"]))
@pytest.fixture
def dashboard_db(settings):
settings.DATABASES["dashboard"]["OPTIONS"] = {
"options": "-c default_transaction_read_only=on -c statement_timeout=100"
}
@pytest.fixture
def execute_sql_permission():
return Permission.objects.get(
content_type__app_label="django_sql_dashboard",
content_type__model="dashboard",
codename="execute_sql",
)
@pytest.fixture
def saved_dashboard(dashboard_db):
dashboard = Dashboard.objects.create(
slug="test",
title="Test dashboard",
description="This [supports markdown](http://example.com/)",
view_policy="public",
)
dashboard.queries.create(sql="select 11 + 33")
dashboard.queries.create(sql="select 22 + 55")
return dashboard
================================================
FILE: django_sql_dashboard/__init__.py
================================================
urls = "django_sql_dashboard.urls"
================================================
FILE: django_sql_dashboard/admin.py
================================================
from html import escape
from django.contrib import admin
from django.utils.safestring import mark_safe
from .models import Dashboard, DashboardQuery
class DashboardQueryInline(admin.TabularInline):
model = DashboardQuery
extra = 1
def has_change_permission(self, request, obj=None):
if obj is None:
return True
return obj.user_can_edit(request.user)
def get_readonly_fields(self, request, obj=None):
if not request.user.has_perm("django_sql_dashboard.execute_sql"):
return ("sql",)
else:
return tuple()
@admin.register(Dashboard)
class DashboardAdmin(admin.ModelAdmin):
list_display = ("slug", "title", "owned_by", "view_policy", "view_dashboard")
inlines = [
DashboardQueryInline,
]
raw_id_fields = ("owned_by",)
fieldsets = (
(
None,
{"fields": ("slug", "title", "description", "owned_by", "created_at")},
),
(
"Permissions",
{"fields": ("view_policy", "edit_policy", "view_group", "edit_group")},
),
)
def view_dashboard(self, obj):
return mark_safe(
'{path}'.format(path=escape(obj.get_absolute_url()))
)
def save_model(self, request, obj, form, change):
if not obj.owned_by_id:
obj.owned_by = request.user
obj.save()
def has_change_permission(self, request, obj=None):
if obj is None:
return True
if request.user.is_superuser:
return True
return obj.user_can_edit(request.user)
def get_readonly_fields(self, request, obj):
readonly_fields = ["created_at"]
if not request.user.is_superuser:
readonly_fields.append("owned_by")
return readonly_fields
def get_queryset(self, request):
if request.user.is_superuser:
# Superusers should be able to see all dashboards.
return super().get_queryset(request)
# Otherwise, show only the dashboards the user has edit access to.
return Dashboard.get_editable_by_user(request.user)
================================================
FILE: django_sql_dashboard/apps.py
================================================
from django.apps import AppConfig
class DjangoSqlDashboardConfig(AppConfig):
name = "django_sql_dashboard"
default_auto_field = "django.db.models.AutoField"
================================================
FILE: django_sql_dashboard/migrations/0001_initial.py
================================================
# Generated by Django 3.1.7 on 2021-03-13 04:32
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = []
operations = [
migrations.CreateModel(
name="Dashboard",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("slug", models.SlugField(unique=True)),
("title", models.CharField(blank=True, max_length=128)),
("description", models.TextField(blank=True)),
],
options={
"permissions": [("execute_sql", "Can execute arbitrary SQL queries")],
},
),
migrations.CreateModel(
name="DashboardQuery",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("sql", models.TextField()),
(
"dashboard",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="queries",
to="django_sql_dashboard.dashboard",
),
),
],
options={
"verbose_name_plural": "Dashboard queries",
"order_with_respect_to": "dashboard",
},
),
]
================================================
FILE: django_sql_dashboard/migrations/0002_dashboard_permissions.py
================================================
# Generated by Django 3.1.7 on 2021-03-16 16:11
import django.db.models.deletion
import django.utils.timezone
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
("django_sql_dashboard", "0001_initial"),
]
operations = [
migrations.AddField(
model_name="dashboard",
name="created_at",
field=models.DateTimeField(default=django.utils.timezone.now),
),
migrations.AddField(
model_name="dashboard",
name="edit_group",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="can_edit_dashboards",
to="auth.group",
),
),
migrations.AddField(
model_name="dashboard",
name="edit_policy",
field=models.CharField(
choices=[
("private", "Private"),
("loggedin", "Logged-in users"),
("group", "Users in group"),
("staff", "Staff users"),
("superuser", "Superusers"),
],
default="private",
max_length=10,
),
),
migrations.AddField(
model_name="dashboard",
name="owned_by",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="owned_dashboards",
to=settings.AUTH_USER_MODEL,
),
),
migrations.AddField(
model_name="dashboard",
name="view_group",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="can_view_dashboards",
to="auth.group",
),
),
migrations.AddField(
model_name="dashboard",
name="view_policy",
field=models.CharField(
choices=[
("private", "Private"),
("public", "Public"),
("unlisted", "Unlisted"),
("loggedin", "Logged-in users"),
("group", "Users in group"),
("staff", "Staff users"),
("superuser", "Superusers"),
],
default="private",
max_length=10,
),
),
]
================================================
FILE: django_sql_dashboard/migrations/0003_update_metadata.py
================================================
# Generated by Django 3.1.7 on 2021-05-08 15:44
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("auth", "0012_alter_user_first_name_max_length"),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
("django_sql_dashboard", "0002_dashboard_permissions"),
]
operations = [
migrations.AlterField(
model_name="dashboard",
name="edit_group",
field=models.ForeignKey(
blank=True,
help_text="Group that can edit, for 'Users in group' policy",
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="can_edit_dashboards",
to="auth.group",
),
),
migrations.AlterField(
model_name="dashboard",
name="edit_policy",
field=models.CharField(
choices=[
("private", "Private"),
("loggedin", "Logged-in users"),
("group", "Users in group"),
("staff", "Staff users"),
("superuser", "Superusers"),
],
default="private",
help_text="Who can edit this dashboard",
max_length=10,
),
),
migrations.AlterField(
model_name="dashboard",
name="owned_by",
field=models.ForeignKey(
blank=True,
help_text="User who owns this dashboard",
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="owned_dashboards",
to=settings.AUTH_USER_MODEL,
),
),
migrations.AlterField(
model_name="dashboard",
name="view_group",
field=models.ForeignKey(
blank=True,
help_text="Group that can view, for 'Users in group' policy",
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="can_view_dashboards",
to="auth.group",
),
),
migrations.AlterField(
model_name="dashboard",
name="view_policy",
field=models.CharField(
choices=[
("private", "Private"),
("public", "Public"),
("unlisted", "Unlisted"),
("loggedin", "Logged-in users"),
("group", "Users in group"),
("staff", "Staff users"),
("superuser", "Superusers"),
],
default="private",
help_text="Who can view this dashboard",
max_length=10,
),
),
]
================================================
FILE: django_sql_dashboard/migrations/0004_add_description_help_text.py
================================================
# Generated by Django 3.2.3 on 2021-05-25 10:58
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_sql_dashboard", "0003_update_metadata"),
]
operations = [
migrations.AlterField(
model_name="dashboard",
name="description",
field=models.TextField(
blank=True, help_text="Optional description (Markdown allowed)"
),
),
]
================================================
FILE: django_sql_dashboard/migrations/__init__.py
================================================
================================================
FILE: django_sql_dashboard/models.py
================================================
from django.conf import settings
from django.db import models
from django.urls import reverse
from django.utils import timezone
class Dashboard(models.Model):
slug = models.SlugField(unique=True)
title = models.CharField(blank=True, max_length=128)
description = models.TextField(
blank=True, help_text="Optional description (Markdown allowed)"
)
owned_by = models.ForeignKey(
settings.AUTH_USER_MODEL,
null=True,
blank=True,
on_delete=models.SET_NULL,
related_name="owned_dashboards",
help_text="User who owns this dashboard",
)
created_at = models.DateTimeField(default=timezone.now)
class ViewPolicies(models.TextChoices):
PRIVATE = ("private", "Private")
PUBLIC = ("public", "Public")
UNLISTED = ("unlisted", "Unlisted")
LOGGEDIN = ("loggedin", "Logged-in users")
GROUP = ("group", "Users in group")
STAFF = ("staff", "Staff users")
SUPERUSER = ("superuser", "Superusers")
class EditPolicies(models.TextChoices):
PRIVATE = ("private", "Private")
LOGGEDIN = ("loggedin", "Logged-in users")
GROUP = ("group", "Users in group")
STAFF = ("staff", "Staff users")
SUPERUSER = ("superuser", "Superusers")
# Permissions
view_policy = models.CharField(
max_length=10,
choices=ViewPolicies.choices,
default=ViewPolicies.PRIVATE,
help_text="Who can view this dashboard",
)
edit_policy = models.CharField(
max_length=10,
choices=EditPolicies.choices,
default=EditPolicies.PRIVATE,
help_text="Who can edit this dashboard",
)
view_group = models.ForeignKey(
"auth.Group",
null=True,
blank=True,
on_delete=models.SET_NULL,
related_name="can_view_dashboards",
help_text="Group that can view, for 'Users in group' policy",
)
edit_group = models.ForeignKey(
"auth.Group",
null=True,
blank=True,
on_delete=models.SET_NULL,
related_name="can_edit_dashboards",
help_text="Group that can edit, for 'Users in group' policy",
)
def __str__(self):
return self.title or self.slug
def view_summary(self):
s = self.get_view_policy_display()
if self.view_policy == "group":
s += ' "{}"'.format(self.view_group)
return s
def get_absolute_url(self):
return reverse("django_sql_dashboard-dashboard", args=[self.slug])
def get_edit_url(self):
return reverse("admin:django_sql_dashboard_dashboard_change", args=(self.id,))
class Meta:
permissions = [("execute_sql", "Can execute arbitrary SQL queries")]
def user_can_edit(self, user):
if not user:
return False
if self.owned_by == user:
return True
if self.edit_policy == self.EditPolicies.LOGGEDIN:
return True
if self.edit_policy == self.EditPolicies.STAFF and user.is_staff:
return True
if self.edit_policy == self.EditPolicies.SUPERUSER and user.is_superuser:
return True
if (
self.edit_policy == self.EditPolicies.GROUP
and self.edit_group
and self.edit_group.user_set.filter(pk=user.pk).exists()
):
return True
return False
@classmethod
def get_editable_by_user(cls, user):
allowed_policies = [cls.EditPolicies.LOGGEDIN]
if user.is_staff:
allowed_policies.append(cls.EditPolicies.STAFF)
if user.is_superuser:
allowed_policies.append(cls.EditPolicies.SUPERUSER)
return (
cls.objects.filter(
models.Q(owned_by=user)
| models.Q(edit_policy__in=allowed_policies)
| models.Q(edit_policy=cls.EditPolicies.GROUP, edit_group__user=user)
)
).distinct()
@classmethod
def get_visible_to_user(cls, user):
allowed_policies = [cls.ViewPolicies.PUBLIC, cls.ViewPolicies.LOGGEDIN]
if user.is_staff:
allowed_policies.append(cls.ViewPolicies.STAFF)
if user.is_superuser:
allowed_policies.append(cls.ViewPolicies.SUPERUSER)
return (
cls.objects.filter(
models.Q(owned_by=user)
| models.Q(view_policy__in=allowed_policies)
| models.Q(view_policy=cls.ViewPolicies.GROUP, view_group__user=user)
)
.annotate(
is_owner=models.ExpressionWrapper(
models.Q(owned_by__exact=user.pk),
output_field=models.BooleanField(),
)
)
.order_by("-is_owner", "slug")
).distinct()
class DashboardQuery(models.Model):
dashboard = models.ForeignKey(
Dashboard, related_name="queries", on_delete=models.CASCADE
)
sql = models.TextField()
def __str__(self):
return self.sql
class Meta:
verbose_name_plural = "Dashboard queries"
order_with_respect_to = "dashboard"
================================================
FILE: django_sql_dashboard/templates/django_sql_dashboard/_css.html
================================================
main {
margin-top: 1em;
font-family: Helvetica, sans-serif;
}
main img {
height: 2em;
}
table {
margin-top: 1em;
border-collapse: collapse;
border-spacing: 0;
}
th {
white-space: nowrap;
text-align: left;
padding: 4px;
padding-right: 1em;
border-right: 1px solid #eee;
}
td {
border-top: 1px solid #aaa;
border-right: 1px solid #eee;
padding: 4px;
vertical-align: top;
white-space: pre-wrap;
}
span.null {
color: #aaa;
font-weight: bold;
}
pre.json {
margin: 0;
}
.btn {
font-weight: 400;
cursor: pointer;
text-align: center;
vertical-align: middle;
border: 1px solid #666;
padding: 0.5em 0.8em;
font-size: 0.9rem;
line-height: 1;
border-radius: 0.25rem;
}
.query-results {
border: 1px solid #666;
border-right: none;
margin: 1em 0;
padding: 1em;
}
.query-results textarea,
.query-results pre.sql {
width: 95%;
border: 2px solid #666;
padding: 0.5em;
}
.pre-results textarea {
height: 4em;
}
div.query-parameters {
width: 80%;
display: grid;
grid-template-columns: min-content auto;
grid-gap: 0.5em;
align-items: center;
margin-bottom: 1em;
}
div.query-parameters input[type="text"] {
border: 1px solid #666;
padding: 0.5em;
}
div.query-parameters label {
text-align: right;
font-weight: bold;
display: inline-block;
padding-right: 2em;
}
.completed-count-total-count {
width: 60%;
background-color: #efefef;
height: 3em;
}
.completed-count-total-count-bar {
background-color: blue;
height: 3em;
}
.save-dashboard-form p {
width: 80%;
}
.save-dashboard-form label {
text-align: left;
font-weight: bold;
display: inline-block;
width: 20%;
}
.save-dashboard-form input {
border: 1px solid #666;
padding: 0.5em;
}
.save-dashboard-form .errorlist {
color: red;
}
.save-dashboard-form .helptext {
color: #666;
white-space: nowrap;
}
ul.dashboard-columns {
column-count: 2;
}
ul.dashboard-columns li {
break-inside: avoid;
margin-bottom: 0.3em;
}
ul.dashboard-columns li p {
margin: 0;
color: #666;
font-size: 0.8em;
}
@media (max-width: 800px) {
ul.dashboard-columns {
column-count: auto;
}
}
svg.dropdown-menu-icon {
display: inline-block;
position: relative;
top: 2px;
cursor: pointer;
opacity: 0.8;
padding-left: 6px;
}
.dropdown-menu {
border: 1px solid #ccc;
border-radius: 4px;
line-height: 1.4;
font-size: 16px;
box-shadow: 2px 2px 2px #aaa;
background-color: #fff;
z-index: 1000;
}
.dropdown-menu ul,
.dropdown-menu li {
list-style-type: none;
margin: 0;
padding: 0;
}
.dropdown-menu li {
border-bottom: 1px solid #ccc;
}
.dropdown-menu li:last-child {
border: none;
}
.dropdown-menu a:link,
.dropdown-menu a:visited,
.dropdown-menu a:hover,
.dropdown-menu a:focus
.dropdown-menu a:active {
text-decoration: none;
display: block;
padding: 4px 8px 2px 8px;
color: #222;
white-space: nowrap;
}
.dropdown-menu a:hover {
background-color: #eee;
}
.dropdown-menu .hook {
display: block;
position: absolute;
top: -5px;
left: 6px;
width: 0;
height: 0;
border-left: 5px solid transparent;
border-right: 5px solid transparent;
border-bottom: 5px solid #666;
}
================================================
FILE: django_sql_dashboard/templates/django_sql_dashboard/_script.html
================================================
================================================
FILE: django_sql_dashboard/templates/django_sql_dashboard/base.html
================================================
'.format(
escape(json.dumps(json.loads(value), indent=2))
)
)
except json.JSONDecodeError:
pass
return mark_safe(urlize(value, nofollow=True, autoescape=True))
================================================
FILE: django_sql_dashboard/urls.py
================================================
from django.urls import path
from .views import dashboard, dashboard_json, dashboard_index
urlpatterns = [
path("", dashboard_index, name="django_sql_dashboard-index"),
path("/", dashboard, name="django_sql_dashboard-dashboard"),
path(".json", dashboard_json, name="django_sql_dashboard-dashboard_json"),
]
================================================
FILE: django_sql_dashboard/utils.py
================================================
import binascii
import json
import re
import urllib.parse
from collections import namedtuple
from django.core import signing
SQL_SALT = "django_sql_dashboard:query"
signer = signing.Signer(salt=SQL_SALT)
def sign_sql(sql):
return signer.sign(sql)
def unsign_sql(signed_sql, try_object=False):
# Returns (sql, signature_verified)
# So we can handle broken signatures
# Usually this will be a regular string
try:
sql = signer.unsign(signed_sql)
return sql, True
except signing.BadSignature:
try:
value, bad_sig = signed_sql.rsplit(signer.sep, 1)
return value, False
except ValueError:
return signed_sql, False
class Row:
def __init__(self, values, columns):
self.values = values
self.columns = columns
self.zipped = dict(zip(columns, values))
def __getitem__(self, key):
if isinstance(key, int):
return self.values[key]
else:
return self.zipped[key]
def __repr__(self):
return json.dumps(self.zipped)
def displayable_rows(rows):
fixed = []
for row in rows:
fixed_row = []
for cell in row:
if isinstance(cell, (dict, list)):
cell = json.dumps(cell, default=str)
fixed_row.append(cell)
fixed.append(fixed_row)
return fixed
_named_parameters_re = re.compile(r"\%\(([^\)]+)\)s")
def extract_named_parameters(sql):
params = _named_parameters_re.findall(sql)
# Validation step: after removing params, are there
# any single `%` symbols that will confuse psycopg2?
without_params = _named_parameters_re.sub("", sql)
without_double_percents = without_params.replace("%%", "")
if "%" in without_double_percents:
raise ValueError(r"Found a single % character")
return params
def check_for_base64_upgrade(queries):
if not queries:
return
# Strip of the timing bit if there is one
queries = [q.split(":")[0] for q in queries]
# If every query is base64-encoded JSON, return a new querystring
if not all(is_valid_base64_json(query) for query in queries):
return
# Need to decode these and upgrade them to ?sql= links
sqls = []
for query in queries:
sqls.append(sign_sql(json.loads(signing.b64_decode(query.encode()))))
return "?" + urllib.parse.urlencode({"sql": sqls}, True)
def is_valid_base64_json(s):
try:
json.loads(signing.b64_decode(s.encode()))
return True
except (json.JSONDecodeError, binascii.Error, UnicodeDecodeError):
return False
_reserved_words = None
def postgresql_reserved_words(connection):
global _reserved_words
if _reserved_words is None:
with connection.cursor() as cursor:
cursor.execute("select word from pg_get_keywords() where catcode = 'R'")
_reserved_words = [row[0] for row in cursor.fetchall()]
return _reserved_words
_sort_re = re.compile('(^.*) order by "[^"]+"( desc)?$', re.DOTALL)
def apply_sort(sql, sort_column, is_desc=False):
match = _sort_re.match(sql)
if match is not None:
sql = match.group(1)
else:
sql = "select * from ({}) as results".format(sql)
return sql + ' order by "{}"{}'.format(sort_column, " desc" if is_desc else "")
================================================
FILE: django_sql_dashboard/views.py
================================================
import csv
import hashlib
import re
import time
from io import StringIO
from urllib.parse import urlencode
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.db import connections
from django.db.utils import ProgrammingError
from django.forms import CharField, ModelForm, Textarea
from django.http.response import (
HttpResponseForbidden,
HttpResponseRedirect,
JsonResponse,
StreamingHttpResponse,
)
from django.shortcuts import get_object_or_404, render
from django.utils.safestring import mark_safe
from psycopg2.extensions import quote_ident
from .models import Dashboard
from .utils import (
apply_sort,
check_for_base64_upgrade,
displayable_rows,
extract_named_parameters,
postgresql_reserved_words,
sign_sql,
unsign_sql,
)
# https://github.com/simonw/django-sql-dashboard/issues/58
MAX_REDIRECT_LENGTH = 1800
class SaveDashboardForm(ModelForm):
slug = CharField(required=False, label="URL", help_text='For example "daily-stats"')
class Meta:
model = Dashboard
fields = (
"title",
"slug",
"description",
"view_policy",
"view_group",
"edit_policy",
"edit_group",
)
widgets = {
"description": Textarea(
attrs={
"placeholder": "Optional description, shown at the top of the dashboard page (Markdown allowed)"
}
)
}
@login_required
def dashboard_index(request):
if not request.user.has_perm("django_sql_dashboard.execute_sql"):
return HttpResponseForbidden("You do not have permission to execute SQL")
sql_queries = []
too_long_so_use_post = False
save_form = SaveDashboardForm(prefix="_save")
if request.method == "POST":
# Is this an export?
if any(
k for k in request.POST.keys() if k.startswith("export_")
) and request.user.has_perm("django_sql_dashboard.execute_sql"):
if not getattr(settings, "DASHBOARD_ENABLE_FULL_EXPORT", None):
return HttpResponseForbidden("The export feature is not enabled")
return export_sql_results(request)
sqls = [sql for sql in request.POST.getlist("sql") if sql.strip()]
saving = False
# How about a save?
if request.POST.get("_save-slug"):
save_form = SaveDashboardForm(request.POST, prefix="_save")
saving = True
if save_form.is_valid():
dashboard = save_form.save(commit=False)
dashboard.owned_by = request.user
dashboard.save()
for sql in sqls:
dashboard.queries.create(sql=sql)
return HttpResponseRedirect(dashboard.get_absolute_url())
# Convert ?sql= into signed values and redirect as GET
other_pairs = [
(key, value)
for key, value in request.POST.items()
if key not in ("sql", "csrfmiddlewaretoken")
and not key.startswith("_save-")
]
signed_sqls = [sign_sql(sql) for sql in sqls if sql.strip()]
params = {
"sql": signed_sqls,
}
params.update(other_pairs)
redirect_path = request.path + "?" + urlencode(params, doseq=True)
# Is this short enough for us to redirect?
too_long_so_use_post = len(redirect_path) > MAX_REDIRECT_LENGTH
if not saving and not too_long_so_use_post:
return HttpResponseRedirect(redirect_path)
else:
sql_queries = sqls
unverified_sql_queries = []
for signed_sql in request.GET.getlist("sql"):
sql, signature_verified = unsign_sql(signed_sql)
if signature_verified:
sql_queries.append(sql)
else:
unverified_sql_queries.append(sql)
if getattr(settings, "DASHBOARD_UPGRADE_OLD_BASE64_LINKS", None):
redirect_querystring = check_for_base64_upgrade(sql_queries)
if redirect_querystring:
return HttpResponseRedirect(request.path + redirect_querystring)
return _dashboard_index(
request,
sql_queries,
unverified_sql_queries=unverified_sql_queries,
too_long_so_use_post=too_long_so_use_post,
extra_context={"save_form": save_form},
)
def _dashboard_index(
request,
sql_queries,
unverified_sql_queries=None,
title=None,
description=None,
dashboard=None,
too_long_so_use_post=False,
template="django_sql_dashboard/dashboard.html",
extra_context=None,
json_mode=False,
):
query_results = []
alias = getattr(settings, "DASHBOARD_DB_ALIAS", "dashboard")
row_limit = getattr(settings, "DASHBOARD_ROW_LIMIT", None) or 100
connection = connections[alias]
reserved_words = postgresql_reserved_words(connection)
with connection.cursor() as tables_cursor:
tables_cursor.execute(
"""
with visible_tables as (
select table_name
from information_schema.tables
where table_schema = 'public'
order by table_name
),
reserved_keywords as (
select word
from pg_get_keywords()
where catcode = 'R'
)
select
information_schema.columns.table_name,
array_to_json(array_agg(cast(column_name as text) order by ordinal_position)) as columns
from
information_schema.columns
join
visible_tables on
information_schema.columns.table_name = visible_tables.table_name
where
information_schema.columns.table_schema = 'public'
group by
information_schema.columns.table_name
order by
information_schema.columns.table_name
"""
)
fetched = tables_cursor.fetchall()
available_tables = [
{
"name": row[0],
"columns": ", ".join(row[1]),
"sql_columns": ", ".join(
[
'"{}"'.format(column) if column in reserved_words else column
for column in row[1]
]
),
}
for row in fetched
]
parameters = []
sql_query_parameter_errors = []
for sql in sql_queries:
try:
extracted = extract_named_parameters(sql)
for p in extracted:
if p not in parameters:
parameters.append(p)
sql_query_parameter_errors.append(False)
except ValueError as e:
if "%" in sql:
sql_query_parameter_errors.append(
r"Invalid query - try escaping single '%' as double '%%'"
)
else:
sql_query_parameter_errors.append(str(e))
parameter_values = {
parameter: request.POST.get(parameter, request.GET.get(parameter, ""))
for parameter in parameters
if parameter != "sql"
}
extra_qs = "&{}".format(urlencode(parameter_values)) if parameter_values else ""
results_index = -1
if sql_queries:
for sql, parameter_error in zip(sql_queries, sql_query_parameter_errors):
results_index += 1
sql = sql.strip().rstrip(";")
base_error_result = {
"index": str(results_index),
"sql": sql,
"textarea_rows": min(5, len(sql.split("\n"))),
"rows": [],
"row_lists": [],
"description": [],
"columns": [],
"column_details": [],
"truncated": False,
"extra_qs": extra_qs,
"error": None,
"templates": ["django_sql_dashboard/widgets/error.html"],
}
if parameter_error:
query_results.append(
dict(
base_error_result,
error=parameter_error,
)
)
continue
if ";" in sql:
query_results.append(
dict(base_error_result, error="';' not allowed in SQL queries")
)
continue
with connection.cursor() as cursor:
duration_ms = None
try:
cursor.execute("BEGIN;")
start = time.perf_counter()
# Running a SELECT prevents future SET TRANSACTION READ WRITE:
cursor.execute("SELECT 1;")
cursor.fetchall()
cursor.execute(sql, parameter_values)
try:
rows = list(cursor.fetchmany(row_limit + 1))
except ProgrammingError as e:
rows = [{"statusmessage": str(cursor.statusmessage)}]
duration_ms = (time.perf_counter() - start) * 1000.0
except Exception as e:
query_results.append(dict(base_error_result, error=str(e)))
else:
templates = ["django_sql_dashboard/widgets/default.html"]
columns = [c.name for c in cursor.description]
template_name = ("-".join(sorted(columns))) + ".html"
if len(template_name) < 255:
templates.insert(
0,
"django_sql_dashboard/widgets/" + template_name,
)
display_rows = displayable_rows(rows[:row_limit])
column_details = [
{
"name": column,
"is_unambiguous": columns.count(column) == 1,
"sort_sql": apply_sort(sql, column),
"sort_desc_sql": apply_sort(sql, column, True),
}
for column in columns
]
query_results.append(
{
"index": str(results_index),
"sql": sql,
"textarea_rows": len(sql.split("\n")),
"rows": [dict(zip(columns, row)) for row in display_rows],
"row_lists": display_rows,
"description": cursor.description,
"columns": columns,
"column_details": column_details,
"truncated": len(rows) == row_limit + 1,
"extra_qs": extra_qs,
"duration_ms": duration_ms,
"templates": templates,
}
)
finally:
cursor.execute("ROLLBACK;")
# Page title, composed of truncated SQL queries
html_title = "SQL Dashboard"
if sql_queries:
html_title = "SQL: " + " [,] ".join(sql_queries)
if dashboard and dashboard.title:
html_title = dashboard.title
# Add named parameter values, if any exist
provided_values = {
key: value for key, value in parameter_values.items() if value.strip()
}
if provided_values:
if len(provided_values) == 1:
html_title += ": {}".format(list(provided_values.values())[0])
else:
html_title += ": {}".format(
", ".join(
"{}={}".format(key, value) for key, value in provided_values.items()
)
)
user_can_execute_sql = request.user.has_perm("django_sql_dashboard.execute_sql")
saved_dashboards = []
if not dashboard:
# Only show saved dashboards on index page
saved_dashboards = [
(dashboard, dashboard.user_can_edit(request.user))
for dashboard in Dashboard.get_visible_to_user(request.user).select_related(
"owned_by", "view_group", "edit_group"
)
]
if json_mode:
return JsonResponse(
{
"title": title or "SQL Dashboard",
"queries": [
{"sql": r["sql"], "rows": r["rows"]} for r in query_results
],
},
json_dumps_params={
"indent": 2,
"default": lambda o: (
o.isoformat() if hasattr(o, "isoformat") else str(o)
),
},
)
context = {
"title": title or "SQL Dashboard",
"html_title": html_title,
"query_results": query_results,
"unverified_sql_queries": unverified_sql_queries,
"available_tables": available_tables,
"description": description,
"dashboard": dashboard,
"saved_dashboard": bool(dashboard),
"user_owns_dashboard": dashboard and request.user == dashboard.owned_by,
"user_can_edit_dashboard": dashboard and dashboard.user_can_edit(request.user),
"user_can_execute_sql": user_can_execute_sql,
"user_can_export_data": getattr(settings, "DASHBOARD_ENABLE_FULL_EXPORT", None)
and user_can_execute_sql,
"parameter_values": parameter_values.items(),
"too_long_so_use_post": too_long_so_use_post,
"saved_dashboards": saved_dashboards,
}
if extra_context:
context.update(extra_context)
response = render(
request,
template,
context,
)
if request.user.is_authenticated:
response["cache-control"] = "private"
response["Content-Security-Policy"] = "frame-ancestors 'self'"
return response
def dashboard_json(request, slug):
disable_json = getattr(settings, "DASHBOARD_DISABLE_JSON", None)
if disable_json:
return HttpResponseForbidden("JSON export is disabled")
return dashboard(request, slug, json_mode=True)
def dashboard(request, slug, json_mode=False):
dashboard = get_object_or_404(Dashboard, slug=slug)
# Can current user see it, based on view_policy?
view_policy = dashboard.view_policy
owner = dashboard.owned_by
denied = HttpResponseForbidden("You cannot access this dashboard")
denied["cache-control"] = "private"
if view_policy == Dashboard.ViewPolicies.PRIVATE:
if request.user != owner:
return denied
elif view_policy == Dashboard.ViewPolicies.LOGGEDIN:
if not request.user.is_authenticated:
return denied
elif view_policy == Dashboard.ViewPolicies.GROUP:
if (not request.user.is_authenticated) or not (
request.user == owner
or request.user.groups.filter(pk=dashboard.view_group_id).exists()
):
return denied
elif view_policy == Dashboard.ViewPolicies.STAFF:
if (not request.user.is_authenticated) or (
request.user != owner and not request.user.is_staff
):
return denied
elif view_policy == Dashboard.ViewPolicies.SUPERUSER:
if (not request.user.is_authenticated) or (
request.user != owner and not request.user.is_superuser
):
return denied
return _dashboard_index(
request,
sql_queries=[query.sql for query in dashboard.queries.all()],
title=dashboard.title,
description=dashboard.description,
dashboard=dashboard,
template="django_sql_dashboard/saved_dashboard.html",
json_mode=json_mode,
)
non_alpha_re = re.compile(r"[^a-zA-Z0-9]")
def export_sql_results(request):
export_key = [k for k in request.POST.keys() if k.startswith("export_")][0]
_, format, sql_index = export_key.split("_")
assert format in ("csv", "tsv")
sqls = request.POST.getlist("sql")
sql = sqls[int(sql_index)]
parameter_values = {
parameter: request.POST.get(parameter, "")
for parameter in extract_named_parameters(sql)
}
alias = getattr(settings, "DASHBOARD_DB_ALIAS", "dashboard")
# Decide on filename
sql_hash = hashlib.sha256(sql.encode("utf-8")).hexdigest()[:6]
filename = non_alpha_re.sub("-", sql.lower()[:30]) + sql_hash
filename_plus_ext = filename + "." + format
connection = connections[alias]
connection.cursor() # To initialize connection
cursor = connection.create_cursor(name="c" + filename.replace("-", "_"))
csvfile = StringIO()
csvwriter = csv.writer(
csvfile,
dialect={
"csv": csv.excel,
"tsv": csv.excel_tab,
}[format],
)
def read_and_flush():
csvfile.seek(0)
data = csvfile.read()
csvfile.seek(0)
csvfile.truncate()
return data
def rows():
try:
cursor.execute(sql, parameter_values)
done_header = False
while True:
records = cursor.fetchmany(size=2000)
if not done_header:
csvwriter.writerow([r.name for r in cursor.description])
yield read_and_flush()
done_header = True
if not records:
break
for record in records:
csvwriter.writerow(record)
yield read_and_flush()
finally:
cursor.close()
response = StreamingHttpResponse(
rows(),
content_type={
"csv": "text/csv",
"tsv": "text/tab-separated-values",
}[format],
)
response["Content-Disposition"] = 'attachment; filename="' + filename_plus_ext + '"'
return response
================================================
FILE: docker-compose.yml
================================================
version: "2"
services:
app:
build: .
links:
- db
volumes:
- .:/app
environment:
- DATABASE_URL=postgres://appuser:test123@db/test_project
- DJANGO_SETTINGS_MODULE=config.settings_interactive
- PYTHONUNBUFFERED=yup
working_dir: /app
entrypoint: ["./test_project/wait-for-postgres.sh"]
ports:
- "${APP_PORT:-8000}:${APP_PORT:-8000}"
command: bash -c "python test_project/manage.py migrate && python test_project/manage.py runserver 0.0.0.0:${APP_PORT:-8000}"
docs:
build: .
volumes:
- .:/app
working_dir: /app/docs
ports:
- "${DOCS_PORT:-8001}:${DOCS_PORT:-8001}"
command: make SPHINXOPTS="--host 0.0.0.0 --port ${DOCS_PORT:-8001}" livehtml
db:
# Note that this database is only used when we use
# test_project interactively; automated tests spin up
# their own database inside the app container.
image: postgres:13-alpine
environment:
- POSTGRES_PASSWORD=test123
- POSTGRES_USER=appuser
- POSTGRES_DB=test_project
================================================
FILE: docs/.gitignore
================================================
_build
================================================
FILE: docs/Makefile
================================================
# Minimal makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
SPHINXPROJ = sqlite-utils
SOURCEDIR = .
BUILDDIR = _build
# Put it first so that "make" without argument is like "make help".
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
livehtml:
sphinx-autobuild -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0)
================================================
FILE: docs/conf.py
================================================
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from subprocess import PIPE, Popen
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
# -- General configuration ------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ["myst_parser"]
# Add any paths that contain templates here, relative to this directory.
templates_path = ["_templates"]
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = ".rst"
# The master toctree document.
master_doc = "index"
# General information about the project.
project = "django-sql-dashboard"
copyright = "2021, Simon Willison"
author = "Simon Willison"
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
pipe = Popen("git describe --tags --always", stdout=PIPE, shell=True)
git_version = pipe.stdout.read().decode("utf8")
if git_version:
version = git_version.rsplit("-", 1)[0]
release = git_version
else:
version = ""
release = ""
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = "sphinx"
# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False
# -- Options for HTML output ----------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = "sphinx_rtd_theme"
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ["_static"]
# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
#
# This is required for the alabaster theme
# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
html_sidebars = {
"**": [
"relations.html", # needs 'show_related': True theme option to display
"searchbox.html",
]
}
# -- Options for HTMLHelp output ------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = "django-sql-dashboard-doc"
# -- Options for LaTeX output ---------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#
# 'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#
# 'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#
# 'preamble': '',
# Latex figure (float) alignment
#
# 'figure_align': 'htbp',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(
master_doc,
"django-sql-dashboard.tex",
"django-sql-dashboard documentation",
"Simon Willison",
"manual",
)
]
# -- Options for manual page output ---------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(
master_doc,
"django-sql-dashboard",
"django-sql-dashboard documentation",
[author],
1,
)
]
# -- Options for Texinfo output -------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(
master_doc,
"django-sql-dashboard",
"django-sql-dashboard documentation",
author,
"django-sql-dashboard",
"Django app for building dashboards using raw SQL queries",
"Miscellaneous",
)
]
================================================
FILE: docs/contributing.md
================================================
# Contributing
To contribute to this library, first checkout the code. Use [uv](https://github.com/astral-sh/uv) to run the tests:
cd django-sql-dashboard
uv run pytest
## Generating new migrations
To generate migrations for model changes:
cd test_project
uv run ./manage.py makemigrations
## Code style
This library uses [Black](https://github.com/psf/black) for code formatting. You can run it like this:
uv run black .
## Documentation
Documentation for this project uses [MyST](https://myst-parser.readthedocs.io/) - it is written in Markdown and rendered using Sphinx.
To build the documentation locally, run the following:
cd docs
uv run --with-requirements requirements.txt make livehtml
This will start a live preview server, using [sphinx-autobuild](https://pypi.org/project/sphinx-autobuild/).
## Using Docker Compose
If you're familiar with Docker--or even if you're not--you may want to consider using our optional Docker Compose setup.
An advantage of this approach is that it relieves you of setting up any dependencies, such as ensuring that you have the proper version of Python and Postgres and so forth. On the downside, however, it does require you to familiarize yourself with Docker, which, while relatively easy to use, still has its own learning curve.
To try out the Docker Compose setup, you will first want to [get Docker][] and [install Docker Compose][].
Then, after checking out the code, run the following:
```
cd django-sql-dashboard
docker-compose build
```
At this point, you can start editing code. To run any development tools such as `pytest` or `black`, just prefix everything with `docker-compose run app`. For instance, to run the test suite, run:
```
docker-compose run app python pytest
```
If this is a hassle, you can instead run a bash shell inside your container:
```
docker-compose run app bash
```
At this point, you'll be in a bash shell inside your container, and can run development tools directly.
[get Docker]: https://docs.docker.com/get-docker/
[install Docker Compose]: https://docs.docker.com/compose/install/
### Using the dashboard interactively
The Docker Compose setup is configured to run a simple test project that you can use to tinker with the dashboard interactively.
To use it, run:
```
docker-compose up
```
Then, in a separate terminal, run:
```
docker-compose run app python test_project/manage.py createsuperuser
```
You will now be prompted to enter details about a new superuser. Once you've done that, you can visit the example app's dashboard at http://localhost:8000/. After entering the credentials for the superuser you just created, you will be able to tinker with the dashboard.
### Editing the documentation
Running `docker-compose up` also starts the documentation system's live preview server. You can visit it at http://localhost:8001/.
### Changing the default ports
If you are already using ports 8000 and/or 8001 for other things, you can change them. To do this, create a file in the repository root called `.env` and populate it with the following:
```
APP_PORT=9000
DOCS_PORT=9001
```
You can change the above port values to whatever makes sense for your setup.
Once you next run `docker-compose up` again, the services will be running on the ports you specified in `.env`.
### Changing the default UID and GID
The default settings assume that the user id (UID) and group id (GID) of the account you're using to develop are both 1000. This is likely to be the case, since that's the UID/GID of the first non-root account on most systems. However, if your account doesn't match this, you can customize the container to use a different UID/GID.
For instance, if your UID and GID are 1001, you can build your container with the following arguments:
```
docker-compose build --build-arg UID=1001 --build-arg GID=1001
```
### Updating
The project's Python dependencies are all baked into the container image, which means that whenever they change (or to be safe, whenever you `git pull` new changes to the codebase), you will want to run:
```
docker-compose build
```
You will also want to restart `docker-compose up`.
### Cleaning up
If you somehow get your Docker Compose setup into a broken state, or you decide that you never use Docker Compose again, you can clean everything up by running:
```
docker-compose down -v
```
================================================
FILE: docs/index.md
================================================
Django SQL Dashboard
--------------------
```{toctree}
---
maxdepth: 3
---
setup
sql
saved-dashboards
widgets
security
contributing
```
```{include} ../README.md
```
================================================
FILE: docs/requirements.txt
================================================
sphinx_rtd_theme
sphinx-autobuild
myst-parser
================================================
FILE: docs/saved-dashboards.md
================================================
# Saved dashboards
A set of SQL queries can be used to create a saved dashboard. Saved dashboards have URLs and support permissions, so you can specify which users are allowed to see which dashboard.
You can create a saved dashboard from the interactive dashboard interface (at `/dashboard/`) - execute some queries, then scroll down to the "Save this dashboard" form.
## View permissions
The following viewing permission policies are available:
- `private`: Only the user who created (owns) the dashboard can view
- `public`: Any user can view
- `unlisted`: Any user can view, but they need to know the URL (this feature is not complete)
- `loggedin`: Any logged-in user can view
- `group`: Any user who is a member of the `view_group` attached to the dashboard can view
- `staff`: Any user who is staff can view
- `superuser`: Any user who is a superuser can view
(edit_permissions)=
## Edit permissions
The edit policy controls which users are allowed to edit a dashboard - defaulting to the user who created that dashboard.
Editing currently takes place through the Django Admin - so only users who are staff members with access to that interface will be able to edit their dashboards.
The full list of edit policy options are:
- `private`: Only the user who created (owns) the dashboard can edit
- `loggedin`: Any logged-in user can edit
- `group`: Any user who is a member of the `edit_group` attached to the dashboard can edit
- `staff`: Any user who is staff can edit
- `superuser`: Any user who is a superuser can edit
Dashboards belong to the user who created them. Only Django super-users can re-assign ownership of dashboards to other users.
## JSON export
If your dashboard is called `/dashboards/demo/` you can add `.json` to get `/dashboards/demo.json` which will return a JSON representation of the dashboard.
The JSON format looks something like this:
```json
{
"title": "Tag word cloud",
"queries": [
{
"sql": "select \"tag\" as wordcloud_word, count(*) as wordcloud_count from (select blog_tag.tag from blog_entry_tags join blog_tag on blog_entry_tags.tag_id = blog_tag.id\r\nunion all\r\nselect blog_tag.tag from blog_blogmark_tags join blog_tag on blog_blogmark_tags.tag_id = blog_tag.id\r\nunion all\r\nselect blog_tag.tag from blog_quotation_tags join blog_tag on blog_quotation_tags.tag_id = blog_tag.id) as results where tag != 'quora' group by \"tag\" order by wordcloud_count desc",
"rows": [
{
"wordcloud_word": "python",
"wordcloud_count": 826
},
{
"wordcloud_word": "javascript",
"wordcloud_count": 604
},
{
"wordcloud_word": "django",
"wordcloud_count": 529
},
{
"wordcloud_word": "security",
"wordcloud_count": 402
},
{
"wordcloud_word": "datasette",
"wordcloud_count": 331
},
{
"wordcloud_word": "projects",
"wordcloud_count": 282
}
],
}
]
}
```
Set the `DASHBOARD_DISABLE_JSON` setting to `True` to disable this feature.
================================================
FILE: docs/security.md
================================================
# Security
Allowing people to execute their own SQL directly against your database is risky business!
The safest way to use this tool is to create a read-only replica of your PostgreSQL database with a read-only role that enforces a statement time-limit for executed queries. Different database providers have different mechanisms for doing this - consult your hosting provider's documentation.
You should only provide access to this tool to people you trust. Malicious users may be able to negatively affect the performance of your servers through constructing SQL queries that deliberately consume large amounts of resources.
Configured correctly, Django SQL Dashboard uses a number of measures to keep your data and your database server safe:
- I strongly recommend creating a dedicated PostgreSQL role for accessing your database with read-only permissions granted to an allow-list of tables. PostgreSQL has extremely robust, well tested permissions which this tool can take full advantage of.
- Likewise, configuring a PostgreSQL-enforced query time limit can reduce the risk of expensive queries affecting the performance of the rest of your site.
- Setting up a read-only reporting replica for use with this tool can provide even stronger isolation from other site traffic.
- Your allow-list of tables should not include tables with sensitive information. Django's auth_user table contains password hashes, and the django_session table contains user session information. Neither should be exposed using this tool.
- Access to the dashboard is controlled by Django's permissions system, which means you can limit access to trusted team members.
- SQL queries can be passed to the dashboard using a ?sql= query string parameter - but this parameter needs to be signed before it will be executed. This should prevent attempts to trick you into executing malevolent SQL queries by sending you crafted links - while still allowing your team to create links to queries that can be securely shared.
- Any time a user views a dashboard page while logged in, `Cache-Control: private` is set on the response to ensure the authenticated dashboard will not be stored in any intermediary HTTP caches
================================================
FILE: docs/setup.md
================================================
# Installation and configuration
## Install using pip
Install this library using `pip`:
$ pip install django-sql-dashboard
### Run migrations
The migrations create tables that store dashboards and queries:
$ ./manage.py migrate
## Configuration
Add `"django_sql_dashboard"` to your `INSTALLED_APPS` in `settings.py`.
Add the following to your `urls.py`:
```python
from django.urls import path, include
import django_sql_dashboard
urlpatterns = [
# ...
path("dashboard/", include(django_sql_dashboard.urls)),
]
```
## Setting up read-only PostgreSQL credentials
The safest way to use this tool is against a dedicated read-only replica of your database - see [security](./security) for more details.
Create a new PostgreSQL user or role that is limited to read-only SELECT access to a specific list of tables.
If your read-only role is called `my-read-only-role`, you can grant access using the following SQL (executed as a privileged user):
```sql
GRANT USAGE ON SCHEMA PUBLIC TO "my-read-only-role";
```
This grants that role the ability to see what tables exist. You then need to grant `SELECT` access to specific tables like this:
```sql
GRANT SELECT ON TABLE
public.locations_location,
public.locations_county,
public.django_content_type,
public.django_migrations
TO "my-read-only-role";
```
Think carefully about which tables you expose to the dashboard - in particular, you should avoid exposing tables that contain sensitive data such as `auth_user` or `django_session`.
If you do want to expose `auth_user` - which can be useful if you want to join other tables against it to see details of the user that created another record - you can grant access to specific columns like so:
```sql
GRANT SELECT(
id, last_login, is_superuser, username, first_name,
last_name, email, is_staff, is_active, date_joined
) ON auth_user TO "my-read-only-role";
```
This will allow queries against everything except for the `password` column.
Note that if you use this pattern the query `select * from auth_user` will return a "permission denied" error. You will need to explicitly list the columns you would like to see from that table instead, for example `select id, username, date_joined from auth_user`.
## Configuring the "dashboard" database alias
Django SQL Dashboard defaults to executing all queries using the `"dashboard"` Django database alias.
You can define this `"dashboard"` database alias in `settings.py`. Your `DATABASES` section should look something like this:
```python
DATABASES = {
"default": {
"ENGINE": "django.db.backends.postgresql",
"NAME": "mydb",
"USER": "read_write_user",
"PASSWORD": "read_write_password",
"HOST": "dbhost.example.com",
"PORT": "5432",
},
"dashboard": {
"ENGINE": "django.db.backends.postgresql",
"NAME": "mydb",
"USER": "read_only_user",
"PASSWORD": "read_only_password",
"HOST": "dbhost.example.com",
"PORT": "5432",
"OPTIONS": {
"options": "-c default_transaction_read_only=on -c statement_timeout=100"
},
},
}
```
In addition to the read-only user and password, pay attention to the `"OPTIONS"` section: this sets a statement timeout of 100ms - queries that take longer than that will be terminated with an error message. It also sets it so transactions will be read-only by default, as an extra layer of protection should your read-only user have more permissions that you intended.
Now visit `/dashboard/` as a staff user to start trying out the dashboard.
### Danger mode: configuration without a read-only database user
Some hosting environments such as Heroku charge extra for the ability to create read-only database users. For smaller projects with dashboard access only made available to trusted users it's possible to configure this tool without a read-only account, using the following options:
```python
# ...
"dashboard": {
"ENGINE": "django.db.backends.postgresql",
"USER": "read_write_user",
# ...
"OPTIONS": {
"options": "-c default_transaction_read_only=on -c statement_timeout=100"
},
},
```
The `-c default_transaction_read_only=on` option here should prevent accidental writes from being executed, but note that dashboard users in this configuration will be able to access _all tables_ including tables that might contain sensitive information. Only use this trick if you are confident you fully understand the implications!
### dj-database-url and django-configurations
If you are using [dj-database-url](https://github.com/jacobian/dj-database-url) or [django-configurations](https://github.com/jazzband/django-configurations) _(with `database` extra requirement)_, your `DATABASES` section should look something like this:
```python
import dj_database_url
# ...
DATABASES = {
"default": dj_database_url.config(env="DATABASE_URL"),
"dashboard": dj_database_url.config(env="DATABASE_DASHBOARD_URL"),
}
```
You can define the two database url variables in your environment like this:
```ini
DATABASE_URL=postgresql://read_write_user:read_write_password@dbhost.example.com:5432/mydb
DATABASE_DASHBOARD_URL=postgresql://read_write_user:read_write_password@dbhost.example.com:5432/mydb?options=-c%20default_transaction_read_only%3Don%20-c%20statement_timeout%3D100
```
## Django permissions
Access to the `/dashboard/` interface is controlled by the Django permissions system. To grant a Django user or group access, grant them the `django_sql_dashboard.execute_sql` permission. This is displayed in the admin interface as:
django_sql_dashboard | dashboard | Can execute arbitrary SQL queries
Dashboard editing is currently handled by the Django admin interface. This means a user needs to have **staff** status (allowing them access to the Django admin interface) in order to edit one of their saved dashboards.
The regular Django permission for "can edit dashboard" is ignored. Instead, a permission system that is specific to Django SQL Dashboard is used to control edit permissions. See {ref}`edit_permissions` for details.
## Additional settings
You can customize the following settings in Django's `settings.py` module:
- `DASHBOARD_DB_ALIAS = "db_alias"` - which database alias to use for executing these queries. Defaults to `"dashboard"`.
- `DASHBOARD_ROW_LIMIT = 1000` - the maximum number of rows that can be returned from a query. This defaults to 100.
- `DASHBOARD_UPGRADE_OLD_BASE64_LINKS` - prior to version 0.8a0 SQL URLs used base64-encoded JSON. If you set this to `True` any hits that include those old URLs will be automatically redirected to the upgraded new version. Use this if you have an existing installation of `django-sql-dashboard` that people already have saved bookmarks for.
- `DASHBOARD_ENABLE_FULL_EXPORT` - set this to `True` to enable the full results CSV/TSV export feature. It defaults to `False`. Enable this feature only if you are confident that the database alias you are using does not have write permissions to anything.
- `DASHBOARD_DISABLE_JSON` - set to `True` to disable the feature where `/dashboard/name-of-dashboard.json` provides a JSON representation of the dashboard. This defaults to `False`.
## Custom templates
The templates used by `django-sql-dashboard` extend a base template called `django_sql_dashboard/base.html`, which provides Django template blocks named `title` and `content`. You can customize the appearance of your dashboard installation by providing your own version of this base template in your own configured `templates/` directory.
================================================
FILE: docs/sql.md
================================================
# Running SQL queries
Visit `/dashboard/` to get started. This interface allows you to execute one or more PostgreSQL SQL queries.
Results will be displayed below each query, limited to a maxmim of 100 rows.
The queries you have executed are encoded into the URL of the page. This means you can bookmark queries and share those links with other people who can access your dashboard.
Note that the queries in the URL are signed using Django's `SECRET_KEY` setting. This means that changing you secret will break your bookmarked URLs.
## SQL parameters
If your SQL query contains `%(name)s` parameters, `django-sql-dashboard` will convert those into form fields on the page and allow users to submit values for them. These will be correctly quoted and escaped in the SQL query.
Given the following SQL query:
```
select * from blog_entry where slug = %(slug)s
```
A form field called `slug` will be displayed, and the user will be able to use that to search for blog entries with that given slug.
Here's a more advanced example:
```sql
select * from location
where state_id = cast(%(state_id)s as integer)
and name ilike '%%' || %(search)s || '%%';
```
Here a form will be displayed with `state_id` and `search` fields.
The values provided by the user will always be treated like strings - so in this example the `state_id` is cast to integer in order to be matched with an integer column.
Any `%` characters - for example in the `ilike` query above - need to be escaped by providing them twice: `%%`.
================================================
FILE: docs/widgets.md
================================================
# Widgets
SQL queries default to displaying as a table. Other forms of display - called widgets - are also available, and are selected based on the names of the columns returned by the query.
## Bar chart: bar_label, bar_quantity
A query that returns columns called `bar_label` and `bar_quantity` will be rendered as a simple bar chart, using [Vega-Lite](https://vega.github.io/vega-lite/).

Bar chart live demo: [simonwillison.net/dashboard/by-month/](https://simonwillison.net/dashboard/by-month/)
SQL example:
```sql
select
county.name as bar_label,
count(*) as bar_quantity
from location
join county on county.id = location.county_id
group by county.name
order by count(*) desc limit 10
```
Or using a static list of values:
```sql
SELECT * FROM (
VALUES (1, 'one'), (2, 'two'), (3, 'three')
) AS t (bar_quantity, bar_label);
```
## Big number: big_number, label
If you want to display the results as a big number accompanied by a label, you can do so by returning `big_number` and `label` columns from your query, for example.
```sql
select 'Number of states' as label, count(*) as big_number from states;
```

Big number live demo: [simonwillison.net/dashboard/big-numbers-demo/](https://simonwillison.net/dashboard/big-numbers-demo/)
## Progress bar: completed_count, total_count
To display a progress bar, return columns `total_count` and `completed_count`.
```sql
select 1203 as total_count, 755 as completed_count;
```
This SQL pattern can be useful for constructing progress bars:
```sql
select (
select count(*) from task
) as total_count, (
select count(*) from task where resolved_at is not null
) as completed_count
```

Progress bar live demo: [simonwillison.net/dashboard/progress-bar-demo/](https://simonwillison.net/dashboard/progress-bar-demo/)
## Word cloud: wordcloud_count, wordcloud_word
To display a word cloud, return a column `wordcloud_word` containing words with a corresponding `wordcloud_count` column with the frequency of those words.
This example generates word clouds for article body text:
```sql
with words as (
select
lower(
(regexp_matches(body, '\w+', 'g'))[1]
) as word
from
articles
)
select
word as wordcloud_word,
count(*) as wordcloud_count
from
words
group by
word
order by
count(*) desc
```
Here's a fun variant that uses PostgreSQL's built-in stemming algorithm to first remove common stop words:
```sql
with words as (
select
lower(
(regexp_matches(to_tsvector('english', body)::text, '[a-z]+', 'g'))[1]
) as word
from
articles
)
select
word as wordcloud_word,
count(*) as wordcloud_count
from
words
group by
word
order by
count(*) desc
```

Word cloud live demo: [simonwillison.net/dashboard/tag-cloud/](https://simonwillison.net/dashboard/tag-cloud/)
## markdown
Return a single column called `markdown` to render the contents as Markdown, for example:
```sql
select '# Number of states: ' || count(*) as markdown from states;
```
## html
Return a single column called `html` to render the contents directly as HTML. This HTML is filtered using [Bleach](https://github.com/mozilla/bleach) so the only tags allowed are `a[href]`, `abbr`, `acronym`, `b`, `blockquote`, `code`, `em`, `i`, `li`, `ol`, `strong`, `ul`, `pre`, `p`, `h1`, `h2`, `h3`, `h4`, `h5`, `h6`.
```sql
select '
Number of states: ' || count(*) || '
' as html from states;
```
# Custom widgets
You can define your own custom widgets by creating templates with special names.
Decide on the column names that you wish to customize for, then sort them alphabetically and join them with hyphens to create your template name.
For example, you could define a widget that handles results returned as `placename`, `geojson` by creating a template called `geojson-placename.html`.
Save that in one of your template directories as `django_sql_dashboard/widgets/geojson-placename.html`.
Any SQL query that returns exactly the columns `placename` and `geojson` will now be rendered by your custom template file.
Within your custom template you will have access to a template variable called `result` with the following keys:
- `result.sql` - the SQL query that is being displayed
- `rows` - a list of rows, where each row is a dictionary mapping columns to their values
- `row_lists` - a list of rows, where each row is a list of the values in that row
- `description` - the psycopg2 cursor description
- `columns` - a list of string column names
- `column_details` - a list of `{"name": column_name, "is_unambiguous": True or False}` dictionaries - `is_unambiguous` is `False` if multiple columns of the same name are returned by this query
- `truncated` - boolean, specifying whether the results were truncated (at 100 items) or not
- `extra_qs` - extra parameters for the page encoded as a query string fragment - so if the page was loaded with `state_id=5` then `extra_qs` would be `&state_id=5`. You can use this to assemble links to further queries, like the "Count" column links in the default table view.
- `duration_ms` - how long the query took, in floating point milliseconds
- `templates` - a list of templates that were considered for rendering this widget
The easiest way to define your custom widget template is to extend the `django_sql_dashboard/widgets/_base_widget.html` base template.
Here is the full implementation of the `big_number`, `label` widget that is included with Django SQL Dashboard, in the `django_sql_dashboard/widgets/big_number-label.html` template file:
```html+django
{% extends "django_sql_dashboard/widgets/_base_widget.html" %}
{% block widget_results %}
{% for row in result.rows %}
{{ row.label }}
{{ row.big_number }}
{% endfor %}
{% endblock %}
```
You can find more examples of widget templates in the [templates/django_sql_dashboard/widgets](https://github.com/simonw/django-sql-dashboard/tree/main/django_sql_dashboard/templates/django_sql_dashboard/widgets) directory.
================================================
FILE: pyproject.toml
================================================
[project]
name = "django-sql-dashboard"
version = "1.2"
description = "Django app for building dashboards using raw SQL queries"
readme = "README.md"
license = "Apache-2.0"
authors = [
{ name = "Simon Willison" }
]
requires-python = ">=3.10"
dependencies = [
"Django>=4.2",
"markdown",
"bleach",
]
[project.urls]
Documentation = "https://django-sql-dashboard.datasette.io/"
Issues = "https://github.com/simonw/django-sql-dashboard/issues"
CI = "https://github.com/simonw/django-sql-dashboard/actions"
Changelog = "https://github.com/simonw/django-sql-dashboard/releases"
Homepage = "https://github.com/simonw/django-sql-dashboard"
[dependency-groups]
dev = [
"black>=22.3.0",
"psycopg2",
"pytest",
"pytest-django>=4.11.1",
"pytest-pythonpath",
"dj-database-url",
"testing.postgresql",
"beautifulsoup4",
"html5lib",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["django_sql_dashboard"]
[tool.isort]
profile = "black"
multi_line_output = 3
[tool.pytest.ini_options]
DJANGO_SETTINGS_MODULE = "config.settings"
pythonpath = ["test_project", "pytest_plugins"]
addopts = "-p pytest_use_postgresql"
================================================
FILE: pytest_plugins/__init__.py
================================================
================================================
FILE: pytest_plugins/pytest_use_postgresql.py
================================================
import os
import pytest
from dj_database_url import parse
from django.conf import settings
from testing.postgresql import Postgresql
postgres = os.environ.get("POSTGRESQL_PATH")
initdb = os.environ.get("INITDB_PATH")
_POSTGRESQL = Postgresql(postgres=postgres, initdb=initdb)
@pytest.hookimpl(tryfirst=True)
def pytest_load_initial_conftests(early_config, parser, args):
os.environ["DJANGO_SETTINGS_MODULE"] = early_config.getini("DJANGO_SETTINGS_MODULE")
settings.DATABASES["default"] = parse(_POSTGRESQL.url())
settings.DATABASES["dashboard"] = parse(_POSTGRESQL.url())
def pytest_unconfigure(config):
_POSTGRESQL.stop()
================================================
FILE: test_project/config/__init__.py
================================================
================================================
FILE: test_project/config/asgi.py
================================================
import os
from django.core.asgi import get_asgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
application = get_asgi_application()
================================================
FILE: test_project/config/settings.py
================================================
import os
from pathlib import Path
import dj_database_url
BASE_DIR = Path(__file__).resolve().parent.parent
SECRET_KEY = "local-testing-insecure-secret"
DEBUG = True
ALLOWED_HOSTS = ["*"]
# Application definition
INSTALLED_APPS = [
"django.contrib.admin",
"django.contrib.auth",
"django.contrib.contenttypes",
"django.contrib.sessions",
"django.contrib.messages",
"django.contrib.staticfiles",
"django_sql_dashboard",
"extra_models",
]
MIDDLEWARE = [
"django.middleware.security.SecurityMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"django.middleware.common.CommonMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
"django.middleware.clickjacking.XFrameOptionsMiddleware",
]
ROOT_URLCONF = "config.urls"
TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",
"DIRS": [],
"APP_DIRS": True,
"OPTIONS": {
"context_processors": [
"django.template.context_processors.debug",
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
],
},
},
]
WSGI_APPLICATION = "config.wsgi.application"
DATABASES = {
"default": dj_database_url.config(),
}
# Password validation
# https://docs.djangoproject.com/en/3.1/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
},
{
"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
},
{
"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
},
{
"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
},
]
# Internationalization
# https://docs.djangoproject.com/en/3.1/topics/i18n/
LANGUAGE_CODE = "en-us"
TIME_ZONE = "UTC"
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.1/howto/static-files/
STATIC_URL = "/static/"
LOGIN_URL = "/admin/login/"
================================================
FILE: test_project/config/settings_interactive.py
================================================
# Normally test_project is used as scaffolding for
# django_sql_dashboard's automated tests. However, it can
# be useful during development to have a sample project to
# tinker with interactively. These Django settings can be
# useful when we want to do that.
from .settings import *
# Just have our dashboard use the exact same credentials for
# our database, there's no need to bother with read-only
# permissions when using test_project interactively.
DATABASES["dashboard"] = DATABASES["default"]
================================================
FILE: test_project/config/urls.py
================================================
from django.contrib import admin
from django.urls import include, path
from django.views.generic.base import RedirectView
import django_sql_dashboard
urlpatterns = [
path("dashboard/", include(django_sql_dashboard.urls)),
path("admin/", admin.site.urls),
path("", RedirectView.as_view(url="/dashboard/")),
]
================================================
FILE: test_project/config/wsgi.py
================================================
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
application = get_wsgi_application()
================================================
FILE: test_project/extra_models/models.py
================================================
from django.db import models
class Switch(models.Model):
name = models.SlugField()
on = models.BooleanField(default=False)
class Meta:
db_table = "switches"
================================================
FILE: test_project/manage.py
================================================
#!/usr/bin/env python
"""Django's command-line utility for administrative tasks."""
import os
import sys
def main():
"""Run administrative tasks."""
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
) from exc
execute_from_command_line(sys.argv)
if __name__ == "__main__":
main()
================================================
FILE: test_project/test_dashboard.py
================================================
import urllib.parse
import pytest
from bs4 import BeautifulSoup
from django.core import signing
from django.db import connections
from django_sql_dashboard.utils import SQL_SALT, is_valid_base64_json, sign_sql
def test_dashboard_submit_sql(admin_client, dashboard_db):
# Test full flow of POST submitting new SQL, having it signed
# and having it redirect to the results page
get_response = admin_client.get("/dashboard/")
assert get_response.status_code == 200
assert get_response["Content-Security-Policy"] == "frame-ancestors 'self'"
sql = "select 14 + 33"
response = admin_client.post(
"/dashboard/",
{
"sql": sql,
"_save-title": "",
"_save-slug": "",
"_save-description": "",
"_save-view_policy": "private",
"_save-view_group": "",
"_save-edit_policy": "private",
"_save-edit_group": "",
},
)
assert response.status_code == 302
# Should redirect to ?sql=signed-value
bits = urllib.parse.parse_qs(response.url.split("?")[1])
assert set(bits.keys()) == {"sql"}
signed_sql = bits["sql"][0]
assert signed_sql == sign_sql(sql)
# GET against this new location should return correct result
get_response = admin_client.get(response.url)
assert get_response.status_code == 200
assert b"47" in get_response.content
def test_invalid_signature_shows_warning(admin_client, dashboard_db):
response1 = admin_client.post("/dashboard/", {"sql": "select 1 + 1"})
signed_sql = urllib.parse.parse_qs(response1.url.split("?")[1])["sql"][0]
# Break the signature and load the page
response2 = admin_client.get(
"/dashboard/?" + urllib.parse.urlencode({"sql": signed_sql[:-1]})
)
html = response2.content.decode("utf-8")
assert ">Unverified SQL<" in html
assert "" in html
def test_dashboard_upgrade_old_base64_links(admin_client, dashboard_db, settings):
old_signed = signing.dumps("select 1 + 1", salt=SQL_SALT)
assert is_valid_base64_json(old_signed.split(":")[0])
# Should do nothing without setting
assert admin_client.get("/dashboard/?sql=" + old_signed).status_code == 200
# With setting should redirect
settings.DASHBOARD_UPGRADE_OLD_BASE64_LINKS = True
response = admin_client.get("/dashboard/?sql=" + old_signed)
assert response.status_code == 302
assert response.url == "/dashboard/?" + urllib.parse.urlencode(
{"sql": sign_sql("select 1 + 1")}
)
def test_dashboard_upgrade_does_not_break_regular_pages(
admin_client, dashboard_db, settings
):
# With setting should redirect
settings.DASHBOARD_UPGRADE_OLD_BASE64_LINKS = True
response = admin_client.get("/dashboard/")
assert response.status_code == 200
def test_saved_dashboard(client, admin_client, dashboard_db, saved_dashboard):
assert admin_client.get("/dashboard/test2/").status_code == 404
response = admin_client.get("/dashboard/test/")
assert response.status_code == 200
assert b"44" in response.content
assert b"77" in response.content
assert b"data-count-url" in response.content
# And test markdown support
assert (
b'supports markdown'
in response.content
)
def test_many_long_column_names(admin_client, dashboard_db):
# https://github.com/simonw/django-sql-dashboard/issues/23
columns = ["column{}".format(i) for i in range(200)]
sql = "select " + ", ".join(
"'{}' as {}".format(column, column) for column in columns
)
response = admin_client.post("/dashboard/", {"sql": sql}, follow=True)
assert response.status_code == 200
@pytest.mark.parametrize(
"sql,expected_error",
(
(
"select * from not_a_table",
'relation "not_a_table" does not exist\nLINE 1: select * from not_a_table\n ^',
),
(
"select 'foo' like 'f%'",
r"Invalid query - try escaping single '%' as double '%%'",
),
(
"select '% completed'",
r"Invalid query - try escaping single '%' as double '%%'",
),
),
)
def test_dashboard_sql_errors(admin_client, sql, expected_error):
response = admin_client.post("/dashboard/", {"sql": sql}, follow=True)
assert response.status_code == 200
soup = BeautifulSoup(response.content, "html5lib")
div = soup.select(".query-results")[0]
assert div["class"] == ["query-results", "query-error"]
assert div.select(".error-message")[0].text.strip() == expected_error
@pytest.mark.parametrize(
"sql,expected_columns,expected_rows",
(
("select 'abc' as one, 'bcd' as one", ["one", "one"], [["abc", "bcd"]]),
("select ARRAY[1, 2, 3]", ["array"], [["[\n 1,\n 2,\n 3\n]"]]),
(
"select ARRAY[TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54+02']",
["array"],
[['[\n "2004-10-19 08:23:54+00:00"\n]']],
),
),
)
def test_dashboard_sql_queries(admin_client, sql, expected_columns, expected_rows):
response = admin_client.post("/dashboard/", {"sql": sql}, follow=True)
assert response.status_code == 200
soup = BeautifulSoup(response.content, "html5lib")
div = soup.select(".query-results")[0]
columns = [th.text.split(" [")[0] for th in div.findAll("th")]
trs = div.find("tbody").findAll("tr")
rows = [[td.text for td in tr.findAll("td")] for tr in trs]
assert columns == expected_columns
assert rows == expected_rows
def test_dashboard_uses_post_if_sql_is_too_long(admin_client):
# Queries longer than 1800 characters do not redirect to GET
short_sql = "select %(start)s::integer + "
long_sql = "select %(start)s::integer + " + "+".join(["1"] * 1801)
assert (
admin_client.post("/dashboard/", {"sql": short_sql, "start": 100}).status_code
== 302
)
response = admin_client.post("/dashboard/", {"sql": long_sql, "start": 100})
assert response.status_code == 200
assert b"1901" in response.content
# And should not have 'count' links
assert b"data-count-url=" not in response.content
@pytest.mark.parametrize(
"path,sqls,args,expected_title",
(
("/dashboard/", [], None, "SQL Dashboard"),
("/dashboard/", ["select 1"], None, "SQL: select 1"),
(
"/dashboard/",
["select %(name)s"],
{"name": "test"},
"SQL: select %(name)s: test",
),
(
"/dashboard/",
["select %(name)s, %(age)s"],
{"name": "test", "age": 5},
"SQL: select %(name)s, %(age)s: name=test, age=5",
),
("/dashboard/", ["select 1", "select 2"], None, "SQL: select 1 [,] select 2"),
("/dashboard/test/", [], None, "Test dashboard"),
("/dashboard/test/", [], {"name": "claire"}, "Test dashboard: claire"),
),
)
def test_dashboard_html_title(
admin_client, saved_dashboard, path, args, sqls, expected_title
):
saved_dashboard.queries.create(sql="select %(name)s")
args = args or {}
if sqls:
args["sql"] = sqls
response = admin_client.post(path, args, follow=True)
else:
response = admin_client.get(path, data=args)
soup = BeautifulSoup(response.content, "html5lib")
assert soup.find("title").text == expected_title
def test_saved_dashboard_errors_sql_not_in_textarea(admin_client, saved_dashboard):
saved_dashboard.queries.create(sql="this is bad")
response = admin_client.get("/dashboard/test/")
html = response.content.decode("utf-8")
assert '
this is bad
' in html
def test_dashboard_show_available_tables(admin_client):
response = admin_client.get("/dashboard/")
soup = BeautifulSoup(response.content, "html5lib")
lis = soup.find("ul").findAll("li")
details = [
{
"table": li.find("a").text,
"columns": li.find("p").text,
"href": li.find("a")["href"],
}
for li in lis
if li.find("a").text.startswith("django_sql_dashboard")
or li.find("a").text == "switches"
]
# Decode the href in each one into a SQL query
for detail in details:
href = detail.pop("href")
detail["href_sql"] = urllib.parse.parse_qs(href)["sql"][0].rsplit(":", 1)[0]
assert details == [
{
"table": "django_sql_dashboard_dashboard",
"columns": "id, slug, title, description, created_at, edit_group_id, edit_policy, owned_by_id, view_group_id, view_policy",
"href_sql": "select id, slug, title, description, created_at, edit_group_id, edit_policy, owned_by_id, view_group_id, view_policy from django_sql_dashboard_dashboard",
},
{
"table": "django_sql_dashboard_dashboardquery",
"columns": "id, sql, dashboard_id, _order",
"href_sql": "select id, sql, dashboard_id, _order from django_sql_dashboard_dashboardquery",
},
{
"table": "switches",
"columns": "id, name, on",
"href_sql": 'select id, name, "on" from switches',
},
]
================================================
FILE: test_project/test_dashboard_permissions.py
================================================
from enum import Enum
import pytest
from bs4 import BeautifulSoup
from django.contrib.auth.models import Group, User
from django_sql_dashboard.models import Dashboard
def test_anonymous_user_redirected_to_login(client):
response = client.get("/dashboard/?sql=select+1")
assert response.status_code == 302
assert response.url == "/admin/login/?next=/dashboard/%3Fsql%3Dselect%2B1"
def test_superusers_allowed(admin_client, dashboard_db):
response = admin_client.get("/dashboard/")
assert response.status_code == 200
assert b"SQL Dashboard" in response.content
def test_must_have_execute_sql_permission(
client, django_user_model, dashboard_db, execute_sql_permission
):
not_staff = django_user_model.objects.create(username="not_staff")
staff_no_permisssion = django_user_model.objects.create(
username="staff_no_permission", is_staff=True
)
staff_with_permission = django_user_model.objects.create(
username="staff_with_permission", is_staff=True
)
staff_with_permission.user_permissions.add(execute_sql_permission)
assert staff_with_permission.has_perm("django_sql_dashboard.execute_sql")
client.force_login(not_staff)
assert client.get("/dashboard/").status_code == 403
client.force_login(staff_no_permisssion)
assert client.get("/dashboard/").status_code == 403
client.force_login(staff_with_permission)
assert client.get("/dashboard/").status_code == 200
def test_user_without_execute_sql_permission_does_not_see_count_links_on_saved_dashboard(
client, django_user_model, execute_sql_permission, dashboard_db
):
dashboard = Dashboard.objects.create(slug="test", view_policy="public")
dashboard.queries.create(sql="select 11 + 34")
user = django_user_model.objects.create(username="regular")
client.force_login(user)
response = client.get("/dashboard/test/")
assert response.status_code == 200
html = response.content.decode("utf-8")
assert "data-count-url=" not in html
# If the user DOES have that permission they get the count links
user.user_permissions.add(execute_sql_permission)
response = client.get("/dashboard/test/")
html = response.content.decode("utf-8")
assert "data-count-url=" in html
def test_saved_dashboard_anonymous_users_denied_by_default(client, dashboard_db):
dashboard = Dashboard.objects.create(slug="test")
dashboard.queries.create(sql="select 11 + 34")
response = client.get("/dashboard/test/")
assert response.status_code == 403
class UserType(Enum):
owner = 1
anon = 2
loggedin = 3
groupmember = 4
staff = 5
superuser = 6
all_user_types = (
UserType.owner,
UserType.anon,
UserType.loggedin,
UserType.groupmember,
UserType.staff,
UserType.superuser,
)
@pytest.mark.parametrize(
"view_policy,user_types_who_can_see",
(
("private", (UserType.owner,)),
("public", all_user_types),
("unlisted", all_user_types),
(
"loggedin",
(
UserType.owner,
UserType.loggedin,
UserType.groupmember,
UserType.staff,
UserType.superuser,
),
),
("group", (UserType.owner, UserType.groupmember)),
("staff", (UserType.owner, UserType.staff, UserType.superuser)),
("superuser", (UserType.owner, UserType.superuser)),
),
)
def test_saved_dashboard_view_permissions(
client,
dashboard_db,
view_policy,
user_types_who_can_see,
django_user_model,
):
users = {
UserType.owner: django_user_model.objects.create(username="owner"),
UserType.anon: None,
UserType.loggedin: django_user_model.objects.create(username="loggedin"),
UserType.groupmember: django_user_model.objects.create(username="groupmember"),
UserType.staff: django_user_model.objects.create(
username="staff", is_staff=True
),
UserType.superuser: django_user_model.objects.create(
username="superuser", is_staff=True, is_superuser=True
),
}
group = Group.objects.create(name="view-group")
users[UserType.groupmember].groups.add(group)
Dashboard.objects.create(
slug="dash",
owned_by=users[UserType.owner],
view_policy=view_policy,
view_group=group,
)
for user_type, user in users.items():
if user is not None:
client.force_login(user)
else:
client.logout()
response = client.get("/dashboard/dash/")
if user_type in user_types_who_can_see:
assert response.status_code == 200
else:
assert response.status_code == 403
if user is not None:
assert response["cache-control"] == "private"
def test_unlisted_dashboard_has_meta_robots(client, dashboard_db):
dashboard = Dashboard.objects.create(slug="unlisted", view_policy="unlisted")
dashboard.queries.create(sql="select 11 + 34")
response = client.get("/dashboard/unlisted/")
assert response.status_code == 200
assert b'' in response.content
dashboard.view_policy = "public"
dashboard.save()
response2 = client.get("/dashboard/unlisted/")
assert response2.status_code == 200
assert b'' not in response2.content
@pytest.mark.parametrize(
"dashboard,expected,expected_if_staff,expected_if_superuser",
(
("owned_by_user", True, True, True),
("owned_by_other_private", False, False, False),
("owned_by_other_public", True, True, True),
("owned_by_other_unlisted", False, False, False),
("owned_by_other_loggedin", True, True, True),
("owned_by_other_group_not_member", False, False, False),
("owned_by_other_group_member", True, True, True),
("owned_by_other_staff", False, True, True),
("owned_by_other_superuser", False, False, True),
),
)
def test_get_visible_to_user(
db, dashboard, expected, expected_if_staff, expected_if_superuser
):
user = User.objects.create(username="test")
other = User.objects.create(username="other")
group_member = Group.objects.create(name="group_member")
user.groups.add(group_member)
group_not_member = Group.objects.create(name="group_not_member")
Dashboard.objects.create(slug="owned_by_user", owned_by=user, view_policy="private")
Dashboard.objects.create(
slug="owned_by_other_private", owned_by=other, view_policy="private"
)
Dashboard.objects.create(
slug="owned_by_other_public", owned_by=other, view_policy="public"
)
Dashboard.objects.create(
slug="owned_by_other_unlisted", owned_by=other, view_policy="unlisted"
)
Dashboard.objects.create(
slug="owned_by_other_loggedin", owned_by=other, view_policy="loggedin"
)
Dashboard.objects.create(
slug="owned_by_other_group_not_member",
owned_by=other,
view_policy="group",
view_group=group_not_member,
)
Dashboard.objects.create(
slug="owned_by_other_group_member",
owned_by=other,
view_policy="group",
view_group=group_member,
)
Dashboard.objects.create(
slug="owned_by_other_staff", owned_by=other, view_policy="staff"
)
Dashboard.objects.create(
slug="owned_by_other_superuser", owned_by=other, view_policy="superuser"
)
visible_dashboards = set(
Dashboard.get_visible_to_user(user).values_list("slug", flat=True)
)
if expected:
assert (
dashboard in visible_dashboards
), "Expected user to be able to see {}".format(dashboard)
else:
assert (
dashboard not in visible_dashboards
), "Expected user not to be able to see {}".format(dashboard)
user.is_staff = True
user.save()
visible_dashboards = set(
Dashboard.get_visible_to_user(user).values_list("slug", flat=True)
)
if expected_if_staff:
assert (
dashboard in visible_dashboards
), "Expected staff user to be able to see {}".format(dashboard)
else:
assert (
dashboard not in visible_dashboards
), "Expected staff user not to be able to see {}".format(dashboard)
user.is_superuser = True
user.save()
visible_dashboards = set(
Dashboard.get_visible_to_user(user).values_list("slug", flat=True)
)
if expected_if_superuser:
assert (
dashboard in visible_dashboards
), "Expected super user to be able to see {}".format(dashboard)
else:
assert (
dashboard not in visible_dashboards
), "Expected super user not to be able to see {}".format(dashboard)
def test_get_visible_to_user_no_dupes(db):
owner = User.objects.create(username="owner", is_staff=True)
group = Group.objects.create(name="group")
for i in range(3):
group.user_set.add(User.objects.create(username="user{}".format(i)))
Dashboard.objects.create(
owned_by=owner,
slug="example",
view_policy="public",
view_group=group,
)
dashboards = list(
Dashboard.get_visible_to_user(owner).values_list("slug", flat=True)
)
# This used to return ["example", "example", "example"]
# Until I fixed https://github.com/simonw/django-sql-dashboard/issues/90
assert dashboards == ["example"]
@pytest.mark.parametrize(
"dashboard,expected,expected_if_staff,expected_if_superuser",
(
("owned_by_user", True, True, True),
("owned_by_other_private", False, False, False),
("owned_by_other_loggedin", True, True, True),
("owned_by_other_group_not_member", False, False, False),
("owned_by_other_group_member", True, True, True),
("owned_by_other_staff", False, True, True),
("owned_by_other_superuser", False, False, True),
),
)
def test_user_can_edit(
db, client, dashboard, expected, expected_if_staff, expected_if_superuser
):
user = User.objects.create(username="test")
other = User.objects.create(username="other")
group_member = Group.objects.create(name="group_member")
user.groups.add(group_member)
group_not_member = Group.objects.create(name="group_not_member")
Dashboard.objects.create(slug="owned_by_user", owned_by=user, edit_policy="private")
Dashboard.objects.create(
slug="owned_by_other_private", owned_by=other, edit_policy="private"
)
Dashboard.objects.create(
slug="owned_by_other_loggedin", owned_by=other, edit_policy="loggedin"
)
Dashboard.objects.create(
slug="owned_by_other_group_not_member",
owned_by=other,
edit_policy="group",
edit_group=group_not_member,
)
Dashboard.objects.create(
slug="owned_by_other_group_member",
owned_by=other,
edit_policy="group",
edit_group=group_member,
)
Dashboard.objects.create(
slug="owned_by_other_staff", owned_by=other, edit_policy="staff"
)
Dashboard.objects.create(
slug="owned_by_other_superuser", owned_by=other, edit_policy="superuser"
)
dashboard_obj = Dashboard.objects.get(slug=dashboard)
dashboard_obj.queries.create(sql="select 1 + 1")
assert dashboard_obj.user_can_edit(user) == expected
if dashboard != "owned_by_other_staff":
# This test doesn't make sense for the 'staff' one, they cannot access admin
# https://github.com/simonw/django-sql-dashboard/issues/44#issuecomment-835653787
can_edit_using_admin = can_user_edit_using_admin(client, user, dashboard_obj)
assert can_edit_using_admin == expected
if can_edit_using_admin:
# Check that they cannot edit the SQL queries, because they do not
# have the execute_sql permisssion
assert not user.has_perm("django_sql_dashboard.execute_sql")
html = get_admin_change_form_html(client, user, dashboard_obj)
soup = BeautifulSoup(html, "html5lib")
assert soup.select("td.field-sql p")[0].text == "select 1 + 1"
user.is_staff = True
user.save()
assert dashboard_obj.user_can_edit(user) == expected_if_staff
assert can_user_edit_using_admin(client, user, dashboard_obj) == expected_if_staff
# Confirm that staff user can see the correct dashboards listed
client.force_login(user)
dashboard_change_list_response = client.get(
"/admin/django_sql_dashboard/dashboard/"
)
change_list_soup = BeautifulSoup(dashboard_change_list_response.content, "html5lib")
visible_in_change_list = [
a.text for a in change_list_soup.select("th.field-slug a")
]
assert set(visible_in_change_list) == {
"owned_by_other_staff",
"owned_by_other_group_member",
"owned_by_other_loggedin",
"owned_by_user",
}
# Promote to superuser
user.is_superuser = True
user.save()
assert dashboard_obj.user_can_edit(user) == expected_if_superuser
assert can_user_edit_using_admin(client, user, dashboard_obj)
def get_admin_change_form_html(client, user, dashboard):
# Only staff can access the admin:
original_is_staff = user.is_staff
user.is_staff = True
user.save()
client.force_login(user)
response = client.get(dashboard.get_edit_url())
if not original_is_staff:
user.is_staff = False
user.save()
return response.content.decode("utf-8")
def can_user_edit_using_admin(client, user, dashboard):
return (
''
in get_admin_change_form_html(client, user, dashboard)
)
def test_superuser_can_reassign_ownership(client, db):
user = User.objects.create(username="test", is_staff=True)
dashboard = Dashboard.objects.create(
slug="dashboard", owned_by=user, view_policy="private", edit_policy="private"
)
client.force_login(user)
response = client.get(dashboard.get_edit_url())
assert (
b'