Repository: explorerhq/django-sql-explorer Branch: master Commit: 7627b426e0b8 Files: 214 Total size: 692.0 KB Directory structure: gitextract_ainzhnq4/ ├── .dockerignore ├── .editorconfig ├── .eslintignore ├── .github/ │ ├── dependabot.yml │ └── workflows/ │ ├── codeql-analysis.yml │ ├── docs.yml │ ├── lint.yml │ ├── publish-pypi.yml │ ├── publish-test.yml │ └── test.yml ├── .gitignore ├── .nvmrc ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── AUTHORS ├── Dockerfile ├── HISTORY.rst ├── LICENSE ├── MANIFEST.in ├── README.rst ├── docker-compose.yml ├── docs/ │ ├── Makefile │ ├── _static/ │ │ └── .directory │ ├── _templates/ │ │ └── .directory │ ├── conf.py │ ├── dependencies.rst │ ├── development.rst │ ├── features.rst │ ├── history.rst │ ├── index.rst │ ├── install.rst │ ├── make.bat │ ├── requirements.txt │ └── settings.rst ├── entrypoint.sh ├── explorer/ │ ├── __init__.py │ ├── actions.py │ ├── admin.py │ ├── app_settings.py │ ├── apps.py │ ├── assistant/ │ │ ├── __init__.py │ │ ├── forms.py │ │ ├── models.py │ │ ├── urls.py │ │ ├── utils.py │ │ └── views.py │ ├── charts.py │ ├── ee/ │ │ ├── LICENSE │ │ ├── __init__.py │ │ ├── db_connections/ │ │ │ ├── __init__.py │ │ │ ├── admin.py │ │ │ ├── create_sqlite.py │ │ │ ├── forms.py │ │ │ ├── mime.py │ │ │ ├── models.py │ │ │ ├── type_infer.py │ │ │ ├── utils.py │ │ │ └── views.py │ │ └── urls.py │ ├── exporters.py │ ├── forms.py │ ├── locale/ │ │ ├── ru/ │ │ │ └── LC_MESSAGES/ │ │ │ ├── django.mo │ │ │ └── django.po │ │ └── zh_Hans/ │ │ └── LC_MESSAGES/ │ │ ├── django.mo │ │ └── django.po │ ├── migrations/ │ │ ├── 0001_initial.py │ │ ├── 0002_auto_20150501_1515.py │ │ ├── 0003_query_snapshot.py │ │ ├── 0004_querylog_duration.py │ │ ├── 0005_auto_20160105_2052.py │ │ ├── 0006_query_connection.py │ │ ├── 0007_querylog_connection.py │ │ ├── 0008_auto_20190308_1642.py │ │ ├── 0009_auto_20201009_0547.py │ │ ├── 0010_sql_required.py │ │ ├── 0011_query_favorites.py │ │ ├── 0012_alter_queryfavorite_query_alter_queryfavorite_user.py │ │ ├── 0013_querylog_error_querylog_success.py │ │ ├── 0014_promptlog.py │ │ ├── 0015_explorervalue.py │ │ ├── 0016_alter_explorervalue_key.py │ │ ├── 0017_databaseconnection.py │ │ ├── 0018_alter_databaseconnection_host_and_more.py │ │ ├── 0019_alter_databaseconnection_engine.py │ │ ├── 0020_databaseconnection_extras_and_more.py │ │ ├── 0021_alter_databaseconnection_password_and_more.py │ │ ├── 0022_databaseconnection_upload_fingerprint.py │ │ ├── 0023_query_database_connection_and_more.py │ │ ├── 0024_auto_20240803_1135.py │ │ ├── 0025_alter_query_database_connection_alter_querylog_database_connection.py │ │ ├── 0026_tabledescription.py │ │ ├── 0027_query_few_shot.py │ │ ├── 0028_promptlog_database_connection_promptlog_user_request.py │ │ └── __init__.py │ ├── models.py │ ├── permissions.py │ ├── schema.py │ ├── src/ │ │ ├── js/ │ │ │ ├── assistant.js │ │ │ ├── codemirror-config.js │ │ │ ├── csrf.js │ │ │ ├── explorer.js │ │ │ ├── favorites.js │ │ │ ├── main.js │ │ │ ├── pivot-setup.js │ │ │ ├── pivot.js │ │ │ ├── query-list.js │ │ │ ├── schema.js │ │ │ ├── schemaService.js │ │ │ ├── table-to-csv.js │ │ │ ├── tableDescription.js │ │ │ └── uploads.js │ │ └── scss/ │ │ ├── assistant.scss │ │ ├── choices.scss │ │ ├── explorer.scss │ │ ├── pivot.css │ │ ├── styles.scss │ │ └── variables.scss │ ├── tasks.py │ ├── telemetry.py │ ├── templates/ │ │ ├── assistant/ │ │ │ ├── table_description_confirm_delete.html │ │ │ ├── table_description_form.html │ │ │ └── table_description_list.html │ │ ├── connections/ │ │ │ ├── connection_upload.html │ │ │ ├── connections.html │ │ │ ├── database_connection_confirm_delete.html │ │ │ ├── database_connection_detail.html │ │ │ └── database_connection_form.html │ │ └── explorer/ │ │ ├── assistant.html │ │ ├── base.html │ │ ├── export_buttons.html │ │ ├── fullscreen.html │ │ ├── params.html │ │ ├── pdf_template.html │ │ ├── play.html │ │ ├── preview_pane.html │ │ ├── query.html │ │ ├── query_confirm_delete.html │ │ ├── query_favorite_button.html │ │ ├── query_favorites.html │ │ ├── query_list.html │ │ ├── querylog_list.html │ │ ├── schema.html │ │ └── schema_error.html │ ├── templatetags/ │ │ ├── __init__.py │ │ ├── explorer_tags.py │ │ └── vite.py │ ├── tests/ │ │ ├── __init__.py │ │ ├── csvs/ │ │ │ ├── all_types.csv │ │ │ ├── dates.csv │ │ │ ├── floats.csv │ │ │ ├── integers.csv │ │ │ ├── mixed.csv │ │ │ ├── rc_sample.csv │ │ │ └── test_case1.csv │ │ ├── factories.py │ │ ├── json/ │ │ │ ├── github.json │ │ │ ├── kings.json │ │ │ └── list.json │ │ ├── settings.py │ │ ├── settings_base.py │ │ ├── test_actions.py │ │ ├── test_apps.py │ │ ├── test_assistant.py │ │ ├── test_create_sqlite.py │ │ ├── test_csrf_cookie_name.py │ │ ├── test_db_connection_utils.py │ │ ├── test_exporters.py │ │ ├── test_forms.py │ │ ├── test_mime.py │ │ ├── test_models.py │ │ ├── test_schema.py │ │ ├── test_tasks.py │ │ ├── test_telemetry.py │ │ ├── test_type_infer.py │ │ ├── test_utils.py │ │ └── test_views.py │ ├── urls.py │ ├── utils.py │ └── views/ │ ├── __init__.py │ ├── auth.py │ ├── create.py │ ├── delete.py │ ├── download.py │ ├── email.py │ ├── export.py │ ├── format_sql.py │ ├── list.py │ ├── mixins.py │ ├── query.py │ ├── query_favorite.py │ ├── schema.py │ ├── stream.py │ └── utils.py ├── manage.py ├── package.json ├── public_key.pem ├── pypi-release-checklist.md ├── requirements/ │ ├── base.txt │ ├── dev.txt │ ├── extra/ │ │ ├── assistant.txt │ │ ├── charts.txt │ │ ├── snapshots.txt │ │ ├── uploads.txt │ │ └── xls.txt │ └── tests.txt ├── ruff.toml ├── setup.cfg ├── setup.py ├── test_project/ │ ├── __init__.py │ ├── celery_config.py │ ├── settings.py │ └── urls.py ├── tox.ini └── vite.config.mjs ================================================ FILE CONTENTS ================================================ ================================================ FILE: .dockerignore ================================================ node_modules ================================================ FILE: .editorconfig ================================================ # http://editorconfig.org root = true [*] charset = utf-8 end_of_line = lf insert_final_newline = true trim_trailing_whitespace = true indent_style = space indent_size = 4 [*.py] max_line_length = 120 [*.toml] indent_size = 2 [*.yaml] indent_size = 2 [*.yml] indent_size = 2 ================================================ FILE: .eslintignore ================================================ /explorer/static/js/src/pivot.js ================================================ FILE: .github/dependabot.yml ================================================ # To get started with Dependabot version updates, you'll need to specify which # package ecosystems to update and where the package manifests are located. # Please see the documentation for all configuration options: # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates version: 2 updates: - package-ecosystem: "github-actions" directory: "/" # Location of package manifests schedule: interval: "weekly" ================================================ FILE: .github/workflows/codeql-analysis.yml ================================================ # For most projects, this workflow file will not need changing; you simply need # to commit it to your repository. # # You may wish to alter this file to override the set of languages analyzed, # or to provide custom queries or build logic. name: "CodeQL" permissions: actions: read contents: read security-events: write on: push: branches: - master - support/3.x pull_request: # The branches below must be a subset of the branches above branches: - master - support/3.x schedule: - cron: '0 2 * * 5' concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: analyze: name: Analyze runs-on: ubuntu-latest strategy: fail-fast: false matrix: # Override automatic language detection by changing the below list # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] language: ['python', 'javascript'] # Learn more... # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection steps: - name: Checkout repository uses: actions/checkout@v4 with: # We must fetch at least the immediate parents so that if this is # a pull request then we can checkout the head. fetch-depth: 2 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # queries: ./path/to/local/query, your-org/your-repo/queries@main # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v3 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines # and modify them (or add more) to build your code if your project # uses a compiled language #- run: | # make bootstrap # make release - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 ================================================ FILE: .github/workflows/docs.yml ================================================ name: Docs on: [push, pull_request] concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: docs: runs-on: ubuntu-latest name: docs steps: - name: Checkout uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: 3.9 - run: python -m pip install -r docs/requirements.txt - name: Build docs run: | cd docs sphinx-build -b html -n -d _build/doctrees . _build/html ================================================ FILE: .github/workflows/lint.yml ================================================ name: Ruff on: push: pull_request: jobs: ruff: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Run ruff run: pipx run ruff check --output-format=github explorer ================================================ FILE: .github/workflows/publish-pypi.yml ================================================ name: Publish Python 🐍 distributions 📦 to pypi on: release: types: - published jobs: build-n-publish: name: Build and publish Python 🐍 distributions 📦 to pypi runs-on: ubuntu-latest environment: name: pypi url: https://pypi.org/p/django-sql-explorer permissions: id-token: write steps: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.12' - uses: actions/setup-node@v4 with: node-version-file: '.nvmrc' - name: Install dependencies run: npm install - name: Build client run: npm run build - name: Install pypa/build run: >- python -m pip install build --user - name: Build a binary wheel and a source tarball run: >- python -m build --sdist --wheel --outdir dist/ . - name: Publish distribution 📦 to PyPI if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@release/v1 ================================================ FILE: .github/workflows/publish-test.yml ================================================ name: Publish Python 🐍 distributions 📦 to TestPyPI on: push: branches: - master - support/3.x jobs: build-n-publish: name: Build and publish Python 🐍 distributions 📦 to TestPyPI runs-on: ubuntu-latest environment: name: test url: https://test.pypi.org/p/django-sql-explorer permissions: id-token: write steps: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.12' - name: Install pypa/build run: >- python -m pip install build --user - uses: actions/setup-node@v4 with: node-version-file: '.nvmrc' - name: Install npm dependencies run: npm install - name: Build client run: npm run build - name: Build a binary wheel and a source tarball run: >- python -m build --sdist --wheel --outdir dist/ . - name: Publish distribution 📦 to Test PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ skip_existing: true ================================================ FILE: .github/workflows/test.yml ================================================ name: Tests on: push: branches: - master - support/3.x pull_request: concurrency: group: ${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: tests: name: Python ${{ matrix.python-version }} runs-on: ubuntu-22.04 strategy: matrix: python-version: - '3.10' - '3.11' - '3.12' steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: pip - uses: actions/setup-node@v4 with: node-version-file: '.nvmrc' - name: Install dependencies run: | npm install python -m pip install --upgrade pip setuptools wheel python -m pip install --upgrade 'tox>=4.0.0rc3' - name: Build client run: npm run build - name: Run tox targets for ${{ matrix.python-version }} run: tox run -f py$(echo ${{ matrix.python-version }} | tr -d .) - name: Upload coverage data uses: actions/upload-artifact@v4 with: name: coverage-data-${{ matrix.python-version }} path: '.coverage*' include-hidden-files: true coverage: name: Coverage runs-on: ubuntu-22.04 needs: tests steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: '3.12' - name: Install dependencies run: | npm install python -m pip install --upgrade coverage[toml] - name: Build client run: npm run build - name: Download data uses: actions/download-artifact@v4 with: pattern: coverage-data-* merge-multiple: true - name: Combine coverage run: | python -m coverage combine python -m coverage html --skip-covered --skip-empty python -m coverage report - name: Upload HTML report uses: actions/upload-artifact@v4 with: name: html-report path: htmlcov ================================================ FILE: .gitignore ================================================ /.idea/ *.pyc *.db /project/ /dist *.egg-info .DS_Store /build *# *~ .coverage* /htmlcov/ *.orig tmp venv/ .venv/ .tox/ node_modules/ explorer/static/ # Sphinx documentation docs/_build/ .env tst tst2 user_dbs/* tmp2 chinook.sqlite model_data.json tst1 tst1-journal tst2 coverage-data-* ================================================ FILE: .nvmrc ================================================ 20.15.1 ================================================ FILE: .pre-commit-config.yaml ================================================ ci: autofix_commit_msg: | ci: auto fixes from pre-commit hooks for more information, see https://pre-commit.ci autofix_prs: false autoupdate_commit_msg: "ci: pre-commit autoupdate" autoupdate_schedule: monthly default_language_version: python: python3.12 repos: - repo: https://github.com/asottile/pyupgrade rev: v3.16.0 hooks: - id: pyupgrade args: ["--py38-plus"] - repo: https://github.com/adamchainz/django-upgrade rev: "1.19.0" hooks: - id: django-upgrade args: [--target-version, "3.2"] - repo: https://github.com/asottile/yesqa rev: v1.5.0 hooks: - id: yesqa - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 hooks: - id: check-merge-conflict - id: mixed-line-ending - repo: https://github.com/astral-sh/ruff-pre-commit rev: "v0.5.0" hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] - id: ruff-format - repo: https://github.com/remastr/pre-commit-django-check-migrations rev: v0.1.0 hooks: - id: check-migrations-created args: [--manage-path=manage.py] additional_dependencies: [django==4.1] - repo: https://github.com/rstcheck/rstcheck rev: v6.2.0 hooks: - id: rstcheck additional_dependencies: - sphinx==6.1.3 - tomli==2.0.1 - repo: https://github.com/pre-commit/mirrors-prettier rev: v3.0.0 hooks: - id: prettier ================================================ FILE: .readthedocs.yaml ================================================ # Read the Docs configuration file # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details version: 2 build: os: ubuntu-22.04 tools: python: "3.11" sphinx: configuration: docs/conf.py fail_on_warning: false formats: - epub - pdf python: install: - requirements: docs/requirements.txt ================================================ FILE: AUTHORS ================================================ The following people have contributed to django-sql-explorer: - Chris Clark - Mark Walker - Lee Brooks - Artyom Chernyakov - Rodney Hawkins - Dane Hillard - Wojtek Jurkowlaniec - Lee Kagiso - Phil Krylov - Grant McConnaughey - Josh Miller - Jens Nistler - Pietro Pilolli - David Sanders - Anton Shutik - Nick Spacek - Stanislav Tarazevich - Ming Hsien Tseng - Jared Proffitt - Brad Melin - Dara Adib - Moe Elias - Illia Volochii - Amir Abedi - Christian Clauss - Shiyan Shirani - Calum Smith - Steven Luoma A full list of contributors can be found on Github; https://github.com/explorerhq/sql-explorer/graphs/contributors ================================================ FILE: Dockerfile ================================================ # Build stage FROM python:3.12.4 as builder WORKDIR /app # Install system dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ build-essential \ && rm -rf /var/lib/apt/lists/* # Install Python dependencies COPY requirements /app/requirements RUN pip install --no-cache-dir -r requirements/dev.txt # Install NVM and Node.js RUN mkdir /usr/local/.nvm ENV NVM_DIR /usr/local/.nvm # This should match the version referenced below in the Run stage, and in entrypoint.sh ENV NODE_VERSION 20.15.1 COPY package.json package-lock.json /app/ RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash \ && . "$NVM_DIR/nvm.sh" \ && nvm install ${NODE_VERSION} \ && nvm use v${NODE_VERSION} \ && nvm alias default v${NODE_VERSION} \ && npm install # Runtime stage FROM python:3.12.4 WORKDIR /app # Copy Python environment from builder COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages COPY --from=builder /usr/local/bin /usr/local/bin # Copy Node.js environment from builder COPY --from=builder /usr/local/.nvm /usr/local/.nvm ENV NVM_DIR /usr/local/.nvm # The version in this path should match the version referenced above in the Run stage, and in entrypoint.sh ENV PATH $NVM_DIR/versions/node/v20.15.1/bin:$PATH COPY --from=builder /app/node_modules /app/node_modules COPY . /app # Run migrations and create initial data RUN python manage.py migrate && \ python manage.py shell <`_. This project adheres to `Semantic Versioning `_. `5.3.0`_ (2024-09-24) =========================== * `#664`_: Improvements to the AI SQL Assistant: - Table Annotations: Write persistent table annotations with descriptive information that will get injected into the prompt for the assistant. For example, if a table is commonly joined to another table through a non-obvious foreign key, you can tell the assistant about it in plain english, as an annotation to that table. Every time that table is deemed 'relevant' to an assistant request, that annotation will be included alongside the schema and sample data. - Few-Shot Examples: Using the small checkbox on the bottom-right of any saved queries, you can designate certain queries as 'few shot examples". When making an assistant request, any designated few-shot examples that reference the same tables as your assistant request will get included as 'reference sql' in the prompt for the LLM. - Autocomplete / multiselect when selecting tables info to send to the SQL Assistant. Much easier and more keyboard focused. - Relevant tables are added client-side visually, in real time, based on what's in the SQL editor and/or any tables mentioned in the assistant request. The dependency on sql_metadata is therefore removed, as server-side SQL parsing is no longer necessary. - Ability to view Assistant request/response history. - Improved system prompt that emphasizes the particular SQL dialect being used. - Addresses issue #657. * `#660`_: Userspace connection migration. - This should be an invisible change, but represents a significant refactor of how connections function. Instead of a weird blend of DatabaseConnection models and underlying Django models (which were the original Explorer connections), this migrates all connections to DatabaseConnection models and implements proper foreign keys to them on the Query and QueryLog models. A data migration creates new DatabaseConnection models based on the configured settings.EXPLORER_CONNECTIONS. Going forward, admins can create new Django-backed DatabaseConnection models by registering the connection in EXPLORER_CONNECTIONS, and then creating a DatabaseConnection model using the Django admin or the user-facing /connections/new/ form, and entering the Django DB alias and setting the connection type to "Django Connection". - The Query.connection and QueryLog.connection fields are deprecated and will be removed in a future release. They are kept around in this release in case there is an unforeseen issue with the migration. Preserving the fields for now ensures there is no data loss in the event that a rollback to an earlier version is required. * Fixed a bug when validating connections to uploaded files. Also added basic locking when downloading files from S3. * On-boarding UI; if no connections or queries are created, the UI walks the user through it a bit. * Keyboard shortcut for formatting the SQL in the editor. - Cmd+Shift+F (Windows: Ctrl+Shift+F) - The format button has been moved tobe a small icon towards the bottom-right of the SQL editor. * `#675`_ - fail gracefully when building the schema if a particular table cant be accessed by the connection `5.2.0`_ (2024-08-19) =========================== * `#651`_: Ability to append an upload to a previously uploaded file/sqlite DB as a new table * Good cache busting and detection of file changes on uploads * Significant documentation improvements to uploads and connections * Separate the upload UI from the 'add connection' UI, as they are materially different * Fix a small bug with bar chart generation, when values are null * Ability to refresh a connection's schema and data (if it's an upload) from the connections list view * `#659`_: Search all queries, even if the header is collapsed. Addresses issue #464 (partially) and #658 (fully). * `#662`_: Refactored dockerfile to use non-root directories. Addresses issue #661. `5.1.1`_ (2024-07-30) =========================== * `#654`_: Bugfix: Parameterized query does not work for viewers * `#653`_: Bugfix: Schema search not visible anymore * Bugfix: Error messages in query.html were floating in the wrong spot * `#555`_: Prevent queries with many thousands of results from being punishingly slow. The number of data points in the chart now matches the number of data points in the preview pane. `5.1.0`_ (2024-07-30) =========================== Major improvements: * `#647`_: Upload json files as data sources (in addition to CSV and SQLite files). Both 'normal' json files, and files structured as a list of json objects (one json object per line) are supported. * `#643`_: Addresses #640 (Snowflake support). Additionally, supports an "extras" field on the userspace DatabaseConnection object, which allows for arbitrary additional connection params to get added. This allows engine-specific (or just more obscure) settings to get injected into the connection. * `#644`_: Dockerfile and docker-compose to run the test_project. Replaces the old start.sh script. Minor improvements: * `#647`_: In the schema explorer, clicking on a field name copies it to the clipboard * `#647`_: Charts are limited to a maximum of 10 series. This significantly speeds up rendering of 'wide' result-sets when charts are enabled. * `#645`_: Removed pie charts, added bar charts. Replaced Seaborn with Matplotlib because it's much lighter weight. Pie charts were overly finicky to get working. Bars are more useful. Will look to continue to expand charting in the future. * `#643`_: After uploading a csv/json/etc, the resulting connection is highlighted in the connection list, making it much clearer what happened. * `#643`_: Fixed some bugs in user connection stuff in general, and improved the UI. Bugfixes and internal improvements: * `#647`_: Robustness to the user uploads feature, in terms of the UI, error handling and logging, and test coverage. * `#648`_: Backwards migration for 0016_alter_explorervalue_key.py * `#649`_: Use a more reliable source of the static files URL * `#635`_: Improved test coverage in tox, so that base requirements are properly used. This would have prevented (for example) issue 631. Additionally, introduced a test to verify that migrations are always generated, which would have prevented #633. * `#636`_: Output rendering bugfix. * `#567`_: Upgrade translate tags in templates to more modern style. `5.0.2`_ (2024-07-3) =========================== * `#633`_: Missing migration * CSS tweaks to tighten up the Query UI `5.0.1`_ (2024-06-26) =========================== * `#631`_: Pandas is only required if EXPLORER_USER_UPLOADS_ENABLED is True `5.0.0`_ (2024-06-25) =========================== * Manage DB connections via the UI (and/or Django Admin). Set EXPLORER_DB_CONNECTIONS_ENABLED to True in settings to enable user-facing connection management. * Upload CSV or SQLite DBs directly, to create additional connections. This functionality has additional dependencies which can be installed with the 'uploads' extra (e.g. pip install django-sql-explorer[uploads]). Then set EXPLORER_USER_UPLOADS_ENABLED to True, and make sure S3_BUCKET is also set up. * The above functionality is managed by a new license, restricting the ability of 3rd parties resell SQL Explorer (commercial usage is absolutely still permitted). * Query List home page is sortable * Select all / deselect all with AI assistant * Assistant tests run reliably in CI/CD * Introduced some branding and styling improvements `4.3.0`_ (2024-05-27) =========================== * Keyboard shortcut to show schema hints (cmd+S / ctrl+S -- note that is a capital "S" so the full kbd commands is cmd+shift+s) * DB-managed LLM prompts (editable in django admin) * Versioned .js bundles (for cache busting) * Automatically populate assistant responses that contain code into the editor * `#616`_: Update schema/assistant tables/autocomplete on connection drop-down change * `#618`_: Import models so that migrations are properly understood by Django * `#619`_: Get CSRF from DOM (instead of cookie) if CSRF_USE_SESSIONS is set `4.2.0`_ (2024-04-26) =========================== * `#609`_: Tracking should be opt-in and not use the SECRET_KEY * `#610`_: Import error (sql_metadata) with 4.1 version * `#612`_: Accessing the database during app initialization * Regex-injection vulnerability * Improved assistant UI `4.1.0`_ (2024-04-23) =========================== * SQL Assistant: Built in query help via OpenAI (or LLM of choice), with relevant schema automatically injected into the prompt. Enable by setting EXPLORER_AI_API_KEY. * Anonymous usage telemetry. Disable by setting EXPLORER_ENABLE_ANONYMOUS_STATS to False. * Refactor pip requirements to make 'extras' more robust and easier to manage. * `#592`_: Support user models with no email fields * `#594`_: Eliminate {% endblock %} ================================================ FILE: explorer/templates/connections/database_connection_confirm_delete.html ================================================ {% extends 'explorer/base.html' %} {% block sql_explorer_content %}

Delete Database Connection

Are you sure you want to delete "{{ object }}"?

{% csrf_token %} Cancel
{% endblock %} ================================================ FILE: explorer/templates/connections/database_connection_detail.html ================================================ {% extends "explorer/base.html" %} {% block sql_explorer_content %}

Connection Details

{% if not object.is_upload %} {% endif %}
Alias {{ object.alias }}
Name {{ object.name }}
Engine {{ object.get_engine_display }}
Host {{ object.host }}
User {{ object.user }}
Port {{ object.port }}
Extras {{ object.extras }}
{% if object.is_upload %} The source of this connection is an uploaded file. {% endif %} Edit
{% endblock %} ================================================ FILE: explorer/templates/connections/database_connection_form.html ================================================ {% extends 'explorer/base.html' %} {% block sql_explorer_content %}

{% if object %}Edit{% else %}Create New{% endif %} Connection

{% if object.is_upload %} The source of this connection is an uploaded file. In all likelihood you should not be editing it. {% endif %}
{% csrf_token %}
{{ form.alias }} Required. How the connection will appear in SQL Explorer.
{{ form.engine }}
{{ form.name }} Required. The name of the database itself.
{{ form.user }}
{{ form.password }}
{{ form.host }}
{{ form.port }}
{{ form.extras }} Optionally provide JSON that will get merged into the final connection object. The result should be a valid Django database connection dictionary. This is somewhat rarely used.
Cancel
{% endblock %} ================================================ FILE: explorer/templates/explorer/assistant.html ================================================ {% load i18n %}
"Ask Assistant" to try and automatically fix the issue. The assistant is already aware of error messages & context.
================================================ FILE: explorer/templates/explorer/base.html ================================================ {% load i18n static %} {% load vite %} {% translate "SQL Explorer" %}{% if query %} - {{ query.title }}{% elif title %} - {{ title }}{% endif %} {% block style %} {% vite_asset 'scss/styles.scss' %} {% endblock style %} {% if vite_dev_mode %}

Looks like Vite isn't running

This is easy to fix, I promise!

You can run:
npm run dev
Then refresh this page, and you'll get all of your styles, JS, and hot-reloading.
If this is the first time you are running the project, then run:
nvm install
nvm use
npm install
npm run dev
{% endif %} {% block sql_explorer_content_takeover %} {% block sql_explorer_content %}{% endblock %} {% endblock %} {% block sql_explorer_footer %}

Powered by SQL Explorer. Rendered at {% now "SHORT_DATETIME_FORMAT" %}

{% endblock %} {% block bottom_script %} {% vite_hmr_client %} {% vite_asset 'js/main.js' %} {% endblock bottom_script %} {% block sql_explorer_scripts %}{% endblock %} ================================================ FILE: explorer/templates/explorer/export_buttons.html ================================================ {% load explorer_tags i18n %}
    {% for key, name in exporters %} {% if query and query.id %} {% if query.params %}
  • {{ name }}
    • {% else %}
  • {{ name }}
    • {% endif %} {% else %}
    • {% endif %} {% endfor %}
================================================ FILE: explorer/templates/explorer/fullscreen.html ================================================ {% load i18n static %} {% load vite %} {% translate "SQL Explorer" %}{% if query %} - {{ query.title }}{% elif title %} - {{ title }}{% endif %} {% block style %} {% vite_asset 'scss/styles.scss' %} {% endblock style %}

{% if query %} {{ query.title }}{% if shared %}  shared{% endif %} {% else %} {% translate "New Query" %} {% endif %}

{% for h in headers %} {% endfor %} {% if data %} {% for row in data %} {% for i in row %} {% if unsafe_rendering %} {% else %} {% endif %} {% endfor %} {% endfor %} {% else %} {% endif %}
{{ h }}
{% autoescape off %}{{ i }}{% endautoescape %}{{ i }}
{% translate "Empty Resultset" %}
================================================ FILE: explorer/templates/explorer/params.html ================================================ {% if params %}
{% for k, v in params.items %}
{% endfor %}
{% endif %} ================================================ FILE: explorer/templates/explorer/pdf_template.html ================================================ {% for h in headers %} {% endfor %} {% for row in data %} {% for col in row %} {% endfor %} {% endfor %}
{{ h }}
{{ col }}
================================================ FILE: explorer/templates/explorer/play.html ================================================ {% extends "explorer/base.html" %} {% load explorer_tags i18n %} {% block sql_explorer_content %}

{% translate "Playground" %}

{% blocktranslate trimmed %} The playground is for experimenting and writing ad-hoc queries. By default, nothing you do here will be saved. {% endblocktranslate %}

{% csrf_token %} {% if error %}
{{ error|escape }}
{% endif %} {{ form.non_field_errors }} {% if can_change %}
{{ form.database_connection }}
{% else %} {# still need to submit the connection, just hide the UI element #}
{{ form.database_connection }}
{% endif %}
{% if ql_id %} {% endif %}
{% export_buttons query %}
{% if assistant_enabled %} {% include 'explorer/assistant.html' %} {% endif %}
{% include 'explorer/preview_pane.html' %} {% endblock %} ================================================ FILE: explorer/templates/explorer/preview_pane.html ================================================ {% load i18n %} {% if headers %}
{% if query.id and query.snapshot and query.snapshots %} {% endif %} {% if data %} {% if charts_enabled and line_chart_svg %} {% endif %} {% endif %}
{% endif %} ================================================ FILE: explorer/templates/explorer/query.html ================================================ {% extends "explorer/base.html" %} {% load explorer_tags i18n %} {% block sql_explorer_content %}
{% if query %} {% query_favorite_button query.id is_favorite 'query_favorite_toggle query_favourite_detail'%} {% endif %}

{% if query %} {{ query.title }} {% else %} {% translate "New Query" %} {% endif %}

{% if shared %}  shared{% endif %} {% if message %}
{{ message }}
{% endif %}
{% if query %}
{% csrf_token %} {% else %} {% csrf_token %} {% endif %} {% if error %}
{{ error|escape }}
{% endif %} {{ form.non_field_errors }}
{% if form.title.errors %}{% for error in form.title.errors %}
{{ error|escape }}
{% endfor %}{% endif %}
{% if can_change %}
{{ form.database_connection }}
{% else %} {# still need to submit the connection, just hide the UI element #}
{{ form.database_connection }}
{% endif %}
{% if form.description.errors %}
{{ form.description.errors }}
{% endif %}
{% if form.sql.errors %} {% for error in form.sql.errors %}
{{ error|escape }}
{% endfor %} {% endif %}
{% if params %}
{% include 'explorer/params.html' %}
{% endif %}
{% if query %}
{% if query and can_change and assistant_enabled %}{{ form.few_shot }} {% translate "Assistant Example" %}{% endif %}
{% endif %}
{% if can_change %} {% export_buttons query %} {% else %} {% export_buttons query %} {% endif %}
{% if assistant_enabled %} {% include 'explorer/assistant.html' %} {% endif %}
{% include 'explorer/preview_pane.html' %}
{% if query.avg_duration %} {% blocktranslate trimmed with avg_duration_display=query.avg_duration_display cuser=query.created_by_user created=form.created_at_time %} Avg. execution: {{ avg_duration_display }}ms. Query created by {{ cuser }} on {{ created }}. {% endblocktranslate %} {% if query %} {% translate "History" %}{% endif %} {% endif %}
{% if query and can_change and tasks_enabled %}{{ form.snapshot }} {% translate "Snapshot" %}{% endif %}
{% endblock %} ================================================ FILE: explorer/templates/explorer/query_confirm_delete.html ================================================ {% extends "explorer/base.html" %} {% load i18n %} {% block sql_explorer_content %}
{% csrf_token %}
{% blocktranslate trimmed with title=object.title %} Are you sure you want to delete "{{ title }}"? {% endblocktranslate %}
{% endblock %} ================================================ FILE: explorer/templates/explorer/query_favorite_button.html ================================================ {% if is_favorite %} {% else %} {% endif %} ================================================ FILE: explorer/templates/explorer/query_favorites.html ================================================ {% extends "explorer/base.html" %} {% load i18n %} {% block sql_explorer_content %}

{% translate "Favorite Queries" %}

{% for favorite in favorites %} {% empty %} {% endfor %}
{% translate "Query" %}
{{ favorite.query.title }}
{% translate "No favorite queries added yet." %}
{% endblock %} ================================================ FILE: explorer/templates/explorer/query_list.html ================================================ {% extends "explorer/base.html" %} {% load explorer_tags i18n static %} {% block sql_explorer_content %}
{% csrf_token %}
{% if connection_count == 0 %}

Welcome to SQL Explorer!

First things first, in order to create queries and start exploring, you'll need to:

Create a Connection

Need help? Check out the documentation{% if hosted %} or contact support{% endif %}.

{% elif object_list|length == 0 %}

Time to create a query

You have {{ connection_count }} connection{% if connection_count > 1 %}s{% endif %} created, now get cracking!

Create a Query or Play Around

Need help? Check out the documentation{% if hosted %} or contact support{% endif %}.

{% else %} {% if recent_queries|length > 0 %}

{% translate "Recently Run by You" %}

{% for object in recent_queries %} {% endfor %}
{% translate "Query" %} {% translate "Last Run" %} CSV
{{ object.query.title }} {{ object.run_at|date:"SHORT_DATETIME_FORMAT" }}
{% endif %}

{% translate "All Queries" %}

{% if tasks_enabled %} {% endif %} {% if can_change %} {% endif %} {% for object in object_list %} {% if object.is_header %} {% else %} {% if tasks_enabled %} {% endif %} {% if can_change %} {% endif %} {% endif %} {% endfor %}
{% translate "Query" %} {% translate "Created" %}{% translate "Email" %}{% translate "CSV" %}{% translate "Play" %} {% translate "Delete" %}{% translate "Favorite" %} {% translate "Last Run" %} {% translate "Run Count" %} {% translate "Connection" %}
{{ object.title }} ({{ object.count }}) {{ object.title }} {{ object.created_at|date:"m/d/y" }} {% if object.created_by_user %} {% blocktranslate trimmed with cuser=object.created_by_user %} by {{cuser}} {% endblocktranslate %} {% endif %} {% query_favorite_button object.id object.is_favorite 'query_favorite_toggle' %} {% if object.ran_successfully %} {% elif object.ran_successfully is not None %} {% endif %} {{ object.last_run_at|date:"m/d/y" }} {{ object.run_count }} {{ object.connection_name }}
{% endif %} {% endblock %} ================================================ FILE: explorer/templates/explorer/querylog_list.html ================================================ {% extends "explorer/base.html" %} {% load i18n %} {% block sql_explorer_content %}

{% blocktranslate with pagenum=page_obj.number %}Recent Query Logs - Page {{pagenum}}{% endblocktranslate %}

{% for object in recent_logs %} {% endfor %}
{% translate "Run At" %} {% translate "Run By" %} {% translate "Database Connection" %} {% translate "Duration" %} SQL {% translate "Query ID" %} {% translate "Playground" %}
{{ object.run_at|date:"SHORT_DATETIME_FORMAT" }} {{ object.run_by_user }} {{ object.database_connection }} {{ object.duration|floatformat:2 }}ms {{ object.sql }} {% if object.query_id %} {% blocktranslate trimmed with query_id=object.query_id %} Query {{ query_id }} {% endblocktranslate %} {% elif object.is_playground %} {% translate "Playground" %} {% else %} -- {% endif %} {% translate "Open" %}
{% if is_paginated %} {% endif %}
{% endblock %} ================================================ FILE: explorer/templates/explorer/schema.html ================================================ {% extends "explorer/base.html" %} {% load i18n %} {% block sql_explorer_content_takeover %}

{% translate "Schema" %}

{% translate "Collapse Tables" %}
{% translate "Expand Tables" %}
    {% for m in schema %}
  • {{ m.0 }}
    {% for c in m.1 %} {% endfor %}
    {{ c.0 }} {{ c.1 }}
    • {% endfor %}
{% endblock %} {% block sql_explorer_footer %}{% endblock %} ================================================ FILE: explorer/templates/explorer/schema_error.html ================================================ {% extends "explorer/base.html" %} {% load i18n %} {% block sql_explorer_content_takeover %}

{% translate "Schema failed to build." %}

{% blocktranslate %} The connection '{{ connection }}' is likely misconfigured or unavailable. {% endblocktranslate %}
{% endblock %} ================================================ FILE: explorer/templatetags/__init__.py ================================================ ================================================ FILE: explorer/templatetags/explorer_tags.py ================================================ from django import template from django.utils.module_loading import import_string from explorer import app_settings register = template.Library() @register.inclusion_tag("explorer/export_buttons.html") def export_buttons(query=None): exporters = [] for name, classname in app_settings.EXPLORER_DATA_EXPORTERS: exporter_class = import_string(classname) exporters.append((name, exporter_class.name)) return { "exporters": exporters, "query": query, } @register.inclusion_tag("explorer/query_favorite_button.html") def query_favorite_button(query_id, is_favorite, extra_classes): return { "query_id": query_id, "is_favorite": is_favorite, "extra_classes": extra_classes } ================================================ FILE: explorer/templatetags/vite.py ================================================ import os from django import template from django.conf import settings from django.contrib.staticfiles.storage import staticfiles_storage from django.utils.safestring import mark_safe from explorer import app_settings, get_version register = template.Library() VITE_OUTPUT_DIR = staticfiles_storage.url("explorer/") VITE_DEV_DIR = "explorer/src/" VITE_SERVER_HOST = getattr(settings, "VITE_SERVER_HOST", "localhost") VITE_SERVER_PORT = getattr(settings, "VITE_SERVER_PORT", "5173") def get_css_link(file: str) -> str: if app_settings.VITE_DEV_MODE is False: file = file.replace(".scss", ".css") base_url = f"{VITE_OUTPUT_DIR}" else: base_url = f"http://{VITE_SERVER_HOST}:{VITE_SERVER_PORT}/{VITE_DEV_DIR}" return mark_safe(f'') def get_script(file: str) -> str: if app_settings.VITE_DEV_MODE is False: file = file.replace(".js", f".{get_version()}.js") return mark_safe(f'') else: base_url = f"http://{VITE_SERVER_HOST}:{VITE_SERVER_PORT}/{VITE_DEV_DIR}" return mark_safe(f'') def get_asset(file: str) -> str: if app_settings.VITE_DEV_MODE is False: return mark_safe(f"{VITE_OUTPUT_DIR}{file}") else: return mark_safe(f"http://{VITE_SERVER_HOST}:{VITE_SERVER_PORT}/{VITE_DEV_DIR}{file}") @register.simple_tag def vite_asset(filename: str): if str(filename).endswith("scss"): if app_settings.VITE_DEV_MODE is False: filename = os.path.basename(filename) return get_css_link(filename) if str(filename).endswith("js"): if app_settings.VITE_DEV_MODE is False: filename = os.path.basename(filename) return get_script(filename) # Non js/scss assets respect directory structure so don't need to do the filename rewrite return get_asset(filename) @register.simple_tag def vite_hmr_client(): if app_settings.VITE_DEV_MODE is False: return "" base_url = f"http://{VITE_SERVER_HOST}:{VITE_SERVER_PORT}/@vite/client" return mark_safe(f'') ================================================ FILE: explorer/tests/__init__.py ================================================ ================================================ FILE: explorer/tests/csvs/all_types.csv ================================================ Dates,Integers,Floats,Strings 2020-01-31,0,42.952198961732414,THNVT 2020-02-29,1,27.66862453654746,JXPSY 2020-03-31,2,79.028965687494,FSTNN 2020-04-30,3,97.00288969016145,BVMNF ,,, "","","" 2020-05-31,4,74.09328128351054,XUMUJ ================================================ FILE: explorer/tests/csvs/dates.csv ================================================ Dates,Values 2024-01-24,0 2024-01-24T18:45:00Z,1 01/24/2024,2 01/24/2024 6:45 PM,3 24/01/2024,4 24/01/2024 18:45,5 24/01/2024 18:45:00,6 "January 24, 2024",7 "Jan 24, 2024",8 2024-01-24T18:45:00-05:00,9 "Thu, 24 Jan 2024 18:45:00 +0000",10 "Thu, 24 Jan 2024 18:45:00 +0000 (GMT)",11 ================================================ FILE: explorer/tests/csvs/floats.csv ================================================ Floats,Values 61.3410231760637,0 79.2367071213973,1 96.93217099083482,2 69.50523191870069,3 ,4 "69.42719764194946",4 "2,000.0128",5 "2.000,0128",5 ================================================ FILE: explorer/tests/csvs/integers.csv ================================================ Integers,More_integers "5,000",0 "6000",1 "",0 "7,200,200",2 ================================================ FILE: explorer/tests/csvs/mixed.csv ================================================ Value1,Value2,Value3 2020-01-32,abc,123 Variety of other dates,def,123 ,, Another,123,12a ================================================ FILE: explorer/tests/csvs/rc_sample.csv ================================================ name,material_type,seating_type,speed,height,length,num_inversions,manufacturer,park,status Goudurix,Steel,Sit Down,75.0,37.0,950.0,7.0,Vekoma,Parc Asterix,status.operating Dream catcher,Steel,Suspended,45.0,25.0,600.0,0.0,Vekoma,Bobbejaanland,status.operating Alucinakis,Steel,Sit Down,30.0,8.0,250.0,0.0,Zamperla,Terra Mítica,status.operating Anaconda,Wooden,Sit Down,85.0,35.0,1200.0,0.0,William J. Cobb,Walygator Parc,status.operating Azteka,Steel,Sit Down,55.0,17.0,500.0,0.0,Soquet,Le Pal,status.operating Bat Coaster,Steel,Inverted,70.0,20.0,400.0,2.0,Pinfari,Nigloland,status.relocated Batman : Arkham Asylum,Steel,Inverted,80.0,32.0,823.0,5.0,B&M,Parque Warner Madrid,status.operating Big Thunder Mountain,Steel,Sit Down,60.0,22.0,1500.0,0.0,Vekoma,Disneyland Park,status.operating EqWalizer,Steel,Sit Down,76.0,36.0,285.0,3.0,Vekoma,Walibi Rhône Alpes,status.operating Calamity Mine,Steel,Sit Down,48.0,14.0,785.0,0.0,Vekoma,Walibi Belgium,status.operating "Casey Jr, le Petit Train du Cirque",Steel,Sit Down,30.0,,,0.0,Vekoma,Disneyland Park,status.operating Cobra,Steel,Sit Down,76.0,36.0,285.0,3.0,Vekoma,Walibi Belgium,status.operating Coccinelle,Steel,Sit Down,36.0,8.0,360.0,0.0,Zierer,Walibi Rhône Alpes,status.operating Coleoz'Arbres,Steel,Sit Down,60.0,,540.0,0.0,Schwarzkopf,Bagatelle,status.closed.definitely Comet,Steel,Sit Down,64.0,24.0,,3.0,Vekoma,Walygator Parc,status.operating Course de Bobsleigh,Steel,Sit Down,65.0,15.0,450.0,0.0,Schwarzkopf,Nigloland,status.relocated Cumbres,Steel,Sit Down,,3.0,,0.0,Miler Coaster,Parque de Atracciones de Madrid,status.closed.definitely Le Dragon de Bei Hai,Steel,Sit Down,,,,0.0,Cavazza Diego,La Mer de Sable,status.closed.definitely Euro Mir,Steel,Spinning,80.0,28.0,980.0,0.0,Mack,Europa Park,status.operating Eurosat,Steel,Sit Down,60.0,26.0,877.0,0.0,Mack,Europa Park,status.retracked Expedition Ge Force,Steel,Sit Down,120.0,53.0,1220.0,0.0,Intamin,Holiday Park,status.operating Le Grand canyon,Steel,Sit Down,50.0,12.0,380.0,0.0,Soquet,Fraispertuis City,status.operating Indiana Jones et le Temple du Péril,Steel,Sit Down,58.0,18.0,566.0,1.0,Intamin,Disneyland Park,status.operating Jaguar,Steel,Inverted,83.0,34.0,689.0,5.0,Vekoma,Isla Magica,status.operating Cop Car Chase (1),Steel,Sit Down,60.0,16.0,620.0,2.0,Intamin,Movie Park Germany,status.closed.definitely Loup Garou,Wooden,Sit Down,80.0,28.0,1035.0,0.0,Vekoma,Walibi Belgium,status.operating Magnus Colossus,Wooden,Sit Down,92.0,38.0,1150.0,0.0,RCCA,Terra Mítica,status.closed.temporarily Oki Doki,Steel,Sit Down,58.0,16.0,436.0,0.0,Vekoma,Bobbejaanland,status.operating SOS Numerobis,Steel,Sit Down,32.0,6.0,200.0,0.0,Zierer,Parc Asterix,status.operating Poseïdon,Steel,Water Coaster,70.0,23.0,836.0,0.0,Mack,Europa Park,status.operating Rock'n Roller Coaster avec Aerosmith,Steel,Sit Down,92.0,24.0,1037.0,3.0,Vekoma,Walt Disney Studios,status.operating La Ronde des Rondins,Steel,Sit Down,26.0,3.0,60.0,0.0,Zierer,Parc Asterix,status.relocated Silverstar,Steel,Sit Down,127.0,73.0,1620.0,0.0,B&M,Europa Park,status.operating Superman la Atraccion de Acero,Steel,Floorless,105.0,50.0,1200.0,7.0,B&M,Parque Warner Madrid,status.operating La Trace du Hourra,Steel,Bobsleigh,60.0,31.0,900.0,0.0,Mack,Parc Asterix,status.operating Stunt Fall,Steel,Sit Down,106.0,58.0,367.0,3.0,Vekoma,Parque Warner Madrid,status.operating Le Tigre de Sibérie,Steel,Sit Down,40.0,13.0,360.0,0.0,Reverchon,Le Pal,status.operating Titánide,Steel,Inverted,80.0,33.0,689.0,5.0,Vekoma,Terra Mítica,status.operating Tom y Jerry,Steel,Sit Down,36.0,8.0,360.0,0.0,Zierer,Parque Warner Madrid,status.operating Psyké underground,Steel,Sit Down,85.0,42.0,260.0,1.0,Schwarzkopf,Walibi Belgium,status.operating Tonnerre de Zeus,Wooden,Sit Down,84.0,30.0,1233.0,0.0,CCI,Parc Asterix,status.operating Tren Bravo (Left),Steel,Sit Down,45.0,6.0,394.0,0.0,Zamperla,Terra Mítica,status.closed.temporarily Typhoon,Steel,Sit Down,80.0,26.0,670.0,4.0,Gerstlauer,Bobbejaanland,status.operating Schweizer Bobbahn,Steel,Bobsleigh,50.0,19.0,487.0,0.0,Mack,Europa Park,status.operating Vampire,Steel,Inverted,80.0,33.0,689.0,5.0,Vekoma,Walibi Belgium,status.operating Le Vol d'Icare,Steel,Sit Down,42.0,11.0,410.0,0.0,Zierer,Parc Asterix,status.operating Wild Train,Steel,Sit Down,70.0,15.0,330.0,0.0,Pax,Parc Saint Paul,status.operating Bandit,Wooden,Sit Down,80.0,28.0,1099.0,0.0,RCCA,Movie Park Germany,status.operating Woodstock Express,Steel,Sit Down,,,220.0,0.0,Zamperla,Walibi Rhône Alpes,status.operating ================================================ FILE: explorer/tests/csvs/test_case1.csv ================================================ STORE,CONTENT_TYPE,EMAIL,CUSTOMER_ID,CREATED_DATE,SHIP_MONTH,SHIP_DATE,SHOPIFY_ORDER_NUMBER,SKU,SHOPIFY_PRODUCT_ID,SHOPIFY_VARIANT_ID,PRODUCT_TITLE,VARIANT_TITLE,PRICE,QUANTITY,SUB_TOTAL_PRICE,SHIPPING_FEE,TAX,DISCOUNT,SHOPIFY_ORDER_ID,GIFTCARD_AMOUNT,REFUNDS,REFUND_TAX,REFUND_SALE,REFUND_DATE,SHIPPING_ADDRESS_PROVINCE_CODE,ORDER_TAG,SHIPPING_ADDRESS_FIRST_NAME,SHIPPING_ADDRESS_LAST_NAME,SHIPPING_ADDRESS_ADDRESS_1,SHIPPING_ADDRESS_ADDRESS_2,SHIPPING_ADDRESS_CITY,SHIPPING_ADDRESS_PROVINCE,SHIPPING_ADDRESS_COUNTRY,SHIPPING_ADDRESS_ZIP,ORDER_TAGS, 556516,HBUS,Vitamin,fdf@yahoo.com,3887814443067,2023-03-31T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3948556,HBVITAkidsleep75,4907580457019,33723289010235,(Vitamin Bundle) Kid's Sleep,Default Title,0,1,0,0,0,0,4857684885563,0,0,0,0,,FL,,Clark,4075493288,Petunia Terrace,113,De Pere,Wisconsin,United States,32771,"Subscription Recurring Order, sent-to-d365, Subscription, FRAUD_APPROVED" 556517,HBUS,Vitamin,dsfs@yahoo.com,3887814443067,2023-03-31T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3948556,HBVITAmenim60,4331305107515,31085422018619,(Vitamin Bundle) Organic Men's Multi + Super Immune,Default Title,0,1,0,0,0,0,4857684885563,0,0,0,0,,FL,,Winter,4075493288,20 Union St,113,Warren,Michigan,United States,32771,"Subscription Recurring Order, sent-to-d365, Subscription, FRAUD_APPROVED" 556518,HBUS,Vitamin,fdsdf@comcast.net,2444580454459,2023-03-31T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3946315,HBVITAkidsleep75,4907580457019,33723289010235,(Vitamin Bundle) Kid's Sleep,Default Title,0,3,0,0,0,0,4857557188667,0,0,0,0,,PA,,Berns,2532085200,Eisenhower Dr,,Port Jefferson,New York,United States,15065,"Subscription, Subscription Recurring Order, FRAUD_APPROVED, sent-to-d365" 556519,HBUS,Vitamin,fdsdf@yahoo.com,3887814443067,2023-03-31T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3948556,HBVITAHSNCol60NV,6830033633339,40285923606587,"(Vitamin Bundle) Hair, Skin, Nails + Collagen",Default Title,0,1,0,0,0,0,4857684885563,0,0,0,0,,FL,,Gantt,4075493288,227 Crooked Oak Rd,113,Port Jefferson,New York,United States,32771,"Subscription Recurring Order, sent-to-d365, Subscription, FRAUD_APPROVED" 556520,HBUS,Vitamin,asd@comcast.net,2444580454459,2023-03-31T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3946315,HBVitaBundle7,4331298521147,31085392724027,Vitamin Bundle,7,65.99,1,65.99,4.99,0,13.2,4857557188667,0,0,0,0,,PA,,Johnson,2532085200,227 Eisenhower Dr,,Salt Lake City,Utah,United States,15065,"Subscription, Subscription Recurring Order, FRAUD_APPROVED, sent-to-d365" 556521,HBUS,Vitamin,fda@comcast.net,2444580454459,2023-03-31T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3946315,HBVITAkid60,4331303469115,31085415006267,(Vitamin Bundle) Organic Kid's Multi,60-count,0,2,0,0,0,0,4857557188667,0,0,0,0,,PA,,Moody,2532085200,Eisenhower Dr,,De Pere,Wisconsin,United States,15065,"Subscription, Subscription Recurring Order, FRAUD_APPROVED, sent-to-d365" 556522,HBUS,Vitamin,fdsa.family@gmail.com,5976811143227,2023-03-31T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3948458,HBVITAimm75,4331307499579,31085430014011,(Vitamin Bundle) Immunity,Default Title,0,1,0,0,0,0,4857645498427,0,0,0,0,,MI,,Clark,6164811698,Petunia Terrace,Nw,De Pere,Wisconsin,United States,49544,"FRAUD_APPROVED, Subscription, Subscription Recurring Order, sent-to-d365" 556523,HBUS,Vitamin,asdf.ahlf@gmail.com,5765744361531,2023-03-31T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3946630,HBVITAwom60,4331304681531,31085420281915,(Vitamin Bundle) Organic Women's Multi,Default Title,0,1,0,0,0,0,4857570000955,0,0,0,0,,IL,,Winter,3863347803,20 Union St,,Warren,Wisconsin,United States,60657,"FRAUD_APPROVED, Subscription, sent-to-d365, Subscription Recurring Order" 556524,HBUS,Vitamin,asdf@yahoo.com,3887814443067,2023-03-31T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3948556,HBVITAadultstr,4907584782395,33723310178363,(Vitamin Bundle) Adult Stress,Default Title,0,1,0,0,0,0,4857684885563,0,0,0,0,,FL,,Berns,4075493288,Eisenhower Dr,113,Port Jefferson,Michigan,United States,32771,"Subscription Recurring Order, sent-to-d365, Subscription, FRAUD_APPROVED" 556525,HBUS,Vitamin,dfgh5@yahoo.com,6074075873339,2023-03-30T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3945418,HBVitaBundle5,4331298521147,31085392691259,Vitamin Bundle,5,49.99,1,49.99,4.99,1.92,15,4856756699195,0,0,0,0,,WI,,Gantt,9204717027,227 Crooked Oak Rd,,Port Jefferson,New York,United States,54115,"FRAUD_APPROVED, Subscription First Order, Subscription, sent-to-d365" 556526,HBUS,Vitamin,gdfg5@gmail.com,5845340389435,2023-03-31T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3946834,HBVITAHSNC60,7110465355835,41164903678011,"(Vitamin Bundle) Hair, Skin + Nails",Default Title,0,1,0,0,0,0,4857578029115,0,0,0,0,,MI,,Johnson,3134422087,227 Eisenhower Dr,,Salt Lake City,New York,United States,48091,"Subscription Recurring Order, FRAUD_APPROVED, Subscription, sent-to-d365" 556527,HBUS,Vitamin,ghgh@gf.com,6086223102011,2023-03-31T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3946172,HBVITAPRENATDHA60,4331306811451,31085428015163,(Vitamin Bundle) Prenatal + DHA,Default Title,0,3,0,0,0,0,4857464619067,0,0,0,0,,NY,,Moody,4153162973,Eisenhower Dr,,De Pere,Utah,United States,11777,"sent-to-d365, Subscription First Order, Subscription, FRAUD_APPROVED" 556528,HBUS,Vitamin,sdfg5@fg.com,6086223102011,2023-03-31T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3946172,HBVitaBundle3,4331298521147,31085392658491,Vitamin Bundle,3,32.99,1,32.99,9.99,0,0,4857464619067,0,0,0,0,,NY,,Clark,4153162973,Petunia Terrace,,De Pere,Wisconsin,United States,11777,"sent-to-d365, Subscription First Order, Subscription, FRAUD_APPROVED" 556529,HBUS,Vitamin,sdfgs@gmail.com,5815550279739,2023-03-30T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3945751,HBVITAwom60,4331304681531,31085420281915,(Vitamin Bundle) Organic Women's Multi,Default Title,0,1,0,0,0,0,4857046335547,0,0,0,0,,UT,,Winter,8016081564,20 Union St,635,Warren,Wisconsin,United States,84108,"sent-to-d365, FRAUD_APPROVED, Subscription Recurring Order, Subscription" 556530,HBUS,Vitamin,sdfb@yahoo.com,6074075873339,2023-03-30T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3945418,HBVITAwom60,4331304681531,31085420281915,(Vitamin Bundle) Organic Women's Multi,Default Title,0,1,0,0,0,0,4856756699195,0,0,0,0,,WI,,Berns,9204717027,Eisenhower Dr,,Port Jefferson,Wisconsin,United States,54115,"FRAUD_APPROVED, Subscription First Order, Subscription, sent-to-d365" 556531,HBUS,Vitamin,brookesdfb4colwell@yahoo.com,6074075873339,2023-03-30T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3945418,HBVITAKIDMOODB60,7012437327931,40867363422267,(Vitamin Bundle) Kid's Mood Boost,Default Title,0,2,0,0,0,0,4856756699195,0,0,0,0,,WI,,Gantt,9204717027,227 Crooked Oak Rd,,Port Jefferson,Michigan,United States,54115,"FRAUD_APPROVED, Subscription First Order, Subscription, sent-to-d365" 556532,HBUS,Vitamin,34g@gmail.com,5747147309115,2023-03-31T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3947011,HBVITAwom60,4331304681531,31085420281915,(Vitamin Bundle) Organic Women's Multi,Default Title,0,2,0,0,0,0,4857585467451,0,0,0,0,,SC,,Johnson,6313126335,227 Eisenhower Dr,,Salt Lake City,New York,United States,29455,"Subscription Recurring Order, sent-to-d365, Subscription, FRAUD_APPROVED" 556533,HBUS,Vitamin,dbfd@yahoo.com,6074075873339,2023-03-30T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3945418,HBVITAkid60,4331303469115,31085415006267,(Vitamin Bundle) Organic Kid's Multi,60-count,0,1,0,0,0,0,4856756699195,0,0,0,0,,WI,,Moody,9204717027,Eisenhower Dr,,De Pere,New York,United States,54115,"FRAUD_APPROVED, Subscription First Order, Subscription, sent-to-d365" 556534,HBUS,Vitamin,sdbsdf@gmail.com,5815550279739,2023-03-30T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3945751,HBVitaBundle3,4331298521147,31085392658491,Vitamin Bundle,3,32.99,1,32.99,9.99,0.99,0,4857046335547,0,0,0,0,,UT,,Clark,8016081564,Petunia Terrace,635,De Pere,Utah,United States,84108,"sent-to-d365, FRAUD_APPROVED, Subscription Recurring Order, Subscription" 556535,HBUS,Vitamin,sdfg3@yahoo.com,6074075873339,2023-03-30T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3945418,HBVITAimm75,4331307499579,31085430014011,(Vitamin Bundle) Immunity,Default Title,0,1,0,0,0,0,4856756699195,0,0,0,0,,WI,,Winter,9204717027,20 Union St,,Warren,Wisconsin,United States,54115,"FRAUD_APPROVED, Subscription First Order, Subscription, sent-to-d365" 556536,HBUS,Vitamin,sfg@gmail.com,3058267258939,2023-03-31T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3946977,HBVITAsle75,4331307204667,31085429358651,(Vitamin Bundle) Sleep Well,Default Title,0,2,0,0,0,0,4857584156731,0,0,0,0,,PA,,Berns,2012596182,Eisenhower Dr,,Port Jefferson,Wisconsin,United States,18414,"Subscription, FRAUD_APPROVED, sent-to-d365, Subscription Recurring Order" 556537,HBUS,Vitamin,sdfg@gmail.com,2971195867195,2023-03-30T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3945648,HBVITAHSNCol60NV,6830033633339,40285923606587,"(Vitamin Bundle) Hair, Skin, Nails + Collagen",Default Title,0,2,0,0,0,0,4856948785211,0,0,0,0,,CA,,Gantt,4044097130,227 Crooked Oak Rd,,Port Jefferson,Wisconsin,United States,91505,"FRAUD_APPROVED, Subscription Recurring Order, sent-to-d365, Subscription" 556538,HBUS,Vitamin,vfsw@gmail.com,2971195867195,2023-03-30T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3945648,HBVITACHL60,6841717850171,40321076789307,(Vitamin Bundle) Organic Chlorophyll,Default Title,0,1,0,0,0,0,4856948785211,0,0,0,0,,CA,,Johnson,4044097130,227 Eisenhower Dr,,Salt Lake City,Michigan,United States,91505,"FRAUD_APPROVED, Subscription Recurring Order, sent-to-d365, Subscription" 556539,HBUS,Vitamin,sdfgg@gmail.com,2971195867195,2023-03-30T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3945648,HBVITAimm75,4331307499579,31085430014011,(Vitamin Bundle) Immunity,Default Title,0,2,0,0,0,0,4856948785211,0,0,0,0,,CA,,Moody,4044097130,Eisenhower Dr,,De Pere,New York,United States,91505,"FRAUD_APPROVED, Subscription Recurring Order, sent-to-d365, Subscription" 556540,HBUS,Vitamin,sdfgbv@gmail.com,5810735513659,2023-03-31T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3947165,HBVitaBundle3,4331298521147,31085392658491,Vitamin Bundle,3,32.99,1,32.99,9.99,0,0,4857591169083,0,0,0,0,,IL,,Clark,8159092206,Petunia Terrace,,De Pere,New York,United States,61108,"Subscription Recurring Order, Subscription, FRAUD_APPROVED, sent-to-d365" 556541,HBUS,Vitamin,sdvf3@gmail.com,3379719831611,2023-03-31T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3947396,HBVitaBundle3,4331298521147,31085392658491,Vitamin Bundle,3,32.99,1,32.99,9.99,0,0,4857600639035,0,0,0,0,,NH,,Winter,5174491774,20 Union St,,Warren,Utah,United States,3833,"FRAUD_APPROVED, Subscription Recurring Order, sent-to-d365, Subscription" 556542,HBUS,Vitamin,34f@gmail.com,3379719831611,2023-03-31T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3947396,HBVITAHSNCol60NV,6830033633339,40285923606587,"(Vitamin Bundle) Hair, Skin, Nails + Collagen",Default Title,0,1,0,0,0,0,4857600639035,0,0,0,0,,NH,,Berns,5174491774,Eisenhower Dr,,Port Jefferson,Wisconsin,United States,3833,"FRAUD_APPROVED, Subscription Recurring Order, sent-to-d365, Subscription" 556543,HBUS,Vitamin,dsfv@gmail.com,3379719831611,2023-03-31T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3947396,HBVITAmenim60,4331305107515,31085422018619,(Vitamin Bundle) Organic Men's Multi + Super Immune,Default Title,0,1,0,0,0,0,4857600639035,0,0,0,0,,NH,,Gantt,5174491774,227 Crooked Oak Rd,,Port Jefferson,Wisconsin,United States,3833,"FRAUD_APPROVED, Subscription Recurring Order, sent-to-d365, Subscription" 556544,HBUS,Vitamin,dsfv35@gmail.com,3379719831611,2023-03-31T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3947396,HBVITAPRENATDHA60,4331306811451,31085428015163,(Vitamin Bundle) Prenatal + DHA,Default Title,0,1,0,0,0,0,4857600639035,0,0,0,0,,NH,,Johnson,5174491774,227 Eisenhower Dr,,Salt Lake City,Wisconsin,United States,3833,"FRAUD_APPROVED, Subscription Recurring Order, sent-to-d365, Subscription" 556545,HBUS,Vitamin,sdfv@gmail.com,5747147309115,2023-03-31T00:00:00.000Z,2023-03-01T00:00:00.000Z,2023-03-31T00:00:00.000Z,3947011,HBVITAACVGW,6785724022843,40072796504123,(Vitamin Bundle) Apple Cider Vinegar,Default Title,0,1,0,0,0,0,4857585467451,0,0,0,0,,SC,,Moody,6313126335,Eisenhower Dr,,De Pere,Michigan,United States,29455,"Subscription Recurring Order, sent-to-d365, Subscription, FRAUD_APPROVED" ================================================ FILE: explorer/tests/factories.py ================================================ from django.conf import settings from factory import Sequence, SubFactory, LazyFunction from factory.django import DjangoModelFactory from explorer.models import Query, QueryLog from explorer.ee.db_connections.utils import default_db_connection_id class UserFactory(DjangoModelFactory): class Meta: model = settings.AUTH_USER_MODEL username = Sequence(lambda n: "User %03d" % n) is_staff = True class SimpleQueryFactory(DjangoModelFactory): class Meta: model = Query title = Sequence(lambda n: f"My simple query {n}") sql = "SELECT 1+1 AS TWO" # same result in postgres and sqlite description = "Doin' math" created_by_user = SubFactory(UserFactory) database_connection_id = LazyFunction(default_db_connection_id) class QueryLogFactory(DjangoModelFactory): class Meta: model = QueryLog sql = "SELECT 2+2 AS FOUR" database_connection_id = LazyFunction(default_db_connection_id) ================================================ FILE: explorer/tests/json/github.json ================================================ [ { "id": 6104546, "node_id": "MDEwOlJlcG9zaXRvcnk2MTA0NTQ2", "name": "-REPONAME", "full_name": "mralexgray/-REPONAME", "private": false, "owner": { "login": "mralexgray", "id": 262517, "node_id": "MDQ6VXNlcjI2MjUxNw==", "avatar_url": "https://avatars.githubusercontent.com/u/262517?v=4", "gravatar_id": "", "url": "https://api.github.com/users/mralexgray", "html_url": "https://github.com/mralexgray", "followers_url": "https://api.github.com/users/mralexgray/followers", "following_url": "https://api.github.com/users/mralexgray/following{/other_user}", "gists_url": "https://api.github.com/users/mralexgray/gists{/gist_id}", "starred_url": "https://api.github.com/users/mralexgray/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/mralexgray/subscriptions", "organizations_url": "https://api.github.com/users/mralexgray/orgs", "repos_url": "https://api.github.com/users/mralexgray/repos", "events_url": "https://api.github.com/users/mralexgray/events{/privacy}", "received_events_url": "https://api.github.com/users/mralexgray/received_events", "type": "User", "site_admin": false }, "html_url": "https://github.com/mralexgray/-REPONAME", "description": null, "fork": false, "url": "https://api.github.com/repos/mralexgray/-REPONAME", "forks_url": "https://api.github.com/repos/mralexgray/-REPONAME/forks", "keys_url": "https://api.github.com/repos/mralexgray/-REPONAME/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/mralexgray/-REPONAME/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/mralexgray/-REPONAME/teams", "hooks_url": "https://api.github.com/repos/mralexgray/-REPONAME/hooks", "issue_events_url": "https://api.github.com/repos/mralexgray/-REPONAME/issues/events{/number}", "events_url": "https://api.github.com/repos/mralexgray/-REPONAME/events", "assignees_url": "https://api.github.com/repos/mralexgray/-REPONAME/assignees{/user}", "branches_url": "https://api.github.com/repos/mralexgray/-REPONAME/branches{/branch}", "tags_url": "https://api.github.com/repos/mralexgray/-REPONAME/tags", "blobs_url": "https://api.github.com/repos/mralexgray/-REPONAME/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/mralexgray/-REPONAME/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/mralexgray/-REPONAME/git/refs{/sha}", "trees_url": "https://api.github.com/repos/mralexgray/-REPONAME/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/mralexgray/-REPONAME/statuses/{sha}", "languages_url": "https://api.github.com/repos/mralexgray/-REPONAME/languages", "stargazers_url": "https://api.github.com/repos/mralexgray/-REPONAME/stargazers", "contributors_url": "https://api.github.com/repos/mralexgray/-REPONAME/contributors", "subscribers_url": "https://api.github.com/repos/mralexgray/-REPONAME/subscribers", "subscription_url": "https://api.github.com/repos/mralexgray/-REPONAME/subscription", "commits_url": "https://api.github.com/repos/mralexgray/-REPONAME/commits{/sha}", "git_commits_url": "https://api.github.com/repos/mralexgray/-REPONAME/git/commits{/sha}", "comments_url": "https://api.github.com/repos/mralexgray/-REPONAME/comments{/number}", "issue_comment_url": "https://api.github.com/repos/mralexgray/-REPONAME/issues/comments{/number}", "contents_url": "https://api.github.com/repos/mralexgray/-REPONAME/contents/{+path}", "compare_url": "https://api.github.com/repos/mralexgray/-REPONAME/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/mralexgray/-REPONAME/merges", "archive_url": "https://api.github.com/repos/mralexgray/-REPONAME/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/mralexgray/-REPONAME/downloads", "issues_url": "https://api.github.com/repos/mralexgray/-REPONAME/issues{/number}", "pulls_url": "https://api.github.com/repos/mralexgray/-REPONAME/pulls{/number}", "milestones_url": "https://api.github.com/repos/mralexgray/-REPONAME/milestones{/number}", "notifications_url": "https://api.github.com/repos/mralexgray/-REPONAME/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/mralexgray/-REPONAME/labels{/name}", "releases_url": "https://api.github.com/repos/mralexgray/-REPONAME/releases{/id}", "deployments_url": "https://api.github.com/repos/mralexgray/-REPONAME/deployments", "created_at": "2012-10-06T16:37:39Z", "updated_at": "2013-01-12T13:39:30Z", "pushed_at": "2012-10-06T16:37:39Z", "git_url": "git://github.com/mralexgray/-REPONAME.git", "ssh_url": "git@github.com:mralexgray/-REPONAME.git", "clone_url": "https://github.com/mralexgray/-REPONAME.git", "svn_url": "https://github.com/mralexgray/-REPONAME", "homepage": null, "size": 48, "stargazers_count": 0, "watchers_count": 0, "language": null, "has_issues": true, "has_projects": true, "has_downloads": true, "has_wiki": true, "has_pages": false, "has_discussions": false, "forks_count": 0, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 0, "license": null, "allow_forking": true, "is_template": false, "web_commit_signoff_required": false, "topics": [ ], "visibility": "public", "forks": 0, "open_issues": 0, "watchers": 0, "default_branch": "master" }, { "id": 104510411, "node_id": "MDEwOlJlcG9zaXRvcnkxMDQ1MTA0MTE=", "name": "...", "full_name": "mralexgray/...", "private": false, "owner": { "login": "mralexgray", "id": 262517, "node_id": "MDQ6VXNlcjI2MjUxNw==", "avatar_url": "https://avatars.githubusercontent.com/u/262517?v=4", "gravatar_id": "", "url": "https://api.github.com/users/mralexgray", "html_url": "https://github.com/mralexgray", "followers_url": "https://api.github.com/users/mralexgray/followers", "following_url": "https://api.github.com/users/mralexgray/following{/other_user}", "gists_url": "https://api.github.com/users/mralexgray/gists{/gist_id}", "starred_url": "https://api.github.com/users/mralexgray/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/mralexgray/subscriptions", "organizations_url": "https://api.github.com/users/mralexgray/orgs", "repos_url": "https://api.github.com/users/mralexgray/repos", "events_url": "https://api.github.com/users/mralexgray/events{/privacy}", "received_events_url": "https://api.github.com/users/mralexgray/received_events", "type": "User", "site_admin": false }, "html_url": "https://github.com/mralexgray/...", "description": ":computer: Public repo for my personal dotfiles.", "fork": true, "url": "https://api.github.com/repos/mralexgray/...", "forks_url": "https://api.github.com/repos/mralexgray/.../forks", "keys_url": "https://api.github.com/repos/mralexgray/.../keys{/key_id}", "collaborators_url": "https://api.github.com/repos/mralexgray/.../collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/mralexgray/.../teams", "hooks_url": "https://api.github.com/repos/mralexgray/.../hooks", "issue_events_url": "https://api.github.com/repos/mralexgray/.../issues/events{/number}", "events_url": "https://api.github.com/repos/mralexgray/.../events", "assignees_url": "https://api.github.com/repos/mralexgray/.../assignees{/user}", "branches_url": "https://api.github.com/repos/mralexgray/.../branches{/branch}", "tags_url": "https://api.github.com/repos/mralexgray/.../tags", "blobs_url": "https://api.github.com/repos/mralexgray/.../git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/mralexgray/.../git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/mralexgray/.../git/refs{/sha}", "trees_url": "https://api.github.com/repos/mralexgray/.../git/trees{/sha}", "statuses_url": "https://api.github.com/repos/mralexgray/.../statuses/{sha}", "languages_url": "https://api.github.com/repos/mralexgray/.../languages", "stargazers_url": "https://api.github.com/repos/mralexgray/.../stargazers", "contributors_url": "https://api.github.com/repos/mralexgray/.../contributors", "subscribers_url": "https://api.github.com/repos/mralexgray/.../subscribers", "subscription_url": "https://api.github.com/repos/mralexgray/.../subscription", "commits_url": "https://api.github.com/repos/mralexgray/.../commits{/sha}", "git_commits_url": "https://api.github.com/repos/mralexgray/.../git/commits{/sha}", "comments_url": "https://api.github.com/repos/mralexgray/.../comments{/number}", "issue_comment_url": "https://api.github.com/repos/mralexgray/.../issues/comments{/number}", "contents_url": "https://api.github.com/repos/mralexgray/.../contents/{+path}", "compare_url": "https://api.github.com/repos/mralexgray/.../compare/{base}...{head}", "merges_url": "https://api.github.com/repos/mralexgray/.../merges", "archive_url": "https://api.github.com/repos/mralexgray/.../{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/mralexgray/.../downloads", "issues_url": "https://api.github.com/repos/mralexgray/.../issues{/number}", "pulls_url": "https://api.github.com/repos/mralexgray/.../pulls{/number}", "milestones_url": "https://api.github.com/repos/mralexgray/.../milestones{/number}", "notifications_url": "https://api.github.com/repos/mralexgray/.../notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/mralexgray/.../labels{/name}", "releases_url": "https://api.github.com/repos/mralexgray/.../releases{/id}", "deployments_url": "https://api.github.com/repos/mralexgray/.../deployments", "created_at": "2017-09-22T19:19:42Z", "updated_at": "2017-09-22T19:20:22Z", "pushed_at": "2017-09-15T08:27:32Z", "git_url": "git://github.com/mralexgray/....git", "ssh_url": "git@github.com:mralexgray/....git", "clone_url": "https://github.com/mralexgray/....git", "svn_url": "https://github.com/mralexgray/...", "homepage": "https://driesvints.com/blog/getting-started-with-dotfiles", "size": 113, "stargazers_count": 0, "watchers_count": 0, "language": "Shell", "has_issues": false, "has_projects": true, "has_downloads": true, "has_wiki": false, "has_pages": false, "has_discussions": false, "forks_count": 0, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 0, "license": { "key": "mit", "name": "MIT License", "spdx_id": "MIT", "url": "https://api.github.com/licenses/mit", "node_id": "MDc6TGljZW5zZTEz" }, "allow_forking": true, "is_template": false, "web_commit_signoff_required": false, "topics": [ ], "visibility": "public", "forks": 0, "open_issues": 0, "watchers": 0, "default_branch": "master" }, { "id": 58656723, "node_id": "MDEwOlJlcG9zaXRvcnk1ODY1NjcyMw==", "name": "2200087-Serial-Protocol", "full_name": "mralexgray/2200087-Serial-Protocol", "private": false, "owner": { "login": "mralexgray", "id": 262517, "node_id": "MDQ6VXNlcjI2MjUxNw==", "avatar_url": "https://avatars.githubusercontent.com/u/262517?v=4", "gravatar_id": "", "url": "https://api.github.com/users/mralexgray", "html_url": "https://github.com/mralexgray", "followers_url": "https://api.github.com/users/mralexgray/followers", "following_url": "https://api.github.com/users/mralexgray/following{/other_user}", "gists_url": "https://api.github.com/users/mralexgray/gists{/gist_id}", "starred_url": "https://api.github.com/users/mralexgray/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/mralexgray/subscriptions", "organizations_url": "https://api.github.com/users/mralexgray/orgs", "repos_url": "https://api.github.com/users/mralexgray/repos", "events_url": "https://api.github.com/users/mralexgray/events{/privacy}", "received_events_url": "https://api.github.com/users/mralexgray/received_events", "type": "User", "site_admin": false }, "html_url": "https://github.com/mralexgray/2200087-Serial-Protocol", "description": "A reverse engineered protocol description and accompanying code for Radioshack's 2200087 multimeter", "fork": true, "url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol", "forks_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/forks", "keys_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/teams", "hooks_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/hooks", "issue_events_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/issues/events{/number}", "events_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/events", "assignees_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/assignees{/user}", "branches_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/branches{/branch}", "tags_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/tags", "blobs_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/git/refs{/sha}", "trees_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/statuses/{sha}", "languages_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/languages", "stargazers_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/stargazers", "contributors_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/contributors", "subscribers_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/subscribers", "subscription_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/subscription", "commits_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/commits{/sha}", "git_commits_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/git/commits{/sha}", "comments_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/comments{/number}", "issue_comment_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/issues/comments{/number}", "contents_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/contents/{+path}", "compare_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/merges", "archive_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/downloads", "issues_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/issues{/number}", "pulls_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/pulls{/number}", "milestones_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/milestones{/number}", "notifications_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/labels{/name}", "releases_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/releases{/id}", "deployments_url": "https://api.github.com/repos/mralexgray/2200087-Serial-Protocol/deployments", "created_at": "2016-05-12T16:05:28Z", "updated_at": "2016-05-12T16:05:30Z", "pushed_at": "2016-05-12T16:07:24Z", "git_url": "git://github.com/mralexgray/2200087-Serial-Protocol.git", "ssh_url": "git@github.com:mralexgray/2200087-Serial-Protocol.git", "clone_url": "https://github.com/mralexgray/2200087-Serial-Protocol.git", "svn_url": "https://github.com/mralexgray/2200087-Serial-Protocol", "homepage": "http://daviddworken.com", "size": 41, "stargazers_count": 0, "watchers_count": 0, "language": "Python", "has_issues": false, "has_projects": true, "has_downloads": true, "has_wiki": true, "has_pages": false, "has_discussions": false, "forks_count": 1, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 0, "license": { "key": "gpl-2.0", "name": "GNU General Public License v2.0", "spdx_id": "GPL-2.0", "url": "https://api.github.com/licenses/gpl-2.0", "node_id": "MDc6TGljZW5zZTg=" }, "allow_forking": true, "is_template": false, "web_commit_signoff_required": false, "topics": [ ], "visibility": "public", "forks": 1, "open_issues": 0, "watchers": 0, "default_branch": "master" }, { "id": 13121042, "node_id": "MDEwOlJlcG9zaXRvcnkxMzEyMTA0Mg==", "name": "ace", "full_name": "mralexgray/ace", "private": false, "owner": { "login": "mralexgray", "id": 262517, "node_id": "MDQ6VXNlcjI2MjUxNw==", "avatar_url": "https://avatars.githubusercontent.com/u/262517?v=4", "gravatar_id": "", "url": "https://api.github.com/users/mralexgray", "html_url": "https://github.com/mralexgray", "followers_url": "https://api.github.com/users/mralexgray/followers", "following_url": "https://api.github.com/users/mralexgray/following{/other_user}", "gists_url": "https://api.github.com/users/mralexgray/gists{/gist_id}", "starred_url": "https://api.github.com/users/mralexgray/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/mralexgray/subscriptions", "organizations_url": "https://api.github.com/users/mralexgray/orgs", "repos_url": "https://api.github.com/users/mralexgray/repos", "events_url": "https://api.github.com/users/mralexgray/events{/privacy}", "received_events_url": "https://api.github.com/users/mralexgray/received_events", "type": "User", "site_admin": false }, "html_url": "https://github.com/mralexgray/ace", "description": "Ace (Ajax.org Cloud9 Editor)", "fork": true, "url": "https://api.github.com/repos/mralexgray/ace", "forks_url": "https://api.github.com/repos/mralexgray/ace/forks", "keys_url": "https://api.github.com/repos/mralexgray/ace/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/mralexgray/ace/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/mralexgray/ace/teams", "hooks_url": "https://api.github.com/repos/mralexgray/ace/hooks", "issue_events_url": "https://api.github.com/repos/mralexgray/ace/issues/events{/number}", "events_url": "https://api.github.com/repos/mralexgray/ace/events", "assignees_url": "https://api.github.com/repos/mralexgray/ace/assignees{/user}", "branches_url": "https://api.github.com/repos/mralexgray/ace/branches{/branch}", "tags_url": "https://api.github.com/repos/mralexgray/ace/tags", "blobs_url": "https://api.github.com/repos/mralexgray/ace/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/mralexgray/ace/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/mralexgray/ace/git/refs{/sha}", "trees_url": "https://api.github.com/repos/mralexgray/ace/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/mralexgray/ace/statuses/{sha}", "languages_url": "https://api.github.com/repos/mralexgray/ace/languages", "stargazers_url": "https://api.github.com/repos/mralexgray/ace/stargazers", "contributors_url": "https://api.github.com/repos/mralexgray/ace/contributors", "subscribers_url": "https://api.github.com/repos/mralexgray/ace/subscribers", "subscription_url": "https://api.github.com/repos/mralexgray/ace/subscription", "commits_url": "https://api.github.com/repos/mralexgray/ace/commits{/sha}", "git_commits_url": "https://api.github.com/repos/mralexgray/ace/git/commits{/sha}", "comments_url": "https://api.github.com/repos/mralexgray/ace/comments{/number}", "issue_comment_url": "https://api.github.com/repos/mralexgray/ace/issues/comments{/number}", "contents_url": "https://api.github.com/repos/mralexgray/ace/contents/{+path}", "compare_url": "https://api.github.com/repos/mralexgray/ace/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/mralexgray/ace/merges", "archive_url": "https://api.github.com/repos/mralexgray/ace/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/mralexgray/ace/downloads", "issues_url": "https://api.github.com/repos/mralexgray/ace/issues{/number}", "pulls_url": "https://api.github.com/repos/mralexgray/ace/pulls{/number}", "milestones_url": "https://api.github.com/repos/mralexgray/ace/milestones{/number}", "notifications_url": "https://api.github.com/repos/mralexgray/ace/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/mralexgray/ace/labels{/name}", "releases_url": "https://api.github.com/repos/mralexgray/ace/releases{/id}", "deployments_url": "https://api.github.com/repos/mralexgray/ace/deployments", "created_at": "2013-09-26T11:58:10Z", "updated_at": "2013-10-26T12:34:49Z", "pushed_at": "2013-10-26T12:34:48Z", "git_url": "git://github.com/mralexgray/ace.git", "ssh_url": "git@github.com:mralexgray/ace.git", "clone_url": "https://github.com/mralexgray/ace.git", "svn_url": "https://github.com/mralexgray/ace", "homepage": "http://ace.c9.io", "size": 21080, "stargazers_count": 0, "watchers_count": 0, "language": "JavaScript", "has_issues": false, "has_projects": true, "has_downloads": true, "has_wiki": true, "has_pages": false, "has_discussions": false, "forks_count": 1, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 0, "license": { "key": "other", "name": "Other", "spdx_id": "NOASSERTION", "url": null, "node_id": "MDc6TGljZW5zZTA=" }, "allow_forking": true, "is_template": false, "web_commit_signoff_required": false, "topics": [ ], "visibility": "public", "forks": 1, "open_issues": 0, "watchers": 0, "default_branch": "master" }, { "id": 10791045, "node_id": "MDEwOlJlcG9zaXRvcnkxMDc5MTA0NQ==", "name": "ACEView", "full_name": "mralexgray/ACEView", "private": false, "owner": { "login": "mralexgray", "id": 262517, "node_id": "MDQ6VXNlcjI2MjUxNw==", "avatar_url": "https://avatars.githubusercontent.com/u/262517?v=4", "gravatar_id": "", "url": "https://api.github.com/users/mralexgray", "html_url": "https://github.com/mralexgray", "followers_url": "https://api.github.com/users/mralexgray/followers", "following_url": "https://api.github.com/users/mralexgray/following{/other_user}", "gists_url": "https://api.github.com/users/mralexgray/gists{/gist_id}", "starred_url": "https://api.github.com/users/mralexgray/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/mralexgray/subscriptions", "organizations_url": "https://api.github.com/users/mralexgray/orgs", "repos_url": "https://api.github.com/users/mralexgray/repos", "events_url": "https://api.github.com/users/mralexgray/events{/privacy}", "received_events_url": "https://api.github.com/users/mralexgray/received_events", "type": "User", "site_admin": false }, "html_url": "https://github.com/mralexgray/ACEView", "description": "Use the wonderful ACE editor in your Cocoa applications", "fork": true, "url": "https://api.github.com/repos/mralexgray/ACEView", "forks_url": "https://api.github.com/repos/mralexgray/ACEView/forks", "keys_url": "https://api.github.com/repos/mralexgray/ACEView/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/mralexgray/ACEView/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/mralexgray/ACEView/teams", "hooks_url": "https://api.github.com/repos/mralexgray/ACEView/hooks", "issue_events_url": "https://api.github.com/repos/mralexgray/ACEView/issues/events{/number}", "events_url": "https://api.github.com/repos/mralexgray/ACEView/events", "assignees_url": "https://api.github.com/repos/mralexgray/ACEView/assignees{/user}", "branches_url": "https://api.github.com/repos/mralexgray/ACEView/branches{/branch}", "tags_url": "https://api.github.com/repos/mralexgray/ACEView/tags", "blobs_url": "https://api.github.com/repos/mralexgray/ACEView/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/mralexgray/ACEView/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/mralexgray/ACEView/git/refs{/sha}", "trees_url": "https://api.github.com/repos/mralexgray/ACEView/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/mralexgray/ACEView/statuses/{sha}", "languages_url": "https://api.github.com/repos/mralexgray/ACEView/languages", "stargazers_url": "https://api.github.com/repos/mralexgray/ACEView/stargazers", "contributors_url": "https://api.github.com/repos/mralexgray/ACEView/contributors", "subscribers_url": "https://api.github.com/repos/mralexgray/ACEView/subscribers", "subscription_url": "https://api.github.com/repos/mralexgray/ACEView/subscription", "commits_url": "https://api.github.com/repos/mralexgray/ACEView/commits{/sha}", "git_commits_url": "https://api.github.com/repos/mralexgray/ACEView/git/commits{/sha}", "comments_url": "https://api.github.com/repos/mralexgray/ACEView/comments{/number}", "issue_comment_url": "https://api.github.com/repos/mralexgray/ACEView/issues/comments{/number}", "contents_url": "https://api.github.com/repos/mralexgray/ACEView/contents/{+path}", "compare_url": "https://api.github.com/repos/mralexgray/ACEView/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/mralexgray/ACEView/merges", "archive_url": "https://api.github.com/repos/mralexgray/ACEView/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/mralexgray/ACEView/downloads", "issues_url": "https://api.github.com/repos/mralexgray/ACEView/issues{/number}", "pulls_url": "https://api.github.com/repos/mralexgray/ACEView/pulls{/number}", "milestones_url": "https://api.github.com/repos/mralexgray/ACEView/milestones{/number}", "notifications_url": "https://api.github.com/repos/mralexgray/ACEView/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/mralexgray/ACEView/labels{/name}", "releases_url": "https://api.github.com/repos/mralexgray/ACEView/releases{/id}", "deployments_url": "https://api.github.com/repos/mralexgray/ACEView/deployments", "created_at": "2013-06-19T12:15:04Z", "updated_at": "2015-11-24T01:14:10Z", "pushed_at": "2014-05-09T01:36:23Z", "git_url": "git://github.com/mralexgray/ACEView.git", "ssh_url": "git@github.com:mralexgray/ACEView.git", "clone_url": "https://github.com/mralexgray/ACEView.git", "svn_url": "https://github.com/mralexgray/ACEView", "homepage": null, "size": 1733, "stargazers_count": 0, "watchers_count": 0, "language": "Objective-C", "has_issues": false, "has_projects": true, "has_downloads": true, "has_wiki": true, "has_pages": false, "has_discussions": false, "forks_count": 1, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 0, "license": { "key": "other", "name": "Other", "spdx_id": "NOASSERTION", "url": null, "node_id": "MDc6TGljZW5zZTA=" }, "allow_forking": true, "is_template": false, "web_commit_signoff_required": false, "topics": [ ], "visibility": "public", "forks": 1, "open_issues": 0, "watchers": 0, "default_branch": "master" }, { "id": 13623648, "node_id": "MDEwOlJlcG9zaXRvcnkxMzYyMzY0OA==", "name": "ActiveLog", "full_name": "mralexgray/ActiveLog", "private": false, "owner": { "login": "mralexgray", "id": 262517, "node_id": "MDQ6VXNlcjI2MjUxNw==", "avatar_url": "https://avatars.githubusercontent.com/u/262517?v=4", "gravatar_id": "", "url": "https://api.github.com/users/mralexgray", "html_url": "https://github.com/mralexgray", "followers_url": "https://api.github.com/users/mralexgray/followers", "following_url": "https://api.github.com/users/mralexgray/following{/other_user}", "gists_url": "https://api.github.com/users/mralexgray/gists{/gist_id}", "starred_url": "https://api.github.com/users/mralexgray/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/mralexgray/subscriptions", "organizations_url": "https://api.github.com/users/mralexgray/orgs", "repos_url": "https://api.github.com/users/mralexgray/repos", "events_url": "https://api.github.com/users/mralexgray/events{/privacy}", "received_events_url": "https://api.github.com/users/mralexgray/received_events", "type": "User", "site_admin": false }, "html_url": "https://github.com/mralexgray/ActiveLog", "description": "Shut up all logs with active filter.", "fork": true, "url": "https://api.github.com/repos/mralexgray/ActiveLog", "forks_url": "https://api.github.com/repos/mralexgray/ActiveLog/forks", "keys_url": "https://api.github.com/repos/mralexgray/ActiveLog/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/mralexgray/ActiveLog/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/mralexgray/ActiveLog/teams", "hooks_url": "https://api.github.com/repos/mralexgray/ActiveLog/hooks", "issue_events_url": "https://api.github.com/repos/mralexgray/ActiveLog/issues/events{/number}", "events_url": "https://api.github.com/repos/mralexgray/ActiveLog/events", "assignees_url": "https://api.github.com/repos/mralexgray/ActiveLog/assignees{/user}", "branches_url": "https://api.github.com/repos/mralexgray/ActiveLog/branches{/branch}", "tags_url": "https://api.github.com/repos/mralexgray/ActiveLog/tags", "blobs_url": "https://api.github.com/repos/mralexgray/ActiveLog/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/mralexgray/ActiveLog/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/mralexgray/ActiveLog/git/refs{/sha}", "trees_url": "https://api.github.com/repos/mralexgray/ActiveLog/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/mralexgray/ActiveLog/statuses/{sha}", "languages_url": "https://api.github.com/repos/mralexgray/ActiveLog/languages", "stargazers_url": "https://api.github.com/repos/mralexgray/ActiveLog/stargazers", "contributors_url": "https://api.github.com/repos/mralexgray/ActiveLog/contributors", "subscribers_url": "https://api.github.com/repos/mralexgray/ActiveLog/subscribers", "subscription_url": "https://api.github.com/repos/mralexgray/ActiveLog/subscription", "commits_url": "https://api.github.com/repos/mralexgray/ActiveLog/commits{/sha}", "git_commits_url": "https://api.github.com/repos/mralexgray/ActiveLog/git/commits{/sha}", "comments_url": "https://api.github.com/repos/mralexgray/ActiveLog/comments{/number}", "issue_comment_url": "https://api.github.com/repos/mralexgray/ActiveLog/issues/comments{/number}", "contents_url": "https://api.github.com/repos/mralexgray/ActiveLog/contents/{+path}", "compare_url": "https://api.github.com/repos/mralexgray/ActiveLog/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/mralexgray/ActiveLog/merges", "archive_url": "https://api.github.com/repos/mralexgray/ActiveLog/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/mralexgray/ActiveLog/downloads", "issues_url": "https://api.github.com/repos/mralexgray/ActiveLog/issues{/number}", "pulls_url": "https://api.github.com/repos/mralexgray/ActiveLog/pulls{/number}", "milestones_url": "https://api.github.com/repos/mralexgray/ActiveLog/milestones{/number}", "notifications_url": "https://api.github.com/repos/mralexgray/ActiveLog/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/mralexgray/ActiveLog/labels{/name}", "releases_url": "https://api.github.com/repos/mralexgray/ActiveLog/releases{/id}", "deployments_url": "https://api.github.com/repos/mralexgray/ActiveLog/deployments", "created_at": "2013-10-16T15:52:37Z", "updated_at": "2013-10-16T15:52:37Z", "pushed_at": "2011-07-03T06:28:59Z", "git_url": "git://github.com/mralexgray/ActiveLog.git", "ssh_url": "git@github.com:mralexgray/ActiveLog.git", "clone_url": "https://github.com/mralexgray/ActiveLog.git", "svn_url": "https://github.com/mralexgray/ActiveLog", "homepage": "http://deepitpro.com/en/articles/ActiveLog/info/", "size": 60, "stargazers_count": 0, "watchers_count": 0, "language": "Objective-C", "has_issues": false, "has_projects": true, "has_downloads": true, "has_wiki": true, "has_pages": false, "has_discussions": false, "forks_count": 0, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 0, "license": null, "allow_forking": true, "is_template": false, "web_commit_signoff_required": false, "topics": [ ], "visibility": "public", "forks": 0, "open_issues": 0, "watchers": 0, "default_branch": "master" }, { "id": 9716210, "node_id": "MDEwOlJlcG9zaXRvcnk5NzE2MjEw", "name": "adium", "full_name": "mralexgray/adium", "private": false, "owner": { "login": "mralexgray", "id": 262517, "node_id": "MDQ6VXNlcjI2MjUxNw==", "avatar_url": "https://avatars.githubusercontent.com/u/262517?v=4", "gravatar_id": "", "url": "https://api.github.com/users/mralexgray", "html_url": "https://github.com/mralexgray", "followers_url": "https://api.github.com/users/mralexgray/followers", "following_url": "https://api.github.com/users/mralexgray/following{/other_user}", "gists_url": "https://api.github.com/users/mralexgray/gists{/gist_id}", "starred_url": "https://api.github.com/users/mralexgray/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/mralexgray/subscriptions", "organizations_url": "https://api.github.com/users/mralexgray/orgs", "repos_url": "https://api.github.com/users/mralexgray/repos", "events_url": "https://api.github.com/users/mralexgray/events{/privacy}", "received_events_url": "https://api.github.com/users/mralexgray/received_events", "type": "User", "site_admin": false }, "html_url": "https://github.com/mralexgray/adium", "description": "Official mirror of hg.adium.im", "fork": false, "url": "https://api.github.com/repos/mralexgray/adium", "forks_url": "https://api.github.com/repos/mralexgray/adium/forks", "keys_url": "https://api.github.com/repos/mralexgray/adium/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/mralexgray/adium/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/mralexgray/adium/teams", "hooks_url": "https://api.github.com/repos/mralexgray/adium/hooks", "issue_events_url": "https://api.github.com/repos/mralexgray/adium/issues/events{/number}", "events_url": "https://api.github.com/repos/mralexgray/adium/events", "assignees_url": "https://api.github.com/repos/mralexgray/adium/assignees{/user}", "branches_url": "https://api.github.com/repos/mralexgray/adium/branches{/branch}", "tags_url": "https://api.github.com/repos/mralexgray/adium/tags", "blobs_url": "https://api.github.com/repos/mralexgray/adium/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/mralexgray/adium/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/mralexgray/adium/git/refs{/sha}", "trees_url": "https://api.github.com/repos/mralexgray/adium/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/mralexgray/adium/statuses/{sha}", "languages_url": "https://api.github.com/repos/mralexgray/adium/languages", "stargazers_url": "https://api.github.com/repos/mralexgray/adium/stargazers", "contributors_url": "https://api.github.com/repos/mralexgray/adium/contributors", "subscribers_url": "https://api.github.com/repos/mralexgray/adium/subscribers", "subscription_url": "https://api.github.com/repos/mralexgray/adium/subscription", "commits_url": "https://api.github.com/repos/mralexgray/adium/commits{/sha}", "git_commits_url": "https://api.github.com/repos/mralexgray/adium/git/commits{/sha}", "comments_url": "https://api.github.com/repos/mralexgray/adium/comments{/number}", "issue_comment_url": "https://api.github.com/repos/mralexgray/adium/issues/comments{/number}", "contents_url": "https://api.github.com/repos/mralexgray/adium/contents/{+path}", "compare_url": "https://api.github.com/repos/mralexgray/adium/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/mralexgray/adium/merges", "archive_url": "https://api.github.com/repos/mralexgray/adium/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/mralexgray/adium/downloads", "issues_url": "https://api.github.com/repos/mralexgray/adium/issues{/number}", "pulls_url": "https://api.github.com/repos/mralexgray/adium/pulls{/number}", "milestones_url": "https://api.github.com/repos/mralexgray/adium/milestones{/number}", "notifications_url": "https://api.github.com/repos/mralexgray/adium/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/mralexgray/adium/labels{/name}", "releases_url": "https://api.github.com/repos/mralexgray/adium/releases{/id}", "deployments_url": "https://api.github.com/repos/mralexgray/adium/deployments", "created_at": "2013-04-27T14:59:33Z", "updated_at": "2019-12-11T06:51:45Z", "pushed_at": "2013-04-26T16:43:53Z", "git_url": "git://github.com/mralexgray/adium.git", "ssh_url": "git@github.com:mralexgray/adium.git", "clone_url": "https://github.com/mralexgray/adium.git", "svn_url": "https://github.com/mralexgray/adium", "homepage": null, "size": 277719, "stargazers_count": 0, "watchers_count": 0, "language": "Objective-C", "has_issues": false, "has_projects": true, "has_downloads": true, "has_wiki": false, "has_pages": false, "has_discussions": false, "forks_count": 36, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 0, "license": { "key": "other", "name": "Other", "spdx_id": "NOASSERTION", "url": null, "node_id": "MDc6TGljZW5zZTA=" }, "allow_forking": true, "is_template": false, "web_commit_signoff_required": false, "topics": [ ], "visibility": "public", "forks": 36, "open_issues": 0, "watchers": 0, "default_branch": "master" } ] ================================================ FILE: explorer/tests/json/kings.json ================================================ [ { "Name": "Edward the Elder", "Country": "United Kingdom", "House": "House of Wessex", "Reign": "899-925", "ID": 1 }, { "Name": "Athelstan", "Country": "United Kingdom", "House": "House of Wessex", "Reign": "925-940", "ID": 2 }, { "Name": "Edmund", "Country": "United Kingdom", "House": "House of Wessex", "Reign": "940-946", "ID": 3 }, { "Name": "Edred", "Country": "United Kingdom", "House": "House of Wessex", "Reign": "946-955", "ID": 4 }, { "Name": "Edwy", "Country": "United Kingdom", "House": "House of Wessex", "Reign": "955-959", "ID": 5 } ] ================================================ FILE: explorer/tests/json/list.json ================================================ {"Item":{"instanceId":{"S":"ba4afa8857d1f836701c4ce8e54b95d84a8e3b92ee52292c7e5b2fb6fe6f1a69"},"time":{"N":"1713969118.917831897735595703125"},"value":{"M":{"unique_connection_count":{"N":"1"},"unsafe_rendering":{"BOOL":false},"total_log_count":{"N":"24"},"total_query_count":{"N":"1"},"debug":{"BOOL":false},"unique_run_by_user_count":{"N":"3"},"default_database":{"S":"mysql"},"tasks_enabled":{"BOOL":false},"transform_count":{"N":"0"},"django_install_date":{"N":"1585856800.150691986083984375"},"version":{"S":"4.1"},"assistant_enabled":{"BOOL":false}}},"name":{"S":"STARTUP_STATS"}}} {"Item":{"instanceId":{"S":"ba4afa8857d1f836701c4ce8e54b95d84a8e3b92ee52292c7e5b2fb6fe6f1a69"},"time":{"N":"1713969149.071483612060546875"},"value":{"M":{"unique_connection_count":{"N":"1"},"unsafe_rendering":{"BOOL":false},"total_log_count":{"N":"24"},"total_query_count":{"N":"1"},"debug":{"BOOL":false},"unique_run_by_user_count":{"N":"3"},"default_database":{"S":"mysql"},"tasks_enabled":{"BOOL":false},"transform_count":{"N":"0"},"django_install_date":{"N":"1585856800.150691986083984375"},"version":{"S":"4.1"},"assistant_enabled":{"BOOL":false}}},"name":{"S":"STARTUP_STATS"}}} {"Item":{"instanceId":{"S":"ba4afa8857d1f836701c4ce8e54b95d84a8e3b92ee52292c7e5b2fb6fe6f1a69"},"time":{"N":"1713969303.64731121063232421875"},"value":{"M":{"unique_connection_count":{"N":"1"},"unsafe_rendering":{"BOOL":false},"total_log_count":{"N":"24"},"total_query_count":{"N":"1"},"debug":{"BOOL":false},"unique_run_by_user_count":{"N":"3"},"default_database":{"S":"mysql"},"tasks_enabled":{"BOOL":false},"transform_count":{"N":"0"},"django_install_date":{"N":"1585856800.150691986083984375"},"version":{"S":"4.1"},"assistant_enabled":{"BOOL":false}}},"name":{"S":"STARTUP_STATS"}}} {"Item":{"instanceId":{"S":"67503169-1466-419a-8995-37e28603998c"},"time":{"N":"1717804322.493606090545654296875"},"value":{"M":{"duration":{"N":"7.7049732208251953125"},"sql_len":{"N":"26"}}},"name":{"S":"QUERY_RUN"}}} {"Item":{"instanceId":{"S":"67503169-1466-419a-8995-37e28603998c"},"time":{"N":"1717804322.5681588649749755859375"},"value":{"M":{"unique_connection_count":{"N":"0"},"total_query_count":{"N":"20"},"debug":{"BOOL":true},"unique_run_by_user_count":{"N":"10"},"default_database":{"S":"postgresql"},"tasks_enabled":{"BOOL":false},"version":{"S":"4.3"},"unsafe_rendering":{"BOOL":false},"total_log_count":{"N":"600"},"explorer_install_quarter":{"S":"Q3-2021"},"transform_count":{"N":"0"},"charts_enabled":{"BOOL":false},"assistant_enabled":{"BOOL":false}}},"name":{"S":"STARTUP_STATS"}}} ================================================ FILE: explorer/tests/settings.py ================================================ from test_project.settings import * # noqa EXPLORER_ENABLE_ANONYMOUS_STATS = False EXPLORER_TASKS_ENABLED = True EXPLORER_AI_API_KEY = "foo" CELERY_BROKER_URL = "redis://localhost:6379/0" CELERY_TASK_ALWAYS_EAGER = True TEST_MODE = True DATABASES = { "default": { "ENGINE": "django.db.backends.sqlite3", "NAME": "tst1", "TEST": { "NAME": "tst1" } }, "alt": { "ENGINE": "django.db.backends.sqlite3", "NAME": "tst2", "TEST": { "NAME": "tst2" } }, "not_registered": { "ENGINE": "django.db.backends.sqlite3", "NAME": "tst3", "TEST": { "NAME": "tst3" } } } EXPLORER_CONNECTIONS = { "SQLite": "default", "Another": "alt", } class PrimaryDatabaseRouter: def allow_migrate(self, db, app_label, model_name=None, **hints): if db == "default": return None return False DATABASE_ROUTERS = ["explorer.tests.settings.PrimaryDatabaseRouter"] ================================================ FILE: explorer/tests/settings_base.py ================================================ from explorer.tests.settings import * # noqa EXPLORER_TASKS_ENABLED = False EXPLORER_USER_UPLOADS_ENABLED = False EXPLORER_CHARTS_ENABLED = False EXPLORER_AI_API_KEY = None ================================================ FILE: explorer/tests/test_actions.py ================================================ import io from zipfile import ZipFile from django.test import TestCase from explorer.actions import generate_report_action from explorer.tests.factories import SimpleQueryFactory class TestSqlQueryActions(TestCase): def test_single_query_is_csv_file(self): expected_csv = "two\r\n2\r\n" r = SimpleQueryFactory() fn = generate_report_action() result = fn(None, None, [r, ]) self.assertEqual(result.content.lower().decode("utf-8-sig"), expected_csv) def test_multiple_queries_are_zip_file(self): expected_csv = "two\r\n2\r\n" q = SimpleQueryFactory() q2 = SimpleQueryFactory() fn = generate_report_action() res = fn(None, None, [q, q2]) z = ZipFile(io.BytesIO(res.content)) got_csv = z.read(z.namelist()[0]) self.assertEqual(len(z.namelist()), 2) self.assertEqual(z.namelist()[0], f"{q.title}.csv") self.assertEqual(got_csv.lower().decode("utf-8-sig"), expected_csv) # if commas are not removed from the filename, then Chrome throws # "duplicate headers received from server" def test_packaging_removes_commas_from_file_name(self): expected = "attachment; filename=query for x y.csv" q = SimpleQueryFactory(title="query for x, y") fn = generate_report_action() res = fn(None, None, [q]) self.assertEqual(res["Content-Disposition"], expected) ================================================ FILE: explorer/tests/test_apps.py ================================================ from io import StringIO from django.test import TestCase from django.core.management import call_command class PendingMigrationsTests(TestCase): def test_no_pending_migrations(self): out = StringIO() try: call_command( "makemigrations", "--check", stdout=out, stderr=StringIO(), ) except SystemExit: # noqa self.fail("Pending migrations:\n" + out.getvalue()) ================================================ FILE: explorer/tests/test_assistant.py ================================================ from explorer.tests.factories import SimpleQueryFactory, QueryLogFactory from unittest.mock import patch, Mock, MagicMock import unittest from explorer import app_settings import json from django.test import TestCase from django.utils import timezone from django.urls import reverse from django.contrib.auth.models import User from django.db import OperationalError from explorer.ee.db_connections.utils import default_db_connection from explorer.ee.db_connections.models import DatabaseConnection from explorer.assistant.utils import ( sample_rows_from_table, ROW_SAMPLE_SIZE, build_prompt, get_relevant_few_shots, get_relevant_annotation, table_schema ) from explorer.assistant.models import TableDescription from explorer.models import PromptLog def conn(): return default_db_connection().as_django_connection() @unittest.skipIf(not app_settings.has_assistant(), "assistant not enabled") class TestAssistantViews(TestCase): def setUp(self): self.user = User.objects.create_superuser( "admin", "admin@admin.com", "pwd" ) self.client.login(username="admin", password="pwd") self.request_data = { "sql": "SELECT * FROM explorer_query", "connection_id": 1, "assistant_request": "Test Request" } @patch("explorer.assistant.utils.openai_client") def test_do_modify_query(self, mocked_openai_client): from explorer.assistant.views import run_assistant # create.return_value should match: resp.choices[0].message mocked_openai_client.return_value.chat.completions.create.return_value = Mock( choices=[Mock(message=Mock(content="smart computer"))]) resp = run_assistant(self.request_data, None) self.assertEqual(resp, "smart computer") @patch("explorer.assistant.utils.openai_client") def test_assistant_help(self, mocked_openai_client): mocked_openai_client.return_value.chat.completions.create.return_value = Mock( choices=[Mock(message=Mock(content="smart computer"))]) resp = self.client.post(reverse("assistant"), data=json.dumps(self.request_data), content_type="application/json") self.assertEqual(json.loads(resp.content)["message"], "smart computer") @unittest.skipIf(not app_settings.has_assistant(), "assistant not enabled") class TestBuildPrompt(TestCase): @patch("explorer.models.ExplorerValue.objects.get_item") def test_build_prompt_with_vendor_only(self, mock_get_item): mock_get_item.return_value.value = "system prompt" result = build_prompt(default_db_connection(), "Help me with SQL", [], sql="SELECT * FROM table;") self.assertIn("sqlite", result["system"]) @patch("explorer.assistant.utils.sample_rows_from_table", return_value="sample data") @patch("explorer.assistant.utils.table_schema", return_value=[]) @patch("explorer.models.ExplorerValue.objects.get_item") def test_build_prompt_with_sql_and_annotation(self, mock_get_item, mock_table_schema, mock_sample_rows): mock_get_item.return_value.value = "system prompt" included_tables = ["foo"] td = TableDescription(database_connection=default_db_connection(), table_name="foo", description="annotated") td.save() result = build_prompt(default_db_connection(), "Help me with SQL", included_tables, sql="SELECT * FROM table;") self.assertIn("Usage Notes:\nannotated", result["user"]) @patch("explorer.assistant.utils.sample_rows_from_table", return_value="sample data") @patch("explorer.assistant.utils.table_schema", return_value=[]) @patch("explorer.models.ExplorerValue.objects.get_item") def test_build_prompt_with_few_shot(self, mock_get_item, mock_table_schema, mock_sample_rows): mock_get_item.return_value.value = "system prompt" included_tables = ["magic"] SimpleQueryFactory(title="Few shot", description="the quick brown fox", sql="select 'magic value';", few_shot=True) result = build_prompt(default_db_connection(), "Help me with SQL", included_tables, sql="SELECT * FROM table;") self.assertIn("Relevant example queries", result["user"]) self.assertIn("magic value", result["user"]) @patch("explorer.assistant.utils.sample_rows_from_table", return_value="sample data") @patch("explorer.models.ExplorerValue.objects.get_item") def test_build_prompt_with_sql_and_error(self, mock_get_item, mock_sample_rows): mock_get_item.return_value.value = "system prompt" included_tables = [] result = build_prompt(default_db_connection(), "Help me with SQL", included_tables, "Syntax error", "SELECT * FROM table;") self.assertIn("## Existing User-Written SQL ##\nSELECT * FROM table;", result["user"]) self.assertIn("## Query Error ##\nSyntax error\n", result["user"]) self.assertIn("## User's Request to Assistant ##\nHelp me with SQL", result["user"]) self.assertIn("system prompt", result["system"]) @patch("explorer.models.ExplorerValue.objects.get_item") def test_build_prompt_with_extra_tables_fitting_window(self, mock_get_item): mock_get_item.return_value.value = "system prompt" included_tables = ["explorer_query"] SimpleQueryFactory() result = build_prompt(default_db_connection(), "Help me with SQL", included_tables, sql="SELECT * FROM table;") self.assertIn("## Information for Table 'explorer_query' ##", result["user"]) self.assertIn("Sample rows:\nid | title", result["user"]) @unittest.skipIf(not app_settings.has_assistant(), "assistant not enabled") class TestPromptContext(TestCase): def test_retrieves_sample_rows(self): SimpleQueryFactory(title="First Query") SimpleQueryFactory(title="Second Query") SimpleQueryFactory(title="Third Query") SimpleQueryFactory(title="Fourth Query") ret = sample_rows_from_table(conn(), "explorer_query") self.assertEqual(len(ret), ROW_SAMPLE_SIZE+1) # includes header row def test_truncates_long_strings(self): c = MagicMock mock_cursor = MagicMock() long_string = "a" * 600 mock_cursor.description = [("col1",), ("col2",)] mock_cursor.fetchall.return_value = [(long_string, "short string")] c.cursor = MagicMock() c.cursor.return_value = mock_cursor ret = sample_rows_from_table(c, "some_table") header, row = ret self.assertEqual(header, ["col1", "col2"]) self.assertEqual(row[0], "a" * 200 + "...") self.assertEqual(row[1], "short string") def test_binary_data(self): long_binary = b"a" * 600 # Mock database connection and cursor c = MagicMock mock_cursor = MagicMock() mock_cursor.description = [("col1",), ("col2",)] mock_cursor.fetchall.return_value = [(long_binary, b"short binary")] c.cursor = MagicMock() c.cursor.return_value = mock_cursor ret = sample_rows_from_table(c, "some_table") header, row = ret self.assertEqual(header, ["col1", "col2"]) self.assertEqual(row[0], "") self.assertEqual(row[1], "") def test_handles_various_data_types(self): # Mock database connection and cursor c = MagicMock mock_cursor = MagicMock() mock_cursor.description = [("col1",), ("col2",), ("col3",)] mock_cursor.fetchall.return_value = [(123, 45.67, "normal string")] c.cursor = MagicMock() c.cursor.return_value = mock_cursor ret = sample_rows_from_table(c, "some_table") header, row = ret self.assertEqual(header, ["col1", "col2", "col3"]) self.assertEqual(row[0], 123) self.assertEqual(row[1], 45.67) self.assertEqual(row[2], "normal string") def test_handles_operational_error(self): c = MagicMock mock_cursor = MagicMock() mock_cursor.execute.side_effect = OperationalError("Test OperationalError") c.cursor = MagicMock() c.cursor.return_value = mock_cursor ret = sample_rows_from_table(c, "some_table") self.assertEqual(ret, [["Test OperationalError"]]) def test_format_rows_from_table(self): from explorer.assistant.utils import format_rows_from_table d = [ ["col1", "col2"], ["val1", "val2"], ] ret = format_rows_from_table(d) self.assertEqual(ret, "col1 | col2\nval1 | val2") def test_schema_info_from_table_names(self): ret = table_schema(default_db_connection(), "explorer_query") expected = [ ("id", "AutoField"), ("title", "CharField"), ("sql", "TextField"), ("description", "TextField"), ("created_at", "DateTimeField"), ("last_run_date", "DateTimeField"), ("created_by_user_id", "IntegerField"), ("snapshot", "BooleanField"), ("connection", "CharField"), ("database_connection_id", "IntegerField"), ("few_shot", "BooleanField")] self.assertEqual(ret, expected) def test_schema_info_from_table_names_case_invariant(self): ret = table_schema(default_db_connection(), "EXPLORER_QUERY") expected = [ ("id", "AutoField"), ("title", "CharField"), ("sql", "TextField"), ("description", "TextField"), ("created_at", "DateTimeField"), ("last_run_date", "DateTimeField"), ("created_by_user_id", "IntegerField"), ("snapshot", "BooleanField"), ("connection", "CharField"), ("database_connection_id", "IntegerField"), ("few_shot", "BooleanField")] self.assertEqual(ret, expected) @unittest.skipIf(not app_settings.has_assistant(), "assistant not enabled") class TestAssistantUtils(TestCase): def test_sample_rows_from_table(self): from explorer.assistant.utils import sample_rows_from_table, format_rows_from_table SimpleQueryFactory(title="First Query") SimpleQueryFactory(title="Second Query") QueryLogFactory() ret = sample_rows_from_table(conn(), "explorer_query") self.assertEqual(len(ret), ROW_SAMPLE_SIZE) ret = format_rows_from_table(ret) self.assertTrue("First Query" in ret) self.assertTrue("Second Query" in ret) def test_sample_rows_from_tables_no_table_match(self): from explorer.assistant.utils import sample_rows_from_table SimpleQueryFactory(title="First Query") SimpleQueryFactory(title="Second Query") ret = sample_rows_from_table(conn(), "banana") self.assertEqual(ret, [["no such table: banana"]]) def test_relevant_few_shots(self): relevant_q1 = SimpleQueryFactory(sql="select * from relevant_table", few_shot=True) relevant_q2 = SimpleQueryFactory(sql="select * from conn.RELEVANT_TABLE limit 10", few_shot=True) irrelevant_q2 = SimpleQueryFactory(sql="select * from conn.RELEVANT_TABLE limit 10", few_shot=False) relevant_q3 = SimpleQueryFactory(sql="select * from conn.another_good_table limit 10", few_shot=True) irrelevant_q1 = SimpleQueryFactory(sql="select * from irrelevant_table") included_tables = ["relevant_table", "ANOTHER_GOOD_TABLE"] res = get_relevant_few_shots(relevant_q1.database_connection, included_tables) res_ids = [td.id for td in res] self.assertIn(relevant_q1.id, res_ids) self.assertIn(relevant_q2.id, res_ids) self.assertIn(relevant_q3.id, res_ids) self.assertNotIn(irrelevant_q1.id, res_ids) self.assertNotIn(irrelevant_q2.id, res_ids) def test_get_relevant_annotations(self): relevant1 = TableDescription( database_connection=default_db_connection(), table_name="fruit" ) relevant2 = TableDescription( database_connection=default_db_connection(), table_name="Vegetables" ) irrelevant = TableDescription( database_connection=default_db_connection(), table_name="animals" ) relevant1.save() relevant2.save() irrelevant.save() res1 = get_relevant_annotation(default_db_connection(), "Fruit") self.assertEqual(relevant1.id, res1.id) res2 = get_relevant_annotation(default_db_connection(), "vegetables") self.assertEqual(relevant2.id, res2.id) class TestAssistantHistoryApiView(TestCase): def setUp(self): self.user = User.objects.create_superuser( "admin", "admin@admin.com", "pwd" ) self.client.login(username="admin", password="pwd") def test_assistant_history_api_view(self): # Create some PromptLogs connection = default_db_connection() PromptLog.objects.create( run_by_user=self.user, database_connection=connection, user_request="Test request 1", response="Test response 1", run_at=timezone.now() ) PromptLog.objects.create( run_by_user=self.user, database_connection=connection, user_request="Test request 2", response="Test response 2", run_at=timezone.now() ) # Make a POST request to the API url = reverse("assistant_history") data = { "connection_id": connection.id } response = self.client.post(url, data=json.dumps(data), content_type="application/json") # Check the response self.assertEqual(response.status_code, 200) response_data = json.loads(response.content) self.assertIn("logs", response_data) self.assertEqual(len(response_data["logs"]), 2) self.assertEqual(response_data["logs"][0]["user_request"], "Test request 2") self.assertEqual(response_data["logs"][0]["response"], "Test response 2") self.assertEqual(response_data["logs"][1]["user_request"], "Test request 1") self.assertEqual(response_data["logs"][1]["response"], "Test response 1") def test_assistant_history_api_view_invalid_json(self): url = reverse("assistant_history") response = self.client.post(url, data="invalid json", content_type="application/json") self.assertEqual(response.status_code, 400) response_data = json.loads(response.content) self.assertEqual(response_data["status"], "error") self.assertEqual(response_data["message"], "Invalid JSON") def test_assistant_history_api_view_no_logs(self): connection = default_db_connection() url = reverse("assistant_history") data = { "connection_id": connection.id } response = self.client.post(url, data=json.dumps(data), content_type="application/json") self.assertEqual(response.status_code, 200) response_data = json.loads(response.content) self.assertIn("logs", response_data) self.assertEqual(len(response_data["logs"]), 0) def test_assistant_history_api_view_filtered_results(self): # Create two users user1 = self.user user2 = User.objects.create_superuser( "admin2", "admin2@admin.com", "pwd" ) # Create two database connections connection1 = default_db_connection() connection2 = DatabaseConnection.objects.create( alias="test_connection", engine="django.db.backends.sqlite3", name=":memory:" ) # Create prompt logs for both users and connections PromptLog.objects.create( run_by_user=user1, database_connection=connection1, user_request="User1 Connection1 request", response="User1 Connection1 response", run_at=timezone.now() ) PromptLog.objects.create( run_by_user=user1, database_connection=connection2, user_request="User1 Connection2 request", response="User1 Connection2 response", run_at=timezone.now() ) PromptLog.objects.create( run_by_user=user2, database_connection=connection1, user_request="User2 Connection1 request", response="User2 Connection1 response", run_at=timezone.now() ) # Make a POST request to the API as user1 url = reverse("assistant_history") data = { "connection_id": connection1.id } response = self.client.post(url, data=json.dumps(data), content_type="application/json") # Check the response self.assertEqual(response.status_code, 200) response_data = json.loads(response.content) self.assertIn("logs", response_data) self.assertEqual(len(response_data["logs"]), 1) self.assertEqual(response_data["logs"][0]["user_request"], "User1 Connection1 request") self.assertEqual(response_data["logs"][0]["response"], "User1 Connection1 response") # Now test with user2 self.client.logout() self.client.login(username="admin2", password="pwd") response = self.client.post(url, data=json.dumps(data), content_type="application/json") # Check the response self.assertEqual(response.status_code, 200) response_data = json.loads(response.content) self.assertIn("logs", response_data) self.assertEqual(len(response_data["logs"]), 1) self.assertEqual(response_data["logs"][0]["user_request"], "User2 Connection1 request") self.assertEqual(response_data["logs"][0]["response"], "User2 Connection1 response") ================================================ FILE: explorer/tests/test_create_sqlite.py ================================================ from django.test import TestCase from django.core.files.uploadedfile import SimpleUploadedFile from unittest import skipIf, mock from explorer.app_settings import EXPLORER_USER_UPLOADS_ENABLED from explorer.ee.db_connections.create_sqlite import parse_to_sqlite, get_names import os import sqlite3 SQLITE_BYTES = b'SQLite format 3\x00\x10\x00\x01\x01\x00@ \x00\x00\x00\x02\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00.n\xba\r\x00\x00\x00\x01\x0f\xb6\x00\x0f\xb6\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00H\x01\x06\x17\x15\x15\x01utabledatadata\x02CREATE TABLE "data" (\n"name" TEXT,\n " title" TEXT\n)\r\x00\x00\x00\x01\x0f\xf3\x00\x0f\xf3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x01\x03\x17\x13chriscto' # noqa PATH = "./test_parse_to_sqlite.db" def write_sqlite_and_get_row(f_bytes, table_name): os.makedirs(os.path.dirname(PATH), exist_ok=True) with open(PATH, "wb") as temp_file: temp_file.write(f_bytes.getvalue()) conn = sqlite3.connect(PATH) cursor = conn.cursor() cursor.execute(f"SELECT * FROM {table_name}") rows = cursor.fetchall() cursor.close() conn.close() os.remove(PATH) return rows @skipIf(not EXPLORER_USER_UPLOADS_ENABLED, reason="User uploads disabled") class TestCreateSqlite(TestCase): #def test_parse_to_sqlite_with_sqlite_file def test_parse_to_sqlite(self): file = SimpleUploadedFile("name.csv", b"name, title\nchris,cto", content_type="text/csv") sqlite_bytes, name = parse_to_sqlite(file, None, user_id=1) rows = write_sqlite_and_get_row(sqlite_bytes, "name") self.assertEqual(rows[0], ("chris", "cto")) self.assertEqual(name, "name_1.db") def test_parse_to_sqlite_with_no_parser(self): file = SimpleUploadedFile("name.db", SQLITE_BYTES, content_type="application/x-sqlite3") sqlite_bytes, name = parse_to_sqlite(file, None, user_id=1) rows = write_sqlite_and_get_row(sqlite_bytes, "data") self.assertEqual(rows[0], ("chris", "cto")) self.assertEqual(name, "name_1.db") class TestGetNames(TestCase): def setUp(self): # Mock file object self.mock_file = mock.MagicMock() self.mock_file.name = "test file name.txt" # Mock append_conn object self.mock_append_conn = mock.MagicMock() self.mock_append_conn.name = "/path/to/existing_db.sqlite" def test_no_append_conn(self): table_name, f_name = get_names(self.mock_file, append_conn=None, user_id=123) self.assertEqual(table_name, "test_file_name") self.assertEqual(f_name, "test_file_name_123.db") def test_with_append_conn(self): table_name, f_name = get_names(self.mock_file, append_conn=self.mock_append_conn, user_id=123) self.assertEqual(table_name, "test_file_name") self.assertEqual(f_name, "existing_db.sqlite") def test_secure_filename(self): self.mock_file.name = "测试文件.txt" table_name, f_name = get_names(self.mock_file, append_conn=None, user_id=123) self.assertEqual(table_name, "_") self.assertEqual(f_name, "__123.db") def test_empty_filename(self): self.mock_file.name = ".txt" with self.assertRaises(ValueError): get_names(self.mock_file, append_conn=None, user_id=123) def test_invalid_extension(self): self.mock_file.name = "filename.exe" with self.assertRaises(ValueError): get_names(self.mock_file, append_conn=None, user_id=123) ================================================ FILE: explorer/tests/test_csrf_cookie_name.py ================================================ from django.test import TestCase, override_settings try: from django.urls import reverse except ImportError: from django.core.urlresolvers import reverse from django.conf import settings from django.contrib.auth.models import User class TestCsrfCookieName(TestCase): def test_csrf_cookie_name_in_context(self): self.user = User.objects.create_superuser("admin", "admin@admin-fake.com", "pwd") self.client.login(username="admin", password="pwd") resp = self.client.get(reverse("explorer_index")) self.assertTrue("csrf_cookie_name" in resp.context) self.assertEqual(resp.context["csrf_cookie_name"], settings.CSRF_COOKIE_NAME) @override_settings(CSRF_COOKIE_NAME="TEST_CSRF_COOKIE_NAME") def test_custom_csrf_cookie_name(self): self.user = User.objects.create_superuser("admin", "admin@admin-fake.com", "pwd") self.client.login(username="admin", password="pwd") resp = self.client.get(reverse("explorer_index")) self.assertTrue("csrf_cookie_name" in resp.context) self.assertEqual(resp.context["csrf_cookie_name"], "TEST_CSRF_COOKIE_NAME") ================================================ FILE: explorer/tests/test_db_connection_utils.py ================================================ from django.test import TestCase from unittest import skipIf from explorer.app_settings import EXPLORER_USER_UPLOADS_ENABLED if EXPLORER_USER_UPLOADS_ENABLED: import pandas as pd import os import sqlite3 from django.db import DatabaseError from explorer.models import DatabaseConnection from unittest.mock import patch, MagicMock from explorer.ee.db_connections.utils import ( pandas_to_sqlite ) @skipIf(not EXPLORER_USER_UPLOADS_ENABLED, "User uploads not enabled") class TestSQLiteConnection(TestCase): @patch("explorer.utils.get_s3_bucket") def test_get_sqlite_for_connection_downloads_file_if_not_exists(self, mock_get_s3_bucket): mock_s3 = MagicMock() mock_get_s3_bucket.return_value = mock_s3 conn = DatabaseConnection( name="test_db.db", host="s3_bucket/test_db.db", engine=DatabaseConnection.SQLITE ) conn.delete_local_sqlite() local_name = conn.local_name conn.as_django_connection() mock_s3.download_file.assert_called_once_with("s3_bucket/test_db.db", local_name) @patch("explorer.utils.get_s3_bucket") def test_get_sqlite_for_connection_skips_download_if_exists(self, mock_get_s3_bucket): mock_s3 = MagicMock() mock_get_s3_bucket.return_value = mock_s3 conn = DatabaseConnection( name="test_db.db", host="s3_bucket/test_db.db", engine=DatabaseConnection.SQLITE ) conn.delete_local_sqlite() local_name = conn.local_name with open(local_name, "wb") as file: file.write(b"\x00" * 10) conn.update_fingerprint() conn.as_django_connection() mock_s3.download_file.assert_not_called() os.remove(local_name) class TestDjangoStyleConnection(TestCase): @patch("explorer.ee.db_connections.models.load_backend") def test_create_django_style_connection_with_extras(self, mock_load_backend): conn = DatabaseConnection( name="test_db", alias="test_db", engine="django.db.backends.postgresql", extras='{"sslmode": "require", "connect_timeout": 10}' ) mock_backend = MagicMock() mock_load_backend.return_value = mock_backend conn.as_django_connection() mock_load_backend.assert_called_once_with("django.db.backends.postgresql") mock_backend.DatabaseWrapper.assert_called_once() args, kwargs = mock_backend.DatabaseWrapper.call_args self.assertEqual(args[0]["sslmode"], "require") self.assertEqual(args[0]["connect_timeout"], 10) @skipIf(not EXPLORER_USER_UPLOADS_ENABLED, "User uploads not enabled") class TestPandasToSQLite(TestCase): def test_pandas_to_sqlite(self): # Create a sample DataFrame data = { "column1": [1, 2, 3], "column2": ["A", "B", "C"] } df = pd.DataFrame(data) # Convert the DataFrame to SQLite and get the BytesIO buffer db_buffer = pandas_to_sqlite(df, "data", "test_pandas_to_sqlite.db") # Write the buffer to a temporary file to simulate reading it back temp_db_path = "temp_test_database.db" with open(temp_db_path, "wb") as f: f.write(db_buffer.getbuffer()) # Connect to the SQLite database and verify its content con = sqlite3.connect(temp_db_path) try: cursor = con.cursor() cursor.execute("SELECT * FROM data") # noqa rows = cursor.fetchall() # Verify the content of the SQLite database self.assertEqual(len(rows), 3) self.assertEqual(rows[0], (1, "A")) self.assertEqual(rows[1], (2, "B")) self.assertEqual(rows[2], (3, "C")) finally: con.close() os.remove(temp_db_path) def test_cant_create_connection_for_unregistered_django_alias(self): conn = DatabaseConnection(alias="not_registered", engine=DatabaseConnection.DJANGO) conn.save() self.assertRaises(DatabaseError, conn.as_django_connection) ================================================ FILE: explorer/tests/test_exporters.py ================================================ import json import unittest from datetime import date, datetime from django.core.serializers.json import DjangoJSONEncoder from django.test import TestCase from django.utils import timezone from explorer.exporters import CSVExporter, ExcelExporter, JSONExporter from explorer.models import QueryResult from explorer.tests.factories import SimpleQueryFactory from explorer.utils import is_xls_writer_available from explorer.ee.db_connections.utils import default_db_connection class TestCsv(TestCase): def test_writing_unicode(self): res = QueryResult( SimpleQueryFactory(sql='select 1 as "a", 2 as ""').sql, default_db_connection().as_django_connection() ) res.execute_query() res.process() res._data = [[1, None], ["Jenét", "1"]] res = CSVExporter(query=None)._get_output(res).getvalue() self.assertEqual( res.encode("utf-8").decode("utf-8-sig"), "a,\r\n1,\r\nJenét,1\r\n" ) def test_custom_delimiter(self): q = SimpleQueryFactory(sql="select 1, 2") exporter = CSVExporter(query=q) res = exporter.get_output(delim="|") self.assertEqual( res.encode("utf-8").decode("utf-8-sig"), "1|2\r\n1|2\r\n" ) def test_writing_bom(self): q = SimpleQueryFactory(sql="select 1, 2") exporter = CSVExporter(query=q) res = exporter.get_output() self.assertEqual(res, "\ufeff1,2\r\n1,2\r\n") class TestJson(TestCase): def test_writing_json(self): res = QueryResult( SimpleQueryFactory(sql='select 1 as "a", 2 as ""').sql, default_db_connection().as_django_connection() ) res.execute_query() res.process() res._data = [[1, None], ["Jenét", "1"]] res = JSONExporter(query=None)._get_output(res).getvalue() expected = [{"a": 1, "": None}, {"a": "Jenét", "": "1"}] self.assertEqual(res, json.dumps(expected)) def test_writing_datetimes(self): res = QueryResult( SimpleQueryFactory(sql='select 1 as "a", 2 as "b"').sql, default_db_connection().as_django_connection() ) res.execute_query() res.process() res._data = [[1, date.today()]] res = JSONExporter(query=None)._get_output(res).getvalue() expected = [{"a": 1, "b": date.today()}] self.assertEqual(res, json.dumps(expected, cls=DjangoJSONEncoder)) class TestExcel(TestCase): @unittest.skipIf(not is_xls_writer_available(), "excel exporter not available") def test_writing_excel(self): """ This is a pretty crap test. It at least exercises the code. If anyone wants to go through the brain damage of actually building an 'expected' xlsx output and comparing it (https://github.com/jmcnamara/XlsxWriter/blob/master/xlsxwriter/ test/helperfunctions.py) by all means submit a pull request! """ res = QueryResult( SimpleQueryFactory( sql='select 1 as "a", 2 as ""', title="\\/*[]:?this title is longer than 32 characters" ).sql, default_db_connection().as_django_connection() ) res.execute_query() res.process() d = datetime.now() d = timezone.make_aware(d, timezone.get_current_timezone()) res._data = [[1, None], ["Jenét", d]] res = ExcelExporter( query=SimpleQueryFactory() )._get_output(res).getvalue() expected = b"PK" self.assertEqual(res[:2], expected) @unittest.skipIf(not is_xls_writer_available(), "excel exporter not available") def test_writing_dict_fields(self): res = QueryResult( SimpleQueryFactory( sql='select 1 as "a", 2 as ""', title="\\/*[]:?this title is longer than 32 characters" ).sql, default_db_connection().as_django_connection() ) res.execute_query() res.process() res._data = [[1, ["foo", "bar"]], [2, {"foo": "bar"}]] res = ExcelExporter( query=SimpleQueryFactory() )._get_output(res).getvalue() expected = b"PK" self.assertEqual(res[:2], expected) ================================================ FILE: explorer/tests/test_forms.py ================================================ from django.db.utils import IntegrityError from django.forms.models import model_to_dict from django.test import TestCase from unittest.mock import patch, MagicMock from explorer.forms import QueryForm from explorer.tests.factories import SimpleQueryFactory from explorer.ee.db_connections.utils import default_db_connection_id class TestFormValidation(TestCase): def test_form_is_valid_with_valid_sql(self): q = SimpleQueryFactory(sql="select 1;", created_by_user_id=None) form = QueryForm(model_to_dict(q)) self.assertTrue(form.is_valid()) def test_form_fails_null(self): with self.assertRaises(IntegrityError): SimpleQueryFactory(sql=None, created_by_user_id=None) def test_form_fails_blank(self): q = SimpleQueryFactory(sql="", created_by_user_id=None) q.params = {} form = QueryForm(model_to_dict(q)) self.assertFalse(form.is_valid()) def test_form_fails_blacklist(self): q = SimpleQueryFactory(sql="delete $$a$$;", created_by_user_id=None) q.params = {} form = QueryForm(model_to_dict(q)) self.assertFalse(form.is_valid()) class QueryFormTestCase(TestCase): def test_valid_form_submission(self): form_data = { "title": "Test Query", "sql": "SELECT * FROM table", "description": "A test query description", "snapshot": False, "database_connection": str(default_db_connection_id()), } form = QueryForm(data=form_data) self.assertTrue(form.is_valid(), msg=form.errors) query = form.save() # Verify that the Query instance was created and is correctly linked to the DatabaseConnection self.assertEqual(query.database_connection_id, default_db_connection_id()) self.assertEqual(query.title, form_data["title"]) self.assertEqual(query.sql, form_data["sql"]) @patch("explorer.forms.default_db_connection") def test_default_connection_first(self, mocked_default_db_connection): dbc = MagicMock() dbc.id = default_db_connection_id() mocked_default_db_connection.return_value = dbc self.assertEqual(default_db_connection_id(), QueryForm().connections[0][0]) dbc = MagicMock() dbc.id = 2 mocked_default_db_connection.return_value = dbc self.assertEqual(2, QueryForm().connections[0][0]) ================================================ FILE: explorer/tests/test_mime.py ================================================ from django.test import TestCase from django.core.files.uploadedfile import SimpleUploadedFile from explorer.ee.db_connections.mime import is_sqlite, is_json, is_json_list, is_csv import io import sqlite3 import os class TestIsCsvFunction(TestCase): def test_is_csv_with_csv_file(self): # Create a SimpleUploadedFile with content_type set to "text/csv" csv_file = SimpleUploadedFile("test.csv", b"column1,column2\n1,A\n2,B", content_type="text/csv") self.assertTrue(is_csv(csv_file)) def test_is_csv_with_non_csv_file(self): # Create a SimpleUploadedFile with content_type set to "text/plain" txt_file = SimpleUploadedFile("test.txt", b"Just some text", content_type="text/plain") self.assertFalse(is_csv(txt_file)) def test_is_csv_with_empty_content_type(self): # Create a SimpleUploadedFile with an empty content_type empty_file = SimpleUploadedFile("test.csv", b"column1,column2\n1,A\n2,B", content_type="") self.assertFalse(is_csv(empty_file)) class TestIsJsonFunction(TestCase): def test_is_json_with_valid_json(self): long_json = '{"key1": "value1", "key2": {"subkey1": "subvalue1", "subkey2": "subvalue2"}, "key3": [1, 2, 3, 4]}' # noqa json_file = SimpleUploadedFile("test.json", long_json.encode("utf-8"), content_type="application/json") self.assertTrue(is_json(json_file)) def test_is_json_with_non_json_file(self): txt_file = SimpleUploadedFile("test.txt", b"Just some text", content_type="text/plain") self.assertFalse(is_json(txt_file)) def test_is_json_with_wrong_extension(self): long_json = '{"key1": "value1", "key2": {"subkey1": "subvalue1", "subkey2": "subvalue2"}, "key3": [1, 2, 3, 4]}' # noqa json_file = SimpleUploadedFile("test.txt", long_json.encode("utf-8"), content_type="application/json") self.assertFalse(is_json(json_file)) def test_is_json_with_empty_content_type(self): long_json = '{"key1": "value1", "key2": {"subkey1": "subvalue1", "subkey2": "subvalue2"}, "key3": [1, 2, 3, 4]}' # noqa json_file = SimpleUploadedFile("test.json", long_json.encode("utf-8"), content_type="") self.assertFalse(is_json(json_file)) class TestIsJsonListFunction(TestCase): def test_is_json_list_with_valid_json_lines(self): json_lines = b'{"key1": "value1"}\n{"key2": "value2"}\n{"key3": {"subkey1": "subvalue1"}}\n' # noqa json_file = SimpleUploadedFile("test.json", json_lines, content_type="application/json") self.assertTrue(is_json_list(json_file)) def test_is_json_list_with_multiline_json(self): json_lines = b'{"key1":\n"value1"}\n{"key2": "value2"}\n{"key3": {"subkey1": "subvalue1"}}\n' # noqa json_file = SimpleUploadedFile("test.json", json_lines, content_type="application/json") self.assertFalse(is_json_list(json_file)) def test_is_json_list_with_non_json_file(self): txt_file = SimpleUploadedFile("test.txt", b"Just some text", content_type="text/plain") self.assertFalse(is_json_list(txt_file)) def test_is_json_list_with_invalid_json_lines(self): # This is actually going to *pass* the check, because it's a shallow file-type check, not a comprehensive # one. That's ok! This type of error will get caught later, when pandas tries to parse it invalid_json_lines = b'{"key1": "value1"}\nNot a JSON content\n{"key3": {"subkey1": "subvalue1"}}\n' # noqa json_file = SimpleUploadedFile("test.json", invalid_json_lines, content_type="application/json") self.assertTrue(is_json_list(json_file)) def test_is_json_list_with_wrong_extension(self): json_lines = b'{"key1": "value1"}\n{"key2": "value2"}\n{"key3": {"subkey1": "subvalue1"}}\n' # noqa json_file = SimpleUploadedFile("test.txt", json_lines, content_type="application/json") self.assertFalse(is_json_list(json_file)) def test_is_json_list_with_empty_file(self): json_file = SimpleUploadedFile("test.json", b"", content_type="application/json") self.assertFalse(is_json_list(json_file)) class IsSqliteTestCase(TestCase): def setUp(self): # Create a SQLite database in a local file and read it into a BytesIO object # It would be nice to do this in memory, but that is not possible. local_path = "local_database.db" try: os.remove(local_path) except Exception as e: # noqa pass conn = sqlite3.connect(local_path) conn.execute("CREATE TABLE test (id INTEGER PRIMARY KEY, name TEXT)") for i in range(5): conn.execute("INSERT INTO test (name) VALUES (?)", (f"name_{i}",)) conn.commit() conn.close() # Read the local SQLite database file into a BytesIO buffer self.sqlite_db = io.BytesIO() with open(local_path, "rb") as f: self.sqlite_db.write(f.read()) self.sqlite_db.seek(0) # Clean up the local file os.remove(local_path) def test_is_sqlite_with_valid_sqlite_file(self): valid_sqlite_file = SimpleUploadedFile("test.sqlite", self.sqlite_db.read(), content_type="application/x-sqlite3") self.assertTrue(is_sqlite(valid_sqlite_file)) def test_is_sqlite_with_invalid_sqlite_file_content_type(self): self.sqlite_db.seek(0) invalid_content_type_file = SimpleUploadedFile("test.sqlite", self.sqlite_db.read(), content_type="text/plain") self.assertFalse(is_sqlite(invalid_content_type_file)) def test_is_sqlite_with_invalid_sqlite_file_header(self): invalid_sqlite_header = b"Invalid header" + b"\x00" * 100 invalid_sqlite_file = SimpleUploadedFile("test.sqlite", invalid_sqlite_header, content_type="application/x-sqlite3") self.assertFalse(is_sqlite(invalid_sqlite_file)) def test_is_sqlite_with_exception_handling(self): class FaultyFile: content_type = "application/x-sqlite3" def seek(self, offset): pass def read(self, num_bytes): raise OSError("Unable to read file") faulty_file = FaultyFile() self.assertFalse(is_sqlite(faulty_file)) ================================================ FILE: explorer/tests/test_models.py ================================================ import unittest import os from unittest.mock import Mock, patch, MagicMock from django.core.exceptions import ValidationError from django.db import IntegrityError from django.test import TestCase from explorer import app_settings from explorer.models import ColumnHeader, ColumnSummary, Query, QueryLog, QueryResult, DatabaseConnection from explorer.tests.factories import SimpleQueryFactory from explorer.ee.db_connections.utils import default_db_connection class TestQueryModel(TestCase): def test_params_get_merged(self): q = SimpleQueryFactory(sql="select '$$foo$$';") q.params = {"foo": "bar", "mux": "qux"} self.assertEqual(q.available_params(), {"foo": "bar"}) def test_default_params_used(self): q = SimpleQueryFactory(sql="select '$$foo:bar$$';") self.assertEqual(q.available_params(), {"foo": "bar"}) def test_default_params_used_even_with_labels(self): q = SimpleQueryFactory(sql="select '$$foo|label:bar$$';") self.assertEqual(q.available_params(), {"foo": "bar"}) def test_default_params_and_labels(self): q = SimpleQueryFactory(sql="select '$$foo|Label:bar$$';") self.assertEqual(q.available_params_w_labels(), {"foo": {"label": "Label", "val": "bar"}}) def test_query_log(self): self.assertEqual(0, QueryLog.objects.count()) q = SimpleQueryFactory() q.log(None) self.assertEqual(1, QueryLog.objects.count()) log = QueryLog.objects.first() self.assertEqual(log.run_by_user, None) self.assertEqual(log.query, q) self.assertFalse(log.is_playground) self.assertEqual(log.database_connection, q.database_connection) def test_query_logs_final_sql(self): q = SimpleQueryFactory(sql="select '$$foo$$';") q.params = {"foo": "bar"} q.log(None) self.assertEqual(1, QueryLog.objects.count()) log = QueryLog.objects.first() self.assertEqual(log.sql, "select 'bar';") def test_playground_query_log(self): query = Query(sql="select 1;", title="Playground") query.log(None) log = QueryLog.objects.first() self.assertTrue(log.is_playground) def test_shared(self): q = SimpleQueryFactory() q2 = SimpleQueryFactory() with self.settings(EXPLORER_USER_QUERY_VIEWS={"foo": [q.id]}): self.assertTrue(q.shared) self.assertFalse(q2.shared) def test_get_run_count(self): q = SimpleQueryFactory() self.assertEqual(q.get_run_count(), 0) expected = 4 for _ in range(0, expected): q.log() self.assertEqual(q.get_run_count(), expected) def test_avg_duration(self): q = SimpleQueryFactory() self.assertIsNone(q.avg_duration()) expected = 2.5 ql = q.log() ql.duration = 2 ql.save() ql = q.log() ql.duration = 3 ql.save() self.assertEqual(q.avg_duration(), expected) def test_log_saves_duration(self): q = SimpleQueryFactory() res, ql = q.execute_with_logging(None) log = QueryLog.objects.first() self.assertEqual(log.duration, res.duration) self.assertTrue(log.success) self.assertIsNone(log.error) def test_log_saves_errors(self): q = SimpleQueryFactory() q.sql = "select wildly invalid query" q.save() try: q.execute_with_logging(None) except Exception: pass log = QueryLog.objects.first() self.assertFalse(log.success) self.assertIsNotNone(log.error) @unittest.skipIf(not app_settings.ENABLE_TASKS, "tasks not enabled") @patch("explorer.models.s3_url") @patch("explorer.models.get_s3_bucket") def test_get_snapshots_sorts_snaps(self, mocked_get_s3_bucket, mocked_s3_url): bucket = Mock() bucket.objects.filter = Mock() k1 = Mock() k1.key = "foo" k1.last_modified = "b" k2 = Mock() k2.key = "bar" k2.last_modified = "a" bucket.objects.filter.return_value = [k1, k2] mocked_get_s3_bucket.return_value = bucket mocked_s3_url.return_value = "http://s3.com/presigned_url" q = SimpleQueryFactory() snaps = q.snapshots self.assertEqual(bucket.objects.filter.call_count, 1) self.assertEqual(snaps[0].url, "http://s3.com/presigned_url") bucket.objects.filter.assert_called_once_with(Prefix=f"query-{q.id}/snap-") def test_final_sql_uses_merged_params(self): q = SimpleQueryFactory(sql="select '$$foo:bar$$', '$$qux$$';") q.params = {"qux": "mux"} expected = "select 'bar', 'mux';" self.assertEqual(q.final_sql(), expected) def test_final_sql_fails_blacklist_with_bad_param(self): q = SimpleQueryFactory(sql="$$command$$ from bar;") q.params = {"command": "delete"} expected = "delete from bar;" self.assertEqual(q.final_sql(), expected) with self.assertRaises(ValidationError): q.execute_query_only() def test_query_will_execute_with_null_database_connection(self): q = SimpleQueryFactory(sql="select 1;") q.database_connection_id = None q.save() q.refresh_from_db() qr = q.execute_query_only() self.assertEqual(qr.data[0], [1]) class TestQueryResults(TestCase): def setUp(self): conn = default_db_connection().as_django_connection() self.qr = QueryResult('select 1 as "foo", "qux" as "mux";', conn) def test_column_access(self): self.qr._data = [[1, 2, 3], [4, 5, 6], [7, 8, 9]] self.assertEqual(self.qr.column(1), [2, 5, 8]) def test_headers(self): self.assertEqual(str(self.qr.headers[0]), "foo") self.assertEqual(str(self.qr.headers[1]), "mux") def test_data(self): self.assertEqual(self.qr.data, [[1, "qux"]]) def test_unicode_with_nulls(self): self.qr._headers = [ColumnHeader("num"), ColumnHeader("char")] self.qr._description = [("num",), ("char",)] self.qr._data = [[2, "a"], [3, None]] self.qr.process() self.assertEqual(self.qr.data, [[2, "a"], [3, None]]) def test_summary_gets_built(self): self.qr.process() self.assertEqual(len([h for h in self.qr.headers if h.summary]), 1) self.assertEqual(str(self.qr.headers[0].summary), "foo") self.assertEqual(self.qr.headers[0].summary.stats["Sum"], 1.0) def test_summary_gets_built_for_multiple_cols(self): self.qr._headers = [ColumnHeader("a"), ColumnHeader("b")] self.qr._description = [("a",), ("b",)] self.qr._data = [[1, 10], [2, 20]] self.qr.process() self.assertEqual(len([h for h in self.qr.headers if h.summary]), 2) self.assertEqual(self.qr.headers[0].summary.stats["Sum"], 3.0) self.assertEqual(self.qr.headers[1].summary.stats["Sum"], 30.0) def test_numeric_detection(self): self.assertEqual(self.qr._get_numerics(), [0]) def test_transforms_are_identified(self): self.qr._headers = [ColumnHeader("foo")] got = self.qr._get_transforms() self.assertEqual([(0, '{0}')], got) def test_transform_alters_row(self): self.qr._headers = [ColumnHeader("foo"), ColumnHeader("qux")] self.qr._data = [[1, 2]] self.qr.process() self.assertEqual(['1', 2], self.qr._data[0]) def test_multiple_transforms(self): self.qr._headers = [ColumnHeader("foo"), ColumnHeader("bar")] self.qr._data = [[1, 2]] self.qr.process() self.assertEqual(['1', "x: 2"], self.qr._data[0]) def test_get_headers_no_results(self): self.qr._description = None self.assertEqual([ColumnHeader("--")][0].title, self.qr._get_headers()[0].title) class TestColumnSummary(TestCase): def test_executes(self): res = ColumnSummary("foo", [1, 2, 3]) self.assertEqual(res.stats, {"Min": 1, "Max": 3, "Avg": 2, "Sum": 6, "NUL": 0}) def test_handles_null_as_zero(self): res = ColumnSummary("foo", [1, None, 5]) self.assertEqual(res.stats, {"Min": 0, "Max": 5, "Avg": 2, "Sum": 6, "NUL": 1}) def test_empty_data(self): res = ColumnSummary("foo", []) self.assertEqual(res.stats, {"Min": 0, "Max": 0, "Avg": 0, "Sum": 0, "NUL": 0}) class TestDatabaseConnection(TestCase): def test_cant_create_a_connection_with_conflicting_name(self): thrown = False try: conn = DatabaseConnection(alias="default") conn.save() except IntegrityError: thrown = True self.assertTrue(thrown) @patch("os.makedirs") @patch("os.path.exists", return_value=False) @patch("os.getcwd", return_value="/mocked/path") def test_local_name_calls_user_dbs_local_dir(self, mock_getcwd, mock_exists, mock_makedirs): connection = DatabaseConnection( alias="test", engine=DatabaseConnection.SQLITE, name="test_db.sqlite3", host="some-s3-bucket", ) local_name = connection.local_name expected_path = "/mocked/path/user_dbs/test_db.sqlite3" # Check if the local_name property returns the correct path self.assertEqual(local_name, expected_path) # Ensure os.makedirs was called once since the directory does not exist mock_makedirs.assert_called_once_with("/mocked/path/user_dbs") @patch("explorer.utils.get_s3_bucket") @patch("explorer.ee.db_connections.models.cache") def test_single_download_triggered(self, mock_cache, mock_get_s3_bucket): # Setup mocks mock_cache.add.return_value = True # Simulate acquiring the lock mock_s3 = MagicMock() mock_get_s3_bucket.return_value = mock_s3 # Call the method instance = DatabaseConnection( alias="test", engine=DatabaseConnection.SQLITE, name="test_db.sqlite3", host="some-s3-bucket", id=123 ) instance.download_sqlite_if_needed() # Assertions mock_s3.download_file.assert_called_once() mock_cache.add.assert_called_once() mock_cache.delete.assert_called_once() @patch("explorer.utils.get_s3_bucket") @patch("explorer.ee.db_connections.models.cache") def test_skip_download_when_locked(self, mock_cache, mock_get_s3_bucket): # Setup mocks mock_cache.add.return_value = False # Simulate that another process has the lock mock_s3 = MagicMock() mock_get_s3_bucket.return_value = mock_s3 # Call the method instance = DatabaseConnection( alias="test", engine=DatabaseConnection.SQLITE, name="test_db.sqlite3", host="some-s3-bucket", id=123 ) instance.download_sqlite_if_needed() # Assertions mock_s3.download_file.assert_not_called() mock_cache.add.assert_called_once() mock_cache.delete.assert_not_called() @patch("explorer.utils.get_s3_bucket") def test_not_downloaded_if_file_exists_and_model_is_unsaved(self, mock_get_s3_bucket): # Note this is NOT being saved to disk, e.g. how a DatabaseValidate would work. connection = DatabaseConnection( alias="test", engine=DatabaseConnection.SQLITE, name="test_db.sqlite3", host="some-s3-bucket", ) def mock_download_file(path, filename): pass mock_s3 = mock_get_s3_bucket.return_value mock_s3.download_file = MagicMock(side_effect=mock_download_file) # write the file with open(connection.local_name, "w") as f: f.write("Initial content") # See if it downloads connection.download_sqlite_if_needed() # And it shouldn't.... mock_s3.download_file.assert_not_called() # ...even though the fingerprints don't match self.assertIsNone(connection.upload_fingerprint) @patch("explorer.utils.get_s3_bucket") def test_fingerprint_is_updated_after_download_and_download_is_not_called_again(self, mock_get_s3_bucket): # Setup mock_s3 = mock_get_s3_bucket.return_value connection = DatabaseConnection.objects.create( alias="test", engine=DatabaseConnection.SQLITE, name="test_db.sqlite3", host="some-s3-bucket", ) # Define a function to mock S3 download def mock_download_file(path, filename): with open(filename, "w") as f: f.write("Initial content") mock_s3.download_file = MagicMock(side_effect=mock_download_file) # First download connection.download_sqlite_if_needed() # Check that the file was "downloaded" (in this case, created) self.assertTrue(os.path.exists(connection.local_name)) # Check that the fingerprint was updated self.assertIsNotNone(connection.upload_fingerprint) initial_fingerprint = connection.upload_fingerprint # Mock S3 download to track calls mock_s3.download_file.reset_mock() # Second attempt to download connection.download_sqlite_if_needed() # Check that download was not called again mock_s3.download_file.assert_not_called() # Check that the fingerprint hasn't changed connection.refresh_from_db() self.assertEqual(connection.upload_fingerprint, initial_fingerprint) # Modify the file to simulate changes with open(connection.local_name, "w") as f: f.write("Modified content") # Third attempt to download connection.download_sqlite_if_needed() # Check that download was called again mock_s3.download_file.assert_called_once() # Check that the fingerprint has been updated back to the original connection.refresh_from_db() self.assertEqual(connection.upload_fingerprint, initial_fingerprint) def test_default_is_set(self): orig_default = default_db_connection() new_default = DatabaseConnection(alias="new1", engine=DatabaseConnection.SQLITE, name="test_db.sqlite3", default=True) new_default.save() orig_default.refresh_from_db() self.assertFalse(orig_default.default) self.assertEqual(new_default.id, default_db_connection().id) self.assertEqual(DatabaseConnection.objects.filter(default=True).count(), 1) ================================================ FILE: explorer/tests/test_schema.py ================================================ from unittest.mock import patch from django.core.cache import cache from django.db import connection from django.test import TestCase from explorer import schema from explorer.ee.db_connections.utils import default_db_connection def conn(): return default_db_connection() class TestSchemaInfo(TestCase): def setUp(self): cache.clear() @patch("explorer.schema._get_includes") @patch("explorer.schema._get_excludes") def test_schema_info_returns_valid_data(self, mocked_excludes, mocked_includes): mocked_includes.return_value = None mocked_excludes.return_value = [] res = schema.schema_info(conn()) assert mocked_includes.called # sanity check: ensure patch worked tables = [x[0] for x in res] self.assertIn("explorer_query", tables) json_res = schema.schema_json_info(conn()) self.assertListEqual(list(json_res.keys()), tables) @patch("explorer.schema._get_includes") @patch("explorer.schema._get_excludes") def test_table_exclusion_list(self, mocked_excludes, mocked_includes): mocked_includes.return_value = None mocked_excludes.return_value = ("explorer_",) res = schema.schema_info(conn()) tables = [x[0] for x in res] self.assertNotIn("explorer_query", tables) @patch("explorer.schema._get_includes") @patch("explorer.schema._get_excludes") def test_app_inclusion_list(self, mocked_excludes, mocked_includes): mocked_includes.return_value = ("auth_",) mocked_excludes.return_value = [] res = schema.schema_info(conn()) tables = [x[0] for x in res] self.assertNotIn("explorer_query", tables) self.assertIn("auth_user", tables) @patch("explorer.schema._get_includes") @patch("explorer.schema._get_excludes") def test_app_inclusion_list_excluded(self, mocked_excludes, mocked_includes): # Inclusion list "wins" mocked_includes.return_value = ("explorer_",) mocked_excludes.return_value = ("explorer_",) res = schema.schema_info(conn()) tables = [x[0] for x in res] self.assertIn("explorer_query", tables) @patch("explorer.schema._include_views") def test_app_include_views(self, mocked_include_views): database_view = setup_sample_database_view() mocked_include_views.return_value = True res = schema.schema_info(conn()) tables = [x[0] for x in res] self.assertIn(database_view, tables) @patch("explorer.schema._include_views") def test_app_exclude_views(self, mocked_include_views): database_view = setup_sample_database_view() mocked_include_views.return_value = False res = schema.schema_info(conn()) tables = [x[0] for x in res] self.assertNotIn(database_view, tables) def test_transform_to_json(self): schema_info = [ ("table1", [("col1", "type1"), ("col2", "type2")]), ("table2", [("col1", "type1"), ("col2", "type2")]), ] json_schema = schema.transform_to_json_schema(schema_info) self.assertEqual(json_schema, { "table1": ["col1", "col2"], "table2": ["col1", "col2"], }) def setup_sample_database_view(): with connection.cursor() as cursor: cursor.execute( "CREATE VIEW IF NOT EXISTS v_explorer_query AS SELECT title, " "sql from explorer_query" ) return "v_explorer_query" ================================================ FILE: explorer/tests/test_tasks.py ================================================ import unittest from datetime import datetime, timedelta from io import StringIO from unittest.mock import patch import os from django.core import mail from django.test import TestCase from django.utils import timezone from explorer import app_settings from explorer.models import QueryLog from explorer.ee.db_connections.models import DatabaseConnection from explorer.tasks import execute_query, snapshot_queries, truncate_querylogs, \ remove_unused_sqlite_dbs from explorer.tests.factories import SimpleQueryFactory class TestTasks(TestCase): @unittest.skipIf(not app_settings.ENABLE_TASKS, "tasks not enabled") @patch("explorer.tasks.s3_csv_upload") def test_async_results(self, mocked_upload): mocked_upload.return_value = "http://s3.com/your-file.csv" q = SimpleQueryFactory( sql='select 1 "a", 2 "b", 3 "c";', title="testquery" ) execute_query(q.id, "cc@epantry.com") output = StringIO() output.write("a,b,c\r\n1,2,3\r\n") self.assertEqual(len(mail.outbox), 2) self.assertIn( "[SQL Explorer] Your query is running", mail.outbox[0].subject ) self.assertIn("[SQL Explorer] Report ", mail.outbox[1].subject) self.assertEqual( mocked_upload .call_args[0][1].getvalue() .decode("utf-8-sig"), output.getvalue() ) self.assertEqual(mocked_upload.call_count, 1) @unittest.skipIf(not app_settings.ENABLE_TASKS, "tasks not enabled") @patch("explorer.tasks.s3_csv_upload") def test_async_results_fails_with_message(self, mocked_upload): mocked_upload.return_value = "http://s3.com/your-file.csv" q = SimpleQueryFactory(sql="select x from foo;", title="testquery") execute_query(q.id, "cc@epantry.com") output = StringIO() output.write("a,b,c\r\n1,2,3\r\n") self.assertEqual(len(mail.outbox), 2) self.assertIn("[SQL Explorer] Error ", mail.outbox[1].subject) self.assertEqual(mocked_upload.call_count, 0) @unittest.skipIf(not app_settings.ENABLE_TASKS, "tasks not enabled") @patch("explorer.tasks.s3_csv_upload") def test_snapshots(self, mocked_upload): mocked_upload.return_value = "http://s3.com/your-file.csv" SimpleQueryFactory(snapshot=True) SimpleQueryFactory(snapshot=True) SimpleQueryFactory(snapshot=True) SimpleQueryFactory(snapshot=False) snapshot_queries() self.assertEqual(mocked_upload.call_count, 3) @unittest.skipIf(not app_settings.ENABLE_TASKS, "tasks not enabled") def test_truncating_querylogs(self): QueryLog(sql="foo").save() delete_time = timezone.make_aware(datetime.now() - timedelta(days=31), timezone.get_default_timezone()) QueryLog.objects.filter(sql="foo").update( run_at=delete_time ) QueryLog(sql="bar").save() ok_time = timezone.make_aware(datetime.now() - timedelta(days=29), timezone.get_default_timezone()) QueryLog.objects.filter(sql="bar").update( run_at=ok_time ) truncate_querylogs(30) self.assertEqual(QueryLog.objects.count(), 1) class RemoveUnusedSQLiteDBsTestCase(TestCase): def set_up_the_things(self, offset): dbc = DatabaseConnection( alias="localconn", name="localconn", engine=DatabaseConnection.SQLITE, host="foo" ) dbc.save() days = app_settings.EXPLORER_PRUNE_LOCAL_UPLOAD_COPY_DAYS_INACTIVITY with open(dbc.local_name, "w") as temp_db: temp_db.write("") recent_time = timezone.make_aware(datetime.now() - timedelta(days=days + offset), timezone.get_default_timezone()) ql = QueryLog(sql="foo", database_connection=dbc) ql.save() QueryLog.objects.filter(id=ql.id).update(run_at=recent_time) # Have to sidestep the auto_add_now return dbc, ql def test_remove_unused_sqlite_dbs(self): dbc, ql = self.set_up_the_things(1) remove_unused_sqlite_dbs() self.assertFalse(os.path.exists(dbc.local_name)) dbc.delete() ql.delete() def test_do_not_remove_recently_used_db(self): dbc, ql = self.set_up_the_things(-1) remove_unused_sqlite_dbs() self.assertTrue(os.path.exists(dbc.local_name)) os.remove(dbc.local_name) dbc.delete() ql.delete() ================================================ FILE: explorer/tests/test_telemetry.py ================================================ from django.test import TestCase from explorer.telemetry import instance_identifier, _gather_summary_stats, Stat, StatNames, _get_install_quarter from unittest.mock import patch, MagicMock from django.core.cache import cache from datetime import datetime class TestTelemetry(TestCase): def setUp(self): cache.delete("last_stat_sent_time") def test_instance_identifier(self): v = instance_identifier() self.assertEqual(len(v), 36) # Doesn't change after calling it again v = instance_identifier() self.assertEqual(len(v), 36) def test_gather_summary_stats(self): res = _gather_summary_stats() self.assertEqual(res["total_query_count"], 0) self.assertEqual(res["default_database"], "sqlite") @patch("explorer.telemetry.threading.Thread") @patch("explorer.app_settings") def test_stats_not_sent_too_frequently(self, mocked_app_settings, mocked_thread): mocked_app_settings.EXPLORER_ENABLE_ANONYMOUS_STATS = True mocked_app_settings.UNSAFE_RENDERING = True mocked_app_settings.EXPLORER_CHARTS_ENABLED = True mocked_app_settings.has_assistant = MagicMock(return_value=True) mocked_app_settings.db_connections_enabled = MagicMock(return_value=True) mocked_app_settings.ENABLE_TASKS = True s1 = Stat(StatNames.QUERY_RUN, {"foo": "bar"}) s2 = Stat(StatNames.QUERY_RUN, {"mux": "qux"}) s3 = Stat(StatNames.QUERY_RUN, {"bar": "baz"}) # once for s1 and once for summary stats s1.track() self.assertEqual(mocked_thread.call_count, 2) # both the s2 track call is suppressed, and the summary stat call s2.track() self.assertEqual(mocked_thread.call_count, 2) # clear the cache, which should cause track() for the stat to work, but not send summary stats cache.clear() s3.track() self.assertEqual(mocked_thread.call_count, 3) @patch("explorer.telemetry.threading.Thread") @patch("explorer.app_settings") def test_stats_not_sent_if_disabled(self, mocked_app_settings, mocked_thread): mocked_app_settings.EXPLORER_ENABLE_ANONYMOUS_STATS = False s1 = Stat(StatNames.QUERY_RUN, {"foo": "bar"}) s1.track() self.assertEqual(mocked_thread.call_count, 0) @patch("explorer.telemetry.MigrationRecorder.Migration.objects.filter") def test_get_install_quarter_with_no_migrations(self, mock_filter): mock_filter.return_value.order_by.return_value.first.return_value = None result = _get_install_quarter() self.assertIsNone(result) @patch("explorer.telemetry.MigrationRecorder.Migration.objects.filter") def test_get_install_quarter_edge_cases(self, mock_filter): # Test edge cases like end of year and start of year dates = [datetime(2022, 12, 31), datetime(2023, 1, 1), datetime(2023, 3, 31), datetime(2023, 4, 1)] results = ["Q4-2022", "Q1-2023", "Q1-2023", "Q2-2023"] for date, expected in zip(dates, results): with self.subTest(date=date): mock_migration = MagicMock() mock_migration.applied = date mock_filter.return_value.order_by.return_value.first.return_value = mock_migration result = _get_install_quarter() self.assertEqual(result, expected) ================================================ FILE: explorer/tests/test_type_infer.py ================================================ from django.test import TestCase from unittest import skipIf from explorer.app_settings import EXPLORER_USER_UPLOADS_ENABLED if EXPLORER_USER_UPLOADS_ENABLED: import pandas as pd import os from explorer.ee.db_connections.type_infer import csv_to_typed_df, json_to_typed_df, json_list_to_typed_df def _get_csv(csv_name): current_script_dir = os.path.dirname(os.path.abspath(__file__)) file_path = os.path.join(current_script_dir, "csvs", csv_name) # Open the file in binary mode and read its contents with open(file_path, "rb") as file: csv_bytes = file.read() return csv_bytes def _get_json(json_name): current_script_dir = os.path.dirname(os.path.abspath(__file__)) file_path = os.path.join(current_script_dir, "json", json_name) # Open the file in binary mode and read its contents with open(file_path, "rb") as file: json_bytes = file.read() return json_bytes @skipIf(not EXPLORER_USER_UPLOADS_ENABLED, "User uploads not enabled") class TestCsvToTypedDf(TestCase): def test_mixed_types(self): df = csv_to_typed_df(_get_csv("mixed.csv")) self.assertTrue(pd.api.types.is_object_dtype(df["Value1"])) self.assertTrue(pd.api.types.is_object_dtype(df["Value2"])) self.assertTrue(pd.api.types.is_object_dtype(df["Value3"])) def test_all_types(self): df = csv_to_typed_df(_get_csv("all_types.csv")) self.assertTrue(pd.api.types.is_datetime64_ns_dtype(df["Dates"])) self.assertTrue(pd.api.types.is_integer_dtype(df["Integers"])) self.assertTrue(pd.api.types.is_float_dtype(df["Floats"])) self.assertTrue(pd.api.types.is_object_dtype(df["Strings"])) def test_integer_parsing(self): df = csv_to_typed_df(_get_csv("integers.csv")) self.assertTrue(pd.api.types.is_integer_dtype(df["Integers"])) self.assertTrue(pd.api.types.is_integer_dtype(df["More_integers"])) def test_float_parsing(self): df = csv_to_typed_df(_get_csv("floats.csv")) self.assertTrue(pd.api.types.is_float_dtype(df["Floats"])) def test_date_parsing(self): # Will not handle these formats: # Unix Timestamp: 1706232300 (Seconds since Unix Epoch - 1970-01-01 00:00:00 UTC) # ISO 8601 Week Number: 2024-W04-3 (Year-WWeekNumber-Weekday) # Day of Year: 2024-024 (Year-DayOfYear) df = csv_to_typed_df(_get_csv("dates.csv")) self.assertTrue(pd.api.types.is_datetime64_ns_dtype(df["Dates"])) @skipIf(not EXPLORER_USER_UPLOADS_ENABLED, "User uploads not enabled") class TestJsonToTypedDf(TestCase): def test_basic_json(self): df = json_to_typed_df(_get_json("kings.json")) self.assertTrue(pd.api.types.is_object_dtype(df["Name"])) self.assertTrue(pd.api.types.is_object_dtype(df["Country"])) self.assertTrue(pd.api.types.is_integer_dtype(df["ID"])) def test_nested_json(self): df = json_to_typed_df(_get_json("github.json")) self.assertTrue(pd.api.types.is_object_dtype(df["subscription_url"])) self.assertTrue(pd.api.types.is_object_dtype(df["topics"])) self.assertTrue(pd.api.types.is_integer_dtype(df["size"])) self.assertTrue(pd.api.types.is_integer_dtype(df["owner.id"])) def test_json_list(self): df = json_list_to_typed_df(_get_json("list.json")) self.assertTrue(pd.api.types.is_integer_dtype(df["Item.value.M.unique_connection_count.N"])) self.assertTrue(pd.api.types.is_object_dtype(df["Item.instanceId.S"])) self.assertEqual(len(df), 5) ================================================ FILE: explorer/tests/test_utils.py ================================================ from unittest.mock import Mock from django.test import TestCase from explorer import app_settings from explorer.tests.factories import SimpleQueryFactory from explorer.utils import ( EXPLORER_PARAM_TOKEN, extract_params, get_params_for_url, get_params_from_request, param, passes_blacklist, shared_dict_update, swap_params, secure_filename ) class TestSqlBlacklist(TestCase): def setUp(self): self.orig = app_settings.EXPLORER_SQL_BLACKLIST def tearDown(self): app_settings.EXPLORER_SQL_BLACKLIST = self.orig def test_overriding_blacklist(self): app_settings.EXPLORER_SQL_BLACKLIST = [] sql = "DELETE FROM some_table;" passes, words = passes_blacklist(sql) self.assertTrue(passes) def test_not_overriding_blacklist(self): sql = "DELETE FROM some_table;" passes, words = passes_blacklist(sql) self.assertFalse(passes) # Various flavors of select - all should be ok def test_select_keywords_as_literals(self): sql = "SELECT * from eventtype where eventtype.value = 'Grant Date';" passes, words = passes_blacklist(sql) self.assertTrue(passes) def test_select_containing_drop_in_word(self): sql = "SELECT * FROM student droptable WHERE name LIKE 'Robert%'" self.assertTrue(passes_blacklist(sql)[0]) def test_select_with_case(self): sql = """SELECT ProductNumber, Name, "Price Range" = CASE WHEN ListPrice = 0 THEN 'Mfg item - not for resale' WHEN ListPrice < 50 THEN 'Under $50' WHEN ListPrice >= 50 and ListPrice < 250 THEN 'Under $250' WHEN ListPrice >= 250 and ListPrice < 1000 THEN 'Under $1000' ELSE 'Over $1000' END FROM Production.Product ORDER BY ProductNumber ; """ passes, words = passes_blacklist(sql) self.assertTrue(passes) def test_select_with_subselect(self): sql = """SELECT a.studentid, a.name, b.total_marks FROM student a, marks b WHERE a.studentid = b.studentid AND b.total_marks > (SELECT total_marks FROM marks WHERE studentid = 'V002'); """ passes, words = passes_blacklist(sql) self.assertTrue(passes) def test_select_with_replace_function(self): sql = "SELECT replace('test string', 'st', '**');" passes, words = passes_blacklist(sql) self.assertTrue(passes) def test_dml_commit(self): sql = "COMMIT TRANSACTION;" passes, words = passes_blacklist(sql) self.assertFalse(passes) def test_dml_delete(self): sql = "'distraction'; deLeTe from table; " \ "SELECT 1+1 AS TWO; drop view foo;" passes, words = passes_blacklist(sql) self.assertFalse(passes) self.assertEqual(len(words), 2) def test_dml_insert(self): sql = "INSERT INTO products (product_no, name, price) VALUES (1, 'Cheese', 9.99);" passes, words = passes_blacklist(sql) self.assertFalse(passes) def test_dml_merge(self): sql = """MERGE INTO wines w USING (VALUES('Chateau Lafite 2003', '24')) v ON v.column1 = w.winename WHEN NOT MATCHED INSERT VALUES(v.column1, v.column2) WHEN MATCHED UPDATE SET stock = stock + v.column2;""" passes, words = passes_blacklist(sql) self.assertFalse(passes) def test_dml_replace(self): sql = "REPLACE INTO test VALUES (1, 'Old', '2014-08-20 18:47:00');" passes, words = passes_blacklist(sql) self.assertFalse(passes) def test_dml_rollback(self): sql = "ROLLBACK TO SAVEPOINT my_savepoint;" passes, words = passes_blacklist(sql) self.assertFalse(passes) def test_dml_set(self): sql = "SET PASSWORD FOR 'user-name-here' = PASSWORD('new-password');" passes, words = passes_blacklist(sql) self.assertFalse(passes) def test_dml_start(self): sql = "START TRANSACTION;" passes, words = passes_blacklist(sql) self.assertFalse(passes) def test_dml_update(self): sql = """UPDATE accounts SET (contact_first_name, contact_last_name) = (SELECT first_name, last_name FROM employees WHERE employees.id = accounts.sales_person);""" passes, words = passes_blacklist(sql) self.assertFalse(passes) def test_dml_upsert(self): sql = "UPSERT INTO Users VALUES (10, 'John', 'Smith', 27, 60000);" passes, words = passes_blacklist(sql) self.assertFalse(passes) def test_ddl_alter(self): sql = """ALTER TABLE foo ALTER COLUMN foo_timestamp DROP DEFAULT, ALTER COLUMN foo_timestamp TYPE timestamp with time zone USING timestamp with time zone 'epoch' + foo_timestamp * interval '1 second', ALTER COLUMN foo_timestamp SET DEFAULT now();""" passes, words = passes_blacklist(sql) self.assertFalse(passes) def test_ddl_create(self): sql = """CREATE TABLE Persons ( PersonID int, LastName varchar(255), FirstName varchar(255), Address varchar(255), City varchar(255) ); """ passes, words = passes_blacklist(sql) self.assertFalse(passes) def test_ddl_drop(self): sql = "DROP TABLE films, distributors;" passes, words = passes_blacklist(sql) self.assertFalse(passes) def test_ddl_rename(self): sql = "RENAME TABLE old_table_name TO new_table_name;" passes, words = passes_blacklist(sql) self.assertFalse(passes) def test_ddl_truncate(self): sql = "TRUNCATE bigtable, othertable RESTART IDENTITY;" passes, words = passes_blacklist(sql) self.assertFalse(passes) def test_dcl_grant(self): sql = "GRANT ALL PRIVILEGES ON kinds TO manuel;" passes, words = passes_blacklist(sql) self.assertFalse(passes) def test_dcl_revoke(self): sql = "REVOKE ALL PRIVILEGES ON kinds FROM manuel;" passes, words = passes_blacklist(sql) self.assertFalse(passes) def test_dcl_revoke_bad_syntax(self): sql = "REVOKE ON kinds; FROM manuel;" passes, words = passes_blacklist(sql) self.assertFalse(passes) class TestParams(TestCase): def test_swappable_params_are_built_correctly(self): expected = EXPLORER_PARAM_TOKEN + "foo" + EXPLORER_PARAM_TOKEN self.assertEqual(expected, param("foo")) def test_params_get_swapped(self): sql = "please Swap $$this$$ and $$THat$$" expected = "please Swap here and there" params = {"this": "here", "that": "there"} got = swap_params(sql, params) self.assertEqual(got, expected) def test_empty_params_does_nothing(self): sql = "please swap $$this$$ and $$that$$" params = None got = swap_params(sql, params) self.assertEqual(got, sql) def test_non_string_param_gets_swapper(self): sql = "please swap $$this$$" expected = "please swap 1" params = {"this": 1} got = swap_params(sql, params) self.assertEqual(got, expected) def _assertSwap(self, tuple): self.assertEqual(extract_params(tuple[0]), tuple[1]) def test_extracting_params(self): tests = [ ("please swap $$this0$$", {"this0": {"default": "", "label": ""}}), ("please swap $$THis0$$", {"this0": {"default": "", "label": ""}}), ("please swap $$this6$$ $$this6:that$$", {"this6": {"default": "that", "label": ""}}), ("please swap $$this_7:foo, bar$$", {"this_7": {"default": "foo, bar", "label": ""}}), ("please swap $$this8:$$", {}), ("do nothing with $$this1 $$", {}), ("do nothing with $$this2 :$$", {}), ("do something with $$this3: $$", {"this3": {"default": " ", "label": ""}}), ("do nothing with $$this4: ", {}), ("do nothing with $$this5$that$$", {}), ("check label $$this|label:val$$", {"this": {"default": "val", "label": "label"}}), ("check case $$this|label Case:Va l$$", {"this": {"default": "Va l", "label": "label Case"}}), ("check label case and unicode $$this|label Case ελληνικά:val Τέστ$$", { "this": {"default": "val Τέστ", "label": "label Case ελληνικά"} }), ] for s in tests: self._assertSwap(s) def test_shared_dict_update(self): source = {"foo": 1, "bar": 2} target = {"bar": None} # ha ha! self.assertEqual({"bar": 2}, shared_dict_update(target, source)) def test_get_params_from_url(self): r = Mock() r.GET = {"params": "foo:bar|qux:mux"} res = get_params_from_request(r) self.assertEqual(res["foo"], "bar") self.assertEqual(res["qux"], "mux") def test_get_params_for_request(self): q = SimpleQueryFactory(params={"a": 1, "b": 2}) # For some reason the order of the params is non-deterministic, # causing the following to periodically fail: # self.assertEqual(get_params_for_url(q), 'a:1|b:2') # So instead we go for the following, convoluted, asserts: res = get_params_for_url(q) res = res.split("|") expected = ["a:1", "b:2"] for e in expected: self.assertIn(e, res) def test_get_params_for_request_empty(self): q = SimpleQueryFactory() self.assertEqual(get_params_for_url(q), None) class TestSecureFilename(TestCase): def test_basic_ascii(self): self.assertEqual(secure_filename("simple_file.txt"), "simple_file.txt") def test_special_characters(self): self.assertEqual(secure_filename("file@name!.txt"), "file_name.txt") def test_leading_trailing_underscores(self): self.assertEqual(secure_filename("_leading.txt"), "leading.txt") self.assertEqual(secure_filename("trailing_.txt"), "trailing.txt") self.assertEqual(secure_filename(".__filename__.txt"), "filename.txt") def test_unicode_characters(self): self.assertEqual(secure_filename("fïléñâmé.txt"), "filename.txt") self.assertEqual(secure_filename("测试文件.txt"), "_.txt") def test_empty_filename(self): with self.assertRaises(ValueError): secure_filename("") def test_bad_extension(self): with self.assertRaises(ValueError): secure_filename("foo.xyz") def test_empty_extension(self): with self.assertRaises(ValueError): secure_filename("foo.") def test_spaces(self): self.assertEqual(secure_filename("file name.txt"), "file_name.txt") ================================================ FILE: explorer/tests/test_views.py ================================================ import importlib import json import time import unittest import os from unittest.mock import Mock, patch, MagicMock from unittest import skipIf from django.contrib.auth.models import User from django.core.files.uploadedfile import SimpleUploadedFile from django.core.cache import cache from django.db import DatabaseError from django.forms.models import model_to_dict from django.shortcuts import redirect from django.test import TestCase from django.urls import reverse from explorer import app_settings from explorer.forms import QueryForm from explorer.app_settings import EXPLORER_TOKEN, EXPLORER_USER_UPLOADS_ENABLED from explorer.models import MSG_FAILED_BLACKLIST, Query, QueryFavorite, QueryLog, DatabaseConnection from explorer.tests.factories import QueryLogFactory, SimpleQueryFactory from explorer.utils import user_can_see_query from explorer.ee.db_connections.utils import default_db_connection from explorer.schema import connection_schema_cache_key, connection_schema_json_cache_key from explorer.assistant.models import TableDescription def reload_app_settings(): """ Reload app settings, otherwise changes from testing context manager won't take effect app_settings are loaded at time of import """ importlib.reload(app_settings) class TestQueryListView(TestCase): def setUp(self): self.user = User.objects.create_superuser( "admin", "admin@admin.com", "pwd" ) self.client.login(username="admin", password="pwd") def test_admin_required(self): self.client.logout() resp = self.client.get(reverse("explorer_index")) self.assertTemplateUsed(resp, "admin/login.html") def test_headers(self): SimpleQueryFactory(title="foo - bar1") SimpleQueryFactory(title="foo - bar2") SimpleQueryFactory(title="foo - bar3") SimpleQueryFactory(title="qux - mux") resp = self.client.get(reverse("explorer_index")) self.assertContains(resp, "foo (3)") self.assertContains(resp, "foo - bar2") self.assertContains(resp, "qux - mux") def test_permissions_show_only_allowed_queries(self): self.client.logout() q1 = SimpleQueryFactory(title="canseethisone") q2 = SimpleQueryFactory(title="nope") user = User.objects.create_user("user", "user@user.com", "pwd") self.client.login(username="user", password="pwd") with self.settings(EXPLORER_USER_QUERY_VIEWS={user.id: [q1.id]}): resp = self.client.get(reverse("explorer_index")) self.assertTemplateUsed(resp, "explorer/query_list.html") self.assertContains(resp, q1.title) self.assertNotContains(resp, q2.title) def test_run_count(self): q = SimpleQueryFactory(title="foo - bar1") for _ in range(0, 4): q.log() resp = self.client.get(reverse("explorer_index")) self.assertContains(resp, "4") class TestQueryCreateView(TestCase): def setUp(self): self.admin = User.objects.create_superuser( "admin", "admin@admin.com", "pwd" ) self.user = User.objects.create_user( "user", "user@user.com", "pwd" ) def test_change_permission_required(self): self.client.login(username="user", password="pwd") resp = self.client.get(reverse("query_create")) self.assertTemplateUsed(resp, "admin/login.html") def test_renders_with_title(self): self.client.login(username="admin", password="pwd") resp = self.client.get(reverse("query_create")) self.assertTemplateUsed(resp, "explorer/query.html") self.assertContains(resp, "New Query") def custom_view(request): return redirect("/custom/login") class TestQueryDetailView(TestCase): databases = ["default", "alt"] def setUp(self): self.user = User.objects.create_superuser( "admin", "admin@admin.com", "pwd" ) self.client.login(username="admin", password="pwd") def test_query_with_bad_sql_renders_error(self): query = SimpleQueryFactory(sql="error") resp = self.client.get( reverse("query_detail", kwargs={"query_id": query.id}) ) self.assertTemplateUsed(resp, "explorer/query.html") self.assertContains(resp, "syntax error") def test_query_with_bad_sql_renders_error_on_save(self): query = SimpleQueryFactory(sql="select 1;") resp = self.client.post( reverse("query_detail", kwargs={"query_id": query.id}), data={"sql": "error"} ) self.assertTemplateUsed(resp, "explorer/query.html") self.assertContains(resp, "syntax error") def test_posting_query_saves_correctly(self): expected = "select 2;" query = SimpleQueryFactory(sql="select 1;") data = model_to_dict(query) data["sql"] = expected self.client.post( reverse("query_detail", kwargs={"query_id": query.id}), data ) self.assertEqual(Query.objects.get(pk=query.id).sql, expected) def test_change_permission_required_to_save_query(self): query = SimpleQueryFactory() expected = query.sql resp = self.client.get( reverse("query_detail", kwargs={"query_id": query.id}) ) self.assertTemplateUsed(resp, "explorer/query.html") self.client.post( reverse("query_detail", kwargs={"query_id": query.id}), {"sql": "select 1;"} ) self.assertEqual(Query.objects.get(pk=query.id).sql, expected) def test_modified_date_gets_updated_after_viewing_query(self): query = SimpleQueryFactory() old = query.last_run_date time.sleep(0.1) self.client.get( reverse("query_detail", kwargs={"query_id": query.id}) ) self.assertNotEqual(old, Query.objects.get(pk=query.id).last_run_date) def test_doesnt_render_results_if_show_is_none(self): query = SimpleQueryFactory(sql="select 6870+1;") resp = self.client.get( reverse( "query_detail", kwargs={"query_id": query.id} ) + "?show=0" ) self.assertTemplateUsed(resp, "explorer/query.html") self.assertNotContains(resp, "6871") def test_doesnt_render_results_if_show_is_none_on_post(self): query = SimpleQueryFactory(sql="select 6870+1;") resp = self.client.post( reverse( "query_detail", kwargs={"query_id": query.id} ) + "?show=0", {"sql": "select 6870+2;"} ) self.assertTemplateUsed(resp, "explorer/query.html") self.assertNotContains(resp, "6872") def test_doesnt_render_results_if_params_and_no_autorun(self): with self.settings(EXPLORER_AUTORUN_QUERY_WITH_PARAMS=False): reload_app_settings() query = SimpleQueryFactory(sql="select 6870+3 where 1=$$myparam:1$$;") resp = self.client.get( reverse( "query_detail", kwargs={"query_id": query.id} ) ) self.assertTemplateUsed(resp, "explorer/query.html") self.assertNotContains(resp, "6873") def test_does_render_results_if_params_and_autorun(self): with self.settings(EXPLORER_AUTORUN_QUERY_WITH_PARAMS=True): reload_app_settings() query = SimpleQueryFactory(sql="select 6870+4 where 1=$$myparam:1$$;") resp = self.client.get( reverse( "query_detail", kwargs={"query_id": query.id} ) ) self.assertTemplateUsed(resp, "explorer/query.html") self.assertContains(resp, "6874") def test_does_render_label_if_params_and_autorun(self): with self.settings(EXPLORER_AUTORUN_QUERY_WITH_PARAMS=True): reload_app_settings() query = SimpleQueryFactory(sql="select 6870+4 where 1=$$myparam|test my param label:1$$;") resp = self.client.get( reverse( "query_detail", kwargs={"query_id": query.id} ) ) self.assertTemplateUsed(resp, "explorer/query.html") self.assertContains(resp, "test my param label") def test_admin_required(self): self.client.logout() query = SimpleQueryFactory() resp = self.client.get( reverse("query_detail", kwargs={"query_id": query.id}) ) self.assertTemplateUsed(resp, "admin/login.html") def test_admin_required_with_explorer_no_permission_setting(self): self.client.logout() query = SimpleQueryFactory() with self.settings(EXPLORER_NO_PERMISSION_VIEW="explorer.tests.test_views.custom_view"): resp = self.client.get( reverse("query_detail", kwargs={"query_id": query.id}) ) self.assertRedirects( resp, "/custom/login", target_status_code=404 ) def test_individual_view_permission(self): self.client.logout() user = User.objects.create_user("user1", "user@user.com", "pwd") self.client.login(username="user1", password="pwd") query = SimpleQueryFactory(sql="select 123+1") with self.settings(EXPLORER_USER_QUERY_VIEWS={user.id: [query.id]}): resp = self.client.get( reverse("query_detail", kwargs={"query_id": query.id}) ) self.assertTemplateUsed(resp, "explorer/query.html") self.assertContains(resp, "124") def test_header_token_auth(self): self.client.logout() query = SimpleQueryFactory(sql="select 123+1") with self.settings(EXPLORER_TOKEN_AUTH_ENABLED=True): resp = self.client.get( reverse("query_detail", kwargs={"query_id": query.id}), **{"HTTP_X_API_TOKEN": EXPLORER_TOKEN} ) self.assertTemplateUsed(resp, "explorer/query.html") self.assertContains(resp, "124") def test_url_token_auth(self): self.client.logout() query = SimpleQueryFactory(sql="select 123+1") with self.settings(EXPLORER_TOKEN_AUTH_ENABLED=True): resp = self.client.get( reverse( "query_detail", kwargs={"query_id": query.id} ) + f"?token={EXPLORER_TOKEN}" ) self.assertTemplateUsed(resp, "explorer/query.html") self.assertContains(resp, "124") def test_user_query_views(self): request = Mock() request.user.is_anonymous = True kwargs = {} self.assertFalse(user_can_see_query(request, **kwargs)) request.user.is_anonymous = True self.assertFalse(user_can_see_query(request, **kwargs)) kwargs = {"query_id": 123} request.user.is_anonymous = False self.assertFalse(user_can_see_query(request, **kwargs)) request.user.id = 99 with self.settings(EXPLORER_USER_QUERY_VIEWS={99: [111, 123]}): self.assertTrue(user_can_see_query(request, **kwargs)) @unittest.skipIf(not app_settings.ENABLE_TASKS, "tasks not enabled") @patch("explorer.models.get_s3_bucket") def test_query_snapshot_renders(self, mocked_conn): conn = Mock() conn.objects.filter = Mock() k1 = Mock() k1.generate_url.return_value = "http://s3.com/foo" k1.last_modified = "2015-01-01" k2 = Mock() k2.generate_url.return_value = "http://s3.com/bar" k2.last_modified = "2015-01-02" conn.objects.filter.return_value = [k1, k2] mocked_conn.return_value = conn query = SimpleQueryFactory(sql="select 1;", snapshot=True) resp = self.client.get( reverse("query_detail", kwargs={"query_id": query.id}) ) self.assertContains(resp, "2015-01-01") self.assertContains(resp, "2015-01-02") def test_failing_blacklist_means_query_doesnt_execute(self): conn = default_db_connection().as_django_connection() start = len(conn.queries) query = SimpleQueryFactory(sql="select 1;") resp = self.client.post( reverse("query_detail", kwargs={"query_id": query.id}), data={"sql": "delete from auth_user;"} ) end = len(conn.queries) self.assertTemplateUsed(resp, "explorer/query.html") self.assertContains(resp, MSG_FAILED_BLACKLIST % "") self.assertEqual(start, end) def test_fullscreen(self): query = SimpleQueryFactory(sql="select 1;") resp = self.client.get( reverse( "query_detail", kwargs={"query_id": query.id} ) + "?fullscreen=1" ) self.assertTemplateUsed(resp, "explorer/fullscreen.html") def test_multiple_connections_integration(self): from explorer.app_settings import EXPLORER_CONNECTIONS c1_alias = EXPLORER_CONNECTIONS["SQLite"] conn = DatabaseConnection.objects.get(alias=c1_alias).as_django_connection() c = conn.cursor() c.execute("CREATE TABLE IF NOT EXISTS animals (name text NOT NULL);") c.execute("INSERT INTO animals ( name ) VALUES ('peacock')") c2_alias = EXPLORER_CONNECTIONS["Another"] conn = DatabaseConnection.objects.get(alias=c2_alias).as_django_connection() c = conn.cursor() c.execute("CREATE TABLE IF NOT EXISTS animals (name text NOT NULL);") c.execute("INSERT INTO animals ( name ) VALUES ('superchicken')") query1 = SimpleQueryFactory( sql="select name from animals;", database_connection_id=DatabaseConnection.objects.get(alias=c1_alias).id ) resp = self.client.get( reverse("query_detail", kwargs={"query_id": query1.id}) ) self.assertContains(resp, "peacock") query2 = SimpleQueryFactory( sql="select name from animals;", database_connection_id=DatabaseConnection.objects.get(alias=c2_alias).id ) resp = self.client.get( reverse("query_detail", kwargs={"query_id": query2.id}) ) self.assertContains(resp, "superchicken") class TestDownloadView(TestCase): def setUp(self): self.query = SimpleQueryFactory(sql="select 1;") self.user = User.objects.create_superuser( "admin", "admin@admin.com", "pwd" ) self.client.login(username="admin", password="pwd") def test_admin_required(self): self.client.logout() resp = self.client.get( reverse("download_query", kwargs={"query_id": self.query.id}) ) self.assertTemplateUsed(resp, "admin/login.html") def test_params_in_download(self): q = SimpleQueryFactory(sql="select '$$foo$$';") url = "{}?params={}".format( reverse("download_query", kwargs={"query_id": q.id}), "foo:123" ) resp = self.client.get(url) self.assertContains(resp, "'123'") def test_download_defaults_to_csv(self): query = SimpleQueryFactory() url = reverse("download_query", args=[query.pk]) response = self.client.get(url) self.assertEqual(response.status_code, 200) self.assertEqual(response["content-type"], "text/csv") def test_download_csv(self): query = SimpleQueryFactory() url = reverse("download_query", args=[query.pk]) + "?format=csv" response = self.client.get(url) self.assertEqual(response.status_code, 200) self.assertEqual(response["content-type"], "text/csv") def test_bad_query_gives_500(self): query = SimpleQueryFactory(sql="bad") url = reverse("download_query", args=[query.pk]) + "?format=csv" response = self.client.get(url) self.assertEqual(response.status_code, 500) def test_download_json(self): query = SimpleQueryFactory() url = reverse("download_query", args=[query.pk]) + "?format=json" response = self.client.get(url) self.assertEqual(response.status_code, 200) self.assertEqual(response["content-type"], "application/json") json_data = json.loads(response.content.decode("utf-8")) self.assertIsInstance(json_data, list) self.assertEqual(len(json_data), 1) self.assertEqual(json_data, [{"TWO": 2}]) class TestQueryPlayground(TestCase): def setUp(self): self.user = User.objects.create_superuser( "admin", "admin@admin.com", "pwd" ) self.client.login(username="admin", password="pwd") def test_empty_playground_renders(self): resp = self.client.get(reverse("explorer_playground")) self.assertEqual(resp.status_code, 200) self.assertTemplateUsed(resp, "explorer/play.html") def test_playground_renders_with_query_sql(self): query = SimpleQueryFactory(sql="select 1;") resp = self.client.get( "{}?query_id={}".format(reverse("explorer_playground"), query.id) ) self.assertTemplateUsed(resp, "explorer/play.html") self.assertContains(resp, "select 1;") def test_playground_renders_with_posted_sql(self): resp = self.client.post( reverse("explorer_playground"), {"sql": "select 1+3400;"} ) self.assertTemplateUsed(resp, "explorer/play.html") self.assertContains(resp, "3401") def test_playground_doesnt_render_with_posted_sql_if_show_is_none(self): resp = self.client.post( reverse("explorer_playground") + "?show=0", {"sql": "select 1+3400;"} ) self.assertTemplateUsed(resp, "explorer/play.html") self.assertNotContains(resp, "3401") def test_playground_renders_with_empty_posted_sql(self): resp = self.client.post(reverse("explorer_playground"), {"sql": ""}) self.assertEqual(resp.status_code, 200) self.assertTemplateUsed(resp, "explorer/play.html") def test_query_with_no_resultset_doesnt_throw_error(self): query = SimpleQueryFactory(sql="") resp = self.client.get( "{}?query_id={}".format(reverse("explorer_playground"), query.id) ) self.assertTemplateUsed(resp, "explorer/play.html") def test_admin_required(self): self.client.logout() resp = self.client.get(reverse("explorer_playground")) self.assertTemplateUsed(resp, "admin/login.html") def test_admin_required_with_no_permission_view_setting(self): self.client.logout() with self.settings(EXPLORER_NO_PERMISSION_VIEW="explorer.tests.test_views.custom_view"): resp = self.client.get(reverse("explorer_playground")) self.assertRedirects( resp, "/custom/login", target_status_code=404 ) def test_loads_query_from_log(self): querylog = QueryLogFactory() resp = self.client.get( "{}?querylog_id={}".format( reverse("explorer_playground"), querylog.id ) ) self.assertContains(resp, "FOUR") def test_fails_blacklist(self): resp = self.client.post( reverse("explorer_playground"), {"sql": "delete from auth_user;"} ) self.assertTemplateUsed(resp, "explorer/play.html") self.assertContains(resp, MSG_FAILED_BLACKLIST % "") def test_fullscreen(self): query = SimpleQueryFactory(sql="") resp = self.client.get( "{}?query_id={}&fullscreen=1".format( reverse("explorer_playground"), query.id ) ) self.assertTemplateUsed(resp, "explorer/fullscreen.html") class TestCSVFromSQL(TestCase): def setUp(self): self.user = User.objects.create_superuser( "admin", "admin@admin.com", "pwd" ) self.client.login(username="admin", password="pwd") def test_admin_required(self): self.client.logout() resp = self.client.post(reverse("download_sql"), {}) self.assertTemplateUsed(resp, "admin/login.html") def test_downloading_from_playground(self): sql = "select 1;" resp = self.client.post(reverse("download_sql"), {"sql": sql}) self.assertIn("attachment", resp["Content-Disposition"]) self.assertEqual("text/csv", resp["content-type"]) ql = QueryLog.objects.first() self.assertIn( f'filename="Playground-{ql.id}.csv"', resp["Content-Disposition"] ) def test_stream_csv_from_query(self): q = SimpleQueryFactory() resp = self.client.get( reverse("stream_query", kwargs={"query_id": q.id}) ) self.assertEqual("text/csv", resp["content-type"]) class TestSQLDownloadViews(TestCase): databases = ["default", "alt"] def setUp(self): self.user = User.objects.create_superuser( "admin", "admin@admin.com", "pwd" ) self.client.login(username="admin", password="pwd") def test_sql_download_csv(self): url = reverse("download_sql") + "?format=csv" response = self.client.post(url, {"sql": "select 1;"}) self.assertEqual(response.status_code, 200) self.assertEqual(response["content-type"], "text/csv") def test_sql_download_respects_connection(self): from explorer.app_settings import EXPLORER_CONNECTIONS c1_alias = EXPLORER_CONNECTIONS["SQLite"] conn = DatabaseConnection.objects.get(alias=c1_alias).as_django_connection() c = conn.cursor() c.execute("CREATE TABLE IF NOT EXISTS animals (name text NOT NULL);") c.execute("INSERT INTO animals ( name ) VALUES ('peacock')") c2_alias = EXPLORER_CONNECTIONS["Another"] conn = DatabaseConnection.objects.get(alias=c2_alias).as_django_connection() c = conn.cursor() c.execute("CREATE TABLE IF NOT EXISTS animals (name text NOT NULL);") c.execute("INSERT INTO animals ( name ) VALUES ('superchicken')") url = reverse("download_sql") + "?format=csv" form_data = {"sql": "select * from animals;", "title": "foo", "database_connection": DatabaseConnection.objects.get(alias=c2_alias).id} form = QueryForm(data=form_data) self.assertTrue(form.is_valid()) response = self.client.post(url, form.data) self.assertEqual(response.status_code, 200) self.assertContains(response, "superchicken") def test_sql_download_csv_with_custom_delim(self): url = reverse("download_sql") + "?format=csv&delim=|" form_data = {"sql": "select 1,2;", "title": "foo", "database_connection": default_db_connection().id} form = QueryForm(data=form_data) self.assertTrue(form.is_valid()) response = self.client.post(url, form.data) self.assertEqual(response.status_code, 200) self.assertEqual(response["content-type"], "text/csv") self.assertEqual(response.content.decode("utf-8-sig"), "1|2\r\n1|2\r\n") def test_sql_download_csv_with_tab_delim(self): url = reverse("download_sql") + "?format=csv&delim=tab" response = self.client.post(url, {"sql": "select 1,2;"}) self.assertEqual(response.status_code, 200) self.assertEqual(response["content-type"], "text/csv") self.assertEqual(response.content.decode("utf-8-sig"), "1\t2\r\n1\t2\r\n") def test_sql_download_csv_with_bad_delim(self): url = reverse("download_sql") + "?format=csv&delim=foo" response = self.client.post(url, {"sql": "select 1,2;"}) self.assertEqual(response.status_code, 200) self.assertEqual(response["content-type"], "text/csv") self.assertEqual(response.content.decode("utf-8-sig"), "1,2\r\n1,2\r\n") def test_sql_download_json(self): url = reverse("download_sql") + "?format=json" response = self.client.post(url, {"sql": "select 1;"}) self.assertEqual(response.status_code, 200) self.assertEqual(response["content-type"], "application/json") class TestSchemaView(TestCase): def setUp(self): cache.clear() self.user = User.objects.create_superuser( "admin", "admin@admin.com", "pwd" ) self.client.login(username="admin", password="pwd") def test_returns_schema_contents(self): resp = self.client.get( reverse("explorer_schema", kwargs={"connection": default_db_connection().id}) ) self.assertContains(resp, "explorer_query") self.assertTemplateUsed(resp, "explorer/schema.html") def test_returns_schema_contents_json(self): resp = self.client.get( reverse("explorer_schema_json", kwargs={"connection": default_db_connection().id}) ) self.assertContains(resp, "explorer_query") self.assertEqual(resp.headers["Content-Type"], "application/json") def test_returns_404_if_conn_doesnt_exist(self): resp = self.client.get( reverse("explorer_schema", kwargs={"connection": "bananas"}) ) self.assertEqual(resp.status_code, 404) def test_admin_required(self): self.client.logout() resp = self.client.get( reverse("explorer_schema", kwargs={"connection": default_db_connection().id}) ) self.assertTemplateUsed(resp, "admin/login.html") class TestFormat(TestCase): def setUp(self): self.user = User.objects.create_superuser( "admin", "admin@admin.com", "pwd" ) self.client.login(username="admin", password="pwd") def test_returns_formatted_sql(self): resp = self.client.post( reverse("format_sql"), data={"sql": "select * from explorer_query"} ) resp = json.loads(resp.content.decode("utf-8")) self.assertIn("\n", resp["formatted"]) self.assertIn("explorer_query", resp["formatted"]) class TestParamsInViews(TestCase): def setUp(self): self.user = User.objects.create_superuser( "admin", "admin@admin.com", "pwd" ) self.client.login(username="admin", password="pwd") self.query = SimpleQueryFactory(sql="select $$swap$$;") def test_retrieving_query_works_with_params(self): resp = self.client.get( reverse( "query_detail", kwargs={"query_id": self.query.id} ) + "?params=swap:123}" ) self.assertContains(resp, "123") def test_saving_non_executing_query_with__wrong_url_params_works(self): q = SimpleQueryFactory(sql="select $$swap$$;") data = model_to_dict(q) url = "{}?params={}".format( reverse("query_detail", kwargs={"query_id": q.id}), "foo:123" ) resp = self.client.post(url, data) self.assertContains(resp, "saved") def test_users_without_change_permissions_can_use_params(self): resp = self.client.get( reverse( "query_detail", kwargs={"query_id": self.query.id} ) + "?params=swap:123}" ) self.assertContains(resp, "123") class TestCreatedBy(TestCase): def setUp(self): self.user = User.objects.create_superuser( "admin", "admin@admin.com", "pwd" ) self.user2 = User.objects.create_superuser( "admin2", "admin2@admin.com", "pwd" ) self.client.login(username="admin", password="pwd") self.query = SimpleQueryFactory.build(created_by_user=self.user) self.data = model_to_dict(self.query) del self.data["id"] self.data["created_by_user_id"] = self.user2.id def test_query_update_doesnt_change_created_user(self): self.query.save() self.client.post( reverse("query_detail", kwargs={"query_id": self.query.id}), self.data ) q = Query.objects.get(id=self.query.id) self.assertEqual(q.created_by_user_id, self.user.id) def test_new_query_gets_created_by_logged_in_user(self): self.client.post(reverse("query_create"), self.data) q = Query.objects.first() self.assertEqual(q.created_by_user_id, self.user.id) class TestQueryLog(TestCase): def setUp(self): self.user = User.objects.create_superuser( "admin", "admin@admin.com", "pwd" ) self.client.login(username="admin", password="pwd") def test_playground_saves_query_to_log(self): self.client.post(reverse("explorer_playground"), {"sql": "select 1;"}) log = QueryLog.objects.first() self.assertTrue(log.is_playground) self.assertEqual(log.sql, "select 1;") # Since it will be saved on the initial query creation, no need to log it def test_creating_query_does_not_save_to_log(self): query = SimpleQueryFactory() self.client.post(reverse("query_create"), model_to_dict(query)) self.assertEqual(0, QueryLog.objects.count()) def test_query_saves_to_log(self): query = SimpleQueryFactory() data = model_to_dict(query) data["sql"] = "select 12345;" self.client.post( reverse("query_detail", kwargs={"query_id": query.id}), data ) self.assertEqual(1, QueryLog.objects.count()) def test_query_gets_logged_and_appears_on_log_page(self): query = SimpleQueryFactory() data = model_to_dict(query) data["sql"] = "select 12345;" self.client.post( reverse("query_detail", kwargs={"query_id": query.id}), data ) resp = self.client.get(reverse("explorer_logs")) self.assertContains(resp, "select 12345;") def test_admin_required(self): self.client.logout() resp = self.client.get(reverse("explorer_logs")) self.assertTemplateUsed(resp, "admin/login.html") def test_is_playground(self): self.assertTrue(QueryLog(sql="foo").is_playground) q = SimpleQueryFactory() self.assertFalse(QueryLog(sql="foo", query_id=q.id).is_playground) class TestEmailQuery(TestCase): def setUp(self): self.user = User.objects.create_superuser( "admin", "admin@admin.com", "pwd" ) self.client.login(username="admin", password="pwd") @patch("explorer.views.email.execute_query") def test_email_calls_task(self, mocked_execute): query = SimpleQueryFactory() url = reverse("email_csv_query", kwargs={"query_id": query.id}) self.client.post( url, data={"email": "foo@bar.com"}, ) self.assertEqual(mocked_execute.delay.call_count, 1) def test_no_email(self): query = SimpleQueryFactory() url = reverse("email_csv_query", kwargs={"query_id": query.id}) response = self.client.post( url, data={}, ) self.assertEqual(response.status_code, 400) class TestQueryFavorites(TestCase): def setUp(self): self.user = User.objects.create_superuser( "admin", "admin@admin.com", "pwd" ) self.client.login(username="admin", password="pwd") self.q = SimpleQueryFactory(title="query for x, y") QueryFavorite.objects.create(user=self.user, query=self.q) def test_returns_favorite_list(self): resp = self.client.get( reverse("query_favorites") ) self.assertContains(resp, "query for x, y") class TestQueryFavorite(TestCase): def setUp(self): self.user = User.objects.create_superuser( "admin", "admin@admin.com", "pwd" ) self.client.login(username="admin", password="pwd") self.q = SimpleQueryFactory(title="query for x, y") def test_toggle(self): resp = self.client.post( reverse("query_favorite", args=(self.q.id,)) ) resp = json.loads(resp.content.decode("utf-8")) self.assertTrue(resp["is_favorite"]) resp = self.client.post( reverse("query_favorite", args=(self.q.id,)) ) resp = json.loads(resp.content.decode("utf-8")) self.assertFalse(resp["is_favorite"]) @skipIf(not EXPLORER_USER_UPLOADS_ENABLED, "User uploads not enabled") class UploadDbViewTest(TestCase): def setUp(self): DatabaseConnection.objects.uploads().delete() self.user = User.objects.create_superuser( "admin", "admin@admin.com", "pwd" ) self.client.login(username="admin", password="pwd") def test_post_csv_file(self): file_content = "col1,col2\nval1,val2\nval3,val4" uploaded_file = SimpleUploadedFile("test.csv", file_content.encode(), content_type="text/csv") self.assertFalse(DatabaseConnection.objects.filter(alias=f"test_{self.user.id}.db").exists()) with patch("explorer.ee.db_connections.type_infer.csv_to_typed_df") as mock_csv_to_typed_df, \ patch("explorer.ee.db_connections.views.upload_sqlite") as mock_upload_sqlite: mock_csv_to_typed_df.return_value = MagicMock() response = self.client.post(reverse("explorer_upload"), {"file": uploaded_file}) self.assertEqual(response.status_code, 200) self.assertJSONEqual(response.content, {"success": True}) self.assertTrue(DatabaseConnection.objects.filter(alias=f"test_{self.user.id}.db").exists()) mock_upload_sqlite.assert_called_once() mock_csv_to_typed_df.assert_called_once() # An end-to-end test that uploads a json file, verifies a connection was created, then issues a query # using that connection and verifies the right data is returned. @patch("explorer.ee.db_connections.views.upload_sqlite") def test_upload_file(self, mock_upload_sqlite): self.assertFalse(DatabaseConnection.objects.filter(alias__contains="kings").exists()) # Upload some JSON file_path = os.path.join(os.getcwd(), "explorer/tests/json/kings.json") with open(file_path, "rb") as f: response = self.client.post(reverse("explorer_upload"), {"file": f}) # Verify that the mock was called and the connection created self.assertEqual(response.status_code, 200) self.assertEqual(mock_upload_sqlite.call_count, 1) # Query it and make sure that the reign of this particular king is indeed in the results. conn = DatabaseConnection.objects.filter(alias__contains="kings").first() resp = self.client.post( reverse("explorer_playground"), {"sql": "select * from kings where Name = 'Athelstan';", "database_connection": conn.id} ) self.assertIn("925-940", resp.content.decode("utf-8")) # Append a new table to the existing connection file_path = os.path.join(os.getcwd(), "explorer/tests/csvs/rc_sample.csv") with open(file_path, "rb") as f: response = self.client.post(reverse("explorer_upload"), {"file": f, "append": conn.id}) # Make sure it got re-uploaded self.assertEqual(response.status_code, 200) self.assertEqual(mock_upload_sqlite.call_count, 2) # Query it and make sure a valid result is in the response. Note this is the *same* connection. resp = self.client.post( reverse("explorer_playground"), {"sql": "select * from rc_sample where material_type = 'Steel';", "database_connection": conn.id} ) self.assertIn("Goudurix", resp.content.decode("utf-8")) # Clean up filesystem os.remove(conn.local_name) def test_post_no_file(self): response = self.client.post(reverse("explorer_upload")) self.assertEqual(response.status_code, 400) self.assertJSONEqual(response.content, {"error": "No file provided"}) def test_delete_existing_connection(self): dbc = DatabaseConnection.objects.create( alias="test.db", engine=DatabaseConnection.SQLITE, name="test.db", host="s3_path/test.db" ) with patch("explorer.ee.db_connections.views.delete_from_s3") as mock_delete_from_s3: response = self.client.delete(reverse("explorer_connection_delete", kwargs={"pk": dbc.id})) self.assertEqual(response.status_code, 302) self.assertFalse(DatabaseConnection.objects.filter(alias="test.db").exists()) mock_delete_from_s3.assert_called_once_with("s3_path/test.db") def test_delete_non_existing_connection(self): response = self.client.delete("/upload/?alias=nonexistent.db") self.assertEqual(response.status_code, 404) @patch("explorer.ee.db_connections.views.EXPLORER_MAX_UPLOAD_SIZE", 1024*1024) def test_post_file_too_large(self): file_content = "a" * (1024 * 1024 + 1) # Slightly larger 1 MB uploaded_file = SimpleUploadedFile("large_file.csv", file_content.encode(), content_type="text/csv") response = self.client.post(reverse("explorer_upload"), {"file": uploaded_file}) self.assertEqual(response.status_code, 400) self.assertJSONEqual(response.content, {"error": "File size exceeds the limit of 1.0 MB"}) @patch("explorer.ee.db_connections.views.parse_to_sqlite") def test_bad_parse_type(self, patched_parse): patched_parse.side_effect = TypeError("didnt work") uploaded_file = SimpleUploadedFile("large_file.csv", ("a"*10).encode(), content_type="text/foo") response = self.client.post(reverse("explorer_upload"), {"file": uploaded_file}) self.assertEqual(json.loads(response.content.decode("utf-8"))["error"], "Error parsing file.") def test_bad_parse_mime(self): uploaded_file = SimpleUploadedFile("large_file.foo", ("a" * 10).encode(), content_type="text/foo") response = self.client.post(reverse("explorer_upload"), {"file": uploaded_file}) self.assertEqual(json.loads(response.content.decode("utf-8"))["error"], "File was not csv, json, or sqlite.") @patch("explorer.ee.db_connections.views.is_sqlite") def test_cant_append_sqlite_to_file(self, patched_is_sqlite): patched_is_sqlite.return_value = True f = SimpleUploadedFile("large_file.foo", ("a" * 10).encode(), content_type="text/foo") dbc = DatabaseConnection.objects.create( alias="test.db", engine=DatabaseConnection.SQLITE, name="test.db", host="s3_path/test.db" ) resp = self.client.post(reverse("explorer_upload"), {"file": f, "append": dbc.id}) self.assertEqual(json.loads(resp.content.decode("utf-8"))["error"], "Can't append a SQLite file to a SQLite file. Only CSV and JSON.") class DatabaseConnectionValidateViewTestCase(TestCase): def setUp(self): self.user = User.objects.create_superuser( "admin", "admin@admin.com", "pwd" ) self.client.login(username="admin", password="pwd") self.url = reverse("explorer_connection_validate") self.valid_data = { "alias": "test_alias", "engine": "django.db.backends.sqlite3", "name": ":memory:", "user": "", "password": "", "host": "", "port": "", } self.invalid_data = { "alias": "", "engine": "", "name": "", "user": "", "password": "", "host": "", "port": "", } def test_validate_connection_success(self): response = self.client.post(self.url, data=self.valid_data) self.assertEqual(response.status_code, 200) self.assertJSONEqual(response.content, {"success": True}) def test_validate_connection_invalid_form(self): response = self.client.post(self.url, data=self.invalid_data) self.assertEqual(response.status_code, 200) self.assertJSONEqual(response.content, {"success": False, "error": "Invalid form data"}) def test_update_existing_connection(self): DatabaseConnection.objects.create(alias="test_alias", engine="django.db.backends.sqlite3", name=":memory:") response = self.client.post(self.url, data=self.valid_data) self.assertEqual(response.status_code, 200) self.assertJSONEqual(response.content, {"success": True}) @patch("explorer.ee.db_connections.models.load_backend") def test_database_connection_error(self, mock_load): mock_load.side_effect = DatabaseError("Connection error") response = self.client.post(self.url, data=self.valid_data) self.assertEqual(response.status_code, 200) self.assertJSONEqual(response.content, {"success": False, "error": "Failed to create explorer connection: Connection error"}) class TestDatabaseConnectionRefreshView(TestCase): def setUp(self): self.user = User.objects.create_superuser( "admin", "admin@admin.com", "pwd" ) self.client.login(username="admin", password="pwd") self.dbc = DatabaseConnection.objects.create( alias="test_alias", engine="django.db.backends.sqlite3", name="test.db", host="foo" ) self.k = connection_schema_cache_key(self.dbc.id) self.kj = connection_schema_json_cache_key(self.dbc.id) cache.set(self.k, "foo") cache.set(self.kj, "foo") def test_refresh_connection(self): # Create a file on disk with open(self.dbc.local_name, "w") as f: f.write("test data") # Ensure the file exists self.assertTrue(os.path.exists(self.dbc.local_name)) # Make a GET call to refresh the connection url = reverse("explorer_connection_refresh", args=[self.dbc.id]) response = self.client.get(url) # Assert the response is successful self.assertEqual(response.status_code, 200) # Assert that the file has been deleted self.assertFalse(os.path.exists(self.dbc.local_name)) # Assert that the cache keys are clear self.assertIsNone(cache.get(self.k)) self.assertIsNone(cache.get(self.kj)) def tearDown(self): # Clean up any files that might have been created if os.path.exists(self.dbc.local_name): os.remove(self.dbc.local_name) # The idea is to render all of these views, to ensure that errors haven't been introduced in the templates class SimpleViewTests(TestCase): def setUp(self): self.user = User.objects.create_superuser( "admin", "admin@admin.com", "pwd" ) self.client.login(username="admin", password="pwd") self.connection = DatabaseConnection.objects.create( alias="test_alias", engine="django.db.backends.sqlite3", name=":memory:", user="", password="", host="", port="" ) def test_database_connection_detail_view(self): response = self.client.get(reverse("explorer_connection_detail", args=[self.connection.pk])) self.assertEqual(response.status_code, 200) def test_database_connection_create_view(self): response = self.client.get(reverse("explorer_connection_create")) self.assertEqual(response.status_code, 200) def test_database_connection_update_view(self): response = self.client.get(reverse("explorer_connection_update", args=[self.connection.pk])) self.assertEqual(response.status_code, 200) def test_database_connections_list_view(self): response = self.client.get(reverse("explorer_connections")) self.assertEqual(response.status_code, 200) def test_database_connection_delete_view(self): response = self.client.get(reverse("explorer_connection_delete", args=[self.connection.pk])) self.assertEqual(response.status_code, 200) def test_database_connection_upload_view(self): response = self.client.get(reverse("explorer_upload_create")) self.assertEqual(response.status_code, 200) def test_table_description_list_view(self): td = TableDescription(database_connection=default_db_connection(), table_name="foo", description="annotated") td.save() response = self.client.get(reverse("table_description_list")) self.assertEqual(response.status_code, 200) response = self.client.get(reverse("table_description_update", args=[td.pk])) self.assertEqual(response.status_code, 200) response = self.client.get(reverse("table_description_create")) self.assertEqual(response.status_code, 200) ================================================ FILE: explorer/urls.py ================================================ from django.urls import path from explorer.ee.urls import ee_urls from explorer.views import ( CreateQueryView, DeleteQueryView, DownloadFromSqlView, DownloadQueryView, EmailCsvQueryView, ListQueryLogView, ListQueryView, PlayQueryView, QueryFavoritesView, QueryFavoriteView, QueryView, SchemaJsonView, SchemaView, StreamQueryView, format_sql ) from explorer.assistant.urls import assistant_urls urlpatterns = [ path( "/", QueryView.as_view(), name="query_detail" ), path( "/download", DownloadQueryView.as_view(), name="download_query" ), path( "/stream", StreamQueryView.as_view(), name="stream_query" ), path("download", DownloadFromSqlView.as_view(), name="download_sql"), path( "/email_csv", EmailCsvQueryView.as_view(), name="email_csv_query" ), path( "/delete", DeleteQueryView.as_view(), name="query_delete" ), path("new/", CreateQueryView.as_view(), name="query_create"), path("play/", PlayQueryView.as_view(), name="explorer_playground"), path( "schema/", SchemaView.as_view(), name="explorer_schema" ), path( "schema.json/", SchemaJsonView.as_view(), name="explorer_schema_json" ), path("logs/", ListQueryLogView.as_view(), name="explorer_logs"), path("format/", format_sql, name="format_sql"), path("favorites/", QueryFavoritesView.as_view(), name="query_favorites"), path("favorite/", QueryFavoriteView.as_view(), name="query_favorite"), path("", ListQueryView.as_view(), name="explorer_index"), ] urlpatterns += assistant_urls urlpatterns += ee_urls ================================================ FILE: explorer/utils.py ================================================ import re import os import unicodedata from collections import deque from typing import Iterable, Tuple from django.contrib.auth import REDIRECT_FIELD_NAME from django.contrib.auth.forms import AuthenticationForm from django.contrib.auth.views import LoginView import sqlparse from sqlparse import format as sql_format from sqlparse.sql import Token, TokenList from sqlparse.tokens import Keyword from explorer import app_settings EXPLORER_PARAM_TOKEN = "$$" def passes_blacklist(sql: str) -> Tuple[bool, Iterable[str]]: sql_strings = sqlparse.split(sql) keyword_tokens = set() for sql_string in sql_strings: statements = sqlparse.parse(sql_string) for statement in statements: for token in walk_tokens(statement): if not token.is_whitespace and not isinstance(token, TokenList): if token.ttype in Keyword: keyword_tokens.add(str(token.value).upper()) fails = [ bl_word for bl_word in app_settings.EXPLORER_SQL_BLACKLIST if bl_word.upper() in keyword_tokens ] return not bool(fails), fails def walk_tokens(token: TokenList) -> Iterable[Token]: """ Generator to walk all tokens in a Statement https://stackoverflow.com/questions/54982118/parse-case-when-statements-with-sqlparse :param token: TokenList """ queue = deque([token]) while queue: token = queue.popleft() if isinstance(token, TokenList): queue.extend(token) yield token def _format_field(field): return field.get_attname_column()[1], field.get_internal_type() def param(name): return f"{EXPLORER_PARAM_TOKEN}{name}{EXPLORER_PARAM_TOKEN}" def swap_params(sql, params): p = params.items() if params else {} for k, v in p: fmt_k = re.escape(str(k).lower()) regex = re.compile(rf"\$\${fmt_k}(?:\|([^\$\:]+))?(?:\:([^\$]+))?\$\$", re.I) sql = regex.sub(str(v), sql) return sql def extract_params(text): regex = re.compile(r"\$\$([a-z0-9_]+)(?:\|([^\$\:]+))?(?:\:([^\$]+))?\$\$", re.IGNORECASE) params = re.findall(regex, text) # Matching will result to ('name', 'label', 'default') return { p[0].lower(): { "label": p[1], "default": p[2] } for p in params if len(p) > 1 } def safe_login_prompt(request): defaults = { "template_name": "admin/login.html", "authentication_form": AuthenticationForm, "extra_context": { "title": "Log in", "app_path": request.get_full_path(), REDIRECT_FIELD_NAME: request.get_full_path(), }, } return LoginView.as_view(**defaults)(request) def shared_dict_update(target, source): for k_d1 in target: if k_d1 in source: target[k_d1] = source[k_d1] return target def safe_cast(val, to_type, default=None): try: return to_type(val) except ValueError: return default def get_int_from_request(request, name, default): val = request.GET.get(name, default) return safe_cast(val, int, default) if val else None def get_params_from_request(request): val = request.GET.get("params", None) try: d = {} tuples = val.split("|") for t in tuples: res = t.split(":") d[res[0]] = res[1] return d except Exception: return None def get_params_for_url(query): if query.params: return "|".join([f"{p}:{v}" for p, v in query.params.items()]) def url_get_rows(request): return get_int_from_request( request, "rows", app_settings.EXPLORER_DEFAULT_ROWS ) def url_get_query_id(request): return get_int_from_request(request, "query_id", None) def url_get_log_id(request): return get_int_from_request(request, "querylog_id", None) def url_get_show(request): return bool(get_int_from_request(request, "show", 1)) def url_get_fullscreen(request): return bool(get_int_from_request(request, "fullscreen", 0)) def url_get_params(request): return get_params_from_request(request) def allowed_query_pks(user_id): return app_settings.EXPLORER_GET_USER_QUERY_VIEWS().get(user_id, []) def user_can_see_query(request, **kwargs): if not request.user.is_anonymous and "query_id" in kwargs: return int(kwargs["query_id"]) in allowed_query_pks(request.user.id) return False def fmt_sql(sql): return sql_format(sql, reindent=True, keyword_case="upper") def noop_decorator(f): return f class InvalidExplorerConnectionException(Exception): pass def delete_from_s3(s3_path): s3_bucket = get_s3_bucket() s3_bucket.delete_objects( Delete={ "Objects": [ {"Key": s3_path} ] } ) def get_s3_bucket(): import boto3 from botocore.client import Config config = Config( signature_version=app_settings.S3_SIGNATURE_VERSION, region_name=app_settings.S3_REGION ) kwargs = {"config": config} # If these are set, use them. Otherwise, boto will use its built-in mechanisms # to provide authentication. if app_settings.S3_ACCESS_KEY and app_settings.S3_SECRET_KEY: kwargs["aws_access_key_id"] = app_settings.S3_ACCESS_KEY kwargs["aws_secret_access_key"] = app_settings.S3_SECRET_KEY if app_settings.S3_ENDPOINT_URL: kwargs["endpoint_url"] = app_settings.S3_ENDPOINT_URL s3 = boto3.resource("s3", **kwargs) return s3.Bucket(name=app_settings.S3_BUCKET) def s3_csv_upload(key, data): if app_settings.S3_DESTINATION: key = "/".join([app_settings.S3_DESTINATION, key]) bucket = get_s3_bucket() bucket.upload_fileobj(data, key, ExtraArgs={"ContentType": "text/csv"}) return s3_url(bucket, key) def s3_url(bucket, key): url = bucket.meta.client.generate_presigned_url( ClientMethod="get_object", Params={"Bucket": app_settings.S3_BUCKET, "Key": key}, ExpiresIn=app_settings.S3_LINK_EXPIRATION) return url def is_xls_writer_available(): try: import xlsxwriter # noqa return True except ImportError: return False def secure_filename(filename): filename, ext = os.path.splitext(filename) if not filename and not ext: raise ValueError("Filename or extension cannot be blank") if ext.lower() not in [".db", ".sqlite", ".sqlite3", ".csv", ".json", ".txt"]: raise ValueError(f"Invalid extension: {ext}") filename = unicodedata.normalize("NFKD", filename).encode("ascii", "ignore").decode("ascii") filename = re.sub(r"[^a-zA-Z0-9_.-]", "_", filename) filename = filename.strip("._") if not filename: # If filename becomes empty, replace it with an underscore filename = "_" return f"{filename}{ext}" ================================================ FILE: explorer/views/__init__.py ================================================ from .auth import PermissionRequiredMixin, SafeLoginView from .create import CreateQueryView from .delete import DeleteQueryView from .download import DownloadFromSqlView, DownloadQueryView from .email import EmailCsvQueryView from .format_sql import format_sql from .list import ListQueryLogView, ListQueryView from .query import PlayQueryView, QueryView from .query_favorite import QueryFavoritesView, QueryFavoriteView from .schema import SchemaJsonView, SchemaView from .stream import StreamQueryView __all__ = [ "CreateQueryView", "DeleteQueryView", "DownloadQueryView", "DownloadFromSqlView", "EmailCsvQueryView", "ListQueryView", "ListQueryLogView", "PermissionRequiredMixin", "PlayQueryView", "QueryView", "SafeLoginView", "StreamQueryView", "SchemaJsonView", "SchemaView", "format_sql", "QueryFavoritesView", "QueryFavoriteView", ] ================================================ FILE: explorer/views/auth.py ================================================ from django.contrib.auth import REDIRECT_FIELD_NAME from django.contrib.auth.views import LoginView from django.core.exceptions import ImproperlyConfigured from explorer import app_settings, permissions class PermissionRequiredMixin: permission_required = None @staticmethod def handle_no_permission(request): return app_settings.EXPLORER_NO_PERMISSION_VIEW()(request) def get_permission_required(self): if self.permission_required is None: raise ImproperlyConfigured( f"{self.__class__.__name__} is missing the permission_required attribute. " f"Define {self.__class__.__name__}.permission_required, or override " f"{self.__class__.__name__}.get_permission_required()." ) return self.permission_required def has_permission(self, request, *args, **kwargs): perms = self.get_permission_required() # TODO: fix the case when the perms is not defined in # permissions module. handler = getattr(permissions, perms) return handler(request, *args, **kwargs) def dispatch(self, request, *args, **kwargs): if not self.has_permission(request, *args, **kwargs): return self.handle_no_permission(request) return super().dispatch(request, *args, **kwargs) class SafeLoginView(LoginView): template_name = "admin/login.html" def safe_login_view_wrapper(request): return SafeLoginView.as_view( extra_context={ "title": "Log in", REDIRECT_FIELD_NAME: request.get_full_path() } )(request) ================================================ FILE: explorer/views/create.py ================================================ from django.views.generic import CreateView from explorer.forms import QueryForm from explorer.views.auth import PermissionRequiredMixin from explorer.views.mixins import ExplorerContextMixin class CreateQueryView(PermissionRequiredMixin, ExplorerContextMixin, CreateView): permission_required = "change_permission" form_class = QueryForm template_name = "explorer/query.html" def form_valid(self, form): form.instance.created_by_user = self.request.user return super().form_valid(form) ================================================ FILE: explorer/views/delete.py ================================================ from django.urls import reverse_lazy from django.views.generic import DeleteView from explorer.models import Query from explorer.views.auth import PermissionRequiredMixin from explorer.views.mixins import ExplorerContextMixin class DeleteQueryView(PermissionRequiredMixin, ExplorerContextMixin, DeleteView): permission_required = "change_permission" model = Query success_url = reverse_lazy("explorer_index") ================================================ FILE: explorer/views/download.py ================================================ from django.shortcuts import get_object_or_404 from django.views.generic.base import View from explorer.models import Query from explorer.views.auth import PermissionRequiredMixin from explorer.views.export import _export from explorer.ee.db_connections.utils import default_db_connection_id class DownloadQueryView(PermissionRequiredMixin, View): permission_required = "view_permission" def get(self, request, query_id, *args, **kwargs): query = get_object_or_404(Query, pk=query_id) return _export(request, query) class DownloadFromSqlView(PermissionRequiredMixin, View): permission_required = "view_permission" def post(self, request, *args, **kwargs): sql = request.POST.get("sql", "") connection = request.POST.get("database_connection", default_db_connection_id()) query = Query(sql=sql, database_connection_id=connection, title="") ql = query.log(request.user) query.title = f"Playground-{ql.id}" return _export(request, query) ================================================ FILE: explorer/views/email.py ================================================ from django.http import JsonResponse from django.views import View from explorer.tasks import execute_query from explorer.views.auth import PermissionRequiredMixin class EmailCsvQueryView(PermissionRequiredMixin, View): permission_required = "view_permission" def post(self, request, query_id, *args, **kwargs): email = request.POST.get("email", None) if not email: return JsonResponse( {"error": "email is required"}, status=400, ) execute_query.delay(query_id, email) return JsonResponse({"message": "message was sent successfully"}) ================================================ FILE: explorer/views/export.py ================================================ from django.db import DatabaseError from django.http import HttpResponse from explorer.exporters import get_exporter_class from explorer.utils import url_get_params def _export(request, query, download=True): _fmt = request.GET.get("format", "csv") exporter_class = get_exporter_class(_fmt) query.params = url_get_params(request) delim = request.GET.get("delim") exporter = exporter_class(query) try: output = exporter.get_output(delim=delim) except DatabaseError as e: msg = f"Error executing query {query.title}: {e}" return HttpResponse( msg, status=500 ) response = HttpResponse( output, content_type=exporter.content_type ) if download: response["Content-Disposition"] = \ f'attachment; filename="{exporter.get_filename()}"' return response ================================================ FILE: explorer/views/format_sql.py ================================================ from django.http import JsonResponse from django.views.decorators.http import require_POST from explorer.utils import fmt_sql @require_POST def format_sql(request): sql = request.POST.get("sql", "") formatted = fmt_sql(sql) return JsonResponse({"formatted": formatted}) ================================================ FILE: explorer/views/list.py ================================================ import re from collections import Counter from django.forms.models import model_to_dict from django.views.generic import ListView from explorer import app_settings from explorer.models import Query, QueryFavorite, QueryLog from explorer.utils import allowed_query_pks, url_get_query_id from explorer.views.auth import PermissionRequiredMixin from explorer.views.mixins import ExplorerContextMixin from explorer.ee.db_connections.models import DatabaseConnection class ListQueryView(PermissionRequiredMixin, ExplorerContextMixin, ListView): permission_required = "view_permission_list" model = Query def recently_viewed(self): qll = QueryLog.objects.filter( run_by_user=self.request.user, query_id__isnull=False ).order_by( "-run_at" ).select_related("query") ret = [] tracker = [] for ql in qll: if len(ret) == app_settings.EXPLORER_RECENT_QUERY_COUNT: break if ql.query_id not in tracker: ret.append(ql) tracker.append(ql.query_id) return ret def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["object_list"] = self._build_queries_and_headers() context["connection_count"] = DatabaseConnection.objects.count() context["recent_queries"] = self.recently_viewed() context["tasks_enabled"] = app_settings.ENABLE_TASKS context["vite_dev_mode"] = app_settings.VITE_DEV_MODE return context def get_queryset(self): if app_settings.EXPLORER_PERMISSION_VIEW(self.request): qs = ( Query.objects.prefetch_related( "created_by_user", "querylog_set" ).all() ) else: qs = ( Query.objects.prefetch_related( "created_by_user", "querylog_set" ).filter(pk__in=allowed_query_pks(self.request.user.id)) ) return qs def _build_queries_and_headers(self): """ Build a list of query information and headers (pseudo-folders) for consumption by the template. Strategy: Look for queries with titles of the form "something - else" (eg. with a ' - ' in the middle) and split on the ' - ', treating the left side as a "header" (or folder). Interleave the headers into the ListView's object_list as appropriate. Ignore headers that only have one child. The front end uses bootstrap's JS Collapse plugin, which necessitates generating CSS classes to map the header onto the child rows, hence the collapse_target variable. To make the return object homogeneous, convert the object_list models into dictionaries for interleaving with the header "objects". This necessitates special handling of 'created_at' and 'created_by_user' because model_to_dict doesn't include non-editable fields (created_at) and will give the int representation of the user instead of the string representation. :return: A list of model dictionaries representing all the query objects, interleaved with header dictionaries. :rtype: list """ dict_list = [] rendered_headers = [] pattern = re.compile(r"[\W_]+") headers = Counter([q.title.split(" - ")[0] for q in self.object_list]) query_favorites_for_user = QueryFavorite.objects.filter(user_id=self.request.user.pk).values_list("query_id", flat=True) for q in self.object_list: model_dict = model_to_dict(q) header = q.title.split(" - ")[0] collapse_target = pattern.sub("", header) if headers[header] > 1 and header not in rendered_headers: dict_list.append({ "title": header, "is_header": True, "is_in_category": False, "collapse_target": collapse_target, "count": headers[header] }) rendered_headers.append(header) lrl = q.last_run_log() model_dict.update({ "is_in_category": headers[header] > 1, "collapse_target": collapse_target, "created_at": q.created_at, "is_header": False, "run_count": q.querylog_set.count(), "connection_name": str(q.database_connection), "ran_successfully": lrl.success, "last_run_at": lrl.run_at, "created_by_user": str(q.created_by_user) if q.created_by_user else None, "is_favorite": q.id in query_favorites_for_user }) dict_list.append(model_dict) return dict_list class ListQueryLogView(PermissionRequiredMixin, ExplorerContextMixin, ListView): context_object_name = "recent_logs" model = QueryLog paginate_by = 20 permission_required = "view_permission" def get_queryset(self): kwargs = {"sql__isnull": False} if url_get_query_id(self.request): kwargs["query_id"] = url_get_query_id(self.request) return QueryLog.objects.filter(**kwargs).all() ================================================ FILE: explorer/views/mixins.py ================================================ from django.conf import settings from django.shortcuts import render from explorer import app_settings class ExplorerContextMixin: def gen_ctx(self): return { "can_view": app_settings.EXPLORER_PERMISSION_VIEW( self.request ), "can_change": app_settings.EXPLORER_PERMISSION_CHANGE( self.request ), "can_manage_connections": app_settings.EXPLORER_PERMISSION_CONNECTIONS( self.request ), "assistant_enabled": app_settings.has_assistant(), "db_connections_enabled": app_settings.db_connections_enabled(), "user_uploads_enabled": app_settings.user_uploads_enabled(), "csrf_cookie_name": settings.CSRF_COOKIE_NAME, "csrf_token_in_dom": settings.CSRF_COOKIE_HTTPONLY or settings.CSRF_USE_SESSIONS, "view_name": self.request.resolver_match.view_name, "hosted": app_settings.EXPLORER_HOSTED } def get_context_data(self, **kwargs): ctx = super().get_context_data(**kwargs) ctx.update(self.gen_ctx()) return ctx def render_template(self, template, ctx): ctx.update(self.gen_ctx()) return render(self.request, template, ctx) ================================================ FILE: explorer/views/query.py ================================================ from django.core.exceptions import ValidationError from django.http import HttpResponseRedirect from django.shortcuts import get_object_or_404 from django.urls import reverse_lazy from django.utils.translation import gettext_lazy as _ from django.views import View from explorer import app_settings from explorer.forms import QueryForm from explorer.models import MSG_FAILED_BLACKLIST, Query, QueryLog from explorer.utils import ( url_get_fullscreen, url_get_log_id, url_get_params, url_get_query_id, url_get_rows, url_get_show, InvalidExplorerConnectionException ) from explorer.views.auth import PermissionRequiredMixin from explorer.views.mixins import ExplorerContextMixin from explorer.views.utils import query_viewmodel from explorer.ee.db_connections.utils import default_db_connection_id class PlayQueryView(PermissionRequiredMixin, ExplorerContextMixin, View): permission_required = "change_permission" def get(self, request): if url_get_query_id(request): query = get_object_or_404(Query, pk=url_get_query_id(request)) return self.render_with_sql(request, query, run_query=False) if url_get_log_id(request): log = get_object_or_404(QueryLog, pk=url_get_log_id(request)) c = log.database_connection_id or "" query = Query(sql=log.sql, title="Playground", database_connection_id=c) return self.render_with_sql(request, query) return self.render() def post(self, request): c = request.POST.get("database_connection", default_db_connection_id()) show = url_get_show(request) sql = request.POST.get("sql", "") query = Query(sql=sql, title="Playground", database_connection_id=c) passes_blacklist, failing_words = query.passes_blacklist() error = MSG_FAILED_BLACKLIST % ", ".join( failing_words ) if not passes_blacklist else None run_query = not bool(error) if show else False return self.render_with_sql( request, query, run_query=run_query, error=error ) def render(self): return self.render_template( "explorer/play.html", { "title": "Playground", "form": QueryForm() } ) def render_with_sql(self, request, query, run_query=True, error=None): rows = url_get_rows(request) fullscreen = url_get_fullscreen(request) template = "fullscreen" if fullscreen else "play" form = QueryForm( request.POST if len(request.POST) else None, instance=query ) return self.render_template( f"explorer/{template}.html", query_viewmodel( request, query, title="Playground", run_query=run_query, error=error, rows=rows, form=form ) ) class QueryView(PermissionRequiredMixin, ExplorerContextMixin, View): permission_required = "view_permission" def get(self, request, query_id): query, form = QueryView.get_instance_and_form(request, query_id) query.save() # updates the modified date show = url_get_show(request) rows = url_get_rows(request) params = query.available_params() if not app_settings.EXPLORER_AUTORUN_QUERY_WITH_PARAMS and params: show = False try: vm = query_viewmodel( request, query, form=form, run_query=show, rows=rows ) except InvalidExplorerConnectionException as e: vm = query_viewmodel( request, query, form=form, run_query=False, error=str(e) ) fullscreen = url_get_fullscreen(request) template = "fullscreen" if fullscreen else "query" return self.render_template( f"explorer/{template}.html", vm ) def post(self, request, query_id): if not app_settings.EXPLORER_PERMISSION_CHANGE(request): return HttpResponseRedirect( reverse_lazy("query_detail", kwargs={"query_id": query_id}) ) show = url_get_show(request) query, form = QueryView.get_instance_and_form(request, query_id) success = form.is_valid() and form.save() try: vm = query_viewmodel( request, query, form=form, run_query=show, rows=url_get_rows(request), message=_("Query saved.") if success else None ) except ValidationError as ve: vm = query_viewmodel( request, query, form=form, run_query=False, rows=url_get_rows(request), error=ve.message ) return self.render_template("explorer/query.html", vm) @staticmethod def get_instance_and_form(request, query_id): query = get_object_or_404(Query.objects.prefetch_related("favorites"), pk=query_id) query.params = url_get_params(request) form = QueryForm( request.POST if len(request.POST) else None, instance=query ) return query, form ================================================ FILE: explorer/views/query_favorite.py ================================================ from django.http import JsonResponse from django.views import View from explorer.models import QueryFavorite from explorer.views.auth import PermissionRequiredMixin from explorer.views.mixins import ExplorerContextMixin class QueryFavoritesView(PermissionRequiredMixin, ExplorerContextMixin, View): permission_required = "view_permission" def get(self, request): favorites = QueryFavorite.objects.filter(user=request.user).select_related("query", "user").order_by( "query__title") return self.render_template( "explorer/query_favorites.html", {"favorites": favorites} ) class QueryFavoriteView(PermissionRequiredMixin, ExplorerContextMixin, View): permission_required = "view_permission" @staticmethod def build_favorite_response(user, query_id): is_favorite = QueryFavorite.objects.filter(user=user, query_id=query_id).exists() data = { "status": "success", "query_id": query_id, "is_favorite": is_favorite } return data def get(self, request, query_id): return JsonResponse(QueryFavoriteView.build_favorite_response(request.user, query_id)) def post(self, request, query_id): # toggle favorite if QueryFavorite.objects.filter(user=request.user, query_id=query_id).exists(): QueryFavorite.objects.filter(user=request.user, query_id=query_id).delete() else: QueryFavorite.objects.get_or_create(user=request.user, query_id=query_id) return JsonResponse(QueryFavoriteView.build_favorite_response(request.user, query_id)) ================================================ FILE: explorer/views/schema.py ================================================ from django.http import Http404, JsonResponse from django.shortcuts import render, get_object_or_404 from django.utils.decorators import method_decorator from django.views import View from django.views.decorators.clickjacking import xframe_options_sameorigin from explorer.ee.db_connections.models import DatabaseConnection from explorer.ee.db_connections.utils import default_db_connection_id from explorer.schema import schema_info, schema_json_info from explorer.views.auth import PermissionRequiredMixin class SchemaView(PermissionRequiredMixin, View): permission_required = "change_permission" @method_decorator(xframe_options_sameorigin) def dispatch(self, *args, **kwargs): return super().dispatch(*args, **kwargs) def get(self, request, *args, **kwargs): connection_id = kwargs.get("connection", default_db_connection_id()) try: connection = DatabaseConnection.objects.get(id=connection_id) except DatabaseConnection.DoesNotExist as e: raise Http404 from e except ValueError as e: raise Http404 from e schema = schema_info(connection) if schema: return render( request, "explorer/schema.html", {"schema": schema} ) else: return render(request, "explorer/schema_error.html", {"connection": connection.alias}) class SchemaJsonView(PermissionRequiredMixin, View): permission_required = "change_permission" def get(self, request, *args, **kwargs): connection = kwargs.get("connection", default_db_connection_id()) conn = get_object_or_404(DatabaseConnection, id=connection) return JsonResponse(schema_json_info(conn)) ================================================ FILE: explorer/views/stream.py ================================================ from django.shortcuts import get_object_or_404 from django.views import View from explorer.models import Query from explorer.views.auth import PermissionRequiredMixin from explorer.views.export import _export from explorer.telemetry import Stat, StatNames class StreamQueryView(PermissionRequiredMixin, View): permission_required = "view_permission" def get(self, request, query_id, *args, **kwargs): query = get_object_or_404(Query, pk=query_id) Stat(StatNames.QUERY_STREAM, { "fmt": request.GET.get("format", "csv"), }).track() return _export(request, query, download=False) ================================================ FILE: explorer/views/utils.py ================================================ from django.db import DatabaseError import logging from explorer import app_settings from explorer.charts import get_chart from explorer.models import QueryFavorite from explorer.schema import schema_json_info logger = logging.getLogger(__name__) def query_viewmodel(request, query, title=None, form=None, message=None, run_query=True, error=None, rows=app_settings.EXPLORER_DEFAULT_ROWS): """ :return: Returns the context required for a view :rtype: dict """ res = None ql = None if run_query: try: res, ql = query.execute_with_logging(request.user) except DatabaseError as e: error = str(e) has_valid_results = not error and res and run_query fullscreen_params = request.GET.copy() if "fullscreen" not in fullscreen_params: fullscreen_params.update({ "fullscreen": 1 }) if "rows" not in fullscreen_params: fullscreen_params.update({ "rows": rows }) if "querylog_id" not in fullscreen_params and ql: fullscreen_params.update({ "querylog_id": ql.id }) user = request.user is_favorite = False if user.is_authenticated and query.pk: is_favorite = QueryFavorite.objects.filter(user=user, query=query).exists() charts = {"line_chart_svg": None, "bar_chart_svg": None} try: if app_settings.EXPLORER_CHARTS_ENABLED and has_valid_results: charts["line_chart_svg"] = get_chart(res,"line", rows) charts["bar_chart_svg"] = get_chart(res,"bar", rows) except TypeError as e: if ql is not None: msg = f"Error generating charts for querylog {ql.id}: {e}" else: msg = f"Error generating charts for query {query.id}: {e}" logger.error(msg) ret = { "tasks_enabled": app_settings.ENABLE_TASKS, "params": query.available_params_w_labels(), "title": title, "shared": query.shared, "query": query, "form": form, "message": message, "error": error, "rows": rows, "query_id": query.id, "data": res.data[:rows] if has_valid_results else None, "headers": res.headers if has_valid_results else None, "total_rows": len(res.data) if has_valid_results else None, "duration": res.duration if has_valid_results else None, "has_stats": len([h for h in res.headers if h.summary]) if has_valid_results else False, "snapshots": query.snapshots if query.snapshot else [], "ql_id": ql.id if ql else None, "unsafe_rendering": app_settings.UNSAFE_RENDERING, "fullscreen_params": fullscreen_params.urlencode(), "charts_enabled": app_settings.EXPLORER_CHARTS_ENABLED, "is_favorite": is_favorite, "show_sql_by_default": app_settings.EXPLORER_SHOW_SQL_BY_DEFAULT, "schema_json": schema_json_info(query.database_connection) if query and query.database_connection else None, } return {**ret, **charts} ================================================ FILE: manage.py ================================================ #!/usr/bin/env python import os import sys from django.core import management sys.path.append(os.path.join(os.path.dirname(__file__), "explorer")) os.environ["DJANGO_SETTINGS_MODULE"] = "test_project.settings" if __name__ == "__main__": management.execute_from_command_line() ================================================ FILE: package.json ================================================ { "name": "django-sql-explorer", "description": "Django SQL Explorer", "type": "module", "scripts": { "dev": "export APP_VERSION=$(python -c 'from explorer import __version__; print(__version__)') && npx vite --config vite.config.mjs", "build": "export APP_VERSION=$(python -c 'from explorer import __version__; print(__version__)') && npx vite build --config vite.config.mjs", "preview": "export APP_VERSION=$(python -c 'from explorer import __version__; print(__version__)') && npx vite preview --config vite.config.mjs" }, "devDependencies": { "sass": "~1.69.0", "vite": "^5.4.6", "vite-plugin-copy": "^0.1.6", "vite-plugin-static-copy": "^1.0.5" }, "repository": { "type": "git", "url": "git+https://github.com/explorerhq/sql-explorer.git" }, "author": "Chris Clark", "license": "MIT", "bugs": { "url": "https://github.com/explorerhq/sql-explorer/issues" }, "homepage": "https://www.sqlexplorer.io", "dependencies": { "@codemirror/lang-sql": "^6.5.4", "@codemirror/language-data": "^6.3.1", "bootstrap": "^5.0.1", "bootstrap-icons": "^1.11.2", "choices.js": "^10.2.0", "codemirror": "^6.0.1", "cookiejs": "^2.1.3", "dompurify": "^3.1.3", "jquery": "^3.7.1", "list.js": "^2.3.1", "marked": "^11.1.1", "sortablejs": "^1.15.2" }, "optionalDependencies": { "@rollup/rollup-linux-x64-gnu": "^4.18.1" } } ================================================ FILE: public_key.pem ================================================ -----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAr1Lfhtla6gpaBlKb2r9s 5udyW6zHt0my9924sFtJ8OWzWpUFzGTDHySclWfKJkxdAS//zVhID0+i3FvkMCAe kldnU9qDvSWx40zebWZ6JlnYA/cEl2XbiXOrLyAueeYZsr0Y1ZomMbFuMee8Kmjl gj/YV+XGzlA3BBT3ajeCsBEYuSUIuGKsn7VUG6fjKX/TsOKNcGOCCXXyE9gcsrSe J4h63eJTN+IZBC78E9f+0y8VqWZ/k1lpmPFLMSwr/Z5oeS1mvadFIikRq6UWNBC7 YZxh7XSsfDSC1oaTSCgWiN+DDvK+mhBC4tpm1vJJ6lFWjS/pUrCoas0BMU5r3QJp oQIDAQAB -----END PUBLIC KEY----- ================================================ FILE: pypi-release-checklist.md ================================================ - [x] Update HISTORY - [x] Update README and check formatting with http://rst.ninjs.org/ - [x] Make sure any new files are included in MANIFEST.in - [x] Update version number in `explorer/__init__.py` - [x] Update any package dependencies in `setup.py` - [x] Commit the changes and add the tag *in master*: ``` git add . git commit -m "Release 1.0.0" git tag -a "1.0.0" git push git push --tags ``` - Be sure to test the built JS source by running `npm run build` and setting `VITE_DEV_MODE = False` in settings.py - [x] Check the PyPI listing page (https://pypi.python.org/pypi/django-sql-explorer) to make sure that the release went out and that the README is displaying properly. ================================================ FILE: requirements/base.txt ================================================ Django>=3.2 sqlparse>=0.4.0 requests>=2.2 django-cryptography-django5==2.2 cryptography>=42.0 ================================================ FILE: requirements/dev.txt ================================================ -r ./base.txt -r ./extra/assistant.txt -r ./extra/charts.txt -r ./extra/snapshots.txt -r ./extra/xls.txt -r ./extra/uploads.txt -r ./tests.txt # The Celery broker that test_project uses. Not required if not using async tasks, or if you have # a Celery config that uses a different broker. redis>=5.0 ================================================ FILE: requirements/extra/assistant.txt ================================================ openai>=1.6.1 ================================================ FILE: requirements/extra/charts.txt ================================================ matplotlib>=3.9 ================================================ FILE: requirements/extra/snapshots.txt ================================================ boto3>=1.30.0 celery>=4.0 ================================================ FILE: requirements/extra/uploads.txt ================================================ python-dateutil>=2.9 pandas>=2.2 boto3>=1.30.0 ================================================ FILE: requirements/extra/xls.txt ================================================ xlsxwriter>=1.3.6 ================================================ FILE: requirements/tests.txt ================================================ -r ./base.txt importlib-metadata<5.0; python_version <= '3.7' coverage factory-boy>=3.1.0 ================================================ FILE: ruff.toml ================================================ line-length = 120 extend-exclude = [ ".ruff_cache", ".env", ".venv", "**migrations/**", ] [lint] select = [ "E", # pycodestyle errors "W", # pycodestyle warnings "F", # pyflakes "I", # isort "C", # flake8-comprehensions "B", # flake8-bugbear "Q", # flake8-quotes "PLE", # pylint error "PLR", # pylint refactor "PLW", # pylint warning "UP", # pyupgrade ] ignore = [ "I001", # Import block is un-sorted or un-formatted (would be nice not to do this) ] [lint.per-file-ignores] "__init__.py" = [ "F401" # unused-import ] "explorer/charts.py" = [ "C419", # Unnecessary list comprehension. "PLR2004", # Magic value used in comparison, consider replacing 2 with a constant variable ] "explorer/exporters.py" = [ "PLW2901", # `for` loop variable `data` overwritten by assignment target ] "explorer/models.py" = [ "C417", # Unnecessary `map` usage (rewrite using a generator expression) ] "explorer/schema.py" = [ "C419", # Unnecessary list comprehension. ] "explorer/tests/test_utils.py" = [ "C416", # Unnecessary `list` comprehension (rewrite using `list()`) ] "explorer/views/utils.py" = [ "PLR0913", # Too many arguments in function definition (8 > 5) ] [lint.isort] combine-as-imports = true known-first-party = [ "explorer", ] extra-standard-library = ["dataclasses"] [lint.pyupgrade] # Preserve types, even if a file imports `from __future__ import annotations`. keep-runtime-typing = true [format] quote-style = "double" indent-style = "space" docstring-code-format = true docstring-code-line-length = 80 ================================================ FILE: setup.cfg ================================================ [coverage:run] branch = True parallel = True omit = explorer/__init__.py, explorer/migrations/*, explorer/tests/*, test_project/*, */setup.py */manage.py test_project/* source = explorer [coverage:paths] source = explorer .tox/*/site-packages [coverage:report] show_missing = True [flake8] max-line-length = 119 exclude = *.egg-info, .eggs, .git, .settings, .tox, .venv, build, data, dist, docs, *migrations*, requirements, tmp [isort] line_length = 119 skip = manage.py, *migrations*, .tox, .eggs, data, .env, .venv include_trailing_comma = true multi_line_output = 5 lines_after_imports = 2 default_section = THIRDPARTY sections = FUTURE, STDLIB, DJANGO, THIRDPARTY, FIRSTPARTY, LOCALFOLDER known_first_party = explorer known_django = django ================================================ FILE: setup.py ================================================ import os import sys from pathlib import Path from setuptools import setup try: from sphinx.setup_command import BuildDoc except ImportError: BuildDoc = None from explorer import get_version name = "django-sql-explorer" version = get_version() release = get_version(True) def requirements(fname): path = os.path.join(os.path.dirname(__file__), "requirements", fname) with open(path) as f: return f.read().splitlines() if sys.argv[-1] == "build": os.system("python setup.py sdist bdist_wheel") print(f"Built release {release} (version {version})") sys.exit() if sys.argv[-1] == "release": os.system("twine upload --skip-existing dist/*") sys.exit() if sys.argv[-1] == "tag": print("Tagging the version:") os.system(f"git tag -a {version} -m 'version {version}'") os.system("git push --tags") sys.exit() this_directory = Path(__file__).parent long_description = (this_directory / "README.rst").read_text() setup( name=name, version=version, author="Chris Clark", author_email="chris@sqlexplorer.io", maintainer="Chris Clark", maintainer_email="chris@sqlexplorer.io", description=("SQL Reporting that Just Works. Fast, simple, and confusion-free." "Write and share queries in a delightful SQL editor, with AI assistance"), license="MIT", keywords="django sql explorer reports reporting csv json database query", url="https://www.sqlexplorer.io", project_urls={ "Changes": "https://django-sql-explorer.readthedocs.io/en/latest/history.html", "Documentation": "https://django-sql-explorer.readthedocs.io/en/latest/", "Issues": "https://github.com/explorerhq/sql-explorer/issues" }, packages=["explorer"], long_description=long_description, long_description_content_type="text/x-rst", classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Topic :: Utilities", "Framework :: Django :: 3.2", "Framework :: Django :: 4.2", "Framework :: Django :: 5.0", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3 :: Only", ], python_requires=">=3.8", install_requires=[ requirements("base.txt"), ], extras_require={ "charts": requirements("extra/charts.txt"), "snapshots": requirements("extra/snapshots.txt"), "xls": requirements("extra/xls.txt"), "assistant": requirements("extra/assistant.txt"), "uploads": requirements("extra/uploads.txt"), }, cmdclass={ "build_sphinx": BuildDoc, }, command_options={ "build_sphinx": { "project": ("setup.py", name), "version": ("setup.py", version), "release": ("setup.py", release), "source_dir": ("setup.py", "docs"), "build_dir": ("setup.py", "./docs/_build") } }, include_package_data=True, zip_safe=False, ) ================================================ FILE: test_project/__init__.py ================================================ try: from .celery_config import app as celery_app __all__ = ["celery_app"] except ImportError: pass ================================================ FILE: test_project/celery_config.py ================================================ import os from celery import Celery from celery.schedules import crontab # Set the default Django settings module for the "celery" program. os.environ.setdefault("DJANGO_SETTINGS_MODULE", "test_project.settings") app = Celery("test_project") # Using a string here means the worker doesn"t have to serialize # the configuration object to child processes. # - namespace="CELERY" means all celery-related configuration keys # should have a `CELERY_` prefix. app.config_from_object("django.conf:settings", namespace="CELERY") # Load task modules from all registered Django apps. app.autodiscover_tasks() app.conf.beat_schedule = { "explorer.tasks.snapshot_queries": { "task": "explorer.tasks.snapshot_queries", "schedule": crontab(hour="1", minute="0") }, "explorer.tasks.truncate_querylogs": { "task": "explorer.tasks.truncate_querylogs", "schedule": crontab(hour="1", minute="10"), "kwargs": {"days": 30} }, "explorer.tasks.remove_unused_sqlite_dbs": { "task": "explorer.tasks.remove_unused_sqlite_dbs", "schedule": crontab(hour="1", minute="20") }, "explorer.tasks.build_async_schemas": { "task": "explorer.tasks.build_async_schemas", "schedule": crontab(hour="1", minute="30") } } ================================================ FILE: test_project/settings.py ================================================ import os USE_TZ = True SECRET_KEY = "shhh" DEBUG = True STATIC_URL = "/static/" VITE_DEV_MODE = True ALLOWED_HOSTS = ["0.0.0.0", "localhost", "127.0.0.1"] BASE_DIR = os.path.dirname(os.path.dirname(__file__)) DATABASES = { "default": { "ENGINE": "django.db.backends.sqlite3", "NAME": "tmp", "TEST": { "NAME": "tmp" } } } EXPLORER_CONNECTIONS = { "Primary": "default", } EXPLORER_DEFAULT_CONNECTION = "default" ROOT_URLCONF = "test_project.urls" PROJECT_PATH = os.path.realpath(os.path.dirname(__file__)) TEMPLATES = [ { "BACKEND": "django.template.backends.django.DjangoTemplates", "DIRS": [], "APP_DIRS": True, "OPTIONS": { "context_processors": [ "django.contrib.auth.context_processors.auth", "django.contrib.messages.context_processors.messages", "django.template.context_processors.static", "django.template.context_processors.request", ], "debug": DEBUG }, }, ] INSTALLED_APPS = ( "django.contrib.auth", "django.contrib.contenttypes", "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", "django.contrib.admin", "explorer", ) STATICFILES_FINDERS = ( "django.contrib.staticfiles.finders.FileSystemFinder", "django.contrib.staticfiles.finders.AppDirectoriesFinder", ) STORAGES = { "default": { "BACKEND": "django.core.files.storage.FileSystemStorage", }, "staticfiles": { "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", }, } AUTHENTICATION_BACKENDS = ( "django.contrib.auth.backends.ModelBackend", ) MIDDLEWARE = [ "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", ] # added to help debug tasks EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" # Explorer-specific EXPLORER_TRANSFORMS = ( ("foo", '{0}'), ("bar", "x: {0}") ) EXPLORER_USER_QUERY_VIEWS = {} # Tasks disabled by default, but if you have celery installed # make sure the broker URL is set correctly EXPLORER_TASKS_ENABLED = False CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL") EXPLORER_S3_BUCKET = os.environ.get("EXPLORER_S3_BUCKET") EXPLORER_S3_ACCESS_KEY = os.environ.get("EXPLORER_S3_ACCESS_KEY") EXPLORER_S3_SECRET_KEY = os.environ.get("EXPLORER_S3_SECRET_KEY") EXPLORER_AI_API_KEY = os.environ.get("AI_API_KEY") EXPLORER_ASSISTANT_BASE_URL = os.environ.get("AI_BASE_URL") EXPLORER_DB_CONNECTIONS_ENABLED = True EXPLORER_USER_UPLOADS_ENABLED = True EXPLORER_CHARTS_ENABLED = True EXPLORER_ASSISTANT_MODEL_NAME = "anthropic/claude-3.5-sonnet" ================================================ FILE: test_project/urls.py ================================================ from django.contrib import admin from django.contrib.staticfiles.urls import staticfiles_urlpatterns from django.urls import path, include # Installing to /explorer/ better mimics likely production setups # Explorer is probably *not* running at the Django project root urlpatterns = [ path("explorer/", include("explorer.urls")) ] admin.autodiscover() urlpatterns += [ path("admin/", admin.site.urls), ] urlpatterns += staticfiles_urlpatterns() ================================================ FILE: tox.ini ================================================ [tox] envlist = flake8 isort {base-reqs,dev}-py{310,311,312}-dj{32,42,50} {base-reqs,dev}-py{311,312}-dj{50,main} skip_missing_interpreters=True [testenv] allowlist_externals = coverage deps = base-reqs: -r requirements/tests.txt dj32: django>=3.2,<4.0 dj42: django>=4.2,<5.0 dj50: django>=5.0,<5.1 djmain: https://github.com/django/django/archive/main.tar.gz dev: -r requirements/dev.txt commands = {envpython} --version base-reqs: coverage run manage.py test --settings=explorer.tests.settings_base --noinput dev: coverage run manage.py test --settings=explorer.tests.settings --noinput ignore_outcome = djmain: True ignore_errors = djmain: True [testenv:flake8] deps = flake8 commands = flake8 [testenv:isort] deps = isort commands = isort --check --diff explorer skip_install = true ================================================ FILE: vite.config.mjs ================================================ import { resolve } from 'path'; import { defineConfig } from 'vite'; import { viteStaticCopy } from 'vite-plugin-static-copy'; export default defineConfig({ plugins: [ viteStaticCopy({ targets: [ { src: 'explorer/src/images/*', dest: 'images' }, ] }) ], root: resolve(__dirname, './'), base: '', server: { host: true, port: 5173, strictPort: true, open: false, watch: { usePolling: true, disableGlobbing: false, }, }, resolve: { extensions: ['.js', '.json'], alias: { '~bootstrap': resolve(__dirname, './node_modules/bootstrap'), '~bootstrap-icons': resolve(__dirname, './node_modules/bootstrap-icons'), }, }, build: { outDir: resolve(__dirname, './explorer/static/explorer'), assetsDir: '', emptyOutDir: true, target: 'es2015', rollupOptions: { input: { main: resolve(__dirname, './explorer/src/js/main.js'), // Some magic here; Vite always builds to styles.css, we named our entrypoint SCSS file the same thing // so that in the base template HTML file we can include 'styles.scss', and rename just the extension // in the vite template tag, and get both the dev and prod builds to work. styles: resolve(__dirname, '/explorer/src/scss/styles.scss'), }, output: { entryFileNames: `[name].${process.env.APP_VERSION}.js`, chunkFileNames: `[name].${process.env.APP_VERSION}.js`, assetFileNames: `[name].[ext]` }, }, }, });