Repository: coleifer/peewee Branch: master Commit: 3abb8c8aaa9d Files: 167 Total size: 2.7 MB Directory structure: gitextract_2t_sjv5p/ ├── .github/ │ └── workflows/ │ ├── tests.yaml │ └── wheels.yaml ├── .gitignore ├── .readthedocs.yaml ├── .travis.yml ├── .travis_deps.sh ├── CHANGELOG.md ├── LICENSE ├── MANIFEST.in ├── README.rst ├── bench.py ├── docs/ │ ├── Makefile │ ├── _themes/ │ │ └── flask/ │ │ ├── layout.html │ │ ├── relations.html │ │ ├── static/ │ │ │ ├── flasky.css_t │ │ │ └── small_flask.css │ │ └── theme.conf │ ├── clubdata.sql │ ├── conf.py │ ├── index.rst │ ├── make.bat │ ├── peewee/ │ │ ├── api.rst │ │ ├── asyncio.rst │ │ ├── contributing.rst │ │ ├── database.rst │ │ ├── db_tools.rst │ │ ├── example.rst │ │ ├── framework_integration.rst │ │ ├── installation.rst │ │ ├── interactive.rst │ │ ├── models.rst │ │ ├── mysql.rst │ │ ├── orm_utils.rst │ │ ├── pool-snippet.rst │ │ ├── postgres.rst │ │ ├── query_builder.rst │ │ ├── query_library.rst │ │ ├── query_operators.rst │ │ ├── querying.rst │ │ ├── quickstart.rst │ │ ├── recipes.rst │ │ ├── relationships.rst │ │ ├── schema.rst │ │ ├── sqlite.rst │ │ ├── transactions.rst │ │ └── writing.rst │ └── requirements.txt ├── examples/ │ ├── adjacency_list.py │ ├── analytics/ │ │ ├── app.py │ │ ├── reports.py │ │ ├── requirements.txt │ │ └── run_example.py │ ├── anomaly_detection.py │ ├── blog/ │ │ ├── app.py │ │ ├── requirements.txt │ │ ├── static/ │ │ │ ├── css/ │ │ │ │ └── hilite.css │ │ │ └── robots.txt │ │ └── templates/ │ │ ├── base.html │ │ ├── create.html │ │ ├── detail.html │ │ ├── edit.html │ │ ├── includes/ │ │ │ └── pagination.html │ │ ├── index.html │ │ ├── login.html │ │ └── logout.html │ ├── diary.py │ ├── graph.py │ ├── hexastore.py │ ├── query_library.py │ ├── reddit_ranking.py │ ├── sqlite_fts_compression.py │ └── twitter/ │ ├── app.py │ ├── requirements.txt │ ├── run_example.py │ ├── static/ │ │ └── style.css │ └── templates/ │ ├── create.html │ ├── homepage.html │ ├── includes/ │ │ ├── message.html │ │ └── pagination.html │ ├── join.html │ ├── layout.html │ ├── login.html │ ├── private_messages.html │ ├── public_messages.html │ ├── user_detail.html │ ├── user_followers.html │ ├── user_following.html │ └── user_list.html ├── peewee.py ├── playhouse/ │ ├── README.md │ ├── __init__.py │ ├── _sqlite_udf.pyx │ ├── apsw_ext.py │ ├── cockroachdb.py │ ├── cysqlite_ext.py │ ├── dataset.py │ ├── db_url.py │ ├── fields.py │ ├── flask_utils.py │ ├── hybrid.py │ ├── kv.py │ ├── migrate.py │ ├── mysql_ext.py │ ├── pool.py │ ├── postgres_ext.py │ ├── pwasyncio.py │ ├── pydantic_utils.py │ ├── reflection.py │ ├── shortcuts.py │ ├── signals.py │ ├── sqlcipher_ext.py │ ├── sqlite_changelog.py │ ├── sqlite_ext.py │ ├── sqlite_udf.py │ ├── sqliteq.py │ └── test_utils.py ├── pwiz.py ├── pyproject.toml ├── runtests.py ├── setup.py └── tests/ ├── __init__.py ├── __main__.py ├── apsw_ext.py ├── base.py ├── base_models.py ├── cockroachdb.py ├── cysqlite_ext.py ├── dataset.py ├── db_tests.py ├── db_url.py ├── expressions.py ├── extra_fields.py ├── fields.py ├── hybrid.py ├── keys.py ├── kv.py ├── manytomany.py ├── migrations.py ├── model_save.py ├── model_sql.py ├── models.py ├── mysql_ext.py ├── pool.py ├── postgres.py ├── postgres_helpers.py ├── prefetch_tests.py ├── pwasyncio.py ├── pwasyncio_stress.py ├── pwiz_integration.py ├── pydantic_utils.py ├── queries.py ├── reflection.py ├── regressions.py ├── results.py ├── returning.py ├── schema.py ├── shortcuts.py ├── signals.py ├── sql.py ├── sqlcipher_ext.py ├── sqlite.py ├── sqlite_changelog.py ├── sqlite_helpers.py ├── sqlite_udf.py ├── sqliteq.py ├── test_utils.py └── transactions.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/workflows/tests.yaml ================================================ name: Tests on: [push] jobs: tests: name: ${{ matrix.peewee-backend }} - ${{ matrix.python-version }} runs-on: ubuntu-latest timeout-minutes: 15 services: mysql: image: mariadb:latest env: MYSQL_ROOT_PASSWORD: peewee MYSQL_DATABASE: peewee_test ports: - 3306:3306 postgres: image: postgres env: POSTGRES_USER: postgres POSTGRES_PASSWORD: peewee POSTGRES_DB: peewee_test ports: - 5432:5432 strategy: fail-fast: false matrix: python-version: [3.9, "3.11", "3.13", "3.14"] peewee-backend: - "sqlite" - "postgresql" - "mysql" include: - python-version: "3.8" peewee-backend: sqlite - python-version: "3.12" peewee-backend: sqlite - python-version: "3.12" peewee-backend: psycopg3 - python-version: "3.13" peewee-backend: psycopg3 - python-version: "3.14" peewee-backend: psycopg3 - python-version: "3.13" peewee-backend: cysqlite - python-version: "3.13" peewee-backend: cockroachdb steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: deps env: PGUSER: postgres PGHOST: 127.0.0.1 PGPASSWORD: peewee run: | sudo apt-get install libsqlite3-dev pip install setuptools psycopg2-binary cython pymysql 'apsw' mysql-connector sqlcipher3 pysqlite3 'psycopg[binary]' gevent pip install greenlet aiosqlite aiomysql asyncpg pydantic python setup.py build_ext -i psql peewee_test -c 'CREATE EXTENSION hstore;' - name: crdb if: ${{ matrix.peewee-backend == 'cockroachdb' }} run: | wget -qO- https://binaries.cockroachdb.com/cockroach-v22.2.6.linux-amd64.tgz | tar xz ./cockroach-v22.2.6.linux-amd64/cockroach start-single-node --insecure --background ./cockroach-v22.2.6.linux-amd64/cockroach sql --insecure -e 'create database peewee_test;' - name: cysqlite if: ${{ matrix.peewee-backend == 'cysqlite' || matrix.peewee-backend == 'sqlite' }} run: | pip install -e 'git+https://github.com/coleifer/cysqlite#egg=cysqlite' - name: runtests ${{ matrix.peewee-backend }} - ${{ matrix.python-version }} env: PEEWEE_TEST_BACKEND: ${{ matrix.peewee-backend }} PGUSER: postgres PGHOST: 127.0.0.1 PGPASSWORD: peewee CI: 1 run: python runtests.py --mysql-user=root --mysql-password=peewee -s -v2 ================================================ FILE: .github/workflows/wheels.yaml ================================================ name: Build wheels on: push: tags: - "[0-9]+.[0-9]+.[0-9]+" - "[0-9]+.[0-9]+.[0-9]+-**" jobs: build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: "3.12" - name: Install build tools run: | python -m pip install -U pip pip install setuptools build - name: Build sdist and wheel env: NO_SQLITE: 1 run: | python -m build . - uses: actions/upload-artifact@v4 with: name: package path: dist/peewee* publish: needs: [build] runs-on: ubuntu-latest steps: - uses: actions/download-artifact@v4 with: name: package path: dist merge-multiple: true - uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} ================================================ FILE: .gitignore ================================================ *.pyc build prof/ docs/_build/ playhouse/*.c playhouse/*.h playhouse/*.so playhouse/tests/peewee_test.db .idea/ MANIFEST peewee_test.db closure.so lsm.so regexp.so ================================================ FILE: .readthedocs.yaml ================================================ version: 2 python: install: - requirements: docs/requirements.txt build: os: ubuntu-22.04 tools: python: "3.11" sphinx: configuration: docs/conf.py ================================================ FILE: .travis.yml ================================================ language: python python: - 2.7 - 3.4 - 3.5 - 3.6 env: - PEEWEE_TEST_BACKEND=sqlite - PEEWEE_TEST_BACKEND=postgresql - PEEWEE_TEST_BACKEND=mysql matrix: include: - python: 3.7 dist: xenial env: PEEWEE_TEST_BACKEND=sqlite - python: 3.7 dist: xenial env: PEEWEE_TEST_BACKEND=postgresql - python: 3.7 dist: xenial env: PEEWEE_TEST_BACKEND=mysql - python: 3.8 dist: xenial - python: 3.7 dist: xenial env: - PEEWEE_TEST_BUILD_SQLITE=1 - PEEWEE_CLOSURE_EXTENSION=/usr/local/lib/closure.so - LSM_EXTENSION=/usr/local/lib/lsm.so before_install: - sudo apt-get install -y tcl-dev - ./.travis_deps.sh - sudo ldconfig script: "python runtests.py -v2" - python: 3.7 dist: xenial env: - PEEWEE_TEST_BACKEND=cockroachdb before_install: - wget -qO- https://binaries.cockroachdb.com/cockroach-v20.1.1.linux-amd64.tgz | tar xvz - ./cockroach-v20.1.1.linux-amd64/cockroach start --insecure --background - ./cockroach-v20.1.1.linux-amd64/cockroach sql --insecure -e 'create database peewee_test;' allow_failures: addons: postgresql: "9.6" mariadb: "10.3" services: - postgresql - mariadb install: "pip install psycopg2-binary Cython pymysql apsw mysql-connector" before_script: - python setup.py build_ext -i - psql -c 'drop database if exists peewee_test;' -U postgres - psql -c 'create database peewee_test;' -U postgres - psql peewee_test -c 'create extension hstore;' -U postgres - mysql -e 'drop user if exists travis@localhost;' - mysql -e 'create user travis@localhost;' - mysql -e 'drop database if exists peewee_test;' - mysql -e 'create database peewee_test;' - mysql -e 'grant all on *.* to travis@localhost;' || true script: "python runtests.py" ================================================ FILE: .travis_deps.sh ================================================ #!/bin/bash setup_sqlite_deps() { wget https://www.sqlite.org/src/tarball/sqlite.tar.gz tar xzf sqlite.tar.gz cd sqlite/ export CFLAGS="-DSQLITE_ENABLE_FTS3 \ -DSQLITE_ENABLE_FTS3_PARENTHESIS \ -DSQLITE_ENABLE_FTS4 \ -DSQLITE_ENABLE_FTS5 \ -DSQLITE_ENABLE_JSON1 \ -DSQLITE_ENABLE_LOAD_EXTENSION \ -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT \ -DSQLITE_TEMP_STORE=3 \ -DSQLITE_USE_URI \ -O2 \ -fPIC" export PREFIX="/usr/local" LIBS="-lm" ./configure \ --disable-tcl \ --enable-shared \ --enable-tempstore=always \ --prefix="$PREFIX" make && sudo make install cd ext/misc/ # Build the transitive closure extension and copy shared library. gcc -fPIC -O2 -lsqlite3 -shared closure.c -o closure.so sudo cp closure.so /usr/local/lib # Build the lsm1 extension and copy shared library. cd ../lsm1 export CFLAGS="-fPIC -O2" TCCX="gcc -fPIC -O2" make lsm.so sudo cp lsm.so /usr/local/lib } if [ -n "$PEEWEE_TEST_BUILD_SQLITE" ]; then setup_sqlite_deps fi ================================================ FILE: CHANGELOG.md ================================================ # Changelog Tracking changes in peewee between versions. For a complete view of all the releases, visit GitHub: https://github.com/coleifer/peewee/releases ## master [View commits](https://github.com/coleifer/peewee/compare/4.0.2...master) ## 4.0.2 * Remove all Python 2.x compatibility code. * Add streaming result cursors to pwasyncio module via `db.iterate(query)`. * Better serialization and deserialization of datetimes and binary data in the DataSet module. Previously binary data was encoded as base64, going forward hex is the new default. For base64 specify `base64_bytes=True`. * Improvements to Postgres `BinaryJSONField`, support atomic removal of sub-elements, as well as alternate helper for extracting sub-elements and querying array length. * [Pydantic integration](https://docs.peewee-orm.com/en/latest/peewee/orm_utils.html#module-playhouse.pydantic_utils) [View commits](https://github.com/coleifer/peewee/compare/4.0.1...4.0.2) ## 4.0.1 * Ensure `gr_context` is set on greenlet in `greenlet_spawn` so that contextvars will be operable in sync handlers. * Removed `SqliteExtDatabase` (it basically served no purpose in 4.0). Use `SqliteDatabase` instead. * Moved driver and extension-specific pooled implementations into the corresponding extension module rather than putting all into `playhouse.pool`. * Restore custom `dumps` option for postgres JSON fields. * Major docs rewrite / reorganization. [View commits](https://github.com/coleifer/peewee/compare/4.0.0...4.0.1) ## 4.0.0 * Adds preliminary support for `asyncio` via a new playhouse extension. See [the documentation](http://docs.peewee-orm.com/en/latest/peewee/asyncio.html) for details. * `PostgresqlDatabase` can use `psycopg` (psycopg3) if it is installed. If both psycopg2 and psycopg3 are installed, Peewee will prefer psycopg2, but this can be controlled by specifying `prefer_psycopg3=True` in the constructor. Same applies to `PostgresqlExtDatabase`. * `Psycopg3Database` class has been moved to `playhouse.postgres_ext` and is now just a thin wrapper around `PostgresqlExtDatabase`. * Postgres JSON operations no longer dump and try to do minimal casts, instead relying on the driver-provided `Json()` wrapper(s). * Adds new `ISODateTimeField` for Sqlite that encodes datetimes in ISO format (more friendly when db is shared with other tools), and also properly reads back UTC offset info. * Remove `playhouse.sqlite_ext.ClosureTable` implementation. * Add a `Model.dirty_field_names` attribute that is safe for membership testing, since testing `x in dirty_fields` returns True if one or more field exists due to operator overloads returning a truthy Expression object. Refs #3028. * Removal of Cython `_sqlite_ext` extension. The C implementations of the FTS rank functions are moved to `sqlite_udf`. Most of the remaining functionality is moved to `playhouse.cysqlite_ext` which supports it natively. Migrating `CSqliteExtDatabase` usage: You can either use `sqlite_ext.SqliteExtDatabase` or try the new `cysqlite_ext.CySqliteDatabase` if you want all the old functionality and are willing to try a new driver. [View commits](https://github.com/coleifer/peewee/compare/3.19.0...4.0.0) ## 3.19.0 * Move to new build system using pyproject and github actions. * No longer build and ship the Sqlite C extensions by default. Users who prefer to use those can install via the sdist `pip install peewee --no-binary :all:`. Rationale about the Sqlite C extensions -- I've started shipping pysqlite3 as a statically-linked, self-contained binary wheel. This means that when using Peewee with the statically-linked pysqlite3, you can end up in a funny situation where the peewee Sqlite extensions are linked against the system libsqlite3, and the pysqlite driver has it's own Sqlite embedded, which does not work. If you are using the system/standard-lib sqlite3 module then the extension works properly, because everything is talking to the same `libsqlite3`. Similarly if you built pysqlite3 to link against the system `libsqlite3` everything also works correctly, though this is not "wheel-friendly". So in order to use the C extensions, you can install Peewee from the sdist and do either of the following: ``` # Use system sqlite and standard-lib `sqlite3` module. $ pip install peewee --no-binary :all: # OR, # Use pysqlite3 linked against the system sqlite. $ pip install pysqlite3 peewee --no-binary :all: ``` I don't believe, besides myself, there were many people using these extensions so hopefully this change is not disruptive! Please let me hear about it if I'm mistaken. Other small changes: * When exporting / "freezing" binary data with the `playhouse.dataset` JSON serializer, encode binary data as base64. ## 3.18.3 * Fix potential regex DoS vulnerability in FTS5 query validation code (#3005). [View commits](https://github.com/coleifer/peewee/compare/3.18.2...3.18.3) ## 3.18.2 Cython 3.1 removes some Python 2 stuff we referenced -- this resolves the issue. Couple other very minor fixes. [View commits](https://github.com/coleifer/peewee/compare/3.18.1...3.18.2) ## 3.18.1 @pypa is such a bunch of clowns. I swear. ![](https://media.charlesleifer.com/blog/photos/p1746027512.1307786.gif) [View commits](https://github.com/coleifer/peewee/compare/3.18.0...3.18.1) ## 3.18.0 The behavior of `postgresql_ext.BinaryJSONField.contains()` has changed. Previously, passing a string to this method would perform a JSON key exists check (`?` operator) instead of JSON contains (`@>` operator). As of 3.18.0, this special-case has been **removed** and the `contains()` method always uses the JSONB contains operator (`@>`). For the **old** behavior of checking whether a key exists, use the `BinaryJSONField.has_key()` method. See #2984 for discussion. * Add options to URL-unquote user and password when using the `db_url` helpers, see #2974 for discussion. * Support using `postgresql://` URLs when connecting to psycopg3. [View commits](https://github.com/coleifer/peewee/compare/3.17.9...3.18.0) ## 3.17.9 * Fix incorrect handling of fk constraint name in migrator. * Fix test-only issue that can occur in Python 3.14a4. [View commits](https://github.com/coleifer/peewee/compare/3.17.8...3.17.9) ## 3.17.8 * Fix regression in behavior of `delete_instance()` when traversing nullable foreign-keys, #2952. Introduced in 3.17.6. **Recommended that you update**. * Fix bug where joins not cloned when going from join-less -> joined query, refs #2941. ## 3.17.7 * Add db_url support for psycopg3 via `psycopg3://`. * Ensure double-quotes are escaped properly when introspecting constraints. * A few documentation-related fixes. [View commits](https://github.com/coleifer/peewee/compare/3.17.6...3.17.7) ## 3.17.6 * Fix bug in recursive `model.delete_instance()` when a table contains foreign-keys at multiple depths of the graph, #2893. * Fix regression in pool behavior on systems where `time.time()` returns identical values for two connections. This adds a no-op comparable sentinel to the heap to prevent any recurrence of this problem, #2901. * Ensure that subqueries inside `CASE` statements generate correct SQL. * Fix regression that broke server-side cursors with Postgres (introduced in 3.16.0). * Fix to ensure compatibility with psycopg3 - the libpq TransactionStatus constants are no longer available on the `Connection` instance. * Fix quoting issue in pwiz that could generate invalid python code for double-quoted string literals used as column defaults. [View commits](https://github.com/coleifer/peewee/compare/3.17.5...3.17.6) ## 3.17.5 This release fixes a build system problem in Python 3.12, #2891. [View commits](https://github.com/coleifer/peewee/compare/3.17.4...3.17.5) ## 3.17.4 * Fix bug that could occur when using CASE inside a function, and one or more of the CASE clauses consisted of a subquery. Refs #2873. new fix in #2872 for regression in truthiness of cursor. * Fix bug in the conversion of TIMESTAMP type in Sqlite on Python 3.12+. * Fix for hybrid properties on subclasses when aliased (#2888). * Many fixes for SqliteQueueDatabase (#2874, #2876, #2877). [View commits](https://github.com/coleifer/peewee/compare/3.17.3...3.17.4) ## 3.17.3 * Better fix for #2871 (extraneous queries when coercing query to list), and [View commits](https://github.com/coleifer/peewee/compare/3.17.2...3.17.3) ## 3.17.2 * Full support for `psycopg3`. * Basic support for Sqlite `jsonb`. * Fix bug where calling `list(query)` resulted in extra queries, #2871 [View commits](https://github.com/coleifer/peewee/compare/3.17.1...3.17.2) ## 3.17.1 * Add bitwise and other helper methods to `BigBitField`, #2802. * Add `add_column_default` and `drop_column_default` migrator methods for specifying a server-side default value, #2803. * The new `star` attribute was causing issues for users who had a field named star on their models. This attribute is now renamed to `__star__`. #2796. * Fix compatibility issues with 3.12 related to utcnow() deprecation. * Add stricter locking on connection pool to prevent race conditions. * Add adapters and converters to Sqlite to replace ones deprecated in 3.12. * Fix bug in `model_to_dict()` when only aliases are present. * Fix version check for Sqlite native drop column support. * Do not specify a `reconnect=` argument to `ping()` if using MySQL 8.x. [View commits](https://github.com/coleifer/peewee/compare/3.17.0...3.17.1) ## 3.17.0 * Only roll-back in the outermost `@db.transaction` decorator/ctx manager if an unhandled exception occurs. Previously, an unhandled exception that occurred in a nested `transaction` context would trigger a rollback. The use of nested `transaction` has long been discouraged in the documentation: the recommendation is to always use `db.atomic`, which will use savepoints to properly handle nested blocks. However, the new behavior should make it easier to reason about transaction boundaries - see #2767 for discussion. * Cover transaction `BEGIN` in the reconnect-mixin. Given that no transaction has been started, reconnecting when beginning a new transaction ensures that a reconnect will occur if it is safe to do so. * Add support for setting `isolation_level` in `db.atomic()` and `db.transaction()` when using Postgres and MySQL/MariaDB, which will apply to the wrapped transaction. Note: Sqlite has supported a similar `lock_type` parameter for some time. * Add support for the Sqlite `SQLITE_DETERMINISTIC` function flag. This allows user-defined Sqlite functions to be used in indexes and may be used by the query planner. * Fix unreported bug in dataset import when inferred field name differs from column name. [View commits](https://github.com/coleifer/peewee/compare/3.16.3...3.17.0) ## 3.16.3 * Support for Cython 3.0. * Add flag to `ManyToManyField` to prevent setting/getting values on unsaved instances. This is worthwhile, since reading or writing a many-to-many has no meaning when the instance is unsaved. * Adds a `star()` helper to `Source` base-class for selecting all columns. * Fix missing `binary` types for mysql-connector and mariadb-connector. * Add `extract()` method to MySQL `JSONField` for extracting a jsonpath. [View commits](https://github.com/coleifer/peewee/compare/3.16.2...3.16.3) ## 3.16.2 Fixes a longstanding issue with thread-safety of various decorators, including `atomic()`, `transaction()`, `savepoint()`. The context-managers are unaffected. See #2709 for details. [View commits](https://github.com/coleifer/peewee/compare/3.16.1...3.16.2) ## 3.16.1 * Add changes required for building against Cython 3.0 and set Cython language-level to 3. * Ensure indexes aren't added to unindexed fields during introspection, #2691. * Ensure we don't redundantly select same PK in prefetch when using PREFETCH_TYPE.JOIN. * In Sqlite migrator, use Sqlite's builtin DROP and RENAME column facilities when possible. This can be overridden by passing `legacy=True` flag. [View commits](https://github.com/coleifer/peewee/compare/3.16.0...3.16.1) ## 3.16.0 This release contains backwards-incompatible changes in the way Peewee initializes connections to the underlying database driver. Previously, peewee implemented autocommit semantics *on-top* of the existing DB-API transactional workflow. Going forward, Peewee instead places the DB-API driver into autocommit mode directly. Why this change? Previously, Peewee emulated autocommit behavior for top-level queries issued outside of a transaction. This necessitated a number of checks which had to be performed each time a query was executed, so as to ensure that we didn't end up with uncommitted writes or, conversely, idle read transactions. By running the underlying driver in autocommit mode, we can eliminate all these checks, since we are already managing transactions ourselves. Behaviorally, there should be no change -- Peewee will still treat top-level queries outside of transactions as being autocommitted, while queries inside of `atomic()` / `with db:` blocks are implicitly committed at the end of the block, or rolled-back if an exception occurs. **How might this affect me?** * If you are using the underlying database connection or cursors, e.g. via `Database.connection()` or `Database.cursor()`, your queries will now be executed in autocommit mode. * The `commit=` argument is deprecated for the `cursor()`, `execute()` and `execute_sql()` methods. * If you have a custom `Database` implementation (whether for a database that is not officially supported, or for the purpose of overriding default behaviors), you will want to ensure that your connections are opened in autocommit mode. Other changes: * Some fixes to help with packaging in Python 3.11. * MySQL `get_columns()` implementation now returns columns in their declared order. [View commits](https://github.com/coleifer/peewee/compare/3.15.4...3.16.0) ## 3.15.4 * Raise an exception in `ReconnectMixin` if connection is lost while inside a transaction (if the transaction was interrupted presumably some changes were lost and explicit intervention is needed). * Add `db.Model` property to reduce boilerplate. * Add support for running `prefetch()` queries with joins instead of subqueries (this helps overcome a MySQL limitation about applying LIMITs to a subquery). * Add SQL `AVG` to whitelist to avoid coercing by default. * Allow arbitrary keywords in metaclass constructor, #2627 * Add a `pyproject.toml` to silence warnings from newer pips when `wheel` package is not available. This release has a small helper for reducing boilerplate in some cases by exposing a base model class as an attribute of the database instance. ```python # old: db = SqliteDatabase('...') class BaseModel(Model): class Meta: database = db class MyModel(BaseModel): pass # new: db = SqliteDatabase('...') class MyModel(db.Model): pass ``` [View commits](https://github.com/coleifer/peewee/compare/3.15.3...3.15.4) ## 3.15.3 * Add `scalars()` query method (complements `scalar()`), roughly equivalent to writing `[t[0] for t in query.tuples()]`. * Small doc improvements * Fix and remove some flaky test assertions with Sqlite INSERT + RETURNING. * Fix innocuous failing Sqlite test on big-endian machines. [View commits](https://github.com/coleifer/peewee/compare/3.15.2...3.15.3) ## 3.15.2 * Fix bug where field-specific conversions were being applied to the pattern used for LIKE / ILIKE operations. Refs #2609 * Fix possible infinite loop when accidentally invoking the `__iter__` method on certain `Column` subclasses. Refs #2606 * Add new helper for specifying which Model a particular selected column-like should be bound to, in queries with joins that select from multiple sources. [View commits](https://github.com/coleifer/peewee/compare/3.15.1...3.15.2) ## 3.15.1 * Fix issue introduced in Sqlite 3.39.0 regarding the propagation of column subtypes in subqueries. * Fix bug where cockroachdb server version was not set when beginning a transaction on an unopened database. [View commits](https://github.com/coleifer/peewee/compare/3.15.0...3.15.1) ## 3.15.0 Rollback behavior change in commit ab43376697 (GH #2026). Peewee will no longer automatically return the cursor `rowcount` for certain bulk-inserts. This should mainly affect users of MySQL and Sqlite who relied on a bulk INSERT returning the `rowcount` (as opposed to the cursor's `lastrowid`). The `rowcount` behavior is still available chaining the ``as_rowcount()`` method: ```python # NOTE: this change only affects MySQL or Sqlite. db = MySQLDatabase(...) # Previously, bulk inserts of the following forms would return the rowcount. query = User.insert_many(...) # Bulk insert. query = User.insert_from(...) # Bulk insert (INSERT INTO .. SELECT FROM). # Previous behavior (peewee 3.12 - 3.14.10): # rows_inserted = query.execute() # New behavior: last_id = query.execute() # To get the old behavior back: rows_inserted = query.as_rowcount().execute() ``` Additionally, in previous versions specifying an empty `.returning()` with Postgres would cause the rowcount to be returned. For Postgres users who wish to receive the rowcount: ```python # NOTE: this change only affects Postgresql. db = PostgresqlDatabase(...) # Previously, an empty returning() would return the rowcount. query = User.insert_many(...) # Bulk insert. query = User.insert_from(...) # Bulk insert (INSERT INTO .. SELECT FROM). # Old behavior: # rows_inserted = query.returning().execute() # To get the rows inserted in 3.15 and newer: rows_inserted = query.as_rowcount().execute() ``` This release contains a fix for a long-standing request to allow data-modifying queries to support CTEs. CTEs are now supported for use with INSERT, DELETE and UPDATE queries - see #2152. Additionally, this release adds better support for using the new `RETURNING` syntax with Sqlite automatically. Specify `returning_clause=True` when initializing your `SqliteDatabase` and all bulk inserts will automatically specify a `RETURNING` clause, returning the newly-inserted primary keys. This functionality requires Sqlite 3.35 or newer. Smaller changes: * Add `shortcuts.insert_where()` helper for generating conditional INSERT with a bit less boilerplate. * Fix bug in `test_utils.count_queres()` which could erroneously include pool events such as connect/disconnect, etc. [View commits](https://github.com/coleifer/peewee/compare/3.14.10...3.15.0) ## 3.14.10 * Add shortcut for conditional insert using sub-select, see #2528 * Add convenience `left_outer_join()` method to query. * Add `selected_columns` property to Select queries. * Add `name` property to Alias instances. * Fix regression in tests introduced by change to DataSet in 3.14.9. [View commits](https://github.com/coleifer/peewee/compare/3.14.9...3.14.10) ## 3.14.9 * Allow calling `table_exists()` with a model-class, refs * Improve `is_connection_usable()` method of `MySQLDatabase` class. * Better support for VIEWs with `playhouse.dataset.DataSet` and sqlite-web. * Support INSERT / ON CONFLICT in `playhosue.kv` for newer Sqlite. * Add `ArrayField.contained_by()` method, a corollary to `contains()` and the `contains_any()` methods. * Support cyclical foreign-key relationships in reflection/introspection, and also for sqlite-web. * Add magic methods for FTS5 field to optimize, rebuild and integrity check the full-text index. * Add fallbacks in `setup.py` in the event distutils is not available. [View commits](https://github.com/coleifer/peewee/compare/3.14.8...3.14.9) ## 3.14.8 Back-out all changes to automatically use RETURNING for `SqliteExtDatabase`, `CSqliteExtDatabase` and `APSWDatabase`. The issue I found is that when a RETURNING cursor is not fully-consumed, any parent SAVEPOINT (and possibly transaction) would not be able to be released. Since this is a backwards-incompatible change, I am going to back it out for now. Returning clause can still be specified for Sqlite, however it just needs to be done so manually rather than having it applied automatically. [View commits](https://github.com/coleifer/peewee/compare/3.14.7...3.14.8) ## 3.14.7 Fix bug in APSW extension with Sqlite 3.35 and newer, due to handling of last insert rowid with RETURNING. Refs #2479. [View commits](https://github.com/coleifer/peewee/compare/3.14.6...3.14.7) ## 3.14.6 Fix pesky bug in new `last_insert_id()` on the `SqliteExtDatabase`. [View commits](https://github.com/coleifer/peewee/compare/3.14.5...3.14.6) ## 3.14.5 This release contains a number of bug-fixes and small improvements. * Only raise `DoesNotExist` when `lazy_load` is enabled on ForeignKeyField, fixes issue #2377. * Add missing convenience method `ModelSelect.get_or_none()` * Allow `ForeignKeyField` to specify a custom `BackrefAccessorClass`, references issue #2391. * Ensure foreign-key-specific conversions are applied on INSERT and UPDATE, fixes #2408. * Add handling of MySQL error 4031 (inactivity timeout) to the `ReconnectMixin` helper class. Fixes #2419. * Support specification of conflict target for ON CONFLICT/DO NOTHING. * Add `encoding` parameter to the DataSet `freeze()` and `thaw()` methods, fixes #2425. * Fix bug which prevented `DeferredForeignKey` from being used as a model's primary key, fixes #2427. * Ensure foreign key's related object cache is cleared when the foreign-key is set to `None`. Fixes #2428. * Allow specification of `(schema, table)` to be used with CREATE TABLE AS..., fixes #2423. * Allow reusing open connections with DataSet, refs #2441. * Add `highlight()` and `snippet()` helpers to Sqlite `SearchField`, for use with full-text search extension. * Preserve user-provided aliases in column names. Fixes #2453. * Add support for Sqlite 3.37 strict tables. * Ensure database is inherited when using `ThreadSafeDatabaseMetadata`, and also adds an implementation in `playhouse.shortcuts` along with basic unit tests. * Better handling of Model's dirty fields when saving, fixes #2466. * Add basic support for MariaDB connector driver in `playhouse.mysql_ext`, refs issue #2471. * Begin a basic implementation for a psycopg3-compatible pg database, refs issue #2473. * Add provisional support for RETURNING when using the appropriate versions of Sqlite or MariaDB. [View commits](https://github.com/coleifer/peewee/compare/3.14.4...3.14.5) ## 3.14.4 This release contains an important fix for a regression introduced by commit ebe3ad5, which affected the way model instances are converted to parameters for use in expressions within a query. The bug could manifest when code uses model instances as parameters in expressions against fields that are not foreign-keys. The issue is described in #2376. [View commits](https://github.com/coleifer/peewee/compare/3.14.3...3.14.4) ## 3.14.3 This release contains a single fix for ensuring NULL values are inserted when issuing a bulk-insert of heterogeneous dictionaries which may be missing explicit NULL values. Fixes issue #2638. [View commits](https://github.com/coleifer/peewee/compare/3.14.2...3.14.3) ## 3.14.2 This is a small release mainly to get some fixes out. * Support for named `Check` and foreign-key constraints. * Better foreign-key introspection for CockroachDB (and Postgres). * Register UUID adapter for Postgres. * Add `fn.array_agg()` to blacklist for automatic value coercion. [View commits](https://github.com/coleifer/peewee/compare/3.14.1...3.14.2) ## 3.14.1 This release contains primarily bugfixes. * Properly delegate to a foreign-key field's `db_value()` function when converting model instances. #2304. * Strip quote marks and parentheses from column names returned by sqlite cursor when a function-call is projected without an alias. #2305. * Fix `DataSet.create_index()` method, #2319. * Fix column-to-model mapping in model-select from subquery with joins, #2320. * Improvements to foreign-key lazy-loading thanks @conqp, #2328. * Preserve and handle `CHECK()` constraints in Sqlite migrator, #2343. * Add `stddev` aggregate function to collection of sqlite user-defined funcs. [View commits](https://github.com/coleifer/peewee/compare/3.14.0...3.14.1) ## 3.14.0 This release has been a bit overdue and there are numerous small improvements and bug-fixes. The bugfix that prompted this release is #2293, which is a regression in the Django-inspired `.filter()` APIs that could cause some filter expressions to be discarded from the generated SQL. Many thanks for the excellent bug report, Jakub. * Add an experimental helper, `shortcuts.resolve_multimodel_query()`, for resolving multiple models used in a compound select query. * Add a `lateral()` method to select query for use with lateral joins, refs issue #2205. * Added support for nested transactions (savepoints) in cockroach-db (requires 20.1 or newer). * Automatically escape wildcards passed to string-matching methods, refs #2224. * Allow index-type to be specified on MySQL, refs #2242. * Added a new API, `converter()` to be used for specifying a function to use to convert a row-value pulled off the cursor, refs #2248. * Add `set()` and `clear()` method to the bitfield flag descriptor, refs #2257. * Add support for `range` types with `IN` and other expressions. * Support CTEs bound to compound select queries, refs #2289. ### Bug-fixes * Fix to return related object id when accessing via the object-id descriptor, when the related object is not populated, refs #2162. * Fix to ensure we do not insert a NULL value for a primary key. * Fix to conditionally set the field/column on an added column in a migration, refs #2171. * Apply field conversion logic to model-class values. Relocates the logic from issue #2131 and fixes #2185. * Clone node before modifying it to be flat in an enclosed nodelist expr, fixes issue #2200. * Fix an invalid item assignment in nodelist, refs #2220. * Fix an incorrect truthiness check used with `save()` and `only=`, refs #2269. * Fix regression in `filter()` where using both `*args` and `**kwargs` caused the expressions passed as `args` to be discarded. See #2293. [View commits](https://github.com/coleifer/peewee/compare/3.13.3...3.14.0) ## 3.13.3 * Allow arbitrary keyword arguments to be passed to `DataSet` constructor, which are then passed to the instrospector. * Allow scalar subqueries to be compared using numeric operands. * Fix `bulk_create()` when model being inserted uses FK identifiers. * Fix `bulk_update()` so that PK values are properly coerced to the right data-type (e.g. UUIDs to strings for Sqlite). * Allow array indices to be used as dict keys, e.g. for the purposes of updating a single array index value. [View commits](https://github.com/coleifer/peewee/compare/3.13.2...3.13.3) ## 3.13.2 * Allow aggregate functions to support an `ORDER BY` clause, via the addition of an `order_by()` method to the function (`fn`) instance. Refs #2094. * Fix `prefetch()` bug, where related "backref" instances were marked as dirty, even though they had no changes. Fixes #2091. * Support `LIMIT 0`. Previously a limit of 0 would be translated into effectively an unlimited query on MySQL. References #2084. * Support indexing into arrays using expressions with Postgres array fields. References #2085. * Ensure postgres introspection methods return the columns for multi-column indexes in the correct order. Fixes #2104. * Add support for arrays of UUIDs to postgres introspection. * Fix introspection of columns w/capitalized table names in postgres (#2110). * Fix to ensure correct exception is raised in SqliteQueueDatabase when iterating over cursor/result-set. * Fix bug comparing subquery against a scalar value. Fixes #2118. * Fix issue resolving composite primary-keys that include foreign-keys when building the model-graph. Fixes #2115. * Allow model-classes to be passed as arguments, e.g., to a table function. Refs #2131. * Ensure postgres `JSONField.concat()` accepts expressions as arguments. [View commits](https://github.com/coleifer/peewee/compare/3.13.1...3.13.2) ## 3.13.1 Fix a regression when specifying keyword arguments to the `atomic()` or `transaction()` helper methods. Note: this only occurs if you were using Sqlite and were explicitly setting the `lock_type=` parameter. [View commits](https://github.com/coleifer/peewee/compare/3.13.0...3.13.1) ## 3.13.0 ### CockroachDB support added This will be a notable release as it adds support for [CockroachDB](https://cockroachlabs.com/), a distributed, horizontally-scalable SQL database. * [CockroachDB usage overview](http://docs.peewee-orm.com/en/latest/peewee/database.html#using-crdb) * [CockroachDB API documentation](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#crdb) ### Other features and fixes * Allow `FOR UPDATE` clause to specify one or more tables (`FOR UPDATE OF...`). * Support for Postgres `LATERAL` join. * Properly wrap exceptions raised during explicit commit/rollback in the appropriate peewee-specific exception class. * Capture original exception object and expose it as `exc.orig` on the wrapped exception. * Properly introspect `SMALLINT` columns in Postgres schema reflection. * More flexible handling of passing database-specific arguments to `atomic()` and `transaction()` context-manager/decorator. * Fix non-deterministic join ordering issue when using the `filter()` API across several tables (#2063). [View commits](https://github.com/coleifer/peewee/compare/3.12.0...3.13.0) ## 3.12.0 * Bulk insert (`insert_many()` and `insert_from()`) will now return the row count instead of the last insert ID. If you are using Postgres, peewee will continue to return a cursor that provides an iterator over the newly-inserted primary-key values by default. This behavior is being retained by default for compatibility. Postgres users can simply specify an empty `returning()` call to disable the cursor and retrieve the rowcount instead. * Migration extension now supports altering a column's data-type, via the new `alter_column_type()` method. * Added `Database.is_connection_usabe()` method, which attempts to look at the status of the underlying DB-API connection to determine whether the connection is usable. * Common table expressions include a `materialized` parameter, which can be used to control Postgres' optimization fencing around CTEs. * Added `BloomFilter.from_buffer()` method for populating a bloom-filter from the output of a previous call to the `to_buffer()` method. * Fixed APSW extension's `commit()` and `rollback()` methods to no-op if the database is in auto-commit mode. * Added `generate_always=` option to the `IdentityField` (defaults to False). [View commits](https://github.com/coleifer/peewee/compare/3.11.2...3.12.0) ## 3.11.2 * Implement `hash` interface for `Alias` instances, allowing them to be used in multi-source queries. [View commits](https://github.com/coleifer/peewee/compare/3.11.1...3.11.2) ## 3.11.1 * Fix bug in new `_pk` / `get_id()` implementation for models that explicitly have disabled a primary-key. [View commits](https://github.com/coleifer/peewee/compare/3.11.0...3.11.1) ## 3.11.0 * Fixes #1991. This particular issue involves joining 3 models together in a chain, where the outer two models are empty. Previously peewee would make the middle model an empty model instance (since a link might be needed from the source model to the outermost model). But since both were empty, it is more correct to make the intervening model a NULL value on the foreign-key field rather than an empty instance. * An unrelated fix came out of the work on #1991 where hashing a model whose primary-key happened to be a foreign-key could trigger the FK resolution query. This patch fixes the `Model._pk` and `get_id()` interfaces so they no longer introduce the possibility of accidentally resolving the FK. * Allow `Field.contains()`, `startswith()` and `endswith()` to compare against another column-like object or expression. * Workaround for MySQL prior to 8 and MariaDB handling of union queries inside of parenthesized expressions (like IN). * Be more permissive in letting invalid values be stored in a field whose type is INTEGER or REAL, since Sqlite allows this. * `TimestampField` resolution cleanup. Now values 0 *and* 1 will resolve to a timestamp resolution of 1 second. Values 2-6 specify the number of decimal places (hundredths to microsecond), or alternatively the resolution can still be provided as a power of 10, e.g. 10, 1000 (millisecond), 1e6 (microsecond). * When self-referential foreign-keys are inherited, the foreign-key on the subclass will also be self-referential (rather than pointing to the parent model). * Add TSV import/export option to the `dataset` extension. * Add item interface to the `dataset.Table` class for doing primary-key lookup, assignment, or deletion. * Extend the mysql `ReconnectMixin` helper to work with mysql-connector. * Fix mapping of double-precision float in postgres schema reflection. Previously it mapped to single-precision, now it correctly uses a double. * Fix issue where `PostgresqlExtDatabase` and `MySQLConnectorDatabase` did not respect the `autoconnect` setting. [View commits](https://github.com/coleifer/peewee/compare/3.10.0...3.11.0) ## 3.10.0 * Add a helper to `playhouse.mysql_ext` for creating `Match` full-text search expressions. * Added date-part properties to `TimestampField` for accessing the year, month, day, etc., within a SQL expression. * Added `to_timestamp()` helper for `DateField` and `DateTimeField` that produces an expression returning a unix timestamp. * Add `autoconnect` parameter to `Database` classes. This parameter defaults to `True` and is compatible with previous versions of Peewee, in which executing a query on a closed database would open a connection automatically. To make it easier to catch inconsistent use of the database connection, this behavior can now be disabled by specifying `autoconnect=False`, making an explicit call to `Database.connect()` needed before executing a query. * Added database-agnostic interface for obtaining a random value. * Allow `isolation_level` to be specified when initializing a Postgres db. * Allow hybrid properties to be used on model aliases. Refs #1969. * Support aggregates with FILTER predicates on the latest Sqlite. #### Changes * More aggressively slot row values into the appropriate field when building objects from the database cursor (rather than using whatever `cursor.description` tells us, which is buggy in older Sqlite). * Be more permissive in what we accept in the `insert_many()` and `insert()` methods. * When implicitly joining a model with multiple foreign-keys, choose the foreign-key whose name matches that of the related model. Previously, this would have raised a `ValueError` stating that multiple FKs existed. * Improved date truncation logic for Sqlite and MySQL to make more compatible with Postgres' `date_trunc()` behavior. Previously, truncating a datetime to month resolution would return `'2019-08'` for example. As of 3.10.0, the Sqlite and MySQL `date_trunc` implementation returns a full datetime, e.g. `'2019-08-01 00:00:00'`. * Apply slightly different logic for casting JSON values with Postgres. Previously, Peewee just wrapped the value in the psycopg2 `Json()` helper. In this version, Peewee now dumps the json to a string and applies an explicit cast to the underlying JSON data-type (e.g. json or jsonb). #### Bug fixes * Save hooks can now be called for models without a primary key. * Fixed bug in the conversion of Python values to JSON when using Postgres. * Fix for differentiating empty values from NULL values in `model_to_dict`. * Fixed a bug referencing primary-key values that required some kind of conversion (e.g., a UUID). See #1979 for details. * Add small jitter to the pool connection timestamp to avoid issues when multiple connections are checked-out at the same exact time. [View commits](https://github.com/coleifer/peewee/compare/3.9.6...3.10.0) ## 3.9.6 * Support nesting the `Database` instance as a context-manager. The outermost block will handle opening and closing the connection along with wrapping everything in a transaction. Nested blocks will use savepoints. * Add new `session_start()`, `session_commit()` and `session_rollback()` interfaces to the Database object to support using transactional controls in situations where a context-manager or decorator is awkward. * Fix error that would arise when attempting to do an empty bulk-insert. * Set `isolation_level=None` in SQLite connection constructor rather than afterwards using the setter. * Add `create_table()` method to `Select` query to implement `CREATE TABLE AS`. * Cleanup some declarations in the Sqlite C extension. * Add new example showing how to implement Reddit's ranking algorithm in SQL. [View commits](https://github.com/coleifer/peewee/compare/3.9.5...3.9.6) ## 3.9.5 * Added small helper for setting timezone when using Postgres. * Improved SQL generation for `VALUES` clause. * Support passing resolution to `TimestampField` as a power-of-10. * Small improvements to `INSERT` queries when the primary-key is not an auto-incrementing integer, but is generated by the database server (eg uuid). * Cleanups to virtual table implementation and python-to-sqlite value conversions. * Fixed bug related to binding previously-unbound models to a database using a context manager, #1913. [View commits](https://github.com/coleifer/peewee/compare/3.9.4...3.9.5) ## 3.9.4 * Add `Model.bulk_update()` method for bulk-updating fields across multiple model instances. [Docs](http://docs.peewee-orm.com/en/latest/peewee/api.html#Model.bulk_update). * Add `lazy_load` parameter to `ForeignKeyField`. When initialized with `lazy_load=False`, the foreign-key will not use an additional query to resolve the related model instance. Instead, if the related model instance is not available, the underlying FK column value is returned (behaving like the "_id" descriptor). * Added `Model.truncate_table()` method. * The `reflection` and `pwiz` extensions now attempt to be smarter about converting database table and column names into snake-case. To disable this, you can set `snake_case=False` when calling the `Introspector.introspect()` method or use the `-L` (legacy naming) option with the `pwiz` script. * Bulk insert via ``insert_many()`` no longer require specification of the fields argument when the inserted rows are lists/tuples. In that case, the fields will be inferred to be all model fields except any auto-increment id. * Add `DatabaseProxy`, which implements several of the `Database` class context managers. This allows you to reference some of the special features of the database object without directly needing to initialize the proxy first. * Add support for window function frame exclusion and added built-in support for the GROUPS frame type. * Add support for chaining window functions by extending a previously-declared window function. * Playhouse Postgresql extension `TSVectorField.match()` method supports an additional argument `plain`, which can be used to control the parsing of the TS query. * Added very minimal `JSONField` to the playhouse MySQL extension. [View commits](https://github.com/coleifer/peewee/compare/3.9.3...3.9.4) ## 3.9.3 * Added cross-database support for `NULLS FIRST/LAST` when specifying the ordering for a query. Previously this was only supported for Postgres. Peewee will now generate an equivalent `CASE` statement for Sqlite and MySQL. * Added [EXCLUDED](http://docs.peewee-orm.com/en/latest/peewee/api.html#EXCLUDED) helper for referring to the `EXCLUDED` namespace used with `INSERT...ON CONFLICT` queries, when referencing values in the conflicting row data. * Added helper method to the model `Metadata` class for setting the table name at run-time. Setting the `Model._meta.table_name` directly may have appeared to work in some situations, but could lead to subtle bugs. The new API is `Model._meta.set_table_name()`. * Enhanced helpers for working with Peewee interactively, [see doc](http://docs.peewee-orm.com/en/latest/peewee/interactive.html). * Fix cache invalidation bug in `DataSet` that was originally reported on the sqlite-web project. * New example script implementing a [hexastore](https://github.com/coleifer/peewee/blob/master/examples/hexastore.py). [View commits](https://github.com/coleifer/peewee/compare/3.9.2...3.9.3) ## 3.9.1 and 3.9.2 Includes a bugfix for an `AttributeError` that occurs when using MySQL with the `MySQLdb` client. The 3.9.2 release includes fixes for a test failure. [View commits](https://github.com/coleifer/peewee/compare/3.9.0...3.9.2) ## 3.9.0 * Added new document describing how to [use peewee interactively](http://docs.peewee-orm.com/en/latest/peewee/interactive.html). * Added convenience functions for generating model classes from a pre-existing database, printing model definitions and printing CREATE TABLE sql for a model. See the "use peewee interactively" section for details. * Added a `__str__` implementation to all `Query` subclasses which converts the query to a string and interpolates the parameters. * Improvements to `sqlite_ext.JSONField` regarding the serialization of data, as well as the addition of options to override the JSON serialization and de-serialization functions. * Added `index_type` parameter to `Field` * Added `DatabaseProxy`, which allows one to use database-specific decorators with an uninitialized `Proxy` object. See #1842 for discussion. Recommend that you update any usage of `Proxy` for deferring database initialization to use the new `DatabaseProxy` class instead. * Added support for `INSERT ... ON CONFLICT` when the conflict target is a partial index (e.g., contains a `WHERE` clause). The `OnConflict` and `on_conflict()` APIs now take an additional `conflict_where` parameter to represent the `WHERE` clause of the partial index in question. See #1860. * Enhanced the `playhouse.kv` extension to use efficient upsert for *all* database engines. Previously upsert was only supported for sqlite and mysql. * Re-added the `orwhere()` query filtering method, which will append the given expressions using `OR` instead of `AND`. See #391 for old discussion. * Added some new examples to the ``examples/`` directory * Added `select_from()` API for wrapping a query and selecting one or more columns from the wrapped subquery. [Docs](http://docs.peewee-orm.com/en/latest/peewee/api.html#SelectQuery.select_from). * Added documentation on using [row values](http://docs.peewee-orm.com/en/latest/peewee/query_operators.html#row-values). * Removed the (defunct) "speedups" C extension, which as of 3.8.2 only contained a barely-faster function for quoting entities. **Bugfixes** * Fix bug in SQL generation when there was a subquery that used a common table expressions. * Enhanced `prefetch()` and fixed bug that could occur when mixing self-referential foreign-keys and model aliases. * MariaDB 10.3.3 introduces backwards-incompatible changes to the SQL used for upsert. Peewee now introspects the MySQL server version at connection time to ensure proper handling of version-specific features. See #1834 for details. * Fixed bug where `TimestampField` would treat zero values as `None` when reading from the database. [View commits](https://github.com/coleifer/peewee/compare/3.8.2...3.9.0) ## 3.8.2 **Backwards-incompatible changes** * The default row-type for `INSERT` queries executed with a non-default `RETURNING` clause has changed from `tuple` to `Model` instances. This makes `INSERT` behavior consistent with `UPDATE` and `DELETE` queries that specify a `RETURNING` clause. To revert back to the old behavior, just append a call to `.tuples()` to your `INSERT ... RETURNING` query. * Removing support for the `table_alias` model `Meta` option. Previously, this attribute could be used to specify a "vanity" alias for a model class in the generated SQL. As a result of some changes to support more robust UPDATE and DELETE queries, supporting this feature will require some re-working. As of the 3.8.0 release, it was broken and resulted in incorrect SQL for UPDATE queries, so now it is removed. **New features** * Added `playhouse.shortcuts.ReconnectMixin`, which can be used to implement automatic reconnect under certain error conditions (notably the MySQL error 2006 - server has gone away). **Bugfixes** * Fix SQL generation bug when using an inline window function in the `ORDER BY` clause of a query. * Fix possible zero-division in user-defined implementation of BM25 ranking algorithm for SQLite full-text search. [View commits](https://github.com/coleifer/peewee/compare/3.8.1...3.8.2) ## 3.8.1 **New features** * Sqlite `SearchField` now supports the `match()` operator, allowing full-text search to be performed on a single column (as opposed to the whole table). **Changes** * Remove minimum passphrase restrictions in SQLCipher integration. **Bugfixes** * Support inheritance of `ManyToManyField` instances. * Ensure operator overloads are invoked when generating filter expressions. * Fix incorrect scoring in Sqlite BM25, BM25f and Lucene ranking algorithms. * Support string field-names in data dictionary when performing an ON CONFLICT ... UPDATE query, which allows field-specific conversions to be applied. References #1815. [View commits](https://github.com/coleifer/peewee/compare/3.8.0...3.8.1) ## 3.8.0 **New features** * Postgres `BinaryJSONField` now supports `has_key()`, `concat()` and `remove()` methods (though remove may require pg10+). * Add `python_value()` method to the SQL-function helper `fn`, to allow specifying a custom function for mapping database values to Python values. **Changes** * Better support for UPDATE ... FROM queries, and more generally, more robust support for UPDATE and RETURNING clauses. This means that the `QualifiedNames` helper is no longer needed for certain types of queries. * The `SqlCipherDatabase` no longer accepts a `kdf_iter` parameter. To configure the various SQLCipher encryption settings, specify the setting values as `pragmas` when initializing the database. * Introspection will now, by default, only strip "_id" from introspected column names if those columns are foreign-keys. See #1799 for discussion. * Allow `UUIDField` and `BinaryUUIDField` to accept hexadecimal UUID strings as well as raw binary UUID bytestrings (in addition to `UUID` instances, which are already supported). * Allow `ForeignKeyField` to be created without an index. * Allow multiple calls to `cast()` to be chained (#1795). * Add logic to ensure foreign-key constraint names that exceed 64 characters are truncated using the same logic as is currently in place for long indexes. * `ManyToManyField` supports foreign-keys to fields other than primary-keys. * When linked against SQLite 3.26 or newer, support `SQLITE_CONSTRAINT` to designate invalid queries against virtual tables. * SQL-generation changes to aid in supporting using queries within expressions following the SELECT statement. **Bugfixes** * Fixed bug in `order_by_extend()`, thanks @nhatHero. * Fixed bug where the `DataSet` CSV import/export did not support non-ASCII characters in Python 3.x. * Fixed bug where `model_to_dict` would attempt to traverse explicitly disabled foreign-key backrefs (#1785). * Fixed bug when attempting to migrate SQLite tables that have a field whose column-name begins with "primary_". * Fixed bug with inheriting deferred foreign-keys. [View commits](https://github.com/coleifer/peewee/compare/3.7.1...3.8.0) ## 3.7.1 **New features** * Added `table_settings` model `Meta` option, which should be a list of strings specifying additional options for `CREATE TABLE`, which are placed *after* the closing parentheses. * Allow specification of `on_update` and `on_delete` behavior for many-to-many relationships when using `ManyToManyField`. **Bugfixes** * Fixed incorrect SQL generation for Postgresql ON CONFLICT clause when the conflict_target is a named constraint (rather than an index expression). This introduces a new keyword-argument to the `on_conflict()` method: `conflict_constraint`, which is currently only supported by Postgresql. Refs issue #1737. * Fixed incorrect SQL for sub-selects used on the right side of `IN` expressions. Previously the query would be assigned an alias, even though an alias was not needed. * Fixed incorrect SQL generation for Model indexes which contain SQL functions as indexed columns. * Fixed bug in the generation of special queries used to perform operations on SQLite FTS5 virtual tables. * Allow `frozenset` to be correctly parameterized as a list of values. * Allow multi-value INSERT queries to specify `columns` as a list of strings. * Support `CROSS JOIN` for model select queries. [View commits](https://github.com/coleifer/peewee/compare/3.7.0...3.7.1) ## 3.7.0 **Backwards-incompatible changes** * Pool database `close_all()` method renamed to `close_idle()` to better reflect the actual behavior. * Databases will now raise `InterfaceError` when `connect()` or `close()` are called on an uninitialized, deferred database object. **New features** * Add methods to the migrations extension to support adding and dropping table constraints. * Add [Model.bulk_create()](http://docs.peewee-orm.com/en/latest/peewee/api.html#Model.bulk_create) method for bulk-inserting unsaved model instances. * Add `close_stale()` method to the connection pool to support closing stale connections. * The `FlaskDB` class in `playhouse.flask_utils` now accepts a `model_class` parameter, which can be used to specify a custom base-class for models. **Bugfixes** * Parentheses were not added to subqueries used in function calls with more than one argument. * Fixed bug when attempting to serialize many-to-many fields which were created initially with a `DeferredThroughModel`, see #1708. * Fixed bug when using the Postgres `ArrayField` with an array of `BlobField`. * Allow `Proxy` databases to be used as a context-manager. * Fixed bug where the APSW driver was referring to the SQLite version from the standard library `sqlite3` driver, rather than from `apsw`. * Reflection library attempts to wrap server-side column defaults in quotation marks if the column data-type is text/varchar. * Missing import in migrations library, which would cause errors when attempting to add indexes whose name exceeded 64 chars. * When using the Postgres connection pool, ensure any open/pending transactions are rolled-back when the connection is recycled. * Even *more* changes to the `setup.py` script. In this case I've added a helper function which will reliably determine if the SQLite3 extensions can be built. This follows the approach taken by the Python YAML package. [View commits](https://github.com/coleifer/peewee/compare/3.6.4...3.7.0) ## 3.6.4 Take a whole new approach, following what `simplejson` does. Allow the `build_ext` command class to fail, and retry without extensions in the event we run into issues building extensions. References #1676. [View commits](https://github.com/coleifer/peewee/compare/3.6.3...3.6.4) ## 3.6.3 Add check in `setup.py` to determine if a C compiler is available before building C extensions. References #1676. [View commits](https://github.com/coleifer/peewee/compare/3.6.2...3.6.3) ## 3.6.2 Use `ctypes.util.find_library` to determine if `libsqlite3` is installed. Should fix problems people are encountering installing when SQLite3 is not available. [View commits](https://github.com/coleifer/peewee/compare/3.6.1...3.6.2) ## 3.6.1 Fixed issue with setup script. [View commits](https://github.com/coleifer/peewee/compare/3.6.0...3.6.1) ## 3.6.0 * Support for Python 3.7, including bugfixes related to new StopIteration handling inside of generators. * Support for specifying `ROWS` or `RANGE` window frame types. For more information, see the new [frame type documentation](http://docs.peewee-orm.com/en/latest/peewee/querying.html#frame-types-range-vs-rows). * Add APIs for user-defined window functions if using [pysqlite3](https://github.com/coleifer/pysqlite3) and sqlite 3.25.0 or newer. * `TimestampField` now uses 64-bit integer data-type for storage. * Added support to `pwiz` and `playhouse.reflection` to enable generating models from VIEWs. * Added lower-level database API for introspecting VIEWs. * Revamped continuous integration setup for better coverage, including 3.7 and 3.8-dev. * Allow building C extensions even if Cython is not installed, by distributing pre-generated C source files. * Switch to using `setuptools` for packaging. [View commits](https://github.com/coleifer/peewee/compare/3.5.2...3.6.0) ## 3.5.2 * New guide to using [window functions in Peewee](http://docs.peewee-orm.com/en/latest/peewee/querying.html#window-functions). * New and improved table name auto-generation. This feature is not backwards compatible, so it is **disabled by default**. To enable, set `legacy_table_names=False` in your model's `Meta` options. For more details, see [table names](http://docs.peewee-orm.com/en/latest/peewee/models.html#table_names) documentation. * Allow passing single fields/columns to window function `order_by` and `partition_by` arguments. * Support for `FILTER (WHERE...)` clauses with window functions and aggregates. * Added `IdentityField` class suitable for use with Postgres 10's new identity column type. It can be used anywhere `AutoField` or `BigAutoField` was being used previously. * Fixed bug creating indexes on tables that are in attached databases (SQLite). * Fixed obscure bug when using `prefetch()` and `ModelAlias` to populate a back-reference related model. [View commits](https://github.com/coleifer/peewee/compare/3.5.1...3.5.2) ## 3.5.1 **New features** * New documentation for working with [relationships](http://docs.peewee-orm.com/en/latest/peewee/relationships.html) in Peewee. * Improved tests and documentation for MySQL upsert functionality. * Allow `database` parameter to be specified with `ModelSelect.get()` method. For discussion, see #1620. * Add `QualifiedNames` helper to peewee module exports. * Add `temporary=` meta option to support temporary tables. * Allow a `Database` object to be passed to constructor of `DataSet` helper. **Bug fixes** * Fixed edge-case where attempting to alias a field to it's underlying column-name (when different), Peewee would not respect the alias and use the field name instead. See #1625 for details and discussion. * Raise a `ValueError` when joining and aliasing the join to a foreign-key's `object_id_name` descriptor. Should prevent accidentally introducing O(n) queries or silently ignoring data from a joined-instance. * Fixed bug for MySQL when creating a foreign-key to a model which used the `BigAutoField` for it's primary-key. * Fixed bugs in the implementation of user-defined aggregates and extensions with the APSW SQLite driver. * Fixed regression introduced in 3.5.0 which ignored custom Model `__repr__()`. * Fixed regression from 2.x in which inserting from a query using a `SQL()` was no longer working. Refs #1645. [View commits](https://github.com/coleifer/peewee/compare/3.5.0...3.5.1) ## 3.5.0 **Backwards-incompatible changes** * Custom Model `repr` no longer use the convention of overriding `__unicode__`, and now use `__str__`. * Redesigned the [sqlite json1 integration](http://docs.peewee-orm.com/en/latest/peewee/sqlite_ext.html#sqlite-json1). and changed some of the APIs and semantics of various `JSONField` methods. The documentation has been expanded to include more examples and the API has been simplified to make it easier to work with. These changes **do not** have any effect on the [Postgresql JSON fields](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#pgjson). **New features** * Better default `repr` for model classes and fields. * `ForeignKeyField()` accepts a new initialization parameter, `deferrable`, for specifying when constraints should be enforced. * `BitField.flag()` can be called without a value parameter for the common use-case of using flags that are powers-of-2. * `SqliteDatabase` pragmas can be specified as a `dict` (previously required a list of 2-tuples). * SQLite `TableFunction` ([docs](http://docs.peewee-orm.com/en/latest/peewee/sqlite_ext.html#sqlite-vtfunc)) will print Python exception tracebacks raised in the `initialize` and `iterate` callbacks, making debugging significantly easier. **Bug fixes** * Fixed bug in `migrator.add_column()` where, if the field being added declared a non-standard index type (e.g., binary json field with GIN index), this index type was not being respected. * Fixed bug in `database.table_exists()` where the implementation did not match the documentation. Implementation has been updated to match the documentation. * Fixed bug in SQLite `TableFunction` implementation which raised errors if the return value of the `iterate()` method was not a `tuple`. [View commits](https://github.com/coleifer/peewee/compare/3.4.0...3.5.0) ## 3.4.0 **Backwards-incompatible changes** * The `regexp()` operation is now case-sensitive for MySQL and Postgres. To perform case-insensitive regexp operations, use `iregexp()`. * The SQLite `BareField()` field-type now supports all column constraints *except* specifying the data-type. Previously it silently ignored any column constraints. * LIMIT and OFFSET parameters are now treated as parameterized values instead of literals. * The `schema` parameter for SQLite database introspection methods is no longer ignored by default. The schema corresponds to the name given to an attached database. * `ArrayField` now accepts a new parameter `field_kwargs`, which is used to pass information to the array field's `field_class` initializer. **New features and other changes** * SQLite backup interface supports specifying page-counts and a user-defined progress handler. * GIL is released when doing backups or during SQLite busy timeouts (when using the peewee SQLite busy-handler). * Add NATURAL join-type to the `JOIN` helper. * Improved identifier quoting to allow specifying distinct open/close-quote characters. Enables adding support for MSSQL, for instance, which uses square brackets, e.g. `[table].[column]`. * Unify timeout interfaces for SQLite databases (use seconds everywhere rather than mixing seconds and milliseconds, which was confusing). * Added `attach()` and `detach()` methods to SQLite database, making it possible to attach additional databases (e.g. an in-memory cache db). [View commits](https://github.com/coleifer/peewee/compare/3.3.4...3.4.0) ## 3.3.4 * Added a `BinaryUUIDField` class for efficiently storing UUIDs in 16-bytes. * Fix dataset's `update_cache()` logic so that when updating a single table that was newly-added, we also ensure that all dependent tables are updated at the same time. Refs coleifer/sqlite-web#42. [View commits](https://github.com/coleifer/peewee/compare/3.3.3...3.3.4) ## 3.3.3 * More efficient implementation of model dependency-graph generation. Improves performance of recursively deleting related objects by omitting unnecessary subqueries. * Added `union()`, `union_all()`, `intersect()` and `except_()` to the `Model`-specific query implementations. This was an oversight that should have been patched in 3.3.2, but is fixed in 3.3.3. * Major cleanup to test runner and standardized test skipping logic to integrate with standard-library `unittest` conventions. [View commits](https://github.com/coleifer/peewee/compare/3.3.2...3.3.3) ## 3.3.2 * Add methods for `union()`, `union_all`, `intersect()` and `except_()`. Previously, these methods were only available as operator overloads. * Removed some Python 2.6-specific support code, as 2.6 is no longer officially supported. * Fixed model-graph resolution logic for deferred foreign-keys. * Better support for UPDATE...FROM queries (Postgresql). [View commits](https://github.com/coleifer/peewee/compare/3.3.1...3.3.2) ## 3.3.1 * Fixed long-standing bug in 3.x regarding using column aliases with queries that utilize the ModelCursorWrapper (typically queries with one or more joins). * Fix typo in model metadata code, thanks @klen. * Add examples of using recursive CTEs to docs. [View commits](https://github.com/coleifer/peewee/compare/3.3.0...3.3.1) ## 3.3.0 * Added support for SQLite's new `ON CONFLICT` clause, which is modelled on the syntax used by Postgresql and will be available in SQLite 3.24.0 and onward. * Added better support for using common table expressions and a cleaner way of implementing recursive CTEs, both of which are also tested with integration tests (as opposed to just checking the generated SQL). * Modernized the CI environment to utilize the latest MariaDB features, so we can test window functions and CTEs with MySQL (when available). * Reorganized and unified the feature-flags in the test suite. [View commits](https://github.com/coleifer/peewee/compare/3.2.5...3.3.0) ## 3.2.5 * Added `ValuesList` for representing values lists. [Docs](http://docs.peewee-orm.com/en/latest/peewee/api.html#ValuesList). * `DateTimeField`, `DateField` and `TimeField` will parse formatted-strings before sending to the database. Previously this only occurred when reading values from the database. [View commits](https://github.com/coleifer/peewee/compare/3.2.4...3.2.5) ## 3.2.4 * Smarter handling of model-graph when dealing with compound queries (union, intersect, etc). #1579. * If the same column-name is selected multiple times, first value wins. #1579. * If `ModelSelect.switch()` is called without any arguments, default to the query's model. Refs #1573. * Fix issue where cloning a ModelSelect query did not result in the joins being cloned. #1576. [View commits](https://github.com/coleifer/peewee/compare/3.2.3...3.2.4) ## 3.2.3 * `pwiz` tool will capture column defaults defined as part of the table schema. * Fixed a misleading error message - #1563. * Ensure `reuse_if_open` parameter has effect on pooled databases. * Added support for on update/delete when migrating foreign-key. * Fixed bug in SQL generation for subqueries in aliased functions #1572. [View commits](https://github.com/coleifer/peewee/compare/3.2.2...3.2.3) ## 3.2.2 * Added support for passing `Model` classes to the `returning()` method when you intend to return all columns for the given model. * Fixed a bug when using user-defined sequences, and the underlying sequence already exists. * Added `drop_sequences` parameter to `drop_table()` method which allows you to conditionally drop any user-defined sequences when dropping the table. [View commits](https://github.com/coleifer/peewee/compare/3.2.1...3.2.2) ## 3.2.1 **Notice:** the default mysql driver for Peewee has changed to [pymysql](https://github.com/PyMySQL/PyMySQL) in version 3.2.1. In previous versions, if both *mysql-python* and *pymysql* were installed, Peewee would use *mysql-python*. As of 3.2.1, if both libraries are installed Peewee will use *pymysql*. * Added new module `playhouse.mysql_ext` which includes `MySQLConnectorDatabase`, a database implementation that works with the [mysql-connector](https://dev.mysql.com/doc/connector-python/en/) driver. * Added new field to `ColumnMetadata` class which captures a database column's default value. `ColumnMetadata` is returned by `Database.get_columns()`. * Added [documentation on making Peewee async](http://docs.peewee-orm.com/en/latest/peewee/database.html#async-with-gevent). [View commits](https://github.com/coleifer/peewee/compare/3.2.0...3.2.1) ## 3.2.0 The 3.2.0 release introduces a potentially backwards-incompatible change. The only users affected will be those that have implemented custom `Field` types with a user-defined `coerce` method. tl/dr: rename the coerce attribute to adapt and you should be set. #### Field.coerce renamed to Field.adapt The `Field.coerce` method has been renamed to `Field.adapt`. The purpose of this method is to convert a value from the application/database into the appropriate Python data-type. For instance, `IntegerField.adapt` is simply the `int` built-in function. The motivation for this change is to support adding metadata to any AST node instructing Peewee to not coerce the associated value. As an example, consider this code: ```python class Note(Model): id = AutoField() # autoincrementing integer primary key. content = TextField() # Query notes table and cast the "id" to a string and store as "id_text" attr. query = Note.select(Note.id.cast('TEXT').alias('id_text'), Note.content) a_note = query.get() print((a_note.id_text, a_note.content)) # Prior to 3.2.0 the CAST is "un-done" because the value gets converted # back to an integer, since the value is associated with the Note.id field: (1, u'some note') # 3.1.7, e.g. -- "id_text" is an integer! # As of 3.2.0, CAST will automatically prevent the conversion of field values, # which is an extension of a more general metadata API that can instruct Peewee # not to convert certain values. (u'1', u'some note') # 3.2.0 -- "id_text" is a string as expected. ``` If you have implemented custom `Field` classes and are using `coerce` to enforce a particular data-type, you can simply rename the attribute to `adapt`. #### Other changes Old versions of SQLite do not strip quotation marks from aliased column names in compound queries (e.g. UNION). Fixed in 3.2.0. [View commits](https://github.com/coleifer/peewee/compare/3.1.7...3.2.0) ## 3.1.7 For all the winblows lusers out there, added an option to skip compilation of the SQLite C extensions during installation. Set env var `NO_SQLITE=1` and run `setup.py install` and you should be able to build without requiring SQLite. [View commits](https://github.com/coleifer/peewee/compare/3.1.6...3.1.7) ## 3.1.6 * Added `rekey()` method to SqlCipher database for changing encryption key and documentation for `set_passphrase()` method. * Added `convert_values` parameter to `ArrayField` constructor, which will cause the array values to be processed using the underlying data-type's conversion logic. * Fixed unreported bug using `TimestampField` with sub-second resolutions. * Fixed bug where options were not being processed when calling `drop_table()`. * Some fixes and improvements to `signals` extension. [View commits](https://github.com/coleifer/peewee/compare/3.1.5...3.1.6) ## 3.1.5 Fixed Python 2/3 incompatibility with `itertools.izip_longest()`. [View commits](https://github.com/coleifer/peewee/compare/3.1.4...3.1.5) ## 3.1.4 * Added `BigAutoField` to support 64-bit auto-incrementing primary keys. * Use Peewee-compatible datetime serialization when exporting JSON from a `DataSet`. Previously the JSON export used ISO-8601 by default. See #1536. * Added `Database.batch_commit` helper to wrap iterators in chunked transactions. See #1539 for discussion. [View commits](https://github.com/coleifer/peewee/compare/3.1.3...3.1.4) ## 3.1.3 * Fixed issue where scope-specific settings were being updated in-place instead of copied. #1534. * Fixed bug where setting a `ForeignKeyField` did not add it to the model's "dirty" fields list. #1530. * Use pre-fetched data when using `prefetch()` with `ManyToManyField`. Thanks to @iBelieve for the patch. #1531. * Use `JSON` data-type for SQLite `JSONField` instances. * Add a `json_contains` function for use with SQLite `json1` extension. * Various documentation updates and additions. [View commits](https://github.com/coleifer/peewee/compare/3.1.2...3.1.3) ## 3.1.2 #### New behavior for INSERT queries with RETURNING clause Investigating #1522, it occurred to me that INSERT queries with non-default *RETURNING* clauses (postgres-only feature) should always return a cursor object. Previously, if executing a single-row INSERT query, the last-inserted row ID would be returned, regardless of what was specified by the RETURNING clause. This change only affects INSERT queries with non-default RETURNING clauses and will cause a cursor to be returned, as opposed to the last-inserted row ID. [View commits](https://github.com/coleifer/peewee/compare/3.1.1...3.1.2) ## 3.1.1 * Fixed bug when using `Model.alias()` when the model defined a particular database schema. * Added `SchemaManager.create_foreign_key` API to simplify adding constraints when dealing with circular foreign-key relationships. Updated docs accordingly. * Improved implementation of `Migrator.add_foreign_key_constraint` so that it can be used with Postgresql (in addition to MySQL). * Added `PickleField` to the `playhouse.fields` module. [Docs](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#PickleField). * Fixed bug in implementation of `CompressedField` when using Python 3. * Added `KeyValue` API in `playhouse.kv` module. [Docs](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#key-value-store). * More test cases for joining on sub-selects or common table expressions. [View commits](https://github.com/coleifer/peewee/compare/3.1.0...3.1.1) ## 3.1.0 #### Backwards-incompatible changes `Database.bind()` has been renamed to `Database.bind_ctx()`, to more closely match the semantics of the corresponding model methods, `Model.bind()` and `Model.bind_ctx()`. The new `Database.bind()` method is a one-time operation that binds the given models to the database. See documentation: * [Database.bind()](http://docs.peewee-orm.com/en/latest/peewee/api.html#Database.bind) * [Database.bind_ctx()](http://docs.peewee-orm.com/en/latest/peewee/api.html#Database.bind_ctx) #### Other changes * Removed Python 2.6 support code from a few places. * Fixed example analytics app code to ensure hstore extension is registered. * Small efficiency improvement to bloom filter. * Removed "attention!" from *README*. [View commits](https://github.com/coleifer/peewee/compare/3.0.20...3.1.0) ## 3.0.20 * Include `schema` (if specified) when checking for table-existence. * Correct placement of ORDER BY / LIMIT clauses in compound select queries. * Fix bug in back-reference lookups when using `filter()` API. * Fix bug in SQL generation for ON CONFLICT queries with Postgres, #1512. [View commits](https://github.com/coleifer/peewee/compare/3.0.19...3.0.20) ## 3.0.19 * Support for more types of mappings in `insert_many()`, refs #1495. * Lots of documentation improvements. * Fix bug when calling `tuples()` on a `ModelRaw` query. This was reported originally as a bug with *sqlite-web* CSV export. See coleifer/sqlite-web#38. [View commits](https://github.com/coleifer/peewee/compare/3.0.18...3.0.19) ## 3.0.18 * Improved error messages when attempting to use a database class for which the corresponding driver is not installed. * Added tests showing the use of custom operator (a-la the docs). * Fixed indentation issue in docs, #1493. * Fixed issue with the SQLite date_part issue, #1494. [View commits](https://github.com/coleifer/peewee/compare/3.0.17...3.0.18) ## 3.0.17 * Fix `schema` inheritance regression, #1485. * Add helper method to postgres migrator for setting search_path, #1353. [View commits](https://github.com/coleifer/peewee/compare/3.0.16...3.0.17) ## 3.0.16 * Improve model graph resolution when iterating results of a query. Refs #1482. * Allow Model._meta.schema to be changed at run-time. #1483. [View commits](https://github.com/coleifer/peewee/compare/3.0.15...3.0.16) ## 3.0.15 * Use same `schema` used for reflection in generated models. * Preserve `pragmas` set on deferred Sqlite database if database is re-initialized without re-specifying pragmas. [View commits](https://github.com/coleifer/peewee/compare/3.0.14...3.0.15) ## 3.0.14 * Fix bug creating model instances on Postgres when model does not have a primary key column. * Extend postgresql reflection to support array types. [View commits](https://github.com/coleifer/peewee/compare/3.0.13...3.0.14) ## 3.0.13 * Fix bug where simple field aliases were being ignored. Fixes #1473. * More strict about column type inference for postgres + pwiz. [View commits](https://github.com/coleifer/peewee/compare/3.0.12...3.0.13) ## 3.0.12 * Fix queries of the form INSERT ... VALUES (SELECT...) so that sub-select is wrapped in parentheses. * Improve model-graph resolution when selecting from multiple tables that are joined by foreign-keys, and an intermediate table is omitted from selection. * Docs update to reflect deletion of post_init signal. [View commits](https://github.com/coleifer/peewee/compare/3.0.11...3.0.12) ## 3.0.11 * Add note to changelog about `cursor()` method. * Add hash method to postgres indexedfield subclasses. * Add TableFunction to sqlite_ext module namespace. * Fix bug regarding NOT IN queries where the right-hand-side is an empty set. * Fallback implementations of bm25f and lucene search ranking algorithms. * Fixed DecimalField issue. * Fixed issue with BlobField when database is a Proxy object. [View commits](https://github.com/coleifer/peewee/compare/3.0.10...3.0.11) ## 3.0.10 * Fix `Database.drop_tables()` signature to support `cascade` argument - #1453. * Fix querying documentation for custom functions - #1454. * Added len() method to `ModelBase` for convenient counting. * Fix bug related to unsaved relation population (thanks @conqp) - #1459. * Fix count() on compound select - #1460. * Support `coerce` keyword argument with `fn.XXX()` - #1463. * Support updating existing model instance with dict_to_model-like API - #1456. * Fix equality tests with ArrayField - #1461. [View commits](https://github.com/coleifer/peewee/compare/3.0.9...3.0.10) ## 3.0.9 * Add deprecation notice if passing `autocommit` as keyword argument to the `Database` initializer. Refs #1452. * Add `JSONPath` and "J" helpers to sqlite extension. [View commits](https://github.com/coleifer/peewee/compare/3.0.8...3.0.9) ## 3.0.8 * Add support for passing `cascade=True` when dropping tables. Fixes #1449. * Fix issues with backrefs and inherited foreign-keys. Fixes #1448. [View commits](https://github.com/coleifer/peewee/compare/3.0.7...3.0.8) ## 3.0.7 * Add `select_extend()` method to extend existing SELECT-ion. [Doc](http://docs.peewee-orm.com/en/latest/peewee/api.html#Select.select_extend). * Accept `set()` as iterable value type, fixes #1445 * Add test for model/field inheritance and fix bug relating to recursion error when inheriting foreign-key field. Fixes #1448. * Fix regression where consecutive calls to `ModelSelect.select()` with no parameters resulted in an empty selection. Fixes #1438. [View commits](https://github.com/coleifer/peewee/compare/3.0.6...3.0.7) ## 3.0.6 Add constraints for ON UPDATE/ON DELETE to foreign-key constraint - #1443. [View commits](https://github.com/coleifer/peewee/compare/3.0.5...3.0.6) ## 3.0.5 Adds Model.index(), a short-hand method for declaring ModelIndex instances. * [Model.index docs](http://docs.peewee-orm.com/en/latest/peewee/api.html#Model.index) * [Model.add_index docs](http://docs.peewee-orm.com/en/latest/peewee/api.html#Model.add_index) * [ModelIndex docs](http://docs.peewee-orm.com/en/latest/peewee/api.html#ModelIndex) [View commits](https://github.com/coleifer/peewee/compare/3.0.4...3.0.5) ## 3.0.4 Re-add a shim for `PrimaryKeyField` (renamed to `AutoField`) and log a deprecation warning if you try to use it. [View commits](https://github.com/coleifer/peewee/compare/3.0.3...3.0.4) ## 3.0.3 Includes fix for bug where column-name to field-name translation was not being done when running select queries on models whose field name differed from the underlying column name (#1437). [View commits](https://github.com/coleifer/peewee/compare/3.0.2...3.0.3) ## 3.0.2 Ensures that the pysqlite headers are included in the source distribution so that certain C extensions can be compiled. [View commits](https://github.com/coleifer/peewee/compare/3.0.0...3.0.2) ## 3.0.0 * Complete rewrite of SQL AST and code-generation. * Inclusion of new, low-level query builder APIs. * List of [backwards-incompatible changes](http://docs.peewee-orm.com/en/latest/peewee/changes.html). [View commits](https://github.com/coleifer/peewee/compare/2.10.2...3.0.0) ## 2.10.2 * Update travis-ci build scripts to use Postgres 9.6 and test against Python 3.6. * Added support for returning `namedtuple` objects when iterating over a cursor. * Added support for specifying the "object id" attribute used when declaring a foreign key. By default, it is `foreign-key-name_id`, but it can now be customized. * Fixed small bug in the calculation of search scores when using the SQLite C extension or the `sqlite_ext` module. * Support literal column names with the `dataset` module. [View commits](https://github.com/coleifer/peewee/compare/2.10.1...2.10.2) ## 2.10.1 Removed `AESEncryptedField`. [View commits](https://github.com/coleifer/peewee/compare/2.10.0...2.10.1) ## 2.10.0 The main change in this release is the removal of the `AESEncryptedField`, which was included as part of the `playhouse.fields` extension. It was brought to my attention that there was some serious potential for security vulnerabilities. Rather than give users a false sense of security, I've decided the best course of action is to remove the field. * Remove the `playhouse.fields.AESEncryptedField` over security concerns described in ticket #1264. * Correctly resolve explicit table dependencies when creating tables, refs #1076. Thanks @maaaks. * Implement not equals comparison for `CompositeKey`. [View commits](https://github.com/coleifer/peewee/compare/2.9.2...2.10.0) ## 2.9.2 * Fixed significant bug in the `savepoint` commit/rollback implementation. Many thanks to @Syeberman for raising the issue. See #1225 for details. * Added support for postgresql `INTERVAL` columns. The new `IntervalField` in the `postgres_ext` module is suitable for storing `datetime.timedelta`. * Fixed bug where missing `sqlite3` library was causing other, unrelated libraries to throw errors when attempting to import. * Added a `case_sensitive` parameter to the SQLite `REGEXP` function implementation. The default is `False`, to preserve backwards-compatibility. * Fixed bug that caused tables not to be created when using the `dataset` extension. See #1213 for details. * Modified `drop_table` to raise an exception if the user attempts to drop tables with `CASCADE` when the database backend does not support it. * Fixed Python3 issue in the `AESEncryptedField`. * Modified the behavior of string-typed fields to treat the addition operator as concatenation. See #1241 for details. [View commits](https://github.com/coleifer/peewee/compare/2.9.1...2.9.2) ## 2.9.1 * Fixed #1218, where the use of `playhouse.flask_utils` was requiring the `sqlite3` module to be installed. * Fixed #1219 regarding the SQL generation for composite key sub-selects, joins, etc. [View commits](https://github.com/coleifer/peewee/compare/2.9.0...2.9.1) ## 2.9.0 In this release there are two notable changes: * The ``Model.create_or_get()`` method was removed. See the [documentation](http://docs.peewee-orm.com/en/latest/peewee/querying.html#create-or-get) for an example of the code one would write to replicate this functionality. * The SQLite closure table extension gained support for many-to-many relationships thanks to a nice PR by @necoro. [Docs](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#ClosureTable). [View commits](https://github.com/coleifer/peewee/compare/2.8.8...2.9.0) ## 2.8.8 This release contains a single important bugfix for a regression in specifying the type of lock to use when opening a SQLite transaction. [View commits](https://github.com/coleifer/peewee/compare/2.8.7...2.8.8) ## 2.8.7 This release contains numerous cleanups. ### Bugs fixed * #1087 - Fixed a misuse of the iteration protocol in the `sqliteq` extension. * Ensure that driver exceptions are wrapped when calling `commit` and `rollback`. * #1096 - Fix representation of recursive foreign key relations when using the `model_to_dict` helper. * #1126 - Allow `pskel` to be installed into `bin` directory. * #1105 - Added a `Tuple()` type to Peewee to enable expressing arbitrary tuple expressions in SQL. * #1133 - Fixed bug in the conversion of objects to `Decimal` instances in the `DecimalField`. * Fixed an issue renaming a unique foreign key in MySQL. * Remove the join predicate from CROSS JOINs. * #1148 - Ensure indexes are created when a column is added using a schema migration. * #1165 - Fix bug where the primary key was being overwritten in queries using the closure-table extension. ### New stuff * Added properties to the `SqliteExtDatabase` to expose common `PRAGMA` settings. For example, to set the cache size to 4MB, `db.cache_size = 1000`. * Clarified documentation on calling `commit()` or `rollback()` from within the scope of an atomic block. [See docs](http://docs.peewee-orm.com/en/latest/peewee/transactions.html#transactions). * Allow table creation dependencies to be specified using new `depends_on` meta option. Refs #1076. * Allow specification of the lock type used in SQLite transactions. Previously this behavior was only present in `playhouse.sqlite_ext.SqliteExtDatabase`, but it now exists in `peewee.SqliteDatabase`. * Added support for `CROSS JOIN` expressions in select queries. * Docs on how to implement [optimistic locking](http://docs.peewee-orm.com/en/latest/peewee/hacks.html#optimistic-locking). * Documented optional dependencies. * Generic support for specifying select queries as locking the selected rows `FOR X`, e.g. `FOR UPDATE` or `FOR SHARE`. * Support for specifying the frame-of-reference in window queries, e.g. specifying `UNBOUNDED PRECEDING`, etc. [See docs](http://docs.peewee-orm.com/en/latest/peewee/api.html#Window). ### Backwards-incompatible changes * As of 9e76c99, an `OperationalError` is raised if the user calls `connect()` on an already-open Database object. Previously, the existing connection would remain open and a new connection would overwrite it, making it impossible to close the previous connection. If you find this is causing breakage in your application, you can switch the `connect()` call to `get_conn()` which will only open a connection if necessary. The error **is** indicative of a real issue, though, so audit your code for places where you may be opening a connection without closing it (module-scope operations, e.g.). [View commits](https://github.com/coleifer/peewee/compare/2.8.5...2.8.7) ## 2.8.6 This release was later removed due to containing a bug. See notes on 2.8.7. ## 2.8.5 This release contains two small bugfixes. * #1081 - fixed the use of parentheses in compound queries on MySQL. * Fixed some grossness in a helper function used by `prefetch` that was clearing out the `GROUP BY` and `HAVING` clauses of sub-queries. [View commits](https://github.com/coleifer/peewee/compare/2.8.4...2.8.5) ## 2.8.4 This release contains bugfixes as well as a new playhouse extension module for working with [SQLite in multi-threaded / concurrent environments](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#sqliteq). The new module is called `playhouse.sqliteq` and it works by serializing queries using a dedicated worker thread (or greenlet). The performance is quite good, hopefully this proves useful to someone besides myself! You can learn more by reading the [sqliteq documentation](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#sqliteq). As a miscellaneous note, I did some major refactoring and cleanup in `ExtQueryResultsWrapper` and it's corollary in the `speedups` module. The code is much easier to read than before. [View commits](https://github.com/coleifer/peewee/compare/2.8.3...2.8.4) ### Bugs fixed * #1061 - @akrs patched a bug in `TimestampField` which affected the accuracy of sub-second timestamps (for resolution > 1). * #1071, small python 3 fix. * #1072, allow `DeferredRelation` to be used multiple times if there are multiple references to a given deferred model. * #1073, fixed regression in the speedups module that caused SQL functions to always coerce return values, regardless of the `coerce` flag. * #1083, another Python 3 issue - this time regarding the use of `exc.message`. [View commits](https://github.com/coleifer/peewee/compare/2.8.3...2.8.4) ## 2.8.3 This release contains bugfixes and a small backwards-incompatible change to the way foreign key `ObjectIdDescriptor` is named (issue #1050). ### Bugs fixed and general changes * #1028 - allow the `ensure_join` method to accept `on` and `join_type` parameters. Thanks @paulbooth. * #1032 - fix bug related to coercing model instances to database parameters when the model's primary key is a foreign key. * #1035 - fix bug introduced in 2.8.2, where I had added some logic to try and restrict the base `Model` class from being treated as a "real" Model. * #1039 - update documentation to clarify that lists *or tuples* are acceptable values when specifying SQLite `PRAGMA` statements. * #1041 - PyPy user was unable to install Peewee. (Who in their right mind would *ever* use PyPy?!) Bug was fixed by removing the pre-generated C files from the distribution. * #1043 - fix bug where the `speedups` C extension was not calling the correct model initialization method, resulting in model instances returned as results of a query having their `dirty` flag incorrectly set. * #1048 - similar to #1043, add logic to ensure that fields with default values are considered dirty when instantiating the model. * #1049 - update URL to [APSW](https://rogerbinns.github.io/apsw). * Fixed unreported bug regarding `TimestampField` with zero values reporting the incorrect datetime. ### New stuff * [djpeewee](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#djpeewee) extension module now works with Django 1.9. * [TimestampField](http://docs.peewee-orm.com/en/latest/peewee/api.html#TimestampField) is now an officially documented field. * #1050 - use the `db_column` of a `ForeignKeyField` for the name of the `ObjectIdDescriptor`, except when the `db_column` and field `name` are the same, in which case the ID descriptor will be named `_id`. [View commits](https://github.com/coleifer/peewee/compare/2.8.2...2.8.3) ## 2.8.2 This release contains mostly bug-fixes, clean-ups, and API enhancements. ### Bugs fixed and general cleanups * #820 - fixed some bugs related to the Cython extension build process. * #858 - allow blanks and perform type conversion when using the `db_url` extension * #922 - ensure that `peewee.OperationalError` is raised consistently when using the `RetryOperationalError` mixin. * #929 - ensure that `pwiz` will import the appropriate extensions when vendor-specific fields are used. * #930 - ensure that `pwiz`-generated models containing `UnknownField` placeholders do not blow up when you instantiate them. * #932 - correctly limit the length of automatically-generated index names. * #933 - fixed bug where `BlobField` could not be used if it's parent model pointed to an uninitialized database `Proxy`. * #935 - greater consistency with the conversion to Python data-types when performing aggregations, annotations, or calling `scalar()`. * #939 - ensure the correct data-types are used when initializing a connection pool. * #947 - fix bug where `Signal` subclasses were not returning rows affected on save. * #951 - better warnings regarding C extension compilation, thanks @dhaase-de. * #968 - fix bug where table names starting with numbers generated invalid table names when using `pwiz`. * #971 - fix bug where parameter was not being used. Thanks @jberkel. * #974 - fixed the way `SqliteExtDatabase` handles the automatic `rowid` (and `docid`) columns. Thanks for alerting me to the issue and providing a failing test case @jberkel. * #976 - fix obscure bug relating to cloning foreign key fields twice. * #981 - allow `set` instances to be used on the right-hand side of `IN` exprs. * #983 - fix behavior where the default `id` primary key was inherited regardless. When users would inadvertently include it in their queries, it would use the table alias of it's parent class. * #992 - add support for `db_column` in `djpeewee` * #995 - fix the behavior of `truncate_date` with Postgresql. Thanks @Zverik. * #1011 - correctly handle `bytes` wrapper used by `PasswordField` to `bytes`. * #1012 - when selecting and joining on multiple models, do not create model instances when the foreign key is NULL. * #1017 - do not coerce the return value of function calls to `COUNT` or `SUM`, since the python driver will already give us the right Python value. * #1018 - use global state to resolve `DeferredRelations`, allowing for a nicer API. Thanks @brenguyen711. * #1022 - attempt to avoid creating invalid Python when using `pwiz` with MySQL database columns containing spaces. Yes, fucking spaces. * #1024 - fix bug in SQLite migrator which had a naive approach to fixing indexes. * #1025 - explicitly check for `None` when determining if the database has been set on `ModelOptions`. Thanks @joeyespo. ### New stuff * Added `TimestampField` for storing datetimes using integers. Greater than second delay is possible through exponentiation. * Added `Database.drop_index()` method. * Added a `max_depth` parameter to the `model_to_dict` function in the `playhouse.shortcuts` extension module. * `SelectQuery.first()` function accepts a parameter `n` which applies a limit to the query and returns the first row. Previously the limit was not applied out of consideration for subsequent iterations, but I believe usage has shown that a limit is more desirable than reserving the option to iterate without a second query. The old behavior is preserved in the new `SelectQuery.peek()` method. * `group_by()`, `order_by()`, `window()` now accept a keyward argument `extend`, which, when set to `True`, will append to the existing values rather than overwriting them. * Query results support negative indexing. * C sources are included now as part of the package. I *think* they should be able to compile for python 2 or 3, on linux or windows...but not positive. * #895 - added the ability to query using the `_id` attribute. * #948 - added documentation about SQLite limits and how they affect * #1009 - allow `DATABASE_URL` as a recognized parameter to the Flask config. `insert_many`. [View commits](https://github.com/coleifer/peewee/compare/2.8.1...2.8.2) ## 2.8.1 This release is long overdue so apologies if you've been waiting on it and running off master. There are numerous bugfixes contained in this release, so I'll list those first this time. ### Bugs fixed * #821 - issue warning if Cython is old * #822 - better handling of MySQL connections point for advanced use-cases. * #313 - support equality/inequality with generic foreign key queries, and ensure `get_or_create` works with GFKs. * #834 - fixed Python3 incompatibilities in the `PasswordField`, thanks @mosquito. * #836 - fix handling of `last_insert_id()` when using `APSWDatabase`. * #845 - add connection hooks to `APSWDatabase`. * #852 - check SQLite library version to avoid calls to missing APIs. * #857 - allow database definition to be deferred when using the connection pool. * #878 - formerly `.limit(0)` had no effect. Now adds `LIMIT 0`. * #879 - implement a `__hash__` method for `Model` * #886 - fix `count()` for compound select queries. * #895 - allow writing to the `foreign_key_id` descriptor to set the foreign key value. * #893 - fix boolean logic bug in `model_to_dict()`. * #904 - fix side-effect in `clean_prefetch_query`, thanks to @p.kamayev * #907 - package includes `pskel` now. * #852 - fix sqlite version check in BerkeleyDB backend. * #919 - add runtime check for `sqlite3` library to match MySQL and Postgres. Thanks @M157q ### New features * Added a number of [SQLite user-defined functions and aggregates](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#sqlite-udf). * Use the DB-API2 `Binary` type for `BlobField`. * Implemented the lucene scoring algorithm in the `sqlite_ext` Cython library. * #825 - allow a custom base class for `ModelOptions`, providing an extension * #830 - added `SmallIntegerField` type. * #838 - allow using a custom descriptor class with `ManyToManyField`. * #855 - merged change from @lez which included docs on using peewee with Pyramid. * #858 - allow arguments to be passed on query-string when using the `db_url` module. Thanks @RealSalmon * #862 - add support for `truncate table`, thanks @dev-zero for the sample code. * Allow the `related_name` model `Meta` option to be a callable that accepts the foreign key field instance. [View commits](https://github.com/coleifer/peewee/compare/2.8.0...2.8.1) ## 2.8.0 This release includes a couple new field types and greatly improved C extension support for both speedups and SQLite enhancements. Also includes some work, suggested by @foxx, to remove some places where `Proxy` was used in favor of more obvious APIs. ### New features * [travis-ci builds](http://travis-ci.org/coleifer/peewee/builds/) now include MySQL and Python 3.5. Dropped support for Python 3.2 and 3.3. Builds also will run the C-extension code. * C extension speedups now enabled by default, includes faster implementations for `dict` and `tuple` `QueryResultWrapper` classes, faster date formatting, and a faster field and model sorting. * C implementations of SQLite functions is now enabled by default. SQLite extension is now compatible with APSW and can be used in standalone form directly from Python. See [SqliteExtDatabase](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#SqliteExtDatabase) for more details. * SQLite C extension now supports `murmurhash2`. * `UUIDField` is now supported for SQLite and MySQL, using `text` and `varchar` respectively, thanks @foxx! * Added `BinaryField`, thanks again, @foxx! * Added `PickledField` to `playhouse.fields`. * `ManyToManyField` now accepts a list of primary keys when adding or removing values from the through relationship. * Added support for SQLite [table-valued functions](http://sqlite.org/vtab.html#tabfunc2) using the [sqlite-vtfunc library](https://github.com/coleifer/sqlite-vtfunc). * Significantly simplified the build process for compiling the C extensions. ### Backwards-incompatible changes * Instead of using a `Proxy` for defining circular foreign key relationships, you now need to use [DeferredRelation](http://docs.peewee-orm.com/en/latest/peewee/api.html#DeferredRelation). * Instead of using a `Proxy` for defining many-to-many through tables, you now need to use [DeferredThroughModel](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#DeferredThroughModel). * SQLite Virtual Models must now use `Meta.extension_module` and `Meta.extension_options` to declare extension and any options. For more details, see [VirtualModel](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#VirtualModel). * MySQL database will now issue `COMMIT` statements for `SELECT` queries. This was not necessary, but added due to an influx of confused users creating GitHub tickets. Hint: learn to user your damn database, it's not magic! ### Bugs fixed Some of these may have been included in a previous release, but since I did not list them I'm listing them here. * #766, fixed bug with PasswordField and Python3. Fuck Python 3. * #768, fixed SortedFieldList and `remove_field()`. Thanks @klen! * #771, clarified docs for APSW. * #773, added docs for request hooks in Pyramid (who uses Pyramid, by the way?). * #774, prefetch() only loads first ForeignKeyField for a given relation. * #782, fixed typo in docs. * #791, foreign keys were not correctly handling coercing to the appropriate python value. * #792, cleaned up some CSV utils code. * #798, cleaned up iteration protocol in QueryResultWrappers. * #806, not really a bug, but MySQL users were clowning around and needed help. [View commits](https://github.com/coleifer/peewee/compare/2.7.4...2.8.0) ## 2.7.4 This is another small release which adds code to automatically build the SQLite C extension if `libsqlite` is available. The release also includes: * Support for `UUIDField` with SQLite. * Support for registering additional database classes with the `db_url` module via `register_database`. * `prefetch()` supports fetching multiple foreign-keys to the same model class. * Added method to validate FTS5 search queries. [View commits](https://github.com/coleifer/peewee/compare/2.7.3...2.7.4) ## 2.7.3 Small release which includes some changes to the BM25 sorting algorithm and the addition of a [`JSONField`](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#JSONField) for use with the new [JSON1 extension](http://sqlite.org/json1.html). ## 2.7.2 People were having trouble building the sqlite extension. I figure enough people are having trouble that I made it a separate command: `python setup.py build_sqlite_ext`. ## 2.7.1 Jacked up the setup.py ## 2.7.0 New APIs, features, and performance improvements. ### Notable changes and new features * [`PasswordField`](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#PasswordField) that uses the `bcrypt` module. * Added new Model [`Meta.only_save_dirty`](http://docs.peewee-orm.com/en/latest/peewee/models.html#model-options-and-table-metadata) flag to, by default, only save fields that have been modified. * Added support for [`upsert()`](http://docs.peewee-orm.com/en/latest/peewee/api.html#InsertQuery.upsert) on MySQL (in addition to SQLite). * Implemented SQLite ranking functions (``rank`` and ``bm25``) in Cython, and changed both the Cython and Python APIs to accept weight values for every column in the search index. This more closely aligns with the APIs provided by FTS5. In fact, made the APIs for FTS4 and FTS5 result ranking compatible. * Major changes to the :ref:`sqlite_ext` module. Function callbacks implemented in Python were implemented in Cython (e.g. date manipulation and regex processing) and will be used if Cython is available when Peewee is installed. * Support for the experimental new [FTS5](http://sqlite.org/fts5.html) SQLite search extension. * Added :py:class:`SearchField` for use with the SQLite FTS extensions. * Added :py:class:`RowIDField` for working with the special ``rowid`` column in SQLite. * Added a model class validation hook to allow model subclasses to perform any validation after class construction. This is currently used to ensure that ``FTS5Model`` subclasses do not violate any rules required by the FTS5 virtual table. ### Bugs fixed * **#751**, fixed some very broken behavior in the MySQL migrator code. Added more tests. * **#718**, added a `RetryOperationalError` mixin that will try automatically reconnecting after a failed query. There was a bug in the previous error handler implementation that made this impossible, which is also fixed. #### Small bugs * #713, fix column name regular expression in SQLite migrator. * #724, fixed `NULL` handling with the Postgresql `JSONField`. * #725, added `__module__` attribute to `DoesNotExist` classes. * #727, removed the `commit_select` logic for MySQL databases. * #730, added documentation for `Meta.order_by` API. * #745, added `cast()` method for casting JSON field values. * #748, added docs and method override to indicate that SQLite does not support adding foreign key constraints after table creation. * Check whether pysqlite or libsqlite were compiled with BerkeleyDB support when using the :py:class:`BerkeleyDatabase`. * Clean up the options passed to SQLite virtual tables on creation. ### Small features * #700, use sensible default if field's declared data-type is not present in the field type map. * #707, allow model to be specified explicitly in `prefetch()`. * #734, automatic testing against python 3.5. * #753, added support for `upsert()` ith MySQL via the `REPLACE INTO ...` statement. * #757, `pwiz`, the schema intropsection tool, will now generate multi-column index declarations. * #756, `pwiz` will capture passwords using the `getpass()` function rather than via the command-line. * Removed `Database.sql_error_handler()`, replaced with the `RetryOperationalError` mixin class. * Documentation for `Meta.order_by` and `Meta.primary_key`. * Better documentation around column and table constraints. * Improved performance for some methods that are called frequently. * Added `coerce` parameter to `BareField` and added documentation. [View commits](https://github.com/coleifer/peewee/compare/2.6.4...2.7.0) ## 2.6.4 Updating so some of the new APIs are available on pypi. ### Bugs fixed * #646, fixed a bug with the Cython speedups not being included in package. * #654, documented how to create models with no primary key. * #659, allow bare `INSERT` statements. * #674, regarding foreign key / one-to-one relationships. * #676, allow `ArrayField` to accept tuples in addition to lists. * #679, fix regarding unsaved relations. * #682, refactored QueryResultWrapper to allow multiple independent iterations over the same underlying result cache. * #692, fix bug with multiple joins to same table + eager loading. * #695, fix bug when connection fails while using an execution context. * #698, use correct column names with non-standard django foreign keys. * #706, return `datetime.time` instead of `timedelta` for MySQL time fields. * #712, fixed SQLite migrator regular expressions. Thanks @sroebert. ### New features * #647, #649, #650, added support for `RETURNING` clauses. Update, Insert and Delete queries can now be called with `RETURNING` to retrieve the rows that were affected. [See docs](http://docs.peewee-orm.com/en/latest/peewee/querying.html#returning-clause). * #685, added web request hook docs. * #691, allowed arbitrary model attributes and methods to be serialized by `model_to_dict()`. [Docs](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#model_to_dict). * #696, allow `model_to_dict()` to introspect query for which fields to serialize. * Added backend-agnostic [truncate_date()](http://docs.peewee-orm.com/en/latest/peewee/api.html#Database.truncate_date) implementation. * Added a `FixedCharField` which uses column type `CHAR`. * Added support for arbitrary `PRAGMA` statements to be run on new SQLite connections. [Docs](http://docs.peewee-orm.com/en/latest/peewee/databases.html#sqlite-pragma). * Removed `berkeley_build.sh` script. See instructions [on my blog instead](http://charlesleifer.com/blog/building-the-python-sqlite-driver-for-use-with-berkeleydb/). [View commits](https://github.com/coleifer/peewee/compare/2.6.2...2.6.4) ## 2.6.2 Just a regular old release. ### Bugs fixed * #641, fixed bug with exception wrapping and Python 2.6 * #634, fixed bug where correct query result wrapper was not being used for certain composite queries. * #625, cleaned up some example code. * #614, fixed bug with `aggregate_rows()` when there are multiple joins to the same table. ### New features * Added [create_or_get()](http://docs.peewee-orm.com/en/latest/peewee/querying.html#create-or-get) as a companion to `get_or_create()`. * Added support for `ON CONFLICT` clauses for `UPDATE` and `INSERT` queries. [Docs](http://docs.peewee-orm.com/en/latest/peewee/api.html#UpdateQuery.on_conflict). * Added a [JSONKeyStore](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#JSONKeyStore) to `playhouse.kv`. * Added Cythonized version of `strip_parens()`, with plans to perhaps move more performance-critical code to Cython in the future. * Added docs on specifying [vendor-specific database parameters](http://docs.peewee-orm.com/en/latest/peewee/database.html#vendor-specific-parameters). * Added docs on specifying [field default values](http://docs.peewee-orm.com/en/latest/peewee/models.html#default-field-values) (both client and server-side). * Added docs on [foreign key field back-references](http://docs.peewee-orm.com/en/latest/peewee/models.html#foreignkeyfield). * Added docs for [models without a primary key](http://docs.peewee-orm.com/en/latest/peewee/models.html#models-without-a-primary-key). * Cleaned up docs on `prefetch()` and `aggregate_rows()`. [View commits](https://github.com/coleifer/peewee/compare/2.6.1...2.6.2) ## 2.6.1 This release contains a number of small fixes and enhancements. ### Bugs fixed * #606, support self-referential joins with `prefetch` and `aggregate_rows()` methods. * #588, accomodate changes in SQLite's `PRAGMA index_list()` return value. * #607, fixed bug where `pwiz` was not passing table names to introspector. * #591, fixed bug with handling of named cursors in older psycopg2 version. * Removed some cruft from the `APSWDatabase` implementation. ### New features * Added [CompressedField](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#CompressedField) and [AESEncryptedField](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#AESEncryptedField) * #609, #610, added Django-style foreign key ID lookup. [Docs](http://docs.peewee-orm.com/en/latest/peewee/models.html#foreignkeyfield). * Added support for [Hybrid Attributes](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#hybrid-attributes) (cool idea courtesy of SQLAlchemy). * Added ``upsert`` keyword argument to the `Model.save()` function (SQLite only). * #587, added support for ``ON CONFLICT`` SQLite clause for `INSERT` and `UPDATE` queries. [Docs](http://docs.peewee-orm.com/en/latest/peewee/api.html#UpdateQuery.on_conflict) * #601, added hook for programmatically defining table names. [Model options docs](http://docs.peewee-orm.com/en/latest/peewee/models.html#model-options-and-table-metadata) * #581, #611, support connection pools with `playhouse.db_url.connect()`. [Docs](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#connect). * Added [Contributing section](http://docs.peewee-orm.com/en/latest/peewee/contributing.html) section to docs. [View commits](https://github.com/coleifer/peewee/compare/2.6.0...2.6.1) ## 2.6.0 This is a tiny update, mainly consisting of a new-and-improved implementation of ``get_or_create()`` ([docs](http://docs.peewee-orm.com/en/latest/peewee/api.html#Model.get_or_create)). ### Backwards-incompatible changes * ``get_or_create()`` now returns a 2-tuple consisting of the model instance and a boolean indicating whether the instance was created. The function now behaves just like the Django equivalent. ### New features * #574, better support for setting the character encoding on Postgresql database connections. Thanks @klen! * Improved implementation of [get_or_create()](http://docs.peewee-orm.com/en/latest/peewee/api.html#Model.get_or_create). [View commits](https://github.com/coleifer/peewee/compare/2.5.1...2.6.0) ## 2.5.1 This is a relatively small release with a few important bugfixes. ### Bugs fixed * #566, fixed a bug regarding parentheses around compound `SELECT` queries (i.e. `UNION`, `INTERSECT`, etc). * Fixed unreported bug where table aliases were not generated correctly for compound `SELECT` queries. * #559, add option to preserve original column order with `pwiz`. Thanks @elgow! * Fixed unreported bug where selecting all columns from a `ModelAlias` does not use the appropriate `FieldAlias` objects. ### New features * #561, added an option for bulk insert queries to return the list of auto-generated primary keys. See [docs for InsertQuery.return_id_list](http://docs.peewee-orm.com/en/latest/peewee/api.html#InsertQuery.return_id_list). * #569, added `parse` function to the `playhouse.db_url` module. Thanks @stt! * Added [hacks](http://docs.peewee-orm.com/en/latest/peewee/hacks.html) section to the docs. Please contribute your hacks! ### Backwards-incompatible changes * Calls to `Node.in_()` and `Node.not_in()` do not take `*args` anymore and instead take a single argument. [View commits](https://github.com/coleifer/peewee/compare/2.5.0...2.5.1) ## 2.5.0 There are a couple new features so I thought I'd bump to 2.5.x. One change Postgres users may be happy to see is the use of `INSERT ... RETURNING` to perform inserts. This should definitely speed up inserts for Postgres, since an extra query is no longer needed to get the new auto-generated primary key. I also added a [new context manager/decorator](http://docs.peewee-orm.com/en/latest/peewee/database.html#using-multiple-databases) that allows you to use a different database for the duration of the wrapped block. ### Bugs fixed * #534, CSV utils was erroneously stripping the primary key from CSV data. * #537, fix upserts when using `insert_many`. * #541, respect `autorollback` with `PostgresqlExtDatabase`. Thanks @davidmcclure. * #551, fix for QueryResultWrapper's implementation of the iterator protocol. * #554, allow SQLite journal_mode to be set at run-time. * Fixed case-sensitivity issue with `DataSet`. ### New features * Added support for [CAST expressions](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#cast). * Added a hook for [extending Node](http://docs.peewee-orm.com/en/latest/peewee/api.html#Node.extend) with custom methods. * `JOIN_` became `JOIN.`, e.g. `.join(JOIN.LEFT_OUTER)`. * `OP_` became `OP.`. * #556, allowed using `+` and `-` prefixes to indicate ascending/descending ordering. * #550, added [Database.initialize_connection()](http://docs.peewee-orm.com/en/latest/peewee/database.html#additional-connection-initialization) hook. * #549, bind selected columns to a particular model. Thanks @jhorman, nice PR! * #531, support for swapping databases at run-time via [Using](http://docs.peewee-orm.com/en/latest/peewee/database.html#using-multiple-databases). * #530, support for SQLCipher and Python3. * New `RowIDField` for `sqlite_ext` playhouse module. This field can be used to interact with SQLite `rowid` fields. * Added `LateralJoin` helper to the `postgres_ext` playhouse module. * New [example blog app](https://github.com/coleifer/peewee/tree/master/examples/blog). [View commits](https://github.com/coleifer/peewee/compare/2.4.7...2.5.0) ## 2.4.7 ### Bugs fixed * #504, Docs updates. * #506, Fixed regression in `aggregate_rows()` * #510, Fixes bug in pwiz overwriting columns. * #514, Correctly cast foreign keys in `prefetch()`. * #515, Simplifies queries issued when doing recursive deletes. * #516, Fix cloning of Field objects. * #519, Aggregate rows now correctly preserves ordering of joined instances. * Unreported, fixed bug to not leave expired connections sitting around in the pool. ### New features * Added support for Postgresql's ``jsonb`` type with [BinaryJSONField](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#BinaryJSONField). * Add some basic [Flask helpers](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#flask-utils). * Add support for `UNION ALL` queries in #512 * Add `SqlCipherExtDatabase`, which combines the sqlcipher database with the sqlite extensions. * Add option to print metadata when generating code with ``pwiz``. [View commits](https://github.com/coleifer/peewee/compare/2.4.6...2.4.7) ## 2.4.6 This is a relatively small release with mostly bug fixes and updates to the documentation. The one new feature I'd like to highlight is the ``ManyToManyField`` ([docs](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#ManyToManyField)). ### Bugs fixed * #503, fixes behavior of `aggregate_rows()` when used with a `CompositeKey`. * #498, fixes value coercion for field aliases. * #492, fixes bug with pwiz and composite primary keys. * #486, correctly handle schemas with reflection module. ### New features * Peewee has a new [ManyToManyField](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#ManyToManyField) available in the ``playhouse.shortcuts`` module. * Peewee now has proper support for *NOT IN* queries through the ``Node.not_in()`` method. * Models now support iteration. This is equivalent to ``Model.select()``. [View commits](https://github.com/coleifer/peewee/compare/2.4.5...2.4.6) ## 2.4.5 I'm excited about this release, as in addition to a number of new features and bugfixes, it also is a step towards cleaner code. I refactored the tests into a number of modules, using a standard set of base test-cases and helpers. I also introduced the `mock` library into the test suite and plan to use it for cleaner tests going forward. There's a lot of work to do to continue cleaning up the tests, but I'm feeling good about the changes. Curiously, the test suite runs faster now. ### Bugs fixed * #471, #482 and #484, all of which had to do with how joins were handled by the `aggregate_rows()` query result wrapper. * #472 removed some needless special-casing in `Model.save()`. * #466 fixed case-sensitive issues with the SQLite migrator. * #474 fixed a handful of bugs that cropped up migrating foreign keys with SQLite. * #475 fixed the behavior of the SQLite migrator regarding auto-generated indexes. * #479 fixed a bug in the code that stripped extra parentheses in the SQL generator. * Fixed a handful of bugs in the APSW extension. ### New features * Added connection abstraction called `ExecutionContext` ([see docs](http://docs.peewee-orm.com/en/latest/peewee/database.html#advanced-connection-management)). * Made all context managers work as decorators (`atomic`, `transaction`, `savepoint`, `execution_context`). * Added explicit methods for `IS NULL` and `IS NOT NULL` queries. The latter was actually necessary since the behavior is different from `NOT IS NULL (...)`. * Allow disabling backref validation (#465) * Made quite a few improvements to the documentation, particularly sections on transactions. * Added caching to the [DataSet](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#dataset) extension, which should improve performance. * Made the SQLite migrator smarter with regards to preserving indexes when a table copy is necessary. [View commits](https://github.com/coleifer/peewee/compare/2.4.4...2.4.5) ## 2.4.4 Biggest news: peewee has a new logo! ![](https://media.charlesleifer.com/blog/photos/peewee-logo-bold.png) * Small documentation updates here and there. ### Backwards-incompatible changes * The argument signature for the `SqliteExtDatabase.aggregate()` decorator changed so that the aggregate name is the first parameter, and the number of parameters is the second parameter. If no values are specified, peewee will choose the name of the class and an un-specified number of arguments (`-1`). * The logic for saving a model with a composite key changed slightly. Previously, if a model had a composite primary key and you called `save()`, only the dirty fields would be saved. ### Bugs fixed * #462 * #465, add hook for disabling backref validation. * #466, fix case-sensitive table names with migration module. * #469, save only dirty fields. ### New features * Lots of enhancements and cleanup to the `playhouse.apsw_ext` module. * The `playhouse.reflection` module now supports introspecting indexes. * Added a model option for disabling backref validation. * Added support for the SQLite [closure table extension](http://charlesleifer.com/blog/querying-tree-structures-in-sqlite-using-python-and-the-transitive-closure-extension/). * Added support for *virtual fields*, which act on dynamically-created virtual table fields. * Added a new example: a virtual table implementation that exposes Redis as a relational database table. * Added a module `playhouse.sqlite_aggregates` that contains a handful of aggregates you may find useful when developing with SQLite. [View commits](https://github.com/coleifer/peewee/compare/2.4.3...2.4.4) ## 2.4.3 This release contains numerous improvements, particularly around the built-in database introspection utilities. Peewee should now also be compatible with PyPy. ### Bugs fixed * #466, table names are case sensitive in the SQLite migrations module. * #465, added option to disable backref validation. * #462, use the schema name consistently with postgres reflection. ### New features * New model *Meta* option to disable backref validation. [See validate_backrefs](http://docs.peewee-orm.com/en/latest/peewee/models.html#model-options-and-table-metadata). * Added documentation on ordering by calculated values. * Added basic PyPy compatibility. * Added logic to close cursors after they have been exhausted. * Structured and consolidated database metadata introspection, including improvements for introspecting indexes. * Added support to [prefetch](http://docs.peewee-orm.com/en/latest/peewee/api.html?highlight=prefetch#prefetch) for traversing *up* the query tree. * Added introspection option to skip invalid models while introspecting. * Added option to limit the tables introspected. * Added closed connection detection to the MySQL connection pool. * Enhancements to passing options to creating virtual tables with SQLite. * Added factory method for generating Closure tables for use with the `transitive_closure` SQLite extension. * Added support for loading SQLite extensions. * Numerous test-suite enhancements and new test-cases. [View commits](https://github.com/coleifer/peewee/compare/2.4.2...2.4.3) ## 2.4.2 This release contains a number of improvements to the `reflection` and `migrate` extension modules. I also added an encrypted *diary* app to the [examples](https://github.com/coleifer/peewee/tree/master/examples) directory. ### Bugs fixed * #449, typo in the db_url extension, thanks to @malea for the fix. * #457 and #458, fixed documentation deficiences. ### New features * Added support for [importing data](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#importing-data) when using the [DataSet extension](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#dataset). * Added an encrypted diary app to the examples. * Better index reconstruction when altering columns on SQLite databases with the [migrate](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#migrate) module. * Support for multi-column primary keys in the [reflection](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#reflection) module. * Close cursors more aggressively when executing SELECT queries. [View commits](https://github.com/coleifer/peewee/compare/2.4.1...2.4.2) ## 2.4.1 This release contains a few small bugfixes. ### Bugs fixed * #448, add hook to the connection pool for detecting closed connections. * #229, fix join attribute detection. * #447, fixed documentation typo. [View commits](https://github.com/coleifer/peewee/compare/2.4.0...2.4.1) ## 2.4.0 This release contains a number of enhancements to the `playhouse` collection of extensions. ### Backwards-incompatible changes As of 2.4.0, most of the introspection logic was moved out of the ``pwiz`` module and into ``playhouse.reflection``. ### New features * Created a new [reflection](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#reflection) extension for introspecting databases. The *reflection* module additionally can generate actual peewee Model classes dynamically. * Created a [dataset](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#dataset) library (based on the [SQLAlchemy project](https://dataset.readthedocs.io/) of the same name). For more info check out the blog post [announcing playhouse.dataset](http://charlesleifer.com/blog/saturday-morning-hacks-dataset-for-peewee/). * Added a [db_url](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#database-url) module which creates `Database` objects from a connection string. * Added [csv dump](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#dumping-csv) functionality to the [CSV utils](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#csv-utils) extension. * Added an [atomic](http://docs.peewee-orm.com/en/latest/peewee/transactions.html#nesting-transactions) context manager to support nested transactions. * Added support for HStore, JSON and TSVector to the `reflection` module. * More documentation updates. ### Bugs fixed * Fixed #440, which fixes a bug where `Model.dirty_fields` did not return an empty set for some subclasses of `QueryResultWrapper`. [View commits](https://github.com/coleifer/peewee/compare/2.3.3...2.4.0) ## 2.3.3 This release contains a lot of improvements to the documentation and a mixed bag of other new features and bugfixes. ### Backwards-incompatible changes As of 2.3.3, all peewee `Database` instances have a default of `True` for the `threadlocals` parameter. This means that a connection is opened for each thread. It seemed to me that by sharing connections across threads caused a lot of confusion to users who weren't aware of (or familiar with) the `threadlocals` parameter. For single-threaded apps the behavior will not be affected, but for multi-threaded applications, if you wish to share your connection across threads you must now specify `threadlocals=False`. For more information, see the [documentation](http://docs.peewee-orm.com/en/latest/peewee/api.html#Database). I also renamed the `Model.get_id()` and `Model.set_id()` convenience methods so as not to conflict with Flask-Login. These methods should have probably been private anyways, and the new methods are named `_get_pk_value()` and `_set_pk_value()`. ### New features * Basic support for [Postgresql full-text search](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#pg-fts). * Helper functions for converting models to dictionaries and unpacking dictionaries into model instances. See [docs](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#model_to_dict). ### Bugs fixed * Fixed #428, documentation formatting error. * Fixed #429, which fixes the way default values are initialized for bulk inserts. * Fixed #432, making the HStore extension optional when using `PostgresqlExtDatabase`. * Fixed #435, allowing peewee to be used with Flask-Login. * Fixed #436, allowing the SQLite date_part and date_trunc functions to correctly handle NULL values. * Fixed #438, in which the ordering of clauses in a Join expression were causing unpredictable behavior when selecting related instances. * Updated the `berkeley_build.sh` script, which was incompatible with the newest version of `bsddb3`. [View commits](https://github.com/coleifer/peewee/compare/2.3.2...2.3.3) ## 2.3.2 This release contains mostly bugfixes. ### Changes in 2.3.2 * Fixed #421, allowing division operations to work correctly in py3k. * Added support for custom json.dumps command, thanks to @alexlatchford. * Fixed some foreign key generation bugs with pwiz in #426. * Fixed a parentheses bug with UNION queries, #422. * Added support for returning partial JSON data-structures from postgresql. [View commits](https://github.com/coleifer/peewee/compare/2.3.1...2.3.2) ## 2.3.1 This release contains a fix for a bug introducted in 2.3.0. Table names are included, unquoted, in update queries now, which is causing some problems when the table name is a keyword. ### Changes in 2.3.1 * [Quote table name / alias](https://github.com/coleifer/peewee/issues/414) [View commits](https://github.com/coleifer/peewee/compare/2.3.0...2.3.1) ## 2.3.0 This release contains a number of bugfixes, enhancements and a rewrite of much of the documentation. ### Changes in 2.3.0 * [New and improved documentation](http://docs.peewee-orm.com/) * Added [aggregate_rows()](http://docs.peewee-orm.com/en/latest/peewee/querying.html#list-users-and-all-their-tweets) method for mitigating N+1 queries. * Query compiler performance improvements and rewrite of table alias internals (51d82fcd and d8d55df04). * Added context-managers and decorators for [counting queries](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#count_queries) and [asserting query counts](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#assert_query_count). * Allow `UPDATE` queries to contain subqueries for values ([example](http://docs.peewee-orm.com/en/latest/peewee/querying.html#atomic-updates)). * Support for `INSERT INTO / SELECT FROM` queries ([docs](http://docs.peewee-orm.com/en/latest/peewee/api.html?highlight=insert_from#Model.insert_from)). * Allow `SqliteDatabase` to set the database's journal mode. * Added method for concatenation ([docs]()). * Moved ``UUIDField`` out of the playhouse and into peewee * Added [pskel](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#pskel) script. * Documentation for [BerkeleyDB](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#berkeleydb). ### Bugs fixed * #340, allow inner query values to be used in outer query joins. * #380, fixed foreign key handling in SQLite migrations. * #389, mark foreign keys as dirty on assignment. * #391, added an ``orwhere()`` method. * #392, fixed ``order_by`` meta option inheritance bug. * #394, fixed UUID and conversion of foreign key values (thanks @alexlatchford). * #395, allow selecting all columns using ``SQL('*')``. * #396, fixed query compiler bug that was adding unnecessary parentheses around expressions. * #405, fixed behavior of ``count()`` when query has a limit or offset. [View commits](https://github.com/coleifer/peewee/compare/2.2.5...2.3.0) ## 2.2.5 This is a small release and contains a handful of fixes. ### Changes in 2.2.5 * Added a `Window` object for creating reusable window definitions. * Added support for `DISTINCT ON (...)`. * Added a BerkeleyDB-backed sqlite `Database` and build script. * Fixed how the `UUIDField` handles `None` values (thanks @alexlatchford). * Fixed various things in the example app. * Added 3.4 to the travis build (thanks @frewsxcv). [View commits](https://github.com/coleifer/peewee/compare/2.2.4...2.2.5) ## 2.2.4 This release contains a complete rewrite of `pwiz` as well as some improvements to the SQLite extension, including support for the BM25 ranking algorithm for full-text searches. I also merged support for sqlcipher, an encrypted SQLite database with many thanks to @thedod! ### Changes in 2.2.4 * Rewrite of `pwiz`, schema introspection utility. * `Model.save()` returns a value indicating the number of modified rows. * Fixed bug with `PostgresqlDatabase.last_insert_id()` leaving a transaction open in autocommit mode (#353). * Added BM25 ranking algorithm for full-text searches with SQLite. [View commits](https://github.com/coleifer/peewee/compare/2.2.3...2.2.4) ## 2.2.3 This release contains a new migrations module in addition to a number of small features and bug fixes. ### Changes in 2.2.3 * New migrations module. * Added a return value to `Model.save()` indicating number of rows affected. * Added a `date_trunc()` method that works for Sqlite. * Added a `Model.sqlall()` class-method to return all the SQL to generate the model / indices. ### Bugs fixed * #342, allow functions to not coerce parameters automatically. * #338, fixed unaliased columns when using Array and Json fields with postgres, thanks @mtwesley. * #331, corrected issue with the way unicode arrays were adapted with psycopg2. * #328, pwiz / mysql bug. * #326, fixed calculation of the alias_map when using subqueries. * #324, bug with `prefetch()` not selecting the correct primary key. [View commits](https://github.com/coleifer/peewee/compare/2.2.2...2.2.3) ## 2.2.1 I've been looking forward to this release, as it contains a couple new features that I've been wanting to add for some time now. Hope you find them useful. ### Changes in 2.2.1 * Window queries using ``OVER`` syntax. * Compound query operations ``UNION``, ``INTERSECT``, ``EXCEPT`` as well as symmetric difference. ### Bugs fixed * #300, pwiz was not correctly interpreting some foreign key constraints in SQLite. * #298, drop table with cascade API was missing. * #294, typo. [View commits](https://github.com/coleifer/peewee/compare/2.2.0...2.2.1) ## 2.2.0 This release contains a large refactoring of the way SQL was generated for both the standard query classes (`Select`, `Insert`, `Update`, `Delete`) as well as for the DDL methods (`create_table`, `create_index`, etc). Instead of joining strings of SQL and manually quoting things, I've created `Clause` objects containing multiple `Node` objects to represent all parts of the query. I also changed the way peewee determins the SQL to represent a field. Now a field implements ``__ddl__`` and ``__ddl_column__`` methods. The former creates the entire field definition, e.g.: "quoted_column_name" [NOT NULL/PRIMARY KEY/DEFAULT NEXTVAL(...)/CONSTRAINTS...] The latter method is responsible just for the column type definition. This might return ``VARCHAR(255)`` or simply ``TEXT``. I've also added support for arbitrary constraints on each field, so you might have: price = DecimalField(decimal_places=2, constraints=[Check('price > 0')]) ### Changes in 2.2.0 * Refactored query generation for both SQL queries and DDL queries. * Support for arbitrary column constraints. * `autorollback` option to the `Database` class that will roll back the transaction before raising an exception. * Added `JSONField` type to the `postgresql_ext` module. * Track fields that are explicitly set, allowing faster saves (thanks @soasme). * Allow the `FROM` clause to be an arbitrary `Node` object (#290). * `schema` is a new `Model.Mketa` option and is used throughout the code. * Allow indexing operation on HStore fields (thanks @zdxerr, #293). ### Bugs fixed * #277 (where calls not chainable with update query) * #278, use `wraps()`, thanks @lucasmarshall * #284, call `prepared()` after `create()`, thanks @soasme. * #286, cursor description issue with pwiz + postgres [View commits](https://github.com/coleifer/peewee/compare/2.1.7...2.2.0) ## 2.1.7 ### Changes in 2.1.7 * Support for savepoints (Sqlite, Postgresql and MySQL) using an API similar to that of transactions. * Common set of exceptions to wrap DB-API 2 driver-specific exception classes, e.g. ``peewee.IntegrityError``. * When pwiz cannot determine the underlying column type, display it in a comment in the generated code. * Support for circular foreign-keys. * Moved ``Proxy`` into peewee (previously in ``playhouse.proxy``). * Renamed ``R()`` to ``SQL()``. * General code cleanup, some new comments and docstrings. ### Bugs fixed * Fixed a small bug in the way errors were handled in transaction context manager. * #257 * #265, nest multiple calls to functions decorated with `@database.commit_on_success`. * #266 * #267 Commits: https://github.com/coleifer/peewee/compare/2.1.6...2.1.7 Released 2013-12-25 ## 2.1.6 Changes included in 2.1.6: * [Lightweight Django integration](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#django-integration). * Added a [csv loader](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#csv-loader) to playhouse. * Register unicode converters per-connection instead of globally when using `pscyopg2`. * Fix for how the related object cache is invalidated (#243). Commits: https://github.com/coleifer/peewee/compare/2.1.5...2.1.6 Released 2013-11-19 ## 2.1.5 ### Summary of new features * Rewrote the ``playhouse.postgres_ext.ServerSideCursor`` helper to work with a single query. [Docs](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#server-side-cursors). * Added error handler hook to the database class, allowing your code to choose how to handle errors executing SQL. [Docs](http://docs.peewee-orm.com/en/latest/peewee/api.html#Database.sql_error_handler). * Allow arbitrary attributes to be stored in ``Model.Meta`` a5e13bb26d6196dbd24ff228f99ff63d9c046f79. * Support for composite primary keys (!!). [How-to](http://docs.peewee-orm.com/en/latest/peewee/cookbook.html#composite-primary-keys) and [API docs](http://docs.peewee-orm.com/en/latest/peewee/api.html#CompositeKey). * Added helper for generating ``CASE`` expressions. [Docs](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#case). * Allow the table alias to be specified as a model ``Meta`` option. * Added ability to specify ``NOWAIT`` when issuing ``SELECT FOR UPDATE`` queries. ### Bug fixes * #147, SQLite auto-increment behavior. * #222 * #223, missing call to ``execute()`` in docs. * #224, python 3 compatibility fix. * #227, was using wrong column type for boolean with MySQL. Commits: https://github.com/coleifer/peewee/compare/2.1.4...2.1.5 Released 2013-10-19 ## 2.1.4 * Small refactor of some components used to represent expressions (mostly better names). * Support for [Array fields](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#ArrayField) in postgresql. * Added notes on [Proxy](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#proxy) * Support for [Server side cursors](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#server-side-cursors) with postgresql. * Code cleanups for more consistency. Commits: https://github.com/coleifer/peewee/compare/2.1.3...2.1.4 Released 2013-08-05 ## 2.1.3 * Added the ``sqlite_ext`` module, including support for virtual tables, full-text search, user-defined functions, collations and aggregates, as well as more granular locking. * Manually convert data-types when doing simple aggregations - fixes issue #208 * Profiled code and dramatically increased performance of benchmarks. * Added a proxy object for lazy database initialization - fixes issue #210 Commits: https://github.com/coleifer/peewee/compare/2.1.2...2.1.3 Released 2013-06-28 ------------------------------------- ## 2.0.0 Major rewrite, see notes here: http://docs.peewee-orm.com/en/latest/peewee/upgrading.html#upgrading ================================================ FILE: LICENSE ================================================ Copyright (c) 2010 Charles Leifer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: MANIFEST.in ================================================ include CHANGELOG.md include LICENSE include README.rst include TODO.rst include pyproject.toml include runtests.py include tests.py include playhouse/*.c include playhouse/*.pyx include playhouse/README.md recursive-include examples * recursive-include docs * ================================================ FILE: README.rst ================================================ .. image:: https://media.charlesleifer.com/blog/photos/peewee4-logo.png peewee ====== Peewee is a simple and small ORM. It has few (but expressive) concepts, making it easy to learn and intuitive to use. * a small, expressive ORM * flexible query-builder that exposes full power of SQL * supports sqlite, mysql, mariadb, postgresql * asyncio support * tons of extensions * use with `flask `__, `fastapi `__, `pydantic `__, and `more `__. New to peewee? These may help: * `Quickstart `_ * `Example twitter app `_ * `Using peewee interactively `_ * `Models and fields `_ * `Querying `_ * `Relationships and joins `_ * `Extensive library of SQL / Peewee examples `_ * `Flask setup `_ or `FastAPI setup `_ Installation: .. code-block:: console pip install peewee Sqlite comes built-in provided by the standard-lib ``sqlite3`` module. Other backends can be installed using the following instead: .. code-block:: console pip install peewee[mysql] # Install peewee with pymysql. pip install peewee[postgres] # Install peewee with psycopg2. pip install peewee[psycopg3] # Install peewee with psycopg3. # AsyncIO implementations. pip install peewee[aiosqlite] # Install peewee with aiosqlite. pip install peewee[aiomysql] # Install peewee with aiomysql. pip install peewee[asyncpg] # Install peewee with asyncpg. Examples -------- Defining models is similar to Django or SQLAlchemy: .. code-block:: python from peewee import * import datetime db = SqliteDatabase('my_database.db') class BaseModel(Model): class Meta: database = db class User(BaseModel): username = CharField(unique=True) class Tweet(BaseModel): user = ForeignKeyField(User, backref='tweets') message = TextField() created_date = DateTimeField(default=datetime.datetime.now) is_published = BooleanField(default=True) Connect to the database and create tables: .. code-block:: python db.connect() db.create_tables([User, Tweet]) Create a few rows: .. code-block:: python charlie = User.create(username='charlie') huey = User(username='huey') huey.save() # No need to set `is_published` or `created_date` since they # will just use the default values we specified. Tweet.create(user=charlie, message='My first tweet') Queries are expressive and composable: .. code-block:: python # A simple query selecting a user. User.get(User.username == 'charlie') # Get tweets created by one of several users. usernames = ['charlie', 'huey', 'mickey'] users = User.select().where(User.username.in_(usernames)) tweets = Tweet.select().where(Tweet.user.in_(users)) # We could accomplish the same using a JOIN: tweets = (Tweet .select() .join(User) .where(User.username.in_(usernames))) # How many tweets were published today? tweets_today = (Tweet .select() .where( (Tweet.created_date >= datetime.date.today()) & (Tweet.is_published == True)) .count()) # Paginate the user table and show me page 3 (users 41-60). User.select().order_by(User.username).paginate(3, 20) # Order users by the number of tweets they've created: tweet_ct = fn.Count(Tweet.id) users = (User .select(User, tweet_ct.alias('ct')) .join(Tweet, JOIN.LEFT_OUTER) .group_by(User) .order_by(tweet_ct.desc())) # Do an atomic update (for illustrative purposes only, imagine a simple # table for tracking a "count" associated with each URL). We don't want to # naively get the save in two separate steps since this is prone to race # conditions. Counter.update(count=Counter.count + 1).where(Counter.url == request.url) Check out the `example twitter app `_. Learning more ------------- Check the `documentation `_ for more examples. Specific question? Come hang out in the #peewee channel on irc.libera.chat, or post to the mailing list, http://groups.google.com/group/peewee-orm . If you would like to report a bug, `create a new issue `_ on GitHub. Still want more info? --------------------- .. image:: https://media.charlesleifer.com/blog/photos/wat.jpg I've written a number of blog posts about building applications and web-services with peewee (and usually Flask). If you'd like to see some real-life applications that use peewee, the following resources may be useful: * `Building a note-taking app with Flask and Peewee `_ as well as `Part 2 `_ and `Part 3 `_. * `Analytics web service built with Flask and Peewee `_. * `Personalized news digest (with a boolean query parser!) `_. * `Structuring Flask apps with Peewee `_. * `Creating a lastpass clone with Flask and Peewee `_. * `Creating a bookmarking web-service that takes screenshots of your bookmarks `_. * `Building a pastebin, wiki and a bookmarking service using Flask and Peewee `_. * `Encrypted databases with Python and SQLCipher `_. * `Dear Diary: An Encrypted, Command-Line Diary with Peewee `_. ================================================ FILE: bench.py ================================================ from peewee import * db = SqliteDatabase(':memory:') #db = PostgresqlDatabase('peewee_test', host='127.0.0.1', port=26257, user='root') #db = PostgresqlDatabase('peewee_test', host='127.0.0.1', user='postgres') #from playhouse.cysqlite_ext import CySqliteDatabase #db = CySqliteDatabase(':memory:') #from playhouse.apsw_ext import APSWDatabase #db = APSWDatabase(':memory:') class Base(Model): class Meta: database = db class Register(Base): value = IntegerField() class Collection(Base): name = TextField() class Item(Base): collection = ForeignKeyField(Collection, backref='items') name = TextField() import functools import time def timed(fn): @functools.wraps(fn) def inner(*args, **kwargs): times = [] N = 20 for i in range(N): start = time.perf_counter() fn(i, *args, **kwargs) times.append(time.perf_counter() - start) print('%0.3f ... %s' % (round(sum(times) / N, 3), fn.__name__)) return inner def populate_register(s, n): for i in range(s, n): Register.create(value=i) def populate_collections(n, n_i): for i in range(n): c = Collection.create(name=str(i)) for j in range(n_i): Item.create(collection=c, name=str(j)) @timed def insert(i): with db.atomic(): populate_register((i * 1000), (i + 1) * 1000) @timed def batch_insert(i): it = range(i * 1000, (i + 1) * 1000) for i in db.batch_commit(it, 100): Register.insert(value=i).execute() @timed def bulk_insert(i): with db.atomic(): for i in range(i * 1000, (i + 1) * 1000, 100): data = [(j,) for j in range(i, i + 100)] Register.insert_many(data, fields=[Register.value]).execute() @timed def bulk_create(i): with db.atomic(): data = [Register(value=i) for i in range(i * 1000, (i + 1) * 1000)] Register.bulk_create(data, batch_size=100) @timed def select(i): query = Register.select() for row in query: pass @timed def select_related_dbapi_raw(i): query = Item.select(Item, Collection).join(Collection) cursor = db.execute(query) for row in cursor: pass @timed def insert_related(i): with db.atomic(): populate_collections(30, 60) @timed def select_related(i): query = Item.select(Item, Collection).join(Collection) for item in query: pass @timed def select_related_left(i): query = Collection.select(Collection, Item).join(Item, JOIN.LEFT_OUTER) for collection in query: pass @timed def select_related_dicts(i): query = Item.select(Item, Collection).join(Collection).dicts() for row in query: pass @timed def select_related_objects(i): query = Item.select(Item, Collection).join(Collection).objects() for item in query: pass @timed def select_prefetch(i): query = prefetch(Collection.select(), Item) for c in query: for i in c.items: pass @timed def select_prefetch_join(i): query = prefetch(Collection.select(), Item, prefetch_type=PREFETCH_TYPE.JOIN) for c in query: for i in c.items: pass if __name__ == '__main__': db.create_tables([Register, Collection, Item]) insert() insert_related() Register.delete().execute() batch_insert() assert Register.select().count() == 20000 Register.delete().execute() bulk_insert() assert Register.select().count() == 20000 Register.delete().execute() bulk_create() assert Register.select().count() == 20000 select() select_related() select_related_left() select_related_objects() select_related_dicts() select_related_dbapi_raw() select_prefetch() select_prefetch_join() db.drop_tables([Register, Collection, Item]) ================================================ FILE: docs/Makefile ================================================ # Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " text to make text files" @echo " man to make manual pages" @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: -rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/peewee.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/peewee.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/peewee" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/peewee" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." make -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." ================================================ FILE: docs/_themes/flask/layout.html ================================================ {%- extends "basic/layout.html" %} {%- block extrahead %} {{ super() }} {% if theme_touch_icon %} {% endif %} {% endblock %} {%- block relbar2 %}{% endblock %} {% block header %} {{ super() }} {% if pagename == 'index' %}
{% endif %} {% endblock %} {%- block footer %}
© Copyright {{ copyright }}. Created using Sphinx.
{% if pagename == 'index' %}
{% endif %} {%- endblock %} ================================================ FILE: docs/_themes/flask/relations.html ================================================

Related Topics

================================================ FILE: docs/_themes/flask/static/flasky.css_t ================================================ /* * flasky.css_t * ~~~~~~~~~~~~ * * :copyright: Copyright 2010 by Armin Ronacher. * :license: Flask Design License, see LICENSE for details. */ {% set page_width = '940px' %} {% set sidebar_width = '220px' %} @import url("basic.css"); /* -- page layout ----------------------------------------------------------- */ body { font-family: 'Georgia', serif; font-size: 17px; background-color: white; color: #000; margin: 0; padding: 0; } div.document { width: {{ page_width }}; margin: 30px auto 0 auto; } div.documentwrapper { float: left; width: 100%; } div.bodywrapper { margin: 0 0 0 {{ sidebar_width }}; } div.sphinxsidebar { width: {{ sidebar_width }}; } hr { border: 1px solid #B1B4B6; } div.body { background-color: #ffffff; color: #3E4349; padding: 0 30px 0 30px; } img.floatingflask { padding: 0 0 10px 10px; float: right; } div.footer { width: {{ page_width }}; margin: 20px auto 30px auto; font-size: 14px; color: #888; text-align: right; } div.footer a { color: #888; } div.related { display: none; } div.sphinxsidebar a { color: #444; text-decoration: none; border-bottom: 1px dotted #999; } div.sphinxsidebar a:hover { border-bottom: 1px solid #999; } div.sphinxsidebar { font-size: 14px; line-height: 1.5; } div.sphinxsidebarwrapper { padding: 18px 10px; } div.sphinxsidebarwrapper p.logo { padding: 0 0 20px 0; margin: 0; text-align: center; } div.sphinxsidebar h3, div.sphinxsidebar h4 { font-family: 'Garamond', 'Georgia', serif; color: #444; font-size: 24px; font-weight: normal; margin: 0 0 5px 0; padding: 0; } div.sphinxsidebar h4 { font-size: 20px; } div.sphinxsidebar h3 a { color: #444; } div.sphinxsidebar p.logo a, div.sphinxsidebar h3 a, div.sphinxsidebar p.logo a:hover, div.sphinxsidebar h3 a:hover { border: none; } div.sphinxsidebar p { color: #555; margin: 10px 0; } div.sphinxsidebar ul { margin: 10px 0; padding: 0; color: #000; } div.sphinxsidebar input { border: 1px solid #ccc; font-family: 'Georgia', serif; font-size: 1em; } /* -- body styles ----------------------------------------------------------- */ a { color: #004B6B; text-decoration: underline; } a:hover { color: #6D4100; text-decoration: underline; } div.body h1, div.body h2, div.body h3, div.body h4, div.body h5, div.body h6 { font-family: 'Garamond', 'Georgia', serif; font-weight: normal; margin: 30px 0px 10px 0px; padding: 0; } {% if theme_index_logo %} div.indexwrapper h1 { text-indent: -999999px; background: url({{ theme_index_logo }}) no-repeat center center; height: {{ theme_index_logo_height }}; } {% endif %} div.body h1 { margin-top: 0; padding-top: 0; font-size: 240%; } div.body h2 { font-size: 180%; } div.body h3 { font-size: 150%; } div.body h4 { font-size: 130%; } div.body h5 { font-size: 100%; } div.body h6 { font-size: 100%; } a.headerlink { color: #ddd; padding: 0 4px; text-decoration: none; } a.headerlink:hover { color: #444; background: #eaeaea; } div.body p, div.body dd, div.body li { line-height: 1.4em; } div.admonition { background: #fafafa; margin: 20px -30px; padding: 10px 30px; border-top: 1px solid #ccc; border-bottom: 1px solid #ccc; } div.admonition tt.xref, div.admonition a tt { border-bottom: 1px solid #fafafa; } dd div.admonition { margin-left: -60px; padding-left: 60px; } div.admonition p.admonition-title { font-family: 'Garamond', 'Georgia', serif; font-weight: normal; font-size: 24px; margin: 0 0 10px 0; padding: 0; line-height: 1; } div.admonition p.last { margin-bottom: 0; } div.highlight { background-color: white; } dt:target, .highlight { background: #FAF3E8; } div.note { background-color: #eee; border: 1px solid #ccc; } div.seealso { background-color: #ffc; border: 1px solid #ff6; } div.topic { background-color: #eee; } p.admonition-title { display: inline; } p.admonition-title:after { content: ":"; } pre, tt { font-family: 'Consolas', 'Menlo', 'Deja Vu Sans Mono', 'Bitstream Vera Sans Mono', monospace; font-size: 0.9em; } img.screenshot { } tt.descname, tt.descclassname { font-size: 0.95em; } tt.descname { padding-right: 0.08em; } img.screenshot { -moz-box-shadow: 2px 2px 4px #eee; -webkit-box-shadow: 2px 2px 4px #eee; box-shadow: 2px 2px 4px #eee; } table.docutils { border: 1px solid #888; -moz-box-shadow: 2px 2px 4px #eee; -webkit-box-shadow: 2px 2px 4px #eee; box-shadow: 2px 2px 4px #eee; } table.docutils td, table.docutils th { border: 1px solid #888; padding: 0.25em 0.7em; } table.field-list, table.footnote { border: none; -moz-box-shadow: none; -webkit-box-shadow: none; box-shadow: none; } table.footnote { margin: 15px 0; width: 100%; border: 1px solid #eee; background: #fdfdfd; font-size: 0.9em; } table.footnote + table.footnote { margin-top: -15px; border-top: none; } table.field-list th { padding: 0 0.8em 0 0; } table.field-list td { padding: 0; } table.footnote td.label { width: 0px; padding: 0.3em 0 0.3em 0.5em; } table.footnote td { padding: 0.3em 0.5em; } dl { margin: 0; padding: 0; } dl dd { margin-left: 30px; } blockquote { margin: 0 0 0 30px; padding: 0; } ul, ol { margin: 10px 0 10px 30px; padding: 0; } pre { background: #eee; padding: 7px 30px; margin: 15px -30px; line-height: 1.3em; } dl pre, blockquote pre, li pre { margin-left: -60px; padding-left: 60px; } dl dl pre { margin-left: -90px; padding-left: 90px; } tt { background-color: #ecf0f3; color: #222; /* padding: 1px 2px; */ } tt.xref, a tt { background-color: #FBFBFB; border-bottom: 1px solid white; } a.reference { text-decoration: none; border-bottom: 1px dotted #004B6B; } a.reference:hover { border-bottom: 1px solid #6D4100; } a.footnote-reference { text-decoration: none; font-size: 0.7em; vertical-align: top; border-bottom: 1px dotted #004B6B; } a.footnote-reference:hover { border-bottom: 1px solid #6D4100; } a:hover tt { background: #EEE; } ================================================ FILE: docs/_themes/flask/static/small_flask.css ================================================ /* * small_flask.css_t * ~~~~~~~~~~~~~~~~~ * * :copyright: Copyright 2010 by Armin Ronacher. * :license: Flask Design License, see LICENSE for details. */ body { margin: 0; padding: 20px 30px; } div.documentwrapper { float: none; background: white; } div.sphinxsidebar { display: block; float: none; width: 102.5%; margin: 50px -30px -20px -30px; padding: 10px 20px; background: #333; color: white; } div.sphinxsidebar h3, div.sphinxsidebar h4, div.sphinxsidebar p, div.sphinxsidebar h3 a { color: white; } div.sphinxsidebar a { color: #aaa; } div.sphinxsidebar p.logo { display: none; } div.document { width: 100%; margin: 0; } div.related { display: block; margin: 0; padding: 10px 0 20px 0; } div.related ul, div.related ul li { margin: 0; padding: 0; } div.footer { display: none; } div.bodywrapper { margin: 0; } div.body { min-height: 0; padding: 0; } ================================================ FILE: docs/_themes/flask/theme.conf ================================================ [theme] inherit = basic stylesheet = flasky.css pygments_style = flask_theme_support.FlaskyStyle [options] index_logo = '' index_logo_height = 120px touch_icon = ================================================ FILE: docs/clubdata.sql ================================================ -- -- PostgreSQL database dump -- --CREATE DATABASE exercises; --\c exercises --CREATE SCHEMA cd; -- Dumped from database version 9.2.0 -- Dumped by pg_dump version 9.2.0 -- Started on 2013-05-19 16:05:10 BST SET client_encoding = 'UTF8'; SET standard_conforming_strings = on; SET check_function_bodies = false; SET client_min_messages = warning; -- -- TOC entry 171 (class 1259 OID 32818) -- Name: bookings; Type: TABLE; Schema: cd; Owner: -; Tablespace: -- CREATE TABLE bookings ( bookid integer NOT NULL, facid integer NOT NULL, memid integer NOT NULL, starttime timestamp without time zone NOT NULL, slots integer NOT NULL ); -- -- TOC entry 169 (class 1259 OID 32770) -- Name: facilities; Type: TABLE; Schema: cd; Owner: -; Tablespace: -- CREATE TABLE facilities ( facid integer NOT NULL, name character varying(100) NOT NULL, membercost numeric NOT NULL, guestcost numeric NOT NULL, initialoutlay numeric NOT NULL, monthlymaintenance numeric NOT NULL ); -- -- TOC entry 170 (class 1259 OID 32800) -- Name: members; Type: TABLE; Schema: cd; Owner: -; Tablespace: -- CREATE TABLE members ( memid integer NOT NULL, surname character varying(200) NOT NULL, firstname character varying(200) NOT NULL, address character varying(300) NOT NULL, zipcode integer NOT NULL, telephone character varying(20) NOT NULL, recommendedby integer, joindate timestamp without time zone NOT NULL ); -- -- TOC entry 2202 (class 0 OID 32818) -- Dependencies: 171 -- Data for Name: bookings; Type: TABLE DATA; Schema: cd; Owner: - -- INSERT INTO bookings (bookid, facid, memid, starttime, slots) VALUES (0, 3, 1, '2012-07-03 11:00:00', 2), (1, 4, 1, '2012-07-03 08:00:00', 2), (2, 6, 0, '2012-07-03 18:00:00', 2), (3, 7, 1, '2012-07-03 19:00:00', 2), (4, 8, 1, '2012-07-03 10:00:00', 1), (5, 8, 1, '2012-07-03 15:00:00', 1), (6, 0, 2, '2012-07-04 09:00:00', 3), (7, 0, 2, '2012-07-04 15:00:00', 3), (8, 4, 3, '2012-07-04 13:30:00', 2), (9, 4, 0, '2012-07-04 15:00:00', 2), (10, 4, 0, '2012-07-04 17:30:00', 2), (11, 6, 0, '2012-07-04 12:30:00', 2), (12, 6, 0, '2012-07-04 14:00:00', 2), (13, 6, 1, '2012-07-04 15:30:00', 2), (14, 7, 2, '2012-07-04 14:00:00', 2), (15, 8, 2, '2012-07-04 12:00:00', 1), (16, 8, 3, '2012-07-04 18:00:00', 1), (17, 1, 0, '2012-07-05 17:30:00', 3), (18, 2, 1, '2012-07-05 09:30:00', 3), (19, 3, 3, '2012-07-05 09:00:00', 2), (20, 3, 1, '2012-07-05 19:00:00', 2), (21, 4, 3, '2012-07-05 18:30:00', 2), (22, 6, 0, '2012-07-05 13:00:00', 2), (23, 6, 1, '2012-07-05 14:30:00', 2), (24, 7, 2, '2012-07-05 18:30:00', 2), (25, 8, 3, '2012-07-05 12:30:00', 1), (26, 0, 0, '2012-07-06 08:00:00', 3), (27, 0, 0, '2012-07-06 14:00:00', 3), (28, 0, 2, '2012-07-06 15:30:00', 3), (29, 2, 1, '2012-07-06 17:00:00', 3), (30, 3, 1, '2012-07-06 11:00:00', 2), (31, 4, 3, '2012-07-06 12:00:00', 2), (32, 6, 1, '2012-07-06 14:00:00', 2), (33, 7, 2, '2012-07-06 08:30:00', 2), (34, 7, 2, '2012-07-06 13:30:00', 2), (35, 8, 3, '2012-07-06 15:30:00', 1), (36, 0, 2, '2012-07-07 08:30:00', 3), (37, 0, 0, '2012-07-07 12:30:00', 3), (38, 0, 2, '2012-07-07 14:30:00', 3), (39, 1, 3, '2012-07-07 08:30:00', 3), (40, 2, 1, '2012-07-07 09:00:00', 3), (41, 2, 1, '2012-07-07 11:30:00', 3), (42, 2, 1, '2012-07-07 16:00:00', 3), (43, 3, 2, '2012-07-07 12:30:00', 2), (44, 4, 3, '2012-07-07 11:30:00', 2), (45, 4, 3, '2012-07-07 14:00:00', 2), (46, 4, 0, '2012-07-07 17:30:00', 2), (47, 6, 0, '2012-07-07 08:30:00', 2), (48, 6, 1, '2012-07-07 10:30:00', 2), (49, 6, 1, '2012-07-07 14:30:00', 2), (50, 6, 0, '2012-07-07 16:00:00', 2), (51, 7, 2, '2012-07-07 11:30:00', 2), (52, 8, 3, '2012-07-07 16:00:00', 1), (53, 8, 3, '2012-07-07 17:30:00', 2), (54, 0, 3, '2012-07-08 13:00:00', 3), (55, 0, 2, '2012-07-08 17:30:00', 3), (56, 1, 1, '2012-07-08 15:00:00', 3), (57, 1, 1, '2012-07-08 17:30:00', 3), (58, 3, 1, '2012-07-08 11:30:00', 2), (59, 3, 3, '2012-07-08 18:30:00', 2), (60, 3, 1, '2012-07-08 19:30:00', 2), (61, 4, 0, '2012-07-08 11:00:00', 2), (62, 4, 2, '2012-07-08 16:30:00', 2), (63, 4, 0, '2012-07-08 18:00:00', 2), (64, 4, 0, '2012-07-08 19:30:00', 2), (65, 6, 0, '2012-07-08 14:00:00', 2), (66, 6, 0, '2012-07-08 18:30:00', 2), (67, 7, 2, '2012-07-08 11:00:00', 2), (68, 7, 1, '2012-07-08 16:30:00', 2), (69, 8, 3, '2012-07-08 10:00:00', 1), (70, 8, 3, '2012-07-08 16:30:00', 1), (71, 0, 2, '2012-07-09 12:30:00', 3), (72, 0, 2, '2012-07-09 15:30:00', 3), (73, 0, 2, '2012-07-09 19:00:00', 3), (74, 1, 0, '2012-07-09 13:00:00', 3), (75, 1, 1, '2012-07-09 19:00:00', 3), (76, 2, 1, '2012-07-09 09:00:00', 6), (77, 2, 0, '2012-07-09 19:00:00', 3), (78, 3, 3, '2012-07-09 17:00:00', 2), (79, 3, 3, '2012-07-09 18:30:00', 2), (80, 4, 2, '2012-07-09 11:00:00', 2), (81, 4, 3, '2012-07-09 14:30:00', 2), (82, 6, 0, '2012-07-09 14:30:00', 2), (83, 7, 1, '2012-07-09 15:30:00', 2), (84, 7, 0, '2012-07-09 18:30:00', 4), (85, 8, 3, '2012-07-09 09:30:00', 1), (86, 8, 3, '2012-07-09 16:30:00', 1), (87, 8, 3, '2012-07-09 20:00:00', 1), (88, 0, 0, '2012-07-10 11:30:00', 3), (89, 0, 0, '2012-07-10 16:00:00', 3), (90, 3, 2, '2012-07-10 08:00:00', 2), (91, 3, 1, '2012-07-10 11:00:00', 2), (92, 3, 3, '2012-07-10 15:30:00', 2), (93, 3, 2, '2012-07-10 16:30:00', 2), (94, 3, 1, '2012-07-10 18:00:00', 2), (95, 4, 0, '2012-07-10 10:00:00', 2), (96, 4, 4, '2012-07-10 11:30:00', 2), (97, 4, 0, '2012-07-10 15:00:00', 2), (98, 4, 3, '2012-07-10 17:00:00', 4), (99, 5, 0, '2012-07-10 08:30:00', 2), (100, 6, 0, '2012-07-10 14:30:00', 2), (101, 6, 0, '2012-07-10 19:00:00', 2), (102, 7, 4, '2012-07-10 08:30:00', 2), (103, 7, 2, '2012-07-10 17:30:00', 2), (104, 8, 0, '2012-07-10 11:30:00', 1), (105, 8, 3, '2012-07-10 12:00:00', 1), (106, 8, 3, '2012-07-10 19:30:00', 1), (107, 0, 4, '2012-07-11 08:00:00', 3), (108, 0, 2, '2012-07-11 10:00:00', 3), (109, 0, 0, '2012-07-11 12:00:00', 3), (110, 0, 0, '2012-07-11 14:00:00', 3), (111, 0, 2, '2012-07-11 15:30:00', 3), (112, 0, 2, '2012-07-11 18:30:00', 3), (113, 1, 0, '2012-07-11 12:30:00', 3), (114, 1, 0, '2012-07-11 16:00:00', 3), (115, 4, 1, '2012-07-11 08:00:00', 2), (116, 4, 0, '2012-07-11 09:00:00', 2), (117, 4, 3, '2012-07-11 11:00:00', 2), (118, 4, 0, '2012-07-11 15:00:00', 2), (119, 5, 4, '2012-07-11 17:00:00', 2), (120, 6, 0, '2012-07-11 14:00:00', 2), (121, 6, 0, '2012-07-11 19:30:00', 2), (122, 7, 0, '2012-07-11 08:00:00', 2), (123, 7, 0, '2012-07-11 14:00:00', 2), (124, 7, 0, '2012-07-11 16:30:00', 2), (125, 8, 4, '2012-07-11 11:00:00', 1), (126, 8, 3, '2012-07-11 13:00:00', 1), (127, 0, 0, '2012-07-12 13:30:00', 3), (128, 0, 2, '2012-07-12 16:30:00', 3), (129, 1, 1, '2012-07-12 11:30:00', 3), (130, 2, 1, '2012-07-12 09:00:00', 3), (131, 2, 1, '2012-07-12 18:30:00', 3), (132, 3, 3, '2012-07-12 18:00:00', 2), (133, 4, 1, '2012-07-12 16:00:00', 2), (134, 6, 0, '2012-07-12 12:00:00', 4), (135, 7, 2, '2012-07-12 08:00:00', 2), (136, 7, 4, '2012-07-12 13:30:00', 2), (137, 7, 4, '2012-07-12 16:00:00', 2), (138, 8, 3, '2012-07-12 16:30:00', 1), (139, 0, 2, '2012-07-13 10:30:00', 3), (140, 0, 4, '2012-07-13 14:00:00', 3), (141, 0, 3, '2012-07-13 17:00:00', 3), (142, 1, 1, '2012-07-13 15:00:00', 3), (143, 2, 1, '2012-07-13 09:00:00', 3), (144, 2, 0, '2012-07-13 15:00:00', 3), (145, 2, 1, '2012-07-13 16:30:00', 3), (146, 4, 0, '2012-07-13 11:00:00', 2), (147, 4, 0, '2012-07-13 13:30:00', 2), (148, 4, 0, '2012-07-13 15:00:00', 2), (149, 4, 3, '2012-07-13 16:00:00', 2), (150, 4, 4, '2012-07-13 17:30:00', 2), (151, 6, 0, '2012-07-13 09:30:00', 2), (152, 7, 0, '2012-07-13 08:00:00', 2), (153, 7, 1, '2012-07-13 11:00:00', 2), (154, 7, 4, '2012-07-13 12:30:00', 2), (155, 8, 0, '2012-07-13 15:30:00', 1), (156, 8, 2, '2012-07-13 18:30:00', 1), (157, 0, 2, '2012-07-14 08:30:00', 3), (158, 0, 4, '2012-07-14 11:30:00', 3), (159, 0, 3, '2012-07-14 15:00:00', 3), (160, 1, 3, '2012-07-14 10:30:00', 3), (161, 1, 3, '2012-07-14 12:30:00', 3), (162, 1, 0, '2012-07-14 14:30:00', 3), (163, 2, 1, '2012-07-14 08:30:00', 3), (164, 3, 2, '2012-07-14 16:00:00', 2), (165, 4, 3, '2012-07-14 08:00:00', 2), (166, 4, 1, '2012-07-14 14:30:00', 2), (167, 6, 0, '2012-07-14 09:30:00', 2), (168, 6, 1, '2012-07-14 12:30:00', 2), (169, 6, 0, '2012-07-14 15:00:00', 2), (170, 7, 2, '2012-07-14 12:30:00', 2), (171, 7, 2, '2012-07-14 15:00:00', 2), (172, 7, 4, '2012-07-14 16:30:00', 2), (173, 7, 1, '2012-07-14 19:00:00', 2), (174, 8, 3, '2012-07-14 09:00:00', 1), (175, 8, 1, '2012-07-14 17:00:00', 1), (176, 0, 2, '2012-07-15 08:00:00', 3), (177, 0, 0, '2012-07-15 16:00:00', 3), (178, 0, 2, '2012-07-15 19:00:00', 3), (179, 1, 0, '2012-07-15 10:00:00', 3), (180, 1, 0, '2012-07-15 12:00:00', 3), (181, 1, 3, '2012-07-15 15:30:00', 3), (182, 2, 1, '2012-07-15 13:00:00', 3), (183, 3, 1, '2012-07-15 17:30:00', 2), (184, 4, 3, '2012-07-15 11:30:00', 2), (185, 4, 0, '2012-07-15 15:00:00', 2), (186, 4, 3, '2012-07-15 17:30:00', 2), (187, 7, 4, '2012-07-15 14:30:00', 2), (188, 7, 4, '2012-07-15 17:00:00', 2), (189, 8, 4, '2012-07-15 10:00:00', 1), (190, 8, 2, '2012-07-15 12:00:00', 1), (191, 8, 3, '2012-07-15 12:30:00', 1), (192, 8, 3, '2012-07-15 13:30:00', 1), (193, 0, 5, '2012-07-16 11:00:00', 3), (194, 0, 5, '2012-07-16 19:00:00', 3), (195, 1, 1, '2012-07-16 08:00:00', 3), (196, 1, 0, '2012-07-16 12:30:00', 3), (197, 2, 1, '2012-07-16 16:30:00', 3), (198, 4, 3, '2012-07-16 09:00:00', 2), (199, 4, 1, '2012-07-16 11:00:00', 2), (200, 4, 3, '2012-07-16 12:00:00', 2), (201, 4, 3, '2012-07-16 17:30:00', 2), (202, 6, 0, '2012-07-16 18:30:00', 2), (203, 7, 4, '2012-07-16 08:00:00', 2), (204, 7, 2, '2012-07-16 11:30:00', 2), (205, 7, 4, '2012-07-16 12:30:00', 2), (206, 7, 5, '2012-07-16 14:00:00', 2), (207, 8, 4, '2012-07-16 12:00:00', 1), (208, 8, 1, '2012-07-16 15:00:00', 1), (209, 8, 4, '2012-07-16 18:00:00', 1), (210, 8, 3, '2012-07-16 19:30:00', 1), (211, 0, 5, '2012-07-17 12:30:00', 3), (212, 0, 5, '2012-07-17 18:00:00', 3), (213, 1, 1, '2012-07-17 10:00:00', 3), (214, 1, 4, '2012-07-17 14:30:00', 3), (215, 2, 5, '2012-07-17 10:30:00', 3), (216, 2, 1, '2012-07-17 12:30:00', 3), (217, 2, 1, '2012-07-17 15:30:00', 3), (218, 2, 2, '2012-07-17 19:00:00', 3), (219, 3, 1, '2012-07-17 14:00:00', 2), (220, 3, 2, '2012-07-17 15:00:00', 2), (221, 4, 0, '2012-07-17 09:00:00', 2), (222, 4, 3, '2012-07-17 10:30:00', 2), (223, 4, 3, '2012-07-17 12:00:00', 2), (224, 4, 5, '2012-07-17 16:00:00', 2), (225, 4, 3, '2012-07-17 18:30:00', 2), (226, 5, 0, '2012-07-17 13:30:00', 2), (227, 6, 4, '2012-07-17 12:00:00', 2), (228, 6, 0, '2012-07-17 14:00:00', 2), (229, 7, 4, '2012-07-17 08:00:00', 2), (230, 7, 5, '2012-07-17 14:00:00', 2), (231, 7, 4, '2012-07-17 16:00:00', 2), (232, 8, 3, '2012-07-17 08:30:00', 1), (233, 8, 2, '2012-07-17 11:00:00', 1), (234, 8, 3, '2012-07-17 11:30:00', 1), (235, 8, 3, '2012-07-17 14:30:00', 1), (236, 8, 0, '2012-07-17 15:00:00', 1), (237, 8, 3, '2012-07-17 15:30:00', 1), (238, 8, 3, '2012-07-17 18:00:00', 1), (239, 8, 3, '2012-07-17 20:00:00', 1), (240, 0, 5, '2012-07-18 13:00:00', 3), (241, 0, 5, '2012-07-18 17:30:00', 3), (242, 1, 0, '2012-07-18 14:00:00', 3), (243, 1, 0, '2012-07-18 16:30:00', 3), (244, 2, 1, '2012-07-18 14:00:00', 3), (245, 3, 2, '2012-07-18 11:30:00', 2), (246, 3, 3, '2012-07-18 19:00:00', 2), (247, 4, 1, '2012-07-18 08:30:00', 2), (248, 4, 4, '2012-07-18 10:00:00', 2), (249, 4, 5, '2012-07-18 19:00:00', 2), (250, 5, 0, '2012-07-18 14:30:00', 2), (251, 6, 0, '2012-07-18 10:30:00', 2), (252, 6, 0, '2012-07-18 13:00:00', 2), (253, 6, 0, '2012-07-18 15:00:00', 2), (254, 6, 1, '2012-07-18 19:30:00', 2), (255, 7, 4, '2012-07-18 08:30:00', 2), (256, 7, 4, '2012-07-18 11:00:00', 2), (257, 8, 3, '2012-07-18 11:00:00', 1), (258, 8, 0, '2012-07-18 13:00:00', 1), (259, 8, 3, '2012-07-18 14:30:00', 1), (260, 8, 4, '2012-07-18 16:00:00', 1), (261, 8, 3, '2012-07-18 16:30:00', 1), (262, 8, 4, '2012-07-18 20:00:00', 1), (263, 0, 2, '2012-07-19 08:30:00', 3), (264, 0, 4, '2012-07-19 10:30:00', 3), (265, 0, 5, '2012-07-19 12:00:00', 3), (266, 0, 0, '2012-07-19 13:30:00', 3), (267, 0, 5, '2012-07-19 16:30:00', 3), (268, 1, 1, '2012-07-19 11:30:00', 3), (269, 1, 0, '2012-07-19 15:00:00', 3), (270, 1, 0, '2012-07-19 18:30:00', 3), (271, 2, 1, '2012-07-19 09:30:00', 3), (272, 2, 0, '2012-07-19 11:30:00', 3), (273, 2, 1, '2012-07-19 14:30:00', 3), (274, 2, 2, '2012-07-19 16:00:00', 3), (275, 3, 3, '2012-07-19 08:30:00', 2), (276, 3, 3, '2012-07-19 17:00:00', 2), (277, 3, 3, '2012-07-19 18:30:00', 2), (278, 4, 3, '2012-07-19 12:00:00', 2), (279, 4, 5, '2012-07-19 14:30:00', 2), (280, 4, 0, '2012-07-19 16:30:00', 2), (281, 4, 1, '2012-07-19 18:30:00', 2), (282, 4, 0, '2012-07-19 19:30:00', 2), (283, 5, 0, '2012-07-19 08:30:00', 2), (284, 6, 4, '2012-07-19 12:30:00', 2), (285, 6, 2, '2012-07-19 14:00:00', 2), (286, 6, 0, '2012-07-19 15:00:00', 2), (287, 6, 0, '2012-07-19 16:30:00', 2), (288, 7, 2, '2012-07-19 13:00:00', 2), (289, 7, 0, '2012-07-19 14:00:00', 2), (290, 7, 0, '2012-07-19 16:30:00', 2), (291, 7, 4, '2012-07-19 17:30:00', 4), (292, 8, 3, '2012-07-19 11:00:00', 1), (293, 8, 1, '2012-07-19 13:30:00', 1), (294, 8, 3, '2012-07-19 14:30:00', 1), (295, 8, 3, '2012-07-19 18:00:00', 1), (296, 8, 3, '2012-07-19 20:00:00', 1), (297, 0, 3, '2012-07-20 08:00:00', 3), (298, 0, 5, '2012-07-20 12:00:00', 3), (299, 0, 5, '2012-07-20 14:00:00', 3), (300, 0, 5, '2012-07-20 17:30:00', 3), (301, 0, 0, '2012-07-20 19:00:00', 3), (302, 1, 2, '2012-07-20 08:30:00', 3), (303, 1, 3, '2012-07-20 12:00:00', 3), (304, 1, 4, '2012-07-20 13:30:00', 3), (305, 2, 1, '2012-07-20 14:30:00', 3), (306, 3, 3, '2012-07-20 15:00:00', 2), (307, 3, 1, '2012-07-20 17:30:00', 2), (308, 4, 5, '2012-07-20 08:00:00', 2), (309, 4, 0, '2012-07-20 13:00:00', 2), (310, 4, 1, '2012-07-20 16:30:00', 2), (311, 4, 0, '2012-07-20 17:30:00', 2), (312, 4, 3, '2012-07-20 18:30:00', 2), (313, 6, 0, '2012-07-20 11:00:00', 2), (314, 6, 4, '2012-07-20 12:30:00', 2), (315, 6, 2, '2012-07-20 15:00:00', 2), (316, 6, 0, '2012-07-20 16:00:00', 4), (317, 7, 2, '2012-07-20 12:30:00', 2), (318, 7, 2, '2012-07-20 16:00:00', 2), (319, 7, 4, '2012-07-20 19:30:00', 2), (320, 8, 1, '2012-07-20 09:00:00', 1), (321, 8, 2, '2012-07-20 12:00:00', 1), (322, 8, 3, '2012-07-20 19:30:00', 1), (323, 0, 0, '2012-07-21 08:00:00', 3), (324, 0, 5, '2012-07-21 11:00:00', 3), (325, 0, 5, '2012-07-21 13:30:00', 3), (326, 0, 4, '2012-07-21 15:30:00', 3), (327, 1, 1, '2012-07-21 09:30:00', 3), (328, 1, 0, '2012-07-21 11:00:00', 3), (329, 2, 0, '2012-07-21 10:30:00', 3), (330, 2, 1, '2012-07-21 13:30:00', 3), (331, 3, 2, '2012-07-21 08:00:00', 2), (332, 4, 0, '2012-07-21 09:00:00', 2), (333, 4, 3, '2012-07-21 10:30:00', 2), (334, 4, 0, '2012-07-21 14:00:00', 4), (335, 4, 3, '2012-07-21 16:00:00', 2), (336, 4, 1, '2012-07-21 17:00:00', 2), (337, 4, 0, '2012-07-21 19:00:00', 2), (338, 6, 4, '2012-07-21 08:00:00', 2), (339, 6, 0, '2012-07-21 09:30:00', 2), (340, 6, 0, '2012-07-21 12:00:00', 2), (341, 8, 3, '2012-07-21 09:30:00', 1), (342, 8, 3, '2012-07-21 11:30:00', 1), (343, 8, 3, '2012-07-21 18:00:00', 2), (344, 8, 3, '2012-07-21 19:30:00', 1), (345, 0, 5, '2012-07-22 10:00:00', 3), (346, 0, 0, '2012-07-22 16:00:00', 3), (347, 0, 2, '2012-07-22 18:00:00', 3), (348, 1, 0, '2012-07-22 08:30:00', 3), (349, 1, 0, '2012-07-22 10:30:00', 3), (350, 1, 5, '2012-07-22 18:30:00', 3), (351, 2, 1, '2012-07-22 08:30:00', 3), (352, 2, 1, '2012-07-22 13:30:00', 3), (353, 2, 1, '2012-07-22 16:30:00', 3), (354, 3, 3, '2012-07-22 11:30:00', 2), (355, 3, 2, '2012-07-22 14:00:00', 2), (356, 4, 4, '2012-07-22 08:00:00', 2), (357, 4, 3, '2012-07-22 10:30:00', 2), (358, 4, 0, '2012-07-22 12:00:00', 2), (359, 4, 5, '2012-07-22 13:00:00', 2), (360, 4, 0, '2012-07-22 16:30:00', 2), (361, 4, 1, '2012-07-22 18:00:00', 2), (362, 4, 3, '2012-07-22 19:30:00', 2), (363, 6, 4, '2012-07-22 10:30:00', 4), (364, 6, 0, '2012-07-22 14:30:00', 2), (365, 6, 0, '2012-07-22 16:30:00', 2), (366, 7, 2, '2012-07-22 10:30:00', 2), (367, 7, 2, '2012-07-22 12:00:00', 2), (368, 8, 3, '2012-07-22 16:00:00', 1), (369, 8, 3, '2012-07-22 17:00:00', 1), (370, 8, 2, '2012-07-22 17:30:00', 1), (371, 0, 0, '2012-07-23 09:30:00', 3), (372, 0, 0, '2012-07-23 12:00:00', 3), (373, 0, 5, '2012-07-23 17:00:00', 3), (374, 1, 1, '2012-07-23 10:00:00', 3), (375, 1, 4, '2012-07-23 12:30:00', 3), (376, 1, 4, '2012-07-23 15:30:00', 3), (377, 1, 0, '2012-07-23 17:00:00', 3), (378, 1, 4, '2012-07-23 19:00:00', 3), (379, 2, 1, '2012-07-23 08:00:00', 3), (380, 2, 5, '2012-07-23 11:30:00', 3), (381, 2, 1, '2012-07-23 13:00:00', 3), (382, 2, 1, '2012-07-23 15:00:00', 3), (383, 3, 2, '2012-07-23 09:30:00', 2), (384, 3, 2, '2012-07-23 19:00:00', 2), (385, 4, 4, '2012-07-23 10:00:00', 2), (386, 4, 0, '2012-07-23 16:30:00', 2), (387, 4, 3, '2012-07-23 19:00:00', 2), (388, 5, 3, '2012-07-23 13:00:00', 2), (389, 6, 0, '2012-07-23 13:30:00', 2), (390, 6, 0, '2012-07-23 15:00:00', 4), (391, 6, 0, '2012-07-23 19:00:00', 2), (392, 7, 5, '2012-07-23 16:00:00', 2), (393, 7, 4, '2012-07-23 18:00:00', 2), (394, 8, 3, '2012-07-23 08:30:00', 3), (395, 8, 3, '2012-07-23 11:00:00', 1), (396, 8, 3, '2012-07-23 14:00:00', 2), (397, 8, 2, '2012-07-23 15:00:00', 1), (398, 0, 0, '2012-07-24 11:00:00', 3), (399, 0, 4, '2012-07-24 13:00:00', 3), (400, 0, 5, '2012-07-24 14:30:00', 3), (401, 1, 4, '2012-07-24 11:00:00', 3), (402, 1, 0, '2012-07-24 16:00:00', 6), (403, 1, 1, '2012-07-24 19:00:00', 3), (404, 2, 1, '2012-07-24 09:00:00', 3), (405, 2, 2, '2012-07-24 12:30:00', 3), (406, 3, 3, '2012-07-24 09:00:00', 2), (407, 3, 3, '2012-07-24 17:30:00', 2), (408, 4, 0, '2012-07-24 08:30:00', 2), (409, 4, 5, '2012-07-24 09:30:00', 2), (410, 4, 0, '2012-07-24 11:30:00', 2), (411, 4, 1, '2012-07-24 14:30:00', 2), (412, 4, 0, '2012-07-24 15:30:00', 2), (413, 4, 0, '2012-07-24 17:30:00', 2), (414, 4, 0, '2012-07-24 19:30:00', 2), (415, 5, 5, '2012-07-24 16:30:00', 2), (416, 6, 0, '2012-07-24 09:30:00', 2), (417, 6, 0, '2012-07-24 14:30:00', 2), (418, 7, 4, '2012-07-24 09:30:00', 2), (419, 7, 5, '2012-07-24 11:30:00', 2), (420, 7, 2, '2012-07-24 16:30:00', 2), (421, 7, 4, '2012-07-24 18:00:00', 2), (422, 7, 2, '2012-07-24 19:30:00', 2), (423, 8, 3, '2012-07-24 08:30:00', 1), (424, 8, 3, '2012-07-24 10:30:00', 2), (425, 8, 3, '2012-07-24 12:00:00', 1), (426, 8, 3, '2012-07-24 14:00:00', 1), (427, 8, 0, '2012-07-24 15:00:00', 1), (428, 8, 4, '2012-07-24 16:30:00', 1), (429, 8, 0, '2012-07-24 20:00:00', 1), (430, 0, 5, '2012-07-25 08:00:00', 3), (431, 0, 0, '2012-07-25 12:30:00', 3), (432, 0, 0, '2012-07-25 16:30:00', 3), (433, 1, 1, '2012-07-25 08:00:00', 3), (434, 1, 0, '2012-07-25 10:30:00', 3), (435, 1, 4, '2012-07-25 15:00:00', 3), (436, 2, 1, '2012-07-25 13:30:00', 3), (437, 2, 1, '2012-07-25 17:30:00', 3), (438, 3, 2, '2012-07-25 10:00:00', 2), (439, 3, 3, '2012-07-25 14:00:00', 4), (440, 3, 3, '2012-07-25 17:00:00', 2), (441, 3, 2, '2012-07-25 18:30:00', 2), (442, 4, 3, '2012-07-25 08:30:00', 2), (443, 4, 0, '2012-07-25 09:30:00', 4), (444, 4, 3, '2012-07-25 11:30:00', 4), (445, 4, 5, '2012-07-25 13:30:00', 4), (446, 4, 3, '2012-07-25 16:00:00', 2), (447, 4, 3, '2012-07-25 18:00:00', 2), (448, 4, 3, '2012-07-25 19:30:00', 2), (449, 5, 0, '2012-07-25 18:30:00', 2), (450, 6, 4, '2012-07-25 08:30:00', 2), (451, 6, 1, '2012-07-25 09:30:00', 2), (452, 6, 0, '2012-07-25 12:00:00', 2), (453, 6, 0, '2012-07-25 13:30:00', 2), (454, 6, 0, '2012-07-25 16:30:00', 4), (455, 6, 5, '2012-07-25 19:00:00', 2), (456, 7, 5, '2012-07-25 10:30:00', 2), (457, 7, 2, '2012-07-25 14:00:00', 2), (458, 7, 2, '2012-07-25 16:00:00', 2), (459, 8, 3, '2012-07-25 08:00:00', 1), (460, 8, 3, '2012-07-25 10:00:00', 1), (461, 8, 4, '2012-07-25 14:30:00', 1), (462, 8, 1, '2012-07-25 16:00:00', 1), (463, 8, 2, '2012-07-25 20:00:00', 1), (464, 0, 4, '2012-07-26 09:00:00', 3), (465, 0, 0, '2012-07-26 11:30:00', 3), (466, 0, 4, '2012-07-26 18:00:00', 3), (467, 1, 8, '2012-07-26 08:00:00', 3), (468, 1, 8, '2012-07-26 11:30:00', 3), (469, 1, 8, '2012-07-26 13:30:00', 3), (470, 1, 1, '2012-07-26 15:00:00', 3), (471, 1, 0, '2012-07-26 16:30:00', 3), (472, 1, 6, '2012-07-26 19:00:00', 3), (473, 2, 1, '2012-07-26 08:30:00', 3), (474, 2, 2, '2012-07-26 11:00:00', 6), (475, 2, 7, '2012-07-26 14:00:00', 3), (476, 2, 2, '2012-07-26 17:00:00', 3), (477, 2, 3, '2012-07-26 19:00:00', 3), (478, 3, 0, '2012-07-26 09:00:00', 2), (479, 3, 0, '2012-07-26 13:30:00', 2), (480, 3, 3, '2012-07-26 16:00:00', 2), (481, 4, 3, '2012-07-26 08:00:00', 2), (482, 4, 6, '2012-07-26 09:00:00', 2), (483, 4, 0, '2012-07-26 12:00:00', 2), (484, 4, 5, '2012-07-26 13:30:00', 2), (485, 4, 6, '2012-07-26 16:00:00', 2), (486, 4, 7, '2012-07-26 17:30:00', 2), (487, 6, 0, '2012-07-26 10:00:00', 4), (488, 6, 0, '2012-07-26 13:00:00', 2), (489, 6, 0, '2012-07-26 19:00:00', 2), (490, 7, 7, '2012-07-26 09:30:00', 2), (491, 7, 6, '2012-07-26 11:00:00', 2), (492, 7, 5, '2012-07-26 12:30:00', 2), (493, 7, 4, '2012-07-26 13:30:00', 2), (494, 7, 5, '2012-07-26 17:00:00', 2), (495, 8, 3, '2012-07-26 12:00:00', 1), (496, 8, 3, '2012-07-26 13:30:00', 1), (497, 8, 2, '2012-07-26 15:00:00', 1), (498, 8, 1, '2012-07-26 16:30:00', 1), (499, 8, 3, '2012-07-26 17:00:00', 1), (500, 0, 4, '2012-07-27 08:00:00', 3), (501, 0, 5, '2012-07-27 11:00:00', 3), (502, 0, 6, '2012-07-27 14:00:00', 3), (503, 0, 6, '2012-07-27 17:30:00', 3), (504, 1, 0, '2012-07-27 10:00:00', 3), (505, 1, 7, '2012-07-27 11:30:00', 3), (506, 1, 0, '2012-07-27 13:00:00', 3), (507, 1, 0, '2012-07-27 15:00:00', 3), (508, 1, 0, '2012-07-27 18:00:00', 3), (509, 2, 1, '2012-07-27 12:00:00', 6), (510, 3, 0, '2012-07-27 10:30:00', 2), (511, 3, 2, '2012-07-27 14:00:00', 2), (512, 3, 3, '2012-07-27 19:00:00', 2), (513, 4, 1, '2012-07-27 10:00:00', 4), (514, 4, 6, '2012-07-27 12:30:00', 2), (515, 4, 0, '2012-07-27 14:00:00', 4), (516, 4, 0, '2012-07-27 16:30:00', 2), (517, 4, 1, '2012-07-27 17:30:00', 2), (518, 4, 0, '2012-07-27 18:30:00', 2), (519, 5, 7, '2012-07-27 18:00:00', 2), (520, 6, 0, '2012-07-27 09:00:00', 2), (521, 6, 5, '2012-07-27 14:00:00', 2), (522, 6, 8, '2012-07-27 16:30:00', 2), (523, 7, 2, '2012-07-27 18:00:00', 2), (524, 7, 4, '2012-07-27 19:00:00', 2), (525, 8, 3, '2012-07-27 09:00:00', 1), (526, 8, 3, '2012-07-27 12:30:00', 1), (527, 8, 3, '2012-07-27 16:00:00', 1), (528, 8, 6, '2012-07-27 16:30:00', 1), (529, 8, 0, '2012-07-27 18:30:00', 1), (530, 0, 7, '2012-07-28 08:00:00', 9), (531, 0, 4, '2012-07-28 13:00:00', 3), (532, 0, 5, '2012-07-28 15:00:00', 3), (533, 0, 2, '2012-07-28 19:00:00', 3), (534, 1, 1, '2012-07-28 08:00:00', 3), (535, 1, 0, '2012-07-28 10:00:00', 3), (536, 1, 0, '2012-07-28 16:00:00', 3), (537, 1, 7, '2012-07-28 17:30:00', 3), (538, 2, 1, '2012-07-28 10:00:00', 3), (539, 2, 1, '2012-07-28 14:00:00', 3), (540, 2, 1, '2012-07-28 17:00:00', 3), (541, 2, 5, '2012-07-28 18:30:00', 3), (542, 3, 3, '2012-07-28 08:30:00', 2), (543, 3, 3, '2012-07-28 15:30:00', 2), (544, 4, 0, '2012-07-28 09:00:00', 2), (545, 4, 3, '2012-07-28 10:30:00', 4), (546, 4, 0, '2012-07-28 12:30:00', 2), (547, 4, 8, '2012-07-28 16:00:00', 2), (548, 4, 0, '2012-07-28 19:00:00', 2), (549, 5, 0, '2012-07-28 18:00:00', 2), (550, 6, 0, '2012-07-28 17:00:00', 2), (551, 6, 0, '2012-07-28 18:30:00', 2), (552, 7, 2, '2012-07-28 09:00:00', 2), (553, 7, 5, '2012-07-28 10:00:00', 2), (554, 7, 6, '2012-07-28 12:30:00', 2), (555, 7, 8, '2012-07-28 17:00:00', 4), (556, 8, 2, '2012-07-28 16:00:00', 1), (557, 8, 3, '2012-07-28 16:30:00', 1), (558, 8, 4, '2012-07-28 19:00:00', 1), (559, 0, 7, '2012-07-29 09:30:00', 3), (560, 0, 2, '2012-07-29 11:00:00', 3), (561, 0, 6, '2012-07-29 13:00:00', 3), (562, 0, 5, '2012-07-29 15:00:00', 3), (563, 0, 0, '2012-07-29 17:00:00', 3), (564, 1, 8, '2012-07-29 09:30:00', 3), (565, 1, 0, '2012-07-29 15:00:00', 3), (566, 1, 8, '2012-07-29 16:30:00', 3), (567, 2, 1, '2012-07-29 08:30:00', 3), (568, 2, 1, '2012-07-29 12:00:00', 6), (569, 2, 1, '2012-07-29 15:30:00', 3), (570, 4, 3, '2012-07-29 08:00:00', 2), (571, 4, 0, '2012-07-29 09:00:00', 2), (572, 4, 3, '2012-07-29 10:30:00', 2), (573, 4, 8, '2012-07-29 11:30:00', 4), (574, 4, 8, '2012-07-29 15:00:00', 2), (575, 4, 0, '2012-07-29 18:30:00', 2), (576, 6, 0, '2012-07-29 09:00:00', 2), (577, 6, 0, '2012-07-29 10:30:00', 2), (578, 6, 6, '2012-07-29 17:30:00', 4), (579, 7, 4, '2012-07-29 16:00:00', 2), (580, 7, 8, '2012-07-29 18:30:00', 2), (581, 8, 3, '2012-07-29 12:30:00', 1), (582, 8, 7, '2012-07-29 13:00:00', 1), (583, 8, 3, '2012-07-29 15:30:00', 1), (584, 8, 3, '2012-07-29 18:00:00', 1), (585, 0, 5, '2012-07-30 14:00:00', 3), (586, 0, 6, '2012-07-30 15:30:00', 3), (587, 0, 7, '2012-07-30 19:00:00', 3), (588, 1, 8, '2012-07-30 08:30:00', 3), (589, 1, 7, '2012-07-30 11:00:00', 3), (590, 1, 2, '2012-07-30 13:30:00', 3), (591, 1, 1, '2012-07-30 15:30:00', 3), (592, 2, 5, '2012-07-30 10:00:00', 3), (593, 2, 8, '2012-07-30 11:30:00', 3), (594, 2, 7, '2012-07-30 15:00:00', 3), (595, 2, 0, '2012-07-30 17:30:00', 3), (596, 3, 3, '2012-07-30 11:30:00', 2), (597, 3, 4, '2012-07-30 16:30:00', 2), (598, 4, 0, '2012-07-30 08:00:00', 2), (599, 4, 0, '2012-07-30 10:30:00', 2), (600, 4, 0, '2012-07-30 12:00:00', 2), (601, 4, 7, '2012-07-30 18:00:00', 2), (602, 4, 3, '2012-07-30 19:30:00', 2), (603, 5, 0, '2012-07-30 12:30:00', 2), (604, 5, 0, '2012-07-30 14:00:00', 2), (605, 6, 0, '2012-07-30 08:30:00', 2), (606, 6, 0, '2012-07-30 12:00:00', 2), (607, 6, 0, '2012-07-30 14:30:00', 2), (608, 6, 0, '2012-07-30 17:30:00', 2), (609, 7, 7, '2012-07-30 08:00:00', 2), (610, 7, 6, '2012-07-30 09:30:00', 2), (611, 7, 8, '2012-07-30 14:30:00', 2), (612, 7, 5, '2012-07-30 16:30:00', 2), (613, 7, 4, '2012-07-30 18:00:00', 2), (614, 7, 6, '2012-07-30 19:00:00', 2), (615, 8, 3, '2012-07-30 08:30:00', 1), (616, 8, 2, '2012-07-30 09:00:00', 1), (617, 8, 2, '2012-07-30 11:00:00', 1), (618, 8, 2, '2012-07-30 12:30:00', 1), (619, 8, 3, '2012-07-30 15:00:00', 2), (620, 8, 5, '2012-07-30 16:00:00', 1), (621, 8, 2, '2012-07-30 16:30:00', 1), (622, 8, 3, '2012-07-30 18:30:00', 1), (623, 8, 1, '2012-07-30 19:30:00', 1), (624, 0, 7, '2012-07-31 09:30:00', 3), (625, 0, 0, '2012-07-31 11:00:00', 3), (626, 0, 0, '2012-07-31 15:00:00', 3), (627, 0, 5, '2012-07-31 17:00:00', 3), (628, 0, 0, '2012-07-31 18:30:00', 3), (629, 1, 0, '2012-07-31 08:00:00', 3), (630, 1, 7, '2012-07-31 13:00:00', 3), (631, 2, 1, '2012-07-31 16:30:00', 3), (632, 3, 3, '2012-07-31 08:30:00', 2), (633, 3, 1, '2012-07-31 13:00:00', 2), (634, 3, 1, '2012-07-31 15:30:00', 2), (635, 4, 8, '2012-07-31 09:30:00', 2), (636, 4, 0, '2012-07-31 11:00:00', 2), (637, 4, 3, '2012-07-31 12:00:00', 2), (638, 4, 2, '2012-07-31 13:00:00', 2), (639, 4, 3, '2012-07-31 14:00:00', 2), (640, 4, 0, '2012-07-31 15:00:00', 2), (641, 4, 6, '2012-07-31 17:00:00', 2), (642, 4, 7, '2012-07-31 18:30:00', 2), (643, 4, 0, '2012-07-31 19:30:00', 2), (644, 6, 0, '2012-07-31 09:00:00', 2), (645, 6, 5, '2012-07-31 10:00:00', 2), (646, 6, 6, '2012-07-31 11:00:00', 2), (647, 6, 0, '2012-07-31 14:30:00', 2), (648, 6, 6, '2012-07-31 16:00:00', 2), (649, 7, 4, '2012-07-31 18:30:00', 2), (650, 8, 3, '2012-07-31 10:00:00', 1), (651, 8, 3, '2012-07-31 11:30:00', 1), (652, 8, 5, '2012-07-31 12:00:00', 1), (653, 8, 7, '2012-07-31 12:30:00', 1), (654, 8, 8, '2012-07-31 13:30:00', 1), (655, 8, 6, '2012-07-31 14:00:00', 1), (656, 8, 4, '2012-07-31 17:00:00', 1), (657, 8, 2, '2012-07-31 17:30:00', 1), (658, 0, 5, '2012-08-01 15:30:00', 3), (659, 0, 5, '2012-08-01 18:00:00', 3), (660, 1, 8, '2012-08-01 09:00:00', 9), (661, 1, 8, '2012-08-01 17:30:00', 3), (662, 2, 1, '2012-08-01 09:30:00', 6), (663, 2, 1, '2012-08-01 14:30:00', 3), (664, 2, 1, '2012-08-01 16:30:00', 3), (665, 3, 7, '2012-08-01 13:00:00', 2), (666, 4, 5, '2012-08-01 08:00:00', 2), (667, 4, 6, '2012-08-01 09:00:00', 2), (668, 4, 0, '2012-08-01 10:30:00', 6), (669, 4, 3, '2012-08-01 13:30:00', 4), (670, 4, 3, '2012-08-01 19:30:00', 2), (671, 5, 7, '2012-08-01 08:30:00', 2), (672, 5, 0, '2012-08-01 14:30:00', 2), (673, 6, 0, '2012-08-01 09:30:00', 2), (674, 6, 0, '2012-08-01 11:00:00', 4), (675, 6, 6, '2012-08-01 14:30:00', 2), (676, 6, 0, '2012-08-01 18:00:00', 2), (677, 7, 4, '2012-08-01 12:30:00', 2), (678, 7, 2, '2012-08-01 16:00:00', 2), (679, 7, 5, '2012-08-01 17:00:00', 2), (680, 8, 3, '2012-08-01 08:30:00', 2), (681, 8, 2, '2012-08-01 09:30:00', 1), (682, 8, 3, '2012-08-01 10:30:00', 1), (683, 8, 3, '2012-08-01 11:30:00', 1), (684, 8, 8, '2012-08-01 13:30:00', 1), (685, 8, 8, '2012-08-01 15:00:00', 1), (686, 8, 3, '2012-08-01 17:00:00', 1), (687, 0, 8, '2012-08-02 08:00:00', 3), (688, 0, 5, '2012-08-02 13:00:00', 3), (689, 0, 7, '2012-08-02 15:30:00', 3), (690, 0, 5, '2012-08-02 18:30:00', 3), (691, 1, 8, '2012-08-02 09:30:00', 3), (692, 1, 8, '2012-08-02 12:00:00', 3), (693, 1, 0, '2012-08-02 13:30:00', 3), (694, 1, 5, '2012-08-02 15:30:00', 3), (695, 1, 0, '2012-08-02 18:00:00', 3), (696, 2, 1, '2012-08-02 09:30:00', 3), (697, 2, 0, '2012-08-02 11:30:00', 3), (698, 2, 3, '2012-08-02 14:00:00', 3), (699, 2, 1, '2012-08-02 19:00:00', 3), (700, 3, 3, '2012-08-02 10:00:00', 2), (701, 3, 2, '2012-08-02 15:00:00', 2), (702, 3, 3, '2012-08-02 17:00:00', 2), (703, 3, 6, '2012-08-02 18:00:00', 2), (704, 3, 4, '2012-08-02 19:30:00', 2), (705, 4, 4, '2012-08-02 10:00:00', 2), (706, 4, 7, '2012-08-02 11:30:00', 2), (707, 4, 5, '2012-08-02 14:30:00', 2), (708, 4, 8, '2012-08-02 15:30:00', 2), (709, 4, 8, '2012-08-02 17:00:00', 2), (710, 4, 3, '2012-08-02 18:30:00', 2), (711, 4, 0, '2012-08-02 19:30:00', 2), (712, 6, 4, '2012-08-02 09:00:00', 2), (713, 6, 0, '2012-08-02 10:00:00', 2), (714, 6, 0, '2012-08-02 11:30:00', 2), (715, 6, 6, '2012-08-02 12:30:00', 2), (716, 6, 8, '2012-08-02 14:00:00', 2), (717, 6, 0, '2012-08-02 17:00:00', 4), (718, 6, 2, '2012-08-02 19:30:00', 2), (719, 7, 7, '2012-08-02 08:00:00', 2), (720, 7, 5, '2012-08-02 11:00:00', 2), (721, 7, 6, '2012-08-02 14:00:00', 2), (722, 7, 4, '2012-08-02 16:00:00', 2), (723, 7, 0, '2012-08-02 18:00:00', 2), (724, 8, 3, '2012-08-02 08:30:00', 1), (725, 8, 3, '2012-08-02 13:00:00', 1), (726, 8, 7, '2012-08-02 15:00:00', 1), (727, 8, 3, '2012-08-02 16:30:00', 1), (728, 8, 7, '2012-08-02 17:00:00', 1), (729, 8, 3, '2012-08-02 19:30:00', 1), (730, 0, 5, '2012-08-03 11:30:00', 3), (731, 0, 0, '2012-08-03 16:00:00', 3), (732, 0, 6, '2012-08-03 18:30:00', 3), (733, 1, 8, '2012-08-03 10:30:00', 3), (734, 1, 0, '2012-08-03 13:00:00', 6), (735, 1, 7, '2012-08-03 16:30:00', 3), (736, 1, 8, '2012-08-03 19:00:00', 3), (737, 2, 8, '2012-08-03 08:30:00', 3), (738, 2, 1, '2012-08-03 11:00:00', 3), (739, 3, 6, '2012-08-03 08:00:00', 2), (740, 3, 2, '2012-08-03 10:00:00', 2), (741, 3, 6, '2012-08-03 12:00:00', 2), (742, 3, 6, '2012-08-03 16:30:00', 2), (743, 3, 2, '2012-08-03 18:30:00', 2), (744, 4, 0, '2012-08-03 09:30:00', 2), (745, 4, 7, '2012-08-03 10:30:00', 2), (746, 4, 0, '2012-08-03 11:30:00', 2), (747, 4, 0, '2012-08-03 13:00:00', 2), (748, 4, 1, '2012-08-03 14:30:00', 2), (749, 4, 3, '2012-08-03 15:30:00', 4), (750, 4, 3, '2012-08-03 18:30:00', 2), (751, 6, 0, '2012-08-03 09:00:00', 2), (752, 6, 0, '2012-08-03 10:30:00', 2), (753, 6, 4, '2012-08-03 12:00:00', 2), (754, 6, 0, '2012-08-03 15:30:00', 2), (755, 6, 1, '2012-08-03 16:30:00', 2), (756, 6, 0, '2012-08-03 19:00:00', 2), (757, 7, 6, '2012-08-03 09:00:00', 2), (758, 7, 6, '2012-08-03 10:30:00', 2), (759, 7, 2, '2012-08-03 13:30:00', 2), (760, 7, 5, '2012-08-03 17:30:00', 2), (761, 8, 8, '2012-08-03 12:00:00', 1), (762, 8, 3, '2012-08-03 12:30:00', 1), (763, 8, 6, '2012-08-03 14:00:00', 1), (764, 8, 3, '2012-08-03 15:00:00', 1), (765, 8, 6, '2012-08-03 15:30:00', 1), (766, 8, 8, '2012-08-03 16:00:00', 1), (767, 8, 0, '2012-08-03 19:00:00', 1), (768, 8, 3, '2012-08-03 19:30:00', 1), (769, 0, 6, '2012-08-04 15:00:00', 3), (770, 1, 9, '2012-08-04 09:30:00', 3), (771, 1, 0, '2012-08-04 11:30:00', 3), (772, 1, 8, '2012-08-04 16:00:00', 3), (773, 1, 0, '2012-08-04 18:30:00', 3), (774, 2, 1, '2012-08-04 08:00:00', 3), (775, 2, 2, '2012-08-04 09:30:00', 3), (776, 2, 1, '2012-08-04 11:00:00', 3), (777, 2, 2, '2012-08-04 16:30:00', 3), (778, 2, 9, '2012-08-04 18:30:00', 3), (779, 3, 6, '2012-08-04 11:30:00', 2), (780, 3, 1, '2012-08-04 15:00:00', 2), (781, 3, 3, '2012-08-04 18:00:00', 2), (782, 3, 4, '2012-08-04 19:00:00', 2), (783, 4, 8, '2012-08-04 08:30:00', 2), (784, 4, 7, '2012-08-04 10:00:00', 2), (785, 4, 0, '2012-08-04 13:30:00', 2), (786, 4, 5, '2012-08-04 14:30:00', 2), (787, 4, 0, '2012-08-04 17:00:00', 2), (788, 4, 5, '2012-08-04 19:30:00', 2), (789, 5, 0, '2012-08-04 12:30:00', 2), (790, 6, 6, '2012-08-04 08:30:00', 2), (791, 6, 5, '2012-08-04 09:30:00', 2), (792, 6, 6, '2012-08-04 12:30:00', 2), (793, 6, 0, '2012-08-04 16:00:00', 2), (794, 6, 0, '2012-08-04 17:30:00', 2), (795, 7, 5, '2012-08-04 08:00:00', 2), (796, 7, 9, '2012-08-04 11:00:00', 2), (797, 7, 7, '2012-08-04 15:00:00', 2), (798, 7, 5, '2012-08-04 18:30:00', 2), (799, 8, 3, '2012-08-04 08:00:00', 1), (800, 8, 3, '2012-08-04 11:00:00', 2), (801, 8, 3, '2012-08-04 13:00:00', 1), (802, 8, 3, '2012-08-04 16:30:00', 1), (803, 8, 6, '2012-08-04 18:00:00', 1), (804, 8, 7, '2012-08-04 18:30:00', 1), (805, 8, 3, '2012-08-04 19:00:00', 1), (806, 0, 2, '2012-08-05 08:00:00', 3), (807, 0, 5, '2012-08-05 09:30:00', 3), (808, 0, 7, '2012-08-05 15:00:00', 3), (809, 0, 7, '2012-08-05 17:30:00', 3), (810, 1, 0, '2012-08-05 08:00:00', 3), (811, 1, 7, '2012-08-05 09:30:00', 3), (812, 1, 9, '2012-08-05 11:00:00', 3), (813, 1, 9, '2012-08-05 15:30:00', 3), (814, 1, 1, '2012-08-05 18:00:00', 3), (815, 2, 1, '2012-08-05 10:00:00', 3), (816, 2, 5, '2012-08-05 11:30:00', 3), (817, 2, 2, '2012-08-05 15:00:00', 3), (818, 2, 8, '2012-08-05 17:00:00', 3), (819, 3, 3, '2012-08-05 09:30:00', 2), (820, 3, 4, '2012-08-05 14:30:00', 2), (821, 3, 3, '2012-08-05 15:30:00', 2), (822, 4, 0, '2012-08-05 08:30:00', 2), (823, 4, 0, '2012-08-05 10:00:00', 2), (824, 4, 0, '2012-08-05 11:30:00', 2), (825, 4, 4, '2012-08-05 16:00:00', 2), (826, 4, 8, '2012-08-05 19:00:00', 2), (827, 6, 0, '2012-08-05 10:00:00', 4), (828, 6, 6, '2012-08-05 13:00:00', 2), (829, 6, 0, '2012-08-05 15:30:00', 2), (830, 7, 2, '2012-08-05 10:30:00', 2), (831, 7, 8, '2012-08-05 15:30:00', 2), (832, 7, 2, '2012-08-05 19:30:00', 2), (833, 8, 0, '2012-08-05 08:30:00', 1), (834, 8, 3, '2012-08-05 13:00:00', 1), (835, 8, 0, '2012-08-05 14:00:00', 1), (836, 8, 3, '2012-08-05 16:30:00', 1), (837, 8, 3, '2012-08-05 17:30:00', 1), (838, 8, 3, '2012-08-05 19:30:00', 2), (839, 0, 7, '2012-08-06 09:00:00', 3), (840, 0, 0, '2012-08-06 10:30:00', 3), (841, 0, 2, '2012-08-06 12:00:00', 3), (842, 0, 0, '2012-08-06 13:30:00', 3), (843, 0, 7, '2012-08-06 15:00:00', 3), (844, 0, 5, '2012-08-06 16:30:00', 3), (845, 0, 7, '2012-08-06 18:00:00', 3), (846, 1, 8, '2012-08-06 08:00:00', 3), (847, 1, 3, '2012-08-06 10:00:00', 3), (848, 1, 0, '2012-08-06 11:30:00', 3), (849, 1, 9, '2012-08-06 14:30:00', 3), (850, 1, 9, '2012-08-06 17:30:00', 3), (851, 2, 1, '2012-08-06 08:30:00', 3), (852, 2, 5, '2012-08-06 10:30:00', 3), (853, 2, 8, '2012-08-06 12:00:00', 3), (854, 2, 8, '2012-08-06 14:00:00', 3), (855, 3, 3, '2012-08-06 08:30:00', 2), (856, 3, 6, '2012-08-06 15:00:00', 2), (857, 3, 6, '2012-08-06 17:00:00', 2), (858, 4, 0, '2012-08-06 08:00:00', 4), (859, 4, 0, '2012-08-06 12:00:00', 2), (860, 4, 7, '2012-08-06 13:30:00', 2), (861, 4, 0, '2012-08-06 16:30:00', 2), (862, 4, 6, '2012-08-06 18:30:00', 2), (863, 5, 0, '2012-08-06 11:00:00', 2), (864, 6, 6, '2012-08-06 09:00:00', 2), (865, 6, 0, '2012-08-06 10:00:00', 2), (866, 6, 6, '2012-08-06 13:00:00', 2), (867, 6, 0, '2012-08-06 14:00:00', 2), (868, 6, 5, '2012-08-06 15:00:00', 2), (869, 7, 8, '2012-08-06 09:30:00', 2), (870, 7, 2, '2012-08-06 11:00:00', 2), (871, 7, 5, '2012-08-06 12:00:00', 4), (872, 7, 4, '2012-08-06 17:30:00', 2), (873, 7, 2, '2012-08-06 19:00:00', 2), (874, 8, 3, '2012-08-06 08:00:00', 1), (875, 8, 4, '2012-08-06 09:00:00', 1), (876, 8, 3, '2012-08-06 09:30:00', 1), (877, 8, 6, '2012-08-06 12:00:00', 1), (878, 8, 1, '2012-08-06 18:00:00', 1), (879, 8, 8, '2012-08-06 18:30:00', 1), (880, 8, 3, '2012-08-06 19:00:00', 1), (881, 0, 10, '2012-08-07 09:00:00', 3), (882, 1, 0, '2012-08-07 08:00:00', 3), (883, 1, 8, '2012-08-07 09:30:00', 3), (884, 1, 7, '2012-08-07 17:00:00', 3), (885, 2, 1, '2012-08-07 09:00:00', 6), (886, 2, 1, '2012-08-07 13:00:00', 3), (887, 2, 10, '2012-08-07 15:00:00', 3), (888, 2, 2, '2012-08-07 18:00:00', 3), (889, 3, 6, '2012-08-07 08:30:00', 2), (890, 3, 6, '2012-08-07 10:00:00', 2), (891, 3, 3, '2012-08-07 11:00:00', 2), (892, 3, 3, '2012-08-07 12:30:00', 2), (893, 3, 3, '2012-08-07 14:30:00', 2), (894, 4, 0, '2012-08-07 08:30:00', 2), (895, 4, 8, '2012-08-07 12:00:00', 2), (896, 4, 8, '2012-08-07 13:30:00', 2), (897, 4, 0, '2012-08-07 15:30:00', 2), (898, 4, 6, '2012-08-07 18:30:00', 2), (899, 6, 1, '2012-08-07 08:00:00', 2), (900, 6, 0, '2012-08-07 14:00:00', 2), (901, 6, 0, '2012-08-07 15:30:00', 2), (902, 6, 0, '2012-08-07 18:00:00', 2), (903, 7, 10, '2012-08-07 12:30:00', 2), (904, 7, 4, '2012-08-07 15:00:00', 2), (905, 7, 9, '2012-08-07 18:30:00', 2), (906, 8, 3, '2012-08-07 08:30:00', 2), (907, 8, 2, '2012-08-07 10:00:00', 1), (908, 8, 3, '2012-08-07 10:30:00', 1), (909, 8, 0, '2012-08-07 11:00:00', 1), (910, 8, 3, '2012-08-07 12:00:00', 1), (911, 8, 2, '2012-08-07 12:30:00', 1), (912, 8, 2, '2012-08-07 14:30:00', 1), (913, 8, 0, '2012-08-07 16:30:00', 1), (914, 8, 3, '2012-08-07 17:00:00', 2), (915, 8, 8, '2012-08-07 19:30:00', 1), (916, 0, 10, '2012-08-08 09:00:00', 3), (917, 0, 6, '2012-08-08 12:30:00', 3), (918, 0, 5, '2012-08-08 14:00:00', 3), (919, 0, 6, '2012-08-08 16:30:00', 3), (920, 1, 0, '2012-08-08 08:30:00', 6), (921, 1, 10, '2012-08-08 11:30:00', 3), (922, 1, 10, '2012-08-08 14:00:00', 3), (923, 1, 9, '2012-08-08 19:00:00', 3), (924, 2, 5, '2012-08-08 08:00:00', 3), (925, 2, 9, '2012-08-08 09:30:00', 3), (926, 2, 7, '2012-08-08 11:00:00', 3), (927, 2, 1, '2012-08-08 14:00:00', 3), (928, 2, 5, '2012-08-08 17:30:00', 3), (929, 2, 1, '2012-08-08 19:00:00', 3), (930, 3, 10, '2012-08-08 08:00:00', 2), (931, 3, 6, '2012-08-08 10:00:00', 2), (932, 3, 0, '2012-08-08 12:00:00', 2), (933, 3, 10, '2012-08-08 15:30:00', 2), (934, 3, 2, '2012-08-08 19:00:00', 2), (935, 4, 6, '2012-08-08 08:00:00', 2), (936, 4, 8, '2012-08-08 11:00:00', 2), (937, 4, 9, '2012-08-08 12:30:00', 4), (938, 4, 0, '2012-08-08 15:00:00', 2), (939, 4, 0, '2012-08-08 16:30:00', 2), (940, 4, 3, '2012-08-08 17:30:00', 2), (941, 5, 0, '2012-08-08 08:00:00', 2), (942, 6, 0, '2012-08-08 09:00:00', 2), (943, 6, 0, '2012-08-08 11:00:00', 2), (944, 6, 8, '2012-08-08 12:30:00', 2), (945, 6, 6, '2012-08-08 15:00:00', 2), (946, 6, 10, '2012-08-08 17:30:00', 2), (947, 6, 0, '2012-08-08 19:00:00', 2), (948, 7, 8, '2012-08-08 08:00:00', 2), (949, 7, 9, '2012-08-08 11:00:00', 2), (950, 7, 7, '2012-08-08 12:30:00', 2), (951, 7, 4, '2012-08-08 14:00:00', 2), (952, 7, 9, '2012-08-08 15:30:00', 2), (953, 7, 10, '2012-08-08 18:30:00', 2), (954, 8, 4, '2012-08-08 08:00:00', 1), (955, 8, 2, '2012-08-08 09:00:00', 1), (956, 8, 3, '2012-08-08 10:00:00', 1), (957, 8, 3, '2012-08-08 11:00:00', 1), (958, 8, 3, '2012-08-08 12:00:00', 1), (959, 8, 2, '2012-08-08 13:00:00', 1), (960, 8, 7, '2012-08-08 16:00:00', 1), (961, 8, 1, '2012-08-08 16:30:00', 1), (962, 8, 3, '2012-08-08 17:00:00', 1), (963, 8, 2, '2012-08-08 17:30:00', 1), (964, 8, 1, '2012-08-08 18:30:00', 1), (965, 0, 6, '2012-08-09 09:30:00', 3), (966, 0, 7, '2012-08-09 16:00:00', 3), (967, 0, 10, '2012-08-09 17:30:00', 3), (968, 1, 10, '2012-08-09 08:00:00', 3), (969, 1, 0, '2012-08-09 10:00:00', 3), (970, 1, 8, '2012-08-09 14:00:00', 3), (971, 1, 0, '2012-08-09 17:00:00', 3), (972, 2, 2, '2012-08-09 09:00:00', 3), (973, 2, 1, '2012-08-09 11:00:00', 3), (974, 2, 9, '2012-08-09 13:00:00', 3), (975, 2, 1, '2012-08-09 14:30:00', 3), (976, 2, 1, '2012-08-09 16:30:00', 3), (977, 3, 10, '2012-08-09 10:00:00', 2), (978, 3, 7, '2012-08-09 13:30:00', 2), (979, 3, 6, '2012-08-09 14:30:00', 2), (980, 3, 2, '2012-08-09 18:00:00', 2), (981, 4, 0, '2012-08-09 09:00:00', 4), (982, 4, 0, '2012-08-09 12:00:00', 4), (983, 4, 10, '2012-08-09 16:30:00', 2), (984, 4, 9, '2012-08-09 17:30:00', 2), (985, 4, 8, '2012-08-09 18:30:00', 2), (986, 4, 10, '2012-08-09 19:30:00', 2), (987, 6, 6, '2012-08-09 11:30:00', 2), (988, 6, 6, '2012-08-09 18:30:00', 2), (989, 7, 8, '2012-08-09 08:00:00', 2), (990, 7, 8, '2012-08-09 10:30:00', 2), (991, 7, 0, '2012-08-09 12:30:00', 2), (992, 7, 6, '2012-08-09 16:00:00', 2), (993, 7, 2, '2012-08-09 17:00:00', 2), (994, 7, 4, '2012-08-09 18:30:00', 2), (995, 8, 4, '2012-08-09 10:00:00', 1), (996, 8, 2, '2012-08-09 11:30:00', 1), (997, 8, 6, '2012-08-09 13:00:00', 1), (998, 8, 3, '2012-08-09 15:00:00', 1), (999, 8, 5, '2012-08-09 15:30:00', 1), (1000, 8, 3, '2012-08-09 17:30:00', 1), (1001, 0, 3, '2012-08-10 08:00:00', 3), (1002, 0, 2, '2012-08-10 09:30:00', 3), (1003, 0, 5, '2012-08-10 11:30:00', 3), (1004, 0, 2, '2012-08-10 13:00:00', 3), (1005, 0, 8, '2012-08-10 16:30:00', 3), (1006, 1, 10, '2012-08-10 08:30:00', 6), (1007, 1, 8, '2012-08-10 12:00:00', 3), (1008, 1, 9, '2012-08-10 14:00:00', 3), (1009, 1, 0, '2012-08-10 16:00:00', 3), (1010, 1, 10, '2012-08-10 18:30:00', 3), (1011, 2, 1, '2012-08-10 08:00:00', 3), (1012, 2, 8, '2012-08-10 09:30:00', 3), (1013, 2, 7, '2012-08-10 17:30:00', 3), (1014, 2, 0, '2012-08-10 19:00:00', 3), (1015, 3, 2, '2012-08-10 08:00:00', 2), (1016, 3, 7, '2012-08-10 10:30:00', 2), (1017, 3, 10, '2012-08-10 11:30:00', 2), (1018, 3, 4, '2012-08-10 14:30:00', 2), (1019, 3, 3, '2012-08-10 18:00:00', 4), (1020, 4, 6, '2012-08-10 08:30:00', 2), (1021, 4, 5, '2012-08-10 10:00:00', 2), (1022, 4, 6, '2012-08-10 12:00:00', 2), (1023, 4, 0, '2012-08-10 13:00:00', 2), (1024, 4, 8, '2012-08-10 14:00:00', 2), (1025, 4, 1, '2012-08-10 15:30:00', 2), (1026, 4, 3, '2012-08-10 16:30:00', 2), (1027, 4, 9, '2012-08-10 19:00:00', 2), (1028, 5, 0, '2012-08-10 13:30:00', 2), (1029, 6, 0, '2012-08-10 09:00:00', 2), (1030, 6, 0, '2012-08-10 11:00:00', 2), (1031, 6, 0, '2012-08-10 12:30:00', 2), (1032, 6, 0, '2012-08-10 15:00:00', 2), (1033, 6, 10, '2012-08-10 16:30:00', 2), (1034, 6, 0, '2012-08-10 18:00:00', 2), (1035, 7, 4, '2012-08-10 09:30:00', 2), (1036, 7, 4, '2012-08-10 11:00:00', 2), (1037, 7, 9, '2012-08-10 13:00:00', 2), (1038, 7, 6, '2012-08-10 15:00:00', 2), (1039, 7, 5, '2012-08-10 16:30:00', 4), (1040, 7, 6, '2012-08-10 18:30:00', 2), (1041, 7, 7, '2012-08-10 19:30:00', 2), (1042, 8, 8, '2012-08-10 09:00:00', 1), (1043, 8, 7, '2012-08-10 10:00:00', 1), (1044, 8, 3, '2012-08-10 11:30:00', 1), (1045, 8, 3, '2012-08-10 12:30:00', 1), (1046, 8, 7, '2012-08-10 14:00:00', 1), (1047, 8, 2, '2012-08-10 14:30:00', 1), (1048, 8, 2, '2012-08-10 15:30:00', 2), (1049, 8, 7, '2012-08-10 17:00:00', 1), (1050, 8, 4, '2012-08-10 17:30:00', 1), (1051, 8, 3, '2012-08-10 20:00:00', 1), (1052, 0, 0, '2012-08-11 08:00:00', 3), (1053, 0, 5, '2012-08-11 10:00:00', 3), (1054, 0, 0, '2012-08-11 12:00:00', 3), (1055, 0, 4, '2012-08-11 13:30:00', 3), (1056, 0, 0, '2012-08-11 15:00:00', 3), (1057, 0, 12, '2012-08-11 16:30:00', 3), (1058, 0, 4, '2012-08-11 18:30:00', 3), (1059, 1, 11, '2012-08-11 08:00:00', 3), (1060, 1, 0, '2012-08-11 10:00:00', 3), (1061, 1, 0, '2012-08-11 12:30:00', 3), (1062, 1, 0, '2012-08-11 14:30:00', 3), (1063, 1, 8, '2012-08-11 16:00:00', 3), (1064, 1, 0, '2012-08-11 17:30:00', 3), (1065, 2, 1, '2012-08-11 09:00:00', 3), (1066, 2, 7, '2012-08-11 11:00:00', 3), (1067, 2, 1, '2012-08-11 18:00:00', 3), (1068, 3, 11, '2012-08-11 12:00:00', 2), (1069, 3, 6, '2012-08-11 14:00:00', 2), (1070, 3, 7, '2012-08-11 17:30:00', 2), (1071, 3, 13, '2012-08-11 19:00:00', 2), (1072, 4, 0, '2012-08-11 10:00:00', 2), (1073, 4, 14, '2012-08-11 11:00:00', 2), (1074, 4, 0, '2012-08-11 12:30:00', 2), (1075, 4, 8, '2012-08-11 14:00:00', 2), (1076, 4, 6, '2012-08-11 16:30:00', 2), (1077, 4, 8, '2012-08-11 18:00:00', 2), (1078, 4, 9, '2012-08-11 19:00:00', 2), (1079, 5, 12, '2012-08-11 19:30:00', 2), (1080, 6, 13, '2012-08-11 08:00:00', 2), (1081, 6, 0, '2012-08-11 09:00:00', 2), (1082, 6, 0, '2012-08-11 14:00:00', 2), (1083, 6, 6, '2012-08-11 15:00:00', 2), (1084, 6, 6, '2012-08-11 17:30:00', 4), (1085, 7, 2, '2012-08-11 08:30:00', 2), (1086, 7, 8, '2012-08-11 11:30:00', 2), (1087, 7, 4, '2012-08-11 15:00:00', 2), (1088, 7, 2, '2012-08-11 16:00:00', 2), (1089, 7, 8, '2012-08-11 19:00:00', 2), (1090, 8, 3, '2012-08-11 08:00:00', 2), (1091, 8, 1, '2012-08-11 11:30:00', 1), (1092, 8, 3, '2012-08-11 12:00:00', 1), (1093, 8, 3, '2012-08-11 13:30:00', 3), (1094, 8, 3, '2012-08-11 16:00:00', 1), (1095, 8, 2, '2012-08-11 17:00:00', 1), (1096, 8, 3, '2012-08-11 17:30:00', 1), (1097, 8, 2, '2012-08-11 18:00:00', 1), (1098, 8, 14, '2012-08-11 19:00:00', 1), (1099, 0, 0, '2012-08-12 08:00:00', 3), (1100, 0, 7, '2012-08-12 10:30:00', 3), (1101, 0, 14, '2012-08-12 13:00:00', 3), (1102, 0, 0, '2012-08-12 14:30:00', 3), (1103, 0, 6, '2012-08-12 16:00:00', 3), (1104, 0, 0, '2012-08-12 17:30:00', 6), (1105, 1, 0, '2012-08-12 10:30:00', 3), (1106, 1, 9, '2012-08-12 13:30:00', 3), (1107, 1, 8, '2012-08-12 19:00:00', 3), (1108, 2, 1, '2012-08-12 11:30:00', 3), (1109, 2, 2, '2012-08-12 13:00:00', 3), (1110, 2, 0, '2012-08-12 14:30:00', 3), (1111, 3, 6, '2012-08-12 08:00:00', 2), (1112, 3, 10, '2012-08-12 09:30:00', 4), (1113, 3, 11, '2012-08-12 14:30:00', 2), (1114, 3, 3, '2012-08-12 17:00:00', 2), (1115, 3, 5, '2012-08-12 19:00:00', 2), (1116, 4, 0, '2012-08-12 09:30:00', 2), (1117, 4, 0, '2012-08-12 12:00:00', 2), (1118, 4, 6, '2012-08-12 13:00:00', 2), (1119, 4, 7, '2012-08-12 16:30:00', 2), (1120, 4, 6, '2012-08-12 18:00:00', 2), (1121, 5, 0, '2012-08-12 09:30:00', 2), (1122, 5, 0, '2012-08-12 12:30:00', 2), (1123, 6, 0, '2012-08-12 09:00:00', 4), (1124, 6, 13, '2012-08-12 13:00:00', 2), (1125, 6, 0, '2012-08-12 14:30:00', 2), (1126, 6, 10, '2012-08-12 17:00:00', 2), (1127, 7, 8, '2012-08-12 11:00:00', 2), (1128, 7, 2, '2012-08-12 12:00:00', 2), (1129, 7, 13, '2012-08-12 14:00:00', 2), (1130, 7, 5, '2012-08-12 15:00:00', 2), (1131, 8, 9, '2012-08-12 08:00:00', 1), (1132, 8, 11, '2012-08-12 10:30:00', 1), (1133, 8, 3, '2012-08-12 12:00:00', 1), (1134, 8, 8, '2012-08-12 12:30:00', 1), (1135, 8, 0, '2012-08-12 13:30:00', 1), (1136, 8, 0, '2012-08-12 14:30:00', 1), (1137, 8, 3, '2012-08-12 19:00:00', 1), (1138, 0, 4, '2012-08-13 08:30:00', 3), (1139, 0, 0, '2012-08-13 11:00:00', 3), (1140, 0, 6, '2012-08-13 15:30:00', 3), (1141, 0, 0, '2012-08-13 18:00:00', 3), (1142, 1, 12, '2012-08-13 08:30:00', 3), (1143, 1, 6, '2012-08-13 11:00:00', 3), (1144, 1, 10, '2012-08-13 12:30:00', 6), (1145, 1, 11, '2012-08-13 15:30:00', 3), (1146, 1, 0, '2012-08-13 17:00:00', 3), (1147, 1, 11, '2012-08-13 19:00:00', 3), (1148, 2, 1, '2012-08-13 08:00:00', 3), (1149, 2, 1, '2012-08-13 11:00:00', 3), (1150, 2, 11, '2012-08-13 13:00:00', 3), (1151, 2, 2, '2012-08-13 14:30:00', 3), (1152, 2, 1, '2012-08-13 17:00:00', 3), (1153, 2, 5, '2012-08-13 19:00:00', 3), (1154, 3, 3, '2012-08-13 08:00:00', 2), (1155, 3, 10, '2012-08-13 11:00:00', 2), (1156, 3, 9, '2012-08-13 12:00:00', 2), (1157, 3, 3, '2012-08-13 13:00:00', 2), (1158, 3, 10, '2012-08-13 15:30:00', 2), (1159, 3, 6, '2012-08-13 17:30:00', 2), (1160, 4, 7, '2012-08-13 08:00:00', 2), (1161, 4, 10, '2012-08-13 09:00:00', 2), (1162, 4, 0, '2012-08-13 10:30:00', 4), (1163, 4, 0, '2012-08-13 14:00:00', 2), (1164, 4, 8, '2012-08-13 15:00:00', 2), (1165, 4, 3, '2012-08-13 16:00:00', 2), (1166, 4, 10, '2012-08-13 19:00:00', 2), (1167, 6, 0, '2012-08-13 08:00:00', 2), (1168, 6, 5, '2012-08-13 12:00:00', 2), (1169, 6, 6, '2012-08-13 13:00:00', 2), (1170, 6, 0, '2012-08-13 17:30:00', 6), (1171, 7, 6, '2012-08-13 08:00:00', 2), (1172, 7, 14, '2012-08-13 09:00:00', 2), (1173, 7, 7, '2012-08-13 10:00:00', 2), (1174, 7, 13, '2012-08-13 13:00:00', 2), (1175, 7, 8, '2012-08-13 14:00:00', 2), (1176, 7, 9, '2012-08-13 17:00:00', 2), (1177, 7, 11, '2012-08-13 18:00:00', 2), (1178, 7, 4, '2012-08-13 19:00:00', 2), (1179, 8, 2, '2012-08-13 08:30:00', 1), (1180, 8, 1, '2012-08-13 10:00:00', 1), (1181, 8, 3, '2012-08-13 11:00:00', 1), (1182, 8, 4, '2012-08-13 12:30:00', 1), (1183, 8, 7, '2012-08-13 14:00:00', 1), (1184, 8, 4, '2012-08-13 15:00:00', 1), (1185, 8, 1, '2012-08-13 16:30:00', 1), (1186, 8, 6, '2012-08-13 17:00:00', 1), (1187, 8, 3, '2012-08-13 18:30:00', 2), (1188, 8, 3, '2012-08-13 20:00:00', 1), (1189, 0, 7, '2012-08-14 09:00:00', 3), (1190, 0, 14, '2012-08-14 10:30:00', 3), (1191, 0, 11, '2012-08-14 13:00:00', 3), (1192, 0, 0, '2012-08-14 15:00:00', 6), (1193, 0, 10, '2012-08-14 18:30:00', 3), (1194, 1, 11, '2012-08-14 10:00:00', 3), (1195, 1, 8, '2012-08-14 11:30:00', 3), (1196, 1, 0, '2012-08-14 16:30:00', 3), (1197, 1, 0, '2012-08-14 18:30:00', 3), (1198, 2, 13, '2012-08-14 08:00:00', 3), (1199, 2, 1, '2012-08-14 10:30:00', 3), (1200, 2, 1, '2012-08-14 13:00:00', 3), (1201, 2, 10, '2012-08-14 15:30:00', 3), (1202, 2, 0, '2012-08-14 17:00:00', 3), (1203, 2, 1, '2012-08-14 19:00:00', 3), (1204, 3, 10, '2012-08-14 10:00:00', 2), (1205, 3, 10, '2012-08-14 13:00:00', 2), (1206, 3, 3, '2012-08-14 18:30:00', 4), (1207, 4, 11, '2012-08-14 08:30:00', 2), (1208, 4, 0, '2012-08-14 11:00:00', 2), (1209, 4, 6, '2012-08-14 12:30:00', 2), (1210, 4, 0, '2012-08-14 14:30:00', 2), (1211, 4, 14, '2012-08-14 16:30:00', 2), (1212, 4, 0, '2012-08-14 18:00:00', 2), (1213, 4, 6, '2012-08-14 19:30:00', 2), (1214, 5, 0, '2012-08-14 12:00:00', 2), (1215, 5, 0, '2012-08-14 13:30:00', 2), (1216, 6, 12, '2012-08-14 09:00:00', 2), (1217, 6, 0, '2012-08-14 12:30:00', 4), (1218, 6, 0, '2012-08-14 16:00:00', 4), (1219, 7, 8, '2012-08-14 09:30:00', 4), (1220, 7, 2, '2012-08-14 11:30:00', 2), (1221, 8, 0, '2012-08-14 08:00:00', 1), (1222, 8, 2, '2012-08-14 08:30:00', 1), (1223, 8, 11, '2012-08-14 09:30:00', 1), (1224, 8, 3, '2012-08-14 11:00:00', 1), (1225, 8, 12, '2012-08-14 12:30:00', 1), (1226, 8, 3, '2012-08-14 13:30:00', 1), (1227, 8, 3, '2012-08-14 16:30:00', 2), (1228, 8, 9, '2012-08-14 18:30:00', 1), (1229, 8, 6, '2012-08-14 19:00:00', 1), (1230, 8, 8, '2012-08-14 19:30:00', 1), (1231, 8, 0, '2012-08-14 20:00:00', 1), (1232, 0, 0, '2012-08-15 08:00:00', 3), (1233, 0, 6, '2012-08-15 11:30:00', 3), (1234, 0, 5, '2012-08-15 13:00:00', 3), (1235, 0, 14, '2012-08-15 15:00:00', 3), (1236, 0, 0, '2012-08-15 16:30:00', 3), (1237, 0, 7, '2012-08-15 18:00:00', 3), (1238, 1, 0, '2012-08-15 08:00:00', 3), (1239, 1, 8, '2012-08-15 09:30:00', 3), (1240, 1, 12, '2012-08-15 11:30:00', 3), (1241, 1, 11, '2012-08-15 14:30:00', 3), (1242, 1, 12, '2012-08-15 16:30:00', 3), (1243, 1, 8, '2012-08-15 18:30:00', 3), (1244, 2, 1, '2012-08-15 08:00:00', 3), (1245, 2, 0, '2012-08-15 10:00:00', 3), (1246, 2, 10, '2012-08-15 12:00:00', 3), (1247, 2, 13, '2012-08-15 13:30:00', 3), (1248, 2, 1, '2012-08-15 15:30:00', 3), (1249, 2, 9, '2012-08-15 18:00:00', 3), (1250, 3, 3, '2012-08-15 11:00:00', 2), (1251, 3, 1, '2012-08-15 13:00:00', 2), (1252, 3, 3, '2012-08-15 14:00:00', 2), (1253, 3, 11, '2012-08-15 16:30:00', 2), (1254, 3, 10, '2012-08-15 18:00:00', 2), (1255, 3, 3, '2012-08-15 19:30:00', 2), (1256, 4, 0, '2012-08-15 08:30:00', 4), (1257, 4, 6, '2012-08-15 10:30:00', 2), (1258, 4, 0, '2012-08-15 13:00:00', 2), (1259, 4, 0, '2012-08-15 15:00:00', 2), (1260, 4, 9, '2012-08-15 16:30:00', 2), (1261, 4, 11, '2012-08-15 18:00:00', 2), (1262, 5, 0, '2012-08-15 12:00:00', 2), (1263, 5, 0, '2012-08-15 16:00:00', 2), (1264, 5, 11, '2012-08-15 19:00:00', 2), (1265, 6, 6, '2012-08-15 08:00:00', 2), (1266, 6, 0, '2012-08-15 10:00:00', 2), (1267, 6, 13, '2012-08-15 11:30:00', 2), (1268, 6, 11, '2012-08-15 12:30:00', 2), (1269, 6, 10, '2012-08-15 13:30:00', 2), (1270, 6, 8, '2012-08-15 15:30:00', 2), (1271, 6, 13, '2012-08-15 17:00:00', 2), (1272, 6, 12, '2012-08-15 18:00:00', 2), (1273, 7, 6, '2012-08-15 15:00:00', 2), (1274, 7, 8, '2012-08-15 17:30:00', 2), (1275, 8, 6, '2012-08-15 09:00:00', 1), (1276, 8, 3, '2012-08-15 10:30:00', 1), (1277, 8, 2, '2012-08-15 11:30:00', 1), (1278, 8, 3, '2012-08-15 13:00:00', 1), (1279, 8, 2, '2012-08-15 14:00:00', 2), (1280, 8, 3, '2012-08-15 15:30:00', 1), (1281, 8, 0, '2012-08-15 16:00:00', 1), (1282, 8, 8, '2012-08-15 17:00:00', 1), (1283, 8, 3, '2012-08-15 17:30:00', 1), (1284, 8, 14, '2012-08-15 19:00:00', 1), (1285, 8, 1, '2012-08-15 19:30:00', 1), (1286, 8, 6, '2012-08-15 20:00:00', 1), (1287, 0, 4, '2012-08-16 08:30:00', 3), (1288, 0, 0, '2012-08-16 11:00:00', 3), (1289, 0, 5, '2012-08-16 12:30:00', 3), (1290, 0, 14, '2012-08-16 14:00:00', 3), (1291, 0, 0, '2012-08-16 15:30:00', 3), (1292, 0, 0, '2012-08-16 17:30:00', 3), (1293, 1, 12, '2012-08-16 08:00:00', 3), (1294, 1, 0, '2012-08-16 13:00:00', 3), (1295, 1, 11, '2012-08-16 14:30:00', 3), (1296, 1, 8, '2012-08-16 16:30:00', 3), (1297, 1, 12, '2012-08-16 18:00:00', 3), (1298, 2, 5, '2012-08-16 08:30:00', 3), (1299, 2, 14, '2012-08-16 10:00:00', 3), (1300, 2, 1, '2012-08-16 13:00:00', 3), (1301, 2, 2, '2012-08-16 15:30:00', 3), (1302, 2, 9, '2012-08-16 17:00:00', 3), (1303, 2, 15, '2012-08-16 18:30:00', 3), (1304, 3, 6, '2012-08-16 11:00:00', 2), (1305, 3, 10, '2012-08-16 16:30:00', 2), (1306, 3, 3, '2012-08-16 17:30:00', 2), (1307, 4, 1, '2012-08-16 08:30:00', 2), (1308, 4, 0, '2012-08-16 11:00:00', 2), (1309, 4, 1, '2012-08-16 12:00:00', 2), (1310, 4, 9, '2012-08-16 13:00:00', 2), (1311, 4, 8, '2012-08-16 14:00:00', 2), (1312, 4, 14, '2012-08-16 15:30:00', 2), (1313, 4, 13, '2012-08-16 18:30:00', 2), (1314, 4, 8, '2012-08-16 19:30:00', 2), (1315, 5, 0, '2012-08-16 11:00:00', 2), (1316, 6, 0, '2012-08-16 08:30:00', 2), (1317, 6, 0, '2012-08-16 11:30:00', 6), (1318, 6, 12, '2012-08-16 15:30:00', 2), (1319, 6, 5, '2012-08-16 17:30:00', 2), (1320, 6, 0, '2012-08-16 18:30:00', 2), (1321, 7, 7, '2012-08-16 10:30:00', 2), (1322, 7, 7, '2012-08-16 13:00:00', 2), (1323, 7, 13, '2012-08-16 14:30:00', 4), (1324, 7, 4, '2012-08-16 16:30:00', 2), (1325, 7, 10, '2012-08-16 18:00:00', 2), (1326, 8, 7, '2012-08-16 08:30:00', 1), (1327, 8, 3, '2012-08-16 09:00:00', 2), (1328, 8, 3, '2012-08-16 10:30:00', 1), (1329, 8, 3, '2012-08-16 12:00:00', 1), (1330, 8, 12, '2012-08-16 13:30:00', 1), (1331, 8, 3, '2012-08-16 14:00:00', 1), (1332, 8, 15, '2012-08-16 14:30:00', 1), (1333, 8, 3, '2012-08-16 15:00:00', 2), (1334, 8, 4, '2012-08-16 16:00:00', 1), (1335, 8, 3, '2012-08-16 19:00:00', 1), (1336, 8, 12, '2012-08-16 19:30:00', 1), (1337, 0, 0, '2012-08-17 08:30:00', 3), (1338, 0, 14, '2012-08-17 12:30:00', 3), (1339, 0, 6, '2012-08-17 14:00:00', 3), (1340, 0, 10, '2012-08-17 16:00:00', 3), (1341, 0, 14, '2012-08-17 17:30:00', 3), (1342, 1, 10, '2012-08-17 08:30:00', 3), (1343, 1, 0, '2012-08-17 11:00:00', 6), (1344, 1, 11, '2012-08-17 15:00:00', 3), (1345, 1, 0, '2012-08-17 17:00:00', 3), (1346, 1, 16, '2012-08-17 19:00:00', 3), (1347, 2, 1, '2012-08-17 09:00:00', 3), (1348, 2, 1, '2012-08-17 12:00:00', 3), (1349, 2, 11, '2012-08-17 13:30:00', 3), (1350, 2, 2, '2012-08-17 16:30:00', 3), (1351, 2, 1, '2012-08-17 18:30:00', 3), (1352, 3, 10, '2012-08-17 10:00:00', 2), (1353, 3, 15, '2012-08-17 11:00:00', 2), (1354, 3, 13, '2012-08-17 14:00:00', 2), (1355, 3, 15, '2012-08-17 17:30:00', 2), (1356, 4, 9, '2012-08-17 08:00:00', 2), (1357, 4, 6, '2012-08-17 09:30:00', 2), (1358, 4, 3, '2012-08-17 12:00:00', 2), (1359, 4, 3, '2012-08-17 13:30:00', 2), (1360, 4, 0, '2012-08-17 14:30:00', 2), (1361, 4, 16, '2012-08-17 15:30:00', 2), (1362, 4, 0, '2012-08-17 16:30:00', 4), (1363, 4, 9, '2012-08-17 19:00:00', 2), (1364, 5, 4, '2012-08-17 13:00:00', 2), (1365, 5, 0, '2012-08-17 15:30:00', 2), (1366, 6, 0, '2012-08-17 08:00:00', 2), (1367, 6, 12, '2012-08-17 09:30:00', 2), (1368, 6, 0, '2012-08-17 11:00:00', 2), (1369, 6, 6, '2012-08-17 12:00:00', 2), (1370, 6, 0, '2012-08-17 15:00:00', 2), (1371, 6, 0, '2012-08-17 17:30:00', 2), (1372, 6, 12, '2012-08-17 18:30:00', 2), (1373, 6, 6, '2012-08-17 19:30:00', 2), (1374, 7, 8, '2012-08-17 08:30:00', 2), (1375, 7, 13, '2012-08-17 12:30:00', 2), (1376, 7, 15, '2012-08-17 14:30:00', 2), (1377, 7, 7, '2012-08-17 16:30:00', 2), (1378, 8, 16, '2012-08-17 08:30:00', 1), (1379, 8, 16, '2012-08-17 10:00:00', 1), (1380, 8, 14, '2012-08-17 11:00:00', 1), (1381, 8, 0, '2012-08-17 12:00:00', 1), (1382, 8, 2, '2012-08-17 14:00:00', 1), (1383, 8, 3, '2012-08-17 14:30:00', 1), (1384, 8, 3, '2012-08-17 15:30:00', 1), (1385, 8, 8, '2012-08-17 16:00:00', 1), (1386, 8, 16, '2012-08-17 16:30:00', 1), (1387, 8, 4, '2012-08-17 18:00:00', 1), (1388, 8, 6, '2012-08-17 18:30:00', 1), (1389, 8, 12, '2012-08-17 19:30:00', 1), (1390, 0, 5, '2012-08-18 08:00:00', 3), (1391, 0, 0, '2012-08-18 11:00:00', 3), (1392, 0, 5, '2012-08-18 12:30:00', 3), (1393, 0, 0, '2012-08-18 14:00:00', 3), (1394, 1, 8, '2012-08-18 09:30:00', 3), (1395, 1, 15, '2012-08-18 12:30:00', 3), (1396, 1, 0, '2012-08-18 14:30:00', 3), (1397, 1, 7, '2012-08-18 17:00:00', 3), (1398, 1, 12, '2012-08-18 18:30:00', 3), (1399, 2, 1, '2012-08-18 08:30:00', 3), (1400, 2, 1, '2012-08-18 11:30:00', 3), (1401, 2, 2, '2012-08-18 16:00:00', 3), (1402, 2, 14, '2012-08-18 18:00:00', 3), (1403, 3, 15, '2012-08-18 08:00:00', 2), (1404, 3, 15, '2012-08-18 11:00:00', 2), (1405, 3, 12, '2012-08-18 13:30:00', 2), (1406, 3, 1, '2012-08-18 19:30:00', 2), (1407, 4, 16, '2012-08-18 08:00:00', 2), (1408, 4, 3, '2012-08-18 09:00:00', 2), (1409, 4, 4, '2012-08-18 10:30:00', 2), (1410, 4, 3, '2012-08-18 11:30:00', 2), (1411, 4, 11, '2012-08-18 12:30:00', 2), (1412, 4, 0, '2012-08-18 13:30:00', 2), (1413, 4, 0, '2012-08-18 15:00:00', 4), (1414, 4, 5, '2012-08-18 17:30:00', 2), (1415, 4, 0, '2012-08-18 18:30:00', 4), (1416, 5, 0, '2012-08-18 11:00:00', 4), (1417, 6, 12, '2012-08-18 09:00:00', 2), (1418, 6, 0, '2012-08-18 11:00:00', 2), (1419, 6, 4, '2012-08-18 12:00:00', 2), (1420, 6, 0, '2012-08-18 13:00:00', 2), (1421, 6, 14, '2012-08-18 14:30:00', 2), (1422, 6, 0, '2012-08-18 16:30:00', 4), (1423, 6, 8, '2012-08-18 19:30:00', 2), (1424, 7, 8, '2012-08-18 12:00:00', 2), (1425, 7, 8, '2012-08-18 13:30:00', 2), (1426, 7, 15, '2012-08-18 15:00:00', 2), (1427, 7, 15, '2012-08-18 16:30:00', 2), (1428, 7, 1, '2012-08-18 18:30:00', 2), (1429, 8, 3, '2012-08-18 08:00:00', 1), (1430, 8, 6, '2012-08-18 08:30:00', 1), (1431, 8, 16, '2012-08-18 09:30:00', 1), (1432, 8, 16, '2012-08-18 11:30:00', 2), (1433, 8, 2, '2012-08-18 12:30:00', 1), (1434, 8, 16, '2012-08-18 13:00:00', 1), (1435, 8, 11, '2012-08-18 13:30:00', 1), (1436, 8, 16, '2012-08-18 14:00:00', 2), (1437, 8, 0, '2012-08-18 16:00:00', 1), (1438, 8, 3, '2012-08-18 16:30:00', 1), (1439, 0, 12, '2012-08-19 08:00:00', 3), (1440, 0, 16, '2012-08-19 10:30:00', 3), (1441, 0, 6, '2012-08-19 13:30:00', 3), (1442, 0, 6, '2012-08-19 17:30:00', 3), (1443, 1, 10, '2012-08-19 08:00:00', 3), (1444, 1, 7, '2012-08-19 11:00:00', 3), (1445, 1, 10, '2012-08-19 12:30:00', 3), (1446, 1, 0, '2012-08-19 15:30:00', 3), (1447, 2, 1, '2012-08-19 09:00:00', 3), (1448, 2, 5, '2012-08-19 12:30:00', 3), (1449, 2, 14, '2012-08-19 16:30:00', 3), (1450, 2, 2, '2012-08-19 18:00:00', 3), (1451, 3, 16, '2012-08-19 08:00:00', 2), (1452, 3, 10, '2012-08-19 09:30:00', 2), (1453, 3, 15, '2012-08-19 11:00:00', 4), (1454, 3, 14, '2012-08-19 15:00:00', 2), (1455, 3, 3, '2012-08-19 18:30:00', 2), (1456, 4, 0, '2012-08-19 09:30:00', 6), (1457, 4, 5, '2012-08-19 14:00:00', 2), (1458, 4, 1, '2012-08-19 15:30:00', 2), (1459, 4, 5, '2012-08-19 16:30:00', 2), (1460, 4, 16, '2012-08-19 17:30:00', 2), (1461, 4, 1, '2012-08-19 19:00:00', 2), (1462, 5, 0, '2012-08-19 17:30:00', 2), (1463, 5, 0, '2012-08-19 19:00:00', 2), (1464, 6, 0, '2012-08-19 09:00:00', 2), (1465, 6, 12, '2012-08-19 10:00:00', 2), (1466, 6, 0, '2012-08-19 12:00:00', 2), (1467, 6, 11, '2012-08-19 13:30:00', 2), (1468, 6, 16, '2012-08-19 14:30:00', 2), (1469, 6, 0, '2012-08-19 16:30:00', 2), (1470, 6, 0, '2012-08-19 18:30:00', 4), (1471, 7, 6, '2012-08-19 08:00:00', 2), (1472, 7, 5, '2012-08-19 11:00:00', 2), (1473, 7, 13, '2012-08-19 13:30:00', 2), (1474, 7, 15, '2012-08-19 17:00:00', 2), (1475, 7, 4, '2012-08-19 18:30:00', 2), (1476, 8, 0, '2012-08-19 10:00:00', 1), (1477, 8, 3, '2012-08-19 11:00:00', 1), (1478, 8, 4, '2012-08-19 12:30:00', 1), (1479, 8, 6, '2012-08-19 13:00:00', 1), (1480, 8, 1, '2012-08-19 15:00:00', 1), (1481, 8, 16, '2012-08-19 15:30:00', 1), (1482, 8, 2, '2012-08-19 17:00:00', 1), (1483, 8, 8, '2012-08-19 17:30:00', 1), (1484, 8, 12, '2012-08-19 19:00:00', 1), (1485, 0, 10, '2012-08-20 08:30:00', 3), (1486, 0, 10, '2012-08-20 10:30:00', 3), (1487, 0, 14, '2012-08-20 12:00:00', 3), (1488, 0, 4, '2012-08-20 14:30:00', 3), (1489, 0, 14, '2012-08-20 16:30:00', 3), (1490, 0, 16, '2012-08-20 19:00:00', 3), (1491, 1, 9, '2012-08-20 08:00:00', 3), (1492, 1, 16, '2012-08-20 09:30:00', 3), (1493, 1, 0, '2012-08-20 12:00:00', 3), (1494, 1, 10, '2012-08-20 13:30:00', 6), (1495, 1, 6, '2012-08-20 16:30:00', 3), (1496, 1, 8, '2012-08-20 18:30:00', 3), (1497, 2, 8, '2012-08-20 08:30:00', 3), (1498, 2, 5, '2012-08-20 10:00:00', 6), (1499, 2, 1, '2012-08-20 13:00:00', 3), (1500, 2, 1, '2012-08-20 15:00:00', 6), (1501, 2, 14, '2012-08-20 19:00:00', 3), (1502, 3, 1, '2012-08-20 08:00:00', 2), (1503, 3, 4, '2012-08-20 11:00:00', 2), (1504, 3, 8, '2012-08-20 12:30:00', 2), (1505, 3, 11, '2012-08-20 15:30:00', 2), (1506, 3, 3, '2012-08-20 17:30:00', 2), (1507, 4, 6, '2012-08-20 08:30:00', 2), (1508, 4, 3, '2012-08-20 09:30:00', 2), (1509, 4, 6, '2012-08-20 10:30:00', 2), (1510, 4, 13, '2012-08-20 11:30:00', 2), (1511, 4, 16, '2012-08-20 12:30:00', 2), (1512, 4, 5, '2012-08-20 13:30:00', 2), (1513, 4, 0, '2012-08-20 14:30:00', 2), (1514, 4, 16, '2012-08-20 16:00:00', 2), (1515, 4, 16, '2012-08-20 17:30:00', 2), (1516, 4, 0, '2012-08-20 18:30:00', 2), (1517, 6, 16, '2012-08-20 08:00:00', 2), (1518, 6, 13, '2012-08-20 09:00:00', 2), (1519, 6, 0, '2012-08-20 10:30:00', 2), (1520, 6, 6, '2012-08-20 11:30:00', 2), (1521, 6, 11, '2012-08-20 12:30:00', 2), (1522, 6, 0, '2012-08-20 14:30:00', 2), (1523, 6, 8, '2012-08-20 16:30:00', 2), (1524, 6, 6, '2012-08-20 19:00:00', 2), (1525, 7, 5, '2012-08-20 08:00:00', 2), (1526, 7, 1, '2012-08-20 11:30:00', 2), (1527, 7, 17, '2012-08-20 12:30:00', 2), (1528, 7, 6, '2012-08-20 14:30:00', 2), (1529, 7, 9, '2012-08-20 16:00:00', 2), (1530, 7, 17, '2012-08-20 17:30:00', 2), (1531, 8, 15, '2012-08-20 10:30:00', 1), (1532, 8, 3, '2012-08-20 11:30:00', 1), (1533, 8, 0, '2012-08-20 13:30:00', 1), (1534, 8, 2, '2012-08-20 14:00:00', 1), (1535, 8, 3, '2012-08-20 17:00:00', 1), (1536, 8, 2, '2012-08-20 18:00:00', 1), (1537, 0, 14, '2012-08-21 09:00:00', 6), (1538, 0, 0, '2012-08-21 13:00:00', 3), (1539, 0, 0, '2012-08-21 18:00:00', 3), (1540, 1, 11, '2012-08-21 09:30:00', 3), (1541, 1, 9, '2012-08-21 11:00:00', 3), (1542, 1, 10, '2012-08-21 12:30:00', 3), (1543, 1, 7, '2012-08-21 14:00:00', 3), (1544, 1, 10, '2012-08-21 16:30:00', 3), (1545, 2, 15, '2012-08-21 08:00:00', 3), (1546, 2, 1, '2012-08-21 09:30:00', 3), (1547, 2, 17, '2012-08-21 11:00:00', 3), (1548, 2, 2, '2012-08-21 12:30:00', 3), (1549, 2, 1, '2012-08-21 15:30:00', 3), (1550, 2, 15, '2012-08-21 17:00:00', 3), (1551, 3, 8, '2012-08-21 10:30:00', 2), (1552, 3, 16, '2012-08-21 12:00:00', 2), (1553, 3, 2, '2012-08-21 16:00:00', 2), (1554, 3, 1, '2012-08-21 18:30:00', 2), (1555, 4, 0, '2012-08-21 08:30:00', 2), (1556, 4, 7, '2012-08-21 10:00:00', 2), (1557, 4, 13, '2012-08-21 11:00:00', 2), (1558, 4, 14, '2012-08-21 12:00:00', 2), (1559, 4, 0, '2012-08-21 13:00:00', 2), (1560, 4, 16, '2012-08-21 14:30:00', 2), (1561, 4, 0, '2012-08-21 16:30:00', 2), (1562, 4, 0, '2012-08-21 18:00:00', 2), (1563, 5, 0, '2012-08-21 08:00:00', 2), (1564, 5, 0, '2012-08-21 18:30:00', 2), (1565, 6, 0, '2012-08-21 09:00:00', 2), (1566, 6, 0, '2012-08-21 10:30:00', 4), (1567, 6, 0, '2012-08-21 14:00:00', 2), (1568, 6, 0, '2012-08-21 15:30:00', 2), (1569, 6, 0, '2012-08-21 17:00:00', 2), (1570, 6, 0, '2012-08-21 19:00:00', 2), (1571, 7, 10, '2012-08-21 09:30:00', 2), (1572, 7, 13, '2012-08-21 13:00:00', 2), (1573, 7, 5, '2012-08-21 15:30:00', 2), (1574, 7, 5, '2012-08-21 17:30:00', 2), (1575, 8, 11, '2012-08-21 08:00:00', 1), (1576, 8, 6, '2012-08-21 09:00:00', 1), (1577, 8, 3, '2012-08-21 09:30:00', 1), (1578, 8, 16, '2012-08-21 10:00:00', 1), (1579, 8, 6, '2012-08-21 10:30:00', 1), (1580, 8, 3, '2012-08-21 11:00:00', 1), (1581, 8, 3, '2012-08-21 12:00:00', 1), (1582, 8, 3, '2012-08-21 13:00:00', 1), (1583, 8, 6, '2012-08-21 13:30:00', 1), (1584, 8, 16, '2012-08-21 16:00:00', 2), (1585, 8, 1, '2012-08-21 19:30:00', 1), (1586, 0, 11, '2012-08-22 08:00:00', 3), (1587, 0, 5, '2012-08-22 10:00:00', 3), (1588, 0, 0, '2012-08-22 11:30:00', 6), (1589, 0, 16, '2012-08-22 15:00:00', 6), (1590, 0, 11, '2012-08-22 18:00:00', 3), (1591, 1, 0, '2012-08-22 08:30:00', 3), (1592, 1, 0, '2012-08-22 10:30:00', 3), (1593, 1, 0, '2012-08-22 13:00:00', 3), (1594, 1, 7, '2012-08-22 15:00:00', 3), (1595, 1, 12, '2012-08-22 17:00:00', 3), (1596, 2, 10, '2012-08-22 09:00:00', 3), (1597, 2, 0, '2012-08-22 10:30:00', 3), (1598, 2, 1, '2012-08-22 12:30:00', 3), (1599, 2, 11, '2012-08-22 15:00:00', 3), (1600, 2, 0, '2012-08-22 16:30:00', 6), (1601, 3, 11, '2012-08-22 10:00:00', 2), (1602, 3, 3, '2012-08-22 11:30:00', 2), (1603, 3, 13, '2012-08-22 13:00:00', 2), (1604, 3, 1, '2012-08-22 14:30:00', 2), (1605, 3, 17, '2012-08-22 15:30:00', 2), (1606, 3, 15, '2012-08-22 16:30:00', 2), (1607, 3, 10, '2012-08-22 18:30:00', 2), (1608, 3, 15, '2012-08-22 19:30:00', 2), (1609, 4, 5, '2012-08-22 08:00:00', 2), (1610, 4, 9, '2012-08-22 09:00:00', 2), (1611, 4, 14, '2012-08-22 10:00:00', 2), (1612, 4, 0, '2012-08-22 11:00:00', 2), (1613, 4, 9, '2012-08-22 12:00:00', 2), (1614, 4, 0, '2012-08-22 14:00:00', 2), (1615, 4, 13, '2012-08-22 15:00:00', 2), (1616, 4, 3, '2012-08-22 16:00:00', 2), (1617, 4, 6, '2012-08-22 17:00:00', 2), (1618, 4, 0, '2012-08-22 18:00:00', 4), (1619, 5, 0, '2012-08-22 18:00:00', 2), (1620, 6, 8, '2012-08-22 08:30:00', 2), (1621, 6, 0, '2012-08-22 09:30:00', 2), (1622, 6, 12, '2012-08-22 11:00:00', 2), (1623, 6, 0, '2012-08-22 12:00:00', 4), (1624, 6, 6, '2012-08-22 14:00:00', 2), (1625, 6, 12, '2012-08-22 15:30:00', 2), (1626, 6, 0, '2012-08-22 18:00:00', 4), (1627, 7, 6, '2012-08-22 09:30:00', 2), (1628, 7, 4, '2012-08-22 11:30:00', 2), (1629, 7, 8, '2012-08-22 15:00:00', 2), (1630, 7, 1, '2012-08-22 16:00:00', 2), (1631, 7, 13, '2012-08-22 17:30:00', 2), (1632, 8, 8, '2012-08-22 08:00:00', 1), (1633, 8, 7, '2012-08-22 11:30:00', 1), (1634, 8, 8, '2012-08-22 12:00:00', 1), (1635, 8, 6, '2012-08-22 12:30:00', 1), (1636, 8, 3, '2012-08-22 15:00:00', 1), (1637, 8, 2, '2012-08-22 15:30:00', 1), (1638, 8, 15, '2012-08-22 16:00:00', 1), (1639, 8, 2, '2012-08-22 17:00:00', 1), (1640, 8, 3, '2012-08-22 19:00:00', 1), (1641, 8, 4, '2012-08-22 19:30:00', 1), (1642, 8, 9, '2012-08-22 20:00:00', 1), (1643, 0, 11, '2012-08-23 08:30:00', 3), (1644, 0, 14, '2012-08-23 11:30:00', 3), (1645, 0, 10, '2012-08-23 13:00:00', 3), (1646, 0, 5, '2012-08-23 15:30:00', 3), (1647, 0, 12, '2012-08-23 17:00:00', 3), (1648, 1, 12, '2012-08-23 09:00:00', 3), (1649, 1, 11, '2012-08-23 10:30:00', 3), (1650, 1, 0, '2012-08-23 13:00:00', 3), (1651, 1, 16, '2012-08-23 14:30:00', 3), (1652, 1, 10, '2012-08-23 16:00:00', 3), (1653, 1, 9, '2012-08-23 17:30:00', 3), (1654, 1, 15, '2012-08-23 19:00:00', 3), (1655, 2, 14, '2012-08-23 09:30:00', 3), (1656, 2, 1, '2012-08-23 11:00:00', 3), (1657, 2, 9, '2012-08-23 13:30:00', 3), (1658, 2, 8, '2012-08-23 15:30:00', 3), (1659, 3, 15, '2012-08-23 09:30:00', 2), (1660, 3, 3, '2012-08-23 10:30:00', 2), (1661, 3, 4, '2012-08-23 14:00:00', 2), (1662, 3, 1, '2012-08-23 15:00:00', 2), (1663, 3, 17, '2012-08-23 16:00:00', 2), (1664, 3, 3, '2012-08-23 17:00:00', 2), (1665, 3, 16, '2012-08-23 19:00:00', 2), (1666, 4, 0, '2012-08-23 08:30:00', 4), (1667, 4, 0, '2012-08-23 11:00:00', 2), (1668, 4, 11, '2012-08-23 12:00:00', 2), (1669, 4, 1, '2012-08-23 13:00:00', 2), (1670, 4, 5, '2012-08-23 14:30:00', 2), (1671, 4, 14, '2012-08-23 15:30:00', 2), (1672, 4, 2, '2012-08-23 16:30:00', 2), (1673, 4, 8, '2012-08-23 17:30:00', 2), (1674, 4, 3, '2012-08-23 18:30:00', 2), (1675, 5, 0, '2012-08-23 12:00:00', 2), (1676, 5, 0, '2012-08-23 16:30:00', 2), (1677, 6, 1, '2012-08-23 08:00:00', 2), (1678, 6, 0, '2012-08-23 09:00:00', 4), (1679, 6, 13, '2012-08-23 13:00:00', 2), (1680, 6, 12, '2012-08-23 14:00:00', 4), (1681, 6, 17, '2012-08-23 18:00:00', 2), (1682, 7, 4, '2012-08-23 11:00:00', 2), (1683, 7, 8, '2012-08-23 14:00:00', 2), (1684, 7, 13, '2012-08-23 16:00:00', 2), (1685, 7, 11, '2012-08-23 17:00:00', 2), (1686, 7, 10, '2012-08-23 18:30:00', 2), (1687, 7, 6, '2012-08-23 19:30:00', 2), (1688, 8, 17, '2012-08-23 09:00:00', 1), (1689, 8, 16, '2012-08-23 09:30:00', 1), (1690, 8, 6, '2012-08-23 10:00:00', 1), (1691, 8, 4, '2012-08-23 10:30:00', 1), (1692, 8, 3, '2012-08-23 13:00:00', 1), (1693, 8, 3, '2012-08-23 14:30:00', 2), (1694, 8, 9, '2012-08-23 15:30:00', 1), (1695, 8, 1, '2012-08-23 16:00:00', 1), (1696, 8, 16, '2012-08-23 17:30:00', 1), (1697, 8, 16, '2012-08-23 18:30:00', 1), (1698, 8, 17, '2012-08-23 19:00:00', 1), (1699, 8, 16, '2012-08-23 20:00:00', 1), (1700, 0, 14, '2012-08-24 09:00:00', 3), (1701, 0, 2, '2012-08-24 11:00:00', 3), (1702, 0, 0, '2012-08-24 12:30:00', 6), (1703, 0, 6, '2012-08-24 15:30:00', 3), (1704, 0, 16, '2012-08-24 17:00:00', 3), (1705, 0, 8, '2012-08-24 19:00:00', 3), (1706, 1, 12, '2012-08-24 08:00:00', 3), (1707, 1, 9, '2012-08-24 09:30:00', 3), (1708, 1, 0, '2012-08-24 11:30:00', 3), (1709, 1, 8, '2012-08-24 13:00:00', 3), (1710, 1, 10, '2012-08-24 15:30:00', 3), (1711, 1, 12, '2012-08-24 18:00:00', 3), (1712, 2, 13, '2012-08-24 08:00:00', 3), (1713, 2, 0, '2012-08-24 11:00:00', 3), (1714, 2, 15, '2012-08-24 13:00:00', 3), (1715, 2, 16, '2012-08-24 15:00:00', 3), (1716, 2, 12, '2012-08-24 16:30:00', 3), (1717, 3, 1, '2012-08-24 08:30:00', 2), (1718, 3, 3, '2012-08-24 11:00:00', 2), (1719, 3, 17, '2012-08-24 14:00:00', 2), (1720, 3, 8, '2012-08-24 16:30:00', 2), (1721, 3, 15, '2012-08-24 17:30:00', 2), (1722, 3, 10, '2012-08-24 18:30:00', 2), (1723, 4, 0, '2012-08-24 08:00:00', 2), (1724, 4, 3, '2012-08-24 10:00:00', 2), (1725, 4, 9, '2012-08-24 12:00:00', 2), (1726, 4, 14, '2012-08-24 13:00:00', 2), (1727, 4, 3, '2012-08-24 14:00:00', 2), (1728, 4, 0, '2012-08-24 17:00:00', 2), (1729, 4, 3, '2012-08-24 18:00:00', 2), (1730, 4, 0, '2012-08-24 19:00:00', 2), (1731, 5, 0, '2012-08-24 18:30:00', 2), (1732, 6, 6, '2012-08-24 09:30:00', 2), (1733, 6, 0, '2012-08-24 11:00:00', 2), (1734, 6, 14, '2012-08-24 12:00:00', 2), (1735, 6, 0, '2012-08-24 14:30:00', 2), (1736, 6, 11, '2012-08-24 17:00:00', 2), (1737, 6, 14, '2012-08-24 18:30:00', 2), (1738, 6, 0, '2012-08-24 19:30:00', 2), (1739, 7, 15, '2012-08-24 09:30:00', 2), (1740, 7, 17, '2012-08-24 13:00:00', 2), (1741, 7, 13, '2012-08-24 14:00:00', 2), (1742, 7, 4, '2012-08-24 17:00:00', 2), (1743, 7, 2, '2012-08-24 18:30:00', 2), (1744, 8, 3, '2012-08-24 08:30:00', 1), (1745, 8, 16, '2012-08-24 11:00:00', 1), (1746, 8, 16, '2012-08-24 13:30:00', 1), (1747, 8, 14, '2012-08-24 14:00:00', 1), (1748, 8, 14, '2012-08-24 17:30:00', 1), (1749, 0, 8, '2012-08-25 08:00:00', 3), (1750, 0, 7, '2012-08-25 11:00:00', 3), (1751, 0, 0, '2012-08-25 12:30:00', 3), (1752, 0, 5, '2012-08-25 14:00:00', 3), (1753, 0, 0, '2012-08-25 15:30:00', 3), (1754, 0, 17, '2012-08-25 17:00:00', 3), (1755, 1, 9, '2012-08-25 08:00:00', 3), (1756, 1, 11, '2012-08-25 11:30:00', 3), (1757, 1, 0, '2012-08-25 13:30:00', 9), (1758, 1, 15, '2012-08-25 18:30:00', 3), (1759, 2, 2, '2012-08-25 08:00:00', 3), (1760, 2, 1, '2012-08-25 09:30:00', 3), (1761, 2, 14, '2012-08-25 11:00:00', 3), (1762, 2, 1, '2012-08-25 12:30:00', 3), (1763, 2, 1, '2012-08-25 16:30:00', 3), (1764, 3, 16, '2012-08-25 08:00:00', 2), (1765, 3, 16, '2012-08-25 09:30:00', 2), (1766, 3, 0, '2012-08-25 12:00:00', 2), (1767, 3, 15, '2012-08-25 14:30:00', 2), (1768, 3, 11, '2012-08-25 18:30:00', 2), (1769, 3, 3, '2012-08-25 19:30:00', 2), (1770, 4, 14, '2012-08-25 08:00:00', 2), (1771, 4, 0, '2012-08-25 09:30:00', 2), (1772, 4, 6, '2012-08-25 10:30:00', 2), (1773, 4, 10, '2012-08-25 11:30:00', 2), (1774, 4, 3, '2012-08-25 12:30:00', 2), (1775, 4, 11, '2012-08-25 14:00:00', 2), (1776, 4, 13, '2012-08-25 15:30:00', 4), (1777, 4, 3, '2012-08-25 17:30:00', 2), (1778, 5, 11, '2012-08-25 08:00:00', 2), (1779, 5, 0, '2012-08-25 14:30:00', 2), (1780, 6, 0, '2012-08-25 08:30:00', 4), (1781, 6, 0, '2012-08-25 11:00:00', 2), (1782, 6, 12, '2012-08-25 14:00:00', 2), (1783, 6, 0, '2012-08-25 18:30:00', 2), (1784, 6, 6, '2012-08-25 19:30:00', 2), (1785, 7, 15, '2012-08-25 08:30:00', 2), (1786, 7, 2, '2012-08-25 09:30:00', 2), (1787, 7, 4, '2012-08-25 11:00:00', 2), (1788, 7, 13, '2012-08-25 14:00:00', 2), (1789, 7, 8, '2012-08-25 15:00:00', 2), (1790, 7, 0, '2012-08-25 19:00:00', 2), (1791, 8, 15, '2012-08-25 08:00:00', 1), (1792, 8, 3, '2012-08-25 09:30:00', 3), (1793, 8, 16, '2012-08-25 11:00:00', 1), (1794, 8, 2, '2012-08-25 12:00:00', 1), (1795, 8, 16, '2012-08-25 12:30:00', 2), (1796, 8, 3, '2012-08-25 13:30:00', 1), (1797, 8, 16, '2012-08-25 14:30:00', 1), (1798, 8, 6, '2012-08-25 15:00:00', 1), (1799, 8, 3, '2012-08-25 15:30:00', 3), (1800, 8, 2, '2012-08-25 17:30:00', 1), (1801, 8, 16, '2012-08-25 19:00:00', 1), (1802, 0, 11, '2012-08-26 08:30:00', 3), (1803, 0, 6, '2012-08-26 10:30:00', 3), (1804, 0, 11, '2012-08-26 12:00:00', 3), (1805, 0, 0, '2012-08-26 15:00:00', 3), (1806, 0, 6, '2012-08-26 17:00:00', 3), (1807, 0, 5, '2012-08-26 19:00:00', 3), (1808, 1, 12, '2012-08-26 08:30:00', 3), (1809, 1, 11, '2012-08-26 10:30:00', 3), (1810, 1, 0, '2012-08-26 13:00:00', 6), (1811, 1, 13, '2012-08-26 16:00:00', 3), (1812, 1, 0, '2012-08-26 17:30:00', 3), (1813, 2, 1, '2012-08-26 08:30:00', 3), (1814, 2, 16, '2012-08-26 10:00:00', 3), (1815, 2, 1, '2012-08-26 11:30:00', 3), (1816, 2, 1, '2012-08-26 15:30:00', 3), (1817, 2, 0, '2012-08-26 17:30:00', 3), (1818, 3, 3, '2012-08-26 08:00:00', 2), (1819, 3, 13, '2012-08-26 13:00:00', 2), (1820, 3, 10, '2012-08-26 16:00:00', 2), (1821, 3, 0, '2012-08-26 18:00:00', 2), (1822, 3, 6, '2012-08-26 19:30:00', 2), (1823, 4, 0, '2012-08-26 08:00:00', 4), (1824, 4, 14, '2012-08-26 10:00:00', 2), (1825, 4, 0, '2012-08-26 11:30:00', 2), (1826, 4, 10, '2012-08-26 13:00:00', 2), (1827, 4, 0, '2012-08-26 15:30:00', 2), (1828, 4, 3, '2012-08-26 18:30:00', 2), (1829, 5, 0, '2012-08-26 08:00:00', 2), (1830, 6, 0, '2012-08-26 08:00:00', 2), (1831, 6, 0, '2012-08-26 11:00:00', 2), (1832, 6, 12, '2012-08-26 12:00:00', 2), (1833, 6, 0, '2012-08-26 15:00:00', 2), (1834, 6, 12, '2012-08-26 16:00:00', 2), (1835, 6, 0, '2012-08-26 18:30:00', 4), (1836, 7, 4, '2012-08-26 08:00:00', 2), (1837, 7, 17, '2012-08-26 10:00:00', 2), (1838, 7, 8, '2012-08-26 11:30:00', 2), (1839, 7, 4, '2012-08-26 13:30:00', 2), (1840, 7, 7, '2012-08-26 16:30:00', 2), (1841, 7, 7, '2012-08-26 18:00:00', 2), (1842, 7, 0, '2012-08-26 19:00:00', 2), (1843, 8, 15, '2012-08-26 08:00:00', 1), (1844, 8, 3, '2012-08-26 09:30:00', 1), (1845, 8, 3, '2012-08-26 10:30:00', 2), (1846, 8, 16, '2012-08-26 11:30:00', 1), (1847, 8, 3, '2012-08-26 12:00:00', 1), (1848, 8, 15, '2012-08-26 12:30:00', 1), (1849, 8, 3, '2012-08-26 14:00:00', 1), (1850, 8, 16, '2012-08-26 14:30:00', 1), (1851, 8, 3, '2012-08-26 15:30:00', 1), (1852, 8, 0, '2012-08-26 16:00:00', 1), (1853, 8, 0, '2012-08-26 17:00:00', 1), (1854, 8, 3, '2012-08-26 18:00:00', 1), (1855, 8, 8, '2012-08-26 20:00:00', 1), (1856, 0, 0, '2012-08-27 09:00:00', 3), (1857, 0, 5, '2012-08-27 10:30:00', 3), (1858, 0, 17, '2012-08-27 13:00:00', 3), (1859, 0, 7, '2012-08-27 15:30:00', 3), (1860, 0, 0, '2012-08-27 17:30:00', 6), (1861, 1, 12, '2012-08-27 08:30:00', 3), (1862, 1, 0, '2012-08-27 11:00:00', 3), (1863, 1, 9, '2012-08-27 12:30:00', 3), (1864, 1, 8, '2012-08-27 14:30:00', 3), (1865, 1, 9, '2012-08-27 16:30:00', 3), (1866, 1, 10, '2012-08-27 18:30:00', 3), (1867, 2, 0, '2012-08-27 08:00:00', 3), (1868, 2, 0, '2012-08-27 11:00:00', 3), (1869, 2, 2, '2012-08-27 14:30:00', 3), (1870, 2, 2, '2012-08-27 16:30:00', 3), (1871, 3, 15, '2012-08-27 09:30:00', 2), (1872, 3, 0, '2012-08-27 11:30:00', 2), (1873, 3, 11, '2012-08-27 14:00:00', 2), (1874, 3, 16, '2012-08-27 17:00:00', 2), (1875, 3, 16, '2012-08-27 19:30:00', 2), (1876, 4, 9, '2012-08-27 08:30:00', 2), (1877, 4, 5, '2012-08-27 09:30:00', 2), (1878, 4, 3, '2012-08-27 10:30:00', 2), (1879, 4, 0, '2012-08-27 12:00:00', 2), (1880, 4, 8, '2012-08-27 13:30:00', 2), (1881, 4, 13, '2012-08-27 14:30:00', 2), (1882, 4, 11, '2012-08-27 15:30:00', 2), (1883, 4, 0, '2012-08-27 16:30:00', 2), (1884, 4, 11, '2012-08-27 18:00:00', 2), (1885, 4, 12, '2012-08-27 19:00:00', 2), (1886, 5, 20, '2012-08-27 09:00:00', 2), (1887, 5, 0, '2012-08-27 10:30:00', 2), (1888, 5, 12, '2012-08-27 14:00:00', 2), (1889, 6, 0, '2012-08-27 08:00:00', 2), (1890, 6, 0, '2012-08-27 09:30:00', 4), (1891, 6, 6, '2012-08-27 13:00:00', 2), (1892, 6, 0, '2012-08-27 15:00:00', 2), (1893, 6, 12, '2012-08-27 16:30:00', 2), (1894, 6, 0, '2012-08-27 17:30:00', 2), (1895, 6, 1, '2012-08-27 19:00:00', 2), (1896, 7, 17, '2012-08-27 09:00:00', 2), (1897, 7, 4, '2012-08-27 10:30:00', 2), (1898, 7, 2, '2012-08-27 12:30:00', 2), (1899, 7, 14, '2012-08-27 13:30:00', 2), (1900, 7, 4, '2012-08-27 14:30:00', 2), (1901, 7, 13, '2012-08-27 17:00:00', 2), (1902, 7, 8, '2012-08-27 18:00:00', 2), (1903, 7, 15, '2012-08-27 19:00:00', 2), (1904, 8, 9, '2012-08-27 08:00:00', 1), (1905, 8, 16, '2012-08-27 10:00:00', 1), (1906, 8, 16, '2012-08-27 11:00:00', 3), (1907, 8, 3, '2012-08-27 13:30:00', 1), (1908, 8, 9, '2012-08-27 14:30:00', 1), (1909, 8, 16, '2012-08-27 15:00:00', 1), (1910, 8, 4, '2012-08-27 15:30:00', 1), (1911, 8, 12, '2012-08-27 16:00:00', 1), (1912, 8, 11, '2012-08-27 17:00:00', 1), (1913, 8, 3, '2012-08-27 20:00:00', 1), (1914, 0, 11, '2012-08-28 08:30:00', 3), (1915, 0, 14, '2012-08-28 10:00:00', 3), (1916, 0, 10, '2012-08-28 11:30:00', 3), (1917, 0, 17, '2012-08-28 14:30:00', 3), (1918, 0, 6, '2012-08-28 16:00:00', 3), (1919, 0, 16, '2012-08-28 17:30:00', 3), (1920, 1, 12, '2012-08-28 08:30:00', 3), (1921, 1, 11, '2012-08-28 13:00:00', 3), (1922, 1, 9, '2012-08-28 14:30:00', 3), (1923, 1, 12, '2012-08-28 19:00:00', 3), (1924, 2, 17, '2012-08-28 08:30:00', 3), (1925, 2, 1, '2012-08-28 10:30:00', 3), (1926, 2, 2, '2012-08-28 12:00:00', 3), (1927, 2, 1, '2012-08-28 13:30:00', 9), (1928, 2, 0, '2012-08-28 18:00:00', 3), (1929, 3, 8, '2012-08-28 11:30:00', 2), (1930, 3, 15, '2012-08-28 13:00:00', 2), (1931, 3, 20, '2012-08-28 14:00:00', 2), (1932, 3, 17, '2012-08-28 18:30:00', 2), (1933, 4, 8, '2012-08-28 08:30:00', 2), (1934, 4, 3, '2012-08-28 10:30:00', 2), (1935, 4, 0, '2012-08-28 11:30:00', 4), (1936, 4, 17, '2012-08-28 13:30:00', 2), (1937, 4, 10, '2012-08-28 15:30:00', 2), (1938, 4, 0, '2012-08-28 16:30:00', 2), (1939, 4, 13, '2012-08-28 18:30:00', 2), (1940, 4, 20, '2012-08-28 19:30:00', 2), (1941, 5, 0, '2012-08-28 09:00:00', 2), (1942, 5, 7, '2012-08-28 10:30:00', 2), (1943, 5, 0, '2012-08-28 16:00:00', 2), (1944, 5, 0, '2012-08-28 18:00:00', 2), (1945, 6, 6, '2012-08-28 08:00:00', 2), (1946, 6, 0, '2012-08-28 10:30:00', 4), (1947, 6, 14, '2012-08-28 12:30:00', 2), (1948, 6, 12, '2012-08-28 18:00:00', 2), (1949, 7, 13, '2012-08-28 08:00:00', 2), (1950, 7, 2, '2012-08-28 09:00:00', 2), (1951, 7, 8, '2012-08-28 10:00:00', 2), (1952, 7, 9, '2012-08-28 13:30:00', 2), (1953, 7, 15, '2012-08-28 14:30:00', 2), (1954, 7, 4, '2012-08-28 17:00:00', 2), (1955, 7, 2, '2012-08-28 18:00:00', 2), (1956, 7, 4, '2012-08-28 19:00:00', 2), (1957, 8, 16, '2012-08-28 08:00:00', 1), (1958, 8, 3, '2012-08-28 09:30:00', 1), (1959, 8, 16, '2012-08-28 10:00:00', 1), (1960, 8, 2, '2012-08-28 11:30:00', 1), (1961, 8, 12, '2012-08-28 12:00:00', 1), (1962, 8, 16, '2012-08-28 13:00:00', 1), (1963, 8, 4, '2012-08-28 13:30:00', 1), (1964, 8, 16, '2012-08-28 15:30:00', 1), (1965, 8, 3, '2012-08-28 17:00:00', 2), (1966, 8, 0, '2012-08-28 19:00:00', 1), (1967, 0, 0, '2012-08-29 08:30:00', 3), (1968, 0, 7, '2012-08-29 11:30:00', 3), (1969, 0, 10, '2012-08-29 13:30:00', 3), (1970, 0, 0, '2012-08-29 16:00:00', 3), (1971, 0, 9, '2012-08-29 17:30:00', 3), (1972, 0, 0, '2012-08-29 19:00:00', 3), (1973, 1, 12, '2012-08-29 08:00:00', 3), (1974, 1, 10, '2012-08-29 10:00:00', 3), (1975, 1, 9, '2012-08-29 13:30:00', 3), (1976, 1, 10, '2012-08-29 16:30:00', 3), (1977, 1, 8, '2012-08-29 18:00:00', 3), (1978, 2, 1, '2012-08-29 08:30:00', 3), (1979, 2, 1, '2012-08-29 10:30:00', 3), (1980, 2, 8, '2012-08-29 12:00:00', 3), (1981, 2, 8, '2012-08-29 14:00:00', 3), (1982, 2, 6, '2012-08-29 15:30:00', 3), (1983, 2, 1, '2012-08-29 17:30:00', 3), (1984, 2, 11, '2012-08-29 19:00:00', 3), (1985, 3, 3, '2012-08-29 08:30:00', 2), (1986, 3, 3, '2012-08-29 10:30:00', 2), (1987, 3, 1, '2012-08-29 14:00:00', 2), (1988, 3, 14, '2012-08-29 16:00:00', 2), (1989, 3, 16, '2012-08-29 17:00:00', 2), (1990, 3, 3, '2012-08-29 18:30:00', 4), (1991, 4, 0, '2012-08-29 08:30:00', 2), (1992, 4, 13, '2012-08-29 10:00:00', 2), (1993, 4, 0, '2012-08-29 11:00:00', 2), (1994, 4, 1, '2012-08-29 12:00:00', 2), (1995, 4, 0, '2012-08-29 13:00:00', 4), (1996, 4, 5, '2012-08-29 15:00:00', 2), (1997, 4, 0, '2012-08-29 16:30:00', 2), (1998, 4, 14, '2012-08-29 18:00:00', 2), (1999, 4, 20, '2012-08-29 19:30:00', 2), (2000, 6, 0, '2012-08-29 08:00:00', 2), (2001, 6, 6, '2012-08-29 10:30:00', 4), (2002, 6, 0, '2012-08-29 13:00:00', 2), (2003, 6, 0, '2012-08-29 15:30:00', 2), (2004, 6, 12, '2012-08-29 17:30:00', 2), (2005, 6, 12, '2012-08-29 19:00:00', 2), (2006, 7, 8, '2012-08-29 10:00:00', 2), (2007, 7, 15, '2012-08-29 13:00:00', 2), (2008, 7, 4, '2012-08-29 15:00:00', 2), (2009, 7, 2, '2012-08-29 16:30:00', 2), (2010, 7, 13, '2012-08-29 17:30:00', 2), (2011, 7, 4, '2012-08-29 18:30:00', 2), (2012, 7, 8, '2012-08-29 19:30:00', 2), (2013, 8, 15, '2012-08-29 08:00:00', 1), (2014, 8, 0, '2012-08-29 11:30:00', 1), (2015, 8, 3, '2012-08-29 13:30:00', 1), (2016, 8, 16, '2012-08-29 14:00:00', 1), (2017, 8, 3, '2012-08-29 15:00:00', 2), (2018, 8, 0, '2012-08-29 17:30:00', 1), (2019, 8, 16, '2012-08-29 18:30:00', 1), (2020, 8, 1, '2012-08-29 19:30:00', 1), (2021, 0, 0, '2012-08-30 08:00:00', 3), (2022, 0, 17, '2012-08-30 09:30:00', 3), (2023, 0, 5, '2012-08-30 12:30:00', 3), (2024, 0, 0, '2012-08-30 14:00:00', 3), (2025, 0, 5, '2012-08-30 16:00:00', 3), (2026, 1, 8, '2012-08-30 08:00:00', 3), (2027, 1, 10, '2012-08-30 12:30:00', 3), (2028, 1, 11, '2012-08-30 14:00:00', 3), (2029, 1, 0, '2012-08-30 16:00:00', 3), (2030, 1, 0, '2012-08-30 19:00:00', 3), (2031, 2, 1, '2012-08-30 11:00:00', 3), (2032, 2, 15, '2012-08-30 12:30:00', 3), (2033, 2, 1, '2012-08-30 14:00:00', 3), (2034, 2, 7, '2012-08-30 17:00:00', 3), (2035, 2, 21, '2012-08-30 19:00:00', 3), (2036, 3, 10, '2012-08-30 08:00:00', 2), (2037, 3, 6, '2012-08-30 09:30:00', 2), (2038, 3, 14, '2012-08-30 12:30:00', 2), (2039, 3, 20, '2012-08-30 15:00:00', 2), (2040, 3, 20, '2012-08-30 16:30:00', 2), (2041, 3, 16, '2012-08-30 17:30:00', 2), (2042, 3, 6, '2012-08-30 19:30:00', 2), (2043, 4, 0, '2012-08-30 08:00:00', 2), (2044, 4, 13, '2012-08-30 09:00:00', 2), (2045, 4, 0, '2012-08-30 10:00:00', 2), (2046, 4, 10, '2012-08-30 14:30:00', 2), (2047, 4, 11, '2012-08-30 15:30:00', 2), (2048, 4, 1, '2012-08-30 16:30:00', 2), (2049, 4, 0, '2012-08-30 18:30:00', 2), (2050, 6, 12, '2012-08-30 08:00:00', 6), (2051, 6, 12, '2012-08-30 11:30:00', 2), (2052, 6, 0, '2012-08-30 13:00:00', 4), (2053, 6, 0, '2012-08-30 15:30:00', 2), (2054, 6, 12, '2012-08-30 16:30:00', 2), (2055, 6, 0, '2012-08-30 17:30:00', 2), (2056, 7, 0, '2012-08-30 11:30:00', 2), (2057, 7, 4, '2012-08-30 14:30:00', 2), (2058, 7, 15, '2012-08-30 17:30:00', 2), (2059, 7, 8, '2012-08-30 19:00:00', 2), (2060, 8, 1, '2012-08-30 08:00:00', 1), (2061, 8, 21, '2012-08-30 10:00:00', 1), (2062, 8, 3, '2012-08-30 10:30:00', 1), (2063, 8, 20, '2012-08-30 11:00:00', 1), (2064, 8, 17, '2012-08-30 12:30:00', 1), (2065, 8, 3, '2012-08-30 13:00:00', 1), (2066, 8, 2, '2012-08-30 14:00:00', 1), (2067, 8, 21, '2012-08-30 15:30:00', 3), (2068, 8, 3, '2012-08-30 18:00:00', 1), (2069, 8, 6, '2012-08-30 19:00:00', 1), (2070, 8, 16, '2012-08-30 19:30:00', 1), (2071, 8, 9, '2012-08-30 20:00:00', 1), (2072, 0, 5, '2012-08-31 09:00:00', 3), (2073, 0, 0, '2012-08-31 10:30:00', 3), (2074, 0, 11, '2012-08-31 12:00:00', 3), (2075, 0, 6, '2012-08-31 14:30:00', 3), (2076, 0, 2, '2012-08-31 16:30:00', 3), (2077, 0, 5, '2012-08-31 19:00:00', 3), (2078, 1, 0, '2012-08-31 08:00:00', 3), (2079, 1, 0, '2012-08-31 10:30:00', 3), (2080, 1, 12, '2012-08-31 12:00:00', 3), (2081, 1, 8, '2012-08-31 13:30:00', 3), (2082, 1, 10, '2012-08-31 15:00:00', 6), (2083, 1, 8, '2012-08-31 18:30:00', 3), (2084, 2, 2, '2012-08-31 08:30:00', 3), (2085, 2, 0, '2012-08-31 11:00:00', 3), (2086, 2, 16, '2012-08-31 12:30:00', 3), (2087, 2, 21, '2012-08-31 14:00:00', 3), (2088, 2, 21, '2012-08-31 17:00:00', 3), (2089, 2, 0, '2012-08-31 19:00:00', 3), (2090, 3, 20, '2012-08-31 09:00:00', 2), (2091, 3, 10, '2012-08-31 10:30:00', 2), (2092, 3, 3, '2012-08-31 12:30:00', 2), (2093, 3, 20, '2012-08-31 19:30:00', 2), (2094, 4, 0, '2012-08-31 08:30:00', 2), (2095, 4, 0, '2012-08-31 10:00:00', 2), (2096, 4, 14, '2012-08-31 12:30:00', 2), (2097, 4, 0, '2012-08-31 13:30:00', 2), (2098, 4, 11, '2012-08-31 14:30:00', 4), (2099, 4, 9, '2012-08-31 16:30:00', 2), (2100, 4, 6, '2012-08-31 18:00:00', 2), (2101, 4, 11, '2012-08-31 19:00:00', 2), (2102, 5, 0, '2012-08-31 09:30:00', 2), (2103, 5, 0, '2012-08-31 11:00:00', 2), (2104, 5, 0, '2012-08-31 15:00:00', 2), (2105, 5, 11, '2012-08-31 17:00:00', 2), (2106, 6, 1, '2012-08-31 09:00:00', 4), (2107, 6, 0, '2012-08-31 11:00:00', 4), (2108, 6, 0, '2012-08-31 14:30:00', 4), (2109, 6, 12, '2012-08-31 18:00:00', 4), (2110, 7, 9, '2012-08-31 08:00:00', 2), (2111, 7, 5, '2012-08-31 11:30:00', 2), (2112, 7, 17, '2012-08-31 13:00:00', 2), (2113, 7, 15, '2012-08-31 15:00:00', 2), (2114, 7, 17, '2012-08-31 16:30:00', 2), (2115, 7, 13, '2012-08-31 17:30:00', 2), (2116, 7, 10, '2012-08-31 18:30:00', 2), (2117, 8, 17, '2012-08-31 08:30:00', 1), (2118, 8, 3, '2012-08-31 10:00:00', 1), (2119, 8, 21, '2012-08-31 12:30:00', 2), (2120, 8, 3, '2012-08-31 13:30:00', 1), (2121, 8, 15, '2012-08-31 14:00:00', 1), (2122, 8, 3, '2012-08-31 14:30:00', 1), (2123, 8, 16, '2012-08-31 16:00:00', 1), (2124, 8, 6, '2012-08-31 16:30:00', 1), (2125, 8, 3, '2012-08-31 17:00:00', 1), (2126, 8, 2, '2012-08-31 18:00:00', 1), (2127, 8, 20, '2012-08-31 18:30:00', 1), (2128, 8, 21, '2012-08-31 19:00:00', 1), (2129, 8, 21, '2012-08-31 20:00:00', 1), (2130, 0, 0, '2012-09-01 08:00:00', 3), (2131, 0, 17, '2012-09-01 11:00:00', 3), (2132, 0, 7, '2012-09-01 12:30:00', 3), (2133, 0, 6, '2012-09-01 15:00:00', 3), (2134, 0, 4, '2012-09-01 17:00:00', 3), (2135, 1, 0, '2012-09-01 08:00:00', 3), (2136, 1, 11, '2012-09-01 09:30:00', 3), (2137, 1, 10, '2012-09-01 11:00:00', 3), (2138, 1, 12, '2012-09-01 14:30:00', 3), (2139, 1, 0, '2012-09-01 16:30:00', 3), (2140, 1, 12, '2012-09-01 19:00:00', 3), (2141, 2, 1, '2012-09-01 09:00:00', 3), (2142, 2, 21, '2012-09-01 13:30:00', 3), (2143, 2, 1, '2012-09-01 16:30:00', 3), (2144, 2, 15, '2012-09-01 18:00:00', 3), (2145, 3, 17, '2012-09-01 08:30:00', 2), (2146, 3, 13, '2012-09-01 09:30:00', 2), (2147, 3, 15, '2012-09-01 10:30:00', 2), (2148, 3, 17, '2012-09-01 12:30:00', 2), (2149, 3, 17, '2012-09-01 14:00:00', 2), (2150, 3, 16, '2012-09-01 15:00:00', 2), (2151, 3, 0, '2012-09-01 16:30:00', 2), (2152, 3, 16, '2012-09-01 18:00:00', 2), (2153, 3, 17, '2012-09-01 19:00:00', 2), (2154, 4, 8, '2012-09-01 08:30:00', 2), (2155, 4, 9, '2012-09-01 11:00:00', 2), (2156, 4, 11, '2012-09-01 12:30:00', 2), (2157, 4, 0, '2012-09-01 13:30:00', 6), (2158, 4, 0, '2012-09-01 17:30:00', 2), (2159, 4, 16, '2012-09-01 19:30:00', 2), (2160, 5, 0, '2012-09-01 09:30:00', 2), (2161, 5, 7, '2012-09-01 15:30:00', 2), (2162, 6, 0, '2012-09-01 09:30:00', 8), (2163, 6, 4, '2012-09-01 15:00:00', 2), (2164, 6, 0, '2012-09-01 16:00:00', 4), (2165, 6, 2, '2012-09-01 18:00:00', 2), (2166, 7, 21, '2012-09-01 08:30:00', 2), (2167, 7, 2, '2012-09-01 11:30:00', 2), (2168, 7, 1, '2012-09-01 14:00:00', 2), (2169, 7, 15, '2012-09-01 15:00:00', 2), (2170, 7, 13, '2012-09-01 17:30:00', 2), (2171, 7, 9, '2012-09-01 19:00:00', 2), (2172, 8, 17, '2012-09-01 10:00:00', 1), (2173, 8, 1, '2012-09-01 10:30:00', 1), (2174, 8, 14, '2012-09-01 11:00:00', 1), (2175, 8, 21, '2012-09-01 11:30:00', 1), (2176, 8, 21, '2012-09-01 15:00:00', 1), (2177, 8, 3, '2012-09-01 16:00:00', 1), (2178, 8, 20, '2012-09-01 18:00:00', 1), (2179, 8, 3, '2012-09-01 18:30:00', 1), (2180, 8, 7, '2012-09-01 19:30:00', 1), (2181, 0, 10, '2012-09-02 08:30:00', 3), (2182, 0, 0, '2012-09-02 10:30:00', 3), (2183, 0, 12, '2012-09-02 12:00:00', 3), (2184, 0, 5, '2012-09-02 15:00:00', 3), (2185, 0, 6, '2012-09-02 18:00:00', 3), (2186, 1, 15, '2012-09-02 08:30:00', 3), (2187, 1, 11, '2012-09-02 12:30:00', 3), (2188, 1, 10, '2012-09-02 16:00:00', 6), (2189, 1, 0, '2012-09-02 19:00:00', 3), (2190, 2, 0, '2012-09-02 09:30:00', 3), (2191, 2, 21, '2012-09-02 11:00:00', 3), (2192, 2, 0, '2012-09-02 12:30:00', 3), (2193, 2, 9, '2012-09-02 15:30:00', 3), (2194, 2, 5, '2012-09-02 17:00:00', 3), (2195, 2, 0, '2012-09-02 19:00:00', 3), (2196, 3, 15, '2012-09-02 13:30:00', 2), (2197, 3, 3, '2012-09-02 14:30:00', 2), (2198, 3, 15, '2012-09-02 16:30:00', 2), (2199, 3, 15, '2012-09-02 18:00:00', 2), (2200, 3, 17, '2012-09-02 19:30:00', 2), (2201, 4, 0, '2012-09-02 08:00:00', 2), (2202, 4, 0, '2012-09-02 09:30:00', 6), (2203, 4, 5, '2012-09-02 12:30:00', 2), (2204, 4, 0, '2012-09-02 13:30:00', 4), (2205, 4, 20, '2012-09-02 15:30:00', 2), (2206, 4, 8, '2012-09-02 16:30:00', 2), (2207, 4, 14, '2012-09-02 17:30:00', 2), (2208, 4, 0, '2012-09-02 18:30:00', 2), (2209, 5, 0, '2012-09-02 09:30:00', 2), (2210, 5, 0, '2012-09-02 11:30:00', 2), (2211, 6, 0, '2012-09-02 08:30:00', 4), (2212, 6, 0, '2012-09-02 11:00:00', 2), (2213, 6, 10, '2012-09-02 14:00:00', 2), (2214, 6, 0, '2012-09-02 15:00:00', 4), (2215, 6, 0, '2012-09-02 17:30:00', 2), (2216, 6, 0, '2012-09-02 19:00:00', 2), (2217, 7, 17, '2012-09-02 08:30:00', 2), (2218, 7, 2, '2012-09-02 10:30:00', 2), (2219, 7, 22, '2012-09-02 11:30:00', 2), (2220, 7, 7, '2012-09-02 13:00:00', 2), (2221, 7, 8, '2012-09-02 14:30:00', 2), (2222, 7, 2, '2012-09-02 16:30:00', 2), (2223, 7, 2, '2012-09-02 18:30:00', 2), (2224, 8, 20, '2012-09-02 08:00:00', 1), (2225, 8, 3, '2012-09-02 08:30:00', 1), (2226, 8, 16, '2012-09-02 09:30:00', 2), (2227, 8, 3, '2012-09-02 10:30:00', 1), (2228, 8, 3, '2012-09-02 11:30:00', 1), (2229, 8, 7, '2012-09-02 12:30:00', 1), (2230, 8, 16, '2012-09-02 13:00:00', 1), (2231, 8, 16, '2012-09-02 16:00:00', 1), (2232, 8, 3, '2012-09-02 17:30:00', 1), (2233, 8, 21, '2012-09-02 18:30:00', 1), (2234, 8, 3, '2012-09-02 19:00:00', 1), (2235, 8, 16, '2012-09-02 20:00:00', 1), (2236, 0, 0, '2012-09-03 08:00:00', 6), (2237, 0, 11, '2012-09-03 11:00:00', 6), (2238, 0, 14, '2012-09-03 14:00:00', 3), (2239, 0, 0, '2012-09-03 15:30:00', 3), (2240, 0, 16, '2012-09-03 18:00:00', 3), (2241, 1, 12, '2012-09-03 08:00:00', 3), (2242, 1, 0, '2012-09-03 10:00:00', 6), (2243, 1, 0, '2012-09-03 13:30:00', 3), (2244, 1, 8, '2012-09-03 15:00:00', 6), (2245, 1, 11, '2012-09-03 18:00:00', 3), (2246, 2, 21, '2012-09-03 08:30:00', 3), (2247, 2, 12, '2012-09-03 10:00:00', 3), (2248, 2, 9, '2012-09-03 12:30:00', 3), (2249, 2, 17, '2012-09-03 14:00:00', 3), (2250, 2, 0, '2012-09-03 19:00:00', 3), (2251, 3, 22, '2012-09-03 09:30:00', 2), (2252, 3, 21, '2012-09-03 11:30:00', 2), (2253, 3, 13, '2012-09-03 12:30:00', 2), (2254, 3, 20, '2012-09-03 13:30:00', 4), (2255, 3, 17, '2012-09-03 17:30:00', 2), (2256, 3, 20, '2012-09-03 19:00:00', 2), (2257, 4, 0, '2012-09-03 08:00:00', 2), (2258, 4, 8, '2012-09-03 09:30:00', 2), (2259, 4, 0, '2012-09-03 11:00:00', 4), (2260, 4, 8, '2012-09-03 13:00:00', 2), (2261, 4, 0, '2012-09-03 15:00:00', 2), (2262, 4, 3, '2012-09-03 16:00:00', 2), (2263, 4, 0, '2012-09-03 17:00:00', 2), (2264, 4, 14, '2012-09-03 19:00:00', 2), (2265, 5, 10, '2012-09-03 11:30:00', 2), (2266, 6, 6, '2012-09-03 11:00:00', 2), (2267, 6, 0, '2012-09-03 12:00:00', 2), (2268, 6, 0, '2012-09-03 13:30:00', 4), (2269, 6, 6, '2012-09-03 16:00:00', 4), (2270, 6, 12, '2012-09-03 18:30:00', 2), (2271, 6, 0, '2012-09-03 19:30:00', 2), (2272, 7, 15, '2012-09-03 09:30:00', 2), (2273, 7, 4, '2012-09-03 12:00:00', 4), (2274, 7, 15, '2012-09-03 15:00:00', 2), (2275, 7, 15, '2012-09-03 17:00:00', 2), (2276, 7, 1, '2012-09-03 18:00:00', 2), (2277, 7, 7, '2012-09-03 19:00:00', 2), (2278, 8, 2, '2012-09-03 08:00:00', 1), (2279, 8, 7, '2012-09-03 08:30:00', 1), (2280, 8, 16, '2012-09-03 10:00:00', 1), (2281, 8, 1, '2012-09-03 10:30:00', 1), (2282, 8, 0, '2012-09-03 11:30:00', 1), (2283, 8, 3, '2012-09-03 13:00:00', 1), (2284, 8, 21, '2012-09-03 14:00:00', 1), (2285, 8, 3, '2012-09-03 15:00:00', 1), (2286, 8, 21, '2012-09-03 15:30:00', 1), (2287, 8, 21, '2012-09-03 17:00:00', 1), (2288, 8, 16, '2012-09-03 17:30:00', 1), (2289, 8, 20, '2012-09-03 18:30:00', 1), (2290, 8, 21, '2012-09-03 20:00:00', 1), (2291, 0, 11, '2012-09-04 08:30:00', 3), (2292, 0, 0, '2012-09-04 10:00:00', 3), (2293, 0, 10, '2012-09-04 11:30:00', 3), (2294, 0, 0, '2012-09-04 13:30:00', 3), (2295, 0, 5, '2012-09-04 15:00:00', 3), (2296, 0, 0, '2012-09-04 16:30:00', 3), (2297, 1, 0, '2012-09-04 10:00:00', 3), (2298, 1, 8, '2012-09-04 12:00:00', 3), (2299, 1, 0, '2012-09-04 14:00:00', 3), (2300, 1, 0, '2012-09-04 16:00:00', 3), (2301, 1, 9, '2012-09-04 17:30:00', 3), (2302, 1, 24, '2012-09-04 19:00:00', 3), (2303, 2, 21, '2012-09-04 08:00:00', 3), (2304, 2, 14, '2012-09-04 09:30:00', 3), (2305, 2, 15, '2012-09-04 11:00:00', 3), (2306, 2, 0, '2012-09-04 12:30:00', 3), (2307, 2, 0, '2012-09-04 15:00:00', 3), (2308, 2, 5, '2012-09-04 16:30:00', 3), (2309, 2, 2, '2012-09-04 18:00:00', 3), (2310, 3, 20, '2012-09-04 10:30:00', 2), (2311, 3, 21, '2012-09-04 11:30:00', 2), (2312, 3, 17, '2012-09-04 13:30:00', 2), (2313, 3, 21, '2012-09-04 15:00:00', 2), (2314, 3, 20, '2012-09-04 17:30:00', 2), (2315, 3, 22, '2012-09-04 18:30:00', 2), (2316, 4, 0, '2012-09-04 08:00:00', 2), (2317, 4, 3, '2012-09-04 10:30:00', 2), (2318, 4, 0, '2012-09-04 11:30:00', 2), (2319, 4, 7, '2012-09-04 12:30:00', 2), (2320, 4, 0, '2012-09-04 13:30:00', 2), (2321, 4, 3, '2012-09-04 15:00:00', 2), (2322, 4, 0, '2012-09-04 16:00:00', 2), (2323, 4, 0, '2012-09-04 17:30:00', 2), (2324, 4, 11, '2012-09-04 18:30:00', 2), (2325, 4, 8, '2012-09-04 19:30:00', 2), (2326, 5, 0, '2012-09-04 09:30:00', 2), (2327, 5, 0, '2012-09-04 12:30:00', 2), (2328, 6, 0, '2012-09-04 08:00:00', 4), (2329, 6, 0, '2012-09-04 11:00:00', 2), (2330, 6, 12, '2012-09-04 12:00:00', 2), (2331, 6, 0, '2012-09-04 13:30:00', 2), (2332, 6, 5, '2012-09-04 18:30:00', 2), (2333, 7, 22, '2012-09-04 08:00:00', 2), (2334, 7, 8, '2012-09-04 09:00:00', 2), (2335, 7, 7, '2012-09-04 10:00:00', 2), (2336, 7, 24, '2012-09-04 11:00:00', 2), (2337, 7, 5, '2012-09-04 13:00:00', 2), (2338, 7, 24, '2012-09-04 16:00:00', 2), (2339, 7, 0, '2012-09-04 17:30:00', 2), (2340, 7, 14, '2012-09-04 19:00:00', 2), (2341, 8, 3, '2012-09-04 08:00:00', 1), (2342, 8, 3, '2012-09-04 09:00:00', 1), (2343, 8, 20, '2012-09-04 09:30:00', 1), (2344, 8, 21, '2012-09-04 10:00:00', 3), (2345, 8, 0, '2012-09-04 13:00:00', 1), (2346, 8, 21, '2012-09-04 13:30:00', 1), (2347, 8, 3, '2012-09-04 14:00:00', 1), (2348, 8, 8, '2012-09-04 15:00:00', 2), (2349, 8, 21, '2012-09-04 16:00:00', 1), (2350, 8, 3, '2012-09-04 18:30:00', 1), (2351, 8, 21, '2012-09-04 19:30:00', 1), (2352, 8, 16, '2012-09-04 20:00:00', 1), (2353, 0, 22, '2012-09-05 08:00:00', 3), (2354, 0, 12, '2012-09-05 09:30:00', 3), (2355, 0, 0, '2012-09-05 11:00:00', 3), (2356, 0, 2, '2012-09-05 14:00:00', 3), (2357, 0, 6, '2012-09-05 15:30:00', 3), (2358, 0, 17, '2012-09-05 18:00:00', 3), (2359, 1, 1, '2012-09-05 08:00:00', 3), (2360, 1, 10, '2012-09-05 09:30:00', 3), (2361, 1, 24, '2012-09-05 12:00:00', 3), (2362, 1, 8, '2012-09-05 15:30:00', 3), (2363, 1, 12, '2012-09-05 18:00:00', 3), (2364, 2, 7, '2012-09-05 08:30:00', 3), (2365, 2, 13, '2012-09-05 11:30:00', 3), (2366, 2, 1, '2012-09-05 13:00:00', 3), (2367, 2, 24, '2012-09-05 16:30:00', 3), (2368, 2, 1, '2012-09-05 18:00:00', 3), (2369, 3, 16, '2012-09-05 08:30:00', 2), (2370, 3, 15, '2012-09-05 09:30:00', 2), (2371, 3, 2, '2012-09-05 12:00:00', 2), (2372, 3, 10, '2012-09-05 15:30:00', 2), (2373, 3, 10, '2012-09-05 19:30:00', 2), (2374, 4, 24, '2012-09-05 08:00:00', 2), (2375, 4, 0, '2012-09-05 09:00:00', 4), (2376, 4, 0, '2012-09-05 11:30:00', 2), (2377, 4, 16, '2012-09-05 12:30:00', 2), (2378, 4, 0, '2012-09-05 13:30:00', 6), (2379, 4, 11, '2012-09-05 17:00:00', 2), (2380, 4, 0, '2012-09-05 18:00:00', 2), (2381, 4, 9, '2012-09-05 19:00:00', 2), (2382, 5, 0, '2012-09-05 09:00:00', 2), (2383, 5, 0, '2012-09-05 11:00:00', 2), (2384, 5, 0, '2012-09-05 12:30:00', 2), (2385, 6, 0, '2012-09-05 08:30:00', 4), (2386, 6, 0, '2012-09-05 11:00:00', 2), (2387, 6, 11, '2012-09-05 13:00:00', 2), (2388, 6, 0, '2012-09-05 14:00:00', 2), (2389, 6, 0, '2012-09-05 15:30:00', 6), (2390, 7, 8, '2012-09-05 08:00:00', 2), (2391, 7, 4, '2012-09-05 10:00:00', 2), (2392, 7, 15, '2012-09-05 11:00:00', 2), (2393, 7, 7, '2012-09-05 13:00:00', 2), (2394, 7, 4, '2012-09-05 15:00:00', 2), (2395, 7, 9, '2012-09-05 16:30:00', 2), (2396, 7, 5, '2012-09-05 18:30:00', 2), (2397, 8, 20, '2012-09-05 09:00:00', 1), (2398, 8, 14, '2012-09-05 10:30:00', 1), (2399, 8, 3, '2012-09-05 11:00:00', 2), (2400, 8, 20, '2012-09-05 13:00:00', 1), (2401, 8, 2, '2012-09-05 13:30:00', 1), (2402, 8, 21, '2012-09-05 14:00:00', 1), (2403, 8, 0, '2012-09-05 14:30:00', 1), (2404, 8, 9, '2012-09-05 15:00:00', 1), (2405, 8, 2, '2012-09-05 15:30:00', 1), (2406, 8, 16, '2012-09-05 16:00:00', 1), (2407, 8, 6, '2012-09-05 17:00:00', 1), (2408, 8, 8, '2012-09-05 17:30:00', 1), (2409, 8, 2, '2012-09-05 19:00:00', 1), (2410, 8, 1, '2012-09-05 20:00:00', 1), (2411, 0, 17, '2012-09-06 08:30:00', 3), (2412, 0, 11, '2012-09-06 10:30:00', 3), (2413, 0, 22, '2012-09-06 12:00:00', 3), (2414, 0, 11, '2012-09-06 16:30:00', 3), (2415, 0, 4, '2012-09-06 19:00:00', 3), (2416, 1, 0, '2012-09-06 08:30:00', 3), (2417, 1, 8, '2012-09-06 10:00:00', 3), (2418, 1, 0, '2012-09-06 11:30:00', 3), (2419, 1, 9, '2012-09-06 13:00:00', 3), (2420, 1, 12, '2012-09-06 16:30:00', 6), (2421, 2, 9, '2012-09-06 08:00:00', 3), (2422, 2, 15, '2012-09-06 09:30:00', 3), (2423, 2, 21, '2012-09-06 12:00:00', 3), (2424, 2, 12, '2012-09-06 13:30:00', 3), (2425, 2, 17, '2012-09-06 15:00:00', 3), (2426, 2, 1, '2012-09-06 17:30:00', 3), (2427, 2, 21, '2012-09-06 19:00:00', 3), (2428, 3, 13, '2012-09-06 08:30:00', 2), (2429, 3, 15, '2012-09-06 11:00:00', 2), (2430, 3, 17, '2012-09-06 13:00:00', 2), (2431, 3, 13, '2012-09-06 14:30:00', 2), (2432, 3, 20, '2012-09-06 15:30:00', 2), (2433, 3, 15, '2012-09-06 17:30:00', 2), (2434, 4, 0, '2012-09-06 08:00:00', 2), (2435, 4, 2, '2012-09-06 09:30:00', 2), (2436, 4, 24, '2012-09-06 10:30:00', 2), (2437, 4, 13, '2012-09-06 12:00:00', 2), (2438, 4, 0, '2012-09-06 13:00:00', 2), (2439, 4, 6, '2012-09-06 14:00:00', 2), (2440, 4, 0, '2012-09-06 15:00:00', 4), (2441, 4, 7, '2012-09-06 17:30:00', 2), (2442, 4, 16, '2012-09-06 18:30:00', 2), (2443, 4, 8, '2012-09-06 19:30:00', 2), (2444, 5, 0, '2012-09-06 11:00:00', 2), (2445, 6, 0, '2012-09-06 09:00:00', 4), (2446, 6, 0, '2012-09-06 11:30:00', 2), (2447, 6, 14, '2012-09-06 12:30:00', 2), (2448, 6, 0, '2012-09-06 13:30:00', 2), (2449, 6, 0, '2012-09-06 15:30:00', 2), (2450, 6, 0, '2012-09-06 17:00:00', 2), (2451, 6, 0, '2012-09-06 18:30:00', 2), (2452, 6, 13, '2012-09-06 19:30:00', 2), (2453, 7, 0, '2012-09-06 09:00:00', 2), (2454, 7, 15, '2012-09-06 12:30:00', 2), (2455, 7, 24, '2012-09-06 16:30:00', 2), (2456, 7, 10, '2012-09-06 17:30:00', 2), (2457, 7, 0, '2012-09-06 18:30:00', 2), (2458, 8, 21, '2012-09-06 09:00:00', 1), (2459, 8, 24, '2012-09-06 09:30:00', 1), (2460, 8, 16, '2012-09-06 10:00:00', 1), (2461, 8, 7, '2012-09-06 11:00:00', 1), (2462, 8, 9, '2012-09-06 11:30:00', 1), (2463, 8, 3, '2012-09-06 12:00:00', 1), (2464, 8, 0, '2012-09-06 13:30:00', 1), (2465, 8, 20, '2012-09-06 14:00:00', 1), (2466, 8, 24, '2012-09-06 15:00:00', 1), (2467, 8, 3, '2012-09-06 16:30:00', 1), (2468, 8, 22, '2012-09-06 17:00:00', 1), (2469, 8, 16, '2012-09-06 18:00:00', 1), (2470, 8, 2, '2012-09-06 19:00:00', 1), (2471, 8, 3, '2012-09-06 19:30:00', 2), (2472, 0, 0, '2012-09-07 08:00:00', 3), (2473, 0, 14, '2012-09-07 09:30:00', 6), (2474, 0, 0, '2012-09-07 12:30:00', 3), (2475, 0, 11, '2012-09-07 14:00:00', 3), (2476, 0, 17, '2012-09-07 16:00:00', 3), (2477, 0, 14, '2012-09-07 18:00:00', 3), (2478, 1, 9, '2012-09-07 08:00:00', 3), (2479, 1, 12, '2012-09-07 11:00:00', 3), (2480, 1, 11, '2012-09-07 12:30:00', 3), (2481, 1, 24, '2012-09-07 14:30:00', 3), (2482, 1, 12, '2012-09-07 16:30:00', 3), (2483, 1, 9, '2012-09-07 18:00:00', 3), (2484, 2, 1, '2012-09-07 08:00:00', 3), (2485, 2, 5, '2012-09-07 09:30:00', 3), (2486, 2, 1, '2012-09-07 11:00:00', 3), (2487, 2, 0, '2012-09-07 12:30:00', 3), (2488, 2, 1, '2012-09-07 14:00:00', 6), (2489, 2, 0, '2012-09-07 17:00:00', 3), (2490, 2, 1, '2012-09-07 19:00:00', 3), (2491, 3, 16, '2012-09-07 08:30:00', 2), (2492, 3, 15, '2012-09-07 11:00:00', 2), (2493, 3, 20, '2012-09-07 14:30:00', 2), (2494, 3, 7, '2012-09-07 17:00:00', 2), (2495, 3, 10, '2012-09-07 19:00:00', 2), (2496, 4, 3, '2012-09-07 08:30:00', 4), (2497, 4, 6, '2012-09-07 11:00:00', 2), (2498, 4, 20, '2012-09-07 12:00:00', 2), (2499, 4, 5, '2012-09-07 13:00:00', 2), (2500, 4, 16, '2012-09-07 14:30:00', 2), (2501, 4, 0, '2012-09-07 16:00:00', 2), (2502, 4, 10, '2012-09-07 18:00:00', 2), (2503, 4, 13, '2012-09-07 19:00:00', 2), (2504, 5, 24, '2012-09-07 11:30:00', 2), (2505, 5, 3, '2012-09-07 14:30:00', 2), (2506, 6, 0, '2012-09-07 09:30:00', 8), (2507, 6, 0, '2012-09-07 14:00:00', 2), (2508, 6, 6, '2012-09-07 16:30:00', 2), (2509, 7, 8, '2012-09-07 09:00:00', 2), (2510, 7, 9, '2012-09-07 11:30:00', 2), (2511, 7, 4, '2012-09-07 13:30:00', 2), (2512, 7, 15, '2012-09-07 15:00:00', 2), (2513, 7, 5, '2012-09-07 17:00:00', 2), (2514, 7, 5, '2012-09-07 19:00:00', 2), (2515, 8, 24, '2012-09-07 08:30:00', 1), (2516, 8, 3, '2012-09-07 10:30:00', 1), (2517, 8, 21, '2012-09-07 11:00:00', 1), (2518, 8, 3, '2012-09-07 11:30:00', 1), (2519, 8, 17, '2012-09-07 12:00:00', 1), (2520, 8, 3, '2012-09-07 13:00:00', 1), (2521, 8, 20, '2012-09-07 13:30:00', 1), (2522, 8, 16, '2012-09-07 14:00:00', 1), (2523, 8, 21, '2012-09-07 14:30:00', 2), (2524, 8, 4, '2012-09-07 15:30:00', 1), (2525, 8, 21, '2012-09-07 16:30:00', 1), (2526, 8, 2, '2012-09-07 18:00:00', 1), (2527, 8, 7, '2012-09-07 18:30:00', 1), (2528, 8, 16, '2012-09-07 20:00:00', 1), (2529, 0, 5, '2012-09-08 08:00:00', 3), (2530, 0, 0, '2012-09-08 09:30:00', 3), (2531, 0, 7, '2012-09-08 11:00:00', 3), (2532, 0, 0, '2012-09-08 12:30:00', 3), (2533, 0, 11, '2012-09-08 15:00:00', 3), (2534, 0, 17, '2012-09-08 16:30:00', 3), (2535, 0, 16, '2012-09-08 18:30:00', 3), (2536, 1, 10, '2012-09-08 08:00:00', 3), (2537, 1, 24, '2012-09-08 09:30:00', 3), (2538, 1, 9, '2012-09-08 11:30:00', 3), (2539, 1, 0, '2012-09-08 13:00:00', 3), (2540, 1, 9, '2012-09-08 15:00:00', 6), (2541, 2, 8, '2012-09-08 08:30:00', 3), (2542, 2, 21, '2012-09-08 10:00:00', 3), (2543, 2, 26, '2012-09-08 13:00:00', 3), (2544, 2, 1, '2012-09-08 15:00:00', 3), (2545, 2, 21, '2012-09-08 16:30:00', 3), (2546, 2, 1, '2012-09-08 18:30:00', 3), (2547, 3, 6, '2012-09-08 08:00:00', 2), (2548, 3, 2, '2012-09-08 09:00:00', 2), (2549, 3, 15, '2012-09-08 10:00:00', 2), (2550, 3, 11, '2012-09-08 12:00:00', 2), (2551, 3, 20, '2012-09-08 13:00:00', 2), (2552, 3, 20, '2012-09-08 16:00:00', 2), (2553, 3, 1, '2012-09-08 17:30:00', 2), (2554, 3, 9, '2012-09-08 18:30:00', 2), (2555, 3, 15, '2012-09-08 19:30:00', 2), (2556, 4, 20, '2012-09-08 08:00:00', 2), (2557, 4, 0, '2012-09-08 09:30:00', 8), (2558, 4, 3, '2012-09-08 13:30:00', 2), (2559, 4, 0, '2012-09-08 14:30:00', 4), (2560, 4, 13, '2012-09-08 16:30:00', 2), (2561, 4, 13, '2012-09-08 18:00:00', 2), (2562, 4, 0, '2012-09-08 19:00:00', 2), (2563, 5, 24, '2012-09-08 15:30:00', 2), (2564, 6, 0, '2012-09-08 09:00:00', 2), (2565, 6, 6, '2012-09-08 13:30:00', 2), (2566, 6, 0, '2012-09-08 16:00:00', 2), (2567, 6, 4, '2012-09-08 17:00:00', 2), (2568, 6, 0, '2012-09-08 18:00:00', 4), (2569, 7, 6, '2012-09-08 09:30:00', 2), (2570, 7, 4, '2012-09-08 11:30:00', 2), (2571, 7, 13, '2012-09-08 12:30:00', 2), (2572, 7, 0, '2012-09-08 13:30:00', 2), (2573, 7, 15, '2012-09-08 14:30:00', 2), (2574, 7, 8, '2012-09-08 15:30:00', 2), (2575, 7, 1, '2012-09-08 16:30:00', 2), (2576, 7, 15, '2012-09-08 18:00:00', 2), (2577, 8, 21, '2012-09-08 08:00:00', 1), (2578, 8, 3, '2012-09-08 08:30:00', 1), (2579, 8, 22, '2012-09-08 09:00:00', 1), (2580, 8, 0, '2012-09-08 09:30:00', 1), (2581, 8, 16, '2012-09-08 10:00:00', 1), (2582, 8, 3, '2012-09-08 10:30:00', 2), (2583, 8, 0, '2012-09-08 11:30:00', 1), (2584, 8, 6, '2012-09-08 13:00:00', 1), (2585, 8, 22, '2012-09-08 15:30:00', 1), (2586, 8, 16, '2012-09-08 16:30:00', 1), (2587, 8, 7, '2012-09-08 17:00:00', 1), (2588, 8, 3, '2012-09-08 17:30:00', 1), (2589, 8, 8, '2012-09-08 19:30:00', 1), (2590, 0, 5, '2012-09-09 08:00:00', 3), (2591, 0, 16, '2012-09-09 09:30:00', 3), (2592, 0, 26, '2012-09-09 12:00:00', 3), (2593, 0, 7, '2012-09-09 15:00:00', 3), (2594, 0, 0, '2012-09-09 17:00:00', 3), (2595, 0, 24, '2012-09-09 18:30:00', 3), (2596, 1, 8, '2012-09-09 08:00:00', 3), (2597, 1, 0, '2012-09-09 10:00:00', 3), (2598, 1, 16, '2012-09-09 13:00:00', 3), (2599, 1, 10, '2012-09-09 14:30:00', 3), (2600, 1, 15, '2012-09-09 16:00:00', 3), (2601, 1, 0, '2012-09-09 17:30:00', 6), (2602, 2, 21, '2012-09-09 08:30:00', 3), (2603, 2, 1, '2012-09-09 11:00:00', 3), (2604, 2, 1, '2012-09-09 13:00:00', 6), (2605, 2, 5, '2012-09-09 16:30:00', 3), (2606, 2, 14, '2012-09-09 18:30:00', 3), (2607, 3, 22, '2012-09-09 09:00:00', 2), (2608, 3, 10, '2012-09-09 10:00:00', 2), (2609, 3, 20, '2012-09-09 13:00:00', 2), (2610, 3, 0, '2012-09-09 15:30:00', 2), (2611, 3, 21, '2012-09-09 16:30:00', 2), (2612, 3, 0, '2012-09-09 18:00:00', 2), (2613, 3, 6, '2012-09-09 19:00:00', 2), (2614, 4, 13, '2012-09-09 08:00:00', 2), (2615, 4, 7, '2012-09-09 09:00:00', 2), (2616, 4, 20, '2012-09-09 10:00:00', 2), (2617, 4, 0, '2012-09-09 11:00:00', 4), (2618, 4, 3, '2012-09-09 13:30:00', 2), (2619, 4, 11, '2012-09-09 14:30:00', 2), (2620, 4, 20, '2012-09-09 15:30:00', 2), (2621, 4, 0, '2012-09-09 17:00:00', 2), (2622, 4, 11, '2012-09-09 18:00:00', 2), (2623, 4, 13, '2012-09-09 19:00:00', 2), (2624, 5, 0, '2012-09-09 14:00:00', 2), (2625, 6, 0, '2012-09-09 08:30:00', 2), (2626, 6, 0, '2012-09-09 11:00:00', 6), (2627, 6, 12, '2012-09-09 14:00:00', 2), (2628, 6, 14, '2012-09-09 15:30:00', 2), (2629, 6, 0, '2012-09-09 16:30:00', 4), (2630, 6, 26, '2012-09-09 18:30:00', 2), (2631, 6, 21, '2012-09-09 19:30:00', 2), (2632, 7, 22, '2012-09-09 08:00:00', 2), (2633, 7, 22, '2012-09-09 10:30:00', 2), (2634, 7, 21, '2012-09-09 14:00:00', 2), (2635, 7, 4, '2012-09-09 17:00:00', 2), (2636, 7, 7, '2012-09-09 18:00:00', 2), (2637, 7, 4, '2012-09-09 19:30:00', 2), (2638, 8, 16, '2012-09-09 08:00:00', 1), (2639, 8, 0, '2012-09-09 08:30:00', 1), (2640, 8, 16, '2012-09-09 09:00:00', 1), (2641, 8, 3, '2012-09-09 09:30:00', 1), (2642, 8, 2, '2012-09-09 10:00:00', 1), (2643, 8, 21, '2012-09-09 10:30:00', 1), (2644, 8, 5, '2012-09-09 11:00:00', 1), (2645, 8, 15, '2012-09-09 11:30:00', 1), (2646, 8, 3, '2012-09-09 12:00:00', 2), (2647, 8, 0, '2012-09-09 13:00:00', 1), (2648, 8, 0, '2012-09-09 14:30:00', 1), (2649, 8, 16, '2012-09-09 16:30:00', 1), (2650, 8, 9, '2012-09-09 17:00:00', 1), (2651, 8, 17, '2012-09-09 17:30:00', 1), (2652, 8, 6, '2012-09-09 18:00:00', 1), (2653, 8, 3, '2012-09-09 18:30:00', 1), (2654, 8, 16, '2012-09-09 19:00:00', 1), (2655, 8, 3, '2012-09-09 19:30:00', 1), (2656, 8, 16, '2012-09-09 20:00:00', 1), (2657, 0, 22, '2012-09-10 10:30:00', 3), (2658, 0, 14, '2012-09-10 12:00:00', 3), (2659, 0, 0, '2012-09-10 13:30:00', 3), (2660, 0, 14, '2012-09-10 15:30:00', 3), (2661, 0, 10, '2012-09-10 18:30:00', 3), (2662, 1, 24, '2012-09-10 08:00:00', 3), (2663, 1, 0, '2012-09-10 09:30:00', 3), (2664, 1, 0, '2012-09-10 13:00:00', 3), (2665, 1, 15, '2012-09-10 14:30:00', 3), (2666, 1, 0, '2012-09-10 16:00:00', 3), (2667, 1, 12, '2012-09-10 17:30:00', 3), (2668, 1, 0, '2012-09-10 19:00:00', 3), (2669, 2, 1, '2012-09-10 09:00:00', 6), (2670, 2, 21, '2012-09-10 12:00:00', 3), (2671, 2, 21, '2012-09-10 14:00:00', 3), (2672, 2, 0, '2012-09-10 15:30:00', 3), (2673, 2, 6, '2012-09-10 19:00:00', 3), (2674, 3, 6, '2012-09-10 08:30:00', 2), (2675, 3, 15, '2012-09-10 09:30:00', 2), (2676, 3, 15, '2012-09-10 11:00:00', 2), (2677, 3, 15, '2012-09-10 13:00:00', 2), (2678, 3, 16, '2012-09-10 15:00:00', 2), (2679, 3, 2, '2012-09-10 16:30:00', 2), (2680, 3, 16, '2012-09-10 17:30:00', 2), (2681, 3, 17, '2012-09-10 18:30:00', 2), (2682, 3, 15, '2012-09-10 19:30:00', 2), (2683, 4, 4, '2012-09-10 08:00:00', 2), (2684, 4, 13, '2012-09-10 09:00:00', 4), (2685, 4, 20, '2012-09-10 11:30:00', 2), (2686, 4, 11, '2012-09-10 12:30:00', 2), (2687, 4, 1, '2012-09-10 13:30:00', 2), (2688, 4, 10, '2012-09-10 14:30:00', 2), (2689, 4, 12, '2012-09-10 15:30:00', 2), (2690, 4, 17, '2012-09-10 17:00:00', 2), (2691, 4, 14, '2012-09-10 18:00:00', 2), (2692, 4, 0, '2012-09-10 19:00:00', 2), (2693, 5, 0, '2012-09-10 10:00:00', 2), (2694, 5, 0, '2012-09-10 11:30:00', 2), (2695, 6, 0, '2012-09-10 08:30:00', 2), (2696, 6, 11, '2012-09-10 09:30:00', 2), (2697, 6, 8, '2012-09-10 11:00:00', 2), (2698, 6, 12, '2012-09-10 12:30:00', 2), (2699, 6, 0, '2012-09-10 14:00:00', 6), (2700, 6, 0, '2012-09-10 17:30:00', 2), (2701, 6, 12, '2012-09-10 19:00:00', 2), (2702, 7, 22, '2012-09-10 09:30:00', 2), (2703, 7, 4, '2012-09-10 11:30:00', 2), (2704, 7, 24, '2012-09-10 15:00:00', 2), (2705, 7, 10, '2012-09-10 16:00:00', 2), (2706, 7, 15, '2012-09-10 17:30:00', 2), (2707, 7, 4, '2012-09-10 18:30:00', 2), (2708, 7, 7, '2012-09-10 19:30:00', 2), (2709, 8, 15, '2012-09-10 08:30:00', 1), (2710, 8, 26, '2012-09-10 10:30:00', 1), (2711, 8, 5, '2012-09-10 12:00:00', 1), (2712, 8, 16, '2012-09-10 12:30:00', 1), (2713, 8, 2, '2012-09-10 13:00:00', 1), (2714, 8, 16, '2012-09-10 13:30:00', 1), (2715, 8, 3, '2012-09-10 15:00:00', 1), (2716, 8, 21, '2012-09-10 15:30:00', 1), (2717, 8, 24, '2012-09-10 16:00:00', 1), (2718, 8, 16, '2012-09-10 16:30:00', 1), (2719, 8, 21, '2012-09-10 17:30:00', 1), (2720, 8, 3, '2012-09-10 19:30:00', 1), (2721, 8, 21, '2012-09-10 20:00:00', 1), (2722, 0, 5, '2012-09-11 09:00:00', 3), (2723, 0, 6, '2012-09-11 10:30:00', 3), (2724, 0, 7, '2012-09-11 12:00:00', 3), (2725, 0, 17, '2012-09-11 14:30:00', 3), (2726, 0, 11, '2012-09-11 16:00:00', 3), (2727, 0, 26, '2012-09-11 19:00:00', 3), (2728, 1, 9, '2012-09-11 08:00:00', 3), (2729, 1, 11, '2012-09-11 09:30:00', 3), (2730, 1, 8, '2012-09-11 11:00:00', 3), (2731, 1, 12, '2012-09-11 12:30:00', 3), (2732, 1, 11, '2012-09-11 14:30:00', 3), (2733, 1, 9, '2012-09-11 16:00:00', 3), (2734, 1, 11, '2012-09-11 17:30:00', 6), (2735, 2, 2, '2012-09-11 11:00:00', 3), (2736, 2, 1, '2012-09-11 12:30:00', 3), (2737, 2, 8, '2012-09-11 14:00:00', 3), (2738, 2, 21, '2012-09-11 17:00:00', 6), (2739, 3, 22, '2012-09-11 08:00:00', 2), (2740, 3, 16, '2012-09-11 09:30:00', 2), (2741, 3, 21, '2012-09-11 11:00:00', 2), (2742, 3, 6, '2012-09-11 12:00:00', 2), (2743, 3, 15, '2012-09-11 13:30:00', 2), (2744, 3, 2, '2012-09-11 18:00:00', 2), (2745, 3, 6, '2012-09-11 19:30:00', 2), (2746, 4, 3, '2012-09-11 08:00:00', 2), (2747, 4, 6, '2012-09-11 09:00:00', 2), (2748, 4, 0, '2012-09-11 10:00:00', 4), (2749, 4, 13, '2012-09-11 12:30:00', 2), (2750, 4, 16, '2012-09-11 13:30:00', 2), (2751, 4, 3, '2012-09-11 14:30:00', 2), (2752, 4, 0, '2012-09-11 15:30:00', 2), (2753, 4, 8, '2012-09-11 16:30:00', 2), (2754, 4, 0, '2012-09-11 18:00:00', 2), (2755, 4, 14, '2012-09-11 19:00:00', 2), (2756, 5, 0, '2012-09-11 11:30:00', 2), (2757, 5, 0, '2012-09-11 18:00:00', 2), (2758, 6, 12, '2012-09-11 08:00:00', 2), (2759, 6, 0, '2012-09-11 09:00:00', 2), (2760, 6, 12, '2012-09-11 10:30:00', 4), (2761, 6, 0, '2012-09-11 12:30:00', 4), (2762, 6, 16, '2012-09-11 14:30:00', 2), (2763, 6, 0, '2012-09-11 15:30:00', 4), (2764, 6, 12, '2012-09-11 17:30:00', 2), (2765, 6, 0, '2012-09-11 18:30:00', 2), (2766, 6, 12, '2012-09-11 19:30:00', 2), (2767, 7, 10, '2012-09-11 08:30:00', 2), (2768, 7, 13, '2012-09-11 09:30:00', 2), (2769, 7, 7, '2012-09-11 11:00:00', 2), (2770, 7, 6, '2012-09-11 13:30:00', 2), (2771, 7, 4, '2012-09-11 14:30:00', 2), (2772, 7, 24, '2012-09-11 16:30:00', 2), (2773, 7, 10, '2012-09-11 18:00:00', 2), (2774, 7, 15, '2012-09-11 19:00:00', 2), (2775, 8, 24, '2012-09-11 08:30:00', 1), (2776, 8, 3, '2012-09-11 09:00:00', 1), (2777, 8, 21, '2012-09-11 09:30:00', 1), (2778, 8, 0, '2012-09-11 10:30:00', 1), (2779, 8, 16, '2012-09-11 11:00:00', 1), (2780, 8, 3, '2012-09-11 12:00:00', 2), (2781, 8, 21, '2012-09-11 13:00:00', 1), (2782, 8, 21, '2012-09-11 14:00:00', 2), (2783, 8, 22, '2012-09-11 15:00:00', 1), (2784, 8, 8, '2012-09-11 15:30:00', 1), (2785, 8, 3, '2012-09-11 17:00:00', 2), (2786, 8, 3, '2012-09-11 18:30:00', 2), (2787, 0, 22, '2012-09-12 08:30:00', 3), (2788, 0, 0, '2012-09-12 10:00:00', 3), (2789, 0, 4, '2012-09-12 11:30:00', 3), (2790, 0, 26, '2012-09-12 13:00:00', 3), (2791, 0, 5, '2012-09-12 15:00:00', 3), (2792, 0, 0, '2012-09-12 16:30:00', 3), (2793, 0, 16, '2012-09-12 18:00:00', 3), (2794, 1, 11, '2012-09-12 08:30:00', 3), (2795, 1, 0, '2012-09-12 10:00:00', 3), (2796, 1, 14, '2012-09-12 12:00:00', 3), (2797, 1, 11, '2012-09-12 13:30:00', 3), (2798, 1, 0, '2012-09-12 15:00:00', 6), (2799, 1, 10, '2012-09-12 18:30:00', 3), (2800, 2, 24, '2012-09-12 08:00:00', 3), (2801, 2, 12, '2012-09-12 09:30:00', 3), (2802, 2, 9, '2012-09-12 11:00:00', 3), (2803, 2, 13, '2012-09-12 14:30:00', 3), (2804, 2, 9, '2012-09-12 16:00:00', 3), (2805, 2, 2, '2012-09-12 17:30:00', 6), (2806, 3, 15, '2012-09-12 09:00:00', 2), (2807, 3, 20, '2012-09-12 12:30:00', 2), (2808, 3, 10, '2012-09-12 13:30:00', 2), (2809, 3, 3, '2012-09-12 14:30:00', 2), (2810, 3, 16, '2012-09-12 15:30:00', 2), (2811, 3, 0, '2012-09-12 19:00:00', 2), (2812, 4, 16, '2012-09-12 08:00:00', 2), (2813, 4, 0, '2012-09-12 09:00:00', 2), (2814, 4, 0, '2012-09-12 10:30:00', 2), (2815, 4, 13, '2012-09-12 11:30:00', 2), (2816, 4, 0, '2012-09-12 12:30:00', 4), (2817, 4, 16, '2012-09-12 14:30:00', 2), (2818, 4, 0, '2012-09-12 15:30:00', 2), (2819, 4, 3, '2012-09-12 16:30:00', 2), (2820, 4, 1, '2012-09-12 17:30:00', 2), (2821, 4, 7, '2012-09-12 19:00:00', 2), (2822, 5, 0, '2012-09-12 16:30:00', 2), (2823, 6, 0, '2012-09-12 08:30:00', 4), (2824, 6, 0, '2012-09-12 11:00:00', 6), (2825, 6, 24, '2012-09-12 14:00:00', 2), (2826, 6, 0, '2012-09-12 15:00:00', 4), (2827, 6, 0, '2012-09-12 17:30:00', 4), (2828, 7, 5, '2012-09-12 08:30:00', 2), (2829, 7, 4, '2012-09-12 09:30:00', 2), (2830, 7, 15, '2012-09-12 10:30:00', 2), (2831, 7, 24, '2012-09-12 13:00:00', 2), (2832, 7, 7, '2012-09-12 15:30:00', 2), (2833, 7, 22, '2012-09-12 17:30:00', 2), (2834, 8, 1, '2012-09-12 10:00:00', 1), (2835, 8, 16, '2012-09-12 11:00:00', 1), (2836, 8, 3, '2012-09-12 11:30:00', 1), (2837, 8, 16, '2012-09-12 12:00:00', 1), (2838, 8, 21, '2012-09-12 12:30:00', 2), (2839, 8, 1, '2012-09-12 13:30:00', 1), (2840, 8, 5, '2012-09-12 14:00:00', 1), (2841, 8, 2, '2012-09-12 14:30:00', 1), (2842, 8, 22, '2012-09-12 15:30:00', 1), (2843, 8, 3, '2012-09-12 16:00:00', 1), (2844, 8, 4, '2012-09-12 18:00:00', 1), (2845, 8, 21, '2012-09-12 18:30:00', 2), (2846, 0, 5, '2012-09-13 09:00:00', 3), (2847, 0, 0, '2012-09-13 10:30:00', 6), (2848, 0, 7, '2012-09-13 13:30:00', 3), (2849, 0, 10, '2012-09-13 16:00:00', 3), (2850, 0, 0, '2012-09-13 17:30:00', 6), (2851, 1, 8, '2012-09-13 08:30:00', 3), (2852, 1, 11, '2012-09-13 10:30:00', 3), (2853, 1, 0, '2012-09-13 12:00:00', 6), (2854, 1, 12, '2012-09-13 15:00:00', 3), (2855, 1, 8, '2012-09-13 16:30:00', 3), (2856, 1, 24, '2012-09-13 18:30:00', 3), (2857, 2, 11, '2012-09-13 08:00:00', 3), (2858, 2, 1, '2012-09-13 09:30:00', 3), (2859, 2, 2, '2012-09-13 11:00:00', 3), (2860, 2, 10, '2012-09-13 13:00:00', 3), (2861, 2, 15, '2012-09-13 14:30:00', 3), (2862, 2, 21, '2012-09-13 16:30:00', 3), (2863, 2, 11, '2012-09-13 18:00:00', 3), (2864, 3, 16, '2012-09-13 08:00:00', 2), (2865, 3, 3, '2012-09-13 09:00:00', 2), (2866, 3, 17, '2012-09-13 10:00:00', 2), (2867, 3, 22, '2012-09-13 11:30:00', 2), (2868, 3, 24, '2012-09-13 13:00:00', 2), (2869, 3, 3, '2012-09-13 14:00:00', 2), (2870, 3, 11, '2012-09-13 16:00:00', 2), (2871, 3, 3, '2012-09-13 17:30:00', 2), (2872, 3, 17, '2012-09-13 18:30:00', 2), (2873, 3, 4, '2012-09-13 19:30:00', 2), (2874, 4, 7, '2012-09-13 08:00:00', 2), (2875, 4, 20, '2012-09-13 09:00:00', 2), (2876, 4, 6, '2012-09-13 10:30:00', 2), (2877, 4, 5, '2012-09-13 11:30:00', 2), (2878, 4, 21, '2012-09-13 12:30:00', 2), (2879, 4, 20, '2012-09-13 14:00:00', 2), (2880, 4, 9, '2012-09-13 15:30:00', 2), (2881, 4, 20, '2012-09-13 17:00:00', 2), (2882, 4, 0, '2012-09-13 18:00:00', 2), (2883, 4, 5, '2012-09-13 19:00:00', 2), (2884, 5, 0, '2012-09-13 08:30:00', 2), (2885, 5, 0, '2012-09-13 16:00:00', 2), (2886, 5, 0, '2012-09-13 19:00:00', 2), (2887, 6, 12, '2012-09-13 09:00:00', 2), (2888, 6, 0, '2012-09-13 10:30:00', 14), (2889, 6, 0, '2012-09-13 18:00:00', 2), (2890, 6, 6, '2012-09-13 19:30:00', 2), (2891, 7, 14, '2012-09-13 08:00:00', 2), (2892, 7, 4, '2012-09-13 09:30:00', 2), (2893, 7, 17, '2012-09-13 12:30:00', 2), (2894, 7, 5, '2012-09-13 13:30:00', 2), (2895, 7, 4, '2012-09-13 14:30:00', 2), (2896, 7, 15, '2012-09-13 17:00:00', 2), (2897, 7, 0, '2012-09-13 18:00:00', 2), (2898, 7, 9, '2012-09-13 19:00:00', 2), (2899, 8, 20, '2012-09-13 08:00:00', 1), (2900, 8, 15, '2012-09-13 09:00:00', 1), (2901, 8, 21, '2012-09-13 09:30:00', 1), (2902, 8, 21, '2012-09-13 10:30:00', 1), (2903, 8, 16, '2012-09-13 11:00:00', 1), (2904, 8, 21, '2012-09-13 11:30:00', 1), (2905, 8, 0, '2012-09-13 12:00:00', 1), (2906, 8, 24, '2012-09-13 12:30:00', 1), (2907, 8, 3, '2012-09-13 13:30:00', 1), (2908, 8, 16, '2012-09-13 14:30:00', 1), (2909, 8, 21, '2012-09-13 15:00:00', 1), (2910, 8, 21, '2012-09-13 16:00:00', 1), (2911, 8, 16, '2012-09-13 18:00:00', 1), (2912, 8, 21, '2012-09-13 18:30:00', 1), (2913, 8, 0, '2012-09-13 19:00:00', 1), (2914, 8, 21, '2012-09-13 19:30:00', 1), (2915, 8, 15, '2012-09-13 20:00:00', 1), (2916, 0, 6, '2012-09-14 08:00:00', 3), (2917, 0, 17, '2012-09-14 10:00:00', 3), (2918, 0, 5, '2012-09-14 12:30:00', 3), (2919, 0, 3, '2012-09-14 14:00:00', 3), (2920, 0, 0, '2012-09-14 16:00:00', 3), (2921, 0, 26, '2012-09-14 17:30:00', 3), (2922, 0, 0, '2012-09-14 19:00:00', 3), (2923, 1, 11, '2012-09-14 08:00:00', 6), (2924, 1, 8, '2012-09-14 11:00:00', 6), (2925, 1, 0, '2012-09-14 14:00:00', 3), (2926, 1, 0, '2012-09-14 17:00:00', 6), (2927, 2, 1, '2012-09-14 08:00:00', 3), (2928, 2, 21, '2012-09-14 11:00:00', 3), (2929, 2, 1, '2012-09-14 13:00:00', 3), (2930, 2, 5, '2012-09-14 16:00:00', 3), (2931, 2, 9, '2012-09-14 18:00:00', 3), (2932, 3, 15, '2012-09-14 08:30:00', 2), (2933, 3, 16, '2012-09-14 11:00:00', 2), (2934, 3, 20, '2012-09-14 12:30:00', 2), (2935, 3, 21, '2012-09-14 18:30:00', 2), (2936, 4, 14, '2012-09-14 08:00:00', 2), (2937, 4, 0, '2012-09-14 09:00:00', 2), (2938, 4, 13, '2012-09-14 11:00:00', 2), (2939, 4, 9, '2012-09-14 12:00:00', 2), (2940, 4, 0, '2012-09-14 13:00:00', 2), (2941, 4, 13, '2012-09-14 14:00:00', 4), (2942, 4, 0, '2012-09-14 16:00:00', 2), (2943, 4, 6, '2012-09-14 18:00:00', 2), (2944, 4, 20, '2012-09-14 19:00:00', 2), (2945, 5, 15, '2012-09-14 09:30:00', 2), (2946, 5, 0, '2012-09-14 11:00:00', 4), (2947, 6, 12, '2012-09-14 08:30:00', 2), (2948, 6, 0, '2012-09-14 09:30:00', 4), (2949, 6, 0, '2012-09-14 12:30:00', 2), (2950, 6, 16, '2012-09-14 14:00:00', 2), (2951, 6, 0, '2012-09-14 15:00:00', 2), (2952, 6, 12, '2012-09-14 16:00:00', 2), (2953, 6, 17, '2012-09-14 17:30:00', 2), (2954, 7, 10, '2012-09-14 08:30:00', 2), (2955, 7, 24, '2012-09-14 12:00:00', 2), (2956, 7, 9, '2012-09-14 13:30:00', 2), (2957, 7, 21, '2012-09-14 16:30:00', 2), (2958, 7, 24, '2012-09-14 18:00:00', 2), (2959, 8, 3, '2012-09-14 08:00:00', 1), (2960, 8, 16, '2012-09-14 08:30:00', 1), (2961, 8, 2, '2012-09-14 09:00:00', 1), (2962, 8, 21, '2012-09-14 09:30:00', 1), (2963, 8, 3, '2012-09-14 10:00:00', 1), (2964, 8, 9, '2012-09-14 10:30:00', 1), (2965, 8, 3, '2012-09-14 11:00:00', 2), (2966, 8, 20, '2012-09-14 12:00:00', 1), (2967, 8, 21, '2012-09-14 13:00:00', 1), (2968, 8, 16, '2012-09-14 13:30:00', 1), (2969, 8, 24, '2012-09-14 14:00:00', 1), (2970, 8, 20, '2012-09-14 15:00:00', 1), (2971, 8, 22, '2012-09-14 15:30:00', 1), (2972, 8, 16, '2012-09-14 16:00:00', 1), (2973, 8, 3, '2012-09-14 16:30:00', 1), (2974, 8, 15, '2012-09-14 17:00:00', 1), (2975, 8, 16, '2012-09-14 17:30:00', 2), (2976, 8, 11, '2012-09-14 19:00:00', 1), (2977, 8, 2, '2012-09-14 19:30:00', 1), (2978, 0, 0, '2012-09-15 08:00:00', 12), (2979, 0, 11, '2012-09-15 14:00:00', 3), (2980, 0, 7, '2012-09-15 16:30:00', 3), (2981, 0, 17, '2012-09-15 18:00:00', 3), (2982, 1, 10, '2012-09-15 08:00:00', 3), (2983, 1, 11, '2012-09-15 10:00:00', 3), (2984, 1, 24, '2012-09-15 13:00:00', 6), (2985, 1, 12, '2012-09-15 16:00:00', 3), (2986, 2, 0, '2012-09-15 08:00:00', 3), (2987, 2, 1, '2012-09-15 10:30:00', 3), (2988, 2, 0, '2012-09-15 12:00:00', 3), (2989, 2, 14, '2012-09-15 13:30:00', 3), (2990, 2, 26, '2012-09-15 15:30:00', 3), (2991, 2, 0, '2012-09-15 17:30:00', 3), (2992, 3, 1, '2012-09-15 08:00:00', 2), (2993, 3, 14, '2012-09-15 09:30:00', 2), (2994, 3, 22, '2012-09-15 10:30:00', 2), (2995, 3, 21, '2012-09-15 11:30:00', 2), (2996, 3, 20, '2012-09-15 12:30:00', 2), (2997, 3, 3, '2012-09-15 14:30:00', 2), (2998, 3, 11, '2012-09-15 15:30:00', 2), (2999, 3, 0, '2012-09-15 17:30:00', 2), (3000, 3, 11, '2012-09-15 19:30:00', 2), (3001, 4, 13, '2012-09-15 08:00:00', 2), (3002, 4, 0, '2012-09-15 09:00:00', 2), (3003, 4, 17, '2012-09-15 10:00:00', 2), (3004, 4, 3, '2012-09-15 11:00:00', 2), (3005, 4, 0, '2012-09-15 12:00:00', 8), (3006, 4, 24, '2012-09-15 16:00:00', 2), (3007, 4, 16, '2012-09-15 17:00:00', 4), (3008, 4, 14, '2012-09-15 19:00:00', 2), (3009, 5, 0, '2012-09-15 12:30:00', 2), (3010, 6, 0, '2012-09-15 08:00:00', 2), (3011, 6, 0, '2012-09-15 09:30:00', 4), (3012, 6, 11, '2012-09-15 11:30:00', 2), (3013, 6, 22, '2012-09-15 12:30:00', 2), (3014, 6, 12, '2012-09-15 14:00:00', 2), (3015, 6, 1, '2012-09-15 15:00:00', 2), (3016, 6, 4, '2012-09-15 16:00:00', 2), (3017, 6, 15, '2012-09-15 17:30:00', 2), (3018, 6, 0, '2012-09-15 18:30:00', 4), (3019, 7, 17, '2012-09-15 08:30:00', 2), (3020, 7, 2, '2012-09-15 09:30:00', 2), (3021, 7, 8, '2012-09-15 10:30:00', 2), (3022, 7, 15, '2012-09-15 13:00:00', 2), (3023, 7, 22, '2012-09-15 14:00:00', 2), (3024, 7, 13, '2012-09-15 15:00:00', 2), (3025, 7, 10, '2012-09-15 16:00:00', 2), (3026, 7, 13, '2012-09-15 19:30:00', 2), (3027, 8, 21, '2012-09-15 08:00:00', 1), (3028, 8, 16, '2012-09-15 08:30:00', 1), (3029, 8, 15, '2012-09-15 09:00:00', 1), (3030, 8, 16, '2012-09-15 09:30:00', 1), (3031, 8, 15, '2012-09-15 10:30:00', 1), (3032, 8, 16, '2012-09-15 11:00:00', 2), (3033, 8, 3, '2012-09-15 12:00:00', 1), (3034, 8, 21, '2012-09-15 12:30:00', 2), (3035, 8, 6, '2012-09-15 13:30:00', 1), (3036, 8, 15, '2012-09-15 15:00:00', 1), (3037, 8, 6, '2012-09-15 15:30:00', 1), (3038, 8, 21, '2012-09-15 16:30:00', 1), (3039, 8, 21, '2012-09-15 19:00:00', 1), (3040, 8, 3, '2012-09-15 19:30:00', 1), (3041, 0, 0, '2012-09-16 08:00:00', 9), (3042, 0, 11, '2012-09-16 12:30:00', 3), (3043, 0, 6, '2012-09-16 14:00:00', 3), (3044, 0, 0, '2012-09-16 15:30:00', 3), (3045, 0, 24, '2012-09-16 17:00:00', 3), (3046, 0, 10, '2012-09-16 18:30:00', 3), (3047, 1, 8, '2012-09-16 08:00:00', 3), (3048, 1, 0, '2012-09-16 09:30:00', 6), (3049, 1, 16, '2012-09-16 12:30:00', 3), (3050, 1, 8, '2012-09-16 14:00:00', 3), (3051, 1, 12, '2012-09-16 15:30:00', 3), (3052, 1, 0, '2012-09-16 17:30:00', 6), (3053, 2, 2, '2012-09-16 08:30:00', 3), (3054, 2, 1, '2012-09-16 10:30:00', 3), (3055, 2, 12, '2012-09-16 12:00:00', 3), (3056, 2, 21, '2012-09-16 13:30:00', 3), (3057, 2, 7, '2012-09-16 15:30:00', 3), (3058, 2, 21, '2012-09-16 17:00:00', 3), (3059, 2, 21, '2012-09-16 19:00:00', 3), (3060, 3, 1, '2012-09-16 09:00:00', 2), (3061, 3, 14, '2012-09-16 10:00:00', 2), (3062, 3, 0, '2012-09-16 13:00:00', 2), (3063, 3, 22, '2012-09-16 16:30:00', 2), (3064, 3, 16, '2012-09-16 17:30:00', 2), (3065, 3, 15, '2012-09-16 18:30:00', 2), (3066, 4, 1, '2012-09-16 08:00:00', 2), (3067, 4, 0, '2012-09-16 09:00:00', 2), (3068, 4, 8, '2012-09-16 10:00:00', 4), (3069, 4, 13, '2012-09-16 12:00:00', 4), (3070, 4, 3, '2012-09-16 14:00:00', 4), (3071, 4, 0, '2012-09-16 16:00:00', 2), (3072, 4, 0, '2012-09-16 17:30:00', 4), (3073, 4, 14, '2012-09-16 19:30:00', 2), (3074, 5, 22, '2012-09-16 08:30:00', 2), (3075, 6, 11, '2012-09-16 08:00:00', 2), (3076, 6, 0, '2012-09-16 09:00:00', 2), (3077, 6, 12, '2012-09-16 10:30:00', 2), (3078, 6, 2, '2012-09-16 12:00:00', 2), (3079, 6, 10, '2012-09-16 13:30:00', 2), (3080, 6, 0, '2012-09-16 14:30:00', 4), (3081, 6, 0, '2012-09-16 17:30:00', 6), (3082, 7, 10, '2012-09-16 08:30:00', 2), (3083, 7, 10, '2012-09-16 10:30:00', 2), (3084, 7, 9, '2012-09-16 11:30:00', 2), (3085, 7, 15, '2012-09-16 12:30:00', 2), (3086, 7, 13, '2012-09-16 14:00:00', 2), (3087, 7, 8, '2012-09-16 15:30:00', 2), (3088, 7, 27, '2012-09-16 16:30:00', 2), (3089, 7, 27, '2012-09-16 19:00:00', 2), (3090, 8, 21, '2012-09-16 09:30:00', 1), (3091, 8, 3, '2012-09-16 10:30:00', 2), (3092, 8, 21, '2012-09-16 12:00:00', 1), (3093, 8, 27, '2012-09-16 13:30:00', 1), (3094, 8, 16, '2012-09-16 14:30:00', 1), (3095, 8, 21, '2012-09-16 15:00:00', 1), (3096, 8, 27, '2012-09-16 15:30:00', 1), (3097, 8, 16, '2012-09-16 16:30:00', 1), (3098, 8, 3, '2012-09-16 17:00:00', 1), (3099, 8, 3, '2012-09-16 18:00:00', 1), (3100, 8, 2, '2012-09-16 19:00:00', 1), (3101, 8, 3, '2012-09-16 20:00:00', 1), (3102, 0, 22, '2012-09-17 08:00:00', 3), (3103, 0, 0, '2012-09-17 09:30:00', 3), (3104, 0, 13, '2012-09-17 11:00:00', 3), (3105, 0, 7, '2012-09-17 14:00:00', 3), (3106, 0, 0, '2012-09-17 16:30:00', 3), (3107, 0, 26, '2012-09-17 18:00:00', 3), (3108, 1, 8, '2012-09-17 08:30:00', 3), (3109, 1, 9, '2012-09-17 10:00:00', 3), (3110, 1, 0, '2012-09-17 11:30:00', 3), (3111, 1, 0, '2012-09-17 13:30:00', 3), (3112, 1, 8, '2012-09-17 16:00:00', 3), (3113, 1, 9, '2012-09-17 17:30:00', 3), (3114, 1, 8, '2012-09-17 19:00:00', 3), (3115, 2, 5, '2012-09-17 08:30:00', 3), (3116, 2, 12, '2012-09-17 10:00:00', 3), (3117, 2, 21, '2012-09-17 12:00:00', 3), (3118, 2, 12, '2012-09-17 13:30:00', 3), (3119, 2, 0, '2012-09-17 15:00:00', 3), (3120, 2, 1, '2012-09-17 18:00:00', 3), (3121, 3, 21, '2012-09-17 08:30:00', 2), (3122, 3, 22, '2012-09-17 09:30:00', 4), (3123, 3, 15, '2012-09-17 12:00:00', 2), (3124, 3, 22, '2012-09-17 14:00:00', 2), (3125, 3, 16, '2012-09-17 16:30:00', 2), (3126, 3, 13, '2012-09-17 17:30:00', 2), (3127, 3, 3, '2012-09-17 18:30:00', 2), (3128, 4, 7, '2012-09-17 08:00:00', 2), (3129, 4, 0, '2012-09-17 09:30:00', 2), (3130, 4, 0, '2012-09-17 11:00:00', 4), (3131, 4, 10, '2012-09-17 13:00:00', 2), (3132, 4, 20, '2012-09-17 14:30:00', 2), (3133, 4, 16, '2012-09-17 15:30:00', 2), (3134, 4, 0, '2012-09-17 16:30:00', 2), (3135, 4, 5, '2012-09-17 17:30:00', 2), (3136, 4, 14, '2012-09-17 18:30:00', 2), (3137, 4, 0, '2012-09-17 19:30:00', 2), (3138, 5, 0, '2012-09-17 12:00:00', 2), (3139, 5, 24, '2012-09-17 15:00:00', 2), (3140, 6, 0, '2012-09-17 08:00:00', 4), (3141, 6, 0, '2012-09-17 10:30:00', 4), (3142, 6, 5, '2012-09-17 12:30:00', 2), (3143, 6, 0, '2012-09-17 13:30:00', 2), (3144, 6, 0, '2012-09-17 15:00:00', 8), (3145, 7, 10, '2012-09-17 08:00:00', 2), (3146, 7, 21, '2012-09-17 10:00:00', 2), (3147, 7, 17, '2012-09-17 11:00:00', 2), (3148, 7, 15, '2012-09-17 14:00:00', 2), (3149, 7, 0, '2012-09-17 15:00:00', 2), (3150, 7, 22, '2012-09-17 16:30:00', 2), (3151, 7, 20, '2012-09-17 18:30:00', 2), (3152, 7, 1, '2012-09-17 19:30:00', 2), (3153, 8, 15, '2012-09-17 09:30:00', 1), (3154, 8, 16, '2012-09-17 11:00:00', 1), (3155, 8, 0, '2012-09-17 12:00:00', 1), (3156, 8, 8, '2012-09-17 12:30:00', 1), (3157, 8, 1, '2012-09-17 13:30:00', 1), (3158, 8, 11, '2012-09-17 14:00:00', 1), (3159, 8, 3, '2012-09-17 15:00:00', 1), (3160, 8, 1, '2012-09-17 15:30:00', 1), (3161, 8, 2, '2012-09-17 16:00:00', 1), (3162, 8, 21, '2012-09-17 16:30:00', 1), (3163, 8, 3, '2012-09-17 17:00:00', 1), (3164, 8, 21, '2012-09-17 17:30:00', 1), (3165, 8, 21, '2012-09-17 19:00:00', 1), (3166, 8, 2, '2012-09-17 20:00:00', 1), (3167, 0, 28, '2012-09-18 09:00:00', 3), (3168, 0, 6, '2012-09-18 10:30:00', 3), (3169, 0, 11, '2012-09-18 12:00:00', 3), (3170, 0, 16, '2012-09-18 13:30:00', 3), (3171, 0, 5, '2012-09-18 16:00:00', 3), (3172, 0, 28, '2012-09-18 17:30:00', 3), (3173, 0, 14, '2012-09-18 19:00:00', 3), (3174, 1, 10, '2012-09-18 08:00:00', 3), (3175, 1, 12, '2012-09-18 09:30:00', 6), (3176, 1, 0, '2012-09-18 13:30:00', 3), (3177, 1, 11, '2012-09-18 16:00:00', 3), (3178, 1, 10, '2012-09-18 18:00:00', 3), (3179, 2, 16, '2012-09-18 08:00:00', 3), (3180, 2, 21, '2012-09-18 10:00:00', 3), (3181, 2, 21, '2012-09-18 13:00:00', 3), (3182, 2, 1, '2012-09-18 14:30:00', 3), (3183, 2, 24, '2012-09-18 16:00:00', 3), (3184, 2, 13, '2012-09-18 17:30:00', 3), (3185, 3, 3, '2012-09-18 08:00:00', 4), (3186, 3, 20, '2012-09-18 10:00:00', 2), (3187, 3, 13, '2012-09-18 11:30:00', 2), (3188, 3, 3, '2012-09-18 13:00:00', 4), (3189, 3, 21, '2012-09-18 17:00:00', 2), (3190, 3, 15, '2012-09-18 18:00:00', 2), (3191, 3, 3, '2012-09-18 19:30:00', 2), (3192, 4, 5, '2012-09-18 08:30:00', 2), (3193, 4, 0, '2012-09-18 09:30:00', 2), (3194, 4, 13, '2012-09-18 10:30:00', 2), (3195, 4, 1, '2012-09-18 11:30:00', 2), (3196, 4, 13, '2012-09-18 12:30:00', 2), (3197, 4, 0, '2012-09-18 15:00:00', 2), (3198, 4, 13, '2012-09-18 16:00:00', 2), (3199, 4, 0, '2012-09-18 17:00:00', 2), (3200, 4, 4, '2012-09-18 18:00:00', 2), (3201, 4, 0, '2012-09-18 19:00:00', 2), (3202, 5, 0, '2012-09-18 08:30:00', 2), (3203, 5, 0, '2012-09-18 18:00:00', 2), (3204, 6, 0, '2012-09-18 08:30:00', 2), (3205, 6, 9, '2012-09-18 09:30:00', 2), (3206, 6, 0, '2012-09-18 11:00:00', 2), (3207, 6, 8, '2012-09-18 14:00:00', 2), (3208, 6, 0, '2012-09-18 15:00:00', 6), (3209, 6, 0, '2012-09-18 18:30:00', 4), (3210, 7, 7, '2012-09-18 10:00:00', 2), (3211, 7, 0, '2012-09-18 11:00:00', 2), (3212, 7, 27, '2012-09-18 16:00:00', 2), (3213, 7, 8, '2012-09-18 18:00:00', 2), (3214, 8, 21, '2012-09-18 08:30:00', 1), (3215, 8, 15, '2012-09-18 10:00:00', 1), (3216, 8, 0, '2012-09-18 10:30:00', 1), (3217, 8, 7, '2012-09-18 11:00:00', 1), (3218, 8, 3, '2012-09-18 12:00:00', 1), (3219, 8, 28, '2012-09-18 13:30:00', 1), (3220, 8, 15, '2012-09-18 14:00:00', 1), (3221, 8, 4, '2012-09-18 14:30:00', 1), (3222, 8, 24, '2012-09-18 15:30:00', 1), (3223, 8, 3, '2012-09-18 16:00:00', 1), (3224, 8, 21, '2012-09-18 16:30:00', 1), (3225, 8, 9, '2012-09-18 17:00:00', 1), (3226, 8, 0, '2012-09-18 17:30:00', 1), (3227, 8, 3, '2012-09-18 18:00:00', 1), (3228, 8, 0, '2012-09-18 19:00:00', 1), (3229, 8, 28, '2012-09-18 20:00:00', 1), (3230, 0, 16, '2012-09-19 08:00:00', 3), (3231, 0, 28, '2012-09-19 09:30:00', 3), (3232, 0, 0, '2012-09-19 11:00:00', 6), (3233, 0, 28, '2012-09-19 15:00:00', 3), (3234, 0, 24, '2012-09-19 16:30:00', 3), (3235, 0, 14, '2012-09-19 18:00:00', 3), (3236, 1, 0, '2012-09-19 09:30:00', 3), (3237, 1, 1, '2012-09-19 11:00:00', 3), (3238, 1, 0, '2012-09-19 13:00:00', 3), (3239, 1, 4, '2012-09-19 16:00:00', 3), (3240, 1, 10, '2012-09-19 18:00:00', 3), (3241, 2, 1, '2012-09-19 08:00:00', 3), (3242, 2, 16, '2012-09-19 10:00:00', 3), (3243, 2, 9, '2012-09-19 11:30:00', 3), (3244, 2, 21, '2012-09-19 13:00:00', 3), (3245, 2, 29, '2012-09-19 14:30:00', 3), (3246, 2, 30, '2012-09-19 17:00:00', 3), (3247, 3, 22, '2012-09-19 08:00:00', 2), (3248, 3, 15, '2012-09-19 09:30:00', 2), (3249, 3, 3, '2012-09-19 10:30:00', 2), (3250, 3, 3, '2012-09-19 12:00:00', 2), (3251, 3, 2, '2012-09-19 13:00:00', 2), (3252, 3, 1, '2012-09-19 16:00:00', 2), (3253, 3, 21, '2012-09-19 19:00:00', 2), (3254, 4, 20, '2012-09-19 08:00:00', 2), (3255, 4, 5, '2012-09-19 09:00:00', 2), (3256, 4, 0, '2012-09-19 10:00:00', 4), (3257, 4, 5, '2012-09-19 12:00:00', 2), (3258, 4, 0, '2012-09-19 13:30:00', 6), (3259, 4, 16, '2012-09-19 17:00:00', 2), (3260, 4, 0, '2012-09-19 18:00:00', 2), (3261, 4, 13, '2012-09-19 19:00:00', 2), (3262, 5, 10, '2012-09-19 10:30:00', 2), (3263, 5, 7, '2012-09-19 12:30:00', 2), (3264, 5, 0, '2012-09-19 13:30:00', 2), (3265, 5, 0, '2012-09-19 15:30:00', 2), (3266, 5, 0, '2012-09-19 19:30:00', 2), (3267, 6, 0, '2012-09-19 08:00:00', 2), (3268, 6, 8, '2012-09-19 09:00:00', 2), (3269, 6, 14, '2012-09-19 10:00:00', 4), (3270, 6, 0, '2012-09-19 12:30:00', 2), (3271, 6, 12, '2012-09-19 13:30:00', 2), (3272, 6, 0, '2012-09-19 15:00:00', 2), (3273, 6, 12, '2012-09-19 16:30:00', 2), (3274, 6, 4, '2012-09-19 17:30:00', 2), (3275, 7, 10, '2012-09-19 08:00:00', 2), (3276, 7, 27, '2012-09-19 09:30:00', 2), (3277, 7, 15, '2012-09-19 10:30:00', 2), (3278, 7, 4, '2012-09-19 13:00:00', 4), (3279, 7, 27, '2012-09-19 15:00:00', 2), (3280, 7, 15, '2012-09-19 16:30:00', 2), (3281, 7, 6, '2012-09-19 18:00:00', 2), (3282, 8, 21, '2012-09-19 08:00:00', 1), (3283, 8, 3, '2012-09-19 08:30:00', 2), (3284, 8, 29, '2012-09-19 09:30:00', 1), (3285, 8, 3, '2012-09-19 10:00:00', 1), (3286, 8, 12, '2012-09-19 10:30:00', 1), (3287, 8, 30, '2012-09-19 11:30:00', 1), (3288, 8, 28, '2012-09-19 12:00:00', 1), (3289, 8, 29, '2012-09-19 12:30:00', 1), (3290, 8, 24, '2012-09-19 13:30:00', 1), (3291, 8, 29, '2012-09-19 14:00:00', 1), (3292, 8, 16, '2012-09-19 14:30:00', 2), (3293, 8, 22, '2012-09-19 17:30:00', 1), (3294, 8, 29, '2012-09-19 18:00:00', 1), (3295, 8, 3, '2012-09-19 18:30:00', 1), (3296, 8, 8, '2012-09-19 19:00:00', 1), (3297, 8, 15, '2012-09-19 19:30:00', 1), (3298, 0, 14, '2012-09-20 08:00:00', 3), (3299, 0, 0, '2012-09-20 09:30:00', 6), (3300, 0, 0, '2012-09-20 13:00:00', 3), (3301, 0, 17, '2012-09-20 15:30:00', 3), (3302, 0, 26, '2012-09-20 17:00:00', 3), (3303, 0, 5, '2012-09-20 18:30:00', 3), (3304, 1, 11, '2012-09-20 08:30:00', 3), (3305, 1, 10, '2012-09-20 10:30:00', 3), (3306, 1, 30, '2012-09-20 12:30:00', 3), (3307, 1, 10, '2012-09-20 14:00:00', 3), (3308, 1, 9, '2012-09-20 16:00:00', 3), (3309, 1, 0, '2012-09-20 17:30:00', 3), (3310, 1, 24, '2012-09-20 19:00:00', 3), (3311, 2, 21, '2012-09-20 08:00:00', 6), (3312, 2, 0, '2012-09-20 11:00:00', 3), (3313, 2, 14, '2012-09-20 12:30:00', 3), (3314, 2, 1, '2012-09-20 14:00:00', 3), (3315, 2, 2, '2012-09-20 15:30:00', 3), (3316, 2, 14, '2012-09-20 17:30:00', 3), (3317, 3, 1, '2012-09-20 09:30:00', 2), (3318, 3, 21, '2012-09-20 15:00:00', 2), (3319, 3, 30, '2012-09-20 18:30:00', 4), (3320, 4, 0, '2012-09-20 08:00:00', 2), (3321, 4, 14, '2012-09-20 09:30:00', 2), (3322, 4, 0, '2012-09-20 10:30:00', 4), (3323, 4, 3, '2012-09-20 12:30:00', 4), (3324, 4, 0, '2012-09-20 15:00:00', 2), (3325, 4, 0, '2012-09-20 16:30:00', 2), (3326, 4, 20, '2012-09-20 18:00:00', 2), (3327, 4, 8, '2012-09-20 19:30:00', 2), (3328, 5, 0, '2012-09-20 11:00:00', 2), (3329, 5, 0, '2012-09-20 13:30:00', 2), (3330, 5, 0, '2012-09-20 16:30:00', 2), (3331, 5, 0, '2012-09-20 18:30:00', 2), (3332, 6, 6, '2012-09-20 08:00:00', 2), (3333, 6, 0, '2012-09-20 09:30:00', 6), (3334, 6, 0, '2012-09-20 14:00:00', 2), (3335, 6, 28, '2012-09-20 15:30:00', 2), (3336, 7, 0, '2012-09-20 08:30:00', 2), (3337, 7, 24, '2012-09-20 09:30:00', 2), (3338, 7, 9, '2012-09-20 13:00:00', 2), (3339, 7, 8, '2012-09-20 14:00:00', 2), (3340, 7, 4, '2012-09-20 15:00:00', 2), (3341, 7, 22, '2012-09-20 16:00:00', 2), (3342, 7, 15, '2012-09-20 17:30:00', 2), (3343, 7, 8, '2012-09-20 18:30:00', 2), (3344, 7, 33, '2012-09-20 19:30:00', 2), (3345, 8, 33, '2012-09-20 08:00:00', 1), (3346, 8, 24, '2012-09-20 08:30:00', 1), (3347, 8, 20, '2012-09-20 09:00:00', 1), (3348, 8, 3, '2012-09-20 10:00:00', 1), (3349, 8, 20, '2012-09-20 10:30:00', 1), (3350, 8, 24, '2012-09-20 11:00:00', 1), (3351, 8, 28, '2012-09-20 11:30:00', 1), (3352, 8, 3, '2012-09-20 12:00:00', 1), (3353, 8, 33, '2012-09-20 12:30:00', 1), (3354, 8, 16, '2012-09-20 13:00:00', 1), (3355, 8, 21, '2012-09-20 13:30:00', 1), (3356, 8, 28, '2012-09-20 14:00:00', 1), (3357, 8, 3, '2012-09-20 16:00:00', 1), (3358, 8, 0, '2012-09-20 19:00:00', 2), (3359, 8, 16, '2012-09-20 20:00:00', 1), (3360, 0, 26, '2012-09-21 08:00:00', 3), (3361, 0, 11, '2012-09-21 09:30:00', 3), (3362, 0, 22, '2012-09-21 12:00:00', 3), (3363, 0, 16, '2012-09-21 13:30:00', 3), (3364, 0, 5, '2012-09-21 15:30:00', 3), (3365, 0, 17, '2012-09-21 17:00:00', 6), (3366, 1, 12, '2012-09-21 08:00:00', 3), (3367, 1, 16, '2012-09-21 10:00:00', 3), (3368, 1, 1, '2012-09-21 11:30:00', 3), (3369, 1, 9, '2012-09-21 14:00:00', 3), (3370, 1, 10, '2012-09-21 16:00:00', 3), (3371, 1, 27, '2012-09-21 18:00:00', 3), (3372, 2, 9, '2012-09-21 09:00:00', 3), (3373, 2, 21, '2012-09-21 10:30:00', 3), (3374, 2, 9, '2012-09-21 12:00:00', 3), (3375, 2, 0, '2012-09-21 14:00:00', 6), (3376, 3, 29, '2012-09-21 09:00:00', 2), (3377, 3, 30, '2012-09-21 10:00:00', 2), (3378, 3, 2, '2012-09-21 11:00:00', 2), (3379, 3, 20, '2012-09-21 13:00:00', 2), (3380, 3, 21, '2012-09-21 14:00:00', 2), (3381, 3, 4, '2012-09-21 15:30:00', 2), (3382, 3, 30, '2012-09-21 16:30:00', 2), (3383, 3, 29, '2012-09-21 18:30:00', 2), (3384, 3, 1, '2012-09-21 19:30:00', 2), (3385, 4, 16, '2012-09-21 08:30:00', 2), (3386, 4, 14, '2012-09-21 09:30:00', 2), (3387, 4, 0, '2012-09-21 10:30:00', 2), (3388, 4, 8, '2012-09-21 11:30:00', 2), (3389, 4, 0, '2012-09-21 13:00:00', 2), (3390, 4, 14, '2012-09-21 14:30:00', 2), (3391, 4, 0, '2012-09-21 15:30:00', 2), (3392, 4, 9, '2012-09-21 16:30:00', 2), (3393, 4, 1, '2012-09-21 17:30:00', 2), (3394, 4, 3, '2012-09-21 18:30:00', 2), (3395, 4, 20, '2012-09-21 19:30:00', 2), (3396, 5, 15, '2012-09-21 12:00:00', 2), (3397, 5, 0, '2012-09-21 17:30:00', 2), (3398, 6, 0, '2012-09-21 09:30:00', 2), (3399, 6, 13, '2012-09-21 10:30:00', 2), (3400, 6, 0, '2012-09-21 11:30:00', 4), (3401, 6, 0, '2012-09-21 14:00:00', 2), (3402, 6, 12, '2012-09-21 15:30:00', 2), (3403, 6, 12, '2012-09-21 17:30:00', 2), (3404, 6, 14, '2012-09-21 18:30:00', 4), (3405, 7, 21, '2012-09-21 08:30:00', 2), (3406, 7, 5, '2012-09-21 09:30:00', 2), (3407, 7, 10, '2012-09-21 11:30:00', 2), (3408, 7, 24, '2012-09-21 13:00:00', 2), (3409, 7, 4, '2012-09-21 14:30:00', 2), (3410, 7, 24, '2012-09-21 16:00:00', 2), (3411, 7, 13, '2012-09-21 17:00:00', 2), (3412, 7, 5, '2012-09-21 19:00:00', 2), (3413, 8, 33, '2012-09-21 08:30:00', 2), (3414, 8, 21, '2012-09-21 09:30:00', 2), (3415, 8, 28, '2012-09-21 10:30:00', 1), (3416, 8, 3, '2012-09-21 11:00:00', 1), (3417, 8, 29, '2012-09-21 12:30:00', 1), (3418, 8, 12, '2012-09-21 13:00:00', 1), (3419, 8, 28, '2012-09-21 14:00:00', 1), (3420, 8, 29, '2012-09-21 14:30:00', 1), (3421, 8, 6, '2012-09-21 15:00:00', 1), (3422, 8, 3, '2012-09-21 16:00:00', 1), (3423, 8, 15, '2012-09-21 16:30:00', 1), (3424, 8, 8, '2012-09-21 17:00:00', 1), (3425, 8, 29, '2012-09-21 18:00:00', 1), (3426, 8, 33, '2012-09-21 18:30:00', 1), (3427, 8, 30, '2012-09-21 19:00:00', 1), (3428, 8, 3, '2012-09-21 19:30:00', 1), (3429, 0, 0, '2012-09-22 08:30:00', 3), (3430, 0, 11, '2012-09-22 10:00:00', 6), (3431, 0, 0, '2012-09-22 13:00:00', 3), (3432, 0, 6, '2012-09-22 15:00:00', 3), (3433, 0, 14, '2012-09-22 16:30:00', 3), (3434, 0, 10, '2012-09-22 18:00:00', 3), (3435, 1, 5, '2012-09-22 09:00:00', 3), (3436, 1, 10, '2012-09-22 11:00:00', 3), (3437, 1, 8, '2012-09-22 12:30:00', 3), (3438, 1, 15, '2012-09-22 14:00:00', 3), (3439, 1, 12, '2012-09-22 16:00:00', 3), (3440, 1, 0, '2012-09-22 17:30:00', 3), (3441, 2, 24, '2012-09-22 08:00:00', 3), (3442, 2, 0, '2012-09-22 09:30:00', 3), (3443, 2, 1, '2012-09-22 11:30:00', 3), (3444, 2, 2, '2012-09-22 13:30:00', 3), (3445, 2, 1, '2012-09-22 15:30:00', 3), (3446, 2, 24, '2012-09-22 17:00:00', 3), (3447, 3, 29, '2012-09-22 08:30:00', 2), (3448, 3, 20, '2012-09-22 11:00:00', 2), (3449, 3, 21, '2012-09-22 13:00:00', 2), (3450, 3, 30, '2012-09-22 16:30:00', 2), (3451, 3, 22, '2012-09-22 18:30:00', 4), (3452, 4, 0, '2012-09-22 08:00:00', 2), (3453, 4, 14, '2012-09-22 09:00:00', 2), (3454, 4, 7, '2012-09-22 10:00:00', 2), (3455, 4, 0, '2012-09-22 11:00:00', 2), (3456, 4, 24, '2012-09-22 12:00:00', 2), (3457, 4, 0, '2012-09-22 13:00:00', 2), (3458, 4, 16, '2012-09-22 14:00:00', 2), (3459, 4, 0, '2012-09-22 15:00:00', 8), (3460, 4, 2, '2012-09-22 19:00:00', 2), (3461, 5, 0, '2012-09-22 12:30:00', 2), (3462, 6, 6, '2012-09-22 09:00:00', 2), (3463, 6, 0, '2012-09-22 10:30:00', 2), (3464, 6, 4, '2012-09-22 11:30:00', 2), (3465, 6, 12, '2012-09-22 12:30:00', 2), (3466, 6, 0, '2012-09-22 13:30:00', 2), (3467, 6, 8, '2012-09-22 14:30:00', 2), (3468, 6, 0, '2012-09-22 15:30:00', 4), (3469, 6, 12, '2012-09-22 17:30:00', 2), (3470, 7, 10, '2012-09-22 08:00:00', 2), (3471, 7, 4, '2012-09-22 09:30:00', 2), (3472, 7, 27, '2012-09-22 11:00:00', 2), (3473, 7, 4, '2012-09-22 13:00:00', 2), (3474, 7, 30, '2012-09-22 15:00:00', 2), (3475, 7, 33, '2012-09-22 16:00:00', 2), (3476, 7, 17, '2012-09-22 18:00:00', 2), (3477, 7, 27, '2012-09-22 19:00:00', 2), (3478, 8, 22, '2012-09-22 08:00:00', 1), (3479, 8, 28, '2012-09-22 08:30:00', 1), (3480, 8, 17, '2012-09-22 09:30:00', 1), (3481, 8, 21, '2012-09-22 10:00:00', 1), (3482, 8, 21, '2012-09-22 11:30:00', 1), (3483, 8, 22, '2012-09-22 12:00:00', 1), (3484, 8, 29, '2012-09-22 13:30:00', 1), (3485, 8, 24, '2012-09-22 15:30:00', 1), (3486, 8, 3, '2012-09-22 16:30:00', 1), (3487, 8, 28, '2012-09-22 17:00:00', 1), (3488, 8, 3, '2012-09-22 18:30:00', 2), (3489, 8, 21, '2012-09-22 19:30:00', 1), (3490, 8, 11, '2012-09-22 20:00:00', 1), (3491, 0, 7, '2012-09-23 08:00:00', 3), (3492, 0, 10, '2012-09-23 09:30:00', 3), (3493, 0, 15, '2012-09-23 11:00:00', 3), (3494, 0, 26, '2012-09-23 12:30:00', 3), (3495, 0, 35, '2012-09-23 14:00:00', 3), (3496, 0, 2, '2012-09-23 16:30:00', 3), (3497, 0, 17, '2012-09-23 18:00:00', 3), (3498, 1, 0, '2012-09-23 08:30:00', 3), (3499, 1, 24, '2012-09-23 10:30:00', 3), (3500, 1, 8, '2012-09-23 12:00:00', 3), (3501, 1, 24, '2012-09-23 13:30:00', 3), (3502, 1, 15, '2012-09-23 15:00:00', 3), (3503, 1, 35, '2012-09-23 16:30:00', 3), (3504, 1, 0, '2012-09-23 18:00:00', 3), (3505, 2, 1, '2012-09-23 08:00:00', 3), (3506, 2, 1, '2012-09-23 11:00:00', 3), (3507, 2, 16, '2012-09-23 12:30:00', 3), (3508, 2, 29, '2012-09-23 14:00:00', 3), (3509, 2, 9, '2012-09-23 15:30:00', 3), (3510, 2, 7, '2012-09-23 17:00:00', 3), (3511, 3, 3, '2012-09-23 08:00:00', 2), (3512, 3, 22, '2012-09-23 09:30:00', 2), (3513, 3, 3, '2012-09-23 10:30:00', 2), (3514, 3, 22, '2012-09-23 12:30:00', 2), (3515, 3, 15, '2012-09-23 13:30:00', 2), (3516, 3, 22, '2012-09-23 15:00:00', 2), (3517, 3, 3, '2012-09-23 16:00:00', 2), (3518, 3, 22, '2012-09-23 17:30:00', 2), (3519, 3, 10, '2012-09-23 19:00:00', 2), (3520, 4, 11, '2012-09-23 08:00:00', 2), (3521, 4, 16, '2012-09-23 09:30:00', 2), (3522, 4, 9, '2012-09-23 10:30:00', 2), (3523, 4, 0, '2012-09-23 11:30:00', 2), (3524, 4, 6, '2012-09-23 12:30:00', 2), (3525, 4, 13, '2012-09-23 13:30:00', 4), (3526, 4, 22, '2012-09-23 16:00:00', 2), (3527, 4, 0, '2012-09-23 17:00:00', 2), (3528, 4, 11, '2012-09-23 18:00:00', 2), (3529, 4, 1, '2012-09-23 19:00:00', 2), (3530, 5, 0, '2012-09-23 15:00:00', 2), (3531, 6, 0, '2012-09-23 08:00:00', 4), (3532, 6, 0, '2012-09-23 10:30:00', 2), (3533, 6, 0, '2012-09-23 12:00:00', 2), (3534, 6, 0, '2012-09-23 13:30:00', 6), (3535, 6, 12, '2012-09-23 16:30:00', 2), (3536, 6, 15, '2012-09-23 17:30:00', 2), (3537, 6, 0, '2012-09-23 19:00:00', 2), (3538, 7, 8, '2012-09-23 09:00:00', 2), (3539, 7, 17, '2012-09-23 10:00:00', 2), (3540, 7, 9, '2012-09-23 11:30:00', 2), (3541, 7, 27, '2012-09-23 15:00:00', 2), (3542, 7, 15, '2012-09-23 16:30:00', 2), (3543, 7, 14, '2012-09-23 17:30:00', 2), (3544, 7, 0, '2012-09-23 19:00:00', 2), (3545, 8, 33, '2012-09-23 08:00:00', 1), (3546, 8, 28, '2012-09-23 08:30:00', 1), (3547, 8, 0, '2012-09-23 09:00:00', 1), (3548, 8, 3, '2012-09-23 09:30:00', 1), (3549, 8, 0, '2012-09-23 10:00:00', 1), (3550, 8, 29, '2012-09-23 10:30:00', 1), (3551, 8, 3, '2012-09-23 11:30:00', 1), (3552, 8, 30, '2012-09-23 12:00:00', 1), (3553, 8, 21, '2012-09-23 13:00:00', 2), (3554, 8, 0, '2012-09-23 15:00:00', 1), (3555, 8, 16, '2012-09-23 15:30:00', 1), (3556, 8, 13, '2012-09-23 16:00:00', 1), (3557, 8, 6, '2012-09-23 16:30:00', 1), (3558, 8, 29, '2012-09-23 17:00:00', 1), (3559, 8, 28, '2012-09-23 17:30:00', 1), (3560, 8, 6, '2012-09-23 18:00:00', 1), (3561, 8, 28, '2012-09-23 19:00:00', 1), (3562, 8, 29, '2012-09-23 19:30:00', 1), (3563, 0, 0, '2012-09-24 08:00:00', 9), (3564, 0, 35, '2012-09-24 12:30:00', 3), (3565, 0, 0, '2012-09-24 14:00:00', 3), (3566, 0, 14, '2012-09-24 15:30:00', 6), (3567, 0, 11, '2012-09-24 18:30:00', 3), (3568, 1, 28, '2012-09-24 08:00:00', 3), (3569, 1, 10, '2012-09-24 09:30:00', 3), (3570, 1, 10, '2012-09-24 12:00:00', 6), (3571, 1, 16, '2012-09-24 15:00:00', 3), (3572, 1, 0, '2012-09-24 16:30:00', 3), (3573, 1, 24, '2012-09-24 19:00:00', 3), (3574, 2, 21, '2012-09-24 08:00:00', 3), (3575, 2, 0, '2012-09-24 09:30:00', 3), (3576, 2, 1, '2012-09-24 11:30:00', 3), (3577, 2, 3, '2012-09-24 13:00:00', 3), (3578, 2, 12, '2012-09-24 14:30:00', 3), (3579, 2, 7, '2012-09-24 16:30:00', 3), (3580, 2, 3, '2012-09-24 18:00:00', 3), (3581, 3, 0, '2012-09-24 08:00:00', 2), (3582, 3, 21, '2012-09-24 09:30:00', 2), (3583, 3, 16, '2012-09-24 12:00:00', 2), (3584, 3, 15, '2012-09-24 13:00:00', 2), (3585, 3, 2, '2012-09-24 14:30:00', 2), (3586, 3, 20, '2012-09-24 15:30:00', 2), (3587, 3, 22, '2012-09-24 17:00:00', 2), (3588, 3, 16, '2012-09-24 18:00:00', 2), (3589, 3, 0, '2012-09-24 19:00:00', 2), (3590, 4, 11, '2012-09-24 08:00:00', 2), (3591, 4, 14, '2012-09-24 09:00:00', 2), (3592, 4, 0, '2012-09-24 10:30:00', 2), (3593, 4, 0, '2012-09-24 12:00:00', 2), (3594, 4, 20, '2012-09-24 13:00:00', 2), (3595, 4, 0, '2012-09-24 14:00:00', 2), (3596, 4, 8, '2012-09-24 15:00:00', 2), (3597, 4, 6, '2012-09-24 16:00:00', 2), (3598, 4, 8, '2012-09-24 17:00:00', 2), (3599, 4, 35, '2012-09-24 18:00:00', 2), (3600, 4, 0, '2012-09-24 19:00:00', 2), (3601, 5, 14, '2012-09-24 11:00:00', 2), (3602, 5, 0, '2012-09-24 14:00:00', 2), (3603, 6, 0, '2012-09-24 08:00:00', 2), (3604, 6, 9, '2012-09-24 09:30:00', 2), (3605, 6, 0, '2012-09-24 10:30:00', 2), (3606, 6, 0, '2012-09-24 12:00:00', 4), (3607, 6, 14, '2012-09-24 14:00:00', 2), (3608, 6, 0, '2012-09-24 15:00:00', 4), (3609, 6, 12, '2012-09-24 17:00:00', 2), (3610, 6, 33, '2012-09-24 18:00:00', 2), (3611, 6, 17, '2012-09-24 19:00:00', 2), (3612, 7, 24, '2012-09-24 08:30:00', 2), (3613, 7, 22, '2012-09-24 13:00:00', 2), (3614, 7, 9, '2012-09-24 14:30:00', 2), (3615, 7, 17, '2012-09-24 15:30:00', 2), (3616, 7, 28, '2012-09-24 16:30:00', 2), (3617, 7, 10, '2012-09-24 17:30:00', 2), (3618, 7, 8, '2012-09-24 19:00:00', 2), (3619, 8, 0, '2012-09-24 08:00:00', 1), (3620, 8, 29, '2012-09-24 08:30:00', 1), (3621, 8, 0, '2012-09-24 09:00:00', 1), (3622, 8, 3, '2012-09-24 09:30:00', 2), (3623, 8, 8, '2012-09-24 10:30:00', 1), (3624, 8, 0, '2012-09-24 12:00:00', 1), (3625, 8, 28, '2012-09-24 12:30:00', 1), (3626, 8, 21, '2012-09-24 13:30:00', 1), (3627, 8, 2, '2012-09-24 14:00:00', 1), (3628, 8, 3, '2012-09-24 14:30:00', 1), (3629, 8, 2, '2012-09-24 16:00:00', 1), (3630, 8, 22, '2012-09-24 16:30:00', 1), (3631, 8, 0, '2012-09-24 17:00:00', 1), (3632, 8, 29, '2012-09-24 17:30:00', 1), (3633, 8, 16, '2012-09-24 19:00:00', 1), (3634, 8, 29, '2012-09-24 19:30:00', 1), (3635, 0, 12, '2012-09-25 08:00:00', 3), (3636, 0, 2, '2012-09-25 09:30:00', 6), (3637, 0, 16, '2012-09-25 13:00:00', 3), (3638, 0, 0, '2012-09-25 15:00:00', 6), (3639, 0, 11, '2012-09-25 18:00:00', 3), (3640, 1, 9, '2012-09-25 08:00:00', 3), (3641, 1, 12, '2012-09-25 10:00:00', 3), (3642, 1, 0, '2012-09-25 11:30:00', 3), (3643, 1, 12, '2012-09-25 13:00:00', 3), (3644, 1, 11, '2012-09-25 14:30:00', 3), (3645, 1, 35, '2012-09-25 16:30:00', 3), (3646, 1, 0, '2012-09-25 18:30:00', 3), (3647, 2, 29, '2012-09-25 08:00:00', 6), (3648, 2, 11, '2012-09-25 11:30:00', 3), (3649, 2, 13, '2012-09-25 14:30:00', 3), (3650, 2, 0, '2012-09-25 16:00:00', 3), (3651, 2, 33, '2012-09-25 17:30:00', 3), (3652, 3, 20, '2012-09-25 08:00:00', 2), (3653, 3, 17, '2012-09-25 11:00:00', 2), (3654, 3, 22, '2012-09-25 12:30:00', 2), (3655, 3, 35, '2012-09-25 13:30:00', 2), (3656, 3, 20, '2012-09-25 17:30:00', 2), (3657, 3, 17, '2012-09-25 19:00:00', 2), (3658, 4, 8, '2012-09-25 08:00:00', 2), (3659, 4, 20, '2012-09-25 09:00:00', 2), (3660, 4, 14, '2012-09-25 10:00:00', 2), (3661, 4, 0, '2012-09-25 11:30:00', 4), (3662, 4, 8, '2012-09-25 13:30:00', 2), (3663, 4, 3, '2012-09-25 14:30:00', 2), (3664, 4, 0, '2012-09-25 15:30:00', 2), (3665, 4, 20, '2012-09-25 16:30:00', 2), (3666, 4, 4, '2012-09-25 17:30:00', 2), (3667, 4, 6, '2012-09-25 18:30:00', 2), (3668, 4, 3, '2012-09-25 19:30:00', 2), (3669, 5, 0, '2012-09-25 08:30:00', 2), (3670, 5, 0, '2012-09-25 11:30:00', 2), (3671, 5, 0, '2012-09-25 16:00:00', 2), (3672, 6, 0, '2012-09-25 08:00:00', 4), (3673, 6, 8, '2012-09-25 10:00:00', 2), (3674, 6, 0, '2012-09-25 11:00:00', 2), (3675, 6, 2, '2012-09-25 12:30:00', 2), (3676, 6, 0, '2012-09-25 14:00:00', 2), (3677, 6, 0, '2012-09-25 15:30:00', 4), (3678, 7, 21, '2012-09-25 09:30:00', 2), (3679, 7, 10, '2012-09-25 12:30:00', 2), (3680, 7, 0, '2012-09-25 13:30:00', 2), (3681, 7, 7, '2012-09-25 14:30:00', 2), (3682, 7, 33, '2012-09-25 15:30:00', 4), (3683, 7, 2, '2012-09-25 18:30:00', 2), (3684, 8, 15, '2012-09-25 08:00:00', 1), (3685, 8, 21, '2012-09-25 09:00:00', 1), (3686, 8, 3, '2012-09-25 09:30:00', 1), (3687, 8, 29, '2012-09-25 12:30:00', 1), (3688, 8, 3, '2012-09-25 13:30:00', 1), (3689, 8, 29, '2012-09-25 14:00:00', 1), (3690, 8, 16, '2012-09-25 15:00:00', 1), (3691, 8, 28, '2012-09-25 15:30:00', 1), (3692, 8, 28, '2012-09-25 17:00:00', 1), (3693, 8, 21, '2012-09-25 17:30:00', 1), (3694, 8, 3, '2012-09-25 18:30:00', 1), (3695, 8, 16, '2012-09-25 19:00:00', 1), (3696, 8, 33, '2012-09-25 19:30:00', 1), (3697, 0, 5, '2012-09-26 08:00:00', 3), (3698, 0, 0, '2012-09-26 09:30:00', 3), (3699, 0, 6, '2012-09-26 11:00:00', 3), (3700, 0, 11, '2012-09-26 13:30:00', 3), (3701, 0, 0, '2012-09-26 15:00:00', 6), (3702, 0, 22, '2012-09-26 18:00:00', 3), (3703, 1, 0, '2012-09-26 08:00:00', 3), (3704, 1, 0, '2012-09-26 10:30:00', 3), (3705, 1, 9, '2012-09-26 12:00:00', 3), (3706, 1, 35, '2012-09-26 14:00:00', 3), (3707, 1, 12, '2012-09-26 15:30:00', 6), (3708, 1, 11, '2012-09-26 18:30:00', 3), (3709, 2, 1, '2012-09-26 08:00:00', 3), (3710, 2, 2, '2012-09-26 09:30:00', 3), (3711, 2, 10, '2012-09-26 11:00:00', 3), (3712, 2, 1, '2012-09-26 12:30:00', 3), (3713, 2, 0, '2012-09-26 14:00:00', 3), (3714, 2, 1, '2012-09-26 15:30:00', 3), (3715, 2, 17, '2012-09-26 17:00:00', 3), (3716, 2, 12, '2012-09-26 18:30:00', 3), (3717, 3, 15, '2012-09-26 11:00:00', 2), (3718, 3, 30, '2012-09-26 12:00:00', 2), (3719, 3, 3, '2012-09-26 15:00:00', 2), (3720, 3, 22, '2012-09-26 16:00:00', 2), (3721, 3, 15, '2012-09-26 18:30:00', 2), (3722, 3, 9, '2012-09-26 19:30:00', 2), (3723, 4, 3, '2012-09-26 08:00:00', 2), (3724, 4, 0, '2012-09-26 09:00:00', 2), (3725, 4, 33, '2012-09-26 10:00:00', 2), (3726, 4, 20, '2012-09-26 11:00:00', 4), (3727, 4, 29, '2012-09-26 13:00:00', 2), (3728, 4, 0, '2012-09-26 14:00:00', 2), (3729, 4, 5, '2012-09-26 15:30:00', 2), (3730, 4, 0, '2012-09-26 16:30:00', 2), (3731, 4, 14, '2012-09-26 17:30:00', 2), (3732, 4, 20, '2012-09-26 18:30:00', 2), (3733, 5, 0, '2012-09-26 18:30:00', 2), (3734, 6, 0, '2012-09-26 08:00:00', 2), (3735, 6, 0, '2012-09-26 09:30:00', 2), (3736, 6, 30, '2012-09-26 10:30:00', 2), (3737, 6, 0, '2012-09-26 11:30:00', 2), (3738, 6, 0, '2012-09-26 13:00:00', 8), (3739, 6, 10, '2012-09-26 17:00:00', 2), (3740, 6, 21, '2012-09-26 18:00:00', 2), (3741, 6, 0, '2012-09-26 19:00:00', 2), (3742, 7, 7, '2012-09-26 09:00:00', 2), (3743, 7, 24, '2012-09-26 10:30:00', 2), (3744, 7, 5, '2012-09-26 11:30:00', 2), (3745, 7, 27, '2012-09-26 14:30:00', 2), (3746, 7, 24, '2012-09-26 16:00:00', 2), (3747, 7, 5, '2012-09-26 17:30:00', 2), (3748, 8, 0, '2012-09-26 08:30:00', 1), (3749, 8, 30, '2012-09-26 09:00:00', 1), (3750, 8, 16, '2012-09-26 09:30:00', 1), (3751, 8, 21, '2012-09-26 10:00:00', 1), (3752, 8, 29, '2012-09-26 10:30:00', 1), (3753, 8, 16, '2012-09-26 11:30:00', 1), (3754, 8, 29, '2012-09-26 12:00:00', 2), (3755, 8, 28, '2012-09-26 13:00:00', 1), (3756, 8, 3, '2012-09-26 14:00:00', 2), (3757, 8, 20, '2012-09-26 15:30:00', 1), (3758, 8, 3, '2012-09-26 16:00:00', 1), (3759, 8, 28, '2012-09-26 17:00:00', 1), (3760, 8, 21, '2012-09-26 19:00:00', 1), (3761, 8, 29, '2012-09-26 19:30:00', 1), (3762, 8, 24, '2012-09-26 20:00:00', 1), (3763, 0, 11, '2012-09-27 09:00:00', 3), (3764, 0, 6, '2012-09-27 11:00:00', 3), (3765, 0, 17, '2012-09-27 13:00:00', 3), (3766, 0, 26, '2012-09-27 16:00:00', 3), (3767, 0, 0, '2012-09-27 17:30:00', 6), (3768, 1, 0, '2012-09-27 08:00:00', 9), (3769, 1, 8, '2012-09-27 12:30:00', 3), (3770, 1, 0, '2012-09-27 14:30:00', 3), (3771, 1, 35, '2012-09-27 16:00:00', 3), (3772, 1, 10, '2012-09-27 17:30:00', 3), (3773, 2, 1, '2012-09-27 08:00:00', 3), (3774, 2, 24, '2012-09-27 10:00:00', 3), (3775, 2, 36, '2012-09-27 11:30:00', 3), (3776, 2, 30, '2012-09-27 15:30:00', 3), (3777, 2, 11, '2012-09-27 17:30:00', 3), (3778, 2, 2, '2012-09-27 19:00:00', 3), (3779, 3, 15, '2012-09-27 08:30:00', 2), (3780, 3, 22, '2012-09-27 09:30:00', 2), (3781, 3, 0, '2012-09-27 12:00:00', 2), (3782, 3, 15, '2012-09-27 13:00:00', 2), (3783, 3, 15, '2012-09-27 15:30:00', 2), (3784, 3, 20, '2012-09-27 17:30:00', 2), (3785, 3, 0, '2012-09-27 18:30:00', 2), (3786, 4, 0, '2012-09-27 08:00:00', 2), (3787, 4, 24, '2012-09-27 09:00:00', 2), (3788, 4, 35, '2012-09-27 10:00:00', 2), (3789, 4, 20, '2012-09-27 11:00:00', 2), (3790, 4, 0, '2012-09-27 12:00:00', 2), (3791, 4, 12, '2012-09-27 13:00:00', 2), (3792, 4, 24, '2012-09-27 14:00:00', 2), (3793, 4, 36, '2012-09-27 15:00:00', 2), (3794, 4, 0, '2012-09-27 16:00:00', 2), (3795, 4, 7, '2012-09-27 17:00:00', 2), (3796, 4, 35, '2012-09-27 18:00:00', 2), (3797, 4, 6, '2012-09-27 19:00:00', 2), (3798, 5, 0, '2012-09-27 10:30:00', 2), (3799, 5, 22, '2012-09-27 16:30:00', 2), (3800, 6, 0, '2012-09-27 08:00:00', 2), (3801, 6, 12, '2012-09-27 09:30:00', 2), (3802, 6, 0, '2012-09-27 10:30:00', 2), (3803, 6, 0, '2012-09-27 12:00:00', 6), (3804, 6, 12, '2012-09-27 15:00:00', 4), (3805, 6, 0, '2012-09-27 19:00:00', 2), (3806, 7, 24, '2012-09-27 08:00:00', 2), (3807, 7, 4, '2012-09-27 09:30:00', 2), (3808, 7, 10, '2012-09-27 12:00:00', 2), (3809, 7, 10, '2012-09-27 13:30:00', 2), (3810, 7, 8, '2012-09-27 15:00:00', 2), (3811, 7, 9, '2012-09-27 16:00:00', 2), (3812, 7, 24, '2012-09-27 18:30:00', 2), (3813, 7, 17, '2012-09-27 19:30:00', 2), (3814, 8, 28, '2012-09-27 08:00:00', 1), (3815, 8, 2, '2012-09-27 08:30:00', 1), (3816, 8, 29, '2012-09-27 09:00:00', 1), (3817, 8, 33, '2012-09-27 10:30:00', 1), (3818, 8, 3, '2012-09-27 11:00:00', 1), (3819, 8, 29, '2012-09-27 13:30:00', 1), (3820, 8, 22, '2012-09-27 14:00:00', 1), (3821, 8, 29, '2012-09-27 15:00:00', 1), (3822, 8, 0, '2012-09-27 15:30:00', 1), (3823, 8, 20, '2012-09-27 16:00:00', 1), (3824, 8, 8, '2012-09-27 16:30:00', 1), (3825, 8, 16, '2012-09-27 17:00:00', 1), (3826, 8, 27, '2012-09-27 18:00:00', 1), (3827, 8, 3, '2012-09-27 19:30:00', 1), (3828, 8, 29, '2012-09-27 20:00:00', 1), (3829, 0, 35, '2012-09-28 08:30:00', 3), (3830, 0, 16, '2012-09-28 10:00:00', 3), (3831, 0, 28, '2012-09-28 11:30:00', 3), (3832, 0, 0, '2012-09-28 13:00:00', 3), (3833, 0, 0, '2012-09-28 15:00:00', 3), (3834, 0, 0, '2012-09-28 17:00:00', 3), (3835, 1, 10, '2012-09-28 08:00:00', 3), (3836, 1, 0, '2012-09-28 09:30:00', 9), (3837, 1, 8, '2012-09-28 14:00:00', 3), (3838, 1, 0, '2012-09-28 15:30:00', 3), (3839, 1, 0, '2012-09-28 17:30:00', 6), (3840, 2, 2, '2012-09-28 08:00:00', 3), (3841, 2, 21, '2012-09-28 09:30:00', 3), (3842, 2, 21, '2012-09-28 11:30:00', 3), (3843, 2, 1, '2012-09-28 13:00:00', 3), (3844, 2, 5, '2012-09-28 14:30:00', 3), (3845, 2, 17, '2012-09-28 16:00:00', 3), (3846, 2, 0, '2012-09-28 17:30:00', 3), (3847, 2, 1, '2012-09-28 19:00:00', 3), (3848, 3, 30, '2012-09-28 09:00:00', 2), (3849, 3, 15, '2012-09-28 10:30:00', 2), (3850, 3, 13, '2012-09-28 11:30:00', 2), (3851, 3, 22, '2012-09-28 12:30:00', 2), (3852, 3, 15, '2012-09-28 14:00:00', 2), (3853, 3, 24, '2012-09-28 15:30:00', 2), (3854, 3, 22, '2012-09-28 17:30:00', 2), (3855, 3, 20, '2012-09-28 19:00:00', 2), (3856, 4, 24, '2012-09-28 08:00:00', 2), (3857, 4, 0, '2012-09-28 09:00:00', 4), (3858, 4, 14, '2012-09-28 11:30:00', 2), (3859, 4, 0, '2012-09-28 12:30:00', 4), (3860, 4, 16, '2012-09-28 14:30:00', 2), (3861, 4, 0, '2012-09-28 15:30:00', 2), (3862, 4, 5, '2012-09-28 16:30:00', 2), (3863, 4, 0, '2012-09-28 17:30:00', 2), (3864, 4, 8, '2012-09-28 18:30:00', 2), (3865, 4, 7, '2012-09-28 19:30:00', 2), (3866, 5, 0, '2012-09-28 10:30:00', 2), (3867, 5, 0, '2012-09-28 13:00:00', 2), (3868, 5, 11, '2012-09-28 16:00:00', 2), (3869, 5, 0, '2012-09-28 17:00:00', 4), (3870, 6, 0, '2012-09-28 08:00:00', 6), (3871, 6, 12, '2012-09-28 11:00:00', 2), (3872, 6, 0, '2012-09-28 12:00:00', 2), (3873, 6, 16, '2012-09-28 13:30:00', 2), (3874, 6, 0, '2012-09-28 14:30:00', 2), (3875, 6, 0, '2012-09-28 16:00:00', 6), (3876, 6, 12, '2012-09-28 19:30:00', 2), (3877, 7, 17, '2012-09-28 08:00:00', 2), (3878, 7, 27, '2012-09-28 09:00:00', 2), (3879, 7, 9, '2012-09-28 12:00:00', 2), (3880, 7, 21, '2012-09-28 13:30:00', 2), (3881, 7, 27, '2012-09-28 15:00:00', 2), (3882, 7, 8, '2012-09-28 16:30:00', 2), (3883, 7, 6, '2012-09-28 18:00:00', 2), (3884, 8, 21, '2012-09-28 08:00:00', 1), (3885, 8, 28, '2012-09-28 09:30:00', 1), (3886, 8, 3, '2012-09-28 10:30:00', 1), (3887, 8, 3, '2012-09-28 12:00:00', 1), (3888, 8, 29, '2012-09-28 12:30:00', 1), (3889, 8, 28, '2012-09-28 13:00:00', 1), (3890, 8, 3, '2012-09-28 13:30:00', 2), (3891, 8, 30, '2012-09-28 15:30:00', 1), (3892, 8, 12, '2012-09-28 16:30:00', 1), (3893, 8, 0, '2012-09-28 17:00:00', 1), (3894, 8, 3, '2012-09-28 17:30:00', 1), (3895, 8, 29, '2012-09-28 18:00:00', 1), (3896, 8, 10, '2012-09-28 18:30:00', 1), (3897, 8, 21, '2012-09-28 19:30:00', 1), (3898, 8, 16, '2012-09-28 20:00:00', 1), (3899, 0, 0, '2012-09-29 08:00:00', 3), (3900, 0, 11, '2012-09-29 11:30:00', 6), (3901, 0, 6, '2012-09-29 14:30:00', 3), (3902, 0, 28, '2012-09-29 16:00:00', 3), (3903, 0, 20, '2012-09-29 17:30:00', 3), (3904, 1, 0, '2012-09-29 10:00:00', 3), (3905, 1, 8, '2012-09-29 11:30:00', 3), (3906, 1, 10, '2012-09-29 13:00:00', 3), (3907, 1, 12, '2012-09-29 14:30:00', 3), (3908, 1, 0, '2012-09-29 16:00:00', 3), (3909, 1, 10, '2012-09-29 18:00:00', 3), (3910, 2, 1, '2012-09-29 08:00:00', 3), (3911, 2, 36, '2012-09-29 09:30:00', 3), (3912, 2, 14, '2012-09-29 11:00:00', 3), (3913, 2, 21, '2012-09-29 12:30:00', 3), (3914, 2, 1, '2012-09-29 14:00:00', 3), (3915, 2, 24, '2012-09-29 15:30:00', 3), (3916, 2, 12, '2012-09-29 17:00:00', 3), (3917, 2, 16, '2012-09-29 18:30:00', 3), (3918, 3, 2, '2012-09-29 08:30:00', 2), (3919, 3, 21, '2012-09-29 09:30:00', 2), (3920, 3, 6, '2012-09-29 11:00:00', 2), (3921, 3, 13, '2012-09-29 13:00:00', 2), (3922, 3, 16, '2012-09-29 14:00:00', 2), (3923, 3, 20, '2012-09-29 16:00:00', 2), (3924, 3, 21, '2012-09-29 19:30:00', 2), (3925, 4, 16, '2012-09-29 08:00:00', 2), (3926, 4, 0, '2012-09-29 09:30:00', 2), (3927, 4, 3, '2012-09-29 10:30:00', 2), (3928, 4, 20, '2012-09-29 11:30:00', 2), (3929, 4, 5, '2012-09-29 12:30:00', 2), (3930, 4, 0, '2012-09-29 13:30:00', 2), (3931, 4, 3, '2012-09-29 14:30:00', 2), (3932, 4, 0, '2012-09-29 15:30:00', 2), (3933, 4, 16, '2012-09-29 16:30:00', 2), (3934, 4, 13, '2012-09-29 17:30:00', 2), (3935, 4, 36, '2012-09-29 18:30:00', 2), (3936, 4, 24, '2012-09-29 19:30:00', 2), (3937, 5, 0, '2012-09-29 12:30:00', 2), (3938, 6, 6, '2012-09-29 08:00:00', 2), (3939, 6, 0, '2012-09-29 09:00:00', 4), (3940, 6, 24, '2012-09-29 11:00:00', 2), (3941, 6, 0, '2012-09-29 12:00:00', 2), (3942, 6, 12, '2012-09-29 13:00:00', 2), (3943, 6, 0, '2012-09-29 14:00:00', 2), (3944, 6, 27, '2012-09-29 17:00:00', 2), (3945, 6, 0, '2012-09-29 18:00:00', 4), (3946, 7, 8, '2012-09-29 08:30:00', 2), (3947, 7, 4, '2012-09-29 10:00:00', 2), (3948, 7, 0, '2012-09-29 12:30:00', 2), (3949, 7, 24, '2012-09-29 13:30:00', 2), (3950, 7, 8, '2012-09-29 14:30:00', 2), (3951, 7, 27, '2012-09-29 15:30:00', 2), (3952, 7, 8, '2012-09-29 16:30:00', 2), (3953, 7, 15, '2012-09-29 18:30:00', 2), (3954, 7, 27, '2012-09-29 19:30:00', 2), (3955, 8, 12, '2012-09-29 08:00:00', 1), (3956, 8, 3, '2012-09-29 08:30:00', 1), (3957, 8, 21, '2012-09-29 09:00:00', 1), (3958, 8, 29, '2012-09-29 10:00:00', 1), (3959, 8, 28, '2012-09-29 10:30:00', 1), (3960, 8, 2, '2012-09-29 11:00:00', 2), (3961, 8, 29, '2012-09-29 12:00:00', 2), (3962, 8, 20, '2012-09-29 13:00:00', 1), (3963, 8, 28, '2012-09-29 13:30:00', 1), (3964, 8, 3, '2012-09-29 14:00:00', 1), (3965, 8, 28, '2012-09-29 14:30:00', 1), (3966, 8, 12, '2012-09-29 16:00:00', 1), (3967, 8, 26, '2012-09-29 16:30:00', 1), (3968, 8, 15, '2012-09-29 17:00:00', 1), (3969, 8, 28, '2012-09-29 17:30:00', 1), (3970, 8, 29, '2012-09-29 18:00:00', 2), (3971, 8, 4, '2012-09-29 19:30:00', 1), (3972, 8, 33, '2012-09-29 20:00:00', 1), (3973, 0, 4, '2012-09-30 08:00:00', 3), (3974, 0, 35, '2012-09-30 09:30:00', 3), (3975, 0, 0, '2012-09-30 11:00:00', 6), (3976, 0, 36, '2012-09-30 14:00:00', 3), (3977, 0, 24, '2012-09-30 16:00:00', 3), (3978, 0, 0, '2012-09-30 17:30:00', 3), (3979, 0, 24, '2012-09-30 19:00:00', 3), (3980, 1, 8, '2012-09-30 08:30:00', 3), (3981, 1, 0, '2012-09-30 10:00:00', 3), (3982, 1, 10, '2012-09-30 11:30:00', 3), (3983, 1, 11, '2012-09-30 13:30:00', 6), (3984, 1, 10, '2012-09-30 16:30:00', 3), (3985, 1, 8, '2012-09-30 18:30:00', 3), (3986, 2, 1, '2012-09-30 08:00:00', 3), (3987, 2, 17, '2012-09-30 09:30:00', 3), (3988, 2, 29, '2012-09-30 11:00:00', 3), (3989, 2, 35, '2012-09-30 12:30:00', 3), (3990, 2, 1, '2012-09-30 14:00:00', 6), (3991, 2, 5, '2012-09-30 17:00:00', 3), (3992, 2, 35, '2012-09-30 18:30:00', 3), (3993, 3, 24, '2012-09-30 08:00:00', 2), (3994, 3, 3, '2012-09-30 09:30:00', 2), (3995, 3, 36, '2012-09-30 10:30:00', 2), (3996, 3, 36, '2012-09-30 12:00:00', 2), (3997, 3, 0, '2012-09-30 14:30:00', 2), (3998, 3, 1, '2012-09-30 18:30:00', 2), (3999, 4, 13, '2012-09-30 08:00:00', 2), (4000, 4, 16, '2012-09-30 09:00:00', 2), (4001, 4, 0, '2012-09-30 10:00:00', 2), (4002, 4, 20, '2012-09-30 11:00:00', 2), (4003, 4, 4, '2012-09-30 12:30:00', 2), (4004, 4, 3, '2012-09-30 13:30:00', 2), (4005, 4, 20, '2012-09-30 15:00:00', 2), (4006, 4, 0, '2012-09-30 16:00:00', 2), (4007, 4, 3, '2012-09-30 17:00:00', 2), (4008, 4, 0, '2012-09-30 18:00:00', 2), (4009, 5, 0, '2012-09-30 11:30:00', 2), (4010, 5, 0, '2012-09-30 19:30:00', 2), (4011, 6, 0, '2012-09-30 08:00:00', 2), (4012, 6, 27, '2012-09-30 09:30:00', 2), (4013, 6, 0, '2012-09-30 11:00:00', 2), (4014, 6, 0, '2012-09-30 12:30:00', 2), (4015, 6, 12, '2012-09-30 14:00:00', 2), (4016, 6, 0, '2012-09-30 15:30:00', 2), (4017, 6, 35, '2012-09-30 16:30:00', 2), (4018, 6, 0, '2012-09-30 17:30:00', 2), (4019, 6, 0, '2012-09-30 19:00:00', 2), (4020, 7, 27, '2012-09-30 08:30:00', 2), (4021, 7, 33, '2012-09-30 09:30:00', 2), (4022, 7, 33, '2012-09-30 11:00:00', 2), (4023, 7, 5, '2012-09-30 14:30:00', 2), (4024, 7, 15, '2012-09-30 16:30:00', 2), (4025, 7, 24, '2012-09-30 17:30:00', 2), (4026, 7, 5, '2012-09-30 19:00:00', 2), (4027, 8, 16, '2012-09-30 08:00:00', 1), (4028, 8, 21, '2012-09-30 08:30:00', 2), (4029, 8, 3, '2012-09-30 10:30:00', 1), (4030, 8, 16, '2012-09-30 11:00:00', 1), (4031, 8, 3, '2012-09-30 11:30:00', 1), (4032, 8, 17, '2012-09-30 12:00:00', 1), (4033, 8, 21, '2012-09-30 12:30:00', 1), (4034, 8, 3, '2012-09-30 13:00:00', 1), (4035, 8, 29, '2012-09-30 13:30:00', 1), (4036, 8, 28, '2012-09-30 14:30:00', 1), (4037, 8, 29, '2012-09-30 15:30:00', 1), (4038, 8, 29, '2012-09-30 16:30:00', 2), (4039, 8, 29, '2012-09-30 18:00:00', 1), (4040, 8, 21, '2012-09-30 18:30:00', 1), (4041, 8, 16, '2012-09-30 19:00:00', 1), (4042, 8, 29, '2012-09-30 19:30:00', 1), (4043, 8, 5, '2013-01-01 15:30:00', 1); -- -- TOC entry 2200 (class 0 OID 32770) -- Dependencies: 169 -- Data for Name: facilities; Type: TABLE DATA; Schema: cd; Owner: - -- INSERT INTO facilities (facid, name, membercost, guestcost, initialoutlay, monthlymaintenance) VALUES (0, 'Tennis Court 1', 5, 25, 10000, 200), (1, 'Tennis Court 2', 5, 25, 8000, 200), (2, 'Badminton Court', 0, 15.5, 4000, 50), (3, 'Table Tennis', 0, 5, 320, 10), (4, 'Massage Room 1', 35, 80, 4000, 3000), (5, 'Massage Room 2', 35, 80, 4000, 3000), (6, 'Squash Court', 3.5, 17.5, 5000, 80), (7, 'Snooker Table', 0, 5, 450, 15), (8, 'Pool Table', 0, 5, 400, 15); -- -- TOC entry 2201 (class 0 OID 32800) -- Dependencies: 170 -- Data for Name: members; Type: TABLE DATA; Schema: cd; Owner: - -- INSERT INTO members (memid, surname, firstname, address, zipcode, telephone, recommendedby, joindate) VALUES (0, 'GUEST', 'GUEST', 'GUEST', 0, '(000) 000-0000', NULL, '2012-07-01 00:00:00'), (1, 'Smith', 'Darren', '8 Bloomsbury Close, Boston', 4321, '555-555-5555', NULL, '2012-07-02 12:02:05'), (2, 'Smith', 'Tracy', '8 Bloomsbury Close, New York', 4321, '555-555-5555', NULL, '2012-07-02 12:08:23'), (3, 'Rownam', 'Tim', '23 Highway Way, Boston', 23423, '(844) 693-0723', NULL, '2012-07-03 09:32:15'), (4, 'Joplette', 'Janice', '20 Crossing Road, New York', 234, '(833) 942-4710', 1, '2012-07-03 10:25:05'), (5, 'Butters', 'Gerald', '1065 Huntingdon Avenue, Boston', 56754, '(844) 078-4130', 1, '2012-07-09 10:44:09'), (6, 'Tracy', 'Burton', '3 Tunisia Drive, Boston', 45678, '(822) 354-9973', NULL, '2012-07-15 08:52:55'), (7, 'Dare', 'Nancy', '6 Hunting Lodge Way, Boston', 10383, '(833) 776-4001', 4, '2012-07-25 08:59:12'), (8, 'Boothe', 'Tim', '3 Bloomsbury Close, Reading, 00234', 234, '(811) 433-2547', 3, '2012-07-25 16:02:35'), (9, 'Stibbons', 'Ponder', '5 Dragons Way, Winchester', 87630, '(833) 160-3900', 6, '2012-07-25 17:09:05'), (10, 'Owen', 'Charles', '52 Cheshire Grove, Winchester, 28563', 28563, '(855) 542-5251', 1, '2012-08-03 19:42:37'), (11, 'Jones', 'David', '976 Gnats Close, Reading', 33862, '(844) 536-8036', 4, '2012-08-06 16:32:55'), (12, 'Baker', 'Anne', '55 Powdery Street, Boston', 80743, '844-076-5141', 9, '2012-08-10 14:23:22'), (13, 'Farrell', 'Jemima', '103 Firth Avenue, North Reading', 57392, '(855) 016-0163', NULL, '2012-08-10 14:28:01'), (14, 'Smith', 'Jack', '252 Binkington Way, Boston', 69302, '(822) 163-3254', 1, '2012-08-10 16:22:05'), (15, 'Bader', 'Florence', '264 Ursula Drive, Westford', 84923, '(833) 499-3527', 9, '2012-08-10 17:52:03'), (16, 'Baker', 'Timothy', '329 James Street, Reading', 58393, '833-941-0824', 13, '2012-08-15 10:34:25'), (17, 'Pinker', 'David', '5 Impreza Road, Boston', 65332, '811 409-6734', 13, '2012-08-16 11:32:47'), (20, 'Genting', 'Matthew', '4 Nunnington Place, Wingfield, Boston', 52365, '(811) 972-1377', 5, '2012-08-19 14:55:55'), (21, 'Mackenzie', 'Anna', '64 Perkington Lane, Reading', 64577, '(822) 661-2898', 1, '2012-08-26 09:32:05'), (22, 'Coplin', 'Joan', '85 Bard Street, Bloomington, Boston', 43533, '(822) 499-2232', 16, '2012-08-29 08:32:41'), (24, 'Sarwin', 'Ramnaresh', '12 Bullington Lane, Boston', 65464, '(822) 413-1470', 15, '2012-09-01 08:44:42'), (26, 'Jones', 'Douglas', '976 Gnats Close, Reading', 11986, '844 536-8036', 11, '2012-09-02 18:43:05'), (27, 'Rumney', 'Henrietta', '3 Burkington Plaza, Boston', 78533, '(822) 989-8876', 20, '2012-09-05 08:42:35'), (28, 'Farrell', 'David', '437 Granite Farm Road, Westford', 43532, '(855) 755-9876', NULL, '2012-09-15 08:22:05'), (29, 'Worthington-Smyth', 'Henry', '55 Jagbi Way, North Reading', 97676, '(855) 894-3758', 2, '2012-09-17 12:27:15'), (30, 'Purview', 'Millicent', '641 Drudgery Close, Burnington, Boston', 34232, '(855) 941-9786', 2, '2012-09-18 19:04:01'), (33, 'Tupperware', 'Hyacinth', '33 Cheerful Plaza, Drake Road, Westford', 68666, '(822) 665-5327', NULL, '2012-09-18 19:32:05'), (35, 'Hunt', 'John', '5 Bullington Lane, Boston', 54333, '(899) 720-6978', 30, '2012-09-19 11:32:45'), (36, 'Crumpet', 'Erica', 'Crimson Road, North Reading', 75655, '(811) 732-4816', 2, '2012-09-22 08:36:38'), (37, 'Smith', 'Darren', '3 Funktown, Denzington, Boston', 66796, '(822) 577-3541', NULL, '2012-09-26 18:08:45'); -- -- TOC entry 2196 (class 2606 OID 32822) -- Name: bookings_pk; Type: CONSTRAINT; Schema: cd; Owner: -; Tablespace: -- ALTER TABLE ONLY bookings ADD CONSTRAINT bookings_pk PRIMARY KEY (bookid); -- -- TOC entry 2192 (class 2606 OID 32777) -- Name: facilities_pk; Type: CONSTRAINT; Schema: cd; Owner: -; Tablespace: -- ALTER TABLE ONLY facilities ADD CONSTRAINT facilities_pk PRIMARY KEY (facid); -- -- TOC entry 2194 (class 2606 OID 32807) -- Name: members_pk; Type: CONSTRAINT; Schema: cd; Owner: -; Tablespace: -- ALTER TABLE ONLY members ADD CONSTRAINT members_pk PRIMARY KEY (memid); -- -- TOC entry 2198 (class 2606 OID 32823) -- Name: fk_bookings_facid; Type: FK CONSTRAINT; Schema: cd; Owner: - -- ALTER TABLE ONLY bookings ADD CONSTRAINT fk_bookings_facid FOREIGN KEY (facid) REFERENCES facilities(facid); -- -- TOC entry 2199 (class 2606 OID 32828) -- Name: fk_bookings_memid; Type: FK CONSTRAINT; Schema: cd; Owner: - -- ALTER TABLE ONLY bookings ADD CONSTRAINT fk_bookings_memid FOREIGN KEY (memid) REFERENCES members(memid); -- -- TOC entry 2197 (class 2606 OID 32808) -- Name: fk_members_recommendedby; Type: FK CONSTRAINT; Schema: cd; Owner: - -- ALTER TABLE ONLY members ADD CONSTRAINT fk_members_recommendedby FOREIGN KEY (recommendedby) REFERENCES members(memid) ON DELETE SET NULL; -- Completed on 2013-05-19 16:05:12 BST -- -- PostgreSQL database dump complete -- CREATE INDEX "bookings.memid_facid" ON bookings USING btree (memid, facid); CREATE INDEX "bookings.facid_memid" ON bookings USING btree (facid, memid); CREATE INDEX "bookings.facid_starttime" ON bookings USING btree (facid, starttime); CREATE INDEX "bookings.memid_starttime" ON bookings USING btree (memid, starttime); CREATE INDEX "bookings.starttime" ON bookings USING btree (starttime); CREATE INDEX "members.joindate" ON members USING btree (joindate); CREATE INDEX "members.recommendedby" ON members USING btree (recommendedby); ANALYZE; ================================================ FILE: docs/conf.py ================================================ # -*- coding: utf-8 -*- # # peewee documentation build configuration file, created by # sphinx-quickstart on Fri Nov 26 11:05:15 2010. # # This file is execfile()d with the current directory set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. #RTD_NEW_THEME = True import sys, os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. #extensions = ['sphinx.ext.autodoc', 'sphinx_rtd_theme'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = 'peewee' copyright = 'charles leifer' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. src_dir = os.path.realpath(os.path.dirname(os.path.dirname(__file__))) sys.path.insert(0, src_dir) from peewee import __version__ version = __version__ # The full version, including alpha/beta/rc tags. release = __version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build'] # The reST default role (used for this markup: `text`) to use for all documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). add_module_names = False # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. #pygments_style = 'pastie' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. #html_theme = 'sphinx_rtd_theme' #html_theme = 'bizstyle' #html_theme_options = { # 'body_max_width': '800px', #} html_theme = 'alabaster' html_theme_options = { 'github_user': 'coleifer', 'github_repo': 'peewee', 'page_width': '1000px', 'sidebar_width': '240px', 'font_size': '15px', } # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = { # 'index_logo': 'peewee4-logo.png' #} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = ['_themes'] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'peeweedoc' # -- Options for LaTeX output -------------------------------------------------- # The paper size ('letter' or 'a4'). #latex_paper_size = 'letter' # The font size ('10pt', '11pt' or '12pt'). #latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ('index', 'peewee.tex', u'peewee Documentation', u'charles leifer', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Additional stuff for the LaTeX preamble. #latex_preamble = '' # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True autodoc_default_flags = ['members', 'show-inheritance'] autodoc_member_order = 'bysource' # -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'peewee', u'peewee Documentation', [u'charles leifer'], 1) ] ================================================ FILE: docs/index.rst ================================================ .. peewee documentation master file, created by sphinx-quickstart on Thu Nov 25 21:20:29 2010. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. peewee ====== .. image:: peewee4-logo.png Peewee is a simple and small ORM. It has few (but expressive) concepts, making it easy to learn and intuitive to use. * a small, expressive ORM * flexible query-builder that exposes full power of SQL * supports :ref:`sqlite, mysql, mariadb, postgresql `. * :ref:`asyncio support ` * tons of extensions * use with :ref:`flask `, :ref:`fastapi `, :ref:`pydantic ` and :ref:`more ` Peewee's source code hosted on `GitHub `_. New to peewee? These may help: * :ref:`Quickstart ` * :ref:`Example twitter app ` * :ref:`Using peewee interactively ` * :ref:`Models and fields ` * :ref:`Querying ` * :ref:`Relationships and joins ` * :ref:`Extensive library of SQL / Peewee examples ` * :ref:`Flask setup ` or :ref:`FastAPI setup ` Contents: --------- .. toctree:: :maxdepth: 2 peewee/installation peewee/quickstart peewee/example peewee/database peewee/models peewee/relationships peewee/querying peewee/writing peewee/query_operators peewee/transactions peewee/schema peewee/asyncio peewee/framework_integration peewee/interactive peewee/query_builder peewee/query_library peewee/api peewee/sqlite peewee/postgres peewee/mysql peewee/db_tools peewee/orm_utils peewee/recipes peewee/contributing Note ---- If you find any bugs, odd behavior, or have an idea for a new feature please don't hesitate to `open an issue `_ on GitHub. Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` ================================================ FILE: docs/make.bat ================================================ @ECHO OFF REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set BUILDDIR=_build set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . if NOT "%PAPER%" == "" ( set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% ) if "%1" == "" goto help if "%1" == "help" ( :help echo.Please use `make ^` where ^ is one of echo. html to make standalone HTML files echo. dirhtml to make HTML files named index.html in directories echo. singlehtml to make a single large HTML file echo. pickle to make pickle files echo. json to make JSON files echo. htmlhelp to make HTML files and a HTML help project echo. qthelp to make HTML files and a qthelp project echo. devhelp to make HTML files and a Devhelp project echo. epub to make an epub echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter echo. text to make text files echo. man to make manual pages echo. changes to make an overview over all changed/added/deprecated items echo. linkcheck to check all external links for integrity echo. doctest to run all doctests embedded in the documentation if enabled goto end ) if "%1" == "clean" ( for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i del /q /s %BUILDDIR%\* goto end ) if "%1" == "html" ( %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html echo. echo.Build finished. The HTML pages are in %BUILDDIR%/html. goto end ) if "%1" == "dirhtml" ( %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml echo. echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. goto end ) if "%1" == "singlehtml" ( %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml echo. echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. goto end ) if "%1" == "pickle" ( %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle echo. echo.Build finished; now you can process the pickle files. goto end ) if "%1" == "json" ( %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json echo. echo.Build finished; now you can process the JSON files. goto end ) if "%1" == "htmlhelp" ( %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp echo. echo.Build finished; now you can run HTML Help Workshop with the ^ .hhp project file in %BUILDDIR%/htmlhelp. goto end ) if "%1" == "qthelp" ( %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp echo. echo.Build finished; now you can run "qcollectiongenerator" with the ^ .qhcp project file in %BUILDDIR%/qthelp, like this: echo.^> qcollectiongenerator %BUILDDIR%\qthelp\peewee.qhcp echo.To view the help file: echo.^> assistant -collectionFile %BUILDDIR%\qthelp\peewee.ghc goto end ) if "%1" == "devhelp" ( %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp echo. echo.Build finished. goto end ) if "%1" == "epub" ( %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub echo. echo.Build finished. The epub file is in %BUILDDIR%/epub. goto end ) if "%1" == "latex" ( %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex echo. echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. goto end ) if "%1" == "text" ( %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text echo. echo.Build finished. The text files are in %BUILDDIR%/text. goto end ) if "%1" == "man" ( %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man echo. echo.Build finished. The manual pages are in %BUILDDIR%/man. goto end ) if "%1" == "changes" ( %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes echo. echo.The overview file is in %BUILDDIR%/changes. goto end ) if "%1" == "linkcheck" ( %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck echo. echo.Link check complete; look for any errors in the above output ^ or in %BUILDDIR%/linkcheck/output.txt. goto end ) if "%1" == "doctest" ( %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest echo. echo.Testing of doctests in the sources finished, look at the ^ results in %BUILDDIR%/doctest/output.txt. goto end ) :end ================================================ FILE: docs/peewee/api.rst ================================================ .. _api: API Reference ============= This document specifies Peewee's APIs. .. contents:: On this page :local: :depth: 1 Database -------- .. class:: Database(database, thread_safe=True, field_types=None, operations=None, autoconnect=True, **kwargs) :param str database: Database name or filename for SQLite (or ``None`` to :ref:`defer initialization `, in which case you must call :meth:`Database.init`, specifying the database name). :param bool thread_safe: Whether to store connection state in a thread-local. :param dict field_types: A mapping of additional field types to support. :param dict operations: A mapping of additional operations to support. :param bool autoconnect: Automatically connect to database if attempting to execute a query on a closed database. :param kwargs: Arbitrary keyword arguments that will be passed to the database driver when a connection is created, for example ``password``, ``host``, etc. The :class:`Database` is responsible for: * Executing queries * Managing connections * Transactions * Introspection The database can be instantiated with ``None`` as the database name if the database is not known until run-time. In this way you can create a database instance and then configure it elsewhere when the settings are known. This is called :ref:`deferred initialization `. Examples: .. code-block:: python # Sqlite database using WAL-mode and 64MB page-cache. db = SqliteDatabase('app.db', pragmas={ 'journal_mode': 'wal', 'cache_size': -64000}) # Postgresql database on remote host. db = PostgresqlDatabase( 'my_app', user='postgres', host='10.8.0.3', password='secret') Deferred initialization example: .. code-block:: python db = PostgresqlDatabase(None) class BaseModel(Model): class Meta: database = db # Read database connection info from env, for example: db_name = os.environ['DATABASE'] db_host = os.environ['PGHOST'] # Initialize database. db.init(db_name, host=db_host, user='postgres') .. attribute:: param = '?' String used as parameter placeholder in SQL queries. .. attribute:: quote = '""' Type of quotation-mark(s) to use to denote entities such as tables or columns, specified as ``''``. .. method:: init(database, **kwargs) :param str database: Database name or filename for SQLite. :param kwargs: Arbitrary keyword arguments that will be passed to the database driver when a connection is created, for example ``password``, ``host``, etc. Initialize a *deferred* database. See :ref:`initializing-database` for more info. .. method:: connect(reuse_if_open=False) :param bool reuse_if_open: Do not raise an exception if a connection is already opened. :return: whether a new connection was opened. :rtype: bool :raises: ``OperationalError`` if connection already open and ``reuse_if_open`` is not set to ``True``. Open a connection to the database. .. code-block:: python db.connect() # Or: db.connect(reuse_if_open=True) .. method:: close() :return: Whether the connection was closed. If the database was already closed, this returns ``False``. :rtype: bool Close the connection to the database. .. code-block:: python if not db.is_closed(): db.close() .. method:: is_closed() :return: return ``True`` if database is closed, ``False`` if open. :rtype: bool .. method:: connection() Return the DB-API driver connection. If a connection is not open, one will be opened. .. code-block:: python db = SqliteDatabase(':memory:') # Get the sqlite3.Connection() instance. conn = db.connection() .. method:: __enter__() .. method:: __exit__(exc_type, exc_val, exc_tb) .. method:: __call__() The database object can be used as a context manager or decorator. 1. Connection opens when context manager / decorated function is entered. 2. Peewee begins a transaction. 3. Control is passed to user for duration of block. 4. Peewee commits transaction if block exits cleanly, otherwise issues a rollback. 5. Peewee closes the connection. 6. Any unhandled exception is raised. .. code-block:: python with db: User.create(username='charlie') # Transaction is committed when the block exits normally, # rolled back if an exception is raised. Decorator: .. code-block:: python @db def demo(): print('closed?', db.is_closed()) demo() # "closed? False" db.is_closed() # True .. method:: connection_context() Create a context-manager or decorator that will hold open a connection for the duration of the wrapped block. Example: .. code-block:: python with db.connection_context(): # Connection is open; no implicit transaction. results = User.select() Decorator: .. code-block:: python @db.connection_context() def load_fixtures(): db.create_tables([User, Tweet]) import_data() .. method:: cursor(named_cursor=None) :param named_cursor: Reserved for internal use by Postgres extension. Return a DB-API ``cursor`` object on the current connection. If a connection is not open, one will be opened. .. method:: execute_sql(sql, params=None) :param str sql: SQL string to execute. :param tuple params: Parameters for query. :return: cursor object. Execute a SQL query and return a cursor over the results. .. code-block:: python db = SqliteDatabase('my_app.db') db.connect() # Example of executing a simple query and ignoring the results. db.execute_sql("ATTACH DATABASE ':memory:' AS cache;") # Example of iterating over the results of a query using the cursor. cursor = db.execute_sql('SELECT * FROM users WHERE status = ?', (ACTIVE,)) for row in cursor.fetchall(): # Do something with row, which is a tuple containing column data. pass .. method:: execute(query, **context_options) :param query: A :class:`Query` instance. :param context_options: Arbitrary options to pass to the SQL generator. :return: cursor object. Execute a SQL query by compiling a ``Query`` instance and executing the resulting SQL. .. code-block:: python query = User.insert({'username': 'Huey'}) db.execute(query) # Equivalent to query.execute() .. method:: last_insert_id(cursor, query_type=None) :param cursor: cursor object. :return: primary key of last-inserted row. .. method:: rows_affected(cursor) :param cursor: cursor object. :return: number of rows modified by query. .. method:: atomic(...) Create a context-manager or decorator which wraps a block of code in a transaction (or savepoint). Calls to :meth:`~Database.atomic` can be nested. Database-specific parameters: :class:`PostgresqlDatabase` and :class:`MySQLDatabase` accept an ``isolation_level`` parameter. :class:`SqliteDatabase` accepts a ``lock_type`` parameter. Refer to :ref:`sqlite-locking` and :ref:`postgres-isolation` for discussion. :param str isolation_level: Isolation strategy: READ UNCOMMITTED, READ COMMITTED, REPEATABLE READ, SERIALIZABLE :param str lock_type: Locking strategy: DEFERRED, IMMEDIATE, EXCLUSIVE. Example code:: with db.atomic() as txn: user = User.create(username='charlie') with db.atomic(): tweet = Tweet.create(user=user, content='Hello') # Both rows are committed when block exits normally. As a decorator: .. code-block:: python @db.atomic() def create_user_with_tweet(username, content): user = User.create(username=username) Tweet.create(user=user, content=content) return user Transactions (and save-points) can be committed or rolled-back within the wrapped block. If this occurs, a new transaction or savepoint is begun: Example: .. code-block:: python with db.atomic() as txn: User.create(username='mickey') txn.commit() # Changes are saved and a new transaction begins. User.create(username='huey') txn.rollback() # "huey" will not be saved. User.create(username='zaizee') # Print the usernames of all users. print([u.username for u in User.select()]) # Prints ["mickey", "zaizee"] If an unhandled exception occurs in the block, the block is rolled-back and the exception propagates. .. method:: transaction(...) Create a context-manager or decorator that runs all queries in the wrapped block in a transaction. Database-specific parameters: :class:`PostgresqlDatabase` and :class:`MySQLDatabase` accept an ``isolation_level`` parameter. :class:`SqliteDatabase` accepts a ``lock_type`` parameter. Refer to :ref:`sqlite-locking` and :ref:`postgres-isolation` for discussion. :param str isolation_level: Isolation strategy: READ UNCOMMITTED, READ COMMITTED, REPEATABLE READ, SERIALIZABLE :param str lock_type: Locking strategy: DEFERRED, IMMEDIATE, EXCLUSIVE. .. code-block:: python with db.transaction() as txn: User.create(username='mickey') txn.commit() # Commit now; a new transaction begins. User.create(username='huey') txn.rollback() # Roll back huey; a new transaction begins. User.create(username='zaizee') # zaizee is committed when the block exits. Transactions can be committed or rolled-back within the wrapped block. If this occurs, a new transaction is begun. .. warning:: If you attempt to nest transactions with peewee using the :meth:`~Database.transaction` context manager, only the outer-most transaction will be used. As this may lead to unpredictable behavior, it is recommended that you use :meth:`~Database.atomic`. .. method:: savepoint() Create a context-manager or decorator that runs all queries in the wrapped block in a savepoint. Savepoints can be nested arbitrarily, but must occur within a transaction. .. code-block:: python with db.transaction() as txn: with db.savepoint() as sp: User.create(username='mickey') with db.savepoint() as sp2: User.create(username='zaizee') sp2.rollback() # "zaizee" is not saved. User.create(username='huey') # mickey and huey were created. Savepoints can be committed or rolled-back within the wrapped block. If this occurs, a new savepoint is begun. .. method:: manual_commit() Create a context-manager or decorator which disables Peewee's transaction management for the wrapped block. Example: .. code-block:: python with db.manual_commit(): db.begin() # Begin transaction explicitly. try: user.delete_instance(recursive=True) except: db.rollback() # Rollback -- an error occurred. raise else: try: db.commit() # Attempt to commit changes. except: db.rollback() # Error committing, rollback. raise The above code is equivalent to the following: .. code-block:: python with db.atomic(): user.delete_instance(recursive=True) .. method:: session_start() Begin a new transaction (without using a context-manager or decorator). This method is useful if you intend to execute a sequence of operations inside a transaction, but using a decorator or context-manager would not be appropriate. It is strongly advised that you use the :meth:`Database.atomic` method whenever possible for managing transactions/savepoints. The ``atomic`` method correctly manages nesting, uses the appropriate construction (e.g., transaction-vs-savepoint), and always cleans up after itself. The :meth:`~Database.session_start` method should only be used if the sequence of operations does not easily lend itself to wrapping using either a context-manager or decorator. You must *always* call either :meth:`~Database.session_commit` or :meth:`~Database.session_rollback` after calling the ``session_start`` method. .. method:: session_commit() Commit any changes made during a transaction begun with :meth:`~Database.session_start`. .. method:: session_rollback() Roll back any changes made during a transaction begun with :meth:`~Database.session_start`. .. method:: begin() Begin a transaction when using manual-commit mode. This method should only be used in conjunction with the :meth:`~Database.manual_commit` context manager. .. method:: commit() Manually commit the currently-active transaction. This method should only be used in conjunction with the :meth:`~Database.manual_commit` context manager. .. method:: rollback() Manually roll-back the currently-active transaction. This method should only be used in conjunction with the :meth:`~Database.manual_commit` context manager. .. method:: in_transaction() :return: whether or not a transaction is currently open. :rtype: bool .. code-block:: python with db.atomic() as tx: assert db.in_transaction() assert not db.in_transaction() # No longer in transaction. .. method:: batch_commit(it, n) :param iterable it: an iterable whose items will be yielded. :param int n: commit every *n* items. :return: an equivalent iterable to the one provided, with the addition that groups of *n* items will be yielded in a transaction. Simplify batching large operations, such as inserts, updates, etc. Pass in an iterable and the number of items-per-batch, and the items will be returned by an equivalent iterator that wraps each batch in a transaction. Example: .. code-block:: python # Some list or iterable containing data to insert. row_data = [{'username': 'u1'}, {'username': 'u2'}, ...] # Insert all data, committing every 100 rows. If, for example, # there are 789 items in the list, then there will be a total of # 8 transactions (7x100 and 1x89). for row in db.batch_commit(row_data, 100): user = User.create(**row) # Now let's suppose we need to do something w/the user. user.call_method() A more efficient option is to batch the data into a multi-value ``INSERT`` statement (for example, using :meth:`Model.insert_many`). Use this approach instead wherever possible: .. code-block:: python with db.atomic(): for idx in range(0, len(row_data), 100): # Insert 100 rows at a time. rows = row_data[idx:idx + 100] User.insert_many(rows).execute() .. method:: table_exists(table, schema=None) :param str table: Table name. :param str schema: Schema name (optional). :return: ``bool`` indicating whether table exists. .. method:: get_tables(schema=None) :param str schema: Schema name (optional). :return: a list of table names in the database. .. method:: get_indexes(table, schema=None) :param str table: Table name. :param str schema: Schema name (optional). Return a list of :class:`IndexMetadata` tuples. Example: .. code-block:: python print(db.get_indexes('entry')) [IndexMetadata( name='entry_public_list', sql='CREATE INDEX "entry_public_list" ...', columns=['timestamp'], unique=False, table='entry'), IndexMetadata( name='entry_slug', sql='CREATE UNIQUE INDEX "entry_slug" ON "entry" ("slug")', columns=['slug'], unique=True, table='entry')] .. method:: get_columns(table, schema=None) :param str table: Table name. :param str schema: Schema name (optional). Return a list of :class:`ColumnMetadata` tuples. Example: .. code-block:: python print(db.get_columns('entry')) [ColumnMetadata( name='id', data_type='INTEGER', null=False, primary_key=True, table='entry', default=None), ColumnMetadata( name='title', data_type='TEXT', null=False, primary_key=False, table='entry', default=None), ...] .. method:: get_primary_keys(table, schema=None) :param str table: Table name. :param str schema: Schema name (optional). Return a list of column names that comprise the primary key. Example: .. code-block:: python print(db.get_primary_keys('entry')) ['id'] .. method:: get_foreign_keys(table, schema=None) :param str table: Table name. :param str schema: Schema name (optional). Return a list of :class:`ForeignKeyMetadata` tuples for keys present on the table. Example: .. code-block:: python print(db.get_foreign_keys('entrytag')) [ForeignKeyMetadata( column='entry_id', dest_table='entry', dest_column='id', table='entrytag'), ...] .. method:: get_views(schema=None) :param str schema: Schema name (optional). Return a list of :class:`ViewMetadata` tuples for VIEWs present in the database. Example: .. code-block:: python print(db.get_views()) [ViewMetadata( name='entries_public', sql='CREATE VIEW entries_public AS SELECT ... '), ...] .. method:: sequence_exists(seq) :param str seq: Name of sequence. :return: Whether sequence exists. :rtype: bool .. code-block:: python if db.sequence_exists('user_id_seq'): print('Sequence found.') .. method:: create_tables(models, **options) :param list models: A list of :class:`Model` classes. :param options: Options to specify when calling :meth:`Model.create_table`. Create tables, indexes and associated constraints for the given list of models. Dependencies are resolved so that tables are created in the appropriate order. .. method:: drop_tables(models, **options) :param list models: A list of :class:`Model` classes. :param kwargs: Options to specify when calling :meth:`Model.drop_table`. Drop tables, indexes and constraints for the given list of models. Dependencies are resolved so that tables are dropped in the appropriate order. .. method:: bind(models, bind_refs=True, bind_backrefs=True) :param list models: One or more :class:`Model` classes to bind. :param bool bind_refs: Bind related models. :param bool bind_backrefs: Bind back-reference related models. Bind the given list of models, and specified relations, to the database. .. code-block:: python def setup_tests(): # Bind models to an in-memory SQLite database. test_db = SqliteDatabase(':memory:') test_db.bind([User, Tweet]) .. method:: bind_ctx(models, bind_refs=True, bind_backrefs=True) :param list models: List of models to bind to the database. :param bool bind_refs: Bind models that are referenced using foreign-keys. :param bool bind_backrefs: Bind models that reference the given model with a foreign-key. Create a context-manager that binds (associates) the given models with the current database for the duration of the wrapped block. Example: .. code-block:: python MODELS = [User, Tweet, Favorite] # Bind the given models to the db for the duration of wrapped block. def use_test_database(fn): @wraps(fn) def inner(self): with test_db.bind_ctx(MODELS): test_db.create_tables(MODELS) try: fn(self) finally: test_db.drop_tables(MODELS) return inner class TestSomething(TestCase): @use_test_database def test_something(self): # ... models are bound to test database ... pass .. method:: extract_date(date_part, date_field) :param str date_part: date part to extract, e.g. 'year'. :param Node date_field: a SQL node containing a date/time, for example a :class:`DateTimeField`. :return: a SQL node representing a function call that will return the provided date part. Provides a compatible interface for extracting a portion of a datetime. Example: .. code-block:: python query = (Tweet.select() .where(db.extract_date('year', Tweet.timestamp) == 2026)) # If Tweet.timestamp is a DateField or DateTimeField we could # also write: query = Tweet.select().where(Tweet.timestamp.year == 2026) .. method:: truncate_date(date_part, date_field) :param str date_part: date part to truncate to, e.g. 'day'. :param Node date_field: a SQL node containing a date/time, for example a :class:`DateTimeField`. :return: a SQL node representing a function call that will return the truncated date part. Provides a compatible interface for truncating a datetime to the given resolution. Example: .. code-block:: python # Report on how many tweets made in each month. query = (Tweet .select( db.truncate_date('month', Tweet.timestamp).alias('month'), fn.COUNT(Tweet.id).alias('count')) .group_by(db.truncate_date('month', Tweet.timestamp))) for row in query: print(row.month, '->', row.count) .. method:: random() :return: a SQL node representing a function call that returns a random value. A compatible interface for calling the appropriate random number generation function provided by the database. For Postgres and Sqlite, this is equivalent to ``fn.random()``, for MySQL ``fn.rand()``. .. class:: SqliteDatabase(database, pragmas=None, regexp_function=False, rank_functions=False, timeout=5, returning_clause=None, **kwargs) :param pragmas: Either a dictionary or a list of 2-tuples containing pragma key and value to set every time a connection is opened. :param bool regexp_function: Make the REGEXP function available. :param bool rank_functions: Make the full-text search ranking functions available (recommended only if using FTS4). :param timeout: Set the busy-timeout on the SQLite driver (in seconds). :param bool returning_clause: Use `RETURNING` clause automatically for bulk INSERT queries (requires Sqlite 3.35 or newer). Sqlite database implementation. :class:`SqliteDatabase` that provides some advanced features only offered by Sqlite. * Register user-defined functions, aggregates, window functions, collations * Load extension modules distributed as shared libraries * Advanced transactions (specify lock type) * For additional features see :class:`CySqliteDatabase`. Example of initializing a database and configuring some PRAGMAs: .. code-block:: python db = SqliteDatabase('my_app.db', pragmas=( ('cache_size', -16000), # 16MB ('journal_mode', 'wal'), # Use write-ahead-log journal mode. )) # Alternatively, pragmas can be specified using a dictionary. db = SqliteDatabase('my_app.db', pragmas={'journal_mode': 'wal'}) .. method:: pragma(key, value=SENTINEL, permanent=False) :param key: Setting name. :param value: New value for the setting (optional). :param permanent: Apply this pragma whenever a connection is opened. Execute a PRAGMA query once on the active connection. If a value is not specified, then the current value will be returned. If ``permanent`` is specified, then the PRAGMA query will also be executed whenever a new connection is opened, ensuring it is always in-effect. .. attribute:: cache_size Get or set the cache_size pragma for the current connection. .. attribute:: foreign_keys Get or set the foreign_keys pragma for the current connection. .. attribute:: journal_mode Get or set the journal_mode pragma. .. attribute:: journal_size_limit Get or set the journal_size_limit pragma. .. attribute:: mmap_size Get or set the mmap_size pragma for the current connection. .. attribute:: page_size Get or set the page_size pragma. .. attribute:: read_uncommitted Get or set the read_uncommitted pragma for the current connection. .. attribute:: synchronous Get or set the synchronous pragma for the current connection. .. attribute:: wal_autocheckpoint Get or set the wal_autocheckpoint pragma for the current connection. .. attribute:: timeout Get or set the busy timeout (seconds). .. method:: func(name=None, num_params=-1, deterministic=None) :param str name: Name of the function (defaults to function name). :param int num_params: Number of parameters the function accepts, or -1 for any number. :param bool deterministic: Whether the function is deterministic for a given input (this is required to use the function in an index). Requires Sqlite 3.20 or newer, and ``sqlite3`` driver support (added to stdlib in Python 3.8). Decorator to register a user-defined scalar function. Example: .. code-block:: python @db.func('title_case') def title_case(s): return s.title() if s else '' Book.select(fn.title_case(Book.title)) .. method:: register_function(fn, name=None, num_params=-1, deterministic=None) :param fn: The user-defined scalar function. :param str name: Name of function (defaults to function name) :param int num_params: Number of arguments the function accepts, or -1 for any number. :param bool deterministic: Whether the function is deterministic for a given input (this is required to use the function in an index). Requires Sqlite 3.20 or newer, and ``sqlite3`` driver support (added to stdlib in Python 3.8). Register a user-defined scalar function. The function will be registered each time a new connection is opened. Additionally, if a connection is already open, the function will be registered with the open connection. .. method:: aggregate(name=None, num_params=-1) :param str name: Name of the aggregate (defaults to class name). :param int num_params: Number of parameters the aggregate accepts, or -1 for any number. Class decorator to register a user-defined aggregate function. Example: .. code-block:: python @db.aggregate('md5') class MD5(object): def initialize(self): self.md5 = hashlib.md5() def step(self, value): self.md5.update(value) def finalize(self): return self.md5.hexdigest() @db.aggregate() class Product(object): '''Like SUM() except calculates cumulative product.''' def __init__(self): self.product = 1 def step(self, value): self.product *= value def finalize(self): return self.product .. method:: register_aggregate(klass, name=None, num_params=-1) :param klass: Class implementing aggregate API. :param str name: Aggregate function name (defaults to name of class). :param int num_params: Number of parameters the aggregate accepts, or -1 for any number. Register a user-defined aggregate function. The function will be registered each time a new connection is opened. Additionally, if a connection is already open, the aggregate will be registered with the open connection. .. method:: window_function(name=None, num_params=-1) :param str name: Name of the window function (defaults to class name). :param int num_params: Number of parameters the function accepts, or -1 for any number. Class decorator to register a user-defined window function. Window functions must define the following methods: * ``step()`` - receive values from a row and update state. * ``inverse()`` - inverse of ``step()`` for the given values. * ``value()`` - return the current value of the window function. * ``finalize()`` - return the final value of the window function. Example: .. code-block:: python @db.window_function('my_sum') class MySum(object): def __init__(self): self._value = 0 def step(self, value): self._value += value def inverse(self, value): self._value -= value def value(self): return self._value def finalize(self): return self._value .. method:: register_window_function(klass, name=None, num_params=-1) :param klass: Class implementing window function API. :param str name: Window function name (defaults to name of class). :param int num_params: Number of parameters the function accepts, or -1 for any number. Register a user-defined window function, requires SQLite >= 3.25.0. The window function will be registered each time a new connection is opened. Additionally, if a connection is already open, the window function will be registered with the open connection. .. method:: collation(name=None) :param str name: Name of collation (defaults to function name) Decorator to register a user-defined collation. Example: .. code-block:: python @db.collation('reverse') def collate_reverse(s1, s2): return -cmp(s1, s2) # Usage: Book.select().order_by(collate_reverse.collation(Book.title)) # Equivalent: Book.select().order_by(Book.title.asc(collation='reverse')) As you might have noticed, the original ``collate_reverse`` function has a special attribute called ``collation`` attached to it. This extra attribute provides a shorthand way to generate the SQL necessary to use our custom collation. .. method:: register_collation(fn, name=None) :param fn: The collation function. :param str name: Name of collation (defaults to function name) Register a user-defined collation. The collation will be registered each time a new connection is opened. Additionally, if a connection is already open, the collation will be registered with the open connection. .. method:: unregister_function(name) :param name: Name of the user-defined scalar function. Unregister the user-defined scalar function. .. method:: unregister_aggregate(name) :param name: Name of the user-defined aggregate. Unregister the user-defined aggregate. .. method:: unregister_window_function(name) :param name: Name of the user-defined window function. Unregister the user-defined window function. .. method:: unregister_collation(name) :param name: Name of the user-defined collation. Unregister the user-defined collation. .. method:: load_extension(extension_module) Load the given extension shared library. Extension will be loaded for the current connection as well as all subsequent connections. .. code-block:: python db = SqliteDatabase('my_app.db') # Load extension in closure.so shared library. db.load_extension('closure') .. method:: unload_extension(extension_module) Unregister extension from being automatically loaded on new connections. .. method:: attach(filename, name) :param str filename: Database to attach (or ``:memory:`` for in-memory) :param str name: Schema name for attached database. :return: boolean indicating success Register another database file that will be attached to every database connection. If the main database is currently connected, the new database will be attached on the open connection. Databases that are attached using this method will be attached every time a database connection is opened. .. method:: detach(name) :param str name: Schema name for attached database. :return: boolean indicating success Unregister another database file that was attached previously with a call to ``attach()``. If the main database is currently connected, the attached database will be detached from the open connection. .. method:: atomic(lock_type=None) :param str lock_type: Locking strategy: DEFERRED, IMMEDIATE, EXCLUSIVE. Create an atomic context-manager / decorator, optionally using the specified locking strategy (default DEFERRED). Lock type only applies to the outermost ``atomic()`` block. .. seealso:: :ref:`sqlite-locking` .. method:: transaction(lock_type=None) :param str lock_type: Locking strategy: DEFERRED, IMMEDIATE, EXCLUSIVE. Create a transaction context-manager / decorator, optionally using the specified locking strategy (default DEFERRED). .. class:: PostgresqlDatabase(database, register_unicode=True, encoding=None, isolation_level=None, prefer_psycopg3=False) Postgresql database implementation. Uses psycopg2 or psycopg3. Additional optional keyword-parameters: :param bool register_unicode: Register unicode types. :param str encoding: Database encoding. :param isolation_level: Isolation level constant, defined in the ``psycopg2.extensions`` module or ``psycopg.IsolationLevel`` enum (psycopg3). Also accepts string which is converted to the matching constant. :type isolation_level: int, str :param bool prefer_psycopg3: If both psycopg2 and psycopg3 are installed, instruct Peewee to prefer psycopg3. Example: .. code-block:: python db = PostgresqlDatabase( 'app', user='postgres', host='10.8.0.1', port=5432, password=os.environ['PGPASSWORD'], isolation_level='SERIALIZABLE') .. method:: interval(val) :param str val: Time interval, e.g. ``'30 minutes'`` :return: expression representing the interval. .. method:: set_time_zone(timezone) :param str timezone: timezone name, e.g. "US/Central". :return: no return value. Set the timezone on the current connection. If no connection is open, then one will be opened. .. method:: set_isolation_level(isolation_level) :param isolation_level: Isolation level constant, defined in the ``psycopg2.extensions`` module or ``psycopg.IsolationLevel`` enum (psycopg3). Also accepts string which is converted to the matching constant. Set to ``None`` to use the server default. :type isolation_level: int, str Example of setting isolation level: .. code-block:: python # psycopg2 or psycopg3 db.set_isolation_level('SERIALIZABLE') # psycopg2 from psycopg2.extensions import ISOLATION_LEVEL_SERIALIZABLE db.set_isolation_level(ISOLATION_LEVEL_SERIALIZABLE) # psycopg3 from psycopg import IsolationLevel db.set_isolation_level(IsolationLevel.SERIALIZABLE) Isolation level values in order of increasing strictness: * READ UNCOMMITTED * READ COMMITTED * REPEATABLE READ * SERIALIZABLE See the `Postgresql transaction isolation docs `__ and Peewee's :ref:`postgres-isolation` for additional discussion. .. method:: atomic(isolation_level=None) :param isolation_level: Isolation strategy: SERIALIZABLE, READ COMMITTED, REPEATABLE READ, READ UNCOMMITTED :type isolation_level: int, str Create an atomic context-manager, optionally using the specified isolation level (if unspecified, the connection default will be used). Isolation level only applies to the outermost ``atomic()`` block. See the `Postgresql transaction isolation docs `__ and Peewee's :ref:`postgres-isolation` for additional discussion. .. method:: transaction(isolation_level=None) :param isolation_level: Isolation strategy: SERIALIZABLE, READ COMMITTED, REPEATABLE READ, READ UNCOMMITTED :type isolation_level: int, str Create a transaction context-manager, optionally using the specified isolation level (if unspecified, the connection default will be used). .. class:: MySQLDatabase(database, **kwargs) MySQL database implementation. Example: .. code-block:: python db = MySQLDatabase('app', host='10.8.0.1') .. method:: atomic(isolation_level=None) :param str isolation_level: Isolation strategy: SERIALIZABLE, READ COMMITTED, REPEATABLE READ, READ UNCOMMITTED Create an atomic context-manager, optionally using the specified isolation level (if unspecified, the server default will be used). Isolation level only applies to the outermost ``atomic()`` block. .. method:: transaction(isolation_level=None) :param str isolation_level: Isolation strategy: SERIALIZABLE, READ COMMITTED, REPEATABLE READ, READ UNCOMMITTED Create a transaction context-manager, optionally using the specified isolation level (if unspecified, the server default will be used). .. _model-api: Model ----- .. class:: Model(**kwargs) :param kwargs: Mapping of field-name to value to initialize model with. Model class provides a high-level abstraction for working with database tables. Models are a one-to-one mapping with a database table (or a table-like object, such as a view). Subclasses of ``Model`` declare any number of :class:`Field` instances as class attributes. These fields correspond to columns on the table. Table-level operations, such as :meth:`~Model.select`, :meth:`~Model.update`, :meth:`~Model.insert` and :meth:`~Model.delete` are implemented as classmethods. Row-level operations, such as :meth:`~Model.save` and :meth:`~Model.delete_instance` are implemented as instancemethods. Example: .. code-block:: python db = SqliteDatabase(':memory:') class User(Model): username = TextField() join_date = DateTimeField(default=datetime.datetime.now) is_admin = BooleanField(default=False) admin = User(username='admin', is_admin=True) admin.save() .. classmethod:: alias([alias=None]) :param str alias: Optional name for alias. :return: :class:`ModelAlias` instance. Create an alias to the model-class. Model aliases allow you to reference the same :class:`Model` multiple times in a query, for example when doing a self-join or sub-query. Example: .. code-block:: python Parent = Category.alias() sq = (Category .select(Category, Parent) .join(Parent, on=(Category.parent == Parent.id)) .where(Parent.name == 'parent category')) .. classmethod:: select(*fields) :param fields: A list of model classes, field instances, functions or expressions. If no arguments are provided, all columns for the given model will be selected by default. :return: :class:`ModelSelect` query. Create a SELECT query. If no fields are explicitly provided, the query will by default SELECT all the fields defined on the model, unless you are using the query as a sub-query, in which case only the primary key will be selected by default. Example of selecting all columns: .. code-block:: python query = User.select().where(User.active == True).order_by(User.username) Example of selecting all columns on *Tweet* and the parent model, *User*. When the ``user`` foreign key is accessed on a *Tweet* instance no additional query will be needed (see :ref:`N+1 ` for more details): .. code-block:: python query = (Tweet .select(Tweet, User) .join(User) .order_by(Tweet.created_date.desc())) for tweet in query: print(tweet.user.username, '->', tweet.content) Example of subquery only selecting the primary key: .. code-block:: python inactive_users = User.select().where(User.active == False) # Here, instead of defaulting to all columns, Peewee will default # to only selecting the primary key. Tweet.delete().where(Tweet.user.in_(inactive_users)).execute() See :ref:`querying` for in-depth discussion. .. classmethod:: update(__data=None, **update) :param dict __data: ``dict`` of fields to values. :param update: Field-name to value mapping. Create an UPDATE query. Example showing users being marked inactive if their registration has expired: .. code-block:: python q = (User .update(active=False) .where(User.registration_expired == True)) q.execute() # Execute the query, returning number of rows updated. Example showing an atomic update: .. code-block:: python q = (PageView .update({PageView.count: PageView.count + 1}) .where(PageView.url == url)) q.execute() # Execute the query. Update queries support :meth:`~WriteQuery.returning` with Postgresql and SQLite to obtain the updated rows: .. code-block:: python query = (User .update(spam=True) .where(User.username.contains('billing')) .returning(User)) for user in query: print(f'Marked {user.username} as spam') See :ref:`updating-records` for additional discussion. When an update query is executed, the number of rows modified will be returned. .. classmethod:: insert(__data=None, **insert) :param dict __data: ``dict`` of fields to values to insert. :param insert: Field-name to value mapping. Create an INSERT query. Insert a new row into the database. If any fields on the model have default values, these values will be used if the fields are not explicitly set in the ``insert`` dictionary. Example showing creation of a new user: .. code-block:: python q = User.insert(username='admin', active=True, registration_expired=False) q.execute() # perform the insert. You can also use :class:`Field` objects as the keys: .. code-block:: python new_id = User.insert({User.username: 'admin'}).execute() If you have a model with a default value on one of the fields, and that field is not specified in the ``insert`` parameter, the default will be used: .. code-block:: python class User(Model): username = CharField() active = BooleanField(default=True) # This INSERT query will automatically specify `active=True`: User.insert(username='charlie') Insert queries support :meth:`~WriteQuery.returning` with Postgresql and SQLite to obtain the inserted rows: .. code-block:: python alice, = (User .insert(username='alice') .returning(User) .execute()) print(f'Added {alice.username} with id = {alice.id}') When an insert query is executed on a table with an auto-incrementing primary key, the primary key of the new row will be returned. .. classmethod:: insert_many(rows, fields=None) :param rows: An iterable that yields rows to insert. :param list fields: List of fields being inserted. :return: number of rows modified (see note). INSERT multiple rows of data. The ``rows`` parameter must be an iterable that yields dictionaries or tuples, where the ordering of the tuple values corresponds to the fields specified in the ``fields`` argument. As with :meth:`~Model.insert`, fields that are not specified in the dictionary will use their default value, if one exists. Due to the nature of bulk inserts, each row must contain the same fields. The following will not work: .. code-block:: python Person.insert_many([ {'first_name': 'Peewee', 'last_name': 'Herman'}, {'first_name': 'Huey'}, # Missing "last_name"! ]).execute() Example of inserting multiple Users: .. code-block:: python data = [ ('charlie', True), ('huey', False), ('zaizee', False)] query = User.insert_many(data, fields=[User.username, User.is_admin]) query.execute() Equivalent example using dictionaries: .. code-block:: python data = [ {'username': 'charlie', 'is_admin': True}, {'username': 'huey', 'is_admin': False}, {'username': 'zaizee', 'is_admin': False}] # Insert new rows. User.insert_many(data).execute() Because the ``rows`` parameter can be an arbitrary iterable, you can also use a generator: .. code-block:: python def get_usernames(): for username in ['charlie', 'huey', 'peewee']: yield {'username': username} User.insert_many(get_usernames()).execute() Insert queries support :meth:`~WriteQuery.returning` with Postgresql and SQLite to obtain the inserted rows: .. code-block:: python query = (User .insert_many([{'username': 'alice'}, {'username': 'bob'}]) .returning(User)) for user in query: print(f'Added {user.username} with id = {user.id}') See :ref:`bulk-inserts` for additional discussion. SQLite has a default limit of bound variables per statement. Additional discussion: :ref:`bulk-inserts`. SQLite documentation: * `Max variable number limit `_ * `SQLite compile-time flags `_ The default return value is the number of rows modified. However, when using Postgresql, Peewee will return a cursor that yields the primary-keys of the inserted rows. To disable this functionality with Postgresql, append ``as_rowcount()`` to your insert. .. classmethod:: insert_from(query, fields) :param Select query: SELECT query to use as source of data. :param fields: Fields to insert data into. :return: number of rows modified (see note). Generates an ``INSERT INTO ... SELECT`` query, copying rows from one table into another without round-tripping data through Python: .. code-block:: python (TweetArchive .insert_from( Tweet.select(Tweet.user, Tweet.message), fields=[TweetArchive.user, TweetArchive.message]) .execute()) See :ref:`bulk-inserts` for additional discussion. The default return value is the number of rows modified. However, when using Postgresql, Peewee will return a cursor that yields the primary-keys of the inserted rows. To disable this functionality with Postgresql, append ``as_rowcount()`` to your insert. .. classmethod:: replace(__data=None, **insert) :param dict __data: ``dict`` of fields to values to insert. :param insert: Field-name to value mapping. SQLite and MySQL support a ``REPLACE`` query, which will replace the row in the event of a conflict: .. code-block:: python class User(BaseModel): username = TextField(unique=True) last_login = DateTimeField(null=True) # Insert, or replace the entire existing row. User.replace(username='huey', last_login=datetime.datetime.now()).execute() # Equivalent using insert(): (User .insert(username='huey', last_login=datetime.datetime.now()) .on_conflict_replace() .execute()) See :ref:`upsert` for additional discussion. ``replace`` deletes and re-inserts, which changes the primary key. Use :meth:`Insert.on_conflict` when the primary key must be preserved, or when only some columns should be updated. .. classmethod:: replace_many(rows, fields=None) :param rows: An iterable that yields rows to insert. :param list fields: List of fields being inserted. INSERT multiple rows of data using REPLACE for conflict-resolution. .. seealso:: * :meth:`Model.insert_many` for syntax and examples. * :ref:`upsert` for additional discussion. ``replace_many`` may delete and re-insert rows, which changes the primary key. Use :meth:`Insert.on_conflict` when the primary key must be preserved, or when only some columns should be updated. .. classmethod:: raw(sql, *params) :param str sql: SQL query to execute. :param params: Parameters for query. Execute a SQL query directly. Example selecting rows from the User table: .. code-block:: python q = User.raw('select id, username from users') for user in q: print(user.id, user.username) Generally the use of ``raw`` is reserved for those cases where you can significantly optimize a select query. .. classmethod:: delete() Create a DELETE query. Example showing the deletion of all inactive users: .. code-block:: python q = User.delete().where(User.active == False) q.execute() # Remove the rows, return number of rows removed. Delete queries support :meth:`~WriteQuery.returning` with Postgresql and SQLite to obtain the deleted rows: .. code-block:: python query = (User .delete() .where(User.username.contains('billing')) .returning(User)) for user in query: print(f'Deleted: {user.username}') .. seealso:: * :ref:`deleting-records` for discussion and example usage. * :meth:`Model.delete_instance` for deleting individual rows. .. classmethod:: create(**query) :param query: Mapping of field-name to value. INSERT new row into table and return corresponding model instance. Example showing the creation of a user (a row will be added to the database): .. code-block:: python user = User.create(username='admin', password='test') .. note:: ``create()`` is a shorthand for instantiate -> save. .. classmethod:: bulk_create(model_list, batch_size=None) :param iterable model_list: a list or other iterable of unsaved :class:`Model` instances. :param int batch_size: number of rows to batch per insert. If unspecified, all models will be inserted in a single query. :return: no return value. Efficiently INSERT multiple unsaved model instances into the database. Unlike :meth:`~Model.insert_many`, which accepts row data as a list of either dictionaries or lists, this method accepts a list of unsaved model instances. Example: .. code-block:: python user_list = [User(username='u%s' % i) for i in range(10)] with db.atomic(): # All 10 users are inserted in a single query. User.bulk_create(user_list) Batches: .. code-block:: python user_list = [User(username='u%s' % i) for i in range(10)] with database.atomic(): # Will execute 4 INSERT queries (3 batches of 3, 1 batch of 1). User.bulk_create(user_list, batch_size=3) * The primary-key value for the newly-created models will only be set if you are using Postgresql (which supports the ``RETURNING`` clause). * SQLite has a limit of bound parameters for a query, typically 999 for Sqlite < 3.32.0, and 32766 for newer versions. * **Strongly recommended** that you wrap the call in a transaction using :meth:`Database.atomic`. Otherwise an error in a batch mid-way through could leave the database in an inconsistent state. .. classmethod:: bulk_update(model_list, fields, batch_size=None) :param iterable model_list: a list or other iterable of :class:`Model` instances. :param list fields: list of fields to update. :param int batch_size: number of rows to batch per insert. If unspecified, all models will be inserted in a single query. :return: total number of rows updated. UPDATE multiple model instances in a single query by generating a ``CASE`` statement mapping ids to new field values. Example: .. code-block:: python # First, create 3 users. u1, u2, u3 = [User.create(username='u%s' % i) for i in (1, 2, 3)] # Now let's modify their usernames. u1.username = 'u1-x' u2.username = 'u2-y' u3.username = 'u3-z' # Update all three rows using a single UPDATE query. User.bulk_update([u1, u2, u3], fields=[User.username]) This will result in executing the following SQL: .. code-block:: sql UPDATE "users" SET "username" = CASE "users"."id" WHEN 1 THEN "u1-x" WHEN 2 THEN "u2-y" WHEN 3 THEN "u3-z" END WHERE "users"."id" IN (1, 2, 3); If you have a large number of objects to update, it is strongly recommended that you specify a ``batch_size`` and wrap the operation in a transaction: .. code-block:: python with database.atomic(): User.bulk_update(user_list, fields=['username'], batch_size=50) ``bulk_update`` may be slower than a direct UPDATE query when the list is very large, because the generated ``CASE`` expression grows proportionally. For updates that can be expressed as a single WHERE clause, the direct :meth:`~Model.update` approach is faster. .. classmethod:: get(*query, **filters) :param query: Zero or more :class:`Expression` objects. :param filters: Mapping of field-name to value for Django-style filter. :raises: :class:`DoesNotExist` :return: Model instance matching the specified filters. Retrieve a single model instance matching the given filters. If no model is returned, a :class:`DoesNotExist` is raised. .. code-block:: python user = User.get(User.username == username, User.active == True) This method is also exposed via the :class:`SelectQuery`, though it takes no parameters: .. code-block:: python active = User.select().where(User.active == True) try: user = (active .where( (User.username == username) & (User.active == True)) .get()) except User.DoesNotExist: user = None The :meth:`~Model.get` method is shorthand for selecting with a limit of 1. It has the added behavior of raising an exception when no matching row is found. If more than one row is found, the first row returned by the database cursor will be used. .. classmethod:: get_or_none(*query, **filters) Identical to :meth:`Model.get` but returns ``None`` if no model matches the given filters. .. code-block:: python active = User.select().where(User.active == True) user = (active .where( (User.username == username) & (User.active == True)) .get_or_none()) .. classmethod:: get_by_id(pk) :param pk: Primary-key value. Short-hand for calling :meth:`Model.get` specifying a lookup by primary key. Raises a :class:`DoesNotExist` if instance with the given primary key value does not exist. Example: .. code-block:: python user = User.get_by_id(1) # Returns user with id = 1. .. classmethod:: set_by_id(key, value) :param key: Primary-key value. :param dict value: Mapping of field to value to update. Short-hand for updating the data with the given primary-key. If no row exists with the given primary key, no exception will be raised. Example: .. code-block:: python # Set "is_admin" to True on user with id=3. User.set_by_id(3, {'is_admin': True}) .. classmethod:: delete_by_id(pk) :param pk: Primary-key value. Short-hand for deleting the row with the given primary-key. If no row exists with the given primary key, no exception will be raised. .. classmethod:: get_or_create(**kwargs) :param kwargs: Mapping of field-name to value. :param defaults: Default values to use if creating a new row. :return: Tuple of :class:`Model` instance and boolean indicating if a new object was created. Attempt to get the row matching the given filters. If no matching row is found, create a new row. .. warning:: Race-conditions are possible when using this method. Example **without** ``get_or_create``: .. code-block:: python # Without `get_or_create`, we might write: try: person = Person.get( (Person.first_name == 'John') & (Person.last_name == 'Lennon')) except Person.DoesNotExist: person = Person.create( first_name='John', last_name='Lennon', birthday=datetime.date(1940, 10, 9)) Equivalent code using ``get_or_create``: .. code-block:: python person, created = Person.get_or_create( first_name='John', last_name='Lennon', defaults={'birthday': datetime.date(1940, 10, 9)}) .. classmethod:: filter(*dq_nodes, **filters) :param dq_nodes: Zero or more :class:`DQ` objects. :param filters: Django-style filters. :return: :class:`ModelSelect` query. .. method:: get_id() :return: The primary-key of the model instance. .. method:: save(force_insert=False, only=None) :param bool force_insert: Force INSERT query. :param list only: Only save the given :class:`Field` instances. :return: Number of rows modified. Save the data in the model instance. By default, the presence of a primary-key value will cause an UPDATE query to be executed. Example showing saving a model instance: .. code-block:: python user = User() user.username = 'some-user' # does not touch the database user.save() # change is persisted to the db When a model uses any primary key OTHER than an auto-incrementing integer it is necessary to specify ``force_insert=True`` when calling ``save()`` with a new instance: .. code-block:: python class Tag(Model): tag = TextField(primary_key=True) t = Tag(tag='python') t.save(force_insert=True) # create() automatically specifies force_insert=True: t = Tag.create(tag='sqlite') .. attribute:: dirty_fields Return list of fields that have been modified. :rtype: list If you just want to persist modified fields, you can call ``model.save(only=model.dirty_fields)``. To **always** save a model's dirty fields, use the Meta option ``only_save_dirty = True``. Any calls to :meth:`Model.save()` will only save the dirty fields by default: .. code-block:: python class Person(Model): first_name = CharField() last_name = CharField() dob = DateField() class Meta: database = db only_save_dirty = True Peewee determines whether a field is "dirty" by observing when the field attribute is set on a model instance. If the field contains a value that is mutable, such as a dictionary instance, and that dictionary is then modified, Peewee will not notice the change. .. warning:: Do not do membership tests on this list, e.g. ``f in dirty_fields`` because if there is one or more fields in the dirty fields list, the field equality override will return a truthy Expression object. If you want to test if a field is dirty, instead check ``f.name in model.dirty_field_names``. .. attribute:: dirty_field_names Return list of field names that have been modified. :rtype: list .. method:: is_dirty() Return boolean indicating whether any fields were manually set. .. method:: delete_instance(recursive=False, delete_nullable=False) :param bool recursive: Delete related models. :param bool delete_nullable: Delete related models that have a null foreign key. If ``False`` nullable relations will be set to NULL. Delete the given instance. Any foreign keys set to cascade on delete will be deleted automatically. For more programmatic control, you can specify ``recursive=True``, which will delete any non-nullable related models (those that *are* nullable will be set to NULL). If you wish to delete all dependencies regardless of whether they are nullable, set ``delete_nullable=True``. Example: .. code-block:: python some_obj.delete_instance() See :ref:`deleting-records` for additional discussion. .. classmethod:: bind(database, bind_refs=True, bind_backrefs=True) :param Database database: database to bind to. :param bool bind_refs: Bind related models. :param bool bind_backrefs: Bind back-reference related models. Bind the model (and specified relations) to the given database. See also: :meth:`Database.bind`. .. classmethod:: bind_ctx(database, bind_refs=True, bind_backrefs=True) Like :meth:`~Model.bind`, but returns a context manager that only binds the models for the duration of the wrapped block. See also: :meth:`Database.bind_ctx`. .. classmethod:: table_exists() :return: boolean indicating whether the table exists. .. classmethod:: create_table(safe=True, **options) :param bool safe: When ``True``, the create table query will include an ``IF NOT EXISTS`` clause. Create the model table, indexes, constraints and sequences. Example: .. code-block:: python with database: SomeModel.create_table() .. classmethod:: drop_table(safe=True, **options) :param bool safe: If set to ``True``, the drop table query will include an ``IF EXISTS`` clause. Drop the model table. .. method:: truncate_table(restart_identity=False, cascade=False) :param bool restart_identity: Restart the id sequence (postgresql-only). :param bool cascade: Truncate related tables as well (postgresql-only). Truncate (delete all rows) for the model. .. classmethod:: index(*fields, unique=False, safe=True, where=None, using=None, name=None) :param fields: Fields to index. :param bool unique: Whether index is UNIQUE. :param bool safe: Whether to add IF NOT EXISTS clause. :param Expression where: Optional WHERE clause for index. :param str using: Index algorithm. :param str name: Optional index name. Expressive method for declaring an index on a model. Wraps the declaration of a :class:`ModelIndex` instance. Examples: .. code-block:: python class Article(Model): name = TextField() timestamp = TimestampField() status = IntegerField() flags = BitField() is_sticky = flags.flag(1) is_favorite = flags.flag(2) # CREATE INDEX ... ON "article" ("name", "timestamp" DESC) idx = Article.index(Article.name, Article.timestamp.desc()) # Be sure to add the index to the model: Article.add_index(idx) # CREATE UNIQUE INDEX ... ON "article" ("timestamp" DESC, "flags" & 2) # WHERE ("status" = 1) idx = (Article .index(Article.timestamp.desc(), Article.flags.bin_and(2), unique=True) .where(Article.status == 1)) # Add index to model: Article.add_index(idx) .. classmethod:: add_index(*args, **kwargs) :param args: a :class:`ModelIndex` instance, Field(s) to index, or a :class:`SQL` instance that contains the SQL for creating the index. :param kwargs: Keyword arguments passed to :class:`ModelIndex` constructor. Add an index to the model's definition. This method does not actually create the index in the database. Rather, it adds the index definition to the model's metadata, so that a subsequent call to :meth:`~Model.create_table` will create the new index (along with the table). Examples: .. code-block:: python class Article(Model): name = TextField() timestamp = TimestampField() status = IntegerField() flags = BitField() is_sticky = flags.flag(1) is_favorite = flags.flag(2) # CREATE INDEX ... ON "article" ("name", "timestamp") WHERE "status" = 1 idx = Article.index(Article.name, Article.timestamp).where(Article.status == 1) Article.add_index(idx) # CREATE UNIQUE INDEX ... ON "article" ("timestamp" DESC, "flags" & 2) ts_flags_idx = Article.index( Article.timestamp.desc(), Article.flags.bin_and(2), unique=True) Article.add_index(ts_flags_idx) # You can also specify a list of fields and use the same keyword # arguments that the ModelIndex constructor accepts: Article.add_index( Article.name, Article.timestamp.desc(), where=(Article.status == 1)) # Or even specify a SQL query directly: Article.add_index(SQL('CREATE INDEX ...')) .. method:: dependencies(search_nullable=False) :param bool search_nullable: Search models related via a nullable foreign key :rtype: Generator expression yielding queries and foreign key fields. Generate a list of queries of dependent models. Yields a 2-tuple containing the query and corresponding foreign key field. Useful for searching dependencies of a model, i.e. things that would be orphaned in the event of a delete. .. method:: __iter__() :return: a :class:`ModelSelect` for the given class. Convenience function for iterating over all instances of a model. Example: .. code-block:: python Setting.insert_many([ {'key': 'host', 'value': '192.168.1.2'}, {'key': 'port', 'value': '1337'}, {'key': 'user', 'value': 'nuggie'}]).execute() # Load settings from db into dict. settings = {setting.key: setting.value for setting in Setting} .. method:: __len__() :return: Count of rows in table. Example: .. code-block:: python n_accounts = len(Account) # Equivalent: n_accounts = Account.select().count() .. class:: ModelAlias(model, alias=None) :param Model model: Model class to reference. :param str alias: (optional) name for alias. Provide a separate reference to a model in a query. With Peewee, we use :meth:`Model.alias` to alias a model class so it can be referenced twice in a single query: .. code-block:: python Owner = User.alias() query = (Favorite .select(Favorite, Tweet.content, User.username, Owner.username) .join_from(Favorite, Owner) # Determine owner of favorite. .join_from(Favorite, Tweet) # Join favorite -> tweet. .join_from(Tweet, User)) # Join tweet -> user. See :ref:`relationships` for additional discussion. .. class:: Metadata(model, database=None, table_name=None, indexes=None, primary_key=None, constraints=None, schema=None, only_save_dirty=False, depends_on=None, options=None, without_rowid=False, strict_tables=False, **kwargs) :param Model model: Model class. :param Database database: database model is bound to. :param str table_name: Specify table name for model. :param list indexes: List of :class:`ModelIndex` objects. :param primary_key: Primary key for model (only specified if this is a :class:`CompositeKey` or ``False`` for no primary key. :param list constraints: List of table constraints. :param str schema: Schema table exists in. :param bool only_save_dirty: When :meth:`~Model.save` is called, only save the fields which have been modified. :param dict options: Arbitrary options for the model. :param bool without_rowid: Specify WITHOUT ROWID (sqlite only). :param bool strict_tables: Specify STRICT (sqlite only, requires 3.37+). :param kwargs: Arbitrary setting attributes and values. Store metadata for a :class:`Model`. This class should not be instantiated directly, but is instantiated using the attributes of a :class:`Model` class' inner ``Meta`` class. Metadata attributes are then available on ``Model._meta``. Example: .. code-block:: python class User(Model): ... class Meta: database = db table_name = 'user' # After class-creation, Meta configuration lives here: isinstance(User._meta, Metadata) # True User._meta.database is db # True User._meta.table_name == 'user' # True .. seealso:: :ref:`model-options` for usage. .. attribute:: table Return a reference to the underlying :class:`Table` object. .. method:: model_graph(refs=True, backrefs=True, depth_first=True) :param bool refs: Follow foreign-key references. :param bool backrefs: Follow foreign-key back-references. :param bool depth_first: Do a depth-first search (``False`` for breadth-first). Traverse the model graph and return a list of 3-tuples, consisting of ``(foreign key field, model class, is_backref)``. .. method:: set_database(database) :param Database database: database object to bind Model to. Bind the model class to the given :class:`Database` instance. .. warning:: This API should not need to be used. Instead, to change a :class:`Model` database at run-time, use one of the following: * :meth:`Model.bind` * :meth:`Model.bind_ctx` (bind for scope of a context manager). * :meth:`Database.bind` * :meth:`Database.bind_ctx` .. method:: set_table_name(table_name) :param str table_name: table name to bind Model to. Bind the model class to the given table name at run-time. .. class:: SubclassAwareMetadata Metadata subclass that tracks :class:`Model` subclasses. Useful for when you need to track all models in a project. Example: .. code-block:: python from peewee import SubclassAwareMetadata class Base(Model): class Meta: database = db model_metadata_class = SubclassAwareMetadata # Create 3 model classes that inherit from Base. class A(Base): pass class B(Base): pass class C(Base): pass # Now let's make a helper for changing the `schema` for each Model. def change_schema(schema): def _update(model): model._meta.schema = schema return _update # Set all models to use "schema1", e.g. "schema1.a", "schema1.b", etc. # Will apply the function to every subclass of Base. Base._meta.map_models(change_schema('schema1')) # Set all models to use "schema2", e.g. "schema2.a", "schema2.b", etc. Base._meta.map_models(change_schema('schema2')) .. method:: map_models(fn) Apply a function to all subclasses. .. seealso:: :class:`~playhouse.shortcuts.ThreadSafeDatabaseMetadata` for a :class:`Metadata` subclass that supports changing the ``database`` attribute at run-time in a multi-threaded environment. .. class:: ModelSelect(model, fields_or_models) :param Model model: Model class to select. :param fields_or_models: List of fields or model classes to select. Model-specific implementation of SELECT query. .. method:: get() :param Database database: database to execute query against. :return: A single row from the database. :raises: ``DoesNotExist`` if row not found. Execute the query and return the first row, if it exists. Multiple calls will result in multiple queries being executed. If no matching row is found, raise ``DoesNotExist``. .. method:: peek(n=1) :param int n: Number of rows to return. :return: A single row if n = 1, else a list of rows. Execute the query and return the given number of rows from the start of the cursor. This function may be called multiple times safely, and will always return the first N rows of results. .. method:: first(n=1) :param int n: Number of rows to return. :return: A single row if n = 1, else a list of rows. Like the :meth:`~ModelSelect.peek` method, except a ``LIMIT`` is applied to the query to ensure that only ``n`` rows are returned. Multiple calls for the same value of ``n`` will not result in multiple executions. The query is altered in-place so it is **not** possible to call :meth:`~ModelSelect.first` and then later iterate over the full result-set using the same query object. Again, this is done to ensure that multiple calls to ``first()`` will not result in multiple query executions. .. method:: scalar(as_tuple=False, as_dict=False) :param bool as_tuple: Return the result as a tuple? :param bool as_dict: Return the result as a dict? :return: Single scalar value. If ``as_tuple = True``, a row tuple is returned. If ``as_dict = True``, a row dict is returned. Return a scalar value from the first row of results. If multiple scalar values are anticipated (e.g. multiple aggregations in a single query) then you may specify ``as_tuple=True`` to get the row tuple. Example: .. code-block:: python query = Note.select(fn.MAX(Note.timestamp)) max_ts = query.scalar() query = Note.select(fn.MAX(Note.timestamp), fn.COUNT(Note.id)) max_ts, n_notes = query.scalar(as_tuple=True) query = Note.select(fn.COUNT(Note.id).alias('count')) assert query.scalar(as_dict=True) == {'count': 123} .. method:: count(clear_limit=False) :param bool clear_limit: Clear any LIMIT clause when counting. :return: Number of rows in the query result-set. Return number of rows in the query result-set. Implemented by running SELECT COUNT(1) FROM (). Example: .. code-block:: python n = Tweet.select().where(Tweet.is_published == True).count() print('%d published tweets' % n) .. method:: exists() :return: Whether any results exist for the current query. Return a boolean indicating whether the current query has any results. Example: .. code-block:: python if User.select().where(User.username == 'Alice').exists(): print('User found') .. method:: dicts(as_dict=True) :param bool as_dict: Specify whether to return rows as dictionaries. Return rows as dictionaries. Example: .. code-block:: python :emphasize-lines: 5, 8 query = (User .select(User.username, fn.COUNT(Tweet.id).alias('tweet_count')) .join(Tweet, JOIN.LEFT_OUTER) .group_by(User.username) .dicts()) for row in query: print(row) # {'username': 'Alice', 'tweet_count': 12} .. method:: tuples(as_tuples=True) :param bool as_tuples: Specify whether to return rows as tuples. Return rows as tuples. Example: .. code-block:: python :emphasize-lines: 5, 8 query = (User .select(User.username, fn.COUNT(Tweet.id).alias('tweet_count')) .join(Tweet, JOIN.LEFT_OUTER) .group_by(User.username) .tuples()) for row in query: print(row) # ('Alice', 12) .. method:: namedtuples(as_namedtuple=True) :param bool as_namedtuple: Specify whether to return rows as named tuples. Return rows as named tuples. Example: .. code-block:: python :emphasize-lines: 5, 8 query = (User .select(User.username, fn.COUNT(Tweet.id).alias('tweet_count')) .join(Tweet, JOIN.LEFT_OUTER) .group_by(User.username) .namedtuples()) for row in query: print(row) # Row(username='Alice', tweet_count=12) .. method:: objects(constructor=None) :param constructor: Constructor (defaults to returning model instances) Return result rows as objects created using the given constructor. The default behavior is to create model instances. This method can be used, when selecting field data from multiple sources/models, to make all data available as attributes on the model being queried (as opposed to constructing the graph of joined model instances). For very complex queries this can have a positive performance impact, especially iterating large result sets. Example: .. code-block:: python query = (Tweet .select(Tweet.id, Tweet.content, User.username) .join(User) .objects()) # Apply all selections to Tweet instance. # NOTE: the username is applied directly to the tweet object # and accessed via `tweet.username`: for tweet in query: print(tweet.id, tweet.content, tweet.username) Similarly, you can use :meth:`~ModelSelect.dicts`, :meth:`~ModelSelect.tuples` or :meth:`~ModelSelect.namedtuples` to achieve even more performance. .. method:: models() Return result rows as :class:`Model` instances, rebuilding the model graph from explicitly selected and joined data. **This is the default**. .. code-block:: python query = (Tweet .select(Tweet, User) .join(User)) # Note that `tweet.user` is populated already since we SELECTed # columns from the joined User model. for tweet in query: print(tweet.user.username, '->', tweet.content) For an in-depth discussion of foreign-keys, joins and relationships between models, refer to :ref:`relationships`. .. method:: join(dest, join_type='INNER', on=None, src=None, attr=None) :param dest: A :class:`Model`, :class:`ModelAlias`, :class:`Select` query, or other object to join to. :param str join_type: Join type, defaults to INNER. :param on: Join predicate or a :class:`ForeignKeyField` to join on. :param src: Explicitly specify the source of the join. If not specified then the current *join context* will be used. :param str attr: Attribute to use when projecting columns from the joined model. Join with another table-like object. Join type may be one of: * ``JOIN.INNER`` * ``JOIN.LEFT_OUTER`` * ``JOIN.RIGHT_OUTER`` * ``JOIN.FULL`` * ``JOIN.FULL_OUTER`` * ``JOIN.CROSS`` Example selecting tweets and joining on user in order to restrict to only those tweets made by "admin" users: .. code-block:: python sq = Tweet.select().join(User).where(User.is_admin == True) Example selecting users and joining on a particular foreign key field. See the :ref:`example app ` for a real-life usage: .. code-block:: python sq = User.select().join(Relationship, on=Relationship.to_user) For an in-depth discussion of foreign-keys, joins and relationships between models, refer to :ref:`relationships`. .. method:: join_from(src, dest, join_type='INNER', on=None, attr=None) :param src: Source for join. :param dest: Table to join to. Use same parameter order as the non-model-specific :meth:`~ModelSelect.join`. Bypasses the *join context* by requiring the join source to be specified. .. method:: switch(ctx=None) :param ctx: A :class:`Model`, :class:`ModelAlias`, subquery, or other object that was joined-on. Switch the *join context* - the source which subsequent calls to :meth:`~ModelSelect.join` will be joined against. Used for specifying multiple joins against a single table. See :ref:`relationships` for additional discussion. If the ``ctx`` is not given, then the query's model will be used. The following example selects from tweet and joins on both user and tweet-flag: .. code-block:: python sq = Tweet.select().join(User).switch(Tweet).join(TweetFlag) # Equivalent (since Tweet is the query's model) sq = Tweet.select().join(User).switch().join(TweetFlag) .. method:: filter(*args, **kwargs) :param args: Zero or more :class:`DQ` objects. :param kwargs: Django-style keyword-argument filters. Use Django-style filters to express a WHERE clause. Joins can be followed by chaining foreign-key fields. The supported operations are: * ``eq`` - equals * ``ne`` - not equals * ``lt``, ``lte`` - less-than, less-than or equal-to * ``gt``, ``gte`` - greater-than, greater-than or equal-to * ``in`` - IN set of values * ``is`` - IS (e.g. IS NULL). * ``like``, ``ilike`` - LIKE and ILIKE (case-insensitive) * ``regexp`` - regular expression match Examples: .. code-block:: python # Get all tweets by user with username="peewee". q = Tweet.filter(user__username='peewee') # Get all posts that are draft or published, and written after 2023. q = Post.filter( (DQ(status='draft') | DQ(status='published')), timestamp__gte=datetime.date(2023, 1, 1)) .. method:: prefetch(*subqueries, prefetch_type=PREFETCH_TYPE.WHERE) :param subqueries: A list of :class:`Model` classes or select queries to prefetch. :param prefetch_type: Query type to use for the subqueries. :return: a list of models with selected relations prefetched. Execute the query, prefetching the given additional resources. Prefetch type may be one of: * ``PREFETCH_TYPE.WHERE`` * ``PREFETCH_TYPE.JOIN`` See also :func:`prefetch` standalone function. Example: .. code-block:: python # Fetch all Users and prefetch their associated tweets. query = User.select().prefetch(Tweet) for user in query: print(user.username) for tweet in user.tweets: print(' *', tweet.content) Because ``prefetch`` must reconstruct a graph of models, it is necessary to be sure that the foreign-key/primary-key of any related models are selected, so that the related objects can be mapped correctly. .. class:: DoesNotExist Base exception class raised when a call to :meth:`Model.get` (or other ``.get()`` method) fails to return a matching result. Model classes have a model-specific subclass as a top-level attribute: .. code-block:: python def get_user(email): try: return User.get(fn.LOWER(User.email) == email.lower()) except User.DoesNotExist: return None .. class:: PeeweeException Base exception class for wrapped DB-API exceptions. .. class:: ImproperlyConfigured Exception raised for configuration issues like missing drivers. .. class:: DatabaseError DataError IntegrityError InterfaceError InternalError NotSupportedError OperationalError ProgrammingError Exception wrappers for DB-API errors raised by the driver. Allows users to catch the Peewee exception wrappers rather than the driver-specific types. .. _fields-api: Fields ------ .. class:: Field(null=False, index=False, unique=False, column_name=None, default=None, primary_key=False, constraints=None, sequence=None, collation=None, unindexed=False, choices=None, help_text=None, verbose_name=None, index_type=None) :param bool null: Field allows NULLs. :param bool index: Create an index on field. :param bool unique: Create a unique index on field. :param str column_name: Specify column name for field. :param default: Default value (enforced in Python, not on server). :param bool primary_key: Field is the primary key. :param list constraints: List of constraints to apply to column, for example: ``[Check('price > 0')]``. :param str sequence: Sequence name for field. :param str collation: Collation name for field. :param bool unindexed: Declare field UNINDEXED (sqlite only). :param list choices: An iterable of 2-tuples mapping column values to display labels. Used for metadata purposes only, to help when displaying a dropdown of choices for field values, for example. :param str help_text: Help-text for field, metadata purposes only. :param str verbose_name: Verbose name for field, metadata purposes only. :param str index_type: Specify index type (postgres only), e.g. 'BRIN'. Fields on a :class:`Model` are analogous to columns on a table. .. attribute:: field_type = '' Attribute used to map this field to a column type, e.g. "INT". See the ``FIELD`` object in the source for more information. .. attribute:: column Retrieve a reference to the underlying :class:`Column` object. .. attribute:: model The model the field is bound to. .. attribute:: name The name of the field. .. method:: db_value(value) Coerce a Python value into a value suitable for storage in the database. Sub-classes operating on special data-types will most likely want to override this method. Example: .. code-block:: python :emphasize-lines: 3 class PickleField(BlobField): # Values going from Python -> Database should be pickled. def db_value(self, value): if value is not None: return pickle.dumps(value, pickle.HIGHEST_PROTOCOL) # Values coming from the Database -> Python should be unpickled. def python_value(self, value): if value is not None: return pickle.loads(value) .. method:: python_value(value) Coerce a value from the database into a Python object. Sub-classes operating on special data-types will most likely want to override this method. Example: .. code-block:: python :emphasize-lines: 8 class PickleField(BlobField): # Values going from Python -> Database should be pickled. def db_value(self, value): if value is not None: return pickle.dumps(value, pickle.HIGHEST_PROTOCOL) # Values coming from the Database -> Python should be unpickled. def python_value(self, value): if value is not None: return pickle.loads(value) .. method:: coerce(value) This method is a shorthand that is used, by default, by both :meth:`~Field.db_value` and :meth:`~Field.python_value`. :param value: arbitrary data from app or backend :rtype: python data type .. class:: IntegerField Field class for storing integers. .. class:: BigIntegerField Field class for storing big integers (if supported by database). .. class:: SmallIntegerField Field class for storing small integers (if supported by database). .. class:: AutoField Field class for storing auto-incrementing primary keys. In SQLite, for performance reasons, the default primary key type simply uses the max existing value + 1 for new values, as opposed to the max ever value + 1. This means deleted records can have their primary keys reused. In conjunction with SQLite having foreign keys disabled by default (meaning ON DELETE is ignored, even if you specify it explicitly), this can lead to surprising and dangerous behaviour. To avoid this, you may want to use one or both of :class:`AutoIncrementField` and ``pragmas=[('foreign_keys', 'on')]`` when you instantiate :class:`SqliteDatabase`. .. class:: BigAutoField Field class for storing auto-incrementing primary keys using 64-bits. .. class:: IdentityField(generate_always=False) :param bool generate_always: if specified, then the identity will always be generated (and specifying the value explicitly during INSERT will raise a programming error). Otherwise, the identity value is only generated as-needed. Field class for storing auto-incrementing primary keys using the Postgresql *IDENTITY* column type. The column definition ends up looking like this: .. code-block:: python id = IdentityField() # "id" INT GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY .. attention:: Requires Postgresql >= 10 .. class:: FloatField Field class for storing floating-point numbers. .. class:: DoubleField Field class for storing double-precision floating-point numbers. .. class:: DecimalField(max_digits=10, decimal_places=5, auto_round=False, rounding=None, **kwargs) :param int max_digits: Maximum digits to store. :param int decimal_places: Maximum precision. :param bool auto_round: Automatically round values. :param rounding: Defaults to ``decimal.DefaultContext.rounding``. Field class for storing decimal numbers. Values are represented as ``decimal.Decimal`` objects. .. class:: CharField(max_length=255) Field class for storing strings. Values that exceed length are NOT truncated automatically. .. class:: FixedCharField Field class for storing fixed-length strings. Values that exceed length are not truncated automatically. .. class:: TextField Field class for storing text. .. class:: BlobField Field class for storing binary data. .. class:: BitField Field class for storing options in a 64-bit integer column. Usage: .. code-block:: python class Post(Model): content = TextField() flags = BitField() is_favorite = flags.flag(1) is_sticky = flags.flag(2) is_minimized = flags.flag(4) is_deleted = flags.flag(8) >>> p = Post() >>> p.is_sticky = True >>> p.is_minimized = True >>> print(p.flags) # Prints 4 | 2 --> "6" 6 >>> p.is_favorite False >>> p.is_sticky True We can use the flags on the Post class to build expressions in queries as well: .. code-block:: python # Generates a WHERE clause that looks like: # WHERE (post.flags & 1 != 0) query = Post.select().where(Post.is_favorite) # Query for sticky + favorite posts: query = Post.select().where(Post.is_sticky & Post.is_favorite) When bulk-updating one or more bits in a :class:`BitField`, you can use bitwise operators to set or clear one or more bits: .. code-block:: python # Set the 4th bit on all Post objects. Post.update(flags=Post.flags | 8).execute() # Clear the 1st and 3rd bits on all Post objects. Post.update(flags=Post.flags & ~(1 | 4)).execute() For simple operations, the flags provide handy ``set()`` and ``clear()`` methods for setting or clearing an individual bit: .. code-block:: python # Set the "is_deleted" bit on all posts. Post.update(flags=Post.is_deleted.set()).execute() # Clear the "is_deleted" bit on all posts. Post.update(flags=Post.is_deleted.clear()).execute() .. method:: flag(value=None) :param int value: Value associated with flag, typically a power of 2. Returns a descriptor that can get or set specific bits in the overall value. When accessed on the class itself, it returns a :class:`Expression` object suitable for use in a query. If the value is not provided, it is assumed that each flag will be an increasing power of 2, so if you had four flags, they would have the values 1, 2, 4, 8. .. class:: BigBitField Field class for storing arbitrarily-large bitmaps in a ``BLOB``. The field will grow the underlying buffer as necessary, ensuring there are enough bytes of data to support the number of bits of data being stored. Example usage: .. code-block:: python class Bitmap(Model): data = BigBitField() bitmap = Bitmap() # Sets the ith bit, e.g. the 1st bit, the 11th bit, the 63rd, etc. bits_to_set = (1, 11, 63, 31, 55, 48, 100, 99) for bit_idx in bits_to_set: bitmap.data.set_bit(bit_idx) # We can test whether a bit is set using "is_set": assert bitmap.data.is_set(11) assert not bitmap.data.is_set(12) # We can clear a bit: bitmap.data.clear_bit(11) assert not bitmap.data.is_set(11) # We can also "toggle" a bit. Recall that the 63rd bit was set earlier. assert bitmap.data.toggle_bit(63) is False assert bitmap.data.toggle_bit(63) is True assert bitmap.data.is_set(63) # BigBitField supports item accessor by bit-number, e.g.: assert bitmap.data[63] bitmap.data[0] = 1 del bitmap.data[0] # We can also combine bitmaps using bitwise operators, e.g. b = Bitmap(data=b'\x01') b.data |= b'\x02' assert list(b.data) == [1, 1, 0, 0, 0, 0, 0, 0] assert len(b.data) == 1 .. method:: clear() Clears the bitmap and sets length to 0. .. method:: set_bit(idx) :param int idx: Bit to set, indexed starting from zero. Sets the *idx*-th bit in the bitmap. .. method:: clear_bit(idx) :param int idx: Bit to clear, indexed starting from zero. Clears the *idx*-th bit in the bitmap. .. method:: toggle_bit(idx) :param int idx: Bit to toggle, indexed starting from zero. :return: Whether the bit is set or not. Toggles the *idx*-th bit in the bitmap and returns whether the bit is set or not. Example: .. code-block:: pycon >>> bitmap = Bitmap() >>> bitmap.data.toggle_bit(10) # Toggle the 10th bit. True >>> bitmap.data.toggle_bit(10) # This will clear the 10th bit. False .. method:: is_set(idx) :param int idx: Bit index, indexed starting from zero. :return: Whether the bit is set or not. Returns boolean indicating whether the *idx*-th bit is set or not. .. method:: __getitem__(idx) Same as :meth:`~BigBitField.is_set` .. method:: __setitem__(idx, value) Set the bit at ``idx`` to value (True or False). .. method:: __delitem__(idx) Same as :meth:`~BigBitField.clear_bit` .. method:: __len__() Return the length of the bitmap **in bytes**. .. method:: __iter__() Returns an iterator yielding 1 or 0 for each bit in the bitmap. .. method:: __and__(other) :param other: Either :class:`BigBitField`, ``bytes``, ``bytearray`` or ``memoryview`` object. :return: bitwise ``and`` of two bitmaps. .. method:: __or__(other) :param other: Either :class:`BigBitField`, ``bytes``, ``bytearray`` or ``memoryview`` object. :return: bitwise ``or`` of two bitmaps. .. method:: __xor__(other) :param other: Either :class:`BigBitField`, ``bytes``, ``bytearray`` or ``memoryview`` object. :return: bitwise ``xor`` of two bitmaps. .. class:: UUIDField Field class for storing ``uuid.UUID`` objects. With Postgres, the underlying column's data-type will be *UUID*. Since SQLite and MySQL do not have a native UUID type, the UUID is stored as a *VARCHAR* instead. .. class:: BinaryUUIDField Field class for storing ``uuid.UUID`` objects efficiently in 16-bytes. Uses the database's *BLOB* data-type (or *VARBINARY* in MySQL, or *BYTEA* in Postgres). .. class:: DateTimeField(formats=None, **kwargs) :param list formats: A list of format strings to use when coercing a string to a date-time. Field class for storing ``datetime.datetime`` objects. Accepts a special parameter ``formats``, which contains a list of formats the datetime can be encoded with (for databases that do not have support for a native datetime data-type). The default supported formats are: .. code-block:: python '%Y-%m-%d %H:%M:%S.%f' # year-month-day hour-minute-second.microsecond '%Y-%m-%d %H:%M:%S' # year-month-day hour-minute-second '%Y-%m-%d' # year-month-day SQLite does not have a native datetime data-type, so datetimes are stored as strings. This is handled transparently by Peewee, but if you have pre-existing data you should ensure it is stored as ``YYYY-mm-dd HH:MM:SS`` or one of the other supported formats. .. attribute:: year Reference the year of the value stored in the column in a query. .. code-block:: python Blog.select().where(Blog.pub_date.year == 2018) .. attribute:: month Reference the month of the value stored in the column in a query. .. attribute:: day Reference the day of the value stored in the column in a query. .. attribute:: hour Reference the hour of the value stored in the column in a query. .. attribute:: minute Reference the minute of the value stored in the column in a query. .. attribute:: second Reference the second of the value stored in the column in a query. .. method:: to_timestamp() Method that returns a database-specific function call that will allow you to work with the given date-time value as a numeric timestamp. This can sometimes simplify tasks like date math in a compatible way. Example: .. code-block:: python # Find all events that are exactly 1 hour long. query = (Event .select() .where((Event.start.to_timestamp() + 3600) == Event.stop.to_timestamp()) .order_by(Event.start)) .. method:: truncate(date_part) :param str date_part: year, month, day, hour, minute or second. :return: expression node to truncate date/time to given resolution. Truncates the value in the column to the given part. This method is useful for finding all rows within a given month, for instance. .. class:: DateField(formats=None, **kwargs) :param list formats: A list of format strings to use when coercing a string to a date. Field class for storing ``datetime.date`` objects. Accepts a special parameter ``formats``, which contains a list of formats the datetime can be encoded with (for databases that do not have support for a native date data-type). The default supported formats are: .. code-block:: python '%Y-%m-%d' # year-month-day '%Y-%m-%d %H:%M:%S' # year-month-day hour-minute-second '%Y-%m-%d %H:%M:%S.%f' # year-month-day hour-minute-second.microsecond .. note:: If the incoming value does not match a format, it is returned as-is. .. attribute:: year Reference the year of the value stored in the column in a query. .. code-block:: python Person.select().where(Person.dob.year == 1983) .. attribute:: month Reference the month of the value stored in the column in a query. .. attribute:: day Reference the day of the value stored in the column in a query. .. method:: to_timestamp() See :meth:`DateTimeField.to_timestamp`. .. method:: truncate(date_part) See :meth:`DateTimeField.truncate`. Note that only *year*, *month*, and *day* are meaningful for :class:`DateField`. .. class:: TimeField(formats=None, **kwargs) :param list formats: A list of format strings to use when coercing a string to a time. Field class for storing ``datetime.time`` objects (not ``timedelta``). Accepts a special parameter ``formats``, which contains a list of formats the datetime can be encoded with (for databases that do not have support for a native time data-type). The default supported formats are: .. code-block:: python '%H:%M:%S.%f' # hour:minute:second.microsecond '%H:%M:%S' # hour:minute:second '%H:%M' # hour:minute '%Y-%m-%d %H:%M:%S.%f' # year-month-day hour-minute-second.microsecond '%Y-%m-%d %H:%M:%S' # year-month-day hour-minute-second .. note:: If the incoming value does not match a format, it is returned as-is. .. attribute:: hour Reference the hour of the value stored in the column in a query. .. code-block:: python evening_events = Event.select().where(Event.time.hour > 17) .. attribute:: minute Reference the minute of the value stored in the column in a query. .. attribute:: second Reference the second of the value stored in the column in a query. .. class:: TimestampField(resolution=1, utc=False, **kwargs) :param resolution: Can be provided as either a power of 10, or as an exponent indicating how many decimal places to store. :param bool utc: Treat timestamps as UTC. Field class for storing date-times as integer timestamps. Sub-second resolution is supported by multiplying by a power of 10 to get an integer. If the ``resolution`` parameter is ``0`` *or* ``1``, then the timestamp is stored using second resolution. A resolution between ``2`` and ``6`` is treated as the number of decimal places, e.g. ``resolution=3`` corresponds to milliseconds. Alternatively, the decimal can be provided as a multiple of 10, such that ``resolution=10`` will store 1/10th of a second resolution. The ``resolution`` parameter can be either 0-6 *or* 10, 100, etc up to 1000000 (for microsecond resolution). This allows sub-second precision while still using an :class:`IntegerField` for storage. The default is second resolution. Also accepts a boolean parameter ``utc``, used to indicate whether the timestamps should be UTC. Default is ``False``. Finally, the field ``default`` is the current timestamp. If you do not want this behavior, then explicitly pass in ``default=None``. .. class:: IPField Field class for storing IPv4 addresses efficiently (as integers). .. class:: BooleanField Field class for storing boolean values. .. class:: BareField(coerce=None, **kwargs) :param coerce: Optional function to use for converting raw values into a specific format. Field class that does not specify a data-type (**SQLite-only**). Since data-types are not enforced, you can declare fields without *any* data-type. It is also common for SQLite virtual tables to use meta-columns or untyped columns, so for those cases as well you may wish to use an untyped field. Accepts a special ``coerce`` parameter, a function that takes a value coming from the database and converts it into the appropriate Python type. .. class:: ForeignKeyField(model, field=None, backref=None, on_delete=None, on_update=None, deferrable=None, object_id_name=None, lazy_load=True, constraint_name=None, **kwargs) :param Model model: Model to reference or the string 'self' if declaring a self-referential foreign key. :param Field field: Field to reference on ``model`` (default is primary key). :param str backref: Accessor name for back-reference, or "+" to disable the back-reference accessor. :param str on_delete: ON DELETE action, e.g. ``'CASCADE'``.. :param str on_update: ON UPDATE action. :param str deferrable: Control when constraint is enforced, e.g. ``'INITIALLY DEFERRED'``. :param str object_id_name: Name for object-id accessor. :param bool lazy_load: Fetch the related object when the foreign-key field attribute is accessed (if it was not already loaded). If this is disabled, accessing the foreign-key field will return the value stored in the foreign-key column. :param str constraint_name: (optional) name to use for foreign-key constraint. Field class for storing a foreign key. .. code-block:: python class User(Model): name = TextField() class Tweet(Model): user = ForeignKeyField(User, backref='tweets') content = TextField() # "user" attribute >>> some_tweet.user # "tweets" backref attribute >>> for tweet in charlie.tweets: ... print(tweet.content) Some tweet Another tweet Yet another tweet For an in-depth discussion of foreign-keys, joins and relationships between models, refer to :ref:`relationships`. Foreign keys do not have a particular ``field_type`` as they will take their field type depending on the type of primary key on the model they are related to. If you manually specify a ``field``, that field must be either a primary key or have a unique constraint. .. note:: Take care with foreign keys in SQLite. By default, ON DELETE has no effect, which can have surprising (and usually unwanted) effects on your database integrity. This can affect you even if you don't specify ``on_delete``, since the default ON DELETE behaviour (to fail without modifying your data) does not happen, and your data can be silently relinked. The safest thing to do is to specify ``pragmas={'foreign_keys': 1}`` when you instantiate :class:`SqliteDatabase`. .. class:: DeferredForeignKey(rel_model_name, **kwargs) :param str rel_model_name: Model name to reference. Field class for representing a deferred foreign key. Useful for circular foreign-key references, for example: .. code-block:: python class Husband(Model): name = TextField() wife = DeferredForeignKey('Wife', deferrable='INITIALLY DEFERRED') class Wife(Model): name = TextField() husband = ForeignKeyField(Husband, deferrable='INITIALLY DEFERRED') In the above example, when the ``Wife`` model is declared, the foreign-key ``Husband.wife`` is automatically resolved and turned into a regular :class:`ForeignKeyField`. :class:`DeferredForeignKey` references are resolved when model classes are declared and created. This means that if you declare a :class:`DeferredForeignKey` to a model class that has already been imported and created, the deferred foreign key instance will never be resolved. For example: .. code-block:: python class User(Model): username = TextField() class Tweet(Model): # This will never actually be resolved, because the User # model has already been declared. user = DeferredForeignKey('user', backref='tweets') content = TextField() In cases like these you should use the regular :class:`ForeignKeyField` *or* you can manually resolve deferred foreign keys like so: .. code-block:: python # Tweet.user will be resolved into a ForeignKeyField: DeferredForeignKey.resolve(User) .. class:: ManyToManyField(model, backref=None, through_model=None, on_delete=None, on_update=None, prevent_unsaved=True) :param Model model: Model to create relationship with. :param str backref: Accessor name for back-reference :param Model through_model: :class:`Model` to use for the intermediary table. If not provided, a simple through table will be automatically created. :param str on_delete: ON DELETE action, e.g. ``'CASCADE'``. Will be used for foreign-keys in through model. :param str on_update: ON UPDATE action. Will be used for foreign-keys in through model. :param bool prevent_unsaved: Raise ``ValueError`` if accessing relation from an unsaved model instance (default True). The :class:`ManyToManyField` provides a simple interface for working with many-to-many relationships, inspired by Django. A many-to-many relationship is typically implemented by creating a junction table with foreign keys to the two models being related. For instance, if you were building a syllabus manager for college students, the relationship between students and courses would be many-to-many. Here is the schema using standard APIs: .. attention:: This is not a field in the sense that there is no column associated with it. Rather, it provides a convenient interface for accessing rows of data related via a through model. Standard way of declaring a many-to-many relationship (without the use of the :class:`ManyToManyField`): .. code-block:: python class Student(Model): name = CharField() class Course(Model): name = CharField() class StudentCourse(Model): student = ForeignKeyField(Student) course = ForeignKeyField(Course) To query the courses for a particular student, you would join through the junction table: .. code-block:: python # List the courses that "Huey" is enrolled in: courses = (Course .select() .join(StudentCourse) .join(Student) .where(Student.name == 'Huey')) for course in courses: print(course.name) The :class:`ManyToManyField` is designed to simplify this use-case by providing a *field-like* API for querying and modifying data in the junction table. Here is how our code looks using :class:`ManyToManyField`: .. code-block:: python class Student(Model): name = CharField() class Course(Model): name = CharField() students = ManyToManyField(Student, backref='courses') It does not matter from Peewee's perspective which model the :class:`ManyToManyField` goes on, since the back-reference is just the mirror image. In order to write valid Python, though, you will need to add the ``ManyToManyField`` on the second model so that the name of the first model is in the scope. We still need a junction table to store the relationships between students and courses. This model can be accessed by calling the :meth:`~ManyToManyField.get_through_model` method. This is useful when creating tables. .. code-block:: python # Create tables for the students, courses, and relationships between # the two. db.create_tables([ Student, Course, Course.students.get_through_model()]) When accessed from a model instance, the :class:`ManyToManyField` exposes a :class:`ModelSelect` representing the set of related objects. Let's use the interactive shell to see how all this works: .. code-block:: pycon >>> huey = Student.get(Student.name == 'huey') >>> [course.name for course in huey.courses] ['English 101', 'CS 101'] >>> engl_101 = Course.get(Course.name == 'English 101') >>> [student.name for student in engl_101.students] ['Huey', 'Mickey', 'Zaizee'] To add new relationships between objects, you can either assign the objects directly to the ``ManyToManyField`` attribute, or call the :meth:`~ManyToManyField.add` method. The difference between the two is that simply assigning will clear out any existing relationships, whereas ``add()`` can preserve existing relationships. .. code-block:: pycon >>> huey.courses = Course.select().where(Course.name.contains('english')) >>> for course in huey.courses.order_by(Course.name): ... print(course.name) English 101 English 151 English 201 English 221 >>> cs_101 = Course.get(Course.name == 'CS 101') >>> cs_151 = Course.get(Course.name == 'CS 151') >>> huey.courses.add([cs_101, cs_151]) >>> [course.name for course in huey.courses.order_by(Course.name)] ['CS 101', 'CS151', 'English 101', 'English 151', 'English 201', 'English 221'] This is quite a few courses, so let's remove the 200-level english courses. To remove objects, use the :meth:`~ManyToManyField.remove` method. .. code-block:: pycon >>> huey.courses.remove(Course.select().where(Course.name.contains('2')) 2 >>> [course.name for course in huey.courses.order_by(Course.name)] ['CS 101', 'CS151', 'English 101', 'English 151'] To remove all relationships from a collection, you can use the :meth:`~SelectQuery.clear` method. Let's say that English 101 is canceled, so we need to remove all the students from it: .. code-block:: pycon >>> engl_101 = Course.get(Course.name == 'English 101') >>> engl_101.students.clear() For an overview of implementing many-to-many relationships using standard Peewee APIs, check out the :ref:`manytomany` section. For all but the most simple cases, you will be better off implementing many-to-many using the standard APIs. .. attribute:: through_model The :class:`Model` representing the many-to-many junction table. Will be auto-generated if not explicitly declared. .. method:: add(value, clear_existing=False) :param value: Either a :class:`Model` instance, a list of model instances, or a :class:`SelectQuery`. :param bool clear_existing: Whether to remove existing relationships. Associate ``value`` with the current instance. You can pass in a single model instance, a list of model instances, or even a :class:`ModelSelect`. Example code: .. code-block:: python # Huey needs to enroll in a bunch of courses, including all # the English classes, and a couple Comp-Sci classes. huey = Student.get(Student.name == 'Huey') # We can add all the objects represented by a query. english_courses = Course.select().where( Course.name.contains('english')) huey.courses.add(english_courses) # We can also add lists of individual objects. cs101 = Course.get(Course.name == 'CS 101') cs151 = Course.get(Course.name == 'CS 151') huey.courses.add([cs101, cs151]) .. method:: remove(value) :param value: Either a :class:`Model` instance, a list of model instances, or a :class:`ModelSelect`. Disassociate ``value`` from the current instance. Like :meth:`~ManyToManyField.add`, you can pass in a model instance, a list of model instances, or even a :class:`ModelSelect`. Example code: .. code-block:: python # Huey is currently enrolled in a lot of english classes # as well as some Comp-Sci. He is changing majors, so we # will remove all his courses. english_courses = Course.select().where( Course.name.contains('english')) huey.courses.remove(english_courses) # Remove the two Comp-Sci classes Huey is enrolled in. cs101 = Course.get(Course.name == 'CS 101') cs151 = Course.get(Course.name == 'CS 151') huey.courses.remove([cs101, cs151]) .. method:: clear() Remove all associated objects. Example code: .. code-block:: python # English 101 is canceled this semester, so remove all # the enrollments. english_101 = Course.get(Course.name == 'English 101') english_101.students.clear() .. method:: get_through_model() Return the :class:`Model` representing the many-to-many junction table. This can be specified manually when the field is being instantiated using the ``through_model`` parameter. If a ``through_model`` is not specified, one will automatically be created. When creating tables for an application that uses :class:`ManyToManyField`, **you must explicitly create the through table**. .. code-block:: python # Get a reference to the automatically-created through table. StudentCourseThrough = Course.students.get_through_model() # Create tables for our two models as well as the through model. db.create_tables([ Student, Course, StudentCourseThrough]) .. class:: DeferredThroughModel() Place-holder for a through-model in cases where, due to a dependency, you cannot declare either a model or a many-to-many field without introducing NameErrors. Example: .. code-block:: python class Note(BaseModel): content = TextField() NoteThroughDeferred = DeferredThroughModel() class User(BaseModel): username = TextField() notes = ManyToManyField(Note, through_model=NoteThroughDeferred) # Cannot declare this before "User" since it has a foreign-key to # the User model. class NoteThrough(BaseModel): note = ForeignKeyField(Note) user = ForeignKeyField(User) # Resolve dependencies. NoteThroughDeferred.set_model(NoteThrough) .. class:: CompositeKey(*field_names) :param field_names: Names of fields that comprise the primary key. A primary key composed of multiple columns. Unlike the other fields, a composite key is defined in the model's ``Meta`` class after the fields have been defined. It takes as parameters the string names of the fields to use as the primary key: .. code-block:: python class BlogTagThrough(Model): blog = ForeignKeyField(Blog, backref='tags') tag = ForeignKeyField(Tag, backref='blogs') class Meta: primary_key = CompositeKey('blog', 'tag') See :ref:`composite-keys` for additional discussion. .. _schema-manager-api: Schema Manager -------------- .. class:: SchemaManager(model, database=None, **context_options) :param Model model: Model class. :param Database database: if ``None`` defaults to model._meta.database. Provides methods for managing the creation and deletion of tables and indexes for the given model. .. method:: create_table(safe=True, **options) :param bool safe: Specify IF NOT EXISTS clause. :param options: Arbitrary options. Execute CREATE TABLE query for the given model. .. method:: drop_table(safe=True, drop_sequences=True, **options) :param bool safe: Specify IF EXISTS clause. :param bool drop_sequences: Drop any sequences associated with the columns on the table (postgres only). :param options: Arbitrary options. Execute DROP TABLE query for the given model. .. method:: truncate_table(restart_identity=False, cascade=False) :param bool restart_identity: Restart the id sequence (postgres-only). :param bool cascade: Truncate related tables as well (postgres-only). Execute TRUNCATE TABLE for the given model. If the database is Sqlite, which does not support TRUNCATE, then an equivalent DELETE query will be executed. .. method:: create_indexes(safe=True) :param bool safe: Specify IF NOT EXISTS clause. Execute CREATE INDEX queries for the indexes defined for the model. .. method:: drop_indexes(safe=True) :param bool safe: Specify IF EXISTS clause. Execute DROP INDEX queries for the indexes defined for the model. .. method:: create_sequence(field) :param Field field: Field instance which specifies a sequence. Create sequence for the given :class:`Field`. .. method:: drop_sequence(field) :param Field field: Field instance which specifies a sequence. Drop sequence for the given :class:`Field`. .. method:: create_foreign_key(field) :param ForeignKeyField field: Foreign-key field constraint to add. Add a foreign-key constraint for the given field. This method should not be necessary in most cases, as foreign-key constraints are created as part of table creation. The exception is when you are creating a circular foreign-key relationship using :class:`DeferredForeignKey`. In those cases, it is necessary to first create the tables, then add the constraint for the deferred foreign-key: .. code-block:: python class Language(Model): name = TextField() selected_snippet = DeferredForeignKey('Snippet') class Snippet(Model): code = TextField() language = ForeignKeyField(Language, backref='snippets') # Creates both tables but does not create the constraint for the # Language.selected_snippet foreign key (because of the circular # dependency). db.create_tables([Language, Snippet]) # Explicitly create the constraint: Language._schema.create_foreign_key(Language.selected_snippet) For more information, see documentation on :ref:`circular-fks`. Because SQLite has limited support for altering existing tables, it is not possible to add a foreign-key constraint to an existing SQLite table. .. method:: create_all(safe=True, **table_options) :param bool safe: Whether to specify IF NOT EXISTS. Create sequence(s), index(es) and table for the model. .. method:: drop_all(safe=True, drop_sequences=True, **options) :param bool safe: Whether to specify IF EXISTS. :param bool drop_sequences: Drop any sequences associated with the columns on the table (postgres only). :param options: Arbitrary options. Drop table for the model and associated indexes. .. class:: Index(name, table, expressions, unique=False, safe=False, where=None, using=None, nulls_distinct=None) :param str name: Index name. :param Table table: Table to create index on. :param expressions: List of columns to index on (or expressions). :param bool unique: Whether index is UNIQUE. :param bool safe: Whether to add IF NOT EXISTS clause. :param Expression where: Optional WHERE clause for index. :param str using: Index algorithm. :param bool nulls_distinct: Postgres-only - specify True (NULLS DISTINCT) or False (NULLS NOT DISTINCT) - controls handling of NULL in unique indexes. .. method:: safe(_safe=True) :param bool _safe: Whether to add IF NOT EXISTS clause. .. method:: where(*expressions) :param expressions: zero or more expressions to include in the WHERE clause. Include the given expressions in the WHERE clause of the index. The expressions will be AND-ed together with any previously-specified WHERE expressions. .. method:: using(_using=None) :param str _using: Specify index algorithm for USING clause. .. method:: nulls_distinct(nulls_distinct=None) :param bool nulls_distinct: specify True (NULLS DISTINCT) or False for (NULLS NOT DISTINCT). Requires Postgres 15 or newer. Control handling of NULL values in unique indexes. .. class:: ModelIndex(model, fields, unique=False, safe=True, where=None, using=None, name=None, nulls_distinct=None) :param Model model: Model class to create index on. :param list fields: Fields to index. :param bool unique: Whether index is UNIQUE. :param bool safe: Whether to add IF NOT EXISTS clause. :param Expression where: Optional WHERE clause for index. :param str using: Index algorithm or type, e.g. 'BRIN', 'GiST' or 'GIN'. :param str name: Optional index name. :param bool nulls_distinct: Postgres-only - specify True (NULLS DISTINCT) or False (NULLS NOT DISTINCT) - controls handling of NULL in unique indexes. Expressive method for declaring an index on a model. Examples: .. code-block:: python class Article(Model): name = TextField() timestamp = TimestampField() status = IntegerField() flags = BitField() is_sticky = flags.flag(1) is_favorite = flags.flag(2) # CREATE INDEX ... ON "article" ("name", "timestamp") idx = ModelIndex(Article, (Article.name, Article.timestamp)) # CREATE INDEX ... ON "article" ("name", "timestamp") WHERE "status" = 1 idx = idx.where(Article.status == 1) # CREATE UNIQUE INDEX ... ON "article" ("timestamp" DESC, "flags" & 2) WHERE "status" = 1 idx = ModelIndex( Article, (Article.timestamp.desc(), Article.flags.bin_and(2)), unique = True).where(Article.status == 1) You can also use :meth:`Model.index`: .. code-block:: python idx = Article.index(Article.name, Article.timestamp).where(Article.status == 1) To add an index to a model definition use :meth:`Model.add_index`: .. code-block:: python idx = Article.index(Article.name, Article.timestamp).where(Article.status == 1) # Add above index definition to the model definition. When you call # Article.create_table() (or database.create_tables([Article])), the # index will be created. Article.add_index(idx) .. _query-builder-api: Query-builder ------------- .. seealso: :ref:`query-builder` .. class:: Node() Base-class for all components which make up the AST for a SQL query. .. staticmethod:: copy(method) Decorator to use with Node methods that mutate the node's state. This allows method-chaining, e.g.: .. code-block:: python query = MyModel.select() new_query = query.where(MyModel.field == 'value') .. method:: unwrap() API for recursively unwrapping "wrapped" nodes. Base case is to return self. .. method:: is_alias() API for determining if a node, at any point, has been explicitly aliased by the user. .. class:: Source(alias=None) A source of row tuples, for example a table, join, or select query. By default provides a "magic" attribute named "c" that is a factory for column/attribute lookups, for example: .. code-block:: python User = Table('users') query = (User .select(User.c.username) .where(User.c.active == True) .order_by(User.c.username)) .. method:: alias(name) Returns a copy of the object with the given alias applied. .. method:: select(*columns) :param columns: :class:`Column` instances, expressions, functions, sub-queries, or anything else that you would like to select. Create a :class:`Select` query on the table. If the table explicitly declares columns and no columns are provided, then by default all the table's defined columns will be selected. .. method:: join(dest, join_type='INNER', on=None) :param Source dest: Join the table with the given destination. :param str join_type: Join type. :param on: Expression to use as join predicate. :return: a :class:`Join` instance. Join type may be one of: * ``JOIN.INNER`` * ``JOIN.LEFT_OUTER`` * ``JOIN.RIGHT_OUTER`` * ``JOIN.FULL`` * ``JOIN.FULL_OUTER`` * ``JOIN.CROSS`` .. method:: left_outer_join(dest, on=None) :param Source dest: Join the table with the given destination. :param on: Expression to use as join predicate. :return: a :class:`Join` instance. Convenience method for calling :meth:`~Source.join` using a LEFT OUTER join. .. class:: BaseTable() Base class for table-like objects, which support JOINs via operator overloading. .. method:: __and__(dest) Perform an INNER join on ``dest``. .. method:: __add__(dest) Perform a LEFT OUTER join on ``dest``. .. method:: __sub__(dest) Perform a RIGHT OUTER join on ``dest``. .. method:: __or__(dest) Perform a FULL OUTER join on ``dest``. .. method:: __mul__(dest) Perform a CROSS join on ``dest``. .. class:: Table(name, columns=None, primary_key=None, schema=None, alias=None) Represents a table in the database (or a table-like object such as a view). :param str name: Database table name :param tuple columns: List of column names (optional). :param str primary_key: Name of primary key column. :param str schema: Schema name used to access table (if necessary). :param str alias: Alias to use for table in SQL queries. If columns are specified, the magic "c" attribute will be disabled. When columns are not explicitly defined, tables have a special attribute "c" which is a factory that provides access to table columns dynamically. Example: .. code-block:: python User = Table('users') query = (User .select(User.c.id, User.c.username) .order_by(User.c.username)) Equivalent example when columns **are** specified: .. code-block:: python User = Table('users', ('id', 'username')) query = (User .select(User.id, User.username) .order_by(User.username)) .. method:: bind(database=None) :param database: :class:`Database` object. Bind this table to the given database (or unbind by leaving empty). When a table is *bound* to a database, queries may be executed against it without the need to specify the database in the query's execute method. .. method:: bind_ctx(database=None) :param database: :class:`Database` object. Return a context manager that will bind the table to the given database for the duration of the wrapped block. .. method:: select(*columns) :param columns: :class:`Column` instances, expressions, functions, sub-queries, or anything else that you would like to select. Create a :class:`Select` query on the table. If the table explicitly declares columns and no columns are provided, then by default all the table's defined columns will be selected. Examples: .. code-block:: python User = Table('users', ('id', 'username')) # Because columns were defined on the Table, we will default to # selecting both of the User table's columns. # Evaluates to SELECT id, username FROM users query = User.select() Note = Table('notes') query = (Note .select(Note.c.content, Note.c.timestamp, User.username) .join(User, on=(Note.c.user_id == User.id)) .where(Note.c.is_published == True) .order_by(Note.c.timestamp.desc())) # Using a function to select users and the number of notes they # have authored. query = (User .select( User.username, fn.COUNT(Note.c.id).alias('n_notes')) .join( Note, JOIN.LEFT_OUTER, on=(User.id == Note.c.user_id)) .order_by(fn.COUNT(Note.c.id).desc())) .. method:: insert(insert=None, columns=None, **kwargs) :param insert: A dictionary mapping column to value, an iterable that yields dictionaries (i.e. list), or a :class:`Select` query. :param list columns: The list of columns to insert into when the data being inserted is not a dictionary. :param kwargs: Mapping of column-name to value. Create a :class:`Insert` query into the table. .. method:: replace(insert=None, columns=None, **kwargs) :param insert: A dictionary mapping column to value, an iterable that yields dictionaries (i.e. list), or a :class:`Select` query. :param list columns: The list of columns to insert into when the data being inserted is not a dictionary. :param kwargs: Mapping of column-name to value. Create a :class:`Insert` query into the table whose conflict resolution method is to replace. .. method:: update(update=None, **kwargs) :param update: A dictionary mapping column to value. :param kwargs: Mapping of column-name to value. Create a :class:`Update` query for the table. .. method:: delete() Create a :class:`Delete` query for the table. .. class:: Join(lhs, rhs, join_type=JOIN.INNER, on=None, alias=None) Represent a JOIN between two table-like objects. :param lhs: Left-hand side of the join. :param rhs: Right-hand side of the join. :param join_type: Type of join. e.g. JOIN.INNER, JOIN.LEFT_OUTER, etc. :param on: Expression describing the join predicate. :param str alias: Alias to apply to joined data. .. method:: on(predicate) :param Expression predicate: join predicate. Specify the predicate expression used for this join. .. class:: ValuesList(values, columns=None, alias=None) Represent a values list that can be used like a table. :param values: a list-of-lists containing the row data to represent. :param list columns: the names to give to the columns in each row. :param str alias: alias to use for values-list. Example: .. code-block:: python data = [(1, 'first'), (2, 'second')] vl = ValuesList(data, columns=('idx', 'name')) query = (vl .select(vl.c.idx, vl.c.name) .order_by(vl.c.idx)) # Yields: # SELECT t1.idx, t1.name # FROM (VALUES (1, 'first'), (2, 'second')) AS t1(idx, name) # ORDER BY t1.idx .. method:: columns(*names) :param names: names to apply to the columns of data. Example: .. code-block:: python vl = ValuesList([(1, 'first'), (2, 'second')]) vl = vl.columns('idx', 'name').alias('v') query = vl.select(vl.c.idx, vl.c.name) # Yields: # SELECT v.idx, v.name # FROM (VALUES (1, 'first'), (2, 'second')) AS v(idx, name) .. class:: CTE(name, query, recursive=False, columns=None) Represent a common-table-expression. For example queries, see :ref:`cte`. :param name: Name for the CTE. :param query: :class:`Select` query describing CTE. :param bool recursive: Whether the CTE is recursive. :param list columns: Explicit list of columns produced by CTE (optional). .. method:: select_from(*columns) Create a SELECT query that utilizes the given common table expression as the source for a new query. :param columns: One or more columns to select from the CTE. :return: :class:`Select` query utilizing the common table expression .. method:: union_all(other) Used on the base-case CTE to construct the recursive term of the CTE. :param other: recursive term, generally a :class:`Select` query. :return: a recursive :class:`CTE` with the given recursive term. .. class:: ColumnBase() Base-class for column-like objects, attributes or expressions. Column-like objects can be composed using various operators and special methods. * ``&``: Logical AND * ``|``: Logical OR * ``+``: Addition * ``-``: Subtraction * ``*``: Multiplication * ``/``: Division * ``^``: Exclusive-OR * ``==``: Equality * ``!=``: Inequality * ``>``: Greater-than * ``<``: Less-than * ``>=``: Greater-than or equal * ``<=``: Less-than or equal * ``<<``: ``IN`` * ``>>``: ``IS`` (i.e. ``IS NULL``) * ``%``: ``LIKE`` * ``**``: ``ILIKE`` * ``bin_and()``: Binary AND * ``bin_or()``: Binary OR * ``in_()``: ``IN`` * ``not_in()``: ``NOT IN`` * ``regexp()``: ``REGEXP`` * ``is_null(True/False)``: ``IS NULL`` or ``IS NOT NULL`` * ``contains(s)``: ``LIKE %s%`` * ``startswith(s)``: ``LIKE s%`` * ``endswith(s)``: ``LIKE %s`` * ``between(low, high)``: ``BETWEEN low AND high`` * ``concat()``: ``||`` .. method:: alias(alias) :param str alias: Alias for the given column-like object. :return: a :class:`Alias` object. Indicate the alias that should be given to the specified column-like object. .. method:: cast(as_type) :param str as_type: Type name to cast to. :return: a :class:`Cast` object. Create a ``CAST`` expression. Example: .. code-block:: python # Cast a text column to integer for comparison. query = (Entry .select() .where(Entry.legacy_code.cast('INTEGER') > 100)) .. method:: asc(collation=None, nulls=None) :param str collation: Collation name to use for sorting. :param str nulls: Sort nulls (FIRST or LAST). :return: an ascending :class:`Ordering` object for the column. .. method:: desc(collation=None, nulls=None) :param str collation: Collation name to use for sorting. :param str nulls: Sort nulls (FIRST or LAST). :return: an descending :class:`Ordering` object for the column. .. method:: __invert__() :return: a :class:`Negated` wrapper for the column. .. class:: Column(source, name) :param Source source: Source for column. :param str name: Column name. Column on a table or a column returned by a sub-query. .. class:: Alias(node, alias) :param Node node: a column-like object. :param str alias: alias to assign to column. Create a named alias for the given column-like object. .. method:: alias(alias=None) :param str alias: new name (or None) for aliased column. Create a new :class:`Alias` for the aliased column-like object. If the new alias is ``None``, then the original column-like object is returned. .. class:: Negated(node) Represents a negated column-like object. .. class:: Value(value, converter=None, unpack=True) :param value: Python object or scalar value. :param converter: Function used to convert value into type the database understands. :param bool unpack: Whether lists or tuples should be unpacked into a list of values or treated as-is. Value to be used in a parameterized query. It is the responsibility of the caller to ensure that the value passed in can be adapted to a type the database driver understands. .. function:: AsIs(value, converter=None) Represents a :class:`Value` that is treated as-is, and passed directly back to the database driver. This may be useful if you are using database extensions that accept native Python data-types and you do not wish Peewee to impose any handling of the values. In the event a converter is in scope for this value, the converter will be applied unless ``converter=False`` (in which case no conversion is applied by Peewee and the value is passed directly to the driver). The Postgres JSON extensions make use of this to pass ``dict`` and ``list`` to the driver, which then handles the JSON serialization more efficiently, for example. .. class:: Cast(node, cast) :param node: A column-like object. :param str cast: Type to cast to. Represents a ``CAST( AS )`` expression. .. class:: Ordering(node, direction, collation=None, nulls=None) :param node: A column-like object. :param str direction: ASC or DESC :param str collation: Collation name to use for sorting. :param str nulls: Sort nulls (FIRST or LAST). Represent ordering by a column-like object. Postgresql supports a non-standard clause ("NULLS FIRST/LAST"). Peewee will automatically use an equivalent ``CASE`` statement for databases that do not support this (Sqlite / MySQL). .. method:: collate(collation=None) :param str collation: Collation name to use for sorting. .. function:: Asc(node, collation=None, nulls=None) Short-hand for instantiating an ascending :class:`Ordering` object. .. function:: Desc(node, collation=None, nulls=None) Short-hand for instantiating an descending :class:`Ordering` object. .. class:: Expression(lhs, op, rhs, flat=True) :param lhs: Left-hand side. :param op: Operation. :param rhs: Right-hand side. :param bool flat: Whether to wrap expression in parentheses. Represent a binary expression of the form (lhs op rhs), e.g. (foo + 1). .. class:: Entity(*path) :param path: Components that make up the dotted-path of the entity name. Represent a quoted entity in a query, such as a table, column, alias. The name may consist of multiple components, e.g. "a_table"."column_name". .. method:: __getattr__(self, attr) Factory method for creating sub-entities. .. class:: SQL(sql, params=None) :param str sql: SQL query string. :param tuple params: Parameters for query (optional). Represent a parameterized SQL query or query-fragment. .. function:: Check(constraint, name=None) :param str constraint: Constraint SQL. :param str name: constraint name. Represent a CHECK constraint. MySQL may not support a ``name`` parameter when inlining the constraint along with the column definition. The solution is to just put the named ``Check`` constraint in the model's ``Meta.constraints`` list instead of in the field instances ``constraints=[...]`` list. .. function:: Default(value) :param value: default value (literal). Represent a DEFAULT constraint. It is important to note that this constraint does not accept a parameterized value, so the value literal must be given. If a string value is intended, it must be quoted. Examples: .. code-block:: python # "added" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP. added = DateTimeField(constraints=[Default('CURRENT_TIMESTAMP')]) # "label" TEXT NOT NULL DEFAULT 'string literal' label = TextField(constraints=[Default("'string literal'")]) # "status" INTEGER NOT NULL DEFAULT 0 status = IntegerField(constraints=[Default(0)]) .. class:: Function(name, arguments, coerce=True, python_value=None) :param str name: Function name. :param tuple arguments: Arguments to function. :param bool coerce: Whether to coerce the function result to a particular data-type when reading function return values from the cursor. :param callable python_value: Function to use for converting the return value from the cursor. Represent an arbitrary SQL function call. Rather than instantiating this class directly, it is recommended to use the ``fn`` helper. Example of using ``fn`` to call an arbitrary SQL function: .. code-block:: python # Query users and count of tweets authored. query = (User .select(User.username, fn.COUNT(Tweet.id).alias('ct')) .join(Tweet, JOIN.LEFT_OUTER, on=(User.id == Tweet.user_id)) .group_by(User.username) .order_by(fn.COUNT(Tweet.id).desc())) .. method:: over(partition_by=None, order_by=None, start=None, end=None, window=None, exclude=None) :param list partition_by: List of columns to partition by. :param list order_by: List of columns / expressions to order window by. :param start: A :class:`SQL` instance or a string expressing the start of the window range. :param end: A :class:`SQL` instance or a string expressing the end of the window range. :param str frame_type: ``Window.RANGE``, ``Window.ROWS`` or ``Window.GROUPS``. :param Window window: A :class:`Window` instance. :param exclude: Frame exclusion, one of ``Window.CURRENT_ROW``, ``Window.GROUP``, ``Window.TIES`` or ``Window.NO_OTHERS``. For an in-depth guide to using window functions with Peewee, see the :ref:`window-functions` section. Examples: .. code-block:: python # Using a simple partition on a single column. query = (Sample .select( Sample.counter, Sample.value, fn.AVG(Sample.value).over([Sample.counter])) .order_by(Sample.counter)) # Equivalent example Using a Window() instance instead. window = Window(partition_by=[Sample.counter]) query = (Sample .select( Sample.counter, Sample.value, fn.AVG(Sample.value).over(window)) .window(window) # Note call to ".window()" .order_by(Sample.counter)) # Example using bounded window. query = (Sample .select(Sample.value, fn.SUM(Sample.value).over( partition_by=[Sample.counter], start=Window.CURRENT_ROW, # current row end=Window.following())) # unbounded following .order_by(Sample.id)) .. method:: filter(where) :param where: Expression for filtering aggregate. Add a ``FILTER (WHERE...)`` clause to an aggregate function. The where expression is evaluated to determine which rows are fed to the aggregate function. This SQL feature is supported for Postgres and SQLite. .. method:: coerce(coerce=True) :param bool coerce: Whether to attempt to coerce function-call result to a Python data-type. When coerce is ``True``, the target data-type is inferred using several heuristics. Read the source for ``BaseModelCursorWrapper._initialize_columns`` method to see how this works. .. method:: python_value(func=None) :param callable python_value: Function to use for converting the return value from the cursor. Specify a particular function to use when converting values returned by the database cursor. For example: .. code-block:: python # Get user and a list of their tweet IDs. The tweet IDs are # returned as a comma-separated string by the db, so we'll split # the result string and convert the values to python ints. convert_ids = lambda s: [int(i) for i in (s or '').split(',') if i] tweet_ids = (fn .GROUP_CONCAT(Tweet.id) .python_value(convert_ids)) query = (User .select(User.username, tweet_ids.alias('tweet_ids')) .join(Tweet) .group_by(User.username)) for user in query: print(user.username, user.tweet_ids) # e.g., # huey [1, 4, 5, 7] # mickey [2, 3, 6] # zaizee [] .. function:: fn() The :func:`fn` helper is actually an instance of :class:`Function` that implements a ``__getattr__`` hook to provide a nice API for calling SQL functions. To create a node representative of a SQL function call, use the function name as an attribute on ``fn`` and then provide the arguments as you would if calling a Python function: .. code-block:: python # List users and the number of tweets they have authored, # from highest-to-lowest: sql_count = fn.COUNT(Tweet.id) query = (User .select(User, sql_count.alias('count')) .join(Tweet, JOIN.LEFT_OUTER) .group_by(User) .order_by(sql_count.desc())) # Get the timestamp of the most recent tweet: query = Tweet.select(fn.MAX(Tweet.timestamp)) max_timestamp = query.scalar() # Retrieve scalar result from query. Function calls can, like anything else, be composed and nested: .. code-block:: python # Get users whose username begins with "A" or "a": a_users = User.select().where(fn.LOWER(fn.SUBSTR(User.username, 1, 1)) == 'a') .. class:: Window(partition_by=None, order_by=None, start=None, end=None, frame_type=None, extends=None, exclude=None, alias=None) :param list partition_by: List of columns to partition by. :param list order_by: List of columns to order by. :param start: A :class:`SQL` instance or a string expressing the start of the window range. :param end: A :class:`SQL` instance or a string expressing the end of the window range. :param str frame_type: ``Window.RANGE``, ``Window.ROWS`` or ``Window.GROUPS``. :param extends: A :class:`Window` definition to extend. Alternately, you may specify the window's alias instead. :param exclude: Frame exclusion, one of ``Window.CURRENT_ROW``, ``Window.GROUP``, ``Window.TIES`` or ``Window.NO_OTHERS``. :param str alias: Alias for the window. Represent a WINDOW clause. For an in-depth guide to using window functions with Peewee, see the :ref:`window-functions` section. .. attribute:: RANGE ROWS GROUPS Specify the window ``frame_type``. See :ref:`window-frame-types`. .. attribute:: CURRENT_ROW Reference to current row for use in start/end clause or the frame exclusion parameter. .. attribute:: NO_OTHERS GROUP TIES Specify the window frame exclusion parameter. .. staticmethod:: preceding(value=None) :param value: Number of rows preceding. If ``None`` is UNBOUNDED. Convenience method for generating SQL suitable for passing in as the ``start`` parameter for a window range. .. staticmethod:: following(value=None) :param value: Number of rows following. If ``None`` is UNBOUNDED. Convenience method for generating SQL suitable for passing in as the ``end`` parameter for a window range. .. method:: as_rows() as_range() as_groups() Specify the frame type. .. method:: extends(window=None) :param Window window: A :class:`Window` definition to extend. Alternately, you may specify the window's alias instead. .. method:: exclude(frame_exclusion=None) :param frame_exclusion: Frame exclusion, one of ``Window.CURRENT_ROW``, ``Window.GROUP``, ``Window.TIES`` or ``Window.NO_OTHERS``. .. method:: alias(alias=None) :param str alias: Alias to use for window. .. function:: Case(predicate, expression_tuples, default=None) :param predicate: Predicate for CASE query (optional). :param expression_tuples: One or more cases to evaluate. :param default: Default value (optional). :return: Representation of CASE statement. Example: .. code-block:: python Number = Table('numbers', ('val',)) num_as_str = Case(Number.val, ( (1, 'one'), (2, 'two'), (3, 'three')), 'a lot') query = Number.select(Number.val, num_as_str.alias('num_str')) Equivalent SQL: .. code-block:: sql SELECT "val", CASE "val" WHEN 1 THEN 'one' WHEN 2 THEN 'two' WHEN 3 THEN 'three' ELSE 'a lot' END AS "num_str" FROM "numbers" Example: .. code-block:: python num_as_str = Case(None, ( (Number.val == 1, 'one'), (Number.val == 2, 'two'), (Number.val == 3, 'three')), 'a lot') query = Number.select(Number.val, num_as_str.alias('num_str')) Equivalent SQL: .. code-block:: sql SELECT "val", CASE WHEN "val" = 1 THEN 'one' WHEN "val" = 2 THEN 'two' WHEN "val" = 3 THEN 'three' ELSE 'a lot' END AS "num_str" FROM "numbers" .. class:: ForUpdate(expr, of=None, nowait=None, skip_locked=None) :param str expr: Either a boolean (``True``) or a string indicating the desired lock strength, e.g. "FOR SHARE" or "FOR NO KEY UPDATE". :param of: One or more models to restrict locking to. :param bool nowait: Specify NOWAIT option when locking. :param bool skip_locked: Specify SKIP LOCKED option when locking. Represent a ``FOR UPDATE`` SQL clause in a :class:`Select` query. Typically you will call :meth:`Select.for_update` rather than using this class directly. .. class:: NodeList(nodes, glue=' ', parens=False) :param list nodes: Zero or more nodes. :param str glue: How to join the nodes when converting to SQL. :param bool parens: Whether to wrap the resulting SQL in parentheses. Represent a list of nodes, a multi-part clause, a list of parameters, etc. .. function:: CommaNodeList(nodes) :param list nodes: Zero or more nodes. :return: a :class:`NodeList` Represent a list of nodes joined by commas. .. function:: EnclosedNodeList(nodes) :param list nodes: Zero or more nodes. :return: a :class:`NodeList` Represent a list of nodes joined by commas and wrapped in parentheses. .. class:: DQ(**query) :param query: Arbitrary filter expressions using Django-style lookups. Represent a composable Django-style filter expression suitable for use with the :meth:`Model.filter` or :meth:`ModelSelect.filter` methods. .. class:: Tuple(*args) Represent a SQL `row value `_. Row-values are supported by most databases. .. class:: OnConflict(action=None, update=None, preserve=None, where=None, conflict_target=None, conflict_where=None, conflict_constraint=None) :param str action: Action to take when resolving conflict. :param update: A dictionary mapping column to new value. :param preserve: A list of columns whose values should be preserved from the original INSERT. See also :class:`EXCLUDED`. :param where: Expression to restrict the conflict resolution. :param conflict_target: Column(s) that comprise the constraint. :param conflict_where: Expressions needed to match the constraint target if it is a partial index (index with a WHERE clause). :param str conflict_constraint: Name of constraint to use for conflict resolution. Currently only supported by Postgres. Represent a conflict resolution clause for a data-modification query. See :ref:`upsert` for detailed discussion. .. method:: preserve(*columns) :param columns: Columns whose values should be preserved. .. method:: update(_data=None, **kwargs) :param dict _data: Dictionary mapping column to new value. :param kwargs: Dictionary mapping column name to new value. The ``update()`` method supports being called with either a dictionary of column-to-value, **or** keyword arguments representing the same. .. method:: where(*expressions) :param expressions: Expressions that restrict the action of the conflict resolution clause. .. method:: conflict_target(*constraints) :param constraints: Column(s) to use as target for conflict resolution. .. method:: conflict_where(*expressions) :param expressions: Expressions that match the conflict target index, in the case the conflict target is a partial index. .. method:: conflict_constraint(constraint) :param str constraint: Name of constraints to use as target for conflict resolution. Currently only supported by Postgres. .. class:: EXCLUDED Helper object that exposes the ``EXCLUDED`` namespace that is used with ``INSERT ... ON CONFLICT`` to reference values in the conflicting data. This is a "magic" helper, such that one uses it by accessing attributes on it that correspond to a particular column. See :meth:`Insert.on_conflict` for example usage. Queries ------- .. class:: BaseQuery() The parent class from which all other query classes are derived. While you will not deal with :class:`BaseQuery` directly in your code, it implements some methods that are common across all query types. .. attribute:: default_row_type = ROW.DICT .. method:: bind(database=None) :param Database database: Database to execute query against. Bind the query to the given database for execution. .. method:: dicts(as_dict=True) :param bool as_dict: Specify whether to return rows as dictionaries. Return rows as dictionaries. .. method:: tuples(as_tuples=True) :param bool as_tuples: Specify whether to return rows as tuples. Return rows as tuples. .. method:: namedtuples(as_namedtuple=True) :param bool as_namedtuple: Specify whether to return rows as named tuples. Return rows as named tuples. .. method:: objects(constructor=None) :param constructor: Function that accepts row dict and returns an arbitrary object. Return rows as arbitrary objects using the given constructor. .. method:: sql() :return: A 2-tuple consisting of the query's SQL and parameters. .. method:: execute(database) :param Database database: Database to execute query against. Not required if query was previously bound to a database. Execute the query and return result (depends on type of query being executed). For example, select queries the return result will be an iterator over the query results. .. method:: iterator(database=None) :param Database database: Database to execute query against. Not required if query was previously bound to a database. Execute the query and return an iterator over the result-set. For large result-sets this method is preferable as rows are not cached in-memory during iteration. Because rows are not cached, the query may only be iterated over once. Subsequent iterations will return empty result-sets as the cursor will have been consumed. Example: .. code-block:: python query = StatTbl.select().order_by(StatTbl.timestamp).tuples() for row in query.iterator(db): process_row(row) .. method:: __iter__() Execute the query and return an iterator over the result-set. Unlike :meth:`~BaseQuery.iterator`, this method will cause rows to be cached in order to allow efficient iteration, indexing and slicing. .. method:: __getitem__(value) :param value: Either an integer index or a slice. Retrieve a row or range of rows from the result-set. .. method:: __len__() Return the number of rows in the result-set. This does not issue a ``COUNT()`` query. Instead, the result-set is loaded as it would be during normal iteration, and the length is determined from the size of the result set. .. class:: RawQuery(sql=None, params=None, **kwargs) :param str sql: SQL query. :param tuple params: Parameters (optional). Create a query by directly specifying the SQL to execute. .. class:: Query(where=None, order_by=None, limit=None, offset=None, **kwargs) :param where: Representation of WHERE clause. :param tuple order_by: Columns or values to order by. :param int limit: Value of LIMIT clause. :param int offset: Value of OFFSET clause. Base-class for queries that support method-chaining APIs. .. method:: with_cte(*cte_list) :param cte_list: zero or more :class:`CTE` objects. Include the given common-table expressions in the query. Any previously specified CTEs will be overwritten. For examples of common-table expressions, see :ref:`cte`. .. method:: cte(name, recursive=False, columns=None) :param str name: Alias for common table expression. :param bool recursive: Will this be a recursive CTE? :param list columns: List of column names (as strings). Indicate that a query will be used as a common table expression. For example, if we are modelling a category tree and are using a parent-link foreign key, we can retrieve all categories and their absolute depths using a recursive CTE: .. code-block:: python class Category(Model): name = TextField() parent = ForeignKeyField('self', backref='children', null=True) # The base case of our recursive CTE will be categories that are at # the root level -- in other words, categories without parents. roots = (Category .select(Category.name, Value(0).alias('level')) .where(Category.parent.is_null()) .cte(name='roots', recursive=True)) # The recursive term will select the category name and increment # the depth, joining on the base term so that the recursive term # consists of all children of the base category. RTerm = Category.alias() recursive = (RTerm .select(RTerm.name, (roots.c.level + 1).alias('level')) .join(roots, on=(RTerm.parent == roots.c.id))) # Express UNION ALL . cte = roots.union_all(recursive) # Select name and level from the recursive CTE. query = (cte .select_from(cte.c.name, cte.c.level) .order_by(cte.c.name)) for category in query: print(category.name, category.level) For more examples of CTEs, see :ref:`cte`. .. method:: where(*expressions) :param expressions: zero or more expressions to include in the WHERE clause. Include the given expressions in the WHERE clause of the query. The expressions will be AND-ed together with any previously-specified WHERE expressions. Example selection users where the username is equal to 'somebody': .. code-block:: python sq = User.select().where(User.username == 'somebody') Example selecting tweets made by users who are either editors or administrators: .. code-block:: python sq = Tweet.select().join(User).where( (User.is_editor == True) | (User.is_admin == True)) Example of deleting tweets by users who are no longer active: .. code-block:: python inactive_users = User.select().where(User.active == False) dq = (Tweet .delete() .where(Tweet.user.in_(inactive_users))) dq.execute() # Return number of tweets deleted. :meth:`~Query.where` calls are chainable. Multiple calls will be "AND"-ed together. .. method:: orwhere(*expressions) :param expressions: zero or more expressions to include in the WHERE clause. Include the given expressions in the WHERE clause of the query. This method is the same as the :meth:`Query.where` method, except that the expressions will be OR-ed together with any previously-specified WHERE expressions. Example: .. code-block:: python query = (User .select() .where(User.is_admin == True) .orwhere(User.is_moderator == True)) .. method:: order_by(*values) :param values: zero or more Column-like objects to order by. Define the ORDER BY clause. Any previously-specified values will be overwritten. .. method:: order_by_extend(*values) :param values: zero or more Column-like objects to order by. Extend any previously-specified ORDER BY clause with the given values. .. method:: limit(value=None) :param int value: specify value for LIMIT clause. .. method:: offset(value=None) :param int value: specify value for OFFSET clause. .. method:: paginate(page, paginate_by=20) :param int page: Page number of results (starting from 1). :param int paginate_by: Rows-per-page. Convenience method for specifying the LIMIT and OFFSET in a more intuitive way. This feature is designed with web-site pagination in mind, so the first page starts with ``page=1``. Example: .. code-block:: python @app.route('/users/') def users(): query = User.select().order_by(User.username) page = request.args.get('page') if page and page.isdigit(): page = max(1, int(page)) else: page = 1 # Render the requested page of results, displaying up to # 20 users per page. return render('users.html', users=query.paginate(page, 20)) .. class:: SelectQuery() Select query helper-class that implements operator-overloads for creating compound queries. .. method:: select_from(*columns) :param columns: one or more columns to select from the inner query. :return: a new query that wraps the calling query. Create a new query that wraps the current (calling) query. For example, suppose you have a simple ``UNION`` query, and need to apply an aggregation on the union result-set. To do this, you need to write something like: .. code-block:: sql SELECT "u"."owner", COUNT("u"."id") AS "ct" FROM ( SELECT "id", "owner", ... FROM "cars" UNION SELECT "id", "owner", ... FROM "motorcycles" UNION SELECT "id", "owner", ... FROM "boats") AS "u" GROUP BY "u"."owner" The :meth:`~SelectQuery.select_from` method is designed to simplify constructing this type of query. Example peewee code: .. code-block:: python class Car(Model): owner = ForeignKeyField(Owner, backref='cars') # ... car-specific fields, etc ... class Motorcycle(Model): owner = ForeignKeyField(Owner, backref='motorcycles') # ... motorcycle-specific fields, etc ... class Boat(Model): owner = ForeignKeyField(Owner, backref='boats') # ... boat-specific fields, etc ... cars = Car.select(Car.owner) motorcycles = Motorcycle.select(Motorcycle.owner) boats = Boat.select(Boat.owner) union = cars | motorcycles | boats query = (union .select_from(union.c.owner, fn.COUNT(union.c.id)) .group_by(union.c.owner)) .. method:: union_all(dest) Create a UNION ALL query with ``dest``. .. method:: __add__(dest) Create a UNION ALL query with ``dest``. .. method:: union(dest) Create a UNION query with ``dest``. .. method:: __or__(dest) Create a UNION query with ``dest``. .. method:: intersect(dest) Create an INTERSECT query with ``dest``. .. method:: __and__(dest) Create an INTERSECT query with ``dest``. .. method:: except_(dest) Create an EXCEPT query with ``dest``. Note that the method name has a trailing "_" character since ``except`` is a Python reserved word. .. method:: __sub__(dest) Create an EXCEPT query with ``dest``. .. class:: SelectBase() Base-class for :class:`Select` and :class:`CompoundSelect` queries. .. method:: peek(database, n=1) :param Database database: database to execute query against. :param int n: Number of rows to return. :return: A single row if n = 1, else a list of rows. Execute the query and return the given number of rows from the start of the cursor. This function may be called multiple times safely, and will always return the first N rows of results. .. method:: first(database, n=1) :param Database database: database to execute query against. :param int n: Number of rows to return. :return: A single row if n = 1, else a list of rows. Like the :meth:`~SelectBase.peek` method, except a ``LIMIT`` is applied to the query to ensure that only ``n`` rows are returned. Multiple calls for the same value of ``n`` will not result in multiple executions. The query is altered in-place so it is not possible to call :meth:`~SelectBase.first` and then later iterate over the full result-set using the same query object. Again, this is done to ensure that multiple calls to ``first()`` will not result in multiple query executions. .. method:: scalar(database, as_tuple=False, as_dict=False) :param Database database: database to execute query against. :param bool as_tuple: Return the result as a tuple? :param bool as_dict: Return the result as a dict? :return: Single scalar value. If ``as_tuple = True``, a row tuple is returned. If ``as_dict = True``, a row dict is returned. Return a scalar value from the first row of results. If multiple scalar values are anticipated (e.g. multiple aggregations in a single query) then you may specify ``as_tuple=True`` to get the row tuple. Example: .. code-block:: python query = Note.select(fn.MAX(Note.timestamp)) max_ts = query.scalar(db) query = Note.select(fn.MAX(Note.timestamp), fn.COUNT(Note.id)) max_ts, n_notes = query.scalar(db, as_tuple=True) query = Note.select(fn.COUNT(Note.id).alias('count')) assert query.scalar(db, as_dict=True) == {'count': 123} .. method:: count(database, clear_limit=False) :param Database database: database to execute query against. :param bool clear_limit: Clear any LIMIT clause when counting. :return: Number of rows in the query result-set. Return number of rows in the query result-set. Implemented by running SELECT COUNT(1) FROM (). .. method:: exists(database) :param Database database: database to execute query against. :return: Whether any results exist for the current query. Return a boolean indicating whether the current query has any results. .. method:: get(database) :param Database database: database to execute query against. :return: A single row from the database or ``None``. Execute the query and return the first row, if it exists. Multiple calls will result in multiple queries being executed. If no matching row is found this method returns ``None`` - this differs from the behavior of :meth:`ModelSelect.get` and :meth:`Model.get`, which raise ``DoesNotExist`` when no row is found. .. class:: CompoundSelectQuery(lhs, op, rhs) :param SelectBase lhs: A Select or CompoundSelect query. :param str op: Operation (e.g. UNION, INTERSECT, EXCEPT). :param SelectBase rhs: A Select or CompoundSelect query. Class representing a compound SELECT query. .. class:: Select(from_list=None, columns=None, group_by=None, having=None, distinct=None, windows=None, for_update=None, **kwargs) :param list from_list: List of sources for FROM clause. :param list columns: Columns or values to select. :param list group_by: List of columns or values to group by. :param Expression having: Expression for HAVING clause. :param distinct: Either a boolean or a list of column-like objects. :param list windows: List of :class:`Window` clauses. :param ForUpdate for_update: indicate SELECT...FOR UPDATE. Class representing a SELECT query. Rather than instantiating this directly, most-commonly you will use a factory method like :meth:`Table.select` or :meth:`Model.select`. Methods on the select query can be chained together. Example selecting some user instances from the database. Only the ``id`` and ``username`` columns are selected. When iterated, will return instances of the ``User`` model: .. code-block:: python query = User.select(User.id, User.username) for user in query: print(user.username) Example selecting users and additionally the number of tweets made by the user. The ``User`` instances returned will have an additional attribute, 'count', that corresponds to the number of tweets made: .. code-block:: python query = (User .select(User, fn.COUNT(Tweet.id).alias('count')) .join(Tweet, JOIN.LEFT_OUTER) .group_by(User)) for user in query: print(user.username, 'has tweeted', user.count, 'times') While it is possible to instantiate :class:`Select` directly, more commonly you will build the query using the method-chaining APIs. .. method:: columns(*columns) :param columns: Zero or more column-like objects to SELECT. Specify which columns or column-like values to SELECT. .. method:: select(*columns) :param columns: Zero or more column-like objects to SELECT. Same as :meth:`Select.columns`, provided for backwards-compatibility. .. method:: select_extend(*columns) :param columns: Zero or more column-like objects to SELECT. Extend the current selection with the given columns. Example: .. code-block:: python def get_users(with_count=False): query = User.select() if with_count: query = (query .select_extend(fn.COUNT(Tweet.id).alias('count')) .join(Tweet, JOIN.LEFT_OUTER) .group_by(User)) return query .. method:: from_(*sources) :param sources: Zero or more sources for the FROM clause. Specify which table-like objects should be used in the FROM clause. .. code-block:: python User = Table('users') Tweet = Table('tweets') query = (User .select(User.c.username, Tweet.c.content) .from_(User, Tweet) .where(User.c.id == Tweet.c.user_id)) for row in query.execute(db): print(row['username'], '->', row['content']) .. method:: join(dest, join_type='INNER', on=None) :param dest: A table or table-like object. :param str join_type: Type of JOIN, default is "INNER". :param Expression on: Join predicate. Join type may be one of: * ``JOIN.INNER`` * ``JOIN.LEFT_OUTER`` * ``JOIN.RIGHT_OUTER`` * ``JOIN.FULL`` * ``JOIN.FULL_OUTER`` * ``JOIN.CROSS`` Express a JOIN: .. code-block:: python User = Table('users', ('id', 'username')) Note = Table('notes', ('id', 'user_id', 'content')) query = (Note .select(Note.content, User.username) .join(User, on=(Note.user_id == User.id))) .. method:: group_by(*columns) :param values: zero or more Column-like objects to group by. Define the GROUP BY clause. Any previously-specified values will be overwritten. Additionally, to specify all columns on a given table, you can pass the table/model object in place of the individual columns. Example: .. code-block:: python query = (User .select(User, fn.Count(Tweet.id).alias('count')) .join(Tweet) .group_by(User)) .. method:: group_by_extend(*columns) :param values: zero or more Column-like objects to group by. Extend the GROUP BY clause with the given columns. .. method:: having(*expressions) :param expressions: zero or more expressions to include in the HAVING clause. Include the given expressions in the HAVING clause of the query. The expressions will be AND-ed together with any previously-specified HAVING expressions. Example: .. code-block:: python # Find users with 100 or more Tweets. query = (User .select(User, fn.Count(Tweet.id).alias('count')) .join(Tweet) .group_by(User) .having(fn.Count(Tweet.id) >= 100)) .. method:: distinct(*columns) :param columns: Zero or more column-like objects. Indicate whether this query should use a DISTINCT clause. By specifying a single value of ``True`` the query will use a simple SELECT DISTINCT. Specifying one or more columns will result in a SELECT DISTINCT ON. .. method:: window(*windows) :param windows: zero or more :class:`Window` objects. Define the WINDOW clause. Any previously-specified values will be overwritten. Example: .. code-block:: python # Equivalent example Using a Window() instance instead. window = Window(partition_by=[Sample.counter]) query = (Sample .select( Sample.counter, Sample.value, fn.AVG(Sample.value).over(window)) .window(window) # Note call to ".window()" .order_by(Sample.counter)) .. method:: for_update(for_update=True, of=None, nowait=None, skip_locked=None) :param for_update: Either a boolean or a string indicating the desired lock strength, e.g. "FOR SHARE" or "FOR NO KEY UPDATE". :param of: One or more models to restrict locking to. :param bool nowait: Specify NOWAIT option when locking. :param bool skip_locked: Specify SKIP LOCKED option when locking. .. class:: _WriteQuery(table, returning=None, **kwargs) :param Table table: Table to write to. :param list returning: List of columns for RETURNING clause. Base-class for write queries. .. method:: returning(*returning) :param returning: Zero or more column-like objects for RETURNING clause Specify the RETURNING clause of query (Postgresql and Sqlite): .. code-block:: python query = (User .insert_many([{'username': 'foo'}, {'username': 'bar'}, {'username': 'baz'}]) .returning(User.id, User.username) .namedtuples()) data = query.execute() for row in data: print('added:', row.username, 'with id=', row.id) .. seealso:: :ref:`returning-clause` for additional discussion. .. class:: Update(table, update=None, **kwargs) :param Table table: Table to update. :param dict update: Data to update. Class representing an UPDATE query. See :ref:`updating-records` for additional discussion. Example: .. code-block:: python PageView = Table('page_views') query = (PageView .update({PageView.c.page_views: PageView.c.page_views + 1}) .where(PageView.c.url == url)) query.execute(database) .. method:: from_(*sources) :param Source sources: one or more :class:`Table`, :class:`Model`, query, or :class:`ValuesList` to join with. Specify additional tables to join with using the UPDATE ... FROM syntax, which is supported by Postgres. The `Postgres documentation `_ provides additional detail, but to summarize: When a ``FROM`` clause is present, what essentially happens is that the target table is joined to the tables mentioned in the from_list, and each output row of the join represents an update operation for the target table. When using ``FROM`` you should ensure that the join produces at most one output row for each row to be modified. Example: .. code-block:: python # Update multiple users in a single query. data = [('huey', True), ('mickey', False), ('zaizee', True)] vl = ValuesList(data, columns=('username', 'is_admin'), alias='vl') # Here we'll update the "is_admin" status of the above users, # "joining" the VALUES() on the "username" column. query = (User .update(is_admin=vl.c.is_admin) .from_(vl) .where(User.username == vl.c.username)) The above query produces the following SQL: .. code-block:: sql UPDATE "users" SET "is_admin" = "vl"."is_admin" FROM ( VALUES ('huey', t), ('mickey', f), ('zaizee', t)) AS "vl"("username", "is_admin") WHERE ("users"."username" = "vl"."username") .. class:: Insert(table, insert=None, columns=None, on_conflict=None, **kwargs) :param Table table: Table to INSERT data into. :param insert: Either a dict, a list, or a query. :param list columns: List of columns when ``insert`` is a list or query. :param on_conflict: Conflict resolution strategy. Class representing an INSERT query. See :ref:`inserting-records` for additional discussion. Example: .. code-block:: python User = Table('users') query = User.insert({User.c.username: 'huey'}) query.execute(database) .. method:: as_rowcount(as_rowcount=True) :param bool as_rowcount: Whether to return the modified row count (as opposed to the last-inserted row id). SQLite and MySQL return the last inserted rowid. Postgresql will return a cursor for iterating over the inserted id(s). If you prefer to receive the inserted row-count, then specify ``as_rowcount()``: .. code-block:: python db = MySQLDatabase(...) query = User.insert_many([...]) # By default, the last rowid is returned: #last_id = query.execute() # To get the modified row-count: rowcount = query.as_rowcount().execute() .. method:: on_conflict_ignore(ignore=True) :param bool ignore: Whether to add ON CONFLICT IGNORE clause. Specify IGNORE conflict resolution strategy. .. method:: on_conflict_replace(replace=True) :param bool replace: Whether to add ON CONFLICT REPLACE clause. Specify REPLACE conflict resolution strategy (SQLite and MySQL only). .. method:: on_conflict(action=None, update=None, preserve=None, where=None, conflict_target=None, conflict_where=None, conflict_constraint=None) :param str action: Action to take when resolving conflict. If blank, action is assumed to be "update". :param update: A dictionary mapping column to new value. :param preserve: A list of columns whose values should be preserved from the original INSERT. :param where: Expression to restrict the conflict resolution. :param conflict_target: Column(s) that comprise the constraint. :param conflict_where: Expressions needed to match the constraint target if it is a partial index (index with a WHERE clause). :param str conflict_constraint: Name of constraint to use for conflict resolution. Currently only supported by Postgres. Specify the parameters for an :class:`OnConflict` clause to use for conflict resolution. See :ref:`upsert` for additional discussion. Examples: .. code-block:: python class User(Model): username = TextField(unique=True) last_login = DateTimeField(null=True) login_count = IntegerField() def log_user_in(username): now = datetime.datetime.now() # INSERT a new row for the user with the current timestamp and # login count set to 1. If the user already exists, then we # will preserve the last_login value from the "insert()" clause # and atomically increment the login-count. userid = (User .insert(username=username, last_login=now, login_count=1) .on_conflict( conflict_target=[User.username], preserve=[User.last_login], update={User.login_count: User.login_count + 1}) .execute()) return userid Example using the special :class:`EXCLUDED` namespace: .. code-block:: python class KV(Model): key = CharField(unique=True) value = IntegerField() # Create one row. KV.create(key='k1', value=1) # Demonstrate usage of EXCLUDED. # Here we will attempt to insert a new value for a given key. If that # key already exists, then we will update its value with the *sum* of its # original value and the value we attempted to insert -- provided that # the new value is larger than the original value. query = (KV.insert(key='k1', value=10) .on_conflict(conflict_target=[KV.key], update={KV.value: KV.value + EXCLUDED.value}, where=(EXCLUDED.value > KV.value))) # Executing the above query will result in the following data being # present in the "kv" table: # (key='k1', value=11) query.execute() # If we attempted to execute the query *again*, then nothing would be # updated, as the new value (10) is now less than the value in the # original row (11). .. class:: Delete() Class representing a DELETE query. See :ref:`deleting-records` for additional discussion. Example: .. code-block:: python Tweet = Table('tweets') # Delete all unpublished tweets older than 30 days. cutoff = datetime.datetime.now() - datetime.timedelta(days=30) query = (Tweet .delete() .where( (Tweet.c.is_published == False) & (Tweet.c.timestamp < cutoff))) nrows = query.execute(database) .. function:: prefetch(sq, *subqueries, prefetch_type=PREFETCH_TYPE.WHERE) :param sq: Query to use as starting-point. :param subqueries: One or more models or :class:`ModelSelect` queries to eagerly fetch. :param prefetch_type: Query type to use for the subqueries. :return: a list of models with selected relations prefetched. Eagerly fetch related objects, allowing efficient querying of multiple tables when a 1-to-many relationship exists. The prefetch type changes how the subqueries are constructed which may be desirable depending on the database engine in use. Prefetch type may be one of: * ``PREFETCH_TYPE.WHERE`` * ``PREFETCH_TYPE.JOIN`` See :ref:`relationships` for in-depth discussion of joining and prefetch. For example, it is simple to query a many-to-1 relationship efficiently: .. code-block:: python query = (Tweet .select(Tweet, User) .join(User)) for tweet in query: # Looking up tweet.user.username does not require a query since # the related user's columns were selected. print(tweet.user.username, '->', tweet.content) To efficiently do the inverse, query users and their tweets, you can use prefetch: .. code-block:: python query = User.select() for user in prefetch(query, Tweet): print(user.username) for tweet in user.tweets: # Does not require additional query. print(' ', tweet.content) Because ``prefetch`` must reconstruct a graph of models, it is necessary to be sure that the foreign-key/primary-key of any related models are selected, so that the related objects can be mapped correctly. Query-builder Internals ----------------------- .. class:: AliasManager() Manages the aliases assigned to :class:`Source` objects in SELECT queries, so as to avoid ambiguous references when multiple sources are used in a single query. .. method:: add(source) Add a source to the AliasManager's internal registry at the current scope. The alias will be automatically generated using the following scheme (where each level of indentation refers to a new scope): :param Source source: Make the manager aware of a new source. If the source has already been added, the call is a no-op. .. method:: get(source, any_depth=False) Return the alias for the source in the current scope. If the source does not have an alias, it will be given the next available alias. :param Source source: The source whose alias should be retrieved. :return: The alias already assigned to the source, or the next available alias. :rtype: str .. method:: __setitem__(source, alias) Manually set the alias for the source at the current scope. :param Source source: The source for which we set the alias. .. method:: push() Push a new scope onto the stack. .. method:: pop() Pop scope from the stack. .. class:: State(scope, parentheses=False, subquery=False, **kwargs) Lightweight object for representing the state at a given scope. During SQL generation, each object visited by the :class:`Context` can inspect the state. The :class:`State` class allows Peewee to do things like: * Use a common interface for field types or SQL expressions, but use vendor-specific data-types or operators. * Compile a :class:`Column` instance into a fully-qualified attribute, as a named alias, etc, depending on the value of the ``scope``. * Ensure parentheses are used appropriately. :param int scope: The scope rules to be applied while the state is active. :param bool parentheses: Wrap the contained SQL in parentheses. :param bool subquery: Whether the current state is a child of an outer query. :param dict kwargs: Arbitrary settings which should be applied in the current state. .. class:: Context(**settings) Converts Peewee structures into parameterized SQL queries. Peewee structures should all implement a `__sql__` method, which will be called by the `Context` class during SQL generation. The `__sql__` method accepts a single parameter, the `Context` instance, which allows for recursive descent and introspection of scope and state. .. attribute:: scope Return the currently-active scope rules. .. attribute:: parentheses Return whether the current state is wrapped in parentheses. .. attribute:: subquery Return whether the current state is the child of another query. .. method:: scope_normal(**kwargs) The default scope. Sources are referred to by alias, columns by dotted-path from the source. .. method:: scope_source(**kwargs) Scope used when defining sources, e.g. in the column list and FROM clause of a SELECT query. This scope is used for defining the fully-qualified name of the source and assigning an alias. .. method:: scope_values(**kwargs) Scope used for UPDATE, INSERT or DELETE queries, where instead of referencing a source by an alias, we refer to it directly. Similarly, since there is a single table, columns do not need to be referenced by dotted-path. .. method:: scope_cte(**kwargs) Scope used when generating the contents of a common-table-expression. Used after a WITH statement, when generating the definition for a CTE (as opposed to merely a reference to one). .. method:: scope_column(**kwargs) Scope used when generating SQL for a column. Ensures that the column is rendered with its correct alias. Was needed because when referencing the inner projection of a sub-select, Peewee would render the full SELECT query as the "source" of the column (instead of the query's alias + . + column). This scope allows us to avoid rendering the full query when we only need the alias. .. method:: sql(obj) Append a composable Node object, sub-context, or other object to the query AST. Python values, such as integers, strings, floats, etc. are treated as parameterized values. :return: The updated Context object. .. method:: literal(keyword) Append a string-literal to the current query AST. :return: The updated Context object. .. method:: parse(node) :param Node node: Instance of a Node subclass. :return: a 2-tuple consisting of (sql, parameters). Convert the given node to a SQL AST and return a 2-tuple consisting of the SQL query and the parameters. .. method:: query() :return: a 2-tuple consisting of (sql, parameters) for the context. Constants and Helpers --------------------- .. class:: Proxy() Create a proxy or placeholder for another object. .. method:: initialize(obj) :param obj: Object to proxy to. Bind the proxy to the given object. Afterwards all attribute lookups and method calls on the proxy will be sent to the given object. Any callbacks that have been registered will be called. .. method:: attach_callback(callback) :param callback: A function that accepts a single parameter, the bound object. :return: self Add a callback to be executed when the proxy is initialized. .. class:: DatabaseProxy() :class:`Proxy` subclass that is suitable to use as a placeholder for a :class:`Database` instance. See :ref:`initializing-database` for details. Example: .. code-block:: python db = DatabaseProxy() class BaseModel(Model): class Meta: database = db # ... some time later ... if app.config['DEBUG']: database = SqliteDatabase('local.db') elif app.config['TESTING']: database = SqliteDatabase(':memory:') else: database = PostgresqlDatabase('production') db.initialize(database) .. function:: chunked(iterable, n) :param iterable: an iterable that is the source of the data to be chunked. :param int n: chunk size :return: a new iterable that yields *n*-length chunks of the source data. Efficient implementation for breaking up large lists of data into smaller-sized chunks. Usage: .. code-block:: python it = range(10) # An iterable that yields 0...9. # Break the iterable into chunks of length 4. for chunk in chunked(it, 4): print(', '.join(str(num) for num in chunk)) # PRINTS: # 0, 1, 2, 3 # 4, 5, 6, 7 # 8, 9 .. class:: IndexMetadata Metadata for indexes returned by :meth:`Database.get_indexes` .. data:: name sql columns unique table .. class:: ColumnMetadata Metadata for columns returned by :meth:`Database.get_columns` .. data:: name data_type null primary_key table default .. class:: ForeignKeyMetadata Metadata for foreign keys returned by :meth:`Database.get_foreign_keys` .. data:: column dest_table dest_column table .. class:: ViewMetadata Metadata for views returned by :meth:`Database.get_views` .. data:: name sql Playhouse Reference ------------------- +---------------------------------------+---------------------------+ | Module | Section | +=======================================+===========================+ | playhouse.sqlite_ext | :ref:`sqlite` | +---------------------------------------+---------------------------+ | playhouse.cysqlite_ext | :ref:`cysqlite-ext` | +---------------------------------------+---------------------------+ | playhouse.sqliteq | :ref:`sqliteq` | +---------------------------------------+---------------------------+ | playhouse.sqlite_udf | :ref:`sqlite-udf` | +---------------------------------------+---------------------------+ | playhouse.apsw_ext | :ref:`apsw` | +---------------------------------------+---------------------------+ | playhouse.sqlcipher_ext | :ref:`sqlcipher` | +---------------------------------------+---------------------------+ | playhouse.postgres_ext | :ref:`postgresql` | +---------------------------------------+---------------------------+ | playhouse.cockroachdb | :ref:`crdb` | +---------------------------------------+---------------------------+ | playhouse.mysql_ext | :ref:`mysql` | +---------------------------------------+---------------------------+ | playhouse.db_url | :ref:`db-url` | +---------------------------------------+---------------------------+ | playhouse.pool | :ref:`pool` | +---------------------------------------+---------------------------+ | playhouse.migrate | :ref:`migrate` | +---------------------------------------+---------------------------+ | playhouse.reflection | :ref:`reflection` | +---------------------------------------+---------------------------+ | playhouse.test_utils | :ref:`test-utils` | +---------------------------------------+---------------------------+ | playhouse.fields | :ref:`extra-fields` | +---------------------------------------+---------------------------+ | playhouse.shortcuts | :ref:`shortcuts` | +---------------------------------------+---------------------------+ | playhouse.hybrid | :ref:`hybrid` | +---------------------------------------+---------------------------+ | playhouse.kv | :ref:`kv` | +---------------------------------------+---------------------------+ | playhouse.signals | :ref:`signals` | +---------------------------------------+---------------------------+ | playhouse.dataset | :ref:`dataset` | +---------------------------------------+---------------------------+ | playhouse.flask_utils | :ref:`flask-utils` | +---------------------------------------+---------------------------+ ================================================ FILE: docs/peewee/asyncio.rst ================================================ .. _pwasyncio: Async Support ============= .. module:: playhouse.pwasyncio Peewee's async extension bridges blocking query execution to the asyncio event loop using ``greenlet``. When database I/O occurs inside a greenlet, control is transparently yielded to the event loop until the driver completes the operation. This allows synchronous Peewee code to run unmodified within an async context. Example ------- ``playhouse.pwasyncio`` contains the async database implementations. Typically this is the only thing you will need in order to use Peewee with asyncio: .. code-block:: python import asyncio from peewee import * from playhouse.pwasyncio import AsyncSqliteDatabase db = AsyncSqliteDatabase('my_app.db') class User(db.Model): name = TextField() Queries must be executed through an async execution method. This ensures that when blocking would occur, control is properly yielded to the event loop. The database context (``async with db``) acquires a connection from the pool and releases it on exit: .. code-block:: python async def main(): async with db: await db.acreate_tables([User]) # Create a new user in a transaction. async with db.atomic() as txn: user = await db.run(User.create, name='Charlie') # Fetch a single row from the database. charlie = await db.get(User.select().where(User.name == 'Charlie')) assert charlie.name == user.name # Execute a query and iterate results. for user in await db.list(User.select().order_by(User.name)): print(user.name) # Async lazy result fetching (uses server-side cursors where # available). query = User.select().order_by(User.name) async for user in db.iterate(query): print(user.name) await db.close_pool() asyncio.run(main()) Installation ------------ Requires Python 3.8 or newer, ``greenlet`` and an async database driver: .. code-block:: shell pip install peewee greenlet pip install aiosqlite # SQLite pip install asyncpg # Postgresql pip install aiomysql # MySQL / MariaDB Supported backends: ================ ============ ==================================== Database Driver Peewee class ================ ============ ==================================== SQLite aiosqlite :class:`AsyncSqliteDatabase` Postgresql asyncpg :class:`AsyncPostgresqlDatabase` MySQL / MariaDB aiomysql :class:`AsyncMySQLDatabase` ================ ============ ==================================== Execution Methods ----------------- ``db.run()`` - general-purpose entry point ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :meth:`~AsyncDatabaseMixin.run` accepts any callable and runs it inside a greenlet bridge. The callable can contain arbitrary synchronous Peewee code, including transactions: .. code-block:: python # Single operation: user = await db.run(User.create, name='Alice') # Multi-step function: def register(username, bio): with db.atomic(): user = User.create(name=username) Profile.create(user=user, bio=bio) return user user = await db.run(register, 'alice', 'Python developer') Use ``db.run()`` when: * You have existing synchronous code you want to call from async. * A single operation involves multiple queries (e.g. a transaction). Async Helper Methods ^^^^^^^^^^^^^^^^^^^^^ For single-query operations, the async helpers are more direct: .. code-block:: python # Execute any query and get its natural return type. cursor = await db.aexecute(query) # Use a transaction: async with db.atomic() as tx: await db.run(User.create, name='Bob') # SELECT and return one model instance (raises DoesNotExist if none). user = await db.get(User.select().where(User.name == 'Alice')) # SELECT and return a list. users = await db.list(User.select().order_by(User.name)) # SELECT and stream results from the database asynchronously. users = [user async for user in db.iterate(User.select())] # SELECT and return a scalar value. count = await db.scalar(User.select(fn.COUNT(User.id))) # Or user shortcut. count = await db.count(User.select()) # CREATE TABLE / DROP TABLE: await db.acreate_tables([User, Tweet]) await db.adrop_tables([User, Tweet]) # Raw SQL: cursor = await db.aexecute_sql('SELECT 1') print(cursor.fetchall()) # [(1,)] Transactions ^^^^^^^^^^^^^ Use ``async with db.atomic()`` for async-aware transactions: .. code-block:: python async with db.atomic(): await db.run(User.create, name='Alice') await db.run(User.create, name='Bob') # Nesting and explicit commit/rollback work. async with db.atomic() as nested: await db.aexecute(User.delete().where(User.name == 'Bob')) await nested.arollback() # Un-delete Bob. # Both Alice and Bob are in the database. Or wrap transactional code in ``db.run()``: .. code-block:: python def create_users(): with db.atomic(): User.create(name='Alice') User.create(name='Bob') with db.atomic() as nested: User.delete().where(User.name == 'Bob').execute() nested.rollback() # Un-delete Bob. await db.run(create_users) # Both Alice and Bob are in the database. Both approaches produce the same result. The ``db.run()`` form is often simpler when the transactional logic involves many inter-dependent queries. Connection Management --------------------- The database context manager (``async with db``) is the recommended way to manage connections. It acquires a connection on entry and releases it on exit: .. code-block:: python async with db: # Connection is available here. pass # Connection released. Explicit control is also available: .. code-block:: python await db.aconnect() # Acquire connection for the current task. # ... queries ... await db.aclose() # Release connection back to pool. Each asyncio task gets its own connection from the pool. **Connections are not shared between tasks**. Each async task will have it's own connection and transaction state - this prevents bugs that may occur when connections are shared and transactions end up interleaved across several running tasks. To shut down completely (e.g. during application teardown): .. code-block:: python await db.close_pool() MySQL and Postgresql ^^^^^^^^^^^^^^^^^^^^ MySQL and Postgresql use the driver's native connection pool. Pool configuration options include: * ``pool_size``: Maximum number of connections * ``pool_min_size``: Minimum pool size * ``acquire_timeout``: Timeout when acquiring a connection .. code-block:: python db = AsyncPostgresqlDatabase( 'peewee_test', host='localhost', user='postgres', pool_size=10, pool_min_size=1, acquire_timeout=10) SQLite ^^^^^^ Peewee provides a simple connection-pooling implementation for SQLite connections. Pool configuration options include: * ``pool_size``: Maximum number of connections * ``acquire_timeout``: Timeout when acquiring a connection SQLite operates on local disk storage, so queries typically execute extremely quickly. The cost of dispatching to a background thread and wrapping in coroutines increases the latency per query. For every query executed, a closure must be created, a future allocated, a queue written-to, a loop ``call_soon_threadsafe()`` issued, and two context switches made. This is the case with `aiosqlite `__. Additionally, SQLite only allows one writer at a time, so while using an async wrapper may keep things responsive while waiting to obtain the write lock, writes will not occur "faster", the bottleneck has merely been moved. Conversely, if you don’t have that much load, the async wrapper adds complexity and overhead for no measurable benefit. To use SQLite in an async environment anyways, it is strongly recommended to use WAL-mode at a minimum, which allows multiple readers to co-exist with a single writer: .. code-block:: python db = AsyncSqliteDatabase('app.db', pragmas={'journal_mode': 'wal'}) Sharp Corners ------------- Lazy foreign key access outside ``db.run()`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Accessing a lazy foreign key attribute triggers a synchronous query if the object has not been populated. Outside a greenlet context, this raises ``MissingGreenletBridge``: .. code-block:: python tweet = await db.get(Tweet.select()) # FAILS: triggers a SELECT outside the greenlet bridge. print(tweet.user.name) # MissingGreenletBridge: Attempted query outside greenlet runner. Fix by selecting the related model in the original query: .. code-block:: python query = Tweet.select(Tweet, User).join(User) tweet = await db.get(query) print(tweet.user.name) # OK - no extra query. Or by wrapping the access in ``db.run()``: .. code-block:: python name = await db.run(lambda: tweet.user.name) The safest approach is to disable lazy-loading on your foreign-key fields and enforce selecting relations via explicit joins. .. code-block:: python class Tweet(db.Model): user = ForeignKeyField(User, backref='tweets', lazy_load=False) ... Iterating back-references outside ``db.run()`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Iterating a back-reference outside a greenlet context also fails for the same reason as above. .. code-block:: python # FAILS: for tweet in user.tweets: print(tweet.content) Solutions: .. code-block:: python # Using db.list(): for tweet in await db.list(user.tweets): print(tweet.content) # Using db.run() with list(): tweets = await db.run(list, user.tweets) # Use prefetch: users = await db.run( prefetch, User.select().where(User.username.in_(('Charlie', 'Huey', 'Mickey'))) Tweet.select()) for user in users: for tweet in user.tweets: # Prefetched - no extra query. print(tweet.content) Any code that triggers a database query must execute via either ``db.run()`` or one of the async helper methods. API Reference ------------- .. class:: AsyncDatabaseMixin(database, pool_size=10, pool_min_size=1, acquire_timeout=10, **kwargs) :param str database: Database name or filename for SQLite. :param int pool_size: Maximum size of the driver-managed connection pool (no-op for SQLite). :param int pool_min_size: Minimum size of the driver-managed connection pool (no-op for SQLite). :param float acquire_timeout: Time (in seconds) to wait for a free connection when acquiring from the pool. :param kwargs: Arbitrary keyword arguments passed to the underlying database driver when creating connections (e.g., ``user``, ``password``, ``host``). Mixin class providing asyncio execution support. Use a driver-specific subclass in application code: * :class:`AsyncSqliteDatabase` * :class:`AsyncPostgresqlDatabase` * :class:`AsyncMySQLDatabase` Each asyncio task maintains its own connection state and transaction stack. Connections are acquired and released back to the pool when the task completes or the database context exits. .. method:: run(fn, *args, **kwargs) :async: :param fn: A synchronous callable. :returns: The return value of ``fn(*args, **kwargs)``. Execute a synchronous callable inside a greenlet and return the result. This is the primary entry point for executing Peewee ORM code in an async context. When database I/O or blocking would occur, control is yielded to the event-loop automatically. Example: .. code-block:: python db = AsyncSqliteDatabase(':memory:') class User(db.Model): username = TextField() def setup_app(): # Ensure table exists and admin user is present at startup. with db: db.create_tables([User]) # Create admin user if does not exist. try: with db.atomic(): User.create(username='admin') except IntegrityError: pass async def main(): await db.run(setup_app) # We can pass arguments to the synchronous callable and get # return values as well. admin_user = await db.run(User.get, User.username == 'admin') .. method:: aconnect() :async: :return: A wrapped async connection. Acquire a connection from the pool for the current task. Typically the connection is not used directly, since the connection will be bound to the task using a task-local. Example: .. code-block:: python # Acquire a connection from the pool which will be used for the # current asyncio task. await db.aconnect() # Run some queries. users = await db.list(User.select().order_by(User.username)) for user in users: print(user.username) # Close connection, which releases it back to the pool. await db.aclose() Typically applications should prefer to use the async context-manager for connection management, e.g.: .. code-block:: python db = AsyncSqliteDatabase(':memory:') async with db: # Connection is obtained from the pool and used for this task. await db.acreate_tables([User, Tweet]) # Context block exits, connection is released back to pool. .. method:: aclose() :async: Release the current task's connection back to the pool. .. method:: close_pool() :async: Close the underlying connection pool and release all active connections. This method should be called during application shutdown. .. method:: __aenter__() __aexit__(exc_type, exc, tb) :async: Async database context, acquiring a connection for the current task for the duration of the wrapped block. .. code-block:: python db = AsyncSqliteDatabase(':memory:') async with db: # Connection is obtained from the pool and used for this task. await db.acreate_tables([User, Tweet]) # Context block exits, connection is released back to pool. .. method:: aexecute(query) :async: :param Query query: a Select, Insert, Update or Delete query. :return: the normal return-value for the query type. Execute any Peewee query object and return its result. Example: .. code-block:: python insert = User.insert(username='Huey') pk = await db.aexecute(insert) update = (Tweet .update(is_published=True) .where(Tweet.timestamp <= datetime.now())) nrows = await db.aexecute(update) spammers = (User .delete() .where(User.username.contains('billing')) .returning(User.username)) for u in await db.aexecute(spammers): print(f'Deleted: {u.username}') .. method:: get(query) :async: :param Query query: a Select query. Execute a SELECT query and return a single model instance. Raises :exc:`~Model.DoesNotExist` if no row matches. Example: .. code-block:: python huey = await db.get(User.select().where(User.username == 'Huey')) # Fetch a model and a relation in single query. query = Tweet.select(Tweet, User).join(User).where(Tweet.id == 123) tweet = await db.get(query) print(tweet.user.username, '->', tweet.content) .. method:: list(query) :async: :param Query query: a Select query, or an Insert, Update or Delete query that utilizes RETURNING. Execute a SELECT (or INSERT/UPDATE/DELETE with RETURNING) and return a list of results. Example: .. code-block:: python query = User.select().order_by(User.username) async for user in db.list(query): print(user.username) .. method:: iterate(query) :async: :param Query query: a Select query to stream results from using an async generator. :meth:`~AsyncDatabaseMixin.iterate` method uses server-side cursors (MySQL and Postgres) to efficiently stream large result-sets. Example: .. code-block:: python query = User.select().order_by(User.username) async for user in db.iterate(query): print(user.username) .. method:: scalar(query) :async: :param Query query: a Select query. Execute a SELECT and return the first column of the first row. Example: .. code-block:: python max_id = await db.scalar(User.select(fn.MAX(User.id))) .. method:: count(query) :async: :param Query query: a Select query. Wrap the query in a SELECT COUNT(...) and return the count of rows. Example: .. code-block:: python tweets = await db.count(Tweet.select().where(Tweet.is_published)) .. method:: exists(query) :async: :param Query query: a Select query. Return boolean whether the query contains any results. .. method:: aprefetch(query, *subqueries) :async: :param Query query: Query to use as starting-point. :param subqueries: One or more models or :class:`ModelSelect` queries to eagerly fetch. :return: a list of models with selected relations prefetched. Eagerly fetch related objects, allowing efficient querying of multiple tables when a 1-to-many relationship exists. .. code-block:: python users = User.select().order_by(User.username) tweets = Tweet.select().order_by(Tweet.timestamp) for user in await db.aprefetch(users, tweets): print(user.username) for tweet in user.tweets: print(' ', tweet.content) .. method:: atomic() Return an async-aware atomic context manager. Supports both ``async with`` and ``with``. Example of async usage: .. code-block:: python async def transfer_funds(src, dest, amount): async with db.atomic() as txn: await db.aexecute( Account .update(balance=Account.balance - amount) .where(Account.id == src.id)) await db.aexecute( Account .update(balance=Account.balance + amount) .where(Account.id == dest.id)) async def main(): await transfer_funds(user1, user2, 100.) Example of sync usage: .. code-block:: python def transfer_funds(src, dest, amount): with db.atomic() as txn: (Account .update(balance=Account.balance - amount) .where(Account.id == src.id) .execute()) (Account .update(balance=Account.balance + amount) .where(Account.id == dest.id) .execute()) async def main(): await db.run(transfer_funds, user1, user2, 100.) .. method:: acreate_tables(models, **options) :async: :param list models: A list of :class:`Model` classes. :param options: Options to specify when calling :meth:`Model.create_table`. Create tables, indexes and associated constraints for the given list of models. Dependencies are resolved so that tables are created in the appropriate order. Example: .. code-block:: python class User(db.Model): ... class Tweet(db.Model): ... async def setup_hook(): async with db: await db.acreate_tables([User, Tweet]) .. method:: adrop_tables(models, **options) :async: :param list models: A list of :class:`Model` classes. :param kwargs: Options to specify when calling :meth:`Model.drop_table`. Drop tables, indexes and constraints for the given list of models. .. method:: aexecute_sql(sql, params=None) :async: :param str sql: SQL query to execute. :param tuple params: Optional query parameters. :returns: A :class:`CursorAdapter` instance. Execute SQL asynchronously. Returns a cursor-like object whose rows are already fetched (call ``.fetchall()`` synchronously). For result streaming, see :meth:`~AsyncDatabaseMixin.iterate`. .. class:: AsyncSqliteDatabase(database, **kwargs) Async SQLite database implementation. Uses ``aiosqlite`` and maintains a single shared connection. Pool-related configuration options are ignored. Inherits from :class:`AsyncDatabaseMixin` and :class:`SqliteDatabase`. .. class:: AsyncPostgresqlDatabase(database, **kwargs) Async Postgresql database implementation. Uses ``asyncpg`` and the driver's native connection pool. Inherits from :class:`AsyncDatabaseMixin` and :class:`PostgresqlDatabase`. .. class:: AsyncMySQLDatabase(database, **kwargs) Async MySQL / MariaDB database implementation. Uses ``aiomysql`` and the driver's native connection pool. Inherits from :class:`AsyncDatabaseMixin` and :class:`MySQLDatabase`. .. class:: MissingGreenletBridge(RuntimeError) Raised when Peewee attempts to execute a query outside a greenlet context. This indicates that a query was triggered outside of ``db.run()`` or an async helper call. ================================================ FILE: docs/peewee/contributing.rst ================================================ .. _contributing: Contributing ============ In order to continually improve, Peewee needs the help of developers like you. Whether it's contributing patches, submitting bug reports, or just asking and answering questions, you are helping to make Peewee a better library. In this document I'll describe some of the ways you can help. Patches ------- Do you have an idea for a new feature, or is there a clunky API you'd like to improve? Before coding it up and submitting a pull-request, `open a new issue `_ on GitHub describing your proposed changes. This doesn't have to be anything formal, just a description of what you'd like to do and why. When you're ready, you can submit a pull-request with your changes. Successful patches will have the following: * Unit tests. * Documentation, both prose form and general :ref:`API documentation `. * Code that conforms stylistically with the rest of the Peewee codebase. Bugs ---- If you've found a bug, please check to see if it has `already been reported `_, and if not `create an issue on GitHub `_. The more information you include, the more quickly the bug will get fixed, so please try to include the following: * Traceback and the error message (please format your code). * Relevant portions of your code or code to reproduce the error * Peewee version: ``python -c "from peewee import __version__; print(__version__)"`` * Which database you're using If you have found a bug in the code and submit a failing test-case, then hats-off to you, you are a hero! Questions --------- If you have questions about how to do something with peewee, then I recommend either: * Ask on StackOverflow. I check SO just about every day for new peewee questions and try to answer them. This has the benefit also of preserving the question and answer for other people to find. * Ask on the mailing list, https://groups.google.com/group/peewee-orm ================================================ FILE: docs/peewee/database.rst ================================================ .. _database: Database ======== The Peewee :class:`Database` object represents a connection to a database. The :class:`Database` class is instantiated with all the information needed to connect to a database. Database responsibilities: * :ref:`connection-lifecycle` * :ref:`executing-sql` * :ref:`Manage database schema ` * :ref:`Manage transactions ` Peewee supports: * SQLite - :class:`SqliteDatabase` using the standard library ``sqlite3``. .. code-block:: python # SQLite database (use WAL journal mode and 64MB cache). db = SqliteDatabase('/path/to/app.db', pragmas={ 'journal_mode': 'wal', 'cache_size': -64000}) * Postgresql - :class:`PostgresqlDatabase` using ``psycopg2`` or ``psycopg3``. .. code-block:: python db = PostgresqlDatabase( 'my_app', user='postgres', password='secret', host='10.8.0.9', port=5432) * MySQL and MariaDB - :class:`MySQLDatabase` using ``pymysql``. .. code-block:: python db = MySQLDatabase( 'my_app', user='app', password='db_password', host='10.8.0.8', port=3306) Using SQLite ------------ To connect to a SQLite database, use :class:`SqliteDatabase`. The first parameter is the filename containing the database, or the string ``':memory:'`` to create an in-memory database. After the database filename specify pragmas or other `sqlite3 parameters `__. .. code-block:: python db = SqliteDatabase('my_app.db', pragmas={'journal_mode': 'wal'}) class BaseModel(Model): """Base model that will use our Sqlite database.""" class Meta: database = db class User(BaseModel): username = TextField() ... SQLite-specific options are set via `pragmas `__. The following settings are recommended for most applications: .. code-block:: python db = SqliteDatabase('my_app.db', pragmas={ 'journal_mode': 'wal', # Allow readers while writer active. 'cache_size': -64000, # 64 MB page cache. 'foreign_keys': 1, # Enforce FK constraints. }) ======================= =================== ================================================ Pragma Recommended value Effect ======================= =================== ================================================ ``journal_mode`` ``wal`` Allow concurrent readers and one writer. ``cache_size`` Negative KiB value E.g. ``-64000`` = 64 MB. ``foreign_keys`` ``1`` Enforce ``FOREIGN KEY`` constraints. ======================= =================== ================================================ .. seealso:: For SQLite-specific features and extensions (JSON, full-text search), see :ref:`sqlite`. Using Postgresql ---------------- To use Peewee with Postgresql install ``psycopg2`` or ``psycopg3``: .. code-block:: shell pip install "psycopg2-binary" # Psycopg2. pip install "psycopg[binary]" # Psycopg3. To connect to a Postgresql database, use :class:`PostgresqlDatabase`. The first parameter is always the name of the database. After the database name specify additional `psycopg2 `__ or `psycopg3 `__ connection parameters: .. code-block:: python db = PostgresqlDatabase( 'my_database', user='postgres', password='secret', host='10.8.0.1', port=5432) class BaseModel(Model): """A base model that will use our Postgresql database""" class Meta: database = db class User(BaseModel): username = CharField() ... The isolation level can be set at initialization time: .. code-block:: python # psycopg2 or psycopg3 db = PostgresqlDatabase('app', isolation_level='SERIALIZABLE') # psycopg2 from psycopg2.extensions import ISOLATION_LEVEL_SERIALIZABLE db = PostgresqlDatabase('app', isolation_level=ISOLATION_LEVEL_SERIALIZABLE) # psycopg3 from psycopg import IsolationLevel db = PostgresqlDatabase('app', isolation_level=IsolationLevel.SERIALIZABLE) .. seealso:: For Postgresql-specific functionality and extensions (arrays, JSONB, full-text search), see :ref:`postgresql`. Using MySQL / MariaDB --------------------- To use Peewee with MySQL or MariaDB install ``pymysql``: .. code-block:: shell pip install pymysql To connect to a MySQL or MariaDB database, use :class:`MySQLDatabase`. The first parameter is always the name of the database. After the database name specify additional `pymysql Connection parameters `__: .. code-block:: python db = MySQLDatabase( 'my_database', host='10.8.0.1', port=3306, connection_timeout=5) class BaseModel(Model): """A base model that will use our MySQL database""" class Meta: database = mysql_db class User(BaseModel): username = CharField() # ... If MySQL drops idle connections (``Error 2006: MySQL server has gone away``), the solution is explicit connection management: open a connection at the start of each unit of work and close it when finished. See :ref:`connection-lifecycle` and :ref:`framework-integration`. Alternate drivers are available for both databases: * :class:`.MySQLConnectorDatabase` - uses ``mysql-connector-python``. * :class:`.MariaDBConnectorDatabase` - uses ``mariadb-connector-python``. .. seealso:: For MySQL-specific functionality and extensions, see :ref:`mysql`. Connection Parameters --------------------- :class:`Database` initialization methods expect the name of the database as the first parameter. Subsequent keyword arguments are passed to the underlying database driver when establishing the connection. With Postgresql it is common to need to specify the ``host``, ``user`` and ``password`` when creating a connection. These should be specified when initializing the database, and they will be passed directly back to ``psycopg`` when creating connections: .. code-block:: python db = PostgresqlDatabase( 'database_name', # Required by Peewee. user='postgres', # Will be passed directly to psycopg. password='secret', # Ditto. host='db.mysite.com') # Ditto. As another example, the ``pymysql`` driver accepts a ``charset`` parameter which is not a standard Peewee :class:`Database` parameter. To set this value, pass in ``charset`` alongside your other settings: .. code-block:: python db = MySQLDatabase('database_name', user='www-data', charset='utf8mb4') Consult your database driver's documentation for the available parameters: * Postgresql: `psycopg2 `__ or `psycopg3 `__ * MySQL: `pymysql `__ * SQLite: `sqlite3 `__ .. _initializing-database: Initializing the Database ------------------------- There are three ways to initialize a database: 1. **Initialize database directly**. Use when connection settings are available at the time the database is declared: .. code-block:: python db = SqliteDatabase('/path/to/app.db') Environment variables, config settings, etc. typically fall into this category as well: .. code-block:: python import os db = PostgresqlDatabase( database=app.config['APP_NAME'], user=os.environ.get('PGUSER') or 'postgres', host=os.environ.get('PGHOST') or '127.0.0.1') 2. **Defer initialization**. This method is needed when a connection setting is not available until run-time **or** it is inconvenient to import connection settings where the database is declared: .. code-block:: python db = PostgresqlDatabase(None) # ... some time later ... db_name = input('Enter database name: ') # Initialize the database now. db.init(db_name, user='postgres', host='10.8.0.1') Attempting to use an uninitialized database will raise an :class:`InterfaceError`. 3. **Proxy**. Use a :class:`DatabaseProxy` and set the database at run-time. This method is needed when the database implementation may change at run-time. For example it may be either Sqlite or Postgresql depending on a command-line option: .. code-block:: python db = DatabaseProxy() # ... some time later ... if app.config['DEBUG']: database = SqliteDatabase('local.db') elif app.config['TESTING']: database = SqliteDatabase(':memory:') else: database = PostgresqlDatabase('production') db.initialize(database) Attempting to use an uninitialized database proxy will raise an ``AttributeError``. .. _binding_database: Changing the database at run-time ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Peewee can also set or change the database at run-time in a different way. This technique is used by the Peewee test suite to **bind** test model classes to various database instances when running tests. There are two sets of complementary methods: * :meth:`Database.bind` and :meth:`Model.bind` - bind one or more models to a database. * :meth:`Database.bind_ctx` and :meth:`Model.bind_ctx` - which are the same as their ``bind()`` counterparts, but return a context-manager and are useful when the database should only be changed temporarily. As an example, we'll declare two models **without** specifying any database: .. code-block:: python class User(Model): username = TextField() class Tweet(Model): user = ForeignKeyField(User, backref='tweets') content = TextField() timestamp = TimestampField() Bind the models to a database at run-time: .. code-block:: python :emphasize-lines: 7, 10 postgres_db = PostgresqlDatabase('my_app', user='postgres') sqlite_db = SqliteDatabase('my_app.db') # At this point, the User and Tweet models are NOT bound to any database. # Bind them to the Postgres database: postgres_db.bind([User, Tweet]) # Temporarily bind them to the sqlite database: with sqlite_db.bind_ctx([User, Tweet]): # User and Tweet are now bound to the sqlite database. assert User._meta.database is sqlite_db # User and Tweet are once again bound to the Postgres database. assert User._meta.database is postgres_db The :meth:`Model.bind` and :meth:`Model.bind_ctx` methods work the same for binding a given model class: .. code-block:: python :emphasize-lines: 3, 9 # Bind the user model to the sqlite db. By default, Peewee will also # bind any models that are related to User via foreign-key as well. User.bind(sqlite_db) assert User._meta.database is sqlite_db assert Tweet._meta.database is sqlite_db # Related models bound too. # Temporarily bind *just* the User model to the postgres db. with User.bind_ctx(postgres_db, bind_backrefs=False): assert User._meta.database is postgres_db assert Tweet._meta.database is sqlite_db # Has not changed. # User is back to being bound to the sqlite_db. assert User._meta.database is sqlite_db Peewee database connections are thread-safe. However, if you plan to **bind** the database at run-time in a multi-threaded application, storing the model's database in a thread-local is necessary. This can be accomplished with the :class:`~playhouse.shortcuts.ThreadSafeDatabaseMetadata`. .. code-block:: python from peewee import * from playhouse.shortcuts import ThreadSafeDatabaseMetadata class BaseModel(Model): class Meta: model_metadata_class = ThreadSafeDatabaseMetadata The database can now be swapped safely while running in a multi-threaded environment using the :meth:`Database.bind` or :meth:`Database.bind_ctx`. Connecting via URL ------------------ The :ref:`db-url` playhouse module provides a :func:`~playhouse.db_url.connect` helper that accepts a database URL and returns the appropriate database instance: .. code-block:: python from playhouse.db_url import connect db = connect(os.environ.get('DATABASE_URL', 'sqlite:///local.db')) Example URLs: * ``sqlite:///my_app.db`` - SQLite file in the current directory. * ``sqlite:///:memory:`` - in-memory SQLite. * ``sqlite:////absolute/path/to/app.db`` - absolute path SQLite. * ``postgresql://user:password@host:5432/dbname`` * ``mysql://user:password@host:3306/dbname`` * :ref:`More examples in the db_url documentation `. .. _connection-lifecycle: Connection Lifecycle -------------------- Applications will generally fall into two categories: * Single-user applications which open a connection at startup and close at exit. * Multi-user or web appliactions, which open a connection per request and close it at the end of the request. To open a connection to a database, use the :meth:`Database.connect` method: .. code-block:: pycon >>> db = SqliteDatabase(':memory:') # In-memory SQLite database. >>> db.connect() True >>> pass # ... do work ... >>> db.close() True If you call ``connect()`` on an already-open database, an :exc:`OperationalError` is raised. Pass ``reuse_if_open=True`` to suppress it: .. code-block:: pycon >>> db.connect(reuse_if_open=True) To close a connection, use the :meth:`Database.close` method: .. code-block:: pycon >>> db.close() True Calling ``close()`` on an already-closed connection will not result in an exception, but will return ``False``: .. code-block:: pycon >>> db.connect() # Open connection. True >>> db.close() # Close connection. True >>> db.close() # Connection already closed, returns False. False Determine whether the database is closed using the :meth:`Database.is_closed` method: .. code-block:: pycon >>> db.is_closed() True Web applications will typically use framework-provided hooks to manage connection lifecycles. .. code-block:: python @app.before_request def _db_connect(): db.connect() @app.teardown_request def _db_close(exc): if not db.is_closed(): db.close() See :ref:`framework-integration` for framework-specific examples. .. tip:: Peewee uses thread local storage to manage connection state, so this pattern can be used with multi-threaded or gevent applications. Peewee's :ref:`asyncio integration ` stores connection state in task-local storage, so the same pattern applies. Context managers ^^^^^^^^^^^^^^^^ The database object can be used as a context manager or decorator. 1. Connection opens when context manager is entered. 2. Peewee begins a transaction. 3. Control is passed to user for duration of block. 4. Peewee commits transaction if block exits cleanly, otherwise issues a rollback. 5. Peewee closes the connection. 6. Any unhandled exception is raised. .. code-block:: python with db: User.create(username='charlie') # Transaction is committed when the block exits normally, # rolled back if an exception is raised. Decorator: .. code-block:: python @db def demo(): print('closed?', db.is_closed()) demo() # "closed? False" db.is_closed() # True To manage the connection lifetime without an implicit transaction, use :meth:`~Database.connection_context`: .. code-block:: python with db.connection_context(): # Connection is open; no implicit transaction. results = User.select() ``connection_context()`` can also decorate a function: .. code-block:: python @db.connection_context() def load_fixtures(): db.create_tables([User, Tweet]) import_data() Using autoconnect ^^^^^^^^^^^^^^^^^ By default Peewee will automatically open a connection if one is not available. This behavior is controlled by the ``autoconnect`` Database parameter. Managing connections explicitly is considered a **best practice**, therefore consider disabling the ``autoconnect`` behavior: .. code-block:: python db = PostgresqlDatabase('app', autoconnect=False) It is helpful to be explicit about connection lifetimes. If a connection cannot be opened, the exception will be caught when the connection is being opened, rather than at query time. Thread safety ^^^^^^^^^^^^^ Database connections and associated transactions are thread-safe. Peewee keeps track of the connection state using thread-local storage, making the Peewee :class:`Database` object safe to use with multiple threads. Each thread will have it's own connection, and as a result any given thread will only have a single connection open at a given time. Peewee's :ref:`asyncio integration ` stores connection state in task-local storage, so the same applies to async applications. DB-API Connection object ^^^^^^^^^^^^^^^^^^^^^^^^ :meth:`Database.connection` returns a reference to the underlying DB-API driver connection. This method will return the currently-open connection object, if one exists, otherwise it will open a new connection. .. code-block:: pycon >>> db.connection() .. _connection-pooling: Connection Pooling ------------------ For web applications that handle many requests, opening and closing a database connection on every request adds latency. A connection pool keeps a set of connections open and lends them out as needed. Pooled database classes are available in :ref:`playhouse.pool `: .. code-block:: python from playhouse.pool import PooledPostgresqlDatabase db = PooledPostgresqlDatabase( 'my_app', user='postgres', max_connections=20, stale_timeout=300, # Recycle connections idle for 5 minutes. ) .. include:: pool-snippet.rst When using a connection pool, :meth:`~Database.connect` and :meth:`~Database.close` do not open and close real connections - they acquire and release connections from the pool. It is therefore essential to call both explicitly (or use a context manager) so connections are returned to the pool for re-use. .. seealso:: :ref:`pool` .. _executing-sql: Executing SQL ------------- SQL queries will typically be executed by calling ``execute()`` on a query constructed using the query-builder APIs (or by simply iterating over a query object in the case of a :class:`Select` query). For cases where you wish to execute SQL directly, use the :meth:`Database.execute_sql`: .. code-block:: python db = SqliteDatabase('my_app.db') db.connect() # Example of executing a simple query and ignoring the results. db.execute_sql("ATTACH DATABASE ':memory:' AS cache;") # Example of iterating over the results of a query using the cursor. cursor = db.execute_sql('SELECT * FROM users WHERE status = ?', (ACTIVE,)) for row in cursor.fetchall(): # Do something with row, which is a tuple containing column data. pass .. _database-errors: Database Errors --------------- The Python DB-API 2.0 spec describes `several types of exceptions `_. Because most database drivers have their own implementations of these exceptions, Peewee simplifies things by providing its own wrappers around any implementation-specific exception classes. That way, you don't need to worry about dealing with driver-specific exception classes, you can just use the ones from peewee: * :class:`DatabaseError` * :class:`DataError` * :class:`IntegrityError` * :class:`InterfaceError` * :class:`InternalError` * :class:`NotSupportedError` * :class:`OperationalError` * :class:`ProgrammingError` All of these error classes extend :class:`PeeweeException`. Logging Queries --------------- Peewee logs every query to the ``peewee`` namespace at ``DEBUG`` level using the standard library ``logging`` module: .. code-block:: python import logging logging.getLogger('peewee').addHandler(logging.StreamHandler()) logging.getLogger('peewee').setLevel(logging.DEBUG) This is the simplest way to verify what queries are being issued during development. .. _testing: Testing Peewee Applications --------------------------- When writing tests for an application that uses Peewee, it may be desirable to use a special database for tests. Another common practice is to run tests against a clean database, which means ensuring tables are empty at the start of each test. Binding models to a database at run-time is described here: :ref:`binding_database`. Example test-case setup: .. code-block:: python # tests.py import unittest from my_app.models import EventLog, Relationship, Tweet, User MODELS = [User, Tweet, EventLog, Relationship] # use an in-memory SQLite for tests. test_db = SqliteDatabase(':memory:') class BaseTestCase(unittest.TestCase): def setUp(self): # Bind model classes to test db. Since we have a complete list of # all models, we do not need to recursively bind dependencies. test_db.bind(MODELS, bind_refs=False, bind_backrefs=False) test_db.connect() test_db.create_tables(MODELS) def tearDown(self): # Not strictly necessary since SQLite in-memory databases only live # for the duration of the connection, and in the next step we close # the connection...but a good practice all the same. test_db.drop_tables(MODELS) # Close connection to db. test_db.close() # If we wanted, we could re-bind the models to their original # database here. But for tests this is probably not necessary. It is recommended to test using the same database backend used in production, so as to avoid any potential compatibility issues. .. seealso:: * :ref:`test-utils` * Peewee's `test-suite `__ Adding a Custom Database Driver --------------------------------- If your database driver conforms to DB-API 2.0, adding Peewee support requires subclassing :class:`Database` and overriding ``_connect``, which must return a connection in autocommit mode: .. code-block:: python from peewee import Database import foodb class FooDatabase(Database): def _connect(self): return foodb.connect(self.database, autocommit=True, **self.connect_params) def get_tables(self): res = self.execute_sql('SHOW TABLES;') return [r[0] for r in res.fetchall()] The minimum Peewee relies on from the driver is: ``Connection.commit``, ``Connection.rollback``, ``Connection.execute``, ``Cursor.description``, and ``Cursor.fetchone``. Everything else can be incrementally added. Other integration points on :class:`Database`: * ``param`` / ``quote`` - parameter placeholder and quoting characters. * ``field_types`` - mapping from Peewee type labels to vendor column types. * ``operations`` - mapping from operations such as ``ILIKE`` to vendor SQL. Refer to the :class:`Database` API reference or the `Peewee source `_ for details. ================================================ FILE: docs/peewee/db_tools.rst ================================================ .. _db-tools: Database Tooling ================ This section covers the playhouse modules for managing connections, database URLs, schema migrations, introspection, code generation, and testing. .. contents:: On this page :local: :depth: 1 .. _db-url: Database URLs ------------- .. module:: playhouse.db_url The ``playhouse.db_url`` module lets you configure Peewee from a connection string, which is common in twelve-factor applications where database credentials live in environment variables. .. code-block:: python import os from playhouse.db_url import connect db = connect(os.environ.get('DATABASE_URL', 'sqlite:////default.db')) Pass additional keyword arguments in the query string: .. code-block:: python db = connect('postgres://user:pass@host/db?max_connections=20') URL format: ``scheme://user:password@host:port/dbname?option=value`` Common schemes: +------------------------+------------------------------------------+ | Scheme | Database class | +========================+==========================================+ | ``sqlite:///path`` | :class:`SqliteDatabase` | +------------------------+------------------------------------------+ | ``postgres://`` | :class:`PostgresqlDatabase` | +------------------------+------------------------------------------+ | ``postgresext://`` | :class:`.PostgresqlExtDatabase` | +------------------------+------------------------------------------+ | ``mysql://`` | :class:`MySQLDatabase` | +------------------------+------------------------------------------+ Connection pool implementations: +-----------------------------+------------------------------------------+ | Scheme | Database class | +=============================+==========================================+ | ``sqlite+pool:///path`` | :class:`.PooledSqliteDatabase` | +-----------------------------+------------------------------------------+ | ``postgres+pool://`` | :class:`.PooledPostgresqlDatabase` | +-----------------------------+------------------------------------------+ | ``postgresext+pool://`` | :class:`.PooledPostgresqlExtDatabase` | +-----------------------------+------------------------------------------+ | ``mysql+pool://`` | :class:`.PooledMySQLDatabase` | +-----------------------------+------------------------------------------+ Alternate drivers: +------------------------------+------------------------------------------+ | Scheme | Database class | +==============================+==========================================+ | ``psycopg3://`` | :class:`.Psycopg3Database` | +------------------------------+------------------------------------------+ | ``psycopg3+pool://`` | :class:`.PooledPsycopg3Database` | +------------------------------+------------------------------------------+ | ``cockroachdb://`` | :class:`.CockroachDatabase` | +------------------------------+------------------------------------------+ | ``cockroachdb+pool://`` | :class:`.PooledCockroachDatabase` | +------------------------------+------------------------------------------+ | ``cysqlite://`` | :class:`.CySqliteDatabase` | +------------------------------+------------------------------------------+ | ``cysqlite+pool://`` | :class:`.PooledCySqliteDatabase` | +------------------------------+------------------------------------------+ | ``apsw://`` | :class:`.APSWDatabase` | +------------------------------+------------------------------------------+ | ``mariadbconnector://`` | :class:`.MariaDBConnectorDatabase` | +------------------------------+------------------------------------------+ | ``mariadbconnector+pool://`` | :class:`.PooledMariaDBConnectorDatabase` | +------------------------------+------------------------------------------+ | ``mysqlconnector://`` | :class:`.MySQLConnectorDatabase` | +------------------------------+------------------------------------------+ | ``mysqlconnector+pool://`` | :class:`.PooledMySQLConnectorDatabase` | +------------------------------+------------------------------------------+ .. function:: connect(url, unquote_password=False, unquote_user=False, **connect_params) :param url: the URL for the database, see examples. :param bool unquote_password: unquote special characters in the password. :param bool unquote_user: unquote special characters in the user. :param connect_params: additional parameters to pass to the Database. Parse ``url`` and return an appropriate :class:`Database` instance. Examples: * ``sqlite:///my_app.db`` - SQLite file in the current directory. * ``sqlite:///:memory:`` - in-memory SQLite. * ``sqlite:////absolute/path/to/app.db`` - absolute path SQLite. * ``postgresql://user:password@host:5432/dbname`` * ``mysql://user:password@host:3306/dbname`` .. function:: parse(url, unquote_password=False, unquote_user=False) :param url: the URL for the database, see :func:`connect` above for examples. :param bool unquote_password: unquote special characters in the password. :param bool unquote_user: unquote special characters in the user. Parse a URL and return a dictionary with ``database``, ``host``, ``port``, ``user``, and ``password`` keys plus any extra connect parameters from the query string. Useful if you need to construct a database class manually: .. code-block:: python params = parse('postgres://user:pass@host:5432/mydb') db = MyCustomDatabase(**params) .. function:: register_database(db_class, *names) :param db_class: A subclass of :class:`Database`. :param names: A list of names to use as the scheme in the URL. Register a custom database class under one or more URL scheme names so that :func:`connect` can instantiate it: .. code-block:: python register_database(FirebirdDatabase, 'firebird') db = connect('firebird://my-firebird-db') .. _pool: Connection Pooling ------------------ .. module:: playhouse.pool The ``playhouse.pool`` module contains a number of :class:`Database` classes that provide connection pooling for Postgresql, MySQL and SQLite databases. The pool works by overriding the methods on the :class:`Database` class that open and close connections to the backend. In multi-threaded applications, each thread gets its own connection; the pool maintains up to ``max_connections`` open connections at any time. In single-threaded applications, a single connection is recycled. The application only needs to ensure that connections are *closed* when work is done - typically at the end of an HTTP request. Closing a pooled connection returns it to the pool rather than actually disconnecting. .. code-block:: python from playhouse.pool import PooledPostgresqlDatabase db = PooledPostgresqlDatabase( 'my_app', user='postgres', max_connections=32, stale_timeout=300) .. tip:: Pooled database implementations may be safely used as drop-in replacements for their non-pooled counterparts. .. include:: pool-snippet.rst .. note:: Applications using Peewee's :ref:`asyncio integration ` do not need to use a special pooled database - the Async databases use a connection pool by default. .. class:: PooledDatabase(database, max_connections=20, stale_timeout=None, timeout=None, **kwargs) Mixin class mixed into the specific backend subclasses above. :param str database: The name of the database or database file. :param int max_connections: Maximum number of concurrent connections. Pass ``None`` for no limit. :param int stale_timeout: Seconds after which an idle connection is considered stale and will be discarded next time it would be reused. :param int timeout: Seconds to block when all connections are in use. ``0`` blocks indefinitely; ``None`` (default) raises immediately. .. note:: Connections will not be closed exactly when they exceed their ``stale_timeout``. Instead, stale connections are only closed when a new connection is requested. .. note:: If the pool is exhausted and no ``timeout`` is configured, a ``ValueError`` is raised. .. method:: manual_close() Close the current connection permanently without returning it to the pool. Use this when a connection has entered a bad state. .. method:: close_idle() Close all pooled connections that are not currently in use. .. method:: close_stale(age=600) :param int age: Age at which a connection should be considered stale. :returns: Number of connections closed. Close in-use connections that have exceeded ``age`` seconds. Use with caution. .. method:: close_all() Close all connections including those currently in use. Use with caution. .. class:: PooledSqliteDatabase(database, max_connections=20, stale_timeout=None, timeout=None, **kwargs) Pool implementation for SQLite databases. Extends :class:`SqliteDatabase`. .. class:: PooledPostgresqlDatabase(database, max_connections=20, stale_timeout=None, timeout=None, **kwargs) Pool implementation for Postgresql databases. Extends :class:`PostgresqlDatabase`. .. class:: PooledMySQLDatabase(database, max_connections=20, stale_timeout=None, timeout=None, **kwargs) Pool implementation for MySQL / MariaDB databases. Extends :class:`MySQLDatabase`. .. _migrate: Schema Migrations ----------------- .. module:: playhouse.migrate The ``playhouse.migrate`` module provides a lightweight API for making incremental schema changes to an existing database without writing raw SQL. The peewee migration philosophy is that tools relying on database introspection, versioning, and auto-detection are often fragile, brittle and unnecessarily complex. Migrations can be written as simple python scripts and executed from the command-line. Since the migrations only depend on your application's :class:`Database` object, migration scripts to not introduce new dependencies. Supported operations: - Add, rename, or drop columns. - Make columns nullable or not nullable. - Change a column's type. - Rename a table. - Add or drop indexes and constraints. - Add or drop column default values. .. seealso:: :ref:`schema` .. code-block:: python from playhouse.migrate import SchemaMigrator, migrate migrator = SchemaMigrator.from_database(db) with db.atomic(): migrate( migrator.add_column('tweet', 'is_published', BooleanField(default=True)), migrator.add_column('user', 'email', CharField(null=True)), migrator.drop_column('user', 'old_bio'), ) .. tip:: Wrap migrations in ``db.atomic()`` to ensure changes are not partially applied. Operations ^^^^^^^^^^ **Add columns:** .. code-block:: python # Non-null fields must supply a default value. migrate( migrator.add_column('comment', 'pub_date', DateTimeField(null=True)), migrator.add_column('comment', 'body', TextField(default='')), ) **Add a foreign key** (the column name must include the ``_id`` suffix that Peewee appends by default): .. code-block:: python user_fk = ForeignKeyField(User, field=User.id, null=True) migrate( migrator.add_column('tweet', 'user_id', user_fk), ) **Rename a column:** .. code-block:: python migrate( migrator.rename_column('story', 'pub_date', 'publish_date'), migrator.rename_column('story', 'mod_date', 'modified_date'), ) **Drop a column:** .. code-block:: python migrate(migrator.drop_column('story', 'old_field')) **Nullable / not nullable:** .. code-block:: python migrate( migrator.drop_not_null('story', 'pub_date'), # Allow NULLs. migrator.add_not_null('story', 'modified_date'), # Disallow NULLs. ) **Change type:** .. code-block:: python # Change a VARCHAR(...) to a TEXT field. migrate(migrator.alter_column_type('person', 'email', TextField())) **Rename table:** .. code-block:: python migrate(migrator.rename_table('story', 'stories')) **Add / drop indexes:** .. code-block:: python # Specify table, column(s), and unique/non-unique. migrate( # Create an index on the `pub_date` column. migrator.add_index('story', ('pub_date',), False), # Normal index. # Create a unique index on the category and title fields. migrator.add_index('story', ('category_id', 'title'), True), # Unique. # Drop the pub-date + status index. migrator.drop_index('story', 'story_pub_date_status'), ) **Add / drop constraints:** .. code-block:: python from peewee import Check # Add a CHECK() constraint to enforce the price cannot be negative. migrate(migrator.add_constraint( 'products', 'price_check', Check('price >= 0'))) # Remove the price check constraint. migrate(migrator.drop_constraint('products', 'price_check')) # Add a UNIQUE constraint on the first and last names. migrate(migrator.add_unique('person', 'first_name', 'last_name')) **Column defaults:** .. code-block:: python # Add a default value: migrate(migrator.add_column_default('entry', 'status', 'draft')) # Use a function (not supported in SQLite): migrate(migrator.add_column_default('entry', 'created_at', fn.NOW())) # SQLite-compatible function syntax: migrate(migrator.add_column_default('entry', 'created_at', 'now()')) # Remove a default: migrate(migrator.drop_column_default('entry', 'status')) .. note:: Postgres users may need to set the search-path when using a non-standard schema. This can be done as follows: .. code-block:: python migrator = PostgresqlMigrator(db) migrate( migrator.set_search_path('my_schema'), migrator.add_column('table', 'field', TextField(default='')), ) Migration API ^^^^^^^^^^^^^ .. function:: migrate(*operations) Execute one or more schema-altering operations. Usage: .. code-block:: python migrate( migrator.add_column('t', 'col', CharField(default='')), migrator.add_index('t', ('col',), False), ) .. class:: SchemaMigrator(database) :param database: a :class:`Database` instance. The :class:`SchemaMigrator` is responsible for generating schema-altering statements. .. classmethod:: from_database(database) :param Database database: database instance to generate migrations for. :return: :class:`SchemaMigrator` instance appropriate to provided database. Factory method that returns the appropriate :class:`SchemaMigrator` subclass for the given database. .. method:: add_column(table, column_name, field) :param str table: Name of the table to add column to. :param str column_name: Name of the new column. :param Field field: A :class:`Field` instance. Add a new column to the provided table. The ``field`` provided will be used to generate the appropriate column definition. If the field is not nullable it must specify a default value. .. note:: For non-null columns, the following occurs: 1. column is added as allowing NULLs 2. ``UPDATE`` query is executed to populate the default value 3. column is changed to NOT NULL .. method:: drop_column(table, column_name, cascade=True) :param str table: Name of the table to drop column from. :param str column_name: Name of the column to drop. :param bool cascade: Whether the column should be dropped with `CASCADE`. .. method:: rename_column(table, old_name, new_name) :param str table: Name of the table containing column to rename. :param str old_name: Current name of the column. :param str new_name: New name for the column. .. method:: add_not_null(table, column) :param str table: Name of table containing column. :param str column: Name of the column to make not nullable. .. method:: drop_not_null(table, column) :param str table: Name of table containing column. :param str column: Name of the column to make nullable. .. method:: add_column_default(table, column, default) :param str table: Name of table containing column. :param str column: Name of the column to add default to. :param default: New default value for column. See notes below. Peewee attempts to properly quote the default if it appears to be a string literal. Otherwise the default will be treated literally. Postgres and MySQL support specifying the default as a peewee expression, e.g. ``fn.NOW()``, but Sqlite users will need to use ``default='now()'`` instead. .. method:: drop_column_default(table, column) :param str table: Name of table containing column. :param str column: Name of the column to remove default from. .. method:: alter_column_type(table, column, field, cast=None) :param str table: Name of the table. :param str column_name: Name of the column to modify. :param Field field: :class:`Field` instance representing new data type. :param cast: (postgres-only) specify a cast expression if the data-types are incompatible, e.g. ``column_name::int``. Can be provided as either a string or a :class:`Cast` instance. Alter the data-type of a column. This method should be used with care, as using incompatible types may not be well-supported by your database. .. method:: rename_table(old_name, new_name) :param str old_name: Current name of the table. :param str new_name: New name for the table. .. method:: add_index(table, columns, unique=False, using=None) :param str table: Name of table on which to create the index. :param list columns: List of columns which should be indexed. :param bool unique: Whether the new index should specify a unique constraint. :param str using: Index type (where supported), e.g. GiST or GIN. .. method:: drop_index(table, index_name) :param str table: Name of the table containing the index to be dropped. :param str index_name: Name of the index to be dropped. .. method:: add_constraint(table, name, constraint) :param str table: Table to add constraint to. :param str name: Name used to identify the constraint. :param constraint: either a :func:`Check` constraint or for adding an arbitrary constraint use :class:`SQL`. .. method:: drop_constraint(table, name) :param str table: Table to drop constraint from. :param str name: Name of constraint to drop. .. method:: add_unique(table, *column_names) :param str table: Table to add constraint to. :param str column_names: One or more columns for UNIQUE constraint. .. class:: PostgresqlMigrator(database) .. method:: set_search_path(schema_name) Set the Postgres search path for subsequent operations. .. class:: SqliteMigrator(database) SQLite has limited support for ``ALTER TABLE`` queries, so the following operations are currently not supported for SQLite: * ``add_constraint`` * ``drop_constraint`` * ``add_unique`` .. class:: MySQLMigrator(database) MySQL-specific subclass. .. _reflection: Reflection ---------- .. module:: playhouse.reflection The ``playhouse.reflection`` module introspects an existing database and generates Peewee model classes from its schema. It is used internally by :ref:`pwiz` and :ref:`dataset`. .. code-block:: python from playhouse.reflection import generate_models db = PostgresqlDatabase('my_app') models = generate_models(db) # Returns {table_name: ModelClass} # list(models.keys()) # ['account', 'customer', 'order', 'orderitem', 'product'] # Get a reference to a generated model. Customer = models['customer'] # Or inject into the current namespace: # globals().update(models) # Query generated models: for customer in Customer.select(): print(customer.name, customer.email) .. function:: generate_models(database, schema=None, **options) :param Database database: database instance to introspect. :param str schema: optional schema to introspect. :param options: arbitrary options, see :meth:`Introspector.generate_models` for details. :returns: a ``dict`` mapping table names to model classes. .. function:: print_model(model) Print a human-readable summary of a model's fields and indexes to stdout. Useful for interactive exploration: .. code-block:: pycon >>> print_model(Tweet) tweet id AUTO PK user INT FK: User.id content TEXT timestamp DATETIME index(es) user_id timestamp .. function:: print_table_sql(model) Print the ``CREATE TABLE`` SQL for a model class (without indexes or constraints): .. code-block:: pycon >>> print_table_sql(Tweet) CREATE TABLE IF NOT EXISTS "tweet" ( "id" INTEGER NOT NULL PRIMARY KEY, "user_id" INTEGER NOT NULL, "content" TEXT NOT NULL, "timestamp" DATETIME NOT NULL, FOREIGN KEY ("user_id") REFERENCES "user" ("id") ) .. class:: Introspector(metadata, schema=None) Metadata can be extracted from a database by instantiating an :class:`Introspector`. Rather than instantiating this class directly, it is recommended to use the factory method :meth:`~Introspector.from_database`. .. classmethod:: from_database(database, schema=None) :param database: a :class:`Database` instance. :param str schema: an optional schema (supported by some databases). Creates an :class:`Introspector` instance suitable for use with the given database. .. code-block:: python db = SqliteDatabase('my_app.db') introspector = Introspector.from_database(db) models = introspector.generate_models() # User and Tweet (assumed to exist in the database) are # peewee Model classes generated from the database schema. User = models['user'] Tweet = models['tweet'] .. method:: generate_models(skip_invalid=False, table_names=None, literal_column_names=False, bare_fields=False, include_views=False) :param bool skip_invalid: Skip tables whose names are not valid Python identifiers. :param list table_names: Only generate models for the given tables. :param bool literal_column_names: Use the exact database column names as field names (rather than converting to Python naming conventions). :param bool bare_fields: Do not attempt to detect field types; use :class:`BareField` for all columns (**SQLite only**). :param bool include_views: Also generate models for views. :return: A dictionary mapping table-names to model classes. Introspect the database, reading in the tables, columns, and foreign key constraints, then generate a dictionary mapping each database table to a dynamically-generated :class:`Model` class. .. _pwiz: pwiz - Model Generator ----------------------- .. module:: pwiz ``pwiz`` is a command-line tool that introspects a database and prints ready-to-use Peewee model code. If you have an existing database, running ``pwiz`` saves significant time generating the initial model definitions. .. code-block:: shell # Introspect a Postgresql database and write models to a file: python -m pwiz -e postgresql -u postgres my_db > models.py # Introspect a SQLite database: python -m pwiz -e sqlite path/to/my.db # Introspect a MySQL database (prompts for password): python -m pwiz -e mysql -u root -P my_db # Introspect only specific tables: python -m pwiz -e postgresql my_db -t user,tweet,follow Command-line options: +--------+-------------------------------------------+-------------------------+ | Option | Meaning | Example | +========+===========================================+=========================+ | ``-e`` | Database backend | ``-e mysql`` | +--------+-------------------------------------------+-------------------------+ | ``-H`` | Host | ``-H 10.0.0.1`` | +--------+-------------------------------------------+-------------------------+ | ``-p`` | Port | ``-p 5432`` | +--------+-------------------------------------------+-------------------------+ | ``-u`` | Username | ``-u postgres`` | +--------+-------------------------------------------+-------------------------+ | ``-P`` | Password (prompts interactively) | | +--------+-------------------------------------------+-------------------------+ | ``-s`` | Schema | ``-s public`` | +--------+-------------------------------------------+-------------------------+ | ``-t`` | Comma-separated list of tables to include | ``-t user,tweet`` | +--------+-------------------------------------------+-------------------------+ | ``-v`` | Include views | | +--------+-------------------------------------------+-------------------------+ | ``-i`` | Embed database info as a comment | | +--------+-------------------------------------------+-------------------------+ | ``-o`` | Preserve original column order | | +--------+-------------------------------------------+-------------------------+ | ``-I`` | Ignore fields whose type is unknown | | +--------+-------------------------------------------+-------------------------+ | ``-L`` | Use legacy table and column naming | | +--------+-------------------------------------------+-------------------------+ Valid ``-e`` values: ``sqlite``, ``mysql``, ``postgresql``. .. warning:: If a password is required to access your database, you will be prompted to enter it using a secure prompt. **The password will be included in the output**. Specifically, at the top of the file a :class:`Database` will be defined along with any required parameters - including the password. Example output for a SQLite database with ``user`` and ``tweet`` tables: .. code-block:: python from peewee import * database = SqliteDatabase('example.db', **{}) class UnknownField(object): def __init__(self, *_, **__): pass class BaseModel(Model): class Meta: database = database class User(BaseModel): username = TextField(unique=True) class Meta: table_name = 'user' class Tweet(BaseModel): content = TextField() timestamp = DateTimeField() user = ForeignKeyField(column_name='user_id', field='id', model=User) class Meta: table_name = 'tweet' Note that ``pwiz`` detects foreign keys, unique constraints, and preserves explicit table names. .. note:: The ``UnknownField`` is a placeholder that is used in the event your schema contains a column declaration that Peewee doesn't know how to map to a field class. .. _test-utils: Test Utilities -------------- .. module:: playhouse.test_utils ``playhouse.test_utils`` provides helpers for testing peewee projects. .. class:: count_queries(only_select=False) Context manager that counts the number of SQL queries executed within its block. :param bool only_select: If ``True``, count only ``SELECT`` queries. .. code-block:: python with count_queries() as counter: user = User.get(User.username == 'alice') tweets = list(user.tweets) # Triggers a second query. assert counter.count == 2 .. attribute:: count Number of queries executed. .. method:: get_queries() Return a list of ``(sql, params)`` 2-tuples for each query executed. .. function:: assert_query_count(expected, only_select=False) Decorator or context manager that raises ``AssertionError`` if the number of queries executed does not match ``expected``. As a decorator: .. code-block:: python class TestAPI(unittest.TestCase): @assert_query_count(1) def test_get_user(self): user = User.get_by_id(1) As a context manager: .. code-block:: python with assert_query_count(3): result = my_function_that_should_make_exactly_three_queries() ================================================ FILE: docs/peewee/example.rst ================================================ .. _example: Example app =========== We'll be building a simple *twitter*-like site. The source code for the example can be found in the ``examples/twitter`` directory. You can also `browse the source-code `__ on github. .. tip:: There is also an example `blog app `__, however it is not covered in this guide. The example app uses the `flask `__ web framework. You will need to install it to run the example: .. code-block:: shell pip install flask Running the Example ------------------- .. image:: tweepee.png After ensuring that flask is installed, ``cd`` into the twitter example directory and execute the ``run_example.py`` script: .. code-block:: shell python run_example.py The example app will be accessible at http://localhost:5000/ Code Structure -------------- For simplicity all example code is contained within a single module, ``examples/twitter/app.py``. For a guide on structuring larger Flask apps with peewee, check out `Structuring Flask Apps `_. .. _example-models: Models ^^^^^^ peewee uses declarative model definitions. Declare a model class for each table. The model class then defines one or more field attributes which correspond to the table's columns. For the twitter clone, there are three models: **User**: Represents a user account and stores the username and password, an email address for generating avatars using *gravatar* and a datetime field indicating when that account was created. **Relationship**: This is a utility model that contains two foreign-keys to the *User* model and stores which users follow one another. **Message**: Analogous to a tweet. The Message model stores the text content of the tweet, when it was created, and who posted it (foreign key to User). If you like UML, these are the tables and relationships: .. image:: schema.jpg In order to create these models we need: 1. declare a :class:`SqliteDatabase` object 2. declare our model classes 3. declare columns as :class:`Field` instances on the model classes .. code-block:: python :emphasize-lines: 3, 8, 13, 23, 39 # create a peewee database instance -- our models will use this database to # persist information database = SqliteDatabase(DATABASE) # model definitions -- the standard "pattern" is to define a base model class # that specifies which database to use. then, any subclasses will automatically # use the correct storage. class BaseModel(Model): class Meta: database = database # the user model specifies its fields (or columns) declaratively, like django class User(BaseModel): username = CharField(unique=True) password = CharField() email = CharField() join_date = DateTimeField() # this model contains two foreign keys to user -- it essentially allows us to # model a "many-to-many" relationship between users. by querying and joining # on different columns we can expose who a user is "related to" and who is # "related to" a given user class Relationship(BaseModel): from_user = ForeignKeyField(User, backref='relationships') to_user = ForeignKeyField(User, backref='related_to') class Meta: # `indexes` is a tuple of 2-tuples, where the 2-tuples are # a tuple of column names to index and a boolean indicating # whether the index is unique or not. indexes = ( # Specify a unique multi-column index on from/to-user. (('from_user', 'to_user'), True), ) # a dead simple one-to-many relationship: one user has 0..n messages, exposed by # the foreign key. a users messages will be accessible as a special attribute, # User.messages. class Message(BaseModel): user = ForeignKeyField(User, backref='messages') content = TextField() pub_date = DateTimeField() .. note:: Note that we create a *BaseModel* class that simply defines what database we would like to use. All other models then extend this class and will also use the correct database connection. Peewee supports many different :ref:`field types ` which map to different column types commonly supported by database engines. Conversion between python types and those used in the database is handled transparently, allowing you to use the following in your application: * Strings (unicode or otherwise) * Integers, floats, and ``Decimal`` numbers. * Boolean values * Dates, times and datetimes * ``None`` (NULL) * Binary data Creating Tables --------------- In order to start using the models, its necessary to **create the tables**. This is a one-time operation and can be done quickly using the interactive interpreter. We can create a small helper function to accomplish this: .. code-block:: python :emphasize-lines: 3 def create_tables(): with database: database.create_tables([User, Relationship, Message]) Open a python shell in the directory alongside the example app and execute the following: .. code-block:: pycon >>> from app import * >>> create_tables() .. attention:: If you encounter an **ImportError** it means that either *flask* or *peewee* was not found and may not be installed correctly. Check the :ref:`installation` document for instructions on installing peewee. Every model has a :meth:`~Model.create_table` classmethod which runs a SQL *CREATE TABLE* statement in the database. This method will create the table, including: * columns * foreign-key constraints * indexes * sequences * check constraints Usually this is something you'll only do once, when a new model is added. Peewee provides a helper method :meth:`Database.create_tables` which will resolve inter-model dependencies and call :meth:`~Model.create_table` on each model, ensuring the tables are created in order. .. note:: Adding, removing or modifying fields after the table has been created will require you to either: * drop the table and re-create it, OR * manually add, drop or modify the columns, OR * use the :ref:`migration tools ` to script your changes. Database Connection ------------------- You may have noticed in the above model code that there is a class defined on the base model named *Meta* that sets the ``database`` attribute. Peewee allows every model to specify which database it uses. There are many :ref:`Meta options ` you can specify which control the behavior of your model. This is a peewee idiom: .. code-block:: python :emphasize-lines: 9, 10, 11 DATABASE = 'tweepee.db' # Create a database instance that will manage the connection and # execute queries database = SqliteDatabase(DATABASE) # Create a base-class all our models will inherit, which defines # the database we'll be using. class BaseModel(Model): class Meta: database = database When developing a web application, it's common to: 1. Open a connection when a request starts. 2. Run your request-handler. 3. Close the connection before returning the response. **You should always manage your connections explicitly**. For instance, if you are using a :ref:`connection pool `, connections will only be recycled correctly if you call :meth:`~Database.connect` and :meth:`~Database.close`. Flask provides connection setup/teardown hooks via decorators: .. code-block:: python :emphasize-lines: 1, 5 @app.before_request def before_request(): database.connect() @app.teardown_request def teardown_request(exc): if not database.is_closed(): database.close() .. seealso:: :ref:`framework-integration` covers setting-up hooks for a variety of popular web frameworks. .. note:: Peewee uses thread local storage to manage connection state, so this pattern can be used with multi-threaded or gevent WSGI servers. Peewee's :ref:`asyncio integration ` stores connection state in task-local storage, so the same pattern applies. Making Queries -------------- In the *User* model there are a few instance methods that encapsulate some user-specific functionality: * ``following()``: who is this user following? * ``followers()``: who is following this user? These methods are similar in their implementation but with an important difference in the SQL *JOIN* and *WHERE* clauses: .. code-block:: python :emphasize-lines: 4, 5, 11, 12 def following(self): return (User .select() .join(Relationship, on=Relationship.to_user) .where(Relationship.from_user == self) .order_by(User.username)) def followers(self): return (User .select() .join(Relationship, on=Relationship.from_user) .where(Relationship.to_user == self) .order_by(User.username)) Storing Data ------------ When a new user wants to join the site we need to make sure the username is available, and if so, create a new ``User`` record. Looking at the ``join()`` view, we can see that our application attempts to create the ``User`` using :meth:`Model.create`. ``User.username`` field has a unique constraint, so if the username is taken the database will raise an :class:`IntegrityError`. .. code-block:: python try: with database.atomic(): # Attempt to create the user. If the username is taken, due to the # unique constraint, the database will raise an IntegrityError. user = User.create( username=request.form['username'], password=md5(request.form['password']).hexdigest(), email=request.form['email'], join_date=datetime.datetime.now()) # mark the user as being 'authenticated' by setting the session vars auth_user(user) return redirect(url_for('homepage')) except IntegrityError: flash('That username is already taken') We will use a similar approach when a user wishes to follow someone. To indicate a following relationship, we create a row in the ``Relationship`` table pointing from one user to another. Due to the unique index on ``(from_user, to_user)``, we will be sure not to end up with duplicate rows: .. code-block:: python user = get_object_or_404(User, username=username) try: with database.atomic(): Relationship.create( from_user=get_current_user(), to_user=user) except IntegrityError: pass Subqueries ---------- If you are logged-in and visit the twitter homepage, you will see tweets from the users that you follow. In order to implement this cleanly, we can use a subquery: .. note:: ``user.following()`` will automatically only select ``User.id`` when it used in a subquery. .. code-block:: python :emphasize-lines: 5 # python code user = get_current_user() messages = (Message .select() .where(Message.user.in_(user.following())) .order_by(Message.pub_date.desc())) This code corresponds to the following SQL query: .. code-block:: sql SELECT t1."id", t1."user_id", t1."content", t1."pub_date" FROM "message" AS t1 WHERE t1."user_id" IN ( SELECT t2."id" FROM "user" AS t2 INNER JOIN "relationship" AS t3 ON t2."id" = t3."to_user_id" WHERE t3."from_user_id" = ? ) Other Topics ------------ There are a couple other neat things going on in the example app that are worth mentioning briefly. * Support for paginating lists of results is implemented in a simple function called ``object_list``. This function is used by all the views that return lists of objects. .. code-block:: python def object_list(template_name, qr, var_name='object_list', **kwargs): kwargs.update( page=int(request.args.get('page', 1)), pages=qr.count() / 20 + 1) kwargs[var_name] = qr.paginate(kwargs['page']) return render_template(template_name, **kwargs) * Simple authentication system with a ``login_required`` decorator. The first function simply adds user data into the current session when a user successfully logs in. The decorator ``login_required`` can be used to wrap view functions, checking for whether the session is authenticated and if not redirecting to the login page. .. code-block:: python def auth_user(user): session['logged_in'] = True session['user'] = user session['username'] = user.username flash('You are logged in as %s' % (user.username)) def login_required(f): @wraps(f) def inner(*args, **kwargs): if not session.get('logged_in'): return redirect(url_for('login')) return f(*args, **kwargs) return inner * Return a 404 response instead of throwing exceptions when an object is not found in the database. .. code-block:: python def get_object_or_404(model, *expressions): try: return model.get(*expressions) except model.DoesNotExist: abort(404) .. tip:: To avoid having to frequently copy/paste :func:`object_list` or :func:`get_object_or_404`, these functions are included as part of the playhouse :ref:`flask extension module `. .. code-block:: python from playhouse.flask_utils import get_object_or_404, object_list More Examples ------------- There are more examples included in the peewee `examples directory `_, including: * `Example blog app `__ using Flask and peewee. Also see `accompanying blog post `__. * `An encrypted command-line diary `_. There is a `companion blog post `__ you might enjoy as well. * `Analytics web-service `_ (like a lite version of Google Analytics). Also check out the `companion blog post `__. .. seealso:: Like these snippets and interested in more? Check out `flask-peewee `__ - a flask plugin that provides a django-like Admin interface, RESTful API, Authentication and more for your peewee models. ================================================ FILE: docs/peewee/framework_integration.rst ================================================ .. _framework-integration: Framework Integration ===================== For web applications, it is common to open a connection when a request is received, and to close the connection when the response is delivered. This document describes how to add hooks to your web app to ensure the database connection is handled properly. These steps will ensure that regardless of whether you're using a simple :class:`SqliteDatabase` or a :class:`PooledPostgresqlDatabase`, peewee will handle the connections correctly. The pattern is always the same: .. code-block:: python # On request start: db.connect() # On request end (success or error): if not db.is_closed(): db.close() Every framework exposes hooks for this. The sections below show the idiomatic approach for each. .. note:: Applications that handle significant traffic should use a :ref:`connection pool ` to avoid the overhead of establishing a new connection per request. Pooled databases can be used as drop-in replacements for their non-pooled counterparts. .. _flask: Flask ----- For a **complete** Flask + Peewee application example, including authentication and other common webapp functionality, see :ref:`example`. There is also a full `blog app `__ and an `analytics app `__ in the project ``examples/`` directory. The **minimal** Flask integration ensures that database connection lifecycles are tied to the request/response cycle via ``before_request`` and ``teardown_request`` hooks: .. code-block:: python from flask import Flask from peewee import * db = SqliteDatabase('my_app.db') app = Flask(__name__) @app.before_request def _db_connect(): db.connect() @app.teardown_request def _db_close(exc): if not db.is_closed(): db.close() ``teardown_request`` is called regardless of whether the request succeeded or raised an exception, making it the correct hook for cleanup. For applications that receive a large number of requests, a connection pool is recommended: .. code-block:: python from flask import Flask from playhouse.pool import PooledPostgresqlDatabase db = PooledPostgresqlDatabase('app', host='10.8.0.1', user='postgres') app = Flask(__name__) # Note that when using the pooled implementation the hooks are the exact # same. Opening and closing the connection simply acquires and releases it # from the pool for the lifetime of the request. @app.before_request def _db_connect(): db.connect() @app.teardown_request def _db_close(exc): if not db.is_closed(): db.close() .. seealso:: The :ref:`flask-utils` extension provides helpers for common tasks like declarative database configuration, object retrieval and pagination. .. _fastapi: FastAPI ------- FastAPI is an async framework and can be used with Peewee's :ref:`pwasyncio` integration or synchronously. Peewee also provides :ref:`pydantic` support, which works well with FastAPI. Quick note on SQLModel ^^^^^^^^^^^^^^^^^^^^^^ FastAPI advocates using SQLModel for database access. SQLModel combines SQLAlchemy and Pydantic into a single class, which may work well for simple examples. There are a few things to watch out for, though: * SQLModel's official tutorial uses synchronous endpoints exclusively, which FastAPI runs on a threadpool. Async usage is listed as an "advanced" topic and is undocumented currently. * Lazy-loading often breaks in async contexts. SQLAlchemy's implicit lazy-loading of relationships can trigger ``MissingGreenlet`` errors when used with async sessions. This can also occur with Peewee, but it's straightforward to avoid by selecting joined relations. * Because SQLModel uses synchronous drivers for DDL and certain operations, you typically need both a sync AND async driver installed, along with separate engine configurations. * SQLModel uses inheritance to manage input, output and table schemas. In practice a single database table often requires three or four model classes, e.g. ``UserBase``, ``User``, ``UserCreate`` and ``UserRead`` - all of this is managed through inheritance. Peewee may provide a simpler experience - there is a single database to manage with built-in pooling, fewer implicit lazy-load gotcha's, and the Pydantic schemas generated with :func:`~playhouse.pydantic_utils.to_pydantic` can be configured to include/exclude fields without inheritance. Field metadata is captured automatically: choice enums, default values, descriptions, titles and type information are captured in the JSON schema and OpenAPI docs. Peewee requires far less machinery to provide real asyncio database access, and of course works equally well for synchronous FastAPI endpoints. Async Example using Pydantic ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Below is a full example FastAPI application demonstrating dependency-injection style hooks, fully :ref:`async query execution `, and :ref:`pydantic integration `: .. code-block:: python # example.py from fastapi import Depends, FastAPI, HTTPException from contextlib import asynccontextmanager from peewee import * from playhouse.pwasyncio import AsyncPostgresqlDatabase from playhouse.pydantic_utils import to_pydantic db = AsyncPostgresqlDatabase('peewee_test') class User(Model): name = CharField(verbose_name='Full Name', help_text='Display name') email = CharField(unique=True) status = IntegerField(default=1, choices=( (1, 'Active'), (2, 'Inactive'), (3, 'Deleted'))) class Meta: database = db # Generate pydantic schemas suitable for create and responses. # Schemas will include metadata from verbose_name, help_text, choices, and # default settings. UserCreate = to_pydantic(User, model_name='UserCreate') UserResponse = to_pydantic(User, exclude_autofield=False, model_name='UserResponse') async def get_db(): async with db: yield db @asynccontextmanager async def lifespan(app): # Create tables (if they don't exist) at application startup. async with db: await db.acreate_tables([User]) yield await db.close_pool() # Shut-down pool and exit. app = FastAPI(lifespan=lifespan) @app.get('/users', response_model=list[UserResponse]) async def list_users(db=Depends(get_db)): rows = await db.list(User.select().dicts()) return [UserResponse(**row) for row in rows] @app.post('/users', response_model=UserResponse) async def create_user(data: UserCreate, db=Depends(get_db)): user = await db.run(User.create, **data.model_dump()) return UserResponse.model_validate(user) @app.get('/users/{user_id}', response_model=UserResponse) async def get_user(user_id: int, db=Depends(get_db)): try: user = await db.get(User.select().where(User.id == user_id)) except User.DoesNotExist: raise HTTPException(status_code=404, detail='User not found') return UserResponse.model_validate(user) Run the example: .. code-block:: console $ fastapi dev example.py Populate and query data: .. code-block:: console $ curl -X POST http://localhost:8000/users \ -H "Content-Type: application/json" \ -d '{"name": "Alice", "email": "alice@example.com"}' {"id":1,"name":"Alice","email":"alice@example.com","status":1} $ curl -X POST http://localhost:8000/users \ -H "Content-Type: application/json" \ -d '{"name": "Bob", "email": "bob@example.com", "status": 2}' {"id":2,"name":"Bob","email":"bob@example.com","status":2} $ curl http://localhost:8000/users [{"id":1,"name":"Alice","email":"alice@example.com","status":1}, {"id":2,"name":"Bob","email":"bob@example.com","status":2}] $ curl http://localhost:8000/users/1 {"id":1,"name":"Alice","email":"alice@example.com","status":1} We can also verify that the pydantic schemas captured our Peewee model metadata: .. code-block:: python >>> UserCreate.model_json_schema() {'properties': { 'name': { 'description': 'Display name', 'title': 'Full Name', 'type': 'string'}, 'email': { 'title': 'Email', 'type': 'string'}, 'status': { 'default': 1, 'description': 'Choices: 1 = Active, 2 = Inactive, 3 = Deleted', 'enum': [1, 2, 3], 'title': 'Status', 'type': 'integer'}}, 'required': ['name', 'email'], 'title': 'UserCreate', 'type': 'object'} .. seealso:: * :ref:`pwasyncio` * :ref:`pydantic` Dependency injection ^^^^^^^^^^^^^^^^^^^^^ The following is a minimal example demonstrating: * Ensure connection is opened and closed automatically for endpoints that use the database. * Create tables/resources when app server starts. * Shut-down connection pool when app server exits. .. code-block:: python from contextlib import asynccontextmanager from fastapi import Depends, FastAPI from peewee import * from playhouse.pwasyncio import * app = FastAPI() db = AsyncPostgresqlDatabase('peewee_test', host='10.8.0.1', user='postgres') async def get_db(): async with db: yield db @asynccontextmanager async def lifespan(app): async with db: await db.acreate_tables([User]) yield await db.close_pool() app = FastAPI(lifespan=lifespan) @app.get('/users') async def list_users(db=Depends(get_db)): return await db.list(User.select().dicts()) Middleware and startup hooks ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The following example demonstrates how to use middleware and startup hooks instead of dependency injection. * Ensure connection is opened and closed for each request. * Create tables/resources when app server starts. * Shut-down connection pool when app server exits. .. code-block:: python from fastapi import FastAPI from peewee import * from playhouse.pwasyncio import * app = FastAPI() db = AsyncPostgresqlDatabase('peewee_test', host='10.8.0.1', user='postgres') @app.middleware('http') async def database_connection(request, call_next): async with db: # Obtain connection from connection pool. response = await call_next(request) return response @app.on_event('startup') async def on_startup(): async with db: await db.acreate_tables([Model1, Model2, Model3, ...]) @app.on_event('shutdown') async def on_shutdown(): await db.close_pool() # Async queries. @app.get('/users') async def list_users(): return await db.list(User.select().dicts()) @app.post('/users') async def create_user(name: str): user = await db.run(User.create, name=name) return {'id': user.id, 'name': user.name} Synchronous FastAPI ^^^^^^^^^^^^^^^^^^^ If you are using synchronous endpoints with FastAPI, you can use the synchronous Peewee database implementations. Here is the above "Full Example" implemented using sync Peewee: .. code-block:: python from fastapi import Depends, FastAPI, HTTPException from contextlib import asynccontextmanager from peewee import * from playhouse.pydantic_utils import to_pydantic db = PostgresqlDatabase('peewee_test') class User(Model): name = CharField(verbose_name='Full Name', help_text='Display name') email = CharField(unique=True) status = IntegerField(default=1, choices=( (1, 'Active'), (2, 'Inactive'), (3, 'Deleted'))) class Meta: database = db # Generate pydantic schemas suitable for create and responses. UserCreate = to_pydantic(User, model_name='UserCreate') UserResponse = to_pydantic(User, exclude_autofield=False, model_name='UserResponse') def get_db(): with db.connection_context(): yield db @asynccontextmanager async def lifespan(app): with db: db.create_tables([User]) yield app = FastAPI(lifespan=lifespan) @app.get('/users', response_model=list[UserResponse]) def list_users(database=Depends(get_db)): rows = User.select().dicts() return [UserResponse(**row) for row in rows] @app.post('/users', response_model=UserResponse) def create_user(data: UserCreate, database=Depends(get_db)): user = User.create(**data.model_dump()) return UserResponse.model_validate(user) @app.get('/users/{user_id}', response_model=UserResponse) def get_user(user_id: int, database=Depends(get_db)): try: user = User.get(User.id == user_id) except User.DoesNotExist: raise HTTPException(status_code=404, detail='User not found') return UserResponse.model_validate(user) .. seealso:: :ref:`pydantic` Django ------ Add a middleware that opens the connection before the view runs and closes it after the response is prepared. Place it first in ``MIDDLEWARE`` so it wraps all other middleware: .. code-block:: python # myproject/middleware.py from myproject.db import database def PeeweeConnectionMiddleware(get_response): def middleware(request): database.connect() try: response = get_response(request) finally: if not database.is_closed(): database.close() return response return middleware .. code-block:: python # settings.py MIDDLEWARE = [ 'myproject.middleware.PeeweeConnectionMiddleware', # ... rest of middleware ... ] Bottle ------ Use the ``before_request`` and ``after_request`` hooks: .. code-block:: python from bottle import hook from peewee import * db = SqliteDatabase('my_app.db') @hook('before_request') def _connect_db(): db.connect() @hook('after_request') def _close_db(): if not db.is_closed(): db.close() Falcon ------ Add a middleware component: .. code-block:: python import falcon from peewee import * db = SqliteDatabase('my_app.db') class DatabaseMiddleware: def process_request(self, req, resp): db.connect() def process_response(self, req, resp, resource, req_succeeded): if not db.is_closed(): db.close() app = falcon.App(middleware=[DatabaseMiddleware()]) Pyramid ------- Set up a custom ``Request`` factory: .. code-block:: python from pyramid.request import Request from peewee import * db = SqliteDatabase('my_app.db') class MyRequest(Request): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) db.connect() self.add_finished_callback(self._close_db) def _close_db(self, request): if not db.is_closed(): db.close() # In your application factory: def main(global_settings, **settings): config = Configurator(settings=settings) config.set_request_factory(MyRequest) Sanic ----- Sanic is an async framework and can be used with Peewee's :ref:`pwasyncio` integration. .. code-block:: python from sanic import Sanic from peewee import * from playhouse.pwasyncio import * app = Sanic('PeeweeApp') db = AsyncPostgresqlDatabase('peewee_test', host='10.8.0.1', user='postgres') @app.on_request async def open_connection(request): await db.aconnect() # Obtain connection from connection pool. @app.on_response async def close_connection(request, response): await db.aclose() # Return connection to pool. @app.before_server_start async def setup_db(app): async with db: await db.acreate_tables([Model1, Model2, Model3, ...]) @app.before_server_stop async def shutdown_db(app): await db.close_pool() Example demonstrating executing an async query: .. code-block:: python from sanic import json @app.get('/message/') async def message(request): # Get the latest message from the database. message = await db.get(Message.select().order_by(Message.id.desc())) return json({'content': message.content, 'id': message.id}) .. seealso:: :ref:`pwasyncio` CherryPy -------- Subscribe to the engine's before/after request events: .. code-block:: python import cherrypy from peewee import * db = SqliteDatabase('my_app.db') def _db_connect(): db.connect() def _db_close(): if not db.is_closed(): db.close() cherrypy.engine.subscribe('before_request', _db_connect) cherrypy.engine.subscribe('after_request', _db_close) General Pattern for Any Framework --------------------------------- If your framework is not listed here, the integration follows the same structure: 1. Find the hook that runs before every request handler. 2. Call ``db.connect()`` there. 3. Find the hook that runs after every request (success and error both). 4. Call ``db.close()`` there if the connection is open. Any WSGI or ASGI middleware that wraps the application callable can also manage this: .. code-block:: python class PeeweeMiddleware: def __init__(self, app, database): self.app = app self.db = database def __call__(self, environ, start_response): self.db.connect() try: return self.app(environ, start_response) finally: if not self.db.is_closed(): self.db.close() # Wrap your WSGI app: application = PeeweeMiddleware(application, db) ================================================ FILE: docs/peewee/installation.rst ================================================ .. _installation: Installing and Testing ====================== Install the latest release from PyPI: .. code-block:: shell pip install peewee Peewee has an optional Sqlite C extension which is not bundled in the default wheel. It provides user-defined ranking functions for use with Sqlite FTS4 and functions for fuzzy string matching. To build from source: .. code-block:: shell pip install peewee --no-binary :all: Installing from Source ---------------------- .. code-block:: shell git clone https://github.com/coleifer/peewee.git cd peewee pip install . Running Tests ------------- .. code-block:: shell python runtests.py python runtests.py --help # Show options. To run tests against Postgres or MySQL create a database named ``peewee_test``. For the Postgres extension tests, enable hstore: .. code-block:: sql CREATE EXTENSION hstore; Supported Drivers ----------------- Peewee works with any database for which a DB-API 2.0 driver exists. The following drivers are supported out of the box: +-----------------------+------------------------+--------------------------------------------+ | Database | Driver | Implementation | +=======================+========================+============================================+ | **Sqlite** | ``sqlite3`` | :class:`SqliteDatabase` | +-----------------------+------------------------+--------------------------------------------+ | **Postgres** | ``psycopg3`` | :class:`PostgresqlDatabase` | +-----------------------+------------------------+--------------------------------------------+ | **Postgres** | ``psycopg2`` | :class:`PostgresqlDatabase` | +-----------------------+------------------------+--------------------------------------------+ | **MySQL** | ``pymysql`` | :class:`MySQLDatabase` | +-----------------------+------------------------+--------------------------------------------+ | Sqlite (async) | ``aiosqlite`` | :class:`.AsyncSqliteDatabase` | +-----------------------+------------------------+--------------------------------------------+ | Postgres (async) | ``asyncpg`` | :class:`.AsyncPostgresqlDatabase` | +-----------------------+------------------------+--------------------------------------------+ | MySQL (async) | ``aiomysql`` | :class:`.AsyncMySQLDatabase` | +-----------------------+------------------------+--------------------------------------------+ | Sqlite (alternate) | ``cysqlite`` | :class:`.CySqliteDatabase` | +-----------------------+------------------------+--------------------------------------------+ | Sqlite (alternate) | ``apsw`` | :class:`.APSWDatabase` | +-----------------------+------------------------+--------------------------------------------+ | SqlCipher | ``sqlcipher3`` | :class:`.SqlCipherDatabase` | +-----------------------+------------------------+--------------------------------------------+ | MySQL (alternate) | ``mysql-connector`` | :class:`.MySQLConnectorDatabase` | +-----------------------+------------------------+--------------------------------------------+ | MariaDB (alternate) | ``mariadb-connector`` | :class:`.MariaDBConnectorDatabase` | +-----------------------+------------------------+--------------------------------------------+ | CockroachDB | ``psycopg`` (2 or 3) | :class:`.CockroachDatabase` | +-----------------------+------------------------+--------------------------------------------+ | Postgres (extensions) | ``psycopg`` (2 or 3) | :class:`.PostgresqlExtDatabase` | +-----------------------+------------------------+--------------------------------------------+ The three bolded rows cover the majority of deployments. All others are optional; install their drivers when needed. ================================================ FILE: docs/peewee/interactive.rst ================================================ .. _interactive: Using Peewee Interactively ========================== Peewee contains helpers for working interactively from a Python interpreter or something like a Jupyter notebook. For this example, we'll assume that we have a pre-existing Sqlite database with the following simple schema: .. code-block:: sql CREATE TABLE IF NOT EXISTS "event" ( "id" INTEGER NOT NULL PRIMARY KEY, "key" TEXT NOT NULL, "timestamp" DATETIME NOT NULL, "metadata" TEXT NOT NULL); To experiment with querying this database from an interactive interpreter session, we would start our interpreter and import the following helpers: * :class:`SqliteDatabase` - to reference the "events.db" * :func:`playhouse.reflection.generate_models` - to generate models from an existing database. * :func:`playhouse.reflection.print_model` - to view the model definition. * :func:`playhouse.reflection.print_table_sql` - to view the table SQL. Our terminal session might look like this: .. code-block:: pycon >>> from peewee import SqliteDatabase >>> from playhouse.reflection import generate_models, print_model, print_table_sql >>> The :func:`~playhouse.reflection.generate_models` function will introspect the database and generate model classes for all the tables that are found. This is a handy way to get started and can save a lot of typing. The function returns a dictionary keyed by the table name, with the generated model as the corresponding value: .. code-block:: pycon >>> db = SqliteDatabase('events.db') >>> models = generate_models(db) >>> list(models.items()) [('events', )] >>> globals().update(models) # Inject models into global namespace. >>> event To take a look at the model definition, which lists the model's fields and data-type, we can use the :func:`~playhouse.reflection.print_model` function: .. code-block:: pycon >>> print_model(event) event id AUTO key TEXT timestamp DATETIME metadata TEXT We can also generate a SQL ``CREATE TABLE`` for the introspected model, if you find that easier to read. This should match the actual table definition in the introspected database: .. code-block:: pycon >>> print_table_sql(event) CREATE TABLE IF NOT EXISTS "event" ( "id" INTEGER NOT NULL PRIMARY KEY, "key" TEXT NOT NULL, "timestamp" DATETIME NOT NULL, "metadata" TEXT NOT NULL) Now that we are familiar with the structure of the table we're working with, we can run some queries on the generated ``event`` model: .. code-block:: pycon >>> for e in event.select().order_by(event.timestamp).limit(5): ... print(e.key, e.timestamp) ... e00 2019-01-01 00:01:00 e01 2019-01-01 00:02:00 e02 2019-01-01 00:03:00 e03 2019-01-01 00:04:00 e04 2019-01-01 00:05:00 >>> event.select(fn.MIN(event.timestamp), fn.MAX(event.timestamp)).scalar(as_tuple=True) (datetime.datetime(2019, 1, 1, 0, 1), datetime.datetime(2019, 1, 1, 1, 0)) >>> event.select().count() # Or, len(event) 60 For more information about these APIs and other similar reflection utilities, see the :ref:`reflection` documentation. To generate an actual Python module containing model definitions for an existing database, you can use the command-line :ref:`pwiz ` tool. Here is a quick example: .. code-block:: shell pwiz -e sqlite events.db > events.py The ``events.py`` file will now be an import-able module containing a database instance (referencing the ``events.db``) along with model definitions for any tables found in the database. ``pwiz`` does some additional nice things like introspecting indexes and adding proper flags for ``NULL``/``NOT NULL`` constraints, etc. The APIs discussed in this section: * :func:`~playhouse.reflection.generate_models` * :func:`~playhouse.reflection.print_model` * :func:`~playhouse.reflection.print_table_sql` More low-level APIs are also available on the :class:`Database` instance: * :meth:`Database.get_tables` * :meth:`Database.get_indexes` * :meth:`Database.get_columns` (for a given table) * :meth:`Database.get_primary_keys` (for a given table) * :meth:`Database.get_foreign_keys` (for a given table) ================================================ FILE: docs/peewee/models.rst ================================================ .. _models: Models and Fields ================= Models and Fields allow Peewee applications to declare the tables and columns they will use, and issue queries using Python. This document explains how to use Peewee to express database tables and columns. :class:`Model` classes, :class:`Field` instances and model instances all map to database concepts: ================= ================================= Python construct Database concept ================= ================================= Model class Table Field instance Column Model instance Row ================= ================================= .. tip:: If you are connecting Peewee to an existing database rather than defining a schema from scratch, the :ref:`pwiz ` tool can generate model definitions automatically by introspecting the database. The following code shows the typical way you will define your database connection and model classes. .. code-block:: python :emphasize-lines: 4, 6, 10 import datetime from peewee import * db = SqliteDatabase('my_app.db') class BaseModel(Model): class Meta: database = db class User(BaseModel): username = CharField(unique=True) class Tweet(BaseModel): user = ForeignKeyField(User, backref='tweets') content = TextField() timestamp = DateTimeField(default=datetime.datetime.now) is_published = BooleanField(default=True) class Favorite(BaseModel): user = ForeignKeyField(User, backref='favorites') tweet = ForeignKeyField(Tweet, backref='favorites') Three things to note: 1. Create an instance of a :class:`Database`. .. code-block:: python db = SqliteDatabase('my_app.db') The ``db`` object will be used to manage the connections to the Sqlite database. In this example we're using :class:`SqliteDatabase`, but you could also use one of the other :ref:`database engines `. 2. Create a base model class which specifies our database. .. code-block:: python class BaseModel(Model): class Meta: database = db **BaseModel** exists only to specify the ``database`` setting in its ``Meta`` class. Because ``Meta.database`` is inheritable, every model that extends ``BaseModel`` will automatically use the same database. This pattern avoids repeating the database assignment on every model class. Model configuration is kept namespaced in a special class called ``Meta``. :ref:`Meta ` configuration is passed on to subclasses, so our project's models will all subclass *BaseModel*. There are :ref:`many different attributes ` you can configure using *Model.Meta*. 3. Declare model classes and fields. .. code-block:: python class User(BaseModel): username = CharField(unique=True) Model definition uses the declarative style seen in other popular ORMs. Note that we are extending the *BaseModel* class so the *User* model will inherit the database connection. We have explicitly defined a single *username* column with a unique constraint. Because we have not specified a primary key, Peewee will automatically add an auto-incrementing integer primary key field named *id*. Model Inheritance ----------------- Model subclasses inherit the ``Meta`` configuration of their parent as well as the parent's fields. Inherited ``Meta`` attributes (such as ``database``) are shared; non-inheritable attributes (such as ``table_name``) are re-derived for each subclass. .. code-block:: python class BaseModel(Model): class Meta: database = db class TimestampedModel(BaseModel): """Adds created/updated timestamps to any subclass.""" created = DateTimeField(default=datetime.datetime.now) updated = DateTimeField(default=datetime.datetime.now) class Article(TimestampedModel): title = TextField() body = TextField() # Article.created and Article.updated are inherited. # Article._meta.database is inherited from BaseModel. Peewee uses a separate table for each concrete model class. There is no notion of inheritance spanning multiple tables. If you subclass a model, both the parent and the child have their own tables. .. _fields: Fields ------ The :class:`Field` class is used to describe the mapping of :class:`Model` attributes to database columns. Each field type has a corresponding SQL storage class (varchar, int, etc). Fields handle conversion between python data types and underlying storage transparently. When creating a :class:`Model` class, fields are defined as class attributes: .. code-block:: python class Tweet(BaseModel): user = ForeignKeyField(User, backref='tweets') content = TextField() timestamp = DateTimeField(default=datetime.datetime.now) is_published = BooleanField(default=True) In the above example, no field specifies ``primary_key=True``. As a result, Peewee will create an auto-incrementing integer primary key named ``id``. Peewee uses :class:`AutoField` to signify an auto-incrementing integer primary key. .. _field_types_table: Field types ^^^^^^^^^^^ ===================== ================= ================= ================= Field Type Sqlite Postgresql MySQL ===================== ================= ================= ================= ``AutoField`` integer serial integer ``BigAutoField`` integer bigserial bigint ``IntegerField`` integer integer integer ``BigIntegerField`` integer bigint bigint ``SmallIntegerField`` integer smallint smallint ``IdentityField`` not supported int identity not supported ``FloatField`` real real real ``DoubleField`` real double precision double precision ``DecimalField`` decimal numeric numeric ``CharField`` varchar varchar varchar ``FixedCharField`` char char char ``TextField`` text text text ``BlobField`` blob bytea blob ``BitField`` integer bigint bigint ``BigBitField`` blob bytea blob ``UUIDField`` text uuid varchar(40) ``BinaryUUIDField`` blob bytea varbinary(16) ``DateTimeField`` datetime timestamp datetime ``DateField`` date date date ``TimeField`` time time time ``TimestampField`` integer integer integer ``IPField`` integer bigint bigint ``BooleanField`` integer boolean bool ``BareField`` untyped not supported not supported ``ForeignKeyField`` integer integer integer ===================== ================= ================= ================= .. seealso:: * SQLite fields for JSON, Full-Text Search: :ref:`sqlite` * Postgresql fields for Arrays, JSON, Full-Text Search, HStore: :ref:`postgresql` * MySQL fields for JSON: :ref:`mysql` * Extra fields (extension): :ref:`extra-fields` * :ref:`custom-fields` Common field parameters ^^^^^^^^^^^^^^^^^^^^^^^ All field types accept the following keyword arguments: ================ ========= ======================================================================= Parameter Default Description ================ ========= ======================================================================= ``null`` ``False`` allow null values ``index`` ``False`` create an index on this column ``unique`` ``False`` create a unique index on this column. See also :ref:`adding composite indexes `. ``column_name`` ``None`` explicitly specify the column name in the database. ``default`` ``None`` any value or callable to use as a default for uninitialized models ``primary_key`` ``False`` primary key for the table ``constraints`` ``None`` one or more constraints, e.g. ``[Check('price > 0')]`` ``sequence`` ``None`` sequence name (if backend supports it) ``collation`` ``None`` collation to use for ordering the field / index ``unindexed`` ``False`` indicate field on virtual table should be unindexed (**SQLite-only**) ``choices`` ``None`` optional iterable containing 2-tuples of ``value``, ``display`` ``help_text`` ``None`` string representing any helpful text for this field ``verbose_name`` ``None`` string representing the "user-friendly" name of this field ``index_type`` ``None`` specify a custom index-type, e.g. for Postgres you might specify a ``'BRIN'`` or ``'GIN'`` index. ================ ========= ======================================================================= Special parameters ^^^^^^^^^^^^^^^^^^ +-----------------------------+------------------------------------------------+ | Field type | Special Parameters | +=============================+================================================+ | :class:`ForeignKeyField` | ``model``, ``field``, ``backref``, | | | ``on_delete``, ``on_update``, ``deferrable`` | | | ``lazy_load`` | +-----------------------------+------------------------------------------------+ | :class:`CharField` | ``max_length`` | +-----------------------------+------------------------------------------------+ | :class:`FixedCharField` | ``max_length`` | +-----------------------------+------------------------------------------------+ | :class:`DateTimeField` | ``formats`` | +-----------------------------+------------------------------------------------+ | :class:`DateField` | ``formats`` | +-----------------------------+------------------------------------------------+ | :class:`TimeField` | ``formats`` | +-----------------------------+------------------------------------------------+ | :class:`TimestampField` | ``resolution``, ``utc`` | +-----------------------------+------------------------------------------------+ | :class:`DecimalField` | ``max_digits``, ``decimal_places``, | | | ``auto_round``, ``rounding`` | +-----------------------------+------------------------------------------------+ | :class:`BareField` | ``adapt`` | +-----------------------------+------------------------------------------------+ .. note:: Both ``default`` and ``choices`` could be implemented at the database level as *DEFAULT* and *CHECK CONSTRAINT* respectively, but any application change would require a schema change. Because of this, ``default`` is implemented purely in python and ``choices`` are not validated but exist for metadata purposes only. To add database (server-side) constraints, use the ``constraints`` parameter: .. code-block:: python class Product(Model): price = DecimalField(max_digits=8, decimal_places=2, constraints=[Check('price >= 0')]) added = DateTimeField(constraints=[Default('CURRENT_TIMESTAMP')]) status = IntegerField(constraints=[Check('status in (0, 1, 2)')]) Default field values ^^^^^^^^^^^^^^^^^^^^ Peewee can provide default values for fields when objects are created. For example to have an ``IntegerField`` default to zero rather than ``NULL``, you could declare the field with a default value: .. code-block:: python class Message(Model): context = TextField() read_count = IntegerField(default=0) created = DateTimeField(default=datetime.datetime.now) For ``read_count``, Peewee uses the literal value ``0``. For ``created``, Peewee calls ``datetime.datetime.now`` at the moment of instantiation - note that the **function itself is passed, not its return value**. **Mutable defaults require a factory function.** If a default value is a mutable object such as a ``list`` or ``dict``, passing it directly means every model instance shares *the same object*. Wrap it in a function instead: .. code-block:: python # Wrong: all instances share one dict. class Config(BaseModel): settings = JSONField(default={}) # Correct: each instance gets a fresh dict. def default_settings(): return {} class Config(BaseModel): settings = JSONField(default=default_settings) The database can also provide the default value for a field. While Peewee does not explicitly provide an API for setting a server-side default value, you can use the ``constraints`` and :func:`Default` to specify the server default: .. code-block:: python class Message(Model): content = TextField() timestamp = DateTimeField(constraints=[Default('CURRENT_TIMESTAMP')]) This produces a ``DEFAULT CURRENT_TIMESTAMP`` clause in the ``CREATE TABLE`` statement. Peewee's own ``default`` parameter produces no DDL; it only operates during Python-side model instantiation. A consequence of using server-generated defaults is that newly-inserted models will not automatically retrieve the new value. This requires a separate query to read back the defaults added by the server. ForeignKeyField --------------- :class:`ForeignKeyField` links a model to another model. It stores the related row's primary key as an integer column and provides a Python-level descriptor that resolves it to a full model instance on access. .. code-block:: python class Tweet(BaseModel): user = ForeignKeyField(User, backref='tweets') content = TextField() The ``backref`` parameter creates a reverse accessor on the target model. With ``backref='tweets'``, every ``User`` instance gains a ``tweets`` attribute that returns a pre-filtered :class:`Select` query of that user's tweets. :class:`ForeignKeyField` accepts referential action parameters: - ``on_delete`` - action to take when the referenced row is deleted. Common values: ``'CASCADE'``, ``'SET NULL'``, ``'RESTRICT'``. - ``on_update`` - action to take when the referenced row's primary key changes. - ``deferrable`` - defers constraint checking to transaction commit (Postgresql and SQLite only). .. warning:: SQLite does not enforce foreign key constraints by default. Enable enforcement by setting the ``foreign_keys`` pragma on connection: .. code-block:: python db = SqliteDatabase('my_app.db', pragmas={'foreign_keys': 1}) .. seealso:: :ref:`relationships` covers how foreign keys behave at runtime, including lazy loading, back-references, and avoiding N+1 query problems. Typically a foreign key will reference the primary key of the related model, but you can specify a particular column by specifying ``field=``. In Peewee, accessing the value of a :class:`ForeignKeyField` will return the entire related object: .. code-block:: python tweets = (Tweet .select(Tweet, User) .join(User) .order_by(Tweet.created_date.desc())) for tweet in tweets: print(tweet.user.username, tweet.message) In the example above the ``User`` data was selected efficiently. If we did not select the ``User``, then an **additional query** would be needed to fetch the associated ``User`` data: .. code-block:: python tweets = (Tweet .select() .order_by(Tweet.created_date.desc()) for tweet in tweets: # WARNING: an additional query will be issued for EACH tweet # to fetch the associated User data. print(tweet.user.username, tweet.message) Sometimes you only need the associated primary key value from the foreign key column. Peewee allows you to access the raw foreign key value by appending ``"_id"`` to the foreign key field's name: .. code-block:: python tweets = Tweet.select() for tweet in tweets: # Instead of "tweet.user", we will just get the raw ID value stored # in the column. print(tweet.user_id, tweet.message) To prevent accidentally resolving a foreign-key and triggering an additional query, :class:`ForeignKeyField` supports an initialization paramater ``lazy_load`` which, when disabled, behaves like the ``"_id"`` attribute: .. code-block:: python class Tweet(Model): # lazy-load disabled: user = ForeignKeyField(User, backref='tweets', lazy_load=False) ... for tweet in Tweet.select(): print(tweet.user, tweet.message) # With lazy-load disabled, accessing tweet.user will NOT perform an extra # query and the user ID value is returned instead. # e.g.: # 1 tweet from user1 # 1 another from user1 # 2 tweet from user2 # However, if we eagerly load the related user object, then the user # foreign key will behave like usual: for tweet in Tweet.select(Tweet, User).join(User): print(tweet.user.username, tweet.message) # user1 tweet from user1 # user1 another from user1 # user2 tweet from user1 ForeignKeyField Back-references ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :class:`ForeignKeyField` allows for a back-reference property to be bound to the target model. This property will be named ``_set`` by default, where ``classname`` is the lowercase name of the model class. This name can be overridden by specifying ``backref=``: .. code-block:: python class Message(Model): from_user = ForeignKeyField(User, backref='outbox') to_user = ForeignKeyField(User, backref='inbox') text = TextField() for message in some_user.outbox: # We are iterating over all Messages whose from_user is some_user. print(message) Back-references are just pre-filtered select queries, so we can add additional behavior like ``order_by()``: .. code-block:: python for message in some_user.inbox.order_by(Message.id): # Iterate over all Messages whose to_user is some_user. print(message) Self-referential foreign keys ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When creating a hierarchical structure it is necessary to create a self-referential foreign key which links a child object to its parent. Because the model class is not defined at the time you instantiate the self-referential foreign key, use the special string ``'self'`` to indicate a self-referential foreign key: .. code-block:: python class Category(Model): name = CharField() parent = ForeignKeyField('self', null=True, backref='children') The foreign key points **upward** to the parent object and the back-reference is named **children**. .. attention:: Self-referential foreign-keys should always be ``null=True``. When querying against a model that contains a self-referential foreign key you may sometimes need to perform a self-join. In those cases you can use :meth:`Model.alias` to create a table reference. Here is how you might query the category and parent model using a self-join: .. code-block:: python Parent = Category.alias() GrandParent = Category.alias() query = (Category .select(Category, Parent) .join(Parent, on=(Category.parent == Parent.id)) .join(GrandParent, on=(Parent.parent == GrandParent.id)) .where(GrandParent.name == 'some category') .order_by(Category.name)) For deeply nested hierarchies, recursive CTEs are more efficient than repeated self-joins. See :ref:`cte`. .. seealso:: :ref:`relationships` Date and Time Fields -------------------- The three fields devoted to working with dates and times have properties to access date attributes like year, month, hour, etc. :class:`DateField` Properties for: ``year``, ``month``, ``day`` :class:`TimeField` Properties for: ``hour``, ``minute``, ``second`` :class:`DateTimeField`: Properties for: ``year``, ``month``, ``day``, ``hour``, ``minute``, ``second`` These properties can be used as an expression in a query. Let's say we have an events table and want to list all the days in the current month which have at least one event: .. code-block:: python # Get the current date. today = datetime.date.today() # Get days that have events for the current month. query = (Event .select(Event.event_date.day.alias('day')) .where( (Event.event_date.year == today.year) & (Event.event_date.month == today.month)) .distinct()) # Group activity by hour of day. query = (PageView .select( PageView.timestamp.hour.alias('hour'), fn.COUNT(PageView.id).alias('n')) .group_by(PageView.timestamp.hour) .order_by(PageView.timestamp.hour)) .. note:: SQLite does not have a native date type, so dates are stored in formatted text columns. To ensure that comparisons work correctly, the dates need to be formatted so they are sorted lexicographically. That is why they are stored, by default, as ``YYYY-MM-DD HH:MM:SS``. :class:`TimestampField` stores a datetime as a Unix timestamp integer. The ``resolution`` parameter controls sub-second precision (default: seconds); ``utc=True`` instructs Peewee to treat stored values as UTC. BitField and BigBitField ------------------------ The :class:`BitField` and :class:`BigBitField` are suitable for storing bitmap data. :class:`BitField` provides a subclass of :class:`IntegerField` that is suitable for storing feature toggles as an integer bitmask. The latter is suitable for storing a bitmap for a large data-set, e.g. expressing membership or bitmap-type data. As an example of using :class:`BitField`, let's say we have a *Post* model and we wish to store certain True/False flags about how the post. We could store all these feature toggles in their own :class:`BooleanField` objects, or we could use a single :class:`BitField` instead: .. code-block:: python class Post(Model): content = TextField() flags = BitField() is_favorite = flags.flag(1) is_sticky = flags.flag(2) is_minimized = flags.flag(4) is_deleted = flags.flag(8) Using these flags is quite simple: .. code-block:: pycon >>> p = Post() >>> p.is_sticky = True >>> p.is_minimized = True >>> print(p.flags) # Prints 4 | 2 --> "6" 6 >>> p.is_favorite False >>> p.is_sticky True We can also use the flags on the Post class to build expressions in queries: .. code-block:: python # Generates a WHERE clause that looks like: # WHERE (post.flags & 1 != 0) favorites = Post.select().where(Post.is_favorite) # Query for sticky + favorite posts: sticky_faves = Post.select().where(Post.is_sticky & Post.is_favorite) Since the :class:`BitField` is stored in an integer, there is a maximum of 64 flags you can represent (64-bits is common size of integer column). For storing arbitrarily large bitmaps, you can instead use :class:`BigBitField`, which uses an automatically managed buffer of bytes, stored in a :class:`BlobField`. When bulk-updating one or more bits in a :class:`BitField`, you can use bitwise operators to set or clear one or more bits: .. code-block:: python # Set the 4th bit on all Post objects. Post.update(flags=Post.flags | 8).execute() # Clear the 1st and 3rd bits on all Post objects. Post.update(flags=Post.flags & ~(1 | 4)).execute() For simple operations, the flags provide handy ``set()`` and ``clear()`` methods for setting or clearing an individual bit: .. code-block:: python # Set the "is_deleted" bit on all posts. Post.update(flags=Post.is_deleted.set()).execute() # Clear the "is_deleted" bit on all posts. Post.update(flags=Post.is_deleted.clear()).execute() Example usage: .. code-block:: python class Bitmap(Model): data = BigBitField() bitmap = Bitmap() # Sets the ith bit, e.g. the 1st bit, the 11th bit, the 63rd, etc. bits_to_set = (1, 11, 63, 31, 55, 48, 100, 99) for bit_idx in bits_to_set: bitmap.data.set_bit(bit_idx) # We can test whether a bit is set using "is_set": assert bitmap.data.is_set(11) assert not bitmap.data.is_set(12) # We can clear a bit: bitmap.data.clear_bit(11) assert not bitmap.data.is_set(11) # We can also "toggle" a bit. Recall that the 63rd bit was set earlier. assert bitmap.data.toggle_bit(63) is False assert bitmap.data.toggle_bit(63) is True assert bitmap.data.is_set(63) # BigBitField supports item accessor by bit-number, e.g.: assert bitmap.data[63] bitmap.data[0] = 1 del bitmap.data[0] # We can also combine bitmaps using bitwise operators, e.g. b = Bitmap(data=b'\x01') b.data |= b'\x02' assert list(b.data) == [1, 1, 0, 0, 0, 0, 0, 0] assert len(b.data) == 1 .. _model-options: Model Settings -------------- Model-specific configuration is placed in a special :class:`Metadata` class called ``Meta``: .. code-block:: python :emphasize-lines: 8, 9 from peewee import * contacts_db = SqliteDatabase('contacts.db') class Person(Model): name = CharField() class Meta: database = contacts_db This instructs Peewee that whenever a query is executed on *Person* to use the contacts database. Once the class is defined metadata settings are accessible at ``ModelClass._meta``: .. code-block:: pycon >>> Person.Meta Traceback (most recent call last): File "", line 1, in AttributeError: type object 'Person' has no attribute 'Meta' >>> Person._meta The :class:`Metadata` class implements several methods which may be of use for retrieving model metadata (such as lists of fields, foreign key relationships, and more). .. code-block:: pycon >>> User._meta.fields {'id': , 'username': } >>> User._meta.primary_key >>> User._meta.database There are several options you can specify as ``Meta`` attributes. While most options are inheritable, some are table-specific and will not be inherited by subclasses. ========================= ====================================================== ============ Option Purpose Inheritable ========================= ====================================================== ============ ``database`` Database instance for this model. Yes ``table_name`` Explicit table name. No ``table_function`` Callable that returns a table name from the class. Yes ``indexes`` Tuple of multi-column index definitions. Yes ``primary_key`` :class:`CompositeKey` or ``False``. Yes ``constraints`` List of table-level constraint expressions. Yes ``schema`` Database schema name. Yes ``only_save_dirty`` Only emit changed fields on ``save()``. Yes ``options`` Extra options for ``CREATE TABLE`` extensions. Yes ``table_settings`` Strings appended after the closing parenthesis in DDL. Yes ``temporary`` Mark as a temporary table. Yes ``legacy_table_names`` Use legacy (non-snake-case) table name generation. Yes ``depends_on`` Declare a dependency on another table for ordering. No ``without_rowid`` SQLite ``WITHOUT ROWID`` tables. No ``strict_tables`` SQLite strict typing (3.37+). Yes ========================= ====================================================== ============ Example of inheritable vs non-inheritable settings: .. code-block:: pycon >>> db = SqliteDatabase(':memory:') >>> class ModelOne(Model): ... class Meta: ... database = db ... table_name = 'model_one_tbl' ... >>> class ModelTwo(ModelOne): ... pass ... >>> ModelOne._meta.database is ModelTwo._meta.database True >>> ModelOne._meta.table_name == ModelTwo._meta.table_name False .. _table-names: Table naming ^^^^^^^^^^^^ By default Peewee derives the table name from the model class name. The exact transformation depends on ``Meta.legacy_table_names``: =================== ========================= ============================== Model class name legacy (default) non-legacy =================== ========================= ============================== ``User`` ``user`` ``user`` ``UserProfile`` ``userprofile`` ``user_profile`` ``APIResponse`` ``apiresponse`` ``api_response`` ``WebHTTPRequest`` ``webhttprequest`` ``web_http_request`` =================== ========================= ============================== New projects should opt into non-legacy naming by setting ``legacy_table_names = False`` on ``BaseModel``. The legacy default exists only for backwards compatibility with existing deployments. .. code-block:: python class BaseModel(Model): class Meta: database = db legacy_table_names = False # Recommended for new projects. To override the table name entirely, use ``table_name``: .. code-block:: python class UserProfile(BaseModel): class Meta: table_name = 'acct_user_profile' # Maps to pre-existing table. To apply a naming convention programmatically across all models, use ``table_function``: .. code-block:: python def prefixed_table_name(model_class): return 'myapp_' + model_class.__name__.lower() class BaseModel(Model): class Meta: database = db table_function = prefixed_table_name class User(BaseModel): pass # Table name: "myapp_user" .. _model_indexes: Indexes and Constraints ----------------------- Peewee can create indexes on single or multiple columns, optionally including a *UNIQUE* constraint. Peewee also supports user-defined constraints on both models and fields. Single-column indexes and constraints ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Single column indexes are defined by specifying ``index=True`` or ``unique=True`` when declaring the Field. Add a unique index on *username* and a normal b-tree index on *email*: .. code-block:: python class User(Model): username = CharField(unique=True) email = CharField(index=True) To add a user-defined constraint on a column, you can specify it using the ``constraints`` parameter. You may wish to specify a default value as part of the schema, or add a ``CHECK`` constraint, for example: .. code-block:: python class Product(Model): name = CharField(unique=True) price = DecimalField(constraints=[Check('price < 10000')]) created = DateTimeField(constraints=[Default('CURRENT_TIMESTAMP')]) Multi-column indexes ^^^^^^^^^^^^^^^^^^^^ Multi-column indexes may be defined as *Meta* attributes using a nested tuple. Each database index is a 2-tuple, the first part of which is a tuple of the names of the fields, the second part a boolean indicating whether the index should be unique. .. code-block:: python class Transaction(Model): from_acct = CharField() to_acct = CharField() amount = DecimalField() date = DateTimeField() class Meta: indexes = ( # create a unique on from/to/date (('from_acct', 'to_acct', 'date'), True), # create a non-unique on from/to (('from_acct', 'to_acct'), False), ) .. note:: Remember to add a **trailing comma** if your tuple of indexes contains only one item: .. code-block:: python class Meta: indexes = ( (('first_name', 'last_name'), True), # Note the trailing comma! ) Partial and expression indexes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Partial indexes, indexes with expressions, and more complex indexes can use the :meth:`Model.add_index` API: .. code-block:: python class Article(BaseModel): name = TextField() timestamp = TimestampField() status = IntegerField() # Add a partial index on name and timestamp where status = 1. Article.add_index(Article.name, Article.timestamp, where=(Article.status == 1)) # Create a unique index on timestamp desc, status & 4. idx = Article.index( Article.timestamp.desc(), Article.flags.bin_and(4), unique=True) Article.add_index(idx) .. note:: SQLite does not support parameterized ``CREATE INDEX`` queries. Partial indexes and expression indexes on SQLite must be written using :class:`SQL`: .. code-block:: python Article.add_index(SQL('CREATE INDEX ... WHERE status = 1')) If the above is cumbersome, you can also pass a :class:`SQL` instance to ``Meta.indexes``: .. code-block:: python class Article(BaseModel): name = TextField() timestamp = TimestampField() status = IntegerField() class Meta: indexes = [ SQL('CREATE INDEX article_published_lookup ON ' 'article (name, timestamp) WHERE status = 1'), ] Primary Keys ------------ Auto-incrementing integer primary key ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If a model declares no primary key, Peewee automatically adds an auto-incrementing integer field named ``id``: .. code-block:: python class Article(BaseModel): title = TextField() # Peewee implicitly adds: id = AutoField() To use a different name for the auto-incrementing primary key, declare an :class:`AutoField` explicitly: .. code-block:: python class Article(BaseModel): article_id = AutoField() title = TextField() .. warning:: A common mistake is writing ``id = IntegerField(primary_key=True)`` when intending an auto-incrementing primary key. This declares a plain integer column whose value the application must supply - the database will not generate it. Use :class:`AutoField` for auto-increment behavior. Non-integer primary keys ^^^^^^^^^^^^^^^^^^^^^^^^ Any field can serve as the primary key by passing ``primary_key=True``: .. code-block:: python class Country(BaseModel): code = CharField(max_length=2, primary_key=True) # e.g. 'US', 'DE' name = TextField() When using a non-auto-incrementing primary key, Peewee cannot distinguish between a new row (needs ``INSERT``) and an existing row (needs ``UPDATE``) by checking whether the primary key is ``None``. On the first save, pass ``force_insert=True`` explicitly: .. code-block:: python country = Country(code='DE', name='Germany') country.save(force_insert=True) # First save: must force INSERT. country.name = 'Deutschland' country.save() # Subsequent saves: UPDATE as normal. :meth:`Model.create` handles this automatically, so it is the simpler option for one-step creation: .. code-block:: python country = Country.create(code='DE', name='Germany') .. _composite-keys: Composite primary keys ^^^^^^^^^^^^^^^^^^^^^^ Use :class:`CompositeKey` in ``Meta.primary_key`` to designate two or more columns as a composite primary key: .. code-block:: python class TweetTag(BaseModel): tweet = ForeignKeyField(Tweet) tag = TextField() class Meta: primary_key = CompositeKey('tweet', 'tag') Composite primary keys are most appropriate for junction tables in many-to-many relationships. Peewee has limited support for foreign keys *to* models with composite primary keys; avoid them in models that other models will reference. Models without a primary key ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ To create a table with no primary key, set ``primary_key = False``: .. code-block:: python class LogEntry(BaseModel): timestamp = DateTimeField() event = TextField() class Meta: primary_key = False Note that :meth:`Model.save` and :meth:`Model.delete_instance` do not work on keyless models, since both require a primary key to target a specific row. Use :meth:`Model.insert`, :meth:`Model.update`, and :meth:`Model.delete` (the class-level query methods) instead. Table Constraints ----------------- Peewee allows arbitrary constraints to :class:`Model` classes. Suppose you have a *people* table with a composite primary key of two columns: the person's first and last name. You wish to have another table relate to the *people* table. To do this define a multi-column foreign key constraint: .. code-block:: python class Person(Model): first = CharField() last = CharField() class Meta: primary_key = CompositeKey('first', 'last') class Pet(Model): owner_first = CharField() owner_last = CharField() pet_name = CharField() class Meta: constraints = [SQL('FOREIGN KEY(owner_first, owner_last) ' 'REFERENCES person(first, last)')] ``CHECK`` constraints can be specified at the table level: .. code-block:: python class Product(Model): name = CharField(unique=True) price = DecimalField() class Meta: constraints = [Check('price < 10000')] Creating Tables --------------- Once models are defined, create their corresponding tables with :meth:`Database.create_tables`: .. code-block:: python db.create_tables([User, Tweet, Favorite]) To create a single table, use :meth:`Model.create_table`: .. code-block:: python Tweet.create_table() .. seealso:: :ref:`schema` for documentation on table creation and other schema management tasks. .. _advanced-model-topics: Advanced Topics --------------- The following sections cover scenarios that arise less frequently. New users can skip this section and return to it when the need arises. .. _circular-fks: Circular foreign key dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Sometimes it happens that you will create a circular dependency between two tables. .. note:: Circular foreign keys should be refactored (by adding an intermediary table, for instance). Adding circular foreign keys with peewee is a bit tricky because at the time you are defining either foreign key, the model it points to will not have been defined yet, causing a ``NameError``. .. code-block:: python :emphasize-lines: 3 class User(Model): username = CharField() favorite_tweet = ForeignKeyField(Tweet, null=True) # NameError!! class Tweet(Model): message = TextField() user = ForeignKeyField(User, backref='tweets') One option is to simply use an :class:`IntegerField` to store the raw ID: .. code-block:: python class User(Model): username = CharField() favorite_tweet_id = IntegerField(null=True) By using :class:`DeferredForeignKey` we can get around the problem and still use a foreign key field: .. code-block:: python class User(BaseModel): username = TextField() favorite_tweet = DeferredForeignKey('Tweet', null=True) class Tweet(BaseModel): user = ForeignKeyField(User, backref='tweets') content = TextField() db.create_tables([User, Tweet]) # Add the constraint that could not be created at table-creation time. User._schema.create_foreign_key(User.favorite_tweet) .. note:: Because SQLite has limited support for altering tables, foreign-key constraints cannot be added to a table after it has been created. Field naming conflicts ^^^^^^^^^^^^^^^^^^^^^^ Several names are reserved by :class:`Model` for built-in methods and attributes (for example ``save``, ``create``, ``delete``, ``update``, ``get``). Declaring a field with one of these names overwrites the method. When the desired column name conflicts with a model method, supply an alternative attribute name and set ``column_name`` explicitly: .. code-block:: python class LogEntry(BaseModel): timestamp = DateTimeField() # "create" and "update" would conflict with Model.create / Model.update. created_at = DateTimeField(column_name='create') updated_at = DateTimeField(column_name='update') The database column is still named ``create`` and ``update``; the Python attributes are ``created_at`` and ``updated_at``. .. _barefield: BareField (SQLite only) ^^^^^^^^^^^^^^^^^^^^^^^ :class:`BareField` declares a column with no type affinity. It is only meaningful with SQLite, which permits untyped columns and virtual table columns. .. code-block:: python class FTSEntry(BaseModel): content = BareField() The optional ``adapt`` parameter specifies a callable that converts values coming from the database into a Python type: .. code-block:: python class RawData(BaseModel): value = BareField(adapt=float) For full-text search virtual tables, use :class:`SearchField` rather than :class:`BareField`. See :ref:`sqlite-fts`. .. _custom-fields: Custom fields ^^^^^^^^^^^^^ A custom field is a subclass of an existing field that overrides the Python-to-database and database-to-Python conversion methods. This is most useful when a database offers a column type that has no built-in Peewee equivalent, or when a standard column type should carry application-specific Python behavior. The two conversion hooks are: - ``db_value(self, value)`` - converts a Python value to the format the database driver expects. - ``python_value(self, value)`` - converts a value from the database driver into the desired Python type. The following example implements a field that stores a ``pathlib.Path`` value as a ``TEXT`` column: .. code-block:: python from pathlib import Path class PathField(TextField): def db_value(self, value): return str(value) if value is not None else None def python_value(self, value): return Path(value) if value is not None else None class Document(BaseModel): path = PathField() doc = Document.create(path=Path('/var/data/report.pdf')) assert isinstance(doc.path, Path) When the database requires a completely new storage type (not a variant of an existing one), set ``field_type`` to the type label and register the label with each database that will use it: .. code-block:: python class PointField(Field): field_type = 'point' # Custom type label. def db_value(self, value): if value is not None: return f'{value[0]},{value[1]}' def python_value(self, value): if value is not None: x, y = value.split(',') return (float(x), float(y)) # Tell Peewee what DDL type to emit for each database. sq_db = SqliteDatabase('mydb', field_types={'point': 'text'}) .. seealso:: :class:`Field` API reference. ================================================ FILE: docs/peewee/mysql.rst ================================================ .. _mysql: MySQL and MariaDB ================= .. module:: playhouse.mysql_ext Peewee provides alternate drivers for MySQL through ``playhouse.mysql_ext``. .. class:: MySQLConnectorDatabase(database, **kwargs) Database implementation using the official `mysql-connector-python `_ driver instead of ``pymysql``. .. code-block:: python from playhouse.mysql_ext import MySQLConnectorDatabase db = MySQLConnectorDatabase('my_db', host='1.2.3.4', user='mysql') .. class:: PooledMySQLConnectorDatabase(database, **kwargs) Connection-pooling variant of :class:`MySQLConnectorDatabase`. .. class:: MariaDBConnectorDatabase(database, **kwargs) Database implementation using the `mariadb-connector `_ driver. .. note:: Does **not** accept ``charset``, ``sql_mode``, or ``use_unicode`` parameters (charset is always ``utf8mb4``). .. code-block:: python from playhouse.mysql_ext import MariaDBConnectorDatabase db = MariaDBConnectorDatabase('my_db', host='1.2.3.4', user='mysql') .. class:: PooledMariaDBConnectorDatabase(database, **kwargs) Connection-pooling variant of :class:`MariaDBConnectorDatabase`. MySQL-specific helpers: .. module:: playhouse.mysql_ext: .. class:: JSONField() Extends :class:`TextField` with transparent JSON encoding/decoding. .. method:: extract(path) Extract a value from a JSON document at the given JSON path (e.g. ``'$.key'``). .. function:: Match(columns, expr, modifier=None) Helper for MySQL full-text search using ``MATCH ... AGAINST`` syntax. :param columns: A single :class:`Field` or a tuple of fields. :param str expr: Full-text search expression. :param str modifier: Optional modifier, e.g. ``'IN BOOLEAN MODE'``. .. code-block:: python from playhouse.mysql_ext import Match Post.select().where( Match((Post.title, Post.body), 'python asyncio', modifier='IN BOOLEAN MODE')) ================================================ FILE: docs/peewee/orm_utils.rst ================================================ .. _orm-utils: ORM Utilities ============= These modules provide higher-level abstractions on top of Peewee's core ORM and work with any database backend. .. contents:: On this page :local: :depth: 1 .. _shortcuts: Shortcuts --------- .. module:: playhouse.shortcuts ``playhouse.shortcuts`` provides helpers for serializing model instances to and from dictionaries, resolving compound queries, and thread-safe database swapping. Model Serialization ^^^^^^^^^^^^^^^^^^^ .. function:: model_to_dict(model, recurse=True, backrefs=False, only=None, exclude=None, extra_attrs=None, fields_from_query=None, max_depth=None, manytomany=False) Convert a model instance to a dictionary. :param bool recurse: Follow foreign keys and include the related object as a nested dict (default: ``True``). :param bool backrefs: Follow back-references and include related collections as nested lists of dicts. :param only: A list or set of field instances to include exclusively. :param exclude: A list or set of field instances to exclude. :param extra_attrs: A list of attribute or method names to include in the output dict. :param Select fields_from_query: Restrict serialization to only the fields that were explicitly selected in the generating query. :param int max_depth: Maximum depth when following relations. :param bool manytomany: Include many-to-many fields. Examples: .. code-block:: python user = User.create(username='alice') model_to_dict(user) # {'id': 1, 'username': 'alice'} model_to_dict(user, backrefs=True) # {'id': 1, 'username': 'alice', 'tweets': []} t = Tweet.create(user=user, content='hello') model_to_dict(t) # {'id': 1, 'content': 'hello', 'user': {'id': 1, 'username': 'alice'}} model_to_dict(t, recurse=False) # {'id': 1, 'content': 'hello', 'user': 1} model_to_dict(user, backrefs=True) # {'id': 1, 'tweets': [{'id': 1, 'content': 'hello'}], 'username': 'alice'} .. note:: If your use case is unusual, write a small custom function rather than trying to coerce ``model_to_dict`` with a complex combination of parameters. .. function:: dict_to_model(model_class, data, ignore_unknown=False) Construct a model instance from a dictionary. Foreign keys may be provided as nested dicts; back-references as lists of dicts. :param Model model_class: The model class to construct. :param dict data: A dictionary of data. Foreign keys can be included as nested dictionaries, and back-references as lists of dictionaries. :param bool ignore_unknown: Allow keys that do not correspond to any field on the model. .. code-block:: python user = dict_to_model(User, {'id': 1, 'username': 'alice'}) user.username # 'alice' # Nested foreign key: tweet = dict_to_model(Tweet, { 'id': 1, 'content': 'hi', 'user': {'id': 1, 'username': 'alice'}}) tweet.user.username # 'alice' .. function:: update_model_from_dict(instance, data, ignore_unknown=False) Update an existing model instance with values from a dictionary. Follows the same rules as :func:`dict_to_model`. :param Model instance: The model instance to update. :param dict data: A dictionary of data. Foreign keys can be included as nested dictionaries, and back-references as lists of dictionaries. :param bool ignore_unknown: Allow keys that do not correspond to any field on the model. Compound Query Resolution ^^^^^^^^^^^^^^^^^^^^^^^^^ .. function:: resolve_multimodel_query(query, key='_model_identifier') Resolve rows from a compound ``UNION`` or similar query to the correct model class. Useful when two tables are unioned and you need each row as an instance of the appropriate model. :param query: A compound :class:`SelectQuery`. :param str key: Name of the column used to identify the model. :returns: An iterable that yields properly typed model instances. Thread-Safe Database Swapping ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: ThreadSafeDatabaseMetadata() Model :class:`Metadata` implementation that enables the ``database`` attribute to safely changed in a multi-threaded application. Use this when your application may swap the active database (e.g. primary / read replica) at runtime across threads: .. code-block:: python from playhouse.shortcuts import ThreadSafeDatabaseMetadata primary = PostgresqlDatabase('main') replica = PostgresqlDatabase('replica') class BaseModel(Model): class Meta: database = primary model_metadata_class = ThreadSafeDatabaseMetadata # Safe to do at runtime from any thread: BaseModel._meta.database = replica .. _pydantic: Pydantic Integration -------------------- .. module:: playhouse.pydantic_utils ``playhouse.pydantic_utils`` generates `Pydantic v2 `_ models from Peewee :class:`Model` classes using the :func:`~playhouse.pydantic_utils.to_pydantic` function. Example ^^^^^^^ .. code-block:: python import datetime from peewee import * from playhouse.pydantic_utils import to_pydantic db = SqliteDatabase(':memory:') class User(db.Model): name = CharField(verbose_name='Full Name', help_text='Display name') age = IntegerField() active = BooleanField(default=True) bio = TextField(null=True) status = CharField( verbose_name='Status', help_text='Record status', choices=[ ('active', 'Active'), ('archived', 'Archived'), ('deleted', 'Deleted'), ]) created = DateTimeField(default=datetime.datetime.now) # Generate a Pydantic model in one call: UserSchema = to_pydantic(User) ``UserSchema`` is a standard Pydantic ``BaseModel``. You can validate data, serialize instances, or populate instances from user data: .. code-block:: python # Validate a dict (e.g. from an HTTP request body). data = UserSchema.model_validate({'name': 'Huey', 'age': 14, 'status': 'active'}) print(data.model_dump()) # {'name': 'Huey', 'age': 14, 'active': True, 'bio': None, 'score': None, # 'status': 'active', 'created': datetime.datetime(...)} # Populate an instance from the validated data. user = User(**validated.dict()) # Validate directly from a Peewee model instance: huey = User.create(name='Huey', age=14, status='active') data = UserSchema.model_validate(huey) How field metadata is mapped ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :func:`to_pydantic` reads the metadata you already set on your Peewee fields and translates it into the Pydantic equivalents: .. list-table:: :header-rows: 1 :widths: 25 75 * - Peewee attribute - Pydantic effect * - ``choices`` - The generated field uses a ``Literal`` type restricted to the choice values, and the available choices are appended to the field description. * - ``default`` / ``default=callable`` - Sets ``default`` or ``default_factory`` on the Pydantic field so it is not required in input data. * - ``null=True`` - Wraps the type in ``Optional[...]`` and defaults to ``None`` when no other default is provided. * - ``verbose_name`` - Becomes the ``title`` in the JSON schema. * - ``help_text`` - Becomes the ``description`` in the JSON schema. Fields with no default and ``null=False`` (default) are **required** in the generated Pydantic model. Field type mapping ^^^^^^^^^^^^^^^^^^ Peewee field types are mapped to a Python type that Pydantic uses for validation. +-------------------------------------------+------------------------+ | Peewee field | Python type | +===========================================+========================+ | ``CharField``, ``FixedCharField``, | ``str`` | | ``TextField`` | | +-------------------------------------------+------------------------+ | ``IntegerField``, ``SmallIntegerField``, | ``int`` | | ``BigIntegerField`` | | +-------------------------------------------+------------------------+ | ``AutoField``, ``BigAutoField`` | ``int`` | +-------------------------------------------+------------------------+ | ``FloatField``, ``DoubleField`` | ``float`` | +-------------------------------------------+------------------------+ | ``DecimalField`` | ``Decimal`` | +-------------------------------------------+------------------------+ | ``BooleanField`` | ``bool`` | +-------------------------------------------+------------------------+ | ``DateTimeField`` | ``datetime.datetime`` | +-------------------------------------------+------------------------+ | ``DateField`` | ``datetime.date`` | +-------------------------------------------+------------------------+ | ``TimeField`` | ``datetime.time`` | +-------------------------------------------+------------------------+ | ``BlobField`` | ``bytes`` | +-------------------------------------------+------------------------+ | ``UUIDField`` | ``uuid.UUID`` | +-------------------------------------------+------------------------+ | ``JSONField``, ``BinaryJSONField`` | ``dict`` | | (SQLite or Postgres extensions) | | +-------------------------------------------+------------------------+ | ``IntervalField`` (Postgres) | ``datetime.timedelta`` | +-------------------------------------------+------------------------+ | ``ForeignKeyField`` | *type of related PK* | +-------------------------------------------+------------------------+ ``AutoField`` and ``BigAutoField`` are excluded from the generated schema by default (``exclude_autofield=True``) - they can be included by passing ``exclude_autofield=False``. ``ForeignKeyField`` resolves through the related model's primary-key field, so a foreign key to a model with an ``AutoField`` PK becomes ``int``. This is overridden when you provide a nested schema via the ``relationships`` parameter. Any field whose ``field_type`` is not present in the map falls back to ``Any``, which means Pydantic will accept any value without validation. If you use custom field type and want strict validation, ensure they set a recognized ``field_type`` or handle the conversion yourself. When a field has ``choices`` defined, the mapped Python type above is **replaced** by a ``Literal`` constrained to the choice values, regardless of the underlying field type. API reference ^^^^^^^^^^^^^^ .. function:: to_pydantic(model_cls, exclude=None, include=None, exclude_autofield=True, model_name=None, relationships=None) Generate a Pydantic ``BaseModel`` class from a Peewee model. :param Model model_cls: Peewee model class. :param exclude: Field names to exclude from the generated schema. :type exclude: set or list :param include: If provided, *only* these field names will appear in the generated schema. All other fields are excluded. :type include: set or list :param bool exclude_autofield: When ``True`` (the default), the auto-incrementing primary-key field is omitted from the schema. Set to ``False`` when you need the ``id`` field in responses. :param str model_name: Name for the generated Pydantic class. Defaults to ``Schema``. :param dict relationships: A mapping that tells ``to_pydantic`` how to handle foreign-key or back-reference fields as nested Pydantic models instead of flat scalar values. See :ref:`pydantic-relationships` below. :returns: A Pydantic ``BaseModel`` subclass configured with ``from_attributes=True``. Generate a Pydantic ``Model`` for the given Peewee ``model_cls``. The generated model will preserve Peewee field metadata: * ``choices`` - restrict acceptable values for field. * ``default`` - provide a default value for field. * ``verbose_name`` - provide a human-readable title for field. * ``help_text`` - provide a human-readable description for field. * ``null`` - control whether field is optional or required. Foreign-key fields are exposed using the underlying column name, and accept a scalar value **unless** you specify the schema for the relation using the ``relationships`` parameter. See below for example. Foreign-key handling ^^^^^^^^^^^^^^^^^^^^ By default, foreign-key fields are exposed using their **underlying column name** (e.g. ``user_id`` rather than ``user``) and accept a plain scalar value, typically an integer primary key. This keeps the schema flat and is a good fit when you are accepting input data: .. code-block:: python class Tweet(db.Model): user = ForeignKeyField(User, backref='tweets') content = TextField() timestamp = DateTimeField(default=datetime.datetime.now) is_published = BooleanField(default=True) TweetSchema = to_pydantic(Tweet) # The schema exposes the column name "user_id", not "user": data = TweetSchema.model_validate({'user_id': 1, 'content': 'hello'}) print(data.model_dump()) # {'user_id': 1, # 'content': 'hello', # 'timestamp': datetime.datetime(...), # 'is_published: True} # Works when validating from a model instance too: tweet = Tweet.create(user=huey, content='hello') data = TweetSchema.model_validate(tweet) print(data.model_dump()) # {'user_id': 1, # 'content': 'hello', # 'timestamp': datetime.datetime(...), # 'is_published: True} .. _pydantic-relationships: Nested relationships ^^^^^^^^^^^^^^^^^^^^ When you wish to embed the related object rather than just its ID, pass a ``relationships`` dict that maps a Peewee :class:`ForeignKeyField` (or backref) to the Pydantic schema that should be used for the nested object. **Nested foreign key** .. code-block:: python # Include the id field so it appears in the response. UserSchema = to_pydantic(User, exclude_autofield=False) TweetResponse = to_pydantic( Tweet, exclude_autofield=False, relationships={Tweet.user: UserSchema}) tweet = Tweet.create(user=huey, content='hello') data = TweetResponse.model_validate(tweet) print(data.model_dump()) # {'id': 1, # 'content': 'hello', # 'user': {'id': 1, 'name': 'Huey', 'age': 14, ...}, # 'timestamp': datetime.datetime(...), # 'is_published': True} .. note:: Validating from a model instance will access ``tweet.user``, which triggers a SELECT query if the relation is not already loaded. To avoid the extra query, use a join: .. code-block:: python tweet = (Tweet .select(Tweet, User) .join(User) .get()) data = TweetResponse.model_validate(tweet) # No additional query. **Nested back-references** Back-references work the same way, but the schema must be wrapped in ``List[...]`` since back-references may contain 0..n records. .. code-block:: python from typing import List # Exclude the "user" FK from the tweet schema to avoid circular nesting. TweetResponse = to_pydantic(Tweet, exclude={'user'}, exclude_autofield=False) UserDetail = to_pydantic( User, exclude_autofield=False, relationships={User.tweets: List[TweetResponse]}) user = User.create(name='Huey', age=14, status='active') Tweet.create(user=user, content='tweet 0') Tweet.create(user=user, content='tweet 1') data = UserDetail.model_validate(user) print(data.model_dump()) # {'id': 1, 'name': 'Huey', ..., # 'tweets': [{'id': 1, 'content': 'tweet 0', ...}, # {'id': 2, 'content': 'tweet 1', ...}]} .. note:: As with foreign keys, accessing a back-reference triggers a query. Use :py:meth:`~ModelSelect.prefetch` to load the collection up front: .. code-block:: python users = (User .select() .where(User.id == 123) .prefetch(Tweet)) data = UserDetail.model_validate(users[0]) # No additional query. JSON schema output ^^^^^^^^^^^^^^^^^^ Because the generated class is a regular Pydantic model, you can call ``model_json_schema()`` to get a JSON-schema dict suitable for OpenAPI docs: .. code-block:: python import json print(json.dumps(UserSchema.model_json_schema(), indent=2)) .. code-block:: json { "properties": { "name": { "description": "Display name", "title": "Full Name", "type": "string" }, "age": { "title": "Age", "type": "integer" }, "active": { "default": true, "title": "Active", "type": "boolean" }, "bio": { "anyOf": [{"type": "string"}, {"type": "null"}], "default": null, "title": "Bio" }, "status": { "description": "Record status | Choices: 'active' = Active, 'archived' = Archived, 'deleted' = Deleted", "enum": ["active", "archived", "deleted"], "title": "Status", "type": "string" }, "created": { "format": "date-time", "title": "Created", "type": "string" } }, "required": ["name", "age", "status"], "title": "UserSchema", "type": "object" } Note that ``name``, ``age``, and ``status`` are the only required fields. All other fields have defaults (``active`` defaults to ``True``, ``bio`` defaults to ``None``, and ``created`` uses a ``default_factory``). .. _hybrid: Hybrid Attributes ----------------- .. module:: playhouse.hybrid A *hybrid attribute* behaves differently depending on whether it is accessed on a model **instance** (executes Python logic) or on the model **class** (generates a SQL expression). This lets you write Python methods that work both as Python computations and as composable SQL clauses. The concept is borrowed from SQLAlchemy's `hybrid extension `_. .. code-block:: python from playhouse.hybrid import hybrid_property, hybrid_method class Interval(Model): start = IntegerField() end = IntegerField() @hybrid_property def length(self): return self.end - self.start @hybrid_method def contains(self, point): return (self.start <= point) & (point < self.end) On an instance, Python arithmetic runs: .. code-block:: python i = Interval(start=1, end=5) i.length # 4 (Python arithmetic) i.contains(3) # True (Python comparison) On the class, SQL is generated: .. code-block:: python Interval.select().where(Interval.length > 5) # WHERE ("end" - "start") > 5 Interval.select().where(Interval.contains(2)) # WHERE ("start" <= 2) AND (2 < "end") When the Python and SQL implementations differ, provide a separate ``expression`` override: .. code-block:: python class Interval(Model): start = IntegerField() end = IntegerField() @hybrid_property def radius(self): return abs(self.length) / 2 # Python: uses Python abs() @radius.expression def radius(cls): return fn.ABS(cls.length) / 2 # SQL: uses fn.ABS() Example: .. code-block:: python query = Interval.select().where(Interval.radius < 3) This query is equivalent to the following SQL: .. code-block:: sql SELECT "t1"."id", "t1"."start", "t1"."end" FROM "interval" AS t1 WHERE ((abs("t1"."end" - "t1"."start") / 2) < 3) .. class:: hybrid_property(fget, fset=None, fdel=None, expr=None) Decorator for defining a property with separate instance and class behaviors. Use ``@prop.expression`` to specify the SQL form when it differs from the Python form. Examples: .. code-block:: python class Interval(Model): start = IntegerField() end = IntegerField() @hybrid_property def length(self): return self.end - self.start @hybrid_property def radius(self): return abs(self.length) / 2 @radius.expression def radius(cls): return fn.ABS(cls.length) / 2 When accessed on an ``Interval`` instance, the ``length`` and ``radius`` properties will behave as you would expect. When accessed as class attributes, though, a SQL expression will be generated instead: .. code-block:: python query = (Interval .select() .where( (Interval.length > 6) & (Interval.radius >= 3))) Would generate the following SQL: .. code-block:: sql SELECT "t1"."id", "t1"."start", "t1"."end" FROM "interval" AS t1 WHERE ( (("t1"."end" - "t1"."start") > 6) AND ((abs("t1"."end" - "t1"."start") / 2) >= 3) ) .. class:: hybrid_method(func, expr=None) Decorator for defining a method with separate instance and class behaviors. Use ``@method.expression`` to specify the SQL form. Example: .. code-block:: python class Interval(Model): start = IntegerField() end = IntegerField() @hybrid_method def contains(self, point): return (self.start <= point) & (point < self.end) When called with an ``Interval`` instance, the ``contains`` method will behave as you would expect. When called as a classmethod, though, a SQL expression will be generated: .. code-block:: python query = Interval.select().where(Interval.contains(2)) Would generate the following SQL: .. code-block:: sql SELECT "t1"."id", "t1"."start", "t1"."end" FROM "interval" AS t1 WHERE (("t1"."start" <= 2) AND (2 < "t1"."end")) .. _kv: Key/Value Store --------------- .. module:: playhouse.kv ``playhouse.kv.KeyValue`` provides a persistent dictionary backed by a Peewee database instance. .. code-block:: python from playhouse.kv import KeyValue KV = KeyValue() # Defaults to an in-memory SQLite database. KV['k1'] = 'v1' KV.update(k2='v2', k3='v3') assert KV['k2'] == 'v2' print(dict(KV)) # {'k1': 'v1', 'k2': 'v2', 'k3': 'v3'} # Expression-based access: for value in KV[KV.key > 'k1']: print(value) # 'v2', 'v3' # Expression-based bulk update: KV[KV.key > 'k1'] = 'updated' # Expression-based deletion: del KV[KV.key > 'k1'] .. class:: KeyValue(key_field=None, value_field=None, ordered=False, database=None, table_name='keyvalue') :param Field key_field: Field for the key. Defaults to :class:`CharField`. Must specify ``primary_key=True``. :param Field value_field: Field for the value. Defaults to :class:`PickleField`. :param bool ordered: Return keys in sorted order when iterating. :param Database database: Database to use. Defaults to an in-memory SQLite database. :param str table_name: Name of the underlying table. The table is created automatically on construction if it does not exist. Supports the standard dictionary interface plus expression-based access. .. method:: __contains__(expr) :param expr: a single key or an expression :returns: Boolean whether key/expression exists. Example: .. code-block:: python kv = KeyValue() kv.update(k1='v1', k2='v2') 'k1' in kv # True 'kx' in kv # False (KV.key < 'k2') in KV # True (KV.key > 'k2') in KV # False .. method:: __len__() :returns: Count of items stored. .. method:: __getitem__(expr) :param expr: a single key or an expression. :returns: value(s) corresponding to key/expression. :raises: ``KeyError`` if single key given and not found. Examples: .. code-block:: python KV = KeyValue() KV.update(k1='v1', k2='v2', k3='v3') KV['k1'] # 'v1' KV['kx'] # KeyError: "kx" not found KV[KV.key > 'k1'] # ['v2', 'v3'] KV[KV.key < 'k1'] # [] .. method:: __setitem__(expr, value) :param expr: a single key or an expression. :param value: value to set for key(s) Set value for the given key. If ``expr`` is an expression, then any keys matching the expression will have their value updated. Example: .. code-block:: python KV = KeyValue() KV.update(k1='v1', k2='v2', k3='v3') KV['k1'] = 'v1-x' print(KV['k1']) # 'v1-x' KV[KV.key >= 'k2'] = 'v99' print(dict(KV)) # {'k1': 'v1-x', 'k2': 'v99', 'k3': 'v99'} .. method:: __delitem__(expr) :param expr: a single key or an expression. Delete the given key. If an expression is given, delete all keys that match the expression. Example: .. code-block:: python KV = KeyValue() KV.update(k1=1, k2=2, k3=3) del KV['k1'] # Deletes "k1". del KV['k1'] # KeyError: "k1" does not exist del KV[KV.key > 'k2'] # Deletes "k3". del KV[KV.key > 'k99'] # Nothing deleted, no keys match. .. method:: keys() :returns: an iterable of all keys in the table. .. method:: values() :returns: an iterable of all values in the table. .. method:: items() :returns: an iterable of all key/value pairs in the table. .. method:: update(__data=None, **mapping) Efficiently bulk-insert or replace the given key/value pairs. Example: .. code-block:: python KV = KeyValue() KV.update(k1=1, k2=2) # Sets 'k1'=1, 'k2'=2. print(dict(KV)) # {'k1': 1, 'k2': 2} KV.update(k2=22, k3=3) # Updates 'k2'->22, sets 'k3'=3. print(dict(KV)) # {'k1': 1, 'k2': 22, 'k3': 3} KV.update({'k2': -2, 'k4': 4}) # Also can pass a dictionary. print(dict(KV)) # {'k1': 1, 'k2': -2, 'k3': 3, 'k4': 4} .. method:: get(expr, default=None) :param expr: a single key or an expression. :param default: default value if key not found. :returns: value of given key/expr or default if single key not found. Get the value at the given key. If the key does not exist, the default value is returned, unless the key is an expression in which case an empty list will be returned. .. method:: pop(expr, default=Sentinel) :param expr: a single key or an expression. :param default: default value if key does not exist. :returns: value of given key/expr or default if single key not found. Get value and delete the given key. If the key does not exist, the default value is returned, unless the key is an expression in which case an empty list is returned. .. method:: clear() Remove all items from the key-value table. .. _signals: Signals ------- .. module:: playhouse.signals ``playhouse.signals`` adds Django-style model lifecycle signals. Models must subclass ``playhouse.signals.Model`` (not ``peewee.Model``) for hooks to fire. .. code-block:: python from playhouse.signals import Model, post_save class MyModel(Model): data = IntegerField() class Meta: database = db @post_save(sender=MyModel) def on_save(model_class, instance, created): if created: notify_new(instance) The following signals are provided: ``pre_save`` Called immediately before an object is saved to the database. Provides an additional keyword argument ``created``, indicating whether the model is being saved for the first time or updated. ``post_save`` Called immediately after an object is saved to the database. Provides an additional keyword argument ``created``, indicating whether the model is being saved for the first time or updated. ``pre_delete`` Called immediately before an object is deleted from the database when :meth:`Model.delete_instance` is used. ``post_delete`` Called immediately after an object is deleted from the database when :meth:`Model.delete_instance` is used. ``pre_init`` Called when a model class is first instantiated .. warning:: Signals fire only through the high-level instance methods (:meth:`~Model.save`, :meth:`~Model.delete_instance`). Bulk operations via :meth:`~Model.insert`, :meth:`~Model.update`, and :meth:`~Model.delete` do not trigger signals because no model instance is involved. Connecting handlers ^^^^^^^^^^^^^^^^^^^ Whenever a signal is dispatched, it will call any handlers that have been registered. This allows totally separate code to respond to events like model save and delete. The :class:`Signal` class provides a :meth:`~Signal.connect` method, which takes a callback function and two optional parameters for "sender" and "name". If specified, the "sender" parameter should be a single model class and allows your callback to only receive signals from that one model class. The "name" parameter is used as a convenient alias in the event you wish to unregister your signal handler. Example: .. code-block:: python @post_save(sender=MyModel, name='project.cache_buster') def cache_bust(sender, instance, created): cache.delete(make_cache_key(instance)) Or connect manually: .. code-block:: python def on_delete(sender, instance): audit_log(instance) pre_delete.connect(on_delete, sender=MyModel) Disconnect by name or reference: .. code-block:: python post_save.disconnect(name='project.cache_buster') pre_delete.disconnect(on_delete) Signal callback signature: - ``pre_init(sender, instance)`` - ``pre_save(sender, instance, created)`` - ``post_save(sender, instance, created)`` - ``pre_delete(sender, instance)`` - ``post_delete(sender, instance)`` .. class:: Signal() Stores a list of receivers (callbacks) and calls them when the "send" method is invoked. .. method:: connect(receiver, name=None, sender=None) :param callable receiver: a callable that takes at least two parameters, a "sender", which is the Model subclass that triggered the signal, and an "instance", which is the actual model instance. :param string name: a short alias :param Model sender: if specified, only instances of this model class will trigger the receiver callback. Add the receiver to the internal list of receivers, which will be called whenever the signal is sent. .. code-block:: python from playhouse.signals import post_save from project.handlers import cache_buster post_save.connect(cache_buster, name='project.cache_buster') .. method:: disconnect(receiver=None, name=None, sender=None) :param callable receiver: the callback to disconnect :param string name: a short alias :param Model sender: disconnect model-specific handler. Disconnect the given receiver (or the receiver with the given name alias) so that it no longer is called. Either the receiver or the name must be provided. .. code-block:: python post_save.disconnect(name='project.cache_buster') .. method:: send(instance, *args, **kwargs) :param instance: a model instance Iterates over the receivers and will call them in the order in which they were connected. If the receiver specified a sender, it will only be called if the instance is an instance of the sender. .. _dataset: DataSet ------- .. module:: playhouse.dataset ``playhouse.dataset`` exposes a dict-oriented API for relational data, modeled after the `dataset library `_. It is useful for quick scripts, data loading, and CSV/JSON import-export. Basic operations: .. code-block:: python from playhouse.dataset import DataSet db = DataSet('sqlite:///data.db') # Access a table (created automatically if it doesn't exist): users = db['user'] # Insert rows with any columns: users.insert(name='Alice', age=30) users.insert(name='Bob', age=25, active=True) # New column added automatically. # Retrieve rows: alice = users.find_one(name='Alice') print(alice) # {'id': 1, 'name': 'Alice', 'age': 30, 'active': None} for user in users: print(user['name']) for admin in users.find(active=True): print(admin['name']) # Bob. # Update: users.update(name='Alice', age=31, columns=['name']) # 'name' is the lookup. # Update all records: users.update(admin=False) # Delete: users.delete(name='Bob') Export and import data: .. code-block:: python # Export to JSON: db.freeze(users.all(), format='json', filename='users.json') # Export CSV to stdout: db.freeze(users.all(), format='csv', file_obj=sys.stdout) # Import from CSV: db.thaw('user', format='csv', filename='import.csv') # Import a JSON file to a new table. db.thaw('new_table', format='json', filename='json-data.json') Transactions: .. code-block:: python # Transactions. with db.transaction() as txn: users.insert(name='Charlie') with db.transaction() as nested_txn: table.update(name='Charlie', favorite_orm='sqlalchemy', columns=['name']) nested_txn.rollback() # JK. Introspection: .. code-block:: python print(db.tables) # ['new_table', 'user'] print(db['user'].columns) # ['id', 'age', 'name', 'active', 'admin', 'favorite_orm'] print(len(db['user'])) # 2 .. class:: DataSet(url, **kwargs) :param url: :ref:`db-url` or a :class:`Database` instance. :param kwargs: additional keyword arguments passed to :meth:`Introspector.generate_models` when introspecting the db. .. attribute:: tables List of table names in the database (computed dynamically). .. method:: __getitem__(table_name) Return a :class:`Table` for the given name. Creates the table if it doesn't exist. .. method:: query(sql, params=None, commit=True) :param str sql: A SQL query. :param list params: Optional parameters for the query. :param bool commit: Whether the query should be committed upon execution. :return: A database cursor. Execute the provided query against the database. .. method:: transaction() Return a context manager representing a transaction. .. method:: freeze(query, format='csv', filename=None, file_obj=None, encoding='utf8', iso8601_datetimes=False, base64_bytes=False, **kwargs) :param query: A :class:`SelectQuery`, generated using :meth:`~Table.all` or `~Table.find`. :param format: Output format. By default, *csv* and *json* are supported. :param filename: Filename to write output to. :param file_obj: File-like object to write output to. :param str encoding: File encoding. :param bool iso8601_datetimes: Encode datetimes and dates in ISO 8601 format. :param bool base64_bytes: Encode binary data as base64. By default hex is used. :param kwargs: Arbitrary parameters for export-specific functionality. Export data to a file. .. method:: thaw(table, format='csv', filename=None, file_obj=None, strict=False, encoding='utf8', iso8601_datetimes=False, base64_bytes=False, **kwargs) :param str table: The name of the table to load data into. :param format: Input format. By default, *csv* and *json* are supported. :param filename: Filename to read data from. :param file_obj: File-like object to read data from. :param bool strict: Whether to store values for columns that do not already exist on the table. :param str encoding: File encoding. :param bool iso8601_datetimes: Decode datetimes and dates from ISO 8601 format. :param bool base64_bytes: Decode BLOB field-data from base64. By default hex is assumed. :param kwargs: Arbitrary parameters for import-specific functionality. Import data from a file into ``table``. If ``strict=False`` (default), new columns are added automatically. .. method:: connect() close() Open or close the underlying database connection. .. class:: Table(dataset, name, model_class) Provides a high-level API for working with rows in a given table. .. attribute:: columns List of column names. .. attribute:: model_class A dynamically-created :class:`Model` class. .. method:: insert(**data) Insert a row, adding new columns as needed. .. method:: update(columns=None, conjunction=None, **data) Update the table using the provided data. If one or more columns are specified in the *columns* parameter, then those columns' values in the *data* dictionary will be used to determine which rows to update. .. code-block:: python # Update all rows. db['users'].update(favorite_orm='peewee') # Only update Huey's record, setting his age to 3. db['users'].update(name='Huey', age=3, columns=['name']) .. method:: find(**query) Return all rows matching equality conditions (all rows if no conditions given). .. method:: find_one(**query) Return the first matching row, or ``None``. .. method:: all() Return all rows. .. method:: delete(**query) Delete matching rows (all rows if no conditions given). .. method:: create_index(columns, unique=False) Create an index on the given columns: .. code-block:: python # Create a unique index on the `username` column. db['users'].create_index(['username'], unique=True) .. method:: freeze(format='csv', filename=None, file_obj=None, encoding='utf8', iso8601_datetimes=False, base64_bytes=False, **kwargs) :param format: Output format. By default, *csv* and *json* are supported. :param filename: Filename to write output to. :param file_obj: File-like object to write output to. :param str encoding: File encoding. :param bool iso8601_datetimes: Encode datetimes and dates in ISO 8601 format. :param bool base64_bytes: Encode binary data as base64. By default hex is used. :param kwargs: Arbitrary parameters for export-specific functionality. .. method:: thaw(format='csv', filename=None, file_obj=None, strict=False, encoding='utf8', iso8601_datetimes=False, base64_bytes=False, **kwargs) :param format: Input format. By default, *csv* and *json* are supported. :param filename: Filename to read data from. :param file_obj: File-like object to read data from. :param bool strict: Whether to store values for columns that do not already exist on the table. :param str encoding: File encoding. :param bool iso8601_datetimes: Decode datetimes and dates from ISO 8601 format. :param bool base64_bytes: Decode BLOB field-data from base64. By default hex is assumed. :param kwargs: Arbitrary parameters for import-specific functionality. .. _extra-fields: Extra Field Types ----------------- .. module:: playhouse.fields ``playhouse.fields`` provides two general-purpose field types. .. class:: CompressedField(compression_level=6, algorithm='zlib', **kwargs) Stores compressed binary data using ``zlib`` or ``bz2``. Extends :class:`BlobField`; compression and decompression are transparent: .. code-block:: python from playhouse.fields import CompressedField class LogEntry(Model): payload = CompressedField(algorithm='zlib', compression_level=9) :param int compression_level: 0-9 (9 is maximum compression). :param str algorithm: ``'zlib'`` or ``'bz2'``. .. class:: PickleField() Stores arbitrary Python objects by pickling them into a :class:`BlobField`. .. code-block:: python from playhouse.fields import PickleField class CachedResult(Model): data = PickleField() CachedResult.create(data={'nested': [1, 2, 3]}) .. _flask-utils: Flask Utilities --------------- .. module:: playhouse.flask_utils ``playhouse.flask_utils`` simplifies Peewee integration with `Flask `_. FlaskDB Wrapper ^^^^^^^^^^^^^^^^ :class:`FlaskDB` handles three boilerplate tasks: 1. Creates a Peewee database instance from Flask's ``app.config``. 2. Provides a ``Model`` base class whose ``Meta.database`` is wired to the Peewee instance. 3. Registers ``before_request`` / ``teardown_request`` hooks that open and close a connection for every request. Basic setup: .. code-block:: python from flask import Flask from playhouse.flask_utils import FlaskDB app = Flask(__name__) app.config['DATABASE'] = 'postgresql://postgres:pw@localhost/my_app' db_wrapper = FlaskDB(app) class User(db_wrapper.Model): username = CharField(unique=True) class Tweet(db_wrapper.Model): user = ForeignKeyField(User, backref='tweets') content = TextField() Access the underlying Peewee database: .. code-block:: python peewee_db = db_wrapper.database @app.route('/transfer', methods=['POST']) def transfer(): with peewee_db.atomic(): # ... transactional logic ... return jsonify({'ok': True}) Application factory pattern: .. code-block:: python db_wrapper = FlaskDB() class User(db_wrapper.Model): username = CharField(unique=True) def create_app(): app = Flask(__name__) app.config['DATABASE'] = 'sqlite:///my_app.db' db_wrapper.init_app(app) return app Configuration via dict or a :class:`Database` instance directly: .. code-block:: python # Dictionary-based (uses playhouse.db_url under the hood): app.config['DATABASE'] = { 'name': 'my_app', 'engine': 'playhouse.pool.PooledPostgresqlDatabase', 'user': 'postgres', 'max_connections': 32, } # Pass a database object: peewee_db = PostgresqlExtDatabase('my_app') db_wrapper = FlaskDB(app, peewee_db) Excluding routes from connection management: .. code-block:: python app.config['FLASKDB_EXCLUDED_ROUTES'] = ('health_check', 'static') .. class:: FlaskDB(app=None, database=None) :param app: Flask application instance (optional; use ``init_app()`` for the factory pattern). :param database: A database URL string, configuration dictionary, or a :class:`Database` instance. .. attribute:: database The underlying :class:`Database` instance. .. attribute:: Model A base :class:`Model` class bound to this database instance. .. method:: init_app(app) Bind to a Flask application (factory pattern). Query Helpers ^^^^^^^^^^^^^ .. function:: get_object_or_404(query_or_model, *query) :param query_or_model: Either a :class:`Model` class or a pre-filtered :class:`SelectQuery`. :param query: Peewee filter expressions. Retrieve a single object matching the given query, or abort with HTTP 404 if no match is found. .. code-block:: python @app.route('/post//') def post_detail(slug): post = get_object_or_404( Post.select().where(Post.published == True), Post.slug == slug) return render_template('post_detail.html', post=post) .. function:: object_list(template_name, query, context_variable='object_list', paginate_by=20, page_var='page', check_bounds=True, **kwargs) Paginate a query and render a template with the results. :param str template_name: Template to render. :param query: :class:`SelectQuery` to paginate. :param str context_variable: Template variable name for the page of objects (default: ``'object_list'``). :param int paginate_by: Items per page. :param str page_var: GET parameter name for the page number. :param bool check_bounds: Return 404 for invalid page numbers. :param kwargs: Extra template context variables. The template receives: - ``object_list`` (or ``context_variable``) - page of objects. - ``page`` - current page number. - ``pagination`` - a :class:`PaginatedQuery` instance. .. code-block:: python @app.route('/posts/') def post_list(): return object_list( 'post_list.html', query=Post.select().where(Post.published == True), paginate_by=10) .. class:: PaginatedQuery(query_or_model, paginate_by, page_var='page', check_bounds=False) :param query_or_model: Either a :class:`Model` or a :class:`SelectQuery` instance containing the collection of records you wish to paginate. :param paginate_by: Number of objects per-page. :param page_var: The name of the ``GET`` argument which contains the page. :param check_bounds: Whether to check that the given page is a valid page. If ``check_bounds`` is ``True`` and an invalid page is specified, then a 404 will be returned. Helper class to perform pagination based on ``GET`` arguments. .. method:: get_page() Return the current page number (1-based; defaults to 1). .. method:: get_page_count() Return the total number of pages. .. method:: get_object_list() Return the :class:`SelectQuery` for the requested page, with appropriate ``LIMIT`` and ``OFFSET`` applied. Returns a 404 if ``check_bounds=True`` and the page is empty. ================================================ FILE: docs/peewee/pool-snippet.rst ================================================ Commonly-used pool implementations: * :class:`PooledPostgresqlDatabase` * :class:`PooledMySQLDatabase` * :class:`PooledSqliteDatabase` Additional implementations: * ``playhouse.cysqlite_ext`` - :class:`PooledCySqliteDatabase` * ``playhouse.mysql_ext`` - :class:`PooledMariaDBConnectorDatabase` * ``playhouse.mysql_ext`` - :class:`PooledMySQLConnectorDatabase` * ``playhouse.postgres_ext`` - :class:`PooledPostgresqlExtDatabase` * ``playhouse.postgres_ext`` - :class:`PooledPsycopg3Database` * ``playhouse.cockroachdb`` - :class:`PooledCockroachDatabase` ================================================ FILE: docs/peewee/postgres.rst ================================================ .. _postgresql: Postgresql ========== .. module:: playhouse.postgres_ext The ``playhouse.postgres_ext`` module exposes Postgresql-specific field types and features that are not available in the standard :class:`PostgresqlDatabase`. .. contents:: On this page :local: :depth: 1 Getting Started --------------- To get started import the ``playhouse.postgres_ext`` module and use the :class:`PostgresqlExtDatabase` database class: .. code-block:: python from playhouse.postgres_ext import * db = PostgresqlExtDatabase('peewee_test', user='postgres') class BaseExtModel(Model): class Meta: database = db .. _postgres-ext-api: PostgresqlExtDatabase --------------------- .. class:: PostgresqlExtDatabase(database, server_side_cursors=False, register_hstore=False, prefer_psycopg3=False, **kwargs) Extends :class:`PostgresqlDatabase` and is required to use: * :class:`ArrayField` * :class:`DateTimeTZField` * :class:`JSONField` / :class:`BinaryJSONField` * :class:`HStoreField` * :class:`TSVectorField` * :ref:`postgres-server-side-cursors` :param str database: Name of database to connect to. :param bool server_side_cursors: Whether ``SELECT`` queries should utilize server-side cursors. :param bool register_hstore: Register the hstore extension. :param bool prefer_psycopg3: If both psycopg2 and psycopg3 are installed, instruct Peewee to prefer psycopg3. When using ``server_side_cursors`` be sure to wrap your queries with :func:`ServerSide`. .. class:: PooledPostgresqlExtDatabase(database, **kwargs) Connection-pooling variant of :class:`PostgresqlExtDatabase`. .. class:: Psycopg3Database(database, **kwargs) Same as :class:`PostgresqlExtDatabase` but specifies ``prefer_psycopg3=True``. .. class:: PooledPsycopg3Database(database, **kwargs) Connection-pooling variant of :class:`Psycopg3Database`. .. _postgres-json: JSON Support ------------ Peewee provides two JSON field types for Postgresql: - :class:`BinaryJSONField` - stores JSON in the efficient binary ``jsonb`` format. Supports key/item access, containment operations. - :class:`JSONField` - stores JSON as text, supports key/item access. Most applications will wish to use :class:`BinaryJSONField` (``JSONB``): * Faster Queries: direct access to data elements without parsing the entire JSON document each time. * Index Support: supports indexing via GiST or GIN. * Faster updates without requiring rewriting the entire document. The only time :class:`JSONField` is preferable is when you must store the exact JSON data verbatim (whitespace, object key ordering). .. code-block:: python from playhouse.postgres_ext import PostgresqlExtDatabase, BinaryJSONField db = PostgresqlExtDatabase('my_app') class Event(Model): data = BinaryJSONField() class Meta: database = db # Store data: Event.create(data={ 'type': 'login', 'user_id': 42, 'request': {'ip': '1.2.3.4'}, 'success': True}) # Filter using a nested key: query = (Event .select() .where(Event.data['request']['ip'] == '1.2.3.4')) # Select, group and order-by JSON values. query = (Event .select(Event.data['user_id'], fn.COUNT(Event.id)) .group_by(Event.data['user_id']) .order_by(Event.data['user_id']) .tuples()) # Retrieve JSON objects. query = (Event .select(Event.data['request'].as_json().alias('request')) .where(Event.data['user_id'] == 42)) for event in query: print(event.request['ip']) .. tip:: Refer to the `Postgresql JSON documentation `__ for in-depth discussion and examples of using JSON and JSONB. BinaryJSONField and JSONField ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. class:: BinaryJSONField(dumps=None, *args, **kwargs) :param dumps: custom implementation of ``json.dumps`` Extends :class:`JSONField` for the ``jsonb`` type. By default BinaryJSONField will use a GiST index. To disable this, initialize the field with ``index=False``. .. method:: as_json() Deserialize and return the JSON value at the given path. .. method:: concat(data) Concatenate the field value with ``data``. Note this is a shallow operation and does not deep-merge nested objects. Example: .. code-block:: python # Add object - if "result" key existed before it is overwritten. (Event .update(data=Event.data.concat({'result': {'success': True}})) .execute()) Nested data can also use ``concat()``: .. code-block:: python # Select the result subkey and merge with additional data: # {'ip': '1.2.3.4'} --> {'ip': '1.2.3.4', 'status': 'ok'} Event.select(Event.data['result'].concat({'status': 'ok'})) .. method:: contains(other) Test whether this field's value contains ``other`` (as a subset). ``other`` may be a partial dict, list, or scalar value. Useful for matching against a partial JSON object or checking if an item is in an array. .. code-block:: python Event.create(data={ 'type': 'rename', 'name': 'new name', 'metadata': {'old_name': 'the old name'}, 'tags': ['t1', 't2', 't3']}) # These queries match the above row: # Search by partial object: Event.select().where(Event.data.contains({'type': 'rename'})) # Partial object and partial array: Event.select().where(Event.data.contains({ 'type': 'rename', 'tags': ['t2', 't1'], })) # Partial array, irrespective of ordering: Event.select().where(Event.data['tags'].contains(['t2', 't1'])) # Search array by individual item: Event.select().where(Event.data['tags'].contains('t1')) To test whether a **key** simply exists, use :meth:`~BinaryJSONField.has_key`: .. code-block:: python Event.select().where(Event.data.has_key('name')) # Or search a sub-key. Event.select().where(Event.data['metadata'].has_key('old_name')) .. method:: contains_any(*keys) Test whether any of ``keys`` is present in the JSON value. .. code-block:: python Event.create(data={ 'type': 'rename', 'name': 'new name', 'metadata': {'old_name': 'the old name'}, 'tags': ['t1', 't2', 't3']}) # These queries match the above row: Event.select().where(Event.data.contains_any('name', 'other')) # Search a nested object: Event.select().where( Event.data['metadata'].contains_any('old_name', 'old_status')) # Search nested object for items in an array: Event.select().where(Event.data['tags'].contains_any('t3', 'tx')) .. method:: contains_all(*keys) Test whether all of ``keys`` are present in the JSON value. .. code-block:: python Event.create(data={ 'type': 'rename', 'name': 'new name', 'metadata': {'old_name': 'the old name'}, 'tags': ['t1', 't2', 't3']}) # These queries match the above row: Event.select().where(Event.data.contains_all('name', 'tags')) # Search nested object for items in an array: Event.select().where(Event.data['tags'].contains_all('t3', 't2')) .. method:: contained_by(other) Test whether this field's value is a subset of ``other``. .. code-block:: python Event.create(data={ 'type': 'login', 'result': {'success': True}}) Event.create(data={ 'type': 'rename', 'name': 'new name', 'metadata': {'old_name': 'the old name'}, 'tags': ['t1', 't2', 't3']}) # Matches the login row. (Event .select() .where(Event.data.contained_by({ 'type': 'login', 'result': {'success': True, 'message': 'OK'}}))) # Match events that have a result w/success=True and/or # error=False: Event.select().where(Event.data['result'].contained_by({ 'success': True, 'error': False}) # Check that tags are subset of the popular tags (matches rename row). popular_tags = ['t3', 't2', 't1', 'tx', 'ty'] Event.select().where(Event.data['tags'].contained_by(popular_tags)) .. method:: has_key(key) Test whether ``key`` exists. .. code-block:: python Event.select().where(Event.data.has_key('result')) Event.select().where(Event.data['result'].has_key('success')) .. method:: remove(*keys) Remove one or more keys from the JSON object. .. code-block:: python # Atomically remove key: Event.update(data=Event.data.remove('result')).execute() # Equivalent to above: Event.update(data=Event.data['result'].remove()).execute() # Remove deeply-nested item: Event.update(data=Event.data['metadata']['prior'].remove()) .. method:: length() Return the length of the JSON array at the given path. .. code-block:: python Event.select().where(Event.data['tags'].length() > 3) .. method:: extract(*path) Extract the JSON data at the given path. .. code-block:: python Event.select().where(Event.data.extract('tags', 0) == 'first_tag') Event.select().where(Event.data.extract('result', 'success') == True) # Equivalent to above. Event.select().where(Event.data['result'].extract('success') == True) .. class:: JSONField(dumps=None, *args, **kwargs) :param dumps: custom implementation of ``json.dumps`` Field that stores and retrieves JSON data. Supports ``__getitem__`` key access for filtering and sub-object retrieval. Consider using the :class:`BinaryJSONField` instead as it offers better performance and more powerful querying options. .. method:: as_json() Deserialize and return the JSON value at the given path. .. method:: concat(data) Concatenate the field value with ``data``. Note this is a shallow operation and does not deep-merge nested objects. See :meth:`BinaryJSONField.concat` for example usage. .. method:: length() Return the length of the JSON array at the given path. See :meth:`BinaryJSONField.length` for example usage. .. method:: extract(*path) Extract the JSON data at the given path. See :meth:`BinaryJSONField.extract` for example usage. .. _postgres-hstore: HStore ------ Postgresql's `hstore `_ extension stores arbitrary key/value pairs in a single column. Enable it by passing ``register_hstore=True`` when initializing the database: .. code-block:: python db = PostgresqlExtDatabase('my_app', register_hstore=True) class Event(Model): data = HStoreField() class Meta: database = db :class:`HStoreField` supports the following operations: * Store and retrieve arbitrary dictionaries * Filter by key(s) or partial dictionary * Update/add one or more keys to an existing dictionary * Delete one or more keys from an existing dictionary * Select keys, values, or zip keys and values * Retrieve a slice of keys/values * Test for the existence of a key * Test that a key has a non-NULL value Example: .. code-block:: python # Create a record with arbitrary attributes: Event.create(data={ 'type': 'register', 'ip': '1.2.3.4', 'email': 'charles@example.com', 'result': 'success', 'referrer': 'google.com'}) Event.create(data={ 'type': 'login', 'ip': '1.2.3.4', 'email': 'charles@example.com', 'result': 'success'}) # Lookup nested values in the data: Event.select().where(Event.data['type'] == 'login') # Filter by a key/value pair: Event.select().where(Event.data.contains({'result': 'success'}) # Filter by key existence: Event.select().where(Event.data.exists('referrer')) # Atomic update - adds new keys, updates existing ones: new_data = Event.data.update({ 'result': 'ok', 'status': 'success'}) (Event .update(data=new_data) .where(Event.data['result'] == 'success') .execute()) # Atomic key deletion: (Event .update(data=Event.data.delete('referrer')) .where(Event.data['referrer'] == 'google.com') .execute()) # Retrieve keys or values as a list: for event in Event.select(Event.id, Event.data.keys().alias('k')): print(event.id, event.k) # Prints: # 1 ['ip', 'type', 'email', 'result', 'status'] # Retrieve a subset of data: query = (Event .select(Event.id, Event.data.slice('ip', 'email').alias('source')) .order_by(Event.data['ip'])) for event in query: print(event.id, event.source) # Prints: # 1 {'ip': '1.2.3.4', 'email': 'charles@example.com'} HStoreField API ^^^^^^^^^^^^^^^ .. class:: HStoreField() By default ``HStoreField`` will use a *GiST* index. To disable this, initialize the field with ``index=False``. .. method:: __getitem__(key) :param str key: get value at given key. Example: .. code-block:: python Event.select().where(Event.data['type'] == 'login') .. method:: contains(value) :param value: value to search for. :type value: dict, list, tuple or string key. Test whether the HStore data contains the given ``dict`` (match keys and values), ``list``/``tuple`` (match keys), or ``str`` key. Example: .. code-block:: python # Contains key/value pairs: Event.select().where(Event.data.contains({'result': 'success'})) # Contains a list of keys: Event.select().where(Event.data.contains(['result', 'status'])) # Contains a single key: Event.select().where(Event.data.contains('result')) .. method:: contains_any(*keys) Test whether the HStore contains any of the given keys. .. method:: exists(key) Test whether key exists in data. .. method:: defined(key) Test whether key is non-NULL in data. .. method:: update(__data=None, **data) :param dict __data: Specify update as a ``dict``. :param data: Specify update as keyword arguments. Perform an in-place, atomic update. .. code-block:: python # Atomic update - adds new keys, updates existing ones: new_data = Event.data.update({ 'result': 'ok', 'status': 'success'}) (Event .update(data=new_data) .where(Event.data['result'] == 'success') .execute()) .. method:: delete(*keys) :param keys: one or more keys to delete from data. .. code-block:: python # Atomic key deletion: (Event .update(data=Event.data.delete('referrer')) .where(Event.data['referrer'] == 'google.com') .execute()) .. method:: slice(*keys) :param str keys: keys to retrieve. Retrieve only the provided key/value pairs: .. code-block:: python query = (Event .select(Event.id, Event.data.slice('ip', 'email').alias('source')) .order_by(Event.data['ip'])) for event in query: print(event.id, event.source) # 1 {'ip': '1.2.3.4', 'email': 'charles@example.com'} .. method:: keys() Return the keys as a list. .. code-block:: python query = Event.select(Event.data.keys().alias('keys')) for event in query: print(event.keys) # ['ip', 'type', 'email', 'result', 'status'] .. method:: values() Return the values as a list. .. method:: items() Return the key-value pairs as a 2-dimensional list. .. code-block:: python query = Event.select(Event.data.items().alias('items')) for event in query: print(event.items) # [['ip', '1.2.3.4'], # ['type', 'register'], # ['email', 'charles@example.com'], # ['result', 'ok'], # ['status', 'success']] .. _postgres-arrays: Arrays ------ .. class:: ArrayField(field_class=IntegerField, field_kwargs=None, dimensions=1, convert_values=False) Stores a Postgresql array of the given field type. :param field_class: a subclass of :class:`Field`, e.g. :class:`IntegerField`. :param dict field_kwargs: arguments to initialize ``field_class``. :param int dimensions: Number of array dimensions. :param bool convert_values: Apply ``field_class`` value conversion to retrieved data. By default ArrayField will use a GIN index. To disable this, initialize the field with ``index=False``. Example: .. code-block:: python class Post(Model): tags = ArrayField(CharField) Post.create(tags=['python', 'peewee', 'postgresql']) Post.create(tags=['python', 'sqlite']) # Get an item by index. Post.select(Post.tags[0].alias('first_tag')) # Get a slice: Post.select(Post.tags[:2].alias('first_two')) Multi-dimensional array example: .. code-block:: python class Outline(Model): points = ArrayField(IntegerField, dimensions=2) Outline.create(points=[[1, 1], [1, 5], [5, 5], [5, 1]]) .. method:: contains(*items) Filter rows where the array contains all of the given values. :param items: One or more items that must be in the given array field. .. code-block:: python Post.select().where(Post.tags.contains('postgresql', 'python')) .. method:: contains_any(*items) Filter rows where the array contains any of the given values. :param items: One or more items to search for in the given array field. .. code-block:: python Post.select().where(Post.tags.contains('postgresql', 'python')) .. _postgres-interval: Interval -------- .. class:: IntervalField(**kwargs) Stores Python ``datetime.timedelta`` instances using Postgresql's native ``INTERVAL`` type. .. code-block:: python from datetime import timedelta class Subscription(Model): duration = IntervalField() Subscription.create(duration=timedelta(days=30)) (Subscription .select() .where(Subscription.duration > timedelta(days=10))) .. _postgres-datetimetz: DateTimeTZ Field ----------------- .. class:: DateTimeTZField(**kwargs) Timezone-aware datetime field using Postgresql's ``TIMESTAMP WITH TIME ZONE`` type. .. code-block:: python class Event(Model): timestamp = DateTimeTZField() now = datetime.datetime.now().astimezone(datetime.timezone.utc) Event.create(timestamp=now) event = Event.get() print(event.timestamp) # 2026-01-02 03:04:05.012345+00:00 .. _postgres-fts: Full-Text Search ---------------- Postgresql full-text search uses the ``tsvector`` and ``tsquery`` types. Peewee offers two approaches: the simple :func:`Match` function (no schema changes required) and the :class:`TSVectorField` for dedicated search columns (better performance). **Simple approach** - no schema changes required: .. code-block:: python from playhouse.postgres_ext import Match def search_posts(term): return Post.select().where( (Post.status == 'published') & Match(Post.body, term)) The :func:`Match` function will automatically convert the left-hand operand to a ``tsvector``, and the right-hand operand to a ``tsquery``. For better performance, create a ``GIN`` index: .. code-block:: sql CREATE INDEX posts_fts ON post USING gin(to_tsvector('english', body)); **Dedicated column** - better performance: .. code-block:: python class Post(Model): body = TextField() search_content = TSVectorField() # Automatically gets a GIN index. # Store a post and populate the search vector: Post.create( body=body_text, search_content=fn.to_tsvector(body_text)) # Search: Post.select().where(Post.search_content.match('python postgresql')) # Search using expressions: terms = 'python & (sqlite | postgres)' Post.select().where(Post.search_content.match(terms)) For more information, see the `Postgres full-text search docs `_. .. function:: Match(field, query) Generate a full-text search expression that converts ``field`` to ``tsvector`` and ``query`` to ``tsquery`` automatically. .. class:: TSVectorField() Field type for storing pre-computed ``tsvector`` data. Automatically created with a GIN index (use ``index=False`` to disable). Data must be explicitly converted to ``tsvector`` on write using ``fn.to_tsvector()``. Example: .. code-block:: python class Post(Model): body = TextField() search_content = TSVectorField() Post.create( body=body_text, search_content=fn.to_tsvector(body_text)) (Post .select() .where(Post.search_content.match('python & (sqlite | postgres)'))) .. method:: match(query, language=None, plain=False) :param str query: Full-text search query. :param str language: Optional language name. :param bool plain: Use the plain (simple) query parser instead of the default one, which supports ``&``, ``|``, and ``!`` operators. .. _postgres-server-side-cursors: Server-Side Cursors ------------------- For large result sets, server-side (named) cursors stream rows from the server rather than loading the entire result into memory. Rows are fetched transparently from the server as you iterate. Refer to your driver documentation for details: * `psycopg2 server-side cursors `__ * `psycopg3 server-side cursors `__ To use server-side (or named) cursors, you must be using :class:`PostgresqlExtDatabase`. Wrap any SELECT query with :func:`ServerSide`: .. code-block:: python from playhouse.postgres_ext import ServerSide # Must be in a transaction to use server-side cursors. with db.atomic(): # Create a normal SELECT query. large_query = PageView.select() # Then wrap in `ServerSide` and iterate. for page_view in ServerSide(large_query): # Do something interesting. pass # At this point server side resources are released. For more granular control or to close the cursor explicitly: .. code-block:: python with db.atomic(): large_query = PageView.select().order_by(PageView.id.desc()) # Rows will be fetched 1000 at-a-time, but iteration is transparent. query = ServerSideQuery(query, array_size=1000) # Read 9500 rows then close server-side cursor. accum = [] for i, obj in enumerate(query.iterator()): if i == 9500: break accum.append(obj) # Release server-side resource. query.close() .. warning:: Server-side cursors live only within a transaction. If you are using psycopg2 (not psycopg3), cursors are declared ``WITH HOLD`` and must be fully exhausted or explicitly closed to release server resources. .. function:: ServerSide(select_query) :param select_query: a :class:`SelectQuery` instance. :rtype generator: Wrap ``select_query`` in a transaction and iterate using :meth:`~BaseQuery.iterator` (disables row caching). .. _crdb: CockroachDB ----------- .. module:: playhouse.cockroachdb `CockroachDB `_ (CRDB) is compatible with Postgresql's wire protocol and is well-supported by Peewee. Use the dedicated :class:`CockroachDatabase` class rather than :class:`PostgresqlDatabase` to get CRDB-specific handling. .. code-block:: python from playhouse.cockroachdb import CockroachDatabase db = CockroachDatabase('my_app', user='root', host='10.8.0.1') If you are using `Cockroach Cloud `_, you may find it easier to specify the connection parameters using a connection-string: .. code-block:: python db = CockroachDatabase('postgresql://root:secret@host:26257/defaultdb...') SSL configuration: .. code-block:: python db = CockroachDatabase( 'my_app', user='root', host='10.8.0.1', sslmode='verify-full', sslrootcert='/path/to/root.crt') # Or, alternatively, specified as part of a connection-string: db = CockroachDatabase('postgresql://root:secret@host:26257/dbname' '?sslmode=verify-full&sslrootcert=/path/to/root.crt' '&options=--cluster=my-cluster-xyz') Key differences from Postgresql ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * **No nested transactions.** CRDB does not support savepoints, so calling :meth:`~Database.atomic` inside another ``atomic()`` block raises an exception. Use :meth:`~Database.transaction` instead, which ignores nested calls and commits only when the outermost block exits. * **Client-side retries.** CRDB may abort transactions due to contention. Use :meth:`~CockroachDatabase.run_transaction` for automatic retries. Special field-types that may be useful when using CRDB: * :class:`~playhouse.cockroachdb.UUIDKeyField` - a primary-key field implementation that uses CRDB's ``UUID`` type with a default randomly-generated UUID. * :class:`~playhouse.cockroachdb.RowIDField` - a primary-key field implementation that uses CRDB's ``INT`` type with a default ``unique_rowid()``. * :class:`~playhouse.postgres_ext.JSONField` - same as the Postgres :class:`~playhouse.postgres_ext.BinaryJSONField`, as CRDB treats all JSON as JSONB. * :class:`~playhouse.postgres_ext.ArrayField` - same as the Postgres extension (but does not support multi-dimensional arrays). Transactions: .. code-block:: python # transaction() is safe to nest; the outer block manages the commit. @db.transaction() def create_user(username): return User.create(username=username) with db.transaction(): create_user('alice') # Nested call is folded into outer transaction. create_user('bob') # Transaction is committed here. Client-side retries: .. code-block:: python from playhouse.cockroachdb import CockroachDatabase db = CockroachDatabase('my_app') def transfer_funds(from_id, to_id, amt): """ Returns a 3-tuple of (success?, from balance, to balance). If there are not sufficient funds, then the original balances are returned. """ def thunk(db_ref): src, dest = (Account .select() .where(Account.id.in_([from_id, to_id]))) if src.id != from_id: src, dest = dest, src # Swap order. # Cannot perform transfer, insufficient funds! if src.balance < amt: return False, src.balance, dest.balance # Update each account, returning the new balance. src, = (Account .update(balance=Account.balance - amt) .where(Account.id == from_id) .returning(Account.balance) .execute()) dest, = (Account .update(balance=Account.balance + amt) .where(Account.id == to_id) .returning(Account.balance) .execute()) return True, src.balance, dest.balance # Perform the queries that comprise a logical transaction. In the # event the transaction fails due to contention, it will be auto- # matically retried (up to 10 times). return db.run_transaction(thunk, max_attempts=10) CRDB API ^^^^^^^^^ .. class:: CockroachDatabase(database, **kwargs) Subclass of :class:`PostgresqlDatabase` for CockroachDB. .. method:: run_transaction(callback, max_attempts=None, system_time=None, priority=None) :param callback: Callable accepting a single ``db`` argument. Must not manage the transaction itself. May be called multiple times. :param int max_attempts: Retry limit. :param datetime system_time: Execute ``AS OF SYSTEM TIME`` with respect to the given value. :param str priority: ``'low'``, ``'normal'``, or ``'high'``. :raises ExceededMaxAttempts: When ``max_attempts`` is exceeded. Execute SQL in a transaction with automatic client-side retries. User-provided ``callback``: * **Must** accept one parameter, the ``db`` instance representing the connection the transaction is running under. * **Must** not attempt to commit, rollback or otherwise manage the transaction. * **May** be called more than one time. * **Should** ideally only contain SQL operations. Additionally, the database must not have any open transactions at the time this function is called, as CRDB does not support nested transactions. Attempting to do so will raise a ``NotImplementedError``. .. class:: PooledCockroachDatabase(database, **kwargs) Connection-pooling variant of :class:`CockroachDatabase`. .. function:: run_transaction(db, callback, max_attempts=None, system_time=None, priority=None) Run SQL in a transaction with automatic client-side retries. See :meth:`CockroachDatabase.run_transaction` for details. This function is equivalent to the identically-named method on the :class:`CockroachDatabase` class. CRDB-specific field types: .. class:: UUIDKeyField() :noindex: UUID primary key auto-populated with CRDB's ``gen_random_uuid()``. .. class:: RowIDField() :noindex: Integer primary key auto-populated with CRDB's ``unique_rowid()``. ================================================ FILE: docs/peewee/query_builder.rst ================================================ .. _query-builder: Query Builder ============= Peewee's high-level :class:`Model` and :class:`Field` APIs are built upon lower-level :class:`Table` and :class:`Column` counterparts. While these lower-level APIs are not documented in as much detail as their high-level counterparts, this document will present an overview with examples that should hopefully allow you to experiment. We'll use the following schema: .. code-block:: sql CREATE TABLE "person" ( "id" INTEGER NOT NULL PRIMARY KEY, "first" TEXT NOT NULL, "last" TEXT NOT NULL); CREATE TABLE "note" ( "id" INTEGER NOT NULL PRIMARY KEY, "person_id" INTEGER NOT NULL, "content" TEXT NOT NULL, "timestamp" DATETIME NOT NULL, FOREIGN KEY ("person_id") REFERENCES "person" ("id")); CREATE TABLE "reminder" ( "id" INTEGER NOT NULL PRIMARY KEY, "note_id" INTEGER NOT NULL, "alarm" DATETIME NOT NULL, FOREIGN KEY ("note_id") REFERENCES "note" ("id")); Declaring tables ---------------- There are two ways we can declare :class:`Table` objects for working with these tables: .. code-block:: python # Explicitly declare columns Person = Table('person', ('id', 'first', 'last')) Note = Table('note', ('id', 'person_id', 'content', 'timestamp')) # Do not declare columns, they will be accessed using magic ".c" attribute Reminder = Table('reminder') Typically we will want to :meth:`~Table.bind` our tables to a database. This saves us having to pass the database explicitly every time we wish to execute a query on the table: .. code-block:: python db = SqliteDatabase('my_app.db') Person = Person.bind(db) Note = Note.bind(db) Reminder = Reminder.bind(db) Select queries -------------- To select the first three notes and print their content, we can write: .. code-block:: python query = Note.select().order_by(Note.timestamp).limit(3) for note_dict in query: print(note_dict['content']) .. note:: By default, rows will be returned as dictionaries. You can use the :meth:`~BaseQuery.tuples`, :meth:`~BaseQuery.namedtuples` or :meth:`~BaseQuery.objects` methods to specify a different container for the row data, if you wish. Because we didn't specify any columns, all the columns we defined in the note's :class:`Table` constructor will be selected. This won't work for Reminder, as we didn't specify any columns at all. To select all notes published in 2018 along with the name of the creator, we will use :meth:`~Select.join`. We'll also request that rows be returned as *namedtuple* objects: .. code-block:: python query = (Note .select(Note.content, Note.timestamp, Person.first, Person.last) .join(Person, on=(Note.person_id == Person.id)) .where(Note.timestamp >= datetime.date(2018, 1, 1)) .order_by(Note.timestamp) .namedtuples()) for row in query: print(row.timestamp, '-', row.content, '-', row.first, row.last) Let's query for the most prolific people, that is, get the people who have created the most notes. This introduces calling a SQL function (COUNT), which is accomplished using the ``fn`` object: .. code-block:: python name = Person.first.concat(' ').concat(Person.last) query = (Person .select(name.alias('name'), fn.COUNT(Note.id).alias('count')) .join(Note, JOIN.LEFT_OUTER, on=(Note.person_id == Person.id)) .group_by(name) .order_by(fn.COUNT(Note.id).desc())) for row in query: print(row['name'], row['count']) There are a couple things to note in the above query: * We store an expression in a variable (``name``), then use it in the query. * We call SQL functions using ``fn.(...)`` passing arguments as if it were a normal Python function. * The :meth:`~ColumnBase.alias` method is used to specify the name used for a column or calculation. As a more complex example, we'll generate a list of all people and the contents and timestamp of their most recently-published note. To do this, we will end up using the Note table twice in different contexts within the same query, which will require us to use a table alias. .. code-block:: python # Start with the query that calculates the timestamp of the most recent # note for each person. NA = Note.alias('na') max_note = (NA .select(NA.person_id, fn.MAX(NA.timestamp).alias('max_ts')) .group_by(NA.person_id) .alias('max_note')) # Now we'll select from the note table, joining on both the subquery and # on the person table to construct the result set. query = (Note .select(Note.content, Note.timestamp, Person.first, Person.last) .join(max_note, on=((max_note.c.person_id == Note.person_id) & (max_note.c.max_ts == Note.timestamp))) .join(Person, on=(Note.person_id == Person.id)) .order_by(Person.first, Person.last)) for row in query.namedtuples(): print(row.first, row.last, ':', row.timestamp, '-', row.content) In the join predicate for the join on the *max_note* subquery, we can reference columns in the subquery using the magical ".c" attribute. So, *max_note.c.max_ts* is translated into "the max_ts column value from the max_note subquery". We can also use the ".c" magic attribute to access columns on tables that do not explicitly define their columns, like we did with the Reminder table. Here's a simple query to get all reminders for today, along with their associated note content: .. code-block:: python today = datetime.date.today() tomorrow = today + datetime.timedelta(days=1) query = (Reminder .select(Reminder.c.alarm, Note.content) .join(Note, on=(Reminder.c.note_id == Note.id)) .where(Reminder.c.alarm.between(today, tomorrow)) .order_by(Reminder.c.alarm)) for row in query: print(row['alarm'], row['content']) .. note:: The ".c" attribute will not work on tables that explicitly define their columns, to prevent confusion. Insert Queries -------------- Inserting data is straightforward. We can specify data to :meth:`~Table.insert` in two different ways (in both cases, the ID of the new row is returned): .. code-block:: python # Using keyword arguments: zaizee_id = Person.insert(first='zaizee', last='cat').execute() # Using column: value mappings: Note.insert({ Note.person_id: zaizee_id, Note.content: 'meeeeowwww', Note.timestamp: datetime.datetime.now()}).execute() It is easy to bulk-insert data, just pass in either: * A list of dictionaries (all must have the same keys/columns). * A list of tuples, if the columns are specified explicitly. Examples: .. code-block:: python people = [ {'first': 'Bob', 'last': 'Foo'}, {'first': 'Herb', 'last': 'Bar'}, {'first': 'Nuggie', 'last': 'Bar'}] # Inserting multiple rows returns the ID of the last-inserted row. last_id = Person.insert(people).execute() # We can also specify row tuples, so long as we tell Peewee which # columns the tuple values correspond to: people = [ ('Bob', 'Foo'), ('Herb', 'Bar'), ('Nuggie', 'Bar')] Person.insert(people, columns=[Person.first, Person.last]).execute() Update Queries -------------- :meth:`~Table.update` queries accept either keyword arguments or a dictionary mapping column to value, just like :meth:`~Table.insert`. Examples: .. code-block:: python # "Bob" changed his last name from "Foo" to "Baze". nrows = (Person .update(last='Baze') .where((Person.first == 'Bob') & (Person.last == 'Foo')) .execute()) # Use dictionary mapping column to value. nrows = (Person .update({Person.last: 'Baze'}) .where((Person.first == 'Bob') & (Person.last == 'Foo')) .execute()) You can also use expressions as the value to perform an atomic update. Imagine we have a *PageView* table and we need to atomically increment the page-view count for some URL: .. code-block:: python # Do an atomic update: (PageView .update({PageView.count: PageView.count + 1}) .where(PageView.url == some_url) .execute()) Delete Queries -------------- :meth:`~Table.delete` queries are simplest of all, as they do not accept any arguments: .. code-block:: python # Delete all notes created before 2018, returning number deleted. n = Note.delete().where(Note.timestamp < datetime.date(2018, 1, 1)).execute() Because DELETE (and UPDATE) queries do not support joins, we can use subqueries to delete rows based on values in related tables. For example, here is how you would delete all notes by anyone whose last name is "Foo": .. code-block:: python # Get the id of all people whose last name is "Foo". foo_people = Person.select(Person.id).where(Person.last == 'Foo') # Delete all notes by any person whose ID is in the previous query. Note.delete().where(Note.person_id.in_(foo_people)).execute() Query Objects ------------- One of the fundamental limitations of the abstractions provided by Peewee 2.x was the absence of a class that represented a structured query with no relation to a given model class. An example of this might be computing aggregate values over a subquery. For example, the :meth:`~SelectBase.count` method, which returns the count of rows in an arbitrary query, is implemented by wrapping the query: .. code-block:: sql SELECT COUNT(1) FROM (...) To accomplish this with Peewee, the implementation is written in this way: .. code-block:: python def count(query): # Select([source1, ... sourcen], [column1, ...columnn]) wrapped = Select(from_list=[query], columns=[fn.COUNT(SQL('1'))]) curs = wrapped.tuples().execute(db) return curs[0][0] # Return first column from first row of result. We can actually express this more concisely using the :meth:`~SelectBase.scalar` method, which is suitable for returning values from aggregate queries: .. code-block:: python def count(query): wrapped = Select(from_list=[query], columns=[fn.COUNT(SQL('1'))]) return wrapped.scalar(db) The :ref:`query-library` document has a more complex example, in which we write a query for a facility with the highest number of available slots booked: The SQL we wish to express is: .. code-block:: sql SELECT facid, total FROM ( SELECT facid, SUM(slots) AS total, rank() OVER (order by SUM(slots) DESC) AS rank FROM bookings GROUP BY facid ) AS ranked WHERE rank = 1 We can express this fairly elegantly by using a plain :class:`Select` for the outer query: .. code-block:: python # Store rank expression in variable for readability. rank_expr = fn.rank().over(order_by=[fn.SUM(Booking.slots).desc()]) subq = (Booking .select(Booking.facility, fn.SUM(Booking.slots).alias('total'), rank_expr.alias('rank')) .group_by(Booking.facility)) # Use a plain "Select" to create outer query. query = (Select(columns=[subq.c.facid, subq.c.total]) .from_(subq) .where(subq.c.rank == 1) .tuples()) # Iterate over the resulting facility ID(s) and total(s): for facid, total in query.execute(db): print(facid, total) For another example, let's create a recursive common table expression to calculate the first 10 fibonacci numbers: .. code-block:: python base = Select(columns=( Value(1).alias('n'), Value(0).alias('fib_n'), Value(1).alias('next_fib_n'))).cte('fibonacci', recursive=True) n = (base.c.n + 1).alias('n') recursive_term = Select(columns=( n, base.c.next_fib_n, base.c.fib_n + base.c.next_fib_n)).from_(base).where(n < 10) fibonacci = base.union_all(recursive_term) query = fibonacci.select_from(fibonacci.c.n, fibonacci.c.fib_n) results = list(query.execute(db)) # Generates the following result list: [{'fib_n': 0, 'n': 1}, {'fib_n': 1, 'n': 2}, {'fib_n': 1, 'n': 3}, {'fib_n': 2, 'n': 4}, {'fib_n': 3, 'n': 5}, {'fib_n': 5, 'n': 6}, {'fib_n': 8, 'n': 7}, {'fib_n': 13, 'n': 8}, {'fib_n': 21, 'n': 9}, {'fib_n': 34, 'n': 10}] More ---- For a description of the various classes used to describe a SQL AST, see the :ref:`query builder API documentation `. If you're interested in learning more, you can also check out the `project source code `_. ================================================ FILE: docs/peewee/query_library.rst ================================================ .. _query-library: Query Examples Library ====================== These query examples are taken from the site `Postgresql Exercises `_. A sample data-set can be found on the `getting started page `__. Direct download: `clubdata.sql `__ .. contents:: On this page :local: :depth: 1 Model Definitions ----------------- Here is a visual representation of the schema used in these examples: .. image:: schema-horizontal.png To begin working with the data, we'll define the model classes that correspond to the tables in the diagram. .. note:: In some cases we explicitly specify column names for a particular field. This is so our models are compatible with the database schema used for the postgres exercises. .. code-block:: python from functools import partial from peewee import * db = PostgresqlDatabase('peewee_test') class BaseModel(Model): class Meta: database = db class Member(BaseModel): memid = AutoField() # Auto-incrementing primary key. surname = CharField() firstname = CharField() address = CharField(max_length=300) zipcode = IntegerField() telephone = CharField() recommendedby = ForeignKeyField('self', backref='recommended', column_name='recommendedby', null=True) joindate = DateTimeField() class Meta: table_name = 'members' # Conveniently declare decimal fields suitable for storing currency. MoneyField = partial(DecimalField, decimal_places=2) class Facility(BaseModel): facid = AutoField() name = CharField() membercost = MoneyField() guestcost = MoneyField() initialoutlay = MoneyField() monthlymaintenance = MoneyField() class Meta: table_name = 'facilities' class Booking(BaseModel): bookid = AutoField() facility = ForeignKeyField(Facility, column_name='facid') member = ForeignKeyField(Member, column_name='memid') starttime = DateTimeField() slots = IntegerField() class Meta: table_name = 'bookings' Schema Creation --------------- If you downloaded the SQL file from the Postgresql Exercises site, then you can load the data into a Postgresql database using the following commands:: createdb peewee_test psql -U postgres -f clubdata.sql -d peewee_test -x -q To create the schema using Peewee, without loading the sample data, you can run the following: .. code-block:: python # Assumes you have created the database "peewee_test" already. db.create_tables([Member, Facility, Booking]) Basic Exercises --------------- This category deals with the basics of SQL. It covers select and where clauses, case expressions, unions, and a few other odds and ends. Retrieve everything ^^^^^^^^^^^^^^^^^^^ Retrieve all information from facilities table. .. code-block:: sql SELECT * FROM facilities .. code-block:: python # By default, when no fields are explicitly passed to select(), all fields # will be selected. query = Facility.select() Retrieve specific columns from a table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Retrieve names of facilities and cost to members. .. code-block:: sql SELECT name, membercost FROM facilities; .. code-block:: python query = Facility.select(Facility.name, Facility.membercost) # To iterate: for facility in query: print(facility.name) Control which rows are retrieved ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Retrieve list of facilities that have a cost to members. .. code-block:: sql SELECT * FROM facilities WHERE membercost > 0 .. code-block:: python query = Facility.select().where(Facility.membercost > 0) Control which rows are retrieved - part 2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Retrieve list of facilities that have a cost to members, and that fee is less than 1/50th of the monthly maintenance cost. Return id, name, cost and monthly-maintenance. .. code-block:: sql SELECT facid, name, membercost, monthlymaintenance FROM facilities WHERE membercost > 0 AND membercost < (monthlymaintenance / 50) .. code-block:: python query = (Facility .select(Facility.facid, Facility.name, Facility.membercost, Facility.monthlymaintenance) .where( (Facility.membercost > 0) & (Facility.membercost < (Facility.monthlymaintenance / 50)))) Basic string searches ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ How can you produce a list of all facilities with the word 'Tennis' in their name? .. code-block:: sql SELECT * FROM facilities WHERE name ILIKE '%tennis%'; .. code-block:: python query = Facility.select().where(Facility.name.contains('tennis')) # OR use the exponent operator. Note: you must include wildcards here: query = Facility.select().where(Facility.name ** '%tennis%') Matching against multiple possible values ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ How can you retrieve the details of facilities with ID 1 and 5? Try to do it without using the OR operator. .. code-block:: sql SELECT * FROM facilities WHERE facid IN (1, 5); .. code-block:: python query = Facility.select().where(Facility.facid.in_([1, 5])) # OR: query = Facility.select().where((Facility.facid == 1) | (Facility.facid == 5)) Classify results into buckets ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ How can you produce a list of facilities, with each labelled as 'cheap' or 'expensive' depending on if their monthly maintenance cost is more than $100? Return the name and monthly maintenance of the facilities in question. .. code-block:: sql SELECT name, CASE WHEN monthlymaintenance > 100 THEN 'expensive' ELSE 'cheap' END FROM facilities; .. code-block:: python cost = Case(None, [(Facility.monthlymaintenance > 100, 'expensive')], 'cheap') query = Facility.select(Facility.name, cost.alias('cost')) .. note:: See documentation :class:`Case` for more examples. Working with dates ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ How can you produce a list of members who joined after the start of September 2012? Return the memid, surname, firstname, and joindate of the members in question. .. code-block:: sql SELECT memid, surname, firstname, joindate FROM members WHERE joindate >= '2012-09-01'; .. code-block:: python query = (Member .select(Member.memid, Member.surname, Member.firstname, Member.joindate) .where(Member.joindate >= datetime.date(2012, 9, 1))) Removing duplicates, and ordering results ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ How can you produce an ordered list of the first 10 surnames in the members table? The list must not contain duplicates. .. code-block:: sql SELECT DISTINCT surname FROM members ORDER BY surname LIMIT 10; .. code-block:: python query = (Member .select(Member.surname) .order_by(Member.surname) .limit(10) .distinct()) Combining results from multiple queries ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You, for some reason, want a combined list of all surnames and all facility names. .. code-block:: sql SELECT surname FROM members UNION SELECT name FROM facilities; .. code-block:: python lhs = Member.select(Member.surname) rhs = Facility.select(Facility.name) query = lhs | rhs Queries can be composed using the following operators: * ``|`` - ``UNION`` * ``+`` - ``UNION ALL`` * ``&`` - ``INTERSECT`` * ``-`` - ``EXCEPT`` Simple aggregation ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You'd like to get the signup date of your last member. How can you retrieve this information? .. code-block:: sql SELECT MAX(join_date) FROM members; .. code-block:: python query = Member.select(fn.MAX(Member.joindate)) # To conveniently obtain a single scalar value, use "scalar()": # max_join_date = query.scalar() More aggregation ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You'd like to get the first and last name of the last member(s) who signed up - not just the date. .. code-block:: sql SELECT firstname, surname, joindate FROM members WHERE joindate = (SELECT MAX(joindate) FROM members); .. code-block:: python # Use "alias()" to reference the same table multiple times in a query. MemberAlias = Member.alias() subq = MemberAlias.select(fn.MAX(MemberAlias.joindate)) query = (Member .select(Member.firstname, Member.surname, Member.joindate) .where(Member.joindate == subq)) Joins and Subqueries -------------------- This category deals primarily with a foundational concept in relational database systems: joining. Joining allows you to combine related information from multiple tables to answer a question. This isn't just beneficial for ease of querying: a lack of join capability encourages denormalisation of data, which increases the complexity of keeping your data internally consistent. This topic covers inner, outer, and self joins, as well as spending a little time on subqueries (queries within queries). Retrieve the start times of members' bookings ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ How can you produce a list of the start times for bookings by members named 'David Farrell'? .. code-block:: sql SELECT starttime FROM bookings INNER JOIN members ON (bookings.memid = members.memid) WHERE surname = 'Farrell' AND firstname = 'David'; .. code-block:: python query = (Booking .select(Booking.starttime) .join(Member) .where((Member.surname == 'Farrell') & (Member.firstname == 'David'))) Work out the start times of bookings for tennis courts ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ How can you produce a list of the start times for bookings for tennis courts, for the date '2012-09-21'? Return a list of start time and facility name pairings, ordered by the time. .. code-block:: sql SELECT starttime, name FROM bookings INNER JOIN facilities ON (bookings.facid = facilities.facid) WHERE date_trunc('day', starttime) = '2012-09-21':: date AND name ILIKE 'tennis%' ORDER BY starttime, name; .. code-block:: python query = (Booking .select(Booking.starttime, Facility.name) .join(Facility) .where( (fn.date_trunc('day', Booking.starttime) == datetime.date(2012, 9, 21)) & Facility.name.startswith('Tennis')) .order_by(Booking.starttime, Facility.name)) # To retrieve the joined facility's name when iterating: for booking in query: print(booking.starttime, booking.facility.name) Produce a list of all members who have recommended another member ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ How can you output a list of all members who have recommended another member? Ensure that there are no duplicates in the list, and that results are ordered by (surname, firstname). .. code-block:: sql SELECT DISTINCT m.firstname, m.surname FROM members AS m2 INNER JOIN members AS m ON (m.memid = m2.recommendedby) ORDER BY m.surname, m.firstname; .. code-block:: python MA = Member.alias() query = (Member .select(Member.firstname, Member.surname) .join(MA, on=(MA.recommendedby == Member.memid)) .order_by(Member.surname, Member.firstname)) Produce a list of all members, along with their recommender ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ How can you output a list of all members, including the individual who recommended them (if any)? Ensure that results are ordered by (surname, firstname). .. code-block:: sql SELECT m.firstname, m.surname, r.firstname, r.surname FROM members AS m LEFT OUTER JOIN members AS r ON (m.recommendedby = r.memid) ORDER BY m.surname, m.firstname .. code-block:: python MA = Member.alias() query = (Member .select(Member.firstname, Member.surname, MA.firstname, MA.surname) .join(MA, JOIN.LEFT_OUTER, on=(Member.recommendedby == MA.memid)) .order_by(Member.surname, Member.firstname)) # To display the recommender's name when iterating: for m in query: print(m.firstname, m.surname) if m.recommendedby: print(' ', m.recommendedby.firstname, m.recommendedby.surname) Produce a list of all members who have used a tennis court ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ How can you produce a list of all members who have used a tennis court? Include in your output the name of the court, and the name of the member formatted as a single column. Ensure no duplicate data, and order by the member name. .. code-block:: sql SELECT DISTINCT m.firstname || ' ' || m.surname AS member, f.name AS facility FROM members AS m INNER JOIN bookings AS b ON (m.memid = b.memid) INNER JOIN facilities AS f ON (b.facid = f.facid) WHERE f.name LIKE 'Tennis%' ORDER BY member, facility; .. code-block:: python fullname = Member.firstname + ' ' + Member.surname query = (Member .select(fullname.alias('member'), Facility.name.alias('facility')) .join(Booking) .join(Facility) .where(Facility.name.startswith('Tennis')) .order_by(fullname, Facility.name) .distinct()) Produce a list of costly bookings ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ How can you produce a list of bookings on the day of 2012-09-14 which will cost the member (or guest) more than $30? Remember that guests have different costs to members (the listed costs are per half-hour 'slot'), and the guest user is always ID 0. Include in your output the name of the facility, the name of the member formatted as a single column, and the cost. Order by descending cost, and do not use any subqueries. .. code-block:: sql SELECT m.firstname || ' ' || m.surname AS member, f.name AS facility, (CASE WHEN m.memid = 0 THEN f.guestcost * b.slots ELSE f.membercost * b.slots END) AS cost FROM members AS m INNER JOIN bookings AS b ON (m.memid = b.memid) INNER JOIN facilities AS f ON (b.facid = f.facid) WHERE (date_trunc('day', b.starttime) = '2012-09-14') AND ((m.memid = 0 AND b.slots * f.guestcost > 30) OR (m.memid > 0 AND b.slots * f.membercost > 30)) ORDER BY cost DESC; .. code-block:: python cost = Case(Member.memid, ( (0, Booking.slots * Facility.guestcost), ), (Booking.slots * Facility.membercost)) fullname = Member.firstname + ' ' + Member.surname query = (Member .select(fullname.alias('member'), Facility.name.alias('facility'), cost.alias('cost')) .join(Booking) .join(Facility) .where( (fn.date_trunc('day', Booking.starttime) == datetime.date(2012, 9, 14)) & (cost > 30)) .order_by(SQL('cost').desc())) # To iterate over the results, it might be easiest to use namedtuples: for row in query.namedtuples(): print(row.member, row.facility, row.cost) Produce a list of all members, along with their recommender, using no joins. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ How can you output a list of all members, including the individual who recommended them (if any), without using any joins? Ensure that there are no duplicates in the list, and that each firstname + surname pairing is formatted as a column and ordered. .. code-block:: sql SELECT DISTINCT m.firstname || ' ' || m.surname AS member, (SELECT r.firstname || ' ' || r.surname FROM members AS r WHERE m.recommendedby = r.memid) AS recommended FROM members AS m ORDER BY member; .. code-block:: python MA = Member.alias() subq = (MA .select(MA.firstname + ' ' + MA.surname) .where(Member.recommendedby == MA.memid)) query = (Member .select(fullname.alias('member'), subq.alias('recommended')) .order_by(fullname)) Produce a list of costly bookings, using a subquery ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The "Produce a list of costly bookings" exercise contained some messy logic: we had to calculate the booking cost in both the WHERE clause and the CASE statement. Try to simplify this calculation using subqueries. .. code-block:: sql SELECT member, facility, cost from ( SELECT m.firstname || ' ' || m.surname as member, f.name as facility, CASE WHEN m.memid = 0 THEN b.slots * f.guestcost ELSE b.slots * f.membercost END AS cost FROM members AS m INNER JOIN bookings AS b ON m.memid = b.memid INNER JOIN facilities AS f ON b.facid = f.facid WHERE date_trunc('day', b.starttime) = '2012-09-14' ) as bookings WHERE cost > 30 ORDER BY cost DESC; .. code-block:: python cost = Case(Member.memid, ( (0, Booking.slots * Facility.guestcost), ), (Booking.slots * Facility.membercost)) iq = (Member .select(fullname.alias('member'), Facility.name.alias('facility'), cost.alias('cost')) .join(Booking) .join(Facility) .where(fn.date_trunc('day', Booking.starttime) == datetime.date(2012, 9, 14))) query = (Member .select(iq.c.member, iq.c.facility, iq.c.cost) .from_(iq) .where(iq.c.cost > 30) .order_by(SQL('cost').desc())) # To iterate, try using dicts: for row in query.dicts(): print(row['member'], row['facility'], row['cost']) Modifying Data -------------- Querying data is all well and good, but at some point you're probably going to want to put data into your database! This section deals with inserting, updating, and deleting information. Operations that alter your data like this are collectively known as Data Manipulation Language, or DML. In previous sections, we returned to you the results of the query you've performed. Since modifications like the ones we're making in this section don't return any query results, we instead show you the updated content of the table you're supposed to be working on. Insert some data into a table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The club is adding a new facility - a spa. We need to add it into the facilities table. Use the following values: facid: 9, Name: 'Spa', membercost: 20, guestcost: 30, initialoutlay: 100000, monthlymaintenance: 800 .. code-block:: sql INSERT INTO "facilities" ("facid", "name", "membercost", "guestcost", "initialoutlay", "monthlymaintenance") VALUES (9, 'Spa', 20, 30, 100000, 800) .. code-block:: python res = Facility.insert({ Facility.facid: 9, Facility.name: 'Spa', Facility.membercost: 20, Facility.guestcost: 30, Facility.initialoutlay: 100000, Facility.monthlymaintenance: 800}).execute() # OR: res = (Facility .insert(facid=9, name='Spa', membercost=20, guestcost=30, initialoutlay=100000, monthlymaintenance=800) .execute()) Insert multiple rows of data into a table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ In the previous exercise, you learned how to add a facility. Now you're going to add multiple facilities in one command. Use the following values: facid: 9, Name: 'Spa', membercost: 20, guestcost: 30, initialoutlay: 100000, monthlymaintenance: 800. facid: 10, Name: 'Squash Court 2', membercost: 3.5, guestcost: 17.5, initialoutlay: 5000, monthlymaintenance: 80. .. code-block:: sql INSERT INTO "facilities" (...) VALUES (9, ...), (10, ...); .. code-block:: python data = [ {'facid': 9, 'name': 'Spa', 'membercost': 20, 'guestcost': 30, 'initialoutlay': 100000, 'monthlymaintenance': 800}, {'facid': 10, 'name': 'Squash Court 2', 'membercost': 3.5, 'guestcost': 17.5, 'initialoutlay': 5000, 'monthlymaintenance': 80}] res = Facility.insert_many(data).execute() Insert calculated data into a table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Let's try adding the spa to the facilities table again. This time, though, we want to automatically generate the value for the next facid, rather than specifying it as a constant. Use the following values for everything else: Name: 'Spa', membercost: 20, guestcost: 30, initialoutlay: 100000, monthlymaintenance: 800. .. code-block:: sql INSERT INTO "facilities" ("facid", "name", "membercost", "guestcost", "initialoutlay", "monthlymaintenance") SELECT (SELECT (MAX("facid") + 1) FROM "facilities") AS _, 'Spa', 20, 30, 100000, 800; .. code-block:: python maxq = Facility.select(fn.MAX(Facility.facid) + 1) subq = Select(columns=(maxq, 'Spa', 20, 30, 100000, 800)) res = Facility.insert_from(subq, Facility._meta.sorted_fields).execute() Update some existing data ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ We made a mistake when entering the data for the second tennis court. The initial outlay was 10000 rather than 8000: you need to alter the data to fix the error. .. code-block:: sql UPDATE facilities SET initialoutlay = 10000 WHERE name = 'Tennis Court 2'; .. code-block:: python res = (Facility .update({Facility.initialoutlay: 10000}) .where(Facility.name == 'Tennis Court 2') .execute()) # OR: res = (Facility .update(initialoutlay=10000) .where(Facility.name == 'Tennis Court 2') .execute()) Update multiple rows and columns at the same time ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ We want to increase the price of the tennis courts for both members and guests. Update the costs to be 6 for members, and 30 for guests. .. code-block:: sql UPDATE facilities SET membercost=6, guestcost=30 WHERE name ILIKE 'Tennis%'; .. code-block:: python nrows = (Facility .update(membercost=6, guestcost=30) .where(Facility.name.startswith('Tennis')) .execute()) Update a row based on the contents of another row ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ We want to alter the price of the second tennis court so that it costs 10% more than the first one. Try to do this without using constant values for the prices, so that we can reuse the statement if we want to. .. code-block:: sql UPDATE facilities SET membercost = (SELECT membercost * 1.1 FROM facilities WHERE facid = 0), guestcost = (SELECT guestcost * 1.1 FROM facilities WHERE facid = 0) WHERE facid = 1; -- OR -- WITH new_prices (nmc, ngc) AS ( SELECT membercost * 1.1, guestcost * 1.1 FROM facilities WHERE name = 'Tennis Court 1') UPDATE facilities SET membercost = new_prices.nmc, guestcost = new_prices.ngc FROM new_prices WHERE name = 'Tennis Court 2' .. code-block:: python sq1 = Facility.select(Facility.membercost * 1.1).where(Facility.facid == 0) sq2 = Facility.select(Facility.guestcost * 1.1).where(Facility.facid == 0) res = (Facility .update(membercost=sq1, guestcost=sq2) .where(Facility.facid == 1) .execute()) # OR: cte = (Facility .select(Facility.membercost * 1.1, Facility.guestcost * 1.1) .where(Facility.name == 'Tennis Court 1') .cte('new_prices', columns=('nmc', 'ngc'))) res = (Facility .update(membercost=SQL('new_prices.nmc'), guestcost=SQL('new_prices.ngc')) .with_cte(cte) .from_(cte) .where(Facility.name == 'Tennis Court 2') .execute()) Delete all bookings ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ As part of a clearout of our database, we want to delete all bookings from the bookings table. .. code-block:: sql DELETE FROM bookings; .. code-block:: python nrows = Booking.delete().execute() Delete a member from the members table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ We want to remove member 37, who has never made a booking, from our database. .. code-block:: sql DELETE FROM members WHERE memid = 37; .. code-block:: python nrows = Member.delete().where(Member.memid == 37).execute() Delete based on a subquery ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ How can we make that more general, to delete all members who have never made a booking? .. code-block:: sql DELETE FROM members WHERE NOT EXISTS ( SELECT * FROM bookings WHERE bookings.memid = members.memid); .. code-block:: python subq = Booking.select().where(Booking.member == Member.memid) nrows = Member.delete().where(~fn.EXISTS(subq)).execute() Aggregation ----------- Aggregation is one of those capabilities that really make you appreciate the power of relational database systems. It allows you to move beyond merely persisting your data, into the realm of asking truly interesting questions that can be used to inform decision making. This category covers aggregation at length, making use of standard grouping as well as more recent window functions. Count the number of facilities ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ For our first foray into aggregates, we're going to stick to something simple. We want to know how many facilities exist - simply produce a total count. .. code-block:: sql SELECT COUNT(facid) FROM facilities; .. code-block:: python query = Facility.select(fn.COUNT(Facility.facid)) count = query.scalar() # OR: count = Facility.select().count() Count the number of expensive facilities ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Produce a count of the number of facilities that have a cost to guests of 10 or more. .. code-block:: sql SELECT COUNT(facid) FROM facilities WHERE guestcost >= 10 .. code-block:: python query = Facility.select(fn.COUNT(Facility.facid)).where(Facility.guestcost >= 10) count = query.scalar() # OR: # count = Facility.select().where(Facility.guestcost >= 10).count() Count the number of recommendations each member makes. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Produce a count of the number of recommendations each member has made. Order by member ID. .. code-block:: sql SELECT recommendedby, COUNT(memid) FROM members WHERE recommendedby IS NOT NULL GROUP BY recommendedby ORDER BY recommendedby .. code-block:: python query = (Member .select(Member.recommendedby, fn.COUNT(Member.memid)) .where(Member.recommendedby.is_null(False)) .group_by(Member.recommendedby) .order_by(Member.recommendedby)) List the total slots booked per facility ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Produce a list of the total number of slots booked per facility. For now, just produce an output table consisting of facility id and slots, sorted by facility id. .. code-block:: sql SELECT facid, SUM(slots) FROM bookings GROUP BY facid ORDER BY facid; .. code-block:: python query = (Booking .select(Booking.facid, fn.SUM(Booking.slots)) .group_by(Booking.facid) .order_by(Booking.facid)) List the total slots booked per facility in a given month ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Produce a list of the total number of slots booked per facility in the month of September 2012. Produce an output table consisting of facility id and slots, sorted by the number of slots. .. code-block:: sql SELECT facid, SUM(slots) FROM bookings WHERE (date_trunc('month', starttime) = '2012-09-01'::dates) GROUP BY facid ORDER BY SUM(slots) .. code-block:: python query = (Booking .select(Booking.facility, fn.SUM(Booking.slots)) .where(fn.date_trunc('month', Booking.starttime) == datetime.date(2012, 9, 1)) .group_by(Booking.facility) .order_by(fn.SUM(Booking.slots))) List the total slots booked per facility per month ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Produce a list of the total number of slots booked per facility per month in the year of 2012. Produce an output table consisting of facility id and slots, sorted by the id and month. .. code-block:: sql SELECT facid, date_part('month', starttime), SUM(slots) FROM bookings WHERE date_part('year', starttime) = 2012 GROUP BY facid, date_part('month', starttime) ORDER BY facid, date_part('month', starttime) .. code-block:: python month = fn.date_part('month', Booking.starttime) query = (Booking .select(Booking.facility, month, fn.SUM(Booking.slots)) .where(fn.date_part('year', Booking.starttime) == 2012) .group_by(Booking.facility, month) .order_by(Booking.facility, month)) Find the count of members who have made at least one booking ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Find the total number of members who have made at least one booking. .. code-block:: sql SELECT COUNT(DISTINCT memid) FROM bookings -- OR -- SELECT COUNT(1) FROM (SELECT DISTINCT memid FROM bookings) AS _ .. code-block:: python query = Booking.select(fn.COUNT(Booking.member.distinct())) # OR: query = Booking.select(Booking.member).distinct() count = query.count() # count() wraps in SELECT COUNT(1) FROM (...) List facilities with more than 1000 slots booked ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Produce a list of facilities with more than 1000 slots booked. Produce an output table consisting of facility id and hours, sorted by facility id. .. code-block:: sql SELECT facid, SUM(slots) FROM bookings GROUP BY facid HAVING SUM(slots) > 1000 ORDER BY facid; .. code-block:: python query = (Booking .select(Booking.facility, fn.SUM(Booking.slots)) .group_by(Booking.facility) .having(fn.SUM(Booking.slots) > 1000) .order_by(Booking.facility)) Find the total revenue of each facility ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Produce a list of facilities along with their total revenue. The output table should consist of facility name and revenue, sorted by revenue. Remember that there's a different cost for guests and members! .. code-block:: sql SELECT f.name, SUM(b.slots * ( CASE WHEN b.memid = 0 THEN f.guestcost ELSE f.membercost END)) AS revenue FROM bookings AS b INNER JOIN facilities AS f ON b.facid = f.facid GROUP BY f.name ORDER BY revenue; .. code-block:: python revenue = fn.SUM(Booking.slots * Case(None, ( (Booking.member == 0, Facility.guestcost), ), Facility.membercost)) query = (Facility .select(Facility.name, revenue.alias('revenue')) .join(Booking) .group_by(Facility.name) .order_by(SQL('revenue'))) Find facilities with a total revenue less than 1000 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Produce a list of facilities with a total revenue less than 1000. Produce an output table consisting of facility name and revenue, sorted by revenue. Remember that there's a different cost for guests and members! .. code-block:: sql SELECT f.name, SUM(b.slots * ( CASE WHEN b.memid = 0 THEN f.guestcost ELSE f.membercost END)) AS revenue FROM bookings AS b INNER JOIN facilities AS f ON b.facid = f.facid GROUP BY f.name HAVING SUM(b.slots * ...) < 1000 ORDER BY revenue; .. code-block:: python # Same definition as previous example. revenue = fn.SUM(Booking.slots * Case(None, ( (Booking.member == 0, Facility.guestcost), ), Facility.membercost)) query = (Facility .select(Facility.name, revenue.alias('revenue')) .join(Booking) .group_by(Facility.name) .having(revenue < 1000) .order_by(SQL('revenue'))) Output the facility id that has the highest number of slots booked ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Output the facility id that has the highest number of slots booked. .. code-block:: sql SELECT facid, SUM(slots) FROM bookings GROUP BY facid ORDER BY SUM(slots) DESC LIMIT 1 .. code-block:: python query = (Booking .select(Booking.facility, fn.SUM(Booking.slots)) .group_by(Booking.facility) .order_by(fn.SUM(Booking.slots).desc()) .limit(1)) # Retrieve multiple scalar values by calling scalar() with as_tuple=True. facid, nslots = query.scalar(as_tuple=True) List the total slots booked per facility per month, part 2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Produce a list of the total number of slots booked per facility per month in the year of 2012. In this version, include output rows containing totals for all months per facility, and a total for all months for all facilities. The output table should consist of facility id, month and slots, sorted by the id and month. When calculating the aggregated values for all months and all facids, return null values in the month and facid columns. Postgres ONLY. .. code-block:: sql SELECT facid, date_part('month', starttime), SUM(slots) FROM booking WHERE date_part('year', starttime) = 2012 GROUP BY ROLLUP(facid, date_part('month', starttime)) ORDER BY facid, date_part('month', starttime) .. code-block:: python month = fn.date_part('month', Booking.starttime) query = (Booking .select(Booking.facility, month.alias('month'), fn.SUM(Booking.slots)) .where(fn.date_part('year', Booking.starttime) == 2012) .group_by(fn.ROLLUP(Booking.facility, month)) .order_by(Booking.facility, month)) List the total hours booked per named facility ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Produce a list of the total number of hours booked per facility, remembering that a slot lasts half an hour. The output table should consist of the facility id, name, and hours booked, sorted by facility id. .. code-block:: sql SELECT f.facid, f.name, SUM(b.slots) * .5 FROM facilities AS f INNER JOIN bookings AS b ON (f.facid = b.facid) GROUP BY f.facid, f.name ORDER BY f.facid .. code-block:: python query = (Facility .select(Facility.facid, Facility.name, fn.SUM(Booking.slots) * .5) .join(Booking) .group_by(Facility.facid, Facility.name) .order_by(Facility.facid)) List each member's first booking after September 1st 2012 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Produce a list of each member name, id, and their first booking after September 1st 2012. Order by member ID. .. code-block:: sql SELECT m.surname, m.firstname, m.memid, min(b.starttime) as starttime FROM members AS m INNER JOIN bookings AS b ON b.memid = m.memid WHERE starttime >= '2012-09-01' GROUP BY m.surname, m.firstname, m.memid ORDER BY m.memid; .. code-block:: python query = (Member .select(Member.surname, Member.firstname, Member.memid, fn.MIN(Booking.starttime).alias('starttime')) .join(Booking) .where(Booking.starttime >= datetime.date(2012, 9, 1)) .group_by(Member.surname, Member.firstname, Member.memid) .order_by(Member.memid)) Produce a list of member names, with each row containing the total member count ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Produce a list of member names, with each row containing the total member count. Order by join date. Postgres ONLY (as written). .. code-block:: sql SELECT COUNT(*) OVER(), firstname, surname FROM members ORDER BY joindate .. code-block:: python query = (Member .select(fn.COUNT(Member.memid).over(), Member.firstname, Member.surname) .order_by(Member.joindate)) Produce a numbered list of members ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Produce a monotonically increasing numbered list of members, ordered by their date of joining. Remember that member IDs are not guaranteed to be sequential. Postgres ONLY (as written). .. code-block:: sql SELECT row_number() OVER (ORDER BY joindate), firstname, surname FROM members ORDER BY joindate; .. code-block:: python query = (Member .select(fn.row_number().over(order_by=[Member.joindate]), Member.firstname, Member.surname) .order_by(Member.joindate)) Output the facility id that has the highest number of slots booked, again ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Output the facility id that has the highest number of slots booked. Ensure that in the event of a tie, all tieing results get output. Postgres ONLY (as written). .. code-block:: sql SELECT facid, total FROM ( SELECT facid, SUM(slots) AS total, rank() OVER (order by SUM(slots) DESC) AS rank FROM bookings GROUP BY facid ) AS ranked WHERE rank = 1 .. code-block:: python rank = fn.rank().over(order_by=[fn.SUM(Booking.slots).desc()]) subq = (Booking .select(Booking.facility, fn.SUM(Booking.slots).alias('total'), rank.alias('rank')) .group_by(Booking.facility)) # Here we use a plain Select() to create our query. query = (Select(columns=[subq.c.facid, subq.c.total]) .from_(subq) .where(subq.c.rank == 1) .bind(db)) # We must bind() it to the database. # To iterate over the query results: for facid, total in query.tuples(): print(facid, total) Rank members by (rounded) hours used ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Produce a list of members, along with the number of hours they've booked in facilities, rounded to the nearest ten hours. Rank them by this rounded figure, producing output of first name, surname, rounded hours, rank. Sort by rank, surname, and first name. Postgres ONLY (as written). .. code-block:: sql SELECT firstname, surname, ((SUM(bks.slots)+10)/20)*10 as hours, rank() over (order by ((sum(bks.slots)+10)/20)*10 desc) as rank FROM members AS mems INNER JOIN bookings AS bks ON mems.memid = bks.memid GROUP BY mems.memid ORDER BY rank, surname, firstname; .. code-block:: python hours = ((fn.SUM(Booking.slots) + 10) / 20) * 10 query = (Member .select(Member.firstname, Member.surname, hours.alias('hours'), fn.rank().over(order_by=[hours.desc()]).alias('rank')) .join(Booking) .group_by(Member.memid) .order_by(SQL('rank'), Member.surname, Member.firstname)) Find the top three revenue generating facilities ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Produce a list of the top three revenue generating facilities (including ties). Output facility name and rank, sorted by rank and facility name. Postgres ONLY (as written). .. code-block:: sql SELECT name, rank FROM ( SELECT f.name, RANK() OVER (ORDER BY SUM( CASE WHEN memid = 0 THEN slots * f.guestcost ELSE slots * f.membercost END) DESC) AS rank FROM bookings INNER JOIN facilities AS f ON bookings.facid = f.facid GROUP BY f.name) AS subq WHERE rank <= 3 ORDER BY rank; .. code-block:: python total_cost = fn.SUM(Case(None, ( (Booking.member == 0, Booking.slots * Facility.guestcost), ), (Booking.slots * Facility.membercost))) subq = (Facility .select(Facility.name, fn.RANK().over(order_by=[total_cost.desc()]).alias('rank')) .join(Booking) .group_by(Facility.name)) query = (Select(columns=[subq.c.name, subq.c.rank]) .from_(subq) .where(subq.c.rank <= 3) .order_by(subq.c.rank) .bind(db)) # Here again we used plain Select, and call bind(). Classify facilities by value ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Classify facilities into equally sized groups of high, average, and low based on their revenue. Order by classification and facility name. Postgres ONLY (as written). .. code-block:: sql SELECT name, CASE class WHEN 1 THEN 'high' WHEN 2 THEN 'average' ELSE 'low' END FROM ( SELECT f.name, ntile(3) OVER (ORDER BY SUM( CASE WHEN memid = 0 THEN slots * f.guestcost ELSE slots * f.membercost END) DESC) AS class FROM bookings INNER JOIN facilities AS f ON bookings.facid = f.facid GROUP BY f.name ) AS subq ORDER BY class, name; .. code-block:: python cost = fn.SUM(Case(None, ( (Booking.member == 0, Booking.slots * Facility.guestcost), ), (Booking.slots * Facility.membercost))) subq = (Facility .select(Facility.name, fn.NTILE(3).over(order_by=[cost.desc()]).alias('klass')) .join(Booking) .group_by(Facility.name)) klass_case = Case(subq.c.klass, [(1, 'high'), (2, 'average')], 'low') query = (Select(columns=[subq.c.name, klass_case]) .from_(subq) .order_by(subq.c.klass, subq.c.name) .bind(db)) Calculate the payback time for each facility ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Based on the 3 complete months of data so far, calculate the amount of time each facility will take to repay its cost of ownership. Remember to take into account ongoing monthly maintenance. Output facility name and payback time in months, order by facility name. Don't worry about differences in month lengths, we're only looking for a rough value here! .. code-block:: sql SELECT f.name, f.initialoutlay / ((SUM(CASE WHEN b.memid = 0 THEN b.slots * f.guestcost ELSE b.slots * f.membercost END) / 3) - f.monthlymaintenance) AS months FROM facilities AS f INNER JOIN bookings AS b on (b.facid = f.facid) GROUP BY f.facid ORDER BY f.name .. code-block:: python # How much money has this facility produced from its bookings? revenue = fn.SUM(Case(None, ( (Booking.member == 0, Booking.slots * Facility.guestcost), ), (Booking.slots * Facility.membercost))) # Subtract monthly maintenance from average monthly revenue. revenue_less_maintenance = (revenue / 3) - Facility.monthlymaintenance # Determine how many months needed to pay off initial outlay. payback_time = Facility.initialoutlay / revenue_less_maintenance query = (Facility .select(Facility.name, payback_time.alias('months')) .join(Booking) .group_by(Facility.facid) .order_by(Facility.name)) But, I hear you ask, what would an automatic version of this look like? One that didn't need to have a hard-coded number of months in it? That's a little more complicated, and involves some date arithmetic. I've factored that out into a CTE to make it a little more clear. .. code-block:: sql with monthdata as ( select mincompletemonth, maxcompletemonth, ((extract(year from maxcompletemonth)*12) + extract(month from maxcompletemonth) - (extract(year from mincompletemonth)*12) - extract(month from mincompletemonth)) as nummonths from ( select date_trunc('month', (select max(starttime) from bookings)) as maxcompletemonth, date_trunc('month', (select min(starttime) from bookings)) as mincompletemonth ) as subq) select name, initialoutlay / (monthlyrevenue - monthlymaintenance) as repaytime from (select f.name as name, f.initialoutlay as initialoutlay, f.monthlymaintenance as monthlymaintenance, sum(case when memid = 0 then slots * f.guestcost else slots * membercost end)/(select nummonths from monthdata) as monthlyrevenue from bookings as b inner join facilities as f on b.facid = f.facid where b.starttime < (select maxcompletemonth from monthdata) group by f.facid ) as subq order by name; .. code-block:: python # First calculate the min and max ranges of bookings. BA = Booking.alias() bounds = BA.select( fn.date_trunc('month', fn.MIN(BA.starttime)).alias('minmonth'), fn.date_trunc('month', fn.MAX(BA.starttime)).alias('maxmonth') ).alias('bounds') # Calculate how many months the range of bookings covers. extract = db.extract_date # Helper for generating EXTRACT .. FROM. q = bounds.select_from( bounds.c.minmonth, bounds.c.maxmonth, ((extract('year', bounds.c.maxmonth) * 12) + extract('month', bounds.c.maxmonth) - (extract('year', bounds.c.minmonth) * 12) - extract('month', bounds.c.minmonth)).alias('nmonths')) # Indicate that we will be using this as a CTE. monthdata = q.cte('monthdata') # Subqueries to retrieve total & max month data from the CTE. nmonths = Select((monthdata,), (monthdata.c.nmonths,)) maxmonth = Select((monthdata,), (monthdata.c.maxmonth,)) # Our familiar revenue calculation. revenue = fn.SUM(Case(None, ( (Booking.member == 0, Booking.slots * Facility.guestcost), ), (Booking.slots * Facility.membercost))) revenue_less_maintenance = (revenue / nmonths) - Facility.monthlymaintenance payback_time = Facility.initialoutlay / revenue_less_maintenance q = (Facility .select( Facility.name, payback_time.alias('payback_time')) .join(Booking) .where(Booking.starttime < maxmonth) .group_by(Facility.facid) .order_by(Facility.name) .with_cte(monthdata)) Dates and Times --------------- Work out the end time of bookings ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Return a list of the start and end time of the last 10 bookings (ordered by the time at which they end, followed by the time at which they start) in the system. .. code-block:: sql SELECT starttime, starttime + slots*(interval '30 minutes') AS endtime FROM bookings ORDER BY endtime DESC, starttime DESC LIMIT 10 .. code-block:: python endtime = Booking.starttime + (Booking.slots * db.interval('30 minutes')) query = (Booking .select(Booking.starttime, endtime.alias('endtime')) .order_by(endtime.desc(), Booking.starttime.desc()) .limit(10)) Return a count of bookings for each month ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Return a count of bookings for each month, sorted by month. .. code-block:: sql SELECT date_trunc('month', starttime) as month, COUNT(*) FROM bookings GROUP BY month ORDER BY month .. code-block:: python month = db.truncate_date('month', Booking.starttime) query = (Booking .select( month.alias('month'), fn.COUNT(Booking.bookid).alias('count')) .group_by(month) .order_by(month)) Work out the utilisation percentage for each facility by month ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Work out the utilisation percentage for each facility by month, sorted by name and month, rounded to 1 decimal place. Opening time is 8am, closing time is 8.30pm. You can treat every month as a full month, regardless of if there were some dates the club was not open. .. code-block:: sql SELECT name, month, round((100*slots)/ cast( 25*(cast((month + interval '1 month') as date) - cast (month as date)) as numeric),1) as utilisation FROM ( SELECT facs.name as name, date_trunc('month', starttime) as month, sum(slots) as slots FROM bookings bks INNER JOIN facilities AS facs ON bks.facid = facs.facid GROUP BY facs.facid, month ) as _ ORDER BY name, month .. code-block:: python # Create the inner query first. month = db.truncate_date('month', Booking.starttime) subq = (Booking .select( Facility.name, month.alias('month'), fn.SUM(Booking.slots).alias('slots')) .join(Facility) .group_by(Facility.facid, month)) # Expression representing the utilization. utilization = fn.ROUND( (100 * subq.c.slots) / Cast(25 * ( Cast(subq.c.month + db.interval('1 month'), 'date') - Cast(subq.c.month, 'date')), 'numeric'), 1) query = (subq .select_from( subq.c.name, subq.c.month, utilization.alias('utilization')) .order_by(subq.c.name, subq.c.month)) String ------ Format the names of members ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Output the names of all members, formatted as 'Surname, Firstname' .. code-block:: sql SELECT surname || ', ' || firstname as name FROM members .. code-block:: python query = Member.select( (Member.surname + ', ' + Member.firstname).alias('name')) Find facilities by a name prefix ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Find all facilities whose name begins with 'Tennis'. Retrieve all columns. .. code-block:: sql SELECT * FROM facilities WHERE name LIKE 'Tennis%'; .. code-block:: python # `startswith()` uses ILIKE (case-insensitive): query = Facility.select().where(Facility.name.startswith('Tennis')) # For case-sensitive search use LIKE explicitly: query = Facility.select().where(Facility.name.like('Tennis%')) Perform a case-insensitive search ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Perform a case-insensitive search to find all facilities whose name begins with 'tennis'. Retrieve all columns. .. code-block:: sql SELECT * FROM facilities WHERE name ILIKE 'tennis%'; -- OR -- SELECT * FROM facilities WHERE UPPER(name) LIKE 'TENNIS%'; .. code-block:: python # `startswith()` uses ILIKE (case-insensitive): query = Facility.select().where(Facility.name.startswith('tennis')) # For case-sensitive search use ILIKE explicitly: query = Facility.select().where(Facility.name.ilike('tennis%')) # Or convert to upper: query = Facility.select().where( fn.upper(Facility.name).like('TENNIS%')) Find telephone numbers with parentheses ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You've noticed that the club's member table has telephone numbers with very inconsistent formatting. You'd like to find all the telephone numbers that contain parentheses, returning the member ID and telephone number sorted by member ID. .. code-block:: sql SELECT memid, telephone FROM members WHERE telephone ~ '[()]'; .. code-block:: python query = (Member .select(Member.memid, Member.telephone) .where(Member.telephone.regexp('[()]'))) Pad zip codes with leading zeroes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The zip codes in our example dataset have had leading zeroes removed from them by virtue of being stored as a numeric type. Retrieve all zip codes from the members table, padding any zip codes less than 5 characters long with leading zeroes. Order by the new zip code. .. code-block:: sql SELECT lpad(cast(zipcode as char(5)),5,'0') AS zip FROM members ORDER BY zip .. code-block:: python # Because we're wrapping an integer field, Peewee will still want to try and # coerce the LPAD() output to an integer, so we need to inform Peewee to # leave the LPAD result as-is. zipcode = fn.lpad(Cast(Member.zipcode, 'char(5)'), 5, '0', coerce=False) query = Member.select(zipcode.alias('zipcode')).order_by(zipcode) Aggregate by last name first initial ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You'd like to produce a count of how many members you have whose surname starts with each letter of the alphabet. Sort by the letter, and don't worry about printing out a letter if the count is 0. .. code-block:: sql SELECT substr(surname, 1, 1) as letter, count(*) as count FROM members GROUP BY letter ORDER BY letter .. code-block:: python initial = fn.SUBSTR(Member.surname, 1, 1) q = (Member .select(initial, fn.COUNT(Member.memid)) .group_by(initial) .order_by(initial)) Clean up telephone numbers ^^^^^^^^^^^^^^^^^^^^^^^^^^^ The telephone numbers in the database are very inconsistently formatted. You'd like to print a list of member ids and numbers that have had '-','(',')', and '' characters removed. Order by member id. .. code-block:: sql SELECT memid, translate(telephone, '-() ', '') as telephone FROM members ORDER BY memid; .. code-block:: python clean = fn.translate(Member.telephone, '-() ', '') q = (Member .select(Member.memid, clean.alias('telephone')) .order_by(Member.memid)) Recursion --------- Common Table Expressions allow us to, effectively, create our own temporary tables for the duration of a query - they're largely a convenience to help us make more readable SQL. Using the WITH RECURSIVE modifier, however, it's possible for us to create recursive queries. This is enormously advantageous for working with tree and graph-structured data - imagine retrieving all of the relations of a graph node to a given depth, for example. Find the upward recommendation chain for member ID 27 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Find the upward recommendation chain for member ID 27: that is, the member who recommended them, and the member who recommended that member, and so on. Return member ID, first name, and surname. Order by descending member id. .. code-block:: sql WITH RECURSIVE recommenders(recommender) as ( SELECT recommendedby FROM members WHERE memid = 27 UNION ALL SELECT mems.recommendedby FROM recommenders recs INNER JOIN members AS mems ON mems.memid = recs.recommender ) SELECT recs.recommender, mems.firstname, mems.surname FROM recommenders AS recs INNER JOIN members AS mems ON recs.recommender = mems.memid ORDER By memid DESC; .. code-block:: python # Base-case of recursive CTE. Get member recommender where memid=27. base = (Member .select(Member.recommendedby) .where(Member.memid == 27) .cte('recommenders', recursive=True, columns=('recommender',))) # Recursive term of CTE. Get recommender of previous recommender. MA = Member.alias() recursive = (MA .select(MA.recommendedby) .join(base, on=(MA.memid == base.c.recommender))) # Combine the base-case with the recursive term. cte = base.union_all(recursive) # Select from the recursive CTE, joining on member to get name info. query = (cte .select_from(cte.c.recommender, Member.firstname, Member.surname) .join(Member, on=(cte.c.recommender == Member.memid)) .order_by(Member.memid.desc())) Find the downward recommendation chain for member ID 1 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Find the downward recommendation chain for member ID 1: that is, the members they recommended, the members those members recommended, and so on. Return member ID and name, and order by ascending member id. .. code-block:: sql WITH RECURSIVE recommendeds(memid) AS ( SELECT memid FROM members WHERE recommendedby = 1 UNION ALL SELECT mems.memid FROM recommendeds recs INNER JOIN members mems ON mems.recommendedby = recs.memid ) SELECT recs.memid, mems.firstname, mems.surname FROM recommendeds recs INNER JOIN members mems ON recs.memid = mems.memid ORDER BY memid .. code-block:: python # Base-case of recursive CTE. Get members recommended by memid=1. base = (Member .select(Member.memid) .where(Member.recommendedby == 1) .cte('recommenders', recursive=True, columns=('memid',))) # Recursive term of CTE. Get recommended by previous recommender. MA = Member.alias() recursive = (MA .select(MA.memid) .join(base, on=(MA.recommendedby == base.c.memid))) # Combine the base-case with the recursive term. cte = base.union_all(recursive) # Select from the recursive CTE, joining on member to get name info. query = (cte .select_from(cte.c.memid, Member.firstname, Member.surname) .join(Member, on=(cte.c.memid == Member.memid)) .order_by(Member.memid)) for row in query: print(row.memid, row.firstname, row.surname) Produce a upward recommendation chain for any member ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Produce a CTE that can return the upward recommendation chain for any member. You should be able to select recommender from recommenders where member=x. Demonstrate it by getting the chains for members 12 and 22. Results table should have member and recommender, ordered by member ascending, recommender descending. .. code-block:: sql WITH RECURSIVE recommenders(recommender, member) AS ( SELECT recommendedby, memid FROM members UNION ALL SELECT mems.recommendedby, recs.member FROM recommenders recs INNER JOIN members mems ON mems.memid = recs.recommender ) SELECT recs.member member, recs.recommender, mems.firstname, mems.surname FROM recommenders recs INNER JOIN members mems ON recs.recommender = mems.memid WHERE recs.member = 22 or recs.member = 12 ORDER BY recs.member ASC, recs.recommender DESC .. code-block:: python # Base-case of recursive CTE. Get member recommender where memid=27. base = (Member .select(Member.recommendedby, Member.memid) .cte('recommenders', recursive=True, columns=('recommender', 'member'))) # Recursive term of CTE. Get recommender of previous recommender. MA = Member.alias() recursive = (MA .select(MA.recommendedby, base.c.member) .join(base, on=(MA.memid == base.c.recommender))) # Combine the base-case with the recursive term. cte = base.union_all(recursive) # Select from the recursive CTE, joining on member to get name info. query = (cte .select_from( cte.c.member, cte.c.recommender, Member.firstname, Member.surname) .join(Member, on=(cte.c.recommender == Member.memid)) .where((cte.c.member == 22) | (cte.c.member == 12)) .order_by(cte.c.member, cte.c.recommender.desc())) ================================================ FILE: docs/peewee/query_operators.rst ================================================ .. _query-operators: Query Operators =============== The following types of comparisons are supported by peewee: ================ ======================================= Comparison Meaning ================ ======================================= ``==`` x equals y ``!=`` x is not equal to y ``<`` x is less than y ``<=`` x is less than or equal to y ``>`` x is greater than y ``>=`` x is greater than or equal to y ``<<`` x IN y, where y is a list or query ``>>`` x IS y, where y is None/NULL ``%`` x LIKE y where y may contain wildcards ``**`` x ILIKE y where y may contain wildcards ``^`` x XOR y ``~`` Unary negation (e.g., NOT x) ================ ======================================= Additional operations are provided as methods: ======================= =============================================== Method Meaning ======================= =============================================== ``.in_(value)`` IN lookup (identical to ``<<``). ``.not_in(value)`` NOT IN lookup. ``.is_null(is_null)`` IS NULL or IS NOT NULL. Accepts boolean param. ``.contains(substr)`` Wild-card search for substring. ``.startswith(prefix)`` Search for values beginning with ``prefix``. ``.endswith(suffix)`` Search for values ending with ``suffix``. ``.between(low, high)`` Search where ``low <= value <= high``. ``.regexp(exp)`` Regular expression match (case-sensitive). ``.iregexp(exp)`` Regular expression match (case-insensitive). ``.bin_and(value)`` Binary AND. ``.bin_or(value)`` Binary OR. ``.concat(other)`` Concatenate two strings or objects using ``||``. ``.distinct()`` Mark column for DISTINCT selection. ``.collate(collation)`` Specify column with the given collation. ``.cast(type)`` Cast the value of the column to the given type. ======================= =============================================== To combine clauses using logical operators, use: ================ ==================== ====================================================== Operator Meaning Example ================ ==================== ====================================================== ``&`` AND ``(User.is_active == True) & (User.is_admin == True)`` ``|`` (pipe) OR ``(User.is_admin) | (User.is_superuser)`` ``~`` NOT (unary negation) ``~(User.username.contains('admin'))`` ================ ==================== ====================================================== Here is how you might use some of these query operators: .. code-block:: python # Find the user whose username is "charlie". User.select().where(User.username == 'charlie') # Find the users whose username is in [charlie, huey, mickey] User.select().where(User.username.in_(['charlie', 'huey', 'mickey'])) # Find users whose salary is between 50k and 60k (inclusive). Employee.select().where(Employee.salary.between(50000, 60000)) Employee.select().where(Employee.name.startswith('C')) Blog.select().where(Blog.title.contains(search_string)) Here is how you might combine expressions. Comparisons can be arbitrarily complex. .. note:: Comparisons must be wrapped in parentheses due to Python operator precedence rules. .. code-block:: python # Find any users who are active administrations. User.select().where( (User.is_admin == True) & (User.is_active == True)) # Find any users who are either administrators or super-users. User.select().where( (User.is_admin == True) | (User.is_superuser == True)) # Alternatively, use the boolean values directly. Here we query users who # are admins and NOT superusers. User.select().where(User.is_admin & ~User.is_superuser) # Find any Tweets by users who are not admins (NOT IN). admins = User.select().where(User.is_admin == True) non_admin_tweets = Tweet.select().where(Tweet.user.not_in(admins)) # Find any users who are not my friends (strangers). friends = User.select().where(User.username.in_(['charlie', 'huey', 'mickey'])) strangers = User.select().where(User.id.not_in(friends)) .. warning:: Although you may be tempted to use python's ``in``, ``and``, ``or``, ``is``, and ``not`` operators in your query expressions, these **will not work.** The return value of an ``in`` expression is always coerced to a boolean value. Similarly, ``and``, ``or`` and ``not`` all treat their arguments as boolean values and cannot be overloaded. So just remember: * Use ``.in_()`` and ``.not_in()`` instead of ``in`` and ``not in`` * Use ``&`` instead of ``and`` * Use ``|`` instead of ``or`` * Use ``~`` instead of ``not`` * Use ``.is_null()`` instead of ``is None`` or ``== None``. * Use ``==`` and ``!=`` for comparing against ``True`` and ``False``, or you may use the implicit value of the expression. * **Don't forget to wrap comparisons in parentheses when using logical operators.** For more examples, see the :ref:`expressions` section. .. note:: **LIKE and ILIKE with SQLite** Because SQLite's ``LIKE`` operation is case-insensitive by default, peewee will use the SQLite ``GLOB`` operation for case-sensitive searches. The glob operation uses asterisks for wildcards as opposed to the usual percent-sign. If you are using SQLite and want case-sensitive partial string matching, remember to use asterisks for the wildcard. Three Valued Logic ------------------ Because of the way SQL handles ``NULL``, there are some special operations available for expressing: * ``IS NULL`` * ``IS NOT NULL`` * ``IN`` * ``NOT IN`` While it would be possible to use the ``IS NULL`` and ``IN`` operators with the negation operator (``~``), sometimes to get the correct semantics you will need to explicitly use ``IS NOT NULL`` and ``NOT IN``. The simplest way to use ``IS NULL`` and ``IN`` is to use the operator overloads: .. code-block:: python # Get all User objects whose last login is NULL. User.select().where(User.last_login >> None) # Get users whose username is in the given list. usernames = ['charlie', 'huey', 'mickey'] User.select().where(User.username << usernames) If you don't like operator overloads, you can call the Field methods instead: .. code-block:: python # Get all User objects whose last login is NULL. User.select().where(User.last_login.is_null(True)) # Get users whose username is in the given list. usernames = ['charlie', 'huey', 'mickey'] User.select().where(User.username.in_(usernames)) To negate the above queries, you can use unary negation, but for the correct semantics use the special ``IS NOT`` and ``NOT IN`` operators: .. code-block:: python # Get all User objects whose last login is *NOT* NULL. User.select().where(User.last_login.is_null(False)) # Get users whose username is *NOT* in the given list. usernames = ['charlie', 'huey', 'mickey'] User.select().where(User.username.not_in(usernames)) .. _custom-operators: User-Defined Operators ----------------------- Because I ran out of python operators to overload, there are some missing operators in peewee, for instance ``modulo``. If you find that you need to support an operator that is not in the table above, it is very easy to add your own. Here is how you might add support for ``modulo`` in SQLite: .. code-block:: python from peewee import * from peewee import Expression # The building block for expressions. def mod(lhs, rhs): # Note: this works with Sqlite, but some drivers may use string- # formatting before sending the query to the database, so you may # need to use '%%' instead here. return Expression(lhs, '%', rhs) Now you can use these custom operators to build richer queries: .. code-block:: python # Users with even ids. User.select().where(mod(User.id, 2) == 0) .. _expressions: Expressions ----------- Peewee is designed to provide a simple, expressive, and pythonic way of constructing SQL queries. This section will provide a quick overview of some common types of expressions. Two common types of objects that are composed to create expressions: * :class:`Field` instances * SQL aggregations and functions using :class:`fn` We will assume a simple "User" model with fields for username and other things. It looks like this: .. code-block:: python class User(Model): username = CharField() is_admin = BooleanField() is_active = BooleanField() last_login = DateTimeField() login_count = IntegerField() failed_logins = IntegerField() Comparisons use the :ref:`query-operators`: .. code-block:: python # username is equal to 'charlie' User.username == 'charlie' # user has logged in less than 5 times User.login_count < 5 Comparisons can be combined using **bitwise** *and* and *or*. Operator precedence is controlled by python and comparisons can be nested to an arbitrary depth: .. code-block:: python # User is both and admin and has logged in today (User.is_admin == True) & (User.last_login >= today) # User's username is either charlie or charles (User.username == 'charlie') | (User.username == 'charles') # User is active and not a superuser. (User.is_active & ~User.is_superuser) Comparisons can be used with functions as well: .. code-block:: python # user's username starts with a 'g' or a 'G': fn.Lower(fn.Substr(User.username, 1, 1)) == 'g' We can do some fairly interesting things, as expressions can be compared against other expressions. Expressions also support arithmetic operations: .. code-block:: python # users who entered the incorrect more than half the time and have logged # in at least 10 times (User.failed_logins > (User.login_count * .5)) & (User.login_count > 10) Expressions allow us to do *atomic updates*: .. code-block:: python # when a user logs in we want to increment their login count: User.update(login_count=User.login_count + 1).where(User.id == user_id) Expressions can be used in all parts of a query, so experiment! Row values ^^^^^^^^^^ Many databases support `row values `_, which are similar to Python `tuple` objects. In Peewee, it is possible to use row-values in expressions via :class:`Tuple`. For example, .. code-block:: python # If for some reason your schema stores dates in separate columns ("year", # "month" and "day"), you can use row-values to find all rows that happened # in a given month: Tuple(Event.year, Event.month) == (2019, 1) The more common use for row-values is to compare against multiple columns from a subquery in a single expression. There are other ways to express these types of queries, but row-values may offer a concise and readable approach. For example, assume we have a table "EventLog" which contains an event type, an event source, and some metadata. We also have an "IncidentLog", which has incident type, incident source, and metadata columns. We can use row-values to correlate incidents with certain events: .. code-block:: python class EventLog(Model): event_type = TextField() source = TextField() data = TextField() timestamp = TimestampField() class IncidentLog(Model): incident_type = TextField() source = TextField() traceback = TextField() timestamp = TimestampField() # Get a list of all the incident types and sources that have occured today. incidents = (IncidentLog .select(IncidentLog.incident_type, IncidentLog.source) .where(IncidentLog.timestamp >= datetime.date.today())) # Find all events that correlate with the type and source of the # incidents that occured today. events = (EventLog .select() .where(Tuple(EventLog.event_type, EventLog.source).in_(incidents)) .order_by(EventLog.timestamp)) Other ways to express this type of query would be to use a :ref:`join ` or to :ref:`join on a subquery `. The above example is there just to give you and idea how :class:`Tuple` might be used. You can also use row-values to update multiple columns in a table, when the new data is derived from a subquery. For an example, see `here `_. SQL Functions ------------- SQL functions, like ``COUNT()`` or ``SUM()``, can be expressed using the :func:`fn` helper: .. code-block:: python # Get all users and the number of tweets they've authored. Sort the # results from most tweets -> fewest tweets. query = (User .select(User, fn.COUNT(Tweet.id).alias('tweet_count')) .join(Tweet, JOIN.LEFT_OUTER) .group_by(User) .order_by(fn.COUNT(Tweet.id).desc())) for user in query: print('%s -- %s tweets' % (user.username, user.tweet_count)) The ``fn`` helper exposes any SQL function as if it were a method. The parameters can be fields, values, subqueries, or even nested functions. Nesting function calls ^^^^^^^^^^^^^^^^^^^^^^ Suppose you need to want to get a list of all users whose username begins with *a*. There are a couple ways to do this, but one method might be to use some SQL functions like *LOWER* and *SUBSTR*. To use arbitrary SQL functions, use the special :func:`fn` object to construct queries: .. code-block:: python # Select the user's id, username and the first letter of their username, lower-cased first_letter = fn.LOWER(fn.SUBSTR(User.username, 1, 1)) query = User.select(User, first_letter.alias('first_letter')) # Alternatively we could select only users whose username begins with 'a' a_users = User.select().where(first_letter == 'a') for user in a_users: print(user.username) SQL Helper ---------- There are times when you may want to simply pass in some arbitrary sql. You can do this using the special :class:`SQL` class. One use-case is when referencing an alias: .. code-block:: python # We'll query the user table and annotate it with a count of tweets for # the given user query = (User .select(User, fn.Count(Tweet.id).alias('ct')) .join(Tweet) .group_by(User)) # Now we will order by the count, which was aliased to "ct" query = query.order_by(SQL('ct')) # You could, of course, also write this as: query = query.order_by(fn.COUNT(Tweet.id)) There are two ways to execute hand-crafted SQL statements with peewee: 1. :meth:`Database.execute_sql` for executing any type of query 2. :class:`RawQuery` for executing ``SELECT`` queries and returning model instances. Security and SQL Injection -------------------------- By default peewee will parameterize queries, so any parameters passed in by the user will be escaped. The only exception to this rule is if you are writing a raw SQL query or are passing in a ``SQL`` object which may contain untrusted data. To mitigate this, ensure that any user-defined data is passed in as a query parameter and not part of the actual SQL query: .. code-block:: python # Bad! DO NOT DO THIS! query = MyModel.raw('SELECT * FROM my_table WHERE data = %s' % user_data) # Bad! DO NOT DO THIS! query = MyModel.select().where(SQL('Some SQL expression %s' % user_data)) Use parameters to prevent SQL injection: .. code-block:: python # Good. `user_data` will be treated as a parameter to the query. query = MyModel.raw('SELECT * FROM my_table WHERE data = %s', user_data) # Good. `user_data` will be treated as a parameter. query = MyModel.select().where(SQL('Some SQL expression %s', user_data)) .. note:: MySQL and Postgresql use ``'%s'`` to denote parameters. SQLite, on the other hand, uses ``'?'``. Be sure to use the character appropriate to your database. You can also find this parameter by checking :attr:`Database.param`. ================================================ FILE: docs/peewee/querying.rst ================================================ .. _querying: Querying ======== This document covers reading data from the database: SELECT queries, filtering, sorting, aggregation, and result-set iteration. Writing data (INSERT, UPDATE, DELETE) is covered in :ref:`writing`. All examples use the following models (see :ref:`models`): .. code-block:: python import datetime from peewee import * db = SqliteDatabase(':memory:') class BaseModel(Model): class Meta: database = db class User(BaseModel): username = TextField(unique=True) class Tweet(BaseModel): user = ForeignKeyField(User, backref='tweets') content = TextField() timestamp = DateTimeField(default=datetime.datetime.now) is_published = BooleanField(default=True) .. seealso:: :ref:`Extensive library of SQL / Peewee examples ` Selecting Records ----------------- :meth:`Model.select` returns a :class:`Select` query. The query is lazy - the database is not queried until you iterate over the result, index, slice or call a method that forces execution. .. code-block:: python # All users. No query issued yet. query = User.select() # Query executes here. for user in query: print(user.username) Iterating over the same query object a second time does not re-query the database: results are cached on the query object. To disable caching (for example, when iterating over a large result set), use :meth:`~BaseQuery.iterator`: .. code-block:: python for user in User.select().iterator(): process(user) # One row at a time, not cached. To select specific columns rather than all columns, pass field expressions to ``select()``: .. code-block:: python for user in User.select(User.username): print(user.username) # user.id is not populated - it was not selected. To select columns from multiple models, pass both model classes or their fields. Peewee reconstructs the model graph from the result set: .. code-block:: python query = Tweet.select(Tweet, User).join(User) for tweet in query: # tweet.user is a fully populated User instance. # No extra query is issued. print(tweet.user.username, '->', tweet.content) .. seealso:: :ref:`relationships` covers joins in detail. Retrieving a Single Record -------------------------- :meth:`Model.get` executes the query and returns the first matching row. If no row matches, :exc:`~Model.DoesNotExist` is raised: .. code-block:: python user = User.get(User.username == 'charlie') # Equivalent long form: user = User.select().where(User.username == 'charlie').get() :meth:`~Model.get_by_id` and the subscript operator are shortcuts for primary-key lookups: .. code-block:: python user = User.get_by_id(1) user = User[1] # Same. :meth:`~Model.get_or_none` returns ``None`` instead of raising an exception when no row is found: .. code-block:: python user = User.get_or_none(User.username == 'charlie') if user is None: print('Not found.') :meth:`~SelectBase.first` returns the first row of a query, or ``None``: .. code-block:: python latest = Tweet.select().order_by(Tweet.timestamp.desc()).first() Get or Create ^^^^^^^^^^^^^ :meth:`~Model.get_or_create` retrieves a matching row, or creates it if it does not exist. It returns a ``(instance, created)`` tuple: .. code-block:: python user, created = User.get_or_create(username='charlie') if created: print('New user created.') Use the ``defaults`` keyword to supply values that are only used during creation, not as lookup keys: .. code-block:: python user, created = User.get_or_create( username='charlie', defaults={'joined': datetime.date.today()}) When uniqueness is enforced by a database constraint, the recommended pattern is to attempt creation first and fall back to retrieval on failure: .. code-block:: python try: with db.atomic(): return User.create(username=username) except IntegrityError: return User.get(User.username == username) This avoids a race window between the lookup and the insert. .. _filtering: Filtering --------- :meth:`~Query.where` accepts expressions built from field comparisons. Peewee overloads Python's comparison operators to produce SQL expressions: .. code-block:: python # Equality User.select().where(User.username == 'charlie') # Inequality Tweet.select().where(Tweet.is_published != False) # Comparison Tweet.select().where(Tweet.timestamp < datetime.datetime(2024, 1, 1)) Peewee uses **bitwise** operators (``&`` and ``|``) rather than logical operators (``and`` and ``or``). The reason for this is that Python coerces the logical operations to a boolean value. This is also the reason why "IN" queries must be expressed using ``.in_()`` rather than the ``in`` operator. .. seealso:: :ref:`Query operations ` to see all operators. Combine conditions with ``&`` (AND) and ``|`` (OR): .. code-block:: python # Published tweets by charlie: query = (Tweet .select() .join(User) .where( (User.username == 'charlie') & (Tweet.is_published == True))) # Tweets by charlie OR huey: query = (Tweet .select() .join(User) .where( (User.username == 'charlie') | (User.username == 'huey'))) Negate a condition with ``~``: .. code-block:: python # All users except charlie: User.select().where(~(User.username == 'charlie')) Calling ``.where()`` multiple times on a query ANDs the conditions: .. code-block:: python # Equivalent to WHERE is_published = 1 AND timestamp > ... query = (Tweet .select() .where(Tweet.is_published == True) .where(Tweet.timestamp > one_week_ago)) Common filtering methods ^^^^^^^^^^^^^^^^^^^^^^^^ ============================================= ==================================== Method SQL equivalent ============================================= ==================================== ``User.username == 'charlie'`` ``username = 'charlie'`` ``User.username != 'charlie'`` ``username != 'charlie'`` ``Tweet.timestamp < dt`` ``timestamp < dt`` ``Tweet.timestamp >= dt`` ``timestamp >= dt`` ``Tweet.timestamp.between(start, end)`` ``timestamp BETWEEN start AND end`` ``User.username.in_(['a', 'b'])`` ``username IN ('a', 'b')`` ``User.username.not_in(['a', 'b'])`` ``username NOT IN ...`` ``User.username.contains('char')`` ``username LIKE '%char%'`` ``User.username.startswith('ch')`` ``username LIKE 'ch%'`` ``User.username.endswith('ie')`` ``username LIKE '%ie'`` ``User.username.regexp(r'^[a-z]+$')`` ``username REGEXP ...`` ``User.username.is_null()`` ``username IS NULL`` ``User.username.is_null(False)`` ``username IS NOT NULL`` ============================================= ==================================== .. note:: ``IN`` queries must use ``.in_()`` rather than Python's ``in`` operator, because Python's ``in`` returns a boolean and cannot be overridden. .. seealso:: :ref:`query-operators` for the full list of supported operators and methods. SQL functions ^^^^^^^^^^^^^ The :class:`fn` helper calls any SQL function by name: .. code-block:: python from peewee import fn # Users whose username starts with a vowel (case-insensitive). vowels = ('a', 'e', 'i', 'o', 'u') query = User.select().where( fn.LOWER(fn.SUBSTR(User.username, 1, 1)).in_(vowels)) # Tweets whose content is less than 10 characters long. query = Tweet.select().where( fn.LENGTH(Tweet.content) < 10) Sorting ------- :meth:`~Query.order_by` specifies the column(s) to sort by: .. code-block:: python # Ascending (default). Tweet.select().order_by(Tweet.timestamp) # Descending. Tweet.select().order_by(Tweet.timestamp.desc()) # Using the + / - prefix operators: Tweet.select().order_by(+Tweet.timestamp) # Ascending. Tweet.select().order_by(-Tweet.timestamp) # Descending. Sort on multiple columns by passing multiple arguments: .. code-block:: python query = (Tweet .select() .join(User) .order_by(User.username, Tweet.timestamp.desc())) Sorting by a calculated or aliased value ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When ordering by an aggregate or expression that appears in ``select()``, reference it by re-using the expression or by wrapping the alias in :class:`SQL`: .. code-block:: python tweet_count = fn.COUNT(Tweet.id) query = (User .select(User.username, tweet_count.alias('num_tweets')) .join(Tweet, JOIN.LEFT_OUTER) .group_by(User.username) .order_by(tweet_count.desc())) # Alternatively, reference the alias string via SQL(): query = (User .select(User.username, fn.COUNT(Tweet.id).alias('num_tweets')) .join(Tweet, JOIN.LEFT_OUTER) .group_by(User.username) .order_by(SQL('num_tweets').desc())) Random ordering ^^^^^^^^^^^^^^^ :meth:`Database.random` provides a database-specific implementation of the random function, which can be used for ordering: .. code-block:: python # Select 5 random winners. LotteryEntry.select().order_by(db.random()).limit(5) Pagination, Limiting, and Offsetting -------------------------------------- :meth:`~Query.limit` and :meth:`~Query.offset` map directly to SQL: .. code-block:: python # First 10 rows. Tweet.select().order_by(Tweet.id).limit(10) # Rows 11-20. Tweet.select().order_by(Tweet.id).limit(10).offset(10) :meth:`~Query.paginate` is a convenience wrapper: .. code-block:: python # Page 3, 20 items per page (rows 41-60). Tweet.select().order_by(Tweet.id).paginate(3, 20) .. attention:: Page numbers are 1-based. Page 1 returns the first ``items_per_page`` rows. Counting -------- :meth:`~SelectBase.count` wraps the query in a ``SELECT COUNT(1) FROM (...)`` and returns an integer: .. code-block:: python total = Tweet.select().count() published = Tweet.select().where(Tweet.is_published == True).count() Aggregates and GROUP BY ------------------------ Use :class:`fn` to call aggregate functions and :meth:`~Select.group_by` to group: .. code-block:: python query = (User .select(User.username, fn.COUNT(Tweet.id).alias('tweet_count')) .join(Tweet, JOIN.LEFT_OUTER) .group_by(User.username) .order_by(SQL('tweet_count').desc())) for user in query: print(user.username, user.tweet_count) Filter groups with :meth:`~Select.having`: .. code-block:: python # Users with more than 5 published tweets. query = (User .select(User.username, fn.COUNT(Tweet.id).alias('n')) .join(Tweet) .where(Tweet.is_published == True) .group_by(User.username) .having(fn.COUNT(Tweet.id) > 5)) Scalar Values ------------- :meth:`~SelectBase.scalar` executes a query and returns the first column of the first row as a Python value. Use it when a query produces a single number or string: .. code-block:: python oldest = Tweet.select(fn.MIN(Tweet.timestamp)).scalar() distinct_users = Tweet.select(fn.COUNT(Tweet.user.distinct())).scalar() Pass ``as_tuple=True`` to retrieve multiple scalar columns: .. code-block:: python min_ts, max_ts = Tweet.select( fn.MIN(Tweet.timestamp), fn.MAX(Tweet.timestamp) ).scalar(as_tuple=True) .. _row-types: Row Types --------- By default, SELECT queries return model instances. Four alternative row types are available by chaining a method before iteration: * :meth:`~BaseQuery.dicts` * :meth:`~BaseQuery.tuples` * :meth:`~BaseQuery.namedtuples` * :meth:`~BaseQuery.objects` Example: .. code-block:: python # Dictionaries. for row in User.select().dicts(): print(row) # {'id': 1, 'username': 'charlie'} # Tuples. for row in User.select().tuples(): print(row) # (1, 'charlie') # Named tuples. for row in User.select().namedtuples(): print(row.username) # Flatten any related data and return model instances. for row in User.select().objects(): print(row.username) # Or pass a constructor callable. for row in User.select().objects(MyUserClass): print(row.my_username) Using tuples or dicts instead of model instances is faster for queries that produce many rows, because Peewee skips constructing model objects. ``objects()`` without an argument returns model instances but does not reconstruct the model graph from joined data, assigning all columns directly onto the primary model. This avoids the overhead of graph reconstruction when you have joined data and don't need nested model instances. .. _large-results: Iterating Over Large Result Sets ---------------------------------- For queries returning many rows, disable result caching with :meth:`~BaseQuery.iterator` to keep memory usage flat: .. code-block:: python # Combine iterator() with tuples() for maximum throughput. query = (Stat .select() .tuples() .iterator()) for stat_tuple in query: write_to_file(stat_tuple) When iterating over joined queries with ``.iterator()``, use ``.objects()`` to avoid the overhead of model-graph reconstruction per row: .. code-block:: python query = (Tweet .select(Tweet.content, User.username) .join(User) .objects() .iterator()) for tweet in query: print(tweet.username, tweet.content) For maximum performance, execute the query and iterate the cursor directly: .. code-block:: python query = Tweet.select(Tweet.content, User.username).join(User) cursor = db.execute(query) for content, username in cursor: print(username, '->', content) .. _window-functions: Window Functions ---------------- A :class:`Window` function refers to an aggregate function that operates on a sliding window of data that is being processed as part of a ``SELECT`` query. Window functions make it possible to do things like: 1. Perform aggregations against subsets of a result-set. 2. Calculate a running total. 3. Rank results. 4. Compare a row value to a value in the preceding (or succeeding!) row(s). peewee comes with support for SQL window functions, which can be created by calling :meth:`Function.over` and passing in your partitioning or ordering parameters. For the following examples, we'll use the following model and sample data: .. code-block:: python class Sample(Model): counter = IntegerField() value = FloatField() data = [(1, 10), (1, 20), (2, 1), (2, 3), (3, 100)] Sample.insert_many(data, fields=[Sample.counter, Sample.value]).execute() Our sample table now contains: === ======== ====== id counter value === ======== ====== 1 1 10.0 2 1 20.0 3 2 1.0 4 2 3.0 5 3 100.0 === ======== ====== Ordered Windows ^^^^^^^^^^^^^^^ Let's calculate a running sum of the ``value`` field. In order for it to be a "running" sum, we need it to be ordered, so we'll order with respect to the Sample's ``id`` field: .. code-block:: python query = Sample.select( Sample.counter, Sample.value, fn.SUM(Sample.value).over(order_by=[Sample.id]).alias('total')) for sample in query: print(sample.counter, sample.value, sample.total) # 1 10. 10. # 1 20. 30. # 2 1. 31. # 2 3. 34. # 3 100 134. For another example, we'll calculate the difference between the current value and the previous value, when ordered by the ``id``: .. code-block:: python difference = Sample.value - fn.LAG(Sample.value, 1).over(order_by=[Sample.id]) query = Sample.select( Sample.counter, Sample.value, difference.alias('diff')) for sample in query: print(sample.counter, sample.value, sample.diff) # 1 10. NULL # 1 20. 10. -- (20 - 10) # 2 1. -19. -- (1 - 20) # 2 3. 2. -- (3 - 1) # 3 100 97. -- (100 - 3) Partitioned Windows ^^^^^^^^^^^^^^^^^^^ Let's calculate the average ``value`` for each distinct "counter" value. Notice that there are three possible values for the ``counter`` field (1, 2, and 3). We can do this by calculating the ``AVG()`` of the ``value`` column over a window that is partitioned depending on the ``counter`` field: .. code-block:: python query = Sample.select( Sample.counter, Sample.value, fn.AVG(Sample.value).over(partition_by=[Sample.counter]).alias('cavg')) for sample in query: print(sample.counter, sample.value, sample.cavg) # 1 10. 15. # 1 20. 15. # 2 1. 2. # 2 3. 2. # 3 100 100. We can use ordering within partitions by specifying both the ``order_by`` and ``partition_by`` parameters. For an example, let's rank the samples by value within each distinct ``counter`` group. .. code-block:: python query = Sample.select( Sample.counter, Sample.value, fn.RANK().over( order_by=[Sample.value], partition_by=[Sample.counter]).alias('rank')) for sample in query: print(sample.counter, sample.value, sample.rank) # 1 10. 1 # 1 20. 2 # 2 1. 1 # 2 3. 2 # 3 100 1 Bounded Windows ^^^^^^^^^^^^^^^ By default, window functions are evaluated using an *unbounded preceding* start for the window, and the *current row* as the end. We can change the bounds of the window our aggregate functions operate on by specifying a ``start`` and/or ``end`` in the call to :meth:`Function.over`. Additionally, Peewee comes with helper-methods on the :class:`Window` object for generating the appropriate boundary references: * :attr:`Window.CURRENT_ROW` - attribute that references the current row. * :meth:`Window.preceding` - specify number of row(s) preceding, or omit number to indicate **all** preceding rows. * :meth:`Window.following` - specify number of row(s) following, or omit number to indicate **all** following rows. To examine how boundaries work, we'll calculate a running total of the ``value`` column, ordered with respect to ``id``, **but** we'll only look the running total of the current row and it's two preceding rows: .. code-block:: python query = Sample.select( Sample.counter, Sample.value, fn.SUM(Sample.value).over( order_by=[Sample.id], start=Window.preceding(2), end=Window.CURRENT_ROW).alias('rsum')) for sample in query: print(sample.counter, sample.value, sample.rsum) # 1 10. 10. # 1 20. 30. -- (20 + 10) # 2 1. 31. -- (1 + 20 + 10) # 2 3. 24. -- (3 + 1 + 20) # 3 100 104. -- (100 + 3 + 1) Technically we did not need to specify the ``end=Window.CURRENT`` because that is the default. It was shown in the example for demonstration. Let's look at another example. In this example we will calculate the "opposite" of a running total, in which the total sum of all values is decreased by the value of the samples, ordered by ``id``. To accomplish this, we'll calculate the sum from the current row to the last row. .. code-block:: python query = Sample.select( Sample.counter, Sample.value, fn.SUM(Sample.value).over( order_by=[Sample.id], start=Window.CURRENT_ROW, end=Window.following()).alias('rsum')) # 1 10. 134. -- (10 + 20 + 1 + 3 + 100) # 1 20. 124. -- (20 + 1 + 3 + 100) # 2 1. 104. -- (1 + 3 + 100) # 2 3. 103. -- (3 + 100) # 3 100 100. -- (100) Filtered Aggregates ^^^^^^^^^^^^^^^^^^^ Aggregate functions may also support filter functions (Postgres and Sqlite 3.25+), which get translated into a ``FILTER (WHERE...)`` clause. Filter expressions are added to an aggregate function with the :meth:`Function.filter` method. For an example, we will calculate the running sum of the ``value`` field with respect to the ``id``, but we will filter-out any samples whose ``counter=2``. .. code-block:: python query = Sample.select( Sample.counter, Sample.value, fn.SUM(Sample.value).filter(Sample.counter != 2).over( order_by=[Sample.id]).alias('csum')) for sample in query: print(sample.counter, sample.value, sample.csum) # 1 10. 10. # 1 20. 30. # 2 1. 30. # 2 3. 30. # 3 100 130. The call to :meth:`~Function.filter` must precede the call to :meth:`~Function.over`. Reusing Window Definitions ^^^^^^^^^^^^^^^^^^^^^^^^^^ If you intend to use the same window definition for multiple aggregates, you can create a :class:`Window` object. The :class:`Window` object takes the same parameters as :meth:`Function.over`, and can be passed to the ``over()`` method in-place of the individual parameters. Here we'll declare a single window, ordered with respect to the sample ``id``, and call several window functions using that window definition: .. code-block:: python win = Window(order_by=[Sample.id]) query = Sample.select( Sample.counter, Sample.value, fn.LEAD(Sample.value).over(win), fn.LAG(Sample.value).over(win), fn.SUM(Sample.value).over(win) ).window(win) # Include our window definition in query. for row in query.tuples(): print(row) # counter value lead() lag() sum() # 1 10. 20. NULL 10. # 1 20. 1. 10. 30. # 2 1. 3. 20. 31. # 2 3. 100. 1. 34. # 3 100. NULL 3. 134. Multiple Window Definitions ^^^^^^^^^^^^^^^^^^^^^^^^^^^ In the previous example, we saw how to declare a :class:`Window` definition and re-use it for multiple different aggregations. You can include as many window definitions as you need in your queries, but it is necessary to ensure each window has a unique alias: .. code-block:: python w1 = Window(order_by=[Sample.id]).alias('w1') w2 = Window(partition_by=[Sample.counter]).alias('w2') query = Sample.select( Sample.counter, Sample.value, fn.SUM(Sample.value).over(w1).alias('rsum'), # Running total. fn.AVG(Sample.value).over(w2).alias('cavg') # Avg per category. ).window(w1, w2) # Include our window definitions. for sample in query: print(sample.counter, sample.value, sample.rsum, sample.cavg) # counter value rsum cavg # 1 10. 10. 15. # 1 20. 30. 15. # 2 1. 31. 2. # 2 3. 34. 2. # 3 100 134. 100. Similarly, if you have multiple window definitions that share similar definitions, it is possible to extend a previously-defined window definition. For example, here we will be partitioning the data-set by the counter value, so we'll be doing our aggregations with respect to the counter. Then we'll define a second window that extends this partitioning, and adds an ordering clause: .. code-block:: python w1 = Window(partition_by=[Sample.counter]).alias('w1') # By extending w1, this window definition will also be partitioned # by "counter". w2 = Window(extends=w1, order_by=[Sample.value.desc()]).alias('w2') query = (Sample .select(Sample.counter, Sample.value, fn.SUM(Sample.value).over(w1).alias('group_sum'), fn.RANK().over(w2).alias('revrank')) .window(w1, w2) .order_by(Sample.id)) for sample in query: print(sample.counter, sample.value, sample.group_sum, sample.revrank) # counter value group_sum revrank # 1 10. 30. 2 # 1 20. 30. 1 # 2 1. 4. 2 # 2 3. 4. 1 # 3 100. 100. 1 .. _window-frame-types: Frame Types: RANGE vs ROWS vs GROUPS ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Depending on the frame type, the database will process ordered groups differently. Let's create two additional ``Sample`` rows to visualize the difference: .. code-block:: pycon >>> Sample.create(counter=1, value=20.) >>> Sample.create(counter=2, value=1.) Our table now contains: === ======== ====== id counter value === ======== ====== 1 1 10.0 2 1 20.0 3 2 1.0 4 2 3.0 5 3 100.0 6 1 20.0 7 2 1.0 === ======== ====== Let's examine the difference by calculating a "running sum" of the samples, ordered with respect to the ``counter`` and ``value`` fields. To specify the frame type, we can use either: * :attr:`Window.RANGE` * :attr:`Window.ROWS` * :attr:`Window.GROUPS` The behavior of :attr:`~Window.RANGE`, when there are logical duplicates, may lead to unexpected results: .. code-block:: python query = Sample.select( Sample.counter, Sample.value, fn.SUM(Sample.value).over( order_by=[Sample.counter, Sample.value], frame_type=Window.RANGE).alias('rsum')) for sample in query.order_by(Sample.counter, Sample.value): print(sample.counter, sample.value, sample.rsum) # counter value rsum # 1 10. 10. # 1 20. 50. # 1 20. 50. # 2 1. 52. # 2 1. 52. # 2 3. 55. # 3 100 155. With the inclusion of the new rows we now have some rows that have duplicate ``category`` and ``value`` values. The :attr:`~Window.RANGE` frame type causes these duplicates to be evaluated together rather than separately. The more expected result can be achieved by using :attr:`~Window.ROWS` as the frame-type: .. code-block:: python query = Sample.select( Sample.counter, Sample.value, fn.SUM(Sample.value).over( order_by=[Sample.counter, Sample.value], frame_type=Window.ROWS).alias('rsum')) for sample in query.order_by(Sample.counter, Sample.value): print(sample.counter, sample.value, sample.rsum) # counter value rsum # 1 10. 10. # 1 20. 30. # 1 20. 50. # 2 1. 51. # 2 1. 52. # 2 3. 55. # 3 100 155. Peewee uses these rules for determining what frame-type to use: * If the user specifies a ``frame_type``, that frame type will be used. * If ``start`` and/or ``end`` boundaries are specified Peewee will default to using ``ROWS``. * If the user did not specify frame type or start/end boundaries, Peewee will use the database default, which is ``RANGE``. The :attr:`Window.GROUPS` frame type looks at the window range specification in terms of groups of rows, based on the ordering term(s). Using ``GROUPS``, we can define the frame so it covers distinct groupings of rows. Let's look at an example: .. code-block:: python query = (Sample .select(Sample.counter, Sample.value, fn.SUM(Sample.value).over( order_by=[Sample.counter, Sample.value], frame_type=Window.GROUPS, start=Window.preceding(1)).alias('gsum')) .order_by(Sample.counter, Sample.value)) for sample in query: print(sample.counter, sample.value, sample.gsum) # counter value gsum # 1 10 10 # 1 20 50 # 1 20 50 (10) + (20+20) # 2 1 42 # 2 1 42 (20+20) + (1+1) # 2 3 5 (1+1) + 3 # 3 100 103 (3) + 100 As you can hopefully infer, the window is grouped by its ordering term, which is ``(counter, value)``. We are looking at a window that extends between one previous group and the current group. For information about the window function APIs, see: * :meth:`Function.over` * :meth:`Function.filter` * :class:`Window` For general information on window functions, read the postgres `window functions tutorial `_ Additionally, the `postgres docs `_ and the `sqlite docs `_ contain a lot of good information. .. _cte: Common Table Expressions ------------------------ A CTE factors out a subquery and gives it a name, making complex queries more readable and sometimes more efficient. CTEs also support recursion. Define a CTE with :meth:`~SelectQuery.cte` and include it with :meth:`~Query.with_cte`: Simple Example ^^^^^^^^^^^^^^ For an example, let's say we have some data points that consist of a key and a floating-point value. Let's define our model and populate some test data: .. code-block:: python class Sample(Model): key = TextField() value = FloatField() data = ( ('a', (1.25, 1.5, 1.75)), ('b', (2.1, 2.3, 2.5, 2.7, 2.9)), ('c', (3.5, 3.5))) # Populate data. for key, values in data: Sample.insert_many([(key, value) for value in values], fields=[Sample.key, Sample.value]).execute() Let's use a CTE to calculate, for each distinct key, which values were above-average for that key. .. code-block:: python # First we'll declare the query that will be used as a CTE. This query # simply determines the average value for each key. cte = (Sample .select(Sample.key, fn.AVG(Sample.value).alias('avg_value')) .group_by(Sample.key) .cte('key_avgs', columns=('key', 'avg_value'))) # Now we'll query the sample table, using our CTE to find rows whose value # exceeds the average for the given key. We'll calculate how far above the # average the given sample's value is, as well. query = (Sample .select(Sample.key, Sample.value) .join(cte, on=(Sample.key == cte.c.key)) .where(Sample.value > cte.c.avg_value) .order_by(Sample.value) .with_cte(cte)) We can iterate over the samples returned by the query to see which samples had above-average values for their given group: .. code-block:: pycon >>> for sample in query: ... print(sample.key, sample.value) # 'a', 1.75 # 'b', 2.7 # 'b', 2.9 Complex Example ^^^^^^^^^^^^^^^ For a more complete example, let's consider the following query which uses multiple CTEs to find per-product sales totals in only the top sales regions. Our model looks like this: .. code-block:: python class Order(Model): region = TextField() amount = FloatField() product = TextField() quantity = IntegerField() Here is how the query might be written in SQL. This example can be found in the `postgresql documentation `_. .. code-block:: sql WITH regional_sales AS ( SELECT region, SUM(amount) AS total_sales FROM orders GROUP BY region ), top_regions AS ( SELECT region FROM regional_sales WHERE total_sales > (SELECT SUM(total_sales) / 10 FROM regional_sales) ) SELECT region, product, SUM(quantity) AS product_units, SUM(amount) AS product_sales FROM orders WHERE region IN (SELECT region FROM top_regions) GROUP BY region, product; With Peewee, we would write: .. code-block:: python reg_sales = (Order .select(Order.region, fn.SUM(Order.amount).alias('total_sales')) .group_by(Order.region) .cte('regional_sales')) top_regions = (reg_sales .select(reg_sales.c.region) .where(reg_sales.c.total_sales > ( reg_sales.select(fn.SUM(reg_sales.c.total_sales) / 10))) .cte('top_regions')) query = (Order .select(Order.region, Order.product, fn.SUM(Order.quantity).alias('product_units'), fn.SUM(Order.amount).alias('product_sales')) .where(Order.region.in_(top_regions.select(top_regions.c.region))) .group_by(Order.region, Order.product) .with_cte(reg_sales, top_regions)) Recursive CTEs ^^^^^^^^^^^^^^ Peewee supports recursive CTEs. Recursive CTEs can be useful when, for example, you have a tree data-structure represented by a parent-link foreign key. Suppose, for example, that we have a hierarchy of categories for an online bookstore. We wish to generate a table showing all categories and their absolute depths, along with the path from the root to the category. We'll assume the following model definition, in which each category has a foreign-key to its immediate parent category: .. code-block:: python class Category(Model): name = TextField() parent = ForeignKeyField('self', backref='children', null=True) To list all categories along with their depth and parents, we can use a recursive CTE: .. code-block:: python # Define the base case of our recursive CTE. This will be categories that # have a null parent foreign-key. Base = Category.alias() level = Value(1).alias('level') path = Base.name.alias('path') base_case = (Base .select(Base.id, Base.name, Base.parent, level, path) .where(Base.parent.is_null()) .cte('base', recursive=True)) # Define the recursive terms. RTerm = Category.alias() rlevel = (base_case.c.level + 1).alias('level') rpath = base_case.c.path.concat('->').concat(RTerm.name).alias('path') recursive = (RTerm .select(RTerm.id, RTerm.name, RTerm.parent, rlevel, rpath) .join(base_case, on=(RTerm.parent == base_case.c.id))) # The recursive CTE is created by taking the base case and UNION ALL with # the recursive term. cte = base_case.union_all(recursive) # We will now query from the CTE to get the categories, their levels, and # their paths. query = (cte .select_from(cte.c.name, cte.c.level, cte.c.path) .order_by(cte.c.path)) # We can now iterate over a list of all categories and print their names, # absolute levels, and path from root -> category. for category in query: print(category.name, category.level, category.path) # Example output: # root, 1, root # p1, 2, root->p1 # c1-1, 3, root->p1->c1-1 # c1-2, 3, root->p1->c1-2 # p2, 2, root->p2 # c2-1, 3, root->p2->c2-1 Data-Modifying CTE ^^^^^^^^^^^^^^^^^^ Peewee supports data-modifying CTE's. Example of using a data-modifying CTE to move data from one table to an archive table, using a single query: .. code-block:: python class Event(Model): name = CharField() timestamp = DateTimeField() class Archive(Model): name = CharField() timestamp = DateTimeField() # Move rows older than 24 hours from the Event table to the Archive. cte = (Event .delete() .where(Event.timestamp < (datetime.now() - timedelta(days=1))) .returning(Event) .cte('moved_rows')) # Create a simple SELECT to get the resulting rows from the CTE. src = Select((cte,), (cte.c.id, cte.c.name, cte.c.timestamp)) # Insert into the archive table whatever data was returned by the DELETE. res = (Archive .insert_from(src, (Archive.id, Archive.name, Archive.timestamp)) .with_cte(cte) .execute()) The above corresponds to, roughly, the following SQL: .. code-block:: sql WITH "moved_rows" AS ( DELETE FROM "event" WHERE ("timestamp" < XXXX-XX-XXTXX:XX:XX) RETURNING "id", "name", "timestamp") INSERT INTO "archive" ("id", "name", "timestamp") SELECT "moved_rows"."id", "moved_rows"."name", "moved_rows"."timestamp" FROM "moved_rows"; For additional examples, refer to the tests in ``models.py`` and ``sql.py``: * https://github.com/coleifer/peewee/blob/master/tests/models.py * https://github.com/coleifer/peewee/blob/master/tests/sql.py ================================================ FILE: docs/peewee/quickstart.rst ================================================ .. _quickstart: Quickstart ========== This guide walks through defining a schema, writing rows, and reading them back. It takes about ten minutes. Every concept introduced here is covered in depth in the following documents. .. tip:: Follow along in an interactive Python session. Model Definition ----------------- A Peewee application starts with a :class:`Database` object and one or more :class:`Model` classes. The database object manages connections; model classes map to tables. .. code-block:: python import datetime from peewee import * # An in-memory SQLite database. Or use PostgresqlDatabase or MySQLDatabase. db = SqliteDatabase(':memory:') class BaseModel(Model): """All models inherit this to share the database connection.""" class Meta: database = db class User(BaseModel): username = TextField(unique=True) class Tweet(BaseModel): user = ForeignKeyField(User, backref='tweets') content = TextField() timestamp = DateTimeField( default=datetime.datetime.now, index=True) Three things to notice: * ``BaseModel`` exists only to carry the ``database`` setting. Every subclass inherits it automatically. * Peewee adds an auto-incrementing integer ``id`` primary key to any model that does not declare its own. * ``ForeignKeyField`` links ``Tweet`` to ``User``. The ``backref='tweets'`` parameter means every ``User`` instance gains a ``tweets`` attribute. Create the Tables ----------------- .. code-block:: python db.connect() db.create_tables([User, Tweet]) :meth:`~Database.create_tables` generates ``CREATE TABLE`` statements for each model. By default ``create_table()`` specifies ``safe=True``, which uses ``CREATE TABLE IF NOT EXISTS``, making it safe to call on every startup. Writing Data ------------ Create a row with :meth:`~Model.create` (one step) or instantiate a model and call :meth:`~Model.save` (two steps): .. code-block:: python # One-step creation - returns the saved instance. charlie = User.create(username='charlie') huey = User.create(username='huey') # Two-step creation. t = Tweet(user=charlie, content='Hello, world!') t.save() Tweet.create(user=charlie, content='My second tweet.') Tweet.create(user=huey, content='meow') To update an existing row, modify attributes and call ``save()`` again: .. code-block:: python charlie.username = 'charlie_admin' charlie.save() To delete a row: .. code-block:: python stale_tweet = Tweet.get(Tweet.content == 'My second tweet.') stale_tweet.delete_instance() Reading Data ------------ Retrieve a single row with :meth:`~Model.get`. It raises :exc:`~Model.DoesNotExist` if no match is found: .. code-block:: python user = User.get(User.username == 'charlie_admin') print(user.id, user.username) Retrieve multiple rows with :meth:`~Model.select`. The result is a lazy query - rows are fetched only when you iterate: .. code-block:: python for tweet in Tweet.select(): print(tweet.content) Filter with :meth:`~Query.where`: .. code-block:: python for tweet in Tweet.select().where(Tweet.user == charlie): print(tweet.content) for tweet in Tweet.select().where(Tweet.timestamp.year == 2026): print(tweet.content) Sort with :meth:`~Query.order_by`: .. code-block:: python for tweet in Tweet.select().order_by(Tweet.timestamp.desc()): print(tweet.timestamp, tweet.content) Join to combine data from related tables in a single query: .. code-block:: python # Fetch each tweet alongside its author's username. # Without the join, accessing tweet.user.username would issue # an extra query per tweet - see the N+1 section in Relationships. query = (Tweet .select(Tweet, User) .join(User) .order_by(Tweet.timestamp.desc())) for tweet in query: print(tweet.user.username, '->', tweet.content) Simple Aggregates ----------------- How many tweets are in the database: .. code-block:: python Tweet.select().count() When the most-recent tweet was added: .. code-block:: python Tweet.select(fn.MAX(Tweet.timestamp)).scalar() Close the Connection -------------------- When done using the database, close the connection: .. code-block:: python db.close() In a web application you would open the connection when a request arrives and close it when the response is sent. See :ref:`framework-integration` for framework-specific patterns. Working with Existing databases ------------------------------- If you have an existing database, peewee can generate models using :ref:`pwiz`. For example to generate models for a Postgres database named ``blog_db``: .. code-block:: shell python -m pwiz -e postgresql blog > blog_models.py What Next --------- Each concept introduced above is covered in full detail in the following documents: * :ref:`database` - connection options, multiple backends, run-time configuration, connection pooling. * :ref:`models` - field types, field parameters, model Meta options, indexes, primary keys. * :ref:`relationships` - how foreign keys work at runtime, joins, the N+1 problem, many-to-many relationships. * :ref:`querying` - the full SELECT API: filtering, sorting, aggregates, window functions, CTEs. * :ref:`writing` - INSERT, UPDATE, DELETE, bulk operations, upsert. * :ref:`transactions` - atomic blocks, nesting, savepoints. For a complete worked example building a small web application, see :ref:`example`. ================================================ FILE: docs/peewee/recipes.rst ================================================ .. _recipes: Recipes ======= Collected patterns for common real-world problems. Each recipe assumes familiarity with :ref:`querying`, :ref:`writing`, and :ref:`relationships`. All examples use the following models: .. code-block:: python import datetime from peewee import * db = SqliteDatabase(':memory:') class BaseModel(Model): class Meta: database = db class User(BaseModel): username = TextField(unique=True) class Tweet(BaseModel): user = ForeignKeyField(User, backref='tweets') content = TextField() timestamp = DateTimeField(default=datetime.datetime.now) created_date = DateTimeField(default=datetime.datetime.now) .. _optimistic-locking: Optimistic Locking ------------------ *Optimistic locking* avoids holding a database lock across the read-modify-write cycle by recording a version number on each row. On write, the update is conditional on the version not having changed. If another process modified the row in the meantime, the update matches zero rows and the conflict is detected in application code. This is a lighter-weight alternative to ``SELECT FOR UPDATE`` (Postgresql) or ``BEGIN IMMEDIATE`` (SQLite) when lock contention is expected to be low. A reusable base class: .. code-block:: python class ConflictDetectedException(Exception): pass class BaseVersionedModel(BaseModel): version = IntegerField(default=1, index=True) def save_optimistic(self): if not self.id: # This is a new record, so the default logic is to perform an # INSERT. Ideally your model would also have a unique # constraint that made it impossible for two INSERTs to happen # at the same time. return self.save() # Update any data that has changed and bump the version counter. field_data = dict(self.__data__) current_version = field_data.pop('version', 1) self._populate_unsaved_relations(field_data) field_data = self._prune_fields(field_data, self.dirty_fields) if not field_data: raise ValueError('No changes have been made.') ModelClass = type(self) field_data['version'] = ModelClass.version + 1 # Atomic increment. updated = (ModelClass .update(**field_data) .where( (ModelClass.version == current_version) & (ModelClass.id == self.id)) .execute()) if updated == 0: # No rows were updated, indicating another process has saved # a new version. raise ConflictDetectedException() else: # Increment local version to match what is now in the db. self.version += 1 return True Usage: .. code-block:: pycon class UserProfile(BaseVersionedModel): username = TextField(unique=True) bio = TextField(default='') >>> u = UserProfile(username='charlie') >>> u.save_optimistic() True >>> u.bio = 'Python developer' >>> u.save_optimistic() True >>> u.version 2 # Simulate a concurrent modification: >>> u2 = UserProfile.get(UserProfile.username == 'charlie') >>> u2.bio = 'Changed by another process' >>> u2.save_optimistic() True # The original instance's version is now stale: >>> u.bio = 'My update' >>> u.save_optimistic() ConflictDetectedException Get-or-Create Safely --------------------- :meth:`~Model.get_or_create` is convenient but has a small race window between the SELECT and the INSERT when the row does not yet exist. Two concurrent processes can both fail the SELECT and both attempt the INSERT, causing one to fail with an ``IntegrityError``. The safe pattern attempts the INSERT first and falls back to a GET on ``IntegrityError``: .. code-block:: python def get_or_create_user(username): try: with db.atomic(): return User.create(username=username), True except IntegrityError: return User.get(User.username == username), False user, created = get_or_create_user('charlie') The ``db.atomic()`` wrapper is important: it ensures that the rollback on ``IntegrityError`` affects only this operation, not any surrounding transaction. .. _top-item-per-group: Top Item Per Group ------------------ These examples find the single most recent tweet for each user. See :ref:`top-n-per-group` below for the generalized N-per-group problem. The most portable approach uses a ``MAX()`` aggregate in a non-correlated subquery, then joins back to the tweet table on both user and timestamp: .. code-block:: python # When referencing a table multiple times, we'll call Model.alias() to create # a secondary reference to the table. TweetAlias = Tweet.alias() # Create a subquery that will calculate the maximum Tweet created_date for # each user. subquery = (TweetAlias .select( TweetAlias.user, fn.MAX(TweetAlias.created_date).alias('max_ts')) .group_by(TweetAlias.user) .alias('tweet_max')) # Query for tweets and join using the subquery to match the tweet's user # and created_date. query = (Tweet .select(Tweet, User) .join(User) .switch(Tweet) .join(subquery, on=( (Tweet.created_date == subquery.c.max_ts) & (Tweet.user == subquery.c.user_id)))) SQLite and MySQL permit a shorter form that groups by a subset of selected columns: .. code-block:: python query = (Tweet .select(Tweet, User) .join(User) .group_by(Tweet.user) .having(Tweet.created_date == fn.MAX(Tweet.created_date))) Postgresql requires the standard subquery form above. .. _top-n-per-group: Top N Per Group --------------- These examples describe several ways to query the top *N* items per group reasonably efficiently. For a thorough discussion of various techniques, check out my blog post `Querying the top N objects per group with Peewee ORM `_. Window functions ^^^^^^^^^^^^^^^^ A ``RANK()`` window function is the cleanest solution. Rank tweets per user by timestamp (newest first), then filter the outer query to the top N ranks: .. code-block:: python TweetAlias = Tweet.alias() ranked = (TweetAlias .select( TweetAlias.content, User.username, fn.RANK().over( partition_by=[TweetAlias.user], order_by=[TweetAlias.created_date.desc()] ).alias('rnk')) .join(User, on=(TweetAlias.user == User.id)) .alias('subq')) query = (Tweet .select(ranked.c.content, ranked.c.username) .from_(ranked) .where(ranked.c.rnk <= 3)) Postgresql - lateral joins ^^^^^^^^^^^^^^^^^^^^^^^^^^^ A ``LATERAL`` join executes a correlated subquery once per row of the driving table. For each user, it selects the three most recent tweets. The desired SQL is: .. code-block:: sql SELECT * FROM (SELECT id, username FROM user) AS uq LEFT JOIN LATERAL (SELECT message, created_date FROM tweet WHERE (user_id = uq.id) ORDER BY created_date DESC LIMIT 3) AS pq ON true To accomplish this with peewee is quite straightforward: .. code-block:: python subq = (Tweet .select(Tweet.message, Tweet.created_date) .where(Tweet.user == User.id) .order_by(Tweet.created_date.desc()) .limit(3)) query = (User .select(User, subq.c.content, subq.c.created_date) .join(subq, JOIN.LEFT_LATERAL) .order_by(User.username, subq.c.created_date.desc())) # We queried from the "perspective" of user, so the rows are User instances # with the addition of a "content" and "created_date" attribute for each of # the (up-to) 3 most-recent tweets for each user. for row in query: print(row.username, row.content, row.created_date) To implement an equivalent query from the "perspective" of the Tweet model, we can instead write: .. code-block:: python # subq is the same as the above example. subq = (Tweet .select(Tweet.message, Tweet.created_date) .where(Tweet.user == User.id) .order_by(Tweet.created_date.desc()) .limit(3)) query = (Tweet .select(User.username, subq.c.content, subq.c.created_date) .from_(User) .join(subq, JOIN.LEFT_LATERAL) .order_by(User.username, subq.c.created_date.desc())) # Each row is a "tweet" instance with an additional "username" attribute. # This will print the (up-to) 3 most-recent tweets from each user. for tweet in query: print(tweet.username, tweet.content, tweet.created_date) Correlated subquery count ^^^^^^^^^^^^^^^^^^^^^^^^^ A correlated subquery that counts tweets newer than the current row can also be used. Rows where fewer than N newer tweets exist are in the top N: .. code-block:: python TweetAlias = Tweet.alias() # Create a correlated subquery that calculates the number of # tweets with a higher (newer) timestamp than the tweet we're # looking at in the outer query. subquery = (TweetAlias .select(fn.COUNT(TweetAlias.id)) .where( (TweetAlias.created_date >= Tweet.created_date) & (TweetAlias.user == Tweet.user))) # Wrap the subquery and filter on the count. query = (Tweet .select(Tweet, User) .join(User) .where(subquery <= 3)) SQLite and MySQL - self-join ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ An alternative: self-join and count newer tweets in the HAVING clause: .. code-block:: python TweetAlias = Tweet.alias() query = (Tweet .select(Tweet.id, Tweet.content, Tweet.user, User.username) .join_from(Tweet, User) .join_from(Tweet, TweetAlias, on=( (TweetAlias.user == Tweet.user) & (TweetAlias.created_date >= Tweet.created_date))) .group_by(Tweet.id, Tweet.content, Tweet.user, User.username) .having(fn.COUNT(Tweet.id) <= 3)) The last example uses a ``LIMIT`` clause in a correlated subquery. .. code-block:: python TweetAlias = Tweet.alias() # The subquery here will calculate, for the user who created the # tweet in the outer loop, the three newest tweets. The expression # will evaluate to `True` if the outer-loop tweet is in the set of # tweets represented by the inner query. query = (Tweet .select(Tweet, User) .join(User) .where(Tweet.id << ( TweetAlias .select(TweetAlias.id) .where(TweetAlias.user == Tweet.user) .order_by(TweetAlias.created_date.desc()) .limit(3)))) For a thorough benchmark comparison of these approaches, see the blog post `Querying the top N objects per group with Peewee ORM `_. Bulk-Loading with Explicit Primary Keys ----------------------------------------- When loading relational data from an external source where primary keys are already assigned, use :meth:`~Model.insert_many` with the ``id`` field included. This avoids the ``auto_increment`` workaround that was common in older Peewee versions: .. code-block:: python data = [(1, 'alice'), (2, 'bob'), (3, 'carol')] fields = [User.id, User.username] with db.atomic(): User.insert_many(data, fields=fields).execute() Because ``insert_many`` never reads rows back, there is no confusion between INSERT and UPDATE paths. Custom SQLite Functions ----------------------- SQLite can be extended with Python functions that are callable from SQL. This is useful for operations SQLite does not natively support. Registering a function with the ``@db.func()`` decorator makes it available immediately after the connection is opened: .. code-block:: python from hashlib import sha256 import os db = SqliteDatabase('my_app.db') def _hash_password(salt, password): return sha256((salt + password).encode()).hexdigest() @db.func() def make_password(raw_password): salt = os.urandom(8).hex() return salt + '$' + _hash_password(salt, raw_password) @db.func() def check_password(raw_password, stored): salt, hsh = stored.split('$', 1) return hsh == _hash_password(salt, raw_password) Store a hashed password: .. code-block:: python User.insert(username='charlie', password=fn.make_password('s3cr3t')).execute() Verify a password at login: .. code-block:: python def login(username, raw_password): try: return (User .select() .where( (User.username == username) & (fn.check_password(raw_password, User.password) == True)) .get()) except User.DoesNotExist: return None .. seealso:: :meth:`SqliteDatabase.func`, :meth:`SqliteDatabase.aggregate`, :meth:`SqliteDatabase.window_function`. Date Arithmetic Across Databases ---------------------------------- Each database implements date arithmetic differently. This section shows how to express "next occurrence of a scheduled task" - defined as ``last_run + interval_seconds`` - for each backend. The schema: .. code-block:: python class Schedule(BaseModel): interval = IntegerField() # Repeat every N seconds. class Task(BaseModel): schedule = ForeignKeyField(Schedule, backref='tasks') command = TextField() last_run = DateTimeField() We want: tasks where ``now >= last_run + interval``. Our desired code would look like: .. code-block:: python next_occurrence = something # ??? how do we define this ??? # We can express the current time as a Python datetime value, or we could # alternatively use the appropriate SQL function/name. now = Value(datetime.datetime.now()) # Or SQL('current_timestamp'), e.g. query = (Task .select(Task, Schedule) .join(Schedule) .where(now >= next_occurrence)) **Postgresql** - multiply a typed interval: .. code-block:: python one_second = SQL("INTERVAL '1 second'") next_run = Task.last_run + (Schedule.interval * one_second) now = Value(datetime.datetime.now()) tasks_due = (Task .select(Task, Schedule) .join(Schedule) .where(now >= next_run)) **MySQL** - use ``DATE_ADD`` with a dynamic INTERVAL expression: .. code-block:: python from peewee import NodeList interval = NodeList((SQL('INTERVAL'), Schedule.interval, SQL('SECOND'))) next_run = fn.DATE_ADD(Task.last_run, interval) now = Value(datetime.datetime.now()) tasks_due = (Task .select(Task, Schedule) .join(Schedule) .where(now >= next_run)) **SQLite** - convert to Unix timestamp, add seconds, convert back: .. code-block:: python next_ts = fn.strftime('%s', Task.last_run) + Schedule.interval next_run = fn.datetime(next_ts, 'unixepoch') now = Value(datetime.datetime.now()) tasks_due = (Task .select(Task, Schedule) .join(Schedule) .where(now >= next_run)) ================================================ FILE: docs/peewee/relationships.rst ================================================ .. _relationships: Relationships and Joins ======================= Relational databases derive most of their power from the ability to link rows across tables. This document explains how Peewee models those links, what happens under the hood when you traverse them, and how to write queries that cross table boundaries efficiently. By the end of this document you will understand: * How :class:`ForeignKeyField` behaves at runtime, not just at schema definition time. * What a back-reference is and when to use one. * What the N+1 problem is and how to recognise it. * How to write joins, including multi-table and self-referential joins. * How many-to-many relationships are modelled. * When to use :func:`prefetch` instead of a join. Model Definitions ----------------- All examples in this document use the following three models. They will be defined once here and reused throughout. .. code-block:: python import datetime from peewee import * db = SqliteDatabase(':memory:') class BaseModel(Model): class Meta: database = db class User(BaseModel): username = TextField() class Tweet(BaseModel): user = ForeignKeyField(User, backref='tweets') content = TextField() timestamp = DateTimeField(default=datetime.datetime.now) class Favorite(BaseModel): user = ForeignKeyField(User, backref='favorites') tweet = ForeignKeyField(Tweet, backref='favorites') A :class:`ForeignKeyField` links one model to another. ``Tweet.user`` links each tweet to the user who wrote it. ``Favorite.user`` and ``Favorite.tweet`` together record which users have favorited which tweets. The following helper populates test data that the examples below will query: .. code-block:: python def create_test_data(): db.create_tables([User, Tweet, Favorite]) users = { name: User.create(username=name) for name in ('huey', 'mickey', 'zaizee') } tweet_data = { 'huey': ('meow', 'hiss', 'purr'), 'mickey': ('woof', 'whine'), 'zaizee': (), } tweets = {} for username, contents in tweet_data.items(): for content in contents: tweets[content] = Tweet.create( user=users[username], content=content) # huey favorites mickey's "whine", # mickey favorites huey's "purr", # zaizee favorites huey's "meow" and "purr". favorite_data = ( ('huey', ['whine']), ('mickey', ['purr']), ('zaizee', ['meow', 'purr']), ) for username, contents in favorite_data: for content in contents: Favorite.create(user=users[username], tweet=tweets[content]) This gives the following data: ========= ============= ================== User Tweet Favorited by ========= ============= ================== huey meow zaizee huey hiss huey purr mickey, zaizee mickey woof mickey whine huey zaizee (no tweets) ========= ============= ================== .. note:: To log every query Peewee executes to the console - useful for verifying query counts while working through this document - add the following before running any queries: .. code-block:: python import logging logging.getLogger('peewee').addHandler(logging.StreamHandler()) logging.getLogger('peewee').setLevel(logging.DEBUG) .. _foreign-keys: Foreign Keys ------------ When you declare a :class:`ForeignKeyField`, Peewee creates two things on the model: a field that stores the raw integer ID value, and a descriptor that resolves that ID into a full model instance on access. .. code-block:: python tweet = Tweet.get(Tweet.content == 'meow') # Accessing .user resolves the foreign key - Peewee issues a SELECT # query to fetch the related User row. print(tweet.user.username) # 'huey' # Accessing .user_id returns the raw integer stored in the column, # without issuing any query. print(tweet.user_id) # 1 The ``_id`` suffix accessor is available for every foreign key field. Use it whenever only the ID value is needed, since it avoids the extra query entirely. Lazy loading ^^^^^^^^^^^^ By default, a :class:`ForeignKeyField` is *lazy-loaded*: the related object is not fetched until the attribute is first accessed, at which point a ``SELECT`` query is issued automatically. This is convenient but can lead to performance problems - see :ref:`nplusone` below. To disable lazy loading on a specific field, pass ``lazy_load=False``. With lazy loading disabled, accessing the attribute returns the raw ID value rather than issuing a query, matching the behaviour of the ``_id`` accessor: .. code-block:: python class Tweet(BaseModel): user = ForeignKeyField(User, backref='tweets', lazy_load=False) for tweet in Tweet.select(): # Returns the integer ID, not a User instance. No extra query. print(tweet.user) # If the User data was eagerly loaded via a join, the full User # instance is accessible as normal, even with lazy_load=False. for tweet in Tweet.select(Tweet, User).join(User): print(tweet.user.username) .. seealso:: :ref:`nplusone` explains when and why disabling lazy loading is useful. .. _backreferences: Back-references --------------- Every :class:`ForeignKeyField` automatically creates a *back-reference* on the related model. The back-reference is a pre-filtered :class:`Select` query that returns all rows pointing at a given instance. In the example schema, ``Tweet.user`` is a foreign key to ``User``. The ``backref='tweets'`` parameter means that every ``User`` instance gains a ``tweets`` attribute, which is a pre-filtered :class:`Select` query: .. code-block:: pycon >>> huey = User.get(User.username == 'huey') >>> huey.tweets # back-reference is a Select query. >>> for tweet in huey.tweets: ... print(tweet.content) meow hiss purr Taking a closer look at ``huey.tweets``, we can see that it is just a simple pre-filtered ``SELECT`` query: .. code-block:: pycon >>> huey.tweets >>> huey.tweets.sql() ('SELECT "t1"."id", "t1"."content", "t1"."timestamp", "t1"."user_id" FROM "tweet" AS "t1" WHERE ("t1"."user_id" = ?)', [1]) A back-reference behaves like any other :class:`Select` query and can be filtered, ordered, and chained: .. code-block:: python recent = (huey.tweets .order_by(Tweet.timestamp.desc()) .limit(2)) If no ``backref`` name is specified, Peewee generates one automatically using the pattern ``_set``. Specifying an explicit ``backref`` is recommended for clarity. .. _nplusone: The N+1 Problem --------------- The *N+1 problem* occurs when code issues one query to fetch a list of N rows, then issues one or more additional queries *per row* to fetch related data - N+1 queries in total instead of one or two. At small scale this is invisible, but at production scale it can make pages that should take milliseconds take seconds. Consider printing every tweet alongside its author's username: .. code-block:: python # Bad: issues 1 query for tweets + 1 query per tweet for the user. for tweet in Tweet.select(): print(tweet.user.username, '->', tweet.content) # Good: only one query is needed. query = (Tweet .select(Tweet, User) .join(User)) for tweet in query: # tweet.user is a User instance populated from the joined data. # No additional query is issued. print(tweet.user.username, '->', tweet.content) Without joining and selecting the related User, each access to ``tweet.user`` triggers a ``SELECT`` on the ``user`` table. With five tweets, this produces six queries. With five thousand tweets, it produces five thousand and one. The same problem can occur when iterating over back-references: .. code-block:: python # Bad: issues 1 query for users + 1 query per user for their tweets. for user in User.select(): print(user.username) for tweet in user.tweets: # A new query for each user. print(' ', tweet.content) # Better: for user in User.select().prefetch(Tweet): print(user.username) for tweet in user.tweets: # Pre-fetched, no additional query. print(' ', tweet.content) Peewee provides two complementary tools for avoiding N+1 queries: * **Joins** - combine rows from multiple tables in a single ``SELECT``. Best when traversing a foreign key *toward* its target (many-to-one direction), for example fetching tweets with their authors. * **Prefetch** - issue one query per table and stitch the results together in Python. Best when traversing a back-reference (one-to-many direction), for example fetching users with all their tweets. Both are covered in the sections below. The choice between them depends on the shape of the query. .. _joins: Joins ----- A SQL join combines columns from two or more tables into a single result set. Peewee's :meth:`~ModelSelect.join` method generates the appropriate ``JOIN`` clause and, when the full result is returned as model instances, reconstructs the model graph automatically. Join context ^^^^^^^^^^^^ Peewee tracks a *join context*: the model from which the next ``join()`` call will depart. At the start of a query the join context is the model being selected from. Each call to ``join()`` moves the join context to the model just joined. .. code-block:: python # Context starts at Tweet. # After .join(User), context moves to User. query = Tweet.select().join(User) When joining through multiple tables in a chain, this is usually what you want. When joining from one model to two different models, the join context needs to be reset explicitly using :meth:`~ModelSelect.switch` or :meth:`~ModelSelect.join_from`. Peewee infers the join predicate (the ``ON`` clause) from the foreign keys defined on the models. If only one foreign key exists between two models, no additional specification is required. If multiple foreign keys exist, the relevant one must be specified explicitly. The following code is equivalent to the prevoius example: .. code-block:: python :emphasize-lines: 3 query = (Tweet .select() .join(User, on=(Tweet.user == User.id)) .where(User.username == 'huey')) Simple joins ^^^^^^^^^^^^ To fetch all of huey's tweets, join from ``Tweet`` to ``User`` and filter on the username: .. code-block:: python query = (Tweet .select() .join(User) .where(User.username == 'huey')) for tweet in query: print(tweet.content) Peewee inferred the join predicate since ``Tweet.user`` is the only key between the two models. To explicitly specify the join predicate use ``on=``: .. code-block:: python query = (Tweet .select() .join(User, on=(Tweet.user == User.id)) .where(User.username == 'huey')) If a ``User`` instance is already available, the back-reference is simpler and equivalent for straightforward cases: .. code-block:: python huey = User.get(User.username == 'huey') for tweet in huey.tweets: print(tweet.content) The join is the better choice when filtering or joining further. The back-reference is more readable for simple access to related rows. Joining across multiple tables ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ To count how many favorites each user has received across all their tweets, a join must traverse ``User -> Tweet -> Favorite``. Because each join moves the context forward, this chain can be written directly: .. code-block:: python # Context: User -> join -> Tweet -> join -> Favorite query = (User .select(User.username, fn.COUNT(Favorite.id).alias('fav_count')) .join(Tweet, JOIN.LEFT_OUTER) .join(Favorite, JOIN.LEFT_OUTER) .group_by(User.username)) for user in query: print(f'{user.username}: {user.fav_count} favorites received') Both joins use ``LEFT OUTER`` because a user may have no tweets, and a tweet may have no favorites - yet both should appear in the result with a count of zero. Switching join context ^^^^^^^^^^^^^^^^^^^^^^ When a query needs to branch - joining from one model to two different models - the join context must be reset manually using :meth:`~ModelSelect.switch`. To find all tweets by huey and how many times each has been favorited: .. code-block:: python # Context: Tweet -> join -> User (context is now User) # switch(Tweet) resets context to Tweet # -> join -> Favorite (context is now Favorite) query = (Tweet .select(Tweet.content, fn.COUNT(Favorite.id).alias('fav_count')) .join(User) .switch(Tweet) .join(Favorite, JOIN.LEFT_OUTER) .where(User.username == 'huey') .group_by(Tweet.content)) for tweet in query: print(f'{tweet.content}: favorited {tweet.fav_count} times') Without the call to ``.switch(Tweet)``, Peewee would attempt to join from ``User`` to ``Favorite`` using ``Favorite.user``, which would produce incorrect results. Using ``join_from`` ^^^^^^^^^^^^^^^^^^^ :meth:`~ModelSelect.join_from` is an alternative to ``switch().join()`` that makes the join source explicit in a single call. The above query can be written equivalently as: .. code-block:: python query = (Tweet .select(Tweet.content, fn.COUNT(Favorite.id).alias('fav_count')) .join_from(Tweet, User) .join_from(Tweet, Favorite, JOIN.LEFT_OUTER) .where(User.username == 'huey') .group_by(Tweet.content)) ``join_from(A, B)`` is equivalent to ``switch(A).join(B)`` and is often more readable when a query branches across several paths. Selecting columns from joined models ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When columns from multiple models are included in ``select()``, Peewee reconstructs the model graph and assigns related model instances to their corresponding attributes. .. code-block:: python query = (Tweet .select(Tweet.content, User.username) .join(User)) for tweet in query: # tweet.user is a User instance populated from the joined data. # No additional query is issued. print(tweet.user.username, '->', tweet.content) # huey -> meow # huey -> hiss # huey -> purr # mickey -> woof # mickey -> whine To make it a bit more obvious that it's doing the correct thing, we can ask Peewee to return the rows as dictionaries. .. code-block:: python query = (Tweet .select(Tweet.content, User.username) .join(User) .dicts()) for row in query: print(row) # {'content': 'meow', 'username': 'huey'} # {'content': 'hiss', 'username': 'huey'} # {'content': 'purr', 'username': 'huey'} # {'content': 'woof', 'username': 'mickey'} # {'content': 'whine', 'username': 'mickey'} Compare these queries to the N+1 version: here, only one query is executed regardless of how many tweets are returned. The attribute name that Peewee uses to store the joined instance follows the foreign key field name (``tweet.user`` in this case). To override it, pass ``attr`` to ``join()``: .. code-block:: python query = (Tweet .select(Tweet.content, User.username) .join(User, attr='author')) for tweet in query: print(tweet.author.username, '->', tweet.content) To flatten all selected columns onto the primary model instance rather than nesting them in a sub-object, append ``.objects()``: .. code-block:: python query = (Tweet .select(Tweet.content, User.username) .join(User) .objects()) for tweet in query: # username is now an attribute on tweet directly. print(tweet.username, '->', tweet.content) # huey -> meow See :ref:`row-types` for the different ways Peewee can return rows. More complex example ^^^^^^^^^^^^^^^^^^^^ As a more complex example, in this query, we will write a single query that selects all the favorites, along with the user who created the favorite, the tweet that was favorited, and that tweet's author. In SQL we would write: .. code-block:: sql SELECT owner.username, tweet.content, author.username AS author FROM favorite INNER JOIN user AS owner ON (favorite.user_id = owner.id) INNER JOIN tweet ON (favorite.tweet_id = tweet.id) INNER JOIN user AS author ON (tweet.user_id = author.id); Note that we are selecting from the user table twice - once in the context of the user who created the favorite, and again as the author of the tweet. With Peewee, we use :meth:`Model.alias` to alias a model class so it can be referenced twice in a single query: .. code-block:: python Owner = User.alias() query = (Favorite .select(Favorite, Tweet.content, User.username, Owner.username) .join_from(Favorite, Owner) # Determine owner of favorite. .join_from(Favorite, Tweet) # Join favorite -> tweet. .join_from(Tweet, User)) # Join tweet -> user. We can iterate over the results and access the joined values in the following way. Note how Peewee has resolved the fields from the various models we selected and reconstructed the model graph: .. code-block:: python for fav in query: print(fav.user.username, 'liked', fav.tweet.content, 'by', fav.tweet.user.username) # huey liked whine by mickey # mickey liked purr by huey # zaizee liked meow by huey # zaizee liked purr by huey .. _join-subquery: Subqueries ^^^^^^^^^^ Peewee allows you to join on any table-like object, including subqueries or common table expressions (see :ref:`cte`). To demonstrate joining on a subquery, let's query for all users and their latest tweet. Here is the SQL: .. code-block:: sql SELECT tweet.*, user.* FROM tweet INNER JOIN ( SELECT latest.user_id, MAX(latest.timestamp) AS max_ts FROM tweet AS latest GROUP BY latest.user_id) AS latest_query ON ((tweet.user_id = latest_query.user_id) AND (tweet.timestamp = latest_query.max_ts)) INNER JOIN user ON (tweet.user_id = user.id) We'll do this by creating a subquery which selects each user and the timestamp of their latest tweet. Then we can query the tweets table in the outer query and join on the user and timestamp combination from the subquery. .. code-block:: python # Define our subquery first. We'll use an alias of the Tweet model, since # we will be querying from the Tweet model directly in the outer query. Latest = Tweet.alias() latest_query = (Latest .select(Latest.user, fn.MAX(Latest.timestamp).alias('max_ts')) .group_by(Latest.user) .alias('latest_query')) # Our join predicate will ensure that we match tweets based on their # timestamp *and* user_id. predicate = ((Tweet.user == latest_query.c.user_id) & (Tweet.timestamp == latest_query.c.max_ts)) # We put it all together, querying from tweet and joining on the subquery # using the above predicate. query = (Tweet .select(Tweet, User) # Select all columns from tweet and user. .join_from(Tweet, latest_query, on=predicate) # Join tweet -> subquery. .join_from(Tweet, User)) # Join from tweet -> user. Iterating over the query, we can see each user and their latest tweet. .. code-block:: python for tweet in query: print(tweet.user.username, '->', tweet.content) # huey -> purr # mickey -> whine There are a couple things you may not have seen before in the code we used to create the query in this section: * We used :meth:`~ModelSelect.join_from` to explicitly specify the join context. We wrote ``.join_from(Tweet, User)``, which is equivalent to ``.switch(Tweet).join(User)``. * We referenced columns in the subquery using the magic ``.c`` attribute, for example ``latest_query.c.max_ts``. The ``.c`` attribute is used to dynamically create column references. * Instead of passing individual fields to ``Tweet.select()``, we passed the ``Tweet`` and ``User`` models. This is shorthand for selecting all fields on the given model. Common-table Expressions ^^^^^^^^^^^^^^^^^^^^^^^^ In the previous section we joined on a subquery, but we could just as easily have used a :ref:`common-table expression (CTE) `. We will repeat the same query as before, listing users and their latest tweets, but this time we will do it using a CTE. Here is the SQL: .. code-block:: sql WITH latest AS ( SELECT user_id, MAX(timestamp) AS max_ts FROM tweet GROUP BY user_id) SELECT tweet.*, user.* FROM tweet INNER JOIN latest ON ((latest.user_id = tweet.user_id) AND (latest.max_ts = tweet.timestamp)) INNER JOIN user ON (tweet.user_id = user.id) This example looks very similar to the previous example with the subquery: .. code-block:: python # Define our CTE first. We'll use an alias of the Tweet model, since # we will be querying from the Tweet model directly in the main query. Latest = Tweet.alias() cte = (Latest .select(Latest.user, fn.MAX(Latest.timestamp).alias('max_ts')) .group_by(Latest.user) .cte('latest')) # Our join predicate will ensure that we match tweets based on their # timestamp *and* user_id. predicate = ((Tweet.user == cte.c.user_id) & (Tweet.timestamp == cte.c.max_ts)) # We put it all together, querying from tweet and joining on the CTE # using the above predicate. query = (Tweet .select(Tweet, User) # Select all columns from tweet and user. .join(cte, on=predicate) # Join tweet -> CTE. .join_from(Tweet, User) # Join from tweet -> user. .with_cte(cte)) We can iterate over the result-set, which consists of the latest tweets for each user: .. code-block:: python for tweet in query: print(tweet.user.username, '->', tweet.content) # huey -> purr # mickey -> whine .. note:: For more information about using CTEs, including information on writing recursive CTEs, see the :ref:`cte` section of the "Querying" document. Multiple foreign keys to the same model ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When two foreign keys on the same model both point at the same target model, Peewee cannot infer which one to use for a join. The field must be specified explicitly. Consider a ``Relationship`` model recording which users follow which other users: .. code-block:: python class Relationship(BaseModel): from_user = ForeignKeyField(User, backref='following') to_user = ForeignKeyField(User, backref='followers') class Meta: indexes = ((('from_user', 'to_user'), True),) To find everyone that ``huey`` follows: .. code-block:: python huey = User.get(User.username == 'huey') following = (User .select() .join(Relationship, on=Relationship.to_user) .where(Relationship.from_user == huey)) To find everyone who follows ``huey``: .. code-block:: python followers = (User .select() .join(Relationship, on=Relationship.from_user) .where(Relationship.to_user == huey)) Passing the field instance to ``on=`` tells Peewee which foreign key column to use for the join. Joining without a foreign key ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A join can be performed on any two tables, even when no :class:`ForeignKeyField` exists between them, by supplying an explicit join predicate as an expression: .. code-block:: python query = (User .select(User, ActivityLog) .join(ActivityLog, on=(User.id == ActivityLog.object_id), attr='log') .where( (ActivityLog.activity_type == 'login') & (User.username == 'huey'))) for user in query: print(user.username, '->', user.log.description) Self-joins ^^^^^^^^^^ A self-join queries a model against an alias of itself. Use :meth:`Model.alias` to create the alias: .. code-block:: python # Find all categories and their immediate parent name. class Category(BaseModel): name = TextField() parent = ForeignKeyField('self', null=True, backref='children') Parent = Category.alias() query = (Category .select(Category.name, Parent.name) .join(Parent, JOIN.LEFT_OUTER, on=(Category.parent == Parent.id)) .order_by(Category.name)) for row in query: print(row.name, 'parent:', row.parent.name if row.parent else 'None') .. seealso:: Recursive queries over self-referential structures are covered in :ref:`cte` using recursive CTEs. .. _manytomany: Many-to-Many Relationships -------------------------- A many-to-many relationship - where one row in table A can relate to many rows in table B *and vice versa* - requires an intermediate *through table* that holds pairs of foreign keys. Manual through table (recommended) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The explicit approach gives full control over the through table and its queries: .. code-block:: python class Student(BaseModel): name = TextField() class Course(BaseModel): title = TextField() class Enrollment(BaseModel): """Through table linking students to courses.""" student = ForeignKeyField(Student, backref='enrollments') course = ForeignKeyField(Course, backref='enrollments') enrolled_on = DateField(default=datetime.date.today) class Meta: indexes = ( (('student', 'course'), True), ) To query all courses a given student is enrolled in: .. code-block:: python huey = Student.get(Student.name == 'Huey') courses = (Course .select() .join(Enrollment) .where(Enrollment.student == huey) .order_by(Course.title)) for course in courses: print(course.title) To query all students in a given course, along with when they enrolled: .. code-block:: python cs101 = Course.get(Course.title == 'CS 101') query = (Student .select(Student, Enrollment.enrolled_on) .join(Enrollment) .where(Enrollment.course == cs101) .order_by(Student.name)) for student in query: print(student.name, student.enrollment.enrolled_on) # To attach enrollment date to the Student for simplicity: for student in query.objects(): print(student.name, student.enrolled_on) Since all data is available via the through table model, this approach is most flexible and handles any querying requirement without special casing. ManyToManyField ^^^^^^^^^^^^^^^ :class:`ManyToManyField` provides a shortcut API that manages the through table automatically. It is suitable for simple cases where the through table requires no extra columns and complex querying is not needed. .. code-block:: python class Student(BaseModel): name = TextField() class Course(BaseModel): title = TextField() students = ManyToManyField(Student, backref='courses') # Retrieve the auto-generated through model if direct access is needed. Enrollment = Course.students.get_through_model() db.create_tables([Student, Course, Enrollment]) huey = Student.create(name='Huey') cs101 = Course.create(title='CS 101') # Adding and removing relationships: huey.courses.add(cs101) huey.courses.add(Course.select().where(Course.title.contains('Math'))) cs101.students.remove(huey) cs101.students.clear() # Removes all students from this course. # Querying through the field: for course in huey.courses.order_by(Course.title): print(course.title) .. warning:: :class:`ManyToManyField` does not work correctly with model inheritance. The through table contains foreign keys back to the original models, and those pointers are not automatically updated for subclasses. For any model that will be subclassed, use an explicit through table instead. .. seealso:: :meth:`ManyToManyField.add`, :meth:`ManyToManyField.remove`, :meth:`ManyToManyField.clear`, :meth:`ManyToManyField.get_through_model`. .. _prefetch: Avoiding N+1 with Prefetch -------------------------- Joins solve the N+1 problem when traversing from the *many* side toward the *one* side - for example, fetching tweets with their authors. Each tweet has exactly one author, so a join produces exactly one result row per tweet. The situation is different when traversing from the *one* side toward the *many* side - for example, fetching users *with all their tweets*. A join in this direction produces one result row *per tweet*, which means users with multiple tweets appear multiple times in the result set. Deduplicating those rows in application code is awkward and error-prone. :func:`prefetch` solves this by issuing one query per table, then stitching the results together in Python. Instead of *O(n)* queries for *n* rows, we will do *O(k)* queries for *k* tables: .. code-block:: python # Two queries total, regardless of how many users or tweets there are: # SELECT * FROM user # SELECT * FROM tweet WHERE user_id IN (...) users = User.select().prefetch(Tweet) # Equivalent to above. users = prefetch(User.select(), Tweet.select()) for user in users: print(user.username) for tweet in user.tweets: # No additional query, user.tweets is a list. print(f' {tweet.content}') The models passed to :func:`prefetch` must be linked by foreign keys. Peewee infers the relationships and assigns the prefetched rows to the appropriate back-reference attribute on each instance. Prefetch can span more than two tables. To fetch users, their tweets, and the favorites on each tweet in three queries: .. code-block:: python users = prefetch(User.select(), Tweet.select(), Favorite.select()) for user in users: for tweet in user.tweets: print(f'{user.username}: {tweet.content} ' f'({len(tweet.favorites)} favorites)') Filtering prefetched rows ^^^^^^^^^^^^^^^^^^^^^^^^^ Both the outer query and the prefetch subqueries can carry ``WHERE`` clauses and other modifiers independently: .. code-block:: python one_week_ago = datetime.date.today() - datetime.timedelta(days=7) users = prefetch( User.select().order_by(User.username), Tweet.select().where(Tweet.timestamp >= one_week_ago), ) The filter on ``Tweet`` applies only to the prefetched tweets; it does not affect which users are returned. Choosing between joins and prefetch ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Use a **join** when: * Traversing from the many side to the one side (tweet -> author). * Filtering on columns in the related table (tweets by users whose username starts with "h"). * Only a subset of related fields is needed. Use **prefetch** when: * Traversing from the one side to the many side (user -> all their tweets). * The full set of related rows is needed for each parent row. * Nesting more than one level of related data (users -> tweets -> favorites). .. note:: ``LIMIT`` on the outer query of a :func:`prefetch` call works as expected. Limiting the *inner* queries (the prefetched tables) is not directly supported and requires a manual approach - see :ref:`top-n-per-group` in the recipes document for techniques. .. seealso:: :func:`prefetch` API reference. ================================================ FILE: docs/peewee/schema.rst ================================================ .. _schema: Schema Management ================= This document covers creating and dropping tables, managing indexes and constraints after the fact, and evolving a schema over time. Creating Tables --------------- Create tables for a list of models with :meth:`Database.create_tables`: .. code-block:: python db.create_tables([User, Tweet, Favorite]) By default Peewee uses ``CREATE TABLE IF NOT EXISTS``, making it safe to call on application startup. To disable this, pass ``safe=False``. .. code-block:: python db.create_tables([User, Tweet, Favorite], safe=False) To create a single table: .. code-block:: python Tweet.create_table() :meth:`Database.create_tables` respects foreign key dependencies: if ``Tweet`` references ``User``, ``User``'s table is created first regardless of the order in which they appear in the list. Indexes declared in ``Meta.indexes`` and via :meth:`Model.add_index` are created along with the table. .. note:: A common pattern in web applications is to call ``db.create_tables(MODELS, safe=True)`` once at startup. This ensures all tables exist without failing on an already- initialized database. It does **not** apply schema changes - for that, see :ref:`migrations`. Dropping Tables --------------- .. code-block:: python db.drop_tables([User, Tweet, Favorite]) By default Peewee uses ``DROP TABLE IF EXISTS``, making it safe to call multiple times. To disable this, pass ``safe=False``. .. code-block:: python db.drop_tables([User, Tweet, Favorite], safe=False) Pass ``cascade=True`` (Postgresql and MySQL) to let the database handle dependency ordering: .. code-block:: python db.drop_tables([User, Tweet, Favorite], cascade=True) To drop a single table: .. code-block:: python User.drop_table() SchemaManager ------------- :class:`SchemaManager` provides finer-grained control over DDL operations. Each model exposes an instance at ``Model._schema``. Creating and dropping indexes independently: .. code-block:: python # Create just the indexes for a model (table already exists). User._schema.create_indexes() # Drop a specific index. User._schema.drop_index(User.username) Adding a foreign key constraint after table creation (useful when circular foreign keys are involved - see :ref:`circular-fks` in the models document): .. code-block:: python # The table exists but the constraint was deferred. User._schema.create_foreign_key(User.favorite_tweet) .. note:: SQLite does not support adding foreign key constraints to existing tables. On SQLite, ``create_foreign_key`` will result in an :class:`OperationalError`. Truncating a table: .. code-block:: python User._schema.truncate_table() # No cascade. User._schema.truncate_table(cascade=True) # Postgresql only. .. seealso:: :class:`SchemaManager` API reference. .. _migrations: Schema Migrations ----------------- Peewee does not include a built-in migration system. For schema changes in an existing deployment - adding columns, dropping columns, renaming tables, modifying indexes - use one of the following approaches. Playhouse migrate module ^^^^^^^^^^^^^^^^^^^^^^^^^ The :ref:`migrate ` module in playhouse provides a set of helper functions for common schema changes, applied through a :class:`SchemaMigrator`: .. code-block:: python from playhouse.migrate import * db = SqliteDatabase(...) migrator = SchemaMigrator.from_database(db) first_name = TextField(default='') last_name = TextField(default='') with db.atomic(): migrate( migrator.add_column('person', 'first_name', first_name), migrator.add_column('person', 'last_name', last_name), migrator.drop_column('person', 'name'), ) Supported operations: - Add, rename, or drop columns. - Make columns nullable or not nullable. - Change a column's type. - Rename a table. - Add or drop indexes and constraints. - Add or drop column default values. .. seealso:: :ref:`migrate` for in-depth examples and API reference. Raw SQL migrations ^^^^^^^^^^^^^^^^^^^ For changes the migrate module does not cover, execute ALTER TABLE statements directly: .. code-block:: python with db.atomic(): db.execute_sql('ALTER TABLE tweet ADD COLUMN view_count INTEGER DEFAULT 0') SQLite limitations ^^^^^^^^^^^^^^^^^^ SQLite has limited ALTER TABLE support. It supports ``ADD COLUMN`` and ``RENAME TABLE`` but not ``DROP COLUMN``, ``RENAME COLUMN``, or constraint changes in older versions (SQLite 3.35.0+ adds ``DROP COLUMN``). For more complex SQLite schema changes, the standard workaround is to: 1. Create a new table with the desired schema. 2. Copy data with ``INSERT INTO new_table SELECT ... FROM old_table``. 3. Drop the old table. 4. Rename the new table. The playhouse :ref:`migrate ` module transparently handles the above workaround for older SQLite versions. Introspecting an Existing Schema ---------------------------------- :meth:`Database.get_tables` returns the names of all tables in the database: .. code-block:: python db.get_tables() # ['user', 'tweet', 'favorite'] :meth:`Database.get_columns` returns column metadata for a table as a list of :class:`ColumnMetadata` instances: .. code-block:: python for col in db.get_columns('tweet'): print(col.name, col.data_type, col.null) :meth:`Database.get_indexes` returns index metadata as a list of :class:`IndexMetadata` instances: .. code-block:: python for idx in db.get_indexes('user'): print(idx.name, idx.columns, idx.unique) :meth:`Database.get_foreign_keys` returns foreign key metadata as a list of :class:`ForeignKeyMetadata` instances: .. code-block:: python for fk in db.get_foreign_keys('tweet'): print(fk.column, '->', fk.dest_table, fk.dest_column) :meth:`Database.get_views` returns a list of views in the database as a list of :class:`ViewMetadata` instances: .. code-block:: python for view in db.get_views(): print(view.name, view.sql) Generating models from an existing database ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The :ref:`pwiz` command-line tool introspects an existing database and emits Python model definitions: .. code-block:: shell python -m pwiz -e postgresql my_database > models.py python -m pwiz -e sqlite my_app.db > models.py The generated models can be used directly or as a starting point for further customization. ================================================ FILE: docs/peewee/sqlite.rst ================================================ .. _sqlite: SQLite ====== The core :class:`SqliteDatabase` handles pragmas, user-defined functions, WAL mode, full-text search and JSON. Because the full-text search and JSON fields are specific to SQLite, these features are provided by ``playhouse.sqlite_ext``. .. contents:: On this page :local: :depth: 1 Implementations --------------- :class:`SqliteDatabase` Core SQLite implementation. Provides: * Pragma support (including WAL-mode) * User-defined functions * ATTACH / DETACH databases * Full-text search * JSON Full-text search and JSON implementations available in ``playhouse.sqlite_ext``. :class:`~playhouse.cysqlite_ext.CySqliteDatabase` (``playhouse.cysqlite_ext``) Extends :class:`SqliteDatabase`, uses `cysqlite `__ driver. * All above functionality * Table-value functions * Commit / Rollback / Update / Progress / Trace hooks * BLOB I/O * Online backups * Can be built `with encryption `__. :class:`~playhouse.apsw_ext.APSWDatabase` (``playhouse.apsw_ext``) Extends :class:`SqliteDatabase`, uses `apsw `__ driver. APSW is a thin C-level driver that exposes the full range of SQLite functionality. :class:`~playhouse.sqlcipher_ext.SqlCipherDatabase` (``playhouse.sqlcipher_ext``) Extends :class:`SqliteDatabase`, uses `sqlcipher3 `__ driver. SQLCipher provides transparent full-database encryption using 256-bit AES, ensuring data on-disk is secure. :class:`~playhouse.sqliteq.SqliteQueueDatabase` (``playhouse.sqliteq``) Extends :class:`SqliteDatabase`. Provides a SQLite database implementation with a long-lived background writer thread. All write operations are managed by a single write connection, preventing timeouts and database locking issues. This implementation is useful when using Sqlite in multi-threaded environments with frequent writes. .. _sqlite-pragma: PRAGMA statements ----------------- SQLite allows run-time configuration through ``PRAGMA`` statements (`SQLite documentation `_). These statements are typically run when a new database connection is created. To specify default ``PRAGMA`` statements for connections: .. code-block:: python db = SqliteDatabase('my_app.db', pragmas={ 'journal_mode': 'wal', 'cache_size': 10000, # 10000 pages, or ~40MB 'foreign_keys': 1, # Enforce foreign-key constraints }) PRAGMAs may also be configured dynamically using either the :meth:`~SqliteDatabase.pragma` method or the special properties exposed on the :class:`SqliteDatabase` object: .. code-block:: python # Set cache size to 64MB for *current connection*. db.pragma('cache_size', -64000) # Same as above. db.cache_size = -64000 # Read the value of several pragmas: print('cache_size:', db.cache_size) print('foreign_keys:', db.foreign_keys) print('journal_mode:', db.journal_mode) print('page_size:', db.page_size) # Set foreign_keys pragma on current connection *AND* on all # connections opened subsequently. db.pragma('foreign_keys', 1, permanent=True) .. attention:: Pragmas set using the :meth:`~SqliteDatabase.pragma` method are not re-applied when a new connection opens. To configure a pragma to be run whenever a new connection is opened, specify ``permanent=True``. .. code-block:: python db.pragma('foreign_keys', 1, permanent=True) .. seealso:: SQLite PRAGMA documentation: https://sqlite.org/pragma.html .. _sqlite-user-functions: User-Defined Functions ---------------------- SQLite can be extended with user-defined Python code. The :class:`SqliteDatabase` class supports a variety of user-defined extensions: Functions User-defined functions accept any number of parameters and return a single value. * :meth:`SqliteDatabase.register_function` * :meth:`SqliteDatabase.func` - decorator. Aggregates Aggregate values across multiple rows and return a single value. * :meth:`SqliteDatabase.register_aggregate` * :meth:`SqliteDatabase.aggregate` - decorator. Window Functions Aggregates which support operating on sliding windows of data. * :meth:`SqliteDatabase.register_window_function` * :meth:`SqliteDatabase.window_function` - decorator. Collations Control how values are ordered and sorted. * :meth:`SqliteDatabase.register_collation` * :meth:`SqliteDatabase.collation` - decorator. Table Functions User-defined tables (requres ``cysqlite``). * :meth:`.CySqliteDatabase.register_table_function` * :meth:`.CySqliteDatabase.table_function` - decorator. Shared Libraries Load an extension from a shared library. * :meth:`SqliteDatabase.load_extension` * :meth:`SqliteDatabase.unload_extension` Function example ^^^^^^^^^^^^^^^^ .. code-block:: python db = SqliteDatabase('analytics.db') from urllib.parse import urlparse @db.func('hostname') def hostname(url): if url is not None: return urlparse(url).netloc # Call this function in our code: # The following finds the most common hostnames of referrers by count: query = (PageView .select(fn.hostname(PageView.referrer), fn.COUNT(PageView.id)) .group_by(fn.hostname(PageView.referrer)) .order_by(fn.COUNT(PageView.id).desc())) Aggregate example ^^^^^^^^^^^^^^^^^ User-defined aggregates must define two methods: * ``step(*values)`` - called once for each row being aggregated. * ``finalize()`` - called only once to produce final aggregate value. .. code-block:: python from hashlib import md5 @db.aggregate('md5') class MD5Checksum(object): def __init__(self): self.checksum = md5() def step(self, value): self.checksum.update(value.encode('utf-8')) def finalize(self): return self.checksum.hexdigest() # Usage: # The following computes an aggregate MD5 checksum for files broken # up into chunks and stored in the database. query = (FileChunk .select(FileChunk.filename, fn.MD5(FileChunk.data)) .group_by(FileChunk.filename) .order_by(FileChunk.filename, FileChunk.sequence)) Window function example ^^^^^^^^^^^^^^^^^^^^^^^ User-defined window functions are simply aggregates with two additional methods: * ``step(*values)`` - called for each row being aggregated. * ``inverse(*values)`` - "invert" the effect of a call to ``step(*values)``. * ``value()`` - return the current value of the aggregate. * ``finalize()`` - return final aggregate value. .. code-block:: python # Window functions are normal aggregates with two additional methods: # inverse(value) - Perform the inverse of step(value). # value() - Report value at current step. @db.aggregate('mysum') class MySum(object): def __init__(self): self._value = 0 def step(self, value): self._value += (value or 0) def inverse(self, value): self._value -= (value or 0) # Do opposite of "step()". def value(self): return self._value def finalize(self): return self._value # e.g., aggregate sum of employee salaries over their department. query = (Employee .select( Employee.department, Employee.salary, fn.mysum(Employee.salary).over( partition_by=[Employee.department])) .order_by(Employee.id)) Collation example ^^^^^^^^^^^^^^^^^ Collations accept two values and provide a value indicating how they should be ordered (e.g. ``cmp(lhs, rhs)``). .. code-block:: python @db.collation('ireverse') def collate_reverse(s1, s2): # Case-insensitive reverse. s1, s2 = s1.lower(), s2.lower() return (s1 < s2) - (s1 > s2) # Equivalent to -cmp(s1, s2) # To use this collation to sort books in reverse order... Book.select().order_by(collate_reverse.collation(Book.title)) # Or... Book.select().order_by(Book.title.asc(collation='reverse')) Table function example ^^^^^^^^^^^^^^^^^^^^^^ Example user-defined table-value function (see `cysqlite TableFunction docs `_ for full details on ``TableFunction``). .. code-block:: python from cysqlite import TableFunction from playhouse.cysqlite_ext import CySqliteDatabase db = CySqliteDatabase('my_app.db') @db.table_function('series') class Series(TableFunction): columns = ['value'] params = ['start', 'stop', 'step'] def initialize(self, start=0, stop=None, step=1): """ Table-functions declare an initialize() method, which is called with whatever arguments the user has called the function with. """ self.start = self.current = start self.stop = stop or float('Inf') self.step = step def iterate(self, idx): """ Iterate is called repeatedly by the SQLite database engine until the required number of rows has been read **or** the function raises a `StopIteration` signalling no more rows are available. """ if self.current > self.stop: raise StopIteration ret, self.current = self.current, self.current + self.step return (ret,) # Usage: cursor = db.execute_sql('SELECT * FROM series(?, ?, ?)', (0, 5, 2)) for value, in cursor: print(value) # Prints: # 0 # 2 # 4 Shared Libraries ^^^^^^^^^^^^^^^^ Example: .. code-block:: python # Load `closure.so` shared library in the current directory. db = SqliteDatabase('my_app.db') db.load_extension('closure') To support shared libraries, your SQLite3 will need to have been compiled with support for run-time loadable extensions. .. _sqlite-locking-mode: Locking Mode for Transactions ----------------------------- SQLite transactions can be opened in three different modes: * *Deferred* (**default**) - only acquires lock when a read or write is performed. The first read creates a `shared lock `_ and the first write creates a `reserved lock `_. Because the acquisition of the lock is deferred until actually needed, it is possible that another thread or process could create a separate transaction and write to the database. * *Immediate* - a `reserved lock `_ is acquired immediately. In this mode, no other connection may write to the database or open an *immediate* or *exclusive* transaction. Other processes can continue to read from the database, however. * *Exclusive* - opens an `exclusive lock `_ which prevents all (except for read uncommitted) connections from accessing the database until the transaction is complete. Example specifying the locking mode: .. code-block:: python db = SqliteDatabase('app.db') with db.atomic('EXCLUSIVE'): read() write() @db.atomic('IMMEDIATE') def some_other_function(): # This function is wrapped in an "IMMEDIATE" transaction. do_something_else() For more information, see the SQLite `locking documentation `_. To learn more about transactions in Peewee, see the :ref:`transactions` documentation. .. danger:: Do not alter the ``isolation_level`` property of the ``sqlite3.Connection`` object. Peewee requires the ``sqlite3`` driver be in autocommit-mode, which is handled automatically by :class:`SqliteDatabase`. .. _cysqlite-ext: CySqlite -------- .. module:: playhouse.cysqlite_ext :class:`CySqliteDatabase` uses the `cysqlite `_ driver, a high-performance alternative to the standard library ``sqlite3`` module. ``cysqlite`` provides additional features and hooks not available with in the standard library ``sqlite3`` driver. Installation: .. code-block:: shell pip install cysqlite Usage: .. code-block:: python from playhouse.cysqlite_ext import CySqliteDatabase db = CySqliteDatabase('my_app.db', pragmas={ 'cache_size': -64000, 'journal_mode': 'wal', 'foreign_keys': 1, }) .. class:: CySqliteDatabase(database, **kwargs) :param list pragmas: A list of 2-tuples containing pragma key and value to set every time a connection is opened. :param timeout: Set the busy-timeout on the SQLite driver (in seconds). :param bool rank_functions: Make search result ranking functions available. Recommended only when using FTS4. :param bool regexp_function: Make the REGEXP function available. .. seealso:: CySqliteDatabase extends :class:`SqliteDatabase` and inherits all methods for declaring user-defined functions, aggregates, window functions, collations, pragmas, etc. Example: .. code-block:: python db = CySqliteDatabase('app.db', pragmas={'journal_mode': 'wal'}) .. method:: table_function(name) Class-decorator for registering a ``cysqlite.TableFunction``. Table functions are user-defined functions that, rather than returning a single, scalar value, can return any number of rows of tabular data. See `cysqlite docs `__ for details on ``TableFunction`` API. .. code-block:: python from cysqlite import TableFunction @db.table_function('series') class Series(TableFunction): columns = ['value'] params = ['start', 'stop', 'step'] def initialize(self, start=0, stop=None, step=1): """ Table-functions declare an initialize() method, which is called with whatever arguments the user has called the function with. """ self.start = self.current = start self.stop = stop or float('Inf') self.step = step def iterate(self, idx): """ Iterate is called repeatedly by the SQLite database engine until the required number of rows has been read **or** the function raises a `StopIteration` signalling no more rows are available. """ if self.current > self.stop: raise StopIteration ret, self.current = self.current, self.current + self.step return (ret,) cursor = db.execute_sql('SELECT * FROM series(?, ?, ?)', (0, 5, 2)) for (value,) in cursor: print(value) # Prints: # 0 # 2 # 4 .. method:: register_table_function(klass, name) :param TableFunction klass: class implementing TableFunction API. :param str name: name for user-defined table function. Register a ``cysqlite.TableFunction`` class with the connection. Table functions are user-defined functions that, rather than returning a single, scalar value, can return any number of rows of tabular data. .. seealso:: * :meth:`CySqliteDatabase.table_function` for example implementation. * `cysqlite docs `__ for details on ``TableFunction`` API. .. method:: unregister_table_function(name) :param name: Name of the user-defined table function. :returns: True or False, depending on whether the function was removed. Unregister the user-defined table function. .. method:: on_commit(fn) :param fn: callable or ``None`` to clear the current hook. Register a callback to be executed whenever a transaction is committed on the current connection. The callback accepts no parameters and the return value is ignored. However, if the callback raises a :class:`ValueError`, the transaction will be aborted and rolled-back. Example: .. code-block:: python db = CySqliteDatabase(':memory:') @db.on_commit def on_commit(): logger.info('COMMITing changes') .. method:: on_rollback(fn) :param fn: callable or ``None`` to clear the current hook. Register a callback to be executed whenever a transaction is rolled back on the current connection. The callback accepts no parameters and the return value is ignored. Example: .. code-block:: python @db.on_rollback def on_rollback(): logger.info('Rolling back changes') .. method:: on_update(fn) :param fn: callable or ``None`` to clear the current hook. Register a callback to be executed whenever the database is written to (via an *UPDATE*, *INSERT* or *DELETE* query). The callback should accept the following parameters: * ``query`` - the type of query, either *INSERT*, *UPDATE* or *DELETE*. * database name - the default database is named *main*. * table name - name of table being modified. * rowid - the rowid of the row being modified. The callback's return value is ignored. Example: .. code-block:: python db = CySqliteDatabase(':memory:') @db.on_update def on_update(query_type, db, table, rowid): # e.g. INSERT row 3 into table users. logger.info('%s row %s into table %s', query_type, rowid, table) .. method:: authorizer(fn) :param fn: callable or ``None`` to clear the current authorizer. Register an authorizer callback. Authorizer callbacks must accept 5 parameters, which vary depending on the operation being checked. * op: operation code, e.g. ``cysqlite.SQLITE_INSERT``. * p1: operation-specific value, e.g. table name for ``SQLITE_INSERT``. * p2: operation-specific value. * p3: database name, e.g. ``"main"``. * p4: inner-most trigger or view responsible for the access attempt if applicable, else ``None``. See `sqlite authorizer documentation `_ for description of authorizer codes and values for parameters p1 and p2. The authorizer callback must return one of: * ``cysqlite.SQLITE_OK``: allow operation. * ``cysqlite.SQLITE_IGNORE``: allow statement compilation but prevent the operation from occuring. * ``cysqlite.SQLITE_DENY``: prevent statement compilation. More details can be found in the `cysqlite docs `__. .. method:: trace(fn, mask=2, expand_sql=True): :param fn: callable or ``None`` to clear the current trace hook. :param int mask: mask of what types of events to trace. Default value corresponds to ``SQLITE_TRACE_PROFILE``. :param bool expand_sql: Pass callback the ``sqlite3_expanded_sql()`` from ``sqlite3_stmt`` (expands bound parameters) Register a trace hook (``sqlite3_trace_v2``). Trace callback must accept 4 parameters, which vary depending on the operation being traced. * event: type of event, e.g. ``SQLITE_TRACE_PROFILE``. * sid: memory address of statement (only ``SQLITE_TRACE_CLOSE``), else -1. * sql: SQL string. If ``expand_sql`` then bound parameters will be expanded (for ``SQLITE_TRACE_CLOSE``, ``sql=None``). * ns: estimated number of nanoseconds the statement took to run (only ``SQLITE_TRACE_PROFILE``), else -1. Any return value from callback is ignored. More details can be found in the `cysqlite docs `__. .. method:: slow_query_log(threshold_ms=50, logger=None, level=logging.WARNING, expand_sql=True) :param threshold_ms: estimated millisecond threshold to log slow queries. :param logger: logging namespace, defaults to ``'peewee.cysqlite_ext'``. :param int level: level for slow query log. :param bool expand_sql: expand bound parameters in SQL query. Register a ``sqlite3_trace_v2`` callback that will log slow queries to the given logger. Overrides previously-registered :py:meth:`~CySqliteDatabase.trace` callback. Automatically re-registered when new connection is opened. .. method:: progress(fn, n=1) :param fn: callable or ``None`` to clear the current progress handler. :param int n: approximate number of VM instructions to execute between calls to the progress handler. Register a progress handler (``sqlite3_progress_handler``). Callback takes no arguments and returns 0 to allow progress to continue or any non-zero value to interrupt progress. More details can be found in the `cysqlite docs `__. .. attribute:: autocommit Property which returns a boolean indicating if autocommit is enabled. By default, this value will be ``True`` except when inside a transaction (or :meth:`~Database.atomic` block). Example: .. code-block:: pycon >>> db = CySqliteDatabase(':memory:') >>> db.autocommit True >>> with db.atomic(): ... print(db.autocommit) ... False >>> db.autocommit True .. method:: backup(destination, pages=None, name=None, progress=None) :param CySqliteDatabase destination: Database object to serve as destination for the backup. :param int pages: Number of pages per iteration. Default value of -1 indicates all pages should be backed-up in a single step. :param str name: Name of source database (may differ if you used ATTACH DATABASE to load multiple databases). Defaults to "main". :param progress: Progress callback, called with three parameters: the number of pages remaining, the total page count, and whether the backup is complete. Example: .. code-block:: python master = CySqliteDatabase('master.db') replica = CySqliteDatabase('replica.db') # Backup the contents of master to replica. master.backup(replica) .. method:: backup_to_file(filename, pages, name, progress) :param filename: Filename to store the database backup. :param int pages: Number of pages per iteration. Default value of -1 indicates all pages should be backed-up in a single step. :param str name: Name of source database (may differ if you used ATTACH DATABASE to load multiple databases). Defaults to "main". :param progress: Progress callback, called with three parameters: the number of pages remaining, the total page count, and whether the backup is complete. Backup the current database to a file. The backed-up data is not a database dump, but an actual SQLite database file. Example: .. code-block:: python db = CySqliteDatabase('app.db') def nightly_backup(): filename = 'backup-%s.db' % (datetime.date.today()) db.backup_to_file(filename) .. method:: blob_open(table, column, rowid, read_only=False) :param str table: Name of table containing data. :param str column: Name of column containing data. :param int rowid: ID of row to retrieve. :param bool read_only: Open the blob for reading only. :param str dbname: Database name (e.g. if multiple databases attached). :returns: ``cysqlite.Blob`` instance which provides efficient access to the underlying binary data. :rtype: cysqlite.Blob See `cysqlite documentation `_ for more details. Example: .. code-block:: python class Image(Model): filename = TextField() data = BlobField() buf_size = 1024 * 1024 * 8 # Allocate 8MB for storing file. rowid = Image.insert({ Image.filename: 'thefile.jpg', Image.data: fn.zeroblob(buf_size), }).execute() # Open the blob, returning a file-like object. blob = db.blob_open('image', 'data', rowid) # Write some data to the blob. blob.write(image_data) img_size = blob.tell() # Read the data back out of the blob. blob.seek(0) image_data = blob.read(img_size) .. class:: PooledCySqliteDatabase(database, **kwargs) Connection-pooling variant of :class:`CySqliteDatabase`. .. _apsw: APSW ---- .. module:: playhouse.apsw_ext `APSW `__ is a thin C wrapper over SQLite's C API that exposes nearly every SQLite feature including virtual tables, virtual filesystems, and BLOB I/O. Installation: .. code-block:: shell pip install apsw Usage: .. code-block:: python from playhouse.apsw_ext import APSWDatabase db = APSWDatabase('my_app.db') class BaseModel(Model): class Meta: database = db Use the ``Field`` subclasses from ``playhouse.apsw_ext`` rather than those from ``peewee`` to ensure correct type adaptation. For example, use ``playhouse.apsw_ext.DateTimeField`` instead of ``peewee.DateTimeField``. .. class:: APSWDatabase(database, **connect_kwargs) Subclass of :class:`SqliteDatabase` using the APSW driver. :param string database: filename of sqlite database :param connect_kwargs: keyword arguments passed to apsw when opening a connection .. method:: register_module(mod_name, mod_inst) Register a virtual table module globally. See the `APSW virtual table documentation `_. :param string mod_name: name to use for module :param object mod_inst: an object implementing the `Virtual Table `_ interface .. method:: unregister_module(mod_name) Unregister a previously registered module. .. _sqlcipher: SQLCipher --------- .. module:: playhouse.sqlcipher_ext `SQLCipher `__ is an encrypted wrapper around SQLite. Peewee exposes it through :class:`SqlCipherDatabase`, which is API-identical to :class:`SqliteDatabase` except for its constructor. Installation: .. code-block:: shell pip install sqlcipher3 Usage: .. code-block:: python from playhouse.sqlcipher_ext import SqlCipherDatabase db = SqlCipherDatabase( 'app.db', passphrase=os.environ['PASSPHRASE'], pragmas={'cache_size': -64000}) Example usage with deferred initialization and passphrase prompt: .. code-block:: python db = SqlCipherDatabase(None) class BaseModel(Model): class Meta: database = db class Secret(BaseModel): value = TextField() # Prompt the user and initialize the database with their passphrase. while True: db.init('my_app.db', passphrase=input('Passphrase: ')) try: db.get_tables() # Will raise if passphrase is wrong. break except DatabaseError as exc: print('Wrong passphrase.') db.init(None) Pragma configuration (e.g. increasing PBKDF2 iterations): .. code-block:: python db = SqlCipherDatabase('my_app.db', passphrase='s3cr3t', pragmas={'kdf_iter': 1_000_000}) SQLCipher can be configured using a number of extension PRAGMAs. The list of PRAGMAs and their descriptions can be found in the `SQLCipher documentation `__. .. class:: SqlCipherDatabase(database, passphrase, **kwargs) :param str database: Path to the encrypted database file. :param str passphrase: Encryption passphrase (should be 8 character minimum; enforce stronger requirements in your application). If the database file does not exist, it is created and encrypted with a key derived from ``passphrase``. If it does exist, ``passphrase`` must match the one used when the file was created. If the passphrase is incorrect, an error will be raised when first attempting to access the database (typically ``DatabaseError: file is not a database``). .. method:: rekey(passphrase) Change the encryption passphrase for the open database. .. _sqliteq: SqliteQueueDatabase ------------------- .. module:: playhouse.sqliteq :class:`SqliteQueueDatabase` serializes all write queries through a single long-lived connection on a dedicated background thread. This allows multiple application threads to write to a SQLite database concurrently without conflict or timeout errors. ``SqliteQueueDatabase`` can be used as a drop-in replacement for the regular :class:`SqliteDatabase` if you want simple **read and write** access to a SQLite database from multiple threads, and do not need transactions. .. code-block:: python from playhouse.sqliteq import SqliteQueueDatabase db = SqliteQueueDatabase( 'my_app.db', use_gevent=False, # Use stdlib threading (default). autostart=True, # Start the writer thread immediately. queue_max_size=64, # Max pending writes before blocking. results_timeout=5.0, # Seconds to wait for a write to complete. pragmas={'journal_mode': 'wal'}) If you set ``autostart=False``, start the writer thread explicitly: .. code-block:: python db.start() Stop the writer thread on application shutdown (waits for pending writes): .. code-block:: python import atexit @atexit.register def _stop(): db.stop() Read queries work as normal. Open and close the connection per-request as you would with any other database. Only writes are funneled through the queue. **Transactions are not supported.** Because writes from different threads are interleaved, there is no way to guarantee that the statements in a transaction from one thread execute atomically without statements from another thread appearing between them. The ``atomic()`` and ``transaction()`` methods raise a ``ValueError`` if called. If you need to temporarily bypass the queue and write directly (for example, during a batch import), use :meth:`~SqliteQueueDatabase.pause` and :meth:`~SqliteQueueDatabase.unpause`. .. class:: SqliteQueueDatabase(database, use_gevent=False, autostart=True, queue_max_size=None, results_timeout=None, **kwargs) :param str database: database filename. :param bool use_gevent: use gevent instead of ``threading``. :param bool autostart: automatically start writer background thread. :param int queue_max_size: maximum size of pending writes queue. :param float results_timeout: timeout for waiting for query results from write thread (seconds). .. method:: start() Start the background writer thread. .. method:: stop() Signal the writer thread to stop. Blocks until all pending writes are flushed. .. method:: is_stopped() Return ``True`` if the writer thread is not running. .. method:: pause() Block until the writer thread finishes its current work, then disconnect it. The calling thread takes over direct database access. Must be followed by a call to :meth:`~SqliteQueueDatabase.unpause`. .. method:: unpause() Resume the writer thread and reconnect the queue. .. _sqlite-fields: SQLite-Specific Fields ---------------------- .. module:: playhouse.sqlite_ext These field classes live in ``playhouse.sqlite_ext`` and can be used with: * :class:`SqliteDatabase` * :class:`.CySqliteDatabase` * :class:`.APSWDatabase` * :class:`.SqlCipherDatabase` * :class:`.SqliteQueueDatabase` .. class:: RowIDField() Primary-key field mapped to SQLite's implicit ``rowid`` column. For more information, see the SQLite documentation on `rowid tables `_. .. code-block:: python class Note(Model): rowid = RowIDField() # Implied primary_key=True. content = TextField() timestamp = TimestampField() RowIDField can be mapped to a different field name, but it's underlying column name will always be ``rowid``. .. code-block:: python class Note(Model): id = RowIDField() ... .. class:: AutoIncrementField() Integer primary key that uses SQLite's ``AUTOINCREMENT`` keyword, guaranteeing the primary key is always strictly increasing even after deletions. Has a small performance cost versus the default :class:`PrimaryKeyField` or :class:`RowIDField`. See the `SQLite AUTOINCREMENT documentation `_ for details. .. class:: ISODateTimeField() Subclass of :class:`DateTimeField` that preserves UTC offset information for timezone-aware datetimes when storing to SQLite's text-based datetime representation. .. class:: TDecimalField(max_digits=10, decimal_places=5, auto_round=False, rounding=None, *args, **kwargs) Subclass of :class:`DecimalField` that stores decimal values in a ``TEXT`` column to avoid any potential loss of precision that may occur when storing in a ``REAL`` (double-precision floating point) column. SQLite does not have a true numeric type, so this field ensures no precision is lost when using Decimals. .. _sqlite-json: SQLite JSON ----------- :class:`~playhouse.sqlite_ext.JSONField` enables storing and querying JSON data in SQLite using the `SQLite json functions `_. .. class:: JSONField(json_dumps=None, json_loads=None, **kwargs) :param json_dumps: Custom JSON serializer. Defaults to ``json.dumps``. :param json_loads: Custom JSON deserializer. Defaults to ``json.loads``. Stores and retrieves JSON data transparently and provides efficient implementations for in-place modification and querying. Data is automatically serialized on write, deserialized on read. Example model: .. code-block:: python from peewee import * from playhouse.sqlite_ext import JSONField db = SqliteDatabase(':memory:') class Config(db.Model): data = JSONField() Config.create_table() # Create two rows. Config.create(data={'timeout': 30, 'retry': {'max': 5}}) Config.create(data={'timeout': 10, 'retry': {'max': 10}}) To access or modify specific object keys or array indexes in a JSON structure, you can treat the :class:`JSONField` as if it were a dictionary/list: .. code-block:: python # Select or order by a JSON value: query = (Config .select(Config, Config.data['timeout'].alias('timeout')) .order_by(Config.data['timeout'].desc())) # Aggregate on nested value: avg = (Config .select(fn.SUM(Config.data['timeout']) / fn.COUNT(Config.id)) .scalar()) # Filter by nested value: Config.select().where(Config.data['retry']['max'] < 8) Data can be atomically updated, written and removed in-place: .. code-block:: python # In-place update (preserves other keys): (Config .update(data=Config.data.update({'timeout': 60})) .where(Config.data['timeout'] >= 30) .execute()) # Set a specific path: (Config .update(data=Config.data['timeout'].set(120)) .where(Config.data['retry']['max'] == 5) .execute()) # Update a specific path with an object. Existing field ("max") will be # preserved in this example. (Config .update(data=Config.data['retry'].update({'backoff': 1})) .execute()) # To overwrite a specific path with an object, use set(): (Config .update(data=Config.data['retry'].set({'allowed': 10})) .execute()) # Remove a key atomically: (Config .update(data=Config.data.update({'retry': None})) .where(Config.id == 1) .execute()) # Another way to remove atomically: (Config .update(data=Config.data['retry'].remove()) .where(Config.id == 2) .execute()) Helpers for other JSON scenarios: .. code-block:: python # Query JSON types: query = (Config .select(Config.data.json_type(), Config.data['timeout'].json_type()) .tuples()) # [('object', 'integer'), ('object', 'integer')] # Query length of an array: cfg1 = Config.create(data={'statuses': [1, 99, 1, 1]}) cfg2 = Config.create(data={'statuses': [1, 1]}) query = (Config .select( Config.data['statuses'], Config.data['statuses'].length()) .tuples()) # [([1, 99, 1, 1], 4), ([1, 1], 2)] Let's add a nested value and then see how to iterate through it's contents recursively using the :meth:`~JSONField.tree` method: .. code-block:: python Config.create(data={'x1': {'y1': 'z1', 'y2': 'z2'}, 'x2': [1, 2]}) tree = Config.data.tree().alias('tree') query = (Config .select(Config.id, tree.c.fullkey, tree.c.value) .from_(Config, tree)) for row in query.tuples(): print(row) (1, '$', {'x1': {'y1': 'z1', 'y2': 'z2'}, 'x2': [1, 2]}), (1, '$.x2', [1, 2]), (1, '$.x2[0]', 1), (1, '$.x2[1]', 2), (1, '$.x1', {'y1': 'z1', 'y2': 'z2'}), (1, '$.x1.y1', 'z1'), (1, '$.x1.y2', 'z2')] The :meth:`~JSONField.tree` and :meth:`~JSONField.children` methods are powerful. For more information on how to utilize them, see the `json1 extension documentation `_. .. method:: __getitem__(item) :param item: Access a specific key or array index in the JSON data. :return: a special object exposing access to the JSON data. :rtype: JSONPath Access a specific key or array index in the JSON data. Returns a :class:`JSONPath` object, which exposes convenient methods for reading or modifying a particular part of a JSON object. Example: .. code-block:: python # If metadata contains {"tags": ["list", "of", "tags"]}, we can # extract the first tag in this way: Post.select(Post, Post.metadata['tags'][0].alias('first_tag')) For more examples see the :class:`JSONPath` API documentation. .. method:: extract(*paths) :param paths: One or more JSON paths to extract. Extract one or more JSON path values. Returns a list when multiple paths are given. .. method:: extract_json(path) :param str path: JSON path Extract the value at the specified path as a JSON data-type. This corresponds to the ``->`` operator added in Sqlite 3.38. .. method:: extract_text(path) :param str path: JSON path Extract the value at the specified path as a SQL data-type. This corresponds to the ``->>`` operator added in Sqlite 3.38. .. method:: set(value, as_json=None) :param value: a scalar value, list, or dictionary. :param bool as_json: force the value to be treated as JSON, in which case it will be serialized as JSON in Python beforehand. By default, lists and dictionaries are treated as JSON to be serialized, while strings and integers are passed as-is. Set the value stored in a :class:`JSONField`. Uses the `json_set() `_ function from the json1 extension. .. method:: replace(value, as_json=None) :param value: a scalar value, list, or dictionary. :param bool as_json: force the value to be treated as JSON, in which case it will be serialized as JSON in Python beforehand. By default, lists and dictionaries are treated as JSON to be serialized, while strings and integers are passed as-is. Replace the existing value stored in a :class:`JSONField`. Will not create if does not exist. Uses the `json_replace() `_ function from the json1 extension. .. method:: insert(value, as_json=None) :param value: a scalar value, list, or dictionary. :param bool as_json: force the value to be treated as JSON, in which case it will be serialized as JSON in Python beforehand. By default, lists and dictionaries are treated as JSON to be serialized, while strings and integers are passed as-is. Insert value into :class:`JSONField`. Will not overwrite existing. Uses the `json_insert() `_ function from the json1 extension. .. method:: append(value, as_json=None) :param value: a scalar value, list, or dictionary. :param bool as_json: force the value to be treated as JSON, in which case it will be serialized as JSON in Python beforehand. By default, lists and dictionaries are treated as JSON to be serialized, while strings and integers are passed as-is. Append to the array stored in a :class:`JSONField`. Uses the `json_set() `_ function from the json1 extension. .. method:: update(data) :param data: a scalar value, list or dictionary to merge with the data currently stored in a :class:`JSONField`. To remove a particular key, set that key to ``None`` in the updated data. Merge new data into the JSON value using the RFC-7396 MergePatch algorithm to apply a patch (``data`` parameter) against the column data. MergePatch can add, modify, or delete elements of a JSON object, which means :meth:`~JSONField.update` is a generalized replacement for both :meth:`~JSONField.set` and :meth:`~JSONField.remove`. MergePatch treats JSON array objects as atomic, so ``update()`` cannot append to an array, nor modify individual elements of an array. For more information as well as examples, see the SQLite `json_patch() `_ function documentation. .. method:: remove() Remove the data stored in the :class:`JSONField`. Uses the `json_remove `_ function from the json1 extension. .. method:: json_type() Return a string identifying the type of value stored in the column. The type returned will be one of: * object * array * integer * real * true * false * text * null <-- the string "null" means an actual NULL value * NULL <-- an actual NULL value means the path was not found Uses the `json_type `_ function from the json1 extension. .. method:: length() Return the length of the array stored in the column. Uses the `json_array_length `_ function from the json1 extension. .. method:: children() The ``children`` function corresponds to ``json_each``, a table-valued function that walks the JSON value provided and returns the immediate children of the top-level array or object. If a path is specified, then that path is treated as the top-most element. The rows returned by calls to ``children()`` have the following attributes: * ``key``: the key of the current element relative to its parent. * ``value``: the value of the current element. * ``type``: one of the data-types (see :meth:`~JSONField.json_type`). * ``atom``: the scalar value for primitive types, ``NULL`` for arrays and objects. * ``id``: a unique ID referencing the current node in the tree. * ``parent``: the ID of the containing node. * ``fullkey``: the full path describing the current element. * ``path``: the path to the container of the current row. Internally this method uses the `json_each `_ (documentation link) function from the json1 extension. Example usage (compare to :meth:`~JSONField.tree` method): .. code-block:: python class KeyData(Model): key = TextField() data = JSONField() KeyData.create(key='a', data={'k1': 'v1', 'x1': {'y1': 'z1'}}) KeyData.create(key='b', data={'x1': {'y1': 'z1', 'y2': 'z2'}}) # We will query the KeyData model for the key and all the # top-level keys and values in it's data field. kd = KeyData.data.children().alias('children') query = (KeyData .select(kd.c.key, kd.c.value, kd.c.fullkey) .from_(KeyData, kd) .order_by(kd.c.key) .tuples()) print(query[:]) # PRINTS: [('a', 'k1', 'v1', '$.k1'), ('a', 'x1', '{"y1":"z1"}', '$.x1'), ('b', 'x1', '{"y1":"z1","y2":"z2"}', '$.x1')] .. method:: tree() The ``tree`` function corresponds to ``json_tree``, a table-valued function that recursively walks the JSON value provided and returns information about the keys at each level. If a path is specified, then that path is treated as the top-most element. The rows returned by calls to ``tree()`` have the same attributes as rows returned by calls to :meth:`~JSONField.children`: * ``key``: the key of the current element relative to its parent. * ``value``: the value of the current element. * ``type``: one of the data-types (see :meth:`~JSONField.json_type`). * ``atom``: the scalar value for primitive types, ``NULL`` for arrays and objects. * ``id``: a unique ID referencing the current node in the tree. * ``parent``: the ID of the containing node. * ``fullkey``: the full path describing the current element. * ``path``: the path to the container of the current row. Internally this method uses the `json_tree `_ (documentation link) function from the json1 extension. Example usage: .. code-block:: python class KeyData(Model): key = TextField() data = JSONField() KeyData.create(key='a', data={'k1': 'v1', 'x1': {'y1': 'z1'}}) KeyData.create(key='b', data={'x1': {'y1': 'z1', 'y2': 'z2'}}) # We will query the KeyData model for the key and all the # keys and values in it's data field, recursively. kd = KeyData.data.tree().alias('tree') query = (KeyData .select(kd.c.key, kd.c.value, kd.c.fullkey) .from_(KeyData, kd) .order_by(kd.c.key) .tuples()) print(query[:]) # PRINTS: [('a', None, '{"k1":"v1","x1":{"y1":"z1"}}', '$'), ('b', None, '{"x1":{"y1":"z1","y2":"z2"}}', '$'), ('a', 'k1', 'v1', '$.k1'), ('a', 'x1', '{"y1":"z1"}', '$.x1'), ('b', 'x1', '{"y1":"z1","y2":"z2"}', '$.x1'), ('a', 'y1', 'z1', '$.x1.y1'), ('b', 'y1', 'z1', '$.x1.y1'), ('b', 'y2', 'z2', '$.x1.y2')] .. class:: JSONPath(field, path=None) :param JSONField field: the field object we intend to access. :param tuple path: Components comprising the JSON path. A convenient, Pythonic way of representing JSON paths for use with :class:`JSONField`. Implements the same methods as :class:`JSONField` but designed for operating on nested items, e.g.: .. code-block:: python Config.create(data={'timeout': 30, 'retries': {'max': 5}}) # Both Config.data['timeout'] and Config.data['retries']['max'] # are instances of JSONPath: query = (Config .select(Config.data['timeout']) .where(Config.data['retries']['max'] < 10)) .. class:: JSONBField(json_dumps=None, json_loads=None, **kwargs) Extends :class:`JSONField` and stores data in the binary ``jsonb`` format (SQLite 3.45.0+). When reading raw column values the data is in its encoded binary form use the :meth:`~JSONBField.json` method to decode: .. code-block:: python # Raw read returns binary: kv = KV.get(KV.key == 'a') kv.value # b"l'k1'v1" # Use .json() to get a Python object: kv = KV.select(KV.value.json()).get() kv.value # {'k1': 'v1'} .. method:: json() Indicate the JSONB field-data should be deserialized and returned as JSON (as opposed to the SQLite binary format). .. _sqlite-fts: Full-Text Search ----------------- Peewee supports :ref:`FTS3, FTS4 ` (legacy, widely available) and :ref:`FTS5 ` full-text search extensions. The general pattern is: 1. Define a :class:`FTSModel` or :class:`FTS5Model` subclass with one or more :class:`SearchField` columns. 2. When a row is created or updated in the source table, insert or update the corresponding row in the search index. 3. Query the index using :meth:`~FTSModel.match` and rank results with :meth:`~FTSModel.bm25` (or :meth:`~FTSModel.rank` for FTS5). Consult the SQLite documentation for FTS query syntax diagrams: * `FTS3 and FTS4 `__ * `FTS5 `__ .. class:: SearchField(unindexed=False, column_name=None) Field type for full-text search virtual tables. Raises an exception if constraints (``null=False``, ``unique=True``, etc.) are specified, since FTS tables do not support them. Pass ``unindexed=True`` to store metadata alongside the search index without indexing it: .. code-block:: python class DocumentIndex(FTSModel): title = SearchField() content = SearchField() tags = SearchField() timestamp = SearchField(unindexed=True) .. method:: match(term) :param str term: full-text search query/terms. :return: a :class:`Expression` corresponding to the ``MATCH`` operator. Sqlite's full-text search supports searching either the full table, including all indexed columns, **or** searching individual columns. The :meth:`~SearchField.match` method can be used to restrict search to a single column: .. code-block:: python # Search *only* the title field and return results ordered by # relevance, using bm25. query = (DocumentIndex .select(DocumentIndex, DocumentIndex.bm25().alias('score')) .where(DocumentIndex.title.match('python')) .order_by(DocumentIndex.bm25())) To search *all* indexed columns, use the :meth:`FTSModel.match` method: .. code-block:: python :emphasize-lines: 5 # Searches *both* the title and body and return results ordered by # relevance, using bm25. query = (DocumentIndex .select(DocumentIndex, DocumentIndex.bm25().alias('score')) .where(DocumentIndex.match('python')) .order_by(DocumentIndex.bm25())) .. method:: highlight(left, right) :param str left: opening tag for highlight, e.g. ``''`` :param str right: closing tag for highlight, e.g. ``''`` When performing a search using the ``MATCH`` operator, FTS5 can return text highlighting matches in a given column. .. code-block:: python # Search for items matching string 'python' and return the title # highlighted with square brackets. query = (SearchIndex .search('python') .select(SearchIndex.title.highlight('[', ']').alias('hi'))) for result in query: print(result.hi) # For example, might print: # Learn [python] the hard way .. method:: snippet(left, right, over_length='...', max_tokens=16) :param str left: opening tag for highlight, e.g. ``''`` :param str right: closing tag for highlight, e.g. ``''`` :param str over_length: text to prepend or append when snippet exceeds the maximum number of tokens. :param int max_tokens: max tokens returned, **must be 1 - 64**. When performing a search using the ``MATCH`` operator, FTS5 can return text with a snippet containing the highlighted match in a given column. .. code-block:: python # Search for items matching string 'python' and return the title # highlighted with square brackets. query = (SearchIndex .search('python') .select(SearchIndex.title.snippet('[', ']').alias('snip'))) for result in query: print(result.snip) .. _sqlite-fts4: FTS4 / ``FTSModel`` ^^^^^^^^^^^^^^^^^^^ FTSModel enables Peewee applications to store data in an efficient full-text search index using SQLite `FTS4 `_. FTSModel caveats: * All queries **except** ``MATCH`` and ``rowid`` lookup require a full table scan. * Constraints, foreign-keys, and indexes are not supported. * All columns are treated as ``TEXT``. * No built-in ranking. Peewee provides several implementations which can be automatically registered by passing ``rank_functions=True`` to ``SqliteDatabase(...)``. * FTSModel ``rowid`` primary key may be declared using :class:`RowIDField`. Lookups on the ``rowid`` are very efficient. Given these constraints all fields besides ``rowid`` should be instances of :class:`SearchField` to ensure correctness. .. tip:: Because of the lack of secondary indexes, it usually makes sense to treat the ``FTSModel.rowid`` primary key as a foreign-key to a row in a normal SQLite table. Example: .. code-block:: python from peewee import * from playhouse.sqlite_ext import FTSModel, SearchField db = SqliteDatabase('app.db', rank_functions=True) class Document(Model): # Canonical source of data, stored in a normal table. author = ForeignKeyField(User, backref='documents') title = TextField(null=False, unique=True) content = TextField(null=False) timestamp = DateTimeField() class Meta: database = db class DocumentIndex(FTSModel): # Full-text search index. rowid = RowIDField() title = SearchField() content = SearchField() author = SearchField(unindexed=True) class Meta: database = db # Use the porter stemming algorithm to tokenize content, optimize # prefix searches of 3 or 4 characters. options = {'tokenize': 'porter unicode61', 'prefix': [3, 4]} Store data by inserting it into the FTS table: .. code-block:: python # Store a document in the index: DocumentIndex.create( rowid=document.id, # Set rowid to match Document's id. title=document.title, content=document.content, author=document.author.get_full_name()) # Equivalent: (DocumentIndex .insert({ 'rowid': document.id, 'title': document.title, 'content': document.content, 'author': document.author.get_full_name()}) .execute()) :class:`FTSModel` provides several shortcuts for full-text search queries: .. code-block:: python # Simple search using basic ranking algorithm. results = DocumentIndex.search('python sqlite') # BM25 search With score and per-column weighting: results = DocumentIndex.search_bm25( 'python sqlite', weights={'title': 2.0, 'content': 1.0}, with_score=True, score_alias='relevance') for r in results: print(r.title, r.relevance) An important method of searching relies on the ``rowid`` of the indexed data matching the document's canonical id. Using this technique we can apply additional filters and retrieve the matching ``Document`` objects efficiently: .. code-block:: python # Search and ensure we only retrieve articles from the last 30 days. cutoff = datetime.datetime.now() - datetime.timedelta(days=30) query = (Document .select() .join( DocumentIndex, on=(Document.id == DocumentIndex.rowid)) .where( (Document.timestamp >= cutoff) & DocumentIndex.match('python sqlite')) .order_by(DocumentIndex.bm25())) .. warning:: All SQL queries on ``FTSModel`` classes will be full-table scans **except** full-text searches and ``rowid`` lookups. .. _sqlite-fts4-external-content: .. topic:: External Content If the primary source of the content you are indexing exists in a separate table, you can save some disk space by instructing SQLite to not store an additional copy of the search index content. To accomplish this, you can specify a table using the ``content`` option. The `FTS4 documentation `_ and `FTS5 documentation `_ have more information. Here is a short example illustrating how to implement this with peewee: .. code-block:: python class Blog(Model): title = TextField() pub_date = DateTimeField(default=datetime.datetime.now) content = TextField() # We want to search this. class Meta: database = db class BlogIndex(FTSModel): # or FTS5Model. content = SearchField() class Meta: database = db options = { 'content': Blog, # Data source. 'content_rowid': Blog.id, # FTS5 only. } db.create_tables([Blog, BlogIndex]) # Now, we can manage content in the BlogIndex. To populate the # search index: BlogIndex.rebuild() # Optimize the index. BlogIndex.optimize() The ``content`` option accepts a :class:`Model` and can reduce the amount of storage used by the database at the expense of requiring more care and attention to keeping data synchronized. .. class:: FTSModel() Base Model class suitable for working with SQLite FTS3 / FTS4. Supports the following options: * ``content``: :class:`Model` containing external content, or empty string for "contentless" * ``prefix``: integer(s). Ex: '2' or '2,3,4' * ``tokenize``: simple, porter, unicode61. Ex: 'porter' Example: .. code-block:: python class DocumentIndex(FTSModel): title = SearchField() body = SearchField() class Meta: database = db options = { 'tokenize': 'porter unicode61', 'prefix': '3', } .. classmethod:: match(term) :param term: Search term or expression. `FTS syntax documentation `__. Generate a SQL expression representing a search for the given term or expression in the table. SQLite uses the ``MATCH`` operator to indicate a full-text search. Example: .. code-block:: python # Search index for "search phrase" and return results ranked # by relevancy using the BM25 algorithm. query = (DocumentIndex .select() .where(DocumentIndex.match('search phrase')) .order_by(DocumentIndex.bm25())) for result in query: print('Result: %s' % result.title) .. classmethod:: search(term, weights=None, with_score=False, score_alias='score', explicit_ordering=False) :param term: Search term or expression. `FTS syntax documentation `__. :param weights: A list of weights for the columns, ordered with respect to the column's position in the table. **Or**, a dictionary keyed by the field or field name and mapped to a value. :param with_score: Whether the score should be returned as part of the ``SELECT`` statement. :param str score_alias: Alias to use for the calculated rank score. This is the attribute you will use to access the score if ``with_score=True``. :param bool explicit_ordering: Order using full SQL function to calculate rank, as opposed to simply referencing the score alias in the ORDER BY clause. Shorthand way of searching for a term and sorting results by the quality of the match. This method uses a simplified algorithm for determining the relevance rank of results. For more sophisticated result ranking, use the :meth:`~FTSModel.search_bm25` method. .. code-block:: python # Simple search. docs = DocumentIndex.search('search term') for result in docs: print(result.title) # More complete example. docs = DocumentIndex.search( 'search term', weights={'title': 2.0, 'content': 1.0}, with_score=True, score_alias='search_score') for result in docs: print(result.title, result.search_score) .. classmethod:: search_bm25(term, weights=None, with_score=False, score_alias='score', explicit_ordering=False) :param term: Search term or expression. `FTS syntax documentation `__. :param weights: A list of weights for the columns, ordered with respect to the column's position in the table. **Or**, a dictionary keyed by the field or field name and mapped to a value. :param with_score: Whether the score should be returned as part of the ``SELECT`` statement. :param str score_alias: Alias to use for the calculated rank score. This is the attribute you will use to access the score if ``with_score=True``. :param bool explicit_ordering: Order using full SQL function to calculate rank, as opposed to simply referencing the score alias in the ORDER BY clause. Shorthand way of searching for a term and sorting results by the quality of the match using the BM25 algorithm. .. attention:: The BM25 ranking algorithm is only available for FTS4. If you are using FTS3, use the :meth:`~FTSModel.search` method instead. .. classmethod:: search_bm25f(term, weights=None, with_score=False, score_alias='score', explicit_ordering=False) Same as :meth:`FTSModel.search_bm25`, but using the BM25f variant of the BM25 ranking algorithm. .. classmethod:: search_lucene(term, weights=None, with_score=False, score_alias='score', explicit_ordering=False) Same as :meth:`FTSModel.search_bm25`, but using the result ranking algorithm from the Lucene search engine. .. classmethod:: rank(col1_weight, col2_weight...coln_weight) :param float col_weight: (Optional) weight to give to the *ith* column of the model. By default all columns have a weight of ``1.0``. Generate an expression that will calculate and return the quality of the search match. This ``rank`` can be used to sort the search results. The ``rank`` function accepts optional parameters that allow you to specify weights for the various columns. If no weights are specified, all columns are considered of equal importance. The algorithm used by :meth:`~FTSModel.rank` is simple and relatively quick. For more sophisticated result ranking, use: * :meth:`~FTSModel.bm25` * :meth:`~FTSModel.bm25f` * :meth:`~FTSModel.lucene` .. code-block:: python query = (DocumentIndex .select( DocumentIndex, DocumentIndex.rank().alias('score')) .where(DocumentIndex.match('search phrase')) .order_by(DocumentIndex.rank())) for search_result in query: print(search_result.title, search_result.score) .. classmethod:: bm25(col1_weight, col2_weight...coln_weight) :param float col_weight: (Optional) weight to give to the *ith* column of the model. By default all columns have a weight of ``1.0``. Generate an expression that will calculate and return the quality of the search match using the `BM25 algorithm `_. This value can be used to sort the search results. Like :meth:`~FTSModel.rank`, ``bm25`` function accepts optional parameters that allow you to specify weights for the various columns. If no weights are specified, all columns are considered of equal importance. The BM25 result ranking algorithm requires FTS4. If you are using FTS3, use :meth:`~FTSModel.rank` instead. .. code-block:: python query = (DocumentIndex .select( DocumentIndex, DocumentIndex.bm25().alias('score')) .where(DocumentIndex.match('search phrase')) .order_by(DocumentIndex.bm25())) for search_result in query: print(search_result.title, search_result.score) The above code example is equivalent to calling the :meth:`~FTSModel.search_bm25` method: .. code-block:: python query = DocumentIndex.search_bm25('search phrase', with_score=True) for search_result in query: print(search_result.title, search_result.score) .. classmethod:: bm25f(col1_weight, col2_weight...coln_weight) Identical to :meth:`~FTSModel.bm25`, except that it uses the BM25f variant of the BM25 ranking algorithm. .. classmethod:: lucene(col1_weight, col2_weight...coln_weight) Identical to :meth:`~FTSModel.bm25`, except that it uses the Lucene search result ranking algorithm. .. classmethod:: rebuild() Rebuild the search index. Only valid when the ``content`` option was specified (content tables). .. classmethod:: optimize() Optimize the index. .. _sqlite-fts5: FTS5 / ``FTS5Model`` ^^^^^^^^^^^^^^^^^^^^ FTS5Model enables Peewee applications to store data in an efficient full-text search index using SQLite `FTS5 `_. FTS5 also comes with native BM25 result ranking. FTS5Model caveats: * All queries **except** ``MATCH`` and ``rowid`` lookup require a full table scan. * Constraints, foreign-keys, and indexes are not supported. All columns **must** be instances of :class:`SearchField`. * FTS5Model ``rowid`` primary key may be declared using :class:`RowIDField`. Lookups on the ``rowid`` are very efficient. .. tip:: Because of the lack of secondary indexes, it usually makes sense to treat the ``FTS5Model.rowid`` primary key as a foreign-key to a row in a normal SQLite table. Example: .. code-block:: python from peewee import * from playhouse.sqlite_ext import FTS5Model, SearchField db = SqliteDatabase('app.db') class Document(Model): # Canonical source of data, stored in a normal table. author = ForeignKeyField(User, backref='documents') title = TextField(null=False, unique=True) content = TextField(null=False) timestamp = DateTimeField() class Meta: database = db class DocumentIndex(FTS5Model): # Full-text search index. rowid = RowIDField() title = SearchField() content = SearchField() author = SearchField(unindexed=True) class Meta: database = db # Use the porter stemming algorithm and unicode tokenizers, # and optimize prefix matches of 3 or 4 characters. options = {'tokenize': 'porter unicode61', 'prefix': [3, 4]} # Check that FTS5 is available: if not DocumentIndex.fts5_installed(): raise RuntimeError('FTS5 is not available in this SQLite build.') Store data by inserting it into the FTS5 table: .. code-block:: python # Store a document in the index: DocumentIndex.create( rowid=document.id, # Set rowid to match Document's id. title=document.title, content=document.content, author=document.author.get_full_name()) # Equivalent: (DocumentIndex .insert({ 'rowid': document.id, 'title': document.title, 'content': document.content, 'author': document.author.get_full_name()}) .execute()) :class:`FTS5Model` provides several shortcuts for full-text search queries: .. code-block:: python # Simple search (BM25, ordered by relevance): results = DocumentIndex.search('python sqlite') # With score and per-column weighting: results = DocumentIndex.search( 'python sqlite', weights={'title': 2.0, 'content': 1.0}, with_score=True, score_alias='relevance') for r in results: print(r.title, r.relevance) # Highlight matches in the title: for r in (DocumentIndex.search('python') .select(DocumentIndex.title.highlight('[', ']').alias('hi'))): print(r.hi) # e.g. "Learn [python] the hard way" .. tip:: An important method of searching relies on the ``rowid`` of the indexed data matching the document's canonical id. Using this technique we can apply additional filters and retrieve the matching ``Document`` objects efficiently: .. code-block:: python # Search and ensure we only retrieve articles from the last 30 days. cutoff = datetime.datetime.now() - datetime.timedelta(days=30) query = (Document .select() .join( DocumentIndex, on=(Document.id == DocumentIndex.rowid)) .where( (Document.timestamp >= cutoff) & DocumentIndex.match('python sqlite')) .order_by(DocumentIndex.rank())) If the primary source of the content you are indexing exists in a separate table, you can save some disk space by instructing SQLite to not store an additional copy of the search index content. See :ref:`External Content ` for implementation details. The `FTS5 documentation `_ has more information. .. class:: FTS5Model() Inherits all :class:`FTSModel` methods plus. Supports the following options: * ``content``: :class:`Model` containing external content, or empty string for "contentless" * ``content_rowid``: :class:`Field` (external content primary key) * ``prefix``: integer(s). Ex: '2' or ``[2, 3]`` * ``tokenize``: simple, porter, unicode61. Ex: 'porter unicode61' Example: .. code-block:: python class DocumentIndex(FTS5Model): title = SearchField() body = SearchField() class Meta: database = db options = { 'tokenize': 'porter unicode61', 'prefix': '3', } .. classmethod:: fts5_installed() Return ``True`` if FTS5 is available. .. classmethod:: match(term) :param term: Search term or expression. `FTS5 syntax documentation `__. Generate a SQL expression representing a search for the given term or expression in the table. SQLite uses the ``MATCH`` operator to indicate a full-text search. Example: .. code-block:: python # Search index for "search phrase" and return results ranked # by relevancy using the BM25 algorithm. query = (DocumentIndex .select() .where(DocumentIndex.match('search phrase')) .order_by(DocumentIndex.rank())) for result in query: print('Result: %s' % result.title) .. classmethod:: search(term, weights=None, with_score=False, score_alias='score') :param term: Search term or expression. `FTS5 syntax documentation `__. :param weights: A list of weights for the columns, ordered with respect to the column's position in the table. **Or**, a dictionary keyed by the field or field name and mapped to a value. :param with_score: Whether the score should be returned as part of the ``SELECT`` statement. :param str score_alias: Alias to use for the calculated rank score. This is the attribute you will use to access the score if ``with_score=True``. :param bool explicit_ordering: Order using full SQL function to calculate rank, as opposed to simply referencing the score alias in the ORDER BY clause. Shorthand way of searching for a term and sorting results by the quality of the match. The ``FTS5`` extension provides a built-in implementation of the BM25 algorithm, which is used to rank the results by relevance. .. code-block:: python # Simple search. docs = DocumentIndex.search('search term') for result in docs: print(result.title) # More complete example. docs = DocumentIndex.search( 'search term', weights={'title': 2.0, 'content': 1.0}, with_score=True, score_alias='search_score') for result in docs: print(result.title, result.search_score) .. classmethod:: search_bm25(term, weights=None, with_score=False, score_alias='score') With FTS5, :meth:`~FTS5Model.search_bm25` is identical to the :meth:`~FTS5Model.search` method. .. classmethod:: rank(col1_weight, col2_weight...coln_weight) :param float col_weight: (Optional) weight to give to the *ith* column of the model. By default all columns have a weight of ``1.0``. Generate an expression that will calculate and return the quality of the search match using the `BM25 algorithm `_. This value can be used to sort the search results. The :meth:`~FTS5Model.rank` function accepts optional parameters that allow you to specify weights for the various columns. If no weights are specified, all columns are considered of equal importance. .. code-block:: python query = (DocumentIndex .select( DocumentIndex, DocumentIndex.rank().alias('score')) .where(DocumentIndex.match('search phrase')) .order_by(DocumentIndex.rank())) for search_result in query: print(search_result.title, search_result.score) The above code example is equivalent to calling the :meth:`~FTS5Model.search` method: .. code-block:: python query = DocumentIndex.search('search phrase', with_score=True) for search_result in query: print(search_result.title, search_result.score) .. classmethod:: bm25(col1_weight, col2_weight...coln_weight) Because FTS5 provides built-in support for BM25, this method is identical to :meth:`~FTS5Model.rank` method. .. classmethod:: VocabModel(table_type='row'|'col'|'instance', table_name=None) :param str table_type: Either 'row', 'col' or 'instance'. :param table_name: Name for the vocab table. If not specified, will be "fts5tablename_v". Generate a model class suitable for accessing the `vocab table `_ corresponding to FTS5 search index. .. classmethod:: rebuild() Rebuild the search index. Only valid when the ``content`` option was specified (content tables). .. classmethod:: optimize() Optimize the index. .. _sqlite-udf: User-Defined Function Collection --------------------------------- .. module:: playhouse.sqlite_udf The ``playhouse.sqlite_udf`` contains a number of functions, aggregates, and table-valued functions grouped into named collections. .. code-block:: python from playhouse.sqlite_udf import register_all, register_groups from playhouse.sqlite_udf import DATE, STRING db = SqliteDatabase('my_app.db') register_all(db) # Register every function. register_groups(db, DATE, STRING) # Register selected groups. # Register individual functions: from playhouse.sqlite_udf import gzip, gunzip db.register_function(gzip, 'gzip') db.register_function(gunzip, 'gunzip') Once registered, call functions via Peewee's ``fn`` namespace or raw SQL: .. code-block:: python # Find most common URL hostnames. query = (Link .select(fn.hostname(Link.url).alias('host'), fn.COUNT(Link.id)) .group_by(fn.hostname(Link.url)) .order_by(fn.COUNT(Link.id).desc()) .tuples()) Available functions ^^^^^^^^^^^^^^^^^^^ **CONTROL_FLOW** .. function:: if_then_else(cond, truthy, falsey=None) Simple ternary-type operator, where, depending on the truthiness of the ``cond`` parameter, either the ``truthy`` or ``falsey`` value will be returned. **DATE** .. function:: strip_tz(date_str) :param date_str: A datetime, encoded as a string. :returns: The datetime with any timezone info stripped off. The time is not adjusted in any way, the timezone is simply removed. .. function:: humandelta(nseconds, glue=', ') :param int nseconds: Number of seconds, total, in timedelta. :param str glue: Fragment to join values. :returns: Easy-to-read description of timedelta. Example, 86471 -> "1 day, 1 minute, 11 seconds" .. function:: mintdiff(datetime_value) :param datetime_value: A date-time. :returns: Minimum difference between any two values in list. *Aggregate*: minimum difference between any two datetimes. .. function:: avgtdiff(datetime_value) :param datetime_value: A date-time. :returns: Average difference between values in list. *Aggregate*: average difference between consecutive values. .. function:: duration(datetime_value) :param datetime_value: A date-time. :returns: Duration from smallest to largest value in list, in seconds. *Aggregate*: duration from the smallest to the largest value, in seconds. .. function:: date_series(start, stop, step_seconds=86400) :param datetime start: Start datetime :param datetime stop: Stop datetime :param int step_seconds: Number of seconds comprising a step. *Table-value function*: returns rows consisting of the date/+time values encountered iterating from start to stop, ``step_seconds`` at a time. Additionally, if start does not have a time component and step_seconds is greater-than-or-equal-to one day (86400 seconds), the values returned will be dates. Conversely, if start does not have a date component, values will be returned as times. Otherwise values are returned as datetimes. Example: .. code-block:: sql SELECT * FROM date_series('2017-01-28', '2017-02-02'); value ----- 2017-01-28 2017-01-29 2017-01-30 2017-01-31 2017-02-01 2017-02-02 **FILE** .. function:: file_ext(filename) :param str filename: Filename to extract extension from. :return: Returns the file extension, including the leading ".". .. function:: file_read(filename) :param str filename: Filename to read. :return: Contents of the file. **HELPER** .. function:: gzip(data, compression=9) :param bytes data: Data to compress. :param int compression: Compression level (9 is max). :returns: Compressed binary data. .. function:: gunzip(data) :param bytes data: Compressed data. :returns: Uncompressed binary data. .. function:: hostname(url) :param str url: URL to extract hostname from. :returns: hostname portion of URL .. function:: toggle(key) :param key: Key to toggle. Toggle a key between True/False state. Example: .. code-block:: pycon >>> toggle('my-key') True >>> toggle('my-key') False >>> toggle('my-key') True .. function:: setting(key, value=None) :param key: Key to set/retrieve. :param value: Value to set. :returns: Value associated with key. Store/retrieve a setting in memory and persist during lifetime of application. To get the current value, specify key. To set a new value, call with key and new value. .. function:: clear_toggles() Clears all state associated with the :func:`toggle` function. .. function:: clear_settings() Clears all state associated with the :func:`setting` function. **MATH** .. function:: randomrange(start, stop=None, step=None) :param int start: Start of range (inclusive) :param int end: End of range(not inclusive) :param int step: Interval at which to return a value. Return a random integer between ``[start, end)``. .. function:: gauss_distribution(mean, sigma) :param float mean: Mean value :param float sigma: Standard deviation .. function:: sqrt(n) Calculate the square root of ``n``. .. function:: tonumber(s) :param str s: String to convert to number. :returns: Integer, floating-point or NULL on failure. .. function:: mode(val) :param val: Numbers in list. :returns: The mode, or most-common, number observed. *Aggregate*: calculates *mode* of values. .. function:: minrange(val) :param val: Value :returns: Min difference between two values. *Aggregate*: minimum distance between two numbers in the sequence. .. function:: avgrange(val) :param val: Value :returns: Average difference between values. *Aggregate*: average distance between consecutive numbers in the sequence. .. function:: range(val) :param val: Value :returns: The range from the smallest to largest value in sequence. *Aggregate*: range of values observed. .. function:: median(val) :param val: Value :returns: The median, or middle, value in a sequence. *Aggregate*: median value of a sequence. .. note:: Only available if you compiled the ``_sqlite_udf`` extension. **STRING** .. function:: substr_count(haystack, needle) Returns number of times ``needle`` appears in ``haystack``. .. function:: strip_chars(haystack, chars) Strips any characters in ``chars`` from beginning and end of ``haystack``. .. function:: damerau_levenshtein_dist(s1, s2) Computes the edit distance from s1 to s2 using the damerau variant of the levenshtein algorithm. .. note:: Only available if you compiled the ``_sqlite_udf`` extension. .. function:: levenshtein_dist(s1, s2) Computes the edit distance from s1 to s2 using the levenshtein algorithm. .. note:: Only available if you compiled the ``_sqlite_udf`` extension. .. function:: str_dist(s1, s2) Computes the edit distance from s1 to s2 using the standard library SequenceMatcher's algorithm. .. note:: Only available if you compiled the ``_sqlite_udf`` extension. .. function:: regex_search(regex, search_string) :param str regex: Regular expression :param str search_string: String to search for instances of regex. *Table-value function*: searches a string for substrings that match the provided ``regex``. Returns rows for each match found. Example: .. code-block:: python SELECT * FROM regex_search('\w+', 'extract words, ignore! symbols'); value ----- extract words ignore symbols ================================================ FILE: docs/peewee/transactions.rst ================================================ .. _transactions: Transactions ============ A *transaction* groups one or more SQL statements into a single unit of work. Either all of the statements succeed and are committed to the database, or none of them are - the database rolls back to the state it was in before the transaction began. Peewee operates in *autocommit mode*: every statement that runs outside an explicit transaction runs in its own implicit transaction. To group statements, use the tools described in this document. db.atomic --------- :meth:`Database.atomic` is the recommended transaction API. :meth:`~Database.atomic` can be used as a context manager or a decorator, and it handles nesting automatically. If an unhandled exception occurs in a wrapped block, the current block will be rolled back. Otherwise the statements will be committed at the end of the block. As a context manager: .. code-block:: python with db.atomic() as txn: user = User.create(username='charlie') tweet = Tweet.create(user=user, content='Hello') # Both rows are committed when block exits normally. As a decorator: .. code-block:: python @db.atomic() def create_user_with_tweet(username, content): user = User.create(username=username) Tweet.create(user=user, content=content) return user If an unhandled exception propagates out of the block, the transaction (or savepoint - see below) is rolled back and the exception continues to propagate: .. code-block:: python with db.atomic() as txn: User.create(username='huey') # User has been INSERTed into the database but the transaction is not # yet committed because we haven't left the scope of the "with" block. raise ValueError('something went wrong') # This exception is unhandled - the transaction will be rolled-back and # the ValueError will be raised. # User('huey') was NOT committed, the transaction rolled-back. # The ValueError is raised here. Manual Commit / Rollback ------------------------ You can commit or roll-back explicitly inside an :meth:`~Database.atomic` block. After calling :meth:`~Transaction.commit` or :meth:`~Transaction.rollback` a new transaction (or savepoint) begins automatically: .. code-block:: python with db.atomic() as txn: try: save_objects() except SaveError: txn.rollback() # Roll back, new transaction starts automatically. log_error() finalize() # Runs in a new transaction. # finalize()'s changes are committed here. Nesting Transactions -------------------- The outermost :meth:`~Database.atomic` block creates a transaction. Any nested ``atomic()`` blocks create *savepoints* instead. A savepoint is a named point within a transaction to which you can roll back without affecting the rest of the transaction. .. code-block:: python with db.atomic(): # Transaction begins. User.create(username='charlie') with db.atomic() as sp: # Savepoint begins. User.create(username='huey') sp.rollback() # Rolls back huey only. User.create(username='alice') # New savepoint begins here. User.create(username='mickey') # Committed: charlie, alice, mickey. huey was rolled back. Savepoints can be nested arbitrarily deep: .. code-block:: python with db.atomic(): with db.atomic(): with db.atomic() as inner: do_something_risky() inner.rollback() # Only the innermost work is lost. do_something_safe() ``atomic()`` tracks the nesting depth internally. You do not need to manage savepoint names or transaction state manually. Explicit Transaction -------------------- :meth:`Database.transaction` opens an explicit transaction that does not nest. Any ``transaction()`` call inside an outer ``transaction()`` block is ignored - only the outermost transaction is active. Use this only when you explicitly need a flat, non-nesting transaction. For most cases, ``atomic()`` is the better choice. If an exception occurs in a wrapped block, the transaction will be rolled back. Otherwise the statements will be committed at the end of the wrapped block. .. code-block:: python with db.transaction() as txn: User.create(username='mickey') txn.commit() # Commit now; a new transaction begins. User.create(username='huey') txn.rollback() # Roll back huey; a new transaction begins. User.create(username='zaizee') # zaizee is committed when the block exits. If you attempt to nest transactions with peewee using the :meth:`~Database.transaction` context manager, only the outer-most transaction will be used. As this may lead to unpredictable behavior, it is recommended that you use :meth:`~Database.atomic`. Explicit Savepoints ------------------- :meth:`Database.savepoint` creates a savepoint within an active transaction. Savepoints must occur within a transaction, but can be nested arbitrarily deep. .. code-block:: python with db.transaction() as txn: with db.savepoint() as sp: User.create(username='mickey') with db.savepoint() as sp2: User.create(username='zaizee') sp2.rollback() # "zaizee" is not saved. User.create(username='huey') # mickey and huey were created. If you manually commit or roll back a savepoint, a new savepoint will automatically begin. Autocommit Mode --------------- Peewee requires the underlying driver to run in autocommit mode and manages transaction boundaries itself. This differs from the DB-API 2.0 default, which starts a transaction implicitly and requires you to commit manually. As a result, Peewee puts all DB-API drivers into *autocommit* mode. In rare cases where you need to take direct control of ``BEGIN``/``COMMIT``/ ``ROLLBACK`` - bypassing Peewee's transaction management entirely - use :meth:`~Database.manual_commit`: .. code-block:: python with db.manual_commit(): db.begin() # Begin transaction explicitly. try: user.delete_instance(recursive=True) except: db.rollback() # Rollback! An error occurred. raise else: try: db.commit() # Commit changes. except: db.rollback() raise ``manual_commit`` suspends Peewee's transaction management for the duration of the block. ``atomic()`` and ``transaction()`` have no effect inside it. This should rarely be needed in application code. .. _sqlite-locking: SQLite Transaction Locking Modes ---------------------------------- SQLite supports three locking modes for transactions. Use these when precise control over read-write locking is required: .. code-block:: python with db.atomic('EXCLUSIVE'): # No other connection can read or write until this commits. do_something() @db.atomic('IMMEDIATE') def load_data(): # No other writer is allowed, but readers can proceed. insert_records() The three modes: * **DEFERRED** (default) - acquires the minimum necessary lock as reads and writes occur. Another writer can intervene between BEGIN and your first write. * **IMMEDIATE** - acquires a write reservation lock at BEGIN. Other writers are blocked; readers can proceed. * **EXCLUSIVE** - acquires an exclusive lock at BEGIN. No other connection can read or write until the transaction completes. .. seealso:: `SQLite locking documentation `_. .. _postgres-isolation: Postgresql Isolation Notes -------------------------- Postgresql supports configurable isolation levels per-transaction, from least to most strict: * READ UNCOMMITTED * READ COMMITTED (default in most deployments) * REPEATABLE READ * SERIALIZABLE See the `Postgresql transaction isolation docs `__ for discussion. The default isolation level is specified when initializing :class:`PostgresqlDatabase`: .. code-block:: python db = PostgresqlDatabase( 'my_app', user='postgres', host='10.8.0.1', port=5432, isolation_level='SERIALIZABLE') # Or use the constants provided by the driver. from psycopg2.extensions import ISOLATION_LEVEL_SERIALIZABLE db = PostgresqlDatabase( ... isolation_level=ISOLATION_LEVEL_SERIALIZABLE) from psycopg import IsolationLevel db = PostgresqlDatabase( ... isolation_level=IsolationLevel.SERIALIZABLE) To control the isolation-level for a transaction, you can pass the desired setting to the outer-most ``atomic()`` block: .. code-block:: python with db.atomic('SERIALIZABLE') as txn: ... Isolation level cannot be specified for nested ``atomic()`` blocks. ================================================ FILE: docs/peewee/writing.rst ================================================ .. _writing: Writing Data ============ This document covers INSERT, UPDATE, and DELETE queries. Reading data is covered in :ref:`querying`. All examples use the canonical schema from :ref:`models`: .. code-block:: python import datetime from peewee import * db = SqliteDatabase(':memory:') class BaseModel(Model): class Meta: database = db class User(BaseModel): username = TextField(unique=True) class Tweet(BaseModel): user = ForeignKeyField(User, backref='tweets') content = TextField() timestamp = DateTimeField(default=datetime.datetime.now) is_published = BooleanField(default=True) Methods which will be discussed: +-----------------+-----------------------------------------------------------+ | Query | Methods | +=================+===========================================================+ | INSERT | * :meth:`Model.create` | | | * :meth:`Model.insert` | | | * :meth:`Model.insert_many` | | | * :meth:`Model.insert_from` | | | * :meth:`Model.replace` (Postgres unsupported) | | | * :meth:`Model.replace_many` (Postgres unsupported) | +-----------------+-----------------------------------------------------------+ | UPDATE | * :meth:`Model.save` | | | * :meth:`Model.update` | +-----------------+-----------------------------------------------------------+ | DELETE | * :meth:`Model.delete_instance` | | | * :meth:`Model.delete` | +-----------------+-----------------------------------------------------------+ .. seealso:: :ref:`Extensive library of SQL / Peewee examples ` .. _inserting-records: Inserting Records ----------------- Creating a single row ^^^^^^^^^^^^^^^^^^^^^ :meth:`~Model.create` inserts a row and returns the saved model instance: .. code-block:: pycon >>> charlie = User.create(username='charlie') >>> charlie.id 1 This will INSERT a new row into the database. The primary key will automatically be retrieved and stored on the model instance. Alternatively, instantiate the model and call :meth:`~Model.save`. The first call to ``save()`` on a new instance performs an INSERT: .. code-block:: pycon >>> user = User(username='huey') >>> user.save() 1 # Returns number of rows modified. >>> user.id 2 After the first save, the model instance holds its primary key. Any subsequent call to ``save()`` performs an UPDATE instead: .. code-block:: pycon >>> user.username = 'Huey' >>> user.save() 1 # Returns number of rows updated. For a foreign key field, pass either the related model instance or its raw primary key value: .. code-block:: python Tweet.create(user=huey, content='Hello!') Tweet.create(user=2, content='Also valid.') To insert without constructing a model instance, use :meth:`~Model.insert`. It returns the primary key of the new row: .. code-block:: pycon >>> User.insert(username='mickey').execute() 3 .. _bulk-inserts: Bulk Inserts ------------ Calling :meth:`Model.create` or :meth:`Model.save` in a loop should be avoided: .. code-block:: python data = [ {'username': 'alice'}, {'username': 'bob'}, {'username': 'carol'}, ] for data_dict in data: User.create(**data_dict) The above is slow: 1. **Does not wrap the loop in a transaction.** Result is each :meth:`~Model.create` happens in its own :ref:`transaction `. 2. **Python interpreter** is getting in the way, and each :class:`Insert` must be generated and parsed into SQL. 3. **Large amount of data** (in terms of raw bytes of SQL) may be sent to the database to parse. 4. **Retrieving the last insert id**, which may not be necessary. You can get a significant speedup by simply wrapping this in a transaction with :meth:`~Database.atomic`: .. code-block:: python :emphasize-lines: 1 with db.atomic(): for data_dict in data: User.create(**data_dict) The fastest way to insert many rows is :meth:`~Model.insert_many`. It accepts a list of dicts or tuples and emits a single multi-row INSERT: .. code-block:: python data = [ {'username': 'alice'}, {'username': 'bob'}, {'username': 'carol'}, ] User.insert_many(data).execute() # Tuples require an explicit field list: data = [('alice',), ('bob',), ('carol',)] User.insert_many(data, fields=[User.username]).execute() Optionally wrap the bulk insert in a transaction: .. code-block:: python with db.atomic(): User.insert_many(data, fields=fields).execute() Insert queries support :meth:`~WriteQuery.returning` with Postgresql and SQLite to obtain the inserted rows: .. code-block:: python query = (User .insert_many([{'username': 'alice'}, {'username': 'bob'}]) .returning(User)) for user in query: print(f'Added {user.username} with id = {user.id}') Batching large data sets ^^^^^^^^^^^^^^^^^^^^^^^^ Depending on the number of rows in your data source, you may need to break it up into chunks. SQLite in particular may have a `limit of 32766 `_ variables-per-query (batch size would then be 32766 // row length). You can write a loop to batch your data into chunks. It is **strongly recommended** you use a :ref:`transaction `: .. code-block:: python from peewee import chunked with db.atomic(): for batch in chunked(data, 100): User.insert_many(batch).execute() :func:`chunked` works on any iterable, including generators. Bulk-creating model instances ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :meth:`~Model.bulk_create` accepts a list of unsaved model instances and inserts them efficiently. Pass ``batch_size`` to avoid hitting database limits: .. code-block:: python users = [User(username=f'user_{i}') for i in range(1000)] with db.atomic(): User.bulk_create(users, batch_size=100) If you are using Postgresql (which supports the ``RETURNING`` clause), then the previously-unsaved model instances will have their new primary key values automatically populated. Other backends will not. Loading from another table ^^^^^^^^^^^^^^^^^^^^^^^^^^ :meth:`~Model.insert_from` generates an ``INSERT INTO ... SELECT`` query, copying rows from one table into another without round-tripping data through Python: .. code-block:: python res = (TweetArchive .insert_from( Tweet.select(Tweet.user, Tweet.message), fields=[TweetArchive.user, TweetArchive.message]) .execute()) The above query is equivalent to the following SQL: .. code-block:: sql INSERT INTO "tweet_archive" ("user_id", "message") SELECT "user_id", "message" FROM "tweet"; .. _updating-records: Updating Records ---------------- Updating a model instance ^^^^^^^^^^^^^^^^^^^^^^^^^^ Modify attributes on a fetched instance and call :meth:`~Model.save` to persist the changes: .. code-block:: python charlie = User.get(User.username == 'charlie') charlie.username = 'charlie_admin' charlie.save() # Issues UPDATE WHERE id = charlie.id By default, ``save()`` re-saves all fields. To only emit changed fields, set ``only_save_dirty = True`` in the model's ``Meta``, or pass only the fields you want to update: .. code-block:: python charlie.username = 'charlie_v2' charlie.save(only=[User.username]) If a model instance does not have a primary key, the first call to :meth:`~Model.save` will perform an INSERT query. Once a model instance has a primary key, subsequent calls to :meth:`~Model.save` result in an *UPDATE*. Updating multiple rows ^^^^^^^^^^^^^^^^^^^^^^ :meth:`~Model.update` issues a single UPDATE that affects every row matching the WHERE clause: .. code-block:: python # Publish all unpublished tweets older than one week. one_week_ago = datetime.datetime.now() - datetime.timedelta(days=7) nrows = (Tweet .update(is_published=True) .where( (Tweet.is_published == False) & (Tweet.timestamp < one_week_ago)) .execute()) The return value is the number of rows affected. Update queries support :meth:`~WriteQuery.returning` with Postgresql and SQLite to obtain the updated rows: .. code-block:: python query = (User .update(spam=True) .where(User.username.contains('billing')) .returning(User)) for user in query: print(f'Marked {user.username} as spam') Because UPDATE queries do not support joins, we can use subqueries to update rows based on values in related tables. For example, unpublish all tweets by users with ``'billing'`` in their username: .. code-block:: python spammers = User.select().where(User.username.contains('billing')) (Tweet .update(is_published=False) .where(Tweet.user.in_(spammers)) .execute()) Atomic updates ^^^^^^^^^^^^^^ Use column expressions in ``update()`` to modify values without a read-modify-write cycle. Performing updates atomically prevents race-conditions: .. code-block:: python # WRONG: reads each row into Python, increments, then saves. # Vulnerable to race conditions; slow on many rows. for stat in Stat.select().where(Stat.url == url): stat.counter += 1 stat.save() # CORRECT: single UPDATE statement, atomic at the database level. Stat.update(counter=Stat.counter + 1).where(Stat.url == url).execute() Any SQL expression is valid on the right-hand side: .. code-block:: python # Give every employee a 10% salary bonus added to their existing bonus. Employee.update(bonus=Employee.bonus + (Employee.salary * 0.10)).execute() # Denormalize a count column from a subquery. tweet_count = (Tweet .select(fn.COUNT(Tweet.id)) .where(Tweet.user == User.id)) User.update(num_tweets=tweet_count).execute() Bulk-updating model instances ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When you have a list of modified model instances and want to update specific fields across all of them in one query, use :meth:`~Model.bulk_update`: .. code-block:: python u1, u2, u3 = User.select().limit(3) u1.username = 'u1-new' u2.username = 'u2-new' u3.username = 'u3-new' User.bulk_update([u1, u2, u3], fields=[User.username]) This emits a single UPDATE using a SQL ``CASE`` expression. For large lists, specify a ``batch_size`` and wrap in a transaction: .. code-block:: python with db.atomic(): User.bulk_update(users, fields=[User.username], batch_size=50) ``bulk_update`` may be slower than a direct UPDATE query when the list is very large, because the generated ``CASE`` expression grows proportionally. For updates that can be expressed as a single WHERE clause, the direct :meth:`~Model.update` approach is faster. .. _upsert: Upsert ------ An *upsert* (INSERT or UPDATE) inserts a new row, or if a unique constraint would be violated, updates the existing row instead. Peewee provides two complementary approaches. ``on_conflict_replace`` - SQLite and MySQL ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ SQLite and MySQL support a ``REPLACE`` query, which will replace the row in the event of a conflict: .. code-block:: python class User(BaseModel): username = TextField(unique=True) last_login = DateTimeField(null=True) # Insert, or replace the entire existing row. User.replace(username='huey', last_login=datetime.datetime.now()).execute() # Equivalent using insert(): (User .insert(username='huey', last_login=datetime.datetime.now()) .on_conflict_replace() .execute()) ``replace`` deletes and re-inserts, which changes the primary key. Use ``on_conflict`` (below) when the primary key must be preserved, or when only some columns should be updated. ``on_conflict`` - all backends ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The :meth:`~Insert.on_conflict` method is much more powerful. .. code-block:: python class User(BaseModel): username = TextField(unique=True) last_login = DateTimeField(null=True) login_count = IntegerField(default=0) now = datetime.datetime.now() (User .insert(username='huey', last_login=now, login_count=1) .on_conflict( # Postgresql and SQLite require identifying the conflicting constraint. # MySQL does not need this. conflict_target=[User.username], # Columns whose values should come from the incoming row: preserve=[User.last_login], # Columns to update using an expression: update={User.login_count: User.login_count + 1}) .execute()) Calling this query repeatedly will increment ``login_count`` atomically and update ``last_login`` on each call, without creating duplicate rows. The ``EXCLUDED`` namespace references the values that would have been inserted if the constraint had not fired. This allows conditional updates: .. code-block:: python class KV(BaseModel): key = TextField(unique=True) value = IntegerField() KV.create(key='k1', value=1) # Demonstrate usage of EXCLUDED. # Here we will attempt to insert a new value for a given key. If that # key already exists, then we will update its value with the *sum* of its # original value and the value we attempted to insert -- provided that # the new value is larger than the original value. query = (KV.insert(key='k1', value=10) .on_conflict(conflict_target=[KV.key], update={KV.value: KV.value + EXCLUDED.value}, where=(EXCLUDED.value > KV.value))) # Executing the above query will result in the following data being # present in the "kv" table: # (key='k1', value=11) query.execute() # If we attempted to execute the query *again*, then nothing would be # updated, as the new value (10) is now less than the value in the # original row (11). There are several important concepts to understand when using ``ON CONFLICT``: * ``conflict_target=``: which column(s) have the UNIQUE constraint. For a user table, this might be the user's email (SQLite and Postgresql only). * ``preserve=``: if a conflict occurs, this parameter is used to indicate which values from the **new** data we wish to update. * ``update=``: if a conflict occurs, this is a mapping of data to apply to the pre-existing row. * ``EXCLUDED``: this "magic" namespace allows you to reference the new data that would have been inserted if the constraint hadn't failed. Full example: .. code-block:: python class User(Model): email = CharField(unique=True) # Unique identifier for user. last_login = DateTimeField() login_count = IntegerField(default=0) ip_log = TextField(default='') # Demonstrates the above 4 concepts. def login(email, ip): rowid = (User .insert({User.email: email, User.last_login: datetime.now(), User.login_count: 1, User.ip_log: ip}) .on_conflict( # If the INSERT fails due to a constraint violation on the # user email, then perform an UPDATE instead. conflict_target=[User.email], # Set the "last_login" to the value we would have inserted # (our call to datetime.now()). preserve=[User.last_login], # Increment the user's login count and prepend the new IP # to the user's ip history. update={User.login_count: User.login_count + 1, User.ip_log: fn.CONCAT(EXCLUDED.ip_log, ',', User.ip_log)}) .execute()) return rowid # This will insert the initial row, returning the new row id (1). print(login('test@example.com', '127.1')) # Because test@example.com exists, this will trigger the UPSERT. The row id # from above is returned again (1). print(login('test@example.com', '127.2')) u = User.get() print(u.login_count, u.ip_log) # Prints "2 127.2,127.1" .. seealso:: :meth:`Insert.on_conflict` and :class:`OnConflict`. ``on_conflict_ignore`` ^^^^^^^^^^^^^^^^^^^^^^ Insert the row, and silently do nothing if a constraint would be violated: .. code-block:: python # Insert if username does not exist; ignore if it does. User.insert(username='huey').on_conflict_ignore().execute() Supported by SQLite, MySQL, and Postgresql. .. _deleting-records: Deleting Records ---------------- Delete a single fetched instance with :meth:`~Model.delete_instance`: .. code-block:: python tweet = Tweet.get_by_id(42) tweet.delete_instance() # Returns number of rows deleted. To delete a row along with all dependent rows (rows in other tables that reference it via foreign key), pass ``recursive=True``: .. code-block:: python # Deletes the user and all their tweets, favorites, etc. with db.atomic(): user.delete_instance(recursive=True) ``recursive=True`` works by querying for dependent rows and deleting them first - it does not rely on ``ON DELETE CASCADE``. For large graphs of related data, this can be slow. Be sure to wrap calls in a :ref:`transaction ` and consider using database-level cascade constraints on the foreign keys. To delete an arbitrary set of rows without fetching them: .. code-block:: python # Delete all unpublished tweets older than 30 days. cutoff = datetime.datetime.now() - datetime.timedelta(days=30) nrows = (Tweet .delete() .where( (Tweet.is_published == False) & (Tweet.timestamp < cutoff)) .execute()) Delete queries support :meth:`~WriteQuery.returning` with Postgresql and SQLite to obtain the deleted rows: .. code-block:: python query = (User .delete() .where(User.username.contains('billing')) .returning(User)) for user in query: print(f'Deleted: {user.username}') Because DELETE queries do not support joins, we can use subqueries to delete rows based on values in related tables. For example, delete all tweets by users with ``'billing'`` in their username: .. code-block:: python spammers = User.select().where(User.username.contains('billing')) (Tweet .delete() .where(Tweet.user.in_(spammers)) .execute()) .. seealso:: * :meth:`Model.delete_instance` * :meth:`Model.delete` * :class:`Delete` .. _returning-clause: Returning Clause ---------------- :class:`PostgresqlDatabase` and :class:`SqliteDatabase` (3.35.0+) support a ``RETURNING`` clause on ``UPDATE``, ``INSERT`` and ``DELETE`` queries. Specifying a ``RETURNING`` clause allows you to iterate over the rows accessed by the query. By default, the return values upon execution of the different queries are: * ``INSERT`` - auto-incrementing primary key value of the newly-inserted row. When not using an auto-incrementing primary key, Postgres will return the new row's primary key, but SQLite and MySQL will not. * ``UPDATE`` - number of rows modified * ``DELETE`` - number of rows deleted When a returning clause is used the return value upon executing a query will be an iterable cursor object, providing access to data that was inserted, updated or deleted by the query. For example, let's say you have an :class:`Update` that deactivates all user accounts whose registration has expired. After deactivating them, you want to send each user an email letting them know their account was deactivated. Rather than writing two queries, a ``SELECT`` and an ``UPDATE``, you can do this in a single ``UPDATE`` query with a ``RETURNING`` clause: .. code-block:: python query = (User .update(is_active=False) .where(User.registration_expired == True) .returning(User)) # Send an email to every user that was deactivated. for deactivate_user in query.execute(): send_deactivation_email(deactivated_user.email) query = (User .delete() .where(User.is_spam == True) .returning(User.id)) for user in query.execute(): print(f'Deleted spam user id: {user.id}') The ``RETURNING`` clause is available on: * :class:`Insert` * :class:`Update` * :class:`Delete` As another example, let's add a user and set their creation-date to the server-generated current timestamp. We'll create and retrieve the new user's ID, Email and the creation timestamp in a single query: .. code-block:: python query = (User .insert(email='foo@bar.com', created=fn.now()) .returning(User)) # Shorthand for all columns on User. # When using RETURNING, execute() returns a cursor. cursor = query.execute() # Get the user object we just inserted and log the data: user = cursor[0] logger.info('Created user %s (id=%s) at %s', user.email, user.id, user.created) By default the cursor will return :class:`Model` instances, but you can specify a different row type: .. code-block:: python data = [{'name': 'charlie'}, {'name': 'huey'}, {'name': 'mickey'}] query = (User .insert_many(data) .returning(User.id, User.username) .dicts()) for new_user in query.execute(): print('Added user "%s", id=%s' % (new_user['username'], new_user['id'])) Just as with :class:`Select` queries, you can specify various :ref:`result row types `. ================================================ FILE: docs/requirements.txt ================================================ docutils<0.18 sphinx-rtd-theme ================================================ FILE: examples/adjacency_list.py ================================================ from peewee import * db = SqliteDatabase(':memory:') class Node(Model): name = TextField() parent = ForeignKeyField('self', backref='children', null=True) class Meta: database = db def __str__(self): return self.name def dump(self, _indent=0): return (' ' * _indent + self.name + '\n' + ''.join(child.dump(_indent + 1) for child in self.children)) db.create_tables([Node]) tree = ('root', ( ('n1', ( ('c11', ()), ('c12', ()))), ('n2', ( ('c21', ()), ('c22', ( ('g221', ()), ('g222', ()))), ('c23', ()), ('c24', ( ('g241', ()), ('g242', ()), ('g243', ()))))))) stack = [(None, tree)] while stack: parent, (name, children) = stack.pop() node = Node.create(name=name, parent=parent) for child_tree in children: stack.insert(0, (node, child_tree)) # Now that we have created the stack, let's eagerly load 4 levels of children. # To show that it works, we'll turn on the query debugger so you can see which # queries are executed. import logging; logger = logging.getLogger('peewee') logger.addHandler(logging.StreamHandler()) logger.setLevel(logging.DEBUG) C = Node.alias('c') G = Node.alias('g') GG = Node.alias('gg') GGG = Node.alias('ggg') roots = Node.select().where(Node.parent.is_null()) pf = prefetch(roots, C, (G, C), (GG, G), (GGG, GG)) for root in pf: print(root.dump()) ================================================ FILE: examples/analytics/app.py ================================================ """ Example "Analytics" app. To start using this on your site, do the following: * Create a postgresql database: createdb analytics * Create an account for each domain you intend to collect analytics for, e.g. Account.create(domain='charlesleifer.com') * Update configuration values marked "TODO", e.g. DOMAIN. * Run this app using the WSGI server of your choice. * Using the appropriate account id, add a ` Take a look at `reports.py` for some interesting queries you can perform on your pageview data. """ import datetime import os from urllib.parse import parse_qsl, urlparse import binascii from flask import Flask, Response, abort, g, request from peewee import * from playhouse.postgres_ext import BinaryJSONField, PostgresqlExtDatabase # Analytics settings. # 1px gif. BEACON = binascii.unhexlify( '47494638396101000100800000dbdfef00000021f90401000000002c00000000010001000002024401003b') DATABASE_NAME = 'analytics' DOMAIN = 'http://analytics.yourdomain.com' # TODO: change me. JAVASCRIPT = """(function(id){ var d=document,i=new Image,e=encodeURIComponent; i.src='%s/a.gif?id='+id+'&url='+e(d.location.href)+'&ref='+e(d.referrer)+'&t='+e(d.title); })(%s)""".replace('\n', '') # Flask settings. DEBUG = bool(os.environ.get('DEBUG')) SECRET_KEY = 'secret - change me' # TODO: change me. app = Flask(__name__) app.config.from_object(__name__) database = PostgresqlExtDatabase(DATABASE_NAME, user='postgres') class BaseModel(Model): class Meta: database = database class Account(BaseModel): domain = CharField() def verify_url(self, url): netloc = urlparse(url).netloc url_domain = '.'.join(netloc.split('.')[-2:]) # Ignore subdomains. return self.domain == url_domain class PageView(BaseModel): account = ForeignKeyField(Account, backref='pageviews') url = TextField() timestamp = DateTimeField(default=datetime.datetime.now) title = TextField(default='') ip = CharField(default='') referrer = TextField(default='') headers = BinaryJSONField() params = BinaryJSONField() @classmethod def create_from_request(cls, account, request): parsed = urlparse(request.args['url']) params = dict(parse_qsl(parsed.query)) return PageView.create( account=account, url=parsed.path, title=request.args.get('t') or '', ip=request.headers.get('x-forwarded-for', request.remote_addr), referrer=request.args.get('ref') or '', headers=dict(request.headers), params=params) @app.route('/a.gif') def analyze(): # Make sure an account id and url were specified. if not request.args.get('id') or not request.args.get('url'): abort(404) # Ensure the account id is valid. try: account = Account.get(Account.id == request.args['id']) except Account.DoesNotExist: abort(404) # Ensure the account id matches the domain of the URL we wish to record. if not account.verify_url(request.args['url']): abort(403) # Store the page-view data in the database. PageView.create_from_request(account, request) # Return a 1px gif. response = Response(app.config['BEACON'], mimetype='image/gif') response.headers['Cache-Control'] = 'private, no-cache' return response @app.route('/a.js') def script(): account_id = request.args.get('id') if account_id: return Response( app.config['JAVASCRIPT'] % (app.config['DOMAIN'], account_id), mimetype='text/javascript') return Response('', mimetype='text/javascript') @app.errorhandler(404) def not_found(e): return Response('

Not found.

') # Request handlers -- these two hooks are provided by flask and we will use them # to create and tear down a database connection on each request. @app.before_request def before_request(): g.db = database g.db.connection() @app.after_request def after_request(response): g.db.close() return response if __name__ == '__main__': database.create_tables([Account, PageView], safe=True) app.run(debug=True) ================================================ FILE: examples/analytics/reports.py ================================================ from peewee import * from app import Account, PageView DEFAULT_ACCOUNT_ID = 1 class Report(object): def __init__(self, account_id=DEFAULT_ACCOUNT_ID): self.account = Account.get(Account.id == account_id) self.date_range = None def get_query(self): query = PageView.select().where(PageView.account == self.account) if self.date_range: query = query.where(PageView.timestamp.between(*self.date_range)) return query def top_pages_by_time_period(self, interval='day'): """ Get a breakdown of top pages per interval, i.e. day url count 2014-01-01 /blog/ 11 2014-01-02 /blog/ 14 2014-01-03 /blog/ 9 """ date_trunc = fn.date_trunc(interval, PageView.timestamp) return (self.get_query() .select( PageView.url, date_trunc.alias(interval), fn.Count(PageView.id).alias('count')) .group_by(PageView.url, date_trunc) .order_by( SQL(interval), SQL('count').desc(), PageView.url)) def cookies(self): """ Retrieve the cookies header from all the users who visited. """ return (self.get_query() .select(PageView.ip, PageView.headers['Cookie']) .where(PageView.headers['Cookie'].is_null(False)) .tuples()) def user_agents(self): """ Retrieve user-agents, sorted by most common to least common. """ return (self.get_query() .select( PageView.headers['User-Agent'], fn.Count(PageView.id)) .group_by(PageView.headers['User-Agent']) .order_by(fn.Count(PageView.id).desc()) .tuples()) def languages(self): """ Retrieve languages, sorted by most common to least common. The Accept-Languages header sometimes looks weird, i.e. "en-US,en;q=0.8,is;q=0.6,da;q=0.4" We will split on the first semi- colon. """ language = PageView.headers['Accept-Language'] first_language = fn.SubStr( language, # String to slice. 1, # Left index. fn.StrPos(language, ';')) return (self.get_query() .select(first_language, fn.Count(PageView.id)) .group_by(first_language) .order_by(fn.Count(PageView.id).desc()) .tuples()) def trail(self): """ Get all visitors by IP and then list the pages they visited in order. """ inner = (self.get_query() .select(PageView.ip, PageView.url) .order_by(PageView.timestamp)) return (PageView .select( PageView.ip, fn.array_agg(PageView.url).alias('urls')) .from_(inner.alias('t1')) .group_by(PageView.ip)) def _referrer_clause(self, domain_only=True): if domain_only: return fn.SubString(Clause( PageView.referrer, SQL('FROM'), '.*://([^/]*)')) return PageView.referrer def top_referrers(self, domain_only=True): """ What domains send us the most traffic? """ referrer = self._referrer_clause(domain_only) return (self.get_query() .select(referrer, fn.Count(PageView.id)) .group_by(referrer) .order_by(fn.Count(PageView.id).desc()) .tuples()) def referrers_for_url(self, domain_only=True): referrer = self._referrer_clause(domain_only) return (self.get_query() .select(PageView.url, referrer, fn.Count(PageView.id)) .group_by(PageView.url, referrer) .order_by(PageView.url, fn.Count(PageView.id).desc()) .tuples()) def referrers_to_url(self, domain_only=True): referrer = self._referrer_clause(domain_only) return (self.get_query() .select(referrer, PageView.url, fn.Count(PageView.id)) .group_by(referrer, PageView.url) .order_by(referrer, fn.Count(PageView.id).desc()) .tuples()) ================================================ FILE: examples/analytics/requirements.txt ================================================ peewee flask psycopg2 ================================================ FILE: examples/analytics/run_example.py ================================================ #!/usr/bin/env python import sys sys.path.insert(0, '../..') from app import app app.run(debug=True) ================================================ FILE: examples/anomaly_detection.py ================================================ import math from peewee import * db = SqliteDatabase(':memory:') class Reg(Model): key = TextField() value = IntegerField() class Meta: database = db db.create_tables([Reg]) # Create a user-defined aggregate function suitable for computing the standard # deviation of a series. @db.aggregate('stddev') class StdDev(object): def __init__(self): self.n = 0 self.values = [] def step(self, value): self.n += 1 self.values.append(value) def finalize(self): if self.n < 2: return 0 mean = sum(self.values) / self.n sqsum = sum((i - mean) ** 2 for i in self.values) return math.sqrt(sqsum / (self.n - 1)) values = [2, 3, 5, 2, 3, 12, 5, 3, 4, 1, 2, 1, -9, 3, 3, 5] Reg.create_table() Reg.insert_many([{'key': 'k%02d' % i, 'value': v} for i, v in enumerate(values)]).execute() # We'll calculate the mean and the standard deviation of the series in a common # table expression, which will then be used by our query to find rows whose # zscore exceeds a certain threshold. cte = (Reg .select(fn.avg(Reg.value), fn.stddev(Reg.value)) .cte('stats', columns=('series_mean', 'series_stddev'))) # The zscore is defined as the (value - mean) / stddev. zscore = (Reg.value - cte.c.series_mean) / cte.c.series_stddev # Find rows which fall outside of 2 standard deviations. threshold = 2 query = (Reg .select(Reg.key, Reg.value, zscore.alias('zscore')) .from_(Reg, cte) .where((zscore >= threshold) | (zscore <= -threshold)) .with_cte(cte)) for row in query: print(row.key, row.value, round(row.zscore, 2)) db.close() ================================================ FILE: examples/blog/app.py ================================================ import datetime import functools import os import re import urllib from flask import (Flask, flash, redirect, render_template, request, Response, session, url_for) from markdown import markdown from markdown.extensions.codehilite import CodeHiliteExtension from markdown.extensions.extra import ExtraExtension from markupsafe import Markup from micawber import bootstrap_basic, parse_html from micawber.cache import Cache as OEmbedCache from peewee import * from playhouse.flask_utils import FlaskDB, get_object_or_404, object_list from playhouse.sqlite_ext import * # Blog configuration values. # You may consider using a one-way hash to generate the password, and then # use the hash again in the login view to perform the comparison. This is just # for simplicity. ADMIN_PASSWORD = 'secret' APP_DIR = os.path.dirname(os.path.realpath(__file__)) # The playhouse.flask_utils.FlaskDB object accepts database URL configuration. DATABASE = 'sqlite:///%s?rank_functions=1' % os.path.join(APP_DIR, 'blog.db') DEBUG = False # The secret key is used internally by Flask to encrypt session data stored # in cookies. Make this unique for your app. SECRET_KEY = 'shhh, secret!' # This is used by micawber, which will attempt to generate rich media # embedded objects with maxwidth=800. SITE_WIDTH = 800 # Create a Flask WSGI app and configure it using values from the module. app = Flask(__name__) app.config.from_object(__name__) # FlaskDB is a wrapper for a peewee database that sets up pre/post-request # hooks for managing database connections. flask_db = FlaskDB(app) # The `database` is the actual peewee database, as opposed to flask_db which is # the wrapper. database = flask_db.database # Configure micawber with the default OEmbed providers (YouTube, Flickr, etc). # We'll use a simple in-memory cache so that multiple requests for the same # video don't require multiple network requests. oembed_providers = bootstrap_basic(OEmbedCache()) class Entry(flask_db.Model): title = CharField() slug = CharField(unique=True) content = TextField() published = BooleanField(index=True) timestamp = DateTimeField(default=datetime.datetime.now, index=True) @property def html_content(self): """ Generate HTML representation of the markdown-formatted blog entry, and also convert any media URLs into rich media objects such as video players or images. """ hilite = CodeHiliteExtension(linenums=False, css_class='highlight') extras = ExtraExtension() markdown_content = markdown(self.content, extensions=[hilite, extras]) oembed_content = parse_html( markdown_content, oembed_providers, urlize_all=True, maxwidth=app.config['SITE_WIDTH']) return Markup(oembed_content) def save(self, *args, **kwargs): # Generate a URL-friendly representation of the entry's title. if not self.slug: self.slug = re.sub(r'[^\w]+', '-', self.title.lower()).strip('-') ret = super(Entry, self).save(*args, **kwargs) # Store search content. self.update_search_index() return ret def update_search_index(self): # Create a row in the FTSEntry table with the post content. This will # allow us to use SQLite's awesome full-text search extension to # search our entries. exists = (FTSEntry .select(FTSEntry.docid) .where(FTSEntry.docid == self.id) .exists()) content = '\n'.join((self.title, self.content)) if exists: (FTSEntry .update({FTSEntry.content: content}) .where(FTSEntry.docid == self.id) .execute()) else: FTSEntry.insert({ FTSEntry.docid: self.id, FTSEntry.content: content}).execute() @classmethod def public(cls): return Entry.select().where(Entry.published == True) @classmethod def drafts(cls): return Entry.select().where(Entry.published == False) @classmethod def search(cls, query): words = [word.strip() for word in query.split() if word.strip()] if not words: # Return an empty query. return Entry.noop() else: search = ' '.join(words) # Query the full-text search index for entries matching the given # search query, then join the actual Entry data on the matching # search result. return (Entry .select(Entry, FTSEntry.rank().alias('score')) .join(FTSEntry, on=(Entry.id == FTSEntry.docid)) .where( FTSEntry.match(search) & (Entry.published == True)) .order_by(SQL('score'))) class FTSEntry(FTSModel): content = TextField() class Meta: database = database def login_required(fn): @functools.wraps(fn) def inner(*args, **kwargs): if session.get('logged_in'): return fn(*args, **kwargs) return redirect(url_for('login', next=request.path)) return inner @app.route('/login/', methods=['GET', 'POST']) def login(): next_url = request.args.get('next') or request.form.get('next') if request.method == 'POST' and request.form.get('password'): password = request.form.get('password') # TODO: If using a one-way hash, you would also hash the user-submitted # password and do the comparison on the hashed versions. if password == app.config['ADMIN_PASSWORD']: session['logged_in'] = True session.permanent = True # Use cookie to store session. flash('You are now logged in.', 'success') return redirect(next_url or url_for('index')) else: flash('Incorrect password.', 'danger') return render_template('login.html', next_url=next_url) @app.route('/logout/', methods=['GET', 'POST']) def logout(): if request.method == 'POST': session.clear() return redirect(url_for('login')) return render_template('logout.html') @app.route('/') def index(): search_query = request.args.get('q') if search_query: query = Entry.search(search_query) else: query = Entry.public().order_by(Entry.timestamp.desc()) # The `object_list` helper will take a base query and then handle # paginating the results if there are more than 20. For more info see # the docs: # http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#object_list return object_list( 'index.html', query, search=search_query, check_bounds=False) def _create_or_edit(entry, template): if request.method == 'POST': entry.title = request.form.get('title') or '' entry.content = request.form.get('content') or '' entry.published = request.form.get('published') or False if not (entry.title and entry.content): flash('Title and Content are required.', 'danger') else: # Wrap the call to save in a transaction so we can roll it back # cleanly in the event of an integrity error. try: with database.atomic(): entry.save() except IntegrityError: flash('Error: this title is already in use.', 'danger') else: flash('Entry saved successfully.', 'success') if entry.published: return redirect(url_for('detail', slug=entry.slug)) else: return redirect(url_for('edit', slug=entry.slug)) return render_template(template, entry=entry) @app.route('/create/', methods=['GET', 'POST']) @login_required def create(): return _create_or_edit(Entry(title='', content=''), 'create.html') @app.route('/drafts/') @login_required def drafts(): query = Entry.drafts().order_by(Entry.timestamp.desc()) return object_list('index.html', query, check_bounds=False) @app.route('//') def detail(slug): if session.get('logged_in'): query = Entry.select() else: query = Entry.public() entry = get_object_or_404(query, Entry.slug == slug) return render_template('detail.html', entry=entry) @app.route('//edit/', methods=['GET', 'POST']) @login_required def edit(slug): entry = get_object_or_404(Entry, Entry.slug == slug) return _create_or_edit(entry, 'edit.html') @app.template_filter('clean_querystring') def clean_querystring(request_args, *keys_to_remove, **new_values): # We'll use this template filter in the pagination include. This filter # will take the current URL and allow us to preserve the arguments in the # querystring while replacing any that we need to overwrite. For instance # if your URL is /?q=search+query&page=2 and we want to preserve the search # term but make a link to page 3, this filter will allow us to do that. querystring = dict((key, value) for key, value in request_args.items()) for key in keys_to_remove: querystring.pop(key, None) querystring.update(new_values) return urllib.urlencode(querystring) @app.errorhandler(404) def not_found(exc): return Response('

Not found

'), 404 def main(): database.create_tables([Entry, FTSEntry], safe=True) app.run(debug=True) if __name__ == '__main__': print('To login, open:') print('http://127.0.0.1:5000/login/') print('password is: %s' % ADMIN_PASSWORD) main() ================================================ FILE: examples/blog/requirements.txt ================================================ flask beautifulsoup4 micawber pygments markdown peewee ================================================ FILE: examples/blog/static/css/hilite.css ================================================ .highlight { background: #040400; color: #FFFFFF; } .highlight span.selection { color: #323232; } .highlight span.gp { color: #9595FF; } .highlight span.vi { color: #9595FF; } .highlight span.kn { color: #00C0D1; } .highlight span.cp { color: #AEE674; } .highlight span.caret { color: #FFFFFF; } .highlight span.no { color: #AEE674; } .highlight span.s2 { color: #BBFB8D; } .highlight span.nb { color: #A7FDB2; } .highlight span.nc { color: #C2ABFF; } .highlight span.nd { color: #AEE674; } .highlight span.s { color: #BBFB8D; } .highlight span.nf { color: #AEE674; } .highlight span.nx { color: #AEE674; } .highlight span.kp { color: #00C0D1; } .highlight span.nt { color: #C2ABFF; } .highlight span.s1 { color: #BBFB8D; } .highlight span.bg { color: #040400; } .highlight span.kt { color: #00C0D1; } .highlight span.support_function { color: #81B864; } .highlight span.ow { color: #EBE1B4; } .highlight span.mf { color: #A1FF24; } .highlight span.bp { color: #9595FF; } .highlight span.fg { color: #FFFFFF; } .highlight span.c1 { color: #3379FF; } .highlight span.kc { color: #9595FF; } .highlight span.c { color: #3379FF; } .highlight span.sx { color: #BBFB8D; } .highlight span.kd { color: #00C0D1; } .highlight span.ss { color: #A1FF24; } .highlight span.sr { color: #BBFB8D; } .highlight span.mo { color: #A1FF24; } .highlight span.mi { color: #A1FF24; } .highlight span.mh { color: #A1FF24; } .highlight span.o { color: #EBE1B4; } .highlight span.si { color: #DA96A3; } .highlight span.sh { color: #BBFB8D; } .highlight span.na { color: #AEE674; } .highlight span.sc { color: #BBFB8D; } .highlight span.k { color: #00C0D1; } .highlight span.se { color: #DA96A3; } .highlight span.sd { color: #54F79C; } ================================================ FILE: examples/blog/static/robots.txt ================================================ User-agent: * ================================================ FILE: examples/blog/templates/base.html ================================================ Blog {% block extra_head %}{% endblock %} {% block extra_scripts %}{% endblock %}
{% for category, message in get_flashed_messages(with_categories=true) %}

{{ message }}

{% endfor %} {% block page_header %}

{% block content_title %}{% endblock %}

{% endblock %} {% block content %}{% endblock %}

Blog, © 2015

================================================ FILE: examples/blog/templates/create.html ================================================ {% extends "base.html" %} {% block title %}Create entry{% endblock %} {% block content_title %}Create entry{% endblock %} {% block content %}
Cancel
{% endblock %} ================================================ FILE: examples/blog/templates/detail.html ================================================ {% extends "base.html" %} {% block title %}{{ entry.title }}{% endblock %} {% block content_title %}{{ entry.title }}{% endblock %} {% block extra_header %} {% if session.logged_in %}
  • Edit entry
    • {% endif %} {% endblock %} {% block content %}

    Created {{ entry.timestamp.strftime('%m/%d/%Y at %I:%M%p') }}

    {{ entry.html_content }} {% endblock %} ================================================ FILE: examples/blog/templates/edit.html ================================================ {% extends "create.html" %} {% block title %}Edit entry{% endblock %} {% block content_title %}Edit entry{% endblock %} {% block form_action %}{{ url_for('edit', slug=entry.slug) }}{% endblock %} {% block save_button %}Save changes{% endblock %} ================================================ FILE: examples/blog/templates/includes/pagination.html ================================================ {% if pagination.get_page_count() > 1 %} {% endif %} ================================================ FILE: examples/blog/templates/index.html ================================================ {% extends "base.html" %} {% block title %}Blog entries{% endblock %} {% block content_title %}{% if search %}Search "{{ search }}"{% else %}Blog entries{% endif %}{% endblock %} {% block content %} {% for entry in object_list %}

    {{ entry.title }}

    Created {{ entry.timestamp.strftime('%m/%d/%Y at %G:%I%p') }}

    {% else %}

    No entries have been created yet.

    {% endfor %} {% include "includes/pagination.html" %} {% endblock %} ================================================ FILE: examples/blog/templates/login.html ================================================ {% extends "base.html" %} {% block title %}Log in{% endblock %} {% block content_title %}Log in{% endblock %} {% block content %}
    {% endblock %} ================================================ FILE: examples/blog/templates/logout.html ================================================ {% extends "base.html" %} {% block title %}Log out{% endblock %} {% block content_title %}Log out{% endblock %} {% block content %}

    Click the button below to log out of the site.

    {% endblock %} ================================================ FILE: examples/diary.py ================================================ #!/usr/bin/env python from collections import OrderedDict import datetime from getpass import getpass import sys from peewee import * from playhouse.sqlcipher_ext import SqlCipherDatabase # Defer initialization of the database until the script is executed from the # command-line. db = SqlCipherDatabase(None) class Entry(Model): content = TextField() timestamp = DateTimeField(default=datetime.datetime.now) class Meta: database = db def initialize(passphrase): db.init('diary.db', passphrase=passphrase) db.create_tables([Entry]) def menu_loop(): choice = None while choice != 'q': for key, value in menu.items(): print('%s) %s' % (key, value.__doc__)) choice = input('Action: ').lower().strip() if choice in menu: menu[choice]() def add_entry(): """Add entry""" print('Enter your entry. Press ctrl+d when finished.') data = sys.stdin.read().strip() if data and input('Save entry? [Yn] ') != 'n': Entry.create(content=data) print('Saved successfully.') def view_entries(search_query=None): """View previous entries""" query = Entry.select().order_by(Entry.timestamp.desc()) if search_query: query = query.where(Entry.content.contains(search_query)) for entry in query: timestamp = entry.timestamp.strftime('%A %B %d, %Y %I:%M%p') print(timestamp) print('=' * len(timestamp)) print(entry.content) print('n) next entry') print('d) delete entry') print('q) return to main menu') action = input('Choice? (Ndq) ').lower().strip() if action == 'q': break elif action == 'd': entry.delete_instance() break def search_entries(): """Search entries""" view_entries(input('Search query: ')) menu = OrderedDict([ ('a', add_entry), ('v', view_entries), ('s', search_entries), ]) if __name__ == '__main__': # Collect the passphrase using a secure method. passphrase = getpass('Enter password: ') if not passphrase: sys.stderr.write('Passphrase required to access diary.\n') sys.stderr.flush() sys.exit(1) elif len(passphrase) < 8: sys.stderr.write('Passphrase must be at least 8 characters.\n') sys.stderr.flush() sys.exit(1) # Initialize the database. initialize(passphrase) menu_loop() ================================================ FILE: examples/graph.py ================================================ from peewee import * db = SqliteDatabase(':memory:') class Base(Model): class Meta: database = db class Node(Base): name = TextField(primary_key=True) def outgoing(self): return (Node .select(Node, Edge.weight) .join(Edge, on=Edge.dest) .where(Edge.src == self) .objects()) def incoming(self): return (Node .select(Node, Edge.weight) .join(Edge, on=Edge.src) .where(Edge.dest == self) .objects()) class Edge(Base): src = ForeignKeyField(Node, backref='outgoing_edges') dest = ForeignKeyField(Node, backref='incoming_edges') weight = FloatField() db.create_tables([Node, Edge]) nodes = [Node.create(name=c) for c in 'abcde'] g = ( ('a', 'b', -1), ('a', 'c', 4), ('b', 'c', 3), ('b', 'd', 2), ('b', 'e', 2), ('d', 'b', 1), ('d', 'c', 5), ('e', 'd', -3)) for src, dest, wt in g: src_n, dest_n = nodes[ord(src) - ord('a')], nodes[ord(dest) - ord('a')] Edge.create(src=src_n, dest=dest_n, weight=wt) def bellman_ford(s): dist = {} pred = {} all_nodes = Node.select() for node in all_nodes: dist[node] = float('inf') pred[node] = None dist[s] = 0 for _ in range(len(all_nodes) - 1): for u in all_nodes: for v in u.outgoing(): potential = dist[u] + v.weight if dist[v] > potential: dist[v] = potential pred[v] = u # Verify no negative-weight cycles. for u in all_nodes: for v in u.outgoing(): assert dist[v] <= dist[u] + v.weight return dist, pred def print_path(s, e): dist, pred = bellman_ford(s) distance = dist[e] route = [e] while e != s: route.append(pred[e]) e = pred[e] print(' -> '.join(v.name for v in route[::-1]) + ' (%s)' % distance) print_path(Node['a'], Node['c']) # a -> b -> c print_path(Node['a'], Node['d']) # a -> b -> e -> d print_path(Node['b'], Node['d']) # b -> e -> d ================================================ FILE: examples/hexastore.py ================================================ try: from functools import reduce except ImportError: pass import operator from peewee import * class Hexastore(object): def __init__(self, database=':memory:', **options): if isinstance(database, str): self.db = SqliteDatabase(database, **options) elif isinstance(database, Database): self.db = database else: raise ValueError('Expected database filename or a Database ' 'instance. Got: %s' % repr(database)) self.v = _VariableFactory() self.G = self.get_model() def get_model(self): class Graph(Model): subj = TextField() pred = TextField() obj = TextField() class Meta: database = self.db indexes = ( (('pred', 'obj'), False), (('obj', 'subj'), False), ) primary_key = CompositeKey('subj', 'pred', 'obj') self.db.create_tables([Graph]) return Graph def store(self, s, p, o): self.G.create(subj=s, pred=p, obj=o) def store_many(self, items): fields = [self.G.subj, self.G.pred, self.G.obj] self.G.insert_many(items, fields=fields).execute() def delete(self, s, p, o): return (self.G.delete() .where(self.G.subj == s, self.G.pred == p, self.G.obj == o) .execute()) def query(self, s=None, p=None, o=None): fields = (self.G.subj, self.G.pred, self.G.obj) expressions = [(f == v) for f, v in zip(fields, (s, p, o)) if v is not None] return self.G.select().where(*expressions) def search(self, *conditions): accum = [] binds = {} variables = set() fields = {'s': 'subj', 'p': 'pred', 'o': 'obj'} for i, condition in enumerate(conditions): if isinstance(condition, dict): condition = (condition['s'], condition['p'], condition['o']) GA = self.G.alias('g%s' % i) for part, val in zip('spo', condition): if isinstance(val, Variable): binds.setdefault(val, []) binds[val].append(getattr(GA, fields[part])) variables.add(val) else: accum.append(getattr(GA, fields[part]) == val) selection = [] sources = set() for var, fields in binds.items(): selection.append(fields[0].alias(var.name)) pairwise = [(fields[i - 1] == fields[i]) for i in range(1, len(fields))] if pairwise: accum.append(reduce(operator.and_, pairwise)) sources.update([field.source for field in fields]) return (self.G .select(*selection) .from_(*list(sources)) .where(*accum) .dicts()) class _VariableFactory(object): def __getattr__(self, name): return Variable(name) __call__ = __getattr__ class Variable(object): __slots__ = ('name',) def __init__(self, name): self.name = name def __hash__(self): return hash(self.name) def __repr__(self): return '' % self.name if __name__ == '__main__': h = Hexastore() data = ( ('charlie', 'likes', 'beanie'), ('charlie', 'likes', 'huey'), ('charlie', 'likes', 'mickey'), ('charlie', 'likes', 'scout'), ('charlie', 'likes', 'zaizee'), ('huey', 'likes', 'charlie'), ('huey', 'likes', 'scout'), ('huey', 'likes', 'zaizee'), ('mickey', 'likes', 'beanie'), ('mickey', 'likes', 'charlie'), ('mickey', 'likes', 'scout'), ('zaizee', 'likes', 'beanie'), ('zaizee', 'likes', 'charlie'), ('zaizee', 'likes', 'scout'), ('charlie', 'lives', 'topeka'), ('beanie', 'lives', 'heaven'), ('huey', 'lives', 'topeka'), ('mickey', 'lives', 'topeka'), ('scout', 'lives', 'heaven'), ('zaizee', 'lives', 'lawrence'), ) h.store_many(data) print('added %s items to store' % len(data)) print('\nwho lives in topeka?') for obj in h.query(p='lives', o='topeka'): print(obj.subj) print('\nmy friends in heaven?') X = h.v.x results = h.search(('charlie', 'likes', X), (X, 'lives', 'heaven')) for result in results: print(result['x']) print('\nmutual friends?') X = h.v.x Y = h.v.y results = h.search((X, 'likes', Y), (Y, 'likes', X)) for result in results: print(result['x'], ' <-> ', result['y']) print('\nliked by both charlie, huey and mickey?') X = h.v.x results = h.search(('charlie', 'likes', X), ('huey', 'likes', X), ('mickey', 'likes', X)) for result in results: print(result['x']) ================================================ FILE: examples/query_library.py ================================================ # Collection of Query Examples. # https://docs.peewee-orm.com/en/latest/peewee/query_library.html from functools import partial from peewee import * db = PostgresqlDatabase('peewee_clubdata') class BaseModel(Model): class Meta: database = db class Member(BaseModel): memid = AutoField() # Auto-incrementing primary key. surname = CharField() firstname = CharField() address = CharField(max_length=300) zipcode = IntegerField() telephone = CharField() recommendedby = ForeignKeyField('self', backref='recommended', column_name='recommendedby', null=True) joindate = DateTimeField() class Meta: table_name = 'members' # Conveniently declare decimal fields suitable for storing currency. MoneyField = partial(DecimalField, decimal_places=2) class Facility(BaseModel): facid = AutoField() name = CharField() membercost = MoneyField() guestcost = MoneyField() initialoutlay = MoneyField() monthlymaintenance = MoneyField() class Meta: table_name = 'facilities' class Booking(BaseModel): bookid = AutoField() facility = ForeignKeyField(Facility, column_name='facid') member = ForeignKeyField(Member, column_name='memid') starttime = DateTimeField() slots = IntegerField() class Meta: table_name = 'bookings' ================================================ FILE: examples/reddit_ranking.py ================================================ import datetime import math from peewee import * from peewee import query_to_string db = SqliteDatabase(':memory:') @db.func('log') def log(n, b): return math.log(n, b) class Base(Model): class Meta: database = db class Post(Base): content = TextField() timestamp = TimestampField() ups = IntegerField(default=0) downs = IntegerField(default=0) db.create_tables([Post]) # Populate with a number of posts. data = ( # Hours ago, ups, downs. (1, 5, 0), (1, 7, 1), (2, 10, 2), (2, 2, 0), (2, 1, 2), (3, 11, 2), (4, 20, 2), (4, 60, 12), (5, 3, 0), (5, 1, 0), (6, 30, 3), (6, 30, 20), (7, 45, 10), (7, 45, 20), (8, 11, 2), (8, 3, 1), ) now = datetime.datetime.now() Post.insert_many([ ('post %2dh %2d up, %2d down' % (hours, ups, downs), now - datetime.timedelta(seconds=hours * 3600), ups, downs) for hours, ups, downs in data]).execute() score = (Post.ups - Post.downs) order = fn.log(fn.max(fn.abs(score), 1), 10) sign = Case(None, ( ((score > 0), 1), ((score < 0), -1)), 0) seconds = (Post.timestamp) - 1134028003 hot = (sign * order) + (seconds / 45000) query = Post.select(Post.content, hot.alias('score')).order_by(SQL('score').desc()) #print(query_to_string(query)) print('Posts, ordered best-to-worse:') for post in query: print(post.content, round(post.score, 3)) ================================================ FILE: examples/sqlite_fts_compression.py ================================================ # # Small example demonstrating the use of zlib compression with the Sqlite # full-text search extension. # import zlib from peewee import * from playhouse.sqlite_ext import * db = SqliteDatabase(':memory:', rank_functions=True) class SearchIndex(FTSModel): content = SearchField() class Meta: database = db @db.func('zlib_compress') def _zlib_compress(data): if data is not None: if isinstance(data, str): data = data.encode('utf8') return zlib.compress(data, 9) @db.func('zlib_decompress') def _zlib_decompress(data): if data is not None: return zlib.decompress(data) SearchIndex.create_table( tokenize='porter', compress='zlib_compress', uncompress='zlib_decompress') phrases = [ 'A faith is a necessity to a man. Woe to him who believes in nothing.', ('All who call on God in true faith, earnestly from the heart, will ' 'certainly be heard, and will receive what they have asked and desired.'), ('Be faithful in small things because it is in them that your strength ' 'lies.'), ('Faith consists in believing when it is beyond the power of reason to ' 'believe.'), ('Faith has to do with things that are not seen and hope with things that ' 'are not at hand.')] for phrase in phrases: SearchIndex.create(content=phrase) # Use the simple ranking algorithm. query = SearchIndex.search('faith things', with_score=True) for row in query: print(round(row.score, 2), row.content.decode('utf8')) print('---') # Use the Okapi-BM25 ranking algorithm. query = SearchIndex.search_bm25('believe', with_score=True) for row in query: print(round(row.score, 2), row.content.decode('utf8')) db.close() ================================================ FILE: examples/twitter/app.py ================================================ import datetime from flask import Flask from flask import g from flask import redirect from flask import request from flask import session from flask import url_for, abort, render_template, flash from functools import wraps from hashlib import md5 from peewee import * # config - aside from our database, the rest is for use by Flask DATABASE = 'tweepee.db' DEBUG = True SECRET_KEY = 'hin6bab8ge25*r=x&+5$0kn=-#log$pt^#@vrqjld!^2ci@g*b' # create a flask application - this ``app`` object will be used to handle # inbound requests, routing them to the proper 'view' functions, etc app = Flask(__name__) app.config.from_object(__name__) # create a peewee database instance -- our models will use this database to # persist information database = SqliteDatabase(DATABASE) # model definitions -- the standard "pattern" is to define a base model class # that specifies which database to use. then, any subclasses will automatically # use the correct storage. for more information, see: # https://charlesleifer.com/docs/peewee/peewee/models.html#model-api-smells-like-django class BaseModel(Model): class Meta: database = database # the user model specifies its fields (or columns) declaratively, like django class User(BaseModel): username = CharField(unique=True) password = CharField() email = CharField() join_date = DateTimeField() # it often makes sense to put convenience methods on model instances, for # example, "give me all the users this user is following": def following(self): # query other users through the "relationship" table return (User .select() .join(Relationship, on=Relationship.to_user) .where(Relationship.from_user == self) .order_by(User.username)) def followers(self): return (User .select() .join(Relationship, on=Relationship.from_user) .where(Relationship.to_user == self) .order_by(User.username)) def is_following(self, user): return (Relationship .select() .where( (Relationship.from_user == self) & (Relationship.to_user == user)) .exists()) def gravatar_url(self, size=80): return 'http://www.gravatar.com/avatar/%s?d=identicon&s=%d' % \ (md5(self.email.strip().lower().encode('utf-8')).hexdigest(), size) # this model contains two foreign keys to user -- it essentially allows us to # model a "many-to-many" relationship between users. by querying and joining # on different columns we can expose who a user is "related to" and who is # "related to" a given user class Relationship(BaseModel): from_user = ForeignKeyField(User, backref='relationships') to_user = ForeignKeyField(User, backref='related_to') class Meta: indexes = ( # Specify a unique multi-column index on from/to-user. (('from_user', 'to_user'), True), ) # a dead simple one-to-many relationship: one user has 0..n messages, exposed by # the foreign key. because we didn't specify, a users messages will be accessible # as a special attribute, User.message_set class Message(BaseModel): user = ForeignKeyField(User, backref='messages') content = TextField() pub_date = DateTimeField() # simple utility function to create tables def create_tables(): with database: database.create_tables([User, Relationship, Message]) # flask provides a "session" object, which allows us to store information across # requests (stored by default in a secure cookie). this function allows us to # mark a user as being logged-in by setting some values in the session data: def auth_user(user): session['logged_in'] = True session['user_id'] = user.id session['username'] = user.username flash('You are logged in as %s' % (user.username)) # get the user from the session def get_current_user(): if session.get('logged_in'): return User.get(User.id == session['user_id']) # view decorator which indicates that the requesting user must be authenticated # before they can access the view. it checks the session to see if they're # logged in, and if not redirects them to the login view. def login_required(f): @wraps(f) def inner(*args, **kwargs): if not session.get('logged_in'): return redirect(url_for('login')) return f(*args, **kwargs) return inner # given a template and a SelectQuery instance, render a paginated list of # objects from the query inside the template def object_list(template_name, qr, var_name='object_list', **kwargs): kwargs.update( page=int(request.args.get('page', 1)), pages=qr.count() / 20 + 1) kwargs[var_name] = qr.paginate(kwargs['page']) return render_template(template_name, **kwargs) # retrieve a single object matching the specified query or 404 -- this uses the # shortcut "get" method on model, which retrieves a single object or raises a # DoesNotExist exception if no matching object exists # https://charlesleifer.com/docs/peewee/peewee/models.html#Model.get) def get_object_or_404(model, *expressions): try: return model.get(*expressions) except model.DoesNotExist: abort(404) # custom template filter -- flask allows you to define these functions and then # they are accessible in the template -- this one returns a boolean whether the # given user is following another user. @app.template_filter('is_following') def is_following(from_user, to_user): return from_user.is_following(to_user) # Request handlers -- these two hooks are provided by flask and we will use them # to create and tear down a database connection on each request. @app.before_request def before_request(): database.connect() @app.teardown_request def teardown_request(exc=None): if not database.is_closed(): database.close() # views -- these are the actual mappings of url to view function @app.route('/') def homepage(): # depending on whether the requesting user is logged in or not, show them # either the public timeline or their own private timeline if session.get('logged_in'): return private_timeline() else: return public_timeline() @app.route('/private/') def private_timeline(): # the private timeline exemplifies the use of a subquery -- we are asking for # messages where the person who created the message is someone the current # user is following. these messages are then ordered newest-first. user = get_current_user() messages = (Message .select() .where(Message.user << user.following()) .order_by(Message.pub_date.desc())) return object_list('private_messages.html', messages, 'message_list') @app.route('/public/') def public_timeline(): # simply display all messages, newest first messages = Message.select().order_by(Message.pub_date.desc()) return object_list('public_messages.html', messages, 'message_list') @app.route('/join/', methods=['GET', 'POST']) def join(): if request.method == 'POST' and request.form['username']: try: with database.atomic(): # Attempt to create the user. If the username is taken, due to the # unique constraint, the database will raise an IntegrityError. user = User.create( username=request.form['username'], password=md5((request.form['password']).encode('utf-8')).hexdigest(), email=request.form['email'], join_date=datetime.datetime.now()) # mark the user as being 'authenticated' by setting the session vars auth_user(user) return redirect(url_for('homepage')) except IntegrityError: flash('That username is already taken') return render_template('join.html') @app.route('/login/', methods=['GET', 'POST']) def login(): if request.method == 'POST' and request.form['username']: try: pw_hash = md5(request.form['password'].encode('utf-8')).hexdigest() user = User.get( (User.username == request.form['username']) & (User.password == pw_hash)) except User.DoesNotExist: flash('The password entered is incorrect') else: auth_user(user) return redirect(url_for('homepage')) return render_template('login.html') @app.route('/logout/') def logout(): session.pop('logged_in', None) flash('You were logged out') return redirect(url_for('homepage')) @app.route('/following/') @login_required def following(): user = get_current_user() return object_list('user_following.html', user.following(), 'user_list') @app.route('/followers/') @login_required def followers(): user = get_current_user() return object_list('user_followers.html', user.followers(), 'user_list') @app.route('/users/') def user_list(): users = User.select().order_by(User.username) return object_list('user_list.html', users, 'user_list') @app.route('/users//') def user_detail(username): # using the "get_object_or_404" shortcut here to get a user with a valid # username or short-circuit and display a 404 if no user exists in the db user = get_object_or_404(User, User.username == username) # get all the users messages ordered newest-first -- note how we're accessing # the messages -- user.message_set. could also have written it as: # Message.select().where(Message.user == user) messages = user.messages.order_by(Message.pub_date.desc()) return object_list('user_detail.html', messages, 'message_list', user=user) @app.route('/users//follow/', methods=['POST']) @login_required def user_follow(username): user = get_object_or_404(User, User.username == username) try: with database.atomic(): Relationship.create( from_user=get_current_user(), to_user=user) except IntegrityError: pass flash('You are following %s' % user.username) return redirect(url_for('user_detail', username=user.username)) @app.route('/users//unfollow/', methods=['POST']) @login_required def user_unfollow(username): user = get_object_or_404(User, User.username == username) (Relationship .delete() .where( (Relationship.from_user == get_current_user()) & (Relationship.to_user == user)) .execute()) flash('You are no longer following %s' % user.username) return redirect(url_for('user_detail', username=user.username)) @app.route('/create/', methods=['GET', 'POST']) @login_required def create(): user = get_current_user() if request.method == 'POST' and request.form['content']: message = Message.create( user=user, content=request.form['content'], pub_date=datetime.datetime.now()) flash('Your message has been created') return redirect(url_for('user_detail', username=user.username)) return render_template('create.html') @app.context_processor def _inject_user(): return {'current_user': get_current_user()} # allow running from the command line if __name__ == '__main__': create_tables() app.run() ================================================ FILE: examples/twitter/requirements.txt ================================================ flask peewee ================================================ FILE: examples/twitter/run_example.py ================================================ #!/usr/bin/env python import sys sys.path.insert(0, '../..') from app import app, create_tables create_tables() app.run() ================================================ FILE: examples/twitter/static/style.css ================================================ body { font-family: sans-serif; background: #eee; } a, h1, h2 { color: #377BA8; } h1, h2 { font-family: 'Georgia', serif; margin: 0; } h1 { border-bottom: 2px solid #eee; } h2 { font-size: 1.2em; } .page { margin: 2em auto; width: 35em; border: 5px solid #ccc; padding: 0.8em; background: white; } .page ul { list-style-type: none; } .page li { clear: both; } .metanav { text-align: right; font-size: 0.8em; padding: 0.3em; margin-bottom: 1em; background: #fafafa; } .flash { background: #CEE5F5; padding: 0.5em; border: 1px solid #AACBE2; } .avatar { display: block; float: left; margin: 0 10px 0 0; } .message-content { min-height: 80px; } ================================================ FILE: examples/twitter/templates/create.html ================================================ {% extends "layout.html" %} {% block body %}

    Create

    Message:
    {% endblock %} ================================================ FILE: examples/twitter/templates/homepage.html ================================================ {% extends "layout.html" %} {% block body %}

    Home

    Welcome to the site!

    {% endblock %} ================================================ FILE: examples/twitter/templates/includes/message.html ================================================

    {{ message.content|urlize }}

    ================================================ FILE: examples/twitter/templates/includes/pagination.html ================================================ {% if page > 1 %} {% endif %} {% if page < pages %} {% endif %} ================================================ FILE: examples/twitter/templates/join.html ================================================ {% extends "layout.html" %} {% block body %}

    Join

    Username:
    Password:
    Email:

    (used for gravatar)

    {% endblock %} ================================================ FILE: examples/twitter/templates/layout.html ================================================ Tweepee

    Tweepee

    {% if not session.logged_in %} log in join {% else %} public timeline create log out {% endif %}
    {% for message in get_flashed_messages() %}
    {{ message }}
    {% endfor %} {% block body %}{% endblock %}
    ================================================ FILE: examples/twitter/templates/login.html ================================================ {% extends "layout.html" %} {% block body %}

    Login

    {% if error %}

    Error: {{ error }}{% endif %}

    Username:
    Password:
    {% endblock %} ================================================ FILE: examples/twitter/templates/private_messages.html ================================================ {% extends "layout.html" %} {% block body %}

    Private Timeline

      {% for message in message_list %}
    • {% include "includes/message.html" %}
      • {% endfor %}
    {% include "includes/pagination.html" %} {% endblock %} ================================================ FILE: examples/twitter/templates/public_messages.html ================================================ {% extends "layout.html" %} {% block body %}

    Public Timeline

      {% for message in message_list %}
    • {% include "includes/message.html" %}
      • {% endfor %}
    {% include "includes/pagination.html" %} {% endblock %} ================================================ FILE: examples/twitter/templates/user_detail.html ================================================ {% extends "layout.html" %} {% block body %}

    Messages from {{ user.username }}

    {% if current_user %} {% if user.username != current_user.username %} {% if current_user|is_following(user) %}
    {% else %}
    {% endif %} {% endif %} {% endif %}
      {% for message in message_list %}
    • {% include "includes/message.html" %}
      • {% endfor %}
    {% include "includes/pagination.html" %} {% endblock %} ================================================ FILE: examples/twitter/templates/user_followers.html ================================================ {% extends "layout.html" %} {% block body %}

    Followers

    {% include "includes/pagination.html" %} {% endblock %} ================================================ FILE: examples/twitter/templates/user_following.html ================================================ {% extends "layout.html" %} {% block body %}

    Following

    {% include "includes/pagination.html" %} {% endblock %} ================================================ FILE: examples/twitter/templates/user_list.html ================================================ {% extends "layout.html" %} {% block body %}

    Users

    {% include "includes/pagination.html" %} {% endblock %} ================================================ FILE: peewee.py ================================================ from bisect import bisect_left from bisect import bisect_right from collections.abc import Callable from collections.abc import Mapping from contextlib import contextmanager from copy import deepcopy from functools import reduce from functools import wraps from inspect import isclass import calendar import collections import datetime import decimal import hashlib import itertools import logging import operator import re import socket import struct import sys import threading import time import types import uuid import warnings try: from pysqlite3 import dbapi2 as pysq3 except ImportError: pysq3 = None try: import sqlite3 except ImportError: sqlite3 = pysq3 else: if pysq3 and pysq3.sqlite_version_info >= sqlite3.sqlite_version_info: sqlite3 = pysq3 try: from psycopg2cffi import compat compat.register() except ImportError: pass try: import psycopg2 from psycopg2 import errors as pg_errors from psycopg2 import extensions as pg_extensions from psycopg2.extras import register_uuid as pg_register_uuid from psycopg2.extras import Json as Json_pg2 pg_register_uuid() except ImportError: psycopg2 = pg_errors = Json_pg2 = None try: import psycopg from psycopg import errors as pg3_errors from psycopg.pq import TransactionStatus from psycopg.types.json import Json as Json_pg3 from psycopg.types.json import Jsonb as Jsonb_pg3 except ImportError: psycopg = pg3_errors = Json_pg3 = Jsonb_pg3 = None mysql_passwd = False try: import pymysql as mysql except ImportError: try: import MySQLdb as mysql mysql_passwd = True except ImportError: mysql = None __version__ = '4.0.2' __all__ = [ 'AnyField', 'AsIs', 'AutoField', 'BareField', 'BigAutoField', 'BigBitField', 'BigIntegerField', 'BinaryUUIDField', 'BitField', 'BlobField', 'BooleanField', 'Case', 'Cast', 'CharField', 'Check', 'chunked', 'Column', 'CompositeKey', 'Context', 'Database', 'DatabaseError', 'DatabaseProxy', 'DataError', 'DateField', 'DateTimeField', 'DecimalField', 'Default', 'DeferredForeignKey', 'DeferredThroughModel', 'DJANGO_MAP', 'DoesNotExist', 'DoubleField', 'DQ', 'EXCLUDED', 'Field', 'FixedCharField', 'FloatField', 'fn', 'ForeignKeyField', 'IdentityField', 'ImproperlyConfigured', 'Index', 'IntegerField', 'IntegrityError', 'InterfaceError', 'InternalError', 'IPField', 'JOIN', 'ManyToManyField', 'Model', 'ModelIndex', 'MySQLDatabase', 'NotSupportedError', 'OP', 'OperationalError', 'PostgresqlDatabase', 'PrimaryKeyField', # XXX: Deprecated, change to AutoField. 'prefetch', 'PREFETCH_TYPE', 'ProgrammingError', 'Proxy', 'QualifiedNames', 'SchemaManager', 'SmallIntegerField', 'Select', 'SQL', 'SqliteDatabase', 'Table', 'TextField', 'TimeField', 'TimestampField', 'Tuple', 'UUIDField', 'Value', 'ValuesList', 'Window', ] logger = logging.getLogger('peewee') logger.addHandler(logging.NullHandler()) callable_ = lambda c: isinstance(c, Callable) multi_types = (list, tuple, frozenset, set, range, types.GeneratorType) def reraise(tp, value, tb=None): if value.__traceback__ is not tb: raise value.with_traceback(tb) raise value # Other compat issues. if sys.version_info < (3, 12): utcfromtimestamp = datetime.datetime.utcfromtimestamp utcnow = datetime.datetime.utcnow else: def utcfromtimestamp(ts): return (datetime.datetime .fromtimestamp(ts, tz=datetime.timezone.utc) .replace(tzinfo=None)) def utcnow(): return (datetime.datetime .now(datetime.timezone.utc) .replace(tzinfo=None)) if sqlite3: sqlite3.register_adapter(decimal.Decimal, str) sqlite3.register_adapter(datetime.date, str) sqlite3.register_adapter(datetime.time, str) if sys.version_info >= (3, 12): # We need to register datetime adapters as these are deprecated. def datetime_adapter(d): return d.isoformat(' ') def convert_date(d): return datetime.date(*map(int, d.split(b'-'))) def convert_timestamp(t): date, time = t.split(b'T') if b'T' in t else t.split(b' ') y, m, d = map(int, date.split(b'-')) t_full = time.split(b'.') hour, minute, second = map(int, t_full[0].split(b':')) if len(t_full) == 2: usec = int('{:0<6.6}'.format(t_full[1].decode())) else: usec = 0 return datetime.datetime(y, m, d, hour, minute, second, usec) sqlite3.register_adapter(datetime.datetime, datetime_adapter) sqlite3.register_converter('date', convert_date) sqlite3.register_converter('timestamp', convert_timestamp) __sqlite_version__ = sqlite3.sqlite_version_info else: __sqlite_version__ = (0, 0, 0) __date_parts__ = set(('year', 'month', 'day', 'hour', 'minute', 'second')) # Sqlite does not support the `date_part` SQL function, so we will define an # implementation in python. __sqlite_datetime_formats__ = ( '%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M:%S.%f', '%Y-%m-%d', '%H:%M:%S', '%H:%M:%S.%f', '%H:%M') __sqlite_date_trunc__ = { 'year': '%Y-01-01 00:00:00', 'month': '%Y-%m-01 00:00:00', 'day': '%Y-%m-%d 00:00:00', 'hour': '%Y-%m-%d %H:00:00', 'minute': '%Y-%m-%d %H:%M:00', 'second': '%Y-%m-%d %H:%M:%S'} __mysql_date_trunc__ = __sqlite_date_trunc__.copy() __mysql_date_trunc__['minute'] = '%Y-%m-%d %H:%i:00' __mysql_date_trunc__['second'] = '%Y-%m-%d %H:%i:%S' def _sqlite_date_part(lookup_type, datetime_string): assert lookup_type in __date_parts__ if not datetime_string: return dt = format_date_time(datetime_string, __sqlite_datetime_formats__) return getattr(dt, lookup_type) def _sqlite_date_trunc(lookup_type, datetime_string): assert lookup_type in __sqlite_date_trunc__ if not datetime_string: return dt = format_date_time(datetime_string, __sqlite_datetime_formats__) return dt.strftime(__sqlite_date_trunc__[lookup_type]) def _sqlite_regexp(regex, value): if value is None: return False return re.search(regex, value) is not None def __deprecated__(s): warnings.warn(s, DeprecationWarning) class attrdict(dict): def __getattr__(self, attr): try: return self[attr] except KeyError: raise AttributeError(attr) def __setattr__(self, attr, value): self[attr] = value def __iadd__(self, rhs): self.update(rhs); return self def __add__(self, rhs): d = attrdict(self); d.update(rhs); return d SENTINEL = object() #: Operations for use in SQL expressions. OP = attrdict( AND='AND', OR='OR', ADD='+', SUB='-', MUL='*', DIV='/', BIN_AND='&', BIN_OR='|', XOR='#', MOD='%', EQ='=', LT='<', LTE='<=', GT='>', GTE='>=', NE='!=', IN='IN', NOT_IN='NOT IN', IS='IS', IS_NOT='IS NOT', LIKE='LIKE', ILIKE='ILIKE', BETWEEN='BETWEEN', REGEXP='REGEXP', IREGEXP='IREGEXP', CONCAT='||', BITWISE_NEGATION='~') # To support "django-style" double-underscore filters, create a mapping between # operation name and operation code, e.g. "__eq" == OP.EQ. DJANGO_MAP = attrdict({ 'eq': operator.eq, 'lt': operator.lt, 'lte': operator.le, 'gt': operator.gt, 'gte': operator.ge, 'ne': operator.ne, 'in': operator.lshift, 'is': lambda l, r: Expression(l, OP.IS, r), 'is_not': lambda l, r: Expression(l, OP.IS_NOT, r), 'like': lambda l, r: Expression(l, OP.LIKE, r), 'ilike': lambda l, r: Expression(l, OP.ILIKE, r), 'regexp': lambda l, r: Expression(l, OP.REGEXP, r), }) #: Mapping of field type to the data-type supported by the database. Databases #: may override or add to this list. FIELD = attrdict( AUTO='INTEGER', BIGAUTO='BIGINT', BIGINT='BIGINT', BLOB='BLOB', BOOL='SMALLINT', CHAR='CHAR', DATE='DATE', DATETIME='DATETIME', DECIMAL='DECIMAL', DEFAULT='', DOUBLE='REAL', FLOAT='REAL', INT='INTEGER', SMALLINT='SMALLINT', TEXT='TEXT', TIME='TIME', UUID='TEXT', UUIDB='BLOB', VARCHAR='VARCHAR') #: Join helpers (for convenience) -- all join types are supported, this object #: is just to help avoid introducing errors by using strings everywhere. JOIN = attrdict( INNER='INNER JOIN', LEFT_OUTER='LEFT OUTER JOIN', RIGHT_OUTER='RIGHT OUTER JOIN', FULL='FULL JOIN', FULL_OUTER='FULL OUTER JOIN', CROSS='CROSS JOIN', NATURAL='NATURAL JOIN', LATERAL='LATERAL', LEFT_LATERAL='LEFT JOIN LATERAL') # Row representations. ROW = attrdict( TUPLE=1, DICT=2, NAMED_TUPLE=3, CONSTRUCTOR=4, MODEL=5) # Query type to use with prefetch PREFETCH_TYPE = attrdict( WHERE=1, JOIN=2) SCOPE_NORMAL = 1 SCOPE_SOURCE = 2 SCOPE_VALUES = 4 SCOPE_CTE = 8 SCOPE_COLUMN = 16 # Rules for parentheses around subqueries in compound select. CSQ_PARENTHESES_NEVER = 0 CSQ_PARENTHESES_ALWAYS = 1 CSQ_PARENTHESES_UNNESTED = 2 # Regular expressions used to convert class names to snake-case table names. # First regex handles acronym followed by word or initial lower-word followed # by a capitalized word. e.g. APIResponse -> API_Response / fooBar -> foo_Bar. # Second regex handles the normal case of two title-cased words. SNAKE_CASE_STEP1 = re.compile('(.)_*([A-Z][a-z]+)') SNAKE_CASE_STEP2 = re.compile('([a-z0-9])_*([A-Z])') # Used for making valid Python identifiers. IDENTIFIER_RE = re.compile(r'[A-Za-z_][A-Za-z0-9_]*') # Helper functions that are used in various parts of the codebase. MODEL_BASE = '_metaclass_helper_' def with_metaclass(meta, base=object): return meta(MODEL_BASE, (base,), {}) def merge_dict(source, overrides): merged = source.copy() if overrides: merged.update(overrides) return merged def quote(path, quote_chars): if len(path) == 1: return path[0].join(quote_chars) return '.'.join([part.join(quote_chars) for part in path]) is_model = lambda o: isclass(o) and issubclass(o, Model) def ensure_tuple(value): if value is not None: return value if isinstance(value, (list, tuple)) else (value,) def ensure_entity(value): if value is not None: return value if isinstance(value, Node) else Entity(value) def make_snake_case(s): first = SNAKE_CASE_STEP1.sub(r'\1_\2', s) return SNAKE_CASE_STEP2.sub(r'\1_\2', first).lower() def make_identifier(s): match_obj = IDENTIFIER_RE.search(s.rsplit('.', 1)[-1]) if match_obj is not None: return match_obj.group() return s def chunked(it, n): marker = object() groups = itertools.zip_longest(*[iter(it)] * n, fillvalue=marker) for group in (list(g) for g in groups): while group and group[-1] is marker: group.pop() yield group class _callable_context_manager(object): def __call__(self, fn): @wraps(fn) def inner(*args, **kwargs): with self: return fn(*args, **kwargs) return inner class Proxy(object): """ Create a proxy or placeholder for another object. """ __slots__ = ('obj', '_callbacks') def __init__(self): self._callbacks = [] self.initialize(None) def initialize(self, obj): self.obj = obj for callback in self._callbacks: callback(obj) def attach_callback(self, callback): self._callbacks.append(callback) return callback def passthrough(method): def inner(self, *args, **kwargs): if self.obj is None: raise AttributeError('Cannot use uninitialized Proxy.') return getattr(self.obj, method)(*args, **kwargs) return inner # Allow proxy to be used as a context-manager. __enter__ = passthrough('__enter__') __exit__ = passthrough('__exit__') def __getattr__(self, attr): if self.obj is None: raise AttributeError('Cannot use uninitialized Proxy.') return getattr(self.obj, attr) def __setattr__(self, attr, value): if attr not in self.__slots__: raise AttributeError('Cannot set attribute on proxy.') return super(Proxy, self).__setattr__(attr, value) class DatabaseProxy(Proxy): """ Proxy implementation specifically for proxying `Database` objects. """ __slots__ = ('obj', '_callbacks', '_Model') def connection_context(self): return ConnectionContext(self) def atomic(self, *args, **kwargs): return _atomic(self, *args, **kwargs) def manual_commit(self): return _manual(self) def transaction(self, *args, **kwargs): return _transaction(self, *args, **kwargs) def savepoint(self): return _savepoint(self) @property def Model(self): if not hasattr(self, '_Model'): class Meta: database = self self._Model = type('BaseModel', (Model,), {'Meta': Meta}) return self._Model class ModelDescriptor(object): pass # SQL Generation. class AliasManager(object): __slots__ = ('_counter', '_current_index', '_mapping') def __init__(self): # A list of dictionaries containing mappings at various depths. self._counter = 0 self._current_index = 0 self._mapping = [] self.push() @property def mapping(self): return self._mapping[self._current_index - 1] def add(self, source): if source not in self.mapping: self._counter += 1 self[source] = 't%d' % self._counter return self.mapping[source] def get(self, source, any_depth=False): if any_depth: for idx in reversed(range(self._current_index)): if source in self._mapping[idx]: return self._mapping[idx][source] return self.add(source) def __getitem__(self, source): return self.get(source) def __setitem__(self, source, alias): self.mapping[source] = alias def push(self): self._current_index += 1 if self._current_index > len(self._mapping): self._mapping.append({}) def pop(self): if self._current_index == 1: raise ValueError('Cannot pop() from empty alias manager.') self._mapping[self._current_index - 1].clear() self._current_index -= 1 class State(collections.namedtuple('_State', ('scope', 'parentheses', 'settings'))): def __new__(cls, scope=SCOPE_NORMAL, parentheses=False, **kwargs): return super(State, cls).__new__(cls, scope, parentheses, kwargs) def __call__(self, scope=None, parentheses=None, **kwargs): # Scope and settings are "inherited" (parentheses is not, however). scope = self.scope if scope is None else scope # Try to avoid unnecessary dict copying. if kwargs and self.settings: settings = self.settings.copy() # Copy original settings dict. settings.update(kwargs) # Update copy with overrides. elif kwargs: settings = kwargs else: settings = self.settings return State(scope, parentheses, **settings) def __getattr__(self, attr_name): return self.settings.get(attr_name) def __scope_context__(scope): @contextmanager def inner(self, **kwargs): with self(scope=scope, **kwargs): yield self return inner class Context(object): __slots__ = ('stack', '_sql', '_values', 'alias_manager', 'state') def __init__(self, **settings): self.stack = [] self._sql = [] self._values = [] self.alias_manager = AliasManager() self.state = State(**settings) def as_new(self): return Context(**self.state.settings) def column_sort_key(self, item): return item[0].get_sort_key(self) @property def scope(self): return self.state.scope @property def parentheses(self): return self.state.parentheses @property def subquery(self): return self.state.subquery def __call__(self, **overrides): if overrides and overrides.get('scope') == self.scope: del overrides['scope'] self.stack.append(self.state) self.state = self.state(**overrides) return self scope_normal = __scope_context__(SCOPE_NORMAL) scope_source = __scope_context__(SCOPE_SOURCE) scope_values = __scope_context__(SCOPE_VALUES) scope_cte = __scope_context__(SCOPE_CTE) scope_column = __scope_context__(SCOPE_COLUMN) def __enter__(self): if self.parentheses: self.literal('(') return self def __exit__(self, exc_type, exc_val, exc_tb): if self.parentheses: self.literal(')') self.state = self.stack.pop() @contextmanager def push_alias(self): self.alias_manager.push() yield self.alias_manager.pop() def sql(self, obj): if isinstance(obj, (Node, Context)): return obj.__sql__(self) elif is_model(obj): return obj._meta.table.__sql__(self) else: return self.sql(Value(obj)) def literal(self, keyword): self._sql.append(keyword) return self def value(self, value, converter=None, add_param=True): if converter: value = converter(value) elif converter is None and self.state.converter: # Explicitly check for None so that "False" can be used to signify # that no conversion should be applied. value = self.state.converter(value) if isinstance(value, Node): with self(converter=None): return self.sql(value) elif is_model(value): # Under certain circumstances, we could end-up treating a model- # class itself as a value. This check ensures that we drop the # table alias into the query instead of trying to parameterize a # model (for instance, passing a model as a function argument). with self.scope_column(): return self.sql(value) if self.state.value_literals: return self.literal(_query_val_transform(value)) self._values.append(value) return self.literal(self.state.param or '?') if add_param else self def __sql__(self, ctx): ctx._sql.extend(self._sql) ctx._values.extend(self._values) return ctx def parse(self, node): return self.sql(node).query() def query(self): return ''.join(self._sql), self._values def query_to_string(query): # NOTE: this function is not exported by default as it might be misused -- # and this misuse could lead to sql injection vulnerabilities. This # function is intended for debugging or logging purposes ONLY. db = getattr(query, '_database', None) if db is not None: ctx = db.get_sql_context() else: ctx = Context() sql, params = ctx.sql(query).query() if not params: return sql param = ctx.state.param or '?' if param == '?': sql = sql.replace('?', '%s') return sql % tuple(map(_query_val_transform, params)) def _query_val_transform(v): # Interpolate parameters. if isinstance(v, (str, datetime.datetime, datetime.date, datetime.time)): v = "'%s'" % str(v).replace("'", "''") elif isinstance(v, bytes): try: v = v.decode('utf8') except UnicodeDecodeError: v = v.decode('raw_unicode_escape') v = "'%s'" % v.replace("'", "''") elif isinstance(v, int): v = '%s' % int(v) # Also handles booleans -> 1 or 0. elif v is None: v = 'NULL' else: v = str(v) return v # AST. class Node(object): _coerce = True __isabstractmethod__ = False # Avoid issue w/abc and __getattr__, eg fn.X def clone(self): obj = self.__class__.__new__(self.__class__) obj.__dict__ = self.__dict__.copy() return obj def __sql__(self, ctx): raise NotImplementedError @staticmethod def copy(method): def inner(self, *args, **kwargs): clone = self.clone() method(clone, *args, **kwargs) return clone return inner def coerce(self, _coerce=True): if _coerce != self._coerce: clone = self.clone() clone._coerce = _coerce return clone return self def is_alias(self): return False def unwrap(self): return self class ColumnFactory(object): __slots__ = ('node',) def __init__(self, node): self.node = node def __getattr__(self, attr): return Column(self.node, attr) __getitem__ = __getattr__ class _DynamicColumn(object): __slots__ = () def __get__(self, instance, instance_type=None): if instance is not None: return ColumnFactory(instance) # Implements __getattr__(). return self class _ExplicitColumn(object): __slots__ = () def __get__(self, instance, instance_type=None): if instance is not None: raise AttributeError( '%s specifies columns explicitly, and does not support ' 'dynamic column lookups.' % instance) return self class Star(Node): def __init__(self, source): self.source = source def __sql__(self, ctx): return ctx.sql(QualifiedNames(self.source)).literal('.*') class Source(Node): c = _DynamicColumn() def __init__(self, alias=None): super(Source, self).__init__() self._alias = alias @Node.copy def alias(self, name): self._alias = name def select(self, *columns): if not columns: columns = (SQL('*'),) return Select((self,), columns) @property def __star__(self): return Star(self) def join(self, dest, join_type=JOIN.INNER, on=None): return Join(self, dest, join_type, on) def left_outer_join(self, dest, on=None): return Join(self, dest, JOIN.LEFT_OUTER, on) def cte(self, name, recursive=False, columns=None, materialized=None): return CTE(name, self, recursive=recursive, columns=columns, materialized=materialized) def get_sort_key(self, ctx): if self._alias: return (self._alias,) return (ctx.alias_manager[self],) def apply_alias(self, ctx): # If we are defining the source, include the "AS alias" declaration. An # alias is created for the source if one is not already defined. if ctx.scope == SCOPE_SOURCE: if self._alias: ctx.alias_manager[self] = self._alias ctx.literal(' AS ').sql(Entity(ctx.alias_manager[self])) return ctx def apply_column(self, ctx): if self._alias: ctx.alias_manager[self] = self._alias return ctx.sql(Entity(ctx.alias_manager[self])) class _HashableSource(object): def __init__(self, *args, **kwargs): super(_HashableSource, self).__init__(*args, **kwargs) self._update_hash() @Node.copy def alias(self, name): self._alias = name self._update_hash() def _update_hash(self): self._hash = self._get_hash() def _get_hash(self): return hash((self.__class__, self._path, self._alias)) def __hash__(self): return self._hash def __eq__(self, other): if isinstance(other, _HashableSource): return self._hash == other._hash return Expression(self, OP.EQ, other) def __ne__(self, other): if isinstance(other, _HashableSource): return self._hash != other._hash return Expression(self, OP.NE, other) def _e(op): def inner(self, rhs): return Expression(self, op, rhs) return inner __lt__ = _e(OP.LT) __le__ = _e(OP.LTE) __gt__ = _e(OP.GT) __ge__ = _e(OP.GTE) def __bind_database__(meth): @wraps(meth) def inner(self, *args, **kwargs): result = meth(self, *args, **kwargs) if self._database: return result.bind(self._database) return result return inner def __join__(join_type=JOIN.INNER, inverted=False): def method(self, other): if inverted: self, other = other, self return Join(self, other, join_type=join_type) return method class BaseTable(Source): __and__ = __join__(JOIN.INNER) __add__ = __join__(JOIN.LEFT_OUTER) __sub__ = __join__(JOIN.RIGHT_OUTER) __or__ = __join__(JOIN.FULL_OUTER) __mul__ = __join__(JOIN.CROSS) __rand__ = __join__(JOIN.INNER, inverted=True) __radd__ = __join__(JOIN.LEFT_OUTER, inverted=True) __rsub__ = __join__(JOIN.RIGHT_OUTER, inverted=True) __ror__ = __join__(JOIN.FULL_OUTER, inverted=True) __rmul__ = __join__(JOIN.CROSS, inverted=True) class _BoundTableContext(object): def __init__(self, table, database): self.table = table self.database = database def __call__(self, fn): @wraps(fn) def inner(*args, **kwargs): with _BoundTableContext(self.table, self.database): return fn(*args, **kwargs) return inner def __enter__(self): self._orig_database = self.table._database self.table.bind(self.database) if self.table._model is not None: self.table._model.bind(self.database) return self.table def __exit__(self, exc_type, exc_val, exc_tb): self.table.bind(self._orig_database) if self.table._model is not None: self.table._model.bind(self._orig_database) class Table(_HashableSource, BaseTable): def __init__(self, name, columns=None, primary_key=None, schema=None, alias=None, _model=None, _database=None): self.__name__ = name self._columns = columns self._primary_key = primary_key self._schema = schema self._path = (schema, name) if schema else (name,) self._model = _model self._database = _database super(Table, self).__init__(alias=alias) # Allow tables to restrict what columns are available. if columns is not None: self.c = _ExplicitColumn() for column in columns: setattr(self, column, Column(self, column)) if primary_key: col_src = self if self._columns else self.c self.primary_key = getattr(col_src, primary_key) else: self.primary_key = None def clone(self): # Ensure a deep copy of the column instances. return Table( self.__name__, columns=self._columns, primary_key=self._primary_key, schema=self._schema, alias=self._alias, _model=self._model, _database=self._database) def bind(self, database=None): self._database = database return self def bind_ctx(self, database=None): return _BoundTableContext(self, database) def _get_hash(self): return hash((self.__class__, self._path, self._alias, self._model)) @__bind_database__ def select(self, *columns): if not columns and self._columns: columns = [Column(self, column) for column in self._columns] return Select((self,), columns) @__bind_database__ def insert(self, insert=None, columns=None, **kwargs): if kwargs: insert = {} if insert is None else insert src = self if self._columns else self.c for key, value in kwargs.items(): insert[getattr(src, key)] = value return Insert(self, insert=insert, columns=columns) @__bind_database__ def replace(self, insert=None, columns=None, **kwargs): return (self .insert(insert=insert, columns=columns) .on_conflict('REPLACE')) @__bind_database__ def update(self, update=None, **kwargs): if kwargs: update = {} if update is None else update for key, value in kwargs.items(): src = self if self._columns else self.c update[getattr(src, key)] = value return Update(self, update=update) @__bind_database__ def delete(self): return Delete(self) def __sql__(self, ctx): if ctx.scope == SCOPE_VALUES: # Return the quoted table name. return ctx.sql(Entity(*self._path)) if self._alias: ctx.alias_manager[self] = self._alias if ctx.scope == SCOPE_SOURCE: # Define the table and its alias. return self.apply_alias(ctx.sql(Entity(*self._path))) else: # Refer to the table using the alias. return self.apply_column(ctx) class Join(BaseTable): def __init__(self, lhs, rhs, join_type=JOIN.INNER, on=None, alias=None): super(Join, self).__init__(alias=alias) self.lhs = lhs self.rhs = rhs self.join_type = join_type self._on = on def on(self, predicate): self._on = predicate return self def __sql__(self, ctx): (ctx .sql(self.lhs) .literal(' %s ' % self.join_type) .sql(self.rhs)) if self._on is not None: ctx.literal(' ON ').sql(self._on) return ctx class ValuesList(_HashableSource, BaseTable): def __init__(self, values, columns=None, alias=None): self._values = values self._columns = columns super(ValuesList, self).__init__(alias=alias) def _get_hash(self): return hash((self.__class__, id(self._values), self._alias)) @Node.copy def columns(self, *names): self._columns = names def __sql__(self, ctx): if self._alias: ctx.alias_manager[self] = self._alias if ctx.scope == SCOPE_SOURCE or ctx.scope == SCOPE_NORMAL: with ctx(parentheses=not ctx.parentheses): ctx = (ctx .literal('VALUES ') .sql(CommaNodeList([ EnclosedNodeList(row) for row in self._values]))) if ctx.scope == SCOPE_SOURCE: ctx.literal(' AS ').sql(Entity(ctx.alias_manager[self])) if self._columns: entities = [Entity(c) for c in self._columns] ctx.sql(EnclosedNodeList(entities)) else: ctx.sql(Entity(ctx.alias_manager[self])) return ctx class CTE(_HashableSource, Source): def __init__(self, name, query, recursive=False, columns=None, materialized=None): self._alias = name self._query = query self._recursive = recursive self._materialized = materialized if columns is not None: columns = [Entity(c) if isinstance(c, str) else c for c in columns] self._columns = columns query._cte_list = () super(CTE, self).__init__(alias=name) def select_from(self, *columns): if not columns: raise ValueError('select_from() must specify one or more columns ' 'from the CTE to select.') query = (Select((self,), columns) .with_cte(self) .bind(self._query._database)) try: query = query.objects(self._query.model) except AttributeError: pass return query def _get_hash(self): return hash((self.__class__, self._alias, id(self._query))) def union_all(self, rhs): clone = self._query.clone() return CTE(self._alias, clone + rhs, self._recursive, self._columns) __add__ = union_all def union(self, rhs): clone = self._query.clone() return CTE(self._alias, clone | rhs, self._recursive, self._columns) __or__ = union def __sql__(self, ctx): if ctx.scope != SCOPE_CTE: return ctx.sql(Entity(self._alias)) with ctx.push_alias(): ctx.alias_manager[self] = self._alias ctx.sql(Entity(self._alias)) if self._columns: ctx.literal(' ').sql(EnclosedNodeList(self._columns)) ctx.literal(' AS ') if self._materialized: ctx.literal('MATERIALIZED ') elif self._materialized is False: ctx.literal('NOT MATERIALIZED ') with ctx.scope_normal(parentheses=True): ctx.sql(self._query) return ctx class ColumnBase(Node): _converter = None @Node.copy def converter(self, converter=None): self._converter = converter def alias(self, alias): if alias: return Alias(self, alias) return self def unalias(self): return self def bind_to(self, dest): return BindTo(self, dest) def cast(self, as_type): return Cast(self, as_type) def asc(self, collation=None, nulls=None): return Asc(self, collation=collation, nulls=nulls) __pos__ = asc def desc(self, collation=None, nulls=None): return Desc(self, collation=collation, nulls=nulls) __neg__ = desc def __invert__(self): return Negated(self) def _e(op, inv=False): """ Lightweight factory which returns a method that builds an Expression consisting of the left-hand and right-hand operands, using `op`. """ def inner(self, rhs): if inv: return Expression(rhs, op, self) return Expression(self, op, rhs) return inner __and__ = _e(OP.AND) __or__ = _e(OP.OR) __add__ = _e(OP.ADD) __sub__ = _e(OP.SUB) __mul__ = _e(OP.MUL) __div__ = __truediv__ = _e(OP.DIV) __xor__ = _e(OP.XOR) __radd__ = _e(OP.ADD, inv=True) __rsub__ = _e(OP.SUB, inv=True) __rmul__ = _e(OP.MUL, inv=True) __rdiv__ = __rtruediv__ = _e(OP.DIV, inv=True) __rand__ = _e(OP.AND, inv=True) __ror__ = _e(OP.OR, inv=True) __rxor__ = _e(OP.XOR, inv=True) def __eq__(self, rhs): op = OP.IS if rhs is None else OP.EQ return Expression(self, op, rhs) def __ne__(self, rhs): op = OP.IS_NOT if rhs is None else OP.NE return Expression(self, op, rhs) __lt__ = _e(OP.LT) __le__ = _e(OP.LTE) __gt__ = _e(OP.GT) __ge__ = _e(OP.GTE) __lshift__ = _e(OP.IN) __rshift__ = _e(OP.IS) __mod__ = _e(OP.LIKE) __pow__ = _e(OP.ILIKE) like = _e(OP.LIKE) ilike = _e(OP.ILIKE) bin_and = _e(OP.BIN_AND) bin_or = _e(OP.BIN_OR) in_ = _e(OP.IN) not_in = _e(OP.NOT_IN) regexp = _e(OP.REGEXP) iregexp = _e(OP.IREGEXP) # Special expressions. def is_null(self, is_null=True): op = OP.IS if is_null else OP.IS_NOT return Expression(self, op, None) def _escape_like_expr(self, s, template): if s.find('_') >= 0 or s.find('%') >= 0 or s.find('\\') >= 0: s = s.replace('\\', '\\\\').replace('_', '\\_').replace('%', '\\%') # Pass the expression and escape string as unconverted values, to # avoid (e.g.) a Json field converter turning the escaped LIKE # pattern into a Json-quoted string. return NodeList(( Value(template % s, converter=False), SQL('ESCAPE'), Value('\\', converter=False))) return template % s def contains(self, rhs): if isinstance(rhs, Node): rhs = Expression('%', OP.CONCAT, Expression(rhs, OP.CONCAT, '%')) else: rhs = self._escape_like_expr(rhs, '%%%s%%') return Expression(self, OP.ILIKE, rhs) def startswith(self, rhs): if isinstance(rhs, Node): rhs = Expression(rhs, OP.CONCAT, '%') else: rhs = self._escape_like_expr(rhs, '%s%%') return Expression(self, OP.ILIKE, rhs) def endswith(self, rhs): if isinstance(rhs, Node): rhs = Expression('%', OP.CONCAT, rhs) else: rhs = self._escape_like_expr(rhs, '%%%s') return Expression(self, OP.ILIKE, rhs) def between(self, lo, hi): return Expression(self, OP.BETWEEN, NodeList((lo, SQL('AND'), hi))) def concat(self, rhs): return StringExpression(self, OP.CONCAT, rhs) def __getitem__(self, item): if isinstance(item, slice): if item.start is None or item.stop is None: raise ValueError('BETWEEN range must have both a start- and ' 'end-point.') return self.between(item.start, item.stop) return self == item __iter__ = None # Prevent infinite loop. def distinct(self): return NodeList((SQL('DISTINCT'), self)) def collate(self, collation): return NodeList((self, SQL('COLLATE %s' % collation))) def get_sort_key(self, ctx): return () class Column(ColumnBase): def __init__(self, source, name): self.source = source self.name = name def get_sort_key(self, ctx): if ctx.scope == SCOPE_VALUES: return (self.name,) else: return self.source.get_sort_key(ctx) + (self.name,) def __hash__(self): return hash((self.source, self.name)) def __sql__(self, ctx): if ctx.scope == SCOPE_VALUES: return ctx.sql(Entity(self.name)) else: with ctx.scope_column(): return ctx.sql(self.source).literal('.').sql(Entity(self.name)) class WrappedNode(ColumnBase): def __init__(self, node): self.node = node self._coerce = getattr(node, '_coerce', True) self._converter = getattr(node, '_converter', None) def is_alias(self): return self.node.is_alias() def unwrap(self): return self.node.unwrap() class EntityFactory(object): __slots__ = ('node',) def __init__(self, node): self.node = node def __getattr__(self, attr): return Entity(self.node, attr) class _DynamicEntity(object): __slots__ = () def __get__(self, instance, instance_type=None): if instance is not None: return EntityFactory(instance._alias) # Implements __getattr__(). return self class Alias(WrappedNode): c = _DynamicEntity() def __init__(self, node, alias): super(Alias, self).__init__(node) self._alias = alias def __hash__(self): return hash(self._alias) @property def name(self): return self._alias @name.setter def name(self, value): self._alias = value def alias(self, alias=None): if alias is None: return self.node else: return Alias(self.node, alias) def unalias(self): return self.node def is_alias(self): return True def __sql__(self, ctx): if ctx.scope == SCOPE_SOURCE: return (ctx .sql(self.node) .literal(' AS ') .sql(Entity(self._alias))) else: return ctx.sql(Entity(self._alias)) class BindTo(WrappedNode): def __init__(self, node, dest): super(BindTo, self).__init__(node) self.dest = dest def __sql__(self, ctx): return ctx.sql(self.node) class Negated(WrappedNode): def __invert__(self): return self.node def __sql__(self, ctx): return ctx.literal('NOT ').sql(self.node) class BitwiseMixin(object): def __and__(self, other): return self.bin_and(other) def __or__(self, other): return self.bin_or(other) def __sub__(self, other): return self.bin_and(other.bin_negated()) def __invert__(self): return BitwiseNegated(self) class BitwiseNegated(BitwiseMixin, WrappedNode): op = OP.BITWISE_NEGATION def __invert__(self): return self.node def __sql__(self, ctx): if ctx.state.operations: op_sql = ctx.state.operations.get(self.op, self.op) else: op_sql = self.op return ctx.literal(op_sql).sql(self.node) class Value(ColumnBase): def __init__(self, value, converter=None, unpack=True): self.value = value self.converter = converter self.multi = unpack and isinstance(self.value, multi_types) if self.multi: self.values = [] for item in self.value: if isinstance(item, Node): self.values.append(item) else: self.values.append(Value(item, self.converter)) else: self.values = None def __sql__(self, ctx): if self.multi: # For multi-part values (e.g. lists of IDs). return ctx.sql(EnclosedNodeList(self.values)) return ctx.value(self.value, self.converter) class ValueLiterals(WrappedNode): def __sql__(self, ctx): with ctx(value_literals=True): return ctx.sql(self.node) def AsIs(value, converter=None): return Value(value, converter, unpack=False) class Cast(WrappedNode): def __init__(self, node, cast): super(Cast, self).__init__(node) self._cast = cast self._coerce = False def __sql__(self, ctx): return (ctx .literal('CAST(') .sql(self.node) .literal(' AS %s)' % self._cast)) class Ordering(WrappedNode): def __init__(self, node, direction, collation=None, nulls=None): super(Ordering, self).__init__(node) self.direction = direction self.collation = collation self.nulls = nulls if nulls and nulls.lower() not in ('first', 'last'): raise ValueError('Ordering nulls= parameter must be "first" or ' '"last", got: %s' % nulls) def collate(self, collation=None): return Ordering(self.node, self.direction, collation) def _null_ordering_case(self, nulls): if nulls.lower() == 'last': ifnull, notnull = 1, 0 elif nulls.lower() == 'first': ifnull, notnull = 0, 1 else: raise ValueError('unsupported value for nulls= ordering.') return Case(None, ((self.node.is_null(), ifnull),), notnull) def __sql__(self, ctx): if self.nulls and not ctx.state.nulls_ordering: ctx.sql(self._null_ordering_case(self.nulls)).literal(', ') ctx.sql(self.node).literal(' %s' % self.direction) if self.collation: ctx.literal(' COLLATE %s' % self.collation) if self.nulls and ctx.state.nulls_ordering: ctx.literal(' NULLS %s' % self.nulls) return ctx def Asc(node, collation=None, nulls=None): return Ordering(node, 'ASC', collation, nulls) def Desc(node, collation=None, nulls=None): return Ordering(node, 'DESC', collation, nulls) class Expression(ColumnBase): def __init__(self, lhs, op, rhs, flat=False): self.lhs = lhs self.op = op self.rhs = rhs self.flat = flat def __sql__(self, ctx): overrides = {'parentheses': not self.flat, 'in_expr': True} # First attempt to unwrap the node on the left-hand-side, so that we # can get at the underlying Field if one is present. node = raw_node = self.lhs if isinstance(raw_node, WrappedNode): node = raw_node.unwrap() # Set up the appropriate converter if we have a field on the left side. if isinstance(node, Field) and raw_node._coerce: overrides['converter'] = node.db_value overrides['is_fk_expr'] = isinstance(node, ForeignKeyField) else: overrides['converter'] = None if ctx.state.operations: op_sql = ctx.state.operations.get(self.op, self.op) else: op_sql = self.op with ctx(**overrides): # Postgresql reports an error for IN/NOT IN (), so convert to # the equivalent boolean expression. op_in = self.op == OP.IN or self.op == OP.NOT_IN rhs = self.rhs if op_in: # if self._is_rhs_empty(rhs, ctx): return ctx.literal('0 = 1' if self.op == OP.IN else '1 = 1') if rhs is None and (self.op == OP.IS or self.op == OP.IS_NOT): rhs = SQL('NULL') return (ctx .sql(self.lhs) .literal(' %s ' % op_sql) .sql(rhs)) def _is_rhs_empty(self, rhs, ctx): if isinstance(rhs, multi_types): return not bool(rhs) elif isinstance(rhs, Value): return (rhs.multi and not rhs.values) else: return ctx.as_new().parse(rhs)[0] == '()' class StringExpression(Expression): def __add__(self, rhs): return self.concat(rhs) def __radd__(self, lhs): return StringExpression(lhs, OP.CONCAT, self) class Entity(ColumnBase): def __init__(self, *path): self._path = [p for p in path if p] def __getattr__(self, attr): return Entity(*self._path + [attr]) def get_sort_key(self, ctx): return tuple(self._path) def __hash__(self): return hash((self.__class__.__name__, tuple(self._path))) def __sql__(self, ctx): quote_chars = ctx.state.quote or '""' q = quote_chars[0] escaped = [p.replace(q, quote_chars) for p in self._path] return ctx.literal(quote(escaped, quote_chars)) class SQL(ColumnBase): def __init__(self, sql, params=None): self.sql = sql self.params = params def __sql__(self, ctx): ctx.literal(self.sql) if self.params: for param in self.params: ctx.value(param, False, add_param=False) return ctx def Check(constraint, name=None): check = SQL('CHECK (%s)' % constraint) if not name: return check return NodeList((SQL('CONSTRAINT'), Entity(name), check)) def Default(value): return SQL('DEFAULT %s' % value) class Function(ColumnBase): no_coerce_functions = set(('sum', 'count', 'avg', 'cast', 'array_agg')) def __init__(self, name, arguments, coerce=True, python_value=None): self.name = name self.arguments = arguments self._filter = None self._order_by = None self._python_value = python_value if name and name.lower() in self.no_coerce_functions: self._coerce = False else: self._coerce = coerce def __getattr__(self, attr): def decorator(*args, **kwargs): return Function(attr, args, **kwargs) return decorator @Node.copy def filter(self, where=None): self._filter = where @Node.copy def order_by(self, *ordering): self._order_by = ordering @Node.copy def python_value(self, func=None): self._python_value = func def over(self, partition_by=None, order_by=None, start=None, end=None, frame_type=None, window=None, exclude=None): if isinstance(partition_by, Window) and window is None: window = partition_by if window is not None: node = WindowAlias(window) else: node = Window(partition_by=partition_by, order_by=order_by, start=start, end=end, frame_type=frame_type, exclude=exclude, _inline=True) return NodeList((self, SQL('OVER'), node)) def __sql__(self, ctx): ctx.literal(self.name) if not len(self.arguments): ctx.literal('()') else: args = self.arguments # If this is an ordered aggregate, then we will modify the last # argument to append the ORDER BY ... clause. We do this to avoid # double-wrapping any expression args in parentheses, as NodeList # has a special check (hack) in place to work around this. if self._order_by: args = list(args) args[-1] = NodeList((args[-1], SQL('ORDER BY'), CommaNodeList(self._order_by))) with ctx(in_function=True, function_arg_count=len(self.arguments)): ctx.sql(EnclosedNodeList([ (arg if isinstance(arg, Node) else Value(arg, False)) for arg in args])) if self._filter: ctx.literal(' FILTER (WHERE ').sql(self._filter).literal(')') return ctx fn = Function(None, None) class Window(Node): # Frame start/end and frame exclusion. CURRENT_ROW = SQL('CURRENT ROW') GROUP = SQL('GROUP') TIES = SQL('TIES') NO_OTHERS = SQL('NO OTHERS') # Frame types. GROUPS = 'GROUPS' RANGE = 'RANGE' ROWS = 'ROWS' def __init__(self, partition_by=None, order_by=None, start=None, end=None, frame_type=None, extends=None, exclude=None, alias=None, _inline=False): super(Window, self).__init__() if start is not None and not isinstance(start, SQL): start = SQL(start) if end is not None and not isinstance(end, SQL): end = SQL(end) self.partition_by = ensure_tuple(partition_by) self.order_by = ensure_tuple(order_by) self.start = start self.end = end if self.start is None and self.end is not None: raise ValueError('Cannot specify WINDOW end without start.') self._alias = alias or 'w' self._inline = _inline self.frame_type = frame_type self._extends = extends self._exclude = exclude def alias(self, alias=None): self._alias = alias or 'w' return self @Node.copy def as_range(self): self.frame_type = Window.RANGE @Node.copy def as_rows(self): self.frame_type = Window.ROWS @Node.copy def as_groups(self): self.frame_type = Window.GROUPS @Node.copy def extends(self, window=None): self._extends = window @Node.copy def exclude(self, frame_exclusion=None): if isinstance(frame_exclusion, str): frame_exclusion = SQL(frame_exclusion) self._exclude = frame_exclusion @staticmethod def following(value=None): if value is None: return SQL('UNBOUNDED FOLLOWING') return SQL('%d FOLLOWING' % value) @staticmethod def preceding(value=None): if value is None: return SQL('UNBOUNDED PRECEDING') return SQL('%d PRECEDING' % value) def __sql__(self, ctx): if ctx.scope != SCOPE_SOURCE and not self._inline: ctx.sql(Entity(self._alias)) ctx.literal(' AS ') with ctx(parentheses=True): parts = [] if self._extends is not None: ext = self._extends if isinstance(ext, Window): ext = Entity(ext._alias) elif isinstance(ext, str): ext = Entity(ext) parts.append(ext) if self.partition_by: parts.extend(( SQL('PARTITION BY'), CommaNodeList(self.partition_by))) if self.order_by: parts.extend(( SQL('ORDER BY'), CommaNodeList(self.order_by))) if self.start is not None and self.end is not None: frame = self.frame_type or 'ROWS' parts.extend(( SQL('%s BETWEEN' % frame), self.start, SQL('AND'), self.end)) elif self.start is not None: parts.extend((SQL(self.frame_type or 'ROWS'), self.start)) elif self.frame_type is not None: parts.append(SQL('%s UNBOUNDED PRECEDING' % self.frame_type)) if self._exclude is not None: parts.extend((SQL('EXCLUDE'), self._exclude)) ctx.sql(NodeList(parts)) return ctx class WindowAlias(Node): def __init__(self, window): self.window = window def alias(self, window_alias): self.window._alias = window_alias return self def __sql__(self, ctx): return ctx.sql(Entity(self.window._alias or 'w')) class _InFunction(Node): def __init__(self, node, in_function=True): self.node = node self.in_function = in_function def __sql__(self, ctx): with ctx(in_function=self.in_function): return ctx.sql(self.node) class Case(ColumnBase): def __init__(self, predicate, expression_tuples, default=None): self.predicate = predicate self.expression_tuples = expression_tuples self.default = default def __sql__(self, ctx): clauses = [SQL('CASE')] if self.predicate is not None: clauses.append(self.predicate) for expr, value in self.expression_tuples: clauses.extend((SQL('WHEN'), expr, SQL('THEN'), _InFunction(value))) if self.default is not None: clauses.extend((SQL('ELSE'), _InFunction(self.default))) clauses.append(SQL('END')) with ctx(in_function=False): return ctx.sql(NodeList(clauses)) class ForUpdate(Node): def __init__(self, expr, of=None, nowait=None, skip_locked=None): expr = 'FOR UPDATE' if expr is True else expr if expr.lower().endswith('nowait'): expr = expr[:-7] # Strip off the "nowait" bit. nowait = True elif expr.lower().endswith('skip locked'): expr = expr[:-12] skip_locked = True if nowait and skip_locked: raise ValueError('Only one of nowait and skip_locked may be used ' 'in a FOR UPDATE clause.') self._expr = expr if of is not None and not isinstance(of, (list, set, tuple)): of = (of,) self._of = of self._nowait = nowait self._skip_locked = skip_locked def __sql__(self, ctx): ctx.literal(self._expr) if self._of is not None: ctx.literal(' OF ').sql(CommaNodeList(self._of)) if self._nowait: ctx.literal(' NOWAIT') elif self._skip_locked: ctx.literal(' SKIP LOCKED') return ctx class NodeList(ColumnBase): def __init__(self, nodes, glue=' ', parens=False): self.nodes = nodes self.glue = glue self.parens = parens def __sql__(self, ctx): n_nodes = len(self.nodes) if n_nodes == 0: return ctx.literal('()') if self.parens else ctx elif self.parens and n_nodes == 1 and \ isinstance(self.nodes[0], Expression) and \ not self.nodes[0].flat: # Hack to avoid double-parentheses. nodes = (self.nodes[0].clone(),) nodes[0].flat = True else: nodes = self.nodes with ctx(parentheses=self.parens): for i in range(n_nodes - 1): ctx.sql(nodes[i]) ctx.literal(self.glue) ctx.sql(nodes[n_nodes - 1]) return ctx def CommaNodeList(nodes): return NodeList(nodes, ', ') def EnclosedNodeList(nodes): return NodeList(nodes, ', ', True) class _Namespace(Node): __slots__ = ('_name',) def __init__(self, name): self._name = name def __getattr__(self, attr): return NamespaceAttribute(self, attr) __getitem__ = __getattr__ class NamespaceAttribute(ColumnBase): def __init__(self, namespace, attribute): self._namespace = namespace self._attribute = attribute def __sql__(self, ctx): return (ctx .literal(self._namespace._name + '.') .sql(Entity(self._attribute))) EXCLUDED = _Namespace('EXCLUDED') class DQ(ColumnBase): def __init__(self, **query): super(DQ, self).__init__() self.query = query self._negated = False @Node.copy def __invert__(self): self._negated = not self._negated def clone(self): node = DQ(**self.query) node._negated = self._negated return node #: Represent a row tuple. Tuple = lambda *a: EnclosedNodeList(a) class QualifiedNames(WrappedNode): def __sql__(self, ctx): with ctx.scope_column(): return ctx.sql(self.node) def qualify_names(node): # Search a node heirarchy to ensure that any column-like objects are # referenced using fully-qualified names. if isinstance(node, Expression): return node.__class__(qualify_names(node.lhs), node.op, qualify_names(node.rhs), node.flat) elif isinstance(node, ColumnBase): return QualifiedNames(node) return node class OnConflict(Node): def __init__(self, action=None, update=None, preserve=None, where=None, conflict_target=None, conflict_where=None, conflict_constraint=None): self._action = action self._update = update self._preserve = ensure_tuple(preserve) self._where = where if conflict_target is not None and conflict_constraint is not None: raise ValueError('only one of "conflict_target" and ' '"conflict_constraint" may be specified.') self._conflict_target = ensure_tuple(conflict_target) self._conflict_where = conflict_where self._conflict_constraint = conflict_constraint def get_conflict_statement(self, ctx, query): return ctx.state.conflict_statement(self, query) def get_conflict_update(self, ctx, query): return ctx.state.conflict_update(self, query) @Node.copy def preserve(self, *columns): self._preserve = columns @Node.copy def update(self, _data=None, **kwargs): if _data and kwargs and not isinstance(_data, dict): raise ValueError('Cannot mix data with keyword arguments in the ' 'OnConflict update method.') _data = _data or {} if kwargs: _data.update(kwargs) self._update = _data @Node.copy def where(self, *expressions): if self._where is not None: expressions = (self._where,) + expressions self._where = reduce(operator.and_, expressions) @Node.copy def conflict_target(self, *constraints): self._conflict_constraint = None self._conflict_target = constraints @Node.copy def conflict_where(self, *expressions): if self._conflict_where is not None: expressions = (self._conflict_where,) + expressions self._conflict_where = reduce(operator.and_, expressions) @Node.copy def conflict_constraint(self, constraint): self._conflict_constraint = constraint self._conflict_target = None def database_required(method): @wraps(method) def inner(self, database=None, *args, **kwargs): database = self._database if database is None else database if not database: raise InterfaceError('Query must be bound to a database in order ' 'to call "%s".' % method.__name__) return method(self, database, *args, **kwargs) return inner # BASE QUERY INTERFACE. class BaseQuery(Node): default_row_type = ROW.DICT def __init__(self, _database=None, **kwargs): self._database = _database self._cursor_wrapper = None self._row_type = None self._constructor = None super(BaseQuery, self).__init__(**kwargs) def bind(self, database=None): self._database = database return self def clone(self): query = super(BaseQuery, self).clone() query._cursor_wrapper = None return query @Node.copy def dicts(self, as_dict=True): self._row_type = ROW.DICT if as_dict else None return self @Node.copy def tuples(self, as_tuple=True): self._row_type = ROW.TUPLE if as_tuple else None return self @Node.copy def namedtuples(self, as_namedtuple=True): self._row_type = ROW.NAMED_TUPLE if as_namedtuple else None return self @Node.copy def objects(self, constructor=None): self._row_type = ROW.CONSTRUCTOR if constructor else None self._constructor = constructor return self def _get_cursor_wrapper(self, cursor): row_type = self._row_type or self.default_row_type if row_type == ROW.DICT: return DictCursorWrapper(cursor) elif row_type == ROW.TUPLE: return CursorWrapper(cursor) elif row_type == ROW.NAMED_TUPLE: return NamedTupleCursorWrapper(cursor) elif row_type == ROW.CONSTRUCTOR: return ObjectCursorWrapper(cursor, self._constructor) else: raise ValueError('Unrecognized row type: "%s".' % row_type) def __sql__(self, ctx): raise NotImplementedError def sql(self): if self._database: context = self._database.get_sql_context() else: context = Context() return context.parse(self) @database_required def execute(self, database): return self._execute(database) def _execute(self, database): raise NotImplementedError def iterator(self, database=None): return iter(self.execute(database).iterator()) def _ensure_execution(self): if self._cursor_wrapper is None: if not self._database: raise ValueError('Query has not been executed.') self.execute() def __iter__(self): self._ensure_execution() return iter(self._cursor_wrapper) def __getitem__(self, value): self._ensure_execution() if isinstance(value, slice): index = value.stop else: index = value if index is not None: index = index + 1 if index >= 0 else 0 self._cursor_wrapper.fill_cache(index) return self._cursor_wrapper.row_cache[value] def __len__(self): self._ensure_execution() return len(self._cursor_wrapper) def __str__(self): return query_to_string(self) class RawQuery(BaseQuery): def __init__(self, sql=None, params=None, **kwargs): super(RawQuery, self).__init__(**kwargs) self._sql = sql self._params = params def __sql__(self, ctx): ctx.literal(self._sql) if self._params: for param in self._params: ctx.value(param, add_param=False) return ctx def _execute(self, database): if self._cursor_wrapper is None: cursor = database.execute(self) self._cursor_wrapper = self._get_cursor_wrapper(cursor) return self._cursor_wrapper class Query(BaseQuery): def __init__(self, where=None, order_by=None, limit=None, offset=None, **kwargs): super(Query, self).__init__(**kwargs) self._where = where self._order_by = order_by self._limit = limit self._offset = offset self._cte_list = None @Node.copy def with_cte(self, *cte_list): self._cte_list = cte_list @Node.copy def where(self, *expressions): if self._where is not None: expressions = (self._where,) + expressions self._where = reduce(operator.and_, expressions) @Node.copy def orwhere(self, *expressions): if self._where is not None: expressions = (self._where,) + expressions self._where = reduce(operator.or_, expressions) @Node.copy def order_by(self, *values): self._order_by = values @Node.copy def order_by_extend(self, *values): self._order_by = ((self._order_by or ()) + values) or None @Node.copy def limit(self, value=None): self._limit = value @Node.copy def offset(self, value=None): self._offset = value @Node.copy def paginate(self, page, paginate_by=20): page = page - 1 if page > 0 else 0 self._limit = paginate_by self._offset = page * paginate_by def _apply_ordering(self, ctx): if self._order_by: (ctx .literal(' ORDER BY ') .sql(CommaNodeList(self._order_by))) if self._limit is not None or (self._offset is not None and ctx.state.limit_max): limit = ctx.state.limit_max if self._limit is None else self._limit ctx.literal(' LIMIT ').sql(limit) if self._offset is not None: ctx.literal(' OFFSET ').sql(self._offset) return ctx def __sql__(self, ctx): if self._cte_list: # The CTE scope is only used at the very beginning of the query, # when we are describing the various CTEs we will be using. recursive = any(cte._recursive for cte in self._cte_list) # Explicitly disable the "subquery" flag here, so as to avoid # unnecessary parentheses around subsequent selects. with ctx.scope_cte(subquery=False): (ctx .literal('WITH RECURSIVE ' if recursive else 'WITH ') .sql(CommaNodeList(self._cte_list)) .literal(' ')) return ctx def __compound_select__(operation, inverted=False): @__bind_database__ def method(self, other): if inverted: self, other = other, self return CompoundSelectQuery(self, operation, other) return method class SelectQuery(Query): union_all = __add__ = __compound_select__('UNION ALL') union = __or__ = __compound_select__('UNION') intersect = __and__ = __compound_select__('INTERSECT') except_ = __sub__ = __compound_select__('EXCEPT') __radd__ = __compound_select__('UNION ALL', inverted=True) __ror__ = __compound_select__('UNION', inverted=True) __rand__ = __compound_select__('INTERSECT', inverted=True) __rsub__ = __compound_select__('EXCEPT', inverted=True) def select_from(self, *columns): if not columns: raise ValueError('select_from() must specify one or more columns.') query = (Select((self,), columns) .bind(self._database)) if getattr(self, 'model', None) is not None: # Bind to the sub-select's model type, if defined. query = query.objects(self.model) return query class SelectBase(_HashableSource, Source, SelectQuery): def _get_hash(self): return hash((self.__class__, self._alias or id(self))) def _execute(self, database): if self._cursor_wrapper is None: cursor = database.execute(self) self._cursor_wrapper = self._get_cursor_wrapper(cursor) return self._cursor_wrapper @database_required def peek(self, database, n=1): rows = self.execute(database)[:n] if rows: return rows[0] if n == 1 else rows @database_required def first(self, database, n=1): if self._limit != n: self._limit = n self._cursor_wrapper = None return self.peek(database, n=n) @database_required def scalar(self, database, as_tuple=False, as_dict=False): if as_dict: return self.dicts().peek(database) row = self.tuples().peek(database) return row[0] if row and not as_tuple else row @database_required def scalars(self, database): for row in self.tuples().execute(database): yield row[0] @database_required def count(self, database, clear_limit=False): clone = self.order_by().alias('_wrapped') if clear_limit: clone._limit = clone._offset = None try: if clone._having is None and clone._group_by is None and \ clone._windows is None and clone._distinct is None and \ clone._simple_distinct is not True: clone = clone.select(SQL('1')) except AttributeError: pass return Select([clone], [fn.COUNT(SQL('1'))]).scalar(database) @database_required def exists(self, database): clone = self.columns(SQL('1')) clone._limit = 1 clone._offset = None return bool(clone.scalar()) @database_required def get(self, database): self._cursor_wrapper = None try: return self.execute(database)[0] except IndexError: pass # QUERY IMPLEMENTATIONS. class CompoundSelectQuery(SelectBase): def __init__(self, lhs, op, rhs): super(CompoundSelectQuery, self).__init__() self.lhs = lhs self.op = op self.rhs = rhs @property def _returning(self): return self.lhs._returning @database_required def exists(self, database): query = Select((self.limit(1),), (SQL('1'),)).bind(database) return bool(query.scalar()) def _get_query_key(self): return (self.lhs.get_query_key(), self.rhs.get_query_key()) def _wrap_parens(self, ctx, subq): csq_setting = ctx.state.compound_select_parentheses if not csq_setting or csq_setting == CSQ_PARENTHESES_NEVER: return False elif csq_setting == CSQ_PARENTHESES_ALWAYS: return True elif csq_setting == CSQ_PARENTHESES_UNNESTED: if ctx.state.in_expr or ctx.state.in_function: # If this compound select query is being used inside an # expression, e.g., an IN or EXISTS(). return False # If the query on the left or right is itself a compound select # query, then we do not apply parentheses. However, if it is a # regular SELECT query, we will apply parentheses. return not isinstance(subq, CompoundSelectQuery) def __sql__(self, ctx): if ctx.scope == SCOPE_COLUMN: return self.apply_column(ctx) # Call parent method to handle any CTEs. super(CompoundSelectQuery, self).__sql__(ctx) outer_parens = ctx.subquery or (ctx.scope == SCOPE_SOURCE) with ctx(parentheses=outer_parens): # Should the left-hand query be wrapped in parentheses? lhs_parens = self._wrap_parens(ctx, self.lhs) with ctx.scope_normal(parentheses=lhs_parens, subquery=False): ctx.sql(self.lhs) ctx.literal(' %s ' % self.op) with ctx.push_alias(): # Should the right-hand query be wrapped in parentheses? rhs_parens = self._wrap_parens(ctx, self.rhs) with ctx.scope_normal(parentheses=rhs_parens, subquery=False): ctx.sql(self.rhs) # Apply ORDER BY, LIMIT, OFFSET. We use the "values" scope so that # entity names are not fully-qualified. This is a bit of a hack, as # we're relying on the logic in Column.__sql__() to not fully # qualify column names. with ctx.scope_values(): self._apply_ordering(ctx) return self.apply_alias(ctx) class Select(SelectBase): def __init__(self, from_list=None, columns=None, group_by=None, having=None, distinct=None, windows=None, for_update=None, lateral=None, **kwargs): super(Select, self).__init__(**kwargs) self._from_list = (list(from_list) if isinstance(from_list, tuple) else from_list) or [] self._returning = columns self._group_by = group_by self._having = having self._windows = None self._for_update = for_update self._lateral = lateral self._distinct = self._simple_distinct = None if distinct: if isinstance(distinct, bool): self._simple_distinct = distinct else: self._distinct = distinct self._cursor_wrapper = None def clone(self): clone = super(Select, self).clone() if clone._from_list: clone._from_list = list(clone._from_list) return clone @Node.copy def columns(self, *columns, **kwargs): self._returning = columns select = columns @Node.copy def select_extend(self, *columns): self._returning = tuple(self._returning) + columns @property def selected_columns(self): return self._returning @selected_columns.setter def selected_columns(self, value): self._returning = value @Node.copy def from_(self, *sources): self._from_list = list(sources) @Node.copy def join(self, dest, join_type=JOIN.INNER, on=None): if not self._from_list: raise ValueError('No sources to join on.') item = self._from_list.pop() if join_type == JOIN.LATERAL or join_type == JOIN.LEFT_LATERAL: on = True self._from_list.append(Join(item, dest, join_type, on)) def left_outer_join(self, dest, on=None): return self.join(dest, JOIN.LEFT_OUTER, on) @Node.copy def group_by(self, *columns): grouping = [] for column in columns: if isinstance(column, Table): if not column._columns: raise ValueError('Cannot pass a table to group_by() that ' 'does not have columns explicitly ' 'declared.') grouping.extend([getattr(column, col_name) for col_name in column._columns]) else: grouping.append(column) self._group_by = grouping def group_by_extend(self, *values): """@Node.copy used from group_by() call""" group_by = tuple(self._group_by or ()) + values return self.group_by(*group_by) @Node.copy def having(self, *expressions): if self._having is not None: expressions = (self._having,) + expressions self._having = reduce(operator.and_, expressions) @Node.copy def distinct(self, *columns): if len(columns) == 1 and (columns[0] is True or columns[0] is False): self._simple_distinct = columns[0] else: self._simple_distinct = False self._distinct = columns @Node.copy def window(self, *windows): self._windows = windows if windows else None @Node.copy def for_update(self, for_update=True, of=None, nowait=None, skip_locked=None): if not for_update and (of is not None or nowait or skip_locked): for_update = True if not for_update: self._for_update = None else: self._for_update = ForUpdate(for_update, of, nowait, skip_locked) @Node.copy def lateral(self, lateral=True): self._lateral = lateral def _get_query_key(self): return self._alias def __sql_selection__(self, ctx, is_subquery=False): return ctx.sql(CommaNodeList(self._returning)) def __sql__(self, ctx): if ctx.scope == SCOPE_COLUMN: return self.apply_column(ctx) if self._lateral and ctx.scope == SCOPE_SOURCE: ctx.literal('LATERAL ') is_subquery = ctx.subquery state = { 'converter': None, 'in_function': False, 'parentheses': is_subquery or (ctx.scope == SCOPE_SOURCE), 'subquery': True, } if ctx.state.in_function and ctx.state.function_arg_count == 1: state['parentheses'] = False with ctx.scope_normal(**state): # Defer calling parent SQL until here. This ensures that any CTEs # for this query will be properly nested if this query is a # sub-select or is used in an expression. See GH#1809 for example. super(Select, self).__sql__(ctx) ctx.literal('SELECT ') if self._simple_distinct or self._distinct is not None: ctx.literal('DISTINCT ') if self._distinct: (ctx .literal('ON ') .sql(EnclosedNodeList(self._distinct)) .literal(' ')) with ctx.scope_source(): ctx = self.__sql_selection__(ctx, is_subquery) if self._from_list: with ctx.scope_source(parentheses=False): ctx.literal(' FROM ').sql(CommaNodeList(self._from_list)) if self._where is not None: ctx.literal(' WHERE ').sql(self._where) if self._group_by: ctx.literal(' GROUP BY ').sql(CommaNodeList(self._group_by)) if self._having is not None: ctx.literal(' HAVING ').sql(self._having) if self._windows is not None: ctx.literal(' WINDOW ') ctx.sql(CommaNodeList(self._windows)) # Apply ORDER BY, LIMIT, OFFSET. self._apply_ordering(ctx) if self._for_update is not None: if not ctx.state.for_update: raise ValueError('FOR UPDATE specified but not supported ' 'by database.') ctx.literal(' ') ctx.sql(self._for_update) # If the subquery is inside a function -or- we are evaluating a # subquery on either side of an expression w/o an explicit alias, do # not generate an alias + AS clause. if ctx.state.in_function or (ctx.state.in_expr and self._alias is None): return ctx return self.apply_alias(ctx) class _WriteQuery(Query): def __init__(self, table, returning=None, **kwargs): self.table = table self._returning = returning self._return_cursor = True if returning else False super(_WriteQuery, self).__init__(**kwargs) def cte(self, name, recursive=False, columns=None, materialized=None): return CTE(name, self, recursive=recursive, columns=columns, materialized=materialized) @Node.copy def returning(self, *returning): self._returning = returning self._return_cursor = True if returning else False def apply_returning(self, ctx): if self._returning: with ctx.scope_source(): ctx.literal(' RETURNING ').sql(CommaNodeList(self._returning)) return ctx def _execute(self, database): if self._returning: cursor = self.execute_returning(database) else: cursor = database.execute(self) return self.handle_result(database, cursor) def execute_returning(self, database): if self._cursor_wrapper is None: cursor = database.execute(self) self._cursor_wrapper = self._get_cursor_wrapper(cursor) return self._cursor_wrapper def handle_result(self, database, cursor): if self._return_cursor: return cursor return database.rows_affected(cursor) def _set_table_alias(self, ctx): ctx.alias_manager[self.table] = self.table.__name__ def __sql__(self, ctx): super(_WriteQuery, self).__sql__(ctx) # We explicitly set the table alias to the table's name, which ensures # that if a sub-select references a column on the outer table, we won't # assign it a new alias (e.g. t2) but will refer to it as table.column. self._set_table_alias(ctx) return ctx class Update(_WriteQuery): def __init__(self, table, update=None, **kwargs): super(Update, self).__init__(table, **kwargs) self._update = update self._from = None @Node.copy def from_(self, *sources): self._from = sources def __sql__(self, ctx): super(Update, self).__sql__(ctx) with ctx.scope_values(subquery=True): ctx.literal('UPDATE ') expressions = [] for k, v in sorted(self._update.items(), key=ctx.column_sort_key): if not isinstance(v, Node): if isinstance(k, Field): v = k.to_value(v) else: v = Value(v, unpack=False) elif isinstance(v, Model) and isinstance(k, ForeignKeyField): # NB: we want to ensure that when passed a model instance # in the context of a foreign-key, we apply the fk-specific # adaptation of the model. v = k.to_value(v) if not isinstance(v, Value): v = qualify_names(v) expressions.append(NodeList((k, SQL('='), v))) (ctx .sql(self.table) .literal(' SET ') .sql(CommaNodeList(expressions))) if self._from: with ctx.scope_source(parentheses=False): ctx.literal(' FROM ').sql(CommaNodeList(self._from)) if self._where: with ctx.scope_normal(): ctx.literal(' WHERE ').sql(self._where) self._apply_ordering(ctx) return self.apply_returning(ctx) class Insert(_WriteQuery): SIMPLE = 0 QUERY = 1 MULTI = 2 class DefaultValuesException(Exception): pass def __init__(self, table, insert=None, columns=None, on_conflict=None, **kwargs): super(Insert, self).__init__(table, **kwargs) self._insert = insert self._columns = columns self._on_conflict = on_conflict self._query_type = None self._as_rowcount = False def where(self, *expressions): raise NotImplementedError('INSERT queries cannot have a WHERE clause.') @Node.copy def as_rowcount(self, _as_rowcount=True): self._as_rowcount = _as_rowcount @Node.copy def on_conflict_ignore(self, ignore=True): self._on_conflict = OnConflict('IGNORE') if ignore else None @Node.copy def on_conflict_replace(self, replace=True): self._on_conflict = OnConflict('REPLACE') if replace else None @Node.copy def on_conflict(self, *args, **kwargs): self._on_conflict = (OnConflict(*args, **kwargs) if (args or kwargs) else None) def _simple_insert(self, ctx): if not self._insert: raise self.DefaultValuesException('Error: no data to insert.') return self._generate_insert((self._insert,), ctx) def get_default_data(self): return {} def get_default_columns(self): if self.table._columns: return [getattr(self.table, col) for col in self.table._columns if col != self.table._primary_key] def _generate_insert(self, insert, ctx): rows_iter = iter(insert) columns = self._columns # Load and organize column defaults (if provided). defaults = self.get_default_data() # First figure out what columns are being inserted (if they weren't # specified explicitly). Resulting columns are normalized and ordered. if not columns: try: row = next(rows_iter) except StopIteration: raise self.DefaultValuesException('Error: no rows to insert.') if not isinstance(row, Mapping): columns = self.get_default_columns() if columns is None: raise ValueError('Bulk insert must specify columns.') else: # Infer column names from the dict of data being inserted. accum = [] for column in row: if isinstance(column, str): column = getattr(self.table, column) accum.append(column) # Add any columns present in the default data that are not # accounted for by the dictionary of row data. column_set = set(accum) for col in (set(defaults) - column_set): accum.append(col) columns = sorted(accum, key=lambda obj: obj.get_sort_key(ctx)) rows_iter = itertools.chain(iter((row,)), rows_iter) else: clean_columns = [] seen = set() for column in columns: if isinstance(column, str): column_obj = getattr(self.table, column) else: column_obj = column clean_columns.append(column_obj) seen.add(column_obj) columns = clean_columns for col in sorted(defaults, key=lambda obj: obj.get_sort_key(ctx)): if col not in seen: columns.append(col) fk_fields = set() nullable_columns = set() value_lookups = {} for column in columns: lookups = [column, column.name] if isinstance(column, Field): if column.name != column.column_name: lookups.append(column.column_name) if column.null: nullable_columns.add(column) if isinstance(column, ForeignKeyField): fk_fields.add(column) value_lookups[column] = lookups ctx.sql(EnclosedNodeList(columns)).literal(' VALUES ') columns_converters = [ (column, column.db_value if isinstance(column, Field) else None) for column in columns] all_values = [] for row in rows_iter: values = [] is_dict = isinstance(row, Mapping) for i, (column, converter) in enumerate(columns_converters): try: if is_dict: # The logic is a bit convoluted, but in order to be # flexible in what we accept (dict keyed by # column/field, field name, or underlying column name), # we try accessing the row data dict using each # possible key. If no match is found, throw an error. for lookup in value_lookups[column]: try: val = row[lookup] except KeyError: pass else: break else: raise KeyError else: val = row[i] except (KeyError, IndexError): if column in defaults: val = defaults[column] if callable_(val): val = val() elif column in nullable_columns: val = None else: raise ValueError('Missing value for %s.' % column.name) if not isinstance(val, Node) or (isinstance(val, Model) and column in fk_fields): val = Value(val, converter=converter, unpack=False) values.append(val) all_values.append(EnclosedNodeList(values)) if not all_values: raise self.DefaultValuesException('Error: no data to insert.') with ctx.scope_values(subquery=True): return ctx.sql(CommaNodeList(all_values)) def _query_insert(self, ctx): return (ctx .sql(EnclosedNodeList(self._columns)) .literal(' ') .sql(self._insert)) def _default_values(self, ctx): if not self._database: return ctx.literal('DEFAULT VALUES') return self._database.default_values_insert(ctx) def __sql__(self, ctx): super(Insert, self).__sql__(ctx) with ctx.scope_values(): stmt = None if self._on_conflict is not None: stmt = self._on_conflict.get_conflict_statement(ctx, self) (ctx .sql(stmt or SQL('INSERT')) .literal(' INTO ') .sql(self.table) .literal(' ')) if isinstance(self._insert, Mapping) and not self._columns: try: self._simple_insert(ctx) except self.DefaultValuesException: self._default_values(ctx) self._query_type = Insert.SIMPLE elif isinstance(self._insert, (SelectQuery, SQL)): self._query_insert(ctx) self._query_type = Insert.QUERY else: self._generate_insert(self._insert, ctx) self._query_type = Insert.MULTI if self._on_conflict is not None: update = self._on_conflict.get_conflict_update(ctx, self) if update is not None: ctx.literal(' ').sql(update) return self.apply_returning(ctx) def _execute(self, database): if self._returning is None and database.returning_clause \ and self.table._primary_key: self._returning = (self.table._primary_key,) try: return super(Insert, self)._execute(database) except self.DefaultValuesException: pass def handle_result(self, database, cursor): if self._return_cursor: return cursor if self._as_rowcount: return database.rows_affected(cursor) return database.last_insert_id(cursor, self._query_type) class Delete(_WriteQuery): def __sql__(self, ctx): super(Delete, self).__sql__(ctx) with ctx.scope_values(subquery=True): ctx.literal('DELETE FROM ').sql(self.table) if self._where is not None: with ctx.scope_normal(): ctx.literal(' WHERE ').sql(self._where) self._apply_ordering(ctx) return self.apply_returning(ctx) class Index(Node): def __init__(self, name, table, expressions, unique=False, safe=False, where=None, using=None, nulls_distinct=None): self._name = name self._table = Entity(table) if not isinstance(table, Table) else table self._expressions = expressions self._where = where self._unique = unique self._safe = safe self._using = using self._nulls_distinct = nulls_distinct if self._nulls_distinct is not None and not self._unique: raise ValueError('NULLS DISTINCT is only available with UNIQUE.') @Node.copy def safe(self, _safe=True): self._safe = _safe @Node.copy def where(self, *expressions): if self._where is not None: expressions = (self._where,) + expressions self._where = reduce(operator.and_, expressions) @Node.copy def using(self, _using=None): self._using = _using @Node.copy def nulls_distinct(self, nulls_distinct=None): if nulls_distinct is not None and not self._unique: raise ValueError('NULLS DISTINCT is only available with UNIQUE.') self._nulls_distinct = nulls_distinct def __sql__(self, ctx): statement = 'CREATE UNIQUE INDEX ' if self._unique else 'CREATE INDEX ' with ctx.scope_values(subquery=True): ctx.literal(statement) if self._safe: ctx.literal('IF NOT EXISTS ') # Sqlite uses CREATE INDEX . ON , whereas most # others use: CREATE INDEX ON .
    . if ctx.state.index_schema_prefix and \ isinstance(self._table, Table) and self._table._schema: index_name = Entity(self._table._schema, self._name) table_name = Entity(self._table.__name__) else: index_name = Entity(self._name) table_name = self._table ctx.sql(index_name) if self._using is not None and \ ctx.state.index_using_precedes_table: ctx.literal(' USING %s' % self._using) # MySQL style. (ctx .literal(' ON ') .sql(table_name) .literal(' ')) if self._using is not None and not \ ctx.state.index_using_precedes_table: ctx.literal('USING %s ' % self._using) # Postgres/default. ctx.sql(EnclosedNodeList([ SQL(expr) if isinstance(expr, str) else expr for expr in self._expressions])) if self._where is not None: ctx.literal(' WHERE ').sql(self._where) if self._nulls_distinct is not None: ctx.literal(' NULLS DISTINCT' if self._nulls_distinct else ' NULLS NOT DISTINCT') return ctx class ModelIndex(Index): def __init__(self, model, fields, unique=False, safe=True, where=None, using=None, name=None, nulls_distinct=None): self._model = model if name is None: name = self._generate_name_from_fields(model, fields) if using is None: for field in fields: if isinstance(field, Field) and hasattr(field, 'index_type'): using = field.index_type super(ModelIndex, self).__init__( name=name, table=model._meta.table, expressions=fields, unique=unique, safe=safe, where=where, using=using, nulls_distinct=nulls_distinct) def _generate_name_from_fields(self, model, fields): accum = [] for field in fields: if isinstance(field, str): accum.append(field.split()[0]) else: if isinstance(field, Node) and not isinstance(field, Field): field = field.unwrap() if isinstance(field, Field): accum.append(field.column_name) if not accum: raise ValueError('Unable to generate a name for the index, please ' 'explicitly specify a name.') clean_field_names = re.sub(r'[^\w]+', '', '_'.join(accum)) meta = model._meta prefix = meta.name if meta.legacy_table_names else meta.table_name return _truncate_constraint_name('_'.join((prefix, clean_field_names))) def _truncate_constraint_name(constraint, maxlen=64): if len(constraint) > maxlen: name_hash = hashlib.md5(constraint.encode('utf-8')).hexdigest() constraint = '%s_%s' % (constraint[:(maxlen - 8)], name_hash[:7]) return constraint # DB-API 2.0 EXCEPTIONS. class PeeweeException(Exception): def __init__(self, *args): if args and isinstance(args[0], Exception): self.orig, args = args[0], args[1:] super(PeeweeException, self).__init__(*args) class ImproperlyConfigured(PeeweeException): pass class DatabaseError(PeeweeException): pass class DataError(DatabaseError): pass class IntegrityError(DatabaseError): pass class InterfaceError(PeeweeException): pass class InternalError(DatabaseError): pass class NotSupportedError(DatabaseError): pass class OperationalError(DatabaseError): pass class ProgrammingError(DatabaseError): pass class ExceptionWrapper(object): __slots__ = ('exceptions',) def __init__(self, exceptions): self.exceptions = exceptions def __enter__(self): pass def __exit__(self, exc_type, exc_value, traceback): if exc_type is None: return # psycopg shits out a million cute error types. Try to catch em all. if pg_errors is not None and exc_type.__name__ not in self.exceptions \ and issubclass(exc_type, pg_errors.Error): exc_type = exc_type.__bases__[0] elif pg3_errors is not None and \ exc_type.__name__ not in self.exceptions \ and issubclass(exc_type, pg3_errors.Error): exc_type = exc_type.__bases__[0] if exc_type.__name__ in self.exceptions: new_type = self.exceptions[exc_type.__name__] exc_args = exc_value.args reraise(new_type, new_type(exc_value, *exc_args), traceback) EXCEPTIONS = { 'ConstraintError': IntegrityError, 'DatabaseError': DatabaseError, 'DataError': DataError, 'IntegrityError': IntegrityError, 'InterfaceError': InterfaceError, 'InternalError': InternalError, 'NotSupportedError': NotSupportedError, 'OperationalError': OperationalError, 'ProgrammingError': ProgrammingError, 'TransactionRollbackError': OperationalError, 'UndefinedFunction': ProgrammingError, 'UniqueViolation': IntegrityError} __exception_wrapper__ = ExceptionWrapper(EXCEPTIONS) # DATABASE INTERFACE AND CONNECTION MANAGEMENT. IndexMetadata = collections.namedtuple( 'IndexMetadata', ('name', 'sql', 'columns', 'unique', 'table')) ColumnMetadata = collections.namedtuple( 'ColumnMetadata', ('name', 'data_type', 'null', 'primary_key', 'table', 'default')) ForeignKeyMetadata = collections.namedtuple( 'ForeignKeyMetadata', ('column', 'dest_table', 'dest_column', 'table')) ViewMetadata = collections.namedtuple('ViewMetadata', ('name', 'sql')) class _ConnectionState(object): def __init__(self, **kwargs): super(_ConnectionState, self).__init__(**kwargs) self.reset() def reset(self): self.closed = True self.conn = None self.ctx = [] self.transactions = [] def set_connection(self, conn): self.conn = conn self.closed = False self.ctx = [] self.transactions = [] class _ConnectionLocal(_ConnectionState, threading.local): pass class _NoopLock(object): __slots__ = () def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): pass class ConnectionContext(object): __slots__ = ('db',) def __init__(self, db): self.db = db def __enter__(self): if self.db.is_closed(): self.db.connect() def __exit__(self, exc_type, exc_val, exc_tb): self.db.close() def __call__(self, fn): @wraps(fn) def inner(*args, **kwargs): with ConnectionContext(self.db): return fn(*args, **kwargs) return inner class Database(_callable_context_manager): context_class = Context field_types = {} operations = {} param = '?' quote = '""' server_version = None # Feature toggles. compound_select_parentheses = CSQ_PARENTHESES_NEVER for_update = False index_schema_prefix = False index_using_precedes_table = False limit_max = None nulls_ordering = False returning_clause = False safe_create_index = True safe_drop_index = True sequences = False truncate_table = True def __init__(self, database, thread_safe=True, autorollback=False, field_types=None, operations=None, autocommit=None, autoconnect=True, **kwargs): self._field_types = merge_dict(FIELD, self.field_types) self._operations = merge_dict(OP, self.operations) if field_types: self._field_types.update(field_types) if operations: self._operations.update(operations) self.autoconnect = autoconnect self.thread_safe = thread_safe if thread_safe: self._state = _ConnectionLocal() self._lock = threading.Lock() else: self._state = _ConnectionState() self._lock = _NoopLock() if autorollback: __deprecated__('Peewee no longer uses the "autorollback" option, ' 'as we always run in autocommit-mode now. This ' 'changes psycopg2\'s semantics so that the conn ' 'is not left in a transaction-aborted state.') if autocommit is not None: __deprecated__('Peewee no longer uses the "autocommit" option, as ' 'the semantics now require it to always be True. ' 'Because some database-drivers also use the ' '"autocommit" parameter, you are receiving a ' 'warning so you may update your code and remove ' 'the parameter, as in the future, specifying ' 'autocommit could impact the behavior of the ' 'database driver you are using.') self.connect_params = {} self.init(database, **kwargs) def init(self, database, **kwargs): if not self.is_closed(): self.close() self.database = database self.connect_params.update(kwargs) self.deferred = not bool(database) def __enter__(self): if self.is_closed(): self.connect() ctx = self.atomic() self._state.ctx.append(ctx) ctx.__enter__() return self def __exit__(self, exc_type, exc_val, exc_tb): ctx = self._state.ctx.pop() try: ctx.__exit__(exc_type, exc_val, exc_tb) finally: if not self._state.ctx: self.close() def connection_context(self): return ConnectionContext(self) def _connect(self): raise NotImplementedError def connect(self, reuse_if_open=False): with self._lock: if self.deferred: raise InterfaceError('Error, database must be initialized ' 'before opening a connection.') if not self._state.closed: if reuse_if_open: return False raise OperationalError('Connection already opened.') self._state.reset() with __exception_wrapper__: self._state.set_connection(self._connect()) if self.server_version is None: self._set_server_version(self._state.conn) self._initialize_connection(self._state.conn) return True def _initialize_connection(self, conn): pass def _set_server_version(self, conn): self.server_version = 0 def close(self): with self._lock: if self.deferred: raise InterfaceError('Error, database must be initialized ' 'before opening a connection.') if self.in_transaction(): raise OperationalError('Attempting to close database while ' 'transaction is open.') is_open = not self._state.closed try: if is_open: with __exception_wrapper__: self._close(self._state.conn) finally: self._state.reset() return is_open def _close(self, conn): conn.close() def is_closed(self): return self._state.closed def is_connection_usable(self): return not self._state.closed def connection(self): if self.is_closed(): self.connect() return self._state.conn def cursor(self, named_cursor=None): if self.is_closed(): if self.autoconnect: self.connect() else: raise InterfaceError('Error, database connection not opened.') return self._state.conn.cursor() def execute_sql(self, sql, params=None): if logger.isEnabledFor(logging.DEBUG): logger.debug((sql, params)) with __exception_wrapper__: cursor = self.cursor() cursor.execute(sql, params or ()) return cursor def execute(self, query, **context_options): ctx = self.get_sql_context(**context_options) sql, params = ctx.sql(query).query() return self.execute_sql(sql, params) def get_context_options(self): return { 'field_types': self._field_types, 'operations': self._operations, 'param': self.param, 'quote': self.quote, 'compound_select_parentheses': self.compound_select_parentheses, 'conflict_statement': self.conflict_statement, 'conflict_update': self.conflict_update, 'for_update': self.for_update, 'index_schema_prefix': self.index_schema_prefix, 'index_using_precedes_table': self.index_using_precedes_table, 'limit_max': self.limit_max, 'nulls_ordering': self.nulls_ordering, } def get_sql_context(self, **context_options): context = self.get_context_options() if context_options: context.update(context_options) return self.context_class(**context) def conflict_statement(self, on_conflict, query): raise NotImplementedError def conflict_update(self, on_conflict, query): raise NotImplementedError def _build_on_conflict_update(self, on_conflict, query): if on_conflict._conflict_target: stmt = SQL('ON CONFLICT') target = EnclosedNodeList([ Entity(col) if isinstance(col, str) else col for col in on_conflict._conflict_target]) if on_conflict._conflict_where is not None: target = NodeList([target, SQL('WHERE'), on_conflict._conflict_where]) else: stmt = SQL('ON CONFLICT ON CONSTRAINT') target = on_conflict._conflict_constraint if isinstance(target, str): target = Entity(target) updates = [] if on_conflict._preserve: for column in on_conflict._preserve: excluded = NodeList((SQL('EXCLUDED'), ensure_entity(column)), glue='.') expression = NodeList((ensure_entity(column), SQL('='), excluded)) updates.append(expression) if on_conflict._update: for k, v in on_conflict._update.items(): if not isinstance(v, Node): # Attempt to resolve string field-names to their respective # field object, to apply data-type conversions. if isinstance(k, str): k = getattr(query.table, k) if isinstance(k, Field): v = k.to_value(v) else: v = Value(v, unpack=False) else: v = QualifiedNames(v) updates.append(NodeList((ensure_entity(k), SQL('='), v))) parts = [stmt, target, SQL('DO UPDATE SET'), CommaNodeList(updates)] if on_conflict._where: parts.extend((SQL('WHERE'), QualifiedNames(on_conflict._where))) return NodeList(parts) def last_insert_id(self, cursor, query_type=None): return cursor.lastrowid def rows_affected(self, cursor): return cursor.rowcount def default_values_insert(self, ctx): return ctx.literal('DEFAULT VALUES') def session_start(self): return self.transaction().__enter__() def session_commit(self): try: txn = self.pop_transaction() except IndexError: return False txn.commit(begin=self.in_transaction()) return True def session_rollback(self): try: txn = self.pop_transaction() except IndexError: return False txn.rollback(begin=self.in_transaction()) return True def in_transaction(self): return bool(self._state.transactions) def push_transaction(self, transaction): self._state.transactions.append(transaction) def pop_transaction(self): return self._state.transactions.pop() def transaction_depth(self): return len(self._state.transactions) def top_transaction(self): if self._state.transactions: return self._state.transactions[-1] def atomic(self, *args, **kwargs): return _atomic(self, *args, **kwargs) def manual_commit(self): return _manual(self) def transaction(self, *args, **kwargs): return _transaction(self, *args, **kwargs) def savepoint(self): return _savepoint(self) def begin(self): if self.is_closed(): self.connect() with __exception_wrapper__: self.cursor().execute('BEGIN') def rollback(self): with __exception_wrapper__: self.cursor().execute('ROLLBACK') def commit(self): with __exception_wrapper__: self.cursor().execute('COMMIT') def batch_commit(self, it, n): for group in chunked(it, n): with self.atomic(): for obj in group: yield obj def table_exists(self, table_name, schema=None): if is_model(table_name): model = table_name table_name = model._meta.table_name schema = model._meta.schema return table_name in self.get_tables(schema=schema) def get_tables(self, schema=None): raise NotImplementedError def get_indexes(self, table, schema=None): raise NotImplementedError def get_columns(self, table, schema=None): raise NotImplementedError def get_primary_keys(self, table, schema=None): raise NotImplementedError def get_foreign_keys(self, table, schema=None): raise NotImplementedError def sequence_exists(self, seq): raise NotImplementedError def create_tables(self, models, **options): for model in sort_models(models): model.create_table(**options) def drop_tables(self, models, **kwargs): for model in reversed(sort_models(models)): model.drop_table(**kwargs) def extract_date(self, date_part, date_field): raise NotImplementedError def truncate_date(self, date_part, date_field): raise NotImplementedError def to_timestamp(self, date_field): raise NotImplementedError def from_timestamp(self, date_field): raise NotImplementedError def random(self): return fn.random() def bind(self, models, bind_refs=True, bind_backrefs=True): for model in models: model.bind(self, bind_refs=bind_refs, bind_backrefs=bind_backrefs) def bind_ctx(self, models, bind_refs=True, bind_backrefs=True): return _BoundModelsContext(models, self, bind_refs, bind_backrefs) def get_noop_select(self, ctx): return ctx.sql(Select().columns(SQL('0')).where(SQL('0'))) @property def Model(self): if not hasattr(self, '_Model'): class Meta: database = self self._Model = type('BaseModel', (Model,), {'Meta': Meta}) return self._Model def __pragma__(name): def __get__(self): return self.pragma(name) def __set__(self, value): return self.pragma(name, value) return property(__get__, __set__) class SqliteDatabase(Database): field_types = { 'BIGAUTO': FIELD.AUTO, 'BIGINT': FIELD.INT, 'BOOL': FIELD.INT, 'DOUBLE': FIELD.FLOAT, 'SMALLINT': FIELD.INT, 'UUID': FIELD.TEXT} operations = { 'LIKE': 'GLOB', 'ILIKE': 'LIKE'} index_schema_prefix = True limit_max = -1 server_version = __sqlite_version__ truncate_table = False def __init__(self, database, pragmas=None, regexp_function=False, rank_functions=False, *args, **kwargs): isolation = kwargs.pop('isolation_level', None) if isolation is not None: raise ImproperlyConfigured('isolation_level must be None when ' 'using peewee.') self._pragmas = pragmas or () super(SqliteDatabase, self).__init__(database, *args, **kwargs) self._aggregates = {} self._collations = {} self._functions = {} self._window_functions = {} self._extensions = set() self._attached = {} self.nulls_ordering = self.server_version >= (3, 30, 0) self.register_function(_sqlite_date_part, 'date_part', 2) self.register_function(_sqlite_date_trunc, 'date_trunc', 2) if regexp_function: self.register_function(_sqlite_regexp, 'regexp', 2) if rank_functions: from playhouse.sqlite_udf import register_udf_groups, RANK register_udf_groups(self, RANK) def init(self, database, pragmas=None, timeout=5, returning_clause=None, **kwargs): if pragmas is not None: self._pragmas = pragmas if isinstance(self._pragmas, dict): self._pragmas = list(self._pragmas.items()) if returning_clause is not None: if __sqlite_version__ < (3, 35, 0): warnings.warn('RETURNING clause requires Sqlite 3.35 or newer') self.returning_clause = returning_clause self._timeout = timeout super(SqliteDatabase, self).init(database, **kwargs) def _set_server_version(self, conn): pass def _connect(self): if sqlite3 is None: raise ImproperlyConfigured('SQLite driver not installed!') conn = sqlite3.connect(self.database, timeout=self._timeout, isolation_level=None, **self.connect_params) try: self._add_conn_hooks(conn) except Exception: conn.close() raise return conn def _add_conn_hooks(self, conn): if self._attached: self._attach_databases(conn) if self._pragmas: self._set_pragmas(conn) self._load_aggregates(conn) self._load_collations(conn) self._load_functions(conn) if self.server_version >= (3, 25, 0): self._load_window_functions(conn) if self._extensions: self._load_extensions(conn) def _set_pragmas(self, conn): cursor = conn.cursor() for pragma, value in self._pragmas: cursor.execute('PRAGMA %s = %s;' % (pragma, value)) cursor.close() def _attach_databases(self, conn): cursor = conn.cursor() for name, db in self._attached.items(): cursor.execute('ATTACH DATABASE "%s" AS "%s"' % (db, name)) cursor.close() def pragma(self, key, value=SENTINEL, permanent=False, schema=None): if schema is not None: key = '"%s".%s' % (schema, key) sql = 'PRAGMA %s' % key if value is not SENTINEL: sql += ' = %s' % (value or 0) if permanent: pragmas = dict(self._pragmas or ()) pragmas[key] = value self._pragmas = list(pragmas.items()) elif permanent: raise ValueError('Cannot specify a permanent pragma without value') row = self.execute_sql(sql).fetchone() if row: return row[0] cache_size = __pragma__('cache_size') foreign_keys = __pragma__('foreign_keys') journal_mode = __pragma__('journal_mode') journal_size_limit = __pragma__('journal_size_limit') mmap_size = __pragma__('mmap_size') page_size = __pragma__('page_size') read_uncommitted = __pragma__('read_uncommitted') synchronous = __pragma__('synchronous') wal_autocheckpoint = __pragma__('wal_autocheckpoint') application_id = __pragma__('application_id') user_version = __pragma__('user_version') data_version = __pragma__('data_version') @property def timeout(self): return self._timeout @timeout.setter def timeout(self, seconds): if self._timeout == seconds: return self._timeout = seconds if not self.is_closed(): # PySQLite multiplies user timeout by 1000, but the unit of the # timeout PRAGMA is actually milliseconds. self.execute_sql('PRAGMA busy_timeout=%d;' % (seconds * 1000)) def _load_aggregates(self, conn): for name, (klass, num_params) in self._aggregates.items(): conn.create_aggregate(name, num_params, klass) def _load_collations(self, conn): for name, fn in self._collations.items(): conn.create_collation(name, fn) def _load_functions(self, conn): for name, (fn, n_params, deterministic) in self._functions.items(): kwargs = {'deterministic': deterministic} if deterministic else {} conn.create_function(name, n_params, fn, **kwargs) def _load_window_functions(self, conn): for name, (klass, num_params) in self._window_functions.items(): conn.create_window_function(name, num_params, klass) def register_aggregate(self, klass, name=None, num_params=-1): self._aggregates[name or klass.__name__.lower()] = (klass, num_params) if not self.is_closed(): self._load_aggregates(self.connection()) def aggregate(self, name=None, num_params=-1): def decorator(klass): self.register_aggregate(klass, name, num_params) return klass return decorator def register_collation(self, fn, name=None): name = name or fn.__name__ def _collation(*args): expressions = args + (SQL('collate %s' % name),) return NodeList(expressions) fn.collation = _collation self._collations[name] = fn if not self.is_closed(): self._load_collations(self.connection()) def collation(self, name=None): def decorator(fn): self.register_collation(fn, name) return fn return decorator def register_function(self, fn, name=None, num_params=-1, deterministic=None): self._functions[name or fn.__name__] = (fn, num_params, deterministic) if not self.is_closed(): self._load_functions(self.connection()) def func(self, name=None, num_params=-1, deterministic=None): def decorator(fn): self.register_function(fn, name, num_params, deterministic) return fn return decorator def register_window_function(self, klass, name=None, num_params=-1): name = name or klass.__name__.lower() self._window_functions[name] = (klass, num_params) if not self.is_closed(): self._load_window_functions(self.connection()) def window_function(self, name=None, num_params=-1): def decorator(klass): self.register_window_function(klass, name, num_params) return klass return decorator def unregister_aggregate(self, name): del(self._aggregates[name]) def unregister_collation(self, name): del(self._collations[name]) def unregister_function(self, name): del(self._functions[name]) def unregister_window_function(self, name): del(self._window_functions[name]) def _load_extensions(self, conn): conn.enable_load_extension(True) for extension in self._extensions: conn.load_extension(extension) def load_extension(self, extension): self._extensions.add(extension) if not self.is_closed(): conn = self.connection() conn.enable_load_extension(True) conn.load_extension(extension) def unload_extension(self, extension): self._extensions.remove(extension) def attach(self, filename, name): if name in self._attached: if self._attached[name] == filename: return False raise OperationalError('schema "%s" already attached.' % name) self._attached[name] = filename if not self.is_closed(): self.execute_sql('ATTACH DATABASE "%s" AS "%s"' % (filename, name)) return True def detach(self, name): if name not in self._attached: return False del self._attached[name] if not self.is_closed(): self.execute_sql('DETACH DATABASE "%s"' % name) return True def last_insert_id(self, cursor, query_type=None): if not self.returning_clause: return cursor.lastrowid elif query_type == Insert.SIMPLE: try: return cursor[0][0] except (IndexError, KeyError, TypeError): pass return cursor def rows_affected(self, cursor): try: return cursor.rowcount except AttributeError: return cursor.cursor.rowcount # This was a RETURNING query. def begin(self, lock_type=None): statement = 'BEGIN %s' % lock_type if lock_type else 'BEGIN' self.execute_sql(statement) def commit(self): with __exception_wrapper__: return self.execute_sql('COMMIT') def rollback(self): with __exception_wrapper__: return self.execute_sql('ROLLBACK') def get_tables(self, schema=None): schema = schema or 'main' cursor = self.execute_sql('SELECT name FROM "%s".sqlite_master WHERE ' 'type=? ORDER BY name' % schema, ('table',)) return [row for row, in cursor.fetchall()] def get_views(self, schema=None): sql = ('SELECT name, sql FROM "%s".sqlite_master WHERE type=? ' 'ORDER BY name') % (schema or 'main') return [ViewMetadata(*row) for row in self.execute_sql(sql, ('view',))] def get_indexes(self, table, schema=None): schema = schema or 'main' query = ('SELECT name, sql FROM "%s".sqlite_master ' 'WHERE tbl_name = ? AND type = ? ORDER BY name') % schema cursor = self.execute_sql(query, (table, 'index')) index_to_sql = dict(cursor.fetchall()) # Determine which indexes have a unique constraint. unique_indexes = set() cursor = self.execute_sql('PRAGMA "%s".index_list("%s")' % (schema, table)) for row in cursor.fetchall(): name = row[1] is_unique = int(row[2]) == 1 if is_unique: unique_indexes.add(name) # Retrieve the indexed columns. index_columns = {} for index_name in sorted(index_to_sql): cursor = self.execute_sql('PRAGMA "%s".index_info("%s")' % (schema, index_name)) index_columns[index_name] = [row[2] for row in cursor.fetchall()] return [ IndexMetadata( name, index_to_sql[name], index_columns[name], name in unique_indexes, table) for name in sorted(index_to_sql)] def get_columns(self, table, schema=None): cursor = self.execute_sql('PRAGMA "%s".table_info("%s")' % (schema or 'main', table)) return [ColumnMetadata(r[1], r[2], not r[3], bool(r[5]), table, r[4]) for r in cursor.fetchall()] def get_primary_keys(self, table, schema=None): cursor = self.execute_sql('PRAGMA "%s".table_info("%s")' % (schema or 'main', table)) return [row[1] for row in filter(lambda r: r[-1], cursor.fetchall())] def get_foreign_keys(self, table, schema=None): cursor = self.execute_sql('PRAGMA "%s".foreign_key_list("%s")' % (schema or 'main', table)) return [ForeignKeyMetadata(row[3], row[2], row[4], table) for row in cursor.fetchall()] def get_binary_type(self): return sqlite3.Binary def conflict_statement(self, on_conflict, query): action = on_conflict._action.lower() if on_conflict._action else '' if action and action not in ('nothing', 'update'): return SQL('INSERT OR %s' % on_conflict._action.upper()) def conflict_update(self, oc, query): # Sqlite prior to 3.24.0 does not support Postgres-style upsert. if self.server_version < (3, 24, 0) and \ any((oc._preserve, oc._update, oc._where, oc._conflict_target, oc._conflict_constraint)): raise ValueError('SQLite does not support specifying which values ' 'to preserve or update.') action = oc._action.lower() if oc._action else '' if action and action not in ('nothing', 'update', ''): return if action == 'nothing': return SQL('ON CONFLICT DO NOTHING') elif not oc._update and not oc._preserve: raise ValueError('If you are not performing any updates (or ' 'preserving any INSERTed values), then the ' 'conflict resolution action should be set to ' '"NOTHING".') elif oc._conflict_constraint: raise ValueError('SQLite does not support specifying named ' 'constraints for conflict resolution.') elif not oc._conflict_target: raise ValueError('SQLite requires that a conflict target be ' 'specified when doing an upsert.') return self._build_on_conflict_update(oc, query) def extract_date(self, date_part, date_field): return fn.date_part(date_part, date_field, python_value=int) def truncate_date(self, date_part, date_field): return fn.date_trunc(date_part, date_field, python_value=simple_date_time) def to_timestamp(self, date_field): return fn.strftime('%s', date_field).cast('integer') def from_timestamp(self, date_field): return fn.datetime(date_field, 'unixepoch') class _BasePsycopgAdapter(object): isolation_levels = {} # Map int -> str. def __init__(self): self.isolation_levels_inv = { v: k for k, v in self.isolation_levels.items()} def isolation_level_int(self, isolation_level): if isinstance(isolation_level, str): return self.isolation_levels_inv[isolation_level] return isolation_level def isolation_level_str(self, isolation_level): if isinstance(isolation_level, int): return self.isolation_levels[isolation_level] return isolation_level class Psycopg2Adapter(_BasePsycopgAdapter): isolation_levels = { 1: 'READ COMMITTED', 2: 'REPEATABLE READ', 3: 'SERIALIZABLE', 4: 'READ UNCOMMITTED', } def __init__(self): super(Psycopg2Adapter, self).__init__() self.json_type = Json_pg2 self.jsonb_type = Json_pg2 self.cast_json_case = True def check_driver(self): if psycopg2 is None: raise ImproperlyConfigured('psycopg2 postgres driver not found.') def get_binary_type(self): return psycopg2.Binary def connect(self, db, **params): if db.database.startswith('postgresql://'): params.setdefault('dsn', db.database) else: params.setdefault('dbname', db.database) conn = psycopg2.connect(**params) if db._register_unicode: pg_extensions.register_type(pg_extensions.UNICODE, conn) pg_extensions.register_type(pg_extensions.UNICODEARRAY, conn) if db._encoding: conn.set_client_encoding(db._encoding) return conn def get_server_version(self, conn): return conn.server_version def is_connection_usable(self, conn): txn_status = conn.get_transaction_status() return txn_status < pg_extensions.TRANSACTION_STATUS_INERROR def is_connection_reusable(self, conn): txn_status = conn.get_transaction_status() # Do not return connection in an error state, as subsequent queries # will all fail. If the status is unknown then we lost the connection # to the server and the connection should not be re-used. if txn_status == pg_extensions.TRANSACTION_STATUS_UNKNOWN: return False elif txn_status == pg_extensions.TRANSACTION_STATUS_INERROR: conn.reset() elif txn_status != pg_extensions.TRANSACTION_STATUS_IDLE: conn.rollback() return True def is_connection_closed(self, conn): txn_status = conn.get_transaction_status() if txn_status == pg_extensions.TRANSACTION_STATUS_UNKNOWN: return True elif txn_status != pg_extensions.TRANSACTION_STATUS_IDLE: conn.rollback() return False class Psycopg3Adapter(_BasePsycopgAdapter): isolation_levels = { 1: 'READ UNCOMMITTED', 2: 'READ COMMITTED', 3: 'REPEATABLE READ', 4: 'SERIALIZABLE', } def __init__(self): super(Psycopg3Adapter, self).__init__() self.json_type = Json_pg3 self.jsonb_type = Jsonb_pg3 self.cast_json_case = False def check_driver(self): if psycopg is None: raise ImproperlyConfigured('psycopg postgres driver not found.') def get_binary_type(self): return psycopg.Binary def connect(self, db, **params): if db.database.startswith('postgresql://'): params.setdefault('conninfo', db.database) else: params.setdefault('dbname', db.database) return psycopg.connect(**params) def get_server_version(self, conn): return conn.pgconn.server_version def is_connection_usable(self, conn): return conn.pgconn.transaction_status < TransactionStatus.INERROR def is_connection_reusable(self, conn): txn_status = conn.pgconn.transaction_status # Do not return connection in an error state, as subsequent queries # will all fail. If the status is unknown then we lost the connection # to the server and the connection should not be re-used. if txn_status == TransactionStatus.UNKNOWN: return False elif txn_status == TransactionStatus.INERROR: conn.reset() elif txn_status != TransactionStatus.IDLE: conn.rollback() return True def is_connection_closed(self, conn): txn_status = conn.pgconn.transaction_status if txn_status == TransactionStatus.UNKNOWN: return True elif txn_status != TransactionStatus.IDLE: conn.rollback() return False class PostgresqlDatabase(Database): field_types = { 'AUTO': 'SERIAL', 'BIGAUTO': 'BIGSERIAL', 'BLOB': 'BYTEA', 'BOOL': 'BOOLEAN', 'DATETIME': 'TIMESTAMP', 'DECIMAL': 'NUMERIC', 'DOUBLE': 'DOUBLE PRECISION', 'UUID': 'UUID', 'UUIDB': 'BYTEA'} operations = {'REGEXP': '~', 'IREGEXP': '~*'} param = '%s' compound_select_parentheses = CSQ_PARENTHESES_ALWAYS for_update = True nulls_ordering = True returning_clause = True sequences = True psycopg2_adapter = Psycopg2Adapter psycopg3_adapter = Psycopg3Adapter def init(self, database, register_unicode=True, encoding=None, isolation_level=None, **kwargs): self._register_unicode = register_unicode self._encoding = encoding prefer_psycopg3 = kwargs.pop('prefer_psycopg3', False) if psycopg is not None and prefer_psycopg3: self._adapter = self.psycopg3_adapter() else: self._adapter = self.psycopg2_adapter() # Accept a string ('READ COMMITTED') or an int constant. Since the # constants vary between psycopg2 & psycopg3 we have to abstract this. self._isolation_level = self._adapter.isolation_level_int( isolation_level) super(PostgresqlDatabase, self).init(database, **kwargs) def _connect(self): self._adapter.check_driver() # Handle connection-strings nicely, since psycopg will accept them, # and they may be easier when lots of parameters are specified. conn = self._adapter.connect(self, **self.connect_params) if self._isolation_level: conn.set_isolation_level(self._isolation_level) conn.autocommit = True return conn def _set_server_version(self, conn): self.server_version = self._adapter.get_server_version(conn) def is_connection_usable(self): if self._state.closed: return False # Returns True if we are idle, running a command, or in an active # connection. If the connection is in an error state or the connection # is otherwise unusable, return False. return self._adapter.is_connection_usable(self._state.conn) def last_insert_id(self, cursor, query_type=None): try: return cursor if query_type != Insert.SIMPLE else cursor[0][0] except (IndexError, KeyError, TypeError): pass def rows_affected(self, cursor): try: return cursor.rowcount except AttributeError: return cursor.cursor.rowcount def begin(self, isolation_level=None): if self.is_closed(): self.connect() if isolation_level: txn_type = self._adapter.isolation_level_str(isolation_level) stmt = 'BEGIN TRANSACTION ISOLATION LEVEL %s' % txn_type else: stmt = 'BEGIN' with __exception_wrapper__: self.cursor().execute(stmt) def get_tables(self, schema=None): query = ('SELECT tablename FROM pg_catalog.pg_tables ' 'WHERE schemaname = %s ORDER BY tablename') cursor = self.execute_sql(query, (schema or 'public',)) return [table for table, in cursor.fetchall()] def get_views(self, schema=None): query = ('SELECT viewname, definition FROM pg_catalog.pg_views ' 'WHERE schemaname = %s ORDER BY viewname') cursor = self.execute_sql(query, (schema or 'public',)) return [ViewMetadata(view_name, sql.strip(' \t;')) for (view_name, sql) in cursor.fetchall()] def get_indexes(self, table, schema=None): query = """ SELECT i.relname, idxs.indexdef, idx.indisunique, array_to_string(ARRAY( SELECT pg_get_indexdef(idx.indexrelid, k + 1, TRUE) FROM generate_subscripts(idx.indkey, 1) AS k ORDER BY k), ',') FROM pg_catalog.pg_class AS t INNER JOIN pg_catalog.pg_index AS idx ON t.oid = idx.indrelid INNER JOIN pg_catalog.pg_class AS i ON idx.indexrelid = i.oid INNER JOIN pg_catalog.pg_indexes AS idxs ON (idxs.tablename = t.relname AND idxs.indexname = i.relname) WHERE t.relname = %s AND t.relkind = %s AND idxs.schemaname = %s ORDER BY idx.indisunique DESC, i.relname;""" cursor = self.execute_sql(query, (table, 'r', schema or 'public')) return [IndexMetadata(name, sql.rstrip(' ;'), columns.split(','), is_unique, table) for name, sql, is_unique, columns in cursor.fetchall()] def get_columns(self, table, schema=None): query = """ SELECT column_name, is_nullable, data_type, column_default FROM information_schema.columns WHERE table_name = %s AND table_schema = %s ORDER BY ordinal_position""" cursor = self.execute_sql(query, (table, schema or 'public')) pks = set(self.get_primary_keys(table, schema)) return [ColumnMetadata(name, dt, null == 'YES', name in pks, table, df) for name, null, dt, df in cursor.fetchall()] def get_primary_keys(self, table, schema=None): query = """ SELECT kc.column_name FROM information_schema.table_constraints AS tc INNER JOIN information_schema.key_column_usage AS kc ON ( tc.table_name = kc.table_name AND tc.table_schema = kc.table_schema AND tc.constraint_name = kc.constraint_name) WHERE tc.constraint_type = %s AND tc.table_name = %s AND tc.table_schema = %s""" ctype = 'PRIMARY KEY' cursor = self.execute_sql(query, (ctype, table, schema or 'public')) return [pk for pk, in cursor.fetchall()] def get_foreign_keys(self, table, schema=None): sql = """ SELECT DISTINCT kcu.column_name, ccu.table_name, ccu.column_name FROM information_schema.table_constraints AS tc JOIN information_schema.key_column_usage AS kcu ON (tc.constraint_name = kcu.constraint_name AND tc.constraint_schema = kcu.constraint_schema AND tc.table_name = kcu.table_name AND tc.table_schema = kcu.table_schema) JOIN information_schema.constraint_column_usage AS ccu ON (ccu.constraint_name = tc.constraint_name AND ccu.constraint_schema = tc.constraint_schema) WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name = %s AND tc.table_schema = %s""" cursor = self.execute_sql(sql, (table, schema or 'public')) return [ForeignKeyMetadata(row[0], row[1], row[2], table) for row in cursor.fetchall()] def sequence_exists(self, sequence): res = self.execute_sql(""" SELECT COUNT(*) FROM pg_class, pg_namespace WHERE relkind='S' AND pg_class.relnamespace = pg_namespace.oid AND relname=%s""", (sequence,)) return bool(res.fetchone()[0]) def get_binary_type(self): return self._adapter.get_binary_type() def conflict_statement(self, on_conflict, query): return def conflict_update(self, oc, query): action = oc._action.lower() if oc._action else '' if action in ('ignore', 'nothing'): parts = [SQL('ON CONFLICT')] if oc._conflict_target: parts.append(EnclosedNodeList([ Entity(col) if isinstance(col, str) else col for col in oc._conflict_target])) parts.append(SQL('DO NOTHING')) return NodeList(parts) elif action and action != 'update': raise ValueError('The only supported actions for conflict ' 'resolution with Postgresql are "ignore" or ' '"update".') elif not oc._update and not oc._preserve: raise ValueError('If you are not performing any updates (or ' 'preserving any INSERTed values), then the ' 'conflict resolution action should be set to ' '"IGNORE".') elif not (oc._conflict_target or oc._conflict_constraint): raise ValueError('Postgres requires that a conflict target be ' 'specified when doing an upsert.') return self._build_on_conflict_update(oc, query) def extract_date(self, date_part, date_field): return fn.EXTRACT(NodeList((SQL(date_part), SQL('FROM'), date_field))) def truncate_date(self, date_part, date_field): return fn.DATE_TRUNC(date_part, date_field) def interval(self, val): return NodeList((SQL('interval'), val)) def to_timestamp(self, date_field): return self.extract_date('EPOCH', date_field) def from_timestamp(self, date_field): # Ironically, here, Postgres means "to the Postgresql timestamp type". return fn.to_timestamp(date_field) def get_noop_select(self, ctx): return ctx.sql(Select().columns(SQL('0')).where(SQL('false'))) def set_time_zone(self, timezone): self.execute_sql('set time zone \'%s\';' % timezone.replace("'", "''")) def set_isolation_level(self, isolation_level): self._isolation_level = self._adapter.isolation_level_int( isolation_level) class MySQLDatabase(Database): field_types = { 'AUTO': 'INTEGER AUTO_INCREMENT', 'BIGAUTO': 'BIGINT AUTO_INCREMENT', 'BOOL': 'BOOL', 'DECIMAL': 'NUMERIC', 'DOUBLE': 'DOUBLE PRECISION', 'FLOAT': 'FLOAT', 'UUID': 'VARCHAR(40)', 'UUIDB': 'VARBINARY(16)'} operations = { 'LIKE': 'LIKE BINARY', 'ILIKE': 'LIKE', 'REGEXP': 'REGEXP BINARY', 'IREGEXP': 'REGEXP', 'XOR': 'XOR'} param = '%s' quote = '``' compound_select_parentheses = CSQ_PARENTHESES_UNNESTED for_update = True index_using_precedes_table = True limit_max = 2 ** 64 - 1 safe_create_index = False safe_drop_index = False sql_mode = 'PIPES_AS_CONCAT' def init(self, database, **kwargs): params = { 'charset': 'utf8', 'sql_mode': self.sql_mode, 'use_unicode': True} params.update(kwargs) if 'password' in params and mysql_passwd: params['passwd'] = params.pop('password') super(MySQLDatabase, self).init(database, **params) def _connect(self): if mysql is None: raise ImproperlyConfigured('MySQL driver not installed!') conn = mysql.connect(db=self.database, autocommit=True, **self.connect_params) return conn def _set_server_version(self, conn): try: version_raw = conn.server_version except AttributeError: version_raw = conn.get_server_info() self.server_version = self._extract_server_version(version_raw) def _extract_server_version(self, version): if isinstance(version, tuple): return version version = version.lower() if 'maria' in version: match_obj = re.search(r'(1\d\.\d+\.\d+)', version) else: match_obj = re.search(r'(\d{1,2}\.\d+\.\d+)', version) if match_obj is not None: return tuple(int(num) for num in match_obj.groups()[0].split('.')) warnings.warn('Unable to determine MySQL version: "%s"' % version) return (0, 0, 0) # Unable to determine version! def is_connection_usable(self): if self._state.closed: return False conn = self._state.conn if hasattr(conn, 'ping'): if self.server_version[0] >= 8: args = () else: args = (False,) try: conn.ping(*args) except Exception: return False return True def default_values_insert(self, ctx): return ctx.literal('() VALUES ()') def begin(self, isolation_level=None): if self.is_closed(): self.connect() with __exception_wrapper__: curs = self.cursor() if isolation_level: curs.execute('SET TRANSACTION ISOLATION LEVEL %s' % isolation_level) curs.execute('BEGIN') def get_tables(self, schema=None): query = ('SELECT table_name FROM information_schema.tables ' 'WHERE table_schema = DATABASE() AND table_type != %s ' 'ORDER BY table_name') return [table for table, in self.execute_sql(query, ('VIEW',))] def get_views(self, schema=None): query = ('SELECT table_name, view_definition ' 'FROM information_schema.views ' 'WHERE table_schema = DATABASE() ORDER BY table_name') cursor = self.execute_sql(query) return [ViewMetadata(*row) for row in cursor.fetchall()] def get_indexes(self, table, schema=None): cursor = self.execute_sql('SHOW INDEX FROM `%s`' % table) unique = set() indexes = {} for row in cursor.fetchall(): if not row[1]: unique.add(row[2]) indexes.setdefault(row[2], []) indexes[row[2]].append(row[4]) return [IndexMetadata(name, None, indexes[name], name in unique, table) for name in indexes] def get_columns(self, table, schema=None): sql = """ SELECT column_name, is_nullable, data_type, column_default FROM information_schema.columns WHERE table_name = %s AND table_schema = DATABASE() ORDER BY ordinal_position""" cursor = self.execute_sql(sql, (table,)) pks = set(self.get_primary_keys(table)) return [ColumnMetadata(name, dt, null == 'YES', name in pks, table, df) for name, null, dt, df in cursor.fetchall()] def get_primary_keys(self, table, schema=None): cursor = self.execute_sql('SHOW INDEX FROM `%s`' % table) return [row[4] for row in filter(lambda row: row[2] == 'PRIMARY', cursor.fetchall())] def get_foreign_keys(self, table, schema=None): query = """ SELECT column_name, referenced_table_name, referenced_column_name FROM information_schema.key_column_usage WHERE table_name = %s AND table_schema = DATABASE() AND referenced_table_name IS NOT NULL AND referenced_column_name IS NOT NULL""" cursor = self.execute_sql(query, (table,)) return [ ForeignKeyMetadata(column, dest_table, dest_column, table) for column, dest_table, dest_column in cursor.fetchall()] def get_binary_type(self): return mysql.Binary def conflict_statement(self, on_conflict, query): if not on_conflict._action: return action = on_conflict._action.lower() if action == 'replace': return SQL('REPLACE') elif action == 'ignore': return SQL('INSERT IGNORE') elif action != 'update': raise ValueError('Un-supported action for conflict resolution. ' 'MySQL supports REPLACE, IGNORE and UPDATE.') def conflict_update(self, on_conflict, query): if on_conflict._where or on_conflict._conflict_target or \ on_conflict._conflict_constraint: raise ValueError('MySQL does not support the specification of ' 'where clauses or conflict targets for conflict ' 'resolution.') updates = [] if on_conflict._preserve: # Here we need to determine which function to use, which varies # depending on the MySQL server version. MySQL and MariaDB prior to # 10.3.3 use "VALUES", while MariaDB 10.3.3+ use "VALUE". version = self.server_version or (0,) if version[0] >= 10 and version >= (10, 3, 3): VALUE_FN = fn.VALUE else: VALUE_FN = fn.VALUES for column in on_conflict._preserve: entity = ensure_entity(column) expression = NodeList(( ensure_entity(column), SQL('='), VALUE_FN(entity))) updates.append(expression) if on_conflict._update: for k, v in on_conflict._update.items(): if not isinstance(v, Node): # Attempt to resolve string field-names to their respective # field object, to apply data-type conversions. if isinstance(k, str): k = getattr(query.table, k) if isinstance(k, Field): v = k.to_value(v) else: v = Value(v, unpack=False) updates.append(NodeList((ensure_entity(k), SQL('='), v))) if updates: return NodeList((SQL('ON DUPLICATE KEY UPDATE'), CommaNodeList(updates))) def extract_date(self, date_part, date_field): return fn.EXTRACT(NodeList((SQL(date_part), SQL('FROM'), date_field))) def truncate_date(self, date_part, date_field): return fn.DATE_FORMAT(date_field, __mysql_date_trunc__[date_part], python_value=simple_date_time) def to_timestamp(self, date_field): return fn.UNIX_TIMESTAMP(date_field) def from_timestamp(self, date_field): return fn.FROM_UNIXTIME(date_field) def random(self): return fn.rand() def get_noop_select(self, ctx): return ctx.literal('DO 0') # TRANSACTION CONTROL. class _manual(object): def __init__(self, db): self.db = db def __call__(self, fn): @wraps(fn) def inner(*args, **kwargs): with _manual(self.db): return fn(*args, **kwargs) return inner def __enter__(self): top = self.db.top_transaction() if top is not None and not isinstance(top, _manual): raise ValueError('Cannot enter manual commit block while a ' 'transaction is active.') self.db.push_transaction(self) def __exit__(self, exc_type, exc_val, exc_tb): if self.db.pop_transaction() is not self: raise ValueError('Transaction stack corrupted while exiting ' 'manual commit block.') class _atomic(object): def __init__(self, db, *args, **kwargs): self.db = db self._transaction_args = (args, kwargs) def __call__(self, fn): @wraps(fn) def inner(*args, **kwargs): a, k = self._transaction_args with _atomic(self.db, *a, **k): return fn(*args, **kwargs) return inner def __enter__(self): if self.db.transaction_depth() == 0: args, kwargs = self._transaction_args self._helper = self.db.transaction(*args, **kwargs) elif isinstance(self.db.top_transaction(), _manual): raise ValueError('Cannot enter atomic commit block while in ' 'manual commit mode.') else: self._helper = self.db.savepoint() return self._helper.__enter__() def __exit__(self, exc_type, exc_val, exc_tb): return self._helper.__exit__(exc_type, exc_val, exc_tb) class _transaction(object): def __init__(self, db, *args, **kwargs): self.db = db self._begin_args = (args, kwargs) def __call__(self, fn): @wraps(fn) def inner(*args, **kwargs): a, k = self._begin_args with _transaction(self.db, *a, **k): return fn(*args, **kwargs) return inner def _begin(self): args, kwargs = self._begin_args self.db.begin(*args, **kwargs) def commit(self, begin=True): self.db.commit() if begin: self._begin() def rollback(self, begin=True): self.db.rollback() if begin: self._begin() def __enter__(self): if self.db.transaction_depth() == 0: self._begin() self.db.push_transaction(self) return self def __exit__(self, exc_type, exc_val, exc_tb): depth = self.db.transaction_depth() self.db.pop_transaction() if exc_type and depth == 1: self.rollback(False) elif depth == 1: try: self.commit(False) except Exception: self.rollback(False) raise class _savepoint(object): def __init__(self, db, sid=None): self.db = db self.sid = sid or 's' + uuid.uuid4().hex self.quoted_sid = self.sid.join(self.db.quote) def __call__(self, fn): @wraps(fn) def inner(*args, **kwargs): with _savepoint(self.db): return fn(*args, **kwargs) return inner def _begin(self): self.db.execute_sql('SAVEPOINT %s;' % self.quoted_sid) def commit(self, begin=True): self.db.execute_sql('RELEASE SAVEPOINT %s;' % self.quoted_sid) if begin: self._begin() def rollback(self, begin=True): self.db.execute_sql('ROLLBACK TO SAVEPOINT %s;' % self.quoted_sid) if begin: self._begin() def __enter__(self): self._begin() return self def __exit__(self, exc_type, exc_val, exc_tb): if exc_type: self.rollback(False) else: try: self.commit(begin=False) except Exception: self.rollback(begin=False) raise # CURSOR REPRESENTATIONS. class CursorWrapper(object): def __init__(self, cursor): self.cursor = cursor self.count = 0 self.index = 0 self.initialized = False self.populated = False self.row_cache = [] def __iter__(self): if self.populated: return iter(self.row_cache) return ResultIterator(self) def __getitem__(self, item): if isinstance(item, slice): stop = item.stop if stop is None or stop < 0: self.fill_cache() else: self.fill_cache(stop) return self.row_cache[item] elif isinstance(item, int): self.fill_cache(item if item > 0 else 0) return self.row_cache[item] else: raise ValueError('CursorWrapper only supports integer and slice ' 'indexes.') def __len__(self): self.fill_cache() return self.count def initialize(self): pass def iterate(self, cache=True): row = self.cursor.fetchone() if row is None: self.populated = True self.cursor.close() raise StopIteration elif not self.initialized: self.initialize() # Lazy initialization. self.initialized = True self.count += 1 result = self.process_row(row) if cache: self.row_cache.append(result) return result def process_row(self, row): return row def iterator(self): """Efficient one-pass iteration over the result set.""" while True: try: yield self.iterate(False) except StopIteration: return def fill_cache(self, n=0): n = n or float('Inf') if n < 0: raise ValueError('Negative values are not supported.') if self.populated or n <= self.count: # We've already filled the requested rows. return iterator = ResultIterator(self) iterator.index = self.count while not self.populated and (n > self.count): try: iterator.next() except StopIteration: break def dedupe_columns(self, columns, valid_identifiers=True): # Try to clean-up messy column descriptions when people do not # provide an alias. The idea is that we take something like: # SUM("t1"."price") -> "price") -> price. Similarly, duplicated column # names will get an integer suffix, e.g. val, val_2, val_3. identifiers = [] duplicates = {} for column in columns: if valid_identifiers: column = make_identifier(column) if column in duplicates: duplicates[column] += 1 column = '%s_%s' % (column, duplicates[column]) else: duplicates[column] = 1 identifiers.append(column) return identifiers class DictCursorWrapper(CursorWrapper): def initialize(self): self.columns = self.dedupe_columns( [col_spec[0] for col_spec in self.cursor.description], valid_identifiers=False) self.ncols = len(self.columns) def _row_to_dict(self, row): result = {} for i in range(self.ncols): result.setdefault(self.columns[i], row[i]) # Do not overwrite. return result process_row = _row_to_dict class NamedTupleCursorWrapper(CursorWrapper): def initialize(self): identifiers = self.dedupe_columns( [col_spec[0] for col_spec in self.cursor.description]) self.tuple_class = collections.namedtuple('Row', identifiers) def process_row(self, row): return self.tuple_class(*row) class ObjectCursorWrapper(DictCursorWrapper): def __init__(self, cursor, constructor): super(ObjectCursorWrapper, self).__init__(cursor) self.constructor = constructor def initialize(self): self.columns = self.dedupe_columns( [col_spec[0] for col_spec in self.cursor.description], valid_identifiers=True) self.ncols = len(self.columns) def process_row(self, row): row_dict = self._row_to_dict(row) return self.constructor(**row_dict) class ResultIterator(object): def __init__(self, cursor_wrapper): self.cursor_wrapper = cursor_wrapper self.index = 0 def __iter__(self): return self def next(self): if self.index < self.cursor_wrapper.count: obj = self.cursor_wrapper.row_cache[self.index] elif not self.cursor_wrapper.populated: self.cursor_wrapper.iterate() obj = self.cursor_wrapper.row_cache[self.index] else: raise StopIteration self.index += 1 return obj __next__ = next # FIELDS class FieldAccessor(object): def __init__(self, model, field, name): self.model = model self.field = field self.name = name def __get__(self, instance, instance_type=None): if instance is not None: try: return instance.__data__[self.name] except KeyError: return return self.field def __set__(self, instance, value): instance.__data__[self.name] = value instance._dirty.add(self.name) class ForeignKeyAccessor(FieldAccessor): def __init__(self, model, field, name): super(ForeignKeyAccessor, self).__init__(model, field, name) self.rel_model = field.rel_model def get_rel_instance(self, instance): value = instance.__data__.get(self.name) if value is not None or self.name in instance.__rel__: if self.name not in instance.__rel__ and self.field.lazy_load: obj = self.rel_model.get(self.field.rel_field == value) instance.__rel__[self.name] = obj return instance.__rel__.get(self.name, value) elif not self.field.null and self.field.lazy_load: raise self.rel_model.DoesNotExist return value def __get__(self, instance, instance_type=None): if instance is not None: return self.get_rel_instance(instance) return self.field def __set__(self, instance, obj): if isinstance(obj, self.rel_model): instance.__data__[self.name] = getattr(obj, self.field.rel_field.name) instance.__rel__[self.name] = obj else: fk_value = instance.__data__.get(self.name) instance.__data__[self.name] = obj if (obj != fk_value or obj is None) and \ self.name in instance.__rel__: del instance.__rel__[self.name] instance._dirty.add(self.name) class BackrefAccessor(object): def __init__(self, field): self.field = field self.model = field.rel_model self.rel_model = field.model def __get__(self, instance, instance_type=None): if instance is not None: dest = self.field.rel_field.name return (self.rel_model .select() .where(self.field == getattr(instance, dest))) return self class ObjectIdAccessor(object): """Gives direct access to the underlying id""" def __init__(self, field): self.field = field def __get__(self, instance, instance_type=None): if instance is not None: value = instance.__data__.get(self.field.name) # Pull the object-id from the related object if it is not set. if value is None and self.field.name in instance.__rel__: rel_obj = instance.__rel__[self.field.name] value = getattr(rel_obj, self.field.rel_field.name) return value return self.field def __set__(self, instance, value): setattr(instance, self.field.name, value) class Field(ColumnBase): _field_counter = 0 _order = 0 accessor_class = FieldAccessor auto_increment = False default_index_type = None field_type = 'DEFAULT' unpack = True def __init__(self, null=False, index=False, unique=False, column_name=None, default=None, primary_key=False, constraints=None, sequence=None, collation=None, unindexed=False, choices=None, help_text=None, verbose_name=None, index_type=None, db_column=None, _hidden=False): if db_column is not None: __deprecated__('"db_column" has been deprecated in favor of ' '"column_name" for Field objects.') column_name = db_column self.null = null self.index = index self.unique = unique self.column_name = column_name self.default = default self.primary_key = primary_key self.constraints = constraints # List of column constraints. self.sequence = sequence # Name of sequence, e.g. foo_id_seq. self.collation = collation self.unindexed = unindexed self.choices = choices self.help_text = help_text self.verbose_name = verbose_name self.index_type = index_type or self.default_index_type self._hidden = _hidden # Used internally for recovering the order in which Fields were defined # on the Model class. Field._field_counter += 1 self._order = Field._field_counter self._sort_key = (self.primary_key and 1 or 2), self._order def __hash__(self): return hash(self.name + '.' + self.model.__name__) def __repr__(self): if hasattr(self, 'model') and getattr(self, 'name', None): return '<%s: %s.%s>' % (type(self).__name__, self.model.__name__, self.name) return '<%s: (unbound)>' % type(self).__name__ def bind(self, model, name, set_attribute=True): self.model = model self.name = self.safe_name = name self.column_name = self.column_name or name if set_attribute: setattr(model, name, self.accessor_class(model, self, name)) @property def column(self): return Column(self.model._meta.table, self.column_name) def adapt(self, value): return value def db_value(self, value): return value if value is None else self.adapt(value) def python_value(self, value): return value if value is None else self.adapt(value) def to_value(self, value, case=False): return Value(value, self.db_value, unpack=False) def get_sort_key(self, ctx): return self._sort_key def __sql__(self, ctx): return ctx.sql(self.column) def get_modifiers(self): pass def ddl_datatype(self, ctx): if ctx and ctx.state.field_types: column_type = ctx.state.field_types.get(self.field_type, self.field_type) else: column_type = self.field_type modifiers = self.get_modifiers() if column_type and modifiers: modifier_literal = ', '.join([str(m) for m in modifiers]) return SQL('%s(%s)' % (column_type, modifier_literal)) else: return SQL(column_type) def ddl(self, ctx): accum = [Entity(self.column_name)] data_type = self.ddl_datatype(ctx) if data_type: accum.append(data_type) if self.unindexed: accum.append(SQL('UNINDEXED')) if not self.null: accum.append(SQL('NOT NULL')) if self.primary_key: accum.append(SQL('PRIMARY KEY')) if self.sequence: accum.append(SQL("DEFAULT NEXTVAL('%s')" % self.sequence)) if self.constraints: accum.extend(self.constraints) if self.collation: accum.append(SQL('COLLATE %s' % self.collation)) return NodeList(accum) class AnyField(Field): field_type = 'ANY' class IntegerField(Field): field_type = 'INT' def adapt(self, value): try: return int(value) except (ValueError, TypeError): return value class BigIntegerField(IntegerField): field_type = 'BIGINT' class SmallIntegerField(IntegerField): field_type = 'SMALLINT' class AutoField(IntegerField): auto_increment = True field_type = 'AUTO' def __init__(self, *args, **kwargs): if kwargs.get('primary_key') is False: raise ValueError('%s must always be a primary key.' % type(self)) kwargs['primary_key'] = True super(AutoField, self).__init__(*args, **kwargs) class BigAutoField(AutoField): field_type = 'BIGAUTO' class IdentityField(AutoField): field_type = 'INT GENERATED BY DEFAULT AS IDENTITY' def __init__(self, generate_always=False, **kwargs): if generate_always: self.field_type = 'INT GENERATED ALWAYS AS IDENTITY' super(IdentityField, self).__init__(**kwargs) class PrimaryKeyField(AutoField): def __init__(self, *args, **kwargs): __deprecated__('"PrimaryKeyField" has been renamed to "AutoField". ' 'Please update your code accordingly as this will be ' 'completely removed in a subsequent release.') super(PrimaryKeyField, self).__init__(*args, **kwargs) class FloatField(Field): field_type = 'FLOAT' def adapt(self, value): try: return float(value) except (ValueError, TypeError): return value class DoubleField(FloatField): field_type = 'DOUBLE' class DecimalField(Field): field_type = 'DECIMAL' def __init__(self, max_digits=10, decimal_places=5, auto_round=False, rounding=None, *args, **kwargs): self.max_digits = max_digits self.decimal_places = decimal_places self.auto_round = auto_round self.rounding = rounding or decimal.DefaultContext.rounding self._exp = decimal.Decimal(10) ** (-self.decimal_places) super(DecimalField, self).__init__(*args, **kwargs) def get_modifiers(self): return [self.max_digits, self.decimal_places] def db_value(self, value): D = decimal.Decimal if not value: return value if value is None else D(0) if self.auto_round: decimal_value = D(str(value)) return decimal_value.quantize(self._exp, rounding=self.rounding) return value def python_value(self, value): if value is not None: if isinstance(value, decimal.Decimal): return value return decimal.Decimal(str(value)) class _StringField(Field): def adapt(self, value): if isinstance(value, str): return value elif isinstance(value, bytes): return value.decode('utf-8') return str(value) def __add__(self, other): return StringExpression(self, OP.CONCAT, other) def __radd__(self, other): return StringExpression(other, OP.CONCAT, self) class CharField(_StringField): field_type = 'VARCHAR' def __init__(self, max_length=255, *args, **kwargs): self.max_length = max_length super(CharField, self).__init__(*args, **kwargs) def get_modifiers(self): return self.max_length and [self.max_length] or None class FixedCharField(CharField): field_type = 'CHAR' def adapt(self, value): value = super(FixedCharField, self).adapt(value) if value: value = value[:self.max_length] return value class TextField(_StringField): field_type = 'TEXT' class FieldDatabaseHook(object): def _db_hook(self, database): raise NotImplementedError('Subclasses must implement') def bind(self, model, name, set_attribute=True): if model._meta.database is not None: if isinstance(model._meta.database, Proxy): model._meta.database.attach_callback(self._db_hook) self._db_hook(None) else: self._db_hook(model._meta.database) else: self._db_hook(None) # Attach a hook to the model metadata; in the event the database is # changed or set at run-time, we will be sure to apply our callback and # use the proper data-type for our database driver. model._meta._db_hooks.append(self._db_hook) return super(FieldDatabaseHook, self).bind(model, name, set_attribute) class BlobField(FieldDatabaseHook, Field): field_type = 'BLOB' def _db_hook(self, database): if database is None: self._constructor = bytearray else: self._constructor = database.get_binary_type() def db_value(self, value): if isinstance(value, str): value = value.encode('raw_unicode_escape') if isinstance(value, bytes): return self._constructor(value) return value class BitField(BitwiseMixin, BigIntegerField): def __init__(self, *args, **kwargs): kwargs.setdefault('default', 0) super(BitField, self).__init__(*args, **kwargs) self.__current_flag = 1 def flag(self, value=None): if value is None: value = self.__current_flag self.__current_flag <<= 1 else: self.__current_flag = value << 1 class FlagDescriptor(ColumnBase): def __init__(self, field, value): self._field = field self._value = value super(FlagDescriptor, self).__init__() def clear(self): return self._field.bin_and(~self._value) def set(self): return self._field.bin_or(self._value) def __get__(self, instance, instance_type=None): if instance is None: return self value = getattr(instance, self._field.name) or 0 return (value & self._value) != 0 def __set__(self, instance, is_set): if is_set not in (True, False): raise ValueError('Value must be either True or False') value = getattr(instance, self._field.name) or 0 if is_set: value |= self._value else: value &= ~self._value setattr(instance, self._field.name, value) def __sql__(self, ctx): return ctx.sql(self._field.bin_and(self._value) != 0) return FlagDescriptor(self, value) class BigBitFieldData(object): def __init__(self, instance, name): self.instance = instance self.name = name value = self.instance.__data__.get(self.name) if not value: value = bytearray() elif not isinstance(value, bytearray): value = bytearray(value) self._buffer = self.instance.__data__[self.name] = value def clear(self): self._buffer.clear() def _ensure_length(self, idx): byte_num, byte_offset = divmod(idx, 8) cur_size = len(self._buffer) if cur_size <= byte_num: self._buffer.extend(b'\x00' * ((byte_num + 1) - cur_size)) return byte_num, byte_offset def set_bit(self, idx): byte_num, byte_offset = self._ensure_length(idx) self._buffer[byte_num] |= (1 << byte_offset) def clear_bit(self, idx): byte_num, byte_offset = self._ensure_length(idx) self._buffer[byte_num] &= ~(1 << byte_offset) def toggle_bit(self, idx): byte_num, byte_offset = self._ensure_length(idx) self._buffer[byte_num] ^= (1 << byte_offset) return bool(self._buffer[byte_num] & (1 << byte_offset)) def is_set(self, idx): byte_num, byte_offset = divmod(idx, 8) cur_size = len(self._buffer) if cur_size <= byte_num: return False return bool(self._buffer[byte_num] & (1 << byte_offset)) __getitem__ = is_set def __setitem__(self, item, value): self.set_bit(item) if value else self.clear_bit(item) __delitem__ = clear_bit def __len__(self): return len(self._buffer) def _get_compatible_data(self, other): if isinstance(other, BigBitFieldData): data = other._buffer elif isinstance(other, (bytes, bytearray, memoryview)): data = other else: raise ValueError('Incompatible data-type') diff = len(data) - len(self) if diff > 0: self._buffer.extend(b'\x00' * diff) return data def _bitwise_op(self, other, op): if isinstance(other, BigBitFieldData): data = other._buffer elif isinstance(other, (bytes, bytearray, memoryview)): data = other else: raise ValueError('Incompatible data-type') buf = bytearray(b'\x00' * max(len(self), len(other))) it = itertools.zip_longest(self._buffer, data, fillvalue=0) for i, (a, b) in enumerate(it): buf[i] = op(a, b) return buf def __and__(self, other): return self._bitwise_op(other, operator.and_) def __or__(self, other): return self._bitwise_op(other, operator.or_) def __xor__(self, other): return self._bitwise_op(other, operator.xor) def __iter__(self): for b in self._buffer: for j in range(8): yield 1 if (b & (1 << j)) else 0 def __repr__(self): return repr(self._buffer) def __bytes__(self): return bytes(self._buffer) class BigBitFieldAccessor(FieldAccessor): def __get__(self, instance, instance_type=None): if instance is None: return self.field return BigBitFieldData(instance, self.name) def __set__(self, instance, value): if isinstance(value, memoryview): value = value.tobytes() elif isinstance(value, bytearray): value = bytes(value) elif isinstance(value, BigBitFieldData): value = bytes(value._buffer) elif isinstance(value, str): value = value.encode('utf-8') elif not isinstance(value, bytes): raise ValueError('Value must be either a bytes, memoryview or ' 'BigBitFieldData instance.') super(BigBitFieldAccessor, self).__set__(instance, value) class BigBitField(BlobField): accessor_class = BigBitFieldAccessor def __init__(self, *args, **kwargs): kwargs.setdefault('default', bytes) super(BigBitField, self).__init__(*args, **kwargs) def db_value(self, value): return bytes(value) if value is not None else value class UUIDField(Field): field_type = 'UUID' def db_value(self, value): if isinstance(value, str) and len(value) == 32: # Hex string. No transformation is necessary. return value elif isinstance(value, bytes) and len(value) == 16: # Allow raw binary representation. value = uuid.UUID(bytes=value) if isinstance(value, uuid.UUID): return value.hex try: return uuid.UUID(value).hex except Exception: return value def python_value(self, value): if isinstance(value, uuid.UUID): return value return uuid.UUID(value) if value is not None else None class BinaryUUIDField(BlobField): field_type = 'UUIDB' def db_value(self, value): if isinstance(value, bytes) and len(value) == 16: # Raw binary value. No transformation is necessary. return self._constructor(value) elif isinstance(value, str) and len(value) == 32: # Allow hex string representation. value = uuid.UUID(hex=value) if isinstance(value, uuid.UUID): return self._constructor(value.bytes) elif value is not None: raise ValueError('value for binary UUID field must be UUID(), ' 'a hexadecimal string, or a bytes object.') def python_value(self, value): if isinstance(value, uuid.UUID): return value elif isinstance(value, memoryview): value = value.tobytes() elif value and not isinstance(value, bytes): value = bytes(value) return uuid.UUID(bytes=value) if value is not None else None def _date_part(date_part): def dec(self): return self.model._meta.database.extract_date(date_part, self) return dec def format_date_time(value, formats, post_process=None): post_process = post_process or (lambda x: x) for fmt in formats: try: return post_process(datetime.datetime.strptime(value, fmt)) except ValueError: pass return value def simple_date_time(value): try: return datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S') except (TypeError, ValueError): return value class _BaseFormattedField(Field): formats = None def __init__(self, formats=None, *args, **kwargs): if formats is not None: self.formats = formats super(_BaseFormattedField, self).__init__(*args, **kwargs) class DateTimeField(_BaseFormattedField): field_type = 'DATETIME' formats = [ '%Y-%m-%d %H:%M:%S.%f', '%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M:%S.%f%z', '%Y-%m-%d %H:%M:%S%z', '%Y-%m-%d', ] def adapt(self, value): if value and isinstance(value, str): return format_date_time(value, self.formats) return value def to_timestamp(self): return self.model._meta.database.to_timestamp(self) def truncate(self, part): return self.model._meta.database.truncate_date(part, self) year = property(_date_part('year')) month = property(_date_part('month')) day = property(_date_part('day')) hour = property(_date_part('hour')) minute = property(_date_part('minute')) second = property(_date_part('second')) class DateField(_BaseFormattedField): field_type = 'DATE' formats = [ '%Y-%m-%d', '%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M:%S.%f', ] def adapt(self, value): if value and isinstance(value, str): pp = lambda x: x.date() return format_date_time(value, self.formats, pp) elif value and isinstance(value, datetime.datetime): return value.date() return value def to_timestamp(self): return self.model._meta.database.to_timestamp(self) def truncate(self, part): return self.model._meta.database.truncate_date(part, self) year = property(_date_part('year')) month = property(_date_part('month')) day = property(_date_part('day')) class TimeField(_BaseFormattedField): field_type = 'TIME' formats = [ '%H:%M:%S.%f', '%H:%M:%S', '%H:%M', '%Y-%m-%d %H:%M:%S.%f', '%Y-%m-%d %H:%M:%S', ] def adapt(self, value): if value: if isinstance(value, str): pp = lambda x: x.time() return format_date_time(value, self.formats, pp) elif isinstance(value, datetime.datetime): return value.time() if value is not None and isinstance(value, datetime.timedelta): return (datetime.datetime.min + value).time() return value hour = property(_date_part('hour')) minute = property(_date_part('minute')) second = property(_date_part('second')) def _timestamp_date_part(date_part): def dec(self): db = self.model._meta.database expr = ((self / Value(self.resolution, converter=False)) if self.resolution > 1 else self) return db.extract_date(date_part, db.from_timestamp(expr)) return dec class TimestampField(BigIntegerField): # Support second -> microsecond resolution. valid_resolutions = [10**i for i in range(7)] def __init__(self, *args, **kwargs): self.resolution = kwargs.pop('resolution', None) if not self.resolution: self.resolution = 1 elif self.resolution in range(2, 7): self.resolution = 10 ** self.resolution elif self.resolution not in self.valid_resolutions: raise ValueError('TimestampField resolution must be one of: %s' % ', '.join(str(i) for i in self.valid_resolutions)) self.ticks_to_microsecond = 1000000 // self.resolution self.utc = kwargs.pop('utc', False) or False dflt = utcnow if self.utc else datetime.datetime.now kwargs.setdefault('default', dflt) super(TimestampField, self).__init__(*args, **kwargs) def local_to_utc(self, dt): # Convert naive local datetime into naive UTC, e.g.: # 2019-03-01T12:00:00 (local=US/Central) -> 2019-03-01T18:00:00. # 2019-05-01T12:00:00 (local=US/Central) -> 2019-05-01T17:00:00. # 2019-03-01T12:00:00 (local=UTC) -> 2019-03-01T12:00:00. return datetime.datetime(*time.gmtime(time.mktime(dt.timetuple()))[:6]) def utc_to_local(self, dt): # Convert a naive UTC datetime into local time, e.g.: # 2019-03-01T18:00:00 (local=US/Central) -> 2019-03-01T12:00:00. # 2019-05-01T17:00:00 (local=US/Central) -> 2019-05-01T12:00:00. # 2019-03-01T12:00:00 (local=UTC) -> 2019-03-01T12:00:00. ts = calendar.timegm(dt.utctimetuple()) return datetime.datetime.fromtimestamp(ts) def get_timestamp(self, value): if self.utc: # If utc-mode is on, then we assume all naive datetimes are in UTC. return calendar.timegm(value.utctimetuple()) else: return time.mktime(value.timetuple()) def db_value(self, value): if value is None: return if isinstance(value, datetime.datetime): pass elif isinstance(value, datetime.date): value = datetime.datetime(value.year, value.month, value.day) else: return int(round(value * self.resolution)) timestamp = self.get_timestamp(value) if self.resolution > 1: timestamp += (value.microsecond * .000001) timestamp *= self.resolution return int(round(timestamp)) def python_value(self, value): if value is not None and isinstance(value, (int, float)): if self.resolution > 1: value, ticks = divmod(value, self.resolution) microseconds = int(ticks * self.ticks_to_microsecond) else: microseconds = 0 if self.utc: value = utcfromtimestamp(value) else: value = datetime.datetime.fromtimestamp(value) if microseconds: value = value.replace(microsecond=microseconds) return value def from_timestamp(self): expr = ((self / Value(self.resolution, converter=False)) if self.resolution > 1 else self) return self.model._meta.database.from_timestamp(expr) year = property(_timestamp_date_part('year')) month = property(_timestamp_date_part('month')) day = property(_timestamp_date_part('day')) hour = property(_timestamp_date_part('hour')) minute = property(_timestamp_date_part('minute')) second = property(_timestamp_date_part('second')) class IPField(BigIntegerField): def db_value(self, val): if val is not None: return struct.unpack('!I', socket.inet_aton(val))[0] def python_value(self, val): if val is not None: return socket.inet_ntoa(struct.pack('!I', val)) class BooleanField(Field): field_type = 'BOOL' adapt = bool class BareField(Field): def __init__(self, adapt=None, *args, **kwargs): super(BareField, self).__init__(*args, **kwargs) if adapt is not None: self.adapt = adapt def ddl_datatype(self, ctx): return class ForeignKeyField(Field): accessor_class = ForeignKeyAccessor backref_accessor_class = BackrefAccessor def __init__(self, model, field=None, backref=None, on_delete=None, on_update=None, deferrable=None, _deferred=None, rel_model=None, to_field=None, object_id_name=None, lazy_load=True, constraint_name=None, related_name=None, *args, **kwargs): kwargs.setdefault('index', True) super(ForeignKeyField, self).__init__(*args, **kwargs) if rel_model is not None: __deprecated__('"rel_model" has been deprecated in favor of ' '"model" for ForeignKeyField objects.') model = rel_model if to_field is not None: __deprecated__('"to_field" has been deprecated in favor of ' '"field" for ForeignKeyField objects.') field = to_field if related_name is not None: __deprecated__('"related_name" has been deprecated in favor of ' '"backref" for Field objects.') backref = related_name self._is_self_reference = model == 'self' self.rel_model = model self.rel_field = field self.declared_backref = backref self.backref = None self.on_delete = on_delete self.on_update = on_update self.deferrable = deferrable self.deferred = _deferred self.object_id_name = object_id_name self.lazy_load = lazy_load self.constraint_name = constraint_name @property def field_type(self): if isinstance(self.rel_field, BigAutoField): return BigIntegerField.field_type elif not isinstance(self.rel_field, AutoField): return self.rel_field.field_type return IntegerField.field_type def get_modifiers(self): if not isinstance(self.rel_field, AutoField): return self.rel_field.get_modifiers() return super(ForeignKeyField, self).get_modifiers() def get_constraint_name(self): return self.constraint_name or 'fk_%s_%s_refs_%s' % ( self.model._meta.table_name, self.column_name, self.rel_model._meta.table_name) def adapt(self, value): return self.rel_field.adapt(value) def db_value(self, value): if isinstance(value, self.rel_model): value = getattr(value, self.rel_field.name) return self.rel_field.db_value(value) def python_value(self, value): if isinstance(value, self.rel_model): return value return self.rel_field.python_value(value) def bind(self, model, name, set_attribute=True): if not self.column_name: self.column_name = name if name.endswith('_id') else name + '_id' if not self.object_id_name: self.object_id_name = self.column_name if self.object_id_name == name: self.object_id_name += '_id' elif self.object_id_name == name: raise ValueError('ForeignKeyField "%s"."%s" specifies an ' 'object_id_name that conflicts with its field ' 'name.' % (model._meta.name, name)) if self._is_self_reference: self.rel_model = model if isinstance(self.rel_field, str): self.rel_field = getattr(self.rel_model, self.rel_field) elif self.rel_field is None: self.rel_field = self.rel_model._meta.primary_key # Bind field before assigning backref, so field is bound when # calling declared_backref() (if callable). super(ForeignKeyField, self).bind(model, name, set_attribute) self.safe_name = self.object_id_name if callable_(self.declared_backref): self.backref = self.declared_backref(self) else: self.backref, self.declared_backref = self.declared_backref, None if not self.backref: self.backref = '%s_set' % model._meta.name if set_attribute: setattr(model, self.object_id_name, ObjectIdAccessor(self)) if self.backref not in '!+': setattr(self.rel_model, self.backref, self.backref_accessor_class(self)) def foreign_key_constraint(self, explicit_name=False): parts = [] if self.constraint_name or explicit_name: name = self.get_constraint_name() parts.extend([ SQL('CONSTRAINT'), Entity(_truncate_constraint_name(name))]) parts.extend([ SQL('FOREIGN KEY'), EnclosedNodeList((self,)), SQL('REFERENCES'), self.rel_model, EnclosedNodeList((self.rel_field,))]) if self.on_delete: parts.append(SQL('ON DELETE %s' % self.on_delete)) if self.on_update: parts.append(SQL('ON UPDATE %s' % self.on_update)) if self.deferrable: parts.append(SQL('DEFERRABLE %s' % self.deferrable)) return NodeList(parts) def __getattr__(self, attr): if attr.startswith('__'): # Prevent recursion error when deep-copying. raise AttributeError('Cannot look-up non-existant "__" methods.') if attr in self.rel_model._meta.fields: return self.rel_model._meta.fields[attr] raise AttributeError('Foreign-key has no attribute %s, nor is it a ' 'valid field on the related model.' % attr) class DeferredForeignKey(Field): _unresolved = set() def __init__(self, rel_model_name, **kwargs): self.field_kwargs = kwargs self.rel_model_name = rel_model_name.lower() DeferredForeignKey._unresolved.add(self) super(DeferredForeignKey, self).__init__( column_name=kwargs.get('column_name'), null=kwargs.get('null'), primary_key=kwargs.get('primary_key')) __hash__ = object.__hash__ def __deepcopy__(self, memo=None): return DeferredForeignKey(self.rel_model_name, **self.field_kwargs) def set_model(self, rel_model): field = ForeignKeyField(rel_model, _deferred=True, **self.field_kwargs) if field.primary_key: # NOTE: this calls add_field() under-the-hood. self.model._meta.set_primary_key(self.name, field) else: self.model._meta.add_field(self.name, field) @staticmethod def resolve(model_cls): unresolved = sorted(DeferredForeignKey._unresolved, key=operator.attrgetter('_order')) for dr in unresolved: if dr.rel_model_name == model_cls.__name__.lower(): dr.set_model(model_cls) DeferredForeignKey._unresolved.discard(dr) class DeferredThroughModel(object): def __init__(self): self._refs = [] def set_field(self, model, field, name): self._refs.append((model, field, name)) def set_model(self, through_model): for src_model, m2mfield, name in self._refs: m2mfield.through_model = through_model src_model._meta.add_field(name, m2mfield) class MetaField(Field): column_name = default = model = name = None primary_key = False class ManyToManyFieldAccessor(FieldAccessor): def __init__(self, model, field, name): super(ManyToManyFieldAccessor, self).__init__(model, field, name) self.model = field.model self.rel_model = field.rel_model self.through_model = field.through_model src_fks = self.through_model._meta.model_refs[self.model] dest_fks = self.through_model._meta.model_refs[self.rel_model] if not src_fks: raise ValueError('Cannot find foreign-key to "%s" on "%s" model.' % (self.model, self.through_model)) elif not dest_fks: raise ValueError('Cannot find foreign-key to "%s" on "%s" model.' % (self.rel_model, self.through_model)) self.src_fk = src_fks[0] self.dest_fk = dest_fks[0] def __get__(self, instance, instance_type=None, force_query=False): if instance is not None: if not force_query and self.src_fk.backref != '+': backref = getattr(instance, self.src_fk.backref) if isinstance(backref, list): return [getattr(obj, self.dest_fk.name) for obj in backref] src_id = getattr(instance, self.src_fk.rel_field.name) if src_id is None and self.field._prevent_unsaved: raise ValueError('Cannot get many-to-many "%s" for unsaved ' 'instance "%s".' % (self.field, instance)) return (ManyToManyQuery(instance, self, self.rel_model) .join(self.through_model) .join(self.model) .where(self.src_fk == src_id)) return self.field def __set__(self, instance, value): src_id = getattr(instance, self.src_fk.rel_field.name) if src_id is None and self.field._prevent_unsaved: raise ValueError('Cannot set many-to-many "%s" for unsaved ' 'instance "%s".' % (self.field, instance)) query = self.__get__(instance, force_query=True) query.add(value, clear_existing=True) class ManyToManyField(MetaField): accessor_class = ManyToManyFieldAccessor def __init__(self, model, backref=None, through_model=None, on_delete=None, on_update=None, prevent_unsaved=True, _is_backref=False): if through_model is not None: if not (isinstance(through_model, DeferredThroughModel) or is_model(through_model)): raise TypeError('Unexpected value for through_model. Expected ' 'Model or DeferredThroughModel.') if not _is_backref and (on_delete is not None or on_update is not None): raise ValueError('Cannot specify on_delete or on_update when ' 'through_model is specified.') self.rel_model = model self.backref = backref self._through_model = through_model self._on_delete = on_delete self._on_update = on_update self._prevent_unsaved = prevent_unsaved self._is_backref = _is_backref def _get_descriptor(self): return ManyToManyFieldAccessor(self) def bind(self, model, name, set_attribute=True): if isinstance(self._through_model, DeferredThroughModel): self._through_model.set_field(model, self, name) return super(ManyToManyField, self).bind(model, name, set_attribute) if not self._is_backref: many_to_many_field = ManyToManyField( self.model, backref=name, through_model=self.through_model, on_delete=self._on_delete, on_update=self._on_update, _is_backref=True) self.backref = self.backref or model._meta.name + 's' self.rel_model._meta.add_field(self.backref, many_to_many_field) def get_models(self): return [model for _, model in sorted(( (self._is_backref, self.model), (not self._is_backref, self.rel_model)))] @property def through_model(self): if self._through_model is None: self._through_model = self._create_through_model() return self._through_model @through_model.setter def through_model(self, value): self._through_model = value def _create_through_model(self): lhs, rhs = self.get_models() tables = [model._meta.table_name for model in (lhs, rhs)] class Meta: database = self.model._meta.database schema = self.model._meta.schema table_name = '%s_%s_through' % tuple(tables) indexes = ( ((lhs._meta.name, rhs._meta.name), True),) params = {'on_delete': self._on_delete, 'on_update': self._on_update} attrs = { lhs._meta.name: ForeignKeyField(lhs, **params), rhs._meta.name: ForeignKeyField(rhs, **params), 'Meta': Meta} klass_name = '%s%sThrough' % (lhs.__name__, rhs.__name__) return type(klass_name, (Model,), attrs) def get_through_model(self): # XXX: Deprecated. Just use the "through_model" property. return self.through_model class VirtualField(MetaField): field_class = None def __init__(self, field_class=None, *args, **kwargs): Field = field_class if field_class is not None else self.field_class self.field_instance = Field() if Field is not None else None super(VirtualField, self).__init__(*args, **kwargs) def db_value(self, value): if self.field_instance is not None: return self.field_instance.db_value(value) return value def python_value(self, value): if self.field_instance is not None: return self.field_instance.python_value(value) return value def bind(self, model, name, set_attribute=True): self.model = model self.column_name = self.name = self.safe_name = name setattr(model, name, self.accessor_class(model, self, name)) class CompositeKey(MetaField): sequence = None def __init__(self, *field_names): self.field_names = field_names self._safe_field_names = None @property def safe_field_names(self): if self._safe_field_names is None: if self.model is None: return self.field_names self._safe_field_names = [self.model._meta.fields[f].safe_name for f in self.field_names] return self._safe_field_names def __get__(self, instance, instance_type=None): if instance is not None: return tuple([getattr(instance, f) for f in self.safe_field_names]) return self def __set__(self, instance, value): if not isinstance(value, (list, tuple)): raise TypeError('A list or tuple must be used to set the value of ' 'a composite primary key.') if len(value) != len(self.field_names): raise ValueError('The length of the value must equal the number ' 'of columns of the composite primary key.') for idx, field_value in enumerate(value): setattr(instance, self.field_names[idx], field_value) def __eq__(self, other): expressions = [(self.model._meta.fields[field] == value) for field, value in zip(self.field_names, other)] return reduce(operator.and_, expressions) def __ne__(self, other): return ~(self == other) def __hash__(self): return hash((self.model.__name__, self.field_names)) def __sql__(self, ctx): # If the composite PK is being selected, do not use parens. Elsewhere, # such as in an expression, we want to use parentheses and treat it as # a row value. parens = ctx.scope != SCOPE_SOURCE return ctx.sql(NodeList([self.model._meta.fields[field] for field in self.field_names], ', ', parens)) def bind(self, model, name, set_attribute=True): self.model = model self.column_name = self.name = self.safe_name = name setattr(model, self.name, self) class _SortedFieldList(object): __slots__ = ('_keys', '_items') def __init__(self): self._keys = [] self._items = [] def __getitem__(self, i): return self._items[i] def __iter__(self): return iter(self._items) def __contains__(self, item): k = item._sort_key i = bisect_left(self._keys, k) j = bisect_right(self._keys, k) return item in self._items[i:j] def index(self, field): return self._keys.index(field._sort_key) def insert(self, item): k = item._sort_key i = bisect_left(self._keys, k) self._keys.insert(i, k) self._items.insert(i, item) def remove(self, item): idx = self.index(item) del self._items[idx] del self._keys[idx] # MODELS class SchemaManager(object): def __init__(self, model, database=None, **context_options): self.model = model self._database = database context_options.setdefault('scope', SCOPE_VALUES) self.context_options = context_options @property def database(self): db = self._database or self.model._meta.database if db is None: raise ImproperlyConfigured('database attribute does not appear to ' 'be set on the model: %s' % self.model) return db @database.setter def database(self, value): self._database = value def _create_context(self): return self.database.get_sql_context(**self.context_options) def _create_table(self, safe=True, **options): is_temp = options.pop('temporary', False) ctx = self._create_context() ctx.literal('CREATE TEMPORARY TABLE ' if is_temp else 'CREATE TABLE ') if safe: ctx.literal('IF NOT EXISTS ') ctx.sql(self.model).literal(' ') columns = [] constraints = [] meta = self.model._meta if meta.composite_key: pk_columns = [meta.fields[field_name].column for field_name in meta.primary_key.field_names] constraints.append(NodeList((SQL('PRIMARY KEY'), EnclosedNodeList(pk_columns)))) for field in meta.sorted_fields: columns.append(field.ddl(ctx)) if isinstance(field, ForeignKeyField) and not field.deferred: constraints.append(field.foreign_key_constraint()) if meta.constraints: constraints.extend(meta.constraints) constraints.extend(self._create_table_option_sql(options)) ctx.sql(EnclosedNodeList(columns + constraints)) if meta.table_settings is not None: table_settings = ensure_tuple(meta.table_settings) for setting in table_settings: if not isinstance(setting, str): raise ValueError('table_settings must be strings') ctx.literal(' ').literal(setting) extra_opts = [] if meta.strict_tables: extra_opts.append('STRICT') if meta.without_rowid: extra_opts.append('WITHOUT ROWID') if extra_opts: ctx.literal(' %s' % ', '.join(extra_opts)) return ctx def _create_table_option_sql(self, options): accum = [] options = merge_dict(self.model._meta.options or {}, options) if not options: return accum for key, value in sorted(options.items()): if not isinstance(value, Node): if is_model(value): value = value._meta.table else: value = SQL(str(value)) accum.append(NodeList((SQL(key), value), glue='=')) return accum def create_table(self, safe=True, **options): self.database.execute(self._create_table(safe=safe, **options)) def _create_table_as(self, table_name, query, safe=True, **meta): ctx = (self._create_context() .literal('CREATE TEMPORARY TABLE ' if meta.get('temporary') else 'CREATE TABLE ')) if safe: ctx.literal('IF NOT EXISTS ') return (ctx .sql(Entity(*ensure_tuple(table_name))) .literal(' AS ') .sql(query)) def create_table_as(self, table_name, query, safe=True, **meta): ctx = self._create_table_as(table_name, query, safe=safe, **meta) self.database.execute(ctx) def _drop_table(self, safe=True, **options): ctx = (self._create_context() .literal('DROP TABLE IF EXISTS ' if safe else 'DROP TABLE ') .sql(self.model)) if options.get('cascade'): ctx = ctx.literal(' CASCADE') elif options.get('restrict'): ctx = ctx.literal(' RESTRICT') return ctx def drop_table(self, safe=True, **options): self.database.execute(self._drop_table(safe=safe, **options)) def _truncate_table(self, restart_identity=False, cascade=False): db = self.database if not db.truncate_table: return (self._create_context() .literal('DELETE FROM ').sql(self.model)) ctx = self._create_context().literal('TRUNCATE TABLE ').sql(self.model) if restart_identity: ctx = ctx.literal(' RESTART IDENTITY') if cascade: ctx = ctx.literal(' CASCADE') return ctx def truncate_table(self, restart_identity=False, cascade=False): self.database.execute(self._truncate_table(restart_identity, cascade)) def _create_indexes(self, safe=True): return [self._create_index(index, safe) for index in self.model._meta.fields_to_index()] def _create_index(self, index, safe=True): if isinstance(index, Index): if not self.database.safe_create_index: index = index.safe(False) elif index._safe != safe: index = index.safe(safe) if isinstance(self._database, SqliteDatabase): # Ensure we do not use value placeholders with Sqlite, as they # are not supported. index = ValueLiterals(index) return self._create_context().sql(index) def create_indexes(self, safe=True): for query in self._create_indexes(safe=safe): self.database.execute(query) def _drop_indexes(self, safe=True): return [self._drop_index(index, safe) for index in self.model._meta.fields_to_index() if isinstance(index, Index)] def _drop_index(self, index, safe): statement = 'DROP INDEX ' if safe and self.database.safe_drop_index: statement += 'IF EXISTS ' if isinstance(index._table, Table) and index._table._schema: index_name = Entity(index._table._schema, index._name) else: index_name = Entity(index._name) return (self ._create_context() .literal(statement) .sql(index_name)) def drop_indexes(self, safe=True): for query in self._drop_indexes(safe=safe): self.database.execute(query) def _check_sequences(self, field): if not field.sequence or not self.database.sequences: raise ValueError('Sequences are either not supported, or are not ' 'defined for "%s".' % field.name) def _sequence_for_field(self, field): if field.model._meta.schema: return Entity(field.model._meta.schema, field.sequence) else: return Entity(field.sequence) def _create_sequence(self, field): self._check_sequences(field) if not self.database.sequence_exists(field.sequence): return (self ._create_context() .literal('CREATE SEQUENCE ') .sql(self._sequence_for_field(field))) def create_sequence(self, field): seq_ctx = self._create_sequence(field) if seq_ctx is not None: self.database.execute(seq_ctx) def _drop_sequence(self, field): self._check_sequences(field) if self.database.sequence_exists(field.sequence): return (self ._create_context() .literal('DROP SEQUENCE ') .sql(self._sequence_for_field(field))) def drop_sequence(self, field): seq_ctx = self._drop_sequence(field) if seq_ctx is not None: self.database.execute(seq_ctx) def _create_foreign_key(self, field): return (self ._create_context() .literal('ALTER TABLE ') .sql(field.model) .literal(' ADD ') .sql(field.foreign_key_constraint(True))) def create_foreign_key(self, field): self.database.execute(self._create_foreign_key(field)) def create_sequences(self): if self.database.sequences: for field in self.model._meta.sorted_fields: if field.sequence: self.create_sequence(field) def create_all(self, safe=True, **table_options): self.create_sequences() self.create_table(safe, **table_options) self.create_indexes(safe=safe) def drop_sequences(self): if self.database.sequences: for field in self.model._meta.sorted_fields: if field.sequence: self.drop_sequence(field) def drop_all(self, safe=True, drop_sequences=True, **options): self.drop_table(safe, **options) if drop_sequences: self.drop_sequences() class Metadata(object): def __init__(self, model, database=None, table_name=None, indexes=None, primary_key=None, constraints=None, schema=None, only_save_dirty=False, depends_on=None, options=None, db_table=None, table_function=None, table_settings=None, without_rowid=False, temporary=False, strict_tables=None, legacy_table_names=True, **kwargs): if db_table is not None: __deprecated__('"db_table" has been deprecated in favor of ' '"table_name" for Models.') table_name = db_table self.model = model self.database = database self.fields = {} self.columns = {} self.combined = {} self._sorted_field_list = _SortedFieldList() self.sorted_fields = [] self.sorted_field_names = [] self.defaults = {} self._default_by_name = {} self._default_dict = {} self._default_callables = {} self._default_callable_list = [] self.name = model.__name__.lower() self.table_function = table_function self.legacy_table_names = legacy_table_names if not table_name: table_name = (self.table_function(model) if self.table_function else self.make_table_name()) self.table_name = table_name self._table = None self.indexes = list(indexes) if indexes else [] self.constraints = constraints self._schema = schema self.primary_key = primary_key self.composite_key = self.auto_increment = None self.only_save_dirty = only_save_dirty self.depends_on = depends_on self.table_settings = table_settings self.without_rowid = without_rowid self.strict_tables = strict_tables self.temporary = temporary self.refs = {} self.backrefs = {} self.model_refs = collections.defaultdict(list) self.model_backrefs = collections.defaultdict(list) self.manytomany = {} self.options = options or {} for key, value in kwargs.items(): setattr(self, key, value) self._additional_keys = set(kwargs.keys()) # Allow objects to register hooks that are called if the model is bound # to a different database. For example, BlobField uses a different # Python data-type depending on the db driver / python version. When # the database changes, we need to update any BlobField so they can use # the appropriate data-type. self._db_hooks = [] def make_table_name(self): if self.legacy_table_names: return re.sub(r'[^\w]+', '_', self.name) return make_snake_case(self.model.__name__) def model_graph(self, refs=True, backrefs=True, depth_first=True): if not refs and not backrefs: raise ValueError('One of `refs` or `backrefs` must be True.') accum = [(None, self.model, None)] seen = set() queue = collections.deque((self,)) method = queue.pop if depth_first else queue.popleft while queue: curr = method() if curr in seen: continue seen.add(curr) if refs: for fk, model in curr.refs.items(): accum.append((fk, model, False)) queue.append(model._meta) if backrefs: for fk, model in curr.backrefs.items(): accum.append((fk, model, True)) queue.append(model._meta) return accum def add_ref(self, field): rel = field.rel_model self.refs[field] = rel self.model_refs[rel].append(field) rel._meta.backrefs[field] = self.model rel._meta.model_backrefs[self.model].append(field) def remove_ref(self, field): rel = field.rel_model del self.refs[field] self.model_refs[rel].remove(field) del rel._meta.backrefs[field] rel._meta.model_backrefs[self.model].remove(field) def add_manytomany(self, field): self.manytomany[field.name] = field def remove_manytomany(self, field): del self.manytomany[field.name] @property def table(self): if self._table is None: self._table = Table( self.table_name, [field.column_name for field in self.sorted_fields], schema=self.schema, _model=self.model, _database=self.database) return self._table @table.setter def table(self, value): raise AttributeError('Cannot set the "table".') @table.deleter def table(self): self._table = None @property def schema(self): return self._schema @schema.setter def schema(self, value): self._schema = value del self.table @property def entity(self): if self._schema: return Entity(self._schema, self.table_name) else: return Entity(self.table_name) def _update_sorted_fields(self): self.sorted_fields = list(self._sorted_field_list) self.sorted_field_names = [f.name for f in self.sorted_fields] def get_rel_for_model(self, model): if isinstance(model, ModelAlias): model = model.model forwardrefs = self.model_refs.get(model, []) backrefs = self.model_backrefs.get(model, []) return (forwardrefs, backrefs) def add_field(self, field_name, field, set_attribute=True): if field_name in self.fields: self.remove_field(field_name) elif field_name in self.manytomany: self.remove_manytomany(self.manytomany[field_name]) if not isinstance(field, MetaField): del self.table field.bind(self.model, field_name, set_attribute) self.fields[field.name] = field self.columns[field.column_name] = field self.combined[field.name] = field self.combined[field.column_name] = field self._sorted_field_list.insert(field) self._update_sorted_fields() if field.default is not None: # This optimization helps speed up model instance construction. self.defaults[field] = field.default if callable_(field.default): self._default_callables[field] = field.default self._default_callable_list.append((field.name, field.default)) else: self._default_dict[field] = field.default self._default_by_name[field.name] = field.default else: field.bind(self.model, field_name, set_attribute) if isinstance(field, ForeignKeyField): self.add_ref(field) elif isinstance(field, ManyToManyField) and field.name: self.add_manytomany(field) def remove_field(self, field_name): if field_name not in self.fields: return del self.table original = self.fields.pop(field_name) del self.columns[original.column_name] del self.combined[field_name] try: del self.combined[original.column_name] except KeyError: pass self._sorted_field_list.remove(original) self._update_sorted_fields() if original.default is not None: del self.defaults[original] if self._default_callables.pop(original, None): for i, (name, _) in enumerate(self._default_callable_list): if name == field_name: self._default_callable_list.pop(i) break else: self._default_dict.pop(original, None) self._default_by_name.pop(original.name, None) if isinstance(original, ForeignKeyField): self.remove_ref(original) def set_primary_key(self, name, field): self.composite_key = isinstance(field, CompositeKey) self.add_field(name, field) self.primary_key = field self.auto_increment = ( field.auto_increment or bool(field.sequence)) def get_primary_keys(self): if self.composite_key: return tuple([self.fields[field_name] for field_name in self.primary_key.field_names]) else: return (self.primary_key,) if self.primary_key is not False else () def get_default_dict(self): dd = self._default_by_name.copy() for field_name, default in self._default_callable_list: dd[field_name] = default() return dd def fields_to_index(self): indexes = [] for f in self.sorted_fields: if f.primary_key: continue if f.index or f.unique: indexes.append(ModelIndex(self.model, (f,), unique=f.unique, using=f.index_type)) for index_obj in self.indexes: if isinstance(index_obj, Node): indexes.append(index_obj) elif isinstance(index_obj, (list, tuple)): index_parts, unique = index_obj fields = [] for part in index_parts: if isinstance(part, str): fields.append(self.combined[part]) elif isinstance(part, Node): fields.append(part) else: raise ValueError('Expected either a field name or a ' 'subclass of Node. Got: %s' % part) indexes.append(ModelIndex(self.model, fields, unique=unique)) return indexes def set_database(self, database): self.database = database self.model._schema._database = database del self.table # Apply any hooks that have been registered. If we have an # uninitialized proxy object, we will treat that as `None`. if isinstance(database, Proxy) and database.obj is None: database = None for hook in self._db_hooks: hook(database) def set_table_name(self, table_name): self.table_name = table_name del self.table class SubclassAwareMetadata(Metadata): models = [] def __init__(self, model, *args, **kwargs): super(SubclassAwareMetadata, self).__init__(model, *args, **kwargs) self.models.append(model) def map_models(self, fn): for model in self.models: fn(model) class DoesNotExist(Exception): pass class ModelBase(type): inheritable = set(['constraints', 'database', 'indexes', 'primary_key', 'options', 'schema', 'table_function', 'temporary', 'only_save_dirty', 'legacy_table_names', 'table_settings', 'strict_tables']) def __new__(cls, name, bases, attrs, **kwargs): if name == MODEL_BASE or bases[0].__name__ == MODEL_BASE: return super(ModelBase, cls).__new__(cls, name, bases, attrs, **kwargs) meta_options = {} meta = attrs.pop('Meta', None) if meta: for k, v in meta.__dict__.items(): if not k.startswith('_'): meta_options[k] = v pk = getattr(meta, 'primary_key', None) pk_name = parent_pk = None # Inherit any field descriptors by deep copying the underlying field # into the attrs of the new model, additionally see if the bases define # inheritable model options and swipe them. for b in bases: if not hasattr(b, '_meta'): continue base_meta = b._meta if parent_pk is None: parent_pk = deepcopy(base_meta.primary_key) all_inheritable = cls.inheritable | base_meta._additional_keys for k in base_meta.__dict__: if k in all_inheritable and k not in meta_options: meta_options[k] = base_meta.__dict__[k] meta_options.setdefault('database', base_meta.database) meta_options.setdefault('schema', base_meta.schema) for (k, v) in b.__dict__.items(): if k in attrs: continue if isinstance(v, FieldAccessor) and not v.field.primary_key: attrs[k] = deepcopy(v.field) sopts = meta_options.pop('schema_options', None) or {} Meta = meta_options.get('model_metadata_class', Metadata) Schema = meta_options.get('schema_manager_class', SchemaManager) # Construct the new class. cls = super(ModelBase, cls).__new__(cls, name, bases, attrs, **kwargs) cls.__data__ = cls.__rel__ = None cls._meta = Meta(cls, **meta_options) cls._schema = Schema(cls, **sopts) fields = [] for key, value in cls.__dict__.items(): if isinstance(value, Field): if value.primary_key and pk: raise ValueError('over-determined primary key %s.' % name) elif value.primary_key: pk, pk_name = value, key else: fields.append((key, value)) if pk is None: if parent_pk is not False: pk, pk_name = ((parent_pk, parent_pk.name) if parent_pk is not None else (AutoField(), 'id')) else: pk = False elif isinstance(pk, CompositeKey): pk_name = '__composite_key__' cls._meta.composite_key = True if pk is not False: cls._meta.set_primary_key(pk_name, pk) for name, field in fields: cls._meta.add_field(name, field) # Create a repr and error class before finalizing. if hasattr(cls, '__str__') and '__repr__' not in attrs: setattr(cls, '__repr__', lambda self: '<%s: %s>' % ( cls.__name__, self.__str__())) exc_name = '%sDoesNotExist' % cls.__name__ exc_attrs = {'__module__': cls.__module__} exception_class = type(exc_name, (DoesNotExist,), exc_attrs) cls.DoesNotExist = exception_class # Call validation hook, allowing additional model validation. cls.validate_model() DeferredForeignKey.resolve(cls) return cls def __repr__(self): return '' % self.__name__ def __iter__(self): return iter(self.select()) def __getitem__(self, key): return self.get_by_id(key) def __setitem__(self, key, value): self.set_by_id(key, value) def __delitem__(self, key): self.delete_by_id(key) def __contains__(self, key): try: self.get_by_id(key) except self.DoesNotExist: return False else: return True def __len__(self): return self.select().count() def __bool__(self): return True __nonzero__ = __bool__ # Python 2. def __sql__(self, ctx): return ctx.sql(self._meta.table) class _BoundModelsContext(object): def __init__(self, models, database, bind_refs, bind_backrefs): self.models = models self.database = database self.bind_refs = bind_refs self.bind_backrefs = bind_backrefs def __enter__(self): self._orig_database = [] for model in self.models: self._orig_database.append(model._meta.database) model.bind(self.database, self.bind_refs, self.bind_backrefs, _exclude=set(self.models)) return self.models def __exit__(self, exc_type, exc_val, exc_tb): for model, db in zip(self.models, self._orig_database): model.bind(db, self.bind_refs, self.bind_backrefs, _exclude=set(self.models)) class Model(with_metaclass(ModelBase, Node)): def __init__(self, *args, **kwargs): if kwargs.pop('__no_default__', None): self.__data__ = {} self._dirty = set() else: self.__data__ = self._meta.get_default_dict() self._dirty = set(self.__data__) self.__rel__ = {} for k in kwargs: setattr(self, k, kwargs[k]) def __str__(self): return str(self._pk) if self._meta.primary_key is not False else 'n/a' @classmethod def validate_model(cls): pass @classmethod def alias(cls, alias=None): return ModelAlias(cls, alias) @classmethod def select(cls, *fields): is_default = not fields if not fields: fields = cls._meta.sorted_fields return ModelSelect(cls, fields, is_default=is_default) @classmethod def _normalize_data(cls, data, kwargs): normalized = {} if data: if not isinstance(data, dict): if kwargs: raise ValueError('Data cannot be mixed with keyword ' 'arguments: %s' % data) return data for key in data: try: field = (key if isinstance(key, Field) else cls._meta.combined[key]) except KeyError: if not isinstance(key, Node): raise ValueError('Unrecognized field name: "%s" in %s.' % (key, data)) field = key normalized[field] = data[key] if kwargs: for key in kwargs: try: normalized[cls._meta.combined[key]] = kwargs[key] except KeyError: normalized[getattr(cls, key)] = kwargs[key] return normalized @classmethod def update(cls, __data=None, **update): return ModelUpdate(cls, cls._normalize_data(__data, update)) @classmethod def insert(cls, __data=None, **insert): return ModelInsert(cls, cls._normalize_data(__data, insert)) @classmethod def insert_many(cls, rows, fields=None): return ModelInsert(cls, insert=rows, columns=fields) @classmethod def insert_from(cls, query, fields): columns = [getattr(cls, field) if isinstance(field, str) else field for field in fields] return ModelInsert(cls, insert=query, columns=columns) @classmethod def replace(cls, __data=None, **insert): return cls.insert(__data, **insert).on_conflict('REPLACE') @classmethod def replace_many(cls, rows, fields=None): return (cls .insert_many(rows=rows, fields=fields) .on_conflict('REPLACE')) @classmethod def raw(cls, sql, *params): return ModelRaw(cls, sql, params) @classmethod def delete(cls): return ModelDelete(cls) @classmethod def create(cls, **query): inst = cls(**query) inst.save(force_insert=True) return inst @classmethod def bulk_create(cls, model_list, batch_size=None): if batch_size is not None: batches = chunked(model_list, batch_size) else: batches = [model_list] field_names = list(cls._meta.sorted_field_names) if cls._meta.auto_increment: pk_name = cls._meta.primary_key.name field_names.remove(pk_name) if cls._meta.database.returning_clause and \ cls._meta.primary_key is not False: pk_fields = cls._meta.get_primary_keys() else: pk_fields = None fields = [cls._meta.fields[field_name] for field_name in field_names] attrs = [] for field in fields: if isinstance(field, ForeignKeyField): attrs.append(field.object_id_name) else: attrs.append(field.name) for batch in batches: accum = ([getattr(model, f) for f in attrs] for model in batch) res = cls.insert_many(accum, fields=fields).execute() if pk_fields and res is not None: for row, model in zip(res, batch): for (pk_field, obj_id) in zip(pk_fields, row): setattr(model, pk_field.name, obj_id) @classmethod def bulk_update(cls, model_list, fields, batch_size=None): if isinstance(cls._meta.primary_key, CompositeKey): raise ValueError('bulk_update() is not supported for models with ' 'a composite primary key.') # First normalize list of fields so all are field instances. fields = [cls._meta.fields[f] if isinstance(f, str) else f for f in fields] # Now collect list of attribute names to use for values. attrs = [field.object_id_name if isinstance(field, ForeignKeyField) else field.name for field in fields] if batch_size is not None: batches = chunked(model_list, batch_size) else: batches = [model_list] n = 0 pk = cls._meta.primary_key for batch in batches: id_list = [model._pk for model in batch] update = {} for field, attr in zip(fields, attrs): accum = [] for model in batch: value = getattr(model, attr) if not isinstance(value, Node): value = field.to_value(value, case=True) accum.append((pk.to_value(model._pk), value)) case = Case(pk, accum) update[field] = case n += (cls.update(update) .where(cls._meta.primary_key.in_(id_list)) .execute()) return n @classmethod def noop(cls): return NoopModelSelect(cls, ()) @classmethod def get(cls, *query, **filters): sq = cls.select() if query: # Handle simple lookup using just the primary key. if len(query) == 1 and isinstance(query[0], int) and \ cls._meta.auto_increment: sq = sq.where(cls._meta.primary_key == query[0]) else: sq = sq.where(*query) if filters: sq = sq.filter(**filters) return sq.get() @classmethod def get_or_none(cls, *query, **filters): try: return cls.get(*query, **filters) except DoesNotExist: pass @classmethod def get_by_id(cls, pk): return cls.get(cls._meta.primary_key == pk) @classmethod def set_by_id(cls, key, value): if key is None: return cls.insert(value).execute() else: return (cls.update(value) .where(cls._meta.primary_key == key).execute()) @classmethod def delete_by_id(cls, pk): return cls.delete().where(cls._meta.primary_key == pk).execute() @classmethod def get_or_create(cls, **kwargs): defaults = kwargs.pop('defaults', {}) query = cls.select() for field, value in kwargs.items(): query = query.where(getattr(cls, field) == value) try: return query.get(), False except cls.DoesNotExist: try: if defaults: kwargs.update(defaults) with cls._meta.database.atomic(): return cls.create(**kwargs), True except IntegrityError as exc: try: return query.get(), False except cls.DoesNotExist: raise exc @classmethod def filter(cls, *dq_nodes, **filters): return cls.select().filter(*dq_nodes, **filters) def get_id(self): # Using getattr(self, pk-name) could accidentally trigger a query if # the primary-key is a foreign-key. So we use the safe_name attribute, # which defaults to the field-name, but will be the object_id_name for # foreign-key fields. if self._meta.primary_key is not False: return getattr(self, self._meta.primary_key.safe_name) _pk = property(get_id) @_pk.setter def _pk(self, value): setattr(self, self._meta.primary_key.name, value) def _pk_expr(self): return self._meta.primary_key == self._pk def _prune_fields(self, field_dict, only): new_data = {} for field in only: if isinstance(field, str): field = self._meta.combined[field] if field.name in field_dict: new_data[field.name] = field_dict[field.name] return new_data def _populate_unsaved_relations(self, field_dict): for foreign_key_field in self._meta.refs: foreign_key = foreign_key_field.name conditions = ( foreign_key in field_dict and field_dict[foreign_key] is None and self.__rel__.get(foreign_key) is not None) if conditions: setattr(self, foreign_key, getattr(self, foreign_key)) field_dict[foreign_key] = self.__data__[foreign_key] def save(self, force_insert=False, only=None): field_dict = self.__data__.copy() if self._meta.primary_key is not False: pk_field = self._meta.primary_key pk_value = self._pk else: pk_field = pk_value = None if only is not None: field_dict = self._prune_fields(field_dict, only) elif self._meta.only_save_dirty and not force_insert: field_dict = self._prune_fields(field_dict, self.dirty_fields) if not field_dict: self._dirty.clear() return False self._populate_unsaved_relations(field_dict) rows = 1 if self._meta.auto_increment and pk_value is None: field_dict.pop(pk_field.name, None) if pk_value is not None and not force_insert: if self._meta.composite_key: for pk_part_name in pk_field.field_names: field_dict.pop(pk_part_name, None) else: field_dict.pop(pk_field.name, None) if not field_dict: raise ValueError('no data to save!') rows = self.update(**field_dict).where(self._pk_expr()).execute() elif pk_field is not None: pk = self.insert(**field_dict).execute() if pk is not None and (self._meta.auto_increment or pk_value is None): self._pk = pk # Although we set the primary-key, do not mark it as dirty. self._dirty.discard(pk_field.name) else: self.insert(**field_dict).execute() self._dirty -= set(field_dict) # Remove any fields we saved. return rows def is_dirty(self): return bool(self._dirty) @property def dirty_fields(self): return [f for f in self._meta.sorted_fields if f.name in self._dirty] @property def dirty_field_names(self): return [f.name for f in self._meta.sorted_fields if f.name in self._dirty] def dependencies(self, search_nullable=True, exclude_null_children=False): model_class = type(self) stack = [(type(self), None)] queries = {} seen = set() while stack: klass, query = stack.pop() if klass in seen: continue seen.add(klass) for fk, rel_model in klass._meta.backrefs.items(): if rel_model is model_class or query is None: node = (fk == self.__data__[fk.rel_field.name]) else: node = fk << query subquery = (rel_model.select(rel_model._meta.primary_key) .where(node)) if not fk.null or search_nullable: queries.setdefault(rel_model, []).append((node, fk)) if fk.null and exclude_null_children: # Do not process additional children of this node, but # include it in the list of dependencies. seen.add(rel_model) else: stack.append((rel_model, subquery)) for m in reversed(sort_models(seen)): for sq, q in queries.get(m, ()): yield sq, q def delete_instance(self, recursive=False, delete_nullable=False): if recursive: for query, fk in self.dependencies(exclude_null_children=not delete_nullable): model = fk.model if fk.null and not delete_nullable: model.update(**{fk.name: None}).where(query).execute() else: model.delete().where(query).execute() return type(self).delete().where(self._pk_expr()).execute() def __hash__(self): return hash((self.__class__, self._pk)) def __eq__(self, other): return ( other.__class__ == self.__class__ and self._pk is not None and self._pk == other._pk) def __ne__(self, other): return not self == other def __sql__(self, ctx): # NOTE: when comparing a foreign-key field whose related-field is not a # primary-key, then doing an equality test for the foreign-key with a # model instance will return the wrong value; since we would return # the primary key for a given model instance. # # This checks to see if we have a converter in the scope, and that we # are converting a foreign-key expression. If so, we hand the model # instance to the converter rather than blindly grabbing the primary- # key. In the event the provided converter fails to handle the model # instance, then we will return the primary-key. if ctx.state.converter is not None and ctx.state.is_fk_expr: try: return ctx.sql(Value(self, converter=ctx.state.converter)) except (TypeError, ValueError): pass return ctx.sql(Value(getattr(self, self._meta.primary_key.name), converter=self._meta.primary_key.db_value)) @classmethod def bind(cls, database, bind_refs=True, bind_backrefs=True, _exclude=None): is_different = cls._meta.database is not database cls._meta.set_database(database) if bind_refs or bind_backrefs: if _exclude is None: _exclude = set() G = cls._meta.model_graph(refs=bind_refs, backrefs=bind_backrefs) for _, model, is_backref in G: if model not in _exclude: model._meta.set_database(database) _exclude.add(model) return is_different @classmethod def bind_ctx(cls, database, bind_refs=True, bind_backrefs=True): return _BoundModelsContext((cls,), database, bind_refs, bind_backrefs) @classmethod def table_exists(cls): M = cls._meta return cls._schema.database.table_exists(M.table.__name__, M.schema) @classmethod def create_table(cls, safe=True, **options): if 'fail_silently' in options: __deprecated__('"fail_silently" has been deprecated in favor of ' '"safe" for the create_table() method.') safe = options.pop('fail_silently') if safe and not cls._schema.database.safe_create_index \ and cls.table_exists(): return if cls._meta.temporary: options.setdefault('temporary', cls._meta.temporary) cls._schema.create_all(safe, **options) @classmethod def drop_table(cls, safe=True, drop_sequences=True, **options): if safe and not cls._schema.database.safe_drop_index \ and not cls.table_exists(): return if cls._meta.temporary: options.setdefault('temporary', cls._meta.temporary) cls._schema.drop_all(safe, drop_sequences, **options) @classmethod def truncate_table(cls, **options): cls._schema.truncate_table(**options) @classmethod def index(cls, *fields, **kwargs): return ModelIndex(cls, fields, **kwargs) @classmethod def add_index(cls, *fields, **kwargs): if len(fields) == 1 and isinstance(fields[0], (SQL, Index)): cls._meta.indexes.append(fields[0]) else: cls._meta.indexes.append(ModelIndex(cls, fields, **kwargs)) class ModelAlias(Node): """Provide a separate reference to a model in a query.""" def __init__(self, model, alias=None): self.__dict__['model'] = model self.__dict__['alias'] = alias def __getattr__(self, attr): # Hack to work-around the fact that properties or other objects # implementing the descriptor protocol (on the model being aliased), # will not work correctly when we use getattr(). So we explicitly pass # the model alias to the descriptor's getter. for b in (self.model,) + self.model.__bases__: try: obj = b.__dict__[attr] if isinstance(obj, ModelDescriptor): return obj.__get__(None, self) except KeyError: continue model_attr = getattr(self.model, attr) if isinstance(model_attr, Field): self.__dict__[attr] = FieldAlias.create(self, model_attr) return self.__dict__[attr] return model_attr def __setattr__(self, attr, value): raise AttributeError('Cannot set attributes on model aliases.') def get_field_aliases(self): return [getattr(self, n) for n in self.model._meta.sorted_field_names] def select(self, *selection): if not selection: selection = self.get_field_aliases() return ModelSelect(self, selection) def __call__(self, **kwargs): return self.model(**kwargs) def __sql__(self, ctx): if ctx.scope == SCOPE_VALUES: # Return the quoted table name. return ctx.sql(self.model) if self.alias: ctx.alias_manager[self] = self.alias if ctx.scope == SCOPE_SOURCE: # Define the table and its alias. return (ctx .sql(self.model._meta.entity) .literal(' AS ') .sql(Entity(ctx.alias_manager[self]))) else: # Refer to the table using the alias. return ctx.sql(Entity(ctx.alias_manager[self])) class FieldAlias(Field): def __init__(self, source, field): self.source = source self.model = source.model self.field = field @classmethod def create(cls, source, field): class _FieldAlias(cls, type(field)): pass return _FieldAlias(source, field) def clone(self): return FieldAlias(self.source, self.field) def adapt(self, value): return self.field.adapt(value) def python_value(self, value): return self.field.python_value(value) def db_value(self, value): return self.field.db_value(value) def __getattr__(self, attr): return self.source if attr == 'model' else getattr(self.field, attr) def __sql__(self, ctx): return ctx.sql(Column(self.source, self.field.column_name)) def sort_models(models): models = set(models) seen = set() ordering = [] def dfs(model): if model in models and model not in seen: seen.add(model) for foreign_key, rel_model in model._meta.refs.items(): # Do not depth-first search deferred foreign-keys as this can # cause tables to be created in the incorrect order. if not foreign_key.deferred: dfs(rel_model) if model._meta.depends_on: for dependency in model._meta.depends_on: dfs(dependency) ordering.append(model) names = lambda m: (m._meta.name, m._meta.table_name) for m in sorted(models, key=names): dfs(m) return ordering class _ModelQueryHelper(object): default_row_type = ROW.MODEL def __init__(self, *args, **kwargs): super(_ModelQueryHelper, self).__init__(*args, **kwargs) if not self._database: self._database = self.model._meta.database @Node.copy def objects(self, constructor=None): self._row_type = ROW.CONSTRUCTOR self._constructor = self.model if constructor is None else constructor @Node.copy def models(self): self._row_type = ROW.MODEL def _get_cursor_wrapper(self, cursor): row_type = self._row_type or self.default_row_type if row_type == ROW.MODEL: return self._get_model_cursor_wrapper(cursor) elif row_type == ROW.DICT: return ModelDictCursorWrapper(cursor, self.model, self._returning) elif row_type == ROW.TUPLE: return ModelTupleCursorWrapper(cursor, self.model, self._returning) elif row_type == ROW.NAMED_TUPLE: return ModelNamedTupleCursorWrapper(cursor, self.model, self._returning) elif row_type == ROW.CONSTRUCTOR: return ModelObjectCursorWrapper(cursor, self.model, self._returning, self._constructor) else: raise ValueError('Unrecognized row type: "%s".' % row_type) def _get_model_cursor_wrapper(self, cursor): return ModelObjectCursorWrapper(cursor, self.model, [], self.model) class ModelRaw(_ModelQueryHelper, RawQuery): def __init__(self, model, sql, params, **kwargs): self.model = model self._returning = () super(ModelRaw, self).__init__(sql=sql, params=params, **kwargs) def get(self): try: return self.execute()[0] except IndexError: sql, params = self.sql() raise self.model.DoesNotExist('%s instance matching query does ' 'not exist:\nSQL: %s\nParams: %s' % (self.model, sql, params)) class BaseModelSelect(_ModelQueryHelper): def union_all(self, rhs): return ModelCompoundSelectQuery(self.model, self, 'UNION ALL', rhs) __add__ = union_all def union(self, rhs): return ModelCompoundSelectQuery(self.model, self, 'UNION', rhs) __or__ = union def intersect(self, rhs): return ModelCompoundSelectQuery(self.model, self, 'INTERSECT', rhs) __and__ = intersect def except_(self, rhs): return ModelCompoundSelectQuery(self.model, self, 'EXCEPT', rhs) __sub__ = except_ def __iter__(self): if not self._cursor_wrapper: self.execute() return iter(self._cursor_wrapper) def prefetch(self, *subqueries, **kwargs): return prefetch(self, *subqueries, **kwargs) def get(self, database=None): clone = self.paginate(1, 1) clone._cursor_wrapper = None try: return clone.execute(database)[0] except IndexError: sql, params = clone.sql() raise self.model.DoesNotExist('%s instance matching query does ' 'not exist:\nSQL: %s\nParams: %s' % (clone.model, sql, params)) def get_or_none(self, database=None): try: return self.get(database=database) except self.model.DoesNotExist: pass @Node.copy def group_by(self, *columns): grouping = [] for column in columns: if is_model(column): grouping.extend(column._meta.sorted_fields) elif isinstance(column, Table): if not column._columns: raise ValueError('Cannot pass a table to group_by() that ' 'does not have columns explicitly ' 'declared.') grouping.extend([getattr(column, col_name) for col_name in column._columns]) else: grouping.append(column) self._group_by = grouping class ModelCompoundSelectQuery(BaseModelSelect, CompoundSelectQuery): def __init__(self, model, *args, **kwargs): self.model = model super(ModelCompoundSelectQuery, self).__init__(*args, **kwargs) def _get_model_cursor_wrapper(self, cursor): return self.lhs._get_model_cursor_wrapper(cursor) def _normalize_model_select(fields_or_models): fields = [] for fm in fields_or_models: if is_model(fm): fields.extend(fm._meta.sorted_fields) elif isinstance(fm, ModelAlias): fields.extend(fm.get_field_aliases()) elif isinstance(fm, Table) and fm._columns: fields.extend([getattr(fm, col) for col in fm._columns]) else: fields.append(fm) return fields class ModelSelect(BaseModelSelect, Select): def __init__(self, model, fields_or_models, is_default=False): self.model = self._join_ctx = model self._joins = {} self._is_default = is_default fields = _normalize_model_select(fields_or_models) super(ModelSelect, self).__init__([model], fields) def clone(self): clone = super(ModelSelect, self).clone() clone._joins = dict(clone._joins) return clone def select(self, *fields_or_models): if fields_or_models or not self._is_default: self._is_default = False fields = _normalize_model_select(fields_or_models) return super(ModelSelect, self).select(*fields) return self def select_extend(self, *columns): self._is_default = False fields = _normalize_model_select(columns) return super(ModelSelect, self).select_extend(*fields) def switch(self, ctx=None): self._join_ctx = self.model if ctx is None else ctx return self def _get_model(self, src): if is_model(src): return src, True elif isinstance(src, Table) and src._model: return src._model, False elif isinstance(src, ModelAlias): return src.model, False elif isinstance(src, ModelSelect): return src.model, False return None, False def _normalize_join(self, src, dest, on, attr): # Allow "on" expression to have an alias that determines the # destination attribute for the joined data. on_alias = isinstance(on, Alias) if on_alias: attr = attr or on._alias on = on.alias() # Obtain references to the source and destination models being joined. src_model, src_is_model = self._get_model(src) dest_model, dest_is_model = self._get_model(dest) if src_model and dest_model: self._join_ctx = dest constructor = dest_model # In the case where the "on" clause is a Column or Field, we will # convert that field into the appropriate predicate expression. if not (src_is_model and dest_is_model) and isinstance(on, Column): if on.source is src: to_field = src_model._meta.columns[on.name] elif on.source is dest: to_field = dest_model._meta.columns[on.name] else: raise AttributeError('"on" clause Column %s does not ' 'belong to %s or %s.' % (on, src_model, dest_model)) on = None elif isinstance(on, Field): to_field = on on = None else: to_field = None fk_field, is_backref = self._generate_on_clause( src_model, dest_model, to_field, on) if on is None: src_attr = 'name' if src_is_model else 'column_name' dest_attr = 'name' if dest_is_model else 'column_name' if is_backref: lhs = getattr(dest, getattr(fk_field, dest_attr)) rhs = getattr(src, getattr(fk_field.rel_field, src_attr)) else: lhs = getattr(src, getattr(fk_field, src_attr)) rhs = getattr(dest, getattr(fk_field.rel_field, dest_attr)) on = (lhs == rhs) if not attr: if fk_field is not None and not is_backref: attr = fk_field.name else: attr = dest_model._meta.name elif on_alias and fk_field is not None and \ attr == fk_field.object_id_name and not is_backref: raise ValueError('Cannot assign join alias to "%s", as this ' 'attribute is the object_id_name for the ' 'foreign-key field "%s"' % (attr, fk_field)) elif isinstance(dest, Source): constructor = dict attr = attr or dest._alias if not attr and isinstance(dest, Table): attr = attr or dest.__name__ return (on, attr, constructor) def _generate_on_clause(self, src, dest, to_field=None, on=None): meta = src._meta is_backref = fk_fields = False # Get all the foreign keys between source and dest, and determine if # the join is via a back-reference. if dest in meta.model_refs: fk_fields = meta.model_refs[dest] elif dest in meta.model_backrefs: fk_fields = meta.model_backrefs[dest] is_backref = True if not fk_fields: if on is not None: return None, False raise ValueError('Unable to find foreign key between %s and %s. ' 'Please specify an explicit join condition.' % (src, dest)) elif to_field is not None: # If the foreign-key field was specified explicitly, remove all # other foreign-key fields from the list. target = (to_field.field if isinstance(to_field, FieldAlias) else to_field) fk_fields = [f for f in fk_fields if ( (f is target) or (is_backref and f.rel_field is to_field))] if len(fk_fields) == 1: return fk_fields[0], is_backref if on is None: # If multiple foreign-keys exist, try using the FK whose name # matches that of the related model. If not, raise an error as this # is ambiguous. for fk in fk_fields: if fk.name == dest._meta.name: return fk, is_backref raise ValueError('More than one foreign key between %s and %s.' ' Please specify which you are joining on.' % (src, dest)) # If there are multiple foreign-keys to choose from and the join # predicate is an expression, we'll try to figure out which # foreign-key field we're joining on so that we can assign to the # correct attribute when resolving the model graph. to_field = None if isinstance(on, Expression): lhs, rhs = on.lhs, on.rhs # Coerce to set() so that we force Python to compare using the # object's hash rather than equality test, which returns a # false-positive due to overriding __eq__. fk_set = set(fk_fields) if isinstance(lhs, Field): lhs_f = lhs.field if isinstance(lhs, FieldAlias) else lhs if lhs_f in fk_set: to_field = lhs_f elif isinstance(rhs, Field): rhs_f = rhs.field if isinstance(rhs, FieldAlias) else rhs if rhs_f in fk_set: to_field = rhs_f return to_field, False @Node.copy def join(self, dest, join_type=JOIN.INNER, on=None, src=None, attr=None): src = self._join_ctx if src is None else src if join_type == JOIN.LATERAL or join_type == JOIN.LEFT_LATERAL: on = True elif join_type != JOIN.CROSS: on, attr, constructor = self._normalize_join(src, dest, on, attr) if attr: self._joins.setdefault(src, []) self._joins[src].append((dest, attr, constructor, join_type)) elif on is not None: raise ValueError('Cannot specify on clause with cross join.') if not self._from_list: raise ValueError('No sources to join on.') item = self._from_list.pop() self._from_list.append(Join(item, dest, join_type, on)) def left_outer_join(self, dest, on=None, src=None, attr=None): return self.join(dest, JOIN.LEFT_OUTER, on, src, attr) def join_from(self, src, dest, join_type=JOIN.INNER, on=None, attr=None): return self.join(dest, join_type, on, src, attr) def _get_model_cursor_wrapper(self, cursor): if len(self._from_list) == 1 and not self._joins: return ModelObjectCursorWrapper(cursor, self.model, self._returning, self.model) return ModelCursorWrapper(cursor, self.model, self._returning, self._from_list, self._joins) def ensure_join(self, lm, rm, on=None, **join_kwargs): join_ctx = self._join_ctx for dest, _, constructor, _ in self._joins.get(lm, []): if dest == rm: return self return self.switch(lm).join(rm, on=on, **join_kwargs).switch(join_ctx) def convert_dict_to_node(self, qdict): accum = [] joins = [] fks = (ForeignKeyField, BackrefAccessor) for key, value in sorted(qdict.items()): curr = self.model if '__' in key and key.rsplit('__', 1)[1] in DJANGO_MAP: key, op = key.rsplit('__', 1) op = DJANGO_MAP[op] elif value is None: op = DJANGO_MAP['is'] else: op = DJANGO_MAP['eq'] if '__' not in key: # Handle simplest case. This avoids joining over-eagerly when a # direct FK lookup is all that is required. model_attr = getattr(curr, key) else: for piece in key.split('__'): for dest, attr, _, _ in self._joins.get(curr, ()): try: model_attr = getattr(curr, piece, None) except Exception: pass if attr == piece or (isinstance(dest, ModelAlias) and dest.alias == piece): curr = dest break else: model_attr = getattr(curr, piece) if value is not None and isinstance(model_attr, fks): curr = model_attr.rel_model joins.append(model_attr) accum.append(op(model_attr, value)) return accum, joins def filter(self, *args, **kwargs): # normalize args and kwargs into a new expression if args and kwargs: dq_node = (reduce(operator.and_, [a.clone() for a in args]) & DQ(**kwargs)) elif args: dq_node = (reduce(operator.and_, [a.clone() for a in args]) & ColumnBase()) elif kwargs: dq_node = DQ(**kwargs) & ColumnBase() else: return self.clone() # dq_node should now be an Expression, lhs = Node(), rhs = ... q = collections.deque([dq_node]) dq_joins = [] seen_joins = set() while q: curr = q.popleft() if not isinstance(curr, Expression): continue for side, piece in (('lhs', curr.lhs), ('rhs', curr.rhs)): if isinstance(piece, DQ): query, joins = self.convert_dict_to_node(piece.query) for join in joins: if join not in seen_joins: dq_joins.append(join) seen_joins.add(join) expression = reduce(operator.and_, query) # Apply values from the DQ object. if piece._negated: expression = Negated(expression) #expression._alias = piece._alias setattr(curr, side, expression) else: q.append(piece) if not args or not kwargs: dq_node = dq_node.lhs query = self.clone() for field in dq_joins: if isinstance(field, ForeignKeyField): lm, rm = field.model, field.rel_model field_obj = field elif isinstance(field, BackrefAccessor): lm, rm = field.model, field.rel_model field_obj = field.field query = query.ensure_join(lm, rm, field_obj) return query.where(dq_node) def create_table(self, name, safe=True, **meta): return self.model._schema.create_table_as(name, self, safe, **meta) def __sql_selection__(self, ctx, is_subquery=False): if self._is_default and is_subquery and len(self._returning) > 1 and \ self.model._meta.primary_key is not False: return ctx.sql(self.model._meta.primary_key) return ctx.sql(CommaNodeList(self._returning)) class NoopModelSelect(ModelSelect): def __sql__(self, ctx): return self.model._meta.database.get_noop_select(ctx) def _get_cursor_wrapper(self, cursor): return CursorWrapper(cursor) class _ModelWriteQueryHelper(_ModelQueryHelper): def __init__(self, model, *args, **kwargs): self.model = model super(_ModelWriteQueryHelper, self).__init__(model, *args, **kwargs) def returning(self, *returning): accum = [] for item in returning: if is_model(item): accum.extend(item._meta.sorted_fields) else: accum.append(item) return super(_ModelWriteQueryHelper, self).returning(*accum) def _set_table_alias(self, ctx): table = self.model._meta.table ctx.alias_manager[table] = table.__name__ class ModelUpdate(_ModelWriteQueryHelper, Update): pass class ModelInsert(_ModelWriteQueryHelper, Insert): default_row_type = ROW.TUPLE def __init__(self, *args, **kwargs): super(ModelInsert, self).__init__(*args, **kwargs) if self._returning is None and self.model._meta.database is not None: if self.model._meta.database.returning_clause: self._returning = self.model._meta.get_primary_keys() def returning(self, *returning): # By default ModelInsert will yield a `tuple` containing the # primary-key of the newly inserted row. But if we are explicitly # specifying a returning clause and have not set a row type, we will # default to returning model instances instead. if returning and self._row_type is None: self._row_type = ROW.MODEL return super(ModelInsert, self).returning(*returning) def get_default_data(self): return self.model._meta.defaults def get_default_columns(self): fields = self.model._meta.sorted_fields return fields[1:] if self.model._meta.auto_increment else fields class ModelDelete(_ModelWriteQueryHelper, Delete): pass class ManyToManyQuery(ModelSelect): def __init__(self, instance, accessor, rel, *args, **kwargs): self._instance = instance self._accessor = accessor self._src_attr = accessor.src_fk.rel_field.name self._dest_attr = accessor.dest_fk.rel_field.name super(ManyToManyQuery, self).__init__(rel, (rel,), *args, **kwargs) def _id_list(self, model_or_id_list): if isinstance(model_or_id_list[0], Model): return [getattr(obj, self._dest_attr) for obj in model_or_id_list] return model_or_id_list def add(self, value, clear_existing=False): if clear_existing: self.clear() accessor = self._accessor src_id = getattr(self._instance, self._src_attr) if isinstance(value, SelectQuery): query = value.columns( Value(src_id), accessor.dest_fk.rel_field) accessor.through_model.insert_from( fields=[accessor.src_fk, accessor.dest_fk], query=query).execute() else: value = ensure_tuple(value) if not value: return inserts = [{ accessor.src_fk.name: src_id, accessor.dest_fk.name: rel_id} for rel_id in self._id_list(value)] accessor.through_model.insert_many(inserts).execute() def remove(self, value): src_id = getattr(self._instance, self._src_attr) if isinstance(value, SelectQuery): column = getattr(value.model, self._dest_attr) subquery = value.columns(column) return (self._accessor.through_model .delete() .where( (self._accessor.dest_fk << subquery) & (self._accessor.src_fk == src_id)) .execute()) else: value = ensure_tuple(value) if not value: return return (self._accessor.through_model .delete() .where( (self._accessor.dest_fk << self._id_list(value)) & (self._accessor.src_fk == src_id)) .execute()) def clear(self): src_id = getattr(self._instance, self._src_attr) return (self._accessor.through_model .delete() .where(self._accessor.src_fk == src_id) .execute()) def safe_python_value(conv_func): def validate(value): try: return conv_func(value) except (TypeError, ValueError): return value return validate class BaseModelCursorWrapper(DictCursorWrapper): def __init__(self, cursor, model, columns): super(BaseModelCursorWrapper, self).__init__(cursor) self.model = model self.select = columns or [] def initialize(self): combined = self.model._meta.combined table = self.model._meta.table description = self.cursor.description self.ncols = len(self.cursor.description) self.columns = [] self.converters = converters = [None] * self.ncols self.fields = fields = [None] * self.ncols for idx, description_item in enumerate(description): column = orig_column = description_item[0] # Try to clean-up messy column descriptions when people do not # provide an alias. The idea is that we take something like: # SUM("t1"."price") -> "price") -> price dot_index = column.rfind('.') if dot_index != -1: column = column[dot_index + 1:] column = column.strip('()"`') self.columns.append(column) # Now we'll see what they selected and see if we can improve the # column-name being returned - e.g. by mapping it to the selected # field's name. try: raw_node = self.select[idx] except IndexError: if column in combined: raw_node = node = combined[column] else: continue else: node = raw_node.unwrap() # If this column was given an alias, then we will use whatever # alias was returned by the cursor. is_alias = raw_node.is_alias() if is_alias: self.columns[idx] = orig_column # Heuristics used to attempt to get the field associated with a # given SELECT column, so that we can accurately convert the value # returned by the database-cursor into a Python object. if isinstance(node, Field): if raw_node._coerce: converters[idx] = node.python_value fields[idx] = node if not is_alias: self.columns[idx] = node.name elif isinstance(node, ColumnBase) and raw_node._converter: converters[idx] = raw_node._converter elif isinstance(node, Function) and node._coerce: if node._python_value is not None: converters[idx] = node._python_value elif node.arguments and isinstance(node.arguments[0], Node): # If the first argument is a field or references a column # on a Model, try using that field's conversion function. # This usually works, but we use "safe_python_value()" so # that if a TypeError or ValueError occurs during # conversion we can just fall-back to the raw cursor value. first = node.arguments[0].unwrap() if isinstance(first, Entity): path = first._path[-1] # Try to look-up by name. first = combined.get(path) if isinstance(first, Field): converters[idx] = safe_python_value(first.python_value) elif column in combined: if node._coerce: converters[idx] = combined[column].python_value if isinstance(node, Column) and node.source == table: fields[idx] = combined[column] self.no_convert = [] self.convert = [] for i in range(self.ncols): if converters[i] is not None: self.convert.append(i) else: self.no_convert.append(i) def process_row(self, row): raise NotImplementedError class ModelDictCursorWrapper(BaseModelCursorWrapper): def initialize(self): super(ModelDictCursorWrapper, self).initialize() self.unique_columns = self.dedupe_columns( self.columns, valid_identifiers=False) def process_row(self, row): result = {} columns = self.unique_columns for i in self.no_convert: result[columns[i]] = row[i] for i in self.convert: result[columns[i]] = self.converters[i](row[i]) return result class ModelTupleCursorWrapper(BaseModelCursorWrapper): constructor = tuple def process_row(self, row): converters = self.converters return self.constructor([ (converters[i](row[i]) if converters[i] is not None else row[i]) for i in range(self.ncols)]) class ModelNamedTupleCursorWrapper(ModelTupleCursorWrapper): def initialize(self): super(ModelNamedTupleCursorWrapper, self).initialize() identifiers = self.dedupe_columns(self.columns) self.impl = collections.namedtuple('Row', identifiers) self.constructor = lambda row: self.impl(*row) class ModelObjectCursorWrapper(ModelDictCursorWrapper): def __init__(self, cursor, model, select, constructor): self.constructor = constructor self.is_model = is_model(constructor) super(ModelObjectCursorWrapper, self).__init__(cursor, model, select) def initialize(self): super(ModelObjectCursorWrapper, self).initialize() self.identifiers = self.dedupe_columns(self.columns) def process_row(self, row): result = {} columns = self.identifiers for i in self.no_convert: result[columns[i]] = row[i] for i in self.convert: result[columns[i]] = self.converters[i](row[i]) if self.is_model: # Clear out any dirty fields before returning to the user. obj = self.constructor(__no_default__=1, **result) obj._dirty.clear() return obj else: return self.constructor(**result) class ModelCursorWrapper(BaseModelCursorWrapper): def __init__(self, cursor, model, select, from_list, joins, dicts=False): super(ModelCursorWrapper, self).__init__(cursor, model, select) self.from_list = from_list self.joins = joins self.dicts = dicts def initialize(self): super(ModelCursorWrapper, self).initialize() selected_src = set([field.model for field in self.fields if field is not None]) select = self.select columns = [make_identifier(c) for c in self.columns] if self.dicts: self.key_to_constructor = {self.model: (dict, False)} else: self.key_to_constructor = {self.model: (self.model, True)} self.src_is_dest = {} self.src_to_dest = [] accum = collections.deque(self.from_list) dests = set() while accum: curr = accum.popleft() if isinstance(curr, Join): accum.append(curr.lhs) accum.append(curr.rhs) continue if curr not in self.joins: continue is_dict = isinstance(curr, dict) for key, attr, constructor, join_type in self.joins[curr]: if self.dicts: constructor = dict if key not in self.key_to_constructor: self.key_to_constructor[key] = (constructor, is_model(constructor)) # (src, attr, dest, is_dict, join_type, is outer?). self.src_to_dest.append(( curr, attr, key, is_dict or self.dicts, join_type, join_type.endswith('OUTER'))) dests.add(key) accum.append(key) # Ensure that we accommodate everything selected. for src in selected_src: if src not in self.key_to_constructor: if is_model(src): self.key_to_constructor[src] = (src, True) elif isinstance(src, ModelAlias): self.key_to_constructor[src] = (src.model, True) # Indicate which sources are also dests. for src, _, dest, _, _, _ in self.src_to_dest: self.src_is_dest[src] = src in dests and (dest in selected_src or src in selected_src) self.column_keys = [] for idx, node in enumerate(select): key = self.model field = self.fields[idx] if field is not None: if isinstance(field, FieldAlias): key = field.source else: key = field.model elif isinstance(node, BindTo): if node.dest not in self.key_to_constructor: raise ValueError('%s specifies bind-to %s, but %s is not ' 'among the selected sources.' % (node.unwrap(), node.dest, node.dest)) key = node.dest else: if isinstance(node, Node): node = node.unwrap() if isinstance(node, Column): key = node.source self.column_keys.append(key) # Pre-compute flat list of key/col/converter for each column index. self._row_spec = tuple( (i, self.column_keys[i], columns[i], self.converters[i]) for i in range(self.ncols)) # Flatten list of key / constructor / is model? flag. self._constructor_list = [ (key, construct, _is_model) for key, (construct, _is_model) in self.key_to_constructor.items()] # Pre-compute join-graph reachability. self._dest_reachable = {} for (src, attr, dest, is_dict, join_type, _) in self.src_to_dest: if dest not in self.joins: continue reachable = set() q = collections.deque([dest]) while q: curr = q.popleft() if curr in self.joins: for _key, _, _, _ in self.joins[curr]: reachable.add(_key) q.append(_key) self._dest_reachable[dest] = frozenset(reachable) def process_row(self, row): objects = {} model_list = [] for key, constructor, _is_model in self._constructor_list: if _is_model: objects[key] = constructor(__no_default__=True) model_list.append(objects[key]) else: objects[key] = constructor() default_instance = objects[self.model] set_keys = set() for idx, key, column, converter in self._row_spec: # Get the instance corresponding to the selected column/value, # falling back to the "root" model instance. instance = objects.get(key, default_instance) column = self.columns[idx] value = row[idx] if value is not None: set_keys.add(key) if converter is not None: value = converter(value) if isinstance(instance, dict): instance[column] = value else: setattr(instance, column, value) # Need to do some analysis on the joins before this. for (src, attr, dest, is_dict, _, is_outer) in self.src_to_dest: instance = objects.get(src) joined_instance = objects.get(dest) if joined_instance is None and dest not in objects: continue # Determine if anything further along in the graph is set. assign = False if dest not in set_keys and dest in self._dest_reachable: assign = bool(self._dest_reachable[dest] & set_keys) # If no fields were set on the destination instance then do not # assign an "empty" instance. if dest not in set_keys and not assign: if is_outer: joined_instance = None else: continue # If no fields were set on either the source or the destination, # then we have nothing to do here. if src not in set_keys and dest not in set_keys and is_outer: continue if is_dict: instance[attr] = joined_instance else: setattr(instance, attr, joined_instance) # When instantiating models from a cursor, we clear the dirty fields. for instance in model_list: instance._dirty.clear() return objects[self.model] class PrefetchQuery(collections.namedtuple('_PrefetchQuery', ( 'query', 'fields', 'is_backref', 'rel_models', 'field_to_name', 'model'))): def __new__(cls, query, fields=None, is_backref=None, rel_models=None, field_to_name=None, model=None): if fields: if is_backref: if rel_models is None: rel_models = [field.model for field in fields] foreign_key_attrs = [field.rel_field.name for field in fields] else: if rel_models is None: rel_models = [field.rel_model for field in fields] foreign_key_attrs = [field.name for field in fields] field_to_name = list(zip(fields, foreign_key_attrs)) model = query.model return super(PrefetchQuery, cls).__new__( cls, query, fields, is_backref, rel_models, field_to_name, model) def populate_instance(self, instance, id_map): if self.is_backref: for field in self.fields: identifier = instance.__data__[field.name] key = (field, identifier) if key in id_map: setattr(instance, field.name, id_map[key]) else: for field, attname in self.field_to_name: identifier = instance.__data__[field.rel_field.name] key = (field, identifier) rel_instances = id_map.get(key, []) for inst in rel_instances: setattr(inst, attname, instance) inst._dirty.clear() setattr(instance, field.backref, rel_instances) def store_instance(self, instance, id_map): for field, attname in self.field_to_name: identity = field.rel_field.python_value(instance.__data__[attname]) key = (field, identity) if self.is_backref: id_map[key] = instance else: id_map.setdefault(key, []) id_map[key].append(instance) def prefetch_add_subquery(sq, subqueries, prefetch_type): fixed_queries = [PrefetchQuery(sq)] for i, subquery in enumerate(subqueries): if isinstance(subquery, tuple): subquery, target_model = subquery else: target_model = None if not isinstance(subquery, Query) and is_model(subquery) or \ isinstance(subquery, ModelAlias): subquery = subquery.select() subquery_model = subquery.model for j in reversed(range(i + 1)): fks = backrefs = None fixed = fixed_queries[j] last_query = fixed.query last_model = last_obj = fixed.model if isinstance(last_model, ModelAlias): last_model = last_model.model rels = subquery_model._meta.model_refs.get(last_model, []) if rels: fks = [getattr(subquery_model, fk.name) for fk in rels] pks = [getattr(last_obj, fk.rel_field.name) for fk in rels] else: backrefs = subquery_model._meta.model_backrefs.get(last_model) if (fks or backrefs) and ((target_model is last_obj) or (target_model is None)): break else: tgt_err = ' using %s' % target_model if target_model else '' raise AttributeError('Error: unable to find foreign key for ' 'query: %s%s' % (subquery, tgt_err)) dest = (target_model,) if target_model else None if fks: if prefetch_type == PREFETCH_TYPE.WHERE: expr = reduce(operator.or_, [ (fk << last_query.select(pk)) for (fk, pk) in zip(fks, pks)]) subquery = subquery.where(expr) elif prefetch_type == PREFETCH_TYPE.JOIN: expr = [] select_pks = set() for fk, pk in zip(fks, pks): expr.append(getattr(last_query.c, pk.column_name) == fk) select_pks.add(pk) subquery = subquery.distinct().join( last_query.select(*select_pks), on=reduce(operator.or_, expr)) fixed_queries.append(PrefetchQuery(subquery, fks, False, dest)) elif backrefs: expr = [] fields = [] for backref in backrefs: rel_field = getattr(subquery_model, backref.rel_field.name) fk_field = getattr(last_obj, backref.name) fields.append((rel_field, fk_field)) if prefetch_type == PREFETCH_TYPE.WHERE: for rel_field, fk_field in fields: expr.append(rel_field << last_query.select(fk_field)) subquery = subquery.where(reduce(operator.or_, expr)) elif prefetch_type == PREFETCH_TYPE.JOIN: select_fks = [] for rel_field, fk_field in fields: select_fks.append(fk_field) target = getattr(last_query.c, fk_field.column_name) expr.append(rel_field == target) subquery = subquery.distinct().join( last_query.select(*select_fks), on=reduce(operator.or_, expr)) fixed_queries.append(PrefetchQuery(subquery, backrefs, True, dest)) return fixed_queries def prefetch(sq, *subqueries, **kwargs): if not subqueries: return sq prefetch_type = kwargs.pop('prefetch_type', PREFETCH_TYPE.WHERE) if kwargs: raise ValueError('Unrecognized arguments: %s' % kwargs) fixed_queries = prefetch_add_subquery(sq, subqueries, prefetch_type) deps = {} rel_map = {} for pq in reversed(fixed_queries): query_model = pq.model if pq.fields: for rel_model in pq.rel_models: rel_map.setdefault(rel_model, []) rel_map[rel_model].append(pq) deps.setdefault(query_model, {}) id_map = deps[query_model] has_relations = bool(rel_map.get(query_model)) for instance in pq.query: if pq.fields: pq.store_instance(instance, id_map) if has_relations: for rel in rel_map[query_model]: rel.populate_instance(instance, deps[rel.model]) return list(pq.query) ================================================ FILE: playhouse/README.md ================================================ ## Playhouse The `playhouse` namespace contains numerous extensions to Peewee. These include vendor-specific database extensions, high-level abstractions to simplify working with databases, and tools for low-level database operations and introspection. ### Vendor extensions * [SQLite extensions](http://docs.peewee-orm.com/en/latest/peewee/sqlite_ext.html) * Full-text search (FTS3/4/5) * BM25 ranking algorithm implemented as SQLite C extension, backported to FTS4 * Virtual tables and C extensions * Closure tables * JSON extension support * LSM1 (key/value database) support * BLOB API * Online backup API * [APSW extensions](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#apsw): use Peewee with the powerful [APSW](https://github.com/rogerbinns/apsw) SQLite driver. * [SQLCipher](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#sqlcipher-ext): encrypted SQLite databases. * [SqliteQ](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#sqliteq): dedicated writer thread for multi-threaded SQLite applications. [More info here](http://charlesleifer.com/blog/multi-threaded-sqlite-without-the-operationalerrors/). * [Postgresql extensions](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#postgres-ext) * JSON and JSONB * HStore * Arrays * Server-side cursors * Full-text search * [MySQL extensions](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#mysql-ext) ### High-level libraries * [Extra fields](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#extra-fields) * Compressed field * PickleField * [Shortcuts / helpers](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#shortcuts) * Model-to-dict serializer * Dict-to-model deserializer * [Hybrid attributes](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#hybrid) * [Signals](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#signals): pre/post-save, pre/post-delete, pre-init. * [Dataset](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#dataset): high-level API for working with databases popuarlized by the [project of the same name](https://dataset.readthedocs.io/). * [Key/Value Store](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#kv): key/value store using SQLite. Supports *smart indexing*, for *Pandas*-style queries. ### Database management and framework support * [pwiz](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#pwiz): generate model code from a pre-existing database. * [Schema migrations](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#migrate): modify your schema using high-level APIs. Even supports dropping or renaming columns in SQLite. * [Connection pool](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#pool): simple connection pooling. * [Reflection](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#reflection): low-level, cross-platform database introspection * [Database URLs](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#db-url): use URLs to connect to database * [Test utils](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#test-utils): helpers for unit-testing Peewee applications. * [Flask utils](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#flask-utils): paginated object lists, database connection management, and more. ================================================ FILE: playhouse/__init__.py ================================================ ================================================ FILE: playhouse/_sqlite_udf.pyx ================================================ # cython: language_level=3 from libc.stdlib cimport free, malloc from libc.math cimport log, sqrt from difflib import SequenceMatcher from random import randint # FTS ranking functions. cdef double *get_weights(int ncol, tuple raw_weights): cdef: int argc = len(raw_weights) int icol double *weights = malloc(sizeof(double) * ncol) for icol in range(ncol): if argc == 0: weights[icol] = 1.0 elif icol < argc: weights[icol] = raw_weights[icol] else: weights[icol] = 0.0 return weights def peewee_rank(py_match_info, *raw_weights): cdef: unsigned int *match_info unsigned int *phrase_info bytes _match_info_buf = bytes(py_match_info) char *match_info_buf = _match_info_buf int nphrase, ncol, icol, iphrase, hits, global_hits int P_O = 0, C_O = 1, X_O = 2 double score = 0.0, weight double *weights match_info = match_info_buf nphrase = match_info[P_O] ncol = match_info[C_O] weights = get_weights(ncol, raw_weights) # matchinfo X value corresponds to, for each phrase in the search query, a # list of 3 values for each column in the search table. # So if we have a two-phrase search query and three columns of data, the # following would be the layout: # p0 : c0=[0, 1, 2], c1=[3, 4, 5], c2=[6, 7, 8] # p1 : c0=[9, 10, 11], c1=[12, 13, 14], c2=[15, 16, 17] for iphrase in range(nphrase): phrase_info = &match_info[X_O + iphrase * ncol * 3] for icol in range(ncol): weight = weights[icol] if weight == 0: continue # The idea is that we count the number of times the phrase appears # in this column of the current row, compared to how many times it # appears in this column across all rows. The ratio of these values # provides a rough way to score based on "high value" terms. hits = phrase_info[3 * icol] global_hits = phrase_info[3 * icol + 1] if hits > 0: score += weight * (hits / global_hits) free(weights) return -1 * score def peewee_lucene(py_match_info, *raw_weights): # Usage: peewee_lucene(matchinfo(table, 'pcnalx'), 1) cdef: unsigned int *match_info bytes _match_info_buf = bytes(py_match_info) char *match_info_buf = _match_info_buf int nphrase, ncol double total_docs, term_frequency double doc_length, docs_with_term, avg_length double idf, weight, rhs, denom double *weights int P_O = 0, C_O = 1, N_O = 2, L_O, X_O int iphrase, icol, x double score = 0.0 match_info = match_info_buf nphrase = match_info[P_O] ncol = match_info[C_O] total_docs = match_info[N_O] L_O = 3 + ncol X_O = L_O + ncol weights = get_weights(ncol, raw_weights) for iphrase in range(nphrase): for icol in range(ncol): weight = weights[icol] if weight == 0: continue doc_length = match_info[L_O + icol] x = X_O + (3 * (icol + iphrase * ncol)) term_frequency = match_info[x] # f(qi) docs_with_term = match_info[x + 2] or 1. # n(qi) idf = log(total_docs / (docs_with_term + 1.)) tf = sqrt(term_frequency) fieldNorms = 1.0 / sqrt(doc_length) score += (idf * tf * fieldNorms) free(weights) return -1 * score def peewee_bm25(py_match_info, *raw_weights): # Usage: peewee_bm25(matchinfo(table, 'pcnalx'), 1) # where the second parameter is the index of the column and # the 3rd and 4th specify k and b. cdef: unsigned int *match_info bytes _match_info_buf = bytes(py_match_info) char *match_info_buf = _match_info_buf int nphrase, ncol double B = 0.75, K = 1.2 double total_docs, term_frequency double doc_length, docs_with_term, avg_length double idf, weight, ratio, num, b_part, denom, pc_score double *weights int P_O = 0, C_O = 1, N_O = 2, A_O = 3, L_O, X_O int iphrase, icol, x double score = 0.0 match_info = match_info_buf # PCNALX = matchinfo format. # P = 1 = phrase count within query. # C = 1 = searchable columns in table. # N = 1 = total rows in table. # A = c = for each column, avg number of tokens # L = c = for each column, length of current row (in tokens) # X = 3 * c * p = for each phrase and table column, # * phrase count within column for current row. # * phrase count within column for all rows. # * total rows for which column contains phrase. nphrase = match_info[P_O] # n ncol = match_info[C_O] total_docs = match_info[N_O] # N L_O = A_O + ncol X_O = L_O + ncol weights = get_weights(ncol, raw_weights) for iphrase in range(nphrase): for icol in range(ncol): weight = weights[icol] if weight == 0: continue x = X_O + (3 * (icol + iphrase * ncol)) term_frequency = match_info[x] # f(qi, D) docs_with_term = match_info[x + 2] # n(qi) # log( (N - n(qi) + 0.5) / (n(qi) + 0.5) ) idf = log( (total_docs - docs_with_term + 0.5) / (docs_with_term + 0.5)) if idf <= 0.0: idf = 1e-6 doc_length = match_info[L_O + icol] # |D| avg_length = match_info[A_O + icol] # avgdl if avg_length == 0: avg_length = 1 ratio = doc_length / avg_length num = term_frequency * (K + 1) b_part = 1 - B + (B * ratio) denom = term_frequency + (K * b_part) pc_score = idf * (num / denom) score += (pc_score * weight) free(weights) return -1 * score def peewee_bm25f(py_match_info, *raw_weights): # Usage: peewee_bm25f(matchinfo(table, 'pcnalx'), 1) # where the second parameter is the index of the column and # the 3rd and 4th specify k and b. cdef: unsigned int *match_info bytes _match_info_buf = bytes(py_match_info) char *match_info_buf = _match_info_buf int nphrase, ncol double B = 0.75, K = 1.2, epsilon double total_docs, term_frequency, docs_with_term double doc_length = 0.0, avg_length = 0.0 double idf, weight, ratio, num, b_part, denom, pc_score double *weights int P_O = 0, C_O = 1, N_O = 2, A_O = 3, L_O, X_O int iphrase, icol, x double score = 0.0 match_info = match_info_buf nphrase = match_info[P_O] # n ncol = match_info[C_O] total_docs = match_info[N_O] # N L_O = A_O + ncol X_O = L_O + ncol for icol in range(ncol): avg_length += match_info[A_O + icol] doc_length += match_info[L_O + icol] epsilon = 1.0 / (total_docs * avg_length) if avg_length == 0: avg_length = 1 ratio = doc_length / avg_length weights = get_weights(ncol, raw_weights) for iphrase in range(nphrase): for icol in range(ncol): weight = weights[icol] if weight == 0: continue x = X_O + (3 * (icol + iphrase * ncol)) term_frequency = match_info[x] # f(qi, D) docs_with_term = match_info[x + 2] # n(qi) # log( (N - n(qi) + 0.5) / (n(qi) + 0.5) ) idf = log( (total_docs - docs_with_term + 0.5) / (docs_with_term + 0.5)) idf = epsilon if idf <= 0 else idf num = term_frequency * (K + 1) b_part = 1 - B + (B * ratio) denom = term_frequency + (K * b_part) pc_score = idf * ((num / denom) + 1.) score += (pc_score * weight) free(weights) return -1 * score # String UDF. def damerau_levenshtein_dist(s1, s2): cdef: int i, j, del_cost, add_cost, sub_cost int s1_len = len(s1), s2_len = len(s2) list one_ago, two_ago, current_row list zeroes = [0] * (s2_len + 1) current_row = list(range(1, s2_len + 2)) current_row[-1] = 0 one_ago = None for i in range(s1_len): two_ago = one_ago one_ago = current_row current_row = list(zeroes) current_row[-1] = i + 1 for j in range(s2_len): del_cost = one_ago[j] + 1 add_cost = current_row[j - 1] + 1 sub_cost = one_ago[j - 1] + (s1[i] != s2[j]) current_row[j] = min(del_cost, add_cost, sub_cost) # Handle transpositions. if (i > 0 and j > 0 and s1[i] == s2[j - 1] and s1[i-1] == s2[j] and s1[i] != s2[j]): current_row[j] = min(current_row[j], two_ago[j - 2] + 1) return current_row[s2_len - 1] # String UDF. def levenshtein_dist(a, b): cdef: int add, delete, change int i, j int n = len(a), m = len(b) list current, previous list zeroes if n > m: a, b = b, a n, m = m, n zeroes = [0] * (m + 1) current = list(range(n + 1)) for i in range(1, m + 1): previous = current current = list(zeroes) current[0] = i for j in range(1, n + 1): add = previous[j] + 1 delete = current[j - 1] + 1 change = previous[j - 1] if a[j - 1] != b[i - 1]: change +=1 current[j] = min(add, delete, change) return current[n] # String UDF. def str_dist(a, b): cdef: int t = 0 for i in SequenceMatcher(None, a, b).get_opcodes(): if i[0] == 'equal': continue t = t + max(i[4] - i[3], i[2] - i[1]) return t # Math Aggregate. cdef class median(object): cdef: int ct list items def __init__(self): self.ct = 0 self.items = [] cdef selectKth(self, int k, int s=0, int e=-1): cdef: int idx if e < 0: e = len(self.items) idx = randint(s, e-1) idx = self.partition_k(idx, s, e) if idx > k: return self.selectKth(k, s, idx) elif idx < k: return self.selectKth(k, idx + 1, e) else: return self.items[idx] cdef int partition_k(self, int pi, int s, int e): cdef: int i, x val = self.items[pi] # Swap pivot w/last item. self.items[e - 1], self.items[pi] = self.items[pi], self.items[e - 1] x = s for i in range(s, e): if self.items[i] < val: self.items[i], self.items[x] = self.items[x], self.items[i] x += 1 self.items[x], self.items[e-1] = self.items[e-1], self.items[x] return x def step(self, item): self.items.append(item) self.ct += 1 def finalize(self): if self.ct == 0: return None elif self.ct < 3: return self.items[0] else: return self.selectKth(self.ct // 2) ================================================ FILE: playhouse/apsw_ext.py ================================================ """ Peewee integration with APSW, "another python sqlite wrapper". Project page: https://rogerbinns.github.io/apsw/ APSW is a really neat library that provides a thin wrapper on top of SQLite's C interface. Here are just a few reasons to use APSW, taken from the documentation: * APSW gives all functionality of SQLite, including virtual tables, virtual file system, blob i/o, backups and file control. * Connections can be shared across threads without any additional locking. * Transactions are managed explicitly by your code. * APSW can handle nested transactions. * Unicode is handled correctly. * APSW is faster. """ import apsw from peewee import * from peewee import __exception_wrapper__ from peewee import BooleanField as _BooleanField from peewee import DateField as _DateField from peewee import DateTimeField as _DateTimeField from peewee import DecimalField as _DecimalField from peewee import Insert from peewee import TimeField as _TimeField from peewee import logger class APSWDatabase(SqliteDatabase): server_version = tuple(int(i) for i in apsw.sqlitelibversion().split('.')) def __init__(self, database, **kwargs): self._modules = {} super(APSWDatabase, self).__init__(database, **kwargs) def register_module(self, mod_name, mod_inst): self._modules[mod_name] = mod_inst if not self.is_closed(): self.connection().createmodule(mod_name, mod_inst) def unregister_module(self, mod_name): del(self._modules[mod_name]) def _connect(self): conn = apsw.Connection(self.database, **self.connect_params) if self._timeout is not None: conn.setbusytimeout(self._timeout * 1000) try: self._add_conn_hooks(conn) except: conn.close() raise return conn def _add_conn_hooks(self, conn): super(APSWDatabase, self)._add_conn_hooks(conn) self._load_modules(conn) # APSW-only. def _load_modules(self, conn): for mod_name, mod_inst in self._modules.items(): conn.createmodule(mod_name, mod_inst) return conn def _load_aggregates(self, conn): for name, (klass, num_params) in self._aggregates.items(): def make_aggregate(): return (klass(), klass.step, klass.finalize) conn.createaggregatefunction(name, make_aggregate) def _load_collations(self, conn): for name, fn in self._collations.items(): conn.createcollation(name, fn) def _load_functions(self, conn): for name, (fn, num_params, deterministic) in self._functions.items(): args = (deterministic,) if deterministic else () conn.createscalarfunction(name, fn, num_params, *args) def _load_extensions(self, conn): conn.enableloadextension(True) for extension in self._extensions: conn.loadextension(extension) def load_extension(self, extension): self._extensions.add(extension) if not self.is_closed(): conn = self.connection() conn.enableloadextension(True) conn.loadextension(extension) def last_insert_id(self, cursor, query_type=None): if not self.returning_clause: return cursor.connection.last_insert_rowid() elif query_type == Insert.SIMPLE: try: return cursor[0][0] except (AttributeError, IndexError, TypeError): pass return cursor def rows_affected(self, cursor): try: return cursor.connection.changes() except AttributeError: return cursor.cursor.connection.changes() # RETURNING query. def begin(self, lock_type='deferred'): self.cursor().execute('begin %s;' % lock_type) def commit(self): with __exception_wrapper__: curs = self.cursor() if curs.connection.getautocommit(): return False curs.execute('commit;') return True def rollback(self): with __exception_wrapper__: curs = self.cursor() if curs.connection.getautocommit(): return False curs.execute('rollback;') return True def nh(s, v): if v is not None: return str(v) class BooleanField(_BooleanField): def db_value(self, v): v = super(BooleanField, self).db_value(v) if v is not None: return v and 1 or 0 class DateField(_DateField): db_value = nh class TimeField(_TimeField): db_value = nh class DateTimeField(_DateTimeField): db_value = nh class DecimalField(_DecimalField): db_value = nh ================================================ FILE: playhouse/cockroachdb.py ================================================ import functools import re import sys from peewee import * from peewee import _atomic from peewee import _manual from peewee import ColumnMetadata # (name, data_type, null, primary_key, table, default) from peewee import EnclosedNodeList from peewee import Entity from peewee import ForeignKeyMetadata # (column, dest_table, dest_column, table). from peewee import IndexMetadata from peewee import NodeList from playhouse.pool import _PooledPostgresqlDatabase try: from playhouse.postgres_ext import ArrayField from playhouse.postgres_ext import BinaryJSONField from playhouse.postgres_ext import IntervalField JSONField = BinaryJSONField except ImportError: # psycopg2 not installed, ignore. ArrayField = BinaryJSONField = IntervalField = JSONField = None NESTED_TX_MIN_VERSION = 200100 TXN_ERR_MSG = ('CockroachDB does not support nested transactions. You may ' 'alternatively use the @transaction context-manager/decorator, ' 'which only wraps the outer-most block in transactional logic. ' 'To run a transaction with automatic retries, use the ' 'run_transaction() helper.') class ExceededMaxAttempts(OperationalError): pass class UUIDKeyField(UUIDField): auto_increment = True def __init__(self, *args, **kwargs): if kwargs.get('constraints'): raise ValueError('%s cannot specify constraints.' % type(self)) kwargs['constraints'] = [SQL('DEFAULT gen_random_uuid()')] kwargs.setdefault('primary_key', True) super(UUIDKeyField, self).__init__(*args, **kwargs) class RowIDField(AutoField): field_type = 'INT' def __init__(self, *args, **kwargs): if kwargs.get('constraints'): raise ValueError('%s cannot specify constraints.' % type(self)) kwargs['constraints'] = [SQL('DEFAULT unique_rowid()')] super(RowIDField, self).__init__(*args, **kwargs) class CockroachDatabase(PostgresqlDatabase): field_types = PostgresqlDatabase.field_types.copy() field_types.update({ 'BLOB': 'BYTES', }) release_after_rollback = True def __init__(self, database, *args, **kwargs): # Unless a DSN or database connection-url were specified, provide # convenient defaults for the user and port. if 'dsn' not in kwargs and (database and not database.startswith('postgresql://')): kwargs.setdefault('user', 'root') kwargs.setdefault('port', 26257) super(CockroachDatabase, self).__init__(database, *args, **kwargs) def _set_server_version(self, conn): curs = conn.cursor() curs.execute('select version()') raw, = curs.fetchone() match_obj = re.match(r'^CockroachDB.+?v(\d+)\.(\d+)\.(\d+)', raw) if match_obj is not None: clean = '%d%02d%02d' % tuple(int(i) for i in match_obj.groups()) self.server_version = int(clean) # 19.1.5 -> 190105. else: # Fallback to use whatever cockroachdb tells us via protocol. super(CockroachDatabase, self)._set_server_version(conn) def _get_pk_constraint(self, table, schema=None): query = ('SELECT constraint_name ' 'FROM information_schema.table_constraints ' 'WHERE table_name = %s AND table_schema = %s ' 'AND constraint_type = %s') cursor = self.execute_sql(query, (table, schema or 'public', 'PRIMARY KEY')) row = cursor.fetchone() return row and row[0] or None def get_indexes(self, table, schema=None): # The primary-key index is returned by default, so we will just strip # it out here. indexes = super(CockroachDatabase, self).get_indexes(table, schema) pkc = self._get_pk_constraint(table, schema) return [idx for idx in indexes if (not pkc) or (idx.name != pkc)] def conflict_statement(self, on_conflict, query): if not on_conflict._action: return action = on_conflict._action.lower() if action in ('replace', 'upsert'): return SQL('UPSERT') elif action not in ('ignore', 'nothing', 'update'): raise ValueError('Un-supported action for conflict resolution. ' 'CockroachDB supports REPLACE (UPSERT), IGNORE ' 'and UPDATE.') def conflict_update(self, oc, query): action = oc._action.lower() if oc._action else '' if action in ('ignore', 'nothing'): parts = [SQL('ON CONFLICT')] if oc._conflict_target: parts.append(EnclosedNodeList([ Entity(col) if isinstance(col, str) else col for col in oc._conflict_target])) parts.append(SQL('DO NOTHING')) return NodeList(parts) elif action in ('replace', 'upsert'): # No special stuff is necessary, this is just indicated by starting # the statement with UPSERT instead of INSERT. return elif oc._conflict_constraint: raise ValueError('CockroachDB does not support the usage of a ' 'constraint name. Use the column(s) instead.') return super(CockroachDatabase, self).conflict_update(oc, query) def extract_date(self, date_part, date_field): return fn.extract(date_part, date_field) def from_timestamp(self, date_field): # CRDB does not allow casting a decimal/float to timestamp, so we first # cast to int, then to timestamptz. return date_field.cast('int').cast('timestamptz') def begin(self, system_time=None, priority=None): super(CockroachDatabase, self).begin() if system_time is not None: self.cursor().execute('SET TRANSACTION AS OF SYSTEM TIME %s', (system_time,)) if priority is not None: priority = priority.lower() if priority not in ('low', 'normal', 'high'): raise ValueError('priority must be low, normal or high') self.cursor().execute('SET TRANSACTION PRIORITY %s' % priority) def atomic(self, system_time=None, priority=None): if self.is_closed(): self.connect() # Side-effect, set server version. if self.server_version < NESTED_TX_MIN_VERSION: return _crdb_atomic(self, system_time, priority) return super(CockroachDatabase, self).atomic(system_time, priority) def savepoint(self): if self.is_closed(): self.connect() # Side-effect, set server version. if self.server_version < NESTED_TX_MIN_VERSION: raise NotImplementedError(TXN_ERR_MSG) return super(CockroachDatabase, self).savepoint() def retry_transaction(self, max_attempts=None, system_time=None, priority=None): def deco(cb): @functools.wraps(cb) def new_fn(): return run_transaction(self, cb, max_attempts, system_time, priority) return new_fn return deco def run_transaction(self, cb, max_attempts=None, system_time=None, priority=None): return run_transaction(self, cb, max_attempts, system_time, priority) class _crdb_atomic(_atomic): def __enter__(self): if self.db.transaction_depth() > 0: if not isinstance(self.db.top_transaction(), _manual): raise NotImplementedError(TXN_ERR_MSG) return super(_crdb_atomic, self).__enter__() def run_transaction(db, callback, max_attempts=None, system_time=None, priority=None): """ Run transactional SQL in a transaction with automatic retries. User-provided `callback`: * Must accept one parameter, the `db` instance representing the connection the transaction is running under. * Must not attempt to commit, rollback or otherwise manage transactions. * May be called more than once. * Should ideally only contain SQL operations. Additionally, the database must not have any open transaction at the time this function is called, as CRDB does not support nested transactions. """ max_attempts = max_attempts or -1 with db.atomic(system_time=system_time, priority=priority) as txn: db.execute_sql('SAVEPOINT cockroach_restart') while max_attempts != 0: try: result = callback(db) db.execute_sql('RELEASE SAVEPOINT cockroach_restart') return result except OperationalError as exc: if exc.orig.pgcode == '40001': max_attempts -= 1 db.execute_sql('ROLLBACK TO SAVEPOINT cockroach_restart') continue raise raise ExceededMaxAttempts(None, 'unable to commit transaction') class PooledCockroachDatabase(_PooledPostgresqlDatabase, CockroachDatabase): pass ================================================ FILE: playhouse/cysqlite_ext.py ================================================ import logging from pathlib import Path from peewee import DecimalField from peewee import ImproperlyConfigured from peewee import OP from peewee import SqliteDatabase from peewee import __exception_wrapper__ from playhouse.pool import _PooledSqliteDatabase from playhouse.sqlite_ext import ( RowIDField, DocIDField, AutoIncrementField, ISODateTimeField, JSONPath, JSONBPath, JSONField, JSONBField, SearchField, VirtualModel, FTSModel, FTS5Model) from playhouse.sqlite_udf import rank try: import cysqlite except ImportError as exc: raise ImportError('cysqlite is not installed') logger = logging.getLogger('peewee') def __status__(flag, return_highwater=False): def getter(self): result = cysqlite.status(flag) return result[1] if return_highwater else result return property(getter) def __dbstatus__(flag, return_highwater=False, return_current=False): """ Expose a sqlite3_dbstatus() call for a particular flag as a property of the Database instance. Unlike sqlite3_status(), the dbstatus properties pertain to the current connection. """ def getter(self): if self._state.conn is None: raise ImproperlyConfigured('database connection not opened.') result = self._state.conn.status(flag) if return_current: return result[0] return result[1] if return_highwater else result return property(getter) class TDecimalField(DecimalField): field_type = 'TEXT' def get_modifiers(self): pass def db_value(self, value): if value is not None: return str(super(DecimalField, self).db_value(value)) class CySqliteDatabase(SqliteDatabase): def __init__(self, database, rank_functions=True, *args, **kwargs): super(CySqliteDatabase, self).__init__(database, *args, **kwargs) self._table_functions = [] self._commit_hook = None self._rollback_hook = None self._update_hook = None self._authorizer = None self._trace = None self._progress = None if rank_functions: self.register_function(cysqlite.rank_bm25, 'fts_bm25') self.register_function(cysqlite.rank_lucene, 'fts_lucene') self.register_function(rank, 'fts_rank') def _connect(self): if cysqlite is None: raise ImproperlyConfigured('cysqlite is not installed.') conn = cysqlite.Connection(self.database, timeout=self._timeout, extensions=True, **self.connect_params) try: self._add_conn_hooks(conn) except Exception: conn.close() raise return conn def _add_conn_hooks(self, conn): if self._commit_hook is not None: conn.commit_hook(self._commit_hook) if self._rollback_hook is not None: conn.rollback_hook(self._rollback_hook) if self._update_hook is not None: conn.update_hook(self._update_hook) if self._authorizer is not None: conn.authorizer(self._authorizer) if self._trace is not None: conn.trace(*self._trace) if self._progress is not None: conn.progress(*self._progress) super(CySqliteDatabase, self)._add_conn_hooks(conn) if self._table_functions: for table_function in self._table_functions: table_function.register(conn) def _set_pragmas(self, conn): for pragma, value in self._pragmas: conn.pragma(pragma, value) def _attach_databases(self, conn): for name, db in self._attached.items(): conn.attach(db, name) def _load_aggregates(self, conn): for name, (klass, num_params) in self._aggregates.items(): conn.create_aggregate(klass, name, num_params) def _load_collations(self, conn): for name, fn in self._collations.items(): conn.create_collation(fn, name) def _load_functions(self, conn): for name, (fn, num_params, deterministic) in self._functions.items(): conn.create_function(fn, name, num_params, deterministic) def _load_window_functions(self, conn): for name, (klass, num_params) in self._window_functions.items(): conn.create_window_function(klass, name, num_params) def register_table_function(self, klass, name=None): if name is not None: klass.name = name self._table_functions.append(klass) if not self.is_closed(): klass.register(self.connection()) def unregister_table_function(self, name): for idx, klass in enumerate(self._table_functions): if klass.name == name: break else: return False self._table_functions.pop(idx) return True def table_function(self, name=None): def decorator(klass): self.register_table_function(klass, name) return klass return decorator def on_commit(self, fn): self._commit_hook = fn if not self.is_closed(): self.connection().commit_hook(fn) return fn def on_rollback(self, fn): self._rollback_hook = fn if not self.is_closed(): self.connection().rollback_hook(fn) return fn def on_update(self, fn): self._update_hook = fn if not self.is_closed(): self.connection().update_hook(fn) return fn def authorizer(self, fn): self._authorizer = fn if not self.is_closed(): self.connection().authorizer(fn) return fn def trace(self, fn, mask=2, expand_sql=True): if fn is None: self._trace = None else: self._trace = (fn, mask, expand_sql) if not self.is_closed(): args = (None,) if fn is None else self._trace self.connection().trace(*args) return fn def slow_query_log(self, threshold_ms=50, logger=None, level=logging.WARNING, expand_sql=True): log = logging.getLogger(logger or 'peewee.cysqlite_ext') def _trace(event, sid, sql, ns): if not sql: return ms = ns / 1000000 if ms >= threshold_ms: log.log(level, 'Slow query %0.1fms: %s', ms, sql) self.trace(_trace, cysqlite.SQLITE_TRACE_PROFILE, expand_sql=expand_sql) return True def progress(self, fn, n=1): if fn is None: self._progress = None else: self._progress = (fn, mask) if not self.is_closed(): args = (None,) if fn is None else self._progress self.connection().progress(*args) return fn def begin(self, lock_type='deferred'): with __exception_wrapper__: self.connection().begin(lock_type) def commit(self): with __exception_wrapper__: self.connection().commit() def rollback(self): with __exception_wrapper__: self.connection().rollback() @property def autocommit(self): return self.connection().autocommit() def blob_open(self, table, column, rowid, read_only=False, dbname=None): return self.connection().blob_open(table, column, rowid, read_only, db_name) def backup(self, destination, pages=None, name=None, progress=None, src_name=None): if isinstance(destination, CySqliteDatabase): conn = destination.connection() elif isinstance(destination, cysqlite.Connection): conn = destination elif isinstance(destination, (str, Path)): return self.backup_to_file(str(destination), pages, name, progress, src_name) return self.connection().backup(conn, pages, name, progress, src_name) def backup_to_file(self, filename, pages=None, name=None, progress=None, src_name=None): return self.connection().backup_to_file(filename, pages, name, progress, src_name) # Status properties. memory_used = __status__(cysqlite.SQLITE_STATUS_MEMORY_USED) malloc_size = __status__(cysqlite.SQLITE_STATUS_MALLOC_SIZE, True) malloc_count = __status__(cysqlite.SQLITE_STATUS_MALLOC_COUNT) pagecache_used = __status__(cysqlite.SQLITE_STATUS_PAGECACHE_USED) pagecache_overflow = __status__( cysqlite.SQLITE_STATUS_PAGECACHE_OVERFLOW) pagecache_size = __status__(cysqlite.SQLITE_STATUS_PAGECACHE_SIZE, True) scratch_used = __status__(cysqlite.SQLITE_STATUS_SCRATCH_USED) scratch_overflow = __status__(cysqlite.SQLITE_STATUS_SCRATCH_OVERFLOW) scratch_size = __status__(cysqlite.SQLITE_STATUS_SCRATCH_SIZE, True) # Connection status properties. lookaside_used = __dbstatus__(cysqlite.SQLITE_DBSTATUS_LOOKASIDE_USED) lookaside_hit = __dbstatus__( cysqlite.SQLITE_DBSTATUS_LOOKASIDE_HIT, True) lookaside_miss = __dbstatus__( cysqlite.SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, True) lookaside_miss_full = __dbstatus__( cysqlite.SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, True) cache_used = __dbstatus__( cysqlite.SQLITE_DBSTATUS_CACHE_USED, False, True) schema_used = __dbstatus__( cysqlite.SQLITE_DBSTATUS_SCHEMA_USED, False, True) statement_used = __dbstatus__( cysqlite.SQLITE_DBSTATUS_STMT_USED, False, True) cache_hit = __dbstatus__( cysqlite.SQLITE_DBSTATUS_CACHE_HIT, False, True) cache_miss = __dbstatus__( cysqlite.SQLITE_DBSTATUS_CACHE_MISS, False, True) cache_write = __dbstatus__( cysqlite.SQLITE_DBSTATUS_CACHE_WRITE, False, True) class PooledCySqliteDatabase(_PooledSqliteDatabase, CySqliteDatabase): pass OP.MATCH = 'MATCH' def _sqlite_regexp(regex, value): return re.search(regex, value) is not None ================================================ FILE: playhouse/dataset.py ================================================ import base64 import csv import datetime import json import operator import uuid from decimal import Decimal from functools import reduce from urllib.parse import urlparse from peewee import * from playhouse.db_url import connect from playhouse.migrate import migrate from playhouse.migrate import SchemaMigrator from playhouse.reflection import Introspector class DataSet(object): def __init__(self, url, include_views=False, **kwargs): if isinstance(url, Database): self._url = None self._database = url self._database_path = self._database.database else: self._url = url parse_result = urlparse(url) self._database_path = parse_result.path[1:] # Connect to the database. self._database = connect(url) # Open a connection if one does not already exist. self._database.connect(reuse_if_open=True) # Introspect the database and generate models. self._introspector = Introspector.from_database(self._database) self._include_views = include_views self._models = self._introspector.generate_models( skip_invalid=True, literal_column_names=True, include_views=self._include_views, **kwargs) self._migrator = SchemaMigrator.from_database(self._database) class BaseModel(Model): class Meta: database = self._database self._base_model = BaseModel self._export_formats = self.get_export_formats() self._import_formats = self.get_import_formats() def __repr__(self): return '' % self._database_path def get_export_formats(self): return { 'csv': CSVExporter, 'json': JSONExporter, 'tsv': TSVExporter} def get_import_formats(self): return { 'csv': CSVImporter, 'json': JSONImporter, 'tsv': TSVImporter} def __getitem__(self, table): if table not in self._models and table in self.tables: self.update_cache(table) return Table(self, table, self._models.get(table)) @property def tables(self): tables = self._database.get_tables() if self._include_views: tables += self.views return tables @property def views(self): return [v.name for v in self._database.get_views()] def __contains__(self, table): return table in self.tables def connect(self, reuse_if_open=False): self._database.connect(reuse_if_open=reuse_if_open) def close(self): self._database.close() def update_cache(self, table=None): if table: dependencies = [table] if table in self._models: model_class = self._models[table] dependencies.extend([ related._meta.table_name for _, related, _ in model_class._meta.model_graph()]) else: dependencies.extend(self.get_table_dependencies(table)) else: dependencies = None # Update all tables. self._models = {} updated = self._introspector.generate_models( skip_invalid=True, table_names=dependencies, literal_column_names=True, include_views=self._include_views) self._models.update(updated) def get_table_dependencies(self, table): stack = [table] accum = [] seen = set() while stack: table = stack.pop() for fk_meta in self._database.get_foreign_keys(table): dest = fk_meta.dest_table if dest not in seen: stack.append(dest) accum.append(dest) return accum def __enter__(self): self.connect() return self def __exit__(self, exc_type, exc_val, exc_tb): if not self._database.is_closed(): self.close() def query(self, sql, params=None): return self._database.execute_sql(sql, params) def transaction(self): return self._database.atomic() def _check_arguments(self, filename, file_obj, format, format_dict): if filename and file_obj: raise ValueError('file is over-specified. Please use either ' 'filename or file_obj, but not both.') if not filename and not file_obj: raise ValueError('A filename or file-like object must be ' 'specified.') if format not in format_dict: valid_formats = ', '.join(sorted(format_dict.keys())) raise ValueError('Unsupported format "%s". Use one of %s.' % ( format, valid_formats)) def freeze(self, query, format='csv', filename=None, file_obj=None, encoding='utf8', iso8601_datetimes=False, base64_bytes=False, **kwargs): self._check_arguments(filename, file_obj, format, self._export_formats) if filename: file_obj = open(filename, 'w', encoding=encoding) exporter = self._export_formats[format]( query, iso8601_datetimes=iso8601_datetimes, base64_bytes=base64_bytes) exporter.export(file_obj, **kwargs) if filename: file_obj.close() def thaw(self, table, format='csv', filename=None, file_obj=None, strict=False, encoding='utf8', iso8601_datetimes=False, base64_bytes=False, **kwargs): self._check_arguments(filename, file_obj, format, self._export_formats) if filename: file_obj = open(filename, 'r', encoding=encoding) importer = self._import_formats[format]( self[table], strict=strict, iso8601_datetimes=iso8601_datetimes, base64_bytes=base64_bytes) count = importer.load(file_obj, **kwargs) if filename: file_obj.close() return count class Table(object): def __init__(self, dataset, name, model_class): self.dataset = dataset self.name = name if model_class is None: model_class = self._create_model() model_class.create_table() self.dataset._models[name] = model_class @property def model_class(self): return self.dataset._models[self.name] def __repr__(self): return '' % self.name def __len__(self): return self.find().count() def __iter__(self): return iter(self.find().iterator()) def _create_model(self): class Meta: table_name = self.name return type( str(self.name), (self.dataset._base_model,), {'Meta': Meta}) def create_index(self, columns, unique=False): index = ModelIndex(self.model_class, columns, unique=unique) self.model_class.add_index(index) self.dataset._database.execute(index) def _guess_field_type(self, value): if isinstance(value, str): return TextField if isinstance(value, (datetime.date, datetime.datetime)): return DateTimeField elif value is True or value is False: return BooleanField elif isinstance(value, int): return IntegerField elif isinstance(value, float): return FloatField elif isinstance(value, Decimal): return DecimalField return TextField @property def columns(self): return [f.name for f in self.model_class._meta.sorted_fields] def _migrate_new_columns(self, data): new_keys = set(data) - set(self.model_class._meta.fields) new_keys -= set(self.model_class._meta.columns) if new_keys: operations = [] for key in new_keys: field_class = self._guess_field_type(data[key]) field = field_class(null=True) operations.append( self.dataset._migrator.add_column(self.name, key, field)) field.bind(self.model_class, key) migrate(*operations) self.dataset.update_cache(self.name) def __getitem__(self, item): try: return self.model_class[item] except self.model_class.DoesNotExist: pass def __setitem__(self, item, value): if not isinstance(value, dict): raise ValueError('Table.__setitem__() value must be a dict') pk = self.model_class._meta.primary_key value[pk.name] = item try: with self.dataset.transaction() as txn: self.insert(**value) except IntegrityError: self.dataset.update_cache(self.name) self.update(columns=[pk.name], **value) def __delitem__(self, item): del self.model_class[item] def insert(self, **data): self._migrate_new_columns(data) return self.model_class.insert(**data).execute() def _apply_where(self, query, filters, conjunction=None): conjunction = conjunction or operator.and_ if filters: expressions = [ (self.model_class._meta.fields[column] == value) for column, value in filters.items()] query = query.where(reduce(conjunction, expressions)) return query def update(self, columns=None, conjunction=None, **data): self._migrate_new_columns(data) filters = {} if columns: for column in columns: filters[column] = data.pop(column) return self._apply_where( self.model_class.update(**data), filters, conjunction).execute() def _query(self, **query): return self._apply_where(self.model_class.select(), query) def find(self, **query): return self._query(**query).dicts() def find_one(self, **query): try: return self.find(**query).get() except self.model_class.DoesNotExist: return None def all(self): return self.find() def delete(self, **query): return self._apply_where(self.model_class.delete(), query).execute() def freeze(self, *args, **kwargs): return self.dataset.freeze(self.all(), *args, **kwargs) def thaw(self, *args, **kwargs): return self.dataset.thaw(self.name, *args, **kwargs) class Exporter(object): def __init__(self, query, iso8601_datetimes=False, base64_bytes=False): self.query = query self.iso8601_datetimes = iso8601_datetimes self.base64_bytes = base64_bytes def export(self, file_obj): raise NotImplementedError _datetime_types = (datetime.datetime, datetime.date, datetime.time) class JSONExporter(Exporter): def _make_default(self): def default(o): if isinstance(o, _datetime_types): if self.iso8601_datetimes: return o.isoformat() else: return str(o) elif isinstance(o, (Decimal, uuid.UUID)): return str(o) elif isinstance(o, bytes): if self.base64_bytes: return base64.urlsafe_b64encode(o).decode('utf8') else: return o.hex() raise TypeError('Unable to serialize %r as JSON' % o) return default def export(self, file_obj, **kwargs): json.dump( list(self.query), file_obj, default=self._make_default(), **kwargs) class CSVExporter(Exporter): def export(self, file_obj, header=True, **kwargs): writer = csv.writer(file_obj, **kwargs) tuples = self.query.tuples().execute() tuples.initialize() if header and getattr(tuples, 'columns', None): writer.writerow([column for column in tuples.columns]) for row in tuples: accum = [] for value in row: if isinstance(value, _datetime_types): if self.iso8601_datetimes: value = value.isoformat() else: value = str(value) elif isinstance(value, (Decimal, uuid.UUID)): value = str(value) elif isinstance(value, bytes): if self.base64_bytes: value = base64.urlsafe_b64encode(value).decode('utf8') else: value = value.hex() accum.append(value) writer.writerow(accum) class TSVExporter(CSVExporter): def export(self, file_obj, header=True, **kwargs): kwargs.setdefault('delimiter', '\t') return super(TSVExporter, self).export(file_obj, header, **kwargs) class Importer(object): def __init__(self, table, strict=False, iso8601_datetimes=False, base64_bytes=False): self.table = table self.strict = strict self.iso8601_datetimes = iso8601_datetimes self.base64_bytes = base64_bytes model = self.table.model_class self.columns = model._meta.columns self.columns.update(model._meta.fields) def load(self, file_obj): raise NotImplementedError class JSONImporter(Importer): def load(self, file_obj, **kwargs): data = json.load(file_obj, **kwargs) count = 0 for row in data: obj = {} for key in row: field = self.columns.get(key) value = row[key] if isinstance(field, DateTimeField) and self.iso8601_datetimes: value = datetime.datetime.fromisoformat(value) elif isinstance(field, DateField) and self.iso8601_datetimes: value = datetime.date.fromisoformat(value) elif isinstance(field, BlobField): if self.base64_bytes: value = base64.urlsafe_b64decode(value.encode('utf8')) else: value = bytes.fromhex(value) if field is not None: value = field.python_value(value) obj[key] = value elif not self.strict: obj[key] = value if obj: self.table.insert(**obj) count += 1 return count class CSVImporter(Importer): def load(self, file_obj, header=True, **kwargs): count = 0 reader = csv.reader(file_obj, **kwargs) header_fields = [] if header: try: header_keys = next(reader) except StopIteration: return count for idx, key in enumerate(header_keys): if key in self.columns or not self.strict: header_fields.append((idx, key, self.columns.get(key))) else: for idx, field in enumerate(self.model._meta.sorted_fields): header_fields.append((idx, field.name, field)) if not header_fields: return count for row in reader: obj = {} for idx, name, field in header_fields: value = row[idx] if field is None: obj[name] = value continue if isinstance(field, DateTimeField) and self.iso8601_datetimes: value = datetime.datetime.fromisoformat(value) elif isinstance(field, DateField) and self.iso8601_datetimes: value = datetime.date.fromisoformat(value) elif isinstance(field, BlobField): if self.base64_bytes: value = base64.urlsafe_b64decode(value.encode('utf8')) else: value = bytes.fromhex(value) obj[field.name] = field.python_value(value) self.table.insert(**obj) count += 1 return count class TSVImporter(CSVImporter): def load(self, file_obj, header=True, **kwargs): kwargs.setdefault('delimiter', '\t') return super(TSVImporter, self).load(file_obj, header, **kwargs) ================================================ FILE: playhouse/db_url.py ================================================ from urllib.parse import parse_qsl, unquote, urlparse from peewee import * from playhouse.pool import PooledMySQLDatabase from playhouse.pool import PooledPostgresqlDatabase from playhouse.pool import PooledSqliteDatabase schemes = { 'mysql': MySQLDatabase, 'mysql+pool': PooledMySQLDatabase, 'postgres': PostgresqlDatabase, 'postgresql': PostgresqlDatabase, 'postgres+pool': PooledPostgresqlDatabase, 'postgresql+pool': PooledPostgresqlDatabase, 'sqlite': SqliteDatabase, 'sqlite+pool': PooledSqliteDatabase, } def register_database(db_class, *names): global schemes for name in names: schemes[name] = db_class def parseresult_to_dict(parsed, unquote_password=False, unquote_user=False): # urlparse in python 2.6 is broken so query will be empty and instead # appended to path complete with '?' path = parsed.path[1:] # Ignore leading '/'. query = parsed.query connect_kwargs = {'database': path} if parsed.username: connect_kwargs['user'] = parsed.username if unquote_user: connect_kwargs['user'] = unquote(connect_kwargs['user']) if parsed.password: connect_kwargs['password'] = parsed.password if unquote_password: connect_kwargs['password'] = unquote(connect_kwargs['password']) if parsed.hostname: connect_kwargs['host'] = parsed.hostname if parsed.port: connect_kwargs['port'] = parsed.port # Adjust parameters for MySQL. if parsed.scheme == 'mysql' and 'password' in connect_kwargs: connect_kwargs['passwd'] = connect_kwargs.pop('password') elif 'sqlite' in parsed.scheme and not connect_kwargs['database']: connect_kwargs['database'] = ':memory:' # Get additional connection args from the query string qs_args = parse_qsl(query, keep_blank_values=True) for key, value in qs_args: if value.lower() == 'false': value = False elif value.lower() == 'true': value = True elif value.isdigit(): value = int(value) elif '.' in value and all(p.isdigit() for p in value.split('.', 1)): try: value = float(value) except ValueError: pass elif value.lower() in ('null', 'none'): value = None connect_kwargs[key] = value return connect_kwargs def parse(url, unquote_password=False, unquote_user=False): parsed = urlparse(url) return parseresult_to_dict(parsed, unquote_password, unquote_user) def connect(url, unquote_password=False, unquote_user=False, **connect_params): parsed = urlparse(url) connect_kwargs = parseresult_to_dict(parsed, unquote_password, unquote_user) connect_kwargs.update(connect_params) database_class = schemes.get(parsed.scheme) if database_class is None: if database_class in schemes: raise RuntimeError('Attempted to use "%s" but a required library ' 'could not be imported.' % parsed.scheme) else: raise RuntimeError('Unrecognized or unsupported scheme: "%s".' % parsed.scheme) return database_class(**connect_kwargs) # Conditionally register additional databases. try: from playhouse.apsw_ext import APSWDatabase register_database(APSWDatabase, 'apsw') except ImportError: pass try: from playhouse.cockroachdb import CockroachDatabase from playhouse.cockroachdb import PooledCockroachDatabase register_database(CockroachDatabase, 'cockroachdb', 'crdb') register_database(PooledCockroachDatabase, 'cockroachdb+pool', 'crdb+pool') except ImportError: pass try: from playhouse.cysqlite_ext import CySqliteDatabase from playhouse.cysqlite_ext import PooledCySqliteDatabase register_database(CySqliteDatabase, 'cysqlite') register_database(PooledCySqliteDatabase, 'cysqlite+pool') except ImportError: pass try: from playhouse.mysql_ext import MariaDBConnectorDatabase from playhouse.mysql_ext import MySQLConnectorDatabase from playhouse.mysql_ext import PooledMariaDBConnectorDatabase from playhouse.mysql_ext import PooledMySQLConnectorDatabase register_database(MariaDBConnectorDatabase, 'mariadbconnector') register_database(MySQLConnectorDatabase, 'mysqlconnector') register_database(PooledMariaDBConnectorDatabase, 'mariadbconnector+pool') register_database(PooledMySQLConnectorDatabase, 'mysqlconnector+pool') except ImportError: pass try: from playhouse.postgres_ext import PooledPostgresqlExtDatabase from playhouse.postgres_ext import PooledPsycopg3Database from playhouse.postgres_ext import PostgresqlExtDatabase from playhouse.postgres_ext import Psycopg3Database register_database( PooledPostgresqlExtDatabase, 'postgresext+pool', 'postgresqlext+pool') register_database( PostgresqlExtDatabase, 'postgresext', 'postgresqlext') register_database(PooledPsycopg3Database, 'psycopg3+pool') register_database(Psycopg3Database, 'psycopg3') except ImportError: pass ================================================ FILE: playhouse/fields.py ================================================ import pickle try: import bz2 except ImportError: bz2 = None try: import zlib except ImportError: zlib = None from peewee import BlobField class CompressedField(BlobField): ZLIB = 'zlib' BZ2 = 'bz2' algorithm_to_import = { ZLIB: zlib, BZ2: bz2, } def __init__(self, compression_level=6, algorithm=ZLIB, *args, **kwargs): self.compression_level = compression_level if algorithm not in self.algorithm_to_import: raise ValueError('Unrecognized algorithm %s' % algorithm) compress_module = self.algorithm_to_import[algorithm] if compress_module is None: raise ValueError('Missing library required for %s.' % algorithm) self.algorithm = algorithm self.compress = compress_module.compress self.decompress = compress_module.decompress super(CompressedField, self).__init__(*args, **kwargs) def python_value(self, value): if value is not None: return self.decompress(value) def db_value(self, value): if value is not None: return self._constructor( self.compress(value, self.compression_level)) class PickleField(BlobField): def python_value(self, value): if value is not None: return pickle.loads(value) def db_value(self, value): if value is not None: pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL) return self._constructor(pickled) ================================================ FILE: playhouse/flask_utils.py ================================================ import math import sys from flask import abort from flask import render_template from flask import request from peewee import Database from peewee import DoesNotExist from peewee import Model from peewee import Proxy from peewee import SelectQuery from playhouse.db_url import connect as db_url_connect class PaginatedQuery(object): def __init__(self, query_or_model, paginate_by, page_var='page', page=None, check_bounds=False): self.paginate_by = paginate_by self.page_var = page_var self.page = page or None self.check_bounds = check_bounds if isinstance(query_or_model, SelectQuery): self.query = query_or_model self.model = self.query.model else: self.model = query_or_model self.query = self.model.select() def get_page(self): if self.page is not None: return self.page curr_page = request.args.get(self.page_var) if curr_page and curr_page.isdigit(): return max(1, int(curr_page)) return 1 def get_page_count(self): if not hasattr(self, '_page_count'): self._page_count = int(math.ceil( float(self.query.count()) / self.paginate_by)) return self._page_count def get_object_list(self): if self.check_bounds and self.get_page() > self.get_page_count(): abort(404) return self.query.paginate(self.get_page(), self.paginate_by) def get_page_range(self, page, total, show=5): # Generate page buttons for a subset of pages, e.g. if the current page # is 4, we have 10 pages, and want to show 5 buttons, this function # returns us: [2, 3, 4, 5, 6] start = max((page - (show // 2)), 1) stop = min(start + show, total) + 1 start = max(min(start, stop - show), 1) return list(range(start, stop)[:show]) def get_object_or_404(query_or_model, *query): if not isinstance(query_or_model, SelectQuery): query_or_model = query_or_model.select() try: return query_or_model.where(*query).get() except DoesNotExist: abort(404) def object_list(template_name, query, context_variable='object_list', paginate_by=20, page_var='page', page=None, check_bounds=True, **kwargs): paginated_query = PaginatedQuery( query, paginate_by=paginate_by, page_var=page_var, page=page, check_bounds=check_bounds) kwargs[context_variable] = paginated_query.get_object_list() return render_template( template_name, pagination=paginated_query, page=paginated_query.get_page(), **kwargs) def get_current_url(): if not request.query_string: return request.path return '%s?%s' % (request.path, request.query_string) def get_next_url(default='/'): if request.args.get('next'): return request.args['next'] elif request.form.get('next'): return request.form['next'] return default class FlaskDB(object): """ Convenience wrapper for configuring a Peewee database for use with a Flask application. Provides a base `Model` class and registers handlers to manage the database connection during the request/response cycle. Usage:: from flask import Flask from peewee import * from playhouse.flask_utils import FlaskDB # The database can be specified using a database URL, or you can pass a # Peewee database instance directly: DATABASE = 'postgresql:///my_app' DATABASE = PostgresqlDatabase('my_app') # If we do not want connection-management on any views, we can specify # the view names using FLASKDB_EXCLUDED_ROUTES. The db connection will # not be opened/closed automatically when these views are requested: FLASKDB_EXCLUDED_ROUTES = ('logout',) app = Flask(__name__) app.config.from_object(__name__) # Now we can configure our FlaskDB: flask_db = FlaskDB(app) # Or use the "deferred initialization" pattern: flask_db = FlaskDB() flask_db.init_app(app) # The `flask_db` provides a base Model-class for easily binding models # to the configured database: class User(flask_db.Model): email = CharField() """ def __init__(self, app=None, database=None, model_class=Model, excluded_routes=None): self.database = None # Reference to actual Peewee database instance. self.base_model_class = model_class self._app = app self._db = database # dict, url, Database, or None (default). self._excluded_routes = excluded_routes or () if app is not None: self.init_app(app) def init_app(self, app): self._app = app if self._db is None: if 'DATABASE' in app.config: initial_db = app.config['DATABASE'] elif 'DATABASE_URL' in app.config: initial_db = app.config['DATABASE_URL'] else: raise ValueError('Missing required configuration data for ' 'database: DATABASE or DATABASE_URL.') else: initial_db = self._db if 'FLASKDB_EXCLUDED_ROUTES' in app.config: self._excluded_routes = app.config['FLASKDB_EXCLUDED_ROUTES'] self._load_database(app, initial_db) self._register_handlers(app) def _load_database(self, app, config_value): if isinstance(config_value, Database): database = config_value elif isinstance(config_value, dict): database = self._load_from_config_dict(dict(config_value)) else: # Assume a database connection URL. database = db_url_connect(config_value) if isinstance(self.database, Proxy): self.database.initialize(database) else: self.database = database def _load_from_config_dict(self, config_dict): try: name = config_dict.pop('name') engine = config_dict.pop('engine') except KeyError: raise RuntimeError('DATABASE configuration must specify a ' '`name` and `engine`.') if '.' in engine: path, class_name = engine.rsplit('.', 1) else: path, class_name = 'peewee', engine try: __import__(path) module = sys.modules[path] database_class = getattr(module, class_name) assert issubclass(database_class, Database) except ImportError: raise RuntimeError('Unable to import %s' % engine) except AttributeError: raise RuntimeError('Database engine not found %s' % engine) except AssertionError: raise RuntimeError('Database engine not a subclass of ' 'peewee.Database: %s' % engine) return database_class(name, **config_dict) def _register_handlers(self, app): app.before_request(self.connect_db) app.teardown_request(self.close_db) def get_model_class(self): if self.database is None: raise RuntimeError('Database must be initialized.') class BaseModel(self.base_model_class): class Meta: database = self.database return BaseModel @property def Model(self): if self._app is None: database = getattr(self, 'database', None) if database is None: self.database = Proxy() if not hasattr(self, '_model_class'): self._model_class = self.get_model_class() return self._model_class def connect_db(self): if self._excluded_routes and request.endpoint in self._excluded_routes: return self.database.connect() def close_db(self, exc): if self._excluded_routes and request.endpoint in self._excluded_routes: return if not self.database.is_closed(): self.database.close() ================================================ FILE: playhouse/hybrid.py ================================================ from peewee import ModelDescriptor # Hybrid methods/attributes, based on similar functionality in SQLAlchemy: # http://docs.sqlalchemy.org/en/improve_toc/orm/extensions/hybrid.html class hybrid_method(ModelDescriptor): def __init__(self, func, expr=None): self.func = func self.expr = expr or func def __get__(self, instance, instance_type): if instance is None: return self.expr.__get__(instance_type, instance_type.__class__) return self.func.__get__(instance, instance_type) def expression(self, expr): self.expr = expr return self class hybrid_property(ModelDescriptor): def __init__(self, fget, fset=None, fdel=None, expr=None): self.fget = fget self.fset = fset self.fdel = fdel self.expr = expr or fget def __get__(self, instance, instance_type): if instance is None: return self.expr(instance_type) return self.fget(instance) def __set__(self, instance, value): if self.fset is None: raise AttributeError('Cannot set attribute.') self.fset(instance, value) def __delete__(self, instance): if self.fdel is None: raise AttributeError('Cannot delete attribute.') self.fdel(instance) def setter(self, fset): self.fset = fset return self def deleter(self, fdel): self.fdel = fdel return self def expression(self, expr): self.expr = expr return self ================================================ FILE: playhouse/kv.py ================================================ import operator from peewee import * from peewee import sqlite3 from peewee import Expression from playhouse.fields import PickleField try: from playhouse.cysqlite_ext import CySqliteDatabase as _SqliteDatabase except ImportError: _SqliteDatabase = SqliteDatabase Sentinel = type('Sentinel', (object,), {}) class KeyValue(object): """ Persistent dictionary. :param Field key_field: field to use for key. Defaults to CharField. :param Field value_field: field to use for value. Defaults to PickleField. :param bool ordered: data should be returned in key-sorted order. :param Database database: database where key/value data is stored. :param str table_name: table name for data. """ def __init__(self, key_field=None, value_field=None, ordered=False, database=None, table_name='keyvalue'): if key_field is None: key_field = CharField(max_length=255, primary_key=True) if not key_field.primary_key: raise ValueError('key_field must have primary_key=True.') if value_field is None: value_field = PickleField() self._key_field = key_field self._value_field = value_field self._ordered = ordered self._database = database or _SqliteDatabase(':memory:') self._table_name = table_name support_on_conflict = (isinstance(self._database, PostgresqlDatabase) or (isinstance(self._database, SqliteDatabase) and self._database.server_version >= (3, 24))) if support_on_conflict: self.upsert = self._postgres_upsert self.update = self._postgres_update else: self.upsert = self._upsert self.update = self._update self.model = self.create_model() self.key = self.model.key self.value = self.model.value # Ensure table exists. self.model.create_table() def create_model(self): class KeyValue(Model): key = self._key_field value = self._value_field class Meta: database = self._database table_name = self._table_name return KeyValue def query(self, *select): query = self.model.select(*select).tuples() if self._ordered: query = query.order_by(self.key) return query def convert_expression(self, expr): if not isinstance(expr, Expression): return (self.key == expr), True return expr, False def __contains__(self, key): expr, _ = self.convert_expression(key) return self.model.select().where(expr).exists() def __len__(self): return len(self.model) def __getitem__(self, expr): converted, is_single = self.convert_expression(expr) query = self.query(self.value).where(converted) item_getter = operator.itemgetter(0) result = [item_getter(row) for row in query] if len(result) == 0 and is_single: raise KeyError(expr) elif is_single: return result[0] return result def _upsert(self, key, value): (self.model .insert(key=key, value=value) .on_conflict('replace') .execute()) def _postgres_upsert(self, key, value): (self.model .insert(key=key, value=value) .on_conflict(conflict_target=[self.key], preserve=[self.value]) .execute()) def __setitem__(self, expr, value): if isinstance(expr, Expression): self.model.update(value=value).where(expr).execute() else: self.upsert(expr, value) def __delitem__(self, expr): converted, _ = self.convert_expression(expr) self.model.delete().where(converted).execute() def __iter__(self): return iter(self.query().execute()) def keys(self): return map(operator.itemgetter(0), self.query(self.key)) def values(self): return map(operator.itemgetter(0), self.query(self.value)) def items(self): return iter(self.query().execute()) def _update(self, __data=None, **mapping): if __data is not None: mapping.update(__data) return (self.model .insert_many(list(mapping.items()), fields=[self.key, self.value]) .on_conflict('replace') .execute()) def _postgres_update(self, __data=None, **mapping): if __data is not None: mapping.update(__data) return (self.model .insert_many(list(mapping.items()), fields=[self.key, self.value]) .on_conflict(conflict_target=[self.key], preserve=[self.value]) .execute()) def get(self, key, default=None): try: return self[key] except KeyError: return default def setdefault(self, key, default=None): try: return self[key] except KeyError: self[key] = default return default def pop(self, key, default=Sentinel): with self._database.atomic(): try: result = self[key] except KeyError: if default is Sentinel: raise return default del self[key] return result def clear(self): self.model.delete().execute() ================================================ FILE: playhouse/migrate.py ================================================ """ Lightweight schema migrations. Example Usage ------------- Instantiate a migrator: # Postgres example: my_db = PostgresqlDatabase(...) migrator = PostgresqlMigrator(my_db) # SQLite example: my_db = SqliteDatabase('my_database.db') migrator = SqliteMigrator(my_db) Then you will use the `migrate` function to run various `Operation`s which are generated by the migrator: migrate( migrator.add_column('some_table', 'column_name', CharField(default='')) ) Migrations are not run inside a transaction, so if you wish the migration to run in a transaction you will need to wrap the call to `migrate` in a transaction block, e.g.: with my_db.transaction(): migrate(...) Supported Operations -------------------- Add new field(s) to an existing model: # Create your field instances. For non-null fields you must specify a # default value. pubdate_field = DateTimeField(null=True) comment_field = TextField(default='') # Run the migration, specifying the database table, field name and field. migrate( migrator.add_column('comment_tbl', 'pub_date', pubdate_field), migrator.add_column('comment_tbl', 'comment', comment_field), ) Renaming a field: # Specify the table, original name of the column, and its new name. migrate( migrator.rename_column('story', 'pub_date', 'publish_date'), migrator.rename_column('story', 'mod_date', 'modified_date'), ) Dropping a field: migrate( migrator.drop_column('story', 'some_old_field'), ) Making a field nullable or not nullable: # Note that when making a field not null that field must not have any # NULL values present. migrate( # Make `pub_date` allow NULL values. migrator.drop_not_null('story', 'pub_date'), # Prevent `modified_date` from containing NULL values. migrator.add_not_null('story', 'modified_date'), ) Renaming a table: migrate( migrator.rename_table('story', 'stories_tbl'), ) Adding an index: # Specify the table, column names, and whether the index should be # UNIQUE or not. migrate( # Create an index on the `pub_date` column. migrator.add_index('story', ('pub_date',), False), # Create a multi-column index on the `pub_date` and `status` fields. migrator.add_index('story', ('pub_date', 'status'), False), # Create a unique index on the category and title fields. migrator.add_index('story', ('category_id', 'title'), True), ) Dropping an index: # Specify the index name. migrate(migrator.drop_index('story', 'story_pub_date_status')) Adding or dropping table constraints: .. code-block:: python # Add a CHECK() constraint to enforce the price cannot be negative. migrate(migrator.add_constraint( 'products', 'price_check', Check('price >= 0'))) # Remove the price check constraint. migrate(migrator.drop_constraint('products', 'price_check')) # Add a UNIQUE constraint on the first and last names. migrate(migrator.add_unique('person', 'first_name', 'last_name')) """ from collections import namedtuple import functools import hashlib import re from peewee import * from peewee import CommaNodeList from peewee import EnclosedNodeList from peewee import Entity from peewee import Expression from peewee import Node from peewee import NodeList from peewee import OP from peewee import callable_ from peewee import sort_models from peewee import sqlite3 from peewee import _truncate_constraint_name try: from playhouse.cockroachdb import CockroachDatabase except ImportError: CockroachDatabase = None class Operation(object): """Encapsulate a single schema altering operation.""" def __init__(self, migrator, method, *args, **kwargs): self.migrator = migrator self.method = method self.args = args self.kwargs = kwargs def execute(self, node): self.migrator.database.execute(node) def _handle_result(self, result): if isinstance(result, (Node, Context)): self.execute(result) elif isinstance(result, Operation): result.run() elif isinstance(result, (list, tuple)): for item in result: self._handle_result(item) def run(self): kwargs = self.kwargs.copy() kwargs['with_context'] = True method = getattr(self.migrator, self.method) self._handle_result(method(*self.args, **kwargs)) def operation(fn): @functools.wraps(fn) def inner(self, *args, **kwargs): with_context = kwargs.pop('with_context', False) if with_context: return fn(self, *args, **kwargs) return Operation(self, fn.__name__, *args, **kwargs) return inner def make_index_name(table_name, columns): index_name = '_'.join((table_name,) + tuple(columns)) if len(index_name) > 64: index_hash = hashlib.md5(index_name.encode('utf-8')).hexdigest() index_name = '%s_%s' % (index_name[:51], index_hash[:12]) return index_name class SchemaMigrator(object): explicit_create_foreign_key = False explicit_delete_foreign_key = False def __init__(self, database): self.database = database def make_context(self): return self.database.get_sql_context() @classmethod def from_database(cls, database): if CockroachDatabase and isinstance(database, CockroachDatabase): return CockroachDBMigrator(database) elif isinstance(database, PostgresqlDatabase): return PostgresqlMigrator(database) elif isinstance(database, MySQLDatabase): return MySQLMigrator(database) elif isinstance(database, SqliteDatabase): return SqliteMigrator(database) raise ValueError('Unsupported database: %s' % database) @operation def apply_default(self, table, column_name, field): default = field.default if callable_(default): default = default() return (self.make_context() .literal('UPDATE ') .sql(Entity(table)) .literal(' SET ') .sql(Expression( Entity(column_name), OP.EQ, field.db_value(default), flat=True))) def _alter_table(self, ctx, table): return ctx.literal('ALTER TABLE ').sql(Entity(table)) def _alter_column(self, ctx, table, column): return (self ._alter_table(ctx, table) .literal(' ALTER COLUMN ') .sql(Entity(column))) @operation def alter_add_column(self, table, column_name, field): # Make field null at first. ctx = self.make_context() field_null, field.null = field.null, True # Set the field's column-name and name, if it is not set or doesn't # match the new value. if field.column_name != column_name: field.name = field.column_name = column_name (self ._alter_table(ctx, table) .literal(' ADD COLUMN ') .sql(field.ddl(ctx))) field.null = field_null if isinstance(field, ForeignKeyField): self.add_inline_fk_sql(ctx, field) return ctx @operation def add_constraint(self, table, name, constraint): return (self ._alter_table(self.make_context(), table) .literal(' ADD CONSTRAINT ') .sql(Entity(name)) .literal(' ') .sql(constraint)) @operation def add_unique(self, table, *column_names): constraint_name = 'uniq_%s' % '_'.join(column_names) constraint = NodeList(( SQL('UNIQUE'), EnclosedNodeList([Entity(column) for column in column_names]))) return self.add_constraint(table, constraint_name, constraint) @operation def drop_constraint(self, table, name): return (self ._alter_table(self.make_context(), table) .literal(' DROP CONSTRAINT ') .sql(Entity(name))) def add_inline_fk_sql(self, ctx, field): ctx = (ctx .literal(' REFERENCES ') .sql(Entity(field.rel_model._meta.table_name)) .literal(' ') .sql(EnclosedNodeList((Entity(field.rel_field.column_name),)))) if field.on_delete is not None: ctx = ctx.literal(' ON DELETE %s' % field.on_delete) if field.on_update is not None: ctx = ctx.literal(' ON UPDATE %s' % field.on_update) return ctx @operation def add_foreign_key_constraint(self, table, column_name, rel, rel_column, on_delete=None, on_update=None, constraint_name=None): constraint = constraint_name or 'fk_%s_%s_refs_%s' % (table, column_name, rel) ctx = (self .make_context() .literal('ALTER TABLE ') .sql(Entity(table)) .literal(' ADD CONSTRAINT ') .sql(Entity(_truncate_constraint_name(constraint))) .literal(' FOREIGN KEY ') .sql(EnclosedNodeList((Entity(column_name),))) .literal(' REFERENCES ') .sql(Entity(rel)) .literal(' (') .sql(Entity(rel_column)) .literal(')')) if on_delete is not None: ctx = ctx.literal(' ON DELETE %s' % on_delete) if on_update is not None: ctx = ctx.literal(' ON UPDATE %s' % on_update) return ctx @operation def add_column(self, table, column_name, field): # Adding a column is complicated by the fact that if there are rows # present and the field is non-null, then we need to first add the # column as a nullable field, then set the value, then add a not null # constraint. if not field.null and field.default is None: raise ValueError('%s is not null but has no default' % column_name) is_foreign_key = isinstance(field, ForeignKeyField) if is_foreign_key and not field.rel_field: raise ValueError('Foreign keys must specify a `field`.') operations = [self.alter_add_column(table, column_name, field)] # In the event the field is *not* nullable, update with the default # value and set not null. if not field.null: operations.extend([ self.apply_default(table, column_name, field), self.add_not_null(table, column_name)]) if is_foreign_key and self.explicit_create_foreign_key: operations.append( self.add_foreign_key_constraint( table, column_name, field.rel_model._meta.table_name, field.rel_field.column_name, field.on_delete, field.on_update)) if field.index or field.unique: using = getattr(field, 'index_type', None) operations.append(self.add_index(table, (column_name,), field.unique, using)) return operations @operation def drop_foreign_key_constraint(self, table, column_name): raise NotImplementedError @operation def drop_column(self, table, column_name, cascade=True): ctx = self.make_context() (self._alter_table(ctx, table) .literal(' DROP COLUMN ') .sql(Entity(column_name))) if cascade: ctx.literal(' CASCADE') fk_columns = [ foreign_key.column for foreign_key in self.database.get_foreign_keys(table)] if column_name in fk_columns and self.explicit_delete_foreign_key: return [self.drop_foreign_key_constraint(table, column_name), ctx] return ctx @operation def rename_column(self, table, old_name, new_name): return (self ._alter_table(self.make_context(), table) .literal(' RENAME COLUMN ') .sql(Entity(old_name)) .literal(' TO ') .sql(Entity(new_name))) @operation def add_not_null(self, table, column): return (self ._alter_column(self.make_context(), table, column) .literal(' SET NOT NULL')) @operation def drop_not_null(self, table, column): return (self ._alter_column(self.make_context(), table, column) .literal(' DROP NOT NULL')) @operation def add_column_default(self, table, column, default): if default is None: raise ValueError('`default` must be not None/NULL.') if callable_(default): default = default() # Try to handle SQL functions and string literals, otherwise pass as a # bound value. if isinstance(default, str) and default.endswith((')', "'")): default = SQL(default) return (self ._alter_table(self.make_context(), table) .literal(' ALTER COLUMN ') .sql(Entity(column)) .literal(' SET DEFAULT ') .sql(default)) @operation def drop_column_default(self, table, column): return (self ._alter_table(self.make_context(), table) .literal(' ALTER COLUMN ') .sql(Entity(column)) .literal(' DROP DEFAULT')) @operation def alter_column_type(self, table, column, field, cast=None): # ALTER TABLE
    ALTER COLUMN ctx = self.make_context() ctx = (self ._alter_column(ctx, table, column) .literal(' TYPE ') .sql(field.ddl_datatype(ctx))) if cast is not None: if not isinstance(cast, Node): cast = SQL(cast) ctx = ctx.literal(' USING ').sql(cast) return ctx @operation def rename_table(self, old_name, new_name): return (self ._alter_table(self.make_context(), old_name) .literal(' RENAME TO ') .sql(Entity(new_name))) @operation def add_index(self, table, columns, unique=False, using=None): ctx = self.make_context() index_name = make_index_name(table, columns) table_obj = Table(table) cols = [getattr(table_obj.c, column) for column in columns] index = Index(index_name, table_obj, cols, unique=unique, using=using) return ctx.sql(index) @operation def drop_index(self, table, index_name): return (self .make_context() .literal('DROP INDEX ') .sql(Entity(index_name))) class PostgresqlMigrator(SchemaMigrator): def _primary_key_columns(self, tbl): query = """ SELECT pg_attribute.attname FROM pg_index, pg_class, pg_attribute WHERE pg_class.oid = %s::regclass AND indrelid = pg_class.oid AND pg_attribute.attrelid = pg_class.oid AND pg_attribute.attnum = any(pg_index.indkey) AND indisprimary; """ cursor = self.database.execute_sql(query, (tbl,)) return [row[0] for row in cursor.fetchall()] @operation def set_search_path(self, schema_name): return (self .make_context() .literal('SET search_path TO ') .sql(Entity(schema_name))) @operation def rename_table(self, old_name, new_name): pk_names = self._primary_key_columns(old_name) ParentClass = super(PostgresqlMigrator, self) operations = [ ParentClass.rename_table(old_name, new_name, with_context=True)] if len(pk_names) == 1: # Check for existence of primary key sequence. seq_name = '%s_%s_seq' % (old_name, pk_names[0]) query = """ SELECT 1 FROM information_schema.sequences WHERE LOWER(sequence_name) = LOWER(%s) """ cursor = self.database.execute_sql(query, (seq_name,)) if bool(cursor.fetchone()): new_seq_name = '%s_%s_seq' % (new_name, pk_names[0]) operations.append(ParentClass.rename_table( seq_name, new_seq_name)) return operations class CockroachDBMigrator(PostgresqlMigrator): explicit_create_foreign_key = True def add_inline_fk_sql(self, ctx, field): pass @operation def drop_index(self, table, index_name): return (self .make_context() .literal('DROP INDEX ') .sql(Entity(index_name)) .literal(' CASCADE')) class MySQLColumn(namedtuple('_Column', ('name', 'definition', 'null', 'pk', 'default', 'extra'))): @property def is_pk(self): return self.pk == 'PRI' @property def is_unique(self): return self.pk == 'UNI' @property def is_null(self): return self.null == 'YES' def sql(self, column_name=None, is_null=None): if is_null is None: is_null = self.is_null if column_name is None: column_name = self.name parts = [ Entity(column_name), SQL(self.definition)] if self.is_unique: parts.append(SQL('UNIQUE')) if is_null: parts.append(SQL('NULL')) else: parts.append(SQL('NOT NULL')) if self.is_pk: parts.append(SQL('PRIMARY KEY')) if self.extra: parts.append(SQL(self.extra)) return NodeList(parts) class MySQLMigrator(SchemaMigrator): explicit_create_foreign_key = True explicit_delete_foreign_key = True def _alter_column(self, ctx, table, column): return (self ._alter_table(ctx, table) .literal(' MODIFY ') .sql(Entity(column))) @operation def rename_table(self, old_name, new_name): return (self .make_context() .literal('RENAME TABLE ') .sql(Entity(old_name)) .literal(' TO ') .sql(Entity(new_name))) def _get_column_definition(self, table, column_name): table_safe = table.replace('`', '``') cursor = self.database.execute_sql('DESCRIBE `%s`;' % table_safe) rows = cursor.fetchall() for row in rows: column = MySQLColumn(*row) if column.name == column_name: return column return False def get_foreign_key_constraint(self, table, column_name): cursor = self.database.execute_sql( ('SELECT constraint_name ' 'FROM information_schema.key_column_usage WHERE ' 'table_schema = DATABASE() AND ' 'table_name = %s AND ' 'column_name = %s AND ' 'referenced_table_name IS NOT NULL AND ' 'referenced_column_name IS NOT NULL;'), (table, column_name)) result = cursor.fetchone() if not result: raise AttributeError( 'Unable to find foreign key constraint for ' '"%s" on table "%s".' % (column_name, table)) return result[0] @operation def drop_foreign_key_constraint(self, table, column_name): fk_constraint = self.get_foreign_key_constraint(table, column_name) return (self ._alter_table(self.make_context(), table) .literal(' DROP FOREIGN KEY ') .sql(Entity(fk_constraint))) def add_inline_fk_sql(self, ctx, field): pass @operation def add_not_null(self, table, column): column_def = self._get_column_definition(table, column) add_not_null = (self ._alter_table(self.make_context(), table) .literal(' MODIFY ') .sql(column_def.sql(is_null=False))) fk_objects = dict( (fk.column, fk) for fk in self.database.get_foreign_keys(table)) if column not in fk_objects: return add_not_null fk_metadata = fk_objects[column] return (self.drop_foreign_key_constraint(table, column), add_not_null, self.add_foreign_key_constraint( table, column, fk_metadata.dest_table, fk_metadata.dest_column)) @operation def drop_not_null(self, table, column): column_def = self._get_column_definition(table, column) if column_def.is_pk: raise ValueError('Primary keys can not be null') return (self ._alter_table(self.make_context(), table) .literal(' MODIFY ') .sql(column_def.sql(is_null=True))) @operation def rename_column(self, table, old_name, new_name): fk_objects = dict( (fk.column, fk) for fk in self.database.get_foreign_keys(table)) is_foreign_key = old_name in fk_objects column = self._get_column_definition(table, old_name) rename_ctx = (self ._alter_table(self.make_context(), table) .literal(' CHANGE ') .sql(Entity(old_name)) .literal(' ') .sql(column.sql(column_name=new_name))) if is_foreign_key: fk_metadata = fk_objects[old_name] return [ self.drop_foreign_key_constraint(table, old_name), rename_ctx, self.add_foreign_key_constraint( table, new_name, fk_metadata.dest_table, fk_metadata.dest_column), ] else: return rename_ctx @operation def alter_column_type(self, table, column, field, cast=None): if cast is not None: raise ValueError('alter_column_type() does not support cast with ' 'MySQL.') ctx = self.make_context() return (self ._alter_table(ctx, table) .literal(' MODIFY ') .sql(Entity(column)) .literal(' ') .sql(field.ddl(ctx))) @operation def drop_index(self, table, index_name): return (self .make_context() .literal('DROP INDEX ') .sql(Entity(index_name)) .literal(' ON ') .sql(Entity(table))) class SqliteMigrator(SchemaMigrator): """ SQLite supports a subset of ALTER TABLE queries, view the docs for the full details http://sqlite.org/lang_altertable.html """ column_re = re.compile(r'(.+?)\((.+)\)') column_split_re = re.compile(r'(?:[^,(]|\([^)]*\))+') column_name_re = re.compile(r'''["`']?([\w]+)''') fk_re = re.compile(r'FOREIGN KEY\s+\("?([\w]+)"?\)\s+', re.I) def _get_column_names(self, table): quoted = table.replace('"', '""') res = self.database.execute_sql('select * from "%s" limit 1' % quoted) return [item[0] for item in res.description] def _get_create_table(self, table): res = self.database.execute_sql( ('select name, sql from sqlite_master ' 'where type=? and LOWER(name)=?'), ['table', table.lower()]) return res.fetchone() @operation def _update_column(self, table, column_to_update, fn): columns = set(column.name.lower() for column in self.database.get_columns(table)) if column_to_update.lower() not in columns: raise ValueError('Column "%s" does not exist on "%s"' % (column_to_update, table)) # Get the SQL used to create the given table. table, create_table = self._get_create_table(table) # Get the indexes and SQL to re-create indexes. indexes = self.database.get_indexes(table) # Make sure the create_table does not contain any newlines or tabs, # allowing the regex to work correctly. create_table = re.sub(r'\s+', ' ', create_table) # Parse out the `CREATE TABLE` and column list portions of the query. raw_create, raw_columns = self.column_re.search(create_table).groups() # Clean up the individual column definitions. split_columns = self.column_split_re.findall(raw_columns) column_defs = [col.strip() for col in split_columns] new_column_defs = [] new_column_names = [] original_column_names = [] constraint_terms = ('foreign ', 'primary ', 'constraint ', 'check ') for column_def in column_defs: column_name, = self.column_name_re.match(column_def).groups() if column_name == column_to_update: new_column_def = fn(column_name, column_def) if new_column_def: new_column_defs.append(new_column_def) original_column_names.append(column_name) column_name, = self.column_name_re.match( new_column_def).groups() new_column_names.append(column_name) else: new_column_defs.append(column_def) # Avoid treating constraints as columns. if not column_def.lower().startswith(constraint_terms): new_column_names.append(column_name) original_column_names.append(column_name) # Create a mapping of original columns to new columns. original_to_new = dict(zip(original_column_names, new_column_names)) new_column = original_to_new.get(column_to_update) fk_filter_fn = lambda column_def: column_def if not new_column: # Remove any foreign keys associated with this column. fk_filter_fn = lambda column_def: None elif new_column != column_to_update: # Update any foreign keys for this column. fk_filter_fn = lambda column_def: self.fk_re.sub( 'FOREIGN KEY ("%s") ' % new_column, column_def) cleaned_columns = [] for column_def in new_column_defs: match = self.fk_re.match(column_def) if match is not None and match.groups()[0] == column_to_update: column_def = fk_filter_fn(column_def) if column_def: cleaned_columns.append(column_def) # Update the name of the new CREATE TABLE query. temp_table = table + '__tmp__' rgx = re.compile(r'("?)%s("?)' % re.escape(table), re.I) create = rgx.sub( r'\1%s\2' % temp_table, raw_create) # Create the new table. columns = ', '.join(cleaned_columns) queries = [ NodeList([SQL('DROP TABLE IF EXISTS'), Entity(temp_table)]), SQL('%s (%s)' % (create.strip(), columns))] # Populate new table. populate_table = NodeList(( SQL('INSERT INTO'), Entity(temp_table), EnclosedNodeList([Entity(col) for col in new_column_names]), SQL('SELECT'), CommaNodeList([Entity(col) for col in original_column_names]), SQL('FROM'), Entity(table))) drop_original = NodeList([SQL('DROP TABLE'), Entity(table)]) # Drop existing table and rename temp table. queries += [ populate_table, drop_original, self.rename_table(temp_table, table)] # Re-create user-defined indexes. User-defined indexes will have a # non-empty SQL attribute. for index in filter(lambda idx: idx.sql, indexes): if column_to_update not in index.columns: queries.append(SQL(index.sql)) elif new_column: sql = self._fix_index(index.sql, column_to_update, new_column) if sql is not None: queries.append(SQL(sql)) return queries def _fix_index(self, sql, column_to_update, new_column): # Split on the name of the column to update. If it splits into two # pieces, then there's no ambiguity and we can simply replace the # old with the new. parts = sql.split(column_to_update) if len(parts) == 2: return sql.replace(column_to_update, new_column) # Find the list of columns in the index expression. lhs, rhs = sql.rsplit('(', 1) # Apply the same "split in two" logic to the column list portion of # the query. if len(rhs.split(column_to_update)) == 2: return '%s(%s' % (lhs, rhs.replace(column_to_update, new_column)) # Strip off the trailing parentheses and go through each column. parts = rhs.rsplit(')', 1)[0].split(',') columns = [part.strip('"`[]\' ') for part in parts] # `columns` looks something like: ['status', 'timestamp" DESC'] # https://www.sqlite.org/lang_keywords.html # Strip out any junk after the column name. clean = [] for column in columns: if re.match(r'%s(?:[\'"`\]]?\s|$)' % re.escape(column_to_update), column): column = new_column + column[len(column_to_update):] clean.append(column) return '%s(%s)' % (lhs, ', '.join('"%s"' % c for c in clean)) @operation def drop_column(self, table, column_name, cascade=True, legacy=False): if sqlite3.sqlite_version_info >= (3, 35, 0) and not legacy: ctx = self.make_context() (self._alter_table(ctx, table) .literal(' DROP COLUMN ') .sql(Entity(column_name))) return ctx return self._update_column(table, column_name, lambda a, b: None) @operation def rename_column(self, table, old_name, new_name, legacy=False): if sqlite3.sqlite_version_info >= (3, 25, 0) and not legacy: return (self ._alter_table(self.make_context(), table) .literal(' RENAME COLUMN ') .sql(Entity(old_name)) .literal(' TO ') .sql(Entity(new_name))) def _rename(column_name, column_def): # Only replace the leading column name identifier to avoid # corrupting type names, defaults, or constraints that happen # to contain the column name as a substring. return re.sub( r'(["\'`]?)%s\1' % re.escape(column_name), r'\g<1>%s\1' % new_name, column_def, count=1) return column_def.replace(column_name, new_name) return self._update_column(table, old_name, _rename) @operation def add_not_null(self, table, column): def _add_not_null(column_name, column_def): return column_def + ' NOT NULL' return self._update_column(table, column, _add_not_null) @operation def drop_not_null(self, table, column): def _drop_not_null(column_name, column_def): return column_def.replace('NOT NULL', '') return self._update_column(table, column, _drop_not_null) @operation def add_column_default(self, table, column, default): if default is None: raise ValueError('`default` must be not None/NULL.') if callable_(default): default = default() if (isinstance(default, str) and not default.endswith((')', "'")) and not default.isdigit()): default = "'%s'" % default.replace("'", "''") def _add_default(column_name, column_def): # Try to handle SQL functions and string literals, otherwise quote. return column_def + ' DEFAULT %s' % default return self._update_column(table, column, _add_default) @operation def drop_column_default(self, table, column): def _drop_default(column_name, column_def): col = re.sub(r'DEFAULT\s+[\w"\'\(\)]+(\s|$)', '', column_def, flags=re.I) return col.strip() return self._update_column(table, column, _drop_default) @operation def alter_column_type(self, table, column, field, cast=None): if cast is not None: raise ValueError('alter_column_type() does not support cast with ' 'Sqlite.') ctx = self.make_context() def _alter_column_type(column_name, column_def): node_list = field.ddl(ctx) sql, _ = ctx.sql(Entity(column)).sql(node_list).query() return sql return self._update_column(table, column, _alter_column_type) @operation def add_constraint(self, table, name, constraint): raise NotImplementedError @operation def drop_constraint(self, table, name): raise NotImplementedError @operation def add_foreign_key_constraint(self, table, column_name, field, on_delete=None, on_update=None): raise NotImplementedError def migrate(*operations, **kwargs): for operation in operations: operation.run() ================================================ FILE: playhouse/mysql_ext.py ================================================ import json try: import mysql.connector as mysql_connector except ImportError: mysql_connector = None try: import mariadb except ImportError: mariadb = None from peewee import ImproperlyConfigured from peewee import Insert from peewee import MySQLDatabase from peewee import Node from peewee import NodeList from peewee import SQL from peewee import TextField from peewee import fn from playhouse.pool import _PooledMySQLDatabase class MySQLConnectorDatabase(MySQLDatabase): def _connect(self): if mysql_connector is None: raise ImproperlyConfigured('MySQL connector not installed!') return mysql_connector.connect(db=self.database, autocommit=True, **self.connect_params) def cursor(self, named_cursor=None): if self.is_closed(): if self.autoconnect: self.connect() else: raise InterfaceError('Error, database connection not opened.') return self._state.conn.cursor(buffered=True) def get_binary_type(self): return mysql_connector.Binary class PooledMySQLConnectorDatabase(_PooledMySQLDatabase, MySQLConnectorDatabase): pass class MariaDBConnectorDatabase(MySQLDatabase): def _connect(self): if mariadb is None: raise ImproperlyConfigured('mariadb connector not installed!') self.connect_params.pop('charset', None) self.connect_params.pop('sql_mode', None) self.connect_params.pop('use_unicode', None) return mariadb.connect(db=self.database, autocommit=True, **self.connect_params) def cursor(self, named_cursor=None): if self.is_closed(): if self.autoconnect: self.connect() else: raise InterfaceError('Error, database connection not opened.') return self._state.conn.cursor(buffered=True) def _set_server_version(self, conn): version = conn.server_version version, point = divmod(version, 100) version, minor = divmod(version, 100) self.server_version = (version, minor, point) if self.server_version >= (10, 5, 0): self.returning_clause = True def last_insert_id(self, cursor, query_type=None): if not self.returning_clause: return cursor.lastrowid elif query_type == Insert.SIMPLE: try: return cursor[0][0] except (AttributeError, IndexError): return cursor.lastrowid return cursor def get_binary_type(self): return mariadb.Binary class PooledMariaDBConnectorDatabase(_PooledMySQLDatabase, MariaDBConnectorDatabase): pass class JSONField(TextField): field_type = 'JSON' def __init__(self, json_dumps=None, json_loads=None, **kwargs): self._json_dumps = json_dumps or json.dumps self._json_loads = json_loads or json.loads super(JSONField, self).__init__(**kwargs) def python_value(self, value): if value is not None: try: return self._json_loads(value) except (TypeError, ValueError): return value def db_value(self, value): if value is not None: if not isinstance(value, Node): value = self._json_dumps(value) return value def extract(self, path): return fn.json_extract(self, path) def Match(columns, expr, modifier=None): if isinstance(columns, (list, tuple)): match = fn.MATCH(*columns) # Tuple of one or more columns / fields. else: match = fn.MATCH(columns) # Single column / field. args = expr if modifier is None else NodeList((expr, SQL(modifier))) return NodeList((match, fn.AGAINST(args))) ================================================ FILE: playhouse/pool.py ================================================ import functools import heapq import logging import threading import time from collections import namedtuple from peewee import MySQLDatabase from peewee import PostgresqlDatabase from peewee import SqliteDatabase logger = logging.getLogger('peewee.pool') def make_int(val): if val is not None and not isinstance(val, (int, float)): return int(val) return val class MaxConnectionsExceeded(ValueError): pass PoolConnection = namedtuple('PoolConnection', ('timestamp', 'connection', 'checked_out')) class _sentinel(object): def __lt__(self, other): return True def locked(fn): @functools.wraps(fn) def inner(self, *args, **kwargs): with self._pool_lock: return fn(self, *args, **kwargs) return inner class PooledDatabase(object): def __init__(self, database, max_connections=20, stale_timeout=None, timeout=None, **kwargs): self._max_connections = make_int(max_connections) self._stale_timeout = make_int(stale_timeout) self._wait_timeout = make_int(timeout) if self._wait_timeout == 0: self._wait_timeout = float('inf') # Lock for pool operations and condition for notifying when connection # is released back to pool. self._pool_lock = threading.RLock() self._pool_available = threading.Condition(self._pool_lock) # Available / idle connections stored in a heap, sorted oldest first. self._connections = [] # Counter used for tie-breaker in heap (so we don't try comparing # connection against connection). self._heap_counter = 0 # Mapping of connection id to PoolConnection. Ordinarily we would want # to use something like a WeakKeyDictionary, but Python typically won't # allow us to create weak references to connection objects. self._in_use = {} # Use the memory address of the connection as the key in the event the # connection object is not hashable. Connections will not get # garbage-collected, however, because a reference to them will persist # in "_in_use" as long as the conn has not been closed. self.conn_key = id super(PooledDatabase, self).__init__(database, **kwargs) def init(self, database, max_connections=None, stale_timeout=None, timeout=None, **connect_kwargs): super(PooledDatabase, self).init(database, **connect_kwargs) if max_connections is not None: self._max_connections = make_int(max_connections) if stale_timeout is not None: self._stale_timeout = make_int(stale_timeout) if timeout is not None: self._wait_timeout = make_int(timeout) if self._wait_timeout == 0: self._wait_timeout = float('inf') def connect(self, reuse_if_open=False): if not self._wait_timeout: return super(PooledDatabase, self).connect(reuse_if_open) deadline = time.monotonic() + self._wait_timeout while True: try: return super(PooledDatabase, self).connect(reuse_if_open) except MaxConnectionsExceeded: remaining = deadline - time.monotonic() if remaining <= 0: raise MaxConnectionsExceeded( 'Max connections exceeded, timed out attempting to ' 'connect.') with self._pool_available: self._pool_available.wait(timeout=min(remaining, 1.0)) @locked def _connect(self): while self._connections: try: # Remove the oldest connection from the heap. ts, _counter, conn = heapq.heappop(self._connections) except IndexError: break key = self.conn_key(conn) if self._is_closed(conn): # Connection closed either by user or by driver - discard. logger.debug('Connection %s was closed, discarding.', key) continue if self._stale_timeout and self._is_stale(ts): logger.debug('Connection %s was stale, closing.', key) self._close_raw(conn) continue # Connection OK to use. self._in_use[key] = PoolConnection(ts, conn, time.time()) return conn if self._max_connections and ( len(self._in_use) >= self._max_connections): raise MaxConnectionsExceeded('Exceeded maximum connections.') conn = super(PooledDatabase, self)._connect() ts = time.time() key = self.conn_key(conn) logger.debug('Created new connection %s.', key) self._in_use[key] = PoolConnection(ts, conn, time.time()) return conn def _is_stale(self, timestamp): # Called on check-out and check-in to ensure the connection has # not outlived the stale timeout. return (time.time() - timestamp) > self._stale_timeout def _is_closed(self, conn): return False def _can_reuse(self, conn): # Called on check-in to make sure the connection can be re-used. return True def _close_raw(self, conn): try: super(PooledDatabase, self)._close(conn) except Exception: logger.debug('Error closing connection %s.', self.conn_key(conn), exc_info=True) @locked def _close(self, conn, close_conn=False): # if close_conn == True, close underlying driver connection and remove # from _in_use tracking. Do not return to available conns. key = self.conn_key(conn) if close_conn: self._in_use.pop(key, None) self._close_raw(conn) return if key not in self._in_use: logger.debug('Connection %s not in use, ignoring close.', key) return pool_conn = self._in_use.pop(key) if self._stale_timeout and self._is_stale(pool_conn.timestamp): logger.debug('Closing stale connection %s on check-in.', key) self._close_raw(conn) elif not self._can_reuse(conn): logger.debug('Connection %s not reusable, closing.', key) self._close_raw(conn) else: logger.debug('Returning %s to pool.', key) self._heap_counter += 1 heapq.heappush(self._connections, (pool_conn.timestamp, self._heap_counter, conn)) # Wake up thread that may be waiting on connection. self._pool_available.notify() @locked def manual_close(self): """ Close the underlying connection without returning it to the pool. """ if self.is_closed(): return False # Obtain reference to the connection in-use by the calling thread. conn = self.connection() key = self.conn_key(conn) # Remove from _in_use so that subsequent self.close() won't try to # restore it to the pool. self._in_use.pop(key, None) self.close() self._close_raw(conn) @locked def close_idle(self): # Close any open connections that are not currently in-use. idle = self._connections self._connections = [] for _, _, conn in idle: self._close_raw(conn) @locked def close_stale(self, age=600): # Close any connections that are in-use but were checked out quite some # time ago and can be considered stale. May close connections in use by # running threads. cutoff = time.time() - age n = 0 for key, pool_conn in list(self._in_use.items()): if pool_conn.checked_out < cutoff: self._close_raw(pool_conn.connection) del self._in_use[key] n += 1 self._pool_available.notify_all() return n @locked def close_all(self): # Close all connections -- available and in-use. Warning: may break any # active connections used by other threads. self.close() self.close_idle() in_use, self._in_use = self._in_use, {} for pool_conn in in_use.values(): self._close_raw(pool_conn.connection) self._pool_available.notify_all() class _PooledMySQLDatabase(PooledDatabase): def _is_closed(self, conn): if self.server_version[0] == 8: args = () else: args = (False,) try: conn.ping(*args) except: return True return False class PooledMySQLDatabase(_PooledMySQLDatabase, MySQLDatabase): pass class _PooledPostgresqlDatabase(PooledDatabase): def _is_closed(self, conn): if conn.closed: return True return self._adapter.is_connection_closed(conn) def _can_reuse(self, conn): return self._adapter.is_connection_reusable(conn) class PooledPostgresqlDatabase(_PooledPostgresqlDatabase, PostgresqlDatabase): pass class _PooledSqliteDatabase(PooledDatabase): def _is_closed(self, conn): try: conn.total_changes except: return True return False class PooledSqliteDatabase(_PooledSqliteDatabase, SqliteDatabase): pass ================================================ FILE: playhouse/postgres_ext.py ================================================ import json import logging import uuid from peewee import * from peewee import ColumnBase from peewee import Expression from peewee import FieldDatabaseHook from peewee import Node from peewee import NodeList from peewee import Psycopg2Adapter from peewee import Psycopg3Adapter from peewee import __exception_wrapper__ from playhouse.pool import _PooledPostgresqlDatabase try: from psycopg2cffi import compat compat.register() except ImportError: pass try: from psycopg2.extras import register_hstore except ImportError: def register_hstore(*args): pass try: from psycopg.types import TypeInfo from psycopg.types.hstore import register_hstore as register_hstore_pg3 except ImportError: def register_hstore_pg3(*args): pass logger = logging.getLogger('peewee') HCONTAINS_DICT = '@>' HCONTAINS_KEYS = '?&' HCONTAINS_KEY = '?' HCONTAINS_ANY_KEY = '?|' HKEY = '->' HUPDATE = '||' ACONTAINS = '@>' ACONTAINED_BY = '<@' ACONTAINS_ANY = '&&' TS_MATCH = '@@' JSONB_CONTAINS = '@>' JSONB_CONTAINED_BY = '<@' JSONB_CONTAINS_KEY = '?' JSONB_CONTAINS_ANY_KEY = '?|' JSONB_CONTAINS_ALL_KEYS = '?&' JSONB_EXISTS = '?' JSONB_REMOVE = '-' JSONB_PATH_REMOVE = '#-' JSONB_PATH = '#>' class Json(Node): # Fallback JSON handler. __slots__ = ('value',) def __init__(self, value, dumps=None): self.value = value self.dumps = dumps or json.dumps def __sql__(self, ctx): return ctx.value(self.value, self.dumps) class _LookupNode(ColumnBase): def __init__(self, node, parts): self.node = node self.parts = parts super(_LookupNode, self).__init__() def clone(self): return type(self)(self.node, list(self.parts)) def __hash__(self): return hash((self.__class__.__name__, id(self))) class ObjectSlice(_LookupNode): @classmethod def create(cls, node, value): if isinstance(value, slice): stop = value.stop - 1 if value.stop is not None else None parts = [value.start or 0, stop] elif isinstance(value, int): parts = [value] elif isinstance(value, Node): parts = value else: # Assumes colon-separated integer indexes. parts = [int(i) for i in value.split(':')] return cls(node, parts) def __sql__(self, ctx): ctx.sql(self.node) if isinstance(self.parts, Node): ctx.literal('[').sql(self.parts).literal(']') else: ctx.literal('[%s]' % ':'.join([str(p + 1) if p is not None else '' for p in self.parts])) return ctx def __getitem__(self, value): return ObjectSlice.create(self, value) class IndexedFieldMixin(object): default_index_type = 'GIN' def __init__(self, *args, **kwargs): kwargs.setdefault('index', True) # By default, use an index. super(IndexedFieldMixin, self).__init__(*args, **kwargs) class ArrayField(IndexedFieldMixin, Field): passthrough = True def __init__(self, field_class=IntegerField, field_kwargs=None, dimensions=1, convert_values=False, *args, **kwargs): self.__field = field_class(**(field_kwargs or {})) self.dimensions = dimensions self.convert_values = convert_values self.field_type = self.__field.field_type super(ArrayField, self).__init__(*args, **kwargs) def bind(self, model, name, set_attribute=True): ret = super(ArrayField, self).bind(model, name, set_attribute) self.__field.bind(model, '__array_%s' % name, False) return ret def ddl_datatype(self, ctx): data_type = self.__field.ddl_datatype(ctx) return NodeList((data_type, SQL('[]' * self.dimensions)), glue='') def db_value(self, value): if value is None or isinstance(value, Node): return value elif self.convert_values: return self._process(self.__field.db_value, value, self.dimensions) else: return value if isinstance(value, list) else list(value) def python_value(self, value): if self.convert_values and value is not None: conv = self.__field.python_value if isinstance(value, list): return self._process(conv, value, self.dimensions) else: return conv(value) else: return value def _process(self, conv, value, dimensions): dimensions -= 1 if dimensions == 0: return [conv(v) for v in value] else: return [self._process(conv, v, dimensions) for v in value] def __getitem__(self, value): return ObjectSlice.create(self, value) def _e(op): def inner(self, rhs): return Expression(self, op, ArrayValue(self, rhs)) return inner __eq__ = _e(OP.EQ) __ne__ = _e(OP.NE) __gt__ = _e(OP.GT) __ge__ = _e(OP.GTE) __lt__ = _e(OP.LT) __le__ = _e(OP.LTE) __hash__ = Field.__hash__ def contains(self, *items): return Expression(self, ACONTAINS, ArrayValue(self, items)) def contains_any(self, *items): return Expression(self, ACONTAINS_ANY, ArrayValue(self, items)) def contained_by(self, *items): return Expression(self, ACONTAINED_BY, ArrayValue(self, items)) class ArrayValue(Node): def __init__(self, field, value): self.field = field self.value = value def __sql__(self, ctx): return (ctx .sql(AsIs(self.value)) .literal('::') .sql(self.field.ddl_datatype(ctx))) class DateTimeTZField(DateTimeField): field_type = 'TIMESTAMPTZ' class HStoreField(IndexedFieldMixin, Field): field_type = 'HSTORE' __hash__ = Field.__hash__ def __getitem__(self, key): return Expression(self, HKEY, Value(key)) def keys(self): return fn.akeys(self) def values(self): return fn.avals(self) def items(self): return fn.hstore_to_matrix(self) def slice(self, *args): return fn.slice(self, AsIs(list(args))) def exists(self, key): return fn.exist(self, key) def defined(self, key): return fn.defined(self, key) def update(self, __data=None, **data): if __data is not None: data.update(__data) return Expression(self, HUPDATE, data) def delete(self, *keys): value = Cast(AsIs(list(keys)), 'text[]') return fn.delete(self, value) def contains(self, value): if isinstance(value, dict): rhs = AsIs(value) return Expression(self, HCONTAINS_DICT, rhs) elif isinstance(value, (list, tuple)): rhs = AsIs(value) return Expression(self, HCONTAINS_KEYS, rhs) return Expression(self, HCONTAINS_KEY, value) def contains_any(self, *keys): return Expression(self, HCONTAINS_ANY_KEY, AsIs(list(keys))) class _JsonLookupBase(_LookupNode): def __init__(self, node, parts, as_json=False): super(_JsonLookupBase, self).__init__(node, parts) self._jsonb = getattr(node, '_json_type', 'jsonb') == 'jsonb' self._as_json = as_json def clone(self): return type(self)(self.node, list(self.parts), self._as_json) @Node.copy def as_json(self, as_json=True): self._as_json = as_json def concat(self, rhs): if not isinstance(rhs, Node): rhs = self.node.json_type(rhs) return Expression(self.as_json(True), OP.CONCAT, rhs) def contains(self, other): if not isinstance(other, Node): other = self.node.json_type(other) return Expression(self.as_json(True), JSONB_CONTAINS, other) def contained_by(self, other): if not isinstance(other, Node): other = self.node.json_type(other) return Expression(self.as_json(True), JSONB_CONTAINED_BY, other) def contains_any(self, *keys): return Expression( self.as_json(True), JSONB_CONTAINS_ANY_KEY, AsIs(list(keys), False)) def contains_all(self, *keys): return Expression( self.as_json(True), JSONB_CONTAINS_ALL_KEYS, AsIs(list(keys), False)) def has_key(self, key): return Expression(self.as_json(True), JSONB_CONTAINS_KEY, key) def remove(self): parts = [str(p) if isinstance(p, int) else p for p in self.parts] value = AsIs(parts, False) return Expression(self.node, JSONB_PATH_REMOVE, value) def length(self): func = fn.jsonb_array_length if self._jsonb else fn.json_array_length return func(self.as_json(True)) def extract(self, *path): path = [str(p) if isinstance(p, int) else p for p in path] func = fn.jsonb_extract_path if self._jsonb else fn.json_extract_path return func(self.as_json(True), *path) def path(self, *keys): return JsonPath(self.as_json(True), keys, as_json=True) class JsonLookup(_JsonLookupBase): def __getitem__(self, value): return JsonLookup(self.node, self.parts + [value], self._as_json) def __sql__(self, ctx): ctx.sql(self.node) for part in self.parts[:-1]: ctx.literal('->').sql(part) if self.parts: (ctx .literal('->' if self._as_json else '->>') .sql(self.parts[-1])) return ctx class JsonPath(_JsonLookupBase): def __sql__(self, ctx): return (ctx .sql(self.node) .literal('#>' if self._as_json else '#>>') .sql(Value('{%s}' % ','.join(map(str, self.parts))))) class JSONField(FieldDatabaseHook, Field): field_type = 'JSON' _json_datatype = 'json' def __init__(self, dumps=None, **kwargs): self._dumps = dumps super(JSONField, self).__init__(**kwargs) def _db_hook(self, database): if database is None or not hasattr(database, '_adapter'): self.json_type = Json self.cast_json_case = True else: self.json_type = database._adapter.json_type self.cast_json_case = database._adapter.cast_json_case if self._dumps: dumps = self._dumps class _Json(self.json_type): def __init__(self, value): super(_Json, self).__init__(value, dumps=dumps) self.json_type = _Json def db_value(self, value): if value is None or isinstance(value, (Node, self.json_type)): return value return self.json_type(value) def to_value(self, value, case=False): # CASE WHEN id = 123 THEN x.json_data fails because the expression is # untyped, so we need an explicit cast with psycopg2. if case and self.cast_json_case: return Cast(self.json_type(value), self._json_datatype) return self.json_type(value) def __getitem__(self, value): return JsonLookup(self, [value]) def path(self, *keys): return JsonPath(self, keys, as_json=True) def concat(self, value): if not isinstance(value, Node): value = self.json_type(value) return super(JSONField, self).concat(value) def length(self): return fn.json_array_length(self) def extract(self, *path): path = [str(p) if isinstance(p, int) else p for p in path] return fn.json_extract_path(self, *path) class BinaryJSONField(IndexedFieldMixin, JSONField): field_type = 'JSONB' _json_datatype = 'jsonb' __hash__ = Field.__hash__ def _db_hook(self, database): if database is None or not hasattr(database, '_adapter'): self.json_type = Json self.cast_json_case = True else: self.json_type = database._adapter.jsonb_type self.cast_json_case = database._adapter.cast_json_case if self._dumps: dumps = self._dumps class _Json(self.json_type): def __init__(self, value): super(_Json, self).__init__(value, dumps=dumps) self.json_type = _Json def contains(self, other): if not isinstance(other, Node): other = self.json_type(other) return Expression(self, JSONB_CONTAINS, other) def contained_by(self, other): if not isinstance(other, Node): other = self.json_type(other) return Expression(self, JSONB_CONTAINED_BY, other) def contains_any(self, *items): return Expression( self, JSONB_CONTAINS_ANY_KEY, AsIs(list(items), False)) def contains_all(self, *items): return Expression( self, JSONB_CONTAINS_ALL_KEYS, AsIs(list(items), False)) def has_key(self, key): return Expression(self, JSONB_CONTAINS_KEY, Value(key, False)) def remove(self, *items): value = Cast(AsIs(list(items), False), 'text[]') return Expression(self, JSONB_REMOVE, value) def length(self): return fn.jsonb_array_length(self) def extract(self, *path): path = [str(p) if isinstance(p, int) else p for p in path] return fn.jsonb_extract_path(self, *path) class TSVectorField(IndexedFieldMixin, TextField): field_type = 'TSVECTOR' __hash__ = Field.__hash__ def match(self, query, language=None, plain=False): params = (language, query) if language is not None else (query,) func = fn.plainto_tsquery if plain else fn.to_tsquery return Expression(self, TS_MATCH, func(*params)) def Match(field, query, language=None): params = (language, query) if language is not None else (query,) field_params = (language, field) if language is not None else (field,) return Expression( fn.to_tsvector(*field_params), TS_MATCH, fn.to_tsquery(*params)) class IntervalField(Field): field_type = 'INTERVAL' class FetchManyCursor(object): __slots__ = ('cursor', 'array_size', 'exhausted', 'iterable') def __init__(self, cursor, array_size=None): self.cursor = cursor self.array_size = array_size or cursor.itersize self.exhausted = False self.iterable = self.row_gen() @property def description(self): return self.cursor.description def close(self): if self.cursor is not None and not self.cursor.closed: self.cursor.close() def row_gen(self): try: while True: rows = self.cursor.fetchmany(self.array_size) if not rows: return for row in rows: yield row finally: self.close() def fetchone(self): if self.exhausted: return try: return next(self.iterable) except StopIteration: self.exhausted = True class ServerSideQuery(Node): def __init__(self, query, array_size=None): self.query = query self.array_size = array_size self._cursor_wrapper = None def __sql__(self, ctx): return self.query.__sql__(ctx) def __iter__(self): if self._cursor_wrapper is None: self._execute(self.query._database) return iter(self._cursor_wrapper.iterator()) def close(self): if self._cursor_wrapper is not None: self._cursor_wrapper.cursor.close() self._cursor_wrapper = None return True return False def iterator(self): if self._cursor_wrapper is None: self._execute(self.query._database) return self._cursor_wrapper.iterator() def _execute(self, database): if self._cursor_wrapper is None: cursor = database.execute(self.query, named_cursor=True, array_size=self.array_size) self._cursor_wrapper = self.query._get_cursor_wrapper(cursor) return self._cursor_wrapper def ServerSide(query, array_size=None): server_side_query = ServerSideQuery(query, array_size=array_size) for row in server_side_query: yield row class _empty_object(object): __slots__ = () def __nonzero__(self): return False __bool__ = __nonzero__ class Psycopg2ExtAdapter(Psycopg2Adapter): def register_hstore(self, conn): register_hstore(conn) def server_side_cursor(self, conn): # psycopg2 does not allow us to use these in autocommit, even if we ARE # inside a transaction - so specify withhold (not desirable!). return conn.cursor(name=str(uuid.uuid1()), withhold=True) class Psycopg3ExtAdapter(Psycopg3Adapter): def register_hstore(self, conn): info = TypeInfo.fetch(conn, 'hstore') register_hstore_pg3(info, conn) def server_side_cursor(self, conn): return conn.cursor(name=str(uuid.uuid1())) class PostgresqlExtDatabase(PostgresqlDatabase): psycopg2_adapter = Psycopg2ExtAdapter psycopg3_adapter = Psycopg3ExtAdapter def __init__(self, *args, **kwargs): self._register_hstore = kwargs.pop('register_hstore', False) self._server_side_cursors = kwargs.pop('server_side_cursors', False) super(PostgresqlExtDatabase, self).__init__(*args, **kwargs) def _connect(self): conn = super(PostgresqlExtDatabase, self)._connect() if self._register_hstore: self._adapter.register_hstore(conn) return conn def cursor(self, named_cursor=None): if self.is_closed(): if self.autoconnect: self.connect() else: raise InterfaceError('Error, database connection not opened.') if named_cursor: return self._adapter.server_side_cursor(self._state.conn) return self._state.conn.cursor() def execute(self, query, named_cursor=False, array_size=None, **context_options): ctx = self.get_sql_context(**context_options) sql, params = ctx.sql(query).query() named_cursor = named_cursor or (self._server_side_cursors and sql[:6].lower() == 'select') cursor = self.execute_sql(sql, params, named_cursor=named_cursor) if named_cursor: cursor = FetchManyCursor(cursor, array_size) return cursor def execute_sql(self, sql, params=None, named_cursor=None): logger.debug((sql, params)) with __exception_wrapper__: cursor = self.cursor(named_cursor=named_cursor) cursor.execute(sql, params or ()) return cursor class PooledPostgresqlExtDatabase(_PooledPostgresqlDatabase, PostgresqlExtDatabase): pass class Psycopg3Database(PostgresqlExtDatabase): def __init__(self, *args, **kwargs): kwargs['prefer_psycopg3'] = True super(Psycopg3Database, self).__init__(*args, **kwargs) class PooledPsycopg3Database(_PooledPostgresqlDatabase, Psycopg3Database): pass ================================================ FILE: playhouse/pwasyncio.py ================================================ import asyncio import collections import contextvars import json import logging from greenlet import greenlet, getcurrent from peewee import * from peewee import _atomic, _savepoint, _transaction from peewee import __exception_wrapper__ from peewee import Node from peewee import Psycopg3Adapter from playhouse.postgres_ext import Json try: import aiosqlite except ImportError: aiosqlite = None try: import asyncpg except ImportError: asyncpg = None try: import aiomysql except ImportError: aiomysql = None logger = logging.getLogger(__name__) class MissingGreenletBridge(RuntimeError): pass async def greenlet_spawn(fn, *args, **kwargs): parent = getcurrent() result = None error = None def runner(): nonlocal result, error try: result = fn(*args, **kwargs) except BaseException as exc: error = exc # Run the sync code in a greenlet - the sync code must use await_() # whenever blocking would occur. await_() transfers a coroutine and control # back up to this runner, which can safely `await` the coroutine before # switching back to the sync code. g = greenlet(runner, parent=parent) g.gr_context = parent.gr_context value = g.switch() while not g.dead: try: value = g.switch(await value) except BaseException as exc: value = g.throw(exc) if error: raise error return result def await_(awaitable): current = getcurrent() parent = current.parent if parent is None: raise MissingGreenletBridge('await_() called outside greenlet_spawn()') return parent.switch(awaitable) class _State(object): __slots__ = ('conn', 'closed', 'transactions', '_task_id') def __init__(self): self._task_id = None self.reset() def reset(self): self.conn = None self.closed = True self.transactions = [] class _ConnectionState(object): def __init__(self): self._cv = contextvars.ContextVar('pwasyncio_state') # Central registry: task-id -> _State. Allows close_pool() to # enumerate *all* live states and release their connections. self._states = {} self._orphaned_conns = [] def _current(self): task = asyncio.current_task() if task is None: raise RuntimeError('Cannot determine current task') tid = id(task) try: state = self._cv.get() if state._task_id == tid: # Re-register if evicted (e.g. by close_pool clearing _states). if tid not in self._states: self._states[tid] = state # Unnecessary to register the callback; task is still # running so the original callback should be present. # task.add_done_callback(self._on_task_done) return state except LookupError: pass if tid in self._states: state = self._states[tid] else: state = _State() state._task_id = tid self._states[tid] = state task.add_done_callback(self._on_task_done) # Cache in the contextvar for subsequent calls for task. self._cv.set(state) return state def _on_task_done(self, task): tid = id(task) state = self._states.pop(tid, None) if state is not None and state.conn is not None and not state.closed: self._orphaned_conns.append(state.conn) state.reset() @property def conn(self): return self._current().conn @property def closed(self): return self._current().closed @property def transactions(self): return self._current().transactions def reset(self): try: state = self._current() except RuntimeError: return state.reset() def set_connection(self, conn): state = self._current() state.conn = conn state.closed = False class _async_transaction_helper(object): async def __aenter__(self): return await self.db.run(self.__enter__) async def __aexit__(self, exc_typ, exc, tb): return await self.db.run(self.__exit__, exc_typ, exc, tb) async def acommit(self): return await self.db.run(self.commit) async def arollback(self): return await self.db.run(self.rollback) class async_atomic(_async_transaction_helper, _atomic): pass class async_transaction(_async_transaction_helper, _transaction): pass class async_savepoint(_async_transaction_helper, _savepoint): pass class AsyncDatabaseMixin(object): def __init__(self, database, **kwargs): self._pool_size = kwargs.pop('pool_size', 10) self._pool_min_size = kwargs.pop('pool_min_size', 1) self._acquire_timeout = kwargs.pop('acquire_timeout', 10) super(AsyncDatabaseMixin, self).__init__(database, **kwargs) self._state = _ConnectionState() self._pool = None self._pool_lock = asyncio.Lock() self._closing = False # Guard against use during shutdown. def execute_sql(self, sql, params=None): try: return await_(self.aexecute_sql(sql, params or ())) except MissingGreenletBridge as exc: raise MissingGreenletBridge( f'Attempted query {sql} ({params}) outside greenlet runner.') \ from exc async def aexecute_sql(self, sql, params=None): conn = await self.aconnect() with __exception_wrapper__: return await conn.execute(sql, params) def connect(self): return await_(self.aconnect()) async def aconnect(self): if self._closing: raise InterfaceError('Database pool is shutting down.') # Drain any connections orphaned by dead tasks. while self._state._orphaned_conns: orphan = self._state._orphaned_conns.pop() await self._pool_release(orphan) conn = self._state.conn if conn is None or conn.conn is None: if conn is not None: # Previous connection was invalidated, release it. await self._pool_release(conn) conn = await self._acquire_conn_async() self._state.set_connection(conn) return conn def close(self): return await_(self.aclose()) async def aclose(self): conn = self._state.conn if conn: self._state.reset() logger.debug('Releasing connection %s to pool.', id(conn)) await self._pool_release(conn) async def _acquire_conn_async(self): async with self._pool_lock: if self._pool is None: self._pool = await self._create_pool_async() conn = await self._pool_acquire() logger.debug('Acquired connection %s from pool.', id(conn)) return conn async def _create_pool_async(self): raise NotImplementedError('Subclasses must implement.') async def _pool_acquire(self): raise NotImplementedError('Subclasses must implement.') async def _pool_release(self, conn): raise NotImplementedError('Subclasses must implement.') async def close_pool(self): self._closing = True try: if self._pool: # Release connections held by any task still in the registry. # We must clear each state BEFORE releasing the connection, # because the await in _pool_release can let the event loop # run pending task-done callbacks. If the callback sees # state.conn still set it will orphan the same connection, # leading to a double-release that overfills the pool queue. for state in list(self._state._states.values()): if state.conn and not state.closed: conn = state.conn state.reset() try: await self._pool_release(conn) except Exception: logger.warning( 'Error releasing connection during pool close', exc_info=True) self._state._states.clear() # Drain any connections orphaned by completed tasks. while self._state._orphaned_conns: orphan = self._state._orphaned_conns.pop() try: await self._pool_release(orphan) except Exception: logger.warning('Error releasing orphaned connection', exc_info=True) await self._pool_close() self._pool = None finally: self._closing = False async def _pool_close(self): raise NotImplementedError('Subclasses must implement.') async def __aenter__(self): await self.run(self.connect) return self async def __aexit__(self, exc_typ, exc, tb): await self.run(self.close) def atomic(self): return async_atomic(self) def transaction(self): return async_transaction(self) def savepoint(self): return async_savepoint(self) async def acreate_tables(self, *args, **kwargs): return await greenlet_spawn(self.create_tables, *args, **kwargs) async def adrop_tables(self, *args, **kwargs): return await greenlet_spawn(self.drop_tables, *args, **kwargs) async def aexecute(self, query): query.bind(self) return await self.run(query.execute) async def get(self, query): return await self.run(query.get) async def list(self, query): return await self.run(list, query) async def scalar(self, query): return await self.run(query.scalar) async def count(self, query): return await self.run(query.count) async def exists(self, query): return await self.run(query.exists) async def aprefetch(self, query, *subqueries): return await self.run(prefetch, query, *subqueries) async def iterate(self, query, buffer_size=None): # Use similar approach to postgres_ext server-side query impl. query.bind(self) sql, params = query.sql() conn = await self.aconnect() cursor = await conn.execute_iter(sql, params or ()) if buffer_size is not None: cursor._buffer_size = buffer_size try: wrapper = query._get_cursor_wrapper(cursor) row_iter = wrapper.iterator() _sentinel = object() # Cursor wrapper `iterator()` calls fetchone() to grab rows from # the internal buffer. `fetchone()` may dispatch do the event loop # to refill buffer (async). while True: row = await greenlet_spawn(next, row_iter, _sentinel) if row is _sentinel: break yield row finally: await cursor.aclose() async def run(self, fn, *args, **kwargs): return await greenlet_spawn(fn, *args, **kwargs) def is_closed(self): try: return self._state.closed except RuntimeError: return True class CursorAdapter(object): DEFAULT_BUFFER_SIZE = 100 def __init__(self, rows=None, lastrowid=None, rowcount=None, description=None, fetch_many=None, cleanup=None, buffer_size=None): self._rows = rows or [] self._idx = 0 self.lastrowid = lastrowid self.rowcount = rowcount if rowcount is not None else len(self._rows) self.description = description or [] # Async server-side cursor support. self._fetch_many = fetch_many self._cleanup = cleanup self._buffer_size = buffer_size or self.DEFAULT_BUFFER_SIZE self._buffer = collections.deque() self._exhausted = False def fetchone(self): if self._fetch_many is not None: return self._lazy_fetchone() if self._idx >= len(self._rows): return row = self._rows[self._idx] self._idx += 1 return row def _lazy_fetchone(self): if not self._buffer: if self._exhausted: return None rows = await_(self._fetch_many(self._buffer_size)) if not rows: self._exhausted = True return None self._buffer.extend(rows) return self._buffer.popleft() def fetchall(self): if self._fetch_many is not None: return list(self) return self._rows def __iter__(self): if self._fetch_many is not None: return _lazy_cursor_iter(self) return iter(self._rows) def close(self): pass async def aclose(self): if self._cleanup is not None: try: await self._cleanup() finally: self._cleanup = None self._fetch_many = None def _lazy_cursor_iter(cursor): while True: row = cursor.fetchone() if row is None: return yield row class DummyCursor(object): def __init__(self, conn): self.conn = conn def execute(self, sql, params=None): return await_(self._async_execute(sql, params)) async def _async_execute(self, sql, params): return await self.conn.execute(sql, params) class AsyncConnectionWrapper(object): def __init__(self, conn): self.conn = conn self._lock = asyncio.Lock() async def execute(self, sql, params=None): async with self._lock: return await self._execute(sql, params) async def _execute(self, sql, params): raise NotImplementedError('Subclasses must implement.') def cursor(self): return DummyCursor(self) async def execute_iter(self, sql, params=None): raise NotImplementedError('Subclasses must implement.') async def close(self): if self.conn: await self.conn.close() self.conn = None class AsyncSqlitePool(object): def __init__(self, database, pool_size=5, on_connect=None, **connect_params): self._database = database self._pool_size = pool_size self._on_connect = on_connect self._connect_params = connect_params self._queue = asyncio.Queue(maxsize=pool_size) self._all_connections = [] self._closed = False async def initialize(self): for _ in range(self._pool_size): conn = await self._create_connection() self._queue.put_nowait(conn) return self async def _create_connection(self): conn = await aiosqlite.connect( self._database, isolation_level=None, **self._connect_params) if self._on_connect is not None: await self._on_connect(conn ) wrapped = AsyncSqliteConnection(conn ) self._all_connections.append(wrapped) return wrapped async def acquire(self, timeout=None): if self._closed: raise InterfaceError('Pool is closed.') return await asyncio.wait_for(self._queue.get(), timeout=timeout) def _conn_is_valid(self, conn): driver_conn = conn.conn if driver_conn is None: return False if not driver_conn._running or not driver_conn._connection: return False return True async def release(self, conn): if self._closed: return elif self._conn_is_valid(conn): await self._queue.put(conn) else: try: self._all_connections.remove(conn) except ValueError: pass await self._queue.put(await self._create_connection()) async def close(self): self._closed = True conns, self._all_connections = list(self._all_connections), [] for conn in conns: try: await conn.close() except Exception: logger.warning('Error closing pooled connection', exc_info=True) class AsyncSqliteConnection(AsyncConnectionWrapper): async def _execute(self, sql, params=None): params = params or () cursor = await self.conn.execute(sql, params) rows = await cursor.fetchall() lastrowid = cursor.lastrowid rowcount = cursor.rowcount description = cursor.description await cursor.close() return CursorAdapter(rows, lastrowid=lastrowid, rowcount=rowcount, description=description) async def execute_iter(self, sql, params=None): await self._lock.acquire() try: cursor = await self.conn.execute(sql, params or ()) except BaseException: self._lock.release() raise lock = self._lock async def fetch_many(count): return await cursor.fetchmany(count) async def cleanup(): try: await cursor.close() finally: lock.release() return CursorAdapter( description=cursor.description, fetch_many=fetch_many, cleanup=cleanup) class AsyncSqliteDatabase(AsyncDatabaseMixin, SqliteDatabase): async def _create_pool_async(self): if aiosqlite is None: raise ImproperlyConfigured('aiosqlite is not installed') pool = AsyncSqlitePool(self.database, pool_size=self._pool_size, on_connect=self._add_conn_hooks) return await pool.initialize() async def _add_conn_hooks(self, conn): if self._pragmas: await self._set_pragmas(conn) if self._functions: await self._load_functions(conn) async def _set_pragmas(self, conn): for pragma, value in self._pragmas: await conn.execute('PRAGMA %s = %s;' % (pragma, value)) async def _load_functions(self, conn): for name, (fn, n_params, deterministic) in self._functions.items(): kwargs = {'deterministic': deterministic} if deterministic else {} await conn.create_function(name, n_params, fn, **kwargs) async def _pool_acquire(self): return await self._pool.acquire(timeout=self._acquire_timeout) async def _pool_release(self, conn): if conn is not None: await self._pool.release(conn) async def _pool_close(self): if self._pool: await self._pool.close() class AsyncMySQLConnection(AsyncConnectionWrapper): async def _execute(self, sql, params=None): params = params or () cursor = await self.conn.cursor() try: await cursor.execute(sql, params) rows = await cursor.fetchall() lastrowid = cursor.lastrowid rowcount = cursor.rowcount description = cursor.description finally: await cursor.close() return CursorAdapter(rows, lastrowid=lastrowid, rowcount=rowcount, description=description) async def execute_iter(self, sql, params=None): await self._lock.acquire() try: # Server-side cursor for unbuffered streaming. cursor = await self.conn.cursor(aiomysql.SSCursor) await cursor.execute(sql, params or ()) except BaseException: self._lock.release() raise lock = self._lock async def fetch_many(count): return await cursor.fetchmany(count) async def cleanup(): try: await cursor.close() finally: lock.release() return CursorAdapter( description=cursor.description, fetch_many=fetch_many, cleanup=cleanup) class AsyncMySQLDatabase(AsyncDatabaseMixin, MySQLDatabase): async def _create_pool_async(self): if aiomysql is None: raise ImproperlyConfigured('aiomysql is not installed') return await aiomysql.create_pool( db=self.database, autocommit=True, minsize=self._pool_min_size, maxsize=self._pool_size, **self.connect_params) async def _pool_acquire(self): conn = await asyncio.wait_for( self._pool.acquire(), timeout=self._acquire_timeout) return AsyncMySQLConnection(conn) async def _pool_release(self, conn): if conn and conn.conn: self._pool.release(conn.conn) async def _pool_close(self): self._pool.close() await self._pool.wait_closed() class AsyncPostgresqlConnection(AsyncConnectionWrapper): async def _execute(self, sql, params=None): # asyncpg uses $1, $2 positional params instead of %s. if params: sql = self._translate_placeholders(sql) records = await self.conn.fetch(sql, *(params or ())) if records: description = [(k,) for k in records[0].keys()] rows = records else: description = [] rows = [] return CursorAdapter(rows, description=description) async def execute_iter(self, sql, params=None): if params: sql = self._translate_placeholders(sql) await self._lock.acquire() try: # NB: asyncpg cursors require an active transaction. # Right now we cannot use peewee-managed transactions because # asyncpg's Cursor._check_ready() requires an asyncpg-managed # transaction be active. # See: https://github.com/MagicStack/asyncpg/issues/1311 tr = self.conn.transaction() await tr.start() stmt = await self.conn.prepare(sql) cursor = await stmt.cursor(*(params or ())) except BaseException: self._lock.release() raise lock = self._lock async def fetch_many(count): return await cursor.fetch(count) async def cleanup(): try: await tr.rollback() except: pass finally: lock.release() return CursorAdapter( fetch_many=fetch_many, cleanup=cleanup, description=[(a.name,) for a in stmt.get_attributes()]) @staticmethod def _translate_placeholders(sql): parts = sql.split('%s') if len(parts) == 1: return sql accum = [parts[0]] for i, part in enumerate(parts[1:], 1): accum.append('$%d' % i) accum.append(part) return ''.join(accum) class AsyncPgAdapter(Psycopg3Adapter): def __init__(self): super(AsyncPgAdapter, self).__init__() self.json_type = Json self.jsonb_type = Json class AsyncPgAtomic(object): def __init__(self, db, *args, **kwargs): self.db = db self._begin_args = (args, kwargs) def __enter__(self): await_(self._abegin()) self.db._state.transactions.append(self) return self def __exit__(self, exc_type, exc_val, exc_tb): self.db._state.transactions.pop() if exc_type: self.rollback(False) else: try: self.commit(False) except: self.rollback(False) raise def commit(self, begin=True): await_(self.acommit(begin)) def rollback(self, begin=True): await_(self.arollback(begin)) async def _abegin(self): a, k = self._begin_args conn = await self.db.aconnect() self._tx = conn.conn.transaction(*a, **k) await self._tx.start() return self._tx async def acommit(self, begin=True): await self._tx.commit() if begin: await self._abegin() async def arollback(self, begin=True): await self._tx.rollback() if begin: await self._abegin() async def __aenter__(self): await self._abegin() self.db._state.transactions.append(self) return self async def __aexit__(self, exc_type, exc_val, exc_tb): self.db._state.transactions.pop() if exc_type: await self.arollback(False) else: try: await self.acommit(False) except: await self.arollback(False) raise class AsyncPostgresqlDatabase(AsyncDatabaseMixin, PostgresqlDatabase): psycopg2_adapter = psycopg3_adapter = AsyncPgAdapter async def register_adapters(self, conn): def decode_json(bval): return json.loads(bval.decode()) await conn.set_type_codec( 'json', encoder=str.encode, decoder=decode_json, schema='pg_catalog', format='binary') def encode_jsonb(val): return b'\x01' + val.encode('utf8') def decode_jsonb(bval): return json.loads(bval[1:].decode()) await conn.set_type_codec( 'jsonb', encoder=encode_jsonb, decoder=decode_jsonb, schema='pg_catalog', format='binary') async def _create_pool_async(self): if asyncpg is None: raise ImproperlyConfigured('asyncpg is not installed') return await asyncpg.create_pool( database=self.database, min_size=self._pool_min_size, max_size=self._pool_size, init=self.register_adapters, **self.connect_params) async def _pool_acquire(self): conn = await asyncio.wait_for( self._pool.acquire(), timeout=self._acquire_timeout) return AsyncPostgresqlConnection(conn) async def _pool_release(self, conn): if conn and conn.conn: await self._pool.release(conn.conn) async def _pool_close(self): await self._pool.close() def atomic(self, *args, **kwargs): return AsyncPgAtomic(self, *args, **kwargs) def transaction(self, *args, **kwargs): return AsyncPgAtomic(self, *args, **kwargs) def savepoint(self, *args, **kwargs): return AsyncPgAtomic(self, *args, **kwargs) ================================================ FILE: playhouse/pydantic_utils.py ================================================ from __future__ import annotations from typing import Any from typing import Literal from typing import Optional from typing import get_origin from peewee import AutoField from peewee import ForeignKeyField from peewee import Model from playhouse.reflection import FieldTypeMap from pydantic import BaseModel from pydantic import ConfigDict from pydantic import Field from pydantic import create_model def choices_to_literal(choices): return Literal[tuple(val for val, label in choices)] def choices_description(choices): return ', '.join(['%r = %s' % (value, label) for value, label in choices]) def get_field_type(field): if isinstance(field, ForeignKeyField): field = field.rel_field return FieldTypeMap.get(field.field_type, Any) def to_pydantic(model_cls, exclude=None, include=None, exclude_autofield=True, model_name=None, relationships=None): exclude = exclude or set() relationships = relationships or {} fields = {} rel_fields = {} backref_fields = {} for field, schema in relationships.items(): if isinstance(field, ForeignKeyField): rel_fields[field.name] = schema else: backref_fields[field.field.backref] = schema for field in model_cls._meta.sorted_fields: name = field.name if name in exclude: continue elif include is not None and name not in include: continue elif exclude_autofield and isinstance(field, AutoField): continue if isinstance(field, ForeignKeyField): if name in rel_fields: schema = rel_fields[name] field_kwargs = {} if field.verbose_name: field_kwargs['title'] = field.verbose_name if field.help_text: field_kwargs['description'] = field.help_text if field.null: schema = Optional[schema] field_kwargs['default'] = None fields[name] = (schema, Field(**field_kwargs)) continue name = field.column_name python_type = get_field_type(field) choices = field.choices if choices: python_type = choices_to_literal(choices) parts = [] if field.help_text: parts.append(field.help_text) if choices: parts.append('Choices: %s' % choices_description(choices)) description = ' | '.join(parts) or None field_kwargs = {} if field.verbose_name: field_kwargs['title'] = field.verbose_name if description: field_kwargs['description'] = description if field.default is not None: if callable(field.default): field_kwargs['default_factory'] = field.default else: field_kwargs['default'] = field.default if field.null: python_type = Optional[python_type] elif field.null: python_type = Optional[python_type] field_kwargs['default'] = None fields[name] = (python_type, Field(**field_kwargs)) for name, schema in backref_fields.items(): origin = get_origin(schema) if origin is not list: raise ValueError('back-references must use a List type') fields[name] = (schema, Field(default_factory=list)) model_name = model_name or ('%sSchema' % model_cls.__name__) return create_model( model_name, __config__=ConfigDict(from_attributes=True), **fields) ================================================ FILE: playhouse/reflection.py ================================================ import datetime import decimal import re import uuid import warnings from collections import OrderedDict from collections import namedtuple from inspect import isclass from peewee import * from peewee import _StringField from peewee import _query_val_transform from peewee import CommaNodeList from peewee import SCOPE_VALUES from peewee import make_snake_case try: from pymysql.constants import FIELD_TYPE except ImportError: try: from MySQLdb.constants import FIELD_TYPE except ImportError: FIELD_TYPE = None try: from playhouse import postgres_ext except ImportError: postgres_ext = None try: from playhouse.cockroachdb import CockroachDatabase except ImportError: CockroachDatabase = None RESERVED_WORDS = set([ 'and', 'as', 'assert', 'break', 'class', 'continue', 'def', 'del', 'elif', 'else', 'except', 'exec', 'finally', 'for', 'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'not', 'or', 'pass', 'print', 'raise', 'return', 'try', 'while', 'with', 'yield', ]) FieldTypeMap = { 'AUTO': int, 'BIGAUTO': int, 'BIGINT': int, 'BLOB': bytes, 'BOOL': bool, 'CHAR': str, 'DATE': datetime.date, 'DATETIME': datetime.datetime, 'DECIMAL': decimal.Decimal, 'DOUBLE': float, 'FLOAT': float, 'INT': int, 'SMALLINT': int, 'TEXT': str, 'TIME': datetime.time, 'UUID': uuid.UUID, 'UUIDB': bytes, 'VARCHAR': str, 'JSON': dict, 'JSONB': dict, 'TIMESTAMPTZ': datetime.datetime, 'INTERVAL': datetime.timedelta, } class UnknownField(object): pass class Column(object): """ Store metadata about a database column. """ primary_key_types = (IntegerField, AutoField) def __init__(self, name, field_class, raw_column_type, nullable, primary_key=False, column_name=None, index=False, unique=False, default=None, extra_parameters=None): self.name = name self.field_class = field_class self.raw_column_type = raw_column_type self.nullable = nullable self.primary_key = primary_key self.column_name = column_name self.index = index self.unique = unique self.default = default self.extra_parameters = extra_parameters # Foreign key metadata. self.rel_model = None self.related_name = None self.to_field = None def __repr__(self): attrs = [ 'field_class', 'raw_column_type', 'nullable', 'primary_key', 'column_name'] keyword_args = ', '.join( '%s=%s' % (attr, getattr(self, attr)) for attr in attrs) return 'Column(%s, %s)' % (self.name, keyword_args) def get_field_parameters(self): params = {} if self.extra_parameters is not None: params.update(self.extra_parameters) # Set up default attributes. if self.nullable: params['null'] = True if self.field_class is ForeignKeyField or self.name != self.column_name: params['column_name'] = "'%s'" % self.column_name if self.primary_key and not issubclass(self.field_class, AutoField): params['primary_key'] = True if self.default is not None: params['constraints'] = '[SQL("DEFAULT %s")]' % \ self.default.replace('"', '\\"') # Handle ForeignKeyField-specific attributes. if self.is_foreign_key(): params['model'] = self.rel_model if self.to_field: params['field'] = "'%s'" % self.to_field if self.related_name: params['backref'] = "'%s'" % self.related_name # Handle indexes on column. if not self.is_primary_key(): if self.unique: params['unique'] = 'True' elif self.index and not self.is_foreign_key(): params['index'] = 'True' return params def is_primary_key(self): return self.field_class is AutoField or self.primary_key def is_foreign_key(self): return self.field_class is ForeignKeyField def is_self_referential_fk(self): return (self.field_class is ForeignKeyField and self.rel_model == "'self'") def set_foreign_key(self, foreign_key, model_names, dest=None, related_name=None): self.foreign_key = foreign_key self.field_class = ForeignKeyField if foreign_key.dest_table == foreign_key.table: self.rel_model = "'self'" else: self.rel_model = model_names[foreign_key.dest_table] self.to_field = dest and dest.name or None self.related_name = related_name or None def get_field(self): # Generate the field definition for this column. field_params = {} for key, value in self.get_field_parameters().items(): if isclass(value) and issubclass(value, Field): value = value.__name__ field_params[key] = value param_str = ', '.join('%s=%s' % (k, v) for k, v in sorted(field_params.items())) field = '%s = %s(%s)' % ( self.name, self.field_class.__name__, param_str) if self.field_class is UnknownField: field = '%s # %s' % (field, self.raw_column_type) return field class Metadata(object): column_map = {} extension_import = '' def __init__(self, database): self.database = database self.requires_extension = False def execute(self, sql, *params): return self.database.execute_sql(sql, params) def get_columns(self, table, schema=None): metadata = OrderedDict( (metadata.name, metadata) for metadata in self.database.get_columns(table, schema)) # Look up the actual column type for each column. column_types, extra_params = self.get_column_types(table, schema) # Look up the primary keys. pk_names = self.get_primary_keys(table, schema) if len(pk_names) == 1: pk = pk_names[0] if column_types[pk] is IntegerField: column_types[pk] = AutoField elif column_types[pk] is BigIntegerField: column_types[pk] = BigAutoField columns = OrderedDict() for name, column_data in metadata.items(): field_class = column_types[name] default = self._clean_default(field_class, column_data.default) columns[name] = Column( name, field_class=field_class, raw_column_type=column_data.data_type, nullable=column_data.null, primary_key=column_data.primary_key, column_name=name, default=default, extra_parameters=extra_params.get(name)) return columns def get_column_types(self, table, schema=None): raise NotImplementedError def _clean_default(self, field_class, default): if default is None or field_class in (AutoField, BigAutoField) or \ default.lower() == 'null': return if issubclass(field_class, _StringField) and \ isinstance(default, str) and not default.startswith("'"): default = "'%s'" % default return default or "''" def get_foreign_keys(self, table, schema=None): return self.database.get_foreign_keys(table, schema) def get_primary_keys(self, table, schema=None): return self.database.get_primary_keys(table, schema) def get_indexes(self, table, schema=None): return self.database.get_indexes(table, schema) class PostgresqlMetadata(Metadata): column_map = { 16: BooleanField, 17: BlobField, 20: BigIntegerField, 21: SmallIntegerField, 23: IntegerField, 25: TextField, 700: FloatField, 701: DoubleField, 1042: CharField, # blank-padded CHAR 1043: CharField, 1082: DateField, 1114: DateTimeField, 1184: DateTimeField, 1083: TimeField, 1266: TimeField, 1700: DecimalField, 2950: UUIDField, # UUID } array_types = { 1000: BooleanField, 1001: BlobField, 1005: SmallIntegerField, 1007: IntegerField, 1009: TextField, 1014: CharField, 1015: CharField, 1016: BigIntegerField, 1115: DateTimeField, 1182: DateField, 1183: TimeField, 2951: UUIDField, } extension_import = 'from playhouse.postgres_ext import *' def __init__(self, database): super(PostgresqlMetadata, self).__init__(database) if postgres_ext is not None: # Attempt to add types like HStore and JSON. cursor = self.execute('select oid, typname, format_type(oid, NULL)' ' from pg_type;') results = cursor.fetchall() for oid, typname, formatted_type in results: if typname == 'json': self.column_map[oid] = postgres_ext.JSONField elif typname == 'jsonb': self.column_map[oid] = postgres_ext.BinaryJSONField elif typname == 'hstore': self.column_map[oid] = postgres_ext.HStoreField elif typname == 'tsvector': self.column_map[oid] = postgres_ext.TSVectorField for oid in self.array_types: self.column_map[oid] = postgres_ext.ArrayField def get_column_types(self, table, schema): column_types = {} extra_params = {} extension_types = set(( postgres_ext.ArrayField, postgres_ext.BinaryJSONField, postgres_ext.JSONField, postgres_ext.TSVectorField, postgres_ext.HStoreField)) if postgres_ext is not None else set() # Look up the actual column type for each column. identifier = '%s."%s"' % (schema, table) cursor = self.execute( 'SELECT attname, atttypid FROM pg_catalog.pg_attribute ' 'WHERE attrelid = %s::regclass AND attnum > %s', identifier, 0) # Store column metadata in dictionary keyed by column name. for name, oid in cursor.fetchall(): column_types[name] = self.column_map.get(oid, UnknownField) if column_types[name] in extension_types: self.requires_extension = True if oid in self.array_types: extra_params[name] = {'field_class': self.array_types[oid]} return column_types, extra_params def get_columns(self, table, schema=None): schema = schema or 'public' return super(PostgresqlMetadata, self).get_columns(table, schema) def get_foreign_keys(self, table, schema=None): schema = schema or 'public' return super(PostgresqlMetadata, self).get_foreign_keys(table, schema) def get_primary_keys(self, table, schema=None): schema = schema or 'public' return super(PostgresqlMetadata, self).get_primary_keys(table, schema) def get_indexes(self, table, schema=None): schema = schema or 'public' return super(PostgresqlMetadata, self).get_indexes(table, schema) class CockroachDBMetadata(PostgresqlMetadata): # CRDB treats INT the same as BIGINT, so we just map bigint type OIDs to # regular IntegerField. column_map = PostgresqlMetadata.column_map.copy() column_map[20] = IntegerField array_types = PostgresqlMetadata.array_types.copy() array_types[1016] = IntegerField extension_import = 'from playhouse.cockroachdb import *' def __init__(self, database): Metadata.__init__(self, database) self.requires_extension = True if postgres_ext is not None: # Attempt to add JSON types. cursor = self.execute('select oid, typname, format_type(oid, NULL)' ' from pg_type;') results = cursor.fetchall() for oid, typname, formatted_type in results: if typname == 'jsonb': self.column_map[oid] = postgres_ext.BinaryJSONField for oid in self.array_types: self.column_map[oid] = postgres_ext.ArrayField class MySQLMetadata(Metadata): if FIELD_TYPE is None: column_map = {} else: column_map = { FIELD_TYPE.BLOB: TextField, FIELD_TYPE.CHAR: CharField, FIELD_TYPE.DATE: DateField, FIELD_TYPE.DATETIME: DateTimeField, FIELD_TYPE.DECIMAL: DecimalField, FIELD_TYPE.DOUBLE: FloatField, FIELD_TYPE.FLOAT: FloatField, FIELD_TYPE.INT24: IntegerField, FIELD_TYPE.LONG_BLOB: TextField, FIELD_TYPE.LONG: IntegerField, FIELD_TYPE.LONGLONG: BigIntegerField, FIELD_TYPE.MEDIUM_BLOB: TextField, FIELD_TYPE.NEWDECIMAL: DecimalField, FIELD_TYPE.SHORT: IntegerField, FIELD_TYPE.STRING: CharField, FIELD_TYPE.TIMESTAMP: DateTimeField, FIELD_TYPE.TIME: TimeField, FIELD_TYPE.TINY_BLOB: TextField, FIELD_TYPE.TINY: IntegerField, FIELD_TYPE.VAR_STRING: CharField, } def __init__(self, database, **kwargs): if 'password' in kwargs: kwargs['passwd'] = kwargs.pop('password') super(MySQLMetadata, self).__init__(database, **kwargs) def get_column_types(self, table, schema=None): column_types = {} # Look up the actual column type for each column. cursor = self.execute('SELECT * FROM `%s` LIMIT 1' % table) # Store column metadata in dictionary keyed by column name. for column_description in cursor.description: name, type_code = column_description[:2] column_types[name] = self.column_map.get(type_code, UnknownField) return column_types, {} class SqliteMetadata(Metadata): column_map = { 'bigint': BigIntegerField, 'blob': BlobField, 'bool': BooleanField, 'boolean': BooleanField, 'char': CharField, 'date': DateField, 'datetime': DateTimeField, 'decimal': DecimalField, 'float': FloatField, 'integer': IntegerField, 'integer unsigned': IntegerField, 'int': IntegerField, 'long': BigIntegerField, 'numeric': DecimalField, 'real': FloatField, 'smallinteger': IntegerField, 'smallint': IntegerField, 'smallint unsigned': IntegerField, 'text': TextField, 'time': TimeField, 'varchar': CharField, } begin = r'(?:["\[\(]+)?' end = r'(?:["\]\)]+)?' re_foreign_key = ( r'(?:FOREIGN KEY\s*)?' r'{begin}(.+?){end}\s+(?:.+\s+)?' r'references\s+{begin}(.+?){end}' r'\s*\(["|\[]?(.+?)["|\]]?\)').format(begin=begin, end=end) re_varchar = r'^\s*(?:var)?char\s*\(\s*(\d+)\s*\)\s*$' def _map_col(self, column_type): raw_column_type = column_type.lower() if raw_column_type in self.column_map: field_class = self.column_map[raw_column_type] elif re.search(self.re_varchar, raw_column_type): field_class = CharField else: column_type = re.sub(r'\(.+\)', '', raw_column_type) if column_type == '': field_class = BareField else: field_class = self.column_map.get(column_type, UnknownField) return field_class def get_column_types(self, table, schema=None): column_types = {} columns = self.database.get_columns(table) for column in columns: column_types[column.name] = self._map_col(column.data_type) return column_types, {} _DatabaseMetadata = namedtuple('_DatabaseMetadata', ( 'columns', 'primary_keys', 'foreign_keys', 'model_names', 'indexes')) class DatabaseMetadata(_DatabaseMetadata): def multi_column_indexes(self, table): accum = [] for index in self.indexes[table]: if len(index.columns) > 1: field_names = [self.columns[table][column].name for column in index.columns if column in self.columns[table]] accum.append((field_names, index.unique)) return accum def column_indexes(self, table): accum = {} for index in self.indexes[table]: if len(index.columns) == 1: accum[index.columns[0]] = index.unique return accum class Introspector(object): pk_classes = [AutoField, IntegerField] def __init__(self, metadata, schema=None): self.metadata = metadata self.schema = schema def __repr__(self): return '' % self.metadata.database @classmethod def from_database(cls, database, schema=None): if isinstance(database, Proxy): if database.obj is None: raise ValueError('Cannot introspect an uninitialized Proxy.') database = database.obj # Reference the proxied db obj. if CockroachDatabase and isinstance(database, CockroachDatabase): metadata = CockroachDBMetadata(database) elif isinstance(database, PostgresqlDatabase): metadata = PostgresqlMetadata(database) elif isinstance(database, MySQLDatabase): metadata = MySQLMetadata(database) elif isinstance(database, SqliteDatabase): metadata = SqliteMetadata(database) else: raise ValueError('Introspection not supported for %r' % database) return cls(metadata, schema=schema) def get_database_class(self): return type(self.metadata.database) def get_database_name(self): return self.metadata.database.database def get_database_kwargs(self): return self.metadata.database.connect_params def get_additional_imports(self): if self.metadata.requires_extension: return '\n' + self.metadata.extension_import return '' def make_model_name(self, table, snake_case=True): if snake_case: table = make_snake_case(table) model = re.sub(r'[^\w]+', '', table) model_name = ''.join(sub.title() for sub in model.split('_')) if not model_name[0].isalpha(): model_name = 'T' + model_name return model_name def make_column_name(self, column, is_foreign_key=False, snake_case=True): column = column.strip() if snake_case: column = make_snake_case(column) column = column.lower() if is_foreign_key: # Strip "_id" from foreign keys, unless the foreign-key happens to # be named "_id", in which case the name is retained. column = re.sub('_id$', '', column) or column # Remove characters that are invalid for Python identifiers. column = re.sub(r'[^\w]+', '_', column) if column in RESERVED_WORDS: column += '_' if len(column) and column[0].isdigit(): column = '_' + column return column def introspect(self, table_names=None, literal_column_names=False, include_views=False, snake_case=True): # Retrieve all the tables in the database. tables = self.metadata.database.get_tables(schema=self.schema) if include_views: views = self.metadata.database.get_views(schema=self.schema) tables.extend([view.name for view in views]) if table_names is not None: tables = [table for table in tables if table in table_names] table_set = set(tables) # Store a mapping of table name -> dictionary of columns. columns = {} # Store a mapping of table name -> set of primary key columns. primary_keys = {} # Store a mapping of table -> foreign keys. foreign_keys = {} # Store a mapping of table name -> model name. model_names = {} # Store a mapping of table name -> indexes. indexes = {} # Gather the columns for each table. for table in tables: table_indexes = self.metadata.get_indexes(table, self.schema) table_columns = self.metadata.get_columns(table, self.schema) try: foreign_keys[table] = self.metadata.get_foreign_keys( table, self.schema) except ValueError as exc: foreign_keys[table] = [] else: # If there is a possibility we could exclude a dependent table, # ensure that we introspect it so FKs will work. if table_names is not None: for foreign_key in foreign_keys[table]: if foreign_key.dest_table not in table_set: tables.append(foreign_key.dest_table) table_set.add(foreign_key.dest_table) model_names[table] = self.make_model_name(table, snake_case) # Collect sets of all the column names as well as all the # foreign-key column names. lower_col_names = set(column_name.lower() for column_name in table_columns) fks = set(fk_col.column for fk_col in foreign_keys[table]) for col_name, column in table_columns.items(): if literal_column_names: new_name = re.sub(r'[^\w]+', '_', col_name) else: new_name = self.make_column_name(col_name, col_name in fks, snake_case) # If we have two columns, "parent" and "parent_id", ensure # that when we don't introduce naming conflicts. lower_name = col_name.lower() if lower_name.endswith('_id') and new_name in lower_col_names: new_name = col_name.lower() column.name = new_name for index in table_indexes: if len(index.columns) == 1: column = index.columns[0] if column in table_columns: table_columns[column].unique = index.unique table_columns[column].index = True primary_keys[table] = self.metadata.get_primary_keys( table, self.schema) columns[table] = table_columns indexes[table] = table_indexes # Gather all instances where we might have a `related_name` conflict, # either due to multiple FKs on a table pointing to the same table, # or a related_name that would conflict with an existing field. related_names = {} sort_fn = lambda foreign_key: foreign_key.column for table in tables: models_referenced = set() for foreign_key in sorted(foreign_keys[table], key=sort_fn): try: column = columns[table][foreign_key.column] except KeyError: continue dest_table = foreign_key.dest_table if dest_table in models_referenced: related_names[column] = '%s_%s_set' % ( dest_table, column.name) else: models_referenced.add(dest_table) # On the second pass convert all foreign keys. for table in tables: for foreign_key in foreign_keys[table]: src = columns[foreign_key.table][foreign_key.column] try: dest = columns[foreign_key.dest_table][ foreign_key.dest_column] except KeyError: dest = None src.set_foreign_key( foreign_key=foreign_key, model_names=model_names, dest=dest, related_name=related_names.get(src)) return DatabaseMetadata( columns, primary_keys, foreign_keys, model_names, indexes) def generate_models(self, skip_invalid=False, table_names=None, literal_column_names=False, bare_fields=False, include_views=False): database = self.introspect(table_names, literal_column_names, include_views) models = {} class BaseModel(Model): class Meta: database = self.metadata.database schema = self.schema pending = set() def _create_model(table, models): pending.add(table) for foreign_key in database.foreign_keys[table]: dest = foreign_key.dest_table if dest not in models and dest != table: if dest in pending: warnings.warn('Possible reference cycle found between ' '%s and %s' % (table, dest)) else: _create_model(dest, models) primary_keys = [] columns = database.columns[table] for column_name, column in columns.items(): if column.primary_key: primary_keys.append(column.name) multi_column_indexes = database.multi_column_indexes(table) column_indexes = database.column_indexes(table) class Meta: indexes = multi_column_indexes table_name = table # Fix models with multi-column primary keys. composite_key = False if len(primary_keys) == 0: if 'id' not in columns: Meta.primary_key = False else: primary_keys = columns.keys() if len(primary_keys) > 1: Meta.primary_key = CompositeKey(*[ field.name for col, field in columns.items() if col in primary_keys]) composite_key = True attrs = {'Meta': Meta} for column_name, column in columns.items(): FieldClass = column.field_class if FieldClass is not ForeignKeyField and bare_fields: FieldClass = BareField elif FieldClass is UnknownField: FieldClass = BareField params = { 'column_name': column_name, 'null': column.nullable} if column.primary_key and composite_key: if FieldClass is AutoField: FieldClass = IntegerField params['primary_key'] = False elif column.primary_key and FieldClass is not AutoField: params['primary_key'] = True if column.is_foreign_key(): if column.is_self_referential_fk(): params['model'] = 'self' else: dest_table = column.foreign_key.dest_table if dest_table in models: params['model'] = models[dest_table] else: FieldClass = DeferredForeignKey params['rel_model_name'] = dest_table if column.to_field: params['field'] = column.to_field # Generate a unique related name. params['backref'] = '%s_%s_rel' % (table, column_name) if column.default is not None: constraint = SQL('DEFAULT %s' % column.default) params['constraints'] = [constraint] if not column.is_primary_key(): if column_name in column_indexes: if column_indexes[column_name]: params['unique'] = True elif not column.is_foreign_key(): params['index'] = True else: params['index'] = False attrs[column.name] = FieldClass(**params) try: models[table] = type(str(table), (BaseModel,), attrs) except ValueError: if not skip_invalid: raise finally: if table in pending: pending.remove(table) # Actually generate Model classes. for table, model in sorted(database.model_names.items()): if table not in models: _create_model(table, models) return models def introspect(database, schema=None): introspector = Introspector.from_database(database, schema=schema) return introspector.introspect() def generate_models(database, schema=None, **options): introspector = Introspector.from_database(database, schema=schema) return introspector.generate_models(**options) def print_model(model, indexes=True, inline_indexes=False): print(model._meta.name) for field in model._meta.sorted_fields: parts = [' %s %s' % (field.name, field.field_type)] if field.primary_key: parts.append(' PK') elif inline_indexes: if field.unique: parts.append(' UNIQUE') elif field.index: parts.append(' INDEX') if isinstance(field, ForeignKeyField): parts.append(' FK: %s.%s' % (field.rel_model.__name__, field.rel_field.name)) print(''.join(parts)) if indexes: index_list = model._meta.fields_to_index() if not index_list: return print('\nindex(es)') for index in index_list: parts = [' '] ctx = model._meta.database.get_sql_context() with ctx.scope_values(param='%s', quote='""'): ctx.sql(CommaNodeList(index._expressions)) if index._where: ctx.literal(' WHERE ') ctx.sql(index._where) sql, params = ctx.query() clean = sql % tuple(map(_query_val_transform, params)) parts.append(clean.replace('"', '')) if index._unique: parts.append(' UNIQUE') print(''.join(parts)) def get_table_sql(model): sql, params = model._schema._create_table().query() if model._meta.database.param != '%s': sql = sql.replace(model._meta.database.param, '%s') # Format and indent the table declaration, simplest possible approach. match_obj = re.match(r'^(.+?\()(.+)(\).*)', sql) create, columns, extra = match_obj.groups() indented = ',\n'.join(' %s' % column for column in columns.split(', ')) clean = '\n'.join((create, indented, extra)).strip() return clean % tuple(map(_query_val_transform, params)) def print_table_sql(model): print(get_table_sql(model)) ================================================ FILE: playhouse/shortcuts.py ================================================ import threading from peewee import * from peewee import Alias from peewee import CompoundSelectQuery from peewee import Metadata from peewee import callable_ _clone_set = lambda s: set(s) if s else set() def model_to_dict(model, recurse=True, backrefs=False, only=None, exclude=None, seen=None, extra_attrs=None, fields_from_query=None, max_depth=None, manytomany=False): """ Convert a model instance (and any related objects) to a dictionary. :param bool recurse: Whether foreign-keys should be recursed. :param bool backrefs: Whether lists of related objects should be recursed. :param only: A list (or set) of field instances indicating which fields should be included. :param exclude: A list (or set) of field instances that should be excluded from the dictionary. :param list extra_attrs: Names of model instance attributes or methods that should be included. :param SelectQuery fields_from_query: Query that was source of model. Take fields explicitly selected by the query and serialize them. :param int max_depth: Maximum depth to recurse, value <= 0 means no max. :param bool manytomany: Process many-to-many fields. """ max_depth = -1 if max_depth is None else max_depth if max_depth == 0: recurse = False only = _clone_set(only) extra_attrs = _clone_set(extra_attrs) should_skip = lambda n: (n in exclude) or (only and (n not in only)) if fields_from_query is not None: only.add('__sentinel__') # Add a placeholder to make non-empty. for item in fields_from_query._returning: if isinstance(item, Field): only.add(item) elif isinstance(item, Alias): extra_attrs.add(item._alias) data = {} exclude = _clone_set(exclude) seen = _clone_set(seen) exclude |= seen model_class = type(model) if manytomany: for name, m2m in model._meta.manytomany.items(): if should_skip(name): continue exclude.update((m2m, m2m.rel_model._meta.manytomany[m2m.backref])) for fkf in m2m.through_model._meta.refs: exclude.add(fkf) accum = [] for rel_obj in getattr(model, name): accum.append(model_to_dict( rel_obj, recurse=recurse, backrefs=backrefs, only=only, exclude=exclude, max_depth=max_depth - 1)) data[name] = accum for field in model._meta.sorted_fields: if should_skip(field): continue field_data = model.__data__.get(field.name) if isinstance(field, ForeignKeyField) and recurse: if field_data is not None: rel_obj = getattr(model, field.name) field_data = model_to_dict( rel_obj, recurse=recurse, backrefs=backrefs, only=only, exclude=exclude, seen=seen | set((field,)), max_depth=max_depth - 1) else: field_data = None data[field.name] = field_data if extra_attrs: for attr_name in extra_attrs: attr = getattr(model, attr_name) if callable_(attr): data[attr_name] = attr() else: data[attr_name] = attr if backrefs and recurse: for foreign_key, rel_model in model._meta.backrefs.items(): if foreign_key.backref == '+': continue descriptor = getattr(model_class, foreign_key.backref) if descriptor in exclude or foreign_key in exclude: continue if only and (descriptor not in only) and (foreign_key not in only): continue accum = [] exclude.add(foreign_key) related_query = getattr(model, foreign_key.backref) for rel_obj in related_query: accum.append(model_to_dict( rel_obj, recurse=recurse, backrefs=backrefs, only=only, exclude=exclude, max_depth=max_depth - 1)) data[foreign_key.backref] = accum return data def update_model_from_dict(instance, data, ignore_unknown=False): meta = instance._meta backrefs = dict([(fk.backref, fk) for fk in meta.backrefs]) for key, value in data.items(): if key in meta.combined: field = meta.combined[key] is_backref = False elif key in backrefs: field = backrefs[key] is_backref = True elif ignore_unknown: setattr(instance, key, value) continue else: raise AttributeError('Unrecognized attribute "%s" for model ' 'class %s.' % (key, type(instance))) is_foreign_key = isinstance(field, ForeignKeyField) if not is_backref and is_foreign_key and isinstance(value, dict): try: rel_instance = instance.__rel__[field.name] except KeyError: rel_instance = field.rel_model() setattr( instance, field.name, update_model_from_dict(rel_instance, value, ignore_unknown)) elif is_backref and isinstance(value, (list, tuple)): instances = [ dict_to_model(field.model, row_data, ignore_unknown) for row_data in value] for rel_instance in instances: setattr(rel_instance, field.name, instance) setattr(instance, field.backref, instances) else: setattr(instance, field.name, value) return instance def dict_to_model(model_class, data, ignore_unknown=False): return update_model_from_dict(model_class(), data, ignore_unknown) def insert_where(cls, data, where=None): """ Helper for generating conditional INSERT queries. For example, prevent INSERTing a new tweet if the user has tweeted within the last hour:: INSERT INTO "tweet" ("user_id", "content", "timestamp") SELECT 234, 'some content', now() WHERE NOT EXISTS ( SELECT 1 FROM "tweet" WHERE user_id = 234 AND timestamp > now() - interval '1 hour') Using this helper: cond = ~fn.EXISTS(Tweet.select().where( Tweet.user == user_obj, Tweet.timestamp > one_hour_ago)) iq = insert_where(Tweet, { Tweet.user: user_obj, Tweet.content: 'some content'}, where=cond) res = iq.execute() """ for field, default in cls._meta.defaults.items(): if field.name in data or field in data: continue value = default() if callable_(default) else default data[field] = value fields, values = zip(*data.items()) sq = Select(columns=values).where(where) return cls.insert_from(sq, fields).as_rowcount() class ReconnectMixin(object): """ Mixin class that attempts to automatically reconnect to the database under certain error conditions. For example, MySQL servers will typically close connections that are idle for 28800 seconds ("wait_timeout" setting). If your application makes use of long-lived connections, you may find your connections are closed after a period of no activity. This mixin will attempt to reconnect automatically when these errors occur. This mixin class probably should not be used with Postgres (unless you REALLY know what you are doing) and definitely has no business being used with Sqlite. If you wish to use with Postgres, you will need to adapt the `reconnect_errors` attribute to something appropriate for Postgres. """ reconnect_errors = ( # Error class, error message fragment (or empty string for all). (OperationalError, '2006'), # MySQL server has gone away. (OperationalError, '2013'), # Lost connection to MySQL server. (OperationalError, '2014'), # Commands out of sync. (OperationalError, '4031'), # Client interaction timeout. # mysql-connector raises a slightly different error when an idle # connection is terminated by the server. This is equivalent to 2013. (OperationalError, 'MySQL Connection not available.'), # Postgres error examples: #(OperationalError, 'terminat'), #(InterfaceError, 'connection already closed'), ) def __init__(self, *args, **kwargs): super(ReconnectMixin, self).__init__(*args, **kwargs) # Normalize the reconnect errors to a more efficient data-structure. self._reconnect_errors = {} for exc_class, err_fragment in self.reconnect_errors: self._reconnect_errors.setdefault(exc_class, []) self._reconnect_errors[exc_class].append(err_fragment.lower()) def execute_sql(self, sql, params=None): return self._reconnect(super(ReconnectMixin, self).execute_sql, sql, params) def begin(self, *args, **kwargs): return self._reconnect(super(ReconnectMixin, self).begin, *args, **kwargs) def _reconnect(self, func, *args, **kwargs): try: return func(*args, **kwargs) except Exception as exc: # If we are in a transaction, do not reconnect silently as # any changes could be lost. if self.in_transaction(): raise exc exc_class = type(exc) if exc_class not in self._reconnect_errors: raise exc exc_repr = str(exc).lower() for err_fragment in self._reconnect_errors[exc_class]: if err_fragment in exc_repr: break else: raise exc if not self.is_closed(): self.close() self.connect() return func(*args, **kwargs) def resolve_multimodel_query(query, key='_model_identifier'): mapping = {} accum = [query] while accum: curr = accum.pop() if isinstance(curr, CompoundSelectQuery): accum.extend((curr.lhs, curr.rhs)) continue model_class = curr.model name = model_class._meta.table_name mapping[name] = model_class curr._returning.append(Value(name).alias(key)) def wrapped_iterator(): for row in query.dicts().iterator(): identifier = row.pop(key) model = mapping[identifier] yield model(**row) return wrapped_iterator() class ThreadSafeDatabaseMetadata(Metadata): """ Metadata class to allow swapping database at run-time in a multi-threaded application. To use: class Base(Model): class Meta: model_metadata_class = ThreadSafeDatabaseMetadata """ def __init__(self, *args, **kwargs): # The database attribute is stored in a thread-local. self._database = None self._table = None self._lock = threading.Lock() self._local = threading.local() super(ThreadSafeDatabaseMetadata, self).__init__(*args, **kwargs) def _get_db(self): return getattr(self._local, 'database', self._database) def _set_db(self, db): if self._database is None: self._database = db self._local.database = db database = property(_get_db, _set_db) def set_database(self, database): with self._lock: return super(ThreadSafeDatabaseMetadata, self).set_database(database) @property def table(self): if getattr(self._local, 'table', None) is None: self._local.table = super(ThreadSafeDatabaseMetadata, self).table return self._local.table @table.setter def table(self, value): raise AttributeError('Cannot set the "table".') @table.deleter def table(self): self._local.table = None ================================================ FILE: playhouse/signals.py ================================================ """ Provide django-style hooks for model events. """ from peewee import Model as _Model class Signal(object): def __init__(self): self._flush() def _flush(self): self._receivers = set() self._receiver_list = [] def connect(self, receiver, name=None, sender=None): name = name or receiver.__name__ key = (name, sender) if key not in self._receivers: self._receivers.add(key) self._receiver_list.append((name, receiver, sender)) else: raise ValueError('receiver named %s (for sender=%s) already ' 'connected' % (name, sender or 'any')) def disconnect(self, receiver=None, name=None, sender=None): if receiver: name = name or receiver.__name__ if not name: raise ValueError('a receiver or a name must be provided') key = (name, sender) if key not in self._receivers: raise ValueError('receiver named %s for sender=%s not found.' % (name, sender or 'any')) self._receivers.remove(key) self._receiver_list = [(n, r, s) for n, r, s in self._receiver_list if (n, s) != key] def __call__(self, name=None, sender=None): def decorator(fn): self.connect(fn, name, sender) return fn return decorator def send(self, instance, *args, **kwargs): sender = type(instance) responses = [] for n, r, s in self._receiver_list: if s is None or isinstance(instance, s): responses.append((r, r(sender, instance, *args, **kwargs))) return responses pre_save = Signal() post_save = Signal() pre_delete = Signal() post_delete = Signal() pre_init = Signal() class Model(_Model): def __init__(self, *args, **kwargs): super(Model, self).__init__(*args, **kwargs) pre_init.send(self) def save(self, *args, **kwargs): pk_value = self._pk if self._meta.primary_key else True created = kwargs.get('force_insert', False) or not bool(pk_value) pre_save.send(self, created=created) ret = super(Model, self).save(*args, **kwargs) post_save.send(self, created=created) return ret def delete_instance(self, *args, **kwargs): pre_delete.send(self) ret = super(Model, self).delete_instance(*args, **kwargs) post_delete.send(self) return ret ================================================ FILE: playhouse/sqlcipher_ext.py ================================================ """ Peewee integration with pysqlcipher. Project page: https://github.com/leapcode/pysqlcipher/ **WARNING!!! EXPERIMENTAL!!!** * Although this extention's code is short, it has not been properly peer-reviewed yet and may have introduced vulnerabilities. Also note that this code relies on pysqlcipher and sqlcipher, and the code there might have vulnerabilities as well, but since these are widely used crypto modules, we can expect "short zero days" there. Example usage: from peewee.playground.ciphersql_ext import SqlCipherDatabase db = SqlCipherDatabase('/path/to/my.db', passphrase="don'tuseme4real") * `passphrase`: should be "long enough". Note that *length beats vocabulary* (much exponential), and even a lowercase-only passphrase like easytorememberyethardforotherstoguess packs more noise than 8 random printable characters and *can* be memorized. When opening an existing database, passphrase should be the one used when the database was created. If the passphrase is incorrect, an exception will only be raised **when you access the database**. If you need to ask for an interactive passphrase, here's example code you can put after the `db = ...` line: try: # Just access the database so that it checks the encryption. db.get_tables() # We're looking for a DatabaseError with a specific error message. except peewee.DatabaseError as e: # Check whether the message *means* "passphrase is wrong" if e.args[0] == 'file is encrypted or is not a database': raise Exception('Developer should Prompt user for passphrase ' 'again.') else: # A different DatabaseError. Raise it. raise e See a more elaborate example with this code at https://gist.github.com/thedod/11048875 """ import datetime import decimal import sys from peewee import * try: from sqlcipher3 import dbapi2 as sqlcipher except ImportError: from pysqlcipher3 import dbapi2 as sqlcipher sqlcipher.register_adapter(decimal.Decimal, str) sqlcipher.register_adapter(datetime.date, str) sqlcipher.register_adapter(datetime.time, str) __sqlcipher_version__ = sqlcipher.sqlite_version_info class _SqlCipherDatabase(object): server_version = __sqlcipher_version__ def _connect(self): params = dict(self.connect_params) passphrase = params.pop('passphrase', '').replace("'", "''") conn = sqlcipher.connect(self.database, isolation_level=None, **params) try: if passphrase: conn.execute("PRAGMA key='%s'" % passphrase) self._add_conn_hooks(conn) except: conn.close() raise return conn def set_passphrase(self, passphrase): if not self.is_closed(): raise ImproperlyConfigured('Cannot set passphrase when database ' 'is open. To change passphrase of an ' 'open database use the rekey() method.') self.connect_params['passphrase'] = passphrase def rekey(self, passphrase): if self.is_closed(): self.connect() self.execute_sql("PRAGMA rekey='%s'" % passphrase.replace("'", "''")) self.connect_params['passphrase'] = passphrase return True class SqlCipherDatabase(_SqlCipherDatabase, SqliteDatabase): pass ================================================ FILE: playhouse/sqlite_changelog.py ================================================ from peewee import * from playhouse.sqlite_ext import JSONField class BaseChangeLog(Model): timestamp = DateTimeField(constraints=[SQL('DEFAULT CURRENT_TIMESTAMP')]) action = TextField() table = TextField() primary_key = IntegerField() changes = JSONField() class ChangeLog(object): # Model class that will serve as the base for the changelog. This model # will be subclassed and mapped to your application database. base_model = BaseChangeLog # Template for the triggers that handle updating the changelog table. # table: table name # action: insert / update / delete # new_old: NEW or OLD (OLD is for DELETE) # primary_key: table primary key column name # column_array: output of build_column_array() # change_table: changelog table name template = """CREATE TRIGGER IF NOT EXISTS %(table)s_changes_%(action)s AFTER %(action)s ON %(table)s BEGIN INSERT INTO %(change_table)s ("action", "table", "primary_key", "changes") SELECT '%(action)s', '%(table)s', %(new_old)s."%(primary_key)s", "changes" FROM ( SELECT json_group_object( col, json_array( case when json_valid("oldval") then json("oldval") else "oldval" end, case when json_valid("newval") then json("newval") else "newval" end) ) AS "changes" FROM ( SELECT json_extract(value, '$[0]') as "col", json_extract(value, '$[1]') as "oldval", json_extract(value, '$[2]') as "newval" FROM json_each(json_array(%(column_array)s)) WHERE "oldval" IS NOT "newval" ) ); END;""" drop_template = 'DROP TRIGGER IF EXISTS %(table)s_changes_%(action)s' _actions = ('INSERT', 'UPDATE', 'DELETE') def __init__(self, db, table_name='changelog'): self.db = db self.table_name = table_name def _build_column_array(self, model, use_old, use_new, skip_fields=None): # Builds a list of SQL expressions for each field we are tracking. This # is used as the data source for change tracking in our trigger. col_array = [] for field in model._meta.sorted_fields: if field.primary_key: continue if skip_fields is not None and field.name in skip_fields: continue column = field.column_name new = 'NULL' if not use_new else 'NEW."%s"' % column old = 'NULL' if not use_old else 'OLD."%s"' % column if isinstance(field, JSONField): # Ensure that values are cast to JSON so that the serialization # is preserved when calculating the old / new. if use_old: old = 'json(%s)' % old if use_new: new = 'json(%s)' % new col_array.append("json_array('%s', %s, %s)" % (column, old, new)) return ', '.join(col_array) def trigger_sql(self, model, action, skip_fields=None): assert action in self._actions use_old = action != 'INSERT' use_new = action != 'DELETE' cols = self._build_column_array(model, use_old, use_new, skip_fields) return self.template % { 'table': model._meta.table_name, 'action': action, 'new_old': 'NEW' if action != 'DELETE' else 'OLD', 'primary_key': model._meta.primary_key.column_name, 'column_array': cols, 'change_table': self.table_name} def drop_trigger_sql(self, model, action): assert action in self._actions return self.drop_template % { 'table': model._meta.table_name, 'action': action} @property def model(self): if not hasattr(self, '_changelog_model'): class ChangeLog(self.base_model): class Meta: database = self.db table_name = self.table_name self._changelog_model = ChangeLog return self._changelog_model def install(self, model, skip_fields=None, drop=True, insert=True, update=True, delete=True, create_table=True): ChangeLog = self.model if create_table: ChangeLog.create_table() actions = list(zip((insert, update, delete), self._actions)) if drop: for _, action in actions: self.db.execute_sql(self.drop_trigger_sql(model, action)) for enabled, action in actions: if enabled: sql = self.trigger_sql(model, action, skip_fields) self.db.execute_sql(sql) ================================================ FILE: playhouse/sqlite_ext.py ================================================ import json import re import sys import warnings from peewee import * from peewee import ColumnBase from peewee import EnclosedNodeList from peewee import Entity from peewee import Expression from peewee import Insert from peewee import Node from peewee import NodeList from peewee import OP from peewee import VirtualField from peewee import merge_dict from peewee import sqlite3 from playhouse.sqlite_udf import JSON from playhouse.sqlite_udf import RANK from playhouse.sqlite_udf import register_udf_groups FTS3_MATCHINFO = 'pcx' FTS4_MATCHINFO = 'pcnalx' if sqlite3 is not None: FTS_VERSION = 4 if sqlite3.sqlite_version_info[:3] >= (3, 7, 4) else 3 else: FTS_VERSION = 3 FTS5_MIN_SQLITE_VERSION = (3, 9, 0) class RowIDField(AutoField): auto_increment = True column_name = name = required_name = 'rowid' def bind(self, model, name, *args): if name != self.required_name: raise ValueError('%s must be named "%s".' % (type(self), self.required_name)) super(RowIDField, self).bind(model, name, *args) class DocIDField(RowIDField): column_name = name = required_name = 'docid' class AutoIncrementField(AutoField): def ddl(self, ctx): node_list = super(AutoIncrementField, self).ddl(ctx) return NodeList((node_list, SQL('AUTOINCREMENT'))) class TDecimalField(DecimalField): field_type = 'TEXT' def get_modifiers(self): pass class ISODateTimeField(DateTimeField): formats = [ '%Y-%m-%dT%H:%M:%S.%f%z', '%Y-%m-%dT%H:%M:%S%z', '%Y-%m-%dT%H:%M:%S.%f', '%Y-%m-%dT%H:%M:%S', '%Y-%m-%d', ] def db_value(self, value): if value: return value.isoformat() class JSONPath(ColumnBase): def __init__(self, field, path=None): super(JSONPath, self).__init__() self._field = field self._path = path or () def _converter(self, value): return self._field.python_value(value) @property def path(self): return Value('$%s' % ''.join(self._path)) def __getitem__(self, idx): if isinstance(idx, int) or idx == '#': item = '[%s]' % idx else: item = '.%s' % idx return type(self)(self._field, self._path + (item,)) def append(self, value, as_json=None): if as_json or isinstance(value, (list, dict)): value = fn.json(self._field._json_dumps(value)) return fn.json_set(self._field, self['#'].path, value) def _json_operation(self, func, value, as_json=None): if as_json or isinstance(value, (list, dict)): value = fn.json(self._field._json_dumps(value)) return func(self._field, self.path, value) def insert(self, value, as_json=None): return self._json_operation(fn.json_insert, value, as_json) def set(self, value, as_json=None): return self._json_operation(fn.json_set, value, as_json) def replace(self, value, as_json=None): return self._json_operation(fn.json_replace, value, as_json) def update(self, value): return self.set(fn.json_patch(self, self._field._json_dumps(value))) def remove(self): return fn.json_remove(self._field, self.path) def json_type(self): return fn.json_type(self._field, self.path) def length(self): return fn.json_array_length(self._field, self.path) def children(self): return fn.json_each(self._field, self.path) def tree(self): return fn.json_tree(self._field, self.path) def __sql__(self, ctx): return ctx.sql(fn.json_extract(self._field, self.path) if self._path else self._field) class JSONBPath(JSONPath): def append(self, value, as_json=None): if as_json or isinstance(value, (list, dict)): value = fn.jsonb(self._field._json_dumps(value)) return fn.jsonb_set(self._field, self['#'].path, value) def _json_operation(self, func, value, as_json=None): if as_json or isinstance(value, (list, dict)): value = fn.jsonb(self._field._json_dumps(value)) return func(self._field, self.path, value) def insert(self, value, as_json=None): return self._json_operation(fn.jsonb_insert, value, as_json) def set(self, value, as_json=None): return self._json_operation(fn.jsonb_set, value, as_json) def replace(self, value, as_json=None): return self._json_operation(fn.jsonb_replace, value, as_json) def update(self, value): return self.set(fn.jsonb_patch(self, self._field._json_dumps(value))) def remove(self): return fn.jsonb_remove(self._field, self.path) def __sql__(self, ctx): return ctx.sql(fn.jsonb_extract(self._field, self.path) if self._path else self._field) class JSONField(TextField): field_type = 'JSON' unpack = False Path = JSONPath def __init__(self, json_dumps=None, json_loads=None, **kwargs): self._json_dumps = json_dumps or json.dumps self._json_loads = json_loads or json.loads super(JSONField, self).__init__(**kwargs) def python_value(self, value): if value is not None: try: return self._json_loads(value) except (TypeError, ValueError): return value def db_value(self, value): if value is not None: if not isinstance(value, Node): value = fn.json(self._json_dumps(value)) return value def _e(op): def inner(self, rhs): if isinstance(rhs, (list, dict)): rhs = AsIs(rhs, self.db_value) return Expression(self, op, rhs) return inner __eq__ = _e(OP.EQ) __ne__ = _e(OP.NE) __gt__ = _e(OP.GT) __ge__ = _e(OP.GTE) __lt__ = _e(OP.LT) __le__ = _e(OP.LTE) __hash__ = Field.__hash__ def __getitem__(self, item): return self.Path(self)[item] def extract(self, *paths): paths = [Value(p, converter=False) for p in paths] return fn.json_extract(self, *paths) def extract_json(self, path): return Expression(self, '->', Value(path, converter=False)) def extract_text(self, path): return Expression(self, '->>', Value(path, converter=False)) def append(self, value, as_json=None): return self.Path(self).append(value, as_json) def insert(self, value, as_json=None): return self.Path(self).insert(value, as_json) def set(self, value, as_json=None): return self.Path(self).set(value, as_json) def replace(self, value, as_json=None): return self.Path(self).replace(value, as_json) def update(self, data): return self.Path(self).update(data) def remove(self, *paths): if not paths: return self.Path(self).remove() return fn.json_remove(self, *paths) def json_type(self): return fn.json_type(self) def length(self, path=None): args = (self, path) if path else (self,) return fn.json_array_length(*args) def children(self): """ Schema of `json_each` and `json_tree`: key, value, type TEXT (object, array, string, etc), atom (value for primitive/scalar types, NULL for array and object) id INTEGER (unique identifier for element) parent INTEGER (unique identifier of parent element or NULL) fullkey TEXT (full path describing element) path TEXT (path to the container of the current element) json JSON hidden (1st input parameter to function) root TEXT hidden (2nd input parameter, path at which to start) """ return fn.json_each(self) def tree(self): return fn.json_tree(self) class JSONBField(JSONField): field_type = 'JSONB' Path = JSONBPath def db_value(self, value): if value is not None: if not isinstance(value, Node): value = fn.jsonb(self._json_dumps(value)) return value def json(self): return fn.json(self) def extract(self, *paths): paths = [Value(p, converter=False) for p in paths] return fn.jsonb_extract(self, *paths) def remove(self, *paths): if not paths: return self.Path(self).remove() return fn.jsonb_remove(self, *paths) class SearchField(Field): def __init__(self, unindexed=False, column_name=None, **k): if k: raise ValueError('SearchField does not accept these keyword ' 'arguments: %s.' % sorted(k)) super(SearchField, self).__init__(unindexed=unindexed, column_name=column_name, null=True) def match(self, term): return match(self, term) @property def fts_column_index(self): if not hasattr(self, '_fts_column_index'): search_fields = [f.name for f in self.model._meta.sorted_fields if isinstance(f, SearchField)] self._fts_column_index = search_fields.index(self.name) return self._fts_column_index def highlight(self, left, right): column_idx = self.fts_column_index return fn.highlight(self.model._meta.entity, column_idx, left, right) def snippet(self, left, right, over_length='...', max_tokens=16): if not (0 < max_tokens < 65): raise ValueError('max_tokens must be between 1 and 64 (inclusive)') column_idx = self.fts_column_index return fn.snippet(self.model._meta.entity, column_idx, left, right, over_length, max_tokens) class VirtualTableSchemaManager(SchemaManager): def _create_virtual_table(self, safe=True, **options): options = self.model.clean_options( merge_dict(self.model._meta.options, options)) # Structure: # CREATE VIRTUAL TABLE # USING # ([prefix_arguments, ...] fields, ... [arguments, ...], [options...]) ctx = self._create_context() ctx.literal('CREATE VIRTUAL TABLE ') if safe: ctx.literal('IF NOT EXISTS ') (ctx .sql(self.model) .literal(' USING ')) ext_module = self.model._meta.extension_module if isinstance(ext_module, Node): return ctx.sql(ext_module) ctx.sql(SQL(ext_module)).literal(' ') arguments = [] meta = self.model._meta if meta.prefix_arguments: arguments.extend([SQL(a) for a in meta.prefix_arguments]) # Constraints, data-types, foreign and primary keys are all omitted. for field in meta.sorted_fields: if isinstance(field, (RowIDField)) or field._hidden: continue field_def = [Entity(field.column_name)] if field.unindexed: field_def.append(SQL('UNINDEXED')) arguments.append(NodeList(field_def)) if meta.arguments: arguments.extend([SQL(a) for a in meta.arguments]) if options: arguments.extend(self._create_table_option_sql(options)) return ctx.sql(EnclosedNodeList(arguments)) def _create_table(self, safe=True, **options): if issubclass(self.model, VirtualModel): return self._create_virtual_table(safe, **options) return super(VirtualTableSchemaManager, self)._create_table( safe, **options) class VirtualModel(Model): class Meta: arguments = None extension_module = None prefix_arguments = None primary_key = False schema_manager_class = VirtualTableSchemaManager @classmethod def clean_options(cls, options): return options class BaseFTSModel(VirtualModel): @classmethod def clean_options(cls, options): content = options.get('content') prefix = options.get('prefix') tokenize = options.get('tokenize') content_rowid = options.get('content_rowid') if isinstance(content, str) and content == '': # Special-case content-less full-text search tables. options['content'] = "''" elif isinstance(content, Field): # Special-case to ensure fields are fully-qualified. options['content'] = Entity(content.model._meta.table_name, content.column_name) if content_rowid is not None: options['content_rowid'] = content_rowid if prefix: if isinstance(prefix, (list, tuple)): prefix = ','.join([str(i) for i in prefix]) options['prefix'] = "'%s'" % prefix.strip("' ") if tokenize and cls._meta.extension_module.lower() == 'fts5': # Tokenizers need to be in quoted string for FTS5, but not for FTS3 # or FTS4. options['tokenize'] = '"%s"' % tokenize return options class FTSModel(BaseFTSModel): """ VirtualModel class for creating tables that use either the FTS3 or FTS4 search extensions. Peewee automatically determines which version of the FTS extension is supported and will use FTS4 if possible. """ # FTS3/4 uses "docid" in the same way a normal table uses "rowid". docid = DocIDField() class Meta: extension_module = 'FTS%s' % FTS_VERSION @classmethod def _fts_cmd(cls, cmd): tbl = cls._meta.table_name res = cls._meta.database.execute_sql( "INSERT INTO %s(%s) VALUES('%s');" % (tbl, tbl, cmd)) return res.fetchone() @classmethod def optimize(cls): return cls._fts_cmd('optimize') @classmethod def rebuild(cls): return cls._fts_cmd('rebuild') @classmethod def integrity_check(cls): return cls._fts_cmd('integrity-check') @classmethod def merge(cls, blocks=200, segments=8): return cls._fts_cmd('merge=%s,%s' % (blocks, segments)) @classmethod def automerge(cls, state=True): return cls._fts_cmd('automerge=%s' % (state and '1' or '0')) @classmethod def match(cls, term): """ Generate a `MATCH` expression appropriate for searching this table. """ return match(cls._meta.entity, term) @classmethod def rank(cls, *weights): matchinfo = fn.matchinfo(cls._meta.entity, FTS3_MATCHINFO) return fn.fts_rank(matchinfo, *weights) @classmethod def bm25(cls, *weights): match_info = fn.matchinfo(cls._meta.entity, FTS4_MATCHINFO) return fn.fts_bm25(match_info, *weights) @classmethod def bm25f(cls, *weights): match_info = fn.matchinfo(cls._meta.entity, FTS4_MATCHINFO) return fn.fts_bm25f(match_info, *weights) @classmethod def lucene(cls, *weights): match_info = fn.matchinfo(cls._meta.entity, FTS4_MATCHINFO) return fn.fts_lucene(match_info, *weights) @classmethod def _search(cls, term, weights, with_score, score_alias, score_fn, explicit_ordering): if not weights: rank = score_fn() elif isinstance(weights, dict): weight_args = [] for field in cls._meta.sorted_fields: # Attempt to get the specified weight of the field by looking # it up using it's field instance followed by name. field_weight = weights.get(field, weights.get(field.name, 1.0)) weight_args.append(field_weight) rank = score_fn(*weight_args) else: rank = score_fn(*weights) selection = () order_by = rank if with_score: selection = (cls, rank.alias(score_alias)) if with_score and not explicit_ordering: order_by = SQL(score_alias) return (cls .select(*selection) .where(cls.match(term)) .order_by(order_by)) @classmethod def search(cls, term, weights=None, with_score=False, score_alias='score', explicit_ordering=False): """Full-text search using selected `term`.""" return cls._search( term, weights, with_score, score_alias, cls.rank, explicit_ordering) @classmethod def search_bm25(cls, term, weights=None, with_score=False, score_alias='score', explicit_ordering=False): """Full-text search for selected `term` using BM25 algorithm.""" return cls._search( term, weights, with_score, score_alias, cls.bm25, explicit_ordering) @classmethod def search_bm25f(cls, term, weights=None, with_score=False, score_alias='score', explicit_ordering=False): """Full-text search for selected `term` using BM25 algorithm.""" return cls._search( term, weights, with_score, score_alias, cls.bm25f, explicit_ordering) @classmethod def search_lucene(cls, term, weights=None, with_score=False, score_alias='score', explicit_ordering=False): """Full-text search for selected `term` using BM25 algorithm.""" return cls._search( term, weights, with_score, score_alias, cls.lucene, explicit_ordering) _alphabet = 'abcdefghijklmnopqrstuvwxyz' _alphanum = (set('\t ,"(){}*:_+0123456789') | set(_alphabet) | set(_alphabet.upper()) | set((chr(26),))) _invalid_ascii = set(chr(p) for p in range(128) if chr(p) not in _alphanum) del _alphabet del _alphanum _quote_re = re.compile(r'[^\s"]+|"[^"\\]*(?:\\.[^"\\]*)*"') class FTS5Model(BaseFTSModel): """ Requires SQLite >= 3.9.0. Table options: content: table name of external content, or empty string for "contentless" content_rowid: column name of external content primary key prefix: integer(s). Ex: '2' or '2 3 4' tokenize: porter, unicode61, ascii. Ex: 'porter unicode61' The unicode tokenizer supports the following parameters: * remove_diacritics (1 or 0, default is 1) * tokenchars (string of characters, e.g. '-_' * separators (string of characters) Parameters are passed as alternating parameter name and value, so: {'tokenize': "unicode61 remove_diacritics 0 tokenchars '-_'"} Content-less tables: If you don't need the full-text content in it's original form, you can specify a content-less table. Searches and auxiliary functions will work as usual, but the only values returned when SELECT-ing can be rowid. Also content-less tables do not support UPDATE or DELETE. External content tables: You can set up triggers to sync these, e.g. -- Create a table. And an external content fts5 table to index it. CREATE TABLE tbl(a INTEGER PRIMARY KEY, b); CREATE VIRTUAL TABLE ft USING fts5(b, content='tbl', content_rowid='a'); -- Triggers to keep the FTS index up to date. CREATE TRIGGER tbl_ai AFTER INSERT ON tbl BEGIN INSERT INTO ft(rowid, b) VALUES (new.a, new.b); END; CREATE TRIGGER tbl_ad AFTER DELETE ON tbl BEGIN INSERT INTO ft(fts_idx, rowid, b) VALUES('delete', old.a, old.b); END; CREATE TRIGGER tbl_au AFTER UPDATE ON tbl BEGIN INSERT INTO ft(fts_idx, rowid, b) VALUES('delete', old.a, old.b); INSERT INTO ft(rowid, b) VALUES (new.a, new.b); END; Built-in auxiliary functions: * bm25(tbl[, weight_0, ... weight_n]) * highlight(tbl, col_idx, prefix, suffix) * snippet(tbl, col_idx, prefix, suffix, ?, max_tokens) """ # FTS5 does not support declared primary keys, but we can use the # implicit rowid. rowid = RowIDField() class Meta: extension_module = 'fts5' _error_messages = { 'field_type': ('Besides the implicit `rowid` column, all columns must ' 'be instances of SearchField'), 'index': 'Secondary indexes are not supported for FTS5 models', 'pk': 'FTS5 models must use the default `rowid` primary key', } @classmethod def validate_model(cls): # Perform FTS5-specific validation and options post-processing. if cls._meta.primary_key.name != 'rowid': raise ImproperlyConfigured(cls._error_messages['pk']) for field in cls._meta.fields.values(): if not isinstance(field, (SearchField, RowIDField)): raise ImproperlyConfigured(cls._error_messages['field_type']) if cls._meta.indexes: raise ImproperlyConfigured(cls._error_messages['index']) @classmethod def fts5_installed(cls): if sqlite3.sqlite_version_info[:3] < FTS5_MIN_SQLITE_VERSION: return False # Test in-memory DB to determine if the FTS5 extension is installed. tmp_db = sqlite3.connect(':memory:') try: tmp_db.execute('CREATE VIRTUAL TABLE fts5test USING fts5 (data);') except: try: tmp_db.enable_load_extension(True) tmp_db.load_extension('fts5') except: return False else: cls._meta.database.load_extension('fts5') finally: tmp_db.close() return True @staticmethod def validate_query(query): """ Simple helper function to indicate whether a search query is a valid FTS5 query. Note: this simply looks at the characters being used, and is not guaranteed to catch all problematic queries. """ tokens = _quote_re.findall(query) for token in tokens: if token.startswith('"') and token.endswith('"'): continue if set(token) & _invalid_ascii: return False return True @staticmethod def clean_query(query, replace=chr(26)): """ Clean a query of invalid tokens. """ accum = [] any_invalid = False tokens = _quote_re.findall(query) for token in tokens: if token.startswith('"') and token.endswith('"'): accum.append(token) continue token_set = set(token) invalid_for_token = token_set & _invalid_ascii if invalid_for_token: any_invalid = True for c in invalid_for_token: token = token.replace(c, replace) accum.append(token) if any_invalid: return ' '.join(accum) return query @classmethod def match(cls, term): """ Generate a `MATCH` expression appropriate for searching this table. """ return match(cls._meta.entity, term) @classmethod def rank(cls, *args): return cls.bm25(*args) if args else SQL('rank') @classmethod def bm25(cls, *weights): return fn.bm25(cls._meta.entity, *weights) @classmethod def search(cls, term, weights=None, with_score=False, score_alias='score', explicit_ordering=False): """Full-text search using selected `term`.""" return cls.search_bm25( FTS5Model.clean_query(term), weights, with_score, score_alias, explicit_ordering) @classmethod def search_bm25(cls, term, weights=None, with_score=False, score_alias='score', explicit_ordering=False): """Full-text search using selected `term`.""" if not weights: rank = SQL('rank') elif isinstance(weights, dict): weight_args = [] for field in cls._meta.sorted_fields: if isinstance(field, SearchField) and not field.unindexed: weight_args.append( weights.get(field, weights.get(field.name, 1.0))) rank = fn.bm25(cls._meta.entity, *weight_args) else: rank = fn.bm25(cls._meta.entity, *weights) selection = () order_by = rank if with_score: selection = (cls, rank.alias(score_alias)) if with_score and not explicit_ordering: order_by = SQL(score_alias) return (cls .select(*selection) .where(cls.match(FTS5Model.clean_query(term))) .order_by(order_by)) @classmethod def _fts_cmd_sql(cls, cmd, **extra_params): tbl = cls._meta.entity columns = [tbl] values = [cmd] for key, value in extra_params.items(): columns.append(Entity(key)) values.append(value) return NodeList(( SQL('INSERT INTO'), cls._meta.entity, EnclosedNodeList(columns), SQL('VALUES'), EnclosedNodeList(values))) @classmethod def _fts_cmd(cls, cmd, **extra_params): query = cls._fts_cmd_sql(cmd, **extra_params) return cls._meta.database.execute(query) @classmethod def automerge(cls, level): if not (0 <= level <= 16): raise ValueError('level must be between 0 and 16') return cls._fts_cmd('automerge', rank=level) @classmethod def merge(cls, npages): return cls._fts_cmd('merge', rank=npages) @classmethod def optimize(cls): return cls._fts_cmd('optimize') @classmethod def rebuild(cls): return cls._fts_cmd('rebuild') @classmethod def set_pgsz(cls, pgsz): return cls._fts_cmd('pgsz', rank=pgsz) @classmethod def set_rank(cls, rank_expression): return cls._fts_cmd('rank', rank=rank_expression) @classmethod def delete_all(cls): return cls._fts_cmd('delete-all') @classmethod def integrity_check(cls, rank=0): return cls._fts_cmd('integrity-check', rank=rank) @classmethod def VocabModel(cls, table_type='row', table=None): if table_type not in ('row', 'col', 'instance'): raise ValueError('table_type must be either "row", "col" or ' '"instance".') attr = '_vocab_model_%s' % table_type if not hasattr(cls, attr): class Meta: database = cls._meta.database table_name = table or cls._meta.table_name + '_v' extension_module = fn.fts5vocab( cls._meta.entity, SQL(table_type)) attrs = { 'term': VirtualField(TextField), 'doc': IntegerField(), 'cnt': IntegerField(), 'rowid': RowIDField(), 'Meta': Meta, } if table_type == 'col': attrs['col'] = VirtualField(TextField) elif table_type == 'instance': attrs['offset'] = VirtualField(IntegerField) class_name = '%sVocab' % cls.__name__ setattr(cls, attr, type(class_name, (VirtualModel,), attrs)) return getattr(cls, attr) OP.MATCH = 'MATCH' def match(lhs, rhs): return Expression(lhs, OP.MATCH, rhs) ================================================ FILE: playhouse/sqlite_udf.py ================================================ import collections import datetime import heapq import json import math import os import random import re import struct import sys import threading import zlib try: from urllib.parse import urlparse except ImportError: from urlparse import urlparse SQLITE_DATETIME_FORMATS = ( '%Y-%m-%d %H:%M:%S.%f', '%Y-%m-%d %H:%M:%S.%f%z', '%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M:%S%z', '%Y-%m-%d', '%H:%M:%S', '%H:%M:%S.%f', '%H:%M') from peewee import format_date_time def format_date_time_sqlite(date_value): return format_date_time(date_value, SQLITE_DATETIME_FORMATS) try: from playhouse import _sqlite_udf as cython_udf except ImportError: cython_udf = None # Group udf by function. CONTROL_FLOW = 'control_flow' DATE = 'date' FILE = 'file' HELPER = 'helpers' JSON = 'json' MATH = 'math' RANK = 'rank' STRING = 'string' AGGREGATE_COLLECTION = {} UDF_COLLECTION = {} class synchronized_dict(dict): def __init__(self, *args, **kwargs): super(synchronized_dict, self).__init__(*args, **kwargs) self._lock = threading.Lock() def __getitem__(self, key): with self._lock: return super(synchronized_dict, self).__getitem__(key) def __setitem__(self, key, value): with self._lock: return super(synchronized_dict, self).__setitem__(key, value) def __delitem__(self, key): with self._lock: return super(synchronized_dict, self).__delitem__(key) STATE = synchronized_dict() SETTINGS = synchronized_dict() # Class and function decorators. def aggregate(*groups): def decorator(klass): for group in groups: AGGREGATE_COLLECTION.setdefault(group, []) AGGREGATE_COLLECTION[group].append(klass) return klass return decorator def udf(group, name=None): def decorator(fn): UDF_COLLECTION.setdefault(group, []) UDF_COLLECTION[group].append((fn, name or fn.__name__)) return fn return decorator # Register aggregates / functions with connection. def register_aggregate_groups(db, *groups): seen = set() for group in groups: klasses = AGGREGATE_COLLECTION.get(group, ()) for klass in klasses: name = getattr(klass, 'name', klass.__name__) if name not in seen: seen.add(name) db.register_aggregate(klass, name) def register_udf_groups(db, *groups): seen = set() for group in groups: functions = UDF_COLLECTION.get(group, ()) for function, name in functions: if name not in seen: seen.add(name) db.register_function(function, name) def register_groups(db, *groups): register_aggregate_groups(db, *groups) register_udf_groups(db, *groups) def register_all(db): register_aggregate_groups(db, *AGGREGATE_COLLECTION) register_udf_groups(db, *UDF_COLLECTION) # Begin actual user-defined functions and aggregates. # Scalar functions. @udf(CONTROL_FLOW) def if_then_else(cond, truthy, falsey=None): if cond: return truthy return falsey @udf(DATE) def strip_tz(date_str): date_str = date_str.replace('T', ' ') tz_idx1 = date_str.find('+') if tz_idx1 != -1: return date_str[:tz_idx1] tz_idx2 = date_str.find('-') if tz_idx2 > 13: return date_str[:tz_idx2] return date_str @udf(DATE) def human_delta(nseconds, glue=', '): parts = ( (86400 * 365, 'year'), (86400 * 30, 'month'), (86400 * 7, 'week'), (86400, 'day'), (3600, 'hour'), (60, 'minute'), (1, 'second'), ) accum = [] for offset, name in parts: val, nseconds = divmod(nseconds, offset) if val: suffix = val != 1 and 's' or '' accum.append('%s %s%s' % (val, name, suffix)) if not accum: return '0 seconds' return glue.join(accum) @udf(FILE) def file_ext(filename): try: res = os.path.splitext(filename) except ValueError: return None return res[1] @udf(FILE) def file_read(filename): try: with open(filename) as fh: return fh.read() except: pass @udf(HELPER) def gzip(data, compression=9): if isinstance(data, str): data = bytes(data.encode('raw_unicode_escape')) return zlib.compress(data, compression) @udf(HELPER) def gunzip(data): return zlib.decompress(data) @udf(HELPER) def hostname(url): parse_result = urlparse(url) if parse_result: return parse_result.netloc @udf(HELPER) def toggle(key): key = key.lower() STATE[key] = ret = not STATE.get(key) return ret @udf(HELPER) def setting(key, value=None): if value is None: return SETTINGS.get(key) else: SETTINGS[key] = value return value @udf(HELPER) def clear_settings(): SETTINGS.clear() @udf(HELPER) def clear_toggles(): STATE.clear() @udf(MATH) def randomrange(start, end=None, step=None): if end is None: start, end = 0, start elif step is None: step = 1 return random.randrange(start, end, step) @udf(MATH) def gauss_distribution(mean, sigma): try: return random.gauss(mean, sigma) except ValueError: return None @udf(MATH) def sqrt(n): try: return math.sqrt(n) except ValueError: return None @udf(MATH) def tonumber(s): try: return int(s) except ValueError: try: return float(s) except: return None @udf(STRING) def substr_count(haystack, needle): if not haystack or not needle: return 0 return haystack.count(needle) @udf(STRING) def strip_chars(haystack, chars): return haystack.strip(chars) @udf(JSON) def json_contains(src_json, obj_json): stack = [] try: stack.append((json.loads(obj_json), json.loads(src_json))) except: # Invalid JSON! return False while stack: obj, src = stack.pop() if isinstance(src, dict): if isinstance(obj, dict): for key in obj: if key not in src: return False stack.append((obj[key], src[key])) elif isinstance(obj, list): for item in obj: if item not in src: return False elif obj not in src: return False elif isinstance(src, list): if isinstance(obj, dict): return False elif isinstance(obj, list): try: for i in range(len(obj)): stack.append((obj[i], src[i])) except IndexError: return False elif obj not in src: return False elif obj != src: return False return True # Aggregates. class _heap_agg(object): def __init__(self): self.heap = [] self.ct = 0 def process(self, value): return value def step(self, value): self.ct += 1 heapq.heappush(self.heap, self.process(value)) class _datetime_heap_agg(_heap_agg): def process(self, value): return format_date_time_sqlite(value) @aggregate(DATE) class mintdiff(_datetime_heap_agg): def finalize(self): dtp = min_diff = None while self.heap: if min_diff is None: if dtp is None: dtp = heapq.heappop(self.heap) continue dt = heapq.heappop(self.heap) diff = dt - dtp if min_diff is None or min_diff > diff: min_diff = diff dtp = dt if min_diff is not None: return min_diff.total_seconds() @aggregate(DATE) class avgtdiff(_datetime_heap_agg): def finalize(self): if self.ct < 1: return elif self.ct == 1: return 0 total = ct = 0 dtp = None while self.heap: if total == 0: if dtp is None: dtp = heapq.heappop(self.heap) continue dt = heapq.heappop(self.heap) diff = dt - dtp ct += 1 total += diff.total_seconds() dtp = dt return float(total) / ct @aggregate(DATE) class duration(object): def __init__(self): self._min = self._max = None def step(self, value): dt = format_date_time_sqlite(value) if self._min is None or dt < self._min: self._min = dt if self._max is None or dt > self._max: self._max = dt def finalize(self): if self._min and self._max: td = (self._max - self._min) return td.total_seconds() return None @aggregate(MATH) class mode(object): def __init__(self): self.items = collections.Counter() def step(self, *args): self.items.update(args) def finalize(self): if self.items: return self.items.most_common(1)[0][0] @aggregate(MATH) class minrange(_heap_agg): def finalize(self): if self.ct == 0: return elif self.ct == 1: return 0 prev = min_diff = None while self.heap: if min_diff is None: if prev is None: prev = heapq.heappop(self.heap) continue curr = heapq.heappop(self.heap) diff = curr - prev if min_diff is None or min_diff > diff: min_diff = diff prev = curr return min_diff @aggregate(MATH) class avgrange(_heap_agg): def finalize(self): if self.ct == 0: return elif self.ct == 1: return 0 total = ct = 0 prev = None while self.heap: if total == 0: if prev is None: prev = heapq.heappop(self.heap) continue curr = heapq.heappop(self.heap) diff = curr - prev ct += 1 total += diff prev = curr return float(total) / ct @aggregate(MATH) class _range(object): name = 'range' def __init__(self): self._min = self._max = None def step(self, value): if self._min is None or value < self._min: self._min = value if self._max is None or value > self._max: self._max = value def finalize(self): if self._min is not None and self._max is not None: return self._max - self._min return None @aggregate(MATH) class stddev(object): def __init__(self): self.n = 0 self.values = [] def step(self, v): self.n += 1 self.values.append(v) def finalize(self): if self.n <= 1: return 0 mean = sum(self.values) / self.n return math.sqrt(sum((i - mean) ** 2 for i in self.values) / (self.n - 1)) def _parse_match_info(buf): # See http://sqlite.org/fts3.html#matchinfo bufsize = len(buf) # Length in bytes. return [struct.unpack('@I', buf[i:i+4])[0] for i in range(0, bufsize, 4)] def get_weights(ncol, raw_weights): if not raw_weights: return [1] * ncol else: weights = [0] * ncol for i, weight in enumerate(raw_weights): weights[i] = weight return weights # Ranking implementation, which parse matchinfo. def rank(raw_match_info, *raw_weights): # Handle match_info called w/default args 'pcx' - based on the example rank # function http://sqlite.org/fts3.html#appendix_a match_info = _parse_match_info(raw_match_info) score = 0.0 p, c = match_info[:2] weights = get_weights(c, raw_weights) # matchinfo X value corresponds to, for each phrase in the search query, a # list of 3 values for each column in the search table. # So if we have a two-phrase search query and three columns of data, the # following would be the layout: # p0 : c0=[0, 1, 2], c1=[3, 4, 5], c2=[6, 7, 8] # p1 : c0=[9, 10, 11], c1=[12, 13, 14], c2=[15, 16, 17] for phrase_num in range(p): phrase_info_idx = 2 + (phrase_num * c * 3) for col_num in range(c): weight = weights[col_num] if not weight: continue col_idx = phrase_info_idx + (col_num * 3) # The idea is that we count the number of times the phrase appears # in this column of the current row, compared to how many times it # appears in this column across all rows. The ratio of these values # provides a rough way to score based on "high value" terms. row_hits = match_info[col_idx] all_rows_hits = match_info[col_idx + 1] if row_hits > 0: score += weight * (float(row_hits) / all_rows_hits) return -score # Okapi BM25 ranking implementation (FTS4 only). def bm25(raw_match_info, *args): """ Usage: # Format string *must* be pcnalx # Second parameter to bm25 specifies the index of the column, on # the table being queries. bm25(matchinfo(document_tbl, 'pcnalx'), 1) AS rank """ match_info = _parse_match_info(raw_match_info) K = 1.2 B = 0.75 score = 0.0 P_O, C_O, N_O, A_O = range(4) # Offsets into the matchinfo buffer. term_count = match_info[P_O] # n col_count = match_info[C_O] total_docs = match_info[N_O] # N L_O = A_O + col_count X_O = L_O + col_count # Worked example of pcnalx for two columns and two phrases, 100 docs total. # { # p = 2 # c = 2 # n = 100 # a0 = 4 -- avg number of tokens for col0, e.g. title # a1 = 40 -- avg number of tokens for col1, e.g. body # l0 = 5 -- curr doc has 5 tokens in col0 # l1 = 30 -- curr doc has 30 tokens in col1 # # x000 -- hits this row for phrase0, col0 # x001 -- hits all rows for phrase0, col0 # x002 -- rows with phrase0 in col0 at least once # # x010 -- hits this row for phrase0, col1 # x011 -- hits all rows for phrase0, col1 # x012 -- rows with phrase0 in col1 at least once # # x100 -- hits this row for phrase1, col0 # x101 -- hits all rows for phrase1, col0 # x102 -- rows with phrase1 in col0 at least once # # x110 -- hits this row for phrase1, col1 # x111 -- hits all rows for phrase1, col1 # x112 -- rows with phrase1 in col1 at least once # } weights = get_weights(col_count, args) for i in range(term_count): for j in range(col_count): weight = weights[j] if weight == 0: continue x = X_O + (3 * (j + i * col_count)) term_frequency = float(match_info[x]) # f(qi, D) docs_with_term = float(match_info[x + 2]) # n(qi) # log( (N - n(qi) + 0.5) / (n(qi) + 0.5) ) idf = math.log( (total_docs - docs_with_term + 0.5) / (docs_with_term + 0.5)) if idf <= 0.0: idf = 1e-6 doc_length = float(match_info[L_O + j]) # |D| avg_length = float(match_info[A_O + j]) or 1. # avgdl ratio = doc_length / avg_length num = term_frequency * (K + 1.0) b_part = 1.0 - B + (B * ratio) denom = term_frequency + (K * b_part) pc_score = idf * (num / denom) score += (pc_score * weight) return -score if cython_udf is not None: rank = udf(RANK, 'fts_rank')(cython_udf.peewee_rank) lucene = udf(RANK, 'fts_lucene')(cython_udf.peewee_lucene) bm25 = udf(RANK, 'fts_bm25')(cython_udf.peewee_bm25) bm25f = udf(RANK, 'fts_bm25f')(cython_udf.peewee_bm25f) damerau_levenshtein_dist = udf(STRING)(cython_udf.damerau_levenshtein_dist) levenshtein_dist = udf(STRING)(cython_udf.levenshtein_dist) str_dist = udf(STRING)(cython_udf.str_dist) median = aggregate(MATH)(cython_udf.median) else: rank = udf(RANK, 'fts_rank')(rank) bm25 = udf(RANK, 'fts_bm25')(bm25) ================================================ FILE: playhouse/sqliteq.py ================================================ import logging import weakref from queue import Queue from threading import local as thread_local from threading import Event from threading import Lock from threading import Thread try: import gevent from gevent import Greenlet as GThread from gevent.event import Event as GEvent from gevent.local import local as greenlet_local from gevent.queue import Queue as GQueue except ImportError: GThread = GQueue = GEvent = None from peewee import SqliteDatabase logger = logging.getLogger('peewee.sqliteq') class ResultTimeout(Exception): pass class WriterPaused(Exception): pass class ShutdownException(Exception): pass class AsyncCursor(object): __slots__ = ('sql', 'params', 'timeout', '_event', '_cursor', '_exc', '_idx', '_rows', '_ready') def __init__(self, event, sql, params, timeout): self._event = event self.sql = sql self.params = params self.timeout = timeout self._cursor = self._exc = self._idx = self._rows = None self._ready = False def set_result(self, cursor, exc=None): self._cursor = cursor self._exc = exc self._idx = 0 self._rows = cursor.fetchall() if exc is None else [] self._event.set() return self def _wait(self, timeout=None): timeout = timeout if timeout is not None else self.timeout if not self._event.wait(timeout=timeout) and timeout is not None: raise ResultTimeout('results not ready, timed out.') if self._exc is not None: raise self._exc self._ready = True def __iter__(self): if not self._ready: self._wait() if self._exc is not None: raise self._exc return self def next(self): if not self._ready: self._wait() try: obj = self._rows[self._idx] except IndexError: raise StopIteration else: self._idx += 1 return obj __next__ = next @property def lastrowid(self): if not self._ready: self._wait() return self._cursor.lastrowid @property def rowcount(self): if not self._ready: self._wait() return self._cursor.rowcount @property def description(self): if not self._ready: self._wait() return self._cursor.description def close(self): if self._cursor is not None: self._cursor.close() self._cursor = None def fetchall(self): return list(self) # Iterating implies waiting until populated. def fetchone(self): if not self._ready: self._wait() try: return next(self) except StopIteration: return None SHUTDOWN = StopIteration QUERY = object() PAUSE = object() UNPAUSE = object() class Writer(object): __slots__ = ('database', 'queue') def __init__(self, database, queue): self.database = database self.queue = queue def run(self): conn = self.database.connection() try: while True: try: if conn is None: # Paused. if self.wait_unpause(): conn = self.database.connection() else: conn = self.loop(conn) except ShutdownException: logger.info('writer received shutdown request, exiting.') return finally: if conn is not None: self.database._close(conn) self.database._state.reset() def wait_unpause(self): op, obj = self.queue.get() if op is UNPAUSE: logger.info('writer unpaused - reconnecting to database.') obj.set() return True elif op is SHUTDOWN: raise ShutdownException() elif op is PAUSE: logger.error('writer received pause, but is already paused.') obj.set() else: obj.set_result(None, WriterPaused()) logger.warning('writer paused, not handling %s', obj) def loop(self, conn): op, obj = self.queue.get() if op is QUERY: self.execute(obj) elif op is PAUSE: logger.info('writer paused - closing database connection.') self.database._close(conn) self.database._state.reset() obj.set() return elif op is UNPAUSE: logger.error('writer received unpause, but is already running.') obj.set() elif op is SHUTDOWN: raise ShutdownException() else: logger.error('writer received unsupported object: %s', obj) return conn def execute(self, obj): logger.debug('received query %s', obj.sql) try: cursor = self.database._execute(obj.sql, obj.params) except Exception as execute_err: cursor = None exc = execute_err # python3 is so fucking lame. else: exc = None return obj.set_result(cursor, exc) class SqliteQueueDatabase(SqliteDatabase): WAL_MODE_ERROR_MESSAGE = ('SQLite must be configured to use the WAL ' 'journal mode when using this feature. WAL mode ' 'allows one or more readers to continue reading ' 'while another connection writes to the ' 'database.') def __init__(self, database, use_gevent=False, autostart=True, queue_max_size=None, results_timeout=None, *args, **kwargs): kwargs['check_same_thread'] = False # Lock around starting and stopping write thread operations. self._qlock = Lock() # Ensure that journal_mode is WAL. This value is passed to the parent # class constructor below. pragmas = self._validate_journal_mode(kwargs.pop('pragmas', None)) # Reference to execute_sql on the parent class. Since we've overridden # execute_sql(), this is just a handy way to reference the real # implementation. Parent = super(SqliteQueueDatabase, self) self._execute = Parent.execute_sql # Call the parent class constructor with our modified pragmas. Parent.__init__(database, pragmas=pragmas, *args, **kwargs) self._autostart = autostart self._results_timeout = results_timeout self._is_stopped = True # Get different objects depending on the threading implementation. self._thread_helper = self.get_thread_impl(use_gevent)(queue_max_size) # Create the writer thread, optionally starting it. self._create_write_queue() if self._autostart: self.start() def get_thread_impl(self, use_gevent): return GreenletHelper if use_gevent else ThreadHelper def _validate_journal_mode(self, pragmas=None): if not pragmas: return {'journal_mode': 'wal'} if not isinstance(pragmas, dict): pragmas = dict((k.lower(), v) for (k, v) in pragmas) if pragmas.get('journal_mode', 'wal').lower() != 'wal': raise ValueError(self.WAL_MODE_ERROR_MESSAGE) pragmas['journal_mode'] = 'wal' return pragmas def _create_write_queue(self): self._write_queue = self._thread_helper.queue() def queue_size(self): return self._write_queue.qsize() def execute_sql(self, sql, params=None, timeout=None): if sql.lower().startswith('select'): return self._execute(sql, params) cursor = AsyncCursor( event=self._thread_helper.event(), sql=sql, params=params, timeout=self._results_timeout if timeout is None else timeout) self._write_queue.put((QUERY, cursor)) return cursor def start(self): with self._qlock: if not self._is_stopped: return False def run(): writer = Writer(self, self._write_queue) writer.run() self._writer = self._thread_helper.thread(run) self._writer.start() self._is_stopped = False return True def stop(self): logger.debug('environment stop requested.') with self._qlock: if self._is_stopped: return False self._write_queue.put((SHUTDOWN, None)) writer = self._writer self._is_stopped = True writer.join() # Empty queue of any remaining tasks. while not self._write_queue.empty(): op, obj = self._write_queue.get() if op is PAUSE or op is UNPAUSE: obj.set() elif op is QUERY: obj.set_result(None, ShutdownException()) return True def is_stopped(self): with self._qlock: return self._is_stopped def pause(self): with self._qlock: if self._is_stopped: return False evt = self._thread_helper.event() self._write_queue.put((PAUSE, evt)) evt.wait() def unpause(self): with self._qlock: if self._is_stopped: return False evt = self._thread_helper.event() self._write_queue.put((UNPAUSE, evt)) evt.wait() def __unsupported__(self, *args, **kwargs): raise ValueError('This method is not supported by %r.' % type(self)) atomic = transaction = savepoint = __unsupported__ class ThreadHelper(object): __slots__ = ('queue_max_size',) def __init__(self, queue_max_size=None): self.queue_max_size = queue_max_size def event(self): return Event() def queue(self, max_size=None): max_size = max_size if max_size is not None else self.queue_max_size return Queue(maxsize=max_size or 0) def thread(self, fn, *args, **kwargs): thread = Thread(target=fn, args=args, kwargs=kwargs) thread.daemon = True return thread class GreenletHelper(ThreadHelper): __slots__ = () def event(self): return GEvent() def queue(self, max_size=None): max_size = max_size if max_size is not None else self.queue_max_size return GQueue(maxsize=max_size or 0) def thread(self, fn, *args, **kwargs): def wrap(*a, **k): gevent.sleep() return fn(*a, **k) return GThread(wrap, *args, **kwargs) ================================================ FILE: playhouse/test_utils.py ================================================ from functools import wraps import logging logger = logging.getLogger('peewee') class _QueryLogHandler(logging.Handler): def __init__(self, *args, **kwargs): self.queries = [] logging.Handler.__init__(self, *args, **kwargs) def emit(self, record): # Counts all entries logged to the "peewee" logger by execute_sql(). if record.name == 'peewee': self.queries.append(record) class count_queries(object): def __init__(self, only_select=False): self.only_select = only_select self.count = 0 def get_queries(self): return self._handler.queries def __enter__(self): self._handler = _QueryLogHandler() logger.setLevel(logging.DEBUG) logger.addHandler(self._handler) return self def __exit__(self, exc_type, exc_val, exc_tb): logger.removeHandler(self._handler) if self.only_select: self.count = len([q for q in self._handler.queries if q.msg[0].startswith('SELECT ')]) else: self.count = len(self._handler.queries) class assert_query_count(count_queries): def __init__(self, expected, only_select=False): super(assert_query_count, self).__init__(only_select=only_select) self.expected = expected def __call__(self, f): @wraps(f) def decorated(*args, **kwds): with self: ret = f(*args, **kwds) self._assert_count() return ret return decorated def _assert_count(self): error_msg = '%s != %s' % (self.count, self.expected) assert self.count == self.expected, error_msg def __exit__(self, exc_type, exc_val, exc_tb): super(assert_query_count, self).__exit__(exc_type, exc_val, exc_tb) self._assert_count() ================================================ FILE: pwiz.py ================================================ #!/usr/bin/env python import datetime import os import sys from getpass import getpass from optparse import OptionParser from peewee import * from peewee import __version__ as peewee_version from playhouse.cockroachdb import CockroachDatabase from playhouse.reflection import * HEADER = """from peewee import *%s database = %s('%s'%s) """ BASE_MODEL = """\ class BaseModel(Model): class Meta: database = database """ UNKNOWN_FIELD = """\ class UnknownField(object): def __init__(self, *_, **__): pass """ DATABASE_ALIASES = { CockroachDatabase: ['cockroach', 'cockroachdb', 'crdb'], MySQLDatabase: ['mysql', 'mysqldb'], PostgresqlDatabase: ['postgres', 'postgresql'], SqliteDatabase: ['sqlite', 'sqlite3'], } DATABASE_MAP = dict((value, key) for key in DATABASE_ALIASES for value in DATABASE_ALIASES[key]) def make_introspector(database_type, database_name, **kwargs): if database_type not in DATABASE_MAP: err('Unrecognized database, must be one of: %s' % ', '.join(DATABASE_MAP.keys())) sys.exit(1) schema = kwargs.pop('schema', None) DatabaseClass = DATABASE_MAP[database_type] db = DatabaseClass(database_name, **kwargs) return Introspector.from_database(db, schema=schema) def print_models(introspector, tables=None, preserve_order=False, include_views=False, ignore_unknown=False, snake_case=True): database = introspector.introspect(table_names=tables, include_views=include_views, snake_case=snake_case) db_kwargs = introspector.get_database_kwargs() header = HEADER % ( introspector.get_additional_imports(), introspector.get_database_class().__name__, introspector.get_database_name().replace('\\', '\\\\'), ', **%s' % repr(db_kwargs) if db_kwargs else '') print(header) if not ignore_unknown: print(UNKNOWN_FIELD) print(BASE_MODEL) def _print_table(table, seen, accum=None): accum = accum or [] foreign_keys = database.foreign_keys[table] for foreign_key in foreign_keys: dest = foreign_key.dest_table # In the event the destination table has already been pushed # for printing, then we have a reference cycle. if dest in accum and table not in accum: print('# Possible reference cycle: %s' % dest) # If this is not a self-referential foreign key, and we have # not already processed the destination table, do so now. if dest not in seen and dest not in accum: seen.add(dest) if dest != table: _print_table(dest, seen, accum + [table]) print('class %s(BaseModel):' % database.model_names[table]) columns = database.columns[table].items() if not preserve_order: columns = sorted(columns) primary_keys = database.primary_keys[table] for name, column in columns: skip = all([ name in primary_keys, name == 'id', len(primary_keys) == 1, column.field_class in introspector.pk_classes]) if skip: continue if column.primary_key and len(primary_keys) > 1: # If we have a CompositeKey, then we do not want to explicitly # mark the columns as being primary keys. column.primary_key = False is_unknown = column.field_class is UnknownField if is_unknown and ignore_unknown: disp = '%s - %s' % (column.name, column.raw_column_type or '?') print(' # %s' % disp) else: print(' %s' % column.get_field()) print('') print(' class Meta:') print(' table_name = \'%s\'' % table) multi_column_indexes = database.multi_column_indexes(table) if multi_column_indexes: print(' indexes = (') for fields, unique in sorted(multi_column_indexes): print(' ((%s), %s),' % ( ', '.join("'%s'" % field for field in fields), unique, )) print(' )') if introspector.schema: print(' schema = \'%s\'' % introspector.schema) if len(primary_keys) > 1: pk_field_names = sorted([ field.name for col, field in columns if col in primary_keys]) pk_list = ', '.join("'%s'" % pk for pk in pk_field_names) print(' primary_key = CompositeKey(%s)' % pk_list) elif not primary_keys: print(' primary_key = False') print('') seen.add(table) seen = set() for table in sorted(database.model_names.keys()): if table not in seen: if not tables or table in tables: _print_table(table, seen) def print_header(cmd_line, introspector): timestamp = datetime.datetime.now() print('# Code generated by:') print('# python -m pwiz %s' % cmd_line) print('# Date: %s' % timestamp.strftime('%B %d, %Y %I:%M%p')) print('# Database: %s' % introspector.get_database_name()) print('# Peewee version: %s' % peewee_version) print('') def err(msg): sys.stderr.write('\033[91m%s\033[0m\n' % msg) sys.stderr.flush() def get_option_parser(): parser = OptionParser(usage='usage: %prog [options] database_name') ao = parser.add_option ao('-H', '--host', dest='host') ao('-p', '--port', dest='port', type='int') ao('-u', '--user', dest='user') ao('-P', '--password', dest='password', action='store_true') engines = sorted(DATABASE_MAP) ao('-e', '--engine', dest='engine', choices=engines, help=('Database type, e.g. sqlite, mysql, postgresql or cockroachdb. ' 'Default is "postgresql".')) ao('-s', '--schema', dest='schema') ao('-t', '--tables', dest='tables', help=('Only generate the specified tables. Multiple table names should ' 'be separated by commas.')) ao('-v', '--views', dest='views', action='store_true', help='Generate model classes for VIEWs in addition to tables.') ao('-i', '--info', dest='info', action='store_true', help=('Add database information and other metadata to top of the ' 'generated file.')) ao('-o', '--preserve-order', action='store_true', dest='preserve_order', help='Model definition column ordering matches source table.') ao('-I', '--ignore-unknown', action='store_true', dest='ignore_unknown', help='Ignore fields whose type cannot be determined.') ao('-L', '--legacy-naming', action='store_true', dest='legacy_naming', help='Use legacy table- and column-name generation.') return parser def get_connect_kwargs(options): ops = ('host', 'port', 'user', 'schema') kwargs = dict((o, getattr(options, o)) for o in ops if getattr(options, o)) if options.password: kwargs['password'] = getpass() return kwargs def main(): raw_argv = sys.argv parser = get_option_parser() options, args = parser.parse_args() if len(args) < 1: err('Missing required parameter "database"') parser.print_help() sys.exit(1) connect = get_connect_kwargs(options) database = args[-1] tables = None if options.tables: tables = [table.strip() for table in options.tables.split(',') if table.strip()] engine = options.engine if engine is None: engine = 'sqlite' if os.path.exists(database) else 'postgresql' introspector = make_introspector(engine, database, **connect) if options.info: cmd_line = ' '.join(raw_argv[1:]) print_header(cmd_line, introspector) print_models(introspector, tables, options.preserve_order, options.views, options.ignore_unknown, not options.legacy_naming) if __name__ == '__main__': main() ================================================ FILE: pyproject.toml ================================================ [build-system] requires = ["setuptools", "wheel", "cython"] build-backend="setuptools.build_meta" [project] name = "peewee" dynamic = ["version"] description = "a little orm" readme = "README.rst" authors = [ { name = "Charles Leifer", email = "coleifer@gmail.com" } ] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Topic :: Database", "Topic :: Software Development :: Libraries :: Python Modules", ] [project.scripts] pwiz = "pwiz:main" [project.urls] Repository = "https://github.com/coleifer/peewee" Documentation = "https://docs.peewee-orm.com/" Changelog = "https://github.com/coleifer/peewee/blob/master/CHANGELOG.md" [project.optional-dependencies] mysql = ["pymysql"] postgres = ["psycopg2-binary"] psycopg3 = ["psycopg[binary]"] cysqlite = ["cysqlite"] aiosqlite = ["aiosqlite ", "greenlet"] aiomysql = ["aiomysql", "greenlet"] asyncpg = ["asyncpg ", "greenlet"] [tool.setuptools] packages = ["playhouse"] py-modules = ["peewee", "pwiz"] exclude-package-data = {"playhouse" = ["*.pyx"]} [tool.setuptools.dynamic] version = { attr = "peewee.__version__" } ================================================ FILE: runtests.py ================================================ #!/usr/bin/env python import optparse import os import shutil import sys import unittest USER = os.environ.get('USER') or 'root' def runtests(suite, verbosity=1, failfast=False): runner = unittest.TextTestRunner(verbosity=verbosity, failfast=failfast) results = runner.run(suite) return results.failures, results.errors def get_option_parser(): usage = 'usage: %prog [-e engine_name, other options] module1, module2 ...' parser = optparse.OptionParser(usage=usage) basic = optparse.OptionGroup(parser, 'Basic test options') basic.add_option( '-e', '--engine', dest='engine', help=('Database engine to test, one of ' '[sqlite, postgres, mysql, mysqlconnector, apsw, sqlcipher,' ' cockroachdb, psycopg3]')) basic.add_option('-v', '--verbosity', dest='verbosity', default=1, type='int', help='Verbosity of output') basic.add_option('-f', '--failfast', action='store_true', default=False, dest='failfast', help='Exit on first failure/error.') basic.add_option('-s', '--slow-tests', action='store_true', default=False, dest='slow_tests', help='Run tests that may be slow.') basic.add_option('-a', '--asyncio', action='store_true', default=False, dest='asyncio_tests', help='Run only asyncio tests.') basic.add_option('-A', '--asyncio-stress', action='store_true', default=False, dest='asyncio_stress_test', help='Run asyncio stress test.') parser.add_option_group(basic) db_param_map = ( ('MySQL', 'MYSQL', ( # param default disp default val ('host', 'localhost', 'localhost'), ('port', '3306', ''), ('user', USER, USER), ('password', 'blank', ''))), ('Postgresql', 'PSQL', ( ('host', 'localhost', os.environ.get('PGHOST', '')), ('port', '5432', ''), ('user', 'postgres', os.environ.get('PGUSER', '')), ('password', 'blank', os.environ.get('PGPASSWORD', '')))), ('CockroachDB', 'CRDB', ( # param default disp default val ('host', 'localhost', 'localhost'), ('port', '26257', ''), ('user', 'root', 'root'), ('password', 'blank', '')))) for name, prefix, param_list in db_param_map: group = optparse.OptionGroup(parser, '%s connection options' % name) for param, default_disp, default_val in param_list: dest = '%s_%s' % (prefix.lower(), param) opt = '--%s-%s' % (prefix.lower(), param) group.add_option(opt, default=default_val, dest=dest, help=( '%s database %s. Default %s.' % (name, param, default_disp))) parser.add_option_group(group) return parser def collect_tests(args): suite = unittest.TestSuite() if not args: import tests module_suite = unittest.TestLoader().loadTestsFromModule(tests) suite.addTest(module_suite) else: cleaned = ['tests.%s' % arg if not arg.startswith('tests.') else arg for arg in args] user_suite = unittest.TestLoader().loadTestsFromNames(cleaned) suite.addTest(user_suite) return suite def collect_asyncio_tests(): try: import aiosqlite except ImportError: raise RuntimeError('Need aiosqlite at minimum to run asyncio tests.') suite = unittest.TestSuite() from tests import pwasyncio module_suite = unittest.TestLoader().loadTestsFromModule(pwasyncio) suite.addTest(module_suite) return suite if __name__ == '__main__': parser = get_option_parser() options, args = parser.parse_args() if options.engine: os.environ['PEEWEE_TEST_BACKEND'] = options.engine for db in ('mysql', 'psql', 'crdb'): for key in ('host', 'port', 'user', 'password'): att_name = '_'.join((db, key)) value = getattr(options, att_name, None) if value: os.environ['PEEWEE_%s' % att_name.upper()] = value os.environ['PEEWEE_TEST_VERBOSITY'] = str(options.verbosity) if options.slow_tests: os.environ['PEEWEE_SLOW_TESTS'] = '1' if options.asyncio_tests: suite = collect_asyncio_tests() elif options.asyncio_stress_test: try: import asyncio from tests.pwasyncio_stress import main except ImportError as exc: print('Error: could not import asyncio stress test: %s' % exc) sys.exit(2) sys.exit(asyncio.run(main())) else: suite = collect_tests(args) failures, errors = runtests(suite, options.verbosity, options.failfast) files_to_delete = [ 'peewee_test.db', 'peewee_test', 'tmp.db', 'peewee_test.bdb.db', 'peewee_test.cipher.db'] paths_to_delete = ['peewee_test.bdb.db-journal'] for filename in files_to_delete: if os.path.exists(filename): os.unlink(filename) for path in paths_to_delete: if os.path.exists(path): shutil.rmtree(path) if errors: sys.exit(2) elif failures: sys.exit(1) sys.exit(0) ================================================ FILE: setup.py ================================================ import platform import os from setuptools import setup from setuptools.extension import Extension try: from Cython.Build import cythonize cython_installed = True except ImportError: cython_installed = False if platform.python_implementation() != 'CPython': extension_support = False elif os.environ.get('NO_SQLITE'): # Retain backward-compat for not building C extensions. extension_support = False else: extension_support = True if cython_installed: src_ext = '.pyx' else: src_ext = '.c' cythonize = lambda obj: obj if extension_support: sqlite_udf_module = Extension( 'playhouse._sqlite_udf', ['playhouse/_sqlite_udf' + src_ext]) ext_modules = cythonize([sqlite_udf_module]) else: ext_modules = [] setup( name='peewee', packages=['playhouse'], py_modules=['peewee', 'pwiz'], ext_modules=ext_modules) ================================================ FILE: tests/__init__.py ================================================ import sys import unittest from peewee import OperationalError # Core modules. from .db_tests import * from .expressions import * from .fields import * from .keys import * from .manytomany import * from .models import * from .model_save import * from .model_sql import * from .prefetch_tests import * from .queries import * from .regressions import * from .results import * from .schema import * from .sql import * from .transactions import * # Extensions. try: from .apsw_ext import * except ImportError: print('Unable to import APSW extension tests, skipping.') try: from .cockroachdb import * except: print('Unable to import CockroachDB tests, skipping.') try: from .cysqlite_ext import * except ImportError: print('Unable to import cysqlite tests, skipping.') from .dataset import * from .db_url import * from .extra_fields import * from .hybrid import * from .kv import * from .migrations import * try: import mysql.connector from .mysql_ext import * except ImportError: print('Unable to import mysql-connector, skipping mysql_ext tests.') from .pool import * try: from .postgres import * except (ImportError, ImproperlyConfigured): print('Unable to import postgres extension tests, skipping.') except OperationalError: print('Postgresql test database "peewee_test" not found, skipping ' 'the postgres_ext tests.') from .pwiz_integration import * from .reflection import * from .returning import * from .shortcuts import * from .signals import * try: from .sqlcipher_ext import * except ImportError: print('Unable to import SQLCipher extension tests, skipping.') try: from .sqlite import * except ImportError: print('Unable to import sqlite extension tests, skipping.') try: from .sqlite_changelog import * except ImportError: print('Unable to import sqlite changelog tests, skipping.') from .sqliteq import * from .sqlite_udf import * from .test_utils import * try: from .pwasyncio import * except (ImportError, SyntaxError): print('Unable to import asyncio tests, skipping.') try: from .pydantic_utils import * except (ImportError, SyntaxError): print('Unable to import pydantic tests, skipping.') ================================================ FILE: tests/__main__.py ================================================ import os import sys import unittest src_dir = os.path.realpath(os.path.dirname(os.path.dirname(__file__))) sys.path.insert(0, src_dir) from tests import * if __name__ == '__main__': print('\x1b[1;31m') print(r""" ______ ______ ______ __ __ ______ ______ /\ == \ /\ ___\ /\ ___\ /\ \ _ \ \ /\ ___\ /\ ___\ \ \ _-/ \ \ __\ \ \ __\ \ \ \/ ".\ \ \ \ __\ \ \ __\ \ \_\ \ \_____\ \ \_____\ \ \__/".~\_\ \ \_____\ \ \_____\ \/_/ \/_____/ \/_____/ \/_/ \/_/ \/_____/ \/_____/ """) print('\x1b[1;33m') print(r""" _ _ _ /\ \ / /\ /\_\\ \ \ / / \ / / / \ \ \ / / /\ \ / / / \ \ \ / / /\ \ \ \ \ \____\ \ \ /_/ / \ \ \ \ \________\ \ \ \ \ \ \ \ \/________/\ \ \ \ \ \ \ \ \ \ \ _ \ \ \___\ \ \ \ \_\/\_\\ \/____\ \ \ \/_/\/_/ \_________\/ """) print('\x1b[0m') unittest.main(argv=sys.argv) ================================================ FILE: tests/apsw_ext.py ================================================ import apsw import datetime from playhouse.apsw_ext import * from .base import ModelTestCase from .base import TestModel database = APSWDatabase(':memory:') class User(TestModel): username = TextField() class Message(TestModel): user = ForeignKeyField(User) message = TextField() pub_date = DateTimeField() published = BooleanField() class VTSource(object): def Create(self, db, modulename, dbname, tablename, *args): schema = 'CREATE TABLE x(value)' return schema, VTable() Connect = Create class VTable(object): def BestIndex(self, *args): return def Open(self): return VTCursor() def Disconnect(self): pass Destroy = Disconnect class VTCursor(object): def Filter(self, *a): self.val = 0 def Eof(self): return False def Rowid(self): return self.val def Column(self, col): return self.val def Next(self): self.val += 1 def Close(self): pass class TestAPSWExtension(ModelTestCase): database = database requires = [User, Message] def test_db_register_module(self): database.register_module('series', VTSource()) database.execute_sql('create virtual table foo using series()') curs = database.execute_sql('select * from foo limit 5;') self.assertEqual([v for v, in curs], [0, 1, 2, 3, 4]) database.unregister_module('series') def test_db_register_function(self): @database.func() def title(s): return s.title() curs = self.database.execute_sql('SELECT title(?)', ('heLLo',)) self.assertEqual(curs.fetchone()[0], 'Hello') def test_db_register_aggregate(self): @database.aggregate() class First(object): def __init__(self): self._value = None def step(self, value): if self._value is None: self._value = value def finalize(self): return self._value with database.atomic(): for i in range(10): User.create(username='u%s' % i) query = User.select(fn.First(User.username)).order_by(User.username) self.assertEqual(query.scalar(), 'u0') def test_db_register_collation(self): @database.collation() def reverse(lhs, rhs): lhs, rhs = lhs.lower(), rhs.lower() if lhs < rhs: return 1 return -1 if rhs > lhs else 0 with database.atomic(): for i in range(3): User.create(username='u%s' % i) query = (User .select(User.username) .order_by(User.username.collate('reverse'))) self.assertEqual([u.username for u in query], ['u2', 'u1', 'u0']) def test_db_pragmas(self): test_db = APSWDatabase(':memory:', pragmas=( ('cache_size', '1337'), )) test_db.connect() cs = test_db.execute_sql('PRAGMA cache_size;').fetchone()[0] self.assertEqual(cs, 1337) def test_select_insert(self): for user in ('u1', 'u2', 'u3'): User.create(username=user) self.assertEqual([x.username for x in User.select()], ['u1', 'u2', 'u3']) dt = datetime.datetime(2012, 1, 1, 11, 11, 11) Message.create(user=User.get(User.username == 'u1'), message='herps', pub_date=dt, published=True) Message.create(user=User.get(User.username == 'u2'), message='derps', pub_date=dt, published=False) m1 = Message.get(Message.message == 'herps') self.assertEqual(m1.user.username, 'u1') self.assertEqual(m1.pub_date, dt) self.assertEqual(m1.published, True) m2 = Message.get(Message.message == 'derps') self.assertEqual(m2.user.username, 'u2') self.assertEqual(m2.pub_date, dt) self.assertEqual(m2.published, False) def test_update_delete(self): u1 = User.create(username='u1') u2 = User.create(username='u2') u1.username = 'u1-modified' u1.save() self.assertEqual(User.select().count(), 2) self.assertEqual(User.get(User.username == 'u1-modified').id, u1.id) u1.delete_instance() self.assertEqual(User.select().count(), 1) def test_transaction_handling(self): dt = datetime.datetime(2012, 1, 1, 11, 11, 11) def do_ctx_mgr_error(): with self.database.transaction(): User.create(username='u1') raise ValueError self.assertRaises(ValueError, do_ctx_mgr_error) self.assertEqual(User.select().count(), 0) def do_ctx_mgr_success(): with self.database.transaction(): u = User.create(username='test') Message.create(message='testing', user=u, pub_date=dt, published=1) do_ctx_mgr_success() self.assertEqual(User.select().count(), 1) self.assertEqual(Message.select().count(), 1) def create_error(): with self.database.atomic(): u = User.create(username='test') Message.create(message='testing', user=u, pub_date=dt, published=1) raise ValueError self.assertRaises(ValueError, create_error) self.assertEqual(User.select().count(), 1) def create_success(): with self.database.atomic(): u = User.create(username='test') Message.create(message='testing', user=u, pub_date=dt, published=1) create_success() self.assertEqual(User.select().count(), 2) self.assertEqual(Message.select().count(), 2) def test_exists_regression(self): User.create(username='u1') self.assertTrue(User.select().where(User.username == 'u1').exists()) self.assertFalse(User.select().where(User.username == 'ux').exists()) ================================================ FILE: tests/base.py ================================================ from contextlib import contextmanager from functools import wraps import datetime import logging import os import re import unittest from peewee import * from peewee import sqlite3 from playhouse.cockroachdb import CockroachDatabase from playhouse.cockroachdb import NESTED_TX_MIN_VERSION from playhouse.mysql_ext import MariaDBConnectorDatabase from playhouse.mysql_ext import MySQLConnectorDatabase try: from playhouse.cysqlite_ext import CySqliteDatabase except ImportError: CySqliteDatabase = None logger = logging.getLogger('peewee') def db_loader(engine, name='peewee_test', db_class=None, **params): if db_class is None: engine_aliases = { SqliteDatabase: ['sqlite', 'sqlite3'], CySqliteDatabase: ['cysqlite'], MySQLDatabase: ['mysql'], PostgresqlDatabase: ['postgres', 'postgresql', 'psycopg3'], MySQLConnectorDatabase: ['mysqlconnector'], MariaDBConnectorDatabase: ['mariadb', 'maridbconnector'], CockroachDatabase: ['cockroach', 'cockroachdb', 'crdb'], } engine_map = dict((alias, db) for db, aliases in engine_aliases.items() for alias in aliases if db is not None) if engine.lower() not in engine_map: raise Exception('Unsupported engine: %s.' % engine) db_class = engine_map[engine.lower()] if issubclass(db_class, SqliteDatabase) and not name.endswith('.db'): name = '%s.db' % name if name != ':memory:' else name elif issubclass(db_class, MySQLDatabase): params.update(MYSQL_PARAMS) elif issubclass(db_class, CockroachDatabase): params.update(CRDB_PARAMS) elif issubclass(db_class, PostgresqlDatabase): params.update(PSQL_PARAMS) return db_class(name, **params) def get_in_memory_db(**params): backend = 'cysqlite' if BACKEND == 'cysqlite' else 'sqlite3' return db_loader(backend, ':memory:', **params) def get_sqlite_db(): backend = 'cysqlite' if BACKEND == 'cysqlite' else 'sqlite3' return db_loader(backend) BACKEND = os.environ.get('PEEWEE_TEST_BACKEND') or 'sqlite' VERBOSITY = int(os.environ.get('PEEWEE_TEST_VERBOSITY') or 1) SLOW_TESTS = bool(os.environ.get('PEEWEE_SLOW_TESTS')) # What family of database are we using. IS_SQLITE = BACKEND.startswith(('sqlite', 'cysqlite')) IS_MYSQL = BACKEND.startswith(('mysql', 'maria')) IS_POSTGRESQL = BACKEND.startswith(('postgres', 'psycopg')) # Specific database or driver. IS_CRDB = BACKEND in ('cockroach', 'cockroachdb', 'crdb') IS_PSYCOPG3 = BACKEND == 'psycopg3' IS_CYSQLITE = BACKEND == 'cysqlite' if IS_MYSQL: try: import pymysql except ImportError: raise ImportError('pymysql is not installed') if BACKEND.startswith('postgres'): try: import psycopg2 except ImportError: raise ImportError('psycopg2 is not installed') if IS_PSYCOPG3: try: import psycopg except ImportError: raise ImportError('psycopg3 is not installed') def make_db_params(key): params = {} env_vars = [(part, 'PEEWEE_%s_%s' % (key, part.upper())) for part in ('host', 'port', 'user', 'password')] for param, env_var in env_vars: value = os.environ.get(env_var) if value: params[param] = int(value) if param == 'port' else value return params CRDB_PARAMS = make_db_params('CRDB') MYSQL_PARAMS = make_db_params('MYSQL') PSQL_PARAMS = make_db_params('PSQL') if IS_PSYCOPG3: PSQL_PARAMS['prefer_psycopg3'] = True if VERBOSITY > 1: handler = logging.StreamHandler() handler.setLevel(logging.INFO) logger.addHandler(handler) if VERBOSITY > 2: handler.setLevel(logging.DEBUG) def new_connection(**kwargs): return db_loader(BACKEND, 'peewee_test', **kwargs) db = new_connection() # Database-specific feature flags. IS_SQLITE_OLD = IS_SQLITE and sqlite3.sqlite_version_info < (3, 18) IS_SQLITE_15 = IS_SQLITE and sqlite3.sqlite_version_info >= (3, 15) IS_SQLITE_24 = IS_SQLITE and sqlite3.sqlite_version_info >= (3, 24) IS_SQLITE_25 = IS_SQLITE and sqlite3.sqlite_version_info >= (3, 25) IS_SQLITE_30 = IS_SQLITE and sqlite3.sqlite_version_info >= (3, 30) IS_SQLITE_35 = IS_SQLITE and sqlite3.sqlite_version_info >= (3, 35) IS_SQLITE_37 = IS_SQLITE and sqlite3.sqlite_version_info >= (3, 37) IS_SQLITE_9 = IS_SQLITE and sqlite3.sqlite_version_info >= (3, 9) IS_MYSQL_ADVANCED_FEATURES = False IS_MYSQL_JSON = False if IS_MYSQL: db.connect() server_info = db.server_version if server_info[0] == 8 or server_info[:2] >= (10, 2): IS_MYSQL_ADVANCED_FEATURES = True elif server_info[0] == 0: logger.warning('Could not determine mysql server version.') if server_info[0] >= 8 or ((5, 7) <= server_info[:2] <= (6, 0)): # Needs actual MySQL - not MariaDB. IS_MYSQL_JSON = True db.close() if not IS_MYSQL_ADVANCED_FEATURES: logger.warning('MySQL too old to test certain advanced features.') if IS_CRDB: db.connect() IS_CRDB_NESTED_TX = db.server_version >= NESTED_TX_MIN_VERSION db.close() else: IS_CRDB_NESTED_TX = False class TestModel(Model): class Meta: database = db legacy_table_names = False def __sql__(q, **state): return Context(**state).sql(q).query() class QueryLogHandler(logging.Handler): def __init__(self, *args, **kwargs): self.queries = [] logging.Handler.__init__(self, *args, **kwargs) def emit(self, record): self.queries.append(record) class BaseTestCase(unittest.TestCase): def setUp(self): self._qh = QueryLogHandler() logger.setLevel(logging.DEBUG) logger.addHandler(self._qh) def tearDown(self): logger.removeHandler(self._qh) def assertIsNone(self, value): self.assertTrue(value is None, '%r is not None' % value) def assertIsNotNone(self, value): self.assertTrue(value is not None, '%r is None' % value) @contextmanager def assertRaisesCtx(self, exceptions): try: yield except Exception as exc: if not isinstance(exc, exceptions): raise AssertionError('Got %s, expected %s' % (exc, exceptions)) else: raise AssertionError('No exception was raised.') def assertSQL(self, query, sql, params=None, **state): database = getattr(self, 'database', None) or db state.setdefault('conflict_statement', database.conflict_statement) state.setdefault('conflict_update', database.conflict_update) qsql, qparams = __sql__(query, **state) self.assertEqual(qsql, sql) if params is not None: self.assertEqual(qparams, params) def assertHistory(self, n, expected): queries = [logrecord.msg for logrecord in self._qh.queries[-n:]] queries = [(sql.replace('%s', '?').replace('`', '"'), params) for sql, params in queries] self.assertEqual(queries, expected) @property def history(self): return self._qh.queries def reset_sql_history(self): self._qh.queries = [] @contextmanager def assertQueryCount(self, num): qc = len(self.history) yield self.assertEqual(len(self.history) - qc, num) class DatabaseTestCase(BaseTestCase): database = db def setUp(self): if not self.database.is_closed(): self.database.close() self.database.connect() super(DatabaseTestCase, self).setUp() def tearDown(self): super(DatabaseTestCase, self).tearDown() self.database.close() def execute(self, sql, params=None): return self.database.execute_sql(sql, params) class ModelDatabaseTestCase(DatabaseTestCase): database = db requires = None def setUp(self): super(ModelDatabaseTestCase, self).setUp() self._db_mapping = {} # Override the model's database object with test db. if self.requires: for model in self.requires: self._db_mapping[model] = model._meta.database model._meta.set_database(self.database) def tearDown(self): # Restore the model's previous database object. if self.requires: for model in self.requires: model._meta.set_database(self._db_mapping[model]) super(ModelDatabaseTestCase, self).tearDown() class ModelTestCase(ModelDatabaseTestCase): database = db requires = None def setUp(self): super(ModelTestCase, self).setUp() if self.requires: self.database.drop_tables(self.requires, safe=True) self.database.create_tables(self.requires) def tearDown(self): # Restore the model's previous database object. try: if self.requires: self.database.drop_tables(self.requires, safe=True) finally: super(ModelTestCase, self).tearDown() def requires_models(*models): def decorator(method): @wraps(method) def inner(self): with self.database.bind_ctx(models, False, False): self.database.drop_tables(models, safe=True) self.database.create_tables(models) try: method(self) finally: try: self.database.drop_tables(models) except: pass return inner return decorator def skip_if(expr, reason='n/a'): def decorator(method): return unittest.skipIf(expr, reason)(method) return decorator def skip_unless(expr, reason='n/a'): def decorator(method): return unittest.skipUnless(expr, reason)(method) return decorator def slow_test(): def decorator(method): return unittest.skipUnless(SLOW_TESTS, 'skipping slow test')(method) return decorator def requires_sqlite(method): return skip_unless(IS_SQLITE, 'requires sqlite')(method) def requires_mysql(method): return skip_unless(IS_MYSQL, 'requires mysql')(method) def requires_postgresql(method): return skip_unless(IS_POSTGRESQL, 'requires postgresql')(method) def requires_pglike(method): return skip_unless(IS_POSTGRESQL or IS_CRDB, 'requires pg-like')(method) ================================================ FILE: tests/base_models.py ================================================ from peewee import * from .base import TestModel class Person(TestModel): first = CharField() last = CharField() dob = DateField(index=True) class Meta: indexes = ( (('first', 'last'), True), ) class Note(TestModel): author = ForeignKeyField(Person) content = TextField() class Category(TestModel): parent = ForeignKeyField('self', backref='children', null=True) name = CharField(max_length=20, primary_key=True) class Relationship(TestModel): from_person = ForeignKeyField(Person, backref='relations') to_person = ForeignKeyField(Person, backref='related_to') class Register(TestModel): value = IntegerField() class User(TestModel): username = CharField() class Meta: table_name = 'users' class Account(TestModel): email = CharField() user = ForeignKeyField(User, backref='accounts', null=True) class Tweet(TestModel): user = ForeignKeyField(User, backref='tweets') content = TextField() timestamp = TimestampField() class Favorite(TestModel): user = ForeignKeyField(User, backref='favorites') tweet = ForeignKeyField(Tweet, backref='favorites') class Sample(TestModel): counter = IntegerField() value = FloatField(default=1.0) class SampleMeta(TestModel): sample = ForeignKeyField(Sample, backref='metadata') value = FloatField(default=0.0) class A(TestModel): a = TextField() class B(TestModel): a = ForeignKeyField(A, backref='bs') b = TextField() class C(TestModel): b = ForeignKeyField(B, backref='cs') c = TextField() class Emp(TestModel): first = CharField() last = CharField() empno = CharField(unique=True) class Meta: indexes = ( (('first', 'last'), True), ) class OCTest(TestModel): a = CharField(unique=True) b = IntegerField(default=0) c = IntegerField(default=0) class UKVP(TestModel): key = TextField() value = IntegerField() extra = IntegerField() class Meta: # Partial index, the WHERE clause must be reflected in the conflict # target. indexes = [ SQL('CREATE UNIQUE INDEX "ukvp_kve" ON "ukvp" ("key", "value") ' 'WHERE "extra" > 1')] class DfltM(TestModel): name = CharField() dflt1 = IntegerField(default=1) dflt2 = IntegerField(default=lambda: 2) dfltn = IntegerField(null=True) ================================================ FILE: tests/cockroachdb.py ================================================ import datetime import uuid from peewee import * from playhouse.cockroachdb import * from .base import IS_CRDB from .base import ModelTestCase from .base import TestModel from .base import db from .base import requires_models from .base import skip_unless from .base_models import User from .postgres_helpers import BaseBinaryJsonFieldTestCase class KV(TestModel): k = TextField(unique=True) v = IntegerField() class Arr(TestModel): title = TextField() tags = ArrayField(TextField, index=False) class JsonModel(TestModel): data = JSONField() class Normal(TestModel): data = TextField() class UID(TestModel): id = UUIDKeyField() title = TextField() class RID(TestModel): id = RowIDField() title = TextField() class UIDNote(TestModel): uid = ForeignKeyField(UID, backref='notes') note = TextField() @skip_unless(IS_CRDB) class TestCockroachDatabase(ModelTestCase): @requires_models(KV) def test_retry_transaction_ok(self): @self.database.retry_transaction() def succeeds(db): k1 = KV.create(k='k1', v=1) k2 = KV.create(k='k2', v=2) return [k1.id, k2.id] id_list = succeeds() self.assertEqual(KV.select().count(), 2) kv_list = [kv.id for kv in KV.select().order_by(KV.k)] self.assertEqual(kv_list, id_list) @requires_models(KV) def test_retry_transfer_example(self): k1 = KV.create(k='k1', v=100) k2 = KV.create(k='k2', v=1) def transfer_funds(from_k, to_k, amt): query = KV.select().where(KV.k.in_((from_k, to_k))) ka, kb = list(query) if from_k != ka.k: ka, kb = kb, ka # Swap order. if ka.v < amt: return False, ka.v, kb.v from_v, = (KV .update(v=KV.v - amt) .where(KV.k == from_k) .returning(KV.v) .execute()) to_v, = (KV .update(v=KV.v + amt) .where(KV.k == to_k) .returning(KV.v) .execute()) return True, from_v.v, to_v.v def thunk(db_ref): return transfer_funds('k1', 'k2', 90) self.assertEqual(run_transaction(self.database, thunk), (True, 10, 91)) def thunk(db_ref): return transfer_funds('k1', 'k2', 5) self.assertEqual(run_transaction(self.database, thunk), (True, 5, 96)) def thunk(db_ref): return transfer_funds('k1', 'k2', 6) self.assertEqual(run_transaction(self.database, thunk), (False, 5, 96)) @requires_models(KV) def test_retry_transfer_example2(self): k1 = KV.create(k='k1', v=100) k2 = KV.create(k='k2', v=1) def transfer_funds(from_k, to_k, amount): def thunk(db_ref): src, dest = KV.select().where(KV.k.in_([from_k, to_k])) if src.k != from_k: src, dest = dest, src if src.v < amount: return False, src.v, dest.v src, = (KV .update(v=KV.v - amount) .where(KV.k == from_k) .returning(KV.v) .execute()) dest, = (KV .update(v=KV.v + amount) .where(KV.k == to_k) .returning(KV.v) .execute()) return True, src.v, dest.v return run_transaction(self.database, thunk, max_attempts=10) self.assertEqual(transfer_funds('k1', 'k2', 90), (True, 10, 91)) self.assertEqual(transfer_funds('k1', 'k2', 11), (False, 10, 91)) self.assertEqual(transfer_funds('k1', 'k2', 10), (True, 0, 101)) @requires_models(KV) def test_retry_transaction_integrityerror(self): KV.create(k='kx', v=0) @self.database.retry_transaction() def fails(db): KV.create(k='k1', v=1) KV.create(k='kx', v=1) with self.assertRaises(IntegrityError): fails() self.assertEqual(KV.select().count(), 1) kv = KV.get(KV.k == 'kx') self.assertEqual(kv.v, 0) @requires_models(KV) def test_run_transaction_helper(self): def succeeds(db): KV.insert_many([('k%s' % i, i) for i in range(10)]).execute() run_transaction(self.database, succeeds) self.assertEqual([(kv.k, kv.v) for kv in KV.select().order_by(KV.k)], [('k%s' % i, i) for i in range(10)]) @requires_models(KV) def test_cannot_nest_run_transaction(self): def insert_row(db): KV.create(k='k1', v=1) with self.database.atomic(): self.assertRaises(Exception, run_transaction, self.database, insert_row) self.assertEqual(KV.select().count(), 0) @requires_models(User) def test_retry_transaction_docs_example(self): def create_user(username): def thunk(db_ref): return User.create(username=username) return self.database.run_transaction(thunk, max_attempts=5) users = [create_user(u) for u in 'abc'] self.assertEqual([u.username for u in users], ['a', 'b', 'c']) query = User.select().order_by(User.username) self.assertEqual([u.username for u in query], ['a', 'b', 'c']) @requires_models(KV) def test_retry_transaction_decorator(self): @self.database.retry_transaction() def retry_decorator(db): content = [] for i in range(5): kv = KV.create(k='k%s' % i, v=i) content.append(kv.k) return content self.assertEqual(retry_decorator(), ['k0', 'k1', 'k2', 'k3', 'k4']) @requires_models(Arr) def test_array_field(self): a1 = Arr.create(title='a1', tags=['t1', 't2']) a2 = Arr.create(title='a2', tags=['t2', 't3']) # Ensure we can read an array back. a1_db = Arr.get(Arr.title == 'a1') self.assertEqual(a1_db.tags, ['t1', 't2']) # Ensure we can filter on arrays. a2_db = Arr.get(Arr.tags == ['t2', 't3']) self.assertEqual(a2_db.id, a2.id) # Item lookups. a1_db = Arr.get(Arr.tags[1] == 't2') self.assertEqual(a1_db.id, a1.id) self.assertRaises(Arr.DoesNotExist, Arr.get, Arr.tags[2] == 'x') @requires_models(Arr) def test_array_field_search(self): def assertAM(where, id_list): query = Arr.select().where(where).order_by(Arr.title) self.assertEqual([a.id for a in query], id_list) data = ( ('a1', ['t1', 't2']), ('a2', ['t2', 't3']), ('a3', ['t3', 't4'])) id_list = Arr.insert_many(data).execute() a1, a2, a3 = [pk for pk, in id_list] assertAM(Value('t2') == fn.ANY(Arr.tags), [a1, a2]) assertAM(Value('t1') == fn.Any(Arr.tags), [a1]) assertAM(Value('tx') == fn.Any(Arr.tags), []) # Use the contains operator explicitly. assertAM(SQL("tags::text[] @> ARRAY['t2']"), [a1, a2]) # Use the porcelain. assertAM(Arr.tags.contains('t2'), [a1, a2]) assertAM(Arr.tags.contains('t3'), [a2, a3]) assertAM(Arr.tags.contains('t1', 't2'), [a1]) assertAM(Arr.tags.contains('t3', 't4'), [a3]) assertAM(Arr.tags.contains('t2', 't3', 't4'), []) assertAM(Arr.tags.contains_any('t2'), [a1, a2]) assertAM(Arr.tags.contains_any('t3'), [a2, a3]) assertAM(Arr.tags.contains_any('t1', 't2'), [a1, a2]) assertAM(Arr.tags.contains_any('t3', 't4'), [a2, a3]) assertAM(Arr.tags.contains_any('t2', 't3', 't4'), [a1, a2, a3]) @requires_models(Arr) def test_array_field_index(self): a1 = Arr.create(title='a1', tags=['a1', 'a2']) a2 = Arr.create(title='a2', tags=['a2', 'a3', 'a4', 'a5']) # NOTE: CRDB does not support array slicing. query = (Arr .select(Arr.tags[1].alias('st')) .order_by(Arr.title)) self.assertEqual([a.st for a in query], ['a2', 'a3']) @requires_models(UID) def test_uuid_key_field(self): # UUID primary-key is automatically populated and returned, and is of # the correct type. u1 = UID.create(title='u1') self.assertTrue(u1.id is not None) self.assertTrue(isinstance(u1.id, uuid.UUID)) # Bulk-insert works as expected. id_list = UID.insert_many([('u2',), ('u3',)]).execute() u2_id, u3_id = [pk for pk, in id_list] self.assertTrue(isinstance(u2_id, uuid.UUID)) # We can perform lookups using UUID() type. u2 = UID.get(UID.id == u2_id) self.assertEqual(u2.title, 'u2') # Get the UUID hex and query using that. u3 = UID.get(UID.id == u3_id.hex) self.assertEqual(u3.title, 'u3') @requires_models(RID) def test_rowid_field(self): r1 = RID.create(title='r1') self.assertTrue(r1.id is not None) # Bulk-insert works as expected. id_list = RID.insert_many([('r2',), ('r3',)]).execute() r2_id, r3_id = [pk for pk, in id_list] r2 = RID.get(RID.id == r2_id) self.assertEqual(r2.title, 'r2') @requires_models(KV) def test_readonly_transaction(self): kv = KV.create(k='k1', v=1) # Table doesn't exist yet. with self.assertRaises((ProgrammingError, InternalError)): with self.database.atomic('-10s'): kv_db = KV.get(KV.k == 'k1') # Cannot write in a read-only transaction with self.assertRaises((ProgrammingError, InternalError)): with self.database.atomic(datetime.datetime.now()): KV.create(k='k2', v=2) # Without system time there are no issues. with self.database.atomic(): kv_db = KV.get(KV.k == 'k1') self.assertEqual(kv.id, kv_db.id) @requires_models(KV) def test_transaction_priority(self): with self.database.atomic(priority='HIGH'): KV.create(k='k1', v=1) with self.assertRaises(IntegrityError): with self.database.atomic(priority='LOW'): KV.create(k='k1', v=2) with self.assertRaises(ValueError): with self.database.atomic(priority='HUH'): KV.create(k='k2', v=2) self.assertEqual(KV.select().count(), 1) kv = KV.get() self.assertEqual((kv.k, kv.v), ('k1', 1)) @requires_models(UID, UIDNote) def test_uuid_key_as_fk(self): # This is covered thoroughly elsewhere, but added here just for fun. u1, u2, u3 = [UID.create(title='u%s' % i) for i in (1, 2, 3)] UIDNote.create(uid=u1, note='u1-1') UIDNote.create(uid=u2, note='u2-1') UIDNote.create(uid=u2, note='u2-2') with self.assertQueryCount(1): query = (UIDNote .select(UIDNote, UID) .join(UID) .where(UID.title == 'u2') .order_by(UIDNote.note)) self.assertEqual([(un.note, un.uid.title) for un in query], [('u2-1', 'u2'), ('u2-2', 'u2')]) query = (UID .select(UID, fn.COUNT(UIDNote.id).alias('note_count')) .join(UIDNote, JOIN.LEFT_OUTER) .group_by(UID) .order_by(fn.COUNT(UIDNote.id).desc())) self.assertEqual([(u.title, u.note_count) for u in query], [('u2', 2), ('u1', 1), ('u3', 0)]) @skip_unless(IS_CRDB) class TestCockroachDatabaseJson(BaseBinaryJsonFieldTestCase, ModelTestCase): database = db M = JsonModel N = Normal requires = [JsonModel, Normal] # General integration tests. class KV2(TestModel): k2 = CharField() v2 = IntegerField() class Post(TestModel): content = TextField() timestamp = DateTimeField(default=datetime.datetime.now) class PostNote(TestModel): post = ForeignKeyField(Post, backref='notes', primary_key=True) note = TextField() @skip_unless(IS_CRDB) class TestCockroachIntegration(ModelTestCase): @requires_models(KV, KV2) def test_compound_select(self): KV.insert_many([('10', 1), ('40', 4)]).execute() KV2.insert_many([('20', 2), ('30', 3)]).execute() lhs = KV.select(KV.k.cast('INT'), KV.v) rhs = KV2.select(KV2.k2.cast('INT'), KV2.v2) query = (lhs | rhs).order_by(SQL('1')) self.assertEqual([(obj.k, obj.v) for obj in query], [(10, 1), (20, 2), (30, 3), (40, 4)]) @requires_models(Post, PostNote) def test_primary_key_as_foreign_key(self): p = Post.create(content='p') n = PostNote.create(post=p, note='n') p_db = Post.select().get() self.assertEqual([n.note for n in p_db.notes], ['n']) with self.assertQueryCount(1): query = (PostNote .select(PostNote, Post) .join(Post)) self.assertEqual([(n.post.content, n.note) for n in query], [('p', 'n')]) @skip_unless(IS_CRDB) class TestEnsureServerVersionSet(ModelTestCase): # References GH ssue #2584. requires = [KV] def test_server_version_set(self): # Mimic state of newly-initialized database. self.database.close() self.database.server_version = None with self.database.atomic() as txn: KV.create(k='k1', v=1) self.assertTrue(self.database.server_version is not None) ================================================ FILE: tests/cysqlite_ext.py ================================================ import glob import os import cysqlite from peewee import * from playhouse.cysqlite_ext import * from .base import BaseTestCase from .base import DatabaseTestCase from .base import TestModel from .base import db_loader from .base import skip_unless database = CySqliteDatabase('peewee_test.db', timeout=100) class CyDatabaseTestCase(DatabaseTestCase): database = database def tearDown(self): super(CyDatabaseTestCase, self).tearDown() for filename in glob.glob(self.database.database + '*'): os.unlink(filename) def execute(self, sql, *params): return self.database.execute_sql(sql, params) class TestCSqliteHelpers(CyDatabaseTestCase): def test_autocommit(self): self.assertTrue(self.database.autocommit) self.database.begin() self.assertFalse(self.database.autocommit) self.database.rollback() self.assertTrue(self.database.autocommit) def test_commit_hook(self): state = {} @self.database.on_commit def on_commit(): state.setdefault('commits', 0) state['commits'] += 1 self.execute('create table register (value text)') self.assertEqual(state['commits'], 1) # Check hook is preserved. self.database.close() self.database.connect() self.execute('insert into register (value) values (?), (?)', 'foo', 'bar') self.assertEqual(state['commits'], 2) curs = self.execute('select * from register order by value;') results = curs.fetchall() self.assertEqual([tuple(r) for r in results], [('bar',), ('foo',)]) self.assertEqual(state['commits'], 2) def test_rollback_hook(self): state = {} @self.database.on_rollback def on_rollback(): state.setdefault('rollbacks', 0) state['rollbacks'] += 1 self.execute('create table register (value text);') self.assertEqual(state, {}) # Check hook is preserved. self.database.close() self.database.connect() self.database.begin() self.execute('insert into register (value) values (?)', 'test') self.database.rollback() self.assertEqual(state, {'rollbacks': 1}) curs = self.execute('select * from register;') self.assertEqual(curs.fetchall(), []) def test_update_hook(self): state = [] @self.database.on_update def on_update(query, db, table, rowid): state.append((query, db, table, rowid)) self.execute('create table register (value text)') self.execute('insert into register (value) values (?), (?)', 'foo', 'bar') self.assertEqual(state, [ ('INSERT', 'main', 'register', 1), ('INSERT', 'main', 'register', 2)]) # Check hook is preserved. self.database.close() self.database.connect() self.execute('update register set value = ? where rowid = ?', 'baz', 1) self.assertEqual(state, [ ('INSERT', 'main', 'register', 1), ('INSERT', 'main', 'register', 2), ('UPDATE', 'main', 'register', 1)]) self.execute('delete from register where rowid=?;', 2) self.assertEqual(state, [ ('INSERT', 'main', 'register', 1), ('INSERT', 'main', 'register', 2), ('UPDATE', 'main', 'register', 1), ('DELETE', 'main', 'register', 2)]) def test_properties(self): self.assertTrue(self.database.cache_used is not None) class TestBackup(CyDatabaseTestCase): backup_filenames = set(('test_backup.db', 'test_backup1.db', 'test_backup2.db')) def tearDown(self): super(TestBackup, self).tearDown() for backup_filename in self.backup_filenames: if os.path.exists(backup_filename): os.unlink(backup_filename) def _populate_test_data(self, nrows=100, db=None): db = self.database if db is None else db db.execute_sql('CREATE TABLE register (id INTEGER NOT NULL PRIMARY KEY' ', value INTEGER NOT NULL)') with db.atomic(): for i in range(nrows): db.execute_sql('INSERT INTO register (value) VALUES (?)', (i,)) def test_backup(self): self._populate_test_data() # Back-up to an in-memory database and verify contents. other_db = CySqliteDatabase(':memory:') self.database.backup(other_db) cursor = other_db.execute_sql('SELECT value FROM register ORDER BY ' 'value;') self.assertEqual([val for val, in cursor.fetchall()], list(range(100))) other_db.close() def test_backup_preserve_pagesize(self): db1 = CySqliteDatabase('test_backup1.db') with db1.connection_context(): db1.page_size = 8192 self._populate_test_data(db=db1) db1.connect() self.assertEqual(db1.page_size, 8192) db2 = CySqliteDatabase('test_backup2.db') db1.backup(db2) self.assertEqual(db2.page_size, 8192) nrows, = db2.execute_sql('select count(*) from register;').fetchone() self.assertEqual(nrows, 100) def test_backup_to_file(self): self._populate_test_data() self.database.backup_to_file('test_backup.db') backup_db = CySqliteDatabase('test_backup.db') cursor = backup_db.execute_sql('SELECT value FROM register ORDER BY ' 'value;') self.assertEqual([val for val, in cursor.fetchall()], list(range(100))) backup_db.close() def test_backup_progress(self): self._populate_test_data() accum = [] def progress(remaining, total, is_done): accum.append((remaining, total, is_done)) other_db = CySqliteDatabase(':memory:') self.database.backup(other_db, pages=1, progress=progress) self.assertTrue(len(accum) > 0) sql = 'select value from register order by value;' self.assertEqual([r for r, in other_db.execute_sql(sql)], list(range(100))) other_db.close() def test_backup_progress_error(self): self._populate_test_data() def broken_progress(remaining, total, is_done): raise ValueError('broken') other_db = CySqliteDatabase(':memory:') self.assertRaises(ValueError, self.database.backup, other_db, progress=broken_progress) other_db.close() class DataTypes(cysqlite.TableFunction): columns = ('key', 'value') params = () name = 'data_types' def initialize(self): self.values = ( None, 1, 2., u'unicode str', b'byte str', False, True) self.idx = 0 self.n = len(self.values) def iterate(self, idx): if idx < self.n: return ('k%s' % idx, self.values[idx]) raise StopIteration @skip_unless(cysqlite.sqlite_version_info >= (3, 9), 'requires sqlite >= 3.9') class TestDataTypesTableFunction(CyDatabaseTestCase): database = db_loader('cysqlite') def test_data_types_table_function(self): self.database.register_table_function(DataTypes) for _ in range(2): cursor = self.database.execute_sql('SELECT key, value FROM ' 'data_types() ORDER BY key') self.assertEqual(cursor.fetchall(), [ ('k0', None), ('k1', 1), ('k2', 2.), ('k3', u'unicode str'), ('k4', b'byte str'), ('k5', 0), ('k6', 1), ]) # Ensure table re-registered after close. self.database.close() self.database.connect() ================================================ FILE: tests/dataset.py ================================================ import csv import datetime import json import operator import os import tempfile from io import StringIO from peewee import * from playhouse.dataset import DataSet from playhouse.dataset import Table from .base import IS_SQLITE_OLD from .base import ModelTestCase from .base import TestModel from .base import get_sqlite_db from .base import requires_models from .base import skip_if db = get_sqlite_db() class User(TestModel): username = TextField(primary_key=True) class Note(TestModel): user = ForeignKeyField(User) content = TextField() timestamp = DateTimeField() status = IntegerField(default=1) data = BlobField() class Category(TestModel): name = TextField() parent = ForeignKeyField('self', null=True) class Bin(TestModel): data = BlobField() ts = DateTimeField() class TestDataSet(ModelTestCase): database = db requires = [User, Note, Category] names = ['charlie', 'huey', 'peewee', 'mickey', 'zaizee'] def setUp(self): if os.path.exists(self.database.database): os.unlink(self.database.database) super(TestDataSet, self).setUp() self.dataset = DataSet('sqlite:///%s' % self.database.database) def tearDown(self): self.dataset.close() super(TestDataSet, self).tearDown() def test_create_index(self): users = self.dataset['users'] users.insert(username='u0') users.create_index(['username'], True) with self.assertRaises(IntegrityError): users.insert(username='u0') def test_pass_database(self): db = SqliteDatabase(':memory:') dataset = DataSet(db) self.assertEqual(dataset._database_path, ':memory:') users = dataset['users'] users.insert(username='charlie') self.assertEqual(list(users), [{'id': 1, 'username': 'charlie'}]) @skip_if(IS_SQLITE_OLD) def test_with_views(self): self.dataset.query('CREATE VIEW notes_public AS ' 'SELECT content, timestamp FROM note ' 'WHERE status = 1 ORDER BY timestamp DESC') try: self.assertTrue('notes_public' in self.dataset.views) self.assertFalse('notes_public' in self.dataset.tables) users = self.dataset['user'] with self.dataset.transaction(): users.insert(username='charlie') users.insert(username='huey') notes = self.dataset['note'] for i, (ct, st) in enumerate([('n1', 1), ('n2', 2), ('n3', 1)]): notes.insert(content=ct, status=st, user_id='charlie', timestamp=datetime.datetime(2022, 1, 1 + i), data=b'') self.assertFalse('notes_public' in self.dataset) # Create a new dataset instance with views enabled. dataset = DataSet(self.dataset._database, include_views=True) self.assertTrue('notes_public' in dataset) public = dataset['notes_public'] self.assertEqual(public.columns, ['content', 'timestamp']) self.assertEqual(list(public), [ {'content': 'n3', 'timestamp': datetime.datetime(2022, 1, 3)}, {'content': 'n1', 'timestamp': datetime.datetime(2022, 1, 1)}]) finally: self.dataset.query('DROP VIEW notes_public') def test_item_apis(self): dataset = DataSet('sqlite:///:memory:') users = dataset['users'] users.insert(username='charlie') self.assertEqual(list(users), [{'id': 1, 'username': 'charlie'}]) users[2] = {'username': 'huey', 'color': 'white'} self.assertEqual(list(users), [ {'id': 1, 'username': 'charlie', 'color': None}, {'id': 2, 'username': 'huey', 'color': 'white'}]) users[2] = {'username': 'huey-x', 'kind': 'cat'} self.assertEqual(list(users), [ {'id': 1, 'username': 'charlie', 'color': None, 'kind': None}, {'id': 2, 'username': 'huey-x', 'color': 'white', 'kind': 'cat'}]) del users[2] self.assertEqual(list(users), [ {'id': 1, 'username': 'charlie', 'color': None, 'kind': None}]) users[1] = {'kind': 'person'} users[2] = {'username': 'zaizee'} users[2] = {'kind': 'cat'} self.assertEqual(list(users), [ {'id': 1, 'username': 'charlie', 'color': None, 'kind': 'person'}, {'id': 2, 'username': 'zaizee', 'color': None, 'kind': 'cat'}]) def create_users(self, n=2): user = self.dataset['user'] for i in range(min(n, len(self.names))): user.insert(username=self.names[i]) def test_special_char_table(self): self.database.execute_sql('CREATE TABLE "hello!!world" ("data" TEXT);') self.database.execute_sql('INSERT INTO "hello!!world" VALUES (?)', ('test',)) ds = DataSet('sqlite:///%s' % self.database.database) table = ds['hello!!world'] model = table.model_class self.assertEqual(model._meta.table_name, 'hello!!world') def test_column_preservation(self): ds = DataSet('sqlite:///:memory:') books = ds['books'] books.insert(book_id='BOOK1') books.insert(bookId='BOOK2') data = [(row['book_id'] or '', row['bookId'] or '') for row in books] self.assertEqual(sorted(data), [ ('', 'BOOK2'), ('BOOK1', '')]) def test_case_insensitive(self): db.execute_sql('CREATE TABLE "SomeTable" (data TEXT);') tables = sorted(self.dataset.tables) self.assertEqual(tables, ['SomeTable', 'category', 'note', 'user']) table = self.dataset['HueyMickey'] self.assertEqual(table.model_class._meta.table_name, 'HueyMickey') tables = sorted(self.dataset.tables) self.assertEqual( tables, ['HueyMickey', 'SomeTable', 'category', 'note', 'user']) # Subsequent lookup succeeds. self.dataset['HueyMickey'] def test_introspect(self): tables = sorted(self.dataset.tables) self.assertEqual(tables, ['category', 'note', 'user']) user = self.dataset['user'] columns = sorted(user.columns) self.assertEqual(columns, ['username']) note = self.dataset['note'] columns = sorted(note.columns) self.assertEqual(columns, ['content', 'data', 'id', 'status', 'timestamp', 'user_id']) category = self.dataset['category'] columns = sorted(category.columns) self.assertEqual(columns, ['id', 'name', 'parent_id']) def test_update_cache(self): self.assertEqual(sorted(self.dataset.tables), ['category', 'note', 'user']) db.execute_sql('create table "foo" (id INTEGER, data TEXT)') Foo = self.dataset['foo'] self.assertEqual(sorted(Foo.columns), ['data', 'id']) self.assertTrue('foo' in self.dataset._models) self.dataset._models['foo'].drop_table() self.dataset.update_cache() self.assertTrue('foo' not in self.database.get_tables()) # This will create the table again. Foo = self.dataset['foo'] self.assertTrue('foo' in self.database.get_tables()) self.assertEqual(Foo.columns, ['id']) def assertQuery(self, query, expected, sort_key='id'): key = operator.itemgetter(sort_key) self.assertEqual( sorted(list(query), key=key), sorted(expected, key=key)) def test_insert(self): self.create_users() user = self.dataset['user'] expected = [ {'username': 'charlie'}, {'username': 'huey'}] self.assertQuery(user.all(), expected, 'username') user.insert(username='mickey', age=5) expected = [ {'username': 'charlie', 'age': None}, {'username': 'huey', 'age': None}, {'username': 'mickey', 'age': 5}] self.assertQuery(user.all(), expected, 'username') query = user.find(username='charlie') expected = [{'username': 'charlie', 'age': None}] self.assertQuery(query, expected, 'username') self.assertEqual( user.find_one(username='mickey'), {'username': 'mickey', 'age': 5}) self.assertTrue(user.find_one(username='xx') is None) def test_update(self): self.create_users() user = self.dataset['user'] self.assertEqual(user.update(favorite_color='green'), 2) expected = [ {'username': 'charlie', 'favorite_color': 'green'}, {'username': 'huey', 'favorite_color': 'green'}] self.assertQuery(user.all(), expected, 'username') res = user.update( favorite_color='blue', username='huey', columns=['username']) self.assertEqual(res, 1) expected[1]['favorite_color'] = 'blue' self.assertQuery(user.all(), expected, 'username') def test_delete(self): self.create_users() user = self.dataset['user'] self.assertEqual(user.delete(username='huey'), 1) self.assertEqual(list(user.all()), [{'username': 'charlie'}]) def test_find(self): self.create_users(5) user = self.dataset['user'] def assertUsernames(query, expected): self.assertEqual( sorted(row['username'] for row in query), sorted(expected)) assertUsernames(user.all(), self.names) assertUsernames(user.find(), self.names) assertUsernames(user.find(username='charlie'), ['charlie']) assertUsernames(user.find(username='missing'), []) user.update(favorite_color='green') for username in ['zaizee', 'huey']: user.update( favorite_color='blue', username=username, columns=['username']) assertUsernames( user.find(favorite_color='green'), ['charlie', 'mickey', 'peewee']) assertUsernames( user.find(favorite_color='blue'), ['zaizee', 'huey']) assertUsernames( user.find(favorite_color='green', username='peewee'), ['peewee']) self.assertEqual( user.find_one(username='charlie'), {'username': 'charlie', 'favorite_color': 'green'}) def test_magic_methods(self): self.create_users(5) user = self.dataset['user'] # __len__() self.assertEqual(len(user), 5) # __iter__() users = sorted([u for u in user], key=operator.itemgetter('username')) self.assertEqual(users[0], {'username': 'charlie'}) self.assertEqual(users[-1], {'username': 'zaizee'}) # __contains__() self.assertTrue('user' in self.dataset) self.assertFalse('missing' in self.dataset) def test_foreign_keys(self): user = self.dataset['user'] user.insert(username='charlie') note = self.dataset['note'] for i in range(1, 4): note.insert( content='note %s' % i, timestamp=datetime.date(2014, 1, i), status=i, user_id='charlie', data=b'') notes = sorted(note.all(), key=operator.itemgetter('id')) self.assertEqual(notes[0], { 'content': 'note 1', 'data': b'', 'id': 1, 'status': 1, 'timestamp': datetime.datetime(2014, 1, 1), 'user_id': 'charlie'}) self.assertEqual(notes[-1], { 'content': 'note 3', 'data': b'', 'id': 3, 'status': 3, 'timestamp': datetime.datetime(2014, 1, 3), 'user_id': 'charlie'}) user.insert(username='mickey') note.update(user_id='mickey', id=3, columns=['id']) self.assertEqual(note.find(user_id='charlie').count(), 2) self.assertEqual(note.find(user_id='mickey').count(), 1) category = self.dataset['category'] category.insert(name='c1') c1 = category.find_one(name='c1') self.assertEqual(c1, {'id': 1, 'name': 'c1', 'parent_id': None}) category.insert(name='c2', parent_id=1) c2 = category.find_one(parent_id=1) self.assertEqual(c2, {'id': 2, 'name': 'c2', 'parent_id': 1}) self.assertEqual(category.delete(parent_id=1), 1) self.assertEqual(list(category.all()), [c1]) def test_transactions(self): user = self.dataset['user'] with self.dataset.transaction() as txn: user.insert(username='u1') with self.dataset.transaction() as txn2: user.insert(username='u2') txn2.rollback() with self.dataset.transaction() as txn3: user.insert(username='u3') with self.dataset.transaction() as txn4: user.insert(username='u4') txn3.rollback() with self.dataset.transaction() as txn5: user.insert(username='u5') with self.dataset.transaction() as txn6: with self.dataset.transaction() as txn7: user.insert(username='u6') txn7.rollback() user.insert(username='u7') user.insert(username='u8') self.assertQuery(user.all(), [ {'username': 'u1'}, {'username': 'u5'}, {'username': 'u7'}, {'username': 'u8'}, ], 'username') def test_export(self): self.create_users() user = self.dataset['user'] buf = StringIO() self.dataset.freeze(user.all(), 'json', file_obj=buf) self.assertEqual(buf.getvalue(), ( '[{"username": "charlie"}, {"username": "huey"}]')) buf = StringIO() self.dataset.freeze(user.all(), 'csv', file_obj=buf) self.assertEqual(buf.getvalue().splitlines(), [ 'username', 'charlie', 'huey']) def test_freeze_thaw_csv_utf8(self): self._test_freeze_thaw_utf8('csv') def test_freeze_thaw_json_utf8(self): self._test_freeze_thaw_utf8('json') def _test_freeze_thaw_utf8(self, fmt): username_bytes = b'\xd0\x92obby' # Bobby with cyrillic "B". username_str = username_bytes.decode('utf8') u = User.create(username=username_str) # Freeze the data as a the given format. user = self.dataset['user'] filename = tempfile.mktemp() # Get a filename. self.dataset.freeze(user.all(), fmt, filename) # Clear out the table and reload. User.delete().execute() self.assertEqual(list(user.all()), []) # Thaw the frozen data. n = user.thaw(format=fmt, filename=filename) self.assertEqual(n, 1) self.assertEqual(list(user.all()), [{'username': username_str}]) def test_freeze_thaw(self): user = self.dataset['user'] user.insert(username='charlie') note = self.dataset['note'] note_ts = datetime.datetime(2017, 1, 2, 3, 4, 5) note.insert(content='foo', timestamp=note_ts, user_id='charlie', status=2, data=b'\xff\x00\xcc') buf = StringIO() self.dataset.freeze(note.all(), 'json', file_obj=buf) self.assertEqual(json.loads(buf.getvalue()), [{ 'id': 1, 'user_id': 'charlie', 'content': 'foo', 'status': 2, 'timestamp': '2017-01-02 03:04:05', 'data': 'ff00cc'}]) note.delete(id=1) self.assertEqual(list(note.all()), []) buf.seek(0) note.thaw(format='json', file_obj=buf) self.assertEqual(list(note.all()), [{ 'id': 1, 'user_id': 'charlie', 'content': 'foo', 'status': 2, 'timestamp': note_ts, 'data': b'\xff\x00\xcc'}]) @requires_models(Bin) def test_freeze_thaw_datatypes_json(self): Bin = self.dataset['bin'] ts = datetime.datetime(2026, 1, 2, 3, 4, 5, tzinfo=datetime.timezone.utc) Bin.insert(data=b'\xff\x00\xcc', ts=ts) buf = StringIO() self.dataset.freeze(Bin.all(), 'json', file_obj=buf) self.assertEqual(json.loads(buf.getvalue()), [{ 'id': 1, 'data': 'ff00cc', 'ts': '2026-01-02 03:04:05+00:00'}]) Bin.delete(id=1) buf.seek(0) Bin.thaw(format='json', file_obj=buf) self.assertEqual(list(Bin.all()), [{ 'id': 1, 'data': b'\xff\x00\xcc', 'ts': ts}]) buf = StringIO() self.dataset.freeze(Bin.all(), 'json', file_obj=buf, iso8601_datetimes=True, base64_bytes=True) self.assertEqual(json.loads(buf.getvalue()), [{ 'id': 1, 'data': '_wDM', 'ts': '2026-01-02T03:04:05+00:00'}]) Bin.delete(id=1) buf.seek(0) Bin.thaw(format='json', file_obj=buf, iso8601_datetimes=True, base64_bytes=True) self.assertEqual(list(Bin.all()), [{ 'id': 1, 'data': b'\xff\x00\xcc', 'ts': ts}]) @requires_models(Bin) def test_freeze_thaw_datatypes_csv(self): Bin = self.dataset['bin'] ts = datetime.datetime(2026, 1, 2, 3, 4, 5, tzinfo=datetime.timezone.utc) Bin.insert(data=b'\xff\x00\xcc', ts=ts) buf = StringIO() self.dataset.freeze(Bin.all(), 'csv', file_obj=buf) self.assertEqual(buf.getvalue().splitlines(), [ 'id,data,ts', '1,ff00cc,2026-01-02 03:04:05+00:00']) Bin.delete(id=1) buf.seek(0) Bin.thaw(format='csv', file_obj=buf) self.assertEqual(list(Bin.all()), [{ 'id': 1, 'data': b'\xff\x00\xcc', 'ts': ts}]) buf = StringIO() self.dataset.freeze(Bin.all(), 'csv', file_obj=buf, iso8601_datetimes=True, base64_bytes=True) self.assertEqual(buf.getvalue().splitlines(), [ 'id,data,ts', '1,_wDM,2026-01-02T03:04:05+00:00']) Bin.delete(id=1) buf.seek(0) Bin.thaw(format='csv', file_obj=buf, iso8601_datetimes=True, base64_bytes=True) self.assertEqual(list(Bin.all()), [{ 'id': 1, 'data': b'\xff\x00\xcc', 'ts': ts}]) def test_table_column_creation(self): table = self.dataset['people'] table.insert(name='charlie') self.assertEqual(table.columns, ['id', 'name']) self.assertEqual(list(table.all()), [{'id': 1, 'name': 'charlie'}]) def test_table_column_creation_field_col(self): table = self.dataset['people'] table.insert(**{'First Name': 'charlie'}) self.assertEqual(table.columns, ['id', 'First_Name']) self.assertEqual(list(table.all()), [{'id': 1, 'First_Name': 'charlie'}]) table.insert(**{'First Name': 'huey'}) self.assertEqual(table.columns, ['id', 'First_Name']) self.assertEqual(list(table.all().order_by(table.model_class.id)), [ {'id': 1, 'First_Name': 'charlie'}, {'id': 2, 'First_Name': 'huey'}]) def test_import_json(self): table = self.dataset['people'] table.insert(name='charlie') data = [ {'name': 'zaizee', 'foo': 1}, {'name': 'huey'}, {'name': 'mickey', 'foo': 2}, {'bar': None}] buf = StringIO() json.dump(data, buf) buf.seek(0) # All rows but the last will be inserted. count = self.dataset.thaw('people', 'json', file_obj=buf, strict=True) self.assertEqual(count, 3) names = [row['name'] for row in self.dataset['people'].all()] self.assertEqual( set(names), set(['charlie', 'huey', 'mickey', 'zaizee'])) # The columns have not changed. self.assertEqual(table.columns, ['id', 'name']) # No rows are inserted because no column overlap between `user` and the # provided data. buf.seek(0) count = self.dataset.thaw('user', 'json', file_obj=buf, strict=True) self.assertEqual(count, 0) # Create a new table and load all data into it. table = self.dataset['more_people'] # All rows and columns will be inserted. buf.seek(0) count = self.dataset.thaw('more_people', 'json', file_obj=buf) self.assertEqual(count, 4) self.assertEqual( set(table.columns), set(['id', 'name', 'bar', 'foo'])) self.assertEqual(sorted(table.all(), key=lambda row: row['id']), [ {'id': 1, 'name': 'zaizee', 'foo': 1, 'bar': None}, {'id': 2, 'name': 'huey', 'foo': None, 'bar': None}, {'id': 3, 'name': 'mickey', 'foo': 2, 'bar': None}, {'id': 4, 'name': None, 'foo': None, 'bar': None}, ]) def test_import_csv(self): table = self.dataset['people'] table.insert(name='charlie') data = [ ('zaizee', 1, None), ('huey', 2, 'foo'), ('mickey', 3, 'baze')] buf = StringIO() writer = csv.writer(buf) writer.writerow(['name', 'foo', 'bar']) writer.writerows(data) buf.seek(0) count = self.dataset.thaw('people', 'csv', file_obj=buf, strict=True) self.assertEqual(count, 3) names = [row['name'] for row in self.dataset['people'].all()] self.assertEqual( set(names), set(['charlie', 'huey', 'mickey', 'zaizee'])) # The columns have not changed. self.assertEqual(table.columns, ['id', 'name']) # No rows are inserted because no column overlap between `user` and the # provided data. buf.seek(0) count = self.dataset.thaw('user', 'csv', file_obj=buf, strict=True) self.assertEqual(count, 0) # Create a new table and load all data into it. table = self.dataset['more_people'] # All rows and columns will be inserted. buf.seek(0) count = self.dataset.thaw('more_people', 'csv', file_obj=buf) self.assertEqual(count, 3) self.assertEqual( set(table.columns), set(['id', 'name', 'bar', 'foo'])) self.assertEqual(sorted(table.all(), key=lambda row: row['id']), [ {'id': 1, 'name': 'zaizee', 'foo': '1', 'bar': ''}, {'id': 2, 'name': 'huey', 'foo': '2', 'bar': 'foo'}, {'id': 3, 'name': 'mickey', 'foo': '3', 'bar': 'baze'}, ]) def test_table_thaw(self): table = self.dataset['people'] data = json.dumps([{'name': 'charlie'}, {'name': 'huey', 'color': 'white'}]) self.assertEqual(table.thaw(file_obj=StringIO(data), format='json'), 2) self.assertEqual(list(table.all()), [ {'id': 1, 'name': 'charlie', 'color': None}, {'id': 2, 'name': 'huey', 'color': 'white'}, ]) def test_creating_tables(self): new_table = self.dataset['new_table'] new_table.insert(data='foo') ref2 = self.dataset['new_table'] self.assertEqual(list(ref2.all()), [{'id': 1, 'data': 'foo'}]) ================================================ FILE: tests/db_tests.py ================================================ from itertools import permutations from queue import Queue import platform import re import threading from peewee import * from peewee import Database from peewee import FIELD from peewee import attrdict from peewee import sort_models from .base import BaseTestCase from .base import DatabaseTestCase from .base import IS_CRDB from .base import IS_MYSQL from .base import IS_POSTGRESQL from .base import IS_SQLITE from .base import ModelTestCase from .base import TestModel from .base import db from .base import get_in_memory_db from .base import get_sqlite_db from .base import new_connection from .base import requires_models from .base import requires_postgresql from .base_models import Category from .base_models import Tweet from .base_models import User class TestDatabase(DatabaseTestCase): database = get_sqlite_db() def test_pragmas(self): self.database.cache_size = -2048 self.assertEqual(self.database.cache_size, -2048) self.database.cache_size = -4096 self.assertEqual(self.database.cache_size, -4096) self.database.foreign_keys = 'on' self.assertEqual(self.database.foreign_keys, 1) self.database.foreign_keys = 'off' self.assertEqual(self.database.foreign_keys, 0) def test_appid_user_version(self): self.assertEqual(self.database.application_id, 0) self.assertEqual(self.database.user_version, 0) self.database.application_id = 1 self.database.user_version = 2 self.assertEqual(self.database.application_id, 1) self.assertEqual(self.database.user_version, 2) self.assertTrue(self.database.close()) self.assertTrue(self.database.connect()) self.assertEqual(self.database.application_id, 1) self.assertEqual(self.database.user_version, 2) def test_timeout_semantics(self): self.assertEqual(self.database.timeout, 5) self.assertEqual(self.database.pragma('busy_timeout'), 5000) self.database.timeout = 2.5 self.assertEqual(self.database.timeout, 2.5) self.assertEqual(self.database.pragma('busy_timeout'), 2500) self.database.close() self.database.connect() self.assertEqual(self.database.timeout, 2.5) self.assertEqual(self.database.pragma('busy_timeout'), 2500) def test_pragmas_deferred(self): pragmas = (('journal_mode', 'wal'),) db = SqliteDatabase(None, pragmas=pragmas) self.assertEqual(db._pragmas, pragmas) # Test pragmas preserved after initializing. db.init(':memory:') self.assertEqual(db._pragmas, pragmas) db = SqliteDatabase(None) self.assertEqual(db._pragmas, ()) # Test pragmas are set and subsequently overwritten. db.init(':memory:', pragmas=pragmas) self.assertEqual(db._pragmas, pragmas) db.init(':memory:', pragmas=()) self.assertEqual(db._pragmas, ()) # Test when specified twice, the previous value is overwritten. db = SqliteDatabase(None, pragmas=pragmas) db.init(':memory:', pragmas=(('cache_size', -8000),)) self.assertEqual(db._pragmas, (('cache_size', -8000),)) def test_pragmas_as_dict(self): pragmas = {'journal_mode': 'wal'} pragma_list = [('journal_mode', 'wal')] db = SqliteDatabase(':memory:', pragmas=pragmas) self.assertEqual(db._pragmas, pragma_list) # Test deferred databases correctly handle pragma dicts. db = SqliteDatabase(None, pragmas=pragmas) self.assertEqual(db._pragmas, pragma_list) db.init(':memory:') self.assertEqual(db._pragmas, pragma_list) db.init(':memory:', pragmas={}) self.assertEqual(db._pragmas, []) def test_pragmas_permanent(self): db = SqliteDatabase(':memory:') db.execute_sql('pragma foreign_keys=0') self.assertEqual(db.foreign_keys, 0) db.pragma('foreign_keys', 1, True) self.assertEqual(db.foreign_keys, 1) db.close() db.connect() self.assertEqual(db.foreign_keys, 1) def test_context_settings(self): class TestDatabase(Database): field_types = {'BIGINT': 'TEST_BIGINT', 'TEXT': 'TEST_TEXT'} operations = {'LIKE': '~', 'NEW': '->>'} param = '$' test_db = TestDatabase(None) state = test_db.get_sql_context().state self.assertEqual(state.field_types['BIGINT'], 'TEST_BIGINT') self.assertEqual(state.field_types['TEXT'], 'TEST_TEXT') self.assertEqual(state.field_types['INT'], FIELD.INT) self.assertEqual(state.field_types['VARCHAR'], FIELD.VARCHAR) self.assertEqual(state.operations['LIKE'], '~') self.assertEqual(state.operations['NEW'], '->>') self.assertEqual(state.operations['ILIKE'], 'ILIKE') self.assertEqual(state.param, '$') self.assertEqual(state.quote, '""') test_db2 = TestDatabase(None, field_types={'BIGINT': 'XXX_BIGINT', 'INT': 'XXX_INT'}) state = test_db2.get_sql_context().state self.assertEqual(state.field_types['BIGINT'], 'XXX_BIGINT') self.assertEqual(state.field_types['TEXT'], 'TEST_TEXT') self.assertEqual(state.field_types['INT'], 'XXX_INT') self.assertEqual(state.field_types['VARCHAR'], FIELD.VARCHAR) def test_connection_state(self): conn = self.database.connection() self.assertFalse(self.database.is_closed()) self.database.close() self.assertTrue(self.database.is_closed()) conn = self.database.connection() self.assertFalse(self.database.is_closed()) def test_db_context_manager(self): self.database.close() self.assertTrue(self.database.is_closed()) with self.database: self.assertFalse(self.database.is_closed()) self.assertTrue(self.database.is_closed()) self.database.connect() self.assertFalse(self.database.is_closed()) # Enter context with an already-open db. with self.database: self.assertFalse(self.database.is_closed()) # Closed after exit. self.assertTrue(self.database.is_closed()) def test_connection_initialization(self): state = {'count': 0} class TestDatabase(SqliteDatabase): def _initialize_connection(self, conn): state['count'] += 1 db = TestDatabase(':memory:') self.assertEqual(state['count'], 0) conn = db.connection() self.assertEqual(state['count'], 1) # Since already connected, nothing happens here. conn = db.connection() self.assertEqual(state['count'], 1) def test_connect_semantics(self): state = {'count': 0} class TestDatabase(SqliteDatabase): def _initialize_connection(self, conn): state['count'] += 1 db = TestDatabase(':memory:') db.connect() self.assertEqual(state['count'], 1) self.assertRaises(OperationalError, db.connect) self.assertEqual(state['count'], 1) self.assertFalse(db.connect(reuse_if_open=True)) self.assertEqual(state['count'], 1) with db: self.assertEqual(state['count'], 1) self.assertFalse(db.is_closed()) self.assertTrue(db.is_closed()) with db: self.assertEqual(state['count'], 2) def test_execute_sql(self): self.database.execute_sql('CREATE TABLE register (val INTEGER);') self.database.execute_sql('INSERT INTO register (val) VALUES (?), (?)', (1337, 31337)) cursor = self.database.execute_sql( 'SELECT val FROM register ORDER BY val') self.assertEqual(cursor.fetchall(), [(1337,), (31337,)]) self.database.execute_sql('DROP TABLE register;') def test_bind_helpers(self): db = get_in_memory_db() alt_db = get_in_memory_db() class Base(Model): class Meta: database = db class A(Base): a = TextField() class B(Base): b = TextField() db.create_tables([A, B]) # Temporarily bind A to alt_db. with alt_db.bind_ctx([A]): self.assertFalse(A.table_exists()) self.assertTrue(B.table_exists()) self.assertTrue(A.table_exists()) self.assertTrue(B.table_exists()) alt_db.bind([A]) self.assertFalse(A.table_exists()) self.assertTrue(B.table_exists()) db.close() alt_db.close() def test_bind_regression(self): class Base(Model): class Meta: database = None class A(Base): pass class B(Base): pass class AB(Base): a = ForeignKeyField(A) b = ForeignKeyField(B) self.assertTrue(A._meta.database is None) db = get_in_memory_db() with db.bind_ctx([A, B]): self.assertEqual(A._meta.database, db) self.assertEqual(B._meta.database, db) self.assertEqual(AB._meta.database, db) self.assertTrue(A._meta.database is None) self.assertTrue(B._meta.database is None) self.assertTrue(AB._meta.database is None) class C(Base): a = ForeignKeyField(A) with db.bind_ctx([C], bind_refs=False): self.assertEqual(C._meta.database, db) self.assertTrue(A._meta.database is None) self.assertTrue(C._meta.database is None) self.assertTrue(A._meta.database is None) def test_batch_commit(self): class PatchCommitDatabase(SqliteDatabase): commits = 0 def begin(self): pass def commit(self): self.commits += 1 db = PatchCommitDatabase(':memory:') def assertBatches(n_objs, batch_size, n_commits): accum = [] source = range(n_objs) db.commits = 0 for item in db.batch_commit(source, batch_size): accum.append(item) self.assertEqual(accum, list(range(n_objs))) self.assertEqual(db.commits, n_commits) assertBatches(12, 1, 12) assertBatches(12, 2, 6) assertBatches(12, 3, 4) assertBatches(12, 4, 3) assertBatches(12, 5, 3) assertBatches(12, 6, 2) assertBatches(12, 7, 2) assertBatches(12, 11, 2) assertBatches(12, 12, 1) assertBatches(12, 13, 1) def test_server_version(self): class FakeDatabase(Database): server_version = None def _connect(self): return 1 def _close(self, conn): pass def _set_server_version(self, conn): self.server_version = (1, 33, 7) db = FakeDatabase(':memory:') self.assertTrue(db.server_version is None) db.connect() self.assertEqual(db.server_version, (1, 33, 7)) db.close() self.assertEqual(db.server_version, (1, 33, 7)) db.server_version = (1, 2, 3) db.connect() self.assertEqual(db.server_version, (1, 2, 3)) db.close() def test_explicit_connect(self): db = get_in_memory_db(autoconnect=False) self.assertRaises(InterfaceError, db.execute_sql, 'pragma cache_size') with db: db.execute_sql('pragma cache_size') self.assertRaises(InterfaceError, db.cursor) class TestThreadSafety(ModelTestCase): # HACK: This workaround increases the Sqlite busy timeout when tests are # being run on certain architectures. if IS_SQLITE and platform.machine() not in ('i386', 'i686', 'x86_64'): database = new_connection(timeout=60) nthreads = 4 nrows = 10 requires = [User] def test_multiple_writers(self): def create_users(idx): for i in range(idx * self.nrows, (idx + 1) * self.nrows): User.create(username='u%d' % i) threads = [] for i in range(self.nthreads): threads.append(threading.Thread(target=create_users, args=(i,))) for t in threads: t.start() for t in threads: t.join() self.assertEqual(User.select().count(), self.nrows * self.nthreads) def test_multiple_readers(self): data = Queue() def read_user_count(n): for i in range(n): data.put(User.select().count()) threads = [] for i in range(self.nthreads): threads.append(threading.Thread(target=read_user_count, args=(self.nrows,))) for t in threads: t.start() for t in threads: t.join() self.assertEqual(data.qsize(), self.nrows * self.nthreads) def test_mt_general(self): def connect_close(): for _ in range(self.nrows): self.database.connect() with self.database.atomic() as txn: self.database.execute_sql('select 1').fetchone() self.database.close() threads = [] for i in range(self.nthreads): threads.append(threading.Thread(target=connect_close)) for t in threads: t.start() for t in threads: t.join() class TestDeferredDatabase(BaseTestCase): def test_deferred_database(self): deferred_db = SqliteDatabase(None) self.assertTrue(deferred_db.deferred) class DeferredModel(Model): class Meta: database = deferred_db self.assertRaises(Exception, deferred_db.connect) query = DeferredModel.select() self.assertRaises(Exception, query.execute) deferred_db.init(':memory:') self.assertFalse(deferred_db.deferred) conn = deferred_db.connect() self.assertFalse(deferred_db.is_closed()) DeferredModel._schema.create_all() self.assertEqual(list(DeferredModel.select()), []) deferred_db.init(None) self.assertTrue(deferred_db.deferred) # The connection was automatically closed. self.assertTrue(deferred_db.is_closed()) class CatToy(TestModel): description = TextField() class Meta: schema = 'huey' @requires_postgresql class TestSchemaNamespace(ModelTestCase): requires = [CatToy] def setUp(self): with self.database: self.execute('CREATE SCHEMA huey;') super(TestSchemaNamespace, self).setUp() def tearDown(self): super(TestSchemaNamespace, self).tearDown() with self.database: self.execute('DROP SCHEMA huey;') def test_schema(self): toy = CatToy.create(description='fur mouse') toy_db = CatToy.select().where(CatToy.id == toy.id).get() self.assertEqual(toy.id, toy_db.id) self.assertEqual(toy.description, toy_db.description) class TestSqliteIsolation(ModelTestCase): database = get_sqlite_db() requires = [User] def test_sqlite_isolation(self): for username in ('u1', 'u2', 'u3'): User.create(username=username) new_db = get_sqlite_db() curs = new_db.execute_sql('SELECT COUNT(*) FROM users') self.assertEqual(curs.fetchone()[0], 3) self.assertEqual(User.select().count(), 3) self.assertEqual(User.delete().execute(), 3) with self.database.atomic(): User.create(username='u4') User.create(username='u5') # Second conn does not see the changes. curs = new_db.execute_sql('SELECT COUNT(*) FROM users') self.assertEqual(curs.fetchone()[0], 0) # Third conn does not see the changes. new_db2 = get_sqlite_db() curs = new_db2.execute_sql('SELECT COUNT(*) FROM users') self.assertEqual(curs.fetchone()[0], 0) # Original connection sees its own changes. self.assertEqual(User.select().count(), 2) curs = new_db.execute_sql('SELECT COUNT(*) FROM users') self.assertEqual(curs.fetchone()[0], 2) class UniqueModel(TestModel): name = CharField(unique=True) class IndexedModel(TestModel): first = CharField() last = CharField() dob = DateField() class Meta: indexes = ( (('first', 'last', 'dob'), True), (('first', 'last'), False), ) class Note(TestModel): content = TextField() ts = DateTimeField() status = IntegerField() class Meta: table_name = 'notes' class Person(TestModel): first = CharField() last = CharField() email = CharField() class Meta: indexes = ( (('last', 'first'), False), ) class TestIntrospection(ModelTestCase): requires = [Category, User, UniqueModel, IndexedModel, Person] def test_table_exists(self): self.assertTrue(self.database.table_exists(User._meta.table_name)) self.assertFalse(self.database.table_exists('nuggies')) self.assertTrue(self.database.table_exists(User)) class X(TestModel): pass self.assertFalse(self.database.table_exists(X)) def test_get_tables(self): tables = self.database.get_tables() required = set(m._meta.table_name for m in self.requires) self.assertTrue(required.issubset(set(tables))) UniqueModel._schema.drop_all() tables = self.database.get_tables() self.assertFalse(UniqueModel._meta.table_name in tables) def test_get_indexes(self): indexes = self.database.get_indexes('unique_model') data = [(index.name, index.columns, index.unique, index.table) for index in indexes if index.name not in ('unique_model_pkey', 'PRIMARY')] self.assertEqual(data, [ ('unique_model_name', ['name'], True, 'unique_model')]) indexes = self.database.get_indexes('indexed_model') data = [(index.name, index.columns, index.unique, index.table) for index in indexes if index.name not in ('indexed_model_pkey', 'PRIMARY')] self.assertEqual(sorted(data), [ ('indexed_model_first_last', ['first', 'last'], False, 'indexed_model'), ('indexed_model_first_last_dob', ['first', 'last', 'dob'], True, 'indexed_model')]) # Multi-column index where columns are in different order than declared # on the table. indexes = self.database.get_indexes('person') data = [(index.name, index.columns, index.unique) for index in indexes if index.name not in ('person_pkey', 'PRIMARY')] self.assertEqual(data, [ ('person_last_first', ['last', 'first'], False)]) def test_get_columns(self): columns = self.database.get_columns('indexed_model') data = [(c.name, c.null, c.primary_key, c.table) for c in columns] self.assertEqual(data, [ ('id', False, True, 'indexed_model'), ('first', False, False, 'indexed_model'), ('last', False, False, 'indexed_model'), ('dob', False, False, 'indexed_model')]) columns = self.database.get_columns('category') data = [(c.name, c.null, c.primary_key, c.table) for c in columns] self.assertEqual(data, [ ('name', False, True, 'category'), ('parent_id', True, False, 'category')]) def test_get_primary_keys(self): primary_keys = self.database.get_primary_keys('users') self.assertEqual(primary_keys, ['id']) primary_keys = self.database.get_primary_keys('category') self.assertEqual(primary_keys, ['name']) @requires_models(Note) def test_get_views(self): def normalize_view_meta(view_meta): sql_ws_norm = re.sub(r'[\n\s]+', ' ', view_meta.sql.strip('; ')) return view_meta.name, (sql_ws_norm .replace('`peewee_test`.', '') .replace('`notes`.', '') .replace('notes.', '') .replace('`', '')) def assertViews(expected): # Create two sample views. self.database.execute_sql('CREATE VIEW notes_public AS ' 'SELECT content, ts FROM notes ' 'WHERE status = 1 ORDER BY ts DESC') self.database.execute_sql('CREATE VIEW notes_deleted AS ' 'SELECT content FROM notes ' 'WHERE status = 9 ORDER BY id DESC') try: views = self.database.get_views() normalized = sorted([normalize_view_meta(v) for v in views]) self.assertEqual(normalized, expected) # Ensure that we can use get_columns to introspect views. columns = self.database.get_columns('notes_deleted') self.assertEqual([c.name for c in columns], ['content']) columns = self.database.get_columns('notes_public') self.assertEqual([c.name for c in columns], ['content', 'ts']) finally: self.database.execute_sql('DROP VIEW notes_public;') self.database.execute_sql('DROP VIEW notes_deleted;') # Unfortunately, all databases seem to represent VIEW definitions # differently internally. if IS_SQLITE: assertViews([ ('notes_deleted', ('CREATE VIEW notes_deleted AS ' 'SELECT content FROM notes ' 'WHERE status = 9 ORDER BY id DESC')), ('notes_public', ('CREATE VIEW notes_public AS ' 'SELECT content, ts FROM notes ' 'WHERE status = 1 ORDER BY ts DESC'))]) elif IS_MYSQL: assertViews([ ('notes_deleted', ('select content AS content from notes ' 'where status = 9 order by id desc')), ('notes_public', ('select content AS content,ts AS ts from notes ' 'where status = 1 order by ts desc'))]) elif IS_POSTGRESQL: assertViews([ ('notes_deleted', ('SELECT content FROM notes ' 'WHERE (status = 9) ORDER BY id DESC')), ('notes_public', ('SELECT content, ts FROM notes ' 'WHERE (status = 1) ORDER BY ts DESC'))]) elif IS_CRDB: assertViews([ ('notes_deleted', ('SELECT content FROM peewee_test.public.notes ' 'WHERE status = 9 ORDER BY id DESC')), ('notes_public', ('SELECT content, ts FROM peewee_test.public.notes ' 'WHERE status = 1 ORDER BY ts DESC'))]) @requires_models(User, Tweet, Category) def test_get_foreign_keys(self): foreign_keys = self.database.get_foreign_keys('tweet') data = [(fk.column, fk.dest_table, fk.dest_column, fk.table) for fk in foreign_keys] self.assertEqual(data, [ ('user_id', 'users', 'id', 'tweet')]) foreign_keys = self.database.get_foreign_keys('category') data = [(fk.column, fk.dest_table, fk.dest_column, fk.table) for fk in foreign_keys] self.assertEqual(data, [ ('parent_id', 'category', 'name', 'category')]) class TestSortModels(BaseTestCase): def test_sort_models(self): class A(Model): pass class B(Model): a = ForeignKeyField(A) class C(Model): b = ForeignKeyField(B) class D(Model): c = ForeignKeyField(C) class E(Model): pass models = [A, B, C, D, E] for list_of_models in permutations(models): sorted_models = sort_models(list_of_models) self.assertEqual(sorted_models, models) def test_sort_models_multi_fk(self): class Inventory(Model): pass class Sheet(Model): inventory = ForeignKeyField(Inventory) class Program(Model): inventory = ForeignKeyField(Inventory) class ProgramSheet(Model): program = ForeignKeyField(Program) sheet = ForeignKeyField(Sheet) class ProgramPart(Model): program_sheet = ForeignKeyField(ProgramSheet) class Offal(Model): program_sheet = ForeignKeyField(ProgramSheet) sheet = ForeignKeyField(Sheet) M = [Inventory, Sheet, Program, ProgramSheet, ProgramPart, Offal] sorted_models = sort_models(M) self.assertEqual(sorted_models, [ Inventory, Program, Sheet, ProgramSheet, Offal, ProgramPart, ]) for list_of_models in permutations(M): self.assertEqual(sort_models(list_of_models), sorted_models) class TestDBProxy(BaseTestCase): def test_proxy_context_manager(self): db = Proxy() class User(Model): username = TextField() class Meta: database = db self.assertRaises(AttributeError, User.create_table) sqlite_db = SqliteDatabase(':memory:') db.initialize(sqlite_db) User.create_table() with db: self.assertFalse(db.is_closed()) self.assertTrue(db.is_closed()) def test_db_proxy(self): db = Proxy() class BaseModel(Model): class Meta: database = db class User(BaseModel): username = TextField() class Tweet(BaseModel): user = ForeignKeyField(User, backref='tweets') message = TextField() sqlite_db = SqliteDatabase(':memory:') db.initialize(sqlite_db) self.assertEqual(User._meta.database.database, ':memory:') self.assertEqual(Tweet._meta.database.database, ':memory:') self.assertTrue(User._meta.database.is_closed()) self.assertTrue(Tweet._meta.database.is_closed()) sqlite_db.connect() self.assertFalse(User._meta.database.is_closed()) self.assertFalse(Tweet._meta.database.is_closed()) sqlite_db.close() def test_proxy_decorator(self): db = DatabaseProxy() @db.connection_context() def with_connection(): self.assertFalse(db.is_closed()) @db.atomic() def with_transaction(): self.assertTrue(db.in_transaction()) @db.manual_commit() def with_manual_commit(): self.assertTrue(db.in_transaction()) db.initialize(SqliteDatabase(':memory:')) with_connection() self.assertTrue(db.is_closed()) with_transaction() self.assertFalse(db.in_transaction()) with_manual_commit() self.assertFalse(db.in_transaction()) def test_proxy_bind_ctx_callbacks(self): db = Proxy() class BaseModel(Model): class Meta: database = db class Hook(BaseModel): data = BlobField() # Attaches hook to configure blob-type. self.assertTrue(Hook.data._constructor is bytearray) class CustomSqliteDB(SqliteDatabase): sentinel = object() def get_binary_type(self): return self.sentinel custom_db = CustomSqliteDB(':memory:') with custom_db.bind_ctx([Hook]): self.assertTrue(Hook.data._constructor is custom_db.sentinel) self.assertTrue(Hook.data._constructor is bytearray) custom_db.bind([Hook]) self.assertTrue(Hook.data._constructor is custom_db.sentinel) class Data(TestModel): key = TextField() value = TextField() class Meta: schema = 'main' class TestAttachDatabase(ModelTestCase): database = get_sqlite_db() requires = [Data] def test_attach(self): database = self.database Data.create(key='k1', value='v1') Data.create(key='k2', value='v2') # Attach an in-memory cache database. database.attach(':memory:', 'cache') # Clone data into the in-memory cache. class CacheData(Data): class Meta: schema = 'cache' self.assertFalse(CacheData.table_exists()) CacheData.create_table(safe=False) self.assertTrue(CacheData.table_exists()) (CacheData .insert_from(Data.select(), fields=[Data.id, Data.key, Data.value]) .execute()) # Update the source data. query = Data.update({Data.value: Data.value + '-x'}) self.assertEqual(query.execute(), 2) # Verify the source data was updated. query = Data.select(Data.key, Data.value).order_by(Data.key) self.assertSQL(query, ( 'SELECT "t1"."key", "t1"."value" ' 'FROM "main"."data" AS "t1" ' 'ORDER BY "t1"."key"'), []) self.assertEqual([v for k, v in query.tuples()], ['v1-x', 'v2-x']) # Verify the cached data reflects the original data, pre-update. query = (CacheData .select(CacheData.key, CacheData.value) .order_by(CacheData.key)) self.assertSQL(query, ( 'SELECT "t1"."key", "t1"."value" ' 'FROM "cache"."cache_data" AS "t1" ' 'ORDER BY "t1"."key"'), []) self.assertEqual([v for k, v in query.tuples()], ['v1', 'v2']) database.close() # On re-connecting, the in-memory database will re-attached. database.connect() # Cache-Data table does not exist. self.assertFalse(CacheData.table_exists()) # Double-check the sqlite master table. curs = database.execute_sql('select * from cache.sqlite_master;') self.assertEqual(curs.fetchall(), []) # Because it's in-memory, the table needs to be re-created. CacheData.create_table(safe=False) self.assertEqual(CacheData.select().count(), 0) # Original data is still there. self.assertEqual(Data.select().count(), 2) def test_attach_detach(self): database = self.database Data.create(key='k1', value='v1') Data.create(key='k2', value='v2') # Attach an in-memory cache database. database.attach(':memory:', 'cache') curs = database.execute_sql('select * from cache.sqlite_master') self.assertEqual(curs.fetchall(), []) self.assertFalse(database.attach(':memory:', 'cache')) self.assertRaises(OperationalError, database.attach, 'foo.db', 'cache') self.assertTrue(database.detach('cache')) self.assertFalse(database.detach('cache')) self.assertRaises(OperationalError, database.execute_sql, 'select * from cache.sqlite_master') def test_sqlite_schema_support(self): class CacheData(Data): class Meta: schema = 'cache' # Attach an in-memory cache database and create the cache table. self.database.attach(':memory:', 'cache') CacheData.create_table() tables = self.database.get_tables() self.assertEqual(tables, ['data']) tables = self.database.get_tables(schema='cache') self.assertEqual(tables, ['cache_data']) class TestDatabaseConnection(DatabaseTestCase): def test_is_connection_usable(self): # Ensure a connection is open. conn = self.database.connection() self.assertTrue(self.database.is_connection_usable()) self.database.close() self.assertFalse(self.database.is_connection_usable()) self.database.connect() self.assertTrue(self.database.is_connection_usable()) @requires_postgresql def test_is_connection_usable_pg(self): self.database.execute_sql('drop table if exists foo') self.database.execute_sql('create table foo (data text not null)') self.assertTrue(self.database.is_connection_usable()) with self.database.atomic() as txn: with self.assertRaises(IntegrityError): self.database.execute_sql('insert into foo (data) values (NULL)') self.assertFalse(self.database.is_closed()) self.assertFalse(self.database.is_connection_usable()) txn.rollback() self.assertTrue(self.database.is_connection_usable()) curs = self.database.execute_sql('select * from foo') self.assertEqual(list(curs), []) self.database.execute_sql('drop table foo') class TestExceptionWrapper(ModelTestCase): database = get_in_memory_db() requires = [User] def test_exception_wrapper(self): exc = None try: User.create(username=None) except IntegrityError as e: exc = e if exc is None: raise Exception('expected integrity error not raised') self.assertTrue(exc.orig.__module__ != 'peewee') class TestModelPropertyHelper(BaseTestCase): def test_model_property(self): database = get_in_memory_db() class M1(database.Model): pass class M2(database.Model): pass class CM1(M1): pass for M in (M1, M2, CM1): self.assertTrue(M._meta.database is database) def test_model_property_on_proxy(self): db = DatabaseProxy() class M1(db.Model): pass class M2(db.Model): pass class CM1(M1): pass test_db = get_in_memory_db() db.initialize(test_db) for M in (M1, M2, CM1): self.assertEqual(M._meta.database.database, ':memory:') class TestChunkedUtility(BaseTestCase): def test_chunked_exact_divisor(self): result = list(chunked(range(6), 3)) self.assertEqual(result, [[0, 1, 2], [3, 4, 5]]) def test_chunked_with_remainder(self): result = list(chunked(range(7), 3)) self.assertEqual(result, [[0, 1, 2], [3, 4, 5], [6]]) def test_chunked_single_element(self): result = list(chunked([42], 5)) self.assertEqual(result, [[42]]) def test_chunked_empty(self): result = list(chunked([], 5)) self.assertEqual(result, []) def test_chunked_size_one(self): result = list(chunked(range(3), 1)) self.assertEqual(result, [[0], [1], [2]]) def test_chunked_generator_input(self): gen = (x * 2 for x in range(5)) result = list(chunked(gen, 2)) self.assertEqual(result, [[0, 2], [4, 6], [8]]) ================================================ FILE: tests/db_url.py ================================================ from peewee import * from playhouse.db_url import connect from playhouse.db_url import parse from .base import BaseTestCase class TestDBUrl(BaseTestCase): def test_db_url_parse(self): cfg = parse('mysql://usr:pwd@hst:123/db') self.assertEqual(cfg['user'], 'usr') self.assertEqual(cfg['passwd'], 'pwd') self.assertEqual(cfg['host'], 'hst') self.assertEqual(cfg['database'], 'db') self.assertEqual(cfg['port'], 123) cfg = parse('postgresql://usr:pwd@hst/db') self.assertEqual(cfg['password'], 'pwd') cfg = parse('mysql+pool://usr:pwd@hst:123/db' '?max_connections=42&stale_timeout=8001.2&zai=&baz=3.4.5' '&boolz=false') self.assertEqual(cfg['user'], 'usr') self.assertEqual(cfg['password'], 'pwd') self.assertEqual(cfg['host'], 'hst') self.assertEqual(cfg['database'], 'db') self.assertEqual(cfg['port'], 123) self.assertEqual(cfg['max_connections'], 42) self.assertEqual(cfg['stale_timeout'], 8001.2) self.assertEqual(cfg['zai'], '') self.assertEqual(cfg['baz'], '3.4.5') self.assertEqual(cfg['boolz'], False) def test_db_url_no_unquoting(self): # By default, neither user nor password is not unescaped. cfg = parse('mysql://usr%40example.com:pwd%23@hst:123/db') self.assertEqual(cfg['user'], 'usr%40example.com') self.assertEqual(cfg['passwd'], 'pwd%23') self.assertEqual(cfg['host'], 'hst') self.assertEqual(cfg['database'], 'db') self.assertEqual(cfg['port'], 123) def test_db_url_quoted_password(self): cfg = parse('mysql://usr:pwd%23%20@hst:123/db', unquote_password=True) self.assertEqual(cfg['user'], 'usr') self.assertEqual(cfg['passwd'], 'pwd# ') self.assertEqual(cfg['host'], 'hst') self.assertEqual(cfg['database'], 'db') self.assertEqual(cfg['port'], 123) def test_db_url_quoted_user(self): cfg = parse('mysql://usr%40example.com:p%40sswd@hst:123/db', unquote_user=True) self.assertEqual(cfg['user'], 'usr@example.com') self.assertEqual(cfg['passwd'], 'p%40sswd') self.assertEqual(cfg['host'], 'hst') self.assertEqual(cfg['database'], 'db') self.assertEqual(cfg['port'], 123) def test_db_url(self): db = connect('sqlite:///:memory:') self.assertTrue(isinstance(db, SqliteDatabase)) self.assertEqual(db.database, ':memory:') db = connect('sqlite:///:memory:', pragmas=( ('journal_mode', 'MEMORY'),)) self.assertTrue(('journal_mode', 'MEMORY') in db._pragmas) #db = connect('sqliteext:///foo/bar.db') #self.assertTrue(isinstance(db, SqliteExtDatabase)) #self.assertEqual(db.database, 'foo/bar.db') db = connect('sqlite:////this/is/absolute.path') self.assertEqual(db.database, '/this/is/absolute.path') db = connect('sqlite://') self.assertTrue(isinstance(db, SqliteDatabase)) self.assertEqual(db.database, ':memory:') db = connect('sqlite:///test.db?p1=1?a&p2=22&p3=xyz') self.assertTrue(isinstance(db, SqliteDatabase)) self.assertEqual(db.database, 'test.db') self.assertEqual(db.connect_params, { 'p1': '1?a', 'p2': 22, 'p3': 'xyz'}) def test_bad_scheme(self): def _test_scheme(): connect('missing:///') self.assertRaises(RuntimeError, _test_scheme) ================================================ FILE: tests/expressions.py ================================================ from peewee import * from .base import IS_SQLITE from .base import ModelTestCase from .base import TestModel from .base import get_in_memory_db from .base import skip_if class Person(TestModel): name = CharField() class BaseNamesTest(ModelTestCase): requires = [Person] def assertNames(self, exp, x): query = Person.select().where(exp).order_by(Person.name) self.assertEqual([p.name for p in query], x) class TestRegexp(BaseNamesTest): @skip_if(IS_SQLITE) def test_regexp_iregexp(self): people = [Person.create(name=name) for name in ('n1', 'n2', 'n3')] self.assertNames(Person.name.regexp('n[1,3]'), ['n1', 'n3']) self.assertNames(Person.name.regexp('N[1,3]'), []) self.assertNames(Person.name.iregexp('n[1,3]'), ['n1', 'n3']) self.assertNames(Person.name.iregexp('N[1,3]'), ['n1', 'n3']) class TestContains(BaseNamesTest): def test_contains_startswith_endswith(self): people = [Person.create(name=n) for n in ('huey', 'mickey', 'zaizee')] self.assertNames(Person.name.contains('ey'), ['huey', 'mickey']) self.assertNames(Person.name.contains('EY'), ['huey', 'mickey']) self.assertNames(Person.name.startswith('m'), ['mickey']) self.assertNames(Person.name.startswith('M'), ['mickey']) self.assertNames(Person.name.endswith('ey'), ['huey', 'mickey']) self.assertNames(Person.name.endswith('EY'), ['huey', 'mickey']) class UpperField(TextField): def db_value(self, value): return fn.UPPER(value) class UpperModel(TestModel): name = UpperField() class TestValueConversion(ModelTestCase): """ Test the conversion of field values using a field's db_value() function. It is possible that a field's `db_value()` function may returns a Node subclass (e.g. a SQL function). These tests verify and document how such conversions are applied in various parts of the query. """ database = get_in_memory_db() requires = [UpperModel] def test_value_conversion(self): # Ensure value is converted on INSERT. insert = UpperModel.insert({UpperModel.name: 'huey'}) self.assertSQL(insert, ( 'INSERT INTO "upper_model" ("name") VALUES (UPPER(?))'), ['huey']) uid = insert.execute() obj = UpperModel.get(UpperModel.id == uid) self.assertEqual(obj.name, 'HUEY') # Ensure value is converted on UPDATE. update = (UpperModel .update({UpperModel.name: 'zaizee'}) .where(UpperModel.id == uid)) self.assertSQL(update, ( 'UPDATE "upper_model" SET "name" = UPPER(?) ' 'WHERE ("upper_model"."id" = ?)'), ['zaizee', uid]) update.execute() # Ensure it works with SELECT (or more generally, WHERE expressions). select = UpperModel.select().where(UpperModel.name == 'zaizee') self.assertSQL(select, ( 'SELECT "t1"."id", "t1"."name" FROM "upper_model" AS "t1" ' 'WHERE ("t1"."name" = UPPER(?))'), ['zaizee']) obj = select.get() self.assertEqual(obj.name, 'ZAIZEE') # Ensure it works with DELETE. delete = UpperModel.delete().where(UpperModel.name == 'zaizee') self.assertSQL(delete, ( 'DELETE FROM "upper_model" ' 'WHERE ("upper_model"."name" = UPPER(?))'), ['zaizee']) self.assertEqual(delete.execute(), 1) def test_value_conversion_mixed(self): um = UpperModel.create(name='huey') # If we apply a function to the field, the conversion is not applied. sq = UpperModel.select().where(fn.SUBSTR(UpperModel.name, 1, 1) == 'h') self.assertSQL(sq, ( 'SELECT "t1"."id", "t1"."name" FROM "upper_model" AS "t1" ' 'WHERE (SUBSTR("t1"."name", ?, ?) = ?)'), [1, 1, 'h']) self.assertRaises(UpperModel.DoesNotExist, sq.get) # If we encapsulate the object as a value, the conversion is applied. sq = UpperModel.select().where(UpperModel.name == Value('huey')) self.assertSQL(sq, ( 'SELECT "t1"."id", "t1"."name" FROM "upper_model" AS "t1" ' 'WHERE ("t1"."name" = UPPER(?))'), ['huey']) self.assertEqual(sq.get().id, um.id) # Unless we explicitly pass converter=False. sq = UpperModel.select().where(UpperModel.name == Value('huey', False)) self.assertSQL(sq, ( 'SELECT "t1"."id", "t1"."name" FROM "upper_model" AS "t1" ' 'WHERE ("t1"."name" = ?)'), ['huey']) self.assertRaises(UpperModel.DoesNotExist, sq.get) # If we specify explicit SQL on the rhs, the conversion is not applied. sq = UpperModel.select().where(UpperModel.name == SQL('?', ['huey'])) self.assertSQL(sq, ( 'SELECT "t1"."id", "t1"."name" FROM "upper_model" AS "t1" ' 'WHERE ("t1"."name" = ?)'), ['huey']) self.assertRaises(UpperModel.DoesNotExist, sq.get) # Function arguments are not coerced. sq = UpperModel.select().where(UpperModel.name == fn.LOWER('huey')) self.assertSQL(sq, ( 'SELECT "t1"."id", "t1"."name" FROM "upper_model" AS "t1" ' 'WHERE ("t1"."name" = LOWER(?))'), ['huey']) self.assertRaises(UpperModel.DoesNotExist, sq.get) def test_value_conversion_query(self): um = UpperModel.create(name='huey') UM = UpperModel.alias() subq = UM.select(UM.name).where(UM.name == 'huey') # Select from WHERE ... IN . query = UpperModel.select().where(UpperModel.name.in_(subq)) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."name" FROM "upper_model" AS "t1" ' 'WHERE ("t1"."name" IN (' 'SELECT "t2"."name" FROM "upper_model" AS "t2" ' 'WHERE ("t2"."name" = UPPER(?))))'), ['huey']) self.assertEqual(query.get().id, um.id) # Join on sub-query. query = (UpperModel .select() .join(subq, on=(UpperModel.name == subq.c.name))) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."name" FROM "upper_model" AS "t1" ' 'INNER JOIN (SELECT "t2"."name" FROM "upper_model" AS "t2" ' 'WHERE ("t2"."name" = UPPER(?))) AS "t3" ' 'ON ("t1"."name" = "t3"."name")'), ['huey']) row = query.tuples().get() self.assertEqual(row, (um.id, 'HUEY')) def test_having_clause(self): query = (UpperModel .select(UpperModel.name, fn.COUNT(UpperModel.id).alias('ct')) .group_by(UpperModel.name) .having(UpperModel.name == 'huey')) self.assertSQL(query, ( 'SELECT "t1"."name", COUNT("t1"."id") AS "ct" ' 'FROM "upper_model" AS "t1" ' 'GROUP BY "t1"."name" ' 'HAVING ("t1"."name" = UPPER(?))'), ['huey']) ================================================ FILE: tests/extra_fields.py ================================================ from peewee import * from playhouse.fields import CompressedField from playhouse.fields import PickleField from .base import db from .base import ModelTestCase from .base import TestModel class Comp(TestModel): key = TextField() data = CompressedField() class Pickled(TestModel): key = TextField() data = PickleField() class TestCompressedField(ModelTestCase): requires = [Comp] def test_compressed_field(self): a = b'a' * 1024 b = b'b' * 1024 Comp.create(data=a, key='a') Comp.create(data=b, key='b') a_db = Comp.get(Comp.key == 'a') self.assertEqual(a_db.data, a) b_db = Comp.get(Comp.key == 'b') self.assertEqual(b_db.data, b) # Get at the underlying data. CompTbl = Table('comp', ('id', 'data', 'key')).bind(self.database) obj = CompTbl.select().where(CompTbl.key == 'a').get() self.assertEqual(obj['key'], 'a') # Ensure that the data actually was compressed. self.assertTrue(len(obj['data']) < 1024) class TestPickleField(ModelTestCase): requires = [Pickled] def test_pickle_field(self): a = {'k1': 'v1', 'k2': [0, 1, 2], 'k3': None} b = 'just a string' Pickled.create(data=a, key='a') Pickled.create(data=b, key='b') a_db = Pickled.get(Pickled.key == 'a') self.assertEqual(a_db.data, a) b_db = Pickled.get(Pickled.key == 'b') self.assertEqual(b_db.data, b) ================================================ FILE: tests/fields.py ================================================ import calendar import datetime import sqlite3 import time import uuid from decimal import Decimal as D from decimal import ROUND_UP from peewee import NodeList from peewee import VirtualField from peewee import * from .base import BaseTestCase from .base import IS_CRDB from .base import IS_MYSQL from .base import IS_POSTGRESQL from .base import IS_SQLITE from .base import ModelTestCase from .base import TestModel from .base import db from .base import get_in_memory_db from .base import requires_models from .base import requires_mysql from .base import requires_pglike from .base import requires_sqlite from .base import skip_if from .base_models import Tweet from .base_models import User class IntModel(TestModel): value = IntegerField() value_null = IntegerField(null=True) class TestCoerce(ModelTestCase): requires = [IntModel] def test_coerce(self): i = IntModel.create(value='1337', value_null=3.14159) i_db = IntModel.get(IntModel.id == i.id) self.assertEqual(i_db.value, 1337) self.assertEqual(i_db.value_null, 3) class DefaultValues(TestModel): data = IntegerField(default=17) data_callable = IntegerField(default=lambda: 1337) class TestTextField(TextField): def first_char(self): return fn.SUBSTR(self, 1, 1) class PhoneBook(TestModel): name = TestTextField() class Bits(TestModel): F_STICKY = 1 F_FAVORITE = 2 F_MINIMIZED = 4 flags = BitField() is_sticky = flags.flag(F_STICKY) is_favorite = flags.flag(F_FAVORITE) is_minimized = flags.flag(F_MINIMIZED) data = BigBitField() class TestDefaultValues(ModelTestCase): requires = [DefaultValues] def test_default_values(self): d = DefaultValues() self.assertEqual(d.data, 17) self.assertEqual(d.data_callable, 1337) d.save() d_db = DefaultValues.get(DefaultValues.id == d.id) self.assertEqual(d_db.data, 17) self.assertEqual(d_db.data_callable, 1337) def test_defaults_create(self): d = DefaultValues.create() self.assertEqual(d.data, 17) self.assertEqual(d.data_callable, 1337) d_db = DefaultValues.get(DefaultValues.id == d.id) self.assertEqual(d_db.data, 17) self.assertEqual(d_db.data_callable, 1337) class TestNullConstraint(ModelTestCase): requires = [IntModel] def test_null(self): i = IntModel.create(value=1) i_db = IntModel.get(IntModel.value == 1) self.assertIsNone(i_db.value_null) def test_empty_value(self): with self.database.atomic(): with self.assertRaisesCtx(IntegrityError): IntModel.create(value=None) class TestIntegerField(ModelTestCase): requires = [IntModel] def test_integer_field(self): i1 = IntModel.create(value=1) i2 = IntModel.create(value=2, value_null=20) vals = [(i.value, i.value_null) for i in IntModel.select().order_by(IntModel.value)] self.assertEqual(vals, [ (1, None), (2, 20)]) class FloatModel(TestModel): value = FloatField() value_null = FloatField(null=True) class TestFloatField(ModelTestCase): requires = [FloatModel] def test_float_field(self): f1 = FloatModel.create(value=1.23) f2 = FloatModel.create(value=3.14, value_null=0.12) query = FloatModel.select().order_by(FloatModel.id) self.assertEqual([(f.value, f.value_null) for f in query], [(1.23, None), (3.14, 0.12)]) class DecimalModel(TestModel): value = DecimalField(decimal_places=2, auto_round=True) value_up = DecimalField(decimal_places=2, auto_round=True, rounding=ROUND_UP, null=True) class TestDecimalField(ModelTestCase): requires = [DecimalModel] def test_decimal_field(self): d1 = DecimalModel.create(value=D('3')) d2 = DecimalModel.create(value=D('100.33')) self.assertEqual(sorted(d.value for d in DecimalModel.select()), [D('3'), D('100.33')]) def test_decimal_rounding(self): d = DecimalModel.create(value=D('1.2345'), value_up=D('1.2345')) d_db = DecimalModel.get(DecimalModel.id == d.id) self.assertEqual(d_db.value, D('1.23')) self.assertEqual(d_db.value_up, D('1.24')) class BoolModel(TestModel): value = BooleanField(null=True) name = CharField() class TestBooleanField(ModelTestCase): requires = [BoolModel] def test_boolean_field(self): BoolModel.create(value=True, name='t') BoolModel.create(value=False, name='f') BoolModel.create(value=None, name='n') vals = sorted((b.name, b.value) for b in BoolModel.select()) self.assertEqual(vals, [ ('f', False), ('n', None), ('t', True)]) class DateModel(TestModel): date = DateField(null=True) time = TimeField(null=True) date_time = DateTimeField(null=True) class CustomDateTimeModel(TestModel): date_time = DateTimeField(formats=[ '%m/%d/%Y %I:%M %p', '%Y-%m-%d %H:%M:%S']) class TestDateFields(ModelTestCase): requires = [DateModel] @requires_models(CustomDateTimeModel) def test_date_time_custom_format(self): cdtm = CustomDateTimeModel.create(date_time='01/02/2003 01:37 PM') cdtm_db = CustomDateTimeModel[cdtm.id] self.assertEqual(cdtm_db.date_time, datetime.datetime(2003, 1, 2, 13, 37, 0)) def test_date_fields(self): dt1 = datetime.datetime(2011, 1, 2, 11, 12, 13, 54321) dt2 = datetime.datetime(2011, 1, 2, 11, 12, 13) d1 = datetime.date(2011, 1, 3) t1 = datetime.time(11, 12, 13, 54321) t2 = datetime.time(11, 12, 13) if isinstance(self.database, MySQLDatabase): dt1 = dt1.replace(microsecond=0) t1 = t1.replace(microsecond=0) dm1 = DateModel.create(date_time=dt1, date=d1, time=t1) dm2 = DateModel.create(date_time=dt2, time=t2) dm1_db = DateModel.get(DateModel.id == dm1.id) self.assertEqual(dm1_db.date, d1) self.assertEqual(dm1_db.date_time, dt1) self.assertEqual(dm1_db.time, t1) dm2_db = DateModel.get(DateModel.id == dm2.id) self.assertEqual(dm2_db.date, None) self.assertEqual(dm2_db.date_time, dt2) self.assertEqual(dm2_db.time, t2) def test_extract_parts(self): dm = DateModel.create( date_time=datetime.datetime(2011, 1, 2, 11, 12, 13, 54321), date=datetime.date(2012, 2, 3), time=datetime.time(3, 13, 37)) query = (DateModel .select(DateModel.date_time.year, DateModel.date_time.month, DateModel.date_time.day, DateModel.date_time.hour, DateModel.date_time.minute, DateModel.date_time.second, DateModel.date.year, DateModel.date.month, DateModel.date.day, DateModel.time.hour, DateModel.time.minute, DateModel.time.second) .tuples()) row, = query if IS_SQLITE or IS_MYSQL: self.assertEqual(row, (2011, 1, 2, 11, 12, 13, 2012, 2, 3, 3, 13, 37)) else: self.assertTrue(row in [ (2011., 1., 2., 11., 12., 13.054321, 2012., 2., 3., 3., 13., 37.), (D('2011'), D('1'), D('2'), D('11'), D('12'), D('13.054321'), D('2012'), D('2'), D('3'), D('3'), D('13'), D('37'))]) def test_truncate_date(self): dm = DateModel.create( date_time=datetime.datetime(2001, 2, 3, 4, 5, 6, 7), date=datetime.date(2002, 3, 4)) accum = [] for p in ('year', 'month', 'day', 'hour', 'minute', 'second'): accum.append(DateModel.date_time.truncate(p)) for p in ('year', 'month', 'day'): accum.append(DateModel.date.truncate(p)) query = DateModel.select(*accum).tuples() data = list(query[0]) # Postgres includes timezone info, so strip that for comparison. if IS_POSTGRESQL or IS_CRDB: data = [dt.replace(tzinfo=None) for dt in data] self.assertEqual(data, [ datetime.datetime(2001, 1, 1, 0, 0, 0), datetime.datetime(2001, 2, 1, 0, 0, 0), datetime.datetime(2001, 2, 3, 0, 0, 0), datetime.datetime(2001, 2, 3, 4, 0, 0), datetime.datetime(2001, 2, 3, 4, 5, 0), datetime.datetime(2001, 2, 3, 4, 5, 6), datetime.datetime(2002, 1, 1, 0, 0, 0), datetime.datetime(2002, 3, 1, 0, 0, 0), datetime.datetime(2002, 3, 4, 0, 0, 0)]) def test_to_timestamp(self): dt = datetime.datetime(2019, 1, 2, 3, 4, 5) ts = calendar.timegm(dt.utctimetuple()) dt2 = datetime.datetime(2019, 1, 3) ts2 = calendar.timegm(dt2.utctimetuple()) DateModel.create(date_time=dt, date=dt2.date()) query = DateModel.select( DateModel.id, DateModel.date_time.to_timestamp().alias('dt_ts'), DateModel.date.to_timestamp().alias('dt2_ts')) obj = query.get() self.assertEqual(obj.dt_ts, ts) self.assertEqual(obj.dt2_ts, ts2) ts3 = ts + 86400 query = (DateModel.select() .where((DateModel.date_time.to_timestamp() + 86400) < ts3)) self.assertRaises(DateModel.DoesNotExist, query.get) query = (DateModel.select() .where((DateModel.date.to_timestamp() + 86400) > ts3)) self.assertEqual(query.get().id, obj.id) def test_distinct_date_part(self): years = (1980, 1990, 2000, 2010) for i, year in enumerate(years): for j in range(i + 1): DateModel.create(date=datetime.date(year, i + 1, 1)) query = (DateModel .select(DateModel.date.year.distinct()) .order_by(DateModel.date.year)) self.assertEqual([year for year, in query.tuples()], [1980, 1990, 2000, 2010]) class U2(TestModel): username = TextField() class T2(TestModel): user = ForeignKeyField(U2, backref='tweets', on_delete='CASCADE') content = TextField() class TestForeignKeyField(ModelTestCase): requires = [User, Tweet] def test_set_fk(self): huey = User.create(username='huey') zaizee = User.create(username='zaizee') # Test resolution of attributes after creation does not trigger SELECT. with self.assertQueryCount(1): tweet = Tweet.create(content='meow', user=huey) self.assertEqual(tweet.user.username, 'huey') # Test we can set to an integer, in which case a query will occur. with self.assertQueryCount(2): tweet = Tweet.create(content='purr', user=zaizee.id) self.assertEqual(tweet.user.username, 'zaizee') # Test we can set the ID accessor directly. with self.assertQueryCount(2): tweet = Tweet.create(content='hiss', user_id=huey.id) self.assertEqual(tweet.user.username, 'huey') def test_follow_attributes(self): huey = User.create(username='huey') Tweet.create(content='meow', user=huey) Tweet.create(content='hiss', user=huey) with self.assertQueryCount(1): query = (Tweet .select(Tweet.content, Tweet.user.username) .join(User) .order_by(Tweet.content)) self.assertEqual([(tweet.content, tweet.user.username) for tweet in query], [('hiss', 'huey'), ('meow', 'huey')]) self.assertRaises(AttributeError, lambda: Tweet.user.foo) def test_disable_backref(self): class Person(TestModel): pass class Pet(TestModel): owner = ForeignKeyField(Person, backref='!') self.assertEqual(Pet.owner.backref, '!') # No attribute/accessor is added to the related model. self.assertRaises(AttributeError, lambda: Person.pet_set) # We still preserve the metadata about the relationship. self.assertTrue(Pet.owner in Person._meta.backrefs) @requires_models(U2, T2) def test_on_delete_behavior(self): if IS_SQLITE: self.database.foreign_keys = 1 with self.database.atomic(): for username in ('u1', 'u2', 'u3'): user = U2.create(username=username) for i in range(3): T2.create(user=user, content='%s-%s' % (username, i)) self.assertEqual(T2.select().count(), 9) U2.delete().where(U2.username == 'u2').execute() self.assertEqual(T2.select().count(), 6) query = (U2 .select(U2.username, fn.COUNT(T2.id).alias('ct')) .join(T2, JOIN.LEFT_OUTER) .group_by(U2.username) .order_by(U2.username)) self.assertEqual([(u.username, u.ct) for u in query], [ ('u1', 3), ('u3', 3)]) class M1(TestModel): name = CharField(primary_key=True) m2 = DeferredForeignKey('M2', deferrable='INITIALLY DEFERRED', on_delete='CASCADE') class M2(TestModel): name = CharField(primary_key=True) m1 = ForeignKeyField(M1, deferrable='INITIALLY DEFERRED', on_delete='CASCADE') @skip_if(IS_MYSQL) @skip_if(IS_CRDB, 'crdb does not support deferred foreign-key constraints') class TestDeferredForeignKey(ModelTestCase): requires = [M1, M2] def test_deferred_foreign_key(self): with self.database.atomic(): m1 = M1.create(name='m1', m2='m2') m2 = M2.create(name='m2', m1='m1') m1_db = M1.get(M1.name == 'm1') self.assertEqual(m1_db.m2.name, 'm2') m2_db = M2.get(M2.name == 'm2') self.assertEqual(m2_db.m1.name, 'm1') class TestDeferredForeignKeyResolution(ModelTestCase): def test_unresolved_deferred_fk(self): class Photo(Model): album = DeferredForeignKey('Album', column_name='id_album') class Meta: database = get_in_memory_db() self.assertSQL(Photo.select(), ( 'SELECT "t1"."id", "t1"."id_album" FROM "photo" AS "t1"'), []) def test_deferred_foreign_key_resolution(self): class Base(Model): class Meta: database = get_in_memory_db() class Photo(Base): album = DeferredForeignKey('Album', column_name='id_album', null=False, backref='pictures') alt_album = DeferredForeignKey('Album', column_name='id_Alt_album', field='alt_id', backref='alt_pix', null=True) class Album(Base): name = TextField() alt_id = IntegerField(column_name='_Alt_id') self.assertTrue(Photo.album.rel_model is Album) self.assertTrue(Photo.album.rel_field is Album.id) self.assertEqual(Photo.album.column_name, 'id_album') self.assertFalse(Photo.album.null) self.assertTrue(Photo.alt_album.rel_model is Album) self.assertTrue(Photo.alt_album.rel_field is Album.alt_id) self.assertEqual(Photo.alt_album.column_name, 'id_Alt_album') self.assertTrue(Photo.alt_album.null) self.assertSQL(Photo._schema._create_table(), ( 'CREATE TABLE IF NOT EXISTS "photo" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"id_album" INTEGER NOT NULL, ' '"id_Alt_album" INTEGER)'), []) self.assertSQL(Photo._schema._create_foreign_key(Photo.album), ( 'ALTER TABLE "photo" ADD CONSTRAINT "fk_photo_id_album_refs_album"' ' FOREIGN KEY ("id_album") REFERENCES "album" ("id")')) self.assertSQL(Photo._schema._create_foreign_key(Photo.alt_album), ( 'ALTER TABLE "photo" ADD CONSTRAINT ' '"fk_photo_id_Alt_album_refs_album"' ' FOREIGN KEY ("id_Alt_album") REFERENCES "album" ("_Alt_id")')) self.assertSQL(Photo.select(), ( 'SELECT "t1"."id", "t1"."id_album", "t1"."id_Alt_album" ' 'FROM "photo" AS "t1"'), []) a = Album(id=3, alt_id=4) self.assertSQL(a.pictures, ( 'SELECT "t1"."id", "t1"."id_album", "t1"."id_Alt_album" ' 'FROM "photo" AS "t1" WHERE ("t1"."id_album" = ?)'), [3]) self.assertSQL(a.alt_pix, ( 'SELECT "t1"."id", "t1"."id_album", "t1"."id_Alt_album" ' 'FROM "photo" AS "t1" WHERE ("t1"."id_Alt_album" = ?)'), [4]) class Composite(TestModel): first = CharField() last = CharField() data = TextField() class Meta: primary_key = CompositeKey('first', 'last') class TestCompositePrimaryKeyField(ModelTestCase): requires = [Composite] def test_composite_primary_key(self): pass class TestFieldFunction(ModelTestCase): requires = [PhoneBook] def setUp(self): super(TestFieldFunction, self).setUp() names = ('huey', 'mickey', 'zaizee', 'beanie', 'scout', 'hallee') for name in names: PhoneBook.create(name=name) def _test_field_function(self, PB): query = (PB .select() .where(PB.name.first_char() == 'h') .order_by(PB.name)) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."name" ' 'FROM "phone_book" AS "t1" ' 'WHERE (SUBSTR("t1"."name", ?, ?) = ?) ' 'ORDER BY "t1"."name"'), [1, 1, 'h']) self.assertEqual([pb.name for pb in query], ['hallee', 'huey']) def test_field_function(self): self._test_field_function(PhoneBook) def test_field_function_alias(self): self._test_field_function(PhoneBook.alias()) class IPModel(TestModel): ip = IPField() ip_null = IPField(null=True) class TestIPField(ModelTestCase): requires = [IPModel] def test_ip_field(self): ips = ('0.0.0.0', '255.255.255.255', '192.168.1.1') for ip in ips: i = IPModel.create(ip=ip) i_db = IPModel.get(ip=ip) self.assertEqual(i_db.ip, ip) self.assertEqual(i_db.ip_null, None) class TestBitFields(ModelTestCase): requires = [Bits] def test_bit_field_update(self): def assertFlags(expected): query = Bits.select().order_by(Bits.id) self.assertEqual([b.flags for b in query], expected) # Bits - flags (1=sticky, 2=favorite, 4=minimized) for i in range(1, 5): Bits.create(flags=i) q = Bits.select((~Bits.flags & 2).alias('bn')).order_by(Bits.id) self.assertEqual([b.bn for b in q], [2, 0, 0, 2]) q = Bits.select().where((Bits.flags & 2) != 0).order_by(Bits.id) self.assertEqual([b.flags for b in q], [2, 3]) Bits.update(flags=Bits.flags & ~2).execute() assertFlags([1, 0, 1, 4]) Bits.update(flags=Bits.flags | 2).execute() assertFlags([3, 2, 3, 6]) Bits.update(flags=Bits.is_favorite.clear()).execute() assertFlags([1, 0, 1, 4]) Bits.update(flags=Bits.is_favorite.set()).execute() assertFlags([3, 2, 3, 6]) # Clear multiple bits in one operation. Bits.update(flags=Bits.flags & ~(1 | 4)).execute() assertFlags([2, 2, 2, 2]) def test_bit_field_auto_flag(self): class Bits2(TestModel): flags = BitField() f1 = flags.flag() # Automatically gets 1. f2 = flags.flag() # 2 f4 = flags.flag() # 4 f16 = flags.flag(16) f32 = flags.flag() # 32 b = Bits2() self.assertEqual(b.flags, 0) b.f1 = True self.assertEqual(b.flags, 1) b.f4 = True self.assertEqual(b.flags, 5) b.f32 = True self.assertEqual(b.flags, 37) def test_bit_field_instance_flags(self): b = Bits() self.assertEqual(b.flags, 0) self.assertFalse(b.is_sticky) self.assertFalse(b.is_favorite) self.assertFalse(b.is_minimized) b.is_sticky = True b.is_minimized = True self.assertEqual(b.flags, 5) # 1 | 4 self.assertTrue(b.is_sticky) self.assertFalse(b.is_favorite) self.assertTrue(b.is_minimized) b.flags = 3 self.assertTrue(b.is_sticky) self.assertTrue(b.is_favorite) self.assertFalse(b.is_minimized) def test_bit_field(self): b1 = Bits.create(flags=1) b2 = Bits.create(flags=2) b3 = Bits.create(flags=3) query = Bits.select().where(Bits.is_sticky).order_by(Bits.id) self.assertEqual([x.id for x in query], [b1.id, b3.id]) query = Bits.select().where(Bits.is_favorite).order_by(Bits.id) self.assertEqual([x.id for x in query], [b2.id, b3.id]) query = Bits.select().where(~Bits.is_favorite).order_by(Bits.id) self.assertEqual([x.id for x in query], [b1.id]) # "&" operator does bitwise and for BitField. query = Bits.select().where((Bits.flags & 1) == 1).order_by(Bits.id) self.assertEqual([x.id for x in query], [b1.id, b3.id]) # Test combining multiple bit expressions. query = Bits.select().where(Bits.is_sticky & Bits.is_favorite) self.assertEqual([x.id for x in query], [b3.id]) query = Bits.select().where(Bits.is_sticky & ~Bits.is_favorite) self.assertEqual([x.id for x in query], [b1.id]) def test_bigbit_field_instance_data(self): b = Bits() values_to_set = (1, 11, 63, 31, 55, 48, 100, 99) for value in values_to_set: b.data.set_bit(value) for i in range(128): self.assertEqual(b.data.is_set(i), i in values_to_set) for i in range(128): b.data.clear_bit(i) buf = bytes(b.data._buffer) self.assertEqual(len(buf), 16) self.assertEqual(bytes(buf), b'\x00' * 16) def test_bigbit_zero_idx(self): b = Bits() b.data.set_bit(0) self.assertTrue(b.data.is_set(0)) b.data.clear_bit(0) self.assertFalse(b.data.is_set(0)) # Out-of-bounds returns False and does not extend data. self.assertFalse(b.data.is_set(1000)) self.assertTrue(len(b.data), 1) def test_bigbit_item_methods(self): b = Bits() idxs = [0, 1, 4, 7, 8, 15, 16, 31, 32, 63] for i in idxs: b.data[i] = True for i in range(64): self.assertEqual(b.data[i], i in idxs) data = list(b.data) self.assertEqual(data, [1 if i in idxs else 0 for i in range(64)]) for i in range(64): del b.data[i] self.assertEqual(len(b.data), 8) self.assertEqual(b.data._buffer, b'\x00' * 8) def test_bigbit_set_clear(self): b = Bits() b.data = b'\x01' for i in range(8): self.assertEqual(b.data[i], i == 0) b.data.clear() self.assertEqual(len(b.data), 0) def test_bigbit_field(self): b = Bits.create() b.data.set_bit(1) b.data.set_bit(3) b.data.set_bit(5) b.save() b_db = Bits.get(Bits.id == b.id) for x in range(7): if x % 2 == 1: self.assertTrue(b_db.data.is_set(x)) else: self.assertFalse(b_db.data.is_set(x)) def test_bigbit_field_bitwise(self): b1 = Bits(data=b'\x11') b2 = Bits(data=b'\x12') b3 = Bits(data=b'\x99') self.assertEqual(b1.data & b2.data, b'\x10') self.assertEqual(b1.data | b2.data, b'\x13') self.assertEqual(b1.data ^ b2.data, b'\x03') self.assertEqual(b1.data & b3.data, b'\x11') self.assertEqual(b1.data | b3.data, b'\x99') self.assertEqual(b1.data ^ b3.data, b'\x88') b1.data &= b2.data self.assertEqual(b1.data._buffer, b'\x10') b1.data |= b2.data self.assertEqual(b1.data._buffer, b'\x12') b1.data ^= b3.data self.assertEqual(b1.data._buffer, b'\x8b') b1.data = b'\x11' self.assertEqual(b1.data & b'\xff\xff', b'\x11\x00') self.assertEqual(b1.data | b'\xff\xff', b'\xff\xff') self.assertEqual(b1.data ^ b'\xff\xff', b'\xee\xff') b1.data = b'\x11\x11' self.assertEqual(b1.data & b'\xff', b'\x11\x00') self.assertEqual(b1.data | b'\xff', b'\xff\x11') self.assertEqual(b1.data ^ b'\xff', b'\xee\x11') def test_bigbit_field_bulk_create(self): b1, b2, b3 = Bits(), Bits(), Bits() b1.data.set_bit(1) b2.data.set_bit(2) b3.data.set_bit(3) Bits.bulk_create([b1, b2, b3]) self.assertEqual(len(Bits), 3) for b in Bits.select(): self.assertEqual(sum(1 if b.data.is_set(i) else 0 for i in (1, 2, 3)), 1) def test_bigbit_field_bulk_update(self): b1, b2, b3 = Bits.create(), Bits.create(), Bits.create() b1.data.set_bit(11) b2.data.set_bit(12) b3.data.set_bit(13) Bits.bulk_update([b1, b2, b3], fields=[Bits.data]) mapping = {b1.id: 11, b2.id: 12, b3.id: 13} for b in Bits.select(): bit = mapping[b.id] self.assertTrue(b.data.is_set(bit)) class BlobModel(TestModel): data = BlobField() class TestBlobField(ModelTestCase): requires = [BlobModel] def test_blob_field(self): b = BlobModel.create(data=b'\xff\x01') b_db = BlobModel.get(BlobModel.data == b'\xff\x01') self.assertEqual(b.id, b_db.id) data = b_db.data if isinstance(data, memoryview): data = data.tobytes() elif not isinstance(data, bytes): data = bytes(data) self.assertEqual(data, b'\xff\x01') def test_blob_on_proxy(self): db = Proxy() class NewBlobModel(Model): data = BlobField() class Meta: database = db db_obj = SqliteDatabase(':memory:') db.initialize(db_obj) self.assertTrue(NewBlobModel.data._constructor is sqlite3.Binary) def test_blob_db_hook(self): sentinel = object() class FakeDatabase(Database): def get_binary_type(self): return sentinel class B(Model): b1 = BlobField() b2 = BlobField() B._meta.set_database(FakeDatabase(None)) self.assertTrue(B.b1._constructor is sentinel) self.assertTrue(B.b2._constructor is sentinel) alt_db = SqliteDatabase(':memory:') with alt_db.bind_ctx([B]): # The constructor has been changed. self.assertTrue(B.b1._constructor is sqlite3.Binary) self.assertTrue(B.b2._constructor is sqlite3.Binary) # The constructor has been restored. self.assertTrue(B.b1._constructor is sentinel) self.assertTrue(B.b2._constructor is sentinel) class BigModel(TestModel): pk = BigAutoField() data = TextField() class TestBigAutoField(ModelTestCase): requires = [BigModel] def test_big_auto_field(self): b1 = BigModel.create(data='b1') b2 = BigModel.create(data='b2') b1_db = BigModel.get(BigModel.pk == b1.pk) b2_db = BigModel.get(BigModel.pk == b2.pk) self.assertTrue(b1_db.pk < b2_db.pk) self.assertTrue(b1_db.data, 'b1') self.assertTrue(b2_db.data, 'b2') class Item(TestModel): price = IntegerField() multiplier = FloatField(default=1.) class Bare(TestModel): key = BareField() value = BareField(adapt=int, null=True) class TestFieldValueHandling(ModelTestCase): requires = [Item] @skip_if(IS_CRDB, 'crdb requires cast to multiply int and float') def test_int_float_multi(self): i = Item.create(price=10, multiplier=0.75) query = (Item .select(Item, (Item.price * Item.multiplier).alias('total')) .where(Item.id == i.id)) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."price", "t1"."multiplier", ' '("t1"."price" * "t1"."multiplier") AS "total" ' 'FROM "item" AS "t1" ' 'WHERE ("t1"."id" = ?)'), [i.id]) i_db = query.get() self.assertEqual(i_db.price, 10) self.assertEqual(i_db.multiplier, .75) self.assertEqual(i_db.total, 7.5) # By default, Peewee will use the Price field (integer) converter to # coerce the value of it's right-hand operand (converting to 0). query = (Item .select(Item, (Item.price * 0.75).alias('total')) .where(Item.id == i.id)) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."price", "t1"."multiplier", ' '("t1"."price" * ?) AS "total" ' 'FROM "item" AS "t1" ' 'WHERE ("t1"."id" = ?)'), [0, i.id]) # We can explicitly pass "False" and the value will not be converted. exp = Item.price * Value(0.75, False) query = (Item .select(Item, exp.alias('total')) .where(Item.id == i.id)) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."price", "t1"."multiplier", ' '("t1"."price" * ?) AS "total" ' 'FROM "item" AS "t1" ' 'WHERE ("t1"."id" = ?)'), [0.75, i.id]) i_db = query.get() self.assertEqual(i_db.price, 10) self.assertEqual(i_db.multiplier, .75) self.assertEqual(i_db.total, 7.5) def test_explicit_cast(self): prices = ((10, 1.1), (5, .5)) for price, multiplier in prices: Item.create(price=price, multiplier=multiplier) text = 'CHAR' if IS_MYSQL else 'TEXT' query = (Item .select(Item.price.cast(text).alias('price_text'), Item.multiplier.cast(text).alias('multiplier_text')) .order_by(Item.id) .dicts()) self.assertEqual(list(query), [ {'price_text': '10', 'multiplier_text': '1.1'}, {'price_text': '5', 'multiplier_text': '0.5'}, ]) item = (Item .select(Item.price.cast(text).alias('price'), Item.multiplier.cast(text).alias('multiplier')) .where(Item.price == 10) .get()) self.assertEqual(item.price, '10') self.assertEqual(item.multiplier, '1.1') @requires_sqlite @requires_models(Bare) def test_bare_model_adapt(self): b1 = Bare.create(key='k1', value=1) b2 = Bare.create(key='k2', value='2') b3 = Bare.create(key='k3', value=None) b1_db = Bare.get(Bare.id == b1.id) self.assertEqual(b1_db.key, 'k1') self.assertEqual(b1_db.value, 1) b2_db = Bare.get(Bare.id == b2.id) self.assertEqual(b2_db.key, 'k2') self.assertEqual(b2_db.value, 2) b3_db = Bare.get(Bare.id == b3.id) self.assertEqual(b3_db.key, 'k3') self.assertTrue(b3_db.value is None) class UUIDModel(TestModel): data = UUIDField(null=True) bdata = BinaryUUIDField(null=True) class TestUUIDField(ModelTestCase): requires = [UUIDModel] def test_uuid_field(self): uu = uuid.uuid4() u = UUIDModel.create(data=uu) u_db = UUIDModel.get(UUIDModel.id == u.id) self.assertEqual(u_db.data, uu) self.assertTrue(u_db.bdata is None) u_db2 = UUIDModel.get(UUIDModel.data == uu) self.assertEqual(u_db2.id, u.id) # Verify we can use hex string. uu = uuid.uuid4() u = UUIDModel.create(data=uu.hex) u_db = UUIDModel.get(UUIDModel.data == uu.hex) self.assertEqual(u.id, u_db.id) self.assertEqual(u_db.data, uu) # Verify we can use raw binary representation. uu = uuid.uuid4() u = UUIDModel.create(data=uu.bytes) u_db = UUIDModel.get(UUIDModel.data == uu.bytes) self.assertEqual(u.id, u_db.id) self.assertEqual(u_db.data, uu) def test_binary_uuid_field(self): uu = uuid.uuid4() u = UUIDModel.create(bdata=uu) u_db = UUIDModel.get(UUIDModel.id == u.id) self.assertEqual(u_db.bdata, uu) self.assertTrue(u_db.data is None) u_db2 = UUIDModel.get(UUIDModel.bdata == uu) self.assertEqual(u_db2.id, u.id) # Verify we can use hex string. uu = uuid.uuid4() u = UUIDModel.create(bdata=uu.hex) u_db = UUIDModel.get(UUIDModel.bdata == uu.hex) self.assertEqual(u.id, u_db.id) self.assertEqual(u_db.bdata, uu) # Verify we can use raw binary representation. uu = uuid.uuid4() u = UUIDModel.create(bdata=uu.bytes) u_db = UUIDModel.get(UUIDModel.bdata == uu.bytes) self.assertEqual(u.id, u_db.id) self.assertEqual(u_db.bdata, uu) class UU1(TestModel): id = UUIDField(default=uuid.uuid4, primary_key=True) name = TextField() class UU2(TestModel): id = UUIDField(default=uuid.uuid4, primary_key=True) u1 = ForeignKeyField(UU1) name = TextField() class TestForeignKeyUUIDField(ModelTestCase): requires = [UU1, UU2] def test_bulk_insert(self): # Create three UU1 instances. UU1.insert_many([{UU1.name: name} for name in 'abc'], fields=[UU1.id, UU1.name]).execute() ua, ub, uc = UU1.select().order_by(UU1.name) # Create several UU2 instances. data = ( ('a1', ua), ('b1', ub), ('b2', ub), ('c1', uc)) iq = UU2.insert_many([{UU2.name: name, UU2.u1: u} for name, u in data], fields=[UU2.id, UU2.name, UU2.u1]) iq.execute() query = UU2.select().order_by(UU2.name) for (name, u1), u2 in zip(data, query): self.assertEqual(u2.name, name) self.assertEqual(u2.u1.id, u1.id) class TSModel(TestModel): ts_s = TimestampField() ts_us = TimestampField(resolution=10 ** 6) ts_ms = TimestampField(resolution=3) # Milliseconds. ts_u = TimestampField(null=True, utc=True) class TSR(TestModel): ts_0 = TimestampField(resolution=0) ts_1 = TimestampField(resolution=1) ts_10 = TimestampField(resolution=10) ts_2 = TimestampField(resolution=2) class TestTimestampField(ModelTestCase): requires = [TSModel] @requires_models(TSR) def test_timestamp_field_resolutions(self): dt = datetime.datetime(2018, 3, 1, 3, 3, 7).replace(microsecond=123456) ts = TSR.create(ts_0=dt, ts_1=dt, ts_10=dt, ts_2=dt) ts_db = TSR[ts.id] # Zero and one are both treated as "seconds" resolution. self.assertEqual(ts_db.ts_0, dt.replace(microsecond=0)) self.assertEqual(ts_db.ts_1, dt.replace(microsecond=0)) self.assertEqual(ts_db.ts_10, dt.replace(microsecond=100000)) self.assertEqual(ts_db.ts_2, dt.replace(microsecond=120000)) def test_timestamp_field(self): dt = datetime.datetime(2018, 3, 1, 3, 3, 7) dt = dt.replace(microsecond=31337) # us=031_337, ms=031. ts = TSModel.create(ts_s=dt, ts_us=dt, ts_ms=dt, ts_u=dt) ts_db = TSModel.get(TSModel.id == ts.id) self.assertEqual(ts_db.ts_s, dt.replace(microsecond=0)) self.assertEqual(ts_db.ts_ms, dt.replace(microsecond=31000)) self.assertEqual(ts_db.ts_us, dt) self.assertEqual(ts_db.ts_u, dt.replace(microsecond=0)) self.assertEqual(TSModel.get(TSModel.ts_s == dt).id, ts.id) self.assertEqual(TSModel.get(TSModel.ts_ms == dt).id, ts.id) self.assertEqual(TSModel.get(TSModel.ts_us == dt).id, ts.id) self.assertEqual(TSModel.get(TSModel.ts_u == dt).id, ts.id) def test_timestamp_field_math(self): dt = datetime.datetime(2019, 1, 2, 3, 4, 5, 31337) ts = TSModel.create(ts_s=dt, ts_us=dt, ts_ms=dt) # Although these fields use different scales for storing the # timestamps, adding "1" has the effect of adding a single second - # the value will be multiplied by the correct scale via the converter. TSModel.update( ts_s=TSModel.ts_s + 1, ts_us=TSModel.ts_us + 1, ts_ms=TSModel.ts_ms + 1).execute() ts_db = TSModel.get(TSModel.id == ts.id) dt2 = dt + datetime.timedelta(seconds=1) self.assertEqual(ts_db.ts_s, dt2.replace(microsecond=0)) self.assertEqual(ts_db.ts_us, dt2) self.assertEqual(ts_db.ts_ms, dt2.replace(microsecond=31000)) def test_timestamp_field_value_as_ts(self): dt = datetime.datetime(2018, 3, 1, 3, 3, 7, 31337) unix_ts = time.mktime(dt.timetuple()) + 0.031337 ts = TSModel.create(ts_s=unix_ts, ts_us=unix_ts, ts_ms=unix_ts, ts_u=unix_ts) # Fetch from the DB and validate the values were stored correctly. ts_db = TSModel[ts.id] self.assertEqual(ts_db.ts_s, dt.replace(microsecond=0)) self.assertEqual(ts_db.ts_ms, dt.replace(microsecond=31000)) self.assertEqual(ts_db.ts_us, dt) utc_dt = TimestampField().local_to_utc(dt) self.assertEqual(ts_db.ts_u, utc_dt) # Verify we can query using a timestamp. self.assertEqual(TSModel.get(TSModel.ts_s == unix_ts).id, ts.id) self.assertEqual(TSModel.get(TSModel.ts_ms == unix_ts).id, ts.id) self.assertEqual(TSModel.get(TSModel.ts_us == unix_ts).id, ts.id) self.assertEqual(TSModel.get(TSModel.ts_u == unix_ts).id, ts.id) def test_timestamp_utc_vs_localtime(self): local_field = TimestampField() utc_field = TimestampField(utc=True) dt = datetime.datetime(2019, 1, 1, 12) unix_ts = int(local_field.get_timestamp(dt)) utc_ts = int(utc_field.get_timestamp(dt)) # Local timestamp is unmodified. Verify that when utc=True, the # timestamp is converted from local time to UTC. self.assertEqual(local_field.db_value(dt), unix_ts) self.assertEqual(utc_field.db_value(dt), utc_ts) self.assertEqual(local_field.python_value(unix_ts), dt) self.assertEqual(utc_field.python_value(utc_ts), dt) # Convert back-and-forth several times. dbv, pyv = local_field.db_value, local_field.python_value self.assertEqual(pyv(dbv(pyv(dbv(dt)))), dt) dbv, pyv = utc_field.db_value, utc_field.python_value self.assertEqual(pyv(dbv(pyv(dbv(dt)))), dt) def test_timestamp_field_parts(self): dt = datetime.datetime(2019, 1, 2, 3, 4, 5) dt_utc = TimestampField().local_to_utc(dt) ts = TSModel.create(ts_s=dt, ts_us=dt, ts_ms=dt, ts_u=dt_utc) fields = (TSModel.ts_s, TSModel.ts_us, TSModel.ts_ms, TSModel.ts_u) attrs = ('year', 'month', 'day', 'hour', 'minute', 'second') selection = [] for field in fields: for attr in attrs: selection.append(getattr(field, attr)) row = TSModel.select(*selection).tuples()[0] # First ensure that all 3 fields are returning the same data. ts_s, ts_us, ts_ms, ts_u = row[:6], row[6:12], row[12:18], row[18:] self.assertEqual(ts_s, ts_us) self.assertEqual(ts_s, ts_ms) self.assertEqual(ts_s, ts_u) # Now validate that the data is correct. We will receive the data back # as a UTC unix timestamp, however! y, m, d, H, M, S = ts_s self.assertEqual(y, 2019) self.assertEqual(m, 1) self.assertEqual(d, dt_utc.day) self.assertEqual(H, dt_utc.hour) self.assertEqual(M, 4) self.assertEqual(S, 5) def test_timestamp_field_from_ts(self): dt = datetime.datetime(2019, 1, 2, 3, 4, 5) dt_utc = TimestampField().local_to_utc(dt) ts = TSModel.create(ts_s=dt, ts_us=dt, ts_ms=dt, ts_u=dt_utc) query = TSModel.select( TSModel.ts_s.from_timestamp().alias('dt_s'), TSModel.ts_us.from_timestamp().alias('dt_us'), TSModel.ts_ms.from_timestamp().alias('dt_ms'), TSModel.ts_u.from_timestamp().alias('dt_u')) # Get row and unpack into variables corresponding to the fields. row = query.tuples()[0] dt_s, dt_us, dt_ms, dt_u = row # Ensure the timestamp values for all 4 fields are the same. self.assertEqual(dt_s, dt_us) self.assertEqual(dt_s, dt_ms) self.assertEqual(dt_s, dt_u) if IS_SQLITE: expected = dt_utc.strftime('%Y-%m-%d %H:%M:%S') self.assertEqual(dt_s, expected) elif IS_POSTGRESQL or IS_CRDB: # Postgres returns an aware UTC datetime. Strip this to compare # against our naive UTC datetime. self.assertEqual(dt_s.replace(tzinfo=None), dt_utc) def test_invalid_resolution(self): self.assertRaises(ValueError, TimestampField, resolution=7) self.assertRaises(ValueError, TimestampField, resolution=20) self.assertRaises(ValueError, TimestampField, resolution=10**7) class ListField(TextField): def db_value(self, value): return ','.join(value) if value else '' def python_value(self, value): return value.split(',') if value else [] class Todo(TestModel): content = TextField() tags = ListField() class TestCustomField(ModelTestCase): requires = [Todo] def test_custom_field(self): t1 = Todo.create(content='t1', tags=['t1-a', 't1-b']) t2 = Todo.create(content='t2', tags=[]) t1_db = Todo.get(Todo.id == t1.id) self.assertEqual(t1_db.tags, ['t1-a', 't1-b']) t2_db = Todo.get(Todo.id == t2.id) self.assertEqual(t2_db.tags, []) t1_db = Todo.get(Todo.tags == AsIs(['t1-a', 't1-b'])) self.assertEqual(t1_db.id, t1.id) t2_db = Todo.get(Todo.tags == AsIs([])) self.assertEqual(t2_db.id, t2.id) class UpperField(TextField): def db_value(self, value): return fn.UPPER(value) class UpperModel(TestModel): name = UpperField() class TestSQLFunctionDBValue(ModelTestCase): database = get_in_memory_db() requires = [UpperModel] def test_sql_function_db_value(self): # Verify that the db function is applied as part of an INSERT. um = UpperModel.create(name='huey') um_db = UpperModel.get(UpperModel.id == um.id) self.assertEqual(um_db.name, 'HUEY') # Verify that the db function is applied as part of an UPDATE. um_db.name = 'zaizee' um_db.save() # Ensure that the name was updated correctly. um_db2 = UpperModel.get(UpperModel.id == um.id) self.assertEqual(um_db2.name, 'ZAIZEE') # Verify that the db function is applied in a WHERE expression. um_db3 = UpperModel.get(UpperModel.name == 'zaiZee') self.assertEqual(um_db3.id, um.id) # If we nest the field in a function, the conversion is not applied. expr = fn.SUBSTR(UpperModel.name, 1, 1) == 'z' self.assertRaises(UpperModel.DoesNotExist, UpperModel.get, expr) class Schedule(TestModel): interval = IntegerField() class Task(TestModel): schedule = ForeignKeyField(Schedule) name = TextField() last_run = DateTimeField() class TestDateTimeMath(ModelTestCase): offset_to_names = ( (-10, ()), (5, ('s1',)), (10, ('s1', 's10')), (11, ('s1', 's10')), (60, ('s1', 's10', 's60')), (61, ('s1', 's10', 's60'))) requires = [Schedule, Task] def setUp(self): super(TestDateTimeMath, self).setUp() with self.database.atomic(): s1 = Schedule.create(interval=1) s10 = Schedule.create(interval=10) s60 = Schedule.create(interval=60) self.dt = datetime.datetime(2019, 1, 1, 12) for s, n in ((s1, 's1'), (s10, 's10'), (s60, 's60')): Task.create(schedule=s, name=n, last_run=self.dt) def _do_test_date_time_math(self, next_occurrence_expression): for offset, names in self.offset_to_names: dt = Value(self.dt + datetime.timedelta(seconds=offset)) query = (Task .select(Task, Schedule) .join(Schedule) .where(dt >= next_occurrence_expression) .order_by(Schedule.interval)) tnames = [task.name for task in query] self.assertEqual(list(names), tnames) @requires_pglike def test_date_time_math_pg(self): second = SQL("INTERVAL '1 second'") next_occurrence = Task.last_run + (Schedule.interval * second) self._do_test_date_time_math(next_occurrence) @requires_sqlite def test_date_time_math_sqlite(self): # Convert to a timestamp, add the scheduled seconds, then convert back # to a datetime string for comparison with the last occurrence. next_ts = Task.last_run.to_timestamp() + Schedule.interval next_occurrence = fn.datetime(next_ts, 'unixepoch') self._do_test_date_time_math(next_occurrence) @requires_mysql def test_date_time_math_mysql(self): nl = NodeList((SQL('INTERVAL'), Schedule.interval, SQL('SECOND'))) next_occurrence = fn.date_add(Task.last_run, nl) self._do_test_date_time_math(next_occurrence) class NQ(TestModel): name = TextField() class NQItem(TestModel): nq = ForeignKeyField(NQ, backref='items') nq_null = ForeignKeyField(NQ, backref='null_items', null=True) nq_lazy = ForeignKeyField(NQ, lazy_load=False, backref='lazy_items') nq_lazy_null = ForeignKeyField(NQ, lazy_load=False, backref='lazy_null_items', null=True) class TestForeignKeyLazyLoad(ModelTestCase): requires = [NQ, NQItem] def setUp(self): super(TestForeignKeyLazyLoad, self).setUp() with self.database.atomic(): a1, a2, a3, a4 = [NQ.create(name='a%s' % i) for i in range(1, 5)] ai = NQItem.create(nq=a1, nq_null=a2, nq_lazy=a3, nq_lazy_null=a4) b = NQ.create(name='b') bi = NQItem.create(nq=b, nq_lazy=b) def test_doesnotexist_lazy_load(self): n = NQ.create(name='n1') i = NQItem.create(nq=n, nq_null=n, nq_lazy=n, nq_lazy_null=n) i_db = NQItem.select(NQItem.id).where(NQItem.nq == n).get() with self.assertQueryCount(0): # Only raise DoesNotExist for non-nullable *and* lazy-load=True. # Otherwise we just return None. self.assertRaises(NQ.DoesNotExist, lambda: i_db.nq) self.assertTrue(i_db.nq_null is None) self.assertTrue(i_db.nq_lazy is None) self.assertTrue(i_db.nq_lazy_null is None) def test_foreign_key_lazy_load(self): a1, a2, a3, a4 = (NQ.select() .where(NQ.name.startswith('a')) .order_by(NQ.name)) b = NQ.get(NQ.name == 'b') ai = NQItem.get(NQItem.nq_id == a1.id) bi = NQItem.get(NQItem.nq_id == b.id) # Accessing the lazy foreign-key fields will not result in any queries # being executed. with self.assertQueryCount(0): self.assertEqual(ai.nq_lazy, a3.id) self.assertEqual(ai.nq_lazy_null, a4.id) self.assertEqual(bi.nq_lazy, b.id) self.assertTrue(bi.nq_lazy_null is None) self.assertTrue(bi.nq_null is None) # Accessing the regular foreign-key fields uses a query to get the # related model instance. with self.assertQueryCount(2): self.assertEqual(ai.nq.id, a1.id) self.assertEqual(ai.nq_null.id, a2.id) with self.assertQueryCount(1): self.assertEqual(bi.nq.id, b.id) def test_fk_lazy_load_related_instance(self): nq = NQ(name='b1') nqi = NQItem(nq=nq, nq_null=nq, nq_lazy=nq, nq_lazy_null=nq) nq.save() nqi.save() with self.assertQueryCount(1): nqi_db = NQItem.get(NQItem.id == nqi.id) self.assertEqual(nqi_db.nq_lazy, nq.id) self.assertEqual(nqi_db.nq_lazy_null, nq.id) def test_fk_lazy_select_related(self): NA, NB, NC, ND = [NQ.alias(a) for a in ('na', 'nb', 'nc', 'nd')] LO = JOIN.LEFT_OUTER query = (NQItem.select(NQItem, NA, NB, NC, ND) .join_from(NQItem, NA, LO, on=NQItem.nq) .join_from(NQItem, NB, LO, on=NQItem.nq_null) .join_from(NQItem, NC, LO, on=NQItem.nq_lazy) .join_from(NQItem, ND, LO, on=NQItem.nq_lazy_null) .order_by(NQItem.id)) # If we explicitly / eagerly select lazy foreign-key models, they # behave just like regular foreign keys. with self.assertQueryCount(1): ai, bi = [ni for ni in query] self.assertEqual(ai.nq.name, 'a1') self.assertEqual(ai.nq_null.name, 'a2') self.assertEqual(ai.nq_lazy.name, 'a3') self.assertEqual(ai.nq_lazy_null.name, 'a4') self.assertEqual(bi.nq.name, 'b') self.assertEqual(bi.nq_lazy.name, 'b') self.assertTrue(bi.nq_null is None) self.assertTrue(bi.nq_lazy_null is None) class SM(TestModel): text_field = TextField() char_field = CharField() class TestStringFields(ModelTestCase): requires = [SM] def test_string_fields(self): bdata = b'b1' udata = b'u1'.decode('utf8') sb = SM.create(text_field=bdata, char_field=bdata) su = SM.create(text_field=udata, char_field=udata) sb_db = SM.get(SM.id == sb.id) self.assertEqual(sb_db.text_field, 'b1') self.assertEqual(sb_db.char_field, 'b1') su_db = SM.get(SM.id == su.id) self.assertEqual(su_db.text_field, 'u1') self.assertEqual(su_db.char_field, 'u1') bvals = (b'b1', u'b1') uvals = (b'u1', u'u1') for field in (SM.text_field, SM.char_field): for bval in bvals: sb_db = SM.get(field == bval) self.assertEqual(sb.id, sb_db.id) for uval in uvals: sb_db = SM.get(field == uval) self.assertEqual(su.id, su_db.id) class InvalidTypes(TestModel): tfield = TextField() ifield = IntegerField() ffield = FloatField() class TestSqliteInvalidDataTypes(ModelTestCase): database = get_in_memory_db() requires = [InvalidTypes] def test_invalid_data_types(self): it = InvalidTypes.create(tfield=100, ifield='five', ffield='pi') it_db1 = InvalidTypes.get(InvalidTypes.tfield == 100) it_db2 = InvalidTypes.get(InvalidTypes.ifield == 'five') it_db3 = InvalidTypes.get(InvalidTypes.ffield == 'pi') self.assertTrue(it.id == it_db1.id == it_db2.id == it_db3.id) self.assertEqual(it_db1.tfield, '100') self.assertEqual(it_db1.ifield, 'five') self.assertEqual(it_db1.ffield, 'pi') class DblSI(TestModel): df = DoubleField() si = SmallIntegerField() class TestDoubleSmallInt(ModelTestCase): database = get_in_memory_db() requires = [DblSI] def test_double_round_trip(self): DblSI.create(df=3.141592653589793, si=0) obj = DblSI.get() self.assertAlmostEqual(obj.df, 3.141592653589793, places=10) def test_small_int_round_trip(self): DblSI.create(df=0, si=32000) DblSI.create(df=0, si=-100) results = (DblSI .select(DblSI.si) .order_by(DblSI.si) .tuples()) self.assertEqual(list(results), [(-100,), (32000,)]) def test_coercion(self): DblSI.create(df=float('inf'), si='42') obj = DblSI.get() self.assertEqual(obj.df, float('inf')) self.assertEqual(obj.si, 42) obj = DblSI.create(df=float('-inf'), si='1.23') obj = DblSI.get(DblSI.id == obj.id) self.assertEqual(obj.df, float('-inf')) self.assertEqual(obj.si, 1) class FC(TestModel): code = FixedCharField(max_length=5) name = CharField() class TestFixedCharFieldIntegration(ModelTestCase): database = get_in_memory_db() requires = [FC] def test_fixed_char_truncates(self): FC.create(code='ABCDEF', name='short') fc = FC.get(FC.code == 'ABCDE') self.assertEqual(fc.code, 'ABCDE') class VF(TestModel): name = TextField() computed = VirtualField(field_class=IntegerField) class TestVirtualFieldBehavior(BaseTestCase): def test_virtual_field_not_in_columns(self): """VirtualField should not appear in the model's SELECT columns.""" fields = VF._meta.sorted_fields field_names = [f.name for f in fields] self.assertIn('name', field_names) # VirtualField should not be in sorted_fields (it's a MetaField). self.assertNotIn('computed', field_names) query = VF.select() self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."name" FROM "vf" AS "t1"')) def test_virtual_field_db_value(self): vf = VF.computed self.assertEqual(vf.db_value('42'), 42) self.assertEqual(vf.python_value('42'), 42) ================================================ FILE: tests/hybrid.py ================================================ from peewee import * from playhouse.hybrid import * from .base import ModelTestCase from .base import TestModel from .base import get_in_memory_db from .base import requires_models class Interval(TestModel): start = IntegerField() end = IntegerField() @hybrid_property def length(self): return self.end - self.start @hybrid_method def contains(self, point): return (self.start <= point) & (point < self.end) @hybrid_property def radius(self): return int(abs(self.length) / 2) @radius.expression def radius(cls): return fn.ABS(cls.length) / 2 class Person(TestModel): first = TextField() last = TextField() @hybrid_property def full_name(self): return self.first + ' ' + self.last class SubPerson(Person): pass class TestHybridProperties(ModelTestCase): database = get_in_memory_db() requires = [Interval, Person] def setUp(self): super(TestHybridProperties, self).setUp() intervals = ( (1, 5), (2, 6), (3, 5), (2, 5)) for start, end in intervals: Interval.create(start=start, end=end) def test_hybrid_property(self): query = Interval.select().where(Interval.length == 4) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."start", "t1"."end" ' 'FROM "interval" AS "t1" ' 'WHERE (("t1"."end" - "t1"."start") = ?)'), [4]) results = sorted((i.start, i.end) for i in query) self.assertEqual(results, [(1, 5), (2, 6)]) query = Interval.select().order_by(Interval.id) self.assertEqual([i.length for i in query], [4, 4, 2, 3]) def test_hybrid_method(self): query = Interval.select().where(Interval.contains(2)) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."start", "t1"."end" ' 'FROM "interval" AS "t1" ' 'WHERE (("t1"."start" <= ?) AND ("t1"."end" > ?))'), [2, 2]) results = sorted((i.start, i.end) for i in query) self.assertEqual(results, [(1, 5), (2, 5), (2, 6)]) query = Interval.select().order_by(Interval.id) self.assertEqual([i.contains(2) for i in query], [1, 1, 0, 1]) def test_expression(self): query = Interval.select().where(Interval.radius == 2) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."start", "t1"."end" ' 'FROM "interval" AS "t1" ' 'WHERE ((ABS("t1"."end" - "t1"."start") / ?) = ?)'), [2, 2]) self.assertEqual(sorted((i.start, i.end) for i in query), [(1, 5), (2, 6)]) query = Interval.select().order_by(Interval.id) self.assertEqual([i.radius for i in query], [2, 2, 1, 1]) def test_string_fields(self): huey = Person.create(first='huey', last='cat') zaizee = Person.create(first='zaizee', last='kitten') self.assertEqual(huey.full_name, 'huey cat') self.assertEqual(zaizee.full_name, 'zaizee kitten') query = Person.select().where(Person.full_name.startswith('huey c')) huey_db = query.get() self.assertEqual(huey_db.id, huey.id) def test_hybrid_model_alias(self): Person.create(first='huey', last='cat') PA = Person.alias() query = PA.select(PA.full_name).where(PA.last == 'cat') self.assertSQL(query, ( 'SELECT (("t1"."first" || ?) || "t1"."last") ' 'FROM "person" AS "t1" WHERE ("t1"."last" = ?)'), [' ', 'cat']) self.assertEqual(query.tuples()[0], ('huey cat',)) @requires_models(SubPerson) def test_hybrid_subclass_model_alias(self): SubPerson.create(first='huey', last='cat') SA = SubPerson.alias() query = SA.select(SA.full_name).where(SA.last == 'cat') self.assertSQL(query, ( 'SELECT (("t1"."first" || ?) || "t1"."last") ' 'FROM "sub_person" AS "t1" WHERE ("t1"."last" = ?)'), [' ', 'cat']) self.assertEqual(query.tuples()[0], ('huey cat',)) class Order(TestModel): name = TextField() @hybrid_property def quantity(self): return sum([item.qt for item in self.items]) @quantity.expression def quantity(cls): return fn.SUM(Item.qt).alias('quantity') class Item(TestModel): order = ForeignKeyField(Order, backref='items') qt = IntegerField() class TestHybridWithRelationship(ModelTestCase): database = get_in_memory_db() requires = [Order, Item] def test_hybrid_with_relationship(self): data = ( ('a', (4, 3, 2, 1)), ('b', (1000, 300, 30, 7)), ('c', ())) for name, qts in data: o = Order.create(name=name) for qt in qts: Item.create(order=o, qt=qt) query = Order.select().order_by(Order.name) self.assertEqual([o.quantity for o in query], [10, 1337, 0]) query = (Order .select(Order.name, Order.quantity.alias('sql_qt')) .join(Item, JOIN.LEFT_OUTER) .group_by(Order.name) .order_by(Order.name)) self.assertEqual([o.sql_qt for o in query], [10, 1337, None]) ================================================ FILE: tests/keys.py ================================================ from peewee import * from .base import IS_MYSQL from .base import IS_SQLITE from .base import ModelTestCase from .base import TestModel from .base import db from .base import get_in_memory_db from .base import requires_sqlite class Package(TestModel): barcode = CharField(unique=True) class PackageItem(TestModel): title = CharField() package = ForeignKeyField(Package, Package.barcode, backref='items') class Manufacturer(TestModel): name = CharField() class Component(TestModel): name = CharField() manufacturer = ForeignKeyField(Manufacturer, null=True) class Computer(TestModel): hard_drive = ForeignKeyField(Component, backref='c1') memory = ForeignKeyField(Component, backref='c2') processor = ForeignKeyField(Component, backref='c3') class User(TestModel): username = CharField() class Meta: table_name = 'users' class Relationship(TestModel): from_user = ForeignKeyField(User, backref='relationships') to_user = ForeignKeyField(User, backref='related_to') class Note(TestModel): user = ForeignKeyField(User, backref='notes') content = TextField() class CompositeKeyModel(TestModel): f1 = CharField() f2 = IntegerField() f3 = FloatField() class Meta: primary_key = CompositeKey('f1', 'f2') class UserThing(TestModel): thing = CharField() user = ForeignKeyField(User, backref='things') class Meta: primary_key = CompositeKey('thing', 'user') class Post(TestModel): title = CharField() class Tag(TestModel): tag = CharField() class TagPostThrough(TestModel): tag = ForeignKeyField(Tag, backref='posts') post = ForeignKeyField(Post, backref='tags') class Meta: primary_key = CompositeKey('tag', 'post') class TagPostThroughAlt(TestModel): tag = ForeignKeyField(Tag, backref='posts_alt') post = ForeignKeyField(Post, backref='tags_alt') class TestForeignKeyToNonPrimaryKey(ModelTestCase): requires = [Package, PackageItem] def setUp(self): super(TestForeignKeyToNonPrimaryKey, self).setUp() for barcode in ['101', '102']: Package.create(barcode=barcode) for i in range(2): PackageItem.create( package=barcode, title='%s-%s' % (barcode, i)) def test_fk_resolution(self): pi = PackageItem.get(PackageItem.title == '101-0') self.assertEqual(pi.__data__['package'], '101') self.assertEqual(pi.package, Package.get(Package.barcode == '101')) def test_select_generation(self): p = Package.get(Package.barcode == '101') self.assertEqual( [item.title for item in p.items.order_by(PackageItem.title)], ['101-0', '101-1']) class TestMultipleForeignKey(ModelTestCase): requires = [Manufacturer, Component, Computer] test_values = [ ['3TB', '16GB', 'i7'], ['128GB', '1GB', 'ARM'], ] def setUp(self): super(TestMultipleForeignKey, self).setUp() intel = Manufacturer.create(name='Intel') amd = Manufacturer.create(name='AMD') kingston = Manufacturer.create(name='Kingston') for hard_drive, memory, processor in self.test_values: c = Computer.create( hard_drive=Component.create(name=hard_drive), memory=Component.create(name=memory, manufacturer=kingston), processor=Component.create(name=processor, manufacturer=intel)) # The 2nd computer has an AMD processor. c.processor.manufacturer = amd c.processor.save() def test_multi_join(self): HDD = Component.alias('hdd') HDDMf = Manufacturer.alias('hddm') Memory = Component.alias('mem') MemoryMf = Manufacturer.alias('memm') Processor = Component.alias('proc') ProcessorMf = Manufacturer.alias('procm') query = (Computer .select( Computer, HDD, Memory, Processor, HDDMf, MemoryMf, ProcessorMf) .join(HDD, on=( Computer.hard_drive_id == HDD.id).alias('hard_drive')) .join( HDDMf, JOIN.LEFT_OUTER, on=(HDD.manufacturer_id == HDDMf.id)) .switch(Computer) .join(Memory, on=( Computer.memory_id == Memory.id).alias('memory')) .join( MemoryMf, JOIN.LEFT_OUTER, on=(Memory.manufacturer_id == MemoryMf.id)) .switch(Computer) .join(Processor, on=( Computer.processor_id == Processor.id).alias('processor')) .join( ProcessorMf, JOIN.LEFT_OUTER, on=(Processor.manufacturer_id == ProcessorMf.id)) .order_by(Computer.id)) with self.assertQueryCount(1): vals = [] manufacturers = [] for computer in query: components = [ computer.hard_drive, computer.memory, computer.processor] vals.append([component.name for component in components]) for component in components: if component.manufacturer: manufacturers.append(component.manufacturer.name) else: manufacturers.append(None) self.assertEqual(vals, self.test_values) self.assertEqual(manufacturers, [ None, 'Kingston', 'Intel', None, 'Kingston', 'AMD', ]) class TestMultipleForeignKeysJoining(ModelTestCase): requires = [User, Relationship] def test_multiple_fks(self): a = User.create(username='a') b = User.create(username='b') c = User.create(username='c') self.assertEqual(list(a.relationships), []) self.assertEqual(list(a.related_to), []) r_ab = Relationship.create(from_user=a, to_user=b) self.assertEqual(list(a.relationships), [r_ab]) self.assertEqual(list(a.related_to), []) self.assertEqual(list(b.relationships), []) self.assertEqual(list(b.related_to), [r_ab]) r_bc = Relationship.create(from_user=b, to_user=c) following = User.select().join( Relationship, on=Relationship.to_user ).where(Relationship.from_user == a) self.assertEqual(list(following), [b]) followers = User.select().join( Relationship, on=Relationship.from_user ).where(Relationship.to_user == a.id) self.assertEqual(list(followers), []) following = User.select().join( Relationship, on=Relationship.to_user ).where(Relationship.from_user == b.id) self.assertEqual(list(following), [c]) followers = User.select().join( Relationship, on=Relationship.from_user ).where(Relationship.to_user == b.id) self.assertEqual(list(followers), [a]) following = User.select().join( Relationship, on=Relationship.to_user ).where(Relationship.from_user == c.id) self.assertEqual(list(following), []) followers = User.select().join( Relationship, on=Relationship.from_user ).where(Relationship.to_user == c.id) self.assertEqual(list(followers), [b]) class TestCompositePrimaryKey(ModelTestCase): requires = [Tag, Post, TagPostThrough, CompositeKeyModel, User, UserThing] def setUp(self): super(TestCompositePrimaryKey, self).setUp() tags = [Tag.create(tag='t%d' % i) for i in range(1, 4)] posts = [Post.create(title='p%d' % i) for i in range(1, 4)] p12 = Post.create(title='p12') for t, p in zip(tags, posts): TagPostThrough.create(tag=t, post=p) TagPostThrough.create(tag=tags[0], post=p12) TagPostThrough.create(tag=tags[1], post=p12) def test_create_table_query(self): query, params = TagPostThrough._schema._create_table().query() sql = ('CREATE TABLE IF NOT EXISTS "tag_post_through" (' '"tag_id" INTEGER NOT NULL, ' '"post_id" INTEGER NOT NULL, ' 'PRIMARY KEY ("tag_id", "post_id"), ' 'FOREIGN KEY ("tag_id") REFERENCES "tag" ("id"), ' 'FOREIGN KEY ("post_id") REFERENCES "post" ("id"))') if IS_MYSQL: sql = sql.replace('"', '`') self.assertEqual(query, sql) def test_get_set_id(self): tpt = (TagPostThrough .select() .join(Tag) .switch(TagPostThrough) .join(Post) .order_by(Tag.tag, Post.title)).get() # Sanity check. self.assertEqual(tpt.tag.tag, 't1') self.assertEqual(tpt.post.title, 'p1') tag = Tag.select().where(Tag.tag == 't1').get() post = Post.select().where(Post.title == 'p1').get() self.assertEqual(tpt._pk, (tag.id, post.id)) # set_id is a no-op. with self.assertRaisesCtx(TypeError): tpt._pk = None self.assertEqual(tpt._pk, (tag.id, post.id)) t3 = Tag.get(Tag.tag == 't3') p3 = Post.get(Post.title == 'p3') tpt._pk = (t3, p3) self.assertEqual(tpt.tag.tag, 't3') self.assertEqual(tpt.post.title, 'p3') def test_querying(self): posts = (Post.select() .join(TagPostThrough) .join(Tag) .where(Tag.tag == 't1') .order_by(Post.title)) self.assertEqual([p.title for p in posts], ['p1', 'p12']) tags = (Tag.select() .join(TagPostThrough) .join(Post) .where(Post.title == 'p12') .order_by(Tag.tag)) self.assertEqual([t.tag for t in tags], ['t1', 't2']) def test_composite_key_model(self): CKM = CompositeKeyModel values = [ ('a', 1, 1.0), ('a', 2, 2.0), ('b', 1, 1.0), ('b', 2, 2.0)] c1, c2, c3, c4 = [ CKM.create(f1=f1, f2=f2, f3=f3) for f1, f2, f3 in values] # Update a single row, giving it a new value for `f3`. CKM.update(f3=3.0).where((CKM.f1 == 'a') & (CKM.f2 == 2)).execute() c = CKM.get((CKM.f1 == 'a') & (CKM.f2 == 2)) self.assertEqual(c.f3, 3.0) # Update the `f3` value and call `save()`, triggering an update. c3.f3 = 4.0 c3.save() c = CKM.get((CKM.f1 == 'b') & (CKM.f2 == 1)) self.assertEqual(c.f3, 4.0) # Only 1 row updated. query = CKM.select().where(CKM.f3 == 4.0) self.assertEqual(query.count(), 1) # Unfortunately this does not work since the original value of the # PK is lost (and hence cannot be used to update). c4.f1 = 'c' c4.save() self.assertRaises( CKM.DoesNotExist, lambda: CKM.get((CKM.f1 == 'c') & (CKM.f2 == 2))) def test_count_composite_key(self): CKM = CompositeKeyModel values = [ ('a', 1, 1.0), ('a', 2, 2.0), ('b', 1, 1.0), ('b', 2, 1.0)] for f1, f2, f3 in values: CKM.create(f1=f1, f2=f2, f3=f3) self.assertEqual(CKM.select().count(), 4) self.assertTrue(CKM.select().where( (CKM.f1 == 'a') & (CKM.f2 == 1)).exists()) self.assertFalse(CKM.select().where( (CKM.f1 == 'a') & (CKM.f2 == 3)).exists()) def test_delete_instance(self): u1, u2 = [User.create(username='u%s' % i) for i in range(2)] ut1 = UserThing.create(thing='t1', user=u1) ut2 = UserThing.create(thing='t2', user=u1) ut3 = UserThing.create(thing='t1', user=u2) ut4 = UserThing.create(thing='t3', user=u2) res = ut1.delete_instance() self.assertEqual(res, 1) self.assertEqual( [x.thing for x in UserThing.select().order_by(UserThing.thing)], ['t1', 't2', 't3']) def test_composite_key_inheritance(self): class Person(TestModel): first = TextField() last = TextField() class Meta: primary_key = CompositeKey('first', 'last') self.assertTrue(isinstance(Person._meta.primary_key, CompositeKey)) self.assertEqual(Person._meta.primary_key.field_names, ('first', 'last')) class Employee(Person): title = TextField() self.assertTrue(isinstance(Employee._meta.primary_key, CompositeKey)) self.assertEqual(Employee._meta.primary_key.field_names, ('first', 'last')) sql = ('CREATE TABLE IF NOT EXISTS "employee" (' '"first" TEXT NOT NULL, "last" TEXT NOT NULL, ' '"title" TEXT NOT NULL, PRIMARY KEY ("first", "last"))') if IS_MYSQL: sql = sql.replace('"', '`') self.assertEqual(Employee._schema._create_table().query(), (sql, [])) class TestForeignKeyConstraints(ModelTestCase): requires = [User, Note] def setUp(self): super(TestForeignKeyConstraints, self).setUp() self.set_foreign_key_pragma(True) def tearDown(self): self.set_foreign_key_pragma(False) super(TestForeignKeyConstraints, self).tearDown() def set_foreign_key_pragma(self, is_enabled): if IS_SQLITE: self.database.foreign_keys = 'on' if is_enabled else 'off' def test_constraint_exists(self): max_id = User.select(fn.MAX(User.id)).scalar() or 0 with self.assertRaisesCtx(IntegrityError): with self.database.atomic(): Note.create(user=max_id + 1, content='test') @requires_sqlite def test_disable_constraint(self): self.set_foreign_key_pragma(False) Note.create(user=0, content='test') class FK_A(TestModel): key = CharField(max_length=16, unique=True) class FK_B(TestModel): fk_a = ForeignKeyField(FK_A, field='key') class TestFKtoNonPKField(ModelTestCase): requires = [FK_A, FK_B] def test_fk_to_non_pk_field(self): a1 = FK_A.create(key='a1') a2 = FK_A.create(key='a2') b1 = FK_B.create(fk_a=a1) b2 = FK_B.create(fk_a=a2) args = (b1.fk_a, b1.fk_a_id, a1, a1.key) for arg in args: query = FK_B.select().where(FK_B.fk_a == arg) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."fk_a_id" FROM "fk_b" AS "t1" ' 'WHERE ("t1"."fk_a_id" = ?)'), ['a1']) b1_db = query.get() self.assertEqual(b1_db.id, b1.id) def test_fk_to_non_pk_insert_update(self): a1 = FK_A.create(key='a1') b1 = FK_B.create(fk_a=a1) self.assertEqual(FK_B.select().where(FK_B.fk_a == a1).count(), 1) exprs = ( {FK_B.fk_a: a1}, {'fk_a': a1}, {FK_B.fk_a: a1.key}, {'fk_a': a1.key}) for n, expr in enumerate(exprs, 2): self.assertTrue(FK_B.insert(expr).execute()) self.assertEqual(FK_B.select().where(FK_B.fk_a == a1).count(), n) a2 = FK_A.create(key='a2') exprs = ( {FK_B.fk_a: a2}, {'fk_a': a2}, {FK_B.fk_a: a2.key}, {'fk_a': a2.key}) b_list = list(FK_B.select().where(FK_B.fk_a == a1)) for i, (b, expr) in enumerate(zip(b_list[1:], exprs), 1): self.assertTrue(FK_B.update(expr).where(FK_B.id == b.id).execute()) self.assertEqual(FK_B.select().where(FK_B.fk_a == a2).count(), i) class TestDeferredForeignKeyIntegration(ModelTestCase): database = get_in_memory_db() def test_deferred_fk_simple(self): class Base(TestModel): class Meta: database = self.database class DFFk(Base): fk = DeferredForeignKey('DFPk') # Deferred key not bound yet. self.assertTrue(isinstance(DFFk.fk, DeferredForeignKey)) class DFPk(Base): pass # Deferred key is bound correctly. self.assertTrue(isinstance(DFFk.fk, ForeignKeyField)) self.assertEqual(DFFk.fk.rel_model, DFPk) self.assertEqual(DFFk._meta.refs, {DFFk.fk: DFPk}) self.assertEqual(DFFk._meta.backrefs, {}) self.assertEqual(DFPk._meta.refs, {}) self.assertEqual(DFPk._meta.backrefs, {DFFk.fk: DFFk}) self.assertSQL(DFFk._schema._create_table(False), ( 'CREATE TABLE "df_fk" ("id" INTEGER NOT NULL PRIMARY KEY, ' '"fk_id" INTEGER NOT NULL)'), []) def test_deferred_fk_as_pk(self): class Base(TestModel): class Meta: database = self.database class DFFk(Base): fk = DeferredForeignKey('DFPk', primary_key=True) # Deferred key not bound yet. self.assertTrue(isinstance(DFFk.fk, DeferredForeignKey)) self.assertTrue(DFFk._meta.primary_key is DFFk.fk) class DFPk(Base): pass # Resolved and primary-key set correctly. self.assertTrue(isinstance(DFFk.fk, ForeignKeyField)) self.assertTrue(DFFk._meta.primary_key is DFFk.fk) self.assertEqual(DFFk.fk.rel_model, DFPk) self.assertEqual(DFFk._meta.refs, {DFFk.fk: DFPk}) self.assertEqual(DFFk._meta.backrefs, {}) self.assertEqual(DFPk._meta.refs, {}) self.assertEqual(DFPk._meta.backrefs, {DFFk.fk: DFFk}) self.assertSQL(DFFk._schema._create_table(False), ( 'CREATE TABLE "df_fk" ("fk_id" INTEGER NOT NULL PRIMARY KEY)'), []) ================================================ FILE: tests/kv.py ================================================ from peewee import IntegerField from playhouse.kv import KeyValue from .base import DatabaseTestCase from .base import db class TestKeyValue(DatabaseTestCase): def setUp(self): super(TestKeyValue, self).setUp() self._kvs = [] def tearDown(self): if self._kvs: self.database.drop_tables([kv.model for kv in self._kvs]) super(TestKeyValue, self).tearDown() def create_kv(self, **kwargs): kv = KeyValue(database=self.database, **kwargs) self._kvs.append(kv) return kv def test_basic_apis(self): KV = self.create_kv() KV['k1'] = 'v1' KV['k2'] = [0, 1, 2] self.assertEqual(KV['k1'], 'v1') self.assertEqual(KV['k2'], [0, 1, 2]) self.assertRaises(KeyError, lambda: KV['k3']) self.assertTrue((KV.key < 'k2') in KV) self.assertFalse((KV.key > 'k2') in KV) del KV['k1'] KV['k3'] = 'v3' self.assertFalse('k1' in KV) self.assertTrue('k3' in KV) self.assertEqual(sorted(KV.keys()), ['k2', 'k3']) self.assertEqual(len(KV), 2) data = dict(KV) self.assertEqual(data, { 'k2': [0, 1, 2], 'k3': 'v3'}) self.assertEqual(dict(KV), dict(KV.items())) self.assertEqual(KV.pop('k2'), [0, 1, 2]) self.assertRaises(KeyError, lambda: KV['k2']) self.assertRaises(KeyError, KV.pop, 'k2') self.assertEqual(KV.get('k3'), 'v3') self.assertTrue(KV.get('kx') is None) self.assertEqual(KV.get('kx', 'vx'), 'vx') self.assertTrue(KV.get('k4') is None) self.assertEqual(KV.setdefault('k4', 'v4'), 'v4') self.assertEqual(KV.get('k4'), 'v4') self.assertEqual(KV.get('k4', 'v5'), 'v4') KV.clear() self.assertEqual(len(KV), 0) def test_update(self): KV = self.create_kv() with self.assertQueryCount(1): KV.update(k1='v1', k2='v2', k3='v3') self.assertEqual(len(KV), 3) with self.assertQueryCount(1): KV.update(k1='v1-x', k3='v3-x', k4='v4') self.assertEqual(len(KV), 4) self.assertEqual(dict(KV), { 'k1': 'v1-x', 'k2': 'v2', 'k3': 'v3-x', 'k4': 'v4'}) KV['k1'] = 'v1-y' self.assertEqual(len(KV), 4) self.assertEqual(dict(KV), { 'k1': 'v1-y', 'k2': 'v2', 'k3': 'v3-x', 'k4': 'v4'}) def test_expressions(self): KV = self.create_kv(value_field=IntegerField(), ordered=True) with self.database.atomic(): for i in range(1, 11): KV['k%d' % i] = i self.assertEqual(KV[KV.key < 'k2'], [1, 10]) self.assertEqual(KV[KV.value > 7], [10, 8, 9]) self.assertEqual(KV[(KV.key > 'k2') & (KV.key < 'k6')], [3, 4, 5]) self.assertEqual(KV[KV.key == 'kx'], []) del KV[KV.key > 'k3'] self.assertEqual(dict(KV), { 'k1': 1, 'k2': 2, 'k3': 3, 'k10': 10}) KV[KV.value > 2] = 99 self.assertEqual(dict(KV), { 'k1': 1, 'k2': 2, 'k3': 99, 'k10': 99}) def test_integer_keys(self): KV = self.create_kv(key_field=IntegerField(primary_key=True), ordered=True) KV[1] = 'v1' KV[2] = 'v2' KV[10] = 'v10' self.assertEqual(list(KV), [(1, 'v1'), (2, 'v2'), (10, 'v10')]) self.assertEqual(list(KV.keys()), [1, 2, 10]) self.assertEqual(list(KV.values()), ['v1', 'v2', 'v10']) del KV[2] KV[1] = 'v1-x' KV[3] = 'v3' self.assertEqual(dict(KV), { 1: 'v1-x', 3: 'v3', 10: 'v10'}) ================================================ FILE: tests/manytomany.py ================================================ from peewee import * from .base import ModelTestCase from .base import TestModel from .base import get_in_memory_db from .base import requires_models from .base_models import Tweet from .base_models import User class User(TestModel): username = TextField(unique=True) class Note(TestModel): text = TextField() users = ManyToManyField(User) NoteUserThrough = Note.users.get_through_model() AltThroughDeferred = DeferredThroughModel() class AltNote(TestModel): text = TextField() users = ManyToManyField(User, through_model=AltThroughDeferred) class AltThroughModel(TestModel): user = ForeignKeyField(User, backref='_xx_rel') note = ForeignKeyField(AltNote, backref='_xx_rel') class Meta: primary_key = CompositeKey('user', 'note') AltThroughDeferred.set_model(AltThroughModel) class Student(TestModel): name = TextField() CourseStudentDeferred = DeferredThroughModel() class Course(TestModel): name = TextField() students = ManyToManyField(Student, backref='+') students2 = ManyToManyField(Student, through_model=CourseStudentDeferred) CourseStudent = Course.students.get_through_model() class CourseStudent2(TestModel): course = ForeignKeyField(Course, backref='+') student = ForeignKeyField(Student, backref='+') CourseStudentDeferred.set_model(CourseStudent2) class Color(TestModel): name = TextField(unique=True) LogoColorDeferred = DeferredThroughModel() class Logo(TestModel): name = TextField(unique=True) colors = ManyToManyField(Color, through_model=LogoColorDeferred) class LogoColor(TestModel): logo = ForeignKeyField(Logo, field=Logo.name) color = ForeignKeyField(Color, field=Color.name) # FK to non-PK column. LogoColorDeferred.set_model(LogoColor) class TestManyToManyFKtoNonPK(ModelTestCase): database = get_in_memory_db() requires = [Color, Logo, LogoColor] def test_manytomany_fk_to_non_pk(self): red = Color.create(name='red') green = Color.create(name='green') blue = Color.create(name='blue') lrg = Logo.create(name='logo-rg') lrb = Logo.create(name='logo-rb') lrgb = Logo.create(name='logo-rgb') lrg.colors.add([red, green]) lrb.colors.add([red, blue]) lrgb.colors.add([red, green, blue]) def assertColors(logo, expected): colors = [c.name for c in logo.colors.order_by(Color.name)] self.assertEqual(colors, expected) assertColors(lrg, ['green', 'red']) assertColors(lrb, ['blue', 'red']) assertColors(lrgb, ['blue', 'green', 'red']) def assertLogos(color, expected): logos = [l.name for l in color.logos.order_by(Logo.name)] self.assertEqual(logos, expected) assertLogos(red, ['logo-rb', 'logo-rg', 'logo-rgb']) assertLogos(green, ['logo-rg', 'logo-rgb']) assertLogos(blue, ['logo-rb', 'logo-rgb']) # Verify we can delete data as well. lrg.colors.remove(red) self.assertEqual([c.name for c in lrg.colors], ['green']) blue.logos.remove(lrb) self.assertEqual([c.name for c in lrb.colors], ['red']) # Verify we can insert using a SELECT query. lrg.colors.add(Color.select().where(Color.name != 'blue'), True) assertColors(lrg, ['green', 'red']) lrb.colors.add(Color.select().where(Color.name == 'blue')) assertColors(lrb, ['blue', 'red']) # Verify we can insert logos using a SELECT query. black = Color.create(name='black') black.logos.add(Logo.select().where(Logo.name != 'logo-rgb')) assertLogos(black, ['logo-rb', 'logo-rg']) assertColors(lrb, ['black', 'blue', 'red']) assertColors(lrg, ['black', 'green', 'red']) assertColors(lrgb, ['blue', 'green', 'red']) # Verify we can delete using a SELECT query. lrg.colors.remove(Color.select().where(Color.name == 'red')) assertColors(lrg, ['black', 'green']) black.logos.remove(Logo.select().where(Logo.name == 'logo-rg')) assertLogos(black, ['logo-rb']) # Verify we can clear. lrg.colors.clear() assertColors(lrg, []) assertColors(lrb, ['black', 'blue', 'red']) # Not affected. black.logos.clear() assertLogos(black, []) assertLogos(red, ['logo-rb', 'logo-rgb']) class TestManyToManyBackrefBehavior(ModelTestCase): database = get_in_memory_db() requires = [Student, Course, CourseStudent, CourseStudent2] def setUp(self): super(TestManyToManyBackrefBehavior, self).setUp() math = Course.create(name='math') engl = Course.create(name='engl') huey, mickey, zaizee = [Student.create(name=name) for name in ('huey', 'mickey', 'zaizee')] # Set up relationships. math.students.add([huey, zaizee]) engl.students.add([mickey]) math.students2.add([mickey]) engl.students2.add([huey, zaizee]) def test_manytomanyfield_disabled_backref(self): math = Course.get(name='math') query = math.students.order_by(Student.name) self.assertEqual([s.name for s in query], ['huey', 'zaizee']) huey = Student.get(name='huey') math.students.remove(huey) self.assertEqual([s.name for s in math.students], ['zaizee']) # The backref is via the CourseStudent2 through-model. self.assertEqual([c.name for c in huey.courses], ['engl']) def test_through_model_disabled_backrefs(self): # Here we're testing the case where the many-to-many field does not # explicitly disable back-references, but the foreign-keys on the # through model have disabled back-references. engl = Course.get(name='engl') query = engl.students2.order_by(Student.name) self.assertEqual([s.name for s in query], ['huey', 'zaizee']) zaizee = Student.get(Student.name == 'zaizee') engl.students2.remove(zaizee) self.assertEqual([s.name for s in engl.students2], ['huey']) math = Course.get(name='math') self.assertEqual([s.name for s in math.students2], ['mickey']) class TestManyToManyInheritance(ModelTestCase): def test_manytomany_inheritance(self): class BaseModel(TestModel): class Meta: database = self.database class User(BaseModel): username = TextField() class Project(BaseModel): name = TextField() users = ManyToManyField(User, backref='projects') def subclass_project(): class VProject(Project): pass # We cannot subclass Project, because the many-to-many field "users" # will be inherited, but the through-model does not contain a # foreign-key to VProject. The through-model in this case is # ProjectUsers, which has foreign-keys to project and user. self.assertRaises(ValueError, subclass_project) PThrough = Project.users.through_model self.assertTrue(PThrough.project.rel_model is Project) self.assertTrue(PThrough.user.rel_model is User) class TestManyToMany(ModelTestCase): database = get_in_memory_db() requires = [User, Note, NoteUserThrough, AltNote, AltThroughModel] user_to_note = { 'gargie': [1, 2], 'huey': [2, 3], 'mickey': [3, 4], 'zaizee': [4, 5], } def setUp(self): super(TestManyToMany, self).setUp() for username in sorted(self.user_to_note): User.create(username=username) for i in range(5): Note.create(text='note-%s' % (i + 1)) def test_through_model(self): self.assertEqual(len(NoteUserThrough._meta.fields), 3) fields = NoteUserThrough._meta.fields self.assertEqual(sorted(fields), ['id', 'note', 'user']) note_field = fields['note'] self.assertEqual(note_field.rel_model, Note) self.assertFalse(note_field.null) user_field = fields['user'] self.assertEqual(user_field.rel_model, User) self.assertFalse(user_field.null) def _set_data(self): for username, notes in self.user_to_note.items(): user = User.get(User.username == username) for note in notes: NoteUserThrough.create( note=Note.get(Note.text == 'note-%s' % note), user=user) def assertNotes(self, query, expected): notes = [note.text for note in query] self.assertEqual(sorted(notes), ['note-%s' % i for i in sorted(expected)]) def assertUsers(self, query, expected): usernames = [user.username for user in query] self.assertEqual(sorted(usernames), sorted(expected)) def test_accessor_query(self): self._set_data() gargie, huey, mickey, zaizee = User.select().order_by(User.username) with self.assertQueryCount(1): self.assertNotes(gargie.notes, [1, 2]) with self.assertQueryCount(1): self.assertNotes(zaizee.notes, [4, 5]) with self.assertQueryCount(2): self.assertNotes(User.create(username='x').notes, []) n1, n2, n3, n4, n5 = Note.select().order_by(Note.text) with self.assertQueryCount(1): self.assertUsers(n1.users, ['gargie']) with self.assertQueryCount(1): self.assertUsers(n2.users, ['gargie', 'huey']) with self.assertQueryCount(1): self.assertUsers(n5.users, ['zaizee']) with self.assertQueryCount(2): self.assertUsers(Note.create(text='x').users, []) def test_prefetch_notes(self): self._set_data() for pt in PREFETCH_TYPE.values(): with self.assertQueryCount(3): gargie, huey, mickey, zaizee = prefetch( User.select().order_by(User.username), NoteUserThrough, Note, prefetch_type=pt) with self.assertQueryCount(0): self.assertNotes(gargie.notes, [1, 2]) with self.assertQueryCount(0): self.assertNotes(zaizee.notes, [4, 5]) with self.assertQueryCount(2): self.assertNotes(User.create(username='x').notes, []) def test_prefetch_users(self): self._set_data() for pt in PREFETCH_TYPE.values(): with self.assertQueryCount(3): n1, n2, n3, n4, n5 = prefetch( Note.select().order_by(Note.text), NoteUserThrough, User, prefetch_type=pt) with self.assertQueryCount(0): self.assertUsers(n1.users, ['gargie']) with self.assertQueryCount(0): self.assertUsers(n2.users, ['gargie', 'huey']) with self.assertQueryCount(0): self.assertUsers(n5.users, ['zaizee']) with self.assertQueryCount(2): self.assertUsers(Note.create(text='x').users, []) def test_query_filtering(self): self._set_data() gargie, huey, mickey, zaizee = User.select().order_by(User.username) with self.assertQueryCount(1): notes = gargie.notes.where(Note.text != 'note-2') self.assertNotes(notes, [1]) def test_set_value(self): self._set_data() gargie = User.get(User.username == 'gargie') huey = User.get(User.username == 'huey') n1, n2, n3, n4, n5 = Note.select().order_by(Note.text) with self.assertQueryCount(2): gargie.notes = n3 self.assertNotes(gargie.notes, [3]) self.assertUsers(n3.users, ['gargie', 'huey', 'mickey']) self.assertUsers(n1.users, []) gargie.notes = [n3, n4] self.assertNotes(gargie.notes, [3, 4]) self.assertUsers(n3.users, ['gargie', 'huey', 'mickey']) self.assertUsers(n4.users, ['gargie', 'mickey', 'zaizee']) def test_set_query(self): huey = User.get(User.username == 'huey') with self.assertQueryCount(2): huey.notes = Note.select().where(~Note.text.endswith('4')) self.assertNotes(huey.notes, [1, 2, 3, 5]) def test_add(self): gargie = User.get(User.username == 'gargie') huey = User.get(User.username == 'huey') n1, n2, n3, n4, n5 = Note.select().order_by(Note.text) gargie.notes.add([n1, n2]) self.assertNotes(gargie.notes, [1, 2]) self.assertUsers(n1.users, ['gargie']) self.assertUsers(n2.users, ['gargie']) for note in [n3, n4, n5]: self.assertUsers(note.users, []) with self.assertQueryCount(1): huey.notes.add(Note.select().where( fn.substr(Note.text, 6, 1) << ['1', '3', '5'])) self.assertNotes(huey.notes, [1, 3, 5]) self.assertUsers(n1.users, ['gargie', 'huey']) self.assertUsers(n2.users, ['gargie']) self.assertUsers(n3.users, ['huey']) self.assertUsers(n4.users, []) self.assertUsers(n5.users, ['huey']) with self.assertQueryCount(1): gargie.notes.add(n4) self.assertNotes(gargie.notes, [1, 2, 4]) with self.assertQueryCount(2): n3.users.add( User.select().where(User.username != 'gargie'), clear_existing=True) self.assertUsers(n3.users, ['huey', 'mickey', 'zaizee']) def test_add_by_pk(self): huey = User.get(User.username == 'huey') n1, n2, n3 = Note.select().order_by(Note.text).limit(3) huey.notes.add([n1.id, n2.id]) self.assertNotes(huey.notes, [1, 2]) self.assertUsers(n1.users, ['huey']) self.assertUsers(n2.users, ['huey']) self.assertUsers(n3.users, []) def test_unique(self): n1 = Note.get(Note.text == 'note-1') huey = User.get(User.username == 'huey') def add_user(note, user): with self.assertQueryCount(1): note.users.add(user) add_user(n1, huey) self.assertRaises(IntegrityError, add_user, n1, huey) add_user(n1, User.get(User.username == 'zaizee')) self.assertUsers(n1.users, ['huey', 'zaizee']) def test_remove(self): self._set_data() gargie, huey, mickey, zaizee = User.select().order_by(User.username) n1, n2, n3, n4, n5 = Note.select().order_by(Note.text) with self.assertQueryCount(1): gargie.notes.remove([n1, n2, n3]) self.assertNotes(gargie.notes, []) self.assertNotes(huey.notes, [2, 3]) with self.assertQueryCount(1): huey.notes.remove(Note.select().where( Note.text << ['note-2', 'note-4', 'note-5'])) self.assertNotes(huey.notes, [3]) self.assertNotes(mickey.notes, [3, 4]) self.assertNotes(zaizee.notes, [4, 5]) with self.assertQueryCount(1): n4.users.remove([gargie, mickey]) self.assertUsers(n4.users, ['zaizee']) with self.assertQueryCount(1): n5.users.remove(User.select()) self.assertUsers(n5.users, []) def test_remove_by_id(self): self._set_data() gargie, huey = User.select().order_by(User.username).limit(2) n1, n2, n3, n4 = Note.select().order_by(Note.text).limit(4) gargie.notes.add([n3, n4]) with self.assertQueryCount(1): gargie.notes.remove([n1.id, n3.id]) self.assertNotes(gargie.notes, [2, 4]) self.assertNotes(huey.notes, [2, 3]) def test_clear(self): gargie = User.get(User.username == 'gargie') huey = User.get(User.username == 'huey') gargie.notes = Note.select() huey.notes = Note.select() self.assertEqual(gargie.notes.count(), 5) self.assertEqual(huey.notes.count(), 5) gargie.notes.clear() self.assertEqual(gargie.notes.count(), 0) self.assertEqual(huey.notes.count(), 5) n1 = Note.get(Note.text == 'note-1') n2 = Note.get(Note.text == 'note-2') n1.users = User.select() n2.users = User.select() self.assertEqual(n1.users.count(), 4) self.assertEqual(n2.users.count(), 4) n1.users.clear() self.assertEqual(n1.users.count(), 0) self.assertEqual(n2.users.count(), 4) def test_manual_through(self): gargie, huey, mickey, zaizee = User.select().order_by(User.username) alt_notes = [] for i in range(5): alt_notes.append(AltNote.create(text='note-%s' % (i + 1))) self.assertNotes(gargie.altnotes, []) for alt_note in alt_notes: self.assertUsers(alt_note.users, []) n1, n2, n3, n4, n5 = alt_notes # Test adding relationships by setting the descriptor. gargie.altnotes = [n1, n2] with self.assertQueryCount(2): huey.altnotes = AltNote.select().where( fn.substr(AltNote.text, 6, 1) << ['1', '3', '5']) mickey.altnotes.add([n1, n4]) with self.assertQueryCount(2): zaizee.altnotes = AltNote.select() # Test that the notes were added correctly. with self.assertQueryCount(1): self.assertNotes(gargie.altnotes, [1, 2]) with self.assertQueryCount(1): self.assertNotes(huey.altnotes, [1, 3, 5]) with self.assertQueryCount(1): self.assertNotes(mickey.altnotes, [1, 4]) with self.assertQueryCount(1): self.assertNotes(zaizee.altnotes, [1, 2, 3, 4, 5]) # Test removing notes. with self.assertQueryCount(1): gargie.altnotes.remove(n1) self.assertNotes(gargie.altnotes, [2]) with self.assertQueryCount(1): huey.altnotes.remove([n1, n2, n3]) self.assertNotes(huey.altnotes, [5]) with self.assertQueryCount(1): sq = (AltNote .select() .where(fn.SUBSTR(AltNote.text, 6, 1) << ['1', '2', '4'])) zaizee.altnotes.remove(sq) self.assertNotes(zaizee.altnotes, [3, 5]) # Test the backside of the relationship. n1.users = User.select().where(User.username != 'gargie') with self.assertQueryCount(1): self.assertUsers(n1.users, ['huey', 'mickey', 'zaizee']) with self.assertQueryCount(1): self.assertUsers(n2.users, ['gargie']) with self.assertQueryCount(1): self.assertUsers(n3.users, ['zaizee']) with self.assertQueryCount(1): self.assertUsers(n4.users, ['mickey']) with self.assertQueryCount(1): self.assertUsers(n5.users, ['huey', 'zaizee']) with self.assertQueryCount(1): n1.users.remove(User.select()) with self.assertQueryCount(1): n5.users.remove([gargie, huey]) with self.assertQueryCount(1): self.assertUsers(n1.users, []) with self.assertQueryCount(1): self.assertUsers(n5.users, ['zaizee']) class Person(TestModel): name = CharField() class Account(TestModel): person = ForeignKeyField(Person, primary_key=True) class AccountList(TestModel): name = CharField() accounts = ManyToManyField(Account, backref='lists') AccountListThrough = AccountList.accounts.get_through_model() class TestForeignKeyPrimaryKeyManyToMany(ModelTestCase): database = get_in_memory_db() requires = [Person, Account, AccountList, AccountListThrough] test_data = ( ('huey', ('cats', 'evil')), ('zaizee', ('cats', 'good')), ('mickey', ('dogs', 'good')), ('zombie', ()), ) def setUp(self): super(TestForeignKeyPrimaryKeyManyToMany, self).setUp() name2list = {} for name, lists in self.test_data: p = Person.create(name=name) a = Account.create(person=p) for l in lists: if l not in name2list: name2list[l] = AccountList.create(name=l) name2list[l].accounts.add(a) def account_for(self, name): return Account.select().join(Person).where(Person.name == name).get() def assertLists(self, l1, l2): self.assertEqual(sorted(list(l1)), sorted(list(l2))) def test_pk_is_fk(self): list2names = {} for name, lists in self.test_data: account = self.account_for(name) self.assertLists([l.name for l in account.lists], lists) for l in lists: list2names.setdefault(l, []) list2names[l].append(name) for list_name, names in list2names.items(): account_list = AccountList.get(AccountList.name == list_name) self.assertLists([s.person.name for s in account_list.accounts], names) def test_empty(self): al = AccountList.create(name='empty') self.assertEqual(list(al.accounts), []) class Permission(TestModel): name = TextField() DeniedThroughDeferred = DeferredThroughModel() class Visitor(TestModel): name = TextField() allowed = ManyToManyField(Permission) denied = ManyToManyField(Permission, through_model=DeniedThroughDeferred) class DeniedThrough(TestModel): permission = ForeignKeyField(Permission) visitor = ForeignKeyField(Visitor) DeniedThroughDeferred.set_model(DeniedThrough) class TestMultipleManyToManySameTables(ModelTestCase): database = get_in_memory_db() requires = [Permission, Visitor, Visitor.allowed.through_model, Visitor.denied.through_model] def test_multiple_manytomany_same_tables(self): p1, p2, p3 = [Permission.create(name=n) for n in ('p1', 'p2', 'p3')] v1, v2, v3 = [Visitor.create(name=n) for n in ('v1', 'v2', 'v3')] v1.allowed.add([p1, p2, p3]) v2.allowed.add(p2) v2.denied.add([p1, p3]) v3.allowed.add(p3) v3.denied.add(p1) accum = [] for v in Visitor.select().order_by(Visitor.name): allowed, denied = [], [] for p in v.allowed.order_by(Permission.name): allowed.append(p.name) for p in v.denied.order_by(Permission.name): denied.append(p.name) accum.append((v.name, allowed, denied)) self.assertEqual(accum, [ ('v1', ['p1', 'p2', 'p3'], []), ('v2', ['p2'], ['p1', 'p3']), ('v3', ['p3'], ['p1'])]) ================================================ FILE: tests/migrations.py ================================================ import datetime import os from functools import partial from peewee import * from playhouse.migrate import * from .base import BaseTestCase from .base import IS_CRDB from .base import IS_MYSQL from .base import IS_POSTGRESQL from .base import IS_PSYCOPG3 from .base import IS_SQLITE from .base import IS_SQLITE_25 from .base import IS_SQLITE_35 from .base import ModelTestCase from .base import TestModel from .base import db from .base import get_in_memory_db from .base import requires_models from .base import requires_pglike from .base import requires_postgresql from .base import requires_sqlite from .base import skip_if from .base import skip_unless try: from psycopg2cffi import compat compat.register() except ImportError: pass class Tag(TestModel): tag = CharField() class Person(TestModel): first_name = CharField() last_name = CharField() dob = DateField(null=True) class User(TestModel): id = CharField(primary_key=True, max_length=20) password = CharField(default='secret') class Meta: table_name = 'users' class Page(TestModel): name = CharField(max_length=100, unique=True, null=True) user = ForeignKeyField(User, null=True, backref='pages') class Session(TestModel): user = ForeignKeyField(User, unique=True, backref='sessions') updated_at = DateField(null=True) class IndexModel(TestModel): first_name = CharField() last_name = CharField() data = IntegerField(unique=True) class Meta: indexes = ( (('first_name', 'last_name'), True), ) class Category(TestModel): name = TextField() class TestSchemaMigration(ModelTestCase): requires = [Person, Tag, User, Page, Session] # Each database behaves slightly differently. _exception_add_not_null = not IS_MYSQL _person_data = [ ('Charlie', 'Leifer', None), ('Huey', 'Kitty', datetime.date(2011, 5, 1)), ('Mickey', 'Dog', datetime.date(2008, 6, 1)), ] def setUp(self): super(TestSchemaMigration, self).setUp() self.migrator = SchemaMigrator.from_database(self.database) def tearDown(self): try: super(TestSchemaMigration, self).tearDown() finally: self.database.close() @requires_pglike def test_add_table_constraint(self): price = FloatField(default=0.) migrate(self.migrator.add_column('tag', 'price', price), self.migrator.add_constraint('tag', 'price_check', Check('price >= 0'))) class Tag2(Model): tag = CharField() price = FloatField(default=0.) class Meta: database = self.database table_name = Tag._meta.table_name with self.database.atomic(): self.assertRaises(IntegrityError, Tag2.create, tag='t1', price=-1) Tag2.create(tag='t1', price=1.0) t1_db = Tag2.get(Tag2.tag == 't1') self.assertEqual(t1_db.price, 1.0) @skip_if(IS_SQLITE) def test_add_unique(self): alt_id = IntegerField(default=0) migrate( self.migrator.add_column('tag', 'alt_id', alt_id), self.migrator.add_unique('tag', 'alt_id')) class Tag2(Model): tag = CharField() alt_id = IntegerField(default=0) class Meta: database = self.database table_name = Tag._meta.table_name Tag2.create(tag='t1', alt_id=1) with self.database.atomic(): self.assertRaises(IntegrityError, Tag2.create, tag='t2', alt_id=1) @requires_pglike def test_drop_table_constraint(self): price = FloatField(default=0.) migrate( self.migrator.add_column('tag', 'price', price), self.migrator.add_constraint('tag', 'price_check', Check('price >= 0'))) class Tag2(Model): tag = CharField() price = FloatField(default=0.) class Meta: database = self.database table_name = Tag._meta.table_name with self.database.atomic(): self.assertRaises(IntegrityError, Tag2.create, tag='t1', price=-1) migrate(self.migrator.drop_constraint('tag', 'price_check')) Tag2.create(tag='t1', price=-1) t1_db = Tag2.get(Tag2.tag == 't1') self.assertEqual(t1_db.price, -1.0) def test_add_column(self): # Create some fields with a variety of NULL / default values. df = DateTimeField(null=True) df_def = DateTimeField(default=datetime.datetime(2012, 1, 1)) cf = CharField(max_length=200, default='') bf = BooleanField(default=True) ff = FloatField(default=0) # Create two rows in the Tag table to test the handling of adding # non-null fields. t1 = Tag.create(tag='t1') t2 = Tag.create(tag='t2') # Convenience function for generating `add_column` migrations. add_column = partial(self.migrator.add_column, 'tag') # Run the migration. migrate( add_column('pub_date', df), add_column('modified_date', df_def), add_column('comment', cf), add_column('is_public', bf), add_column('popularity', ff)) # Create a new tag model to represent the fields we added. class NewTag(Model): tag = CharField() pub_date = df modified_date = df_def comment = cf is_public = bf popularity = ff class Meta: database = self.database table_name = Tag._meta.table_name query = (NewTag .select( NewTag.id, NewTag.tag, NewTag.pub_date, NewTag.modified_date, NewTag.comment, NewTag.is_public, NewTag.popularity) .order_by(NewTag.tag.asc())) # Verify the resulting rows are correct. self.assertEqual(list(query.tuples()), [ (t1.id, 't1', None, datetime.datetime(2012, 1, 1), '', True, 0.0), (t2.id, 't2', None, datetime.datetime(2012, 1, 1), '', True, 0.0), ]) @skip_if(IS_MYSQL, 'mysql does not support CHECK()') def test_add_column_constraint(self): cf = CharField(null=True, constraints=[SQL('default \'foo\'')]) ff = FloatField(default=0., constraints=[Check('val < 1.0')]) t1 = Tag.create(tag='t1') migrate( self.migrator.add_column('tag', 'misc', cf), self.migrator.add_column('tag', 'val', ff)) class NewTag(Model): tag = CharField() misc = CharField() val = FloatField() class Meta: database = self.database table_name = Tag._meta.table_name t1_db = NewTag.get(NewTag.tag == 't1') self.assertEqual(t1_db.misc, 'foo') self.assertEqual(t1_db.val, 0.) with self.database.atomic(): self.assertRaises(IntegrityError, NewTag.create, tag='t2', misc='bar', val=2.) NewTag.create(tag='t3', misc='baz', val=0.9) t3_db = NewTag.get(NewTag.tag == 't3') self.assertEqual(t3_db.misc, 'baz') self.assertEqual(t3_db.val, 0.9) def _create_people(self): for first, last, dob in self._person_data: Person.create(first_name=first, last_name=last, dob=dob) def get_column_names(self, tbl): cursor = self.database.execute_sql('select * from %s limit 1' % tbl) return set([col[0] for col in cursor.description]) def test_drop_column(self, legacy=False): kw = {'legacy': legacy} if IS_SQLITE else {} self._create_people() migrate( self.migrator.drop_column('person', 'last_name', **kw), self.migrator.drop_column('person', 'dob', **kw)) column_names = self.get_column_names('person') self.assertEqual(column_names, set(['id', 'first_name'])) User.create(id='charlie', password='12345') User.create(id='huey', password='meow') migrate(self.migrator.drop_column('users', 'password', **kw)) column_names = self.get_column_names('users') self.assertEqual(column_names, set(['id'])) data = [row for row in User.select(User.id).order_by(User.id).tuples()] self.assertEqual(data, [ ('charlie',), ('huey',),]) @skip_unless(IS_SQLITE_35, 'Requires sqlite 3.35 or newer') def test_drop_column_sqlite_legacy(self): self.test_drop_column(legacy=True) def test_rename_column(self, legacy=False): kw = {'legacy': legacy} if IS_SQLITE else {} self._create_people() migrate( self.migrator.rename_column('person', 'first_name', 'first', **kw), self.migrator.rename_column('person', 'last_name', 'last', **kw)) column_names = self.get_column_names('person') self.assertEqual(column_names, set(['id', 'first', 'last', 'dob'])) class NewPerson(Model): first = CharField() last = CharField() dob = DateField() class Meta: database = self.database table_name = Person._meta.table_name query = (NewPerson .select( NewPerson.first, NewPerson.last, NewPerson.dob) .order_by(NewPerson.first)) self.assertEqual(list(query.tuples()), self._person_data) @skip_unless(IS_SQLITE_25, 'Requires sqlite 3.25 or newer') def test_rename_column_sqlite_legacy(self): self.test_rename_column(legacy=True) def test_rename_gh380(self, legacy=False): kw = {'legacy': legacy} if IS_SQLITE else {} u1 = User.create(id='charlie') u2 = User.create(id='huey') p1 = Page.create(name='p1-1', user=u1) p2 = Page.create(name='p2-1', user=u1) p3 = Page.create(name='p3-2', user=u2) migrate(self.migrator.rename_column('page', 'name', 'title', **kw)) column_names = self.get_column_names('page') self.assertEqual(column_names, set(['id', 'title', 'user_id'])) class NewPage(Model): title = CharField(max_length=100, unique=True, null=True) user = ForeignKeyField(User, null=True, backref='newpages') class Meta: database = self.database table_name = Page._meta.table_name query = (NewPage .select( NewPage.title, NewPage.user) .order_by(NewPage.title)) self.assertEqual( [(np.title, np.user.id) for np in query], [('p1-1', 'charlie'), ('p2-1', 'charlie'), ('p3-2', 'huey')]) @skip_unless(IS_SQLITE_25, 'Requires sqlite 3.25 or newer') def test_rename_gh380_sqlite_legacy(self): self.test_rename_gh380(legacy=True) @skip_if(IS_PSYCOPG3, 'Psycopg3 chokes on the default value.') def test_add_default_drop_default(self): with self.database.transaction(): migrate(self.migrator.add_column_default('person', 'first_name', default='x')) p = Person.create(last_name='Last') p_db = Person.get(Person.last_name == 'Last') self.assertEqual(p_db.first_name, 'x') with self.database.transaction(): migrate(self.migrator.drop_column_default('person', 'first_name')) if IS_MYSQL: # MySQL, even though the column is NOT NULL, does not seem to be # enforcing the constraint(?). Person.create(last_name='Last2') p_db = Person.get(Person.last_name == 'Last2') self.assertEqual(p_db.first_name, '') else: with self.assertRaises(IntegrityError): with self.database.transaction(): Person.create(last_name='Last2') def test_add_not_null(self): self._create_people() def addNotNull(): with self.database.transaction(): migrate(self.migrator.add_not_null('person', 'dob')) # We cannot make the `dob` field not null because there is currently # a null value there. if self._exception_add_not_null: with self.assertRaisesCtx((IntegrityError, InternalError)): addNotNull() (Person .update(dob=datetime.date(2000, 1, 2)) .where(Person.dob >> None) .execute()) # Now we can make the column not null. addNotNull() # And attempting to insert a null value results in an integrity error. with self.database.transaction(): with self.assertRaisesCtx((IntegrityError, OperationalError)): Person.create( first_name='Kirby', last_name='Snazebrauer', dob=None) def test_drop_not_null(self): self._create_people() migrate( self.migrator.drop_not_null('person', 'first_name'), self.migrator.drop_not_null('person', 'last_name')) p = Person.create(first_name=None, last_name=None) query = (Person .select() .where( (Person.first_name >> None) & (Person.last_name >> None))) self.assertEqual(query.count(), 1) def test_modify_not_null_foreign_key(self): user = User.create(id='charlie') Page.create(name='null user') Page.create(name='charlie', user=user) def addNotNull(): with self.database.transaction(): migrate(self.migrator.add_not_null('page', 'user_id')) if self._exception_add_not_null: with self.assertRaisesCtx((IntegrityError, InternalError)): addNotNull() with self.database.transaction(): Page.update(user=user).where(Page.user.is_null()).execute() addNotNull() # And attempting to insert a null value results in an integrity error. with self.database.transaction(): with self.assertRaisesCtx((OperationalError, IntegrityError)): Page.create( name='fails', user=None) # Now we will drop it. with self.database.transaction(): migrate(self.migrator.drop_not_null('page', 'user_id')) self.assertEqual(Page.select().where(Page.user.is_null()).count(), 0) Page.create(name='succeeds', user=None) self.assertEqual(Page.select().where(Page.user.is_null()).count(), 1) def test_rename_table(self): t1 = Tag.create(tag='t1') t2 = Tag.create(tag='t2') # Move the tag data into a new model/table. class Tag_asdf(Tag): pass self.assertEqual(Tag_asdf._meta.table_name, 'tag_asdf') # Drop the new table just to be safe. Tag_asdf._schema.drop_all(True) # Rename the tag table. migrate(self.migrator.rename_table('tag', 'tag_asdf')) # Verify the data was moved. query = (Tag_asdf .select() .order_by(Tag_asdf.tag)) self.assertEqual([t.tag for t in query], ['t1', 't2']) # Verify the old table is gone. with self.database.transaction(): self.assertRaises( DatabaseError, Tag.create, tag='t3') self.database.execute_sql('drop table tag_asdf') def test_add_index(self): # Create a unique index on first and last names. columns = ('first_name', 'last_name') migrate(self.migrator.add_index('person', columns, True)) Person.create(first_name='first', last_name='last') with self.database.transaction(): with self.assertRaisesCtx((IntegrityError, InternalError)): Person.create(first_name='first', last_name='last') def test_add_unique_column(self): uf = CharField(default='', unique=True) # Run the migration. migrate(self.migrator.add_column('tag', 'unique_field', uf)) # Create a new tag model to represent the fields we added. class NewTag(Model): tag = CharField() unique_field = uf class Meta: database = self.database table_name = Tag._meta.table_name NewTag.create(tag='t1', unique_field='u1') NewTag.create(tag='t2', unique_field='u2') with self.database.atomic(): self.assertRaises(IntegrityError, NewTag.create, tag='t3', unique_field='u1') def test_drop_index(self): # Create a unique index. self.test_add_index() # Now drop the unique index. migrate( self.migrator.drop_index('person', 'person_first_name_last_name')) Person.create(first_name='first', last_name='last') query = (Person .select() .where( (Person.first_name == 'first') & (Person.last_name == 'last'))) self.assertEqual(query.count(), 2) def test_add_and_remove(self): operations = [] field = CharField(default='foo') for i in range(10): operations.append(self.migrator.add_column('tag', 'foo', field)) operations.append(self.migrator.drop_column('tag', 'foo')) migrate(*operations) col_names = self.get_column_names('tag') self.assertEqual(col_names, set(['id', 'tag'])) def test_multiple_operations(self): self.database.execute_sql('drop table if exists person_baze;') self.database.execute_sql('drop table if exists person_nugg;') self._create_people() field_n = CharField(null=True) field_d = CharField(default='test') operations = [ self.migrator.add_column('person', 'field_null', field_n), self.migrator.drop_column('person', 'first_name'), self.migrator.add_column('person', 'field_default', field_d), self.migrator.rename_table('person', 'person_baze'), self.migrator.rename_table('person_baze', 'person_nugg'), self.migrator.rename_column('person_nugg', 'last_name', 'last'), self.migrator.add_index('person_nugg', ('last',), True), ] migrate(*operations) class PersonNugg(Model): field_null = field_n field_default = field_d last = CharField() dob = DateField(null=True) class Meta: database = self.database table_name = 'person_nugg' people = (PersonNugg .select( PersonNugg.field_null, PersonNugg.field_default, PersonNugg.last, PersonNugg.dob) .order_by(PersonNugg.last) .tuples()) expected = [ (None, 'test', 'Dog', datetime.date(2008, 6, 1)), (None, 'test', 'Kitty', datetime.date(2011, 5, 1)), (None, 'test', 'Leifer', None), ] self.assertEqual(list(people), expected) with self.database.transaction(): self.assertRaises( IntegrityError, PersonNugg.create, last='Leifer', field_default='bazer') self.database.execute_sql('drop table person_nugg;') def test_add_foreign_key(self): if hasattr(Person, 'newtag_set'): delattr(Person, 'newtag_set') # Ensure no foreign keys are present at the beginning of the test. self.assertEqual(self.database.get_foreign_keys('tag'), []) field = ForeignKeyField(Person, field=Person.id, null=True) migrate(self.migrator.add_column('tag', 'person_id', field)) class NewTag(Tag): person = field class Meta: table_name = 'tag' p = Person.create(first_name='First', last_name='Last') t1 = NewTag.create(tag='t1', person=p) t2 = NewTag.create(tag='t2') t1_db = NewTag.get(NewTag.tag == 't1') self.assertEqual(t1_db.person, p) t2_db = NewTag.get(NewTag.tag == 't2') self.assertIsNone(t2_db.person) foreign_keys = self.database.get_foreign_keys('tag') self.assertEqual(len(foreign_keys), 1) foreign_key = foreign_keys[0] self.assertEqual(foreign_key.column, 'person_id') self.assertEqual(foreign_key.dest_column, 'id') self.assertEqual(foreign_key.dest_table, 'person') def test_drop_foreign_key(self): kw = {'legacy': True} if IS_SQLITE else {} migrate(self.migrator.drop_column('page', 'user_id', **kw)) columns = self.database.get_columns('page') self.assertEqual( sorted(column.name for column in columns), ['id', 'name']) self.assertEqual(self.database.get_foreign_keys('page'), []) def test_rename_foreign_key(self): migrate(self.migrator.rename_column('page', 'user_id', 'huey_id')) columns = self.database.get_columns('page') self.assertEqual( sorted(column.name for column in columns), ['huey_id', 'id', 'name']) foreign_keys = self.database.get_foreign_keys('page') self.assertEqual(len(foreign_keys), 1) foreign_key = foreign_keys[0] self.assertEqual(foreign_key.column, 'huey_id') self.assertEqual(foreign_key.dest_column, 'id') self.assertEqual(foreign_key.dest_table, 'users') def test_rename_unique_foreign_key(self): migrate(self.migrator.rename_column('session', 'user_id', 'huey_id')) columns = self.database.get_columns('session') self.assertEqual( sorted(column.name for column in columns), ['huey_id', 'id', 'updated_at']) foreign_keys = self.database.get_foreign_keys('session') self.assertEqual(len(foreign_keys), 1) foreign_key = foreign_keys[0] self.assertEqual(foreign_key.column, 'huey_id') self.assertEqual(foreign_key.dest_column, 'id') self.assertEqual(foreign_key.dest_table, 'users') @requires_pglike @requires_models(Tag) def test_add_column_with_index_type(self): from playhouse.postgres_ext import BinaryJSONField self.reset_sql_history() field = BinaryJSONField(default=dict, index=True, null=True) migrate(self.migrator.add_column('tag', 'metadata', field)) queries = [x.msg for x in self.history] self.assertEqual(queries, [ ('ALTER TABLE "tag" ADD COLUMN "metadata" JSONB', []), ('CREATE INDEX "tag_metadata" ON "tag" USING GIN ("metadata")', []), ]) @skip_if(IS_CRDB, 'crdb is still finnicky about changing types.') def test_alter_column_type(self): # Convert varchar to text. field = TextField() migrate(self.migrator.alter_column_type('tag', 'tag', field)) _, tag = self.database.get_columns('tag') # name, type, null?, primary-key?, table, default. data_type = 'TEXT' if IS_SQLITE else 'text' self.assertEqual(tag, ('tag', data_type, False, False, 'tag', None)) # Convert date to datetime. field = DateTimeField() migrate(self.migrator.alter_column_type('person', 'dob', field)) _, _, _, dob = self.database.get_columns('person') if IS_POSTGRESQL or IS_CRDB: self.assertTrue(dob.data_type.startswith('timestamp')) else: self.assertEqual(dob.data_type.lower(), 'datetime') # Convert text to integer. field = IntegerField() cast = '(tag::integer)' if IS_POSTGRESQL or IS_CRDB else None migrate(self.migrator.alter_column_type('tag', 'tag', field, cast)) _, tag = self.database.get_columns('tag') if IS_SQLITE: data_type = 'INTEGER' elif IS_MYSQL: data_type = 'int' else: data_type = 'integer' self.assertEqual(tag, ('tag', data_type, False, False, 'tag', None)) @requires_sqlite def test_valid_column_required(self): self.assertRaises( (OperationalError, ValueError), migrate, self.migrator.drop_column('page', 'column_does_not_exist')) self.assertRaises( (OperationalError, ValueError), migrate, self.migrator.rename_column('page', 'xx', 'yy')) @requires_sqlite @requires_models(IndexModel) def test_table_case_insensitive(self): migrate(self.migrator.drop_column('PaGe', 'name', legacy=True)) column_names = self.get_column_names('page') self.assertEqual(column_names, set(['id', 'user_id'])) testing_field = CharField(default='xx') migrate(self.migrator.add_column('pAGE', 'testing', testing_field)) column_names = self.get_column_names('page') self.assertEqual(column_names, set(['id', 'user_id', 'testing'])) migrate(self.migrator.drop_column('indeX_mOdel', 'first_name', legacy=True)) indexes = self.migrator.database.get_indexes('index_model') self.assertEqual(len(indexes), 1) self.assertEqual(indexes[0].name, 'index_model_data') @requires_sqlite @requires_models(IndexModel) def test_add_column_indexed_table(self): # Ensure that columns can be added to tables that have indexes. field = CharField(default='') migrate(self.migrator.add_column('index_model', 'foo', field)) db = self.migrator.database columns = db.get_columns('index_model') self.assertEqual(sorted(column.name for column in columns), ['data', 'first_name', 'foo', 'id', 'last_name']) indexes = db.get_indexes('index_model') self.assertEqual( sorted((index.name, index.columns) for index in indexes), [('index_model_data', ['data']), ('index_model_first_name_last_name', ['first_name', 'last_name'])]) @requires_sqlite def test_rename_column_to_table_name(self): db = self.migrator.database columns = lambda: sorted(col.name for col in db.get_columns('page')) indexes = lambda: sorted((idx.name, idx.columns) for idx in db.get_indexes('page')) orig_columns = columns() orig_indexes = indexes() # Rename "page"."name" to "page"."page". migrate(self.migrator.rename_column('page', 'name', 'page')) # Ensure that the index on "name" is preserved, and that the index on # the user_id foreign key is also preserved. self.assertEqual(columns(), ['id', 'page', 'user_id']) self.assertEqual(indexes(), [ ('page_name', ['page']), ('page_user_id', ['user_id'])]) # Revert the operation and verify migrate(self.migrator.rename_column('page', 'page', 'name')) self.assertEqual(columns(), orig_columns) self.assertEqual(indexes(), orig_indexes) @requires_sqlite @requires_models(Category) def test_add_fk_with_constraints(self): self.reset_sql_history() field = ForeignKeyField(Category, Category.id, backref='children', null=True, on_delete='SET NULL') migrate(self.migrator.add_column( Category._meta.table_name, 'parent_id', field)) queries = [x.msg for x in self.history] self.assertEqual(queries, [ ('ALTER TABLE "category" ADD COLUMN "parent_id" ' 'INTEGER REFERENCES "category" ("id") ON DELETE SET NULL', []), ('CREATE INDEX "category_parent_id" ON "category" ("parent_id")', []), ]) @requires_sqlite @requires_models(IndexModel) def test_index_preservation(self): self.reset_sql_history() migrate(self.migrator.rename_column( 'index_model', 'first_name', 'first', legacy=True)) queries = [x.msg for x in self.history] self.assertEqual(queries, [ # Get all the columns. ('PRAGMA "main".table_info("index_model")', None), # Get the table definition. ('select name, sql from sqlite_master ' 'where type=? and LOWER(name)=?', ['table', 'index_model']), # Get the indexes and indexed columns for the table. ('SELECT name, sql FROM "main".sqlite_master ' 'WHERE tbl_name = ? AND type = ? ORDER BY name', ('index_model', 'index')), ('PRAGMA "main".index_list("index_model")', None), ('PRAGMA "main".index_info("index_model_data")', None), ('PRAGMA "main".index_info("index_model_first_name_last_name")', None), # Drop any temporary table, if it exists. ('DROP TABLE IF EXISTS "index_model__tmp__"', []), # Create a temporary table with the renamed column. ('CREATE TABLE "index_model__tmp__" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"first" VARCHAR(255) NOT NULL, ' '"last_name" VARCHAR(255) NOT NULL, ' '"data" INTEGER NOT NULL)', []), # Copy data from original table into temporary table. ('INSERT INTO "index_model__tmp__" ' '("id", "first", "last_name", "data") ' 'SELECT "id", "first_name", "last_name", "data" ' 'FROM "index_model"', []), # Drop the original table. ('DROP TABLE "index_model"', []), # Rename the temporary table, replacing the original. ('ALTER TABLE "index_model__tmp__" RENAME TO "index_model"', []), # Re-create the indexes. ('CREATE UNIQUE INDEX "index_model_data" ' 'ON "index_model" ("data")', []), ('CREATE UNIQUE INDEX "index_model_first_name_last_name" ' 'ON "index_model" ("first", "last_name")', []) ]) @requires_sqlite @requires_models(User, Page) def test_modify_fk_constraint(self): self.reset_sql_history() new_fk = ForeignKeyField(User, User.id, null=True, on_delete='CASCADE') migrate( self.migrator.drop_column('page', 'user_id', legacy=True), self.migrator.add_column('page', 'user_id', new_fk)) queries = [x.msg for x in self.history] self.assertEqual(queries, [ # Get all columns for table. ('PRAGMA "main".table_info("page")', None), # Get the SQL used to generate the table and indexes. ('select name, sql from sqlite_master ' 'where type=? and LOWER(name)=?', ['table', 'page']), ('SELECT name, sql FROM "main".sqlite_master ' 'WHERE tbl_name = ? AND type = ? ORDER BY name', ('page', 'index')), # Get the indexes and indexed columns for the table. ('PRAGMA "main".index_list("page")', None), ('PRAGMA "main".index_info("page_name")', None), ('PRAGMA "main".index_info("page_user_id")', None), #('PRAGMA "main".foreign_key_list("page")', None), # Clear out a temp table and create it w/o the user_id FK. ('DROP TABLE IF EXISTS "page__tmp__"', []), ('CREATE TABLE "page__tmp__" (' '"id" INTEGER NOT NULL PRIMARY KEY, "name" VARCHAR(100))', []), # Copy data into the temp table, drop the original and rename # the temp -> original. Recreate index(es). ('INSERT INTO "page__tmp__" ("id", "name") ' 'SELECT "id", "name" FROM "page"', []), ('DROP TABLE "page"', []), ('ALTER TABLE "page__tmp__" RENAME TO "page"', []), ('CREATE UNIQUE INDEX "page_name" ON "page" ("name")', []), # Add new foreign-key field with appropriate constraint. ('ALTER TABLE "page" ADD COLUMN "user_id" VARCHAR(20) ' 'REFERENCES "users" ("id") ON DELETE CASCADE', []), ('CREATE INDEX "page_user_id" ON "page" ("user_id")', []), ]) self.database.pragma('foreign_keys', 1) huey = User.create(id='huey') huey_page = Page.create(user=huey, name='huey page') self.assertEqual(Page.select().count(), 1) # Deleting the user will cascade to the associated page. User.delete().where(User.id == 'huey').execute() self.assertEqual(Page.select().count(), 0) def test_make_index_name(self): self.assertEqual(make_index_name('table', ['column']), 'table_column') def test_make_index_name_long(self): columns = [ 'very_long_column_name_number_1', 'very_long_column_name_number_2', 'very_long_column_name_number_3', 'very_long_column_name_number_4' ] name = make_index_name('very_long_table_name', columns) self.assertEqual(len(name), 64) class BadNames(TestModel): primary_data = TextField() foreign_data = TextField() data = TextField() class Meta: constraints = [ SQL('CONSTRAINT const1 UNIQUE (primary_data)'), SQL('CONSTRAINT const2 UNIQUE (foreign_data)')] class HasChecks(TestModel): key = TextField() value = IntegerField() class Meta: constraints = [ SQL("CHECK (key != '')"), SQL('CHECK (value > 0)')] class TestSqliteColumnNameRegression(ModelTestCase): database = get_in_memory_db() requires = [BadNames, HasChecks] def test_sqlite_check_constraints(self): HasChecks.create(key='k1', value=1) migrator = SchemaMigrator.from_database(self.database) extra = TextField(default='') migrate(migrator.add_column('has_checks', 'extra', extra)) columns = self.database.get_columns('has_checks') self.assertEqual([c.name for c in columns], ['id', 'key', 'value', 'extra']) HC = Table('has_checks', ('id', 'key', 'value', 'extra')) HC = HC.bind(self.database) # Sanity-check: ensure we can create a new row. data = {'key': 'k2', 'value': 2, 'extra': 'x2'} self.assertTrue(HC.insert(data).execute()) # Check constraints preserved. data = {'key': 'k0', 'value': 0, 'extra': 'x0'} self.assertRaises(IntegrityError, HC.insert(data).execute) data = {'key': '', 'value': 3, 'extra': 'x3'} self.assertRaises(IntegrityError, HC.insert(data).execute) def test_sqlite_column_name_constraint_regression(self): BadNames.create(primary_data='pd', foreign_data='fd', data='d') migrator = SchemaMigrator.from_database(self.database) new_data = TextField(default='foo') migrate(migrator.add_column('bad_names', 'new_data', new_data), migrator.drop_column('bad_names', 'data')) columns = self.database.get_columns('bad_names') column_names = [column.name for column in columns] self.assertEqual(column_names, ['id', 'primary_data', 'foreign_data', 'new_data']) BNT = Table('bad_names', ('id', 'primary_data', 'foreign_data', 'new_data')).bind(self.database) self.assertEqual([row for row in BNT.select()], [{ 'id': 1, 'primary_data': 'pd', 'foreign_data': 'fd', 'new_data': 'foo'}]) # Verify constraints were carried over. data = {'primary_data': 'pd', 'foreign_data': 'xx', 'new_data': 'd'} self.assertRaises(IntegrityError, BNT.insert(data).execute) data.update(primary_data='px', foreign_data='fd') self.assertRaises(IntegrityError, BNT.insert(data).execute) data.update(foreign_data='fx') self.assertTrue(BNT.insert(data).execute()) ================================================ FILE: tests/model_save.py ================================================ from peewee import * from .base import ModelTestCase from .base import TestModel from .base import requires_pglike class T1(TestModel): pk = AutoField() value = IntegerField() class T2(TestModel): pk = IntegerField(constraints=[SQL('DEFAULT 3')], primary_key=True) value = IntegerField() class T3(TestModel): pk = IntegerField(primary_key=True) value = IntegerField() class T4(TestModel): pk1 = IntegerField() pk2 = IntegerField() value = IntegerField() class Meta: primary_key = CompositeKey('pk1', 'pk2') class T5(TestModel): val = IntegerField(null=True) class TestPrimaryKeySaveHandling(ModelTestCase): requires = [T1, T2, T3, T4] def test_auto_field(self): # AutoField will be inserted if the PK is not set, after which the new # ID will be populated. t11 = T1(value=1) self.assertEqual(t11.save(), 1) self.assertTrue(t11.pk is not None) # Calling save() a second time will issue an update. t11.value = 100 self.assertEqual(t11.save(), 1) # Verify the record was updated. t11_db = T1[t11.pk] self.assertEqual(t11_db.value, 100) # We can explicitly specify the value of an auto-incrementing # primary-key, but we must be sure to call save(force_insert=True), # otherwise peewee will attempt to do an update. t12 = T1(pk=1337, value=2) self.assertEqual(t12.save(), 0) self.assertEqual(T1.select().count(), 1) self.assertEqual(t12.save(force_insert=True), 1) # Attempting to force-insert an already-existing PK will fail with an # integrity error. with self.database.atomic(): with self.assertRaises(IntegrityError): t12.value = 3 t12.save(force_insert=True) query = T1.select().order_by(T1.value).tuples() self.assertEqual(list(query), [(1337, 2), (t11.pk, 100)]) @requires_pglike def test_server_default_pk(self): # The new value of the primary-key will be returned to us, since # postgres supports RETURNING. t2 = T2(value=1) self.assertEqual(t2.save(), 1) self.assertEqual(t2.pk, 3) # Saving after the PK is set will issue an update. t2.value = 100 self.assertEqual(t2.save(), 1) t2_db = T2[3] self.assertEqual(t2_db.value, 100) # If we just set the pk and try to save, peewee issues an update which # doesn't have any effect. t22 = T2(pk=2, value=20) self.assertEqual(t22.save(), 0) self.assertEqual(T2.select().count(), 1) # We can force-insert the value we specify explicitly. self.assertEqual(t22.save(force_insert=True), 1) self.assertEqual(T2[2].value, 20) def test_integer_field_pk(self): # For a non-auto-incrementing primary key, we have to use force_insert. t3 = T3(pk=2, value=1) self.assertEqual(t3.save(), 0) # Oops, attempts to do an update. self.assertEqual(T3.select().count(), 0) # Force to be an insert. self.assertEqual(t3.save(force_insert=True), 1) # Now we can update the value and call save() to issue an update. t3.value = 100 self.assertEqual(t3.save(), 1) # Verify data is correct. t3_db = T3[2] self.assertEqual(t3_db.value, 100) def test_composite_pk(self): t4 = T4(pk1=1, pk2=2, value=10) # Will attempt to do an update on non-existant rows. self.assertEqual(t4.save(), 0) self.assertEqual(t4.save(force_insert=True), 1) # Modifying part of the composite PK and attempt an update will fail. t4.pk2 = 3 t4.value = 30 self.assertEqual(t4.save(), 0) t4.pk2 = 2 self.assertEqual(t4.save(), 1) t4_db = T4[1, 2] self.assertEqual(t4_db.value, 30) @requires_pglike def test_returning_object(self): query = T2.insert(value=10).returning(T2).objects() t2_db, = list(query) self.assertEqual(t2_db.pk, 3) self.assertEqual(t2_db.value, 10) class TestSaveNoData(ModelTestCase): requires = [T5] def test_save_no_data(self): t5 = T5.create() self.assertTrue(t5.id >= 1) t5.val = 3 t5.save() t5_db = T5.get(T5.id == t5.id) self.assertEqual(t5_db.val, 3) t5.val = None t5.save() t5_db = T5.get(T5.id == t5.id) self.assertTrue(t5_db.val is None) def test_save_no_data2(self): t5 = T5.create() t5_db = T5.get(T5.id == t5.id) t5_db.save() t5_db = T5.get(T5.id == t5.id) self.assertTrue(t5_db.val is None) def test_save_no_data3(self): t5 = T5.create() self.assertRaises(ValueError, t5.save) def test_save_only_no_data(self): t5 = T5.create(val=1) t5.val = 2 self.assertRaises(ValueError, t5.save, only=[]) t5_db = T5.get(T5.id == t5.id) self.assertEqual(t5_db.val, 1) ================================================ FILE: tests/model_sql.py ================================================ import datetime from peewee import * from peewee import Alias from peewee import Database from peewee import ModelIndex from .base import get_in_memory_db from .base import requires_pglike from .base import skip_if from .base import BaseTestCase from .base import IS_CRDB from .base import ModelDatabaseTestCase from .base import TestModel from .base import __sql__ from .base_models import * class CKM(TestModel): category = CharField() key = CharField() value = IntegerField() class Meta: primary_key = CompositeKey('category', 'key') class TestModelSQL(ModelDatabaseTestCase): database = get_in_memory_db() requires = [Category, CKM, Note, Person, Relationship, Sample, User, DfltM] def test_select(self): query = (Person .select( Person.first, Person.last, fn.COUNT(Note.id).alias('ct')) .join(Note) .where((Person.last == 'Leifer') & (Person.id < 4))) self.assertSQL(query, ( 'SELECT "t1"."first", "t1"."last", COUNT("t2"."id") AS "ct" ' 'FROM "person" AS "t1" ' 'INNER JOIN "note" AS "t2" ON ("t2"."author_id" = "t1"."id") ' 'WHERE (' '("t1"."last" = ?) AND ' '("t1"."id" < ?))'), ['Leifer', 4]) def test_reselect(self): sql = 'SELECT "t1"."name", "t1"."parent_id" FROM "category" AS "t1"' query = Category.select() self.assertSQL(query, sql, []) query2 = query.select() self.assertSQL(query2, sql, []) query = Category.select(Category.name, Category.parent) self.assertSQL(query, sql, []) query2 = query.select() self.assertSQL(query2, 'SELECT FROM "category" AS "t1"', []) query = query2.select(Category.name) self.assertSQL(query, 'SELECT "t1"."name" FROM "category" AS "t1"', []) def test_select_extend(self): query = Note.select() ext = query.join(Person).select_extend(Person) self.assertSQL(ext, ( 'SELECT "t1"."id", "t1"."author_id", "t1"."content", "t2"."id", ' '"t2"."first", "t2"."last", "t2"."dob" ' 'FROM "note" AS "t1" INNER JOIN "person" AS "t2" ' 'ON ("t1"."author_id" = "t2"."id")'), []) def test_selected_columns(self): query = (Person .select( Person.first, Person.last, fn.COUNT(Note.id).alias('ct')) .join(Note)) f_first, f_last, f_ct = query.selected_columns self.assertEqual(f_first.name, 'first') self.assertTrue(f_first.model is Person) self.assertEqual(f_last.name, 'last') self.assertTrue(f_last.model is Person) self.assertTrue(isinstance(f_ct, Alias)) f_ct = f_ct.unwrap() self.assertEqual(f_ct.name, 'COUNT') f_nid, = f_ct.arguments self.assertEqual(f_nid.name, 'id') self.assertTrue(f_nid.model is Note) query.selected_columns = (Person.first,) f_first, = query.selected_columns self.assertEqual(f_first.name, 'first') self.assertTrue(f_first.model is Person) def test_where_coerce(self): query = Person.select(Person.last).where(Person.id == '1337') self.assertSQL(query, ( 'SELECT "t1"."last" FROM "person" AS "t1" ' 'WHERE ("t1"."id" = ?)'), [1337]) query = Person.select(Person.last).where(Person.id < (Person.id - '5')) self.assertSQL(query, ( 'SELECT "t1"."last" FROM "person" AS "t1" ' 'WHERE ("t1"."id" < ("t1"."id" - ?))'), [5]) query = Person.select(Person.last).where(Person.first == b'foo') self.assertSQL(query, ( 'SELECT "t1"."last" FROM "person" AS "t1" ' 'WHERE ("t1"."first" = ?)'), ['foo']) def test_group_by(self): query = (User .select(User, fn.COUNT(Tweet.id).alias('tweet_count')) .join(Tweet, JOIN.LEFT_OUTER) .group_by(User)) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username", ' 'COUNT("t2"."id") AS "tweet_count" ' 'FROM "users" AS "t1" ' 'LEFT OUTER JOIN "tweet" AS "t2" ON ("t2"."user_id" = "t1"."id") ' 'GROUP BY "t1"."id", "t1"."username"'), []) def test_group_by_extend(self): query = (User .select(User, fn.COUNT(Tweet.id).alias('tweet_count')) .join(Tweet, JOIN.LEFT_OUTER) .group_by_extend(User.id).group_by_extend(User.username)) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username", ' 'COUNT("t2"."id") AS "tweet_count" ' 'FROM "users" AS "t1" ' 'LEFT OUTER JOIN "tweet" AS "t2" ON ("t2"."user_id" = "t1"."id") ' 'GROUP BY "t1"."id", "t1"."username"'), []) def test_order_by(self): query = (User .select() .order_by(User.username.desc(), User.id)) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username" FROM "users" AS "t1" ' 'ORDER BY "t1"."username" DESC, "t1"."id"'), []) def test_order_by_extend(self): query = (User .select() .order_by_extend(User.username.desc()) .order_by_extend(User.id)) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username" FROM "users" AS "t1" ' 'ORDER BY "t1"."username" DESC, "t1"."id"'), []) def test_paginate(self): # Get the first page, default is limit of 20. query = User.select().paginate(1) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username" FROM "users" AS "t1" ' 'LIMIT ? OFFSET ?'), [20, 0]) # Page 3 contains rows 31-45. query = User.select().paginate(3, 15) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username" FROM "users" AS "t1" ' 'LIMIT ? OFFSET ?'), [15, 30]) def test_subquery_correction(self): users = User.select().where(User.username.in_(['foo', 'bar'])) query = Tweet.select().where(Tweet.user.in_(users)) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."user_id", "t1"."content", ' '"t1"."timestamp" ' 'FROM "tweet" AS "t1" ' 'WHERE ("t1"."user_id" IN (' 'SELECT "t2"."id" FROM "users" AS "t2" ' 'WHERE ("t2"."username" IN (?, ?))))'), ['foo', 'bar']) def test_value_flattening(self): sql = ('SELECT "t1"."id", "t1"."username" FROM "users" AS "t1" ' 'WHERE ("t1"."username" IN (?, ?))') expected = (sql, ['foo', 'bar']) users = User.select().where(User.username.in_(['foo', 'bar'])) self.assertSQL(users, *expected) users = User.select().where(User.username.in_(('foo', 'bar'))) self.assertSQL(users, *expected) users = User.select().where(User.username.in_(set(['foo', 'bar']))) # Sets are unordered so params may be in either order: sql, params = __sql__(users) self.assertEqual(sql, expected[0]) self.assertTrue(params in (['foo', 'bar'], ['bar', 'foo'])) def test_model_select_from(self): inner = (User .select(User.id, User.username) .where(User.username == 'x')) query = inner.select_from(inner.c.username) self.assertSQL(query, ( 'SELECT "t1"."username" FROM (' 'SELECT "t2"."id", "t2"."username" ' 'FROM "users" AS "t2" ' 'WHERE ("t2"."username" = ?)) AS "t1"'), ['x']) def test_join_ctx(self): query = Tweet.select(Tweet.id).join(Favorite).switch(Tweet).join(User) self.assertSQL(query, ( 'SELECT "t1"."id" FROM "tweet" AS "t1" ' 'INNER JOIN "favorite" AS "t2" ON ("t2"."tweet_id" = "t1"."id") ' 'INNER JOIN "users" AS "t3" ON ("t1"."user_id" = "t3"."id")'), []) query = Tweet.select(Tweet.id).join(User).switch(Tweet).join(Favorite) self.assertSQL(query, ( 'SELECT "t1"."id" FROM "tweet" AS "t1" ' 'INNER JOIN "users" AS "t2" ON ("t1"."user_id" = "t2"."id") ' 'INNER JOIN "favorite" AS "t3" ON ("t3"."tweet_id" = "t1"."id")'), []) query = Tweet.select(Tweet.id).left_outer_join(Favorite).switch(Tweet).left_outer_join(User) self.assertSQL(query, ( 'SELECT "t1"."id" FROM "tweet" AS "t1" ' 'LEFT OUTER JOIN "favorite" AS "t2" ON ("t2"."tweet_id" = "t1"."id") ' 'LEFT OUTER JOIN "users" AS "t3" ON ("t1"."user_id" = "t3"."id")'), []) query = Tweet.select(Tweet.id).left_outer_join(User).switch(Tweet).left_outer_join(Favorite) self.assertSQL(query, ( 'SELECT "t1"."id" FROM "tweet" AS "t1" ' 'LEFT OUTER JOIN "users" AS "t2" ON ("t1"."user_id" = "t2"."id") ' 'LEFT OUTER JOIN "favorite" AS "t3" ON ("t3"."tweet_id" = "t1"."id")'), []) def test_model_alias(self): TA = Tweet.alias() query = (User .select(User, fn.COUNT(TA.id).alias('tc')) .join(TA, on=(User.id == TA.user)) .group_by(User)) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username", COUNT("t2"."id") AS "tc" ' 'FROM "users" AS "t1" ' 'INNER JOIN "tweet" AS "t2" ON ("t1"."id" = "t2"."user_id") ' 'GROUP BY "t1"."id", "t1"."username"'), []) def test_model_alias_with_schema(self): class Note(TestModel): content = TextField() class Meta: schema = 'notes' query = Note.select() self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."content" ' 'FROM "notes"."note" AS "t1"'), []) query = Note.alias('na').select() self.assertSQL(query, ( 'SELECT "na"."id", "na"."content" ' 'FROM "notes"."note" AS "na"'), []) def test_model_alias_join_with_schema(self): class Note(TestModel): content = TextField() class Meta: schema = 'notes' NA = Note.alias('na') query = (Note .select(Note.content, NA.content) .join(NA, on=(NA.id == Note.id))) self.assertSQL(query, ( 'SELECT "t1"."content", "na"."content" ' 'FROM "notes"."note" AS "t1" ' 'INNER JOIN "notes"."note" AS "na" ' 'ON ("na"."id" = "t1"."id")'), []) def test_filter_simple(self): query = User.filter(username='huey') self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username" FROM "users" AS "t1" ' 'WHERE ("t1"."username" = ?)'), ['huey']) query = User.filter(username='huey', id__gte=1, id__lt=5) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username" FROM "users" AS "t1" ' 'WHERE ((("t1"."id" >= ?) AND ("t1"."id" < ?)) AND ' '("t1"."username" = ?))'), [1, 5, 'huey']) query = User.filter(~DQ(id=1), username__in=('foo', 'bar')) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username" FROM "users" AS "t1" ' 'WHERE (NOT ("t1"."id" = ?) AND ("t1"."username" IN (?, ?)))'), [1, 'foo', 'bar']) query = User.filter((DQ(id=1) | DQ(id=2)), username__in=('foo', 'bar')) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username" FROM "users" AS "t1" ' 'WHERE ((("t1"."id" = ?) OR ("t1"."id" = ?)) AND ' '("t1"."username" IN (?, ?)))'), [1, 2, 'foo', 'bar']) def test_filter_expressions(self): query = User.filter( DQ(username__in=['huey', 'zaizee']) | (DQ(id__gt=2) & DQ(id__lt=4))) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username" ' 'FROM "users" AS "t1" ' 'WHERE (("t1"."username" IN (?, ?)) OR ' '(("t1"."id" > ?) AND ("t1"."id" < ?)))'), ['huey', 'zaizee', 2, 4]) def test_filter_join(self): query = Tweet.select(Tweet.content).filter(user__username='huey') self.assertSQL(query, ( 'SELECT "t1"."content" FROM "tweet" AS "t1" ' 'INNER JOIN "users" AS "t2" ON ("t1"."user_id" = "t2"."id") ' 'WHERE ("t2"."username" = ?)'), ['huey']) UA = User.alias('ua') query = (Tweet .select(Tweet.content) .join(UA) .filter(ua__username='huey')) self.assertSQL(query, ( 'SELECT "t1"."content" FROM "tweet" AS "t1" ' 'INNER JOIN "users" AS "ua" ON ("t1"."user_id" = "ua"."id") ' 'WHERE ("ua"."username" = ?)'), ['huey']) def test_filter_with_or_across_joins(self): query = (Tweet .select(Tweet.content) .filter( DQ(user__username='huey') | DQ(content__like='%hello%'))) self.assertSQL(query, ( 'SELECT "t1"."content" FROM "tweet" AS "t1" ' 'INNER JOIN "users" AS "t2" ' 'ON ("t1"."user_id" = "t2"."id") ' 'WHERE (("t2"."username" = ?) OR ' '("t1"."content" LIKE ?))'), ['huey', '%hello%']) def test_filter_join_combine_models(self): query = (Tweet .select(Tweet.content) .filter(user__username='huey') .filter(DQ(user__id__gte=1) | DQ(id__lt=5))) self.assertSQL(query, ( 'SELECT "t1"."content" FROM "tweet" AS "t1" ' 'INNER JOIN "users" AS "t2" ON ("t1"."user_id" = "t2"."id") ' 'WHERE (("t2"."username" = ?) AND ' '(("t2"."id" >= ?) OR ("t1"."id" < ?)))'), ['huey', 1, 5]) def test_mix_filter_methods(self): query = (User .select(User, fn.COUNT(Tweet.id).alias('count')) .filter(username__in=('huey', 'zaizee')) .join(Tweet, JOIN.LEFT_OUTER) .group_by(User.id, User.username) .order_by(fn.COUNT(Tweet.id).desc())) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username", COUNT("t2"."id") AS "count" ' 'FROM "users" AS "t1" ' 'LEFT OUTER JOIN "tweet" AS "t2" ON ("t2"."user_id" = "t1"."id") ' 'WHERE ("t1"."username" IN (?, ?)) ' 'GROUP BY "t1"."id", "t1"."username" ' 'ORDER BY COUNT("t2"."id") DESC'), ['huey', 'zaizee']) def test_join_parent(self): query = (Category .select() .where(Category.parent == 'test')) self.assertSQL(query, ( 'SELECT "t1"."name", "t1"."parent_id" FROM "category" AS "t1" ' 'WHERE ("t1"."parent_id" = ?)'), ['test']) query = Category.filter(parent='test') self.assertSQL(query, ( 'SELECT "t1"."name", "t1"."parent_id" FROM "category" AS "t1" ' 'WHERE ("t1"."parent_id" = ?)'), ['test']) def test_cross_join(self): class A(TestModel): id = IntegerField(primary_key=True) class B(TestModel): id = IntegerField(primary_key=True) query = (A .select(A.id.alias('aid'), B.id.alias('bid')) .join(B, JOIN.CROSS) .order_by(A.id, B.id)) self.assertSQL(query, ( 'SELECT "t1"."id" AS "aid", "t2"."id" AS "bid" ' 'FROM "a" AS "t1" ' 'CROSS JOIN "b" AS "t2" ' 'ORDER BY "t1"."id", "t2"."id"'), []) def test_join_expr(self): class User(TestModel): username = TextField(primary_key=True) class Tweet(TestModel): user = ForeignKeyField(User, backref='tweets') content = TextField() sql = ('SELECT "t1"."id", "t1"."user_id", "t1"."content", ' '"t2"."username" FROM "tweet" AS "t1" ' 'INNER JOIN "user" AS "t2" ' 'ON ("t1"."user_id" = "t2"."username")') query = Tweet.select(Tweet, User).join(User) self.assertSQL(query, sql, []) query = (Tweet .select(Tweet, User) .join(User, on=(Tweet.user == User.username))) self.assertSQL(query, sql, []) join_expr = ((Tweet.user == User.username) & (Value(1) == 1)) query = Tweet.select().join(User, on=join_expr) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."user_id", "t1"."content" ' 'FROM "tweet" AS "t1" ' 'INNER JOIN "user" AS "t2" ' 'ON (("t1"."user_id" = "t2"."username") AND (? = ?))'), [1, 1]) def test_join_multiple_fks(self): class A(TestModel): name = TextField() class B(TestModel): name = TextField(primary_key=True) a1 = ForeignKeyField(A, backref='b_set1') a2 = ForeignKeyField(A, field=A.name, backref='b_set2') A1 = A.alias('a1') A2 = A.alias('a2') sql = ('SELECT "t1"."name", "t1"."a1_id", "t1"."a2_id", ' '"a1"."id", "a1"."name", "a2"."id", "a2"."name" ' 'FROM "b" AS "t1" ' 'INNER JOIN "a" AS "a1" ON ("t1"."a1_id" = "a1"."id") ' 'INNER JOIN "a" AS "a2" ON ("t1"."a2_id" = "a2"."name")') query = (B.select(B, A1, A2) .join_from(B, A1, on=B.a1) .join_from(B, A2, on=B.a2)) self.assertSQL(query, sql, []) query = (B.select(B, A1, A2) .join(A1, on=(B.a1 == A1.id)).switch(B) .join(A2, on=(B.a2 == A2.name))) self.assertSQL(query, sql, []) jx1 = (B.a1 == A1.id) & (Value(1) == 1) jx2 = (Value(1) == 1) & (B.a2 == A2.name) query = (B.select() .join(A1, on=jx1).switch(B) .join(A2, on=jx2)) self.assertSQL(query, ( 'SELECT "t1"."name", "t1"."a1_id", "t1"."a2_id" ' 'FROM "b" AS "t1" ' 'INNER JOIN "a" AS "a1" ' 'ON (("t1"."a1_id" = "a1"."id") AND (? = ?)) ' 'INNER JOIN "a" AS "a2" ' 'ON ((? = ?) AND ("t1"."a2_id" = "a2"."name"))'), [1, 1, 1, 1]) def test_raw(self): query = (Person .raw('SELECT first, last, dob FROM person ' 'WHERE first = ? AND substr(last, 1, 1) = ? ' 'ORDER BY last', 'huey', 'l')) self.assertSQL(query, ( 'SELECT first, last, dob FROM person ' 'WHERE first = ? AND substr(last, 1, 1) = ? ' 'ORDER BY last'), ['huey', 'l']) def test_insert(self): query = (Person .insert({Person.first: 'huey', Person.last: 'cat', Person.dob: datetime.date(2011, 1, 1)})) self.assertSQL(query, ( 'INSERT INTO "person" ("first", "last", "dob") ' 'VALUES (?, ?, ?)'), ['huey', 'cat', datetime.date(2011, 1, 1)]) query = (Note .insert({Note.author: Person(id=1337), Note.content: 'leet'})) self.assertSQL(query, ( 'INSERT INTO "note" ("author_id", "content") ' 'VALUES (?, ?)'), [1337, 'leet']) query = Person.insert(first='huey', last='cat') self.assertSQL(query, ( 'INSERT INTO "person" ("first", "last") VALUES (?, ?)'), ['huey', 'cat']) def test_replace(self): query = (Person .replace({Person.first: 'huey', Person.last: 'cat'})) self.assertSQL(query, ( 'INSERT OR REPLACE INTO "person" ("first", "last") ' 'VALUES (?, ?)'), ['huey', 'cat']) def test_insert_many(self): query = (Note .insert_many(( {Note.author: Person(id=1), Note.content: 'note-1'}, {Note.author: Person(id=2), Note.content: 'note-2'}, {Note.author: Person(id=3), Note.content: 'note-3'}))) self.assertSQL(query, ( 'INSERT INTO "note" ("author_id", "content") ' 'VALUES (?, ?), (?, ?), (?, ?)'), [1, 'note-1', 2, 'note-2', 3, 'note-3']) query = (Note .insert_many(( {'author': Person(id=1), 'content': 'note-1'}, {'author': Person(id=2), 'content': 'note-2'}))) self.assertSQL(query, ( 'INSERT INTO "note" ("author_id", "content") ' 'VALUES (?, ?), (?, ?)'), [1, 'note-1', 2, 'note-2']) def test_insert_many_defaults(self): # Verify fields are inferred and values are read correctly, when # partial data is given and a field has default values. s2 = {'counter': 2, 'value': 2.} s3 = {'counter': 3} self.assertSQL(Sample.insert_many([s2, s3]), ( 'INSERT INTO "sample" ("counter", "value") VALUES (?, ?), (?, ?)'), [2, 2., 3, 1.]) self.assertSQL(Sample.insert_many([s3, s2]), ( 'INSERT INTO "sample" ("counter", "value") VALUES (?, ?), (?, ?)'), [3, 1., 2, 2.]) def test_insert_many_defaults_nulls(self): data = [ {'name': 'd1'}, {'name': 'd2', 'dflt1': 10}, {'name': 'd3', 'dflt2': 30}, {'name': 'd4', 'dfltn': 40}] fields = [DfltM.name, DfltM.dflt1, DfltM.dflt2, DfltM.dfltn] self.assertSQL(DfltM.insert_many(data, fields=fields), ( 'INSERT INTO "dflt_m" ("name", "dflt1", "dflt2", "dfltn") VALUES ' '(?, ?, ?, ?), (?, ?, ?, ?), (?, ?, ?, ?), (?, ?, ?, ?)'), ['d1', 1, 2, None, 'd2', 10, 2, None, 'd3', 1, 30, None, 'd4', 1, 2, 40]) def test_insert_many_list_with_fields(self): data = [(i,) for i in ('charlie', 'huey', 'zaizee')] query = User.insert_many(data, fields=[User.username]) self.assertSQL(query, ( 'INSERT INTO "users" ("username") VALUES (?), (?), (?)'), ['charlie', 'huey', 'zaizee']) # Use field name instead of field obj. query = User.insert_many(data, fields=['username']) self.assertSQL(query, ( 'INSERT INTO "users" ("username") VALUES (?), (?), (?)'), ['charlie', 'huey', 'zaizee']) def test_insert_many_infer_fields(self): data = [('f1', 'l1', '1980-01-01'), ('f2', 'l2', '1980-02-02')] self.assertSQL(Person.insert_many(data), ( 'INSERT INTO "person" ("first", "last", "dob") ' 'VALUES (?, ?, ?), (?, ?, ?)'), ['f1', 'l1', datetime.date(1980, 1, 1), 'f2', 'l2', datetime.date(1980, 2, 2)]) # When primary key is not auto-increment, PKs are included. data = [('c1', 'k1', 1), ('c2', 'k2', 2)] self.assertSQL(CKM.insert_many(data), ( 'INSERT INTO "ckm" ("category", "key", "value") ' 'VALUES (?, ?, ?), (?, ?, ?)'), ['c1', 'k1', 1, 'c2', 'k2', 2]) def test_insert_query(self): select = (Person .select(Person.id, Person.first) .where(Person.last == 'cat')) query = Note.insert_from(select, (Note.author, Note.content)) self.assertSQL(query, ('INSERT INTO "note" ("author_id", "content") ' 'SELECT "t1"."id", "t1"."first" ' 'FROM "person" AS "t1" ' 'WHERE ("t1"."last" = ?)'), ['cat']) query = Note.insert_from(select, ('author', 'content')) self.assertSQL(query, ('INSERT INTO "note" ("author_id", "content") ' 'SELECT "t1"."id", "t1"."first" ' 'FROM "person" AS "t1" ' 'WHERE ("t1"."last" = ?)'), ['cat']) def test_insert_returning(self): class TestDB(Database): returning_clause = True class User(Model): username = CharField() class Meta: database = TestDB(None) query = User.insert({User.username: 'zaizee'}) self.assertSQL(query, ( 'INSERT INTO "user" ("username") ' 'VALUES (?) RETURNING "user"."id"'), ['zaizee']) class Person(Model): name = CharField() ssn = CharField(primary_key=True) class Meta: database = TestDB(None) query = Person.insert({Person.name: 'charlie', Person.ssn: '123'}) self.assertSQL(query, ( 'INSERT INTO "person" ("ssn", "name") VALUES (?, ?) ' 'RETURNING "person"."ssn"'), ['123', 'charlie']) query = Person.insert({Person.name: 'huey'}).returning() self.assertSQL(query, ( 'INSERT INTO "person" ("name") VALUES (?)'), ['huey']) query = (Person .insert({Person.name: 'foo'}) .returning(Person.ssn.alias('new_ssn'))) self.assertSQL(query, ( 'INSERT INTO "person" ("name") VALUES (?) ' 'RETURNING "person"."ssn" AS "new_ssn"'), ['foo']) def test_insert_get_field_values(self): class User(TestModel): username = TextField(primary_key=True) class Meta: database = self.database class Tweet(TestModel): user = ForeignKeyField(User) content = TextField() class Meta: database = self.database queries = ( User.insert(username='a'), User.insert({'username': 'a'}), User.insert({User.username: 'a'})) for query in queries: self.assertSQL(query, ('INSERT INTO "user" ("username") ' 'VALUES (?)'), ['a']) # Verify that we can provide all kinds of combinations to the # constructor to INSERT and it will map the parameters correctly # without losing values. a = User(username='a') queries = ( Tweet.insert(user=a, content='ca'), Tweet.insert({'user': a, 'content': 'ca'}), Tweet.insert({Tweet.user: a, 'content': 'ca'}), Tweet.insert({'user': a, Tweet.content: 'ca'}), Tweet.insert({Tweet.user: a, Tweet.content: 'ca'}), Tweet.insert({Tweet.user: a}, content='ca'), Tweet.insert({Tweet.content: 'ca'}, user=a), Tweet.insert({'user': a}, content='ca'), Tweet.insert({'content': 'ca'}, user=a), # Also test using the foreign-key descriptor and column name. Tweet.insert({Tweet.user_id: a, Tweet.content: 'ca'}), Tweet.insert(user_id=a, content='ca'), Tweet.insert({'user_id': a, 'content': 'ca'})) for query in queries: self.assertSQL(query, ('INSERT INTO "tweet" ("user_id", "content")' ' VALUES (?, ?)'), ['a', 'ca']) def test_insert_many_get_field_values(self): class User(TestModel): username = TextField(primary_key=True) class Meta: database = self.database class Tweet(TestModel): user = ForeignKeyField(User) content = TextField() class Meta: database = self.database # Ensure we can handle any combination of insert-data key and field # list value. pairs = ((User.username, 'username'), ('username', User.username), ('username', 'username'), (User.username, User.username)) for dict_key, fields_key in pairs: iq = User.insert_many([{dict_key: u} for u in 'abc'], fields=[fields_key]) self.assertSQL(iq, ( 'INSERT INTO "user" ("username") VALUES (?), (?), (?)'), ['a', 'b', 'c']) a, b = User(username='a'), User(username='b') user_content = ( (a, 'ca1'), (a, 'ca2'), (b, 'cb1'), ('a', 'ca3')) # Specify user id directly. # Ensure we can mix-and-match key type within insert-data. pairs = (('user', 'content'), (Tweet.user, Tweet.content), (Tweet.user, 'content'), ('user', Tweet.content), ('user_id', 'content'), (Tweet.user_id, Tweet.content)) for ukey, ckey in pairs: iq = Tweet.insert_many([{ukey: u, ckey: c} for u, c in user_content]) self.assertSQL(iq, ( 'INSERT INTO "tweet" ("user_id", "content") VALUES ' '(?, ?), (?, ?), (?, ?), (?, ?)'), ['a', 'ca1', 'a', 'ca2', 'b', 'cb1', 'a', 'ca3']) def test_insert_many_dict_and_list(self): class R(TestModel): k = TextField(column_name='key') v = IntegerField(column_name='value', default=0) class Meta: database = self.database data = ( {'k': 'k1', 'v': 1}, {R.k: 'k2', R.v: 2}, {'key': 'k3', 'value': 3}, ('k4', 4), ('k5', '5'), # Will be converted properly. {R.k: 'k6', R.v: '6'}, {'key': 'k7', 'value': '7'}, {'k': 'kx'}, ('ky',)) param_str = ', '.join('(?, ?)' for _ in range(len(data))) queries = ( R.insert_many(data), R.insert_many(data, fields=[R.k, R.v]), R.insert_many(data, fields=['k', 'v'])) for query in queries: self.assertSQL(query, ( 'INSERT INTO "r" ("key", "value") VALUES %s' % param_str), ['k1', 1, 'k2', 2, 'k3', 3, 'k4', 4, 'k5', 5, 'k6', 6, 'k7', 7, 'kx', 0, 'ky', 0]) def test_insert_modelalias(self): UA = User.alias('ua') self.assertSQL(UA.insert({UA.username: 'huey'}), ( 'INSERT INTO "users" ("username") VALUES (?)'), ['huey']) self.assertSQL(UA.insert(username='huey'), ( 'INSERT INTO "users" ("username") VALUES (?)'), ['huey']) def test_update(self): class Stat(TestModel): url = TextField() count = IntegerField() timestamp = TimestampField(utc=True) query = (Stat .update({Stat.count: Stat.count + 1, Stat.timestamp: datetime.datetime(2017, 1, 1)}) .where(Stat.url == '/peewee')) self.assertSQL(query, ( 'UPDATE "stat" SET "count" = ("stat"."count" + ?), ' '"timestamp" = ? ' 'WHERE ("stat"."url" = ?)'), [1, 1483228800, '/peewee']) query = (Stat .update(count=Stat.count + 1) .where(Stat.url == '/peewee')) self.assertSQL(query, ( 'UPDATE "stat" SET "count" = ("stat"."count" + ?) ' 'WHERE ("stat"."url" = ?)'), [1, '/peewee']) def test_update_subquery(self): class U(TestModel): username = TextField() flood_count = IntegerField() class T(TestModel): user = ForeignKeyField(U) ctq = T.select(fn.COUNT(T.id) / 100).where(T.user == U.id) subq = (T .select(T.user) .group_by(T.user) .having(fn.COUNT(T.id) > 100)) query = (U .update({U.flood_count: ctq}) .where(U.id.in_(subq))) self.assertSQL(query, ( 'UPDATE "u" SET "flood_count" = (' 'SELECT (COUNT("t1"."id") / ?) FROM "t" AS "t1" ' 'WHERE ("t1"."user_id" = "u"."id")) ' 'WHERE ("u"."id" IN (' 'SELECT "t1"."user_id" FROM "t" AS "t1" ' 'GROUP BY "t1"."user_id" ' 'HAVING (COUNT("t1"."id") > ?)))'), [100, 100]) def test_update_from(self): class SalesPerson(TestModel): first = TextField() last = TextField() class Account(TestModel): contact_first = TextField() contact_last = TextField() sales = ForeignKeyField(SalesPerson) query = (Account .update(contact_first=SalesPerson.first, contact_last=SalesPerson.last) .from_(SalesPerson) .where(Account.sales == SalesPerson.id)) self.assertSQL(query, ( 'UPDATE "account" SET ' '"contact_first" = "t1"."first", ' '"contact_last" = "t1"."last" ' 'FROM "sales_person" AS "t1" ' 'WHERE ("account"."sales_id" = "t1"."id")'), []) query = (User .update({User.username: Tweet.content}) .from_(Tweet) .where(Tweet.content == 'tx')) self.assertSQL(query, ( 'UPDATE "users" SET "username" = "t1"."content" ' 'FROM "tweet" AS "t1" WHERE ("t1"."content" = ?)'), ['tx']) def test_update_from_qualnames(self): data = [(1, 'u1-x'), (2, 'u2-x')] vl = ValuesList(data, columns=('id', 'username'), alias='tmp') query = (User .update({User.username: vl.c.username}) .from_(vl) .where(User.id == vl.c.id)) self.assertSQL(query, ( 'UPDATE "users" SET "username" = "tmp"."username" ' 'FROM (VALUES (?, ?), (?, ?)) AS "tmp"("id", "username") ' 'WHERE ("users"."id" = "tmp"."id")'), [1, 'u1-x', 2, 'u2-x']) def test_update_from_subselect(self): data = [(1, 'u1-x'), (2, 'u2-x')] vl = ValuesList(data, columns=('id', 'username'), alias='tmp') subq = vl.select(vl.c.id, vl.c.username) query = (User .update({User.username: subq.c.username}) .from_(subq) .where(User.id == subq.c.id)) self.assertSQL(query, ( 'UPDATE "users" SET "username" = "t1"."username" FROM (' 'SELECT "tmp"."id", "tmp"."username" ' 'FROM (VALUES (?, ?), (?, ?)) AS "tmp"("id", "username")) AS "t1" ' 'WHERE ("users"."id" = "t1"."id")'), [1, 'u1-x', 2, 'u2-x']) def test_delete(self): query = (Note .delete() .where(Note.author << (Person.select(Person.id) .where(Person.last == 'cat')))) self.assertSQL(query, ('DELETE FROM "note" ' 'WHERE ("note"."author_id" IN (' 'SELECT "t1"."id" FROM "person" AS "t1" ' 'WHERE ("t1"."last" = ?)))'), ['cat']) query = Note.delete().where(Note.author == Person(id=123)) self.assertSQL(query, ( 'DELETE FROM "note" WHERE ("note"."author_id" = ?)'), [123]) def test_delete_recursive(self): class User(TestModel): username = CharField() class Tweet(TestModel): user = ForeignKeyField(User, backref='tweets') content = TextField() class Relationship(TestModel): from_user = ForeignKeyField(User, backref='relationships') to_user = ForeignKeyField(User, backref='related_to') class Like(TestModel): user = ForeignKeyField(User) tweet = ForeignKeyField(Tweet) queries = list(User(id=1).dependencies()) accum = [] for expr, fk in list(queries): query = fk.model.delete().where(expr) accum.append(__sql__(query)) self.assertEqual(sorted(accum), [ ('DELETE FROM "like" WHERE (' '"like"."tweet_id" IN (' 'SELECT "t1"."id" FROM "tweet" AS "t1" WHERE (' '"t1"."user_id" = ?)))', [1]), ('DELETE FROM "like" WHERE ("like"."user_id" = ?)', [1]), ('DELETE FROM "relationship" ' 'WHERE ("relationship"."from_user_id" = ?)', [1]), ('DELETE FROM "relationship" ' 'WHERE ("relationship"."to_user_id" = ?)', [1]), ('DELETE FROM "tweet" WHERE ("tweet"."user_id" = ?)', [1]), ]) def test_aliases(self): class A(TestModel): a = CharField() class B(TestModel): b = CharField() a_link = ForeignKeyField(A) class C(TestModel): c = CharField() b_link = ForeignKeyField(B) class D(TestModel): d = CharField() c_link = ForeignKeyField(C) query = (D .select(D.d, C.c) .join(C) .where(C.b_link << ( B.select(B.id).join(A).where(A.a == 'a')))) self.assertSQL(query, ( 'SELECT "t1"."d", "t2"."c" ' 'FROM "d" AS "t1" ' 'INNER JOIN "c" AS "t2" ON ("t1"."c_link_id" = "t2"."id") ' 'WHERE ("t2"."b_link_id" IN (' 'SELECT "t3"."id" FROM "b" AS "t3" ' 'INNER JOIN "a" AS "t4" ON ("t3"."a_link_id" = "t4"."id") ' 'WHERE ("t4"."a" = ?)))'), ['a']) def test_schema(self): class WithSchema(TestModel): data = CharField(primary_key=True) class Meta: schema = 'huey' query = WithSchema.select().where(WithSchema.data == 'zaizee') self.assertSQL(query, ( 'SELECT "t1"."data" ' 'FROM "huey"."with_schema" AS "t1" ' 'WHERE ("t1"."data" = ?)'), ['zaizee']) @requires_pglike class TestOnConflictSQL(ModelDatabaseTestCase): requires = [Emp, OCTest, UKVP] def test_atomic_update(self): query = OCTest.insert(a='foo', b=1).on_conflict( conflict_target=(OCTest.a,), update={OCTest.b: OCTest.b + 2}) self.assertSQL(query, ( 'INSERT INTO "oc_test" ("a", "b", "c") VALUES (?, ?, ?) ' 'ON CONFLICT ("a") ' 'DO UPDATE SET "b" = ("oc_test"."b" + ?) ' 'RETURNING "oc_test"."id"'), ['foo', 1, 0, 2]) def test_on_conflict_do_nothing(self): query = OCTest.insert(a='foo', b=1).on_conflict(action='IGNORE') self.assertSQL(query, ( 'INSERT INTO "oc_test" ("a", "b", "c") VALUES (?, ?, ?) ' 'ON CONFLICT DO NOTHING ' 'RETURNING "oc_test"."id"'), ['foo', 1, 0]) query = OCTest.insert(a='foo', b=1).on_conflict( conflict_target=(OCTest.a,), action='IGNORE') self.assertSQL(query, ( 'INSERT INTO "oc_test" ("a", "b", "c") VALUES (?, ?, ?) ' 'ON CONFLICT ("a") DO NOTHING ' 'RETURNING "oc_test"."id"'), ['foo', 1, 0]) def test_update_where_clause(self): # Add a new row with the given "a" value. If a conflict occurs, # re-insert with b=b+2 so long as the original b < 3. query = OCTest.insert(a='foo', b=1).on_conflict( conflict_target=(OCTest.a,), update={OCTest.b: OCTest.b + 2}, where=(OCTest.b < 3)) self.assertSQL(query, ( 'INSERT INTO "oc_test" ("a", "b", "c") VALUES (?, ?, ?) ' 'ON CONFLICT ("a") DO UPDATE SET "b" = ("oc_test"."b" + ?) ' 'WHERE ("oc_test"."b" < ?) ' 'RETURNING "oc_test"."id"'), ['foo', 1, 0, 2, 3]) def test_conflict_target_constraint_where(self): fields = [UKVP.key, UKVP.value, UKVP.extra] data = [('k1', 1, 2), ('k2', 2, 3)] query = (UKVP.insert_many(data, fields) .on_conflict(conflict_target=(UKVP.key, UKVP.value), conflict_where=(UKVP.extra > 1), preserve=(UKVP.extra,), where=(UKVP.key != 'kx'))) self.assertSQL(query, ( 'INSERT INTO "ukvp" ("key", "value", "extra") ' 'VALUES (?, ?, ?), (?, ?, ?) ' 'ON CONFLICT ("key", "value") WHERE ("extra" > ?) ' 'DO UPDATE SET "extra" = EXCLUDED."extra" ' 'WHERE ("ukvp"."key" != ?) RETURNING "ukvp"."id"'), ['k1', 1, 2, 'k2', 2, 3, 1, 'kx']) def test_preserve_and_update(self): query = (UKVP .insert(key='k1', value=1, extra=10) .on_conflict( conflict_target=(UKVP.key,), preserve=(UKVP.value,), update={UKVP.extra: UKVP.extra + 1})) self.assertSQL(query, ( 'INSERT INTO "ukvp" ("key", "value", "extra") ' 'VALUES (?, ?, ?) ' 'ON CONFLICT ("key") DO UPDATE SET ' '"value" = EXCLUDED."value", ' '"extra" = ("ukvp"."extra" + ?) ' 'RETURNING "ukvp"."id"'), ['k1', 1, 10, 1]) def test_preserve_with_where(self): query = (UKVP .insert(key='k1', value=1, extra=10) .on_conflict( conflict_target=(UKVP.key,), preserve=(UKVP.value,), where=(UKVP.extra < 100))) self.assertSQL(query, ( 'INSERT INTO "ukvp" ("key", "value", "extra") ' 'VALUES (?, ?, ?) ' 'ON CONFLICT ("key") DO UPDATE SET ' '"value" = EXCLUDED."value" ' 'WHERE ("ukvp"."extra" < ?) ' 'RETURNING "ukvp"."id"'), ['k1', 1, 10, 100]) @skip_if(IS_CRDB, 'crdb lol') def test_on_conflict_named_constraint(self): query = (UKVP .insert(key='k1', value=1) .on_conflict( conflict_constraint='ukvp_key', update={UKVP.value: UKVP.value + 1})) self.assertSQL(query, ( 'INSERT INTO "ukvp" ("key", "value") VALUES (?, ?) ' 'ON CONFLICT ON CONSTRAINT "ukvp_key" ' 'DO UPDATE SET "value" = ("ukvp"."value" + ?) ' 'RETURNING "ukvp"."id"'), ['k1', 1, 1]) class TestStringsForFieldsa(ModelDatabaseTestCase): database = get_in_memory_db() requires = [Note, Person, Relationship] def test_insert(self): qkwargs = Person.insert(first='huey', last='kitty') qliteral = Person.insert({'first': 'huey', 'last': 'kitty'}) for query in (qkwargs, qliteral): self.assertSQL(query, ( 'INSERT INTO "person" ("first", "last") VALUES (?, ?)'), ['huey', 'kitty']) def test_insert_many(self): data = [ {'first': 'huey', 'last': 'cat'}, {'first': 'zaizee', 'last': 'cat'}, {'first': 'mickey', 'last': 'dog'}] query = Person.insert_many(data) self.assertSQL(query, ( 'INSERT INTO "person" ("first", "last") VALUES (?, ?), (?, ?), ' '(?, ?)'), ['huey', 'cat', 'zaizee', 'cat', 'mickey', 'dog']) def test_update(self): qkwargs = Person.update(last='kitty').where(Person.last == 'cat') qliteral = Person.update({'last': 'kitty'}).where(Person.last == 'cat') for query in (qkwargs, qliteral): self.assertSQL(query, ( 'UPDATE "person" SET "last" = ? WHERE ("person"."last" = ?)'), ['kitty', 'cat']) compound_db = get_in_memory_db() class CompoundTestModel(Model): class Meta: database = compound_db class Alpha(CompoundTestModel): alpha = IntegerField() class Beta(CompoundTestModel): beta = IntegerField() other = IntegerField(default=0) class Gamma(CompoundTestModel): gamma = IntegerField() other = IntegerField(default=1) class TestModelCompoundSelect(BaseTestCase): def test_unions(self): lhs = Alpha.select(Alpha.alpha) rhs = Beta.select(Beta.beta) self.assertSQL((lhs | rhs), ( 'SELECT "t1"."alpha" FROM "alpha" AS "t1" UNION ' 'SELECT "t2"."beta" FROM "beta" AS "t2"'), []) rrhs = Gamma.select(Gamma.gamma) query = (lhs | (rhs | rrhs)) self.assertSQL(query, ( 'SELECT "t1"."alpha" FROM "alpha" AS "t1" UNION ' 'SELECT "t2"."beta" FROM "beta" AS "t2" UNION ' 'SELECT "t3"."gamma" FROM "gamma" AS "t3"'), []) def test_union_same_model(self): q1 = Alpha.select(Alpha.alpha) q2 = Alpha.select(Alpha.alpha) q3 = Alpha.select(Alpha.alpha) compound = (q1 | q2) | q3 self.assertSQL(compound, ( 'SELECT "t1"."alpha" FROM "alpha" AS "t1" UNION ' 'SELECT "t2"."alpha" FROM "alpha" AS "t2" UNION ' 'SELECT "t3"."alpha" FROM "alpha" AS "t3"'), []) compound = q1 | (q2 | q3) self.assertSQL(compound, ( 'SELECT "t1"."alpha" FROM "alpha" AS "t1" UNION ' 'SELECT "t2"."alpha" FROM "alpha" AS "t2" UNION ' 'SELECT "t3"."alpha" FROM "alpha" AS "t3"'), []) def test_where(self): q1 = Alpha.select(Alpha.alpha).where(Alpha.alpha < 2) q2 = Alpha.select(Alpha.alpha).where(Alpha.alpha > 5) compound = q1 | q2 self.assertSQL(compound, ( 'SELECT "t1"."alpha" FROM "alpha" AS "t1" ' 'WHERE ("t1"."alpha" < ?) ' 'UNION ' 'SELECT "t2"."alpha" FROM "alpha" AS "t2" ' 'WHERE ("t2"."alpha" > ?)'), [2, 5]) q3 = Beta.select(Beta.beta).where(Beta.beta < 3) q4 = Beta.select(Beta.beta).where(Beta.beta > 4) compound = q1 | q3 self.assertSQL(compound, ( 'SELECT "t1"."alpha" FROM "alpha" AS "t1" ' 'WHERE ("t1"."alpha" < ?) ' 'UNION ' 'SELECT "t2"."beta" FROM "beta" AS "t2" ' 'WHERE ("t2"."beta" < ?)'), [2, 3]) compound = q1 | q3 | q2 | q4 self.assertSQL(compound, ( 'SELECT "t1"."alpha" FROM "alpha" AS "t1" ' 'WHERE ("t1"."alpha" < ?) ' 'UNION ' 'SELECT "t2"."beta" FROM "beta" AS "t2" ' 'WHERE ("t2"."beta" < ?) ' 'UNION ' 'SELECT "t3"."alpha" FROM "alpha" AS "t3" ' 'WHERE ("t3"."alpha" > ?) ' 'UNION ' 'SELECT "t4"."beta" FROM "beta" AS "t4" ' 'WHERE ("t4"."beta" > ?)'), [2, 3, 5, 4]) def test_limit(self): lhs = Alpha.select(Alpha.alpha).order_by(Alpha.alpha).limit(3) rhs = Beta.select(Beta.beta).order_by(Beta.beta).limit(4) compound = (lhs | rhs).limit(5) # This may be invalid SQL, but this at least documents the behavior. self.assertSQL(compound, ( 'SELECT "t1"."alpha" FROM "alpha" AS "t1" ' 'ORDER BY "t1"."alpha" LIMIT ? UNION ' 'SELECT "t2"."beta" FROM "beta" AS "t2" ' 'ORDER BY "t2"."beta" LIMIT ? LIMIT ?'), [3, 4, 5]) def test_union_from(self): lhs = Alpha.select(Alpha.alpha).where(Alpha.alpha < 2) rhs = Alpha.select(Alpha.alpha).where(Alpha.alpha > 5) compound = (lhs | rhs).alias('cq') query = Alpha.select(compound.c.alpha).from_(compound) self.assertSQL(query, ( 'SELECT "cq"."alpha" FROM (' 'SELECT "t1"."alpha" FROM "alpha" AS "t1" ' 'WHERE ("t1"."alpha" < ?) ' 'UNION ' 'SELECT "t2"."alpha" FROM "alpha" AS "t2" ' 'WHERE ("t2"."alpha" > ?)) AS "cq"'), [2, 5]) b = Beta.select(Beta.beta).where(Beta.beta < 3) g = Gamma.select(Gamma.gamma).where(Gamma.gamma < 0) compound = (lhs | b | g).alias('cq') query = Alpha.select(SQL('1')).from_(compound) self.assertSQL(query, ( 'SELECT 1 FROM (' 'SELECT "t1"."alpha" FROM "alpha" AS "t1" ' 'WHERE ("t1"."alpha" < ?) ' 'UNION SELECT "t2"."beta" FROM "beta" AS "t2" ' 'WHERE ("t2"."beta" < ?) ' 'UNION SELECT "t3"."gamma" FROM "gamma" AS "t3" ' 'WHERE ("t3"."gamma" < ?)) AS "cq"'), [2, 3, 0]) def test_parentheses(self): query = (Alpha.select().where(Alpha.alpha < 2) | Beta.select(Beta.id, Beta.beta).where(Beta.beta > 3)) self.assertSQL(query, ( '(SELECT "t1"."id", "t1"."alpha" FROM "alpha" AS "t1" ' 'WHERE ("t1"."alpha" < ?)) ' 'UNION ' '(SELECT "t2"."id", "t2"."beta" FROM "beta" AS "t2" ' 'WHERE ("t2"."beta" > ?))'), [2, 3], compound_select_parentheses=True) def test_where_in(self): union = (Alpha.select(Alpha.alpha) | Beta.select(Beta.beta)) query = Alpha.select().where(Alpha.alpha << union) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."alpha" ' 'FROM "alpha" AS "t1" ' 'WHERE ("t1"."alpha" IN ' '(SELECT "t1"."alpha" FROM "alpha" AS "t1" ' 'UNION ' 'SELECT "t2"."beta" FROM "beta" AS "t2"))'), []) class TestModelIndex(BaseTestCase): database = SqliteDatabase(None) def test_model_index(self): class Article(Model): name = TextField() timestamp = TimestampField() status = IntegerField() flags = IntegerField() aidx = ModelIndex(Article, (Article.name, Article.timestamp),) self.assertSQL(aidx, ( 'CREATE INDEX IF NOT EXISTS "article_name_timestamp" ON "article" ' '("name", "timestamp")'), []) aidx = aidx.where(Article.status == 1) self.assertSQL(aidx, ( 'CREATE INDEX IF NOT EXISTS "article_name_timestamp" ON "article" ' '("name", "timestamp") ' 'WHERE ("status" = ?)'), [1]) aidx = ModelIndex(Article, (Article.timestamp.desc(), Article.flags.bin_and(4)), unique=True) self.assertSQL(aidx, ( 'CREATE UNIQUE INDEX IF NOT EXISTS "article_timestamp" ' 'ON "article" ("timestamp" DESC, ("flags" & ?))'), [4]) def test_unique_index_nulls(self): class A(Model): a = CharField() b = CharField() class Meta: database = self.database idx = ModelIndex(A, ('a', 'b'), unique=True) self.assertSQL(A._schema._create_index(idx), ( 'CREATE UNIQUE INDEX IF NOT EXISTS ' '"a_a_b" ON "a" (a, b)')) idx = idx.nulls_distinct(False) self.assertSQL(A._schema._create_index(idx), ( 'CREATE UNIQUE INDEX IF NOT EXISTS ' '"a_a_b" ON "a" (a, b) NULLS NOT DISTINCT')) idx = idx.nulls_distinct(True) self.assertSQL(A._schema._create_index(idx), ( 'CREATE UNIQUE INDEX IF NOT EXISTS ' '"a_a_b" ON "a" (a, b) NULLS DISTINCT')) idx._unique = False self.assertRaises(ValueError, lambda: idx.nulls_distinct(True)) self.assertRaises(ValueError, lambda: idx.nulls_distinct(False)) class TestModelArgument(BaseTestCase): database = SqliteDatabase(None) def test_model_as_argument(self): class Post(TestModel): content = TextField() timestamp = DateTimeField() class Meta: database = self.database query = (Post .select(Post.id, fn.score(Post).alias('score')) .order_by(Post.timestamp)) self.assertSQL(query, ( 'SELECT "t1"."id", score("t1") AS "score" ' 'FROM "post" AS "t1" ORDER BY "t1"."timestamp"'), []) ================================================ FILE: tests/models.py ================================================ import datetime import threading import time import unittest from unittest import mock from peewee import * from peewee import Entity from peewee import NodeList from peewee import SubclassAwareMetadata from peewee import sort_models from .base import db from .base import get_in_memory_db from .base import new_connection from .base import requires_models from .base import requires_mysql from .base import requires_pglike from .base import requires_postgresql from .base import requires_sqlite from .base import skip_if from .base import skip_unless from .base import BaseTestCase from .base import IS_CRDB from .base import IS_MYSQL from .base import IS_MYSQL_ADVANCED_FEATURES from .base import IS_POSTGRESQL from .base import IS_SQLITE from .base import IS_SQLITE_OLD from .base import IS_SQLITE_15 # Row-values. from .base import IS_SQLITE_24 # Upsert. from .base import IS_SQLITE_25 # Window functions. from .base import IS_SQLITE_30 # FILTER clause functions. from .base import IS_SQLITE_9 from .base import ModelTestCase from .base import TestModel from .base_models import * class Color(TestModel): name = CharField(primary_key=True) is_neutral = BooleanField(default=False) class Post(TestModel): content = TextField(column_name='Content') timestamp = DateTimeField(column_name='TimeStamp', default=datetime.datetime.now) class PostNote(TestModel): post = ForeignKeyField(Post, backref='notes', primary_key=True) note = TextField() class Point(TestModel): x = IntegerField() y = IntegerField() class Meta: primary_key = False class CPK(TestModel): key = CharField() value = IntegerField() extra = IntegerField() class Meta: primary_key = CompositeKey('key', 'value') class City(TestModel): name = CharField() class Venue(TestModel): name = CharField() city = ForeignKeyField(City, backref='venues') city_n = ForeignKeyField(City, backref='venues_n', null=True) class Event(TestModel): name = CharField() venue = ForeignKeyField(Venue, backref='events', null=True) class TestModelAPIs(ModelTestCase): def add_user(self, username): return User.create(username=username) def add_tweets(self, user, *tweets): accum = [] for tweet in tweets: accum.append(Tweet.create(user=user, content=tweet)) return accum @requires_models(Point) def test_no_primary_key(self): p11 = Point.create(x=1, y=1) p33 = Point.create(x=3, y=3) p_db = Point.get((Point.x == 3) & (Point.y == 3)) self.assertEqual(p_db.x, 3) self.assertEqual(p_db.y, 3) @requires_models(Post, PostNote) def test_pk_is_fk(self): with self.database.atomic(): p1 = Post.create(content='p1') p2 = Post.create(content='p2') p1n = PostNote.create(post=p1, note='p1n') p2n = PostNote.create(post=p2, note='p2n') with self.assertQueryCount(2): pn = PostNote.get(PostNote.note == 'p1n') self.assertEqual(pn.post.content, 'p1') with self.assertQueryCount(1): pn = (PostNote .select(PostNote, Post) .join(Post) .where(PostNote.note == 'p2n') .get()) self.assertEqual(pn.post.content, 'p2') if not IS_SQLITE: exc_class = (ProgrammingError, IntegrityError) with self.database.atomic() as txn: self.assertRaises(exc_class, PostNote.create, note='pxn') txn.rollback() @requires_models(User, Tweet) def test_assertQueryCount(self): self.add_tweets(self.add_user('charlie'), 'foo', 'bar', 'baz') def do_test(n): with self.assertQueryCount(n): authors = [tweet.user.username for tweet in Tweet.select()] self.assertRaises(AssertionError, do_test, 1) self.assertRaises(AssertionError, do_test, 3) do_test(4) self.assertRaises(AssertionError, do_test, 5) @requires_models(Post) def test_column_field_translation(self): ts = datetime.datetime(2017, 2, 1, 13, 37) ts2 = datetime.datetime(2017, 2, 2, 13, 37) p = Post.create(content='p1', timestamp=ts) p2 = Post.create(content='p2', timestamp=ts2) p_db = Post.get(Post.content == 'p1') self.assertEqual(p_db.content, 'p1') self.assertEqual(p_db.timestamp, ts) pd1, pd2 = Post.select().order_by(Post.id).dicts() self.assertEqual(pd1['content'], 'p1') self.assertEqual(pd1['timestamp'], ts) self.assertEqual(pd2['content'], 'p2') self.assertEqual(pd2['timestamp'], ts2) @requires_models(User) def test_insert_many(self): data = [('u%02d' % i,) for i in range(100)] with self.database.atomic(): for chunk in chunked(data, 10): User.insert_many(chunk).execute() self.assertEqual(User.select().count(), 100) names = [u.username for u in User.select().order_by(User.username)] self.assertEqual(names, ['u%02d' % i for i in range(100)]) @requires_models(DfltM) def test_insert_many_defaults_nullable(self): data = [ {'name': 'd1'}, {'name': 'd2', 'dflt1': 10}, {'name': 'd3', 'dflt2': 30}, {'name': 'd4', 'dfltn': 40}] fields = [DfltM.name, DfltM.dflt1, DfltM.dflt2, DfltM.dfltn] DfltM.insert_many(data, fields).execute() expected = [ ('d1', 1, 2, None), ('d2', 10, 2, None), ('d3', 1, 30, None), ('d4', 1, 2, 40)] query = DfltM.select().order_by(DfltM.name) actual = [(d.name, d.dflt1, d.dflt2, d.dfltn) for d in query] self.assertEqual(actual, expected) @requires_models(User, Tweet) def test_create(self): with self.assertQueryCount(1): huey = self.add_user('huey') self.assertEqual(huey.username, 'huey') self.assertTrue(isinstance(huey.id, int)) self.assertTrue(huey.id > 0) with self.assertQueryCount(1): tweet = Tweet.create(user=huey, content='meow') self.assertEqual(tweet.user.id, huey.id) self.assertEqual(tweet.user.username, 'huey') self.assertEqual(tweet.content, 'meow') self.assertTrue(isinstance(tweet.id, int)) self.assertTrue(tweet.id > 0) @requires_models(User) def test_bulk_create(self): users = [User(username='u%s' % i) for i in range(5)] self.assertEqual(User.select().count(), 0) with self.assertQueryCount(1): User.bulk_create(users) self.assertEqual(User.select().count(), 5) self.assertEqual([u.username for u in User.select().order_by(User.id)], ['u0', 'u1', 'u2', 'u3', 'u4']) if IS_POSTGRESQL: self.assertEqual([u.id for u in User.select().order_by(User.id)], [user.id for user in users]) @requires_models(User) def test_bulk_create_empty(self): self.assertEqual(User.select().count(), 0) User.bulk_create([]) @requires_models(User) def test_bulk_create_batching(self): users = [User(username=str(i)) for i in range(10)] with self.assertQueryCount(4): User.bulk_create(users, 3) self.assertEqual(User.select().count(), 10) self.assertEqual([u.username for u in User.select().order_by(User.id)], list('0123456789')) if IS_POSTGRESQL: self.assertEqual([u.id for u in User.select().order_by(User.id)], [user.id for user in users]) @requires_models(Person) def test_bulk_create_error(self): people = [Person(first='a', last='b'), Person(first='b', last='c'), Person(first='a', last='b')] with self.assertRaises(IntegrityError): with self.database.atomic(): Person.bulk_create(people) self.assertEqual(Person.select().count(), 0) @requires_models(CPK) def test_bulk_create_composite_key(self): self.assertEqual(CPK.select().count(), 0) items = [CPK(key='k1', value=1, extra=1), CPK(key='k2', value=2, extra=2)] CPK.bulk_create(items) self.assertEqual([(c.key, c.value, c.extra) for c in items], [('k1', 1, 1), ('k2', 2, 2)]) query = CPK.select().order_by(CPK.key).tuples() self.assertEqual(list(query), [('k1', 1, 1), ('k2', 2, 2)]) @requires_models(Person) def test_bulk_update(self): data = [('f%s' % i, 'l%s' % i, datetime.date(1980, i, i)) for i in range(1, 5)] Person.insert_many(data).execute() p1, p2, p3, p4 = list(Person.select().order_by(Person.id)) p1.first = 'f1-x' p1.last = 'l1-x' p2.first = 'f2-y' p3.last = 'l3-z' with self.assertQueryCount(1): n = Person.bulk_update([p1, p2, p3, p4], ['first', 'last']) self.assertEqual(n, 3 if IS_MYSQL else 4) query = Person.select().order_by(Person.id) self.assertEqual([(p.first, p.last) for p in query], [ ('f1-x', 'l1-x'), ('f2-y', 'l2'), ('f3', 'l3-z'), ('f4', 'l4')]) # Modify multiple fields, but only update "first". p1.first = 'f1-x2' p1.last = 'l1-x2' p2.first = 'f2-y2' p3.last = 'f3-z2' with self.assertQueryCount(2): # Two batches, so two queries. n = Person.bulk_update([p1, p2, p3, p4], [Person.first], 2) self.assertEqual(n, 2 if IS_MYSQL else 4) query = Person.select().order_by(Person.id) self.assertEqual([(p.first, p.last) for p in query], [ ('f1-x2', 'l1-x'), ('f2-y2', 'l2'), ('f3', 'l3-z'), ('f4', 'l4')]) @requires_models(User, Tweet) def test_bulk_update_foreign_key(self): for username in ('charlie', 'huey', 'zaizee'): user = User.create(username=username) for i in range(2): Tweet.create(user=user, content='%s-%s' % (username, i)) c, h, z = list(User.select().order_by(User.id)) c0, c1, h0, h1, z0, z1 = list(Tweet.select().order_by(Tweet.id)) c0.content = 'charlie-0x' c1.user = h h0.user = z h1.content = 'huey-1x' z0.user = c z0.content = 'zaizee-0x' with self.assertQueryCount(1): Tweet.bulk_update([c0, c1, h0, h1, z0, z1], ['user', 'content']) query = (Tweet .select(Tweet.content, User.username) .join(User) .order_by(Tweet.id) .objects()) self.assertEqual([(t.username, t.content) for t in query], [ ('charlie', 'charlie-0x'), ('huey', 'charlie-1'), ('zaizee', 'huey-0'), ('huey', 'huey-1x'), ('charlie', 'zaizee-0x'), ('zaizee', 'zaizee-1')]) @requires_models(Person) def test_bulk_update_integrityerror(self): people = [Person(first='f%s' % i, last='l%s' % i, dob='1980-01-01') for i in range(10)] Person.bulk_create(people) # Get list of people w/the IDs populated. They will not be set if the # underlying DB is Sqlite or MySQL. people = list(Person.select().order_by(Person.id)) # First we'll just modify all the first and last names. for person in people: person.first += '-x' person.last += '-x' # Now we'll introduce an issue that will cause an integrity error. p3, p7 = people[3], people[7] p3.first = p7.first = 'fx' p3.last = p7.last = 'lx' with self.assertRaises(IntegrityError): with self.assertQueryCount(1): with self.database.atomic(): Person.bulk_update(people, fields=['first', 'last']) with self.assertRaises(IntegrityError): # 10 objects, batch size=4, so 0-3, 4-7, 8&9. But we never get to 8 # and 9 because of the integrity error processing the 2nd batch. with self.assertQueryCount(2): with self.database.atomic(): Person.bulk_update(people, ['first', 'last'], 4) # Ensure no changes were made. vals = [(p.first, p.last) for p in Person.select().order_by(Person.id)] self.assertEqual(vals, [('f%s' % i, 'l%s' % i) for i in range(10)]) @requires_models(User, Tweet) def test_bulk_update_apply_dbvalue(self): u = User.create(username='u') t1, t2, t3 = [Tweet.create(user=u, content=str(i)) for i in (1, 2, 3)] # If we don't end up applying the field's db_value() to these timestamp # values, then we will end up with bad data or an error when attempting # to do the update. t1.timestamp = datetime.datetime(2019, 1, 2, 3, 4, 5) t2.timestamp = datetime.date(2019, 1, 3) t3.timestamp = 1337133700 # 2012-05-15T21:1:40. t3_dt = datetime.datetime.fromtimestamp(1337133700) Tweet.bulk_update([t1, t2, t3], fields=['timestamp']) # Ensure that the values were handled appropriately. t1, t2, t3 = list(Tweet.select().order_by(Tweet.id)) self.assertEqual(t1.timestamp, datetime.datetime(2019, 1, 2, 3, 4, 5)) self.assertEqual(t2.timestamp, datetime.datetime(2019, 1, 3, 0, 0, 0)) self.assertEqual(t3.timestamp, t3_dt) @skip_if(IS_SQLITE_OLD or IS_MYSQL or IS_CRDB) @requires_models(CPK) def test_bulk_update_cte(self): CPK.insert_many([('k1', 1, 1), ('k2', 2, 2), ('k3', 3, 3)]).execute() # We can also do a bulk-update using ValuesList when the primary-key of # the model is a composite-pk. new_values = [('k1', 1, 10), ('k3', 3, 30)] cte = ValuesList(new_values).cte('new_values', columns=('k', 'v', 'x')) # We have to use a subquery to update the individual column, as SQLite # does not support UPDATE/FROM syntax. subq = (cte .select(cte.c.x) .where(CPK._meta.primary_key == (cte.c.k, cte.c.v))) # Perform the update, assigning extra the new value from the values # list, and restricting the overall update using the composite pk. res = (CPK .update(extra=subq) .where(CPK._meta.primary_key.in_(cte.select(cte.c.k, cte.c.v))) .with_cte(cte) .execute()) self.assertEqual(list(sorted(CPK.select().tuples())), [ ('k1', 1, 10), ('k2', 2, 2), ('k3', 3, 30)]) @requires_models(User) def test_insert_rowcount(self): User.create(username='u0') # Ensure that last insert ID != rowcount. iq = User.insert_many([(u,) for u in ('u1', 'u2', 'u3')]) self.assertEqual(iq.as_rowcount().execute(), 3) # Now explicitly specify empty returning() for all DBs. iq = User.insert_many([(u,) for u in ('u4', 'u5')]).returning() self.assertEqual(iq.as_rowcount().execute(), 2) query = (User .select(User.username.concat('-x')) .where(User.username.in_(['u1', 'u2']))) iq = User.insert_from(query, ['username']) self.assertEqual(iq.as_rowcount().execute(), 2) query = (User .select(User.username.concat('-y')) .where(User.username.in_(['u3', 'u4']))) iq = User.insert_from(query, ['username']).returning() self.assertEqual(iq.as_rowcount().execute(), 2) query = User.insert({'username': 'u5'}) self.assertEqual(query.as_rowcount().execute(), 1) @skip_if(IS_POSTGRESQL or IS_CRDB, 'requires sqlite or mysql') @requires_models(Emp) def test_replace_rowcount(self): Emp.create(first='beanie', last='cat', empno='998') data = [ ('beanie', 'cat', '999'), ('mickey', 'dog', '123')] fields = (Emp.first, Emp.last, Emp.empno) # MySQL returns 3, Sqlite 2. However, older stdlib sqlite3 does not # work properly, so we don't assert a result count here. Emp.replace_many(data, fields=fields).execute() query = Emp.select(Emp.first, Emp.last, Emp.empno).order_by(Emp.last) self.assertEqual(list(query.tuples()), [ ('beanie', 'cat', '999'), ('mickey', 'dog', '123')]) @requires_models(User, Tweet) def test_get_shortcut(self): huey = self.add_user('huey') self.add_tweets(huey, 'meow', 'purr', 'wheeze') mickey = self.add_user('mickey') self.add_tweets(mickey, 'woof', 'yip') # Lookup using just the ID. huey_db = User.get(huey.id) self.assertEqual(huey.id, huey_db.id) # Lookup using an expression. huey_db = User.get(User.username == 'huey') self.assertEqual(huey.id, huey_db.id) mickey_db = User.get(User.username == 'mickey') self.assertEqual(mickey.id, mickey_db.id) self.assertEqual(User.get(username='mickey').id, mickey.id) # No results is an exception. self.assertRaises(User.DoesNotExist, User.get, User.username == 'x') # Multiple results is OK. tweet = Tweet.get(Tweet.user == huey_db) self.assertTrue(tweet.content in ('meow', 'purr', 'wheeze')) # We cannot traverse a join like this. @self.database.atomic() def has_error(): Tweet.get(User.username == 'huey') self.assertRaises(Exception, has_error) # This is OK, though. tweet = Tweet.get(user__username='mickey') self.assertTrue(tweet.content in ('woof', 'yip')) tweet = Tweet.get(content__ilike='w%', user__username__ilike='%ck%') self.assertEqual(tweet.content, 'woof') @requires_models(User) def test_get_with_alias(self): huey = self.add_user('huey') query = (User .select(User.username.alias('name')) .where(User.username == 'huey')) obj = query.dicts().get() self.assertEqual(obj, {'name': 'huey'}) obj = query.objects().get() self.assertEqual(obj.name, 'huey') @requires_models(User, Tweet) def test_get_or_none(self): huey = self.add_user('huey') self.assertEqual(User.get_or_none(User.username == 'huey').username, 'huey') self.assertIsNone(User.get_or_none(User.username == 'foo')) @requires_models(User, Tweet) def test_model_select_get_or_none(self): huey = self.add_user('huey') huey_db = User.select().where(User.username == 'huey').get_or_none() self.assertEqual(huey_db.username, 'huey') self.assertIsNone( User.select().where(User.username == 'foo').get_or_none()) @requires_models(User, Color) def test_get_by_id(self): huey = self.add_user('huey') self.assertEqual(User.get_by_id(huey.id).username, 'huey') Color.insert_many([ {'name': 'red', 'is_neutral': False}, {'name': 'blue', 'is_neutral': False}]).execute() self.assertEqual(Color.get_by_id('red').name, 'red') self.assertRaises(Color.DoesNotExist, Color.get_by_id, 'green') self.assertEqual(Color['red'].name, 'red') self.assertRaises(Color.DoesNotExist, lambda: Color['green']) @requires_models(User, Color) def test_get_set_item(self): huey = self.add_user('huey') huey_db = User[huey.id] self.assertEqual(huey_db.username, 'huey') User[huey.id] = {'username': 'huey-x'} huey_db = User[huey.id] self.assertEqual(huey_db.username, 'huey-x') del User[huey.id] self.assertEqual(len(User), 0) # Allow creation by specifying None for key. User[None] = {'username': 'zaizee'} User.get(User.username == 'zaizee') @requires_models(User) def test_get_or_create(self): huey, created = User.get_or_create(username='huey') self.assertTrue(created) huey2, created2 = User.get_or_create(username='huey') self.assertFalse(created2) self.assertEqual(huey.id, huey2.id) @requires_models(Category) def test_get_or_create_self_referential_fk(self): parent = Category.create(name='parent') child, created = Category.get_or_create(parent=parent, name='child') child_db = Category.get(Category.parent == parent) self.assertEqual(child_db.parent.name, 'parent') self.assertEqual(child_db.name, 'child') @requires_models(Person) def test_get_or_create_defaults(self): p, created = Person.get_or_create(first='huey', defaults={ 'last': 'cat', 'dob': datetime.date(2010, 7, 1)}) self.assertTrue(created) p_db = Person.get(Person.first == 'huey') self.assertEqual(p_db.first, 'huey') self.assertEqual(p_db.last, 'cat') self.assertEqual(p_db.dob, datetime.date(2010, 7, 1)) p2, created = Person.get_or_create(first='huey', defaults={ 'last': 'kitten', 'dob': datetime.date(2020, 1, 1)}) self.assertFalse(created) self.assertEqual(p2.first, 'huey') self.assertEqual(p2.last, 'cat') self.assertEqual(p2.dob, datetime.date(2010, 7, 1)) @requires_models(Person) def test_save(self): huey = Person(first='huey', last='cat', dob=datetime.date(2010, 7, 1)) self.assertTrue(huey.save() > 0) self.assertTrue(huey.id is not None) # Ensure PK is set. orig_id = huey.id # Test initial save (INSERT) worked and data is all present. huey_db = Person.get(first='huey', last='cat') self.assertEqual(huey_db.id, huey.id) self.assertEqual(huey_db.first, 'huey') self.assertEqual(huey_db.last, 'cat') self.assertEqual(huey_db.dob, datetime.date(2010, 7, 1)) # Make a change and do a second save (UPDATE). huey.dob = datetime.date(2010, 7, 2) self.assertTrue(huey.save() > 0) self.assertEqual(huey.id, orig_id) # Test UPDATE worked correctly. huey_db = Person.get(first='huey', last='cat') self.assertEqual(huey_db.id, huey.id) self.assertEqual(huey_db.first, 'huey') self.assertEqual(huey_db.last, 'cat') self.assertEqual(huey_db.dob, datetime.date(2010, 7, 2)) self.assertEqual(Person.select().count(), 1) @requires_models(Person) def test_save_only(self): huey = Person(first='huey', last='cat', dob=datetime.date(2010, 7, 1)) huey.save() huey.first = 'huker' huey.last = 'kitten' self.assertTrue(huey.save(only=('first',)) > 0) huey_db = Person.get_by_id(huey.id) self.assertEqual(huey_db.first, 'huker') self.assertEqual(huey_db.last, 'cat') self.assertEqual(huey_db.dob, datetime.date(2010, 7, 1)) huey.first = 'hubie' self.assertTrue(huey.save(only=[Person.last]) > 0) huey_db = Person.get_by_id(huey.id) self.assertEqual(huey_db.first, 'huker') self.assertEqual(huey_db.last, 'kitten') self.assertEqual(huey_db.dob, datetime.date(2010, 7, 1)) self.assertEqual(Person.select().count(), 1) @requires_models(Color, User) def test_save_force(self): huey = User(username='huey') self.assertTrue(huey.save() > 0) huey_id = huey.id huey.username = 'zaizee' self.assertTrue(huey.save(force_insert=True, only=('username',)) > 0) zaizee_id = huey.id self.assertTrue(huey_id != zaizee_id) query = User.select().order_by(User.username) self.assertEqual([user.username for user in query], ['huey', 'zaizee']) color = Color(name='red') self.assertFalse(bool(color.save())) self.assertEqual(Color.select().count(), 0) color = Color(name='blue') color.save(force_insert=True) self.assertEqual(Color.select().count(), 1) with self.database.atomic(): self.assertRaises(IntegrityError, color.save, force_insert=True) @requires_models(User, Tweet) def test_populate_unsaved_relations(self): user = User(username='charlie') tweet = Tweet(user=user, content='foo') self.assertTrue(user.save()) self.assertTrue(user.id is not None) with self.assertQueryCount(1): self.assertEqual(tweet.user_id, user.id) self.assertTrue(tweet.save()) self.assertEqual(tweet.user_id, user.id) tweet_db = Tweet.get(Tweet.content == 'foo') self.assertEqual(tweet_db.user.username, 'charlie') @requires_models(User, Tweet) def test_model_select(self): huey = self.add_user('huey') mickey = self.add_user('mickey') zaizee = self.add_user('zaizee') self.add_tweets(huey, 'meow', 'hiss', 'purr') self.add_tweets(mickey, 'woof', 'whine') with self.assertQueryCount(1): query = (Tweet .select(Tweet.content, User.username) .join(User) .order_by(User.username, Tweet.content)) self.assertSQL(query, ( 'SELECT "t1"."content", "t2"."username" ' 'FROM "tweet" AS "t1" ' 'INNER JOIN "users" AS "t2" ' 'ON ("t1"."user_id" = "t2"."id") ' 'ORDER BY "t2"."username", "t1"."content"'), []) tweets = list(query) self.assertEqual([(t.content, t.user.username) for t in tweets], [ ('hiss', 'huey'), ('meow', 'huey'), ('purr', 'huey'), ('whine', 'mickey'), ('woof', 'mickey')]) @requires_pglike @requires_models(User, Tweet) def test_distinct_on(self): u1, u2 = self.add_user('u1'), self.add_user('u2') self.add_tweets(u1, 'u1-t1', 'u1-t2', 'u1-t3') self.add_tweets(u2, 'u2-t1') query = (Tweet .select(Tweet.user, Tweet.content) .join(User) .distinct(Tweet.user) .order_by(Tweet.user, Tweet.timestamp)) self.assertEqual([(t.user_id, t.content) for t in query], [(u1.id, 'u1-t1'), (u2.id, 'u2-t1')]) @requires_models(User, Tweet, Favorite) def test_join_two_fks(self): with self.database.atomic(): huey = self.add_user('huey') mickey = self.add_user('mickey') h_m, h_p, h_h = self.add_tweets(huey, 'meow', 'purr', 'hiss') m_w, m_b = self.add_tweets(mickey, 'woof', 'bark') Favorite.create(user=huey, tweet=m_w) Favorite.create(user=mickey, tweet=h_m) Favorite.create(user=mickey, tweet=h_p) with self.assertQueryCount(1): UA = User.alias() query = (Favorite .select(Favorite, Tweet, User, UA) .join(Tweet) .join(User) .switch(Favorite) .join(UA, on=Favorite.user) .order_by(Favorite.id)) accum = [(f.tweet.user.username, f.tweet.content, f.user.username) for f in query] self.assertEqual(accum, [ ('mickey', 'woof', 'huey'), ('huey', 'meow', 'mickey'), ('huey', 'purr', 'mickey')]) with self.assertQueryCount(5): # Test intermediate models not selected. query = (Favorite .select() .join(Tweet) .switch(Favorite) .join(User) .where(User.username == 'mickey') .order_by(Favorite.id)) accum = [(f.user.username, f.tweet.content) for f in query] self.assertEqual(accum, [('mickey', 'meow'), ('mickey', 'purr')]) @requires_models(A, B, C) def test_join_issue_1482(self): a1 = A.create(a='a1') b1 = B.create(a=a1, b='b1') c1 = C.create(b=b1, c='c1') with self.assertQueryCount(3): query = C.select().join(B).join(A).where(A.a == 'a1') accum = [(c.c, c.b.b, c.b.a.a) for c in query] self.assertEqual(accum, [('c1', 'b1', 'a1')]) @requires_models(A, B, C) def test_join_empty_intermediate_model(self): a1 = A.create(a='a1') a2 = A.create(a='a2') b11 = B.create(a=a1, b='b11') b12 = B.create(a=a1, b='b12') b21 = B.create(a=a2, b='b21') c111 = C.create(b=b11, c='c111') c112 = C.create(b=b11, c='c112') c211 = C.create(b=b21, c='c211') with self.assertQueryCount(1): query = C.select(C, A.a).join(B).join(A).order_by(C.c) accum = [(c.c, c.b.a.a) for c in query] self.assertEqual(accum, [ ('c111', 'a1'), ('c112', 'a1'), ('c211', 'a2')]) with self.assertQueryCount(1): query = C.select(C, B, A).join(B).join(A).order_by(C.c) accum = [(c.c, c.b.b, c.b.a.a) for c in query] self.assertEqual(accum, [ ('c111', 'b11', 'a1'), ('c112', 'b11', 'a1'), ('c211', 'b21', 'a2')]) @requires_models(City, Venue, Event) def test_join_empty_relations(self): with self.database.atomic(): city = City.create(name='Topeka') venue1 = Venue.create(name='House', city=city, city_n=city) venue2 = Venue.create(name='Nowhere', city=city, city_n=None) event1 = Event.create(name='House Party', venue=venue1) event2 = Event.create(name='Holiday') event3 = Event.create(name='Nowhere Party', venue=venue2) with self.assertQueryCount(1): query = (Event .select(Event, Venue, City) .join(Venue, JOIN.LEFT_OUTER) .join(City, JOIN.LEFT_OUTER, on=Venue.city) .order_by(Event.id)) # Here we have two left-outer joins, and the second Event # ("Holiday"), does not have an associated Venue (hence, no City). # Peewee would attach an empty Venue() model to the event, however. # It did this since we are selecting from Venue/City and Venue is # an intermediary model. It is more correct for Event.venue to be # None in this case. This is now patched / fixed. r = [(e.name, e.venue is not None and e.venue.city.name or None) for e in query] self.assertEqual(r, [ ('House Party', 'Topeka'), ('Holiday', None), ('Nowhere Party', 'Topeka')]) with self.assertQueryCount(1): query = (Event .select(Event, Venue, City) .join(Venue, JOIN.INNER) .join(City, JOIN.LEFT_OUTER, on=Venue.city_n) .order_by(Event.id)) # Here we have an inner join and a left-outer join. The furthest # object (City) will be NULL for the "Nowhere Party". Make sure # that the object is left as None and not populated with an empty # City instance. accum = [] for event in query: city_name = event.venue.city_n and event.venue.city_n.name accum.append((event.name, event.venue.name, city_name)) self.assertEqual(accum, [ ('House Party', 'House', 'Topeka'), ('Nowhere Party', 'Nowhere', None)]) @requires_models(Relationship, Person) def test_join_same_model_twice(self): d = datetime.date(2010, 1, 1) huey = Person.create(first='huey', last='cat', dob=d) zaizee = Person.create(first='zaizee', last='cat', dob=d) mickey = Person.create(first='mickey', last='dog', dob=d) relationships = ( (huey, zaizee), (zaizee, huey), (mickey, huey), ) for src, dest in relationships: Relationship.create(from_person=src, to_person=dest) PA = Person.alias() with self.assertQueryCount(1): query = (Relationship .select(Relationship, Person, PA) .join(Person, on=Relationship.from_person) .switch(Relationship) .join(PA, on=Relationship.to_person) .order_by(Relationship.id)) results = [(r.from_person.first, r.to_person.first) for r in query] self.assertEqual(results, [ ('huey', 'zaizee'), ('zaizee', 'huey'), ('mickey', 'huey')]) @requires_models(User, Tweet) def test_join_to_dict(self): huey = self.add_user('huey') mickey = self.add_user('mickey') self.add_tweets(huey, 'meow', 'hiss', 'purr') self.add_tweets(mickey, 'woof') with self.assertQueryCount(1): q = Select((User,), (User.id, User.username,)) query = (Tweet .select(Tweet.content, q.c.username) .join(q, on=(Tweet.user == q.c.id), attr='u') .order_by(q.c.username, Tweet.content)) self.assertSQL(query, ( 'SELECT "t1"."content", "t2"."username" FROM "tweet" AS "t1" ' 'INNER JOIN (SELECT "t3"."id", "t3"."username" FROM "users" ' 'AS "t3") AS "t2" ON ("t1"."user_id" = "t2"."id") ' 'ORDER BY "t2"."username", "t1"."content"'), []) tweets = list(query) self.assertEqual([(t.content, t.u) for t in tweets], [ ('hiss', {'username': 'huey'}), ('meow', {'username': 'huey'}), ('purr', {'username': 'huey'}), ('woof', {'username': 'mickey'})]) @requires_models(User) def test_peek(self): for username in ('huey', 'mickey', 'zaizee'): self.add_user(username) with self.assertQueryCount(1): query = User.select(User.username).order_by(User.username).dicts() self.assertEqual(query.peek(n=1), {'username': 'huey'}) self.assertEqual(query.peek(n=2), [{'username': 'huey'}, {'username': 'mickey'}]) @requires_models(User) def test_first(self): for u in 'abc': self.add_user(u) # Multiple calls to first() do not result in multiple executions. with self.assertQueryCount(1): q = User.select().order_by(User.username) self.assertEqual(q.first().username, 'a') self.assertEqual(q.first().username, 'a') @requires_models(User, Tweet, Favorite) def test_multi_join(self): u1 = User.create(username='u1') u2 = User.create(username='u2') u3 = User.create(username='u3') t1_1 = Tweet.create(user=u1, content='t1-1') t1_2 = Tweet.create(user=u1, content='t1-2') t2_1 = Tweet.create(user=u2, content='t2-1') t2_2 = Tweet.create(user=u2, content='t2-2') favorites = ((u1, t2_1), (u1, t2_2), (u2, t1_1), (u3, t1_2), (u3, t2_2)) for user, tweet in favorites: Favorite.create(user=user, tweet=tweet) TweetUser = User.alias('u2') with self.assertQueryCount(1): query = (Favorite .select(Favorite.id, Tweet.content, User.username, TweetUser.username) .join(Tweet) .join(TweetUser, on=(Tweet.user == TweetUser.id)) .switch(Favorite) .join(User) .order_by(Tweet.content, Favorite.id)) self.assertSQL(query, ( 'SELECT ' '"t1"."id", "t2"."content", "t3"."username", "u2"."username" ' 'FROM "favorite" AS "t1" ' 'INNER JOIN "tweet" AS "t2" ON ("t1"."tweet_id" = "t2"."id") ' 'INNER JOIN "users" AS "u2" ON ("t2"."user_id" = "u2"."id") ' 'INNER JOIN "users" AS "t3" ON ("t1"."user_id" = "t3"."id") ' 'ORDER BY "t2"."content", "t1"."id"'), []) accum = [(f.tweet.user.username, f.tweet.content, f.user.username) for f in query] self.assertEqual(accum, [ ('u1', 't1-1', 'u2'), ('u1', 't1-2', 'u3'), ('u2', 't2-1', 'u1'), ('u2', 't2-2', 'u1'), ('u2', 't2-2', 'u3')]) res = query.count() self.assertEqual(res, 5) def _create_user_tweets(self): data = (('huey', ('meow', 'purr', 'hiss')), ('zaizee', ()), ('mickey', ('woof', 'grr'))) with self.database.atomic(): ts = int(time.time()) for username, tweets in data: user = User.create(username=username) for tweet in tweets: Tweet.create(user=user, content=tweet, timestamp=ts) ts += 1 @requires_models(User, Tweet) def test_join_subquery(self): self._create_user_tweets() # Select note user and timestamp of most recent tweet. with self.assertQueryCount(1): TA = Tweet.alias() max_q = (TA .select(TA.user, fn.MAX(TA.timestamp).alias('max_ts')) .group_by(TA.user) .alias('max_q')) predicate = ((Tweet.user == max_q.c.user_id) & (Tweet.timestamp == max_q.c.max_ts)) latest = (Tweet .select(Tweet.user, Tweet.content, Tweet.timestamp) .join(max_q, on=predicate) .alias('latest')) query = (User .select(User, latest.c.content, latest.c.timestamp) .join(latest, on=(User.id == latest.c.user_id))) data = [(user.username, user.tweet.content) for user in query] # Failing on travis-ci...old SQLite? if not IS_SQLITE_OLD: self.assertEqual(data, [ ('huey', 'hiss'), ('mickey', 'grr')]) with self.assertQueryCount(1): query = (Tweet .select(Tweet, User) .join(max_q, on=predicate) .switch(Tweet) .join(User)) data = [(note.user.username, note.content) for note in query] self.assertEqual(data, [ ('huey', 'hiss'), ('mickey', 'grr')]) @requires_models(User, Tweet) def test_join_subquery_2(self): self._create_user_tweets() with self.assertQueryCount(1): users = (User .select(User.id, User.username) .where(User.username.in_(['huey', 'zaizee']))) query = (Tweet .select(Tweet.content.alias('content'), users.c.username.alias('username')) .join(users, on=(Tweet.user == users.c.id)) .order_by(Tweet.id)) self.assertSQL(query, ( 'SELECT "t1"."content" AS "content", ' '"t2"."username" AS "username"' ' FROM "tweet" AS "t1" ' 'INNER JOIN (SELECT "t3"."id", "t3"."username" ' 'FROM "users" AS "t3" ' 'WHERE ("t3"."username" IN (?, ?))) AS "t2" ' 'ON ("t1"."user_id" = "t2"."id") ' 'ORDER BY "t1"."id"'), ['huey', 'zaizee']) results = [(t.content, t.user.username) for t in query] self.assertEqual(results, [ ('meow', 'huey'), ('purr', 'huey'), ('hiss', 'huey')]) @skip_if(IS_SQLITE_OLD or (IS_MYSQL and not IS_MYSQL_ADVANCED_FEATURES)) @requires_models(User, Tweet) def test_join_subquery_cte(self): self._create_user_tweets() cte = (User .select(User.id, User.username) .where(User.username.in_(['huey', 'zaizee']))\ .cte('cats')) with self.assertQueryCount(1): # Attempt join with subquery as common-table expression. query = (Tweet .select(Tweet.content, cte.c.username) .join(cte, on=(Tweet.user == cte.c.id)) .order_by(Tweet.id) .with_cte(cte)) self.assertSQL(query, ( 'WITH "cats" AS (' 'SELECT "t1"."id", "t1"."username" FROM "users" AS "t1" ' 'WHERE ("t1"."username" IN (?, ?))) ' 'SELECT "t2"."content", "cats"."username" FROM "tweet" AS "t2" ' 'INNER JOIN "cats" ON ("t2"."user_id" = "cats"."id") ' 'ORDER BY "t2"."id"'), ['huey', 'zaizee']) self.assertEqual([t.content for t in query], ['meow', 'purr', 'hiss']) @skip_if(IS_MYSQL) # MariaDB does not support LIMIT in subqueries! @requires_models(User) def test_subquery_emulate_window(self): # We have duplicated users. Select a maximum of 2 instances of the # username. name2count = { 'beanie': 6, 'huey': 5, 'mickey': 3, 'pipey': 1, 'zaizee': 4} names = [] for name, count in sorted(name2count.items()): names += [name] * count User.insert_many([(i, n) for i, n in enumerate(names, 1)], [User.id, User.username]).execute() # The results we are trying to obtain. expected = [ ('beanie', 1), ('beanie', 2), ('huey', 7), ('huey', 8), ('mickey', 12), ('mickey', 13), ('pipey', 15), ('zaizee', 16), ('zaizee', 17)] with self.assertQueryCount(1): # Using a self-join. UA = User.alias() query = (User .select(User.username, UA.id) .join(UA, on=((UA.username == User.username) & (UA.id >= User.id))) .group_by(User.username, UA.id) .having(fn.COUNT(UA.id) < 3) .order_by(User.username, UA.id)) self.assertEqual(query.tuples()[:], expected) with self.assertQueryCount(1): # Using a correlated subquery. subq = (UA .select(UA.id) .where(User.username == UA.username) .order_by(UA.id) .limit(2)) query = (User .select(User.username, User.id) .where(User.id.in_(subq.alias('subq'))) .order_by(User.username, User.id)) self.assertEqual(query.tuples()[:], expected) @requires_models(User, Tweet) def test_subquery_alias_selection(self): data = ( ('huey', ('meow', 'hiss', 'purr')), ('mickey', ('woof', 'bark')), ('zaizee', ())) with self.database.atomic(): for username, tweets in data: user = User.create(username=username) for tweet in tweets: Tweet.create(user=user, content=tweet) with self.assertQueryCount(1): subq = (Tweet .select(fn.COUNT(Tweet.id)) .where(Tweet.user == User.id)) query = (User .select(User.username, subq.alias('tweet_count')) .order_by(User.id)) self.assertEqual([(u.username, u.tweet_count) for u in query], [ ('huey', 3), ('mickey', 2), ('zaizee', 0)]) @requires_pglike @requires_models(User) def test_join_on_valueslist(self): for username in ('huey', 'mickey', 'zaizee'): User.create(username=username) vl = ValuesList([('huey',), ('zaizee',)], columns=['username']) with self.assertQueryCount(1): query = (User .select(vl.c.username) .join(vl, on=(User.username == vl.c.username)) .order_by(vl.c.username.desc())) self.assertEqual([u.username for u in query], ['zaizee', 'huey']) @skip_if(IS_SQLITE_OLD or IS_MYSQL or IS_CRDB) @requires_models(User) def test_multi_update(self): data = [(i, 'u%s' % i) for i in range(1, 4)] User.insert_many(data, fields=[User.id, User.username]).execute() data = [(i, 'u%sx' % i) for i in range(1, 3)] vl = ValuesList(data) cte = vl.select().cte('uv', columns=('id', 'username')) subq = cte.select(cte.c.username).where(cte.c.id == User.id) res = (User .update(username=subq) .where(User.id.in_(cte.select(cte.c.id))) .with_cte(cte) .execute()) query = User.select().order_by(User.id) self.assertEqual([(u.id, u.username) for u in query], [ (1, 'u1x'), (2, 'u2x'), (3, 'u3')]) @requires_models(User, Tweet) def test_insert_query_value(self): huey = self.add_user('huey') query = User.select(User.id).where(User.username == 'huey') tid = Tweet.insert(content='meow', user=query).execute() tweet = Tweet[tid] self.assertEqual(tweet.user.id, huey.id) self.assertEqual(tweet.user.username, 'huey') @skip_if(IS_SQLITE and not IS_SQLITE_9, 'requires sqlite >= 3.9') @requires_models(Register) def test_compound_select(self): for i in range(10): Register.create(value=i) q1 = Register.select().where(Register.value < 2) q2 = Register.select().where(Register.value > 7) c1 = (q1 | q2).order_by(SQL('2')) self.assertSQL(c1, ( 'SELECT "t1"."id", "t1"."value" FROM "register" AS "t1" ' 'WHERE ("t1"."value" < ?) UNION ' 'SELECT "t2"."id", "t2"."value" FROM "register" AS "t2" ' 'WHERE ("t2"."value" > ?) ORDER BY 2'), [2, 7]) self.assertEqual([row.value for row in c1], [0, 1, 8, 9], [row.__data__ for row in c1]) self.assertEqual(c1.count(), 4) q3 = Register.select().where(Register.value == 5) c2 = (c1.order_by() | q3).order_by(SQL('2')) self.assertSQL(c2, ( 'SELECT "t1"."id", "t1"."value" FROM "register" AS "t1" ' 'WHERE ("t1"."value" < ?) UNION ' 'SELECT "t2"."id", "t2"."value" FROM "register" AS "t2" ' 'WHERE ("t2"."value" > ?) UNION ' 'SELECT "t3"."id", "t3"."value" FROM "register" AS "t3" ' 'WHERE ("t3"."value" = ?) ORDER BY 2'), [2, 7, 5]) self.assertEqual([row.value for row in c2], [0, 1, 5, 8, 9]) self.assertEqual(c2.count(), 5) @requires_models(User, Tweet) def test_union_column_resolution(self): u1 = User.create(id=1, username='u1') u2 = User.create(id=2, username='u2') q1 = User.select().where(User.id == 1) q2 = User.select() union = q1 | q2 self.assertSQL(union, ( 'SELECT "t1"."id", "t1"."username" FROM "users" AS "t1" ' 'WHERE ("t1"."id" = ?) ' 'UNION ' 'SELECT "t2"."id", "t2"."username" FROM "users" AS "t2"'), [1]) results = [(user.id, user.username) for user in union] self.assertEqual(sorted(results), [ (1, 'u1'), (2, 'u2')]) t1_1 = Tweet.create(id=1, user=u1, content='u1-t1') t1_2 = Tweet.create(id=2, user=u1, content='u1-t2') t2_1 = Tweet.create(id=3, user=u2, content='u2-t1') with self.assertQueryCount(1): q1 = Tweet.select(Tweet, User).join(User).where(User.id == 1) q2 = Tweet.select(Tweet, User).join(User) union = q1 | q2 self.assertSQL(union, ( 'SELECT "t1"."id", "t1"."user_id", "t1"."content", ' '"t1"."timestamp", "t2"."id", "t2"."username" ' 'FROM "tweet" AS "t1" ' 'INNER JOIN "users" AS "t2" ON ("t1"."user_id" = "t2"."id") ' 'WHERE ("t2"."id" = ?) ' 'UNION ' 'SELECT "t3"."id", "t3"."user_id", "t3"."content", ' '"t3"."timestamp", "t4"."id", "t4"."username" ' 'FROM "tweet" AS "t3" ' 'INNER JOIN "users" AS "t4" ON ("t3"."user_id" = "t4"."id")'), [1]) results = [(t.id, t.content, t.user.username) for t in union] self.assertEqual(sorted(results), [ (1, 'u1-t1', 'u1'), (2, 'u1-t2', 'u1'), (3, 'u2-t1', 'u2')]) with self.assertQueryCount(1): union_flat = (q1 | q2).objects() results = list(results) results = [(t.id, t.content, t.username, t.id_2) for t in union_flat] self.assertEqual(sorted(results), [ (1, 'u1-t1', 'u1', 1), (2, 'u1-t2', 'u1', 1), (3, 'u2-t1', 'u2', 2)]) @requires_models(User, Tweet) def test_compound_select_as_subquery(self): with self.database.atomic(): for i in range(5): user = User.create(username='u%s' % i) for j in range(i * 2): Tweet.create(user=user, content='t%s-%s' % (i, j)) q1 = (Tweet .select(Tweet.id, Tweet.content, User.username) .join(User) .where(User.username == 'u3')) q2 = (Tweet .select(Tweet.id, Tweet.content, User.username) .join(User) .where(User.username.in_(['u2', 'u4']))) union = (q1 | q2) q = (union .select_from(union.c.username, fn.COUNT(union.c.id).alias('ct')) .group_by(union.c.username) .order_by(fn.COUNT(union.c.id).desc()) .dicts()) self.assertEqual(list(q), [ {'username': 'u4', 'ct': 8}, {'username': 'u3', 'ct': 6}, {'username': 'u2', 'ct': 4}]) @requires_models(User, Tweet) def test_union_with_join(self): u1, u2 = [User.create(username='u%s' % i) for i in (1, 2)] for u, ts in ((u1, ('t1', 't2')), (u2, ('t1',))): for t in ts: Tweet.create(user=u, content='%s-%s' % (u.username, t)) with self.assertQueryCount(1): q1 = (User .select(User, Tweet) .join(Tweet, on=(Tweet.user == User.id).alias('foo'))) q2 = (User .select(User, Tweet) .join(Tweet, on=(Tweet.user == User.id).alias('foo'))) self.assertEqual( sorted([(user.username, user.foo.content) for user in q1]), [('u1', 'u1-t1'), ('u1', 'u1-t2'), ('u2', 'u2-t1')]) with self.assertQueryCount(1): uq = q1.union_all(q2) result = [(user.username, user.foo.content) for user in uq] self.assertEqual(sorted(result), [ ('u1', 'u1-t1'), ('u1', 'u1-t1'), ('u1', 'u1-t2'), ('u1', 'u1-t2'), ('u2', 'u2-t1'), ('u2', 'u2-t1'), ]) @skip_if(IS_SQLITE_OLD or (IS_MYSQL and not IS_MYSQL_ADVANCED_FEATURES)) @requires_models(User) def test_union_cte(self): with self.database.atomic(): (User .insert_many({'username': 'u%s' % i} for i in range(10)) .execute()) lhs = User.select().where(User.username.in_(['u1', 'u3'])) rhs = User.select().where(User.username.in_(['u5', 'u7'])) u_cte = (lhs | rhs).cte('users_union') query = (User .select(User.username) .join(u_cte, on=(User.id == u_cte.c.id)) .where(User.username.in_(['u1', 'u7'])) .with_cte(u_cte)) self.assertEqual(sorted([u.username for u in query]), ['u1', 'u7']) @requires_models(Category) def test_self_referential_fk(self): self.assertTrue(Category.parent.rel_model is Category) root = Category.create(name='root') c1 = Category.create(parent=root, name='child-1') c2 = Category.create(parent=root, name='child-2') with self.assertQueryCount(1): Parent = Category.alias('p') query = (Category .select( Parent.name, Category.name) .where(Category.parent == root) .order_by(Category.name)) query = query.join(Parent, on=(Category.parent == Parent.name)) c1_db, c2_db = list(query) self.assertEqual(c1_db.name, 'child-1') self.assertEqual(c1_db.parent.name, 'root') self.assertEqual(c2_db.name, 'child-2') self.assertEqual(c2_db.parent.name, 'root') @requires_models(Category) def test_empty_joined_instance(self): root = Category.create(name='a') c1 = Category.create(name='c1', parent=root) c2 = Category.create(name='c2', parent=root) with self.assertQueryCount(1): Parent = Category.alias('p') query = (Category .select(Category, Parent) .join(Parent, JOIN.LEFT_OUTER, on=(Category.parent == Parent.name)) .order_by(Category.name)) result = [(category.name, category.parent is None) for category in query] self.assertEqual(result, [('a', True), ('c1', False), ('c2', False)]) @requires_models(User, Tweet) def test_from_multi_table(self): self.add_tweets(self.add_user('huey'), 'meow', 'hiss', 'purr') self.add_tweets(self.add_user('mickey'), 'woof', 'wheeze') with self.assertQueryCount(1): query = (Tweet .select(Tweet, User) .from_(Tweet, User) .where( (Tweet.user == User.id) & (User.username == 'huey')) .order_by(Tweet.id) .dicts()) self.assertEqual([t['content'] for t in query], ['meow', 'hiss', 'purr']) self.assertEqual([t['username'] for t in query], ['huey', 'huey', 'huey']) @requires_models(Point) def test_subquery_in_select_expression(self): for x, y in ((1, 1), (1, 2), (10, 10), (10, 20)): Point.create(x=x, y=y) with self.assertQueryCount(1): PA = Point.alias('pa') subq = PA.select(fn.SUM(PA.y)).where(PA.x == Point.x) query = (Point .select(Point.x, Point.y, subq.alias('sy')) .order_by(Point.x, Point.y)) self.assertEqual(list(query.tuples()), [ (1, 1, 3), (1, 2, 3), (10, 10, 30), (10, 20, 30)]) with self.assertQueryCount(1): query = (Point .select(Point.x, (Point.y + subq).alias('sy')) .order_by(Point.x, Point.y)) self.assertEqual(list(query.tuples()), [ (1, 4), (1, 5), (10, 40), (10, 50)]) @requires_models(User, Tweet) def test_filtering(self): with self.database.atomic(): huey = self.add_user('huey') mickey = self.add_user('mickey') self.add_tweets(huey, 'meow', 'hiss', 'purr') self.add_tweets(mickey, 'woof', 'wheeze') with self.assertQueryCount(1): query = Tweet.filter(user__username='huey').order_by(Tweet.content) self.assertEqual([row.content for row in query], ['hiss', 'meow', 'purr']) with self.assertQueryCount(1): query = User.filter(tweets__content__ilike='w%') self.assertEqual([user.username for user in query], ['mickey', 'mickey']) def test_deferred_fk(self): class Note(TestModel): foo = DeferredForeignKey('Foo', backref='notes') class Foo(TestModel): note = ForeignKeyField(Note) self.assertTrue(Note.foo.rel_model is Foo) self.assertTrue(Foo.note.rel_model is Note) f = Foo(id=1337) self.assertSQL(f.notes, ( 'SELECT "t1"."id", "t1"."foo_id" FROM "note" AS "t1" ' 'WHERE ("t1"."foo_id" = ?)'), [1337]) def test_deferred_fk_dependency_graph(self): class AUser(TestModel): foo = DeferredForeignKey('Tweet') class ZTweet(TestModel): user = ForeignKeyField(AUser, backref='ztweets') self.assertEqual(sort_models([AUser, ZTweet]), [AUser, ZTweet]) def test_table_schema(self): class Schema(TestModel): pass self.assertTrue(Schema._meta.schema is None) self.assertSQL(Schema.select(), ( 'SELECT "t1"."id" FROM "schema" AS "t1"'), []) Schema._meta.schema = 'test' self.assertSQL(Schema.select(), ( 'SELECT "t1"."id" FROM "test"."schema" AS "t1"'), []) Schema._meta.schema = 'another' self.assertSQL(Schema.select(), ( 'SELECT "t1"."id" FROM "another"."schema" AS "t1"'), []) @requires_models(User) def test_noop(self): query = User.noop() self.assertEqual(list(query), []) @requires_models(User) def test_iteration(self): self.assertEqual(list(User), []) self.assertEqual(len(User), 0) self.assertTrue(User) User.insert_many((['charlie'], ['huey']), [User.username]).execute() self.assertEqual(sorted(u.username for u in User), ['charlie', 'huey']) self.assertEqual(len(User), 2) self.assertTrue(User) @requires_models(User) def test_iterator(self): users = ['charlie', 'huey', 'zaizee'] with self.database.atomic(): for username in users: User.create(username=username) with self.assertQueryCount(1): query = User.select().order_by(User.username).iterator() self.assertEqual([u.username for u in query], users) self.assertEqual(list(query), []) @requires_models(User) def test_select_count(self): users = [self.add_user(u) for u in ('huey', 'charlie', 'mickey')] self.assertEqual(User.select().count(), 3) qr = User.select().execute() self.assertEqual(qr.count, 0) list(qr) self.assertEqual(qr.count, 3) @requires_models(User) def test_batch_commit(self): commit_method = self.database.commit def assertBatch(n_rows, batch_size, n_commits): User.delete().execute() user_data = [{'username': 'u%s' % i} for i in range(n_rows)] with mock.patch.object(self.database, 'commit') as mock_commit: mock_commit.side_effect = commit_method for row in self.database.batch_commit(user_data, batch_size): User.create(**row) self.assertEqual(mock_commit.call_count, n_commits) self.assertEqual(User.select().count(), n_rows) assertBatch(6, 1, 6) assertBatch(6, 2, 3) assertBatch(6, 3, 2) assertBatch(6, 4, 2) assertBatch(6, 6, 1) assertBatch(6, 7, 1) class TestRaw(ModelTestCase): database = get_in_memory_db() requires = [User] def test_raw(self): with self.database.atomic(): for username in ('charlie', 'chuck', 'huey', 'zaizee'): User.create(username=username) query = (User .raw('SELECT username, SUBSTR(username, 1, 1) AS first ' 'FROM users ' 'WHERE SUBSTR(username, 1, 1) = ? ' 'ORDER BY username DESC', 'c')) self.assertEqual([(row.username, row.first) for row in query], [('chuck', 'c'), ('charlie', 'c')]) def test_raw_iterator(self): (User .insert_many([('charlie',), ('huey',)], fields=[User.username]) .execute()) with self.assertQueryCount(1): query = User.raw('SELECT * FROM users ORDER BY id') results = [user.username for user in query.iterator()] self.assertEqual(results, ['charlie', 'huey']) # Since we used iterator(), the results were not cached. self.assertEqual([u.username for u in query], []) class TestDeleteInstance(ModelTestCase): database = get_in_memory_db() requires = [User, Account, Tweet, Favorite] def setUp(self): super(TestDeleteInstance, self).setUp() with self.database.atomic(): huey = User.create(username='huey') acct = Account.create(user=huey, email='huey@meow.com') for content in ('meow', 'purr'): Tweet.create(user=huey, content=content) mickey = User.create(username='mickey') woof = Tweet.create(user=mickey, content='woof') Favorite.create(user=huey, tweet=woof) Favorite.create(user=mickey, tweet=Tweet.create(user=huey, content='hiss')) def test_delete_instance_recursive(self): huey = User.get(User.username == 'huey') a = [] for d in huey.dependencies(): a.append(d) with self.assertQueryCount(5): huey.delete_instance(recursive=True) self.assertHistory(5, [ ('DELETE FROM "favorite" WHERE ("favorite"."user_id" = ?)', [huey.id]), ('DELETE FROM "favorite" WHERE (' '"favorite"."tweet_id" IN (' 'SELECT "t1"."id" FROM "tweet" AS "t1" WHERE (' '"t1"."user_id" = ?)))', [huey.id]), ('DELETE FROM "tweet" WHERE ("tweet"."user_id" = ?)', [huey.id]), ('UPDATE "account" SET "user_id" = ? ' 'WHERE ("account"."user_id" = ?)', [None, huey.id]), ('DELETE FROM "users" WHERE ("users"."id" = ?)', [huey.id]), ]) # Only one user left. self.assertEqual(User.select().count(), 1) # Huey's account has had the FK cleared out. acct = Account.get(Account.email == 'huey@meow.com') self.assertTrue(acct.user is None) # Huey owned a favorite and one of huey's tweets was the other fav. self.assertEqual(Favorite.select().count(), 0) # The only tweet left is mickey's. self.assertEqual(Tweet.select().count(), 1) tweet = Tweet.get() self.assertEqual(tweet.content, 'woof') def test_delete_nullable(self): huey = User.get(User.username == 'huey') # Favorite -> Tweet -> User (other users' favorites of huey's tweets) # Favorite -> User (huey's favorite tweets) # Account -> User (huey's account) # User ... for a total of 5. Favorite x2, Tweet, Account, User. with self.assertQueryCount(5): huey.delete_instance(recursive=True, delete_nullable=True) # Get the last 5 delete queries. self.assertHistory(5, [ ('DELETE FROM "favorite" WHERE ("favorite"."user_id" = ?)', [huey.id]), ('DELETE FROM "favorite" WHERE (' '"favorite"."tweet_id" IN (' 'SELECT "t1"."id" FROM "tweet" AS "t1" WHERE (' '"t1"."user_id" = ?)))', [huey.id]), ('DELETE FROM "tweet" WHERE ("tweet"."user_id" = ?)', [huey.id]), ('DELETE FROM "account" WHERE ("account"."user_id" = ?)', [huey.id]), ('DELETE FROM "users" WHERE ("users"."id" = ?)', [huey.id]), ]) self.assertEqual(User.select().count(), 1) self.assertEqual(Account.select().count(), 0) self.assertEqual(Favorite.select().count(), 0) self.assertEqual(Tweet.select().count(), 1) tweet = Tweet.get() self.assertEqual(tweet.content, 'woof') def incrementer(): d = {'value': 0} def increment(): d['value'] += 1 return d['value'] return increment class AutoCounter(TestModel): counter = IntegerField(default=incrementer()) control = IntegerField(default=1) class TestDefaultDirtyBehavior(ModelTestCase): database = get_in_memory_db() requires = [AutoCounter] def tearDown(self): super(TestDefaultDirtyBehavior, self).tearDown() AutoCounter._meta.only_save_dirty = False def test_default_dirty(self): AutoCounter._meta.only_save_dirty = True ac = AutoCounter() ac.save() self.assertEqual(ac.counter, 1) self.assertEqual(ac.control, 1) ac_db = AutoCounter.get((AutoCounter.counter == 1) & (AutoCounter.control == 1)) self.assertEqual(ac_db.counter, 1) self.assertEqual(ac_db.control, 1) # No changes. self.assertFalse(ac_db.save()) ac = AutoCounter.create() self.assertEqual(ac.counter, 2) self.assertEqual(ac.control, 1) AutoCounter._meta.only_save_dirty = False ac = AutoCounter() self.assertEqual(ac.counter, 3) self.assertEqual(ac.control, 1) ac.save() ac_db = AutoCounter.get(AutoCounter.id == ac.id) self.assertEqual(ac_db.counter, 3) @requires_models(Person) def test_save_only_dirty(self): today = datetime.date.today() try: for only_save_dirty in (False, True): Person._meta.only_save_dirty = only_save_dirty p = Person.create(first='f', last='l', dob=today) p.first = 'f2' p.last = 'l2' p.save(only=[Person.first]) self.assertEqual(p.dirty_fields, [Person.last]) self.assertFalse('first' in p.dirty_field_names) self.assertTrue('last' in p.dirty_field_names) p_db = Person.get(Person.id == p.id) self.assertEqual((p_db.first, p_db.last), ('f2', 'l')) p.save() self.assertEqual(p.dirty_fields, []) p_db = Person.get(Person.id == p.id) self.assertEqual((p_db.first, p_db.last), ('f2', 'l2')) p.delete_instance() finally: # Reset only_save_dirty property for other tests. Person._meta.only_save_dirty = False class TestDefaultValues(ModelTestCase): database = get_in_memory_db() requires = [Sample, SampleMeta] def test_default_present_on_insert(self): # Although value is not specified, it has a default, which is included # in the INSERT. query = Sample.insert(counter=0) self.assertSQL(query, ( 'INSERT INTO "sample" ("counter", "value") ' 'VALUES (?, ?)'), [0, 1.0]) # Default values are also included when doing bulk inserts. query = Sample.insert_many([ {'counter': '0'}, {'counter': 1, 'value': 2}, {'counter': '2'}]) self.assertSQL(query, ( 'INSERT INTO "sample" ("counter", "value") ' 'VALUES (?, ?), (?, ?), (?, ?)'), [0, 1.0, 1, 2.0, 2, 1.0]) query = Sample.insert_many([(0,), (1, 2.)], fields=[Sample.counter]) self.assertSQL(query, ( 'INSERT INTO "sample" ("counter", "value") ' 'VALUES (?, ?), (?, ?)'), [0, 1.0, 1, 2.0]) def test_default_present_on_create(self): s = Sample.create(counter=3) s_db = Sample.get(Sample.counter == 3) self.assertEqual(s_db.value, 1.) def test_defaults_from_cursor(self): s = Sample.create(counter=1) sm1 = SampleMeta.create(sample=s, value=1.) sm2 = SampleMeta.create(sample=s, value=2.) # Defaults are not present when doing a read query. with self.assertQueryCount(1): # Simple query. query = (SampleMeta.select(SampleMeta.sample) .order_by(SampleMeta.value)) sm1_db, sm2_db = list(query) self.assertIsNone(sm1_db.value) self.assertIsNone(sm2_db.value) with self.assertQueryCount(1): # Join-graph query. query = (SampleMeta .select(SampleMeta.sample, Sample.counter) .join(Sample) .order_by(SampleMeta.value)) sm1_db, sm2_db = list(query) self.assertIsNone(sm1_db.value) self.assertIsNone(sm2_db.value) self.assertIsNone(sm1_db.sample.value) self.assertIsNone(sm2_db.sample.value) self.assertEqual(sm1_db.sample.counter, 1) self.assertEqual(sm2_db.sample.counter, 1) class TestFunctionCoerce(ModelTestCase): database = get_in_memory_db() requires = [Sample] def test_coerce(self): for i in range(3): Sample.create(counter=i, value=i) counter_group = fn.GROUP_CONCAT(Sample.counter).coerce(False) query = Sample.select(counter_group.alias('counter')) self.assertEqual(query.get().counter, '0,1,2') query = Sample.select(counter_group.alias('counter_group')) self.assertEqual(query.get().counter_group, '0,1,2') query = Sample.select(counter_group) self.assertEqual(query.scalar(), '0,1,2') def test_scalar(self): for i in range(4): Sample.create(counter=i, value=i) query = Sample.select(fn.SUM(Sample.counter).alias('total')) self.assertEqual(query.scalar(), 6) self.assertEqual(query.scalar(as_tuple=True), (6,)) self.assertEqual(query.scalar(as_dict=True), {'total': 6}) Sample.delete().execute() self.assertTrue(query.scalar() is None) self.assertEqual(query.scalar(as_tuple=True), (None,)) self.assertEqual(query.scalar(as_dict=True), {'total': None}) def test_safe_python_value(self): for i in range(3): Sample.create(counter=i, value=i) counter_group = fn.GROUP_CONCAT(Sample.counter) query = Sample.select(counter_group.alias('counter')) self.assertEqual(query.get().counter, '0,1,2') self.assertEqual(query.scalar(), '0,1,2') query = Sample.select(counter_group.alias('counter_group')) self.assertEqual(query.get().counter_group, '0,1,2') self.assertEqual(query.scalar(), '0,1,2') def test_conv_using_python_value(self): for i in range(3): Sample.create(counter=i, value=i) counter = (fn .GROUP_CONCAT(Sample.counter) .python_value(lambda x: [int(i) for i in x.split(',')])) query = Sample.select(counter.alias('counter')) self.assertEqual(query.get().counter, [0, 1, 2]) query = Sample.select(counter.alias('counter_group')) self.assertEqual(query.get().counter_group, [0, 1, 2]) query = Sample.select(counter) self.assertEqual(query.scalar(), [0, 1, 2]) @requires_models(Category, Sample) def test_no_coerce_count_avg(self): for i in range(10): Category.create(name=str(i)) # COUNT() does not result in the value being coerced. query = Category.select(fn.COUNT(Category.name)) self.assertEqual(query.scalar(), 10) # Force the value to be coerced using the field's db_value(). query = Category.select(fn.COUNT(Category.name).coerce(True)) self.assertEqual(query.scalar(), '10') # Ensure avg over an integer field is returned as a float. Sample.insert_many([(1, 0), (2, 0)]).execute() query = Sample.select(fn.AVG(Sample.counter).alias('a')) self.assertEqual(query.get().a, 1.5) class TestJoinModelAlias(ModelTestCase): data = ( ('huey', 'meow'), ('huey', 'purr'), ('zaizee', 'hiss'), ('mickey', 'woof')) requires = [User, Tweet] def setUp(self): super(TestJoinModelAlias, self).setUp() users = {} for pk, (username, tweet) in enumerate(self.data, 1): if username not in users: user = User.create(id=len(users) + 1, username=username) users[username] = user else: user = users[username] Tweet.create(id=pk, user=user, content=tweet) def _test_query(self, alias_expr): UA = alias_expr() return (Tweet .select(Tweet, UA) .order_by(UA.username, Tweet.content)) def assertTweets(self, query, user_attr='user'): with self.assertQueryCount(1): data = [(getattr(tweet, user_attr).username, tweet.content) for tweet in query] self.assertEqual(sorted(self.data), data) def test_control(self): self.assertTweets(self._test_query(lambda: User).join(User)) def test_join_aliased_columns(self): query = (Tweet .select(Tweet.id.alias('tweet_id'), Tweet.content) .order_by(Tweet.id)) self.assertEqual([(t.tweet_id, t.content) for t in query], [ (1, 'meow'), (2, 'purr'), (3, 'hiss'), (4, 'woof')]) query = (Tweet .select(Tweet.id.alias('tweet_id'), Tweet.content) .join(User) .where(User.username == 'huey') .order_by(Tweet.id)) self.assertEqual([(t.tweet_id, t.content) for t in query], [ (1, 'meow'), (2, 'purr')]) def test_join(self): UA = User.alias('ua') query = self._test_query(lambda: UA).join(UA) self.assertTweets(query) def test_join_on(self): UA = User.alias('ua') query = self._test_query(lambda: UA).join(UA, on=(Tweet.user == UA.id)) self.assertTweets(query) def test_join_on_field(self): UA = User.alias('ua') query = self._test_query(lambda: UA) query = query.join(UA, on=Tweet.user) self.assertTweets(query) def test_join_on_alias(self): UA = User.alias('ua') query = self._test_query(lambda: UA) query = query.join(UA, on=(Tweet.user == UA.id).alias('foo')) self.assertTweets(query, 'foo') def test_join_attr(self): UA = User.alias('ua') query = self._test_query(lambda: UA).join(UA, attr='baz') self.assertTweets(query, 'baz') def test_join_on_alias_attr(self): UA = User.alias('ua') q = self._test_query(lambda: UA) q = q.join(UA, on=(Tweet.user == UA.id).alias('foo'), attr='bar') self.assertTweets(q, 'bar') def _test_query_backref(self, alias_expr): TA = alias_expr() return (User .select(User, TA) .order_by(User.username, TA.content)) def assertUsers(self, query, tweet_attr='tweet'): with self.assertQueryCount(1): data = [(user.username, getattr(user, tweet_attr).content) for user in query] self.assertEqual(sorted(self.data), data) def test_control_backref(self): self.assertUsers(self._test_query_backref(lambda: Tweet).join(Tweet)) def test_join_backref(self): TA = Tweet.alias('ta') query = self._test_query_backref(lambda: TA).join(TA) self.assertUsers(query) def test_join_on_backref(self): TA = Tweet.alias('ta') query = self._test_query_backref(lambda: TA) query = query.join(TA, on=(User.id == TA.user_id)) self.assertUsers(query) def test_join_on_field_backref(self): TA = Tweet.alias('ta') query = self._test_query_backref(lambda: TA) query = query.join(TA, on=TA.user) self.assertUsers(query) def test_join_on_alias_backref(self): TA = Tweet.alias('ta') query = self._test_query_backref(lambda: TA) query = query.join(TA, on=(User.id == TA.user_id).alias('foo')) self.assertUsers(query, 'foo') def test_join_attr_backref(self): TA = Tweet.alias('ta') query = self._test_query_backref(lambda: TA).join(TA, attr='baz') self.assertUsers(query, 'baz') def test_join_alias_twice(self): # Test that a model-alias can be both the source and the dest by # joining from User -> Tweet -> User (as "foo"). TA = Tweet.alias('ta') UA = User.alias('ua') with self.assertQueryCount(1): query = (User .select(User, TA, UA) .join(TA) .join(UA, on=(TA.user_id == UA.id).alias('foo')) .order_by(User.username, TA.content)) data = [(row.username, row.tweet.content, row.tweet.foo.username) for row in query] self.assertEqual(data, [ ('huey', 'meow', 'huey'), ('huey', 'purr', 'huey'), ('mickey', 'woof', 'mickey'), ('zaizee', 'hiss', 'zaizee')]) def test_alias_filter(self): UA = User.alias('ua') lookups = ({'ua__username': 'huey'}, {'user__username': 'huey'}) for lookup in lookups: with self.assertQueryCount(1): query = (Tweet .select(Tweet.content, UA.username) .join(UA) .filter(**lookup) .order_by(Tweet.content)) self.assertSQL(query, ( 'SELECT "t1"."content", "ua"."username" ' 'FROM "tweet" AS "t1" ' 'INNER JOIN "users" AS "ua" ' 'ON ("t1"."user_id" = "ua"."id") ' 'WHERE ("ua"."username" = ?) ' 'ORDER BY "t1"."content"'), ['huey']) data = [(t.content, t.user.username) for t in query] self.assertEqual(data, [('meow', 'huey'), ('purr', 'huey')]) @skip_unless( IS_POSTGRESQL or IS_MYSQL_ADVANCED_FEATURES or IS_SQLITE_25 or IS_CRDB, 'window function') class TestWindowFunctionIntegration(ModelTestCase): requires = [Sample] def setUp(self): super(TestWindowFunctionIntegration, self).setUp() values = ((1, 10), (1, 20), (2, 1), (2, 3), (3, 100)) with self.database.atomic(): for counter, value in values: Sample.create(counter=counter, value=value) def test_simple_partition(self): query = (Sample .select(Sample.counter, Sample.value, fn.AVG(Sample.value).over( partition_by=[Sample.counter])) .order_by(Sample.counter, Sample.value) .tuples()) expected = [ (1, 10., 15.), (1, 20., 15.), (2, 1., 2.), (2, 3., 2.), (3, 100., 100.)] self.assertEqual(list(query), expected) window = Window(partition_by=[Sample.counter]) query = (Sample .select(Sample.counter, Sample.value, fn.AVG(Sample.value).over(window)) .window(window) .order_by(Sample.counter, Sample.value) .tuples()) self.assertEqual(list(query), expected) def test_mixed_ordering(self): s = fn.SUM(Sample.value).over(order_by=[Sample.value]) query = (Sample .select(Sample.counter, Sample.value, s.alias('rtotal')) .order_by(Sample.id)) # We end up with window going 1., 3., 10., 20., 100.. # So: # 1 | 10 | (1 + 3 + 10) # 1 | 20 | (1 + 3 + 10 + 20) # 2 | 1 | (1) # 2 | 3 | (1 + 3) # 3 | 100 | (1 + 3 + 10 + 20 + 100) self.assertEqual([(r.counter, r.value, r.rtotal) for r in query], [ (1, 10., 14.), (1, 20., 34.), (2, 1., 1.), (2, 3., 4.), (3, 100., 134.)]) def test_reuse_window(self): w = Window(order_by=[Sample.value]) with self.database.atomic(): Sample.delete().execute() for i in range(10): Sample.create(counter=i, value=10 * i) query = (Sample .select(Sample.counter, Sample.value, fn.NTILE(4).over(w).alias('quartile'), fn.NTILE(5).over(w).alias('quintile'), fn.NTILE(100).over(w).alias('percentile')) .window(w) .order_by(Sample.id)) results = [(r.counter, r.value, r.quartile, r.quintile, r.percentile) for r in query] self.assertEqual(results, [ # ct, v, 4tile, 5tile, 100tile (0, 0., 1, 1, 1), (1, 10., 1, 1, 2), (2, 20., 1, 2, 3), (3, 30., 2, 2, 4), (4, 40., 2, 3, 5), (5, 50., 2, 3, 6), (6, 60., 3, 4, 7), (7, 70., 3, 4, 8), (8, 80., 4, 5, 9), (9, 90., 4, 5, 10), ]) def test_ordered_window(self): window = Window(partition_by=[Sample.counter], order_by=[Sample.value.desc()]) query = (Sample .select(Sample.counter, Sample.value, fn.RANK().over(window=window).alias('rank')) .window(window) .order_by(Sample.counter, fn.RANK().over(window=window)) .tuples()) self.assertEqual(list(query), [ (1, 20., 1), (1, 10., 2), (2, 3., 1), (2, 1., 2), (3, 100., 1)]) def test_two_windows(self): w1 = Window(partition_by=[Sample.counter]).alias('w1') w2 = Window(order_by=[Sample.counter]).alias('w2') query = (Sample .select(Sample.counter, Sample.value, fn.AVG(Sample.value).over(window=w1), fn.RANK().over(window=w2)) .window(w1, w2) .order_by(Sample.id) .tuples()) self.assertEqual(list(query), [ (1, 10., 15., 1), (1, 20., 15., 1), (2, 1., 2., 3), (2, 3., 2., 3), (3, 100., 100., 5)]) def test_empty_over(self): query = (Sample .select(Sample.counter, Sample.value, fn.LAG(Sample.counter, 1).over(order_by=[Sample.id])) .order_by(Sample.id) .tuples()) self.assertEqual(list(query), [ (1, 10., None), (1, 20., 1), (2, 1., 1), (2, 3., 2), (3, 100., 2)]) def test_bounds(self): query = (Sample .select(Sample.value, fn.SUM(Sample.value).over( partition_by=[Sample.counter], start=Window.preceding(), end=Window.following(1))) .order_by(Sample.id) .tuples()) self.assertEqual(list(query), [ (10., 30.), (20., 30.), (1., 4.), (3., 4.), (100., 100.)]) query = (Sample .select(Sample.counter, Sample.value, fn.SUM(Sample.value).over( order_by=[Sample.id], start=Window.preceding(2))) .order_by(Sample.id) .tuples()) self.assertEqual(list(query), [ (1, 10., 10.), (1, 20., 30.), (2, 1., 31.), (2, 3., 24.), (3, 100., 104.)]) def test_frame_types(self): Sample.create(counter=1, value=20.) Sample.create(counter=2, value=1.) # Observe logical peer handling. # Defaults to RANGE. query = (Sample .select(Sample.counter, Sample.value, fn.SUM(Sample.value).over( order_by=[Sample.counter, Sample.value])) .order_by(Sample.id)) self.assertEqual(list(query.tuples()), [ (1, 10., 10.), (1, 20., 50.), (2, 1., 52.), (2, 3., 55.), (3, 100., 155.), (1, 20., 50.), (2, 1., 52.)]) # Explicitly specify ROWS. query = (Sample .select(Sample.counter, Sample.value, fn.SUM(Sample.value).over( order_by=[Sample.counter, Sample.value], frame_type=Window.ROWS)) .order_by(Sample.counter, Sample.value)) self.assertEqual(list(query.tuples()), [ (1, 10., 10.), (1, 20., 30.), (1, 20., 50.), (2, 1., 51.), (2, 1., 52.), (2, 3., 55.), (3, 100., 155.)]) # Including a boundary results in ROWS. query = (Sample .select(Sample.counter, Sample.value, fn.SUM(Sample.value).over( order_by=[Sample.counter, Sample.value], start=Window.preceding(2))) .order_by(Sample.counter, Sample.value)) self.assertEqual(list(query.tuples()), [ (1, 10., 10.), (1, 20., 30.), (1, 20., 50.), (2, 1., 41.), (2, 1., 22.), (2, 3., 5.), (3, 100., 104.)]) @skip_if(IS_MYSQL, 'requires OVER() with FILTER') def test_filter_clause(self): condsum = fn.SUM(Sample.value).filter(Sample.counter > 1).over( order_by=[Sample.id], start=Window.preceding(1)) query = (Sample .select(Sample.counter, Sample.value, condsum.alias('cs')) .order_by(Sample.value)) self.assertEqual(list(query.tuples()), [ (2, 1., 1.), (2, 3., 4.), (1, 10., None), (1, 20., None), (3, 100., 103.), ]) @skip_if(IS_MYSQL or (IS_SQLITE and not IS_SQLITE_30), 'requires FILTER with aggregates') def test_filter_with_aggregate(self): condsum = fn.SUM(Sample.value).filter(Sample.counter > 1) query = (Sample .select(Sample.counter, condsum.alias('cs')) .group_by(Sample.counter) .order_by(Sample.counter)) self.assertEqual(list(query.tuples()), [ (1, None), (2, 4.), (3, 100.)]) def test_row_number(self): query = (Sample .select(Sample.counter, fn.ROW_NUMBER().over( order_by=[Sample.counter]).alias('rn')) .order_by(Sample.counter) .tuples()) self.assertEqual(list(query), [(1, 1), (1, 2), (2, 3), (2, 4), (3, 5)]) def test_sum_with_frame(self): w = Window(order_by=[Sample.counter], frame_type=Window.ROWS, start=Window.preceding(1), end=Window.CURRENT_ROW) query = (Sample .select(Sample.counter, fn.SUM(Sample.value).over(w).alias('rsum')) .window(w) .order_by(Sample.counter) .tuples()) results = list(query) # Each row sums current + previous row's value. self.assertEqual(results, [ (1, 10.0), # just 10 (1, 30.0), # 10 + 20 (2, 21.0), # 20 + 1 (2, 4.0), # 1 + 3 (3, 103.0)]) # 3 + 100 def test_lag_lead(self): query = (Sample .select(Sample.counter, fn.LAG(Sample.value, 1).over( order_by=[Sample.counter]).alias('prev'), fn.LEAD(Sample.value, 1).over( order_by=[Sample.counter]).alias('next')) .order_by(Sample.counter) .tuples()) results = list(query) self.assertEqual(results, [ (1, None, 20.0), (1, 10.0, 1.0), (2, 20.0, 3.0), (2, 1.0, 100.0), (3, 3.0, None)]) #values = ((1, 10), (1, 20), (2, 1), (2, 3), (3, 100)) @skip_if(IS_SQLITE or (IS_MYSQL and not IS_MYSQL_ADVANCED_FEATURES)) @skip_unless(db.for_update, 'requires for update') class TestForUpdateIntegration(ModelTestCase): requires = [User, Tweet] def setUp(self): super(TestForUpdateIntegration, self).setUp() self.alt_db = new_connection() class AltUser(User): class Meta: database = self.alt_db table_name = User._meta.table_name class AltTweet(Tweet): class Meta: database = self.alt_db table_name = Tweet._meta.table_name self.AltUser = AltUser self.AltTweet = AltTweet def tearDown(self): self.alt_db.close() super(TestForUpdateIntegration, self).tearDown() @skip_if(IS_CRDB, 'crdb locks-up on this test, blocking reads') def test_for_update(self): with self.database.atomic(): User.create(username='huey') zaizee = User.create(username='zaizee') AltUser = self.AltUser with self.database.manual_commit(): self.database.begin() users = (User.select().where(User.username == 'zaizee') .for_update() .execute()) updated = (User .update(username='ziggy') .where(User.username == 'zaizee') .execute()) self.assertEqual(updated, 1) if IS_POSTGRESQL: nrows = (AltUser .update(username='huey-x') .where(AltUser.username == 'huey') .execute()) self.assertEqual(nrows, 1) query = (AltUser .select(AltUser.username) .where(AltUser.id == zaizee.id)) self.assertEqual(query.get().username, 'zaizee') self.database.commit() self.assertEqual(query.get().username, 'ziggy') def test_for_update_blocking(self): User.create(username='u1') AltUser = self.AltUser evt = threading.Event() def run_in_thread(): with self.alt_db.atomic(): evt.wait() n = (AltUser.update(username='u1-y') .where(AltUser.username == 'u1') .execute()) self.assertEqual(n, 0) t = threading.Thread(target=run_in_thread) t.daemon = True t.start() with self.database.atomic() as txn: q = (User.select() .where(User.username == 'u1') .for_update() .execute()) evt.set() n = (User.update(username='u1-x') .where(User.username == 'u1') .execute()) self.assertEqual(n, 1) t.join(timeout=5) u = User.get() self.assertEqual(u.username, 'u1-x') def test_for_update_nested(self): User.insert_many([(u,) for u in 'abc']).execute() subq = User.select().where(User.username != 'b').for_update() nrows = (User .delete() .where(User.id.in_(subq)) .execute()) self.assertEqual(nrows, 2) def test_for_update_nowait(self): User.create(username='huey') zaizee = User.create(username='zaizee') AltUser = self.AltUser with self.database.manual_commit(): self.database.begin() users = (User .select(User.username) .where(User.username == 'zaizee') .for_update(nowait=True) .execute()) def will_fail(): return (AltUser .select() .where(AltUser.username == 'zaizee') .for_update(nowait=True) .get()) self.assertRaises((OperationalError, InternalError), will_fail) self.database.commit() @requires_postgresql @requires_models(User, Tweet) def test_for_update_of(self): h = User.create(username='huey') z = User.create(username='zaizee') Tweet.create(user=h, content='h') Tweet.create(user=z, content='z') AltUser, AltTweet = self.AltUser, self.AltTweet with self.database.manual_commit(): self.database.begin() # Lock tweets by huey. query = (Tweet .select() .join(User) .where(User.username == 'huey') .for_update(of=Tweet, nowait=True)) qr = query.execute() # No problem updating zaizee's tweet or huey's user. nrows = (AltTweet .update(content='zx') .where(AltTweet.user == z.id) .execute()) self.assertEqual(nrows, 1) nrows = (AltUser .update(username='huey-x') .where(AltUser.username == 'huey') .execute()) self.assertEqual(nrows, 1) def will_fail(): (AltTweet .select() .where(AltTweet.user == h) .for_update(nowait=True) .get()) self.assertRaises((OperationalError, InternalError), will_fail) self.database.commit() query = Tweet.select(Tweet, User).join(User).order_by(Tweet.id) self.assertEqual([(t.content, t.user.username) for t in query], [('h', 'huey-x'), ('zx', 'zaizee')]) class ServerDefault(TestModel): timestamp = DateTimeField(constraints=[SQL('default (now())')]) @requires_postgresql class TestReturningIntegration(ModelTestCase): requires = [User] def test_simple_returning(self): query = User.insert(username='charlie') self.assertSQL(query, ( 'INSERT INTO "users" ("username") VALUES (?) ' 'RETURNING "users"."id"'), ['charlie']) self.assertEqual(query.execute(), 1) # By default returns a tuple. query = User.insert(username='huey') self.assertEqual(query.execute(), 2) self.assertEqual(list(query), [(2,)]) # If we specify a returning clause we get user instances. query = User.insert(username='snoobie').returning(User) query.execute() self.assertEqual([x.username for x in query], ['snoobie']) query = (User .insert(username='zaizee') .returning(User.id, User.username) .dicts()) self.assertSQL(query, ( 'INSERT INTO "users" ("username") VALUES (?) ' 'RETURNING "users"."id", "users"."username"'), ['zaizee']) cursor = query.execute() row, = list(cursor) self.assertEqual(row, {'id': 4, 'username': 'zaizee'}) query = (User .insert(username='mickey') .returning(User) .objects()) self.assertSQL(query, ( 'INSERT INTO "users" ("username") VALUES (?) ' 'RETURNING "users"."id", "users"."username"'), ['mickey']) cursor = query.execute() row, = list(cursor) self.assertEqual(row.id, 5) self.assertEqual(row.username, 'mickey') # Can specify aliases. query = (User .insert(username='sipp') .returning(User.username.alias('new_username'))) self.assertEqual([x.new_username for x in query.execute()], ['sipp']) # Minimal test with insert_many. query = User.insert_many([('u7',), ('u8',)]) self.assertEqual([r for r, in query.execute()], [7, 8]) # Test with insert / on conflict. query = (User .insert_many([(7, 'u7',), (9, 'u9',)], [User.id, User.username]) .on_conflict(conflict_target=[User.id], update={User.username: User.username + 'x'}) .returning(User)) self.assertEqual([(x.id, x.username) for x in query], [(7, 'u7x'), (9, 'u9')]) def test_simple_returning_insert_update_delete(self): res = User.insert(username='charlie').returning(User).execute() self.assertEqual([u.username for u in res], ['charlie']) res = (User .update(username='charlie2') .where(User.id == 1) .returning(User) .execute()) # Subsequent iterations are cached. for _ in range(2): self.assertEqual([u.username for u in res], ['charlie2']) res = (User .delete() .where(User.id == 1) .returning(User) .execute()) # Subsequent iterations are cached. for _ in range(2): self.assertEqual([u.username for u in res], ['charlie2']) def test_simple_insert_update_delete_no_returning(self): query = User.insert(username='charlie') self.assertEqual(query.execute(), 1) query = User.insert(username='huey') self.assertEqual(query.execute(), 2) query = User.update(username='huey2').where(User.username == 'huey') self.assertEqual(query.execute(), 1) self.assertEqual(query.execute(), 0) # No rows updated! query = User.delete().where(User.username == 'huey2') self.assertEqual(query.execute(), 1) self.assertEqual(query.execute(), 0) # No rows updated! @requires_models(ServerDefault) def test_returning_server_defaults(self): query = (ServerDefault .insert() .returning(ServerDefault.id, ServerDefault.timestamp)) self.assertSQL(query, ( 'INSERT INTO "server_default" ' 'DEFAULT VALUES ' 'RETURNING "server_default"."id", "server_default"."timestamp"'), []) with self.assertQueryCount(1): cursor = query.dicts().execute() row, = list(cursor) self.assertTrue(row['timestamp'] is not None) obj = ServerDefault.get(ServerDefault.id == row['id']) self.assertEqual(obj.timestamp, row['timestamp']) def test_no_return(self): query = User.insert(username='huey').returning() self.assertIsNone(query.execute()) user = User.get(User.username == 'huey') self.assertEqual(user.username, 'huey') self.assertTrue(user.id >= 1) @requires_models(Category) def test_non_int_pk_returning(self): query = Category.insert(name='root') self.assertSQL(query, ( 'INSERT INTO "category" ("name") VALUES (?) ' 'RETURNING "category"."name"'), ['root']) self.assertEqual(query.execute(), 'root') def test_returning_multi(self): data = [{'username': 'huey'}, {'username': 'mickey'}] query = User.insert_many(data) self.assertSQL(query, ( 'INSERT INTO "users" ("username") VALUES (?), (?) ' 'RETURNING "users"."id"'), ['huey', 'mickey']) data = query.execute() # Check that the result wrapper is correctly set up. self.assertTrue(len(data.select) == 1 and data.select[0] is User.id) self.assertEqual(list(data), [(1,), (2,)]) query = (User .insert_many([{'username': 'foo'}, {'username': 'bar'}, {'username': 'baz'}]) .returning(User.id, User.username) .namedtuples()) data = query.execute() self.assertEqual([(row.id, row.username) for row in data], [ (3, 'foo'), (4, 'bar'), (5, 'baz')]) @requires_models(Category) def test_returning_query(self): for name in ('huey', 'mickey', 'zaizee'): Category.create(name=name) source = Category.select(Category.name).order_by(Category.name) query = User.insert_from(source, (User.username,)) self.assertSQL(query, ( 'INSERT INTO "users" ("username") ' 'SELECT "t1"."name" FROM "category" AS "t1" ORDER BY "t1"."name" ' 'RETURNING "users"."id"'), []) data = query.execute() # Check that the result wrapper is correctly set up. self.assertTrue(len(data.select) == 1 and data.select[0] is User.id) self.assertEqual(list(data), [(1,), (2,), (3,)]) def test_update_returning(self): id_list = User.insert_many([{'username': 'huey'}, {'username': 'zaizee'}]).execute() huey_id, zaizee_id = [pk for pk, in id_list] query = (User .update(username='ziggy') .where(User.username == 'zaizee') .returning(User.id, User.username)) self.assertSQL(query, ( 'UPDATE "users" SET "username" = ? ' 'WHERE ("users"."username" = ?) ' 'RETURNING "users"."id", "users"."username"'), ['ziggy', 'zaizee']) data = query.execute() user = data[0] self.assertEqual(user.username, 'ziggy') self.assertEqual(user.id, zaizee_id) def test_delete_returning(self): id_list = User.insert_many([{'username': 'huey'}, {'username': 'zaizee'}]).execute() huey_id, zaizee_id = [pk for pk, in id_list] query = (User .delete() .where(User.username == 'zaizee') .returning(User.id, User.username)) self.assertSQL(query, ( 'DELETE FROM "users" WHERE ("users"."username" = ?) ' 'RETURNING "users"."id", "users"."username"'), ['zaizee']) data = query.execute() user = data[0] self.assertEqual(user.username, 'zaizee') self.assertEqual(user.id, zaizee_id) class Member(TestModel): name = TextField() recommendedby = ForeignKeyField('self', null=True) class TestCTEIntegration(ModelTestCase): requires = [Category] def setUp(self): super(TestCTEIntegration, self).setUp() CC = Category.create root = CC(name='root') p1 = CC(name='p1', parent=root) p2 = CC(name='p2', parent=root) p3 = CC(name='p3', parent=root) c11 = CC(name='c11', parent=p1) c12 = CC(name='c12', parent=p1) c31 = CC(name='c31', parent=p3) @skip_if(IS_SQLITE_OLD or (IS_MYSQL and not IS_MYSQL_ADVANCED_FEATURES) or IS_CRDB) @requires_models(Member) def test_docs_example(self): f = Member.create(name='founder') gen2_1 = Member.create(name='g2-1', recommendedby=f) gen2_2 = Member.create(name='g2-2', recommendedby=f) gen2_3 = Member.create(name='g2-3', recommendedby=f) gen3_1_1 = Member.create(name='g3-1-1', recommendedby=gen2_1) gen3_1_2 = Member.create(name='g3-1-2', recommendedby=gen2_1) gen3_3_1 = Member.create(name='g3-3-1', recommendedby=gen2_3) # Get recommender chain for 331. base = (Member .select(Member.recommendedby) .where(Member.id == gen3_3_1.id) .cte('recommenders', recursive=True, columns=('recommender',))) MA = Member.alias() recursive = (MA .select(MA.recommendedby) .join(base, on=(MA.id == base.c.recommender))) cte = base.union_all(recursive) query = (cte .select_from(cte.c.recommender, Member.name) .join(Member, on=(cte.c.recommender == Member.id)) .order_by(Member.id.desc())) self.assertEqual([m.name for m in query], ['g2-3', 'founder']) @skip_if(IS_SQLITE_OLD or (IS_MYSQL and not IS_MYSQL_ADVANCED_FEATURES)) def test_simple_cte(self): cte = (Category .select(Category.name, Category.parent) .cte('catz', columns=('name', 'parent'))) cte_sql = ('WITH "catz" ("name", "parent") AS (' 'SELECT "t1"."name", "t1"."parent_id" ' 'FROM "category" AS "t1") ' 'SELECT "catz"."name", "catz"."parent" AS "pname" ' 'FROM "catz" ' 'ORDER BY "catz"."name"') query = (Category .select(cte.c.name, cte.c.parent.alias('pname')) .from_(cte) .order_by(cte.c.name) .with_cte(cte)) self.assertSQL(query, cte_sql, []) query2 = (cte.select_from(cte.c.name, cte.c.parent.alias('pname')) .order_by(cte.c.name)) self.assertSQL(query2, cte_sql, []) self.assertEqual([(row.name, row.pname) for row in query], [ ('c11', 'p1'), ('c12', 'p1'), ('c31', 'p3'), ('p1', 'root'), ('p2', 'root'), ('p3', 'root'), ('root', None)]) self.assertEqual([(row.name, row.pname) for row in query], [(row.name, row.pname) for row in query2]) @skip_if(IS_SQLITE_OLD or (IS_MYSQL and not IS_MYSQL_ADVANCED_FEATURES)) def test_cte_join(self): cte = (Category .select(Category.name) .cte('parents', columns=('name',))) query = (Category .select(Category.name, cte.c.name.alias('pname')) .join(cte, on=(Category.parent == cte.c.name)) .order_by(Category.name) .with_cte(cte)) self.assertSQL(query, ( 'WITH "parents" ("name") AS (' 'SELECT "t1"."name" FROM "category" AS "t1") ' 'SELECT "t2"."name", "parents"."name" AS "pname" ' 'FROM "category" AS "t2" ' 'INNER JOIN "parents" ON ("t2"."parent_id" = "parents"."name") ' 'ORDER BY "t2"."name"'), []) self.assertEqual([(c.name, c.parents['pname']) for c in query], [ ('c11', 'p1'), ('c12', 'p1'), ('c31', 'p3'), ('p1', 'root'), ('p2', 'root'), ('p3', 'root'), ]) @skip_if(IS_SQLITE_OLD or IS_MYSQL or IS_CRDB, 'requires recursive cte') def test_recursive_cte(self): def get_parents(cname): C1 = Category.alias() C2 = Category.alias() level = SQL('1').cast('integer').alias('level') path = C1.name.cast('text').alias('path') base = (C1 .select(C1.name, C1.parent, level, path) .where(C1.name == cname) .cte('parents', recursive=True)) rlevel = (base.c.level + 1).alias('level') rpath = base.c.path.concat('->').concat(C2.name).alias('path') recursive = (C2 .select(C2.name, C2.parent, rlevel, rpath) .from_(base) .join(C2, on=(C2.name == base.c.parent_id))) cte = base + recursive query = (cte .select_from(cte.c.name, cte.c.level, cte.c.path) .order_by(cte.c.level)) self.assertSQL(query, ( 'WITH RECURSIVE "parents" AS (' 'SELECT "t1"."name", "t1"."parent_id", ' 'CAST(1 AS integer) AS "level", ' 'CAST("t1"."name" AS text) AS "path" ' 'FROM "category" AS "t1" ' 'WHERE ("t1"."name" = ?) ' 'UNION ALL ' 'SELECT "t2"."name", "t2"."parent_id", ' '("parents"."level" + ?) AS "level", ' '(("parents"."path" || ?) || "t2"."name") AS "path" ' 'FROM "parents" ' 'INNER JOIN "category" AS "t2" ' 'ON ("t2"."name" = "parents"."parent_id")) ' 'SELECT "parents"."name", "parents"."level", "parents"."path" ' 'FROM "parents" ' 'ORDER BY "parents"."level"'), [cname, 1, '->']) return query data = [row for row in get_parents('c31').tuples()] self.assertEqual(data, [ ('c31', 1, 'c31'), ('p3', 2, 'c31->p3'), ('root', 3, 'c31->p3->root')]) data = [(c.name, c.level, c.path) for c in get_parents('c12').namedtuples()] self.assertEqual(data, [ ('c12', 1, 'c12'), ('p1', 2, 'c12->p1'), ('root', 3, 'c12->p1->root')]) query = get_parents('root') data = [(r.name, r.level, r.path) for r in query] self.assertEqual(data, [('root', 1, 'root')]) @skip_if(IS_SQLITE_OLD or IS_MYSQL or IS_CRDB, 'requires recursive cte') def test_recursive_cte2(self): hierarchy = (Category .select(Category.name, Value(0).alias('level')) .where(Category.parent.is_null(True)) .cte(name='hierarchy', recursive=True)) C = Category.alias() recursive = (C .select(C.name, (hierarchy.c.level + 1).alias('level')) .join(hierarchy, on=(C.parent == hierarchy.c.name))) cte = hierarchy.union_all(recursive) query = (cte .select_from(cte.c.name, cte.c.level) .order_by(cte.c.name)) self.assertEqual([(r.name, r.level) for r in query], [ ('c11', 2), ('c12', 2), ('c31', 2), ('p1', 1), ('p2', 1), ('p3', 1), ('root', 0)]) @skip_if(IS_SQLITE_OLD or IS_MYSQL or IS_CRDB, 'requires recursive cte') def test_recursive_cte_docs_example(self): # Define the base case of our recursive CTE. This will be categories that # have a null parent foreign-key. Base = Category.alias() level = Value(1).cast('integer').alias('level') path = Base.name.cast('text').alias('path') base_case = (Base .select(Base.name, Base.parent, level, path) .where(Base.parent.is_null()) .cte('base', recursive=True)) # Define the recursive terms. RTerm = Category.alias() rlevel = (base_case.c.level + 1).alias('level') rpath = base_case.c.path.concat('->').concat(RTerm.name).alias('path') recursive = (RTerm .select(RTerm.name, RTerm.parent, rlevel, rpath) .join(base_case, on=(RTerm.parent == base_case.c.name))) # The recursive CTE is created by taking the base case and UNION ALL with # the recursive term. cte = base_case.union_all(recursive) # We will now query from the CTE to get the categories, their levels, and # their paths. query = (cte .select_from(cte.c.name, cte.c.level, cte.c.path) .order_by(cte.c.path)) data = [(obj.name, obj.level, obj.path) for obj in query] self.assertEqual(data, [ ('root', 1, 'root'), ('p1', 2, 'root->p1'), ('c11', 3, 'root->p1->c11'), ('c12', 3, 'root->p1->c12'), ('p2', 2, 'root->p2'), ('p3', 2, 'root->p3'), ('c31', 3, 'root->p3->c31')]) @requires_models(Sample) @skip_if(IS_SQLITE_OLD or IS_MYSQL, 'sqlite too old for ctes, mysql flaky') def test_cte_reuse_aggregate(self): data = ( (1, (1.25, 1.5, 1.75)), (2, (2.1, 2.3, 2.5, 2.7, 2.9)), (3, (3.5, 3.5))) with self.database.atomic(): for counter, values in data: (Sample .insert_many([(counter, value) for value in values], fields=[Sample.counter, Sample.value]) .execute()) cte = (Sample .select(Sample.counter, fn.AVG(Sample.value).alias('avg_value')) .group_by(Sample.counter) .cte('count_to_avg', columns=('counter', 'avg_value'))) query = (Sample .select(Sample.counter, (Sample.value - cte.c.avg_value).alias('diff')) .join(cte, on=(Sample.counter == cte.c.counter)) .where(Sample.value > cte.c.avg_value) .order_by(Sample.value) .with_cte(cte)) self.assertEqual([(a, round(b, 2)) for a, b in query.tuples()], [ (1, .25), (2, .2), (2, .4)]) @skip_if(IS_SQLITE_OLD or IS_MYSQL) @requires_models(Sample) def test_cte_with_aggregate_filter(self): for i in range(1, 11): Sample.create(counter=i, value=float(i * i)) cte = (Sample .select(Sample.counter, Sample.value) .where(Sample.counter <= 5) .cte('small')) query = (cte .select_from(fn.SUM(cte.c.value).alias('total')) .where(cte.c.counter > 2)) result = query.scalar() # sum of 3^2 + 4^2 + 5^2 = 9 + 16 + 25 = 50 self.assertEqual(result, 50.0) @skip_if(not IS_SQLITE_15, 'requires row-values') class TestTupleComparison(ModelTestCase): requires = [User] def test_tuples(self): ua, ub, uc = [User.create(username=username) for username in 'abc'] query = User.select().where( Tuple(User.username, User.id) == ('b', ub.id)) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username" FROM "users" AS "t1" ' 'WHERE (("t1"."username", "t1"."id") = (?, ?))'), ['b', ub.id]) self.assertEqual(query.count(), 1) obj = query.get() self.assertEqual(obj, ub) def test_tuple_subquery(self): ua, ub, uc = [User.create(username=username) for username in 'abc'] UA = User.alias() subquery = (UA .select(UA.username, UA.id) .where(UA.username != 'b')) query = (User .select(User.username) .where(Tuple(User.username, User.id).in_(subquery)) .order_by(User.username)) self.assertEqual([u.username for u in query], ['a', 'c']) @requires_models(CPK) def test_row_value_composite_key(self): CPK.insert_many([('k1', 1, 1), ('k2', 2, 2), ('k3', 3, 3)]).execute() cpk = CPK.get(CPK._meta.primary_key == ('k2', 2)) self.assertEqual(cpk._pk, ('k2', 2)) cpk = CPK['k3', 3] self.assertEqual(cpk._pk, ('k3', 3)) uq = CPK.update(extra=20).where(CPK._meta.primary_key != ('k2', 2)) uq.execute() self.assertEqual(list(sorted(CPK.select().tuples())), [ ('k1', 1, 20), ('k2', 2, 2), ('k3', 3, 20)]) class TestModelGraph(BaseTestCase): def test_bind_model_database(self): class User(Model): pass class Tweet(Model): user = ForeignKeyField(User) class Relationship(Model): from_user = ForeignKeyField(User, backref='relationships') to_user = ForeignKeyField(User, backref='related_to') class Flag(Model): tweet = ForeignKeyField(Tweet) class Unrelated(Model): pass fake_db = SqliteDatabase(None) User.bind(fake_db) for model in (User, Tweet, Relationship, Flag): self.assertTrue(model._meta.database is fake_db) self.assertTrue(Unrelated._meta.database is None) User.bind(None) with User.bind_ctx(fake_db) as (FUser,): self.assertTrue(FUser._meta.database is fake_db) self.assertTrue(Unrelated._meta.database is None) self.assertTrue(User._meta.database is None) class TestFieldInheritance(BaseTestCase): def test_field_inheritance(self): class BaseModel(Model): class Meta: database = get_in_memory_db() class BasePost(BaseModel): content = TextField() timestamp = TimestampField() class Photo(BasePost): image = TextField() class Note(BasePost): category = TextField() self.assertEqual(BasePost._meta.sorted_field_names, ['id', 'content', 'timestamp']) self.assertEqual(BasePost._meta.sorted_fields, [ BasePost.id, BasePost.content, BasePost.timestamp]) self.assertEqual(Photo._meta.sorted_field_names, ['id', 'content', 'timestamp', 'image']) self.assertEqual(Photo._meta.sorted_fields, [ Photo.id, Photo.content, Photo.timestamp, Photo.image]) self.assertEqual(Note._meta.sorted_field_names, ['id', 'content', 'timestamp', 'category']) self.assertEqual(Note._meta.sorted_fields, [ Note.id, Note.content, Note.timestamp, Note.category]) self.assertTrue(id(Photo.id) != id(Note.id)) def test_foreign_key_field_inheritance(self): class BaseModel(Model): class Meta: database = get_in_memory_db() class Category(BaseModel): name = TextField() class BasePost(BaseModel): category = ForeignKeyField(Category) timestamp = TimestampField() class Photo(BasePost): image = TextField() class Note(BasePost): content = TextField() self.assertEqual(BasePost._meta.sorted_field_names, ['id', 'category', 'timestamp']) self.assertEqual(BasePost._meta.sorted_fields, [ BasePost.id, BasePost.category, BasePost.timestamp]) self.assertEqual(Photo._meta.sorted_field_names, ['id', 'category', 'timestamp', 'image']) self.assertEqual(Photo._meta.sorted_fields, [ Photo.id, Photo.category, Photo.timestamp, Photo.image]) self.assertEqual(Note._meta.sorted_field_names, ['id', 'category', 'timestamp', 'content']) self.assertEqual(Note._meta.sorted_fields, [ Note.id, Note.category, Note.timestamp, Note.content]) self.assertEqual(Category._meta.backrefs, { BasePost.category: BasePost, Photo.category: Photo, Note.category: Note}) self.assertEqual(BasePost._meta.refs, {BasePost.category: Category}) self.assertEqual(Photo._meta.refs, {Photo.category: Category}) self.assertEqual(Note._meta.refs, {Note.category: Category}) self.assertEqual(BasePost.category.backref, 'basepost_set') self.assertEqual(Photo.category.backref, 'photo_set') self.assertEqual(Note.category.backref, 'note_set') def test_foreign_key_pk_inheritance(self): class BaseModel(Model): class Meta: database = get_in_memory_db() class Account(BaseModel): pass class BaseUser(BaseModel): account = ForeignKeyField(Account, primary_key=True) class User(BaseUser): username = TextField() class Admin(BaseUser): role = TextField() self.assertEqual(Account._meta.backrefs, { Admin.account: Admin, User.account: User, BaseUser.account: BaseUser}) self.assertEqual(BaseUser.account.backref, 'baseuser_set') self.assertEqual(User.account.backref, 'user_set') self.assertEqual(Admin.account.backref, 'admin_set') self.assertTrue(Account.user_set.model is Account) self.assertTrue(Account.admin_set.model is Account) self.assertTrue(Account.user_set.rel_model is User) self.assertTrue(Account.admin_set.rel_model is Admin) self.assertSQL(Account._schema._create_table(), ( 'CREATE TABLE IF NOT EXISTS "account" (' '"id" INTEGER NOT NULL PRIMARY KEY)'), []) self.assertSQL(User._schema._create_table(), ( 'CREATE TABLE IF NOT EXISTS "user" (' '"account_id" INTEGER NOT NULL PRIMARY KEY, ' '"username" TEXT NOT NULL, ' 'FOREIGN KEY ("account_id") REFERENCES "account" ("id"))'), []) self.assertSQL(Admin._schema._create_table(), ( 'CREATE TABLE IF NOT EXISTS "admin" (' '"account_id" INTEGER NOT NULL PRIMARY KEY, ' '"role" TEXT NOT NULL, ' 'FOREIGN KEY ("account_id") REFERENCES "account" ("id"))'), []) def test_backref_inheritance(self): class Category(TestModel): pass def backref(fk_field): return '%ss' % fk_field.model._meta.name class BasePost(TestModel): category = ForeignKeyField(Category, backref=backref) class Note(BasePost): pass class Photo(BasePost): pass self.assertEqual(Category._meta.backrefs, { BasePost.category: BasePost, Note.category: Note, Photo.category: Photo}) self.assertEqual(BasePost.category.backref, 'baseposts') self.assertEqual(Note.category.backref, 'notes') self.assertEqual(Photo.category.backref, 'photos') self.assertTrue(Category.baseposts.rel_model is BasePost) self.assertTrue(Category.baseposts.model is Category) self.assertTrue(Category.notes.rel_model is Note) self.assertTrue(Category.notes.model is Category) self.assertTrue(Category.photos.rel_model is Photo) self.assertTrue(Category.photos.model is Category) class BaseItem(TestModel): category = ForeignKeyField(Category, backref='items') class ItemA(BaseItem): pass class ItemB(BaseItem): pass self.assertEqual(BaseItem.category.backref, 'items') self.assertEqual(ItemA.category.backref, 'itema_set') self.assertEqual(ItemB.category.backref, 'itemb_set') self.assertTrue(Category.items.rel_model is BaseItem) self.assertTrue(Category.itema_set.rel_model is ItemA) self.assertTrue(Category.itema_set.model is Category) self.assertTrue(Category.itemb_set.rel_model is ItemB) self.assertTrue(Category.itemb_set.model is Category) @skip_if(IS_SQLITE, 'sqlite is not supported') @skip_if(IS_MYSQL, 'mysql is not raising this error(?)') @skip_if(IS_CRDB, 'crdb is not raising the error in this test(?)') def test_deferred_fk_creation(self): class B(TestModel): a = DeferredForeignKey('A', null=True) b = TextField() class A(TestModel): a = TextField() db.create_tables([A, B]) try: # Test that we can create B with null "a_id" column: a = A.create(a='a') b = B.create(b='b') # Test that we can create B that has no corresponding A: fake_a = A(id=31337) b2 = B.create(a=fake_a, b='b2') b2_db = B.get(B.a == fake_a) self.assertEqual(b2_db.b, 'b2') # Ensure error occurs trying to create_foreign_key. with db.atomic(): self.assertRaises( IntegrityError, B._schema.create_foreign_key, B.a) b2_db.delete_instance() # We can now create the foreign key. B._schema.create_foreign_key(B.a) # The foreign-key is enforced: with db.atomic(): self.assertRaises(IntegrityError, B.create, a=fake_a, b='b3') finally: db.drop_tables([A, B]) class TestMetaTableName(BaseTestCase): def test_table_name_behavior(self): def make_model(model_name, table=None): class Meta: legacy_table_names = False table_name = table return type(model_name, (Model,), {'Meta': Meta}) def assertTableName(expected, model_name, table_name=None): model_class = make_model(model_name, table_name) self.assertEqual(model_class._meta.table_name, expected) assertTableName('users', 'User', 'users') assertTableName('tweet', 'Tweet') assertTableName('user_profile', 'UserProfile') assertTableName('activity_log_status', 'ActivityLogStatus') assertTableName('camel_case', 'CamelCase') assertTableName('camel_camel_case', 'CamelCamelCase') assertTableName('camel2_camel2_case', 'Camel2Camel2Case') assertTableName('http_request', 'HTTPRequest') assertTableName('api_response', 'APIResponse') assertTableName('api_response', 'API_Response') assertTableName('web_http_request', 'WebHTTPRequest') assertTableName('get_http_response_code', 'getHTTPResponseCode') assertTableName('foo_bar', 'foo_Bar') assertTableName('foo_bar', 'Foo__Bar') class TestMetaInheritance(BaseTestCase): def test_table_name(self): class Foo(Model): class Meta: def table_function(klass): return 'xxx_%s' % klass.__name__.lower() class Bar(Foo): pass class Baze(Foo): class Meta: table_name = 'yyy_baze' class Biz(Baze): pass class Nug(Foo): class Meta: def table_function(klass): return 'zzz_%s' % klass.__name__.lower() self.assertEqual(Foo._meta.table_name, 'xxx_foo') self.assertEqual(Bar._meta.table_name, 'xxx_bar') self.assertEqual(Baze._meta.table_name, 'yyy_baze') self.assertEqual(Biz._meta.table_name, 'xxx_biz') self.assertEqual(Nug._meta.table_name, 'zzz_nug') def test_composite_key_inheritance(self): class Foo(Model): key = TextField() value = TextField() class Meta: primary_key = CompositeKey('key', 'value') class Bar(Foo): pass class Baze(Foo): value = IntegerField() foo = Foo(key='k1', value='v1') self.assertEqual(foo.__composite_key__, ('k1', 'v1')) bar = Bar(key='k2', value='v2') self.assertEqual(bar.__composite_key__, ('k2', 'v2')) baze = Baze(key='k3', value=3) self.assertEqual(baze.__composite_key__, ('k3', 3)) def test_no_primary_key_inheritable(self): class Foo(Model): data = TextField() class Meta: primary_key = False class Bar(Foo): pass class Baze(Foo): pk = AutoField() class Zai(Foo): zee = TextField(primary_key=True) self.assertFalse(Foo._meta.primary_key) self.assertEqual(Foo._meta.sorted_field_names, ['data']) self.assertFalse(Bar._meta.primary_key) self.assertEqual(Bar._meta.sorted_field_names, ['data']) self.assertTrue(Baze._meta.primary_key is Baze.pk) self.assertEqual(Baze._meta.sorted_field_names, ['pk', 'data']) self.assertTrue(Zai._meta.primary_key is Zai.zee) self.assertEqual(Zai._meta.sorted_field_names, ['zee', 'data']) def test_inheritance(self): db = SqliteDatabase(':memory:') class Base(Model): class Meta: constraints = ['c1', 'c2'] database = db indexes = ( (('username',), True), ) only_save_dirty = True options = {'key': 'value'} schema = 'magic' class Child(Base): pass class GrandChild(Child): pass for ModelClass in (Child, GrandChild): self.assertEqual(ModelClass._meta.constraints, ['c1', 'c2']) self.assertTrue(ModelClass._meta.database is db) self.assertEqual(ModelClass._meta.indexes, [(('username',), True)]) self.assertEqual(ModelClass._meta.options, {'key': 'value'}) self.assertTrue(ModelClass._meta.only_save_dirty) self.assertEqual(ModelClass._meta.schema, 'magic') class Overrides(Base): class Meta: constraints = None indexes = None only_save_dirty = False options = {'foo': 'bar'} schema = None self.assertTrue(Overrides._meta.constraints is None) self.assertEqual(Overrides._meta.indexes, []) self.assertFalse(Overrides._meta.only_save_dirty) self.assertEqual(Overrides._meta.options, {'foo': 'bar'}) self.assertTrue(Overrides._meta.schema is None) def test_temporary_inheritance(self): class T0(TestModel): pass class T1(TestModel): class Meta: temporary = True class T2(T1): pass class T3(T1): class Meta: temporary = False self.assertFalse(T0._meta.temporary) self.assertTrue(T1._meta.temporary) self.assertTrue(T2._meta.temporary) self.assertFalse(T3._meta.temporary) class TestModelMetadataMisc(BaseTestCase): database = get_in_memory_db() def test_subclass_aware_metadata(self): class SchemaPropagateMetadata(SubclassAwareMetadata): @property def schema(self): return self._schema @schema.setter def schema(self, value): # self.models is a singleton, essentially, shared among all # classes that use this metadata implementation. for model in self.models: model._meta._schema = value class Base(Model): class Meta: database = self.database model_metadata_class = SchemaPropagateMetadata class User(Base): username = TextField() class Tweet(Base): user = ForeignKeyField(User, backref='tweets') content = TextField() self.assertTrue(User._meta.schema is None) self.assertTrue(Tweet._meta.schema is None) Base._meta.schema = 'temp' self.assertEqual(User._meta.schema, 'temp') self.assertEqual(Tweet._meta.schema, 'temp') User._meta.schema = None for model in (Base, User, Tweet): self.assertTrue(model._meta.schema is None) class TestModelSetDatabase(BaseTestCase): def test_set_database(self): class Register(Model): value = IntegerField() db_a = get_in_memory_db() db_b = get_in_memory_db() Register._meta.set_database(db_a) Register.create_table() Register._meta.set_database(db_b) self.assertFalse(Register.table_exists()) self.assertEqual(db_a.get_tables(), ['register']) self.assertEqual(db_b.get_tables(), []) db_a.close() db_b.close() class TestForeignKeyFieldDescriptors(BaseTestCase): def test_foreign_key_field_descriptors(self): class User(Model): pass class T0(Model): user = ForeignKeyField(User) class T1(Model): user = ForeignKeyField(User, column_name='uid') class T2(Model): user = ForeignKeyField(User, object_id_name='uid') class T3(Model): user = ForeignKeyField(User, column_name='x', object_id_name='uid') class T4(Model): foo = ForeignKeyField(User, column_name='user') class T5(Model): foo = ForeignKeyField(User, object_id_name='uid') self.assertEqual(T0.user.object_id_name, 'user_id') self.assertEqual(T1.user.object_id_name, 'uid') self.assertEqual(T2.user.object_id_name, 'uid') self.assertEqual(T3.user.object_id_name, 'uid') self.assertEqual(T4.foo.object_id_name, 'user') self.assertEqual(T5.foo.object_id_name, 'uid') user = User(id=1337) self.assertEqual(T0(user=user).user_id, 1337) self.assertEqual(T1(user=user).uid, 1337) self.assertEqual(T2(user=user).uid, 1337) self.assertEqual(T3(user=user).uid, 1337) self.assertEqual(T4(foo=user).user, 1337) self.assertEqual(T5(foo=user).uid, 1337) def conflicts_with_field(): class TE(Model): user = ForeignKeyField(User, object_id_name='user') self.assertRaises(ValueError, conflicts_with_field) def test_column_name(self): class User(Model): pass class T1(Model): user = ForeignKeyField(User, column_name='user') self.assertEqual(T1.user.column_name, 'user') self.assertEqual(T1.user.object_id_name, 'user_id') class TestModelAliasFieldProperties(ModelTestCase): database = get_in_memory_db() def test_field_properties(self): class Person(TestModel): name = TextField() dob = DateField() class Meta: database = self.database class Job(TestModel): worker = ForeignKeyField(Person, backref='jobs') client = ForeignKeyField(Person, backref='jobs_hired') class Meta: database = self.database Worker = Person.alias() Client = Person.alias() expected_sql = ( 'SELECT "t1"."id", "t1"."worker_id", "t1"."client_id" ' 'FROM "job" AS "t1" ' 'INNER JOIN "person" AS "t2" ON ("t1"."client_id" = "t2"."id") ' 'INNER JOIN "person" AS "t3" ON ("t1"."worker_id" = "t3"."id") ' 'WHERE (date_part(?, "t2"."dob") = ?)') expected_params = ['year', 1983] query = (Job .select() .join(Client, on=(Job.client == Client.id)) .switch(Job) .join(Worker, on=(Job.worker == Worker.id)) .where(Client.dob.year == 1983)) self.assertSQL(query, expected_sql, expected_params) query = (Job .select() .join(Client, on=(Job.client == Client.id)) .switch(Job) .join(Person, on=(Job.worker == Person.id)) .where(Client.dob.year == 1983)) self.assertSQL(query, expected_sql, expected_params) query = (Job .select() .join(Person, on=(Job.client == Person.id)) .switch(Job) .join(Worker, on=(Job.worker == Worker.id)) .where(Person.dob.year == 1983)) self.assertSQL(query, expected_sql, expected_params) class OnConflictTests(object): requires = [Emp] test_data = ( ('huey', 'cat', '123'), ('zaizee', 'cat', '124'), ('mickey', 'dog', '125'), ) def setUp(self): super(OnConflictTests, self).setUp() for first, last, empno in self.test_data: Emp.create(first=first, last=last, empno=empno) def assertData(self, expected): query = (Emp .select(Emp.first, Emp.last, Emp.empno) .order_by(Emp.id) .tuples()) self.assertEqual(list(query), expected) def test_ignore(self): query = (Emp .insert(first='foo', last='bar', empno='123') .on_conflict('ignore') .execute()) self.assertData(list(self.test_data)) def requires_upsert(m): return skip_unless(IS_SQLITE_24 or IS_POSTGRESQL or IS_CRDB, 'requires upsert')(m) class KV(TestModel): key = CharField(unique=True) value = IntegerField() class PGOnConflictTests(OnConflictTests): @requires_upsert def test_update(self): # Conflict on empno - we'll preserve name and update the ID. This will # overwrite the previous row and set a new ID. res = (Emp .insert(first='foo', last='bar', empno='125') .on_conflict( conflict_target=(Emp.empno,), preserve=(Emp.first, Emp.last), update={Emp.empno: '125.1'}) .execute()) self.assertData([ ('huey', 'cat', '123'), ('zaizee', 'cat', '124'), ('foo', 'bar', '125.1')]) # Conflicts on first/last name. The first name is preserved while the # last-name is updated. The new empno is thrown out. res = (Emp .insert(first='foo', last='bar', empno='126') .on_conflict( conflict_target=(Emp.first, Emp.last), preserve=(Emp.first,), update={Emp.last: 'baze'}) .execute()) self.assertData([ ('huey', 'cat', '123'), ('zaizee', 'cat', '124'), ('foo', 'baze', '125.1')]) @requires_upsert @requires_models(OCTest) def test_update_ignore_with_conflict_target(self): query = OCTest.insert(a='foo', b=1).on_conflict( action='IGNORE', conflict_target=(OCTest.a,)) rowid1 = query.execute() self.assertTrue(rowid1 is not None) query.clone().execute() # Nothing happens, insert is ignored. self.assertEqual(OCTest.select().count(), 1) OCTest.insert(a='foo', b=2).on_conflict_ignore().execute() self.assertEqual(OCTest.select().count(), 1) OCTest.insert(a='bar', b=1).on_conflict_ignore().execute() self.assertEqual(OCTest.select().count(), 2) @requires_upsert @requires_models(OCTest) def test_update_atomic(self): # Add a new row with the given "a" value. If a conflict occurs, # re-insert with b=b+2. query = OCTest.insert(a='foo', b=1).on_conflict( conflict_target=(OCTest.a,), update={OCTest.b: OCTest.b + 2}) # First execution returns rowid=1. Second execution hits the conflict- # resolution, and will update the value in "b" from 1 -> 3. rowid1 = query.execute() rowid2 = query.clone().execute() self.assertEqual(rowid1, rowid2) obj = OCTest.get() self.assertEqual(obj.a, 'foo') self.assertEqual(obj.b, 3) query = OCTest.insert(a='foo', b=4, c=5).on_conflict( conflict_target=[OCTest.a], preserve=[OCTest.c], update={OCTest.b: OCTest.b + 100}) self.assertEqual(query.execute(), rowid2) obj = OCTest.get() self.assertEqual(obj.a, 'foo') self.assertEqual(obj.b, 103) self.assertEqual(obj.c, 5) @requires_upsert @requires_models(OCTest) def test_update_where_clause(self): # Add a new row with the given "a" value. If a conflict occurs, # re-insert with b=b+2 so long as the original b < 3. query = OCTest.insert(a='foo', b=1).on_conflict( conflict_target=(OCTest.a,), update={OCTest.b: OCTest.b + 2}, where=(OCTest.b < 3)) # First execution returns rowid=1. Second execution hits the conflict- # resolution, and will update the value in "b" from 1 -> 3. rowid1 = query.execute() rowid2 = query.clone().execute() self.assertEqual(rowid1, rowid2) obj = OCTest.get() self.assertEqual(obj.a, 'foo') self.assertEqual(obj.b, 3) # Third execution also returns rowid=1. The WHERE clause prevents us # from updating "b" again. If this is SQLite, we get the rowid back, if # this is Postgresql we get None (since nothing happened). rowid3 = query.clone().execute() if IS_SQLITE: self.assertEqual(rowid1, rowid3) else: self.assertTrue(rowid3 is None) # Because we didn't satisfy the WHERE clause, the value in "b" is # not incremented again. obj = OCTest.get() self.assertEqual(obj.a, 'foo') self.assertEqual(obj.b, 3) @requires_upsert @requires_models(Emp) # Has unique on first/last, unique on empno. def test_conflict_update_excluded(self): e1 = Emp.create(first='huey', last='c', empno='10') e2 = Emp.create(first='zaizee', last='c', empno='20') res = (Emp.insert(first='huey', last='c', empno='30') .on_conflict(conflict_target=(Emp.first, Emp.last), update={Emp.empno: Emp.empno + EXCLUDED.empno}, where=(EXCLUDED.empno != Emp.empno)) .execute()) data = sorted(Emp.select(Emp.first, Emp.last, Emp.empno).tuples()) self.assertEqual(data, [('huey', 'c', '1030'), ('zaizee', 'c', '20')]) @requires_upsert @requires_models(KV) def test_conflict_update_excluded2(self): KV.create(key='k1', value=1) query = (KV.insert(key='k1', value=10) .on_conflict(conflict_target=[KV.key], update={KV.value: KV.value + EXCLUDED.value}, where=(EXCLUDED.value > KV.value))) query.execute() self.assertEqual(KV.select(KV.key, KV.value).tuples()[:], [('k1', 11)]) # Running it again will have no effect this time, since the new value # (10) is not greater than the pre-existing row value (11). query.execute() self.assertEqual(KV.select(KV.key, KV.value).tuples()[:], [('k1', 11)]) @requires_upsert @skip_if(IS_CRDB, 'crdb does not support the WHERE clause') @requires_models(UKVP) def test_conflict_target_constraint_where(self): u1 = UKVP.create(key='k1', value=1, extra=1) u2 = UKVP.create(key='k2', value=2, extra=2) fields = [UKVP.key, UKVP.value, UKVP.extra] data = [('k1', 1, 2), ('k2', 2, 3)] # XXX: SQLite does not seem to accept parameterized values for the # conflict target WHERE clause (e.g., the partial index). So we have to # express this literally as ("extra" > 1) rather than using an # expression which will be parameterized. Hopefully SQLite's authors # decide this is a bug and fix it. if IS_SQLITE: conflict_where = UKVP.extra > SQL('1') else: conflict_where = UKVP.extra > 1 res = (UKVP.insert_many(data, fields) .on_conflict(conflict_target=(UKVP.key, UKVP.value), conflict_where=conflict_where, preserve=(UKVP.extra,)) .execute()) # How many rows exist? The first one would not have triggered the # conflict resolution, since the existing k1/1 row's "extra" value was # not greater than 1, thus it did not satisfy the index condition. # The second row (k2/2/3) would have triggered the resolution. self.assertEqual(UKVP.select().count(), 3) query = (UKVP .select(UKVP.key, UKVP.value, UKVP.extra) .order_by(UKVP.key, UKVP.value, UKVP.extra) .tuples()) self.assertEqual(list(query), [ ('k1', 1, 1), ('k1', 1, 2), ('k2', 2, 3)]) # Verify the primary-key of k2 did not change. u2_db = UKVP.get(UKVP.key == 'k2') self.assertEqual(u2_db.id, u2.id) @requires_mysql class TestUpsertMySQL(OnConflictTests, ModelTestCase): def test_replace(self): # Unique constraint on first/last would fail - replace. query = (Emp .insert(first='mickey', last='dog', empno='1337') .on_conflict('replace') .execute()) self.assertData([ ('huey', 'cat', '123'), ('zaizee', 'cat', '124'), ('mickey', 'dog', '1337')]) # Unique constraint on empno would fail - replace. query = (Emp .insert(first='nuggie', last='dog', empno='123') .on_conflict('replace') .execute()) self.assertData([ ('zaizee', 'cat', '124'), ('mickey', 'dog', '1337'), ('nuggie', 'dog', '123')]) # No problems, data added. query = (Emp .insert(first='beanie', last='cat', empno='126') .on_conflict('replace') .execute()) self.assertData([ ('zaizee', 'cat', '124'), ('mickey', 'dog', '1337'), ('nuggie', 'dog', '123'), ('beanie', 'cat', '126')]) @requires_models(OCTest) def test_update(self): pk = (OCTest .insert(a='a', b=3) .on_conflict(update={OCTest.b: 1337}) .execute()) oc = OCTest.get(OCTest.a == 'a') self.assertEqual(oc.b, 3) pk2 = (OCTest .insert(a='a', b=4) .on_conflict(update={OCTest.b: OCTest.b + 10}) .execute()) self.assertEqual(pk, pk2) self.assertEqual(OCTest.select().count(), 1) oc = OCTest.get(OCTest.a == 'a') self.assertEqual(oc.b, 13) pk3 = (OCTest .insert(a='a2', b=5) .on_conflict(update={OCTest.b: 1337}) .execute()) self.assertTrue(pk3 != pk2) self.assertEqual(OCTest.select().count(), 2) oc = OCTest.get(OCTest.a == 'a2') self.assertEqual(oc.b, 5) @requires_models(OCTest) def test_update_preserve(self): OCTest.create(a='a', b=3) pk = (OCTest .insert(a='a', b=4) .on_conflict(preserve=[OCTest.b]) .execute()) oc = OCTest.get(OCTest.a == 'a') self.assertEqual(oc.b, 4) pk2 = (OCTest .insert(a='a', b=5, c=6) .on_conflict( preserve=[OCTest.c], update={OCTest.b: OCTest.b + 100}) .execute()) self.assertEqual(pk, pk2) self.assertEqual(OCTest.select().count(), 1) oc = OCTest.get(OCTest.a == 'a') self.assertEqual(oc.b, 104) self.assertEqual(oc.c, 6) class TestReplaceSqlite(OnConflictTests, ModelTestCase): database = get_in_memory_db() def test_replace(self): # Unique constraint on first/last would fail - replace. query = (Emp .insert(first='mickey', last='dog', empno='1337') .on_conflict('replace') .execute()) self.assertData([ ('huey', 'cat', '123'), ('zaizee', 'cat', '124'), ('mickey', 'dog', '1337')]) # Unique constraint on empno would fail - replace. query = (Emp .insert(first='nuggie', last='dog', empno='123') .on_conflict('replace') .execute()) self.assertData([ ('zaizee', 'cat', '124'), ('mickey', 'dog', '1337'), ('nuggie', 'dog', '123')]) # No problems, data added. query = (Emp .insert(first='beanie', last='cat', empno='126') .on_conflict('replace') .execute()) self.assertData([ ('zaizee', 'cat', '124'), ('mickey', 'dog', '1337'), ('nuggie', 'dog', '123'), ('beanie', 'cat', '126')]) def test_model_replace(self): Emp.replace(first='mickey', last='dog', empno='1337').execute() self.assertData([ ('huey', 'cat', '123'), ('zaizee', 'cat', '124'), ('mickey', 'dog', '1337')]) Emp.replace(first='beanie', last='cat', empno='999').execute() self.assertData([ ('huey', 'cat', '123'), ('zaizee', 'cat', '124'), ('mickey', 'dog', '1337'), ('beanie', 'cat', '999')]) Emp.replace_many([('h', 'cat', '123'), ('z', 'cat', '124'), ('b', 'cat', '125')], fields=[Emp.first, Emp.last, Emp.empno]).execute() self.assertData([ ('mickey', 'dog', '1337'), ('beanie', 'cat', '999'), ('h', 'cat', '123'), ('z', 'cat', '124'), ('b', 'cat', '125')]) @requires_sqlite class TestUpsertSqlite(PGOnConflictTests, ModelTestCase): database = get_in_memory_db() @skip_if(IS_SQLITE_24, 'requires sqlite < 3.24') def test_no_preserve_update_where(self): # Ensure on SQLite < 3.24 we cannot update or preserve values. base = Emp.insert(first='foo', last='bar', empno='125') preserve = base.on_conflict(preserve=[Emp.last]) self.assertRaises(ValueError, preserve.execute) update = base.on_conflict(update={Emp.empno: 'xxx'}) self.assertRaises(ValueError, update.execute) where = base.on_conflict(where=(Emp.id > 10)) self.assertRaises(ValueError, where.execute) @skip_unless(IS_SQLITE_24, 'requires sqlite >= 3.24') def test_update_meets_requirements(self): # Ensure that on >= 3.24 any updates meet the minimum criteria. base = Emp.insert(first='foo', last='bar', empno='125') # Must specify update or preserve. no_update_preserve = base.on_conflict(conflict_target=(Emp.empno,)) self.assertRaises(ValueError, no_update_preserve.execute) # Must specify a conflict target. no_conflict_target = base.on_conflict(update={Emp.empno: '125.1'}) self.assertRaises(ValueError, no_conflict_target.execute) @skip_unless(IS_SQLITE_24, 'requires sqlite >= 3.24') def test_do_nothing(self): query = (Emp .insert(first='foo', last='bar', empno='123') .on_conflict('nothing')) self.assertSQL(query, ( 'INSERT INTO "emp" ("first", "last", "empno") ' 'VALUES (?, ?, ?) ON CONFLICT DO NOTHING'), ['foo', 'bar', '123']) query.execute() # Conflict occurs with empno='123'. self.assertData(list(self.test_data)) class UKV(TestModel): key = TextField() value = TextField() extra = TextField(default='') class Meta: constraints = [ SQL('constraint ukv_key_value unique(key, value)'), ] class UKVRel(TestModel): key = TextField() value = TextField() extra = TextField() class Meta: indexes = ( (('key', 'value'), True), ) @requires_pglike class TestUpsertPostgresql(PGOnConflictTests, ModelTestCase): @requires_postgresql @requires_models(UKV) def test_conflict_target_constraint(self): u1 = UKV.create(key='k1', value='v1') u2 = UKV.create(key='k2', value='v2') ret = (UKV.insert(key='k1', value='v1', extra='e1') .on_conflict(conflict_target=(UKV.key, UKV.value), preserve=(UKV.extra,)) .execute()) self.assertEqual(ret, u1.id) # Changes were saved successfully. u1_db = UKV.get(UKV.key == 'k1') self.assertEqual(u1_db.key, 'k1') self.assertEqual(u1_db.value, 'v1') self.assertEqual(u1_db.extra, 'e1') self.assertEqual(UKV.select().count(), 2) ret = (UKV.insert(key='k2', value='v2', extra='e2') .on_conflict(conflict_constraint='ukv_key_value', preserve=(UKV.extra,)) .execute()) self.assertEqual(ret, u2.id) # Changes were saved successfully. u2_db = UKV.get(UKV.key == 'k2') self.assertEqual(u2_db.key, 'k2') self.assertEqual(u2_db.value, 'v2') self.assertEqual(u2_db.extra, 'e2') self.assertEqual(UKV.select().count(), 2) ret = (UKV.insert(key='k3', value='v3', extra='e3') .on_conflict(conflict_target=[UKV.key, UKV.value], preserve=[UKV.extra]) .execute()) self.assertTrue(ret > u2_db.id) self.assertEqual(UKV.select().count(), 3) @requires_models(UKV, UKVRel) def test_conflict_ambiguous_column(self): # k1/v1/e1, k2/v2/e0, k3/v3/e1 for i in [1, 2, 3]: UKV.create(key='k%s' % i, value='v%s' % i, extra='e%s' % (i % 2)) UKVRel.create(key='k1', value='v1', extra='x1') UKVRel.create(key='k2', value='v2', extra='x2') subq = UKV.select(UKV.key, UKV.value, UKV.extra) query = (UKVRel .insert_from(subq, [UKVRel.key, UKVRel.value, UKVRel.extra]) .on_conflict(conflict_target=[UKVRel.key, UKVRel.value], preserve=[UKVRel.extra], where=(UKVRel.key != 'k2'))) self.assertSQL(query, ( 'INSERT INTO "ukv_rel" ("key", "value", "extra") ' 'SELECT "t1"."key", "t1"."value", "t1"."extra" FROM "ukv" AS "t1" ' 'ON CONFLICT ("key", "value") DO UPDATE ' 'SET "extra" = EXCLUDED."extra" ' 'WHERE ("ukv_rel"."key" != ?) RETURNING "ukv_rel"."id"'), ['k2']) query.execute() query = (UKVRel .select(UKVRel.key, UKVRel.value, UKVRel.extra) .order_by(UKVRel.key)) self.assertEqual(list(query.tuples()), [ ('k1', 'v1', 'e1'), ('k2', 'v2', 'x2'), ('k3', 'v3', 'e1')]) @requires_models(Emp) def test_upsert_preserves_existing(self): #Emp.create(first='beanie', last='cat', empno='998') Emp.create(first='beanie', last='cat', empno='999') (Emp .insert(first='huey', last='kitten', empno='999') .on_conflict( conflict_target=(Emp.empno,), preserve=(Emp.last,)) .execute()) obj = Emp.get(Emp.empno == '999') self.assertEqual(obj.first, 'beanie') # last was NOT preserved, so it gets the val from the insert. self.assertEqual(obj.last, 'kitten') @requires_models(Emp) def test_upsert_update_expression(self): Emp.create(first='huey', last='cat', empno='999') (Emp .insert(first='hueky', last='kitten', empno='999') .on_conflict( conflict_target=(Emp.empno,), update={Emp.first: Emp.first + 'yyy', Emp.last: Emp.last + 'lands'}) .execute()) obj = Emp.get(Emp.empno == '999') self.assertEqual(obj.first, 'hueyyyy') self.assertEqual(obj.last, 'catlands') class TestJoinSubquery(ModelTestCase): requires = [Person, Relationship] def test_join_subquery(self): # Set up some relationships such that there exists a relationship from # the left-hand to the right-hand name. data = ( ('charlie', None), ('huey', 'charlie'), ('mickey', 'charlie'), ('zaizee', 'charlie'), ('zaizee', 'huey')) people = {} def get_person(name): if name not in people: people[name] = Person.create(first=name, last=name, dob=datetime.date(2017, 1, 1)) return people[name] for person, related_to in data: p1 = get_person(person) if related_to is not None: p2 = get_person(related_to) Relationship.create(from_person=p1, to_person=p2) # Create the subquery. Friend = Person.alias('friend') subq = (Relationship .select(Friend.first.alias('friend_name'), Relationship.from_person) .join(Friend, on=(Relationship.to_person == Friend.id)) .alias('subq')) # Outer query does a LEFT OUTER JOIN. We join on the subquery because # it uses an INNER JOIN, saving us doing two LEFT OUTER joins in the # single query. query = (Person .select(Person.first, subq.c.friend_name) .join(subq, JOIN.LEFT_OUTER, on=(Person.id == subq.c.from_person_id)) .order_by(Person.first, subq.c.friend_name)) self.assertSQL(query, ( 'SELECT "t1"."first", "subq"."friend_name" ' 'FROM "person" AS "t1" ' 'LEFT OUTER JOIN (' 'SELECT "friend"."first" AS "friend_name", "t2"."from_person_id" ' 'FROM "relationship" AS "t2" ' 'INNER JOIN "person" AS "friend" ' 'ON ("t2"."to_person_id" = "friend"."id")) AS "subq" ' 'ON ("t1"."id" = "subq"."from_person_id") ' 'ORDER BY "t1"."first", "subq"."friend_name"'), []) db_data = [row for row in query.tuples()] self.assertEqual(db_data, list(data)) class TestSumCase(ModelTestCase): @requires_models(User) def test_sum_case(self): for username in ('charlie', 'huey', 'zaizee'): User.create(username=username) case = Case(None, [(User.username.endswith('e'), 1)], 0) e_sum = fn.SUM(case) query = (User .select(User.username, e_sum.alias('e_sum')) .group_by(User.username) .order_by(User.username)) self.assertSQL(query, ( 'SELECT "t1"."username", ' 'SUM(CASE WHEN ("t1"."username" ILIKE ?) THEN ? ELSE ? END) ' 'AS "e_sum" ' 'FROM "users" AS "t1" ' 'GROUP BY "t1"."username" ' 'ORDER BY "t1"."username"'), ['%e', 1, 0]) data = [(user.username, user.e_sum) for user in query] self.assertEqual(data, [ ('charlie', 1), ('huey', 0), ('zaizee', 1)]) class TUser(TestModel): username = TextField() class Transaction(TestModel): user = ForeignKeyField(TUser, backref='transactions') amount = FloatField(default=0.) class TestMaxAlias(ModelTestCase): requires = [Transaction, TUser] def test_max_alias(self): with self.database.atomic(): charlie = TUser.create(username='charlie') huey = TUser.create(username='huey') data = ( (charlie, 10.), (charlie, 20.), (charlie, 30.), (huey, 1.5), (huey, 2.5)) for user, amount in data: Transaction.create(user=user, amount=amount) with self.assertQueryCount(1): amount = fn.MAX(Transaction.amount).alias('amount') query = (Transaction .select(amount, TUser.username) .join(TUser) .group_by(TUser.username) .order_by(TUser.username)) data = [(txn.amount, txn.user.username) for txn in query] self.assertEqual(data, [ (30., 'charlie'), (2.5, 'huey')]) class CNote(TestModel): content = TextField() timestamp = TimestampField() class CFile(TestModel): filename = CharField(primary_key=True) data = TextField() timestamp = TimestampField() class TestCompoundSelectModels(ModelTestCase): requires = [CFile, CNote] def setUp(self): super(TestCompoundSelectModels, self).setUp() def generate_ts(): i = [0] def _inner(): i[0] += 1 return datetime.datetime(2018, 1, i[0]) return _inner make_ts = generate_ts() self.ts = lambda i: datetime.datetime(2018, 1, i) with self.database.atomic(): for i, content in enumerate(('note-a', 'note-b', 'note-c'), 1): CNote.create(id=i, content=content, timestamp=make_ts()) file_data = ( ('peewee.txt', 'peewee orm'), ('walrus.txt', 'walrus redis toolkit'), ('huey.txt', 'huey task queue')) for filename, data in file_data: CFile.create(filename=filename, data=data, timestamp=make_ts()) def test_mix_models_with_model_row_type(self): cast = 'CHAR' if IS_MYSQL else 'TEXT' lhs = CNote.select(CNote.id.cast(cast).alias('id_text'), CNote.content, CNote.timestamp) rhs = CFile.select(CFile.filename, CFile.data, CFile.timestamp) query = (lhs | rhs).order_by(SQL('timestamp')).limit(4) data = [(n.id_text, n.content, n.timestamp) for n in query] self.assertEqual(data, [ ('1', 'note-a', self.ts(1)), ('2', 'note-b', self.ts(2)), ('3', 'note-c', self.ts(3)), ('peewee.txt', 'peewee orm', self.ts(4))]) def test_mixed_models_tuple_row_type(self): cast = 'CHAR' if IS_MYSQL else 'TEXT' lhs = CNote.select(CNote.id.cast(cast).alias('id'), CNote.content, CNote.timestamp) rhs = CFile.select(CFile.filename, CFile.data, CFile.timestamp) query = (lhs | rhs).order_by(SQL('timestamp')).limit(5) self.assertEqual(list(query.tuples()), [ ('1', 'note-a', self.ts(1)), ('2', 'note-b', self.ts(2)), ('3', 'note-c', self.ts(3)), ('peewee.txt', 'peewee orm', self.ts(4)), ('walrus.txt', 'walrus redis toolkit', self.ts(5))]) def test_mixed_models_dict_row_type(self): notes = CNote.select(CNote.content, CNote.timestamp) files = CFile.select(CFile.filename, CFile.timestamp) query = (notes | files).order_by(SQL('timestamp').desc()).limit(4) self.assertEqual(list(query.dicts()), [ {'content': 'huey.txt', 'timestamp': self.ts(6)}, {'content': 'walrus.txt', 'timestamp': self.ts(5)}, {'content': 'peewee.txt', 'timestamp': self.ts(4)}, {'content': 'note-c', 'timestamp': self.ts(3)}]) class SequenceModel(TestModel): seq_id = IntegerField(sequence='seq_id_sequence') key = TextField() @requires_pglike class TestSequence(ModelTestCase): requires = [SequenceModel] def test_create_table(self): query = SequenceModel._schema._create_table() self.assertSQL(query, ( 'CREATE TABLE IF NOT EXISTS "sequence_model" (' '"id" SERIAL NOT NULL PRIMARY KEY, ' '"seq_id" INTEGER NOT NULL DEFAULT NEXTVAL(\'seq_id_sequence\'), ' '"key" TEXT NOT NULL)'), []) def test_sequence(self): for key in ('k1', 'k2', 'k3'): SequenceModel.create(key=key) s1, s2, s3 = SequenceModel.select().order_by(SequenceModel.key) self.assertEqual(s1.seq_id, 1) self.assertEqual(s2.seq_id, 2) self.assertEqual(s3.seq_id, 3) @requires_postgresql class TestUpdateFromIntegration(ModelTestCase): requires = [User] def test_update_from(self): u1, u2 = [User.create(username=username) for username in ('u1', 'u2')] data = [(u1.id, 'u1-x'), (u2.id, 'u2-x')] vl = ValuesList(data, columns=('id', 'username'), alias='tmp') (User .update({User.username: vl.c.username}) .from_(vl) .where(User.id == vl.c.id) .execute()) usernames = [u.username for u in User.select().order_by(User.username)] self.assertEqual(usernames, ['u1-x', 'u2-x']) def test_update_from_subselect(self): u1, u2 = [User.create(username=username) for username in ('u1', 'u2')] data = [(u1.id, 'u1-y'), (u2.id, 'u2-y')] vl = ValuesList(data, columns=('id', 'username'), alias='tmp') subq = vl.select(vl.c.id, vl.c.username) (User .update({User.username: subq.c.username}) .from_(subq) .where(User.id == subq.c.id) .execute()) usernames = [u.username for u in User.select().order_by(User.username)] self.assertEqual(usernames, ['u1-y', 'u2-y']) @requires_models(User, Tweet) def test_update_from_simple(self): u = User.create(username='u1') t1 = Tweet.create(user=u, content='t1') t2 = Tweet.create(user=u, content='t2') (User .update({User.username: Tweet.content}) .from_(Tweet) .where(Tweet.content == 't2') .execute()) self.assertEqual(User.get(User.id == u.id).username, 't2') @requires_postgresql class TestLateralJoin(ModelTestCase): requires = [User, Tweet] def test_lateral_join(self): with self.database.atomic(): for i in range(3): u = User.create(username='u%s' % i) for j in range(4): Tweet.create(user=u, content='u%s-t%s' % (i, j)) # GOAL: query users and their 2 most-recent tweets (by ID). TA = Tweet.alias() # The "outer loop" will be iterating over the users whose tweets we are # trying to find. user_query = (User .select(User.id, User.username) .order_by(User.id) .alias('uq')) # The inner loop will select tweets and is correlated to the outer loop # via the WHERE clause. Note that we are using a LIMIT clause. tweet_query = (TA .select(TA.id, TA.content) .where(TA.user == user_query.c.id) .order_by(TA.id.desc()) .limit(2) .alias('pq')) join = NodeList((user_query, SQL('LEFT JOIN LATERAL'), tweet_query, SQL('ON %s', [True]))) query = (Tweet .select(user_query.c.username, tweet_query.c.content) .from_(join) .dicts()) self.assertEqual([row for row in query], [ {'username': 'u0', 'content': 'u0-t3'}, {'username': 'u0', 'content': 'u0-t2'}, {'username': 'u1', 'content': 'u1-t3'}, {'username': 'u1', 'content': 'u1-t2'}, {'username': 'u2', 'content': 'u2-t3'}, {'username': 'u2', 'content': 'u2-t2'}]) class Task(TestModel): heading = ForeignKeyField('self', backref='tasks', null=True) project = ForeignKeyField('self', backref='projects', null=True) title = TextField() type = IntegerField() PROJECT = 1 HEADING = 2 class TestMultiSelfJoin(ModelTestCase): requires = [Task] def setUp(self): super(TestMultiSelfJoin, self).setUp() with self.database.atomic(): p_dev = Task.create(title='dev', type=Task.PROJECT) p_p = Task.create(title='peewee', project=p_dev, type=Task.PROJECT) p_h = Task.create(title='huey', project=p_dev, type=Task.PROJECT) heading_data = ( ('peewee-1', p_p, 2), ('peewee-2', p_p, 0), ('huey-1', p_h, 1), ('huey-2', p_h, 1)) for title, proj, n_subtasks in heading_data: t = Task.create(title=title, project=proj, type=Task.HEADING) for i in range(n_subtasks): Task.create(title='%s-%s' % (title, i + 1), project=proj, heading=t, type=Task.HEADING) def test_multi_self_join(self): Project = Task.alias() Heading = Task.alias() query = (Task .select(Task, Project, Heading) .join(Heading, JOIN.LEFT_OUTER, on=(Task.heading == Heading.id).alias('heading')) .switch(Task) .join(Project, JOIN.LEFT_OUTER, on=(Task.project == Project.id).alias('project')) .order_by(Task.id)) with self.assertQueryCount(1): accum = [] for task in query: h_title = task.heading.title if task.heading else None p_title = task.project.title if task.project else None accum.append((task.title, h_title, p_title)) self.assertEqual(accum, [ # title - heading - project ('dev', None, None), ('peewee', None, 'dev'), ('huey', None, 'dev'), ('peewee-1', None, 'peewee'), ('peewee-1-1', 'peewee-1', 'peewee'), ('peewee-1-2', 'peewee-1', 'peewee'), ('peewee-2', None, 'peewee'), ('huey-1', None, 'huey'), ('huey-1-1', 'huey-1', 'huey'), ('huey-2', None, 'huey'), ('huey-2-1', 'huey-2', 'huey'), ]) class Product(TestModel): name = TextField() price = IntegerField() flags = IntegerField(constraints=[SQL('DEFAULT 99')]) status = CharField(constraints=[Check("status IN ('a', 'b', 'c')")]) class Meta: constraints = [Check('price > 0')] class TestModelConstraints(ModelTestCase): requires = [Product] def test_model_constraints(self): p = Product.create(name='p1', price=1, status='a') self.assertTrue(p.flags is None) # Price was saved successfully, flags got server-side default value. p_db = Product.get(Product.id == p.id) self.assertEqual(p_db.price, 1) self.assertEqual(p_db.flags, 99) self.assertEqual(p_db.status, 'a') # Cannot update price with invalid value, must be > 0. with self.database.atomic(): p.price = -1 self.assertRaises(DatabaseError, p.save) # Nor can we create a new product with an invalid price. with self.database.atomic(): self.assertRaises(DatabaseError, Product.create, name='p2', price=0, status='a') # Cannot set status to a value other than 1, 2 or 3. with self.database.atomic(): p.price = 1 p.status = 'd' self.assertRaises(DatabaseError, p.save) # Cannot create a new product with invalid status. with self.database.atomic(): self.assertRaises(DatabaseError, Product.create, name='p3', price=1, status='x') class TestModelFieldReprs(BaseTestCase): def test_model_reprs(self): class User(Model): username = TextField(primary_key=True) class Tweet(Model): user = ForeignKeyField(User, backref='tweets') content = TextField() timestamp = TimestampField() class EAV(Model): entity = TextField() attribute = TextField() value = TextField() class Meta: primary_key = CompositeKey('entity', 'attribute') class NoPK(Model): key = TextField() class Meta: primary_key = False self.assertEqual(repr(User), '') self.assertEqual(repr(Tweet), '') self.assertEqual(repr(EAV), '') self.assertEqual(repr(NoPK), '') self.assertEqual(repr(User()), '') self.assertEqual(repr(Tweet()), '') self.assertEqual(repr(EAV()), '') self.assertEqual(repr(NoPK()), '') self.assertEqual(repr(User(username='huey')), '') self.assertEqual(repr(Tweet(id=1337)), '') self.assertEqual(repr(EAV(entity='e', attribute='a')), "") self.assertEqual(repr(NoPK(key='k')), '') self.assertEqual(repr(User.username), '') self.assertEqual(repr(Tweet.user), '') self.assertEqual(repr(EAV.entity), '') self.assertEqual(repr(TextField()), '') def test_model_str_method(self): class User(Model): username = TextField(primary_key=True) def __str__(self): return self.username.title() u = User(username='charlie') self.assertEqual(repr(u), '') class TestGetWithSecondDatabase(ModelTestCase): database = get_in_memory_db() requires = [User] def test_get_with_second_database(self): User.create(username='huey') query = User.select().where(User.username == 'huey') self.assertEqual(query.get().username, 'huey') alt_db = get_in_memory_db() with User.bind_ctx(alt_db): User.create_table() self.assertRaises(User.DoesNotExist, query.get, alt_db) with User.bind_ctx(alt_db): User.create(username='zaizee') query = User.select().where(User.username == 'zaizee') self.assertRaises(User.DoesNotExist, query.get) self.assertEqual(query.get(alt_db).username, 'zaizee') class TestMixModelsTables(ModelTestCase): database = get_in_memory_db() requires = [User] def test_mix_models_tables(self): Tbl = User._meta.table self.assertEqual(Tbl.insert({Tbl.username: 'huey'}).execute(), 1) huey = Tbl.select(User.username).get() self.assertEqual(huey, {'username': 'huey'}) huey = User.select(Tbl.username).get() self.assertEqual(huey.username, 'huey') Tbl.update(username='huey-x').where(Tbl.username == 'huey').execute() self.assertEqual(User.select().get().username, 'huey-x') Tbl.delete().where(User.username == 'huey-x').execute() self.assertEqual(Tbl.select().count(), 0) class TestDatabaseExecuteQuery(ModelTestCase): database = get_in_memory_db() requires = [User] def test_execute_query(self): for username in ('huey', 'zaizee'): User.create(username=username) query = User.select().order_by(User.username.desc()) cursor = self.database.execute(query) self.assertEqual([row[1] for row in cursor], ['zaizee', 'huey']) class Datum(TestModel): key = TextField() value = IntegerField(null=True) class TestNullOrdering(ModelTestCase): requires = [Datum] def test_null_ordering(self): values = [('k1', 1), ('ka', None), ('k2', 2), ('kb', None)] Datum.insert_many(values, fields=[Datum.key, Datum.value]).execute() def assertOrder(ordering, expected): query = Datum.select().order_by(*ordering) self.assertEqual([d.key for d in query], expected) # Ascending order. nulls_last = (Datum.value.asc(nulls='last'), Datum.key) assertOrder(nulls_last, ['k1', 'k2', 'ka', 'kb']) nulls_first = (Datum.value.asc(nulls='first'), Datum.key) assertOrder(nulls_first, ['ka', 'kb', 'k1', 'k2']) # Descending order. nulls_last = (Datum.value.desc(nulls='last'), Datum.key) assertOrder(nulls_last, ['k2', 'k1', 'ka', 'kb']) nulls_first = (Datum.value.desc(nulls='first'), Datum.key) assertOrder(nulls_first, ['ka', 'kb', 'k2', 'k1']) # Invalid values. self.assertRaises(ValueError, Datum.value.desc, nulls='bar') self.assertRaises(ValueError, Datum.value.asc, nulls='foo') class Student(TestModel): name = TextField() class Course(TestModel): name = TextField() class Attendance(TestModel): student = ForeignKeyField(Student) course = ForeignKeyField(Course) class TestManyToManyJoining(ModelTestCase): requires = [Student, Course, Attendance] def setUp(self): super(TestManyToManyJoining, self).setUp() data = ( ('charlie', ('eng101', 'cs101', 'cs111')), ('huey', ('cats1', 'cats2', 'cats3')), ('zaizee', ('cats2', 'cats3'))) c = {} with self.database.atomic(): for name, courses in data: student = Student.create(name=name) for course in courses: if course not in c: c[course] = Course.create(name=course) Attendance.create(student=student, course=c[course]) def assertQuery(self, query): with self.assertQueryCount(1): query = query.order_by(Attendance.id) results = [(a.student.name, a.course.name) for a in query] self.assertEqual(results, [ ('charlie', 'eng101'), ('charlie', 'cs101'), ('charlie', 'cs111'), ('huey', 'cats1'), ('huey', 'cats2'), ('zaizee', 'cats2')]) def test_join_subquery(self): courses = (Course .select(Course.id, Course.name) .order_by(Course.id) .limit(5)) query = (Attendance .select(Attendance, Student, courses.c.name) .join_from(Attendance, Student) .join_from(Attendance, courses, on=(Attendance.course == courses.c.id))) self.assertQuery(query) @skip_if(IS_MYSQL) def test_join_where_subquery(self): courses = Course.select().order_by(Course.id).limit(5) query = (Attendance .select(Attendance, Student, Course) .join_from(Attendance, Student) .join_from(Attendance, Course) .where(Attendance.course.in_(courses))) self.assertQuery(query) class TestColumnNameStripping(ModelTestCase): database = get_in_memory_db() requires = [Person] def test_column_name_stripping(self): d1 = datetime.date(1990, 1, 1) d2 = datetime.date(1990, 1, 1) p1 = Person.create(first='f1', last='l1', dob=d1) p2 = Person.create(first='f2', last='l2', dob=d2) query = Person.select( fn.MIN(Person.dob), fn.MAX(Person.dob).alias('mdob')) # Get the row as a model. row = query.get() self.assertEqual(row.dob, d1) self.assertEqual(row.mdob, d2) row = query.dicts().get() self.assertEqual(row['dob'], d1) self.assertEqual(row['mdob'], d2) class VL(TestModel): n = IntegerField() s = CharField() @skip_if(IS_SQLITE_OLD or IS_MYSQL or IS_CRDB) class TestValuesListIntegration(ModelTestCase): requires = [VL] _data = [(1, 'one'), (2, 'two'), (3, 'three')] def test_insert_into_select_from_vl(self): vl = ValuesList(self._data) cte = vl.cte('newvals', columns=['n', 's']) res = (VL .insert_from(cte.select(cte.c.n, cte.c.s), fields=[VL.n, VL.s]) .with_cte(cte) .execute()) vq = VL.select().order_by(VL.n) self.assertEqual([(v.n, v.s) for v in vq], self._data) def test_update_vl_cte(self): VL.insert_many(self._data).execute() new_values = [(1, 'One'), (3, 'Three'), (4, 'Four')] cte = ValuesList(new_values).cte('new_values', columns=('n', 's')) # We have to use a subquery to update the individual column, as SQLite # does not support UPDATE/FROM syntax. subq = (cte .select(cte.c.s) .where(VL.n == cte.c.n)) # Perform the update, assigning extra the new value from the values # list, and restricting the overall update using the composite pk. res = (VL .update(s=subq) .where(VL.n.in_(cte.select(cte.c.n))) .with_cte(cte) .execute()) vq = VL.select().order_by(VL.n) self.assertEqual([(v.n, v.s) for v in vq], [ (1, 'One'), (2, 'two'), (3, 'Three')]) def test_values_list(self): vl = ValuesList(self._data) query = vl.select(SQL('*')) self.assertEqual(list(query.tuples().bind(self.database)), self._data) @requires_postgresql def test_values_list_named_columns(self): vl = ValuesList(self._data).columns('idx', 'name') query = (vl .select(vl.c.idx, vl.c.name) .order_by(vl.c.idx.desc())) self.assertEqual(list(query.tuples().bind(self.database)), self._data[::-1]) def test_values_list_named_columns_in_cte(self): vl = ValuesList(self._data) cte = vl.cte('val', columns=('idx', 'name')) query = (cte .select(cte.c.idx, cte.c.name) .order_by(cte.c.idx.desc()) .with_cte(cte)) self.assertEqual(list(query.tuples().bind(self.database)), self._data[::-1]) def test_named_values_list(self): vl = ValuesList(self._data).alias('vl') query = vl.select() self.assertEqual(list(query.tuples().bind(self.database)), self._data) class C_Product(TestModel): name = CharField() price = IntegerField(default=0) class C_Archive(TestModel): name = CharField() price = IntegerField(default=0) class C_Part(TestModel): part = CharField(primary_key=True) sub_part = ForeignKeyField('self', null=True) @skip_unless(IS_POSTGRESQL) class TestDataModifyingCTEIntegration(ModelTestCase): requires = [C_Product, C_Archive, C_Part] def setUp(self): super(TestDataModifyingCTEIntegration, self).setUp() for i in range(5): C_Product.create(name='p%s' % i, price=i) mp1_c_g = C_Part.create(part='mp1-c-g') mp1_c = C_Part.create(part='mp1-c', sub_part=mp1_c_g) mp1 = C_Part.create(part='mp1', sub_part=mp1_c) mp2_c_g = C_Part.create(part='mp2-c-g') mp2_c = C_Part.create(part='mp2-c', sub_part=mp2_c_g) mp2 = C_Part.create(part='mp2', sub_part=mp2_c) def test_data_modifying_cte_delete(self): query = (C_Product.delete() .where(C_Product.price < 3) .returning(C_Product)) cte = query.cte('moved_rows') src = Select((cte,), (cte.c.id, cte.c.name, cte.c.price)) res = (C_Archive .insert_from(src, (C_Archive.id, C_Archive.name, C_Archive.price)) .with_cte(cte) .execute()) self.assertEqual(len(list(res)), 3) self.assertEqual( sorted([(p.name, p.price) for p in C_Product.select()]), [('p3', 3), ('p4', 4)]) self.assertEqual( sorted([(p.name, p.price) for p in C_Archive.select()]), [('p0', 0), ('p1', 1), ('p2', 2)]) base = (C_Part .select(C_Part.sub_part, C_Part.part) .where(C_Part.part == 'mp1') .cte('included_parts', recursive=True, columns=('sub_part', 'part'))) PA = C_Part.alias('p') recursive = (PA .select(PA.sub_part, PA.part) .join(base, on=(PA.part == base.c.sub_part))) cte = base.union_all(recursive) sq = Select((cte,), (cte.c.part,)) res = (C_Part.delete() .where(C_Part.part.in_(sq)) .with_cte(cte) .execute()) self.assertEqual(sorted([p.part for p in C_Part.select()]), ['mp2', 'mp2-c', 'mp2-c-g']) def test_data_modifying_cte_update(self): # Populate archive table w/copy of data in product. C_Archive.insert_from( C_Product.select(), (C_Product.id, C_Product.name, C_Product.price)).execute() query = (C_Product .update(price=C_Product.price * 2) .returning(C_Product.id, C_Product.name, C_Product.price)) cte = query.cte('t') sq = cte.select_from(cte.c.id, cte.c.name, cte.c.price) self.assertEqual(sorted([(x.name, x.price) for x in sq]), [ ('p0', 0), ('p1', 2), ('p2', 4), ('p3', 6), ('p4', 8)]) # Ensure changes were persisted. self.assertEqual(sorted([(x.name, x.price) for x in C_Product]), [ ('p0', 0), ('p1', 2), ('p2', 4), ('p3', 6), ('p4', 8)]) sq = Select((cte,), (cte.c.id, cte.c.price)) res = (C_Archive .update(price=sq.c.price) .from_(sq) .where(C_Archive.id == sq.c.id) .with_cte(cte) .execute()) self.assertEqual(sorted([(x.name, x.price) for x in C_Product]), [ ('p0', 0), ('p1', 4), ('p2', 8), ('p3', 12), ('p4', 16)]) self.assertEqual(sorted([(x.name, x.price) for x in C_Archive]), [ ('p0', 0), ('p1', 4), ('p2', 8), ('p3', 12), ('p4', 16)]) def test_data_modifying_cte_insert(self): query = (C_Product .insert({'name': 'p5', 'price': 5}) .returning(C_Product.id, C_Product.name, C_Product.price)) cte = query.cte('t') sq = cte.select_from(cte.c.id, cte.c.name, cte.c.price) self.assertEqual([(p.name, p.price) for p in sq], [('p5', 5)]) query = (C_Product .insert({'name': 'p6', 'price': 6}) .returning(C_Product.id, C_Product.name, C_Product.price)) cte = query.cte('t') sq = Select((cte,), (cte.c.id, cte.c.name, cte.c.price)) res = (C_Archive .insert_from(sq, (sq.c.id, sq.c.name, sq.c.price)) .with_cte(cte) .execute()) self.assertEqual([(p.name, p.price) for p in C_Archive], [('p6', 6)]) self.assertEqual(sorted([(p.name, p.price) for p in C_Product]), [ ('p0', 0), ('p1', 1), ('p2', 2), ('p3', 3), ('p4', 4), ('p5', 5), ('p6', 6)]) class TestBindTo(ModelTestCase): requires = [User, Tweet] def test_bind_to(self): for i in (1, 2, 3): user = User.create(username='u%s' % i) Tweet.create(user=user, content='t%s' % i) # Alias to a particular field-name. name = Case(User.username, [ ('u1', 'user 1'), ('u2', 'user 2')], 'someone else') q = (Tweet .select(Tweet.content, name.alias('username').bind_to(User)) .join(User) .order_by(Tweet.content)) with self.assertQueryCount(1): self.assertEqual([(t.content, t.user.username) for t in q], [ ('t1', 'user 1'), ('t2', 'user 2'), ('t3', 'someone else')]) # Use a different alias. q = (Tweet .select(Tweet.content, name.alias('display').bind_to(User)) .join(User) .order_by(Tweet.content)) with self.assertQueryCount(1): self.assertEqual([(t.content, t.user.display) for t in q], [ ('t1', 'user 1'), ('t2', 'user 2'), ('t3', 'someone else')]) # Ensure works with model and field aliases. TA, UA = Tweet.alias(), User.alias() name = Case(UA.username, [ ('u1', 'user 1'), ('u2', 'user 2')], 'someone else') q = (TA .select(TA.content, name.alias('display').bind_to(UA)) .join(UA, on=(UA.id == TA.user)) .order_by(TA.content)) with self.assertQueryCount(1): self.assertEqual([(t.content, t.user.display) for t in q], [ ('t1', 'user 1'), ('t2', 'user 2'), ('t3', 'someone else')]) class CascadeParent(TestModel): name = TextField() class CascadeChild(TestModel): parent = ForeignKeyField(CascadeParent, backref='children', on_delete='CASCADE') data = TextField() class TestCascadeDeleteIntegration(ModelTestCase): requires = [CascadeParent, CascadeChild] def setUp(self): super(TestCascadeDeleteIntegration, self).setUp() if IS_SQLITE: self.database.pragma('foreign_keys', 1) def test_cascade_delete(self): p1 = CascadeParent.create(name='p1') p2 = CascadeParent.create(name='p2') CascadeChild.create(parent=p1, data='c1') CascadeChild.create(parent=p1, data='c2') CascadeChild.create(parent=p2, data='c3') self.assertEqual(CascadeChild.select().count(), 3) p1.delete_instance() self.assertEqual(CascadeChild.select().count(), 1) self.assertEqual(CascadeChild.get().data, 'c3') ================================================ FILE: tests/mysql_ext.py ================================================ import datetime from peewee import * from playhouse.mysql_ext import JSONField from playhouse.mysql_ext import Match from .base import IS_MYSQL_JSON from .base import ModelDatabaseTestCase from .base import ModelTestCase from .base import TestModel from .base import db_loader from .base import requires_mysql from .base import skip_if from .base import skip_unless try: import mariadb except ImportError: mariadb = mariadb_db = None else: mariadb_db = db_loader('mariadb') try: import mysql.connector as mysql_connector except ImportError: mysql_connector = None mysql_ext_db = db_loader('mysqlconnector') class Person(TestModel): first = CharField() last = CharField() dob = DateField(default=datetime.date(2000, 1, 1)) class Note(TestModel): person = ForeignKeyField(Person, backref='notes') content = TextField() timestamp = DateTimeField(default=datetime.datetime.now) class KJ(TestModel): key = CharField(primary_key=True, max_length=100) data = JSONField() @requires_mysql @skip_if(mysql_connector is None, 'mysql-connector not installed') class TestMySQLConnector(ModelTestCase): database = mysql_ext_db requires = [Person, Note] def test_basic_operations(self): with self.database.atomic(): charlie, huey, zaizee = [Person.create(first=f, last='leifer') for f in ('charlie', 'huey', 'zaizee')] # Use nested-transaction. with self.database.atomic(): data = ( (charlie, ('foo', 'bar', 'zai')), (huey, ('meow', 'purr', 'hiss')), (zaizee, ())) for person, notes in data: for note in notes: Note.create(person=person, content=note) with self.database.atomic() as sp: Person.create(first='x', last='y') sp.rollback() people = Person.select().order_by(Person.first) self.assertEqual([person.first for person in people], ['charlie', 'huey', 'zaizee']) with self.assertQueryCount(1): notes = (Note .select(Note, Person) .join(Person) .order_by(Note.content)) self.assertEqual([(n.person.first, n.content) for n in notes], [ ('charlie', 'bar'), ('charlie', 'foo'), ('huey', 'hiss'), ('huey', 'meow'), ('huey', 'purr'), ('charlie', 'zai')]) @requires_mysql @skip_if(mariadb is None, 'mariadb connector not installed') class TestMariaDBConnector(TestMySQLConnector): database = mariadb_db @requires_mysql @skip_unless(IS_MYSQL_JSON, 'requires MySQL 5.7+ or 8.x') class TestMySQLJSONField(ModelTestCase): requires = [KJ] def test_mysql_json_field(self): values = ( 0, 1.0, 2.3, True, False, 'string', ['foo', 'bar', 'baz'], {'k1': 'v1', 'k2': 'v2'}, {'k3': [0, 1.0, 2.3], 'k4': {'x1': 'y1', 'x2': 'y2'}}) for i, value in enumerate(values): # Verify data can be written. kj = KJ.create(key='k%s' % i, data=value) # Verify value is deserialized correctly. kj_db = KJ['k%s' % i] self.assertEqual(kj_db.data, value) kj = KJ.select().where(KJ.data.extract('$.k1') == 'v1').get() self.assertEqual(kj.key, 'k7') with self.assertRaises(IntegrityError): KJ.create(key='kx', data=None) @requires_mysql class TestMatchExpression(ModelDatabaseTestCase): requires = [Person] def test_match_expression(self): query = (Person .select() .where(Match(Person.first, 'charlie'))) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."first", "t1"."last", "t1"."dob" ' 'FROM "person" AS "t1" ' 'WHERE MATCH("t1"."first") AGAINST(?)'), ['charlie']) query = (Person .select() .where(Match((Person.first, Person.last), 'huey AND zaizee', 'IN BOOLEAN MODE'))) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."first", "t1"."last", "t1"."dob" ' 'FROM "person" AS "t1" ' 'WHERE MATCH("t1"."first", "t1"."last") ' 'AGAINST(? IN BOOLEAN MODE)'), ['huey AND zaizee']) ================================================ FILE: tests/pool.py ================================================ import heapq import os import threading import time from peewee import * from peewee import _savepoint from peewee import _transaction from playhouse.cockroachdb import PooledCockroachDatabase from playhouse.pool import * from .base import BACKEND from .base import BaseTestCase from .base import IS_CRDB from .base import IS_MYSQL from .base import IS_POSTGRESQL from .base import IS_SQLITE from .base import ModelTestCase from .base import db_loader from .base_models import Register class FakeTransaction(_transaction): def _add_history(self, message): self.db.transaction_history.append( '%s%s' % (message, self._conn)) def __enter__(self): self._conn = self.db.connection() self._add_history('O') self.db.push_transaction(self) def __exit__(self, *args): self._add_history('X') self.db.pop_transaction() class FakeDatabase(SqliteDatabase): def __init__(self, *args, **kwargs): self.counter = self.closed_counter = kwargs.pop('counter', 0) self.transaction_history = [] super(FakeDatabase, self).__init__(*args, **kwargs) def _connect(self): self.counter += 1 return self.counter def _close(self, conn): self.closed_counter += 1 def transaction(self): return FakeTransaction(self) class FakePooledDatabase(PooledDatabase, FakeDatabase): def __init__(self, *args, **kwargs): super(FakePooledDatabase, self).__init__(*args, **kwargs) self.conn_key = lambda conn: conn class PooledTestDatabase(PooledDatabase, SqliteDatabase): pass def push_conn(db, timestamp, conn): # Push a connection onto the pool heap with a proper monotonic counter. db._heap_counter += 1 heapq.heappush(db._connections, (timestamp, db._heap_counter, conn)) class TestPooledDatabase(BaseTestCase): def setUp(self): super(TestPooledDatabase, self).setUp() self.db = FakePooledDatabase('testing') def test_connection_pool(self): # Closing and reopening a connection returns us the same conn. self.assertEqual(self.db.connection(), 1) self.assertEqual(self.db.connection(), 1) self.db.close() self.db.connect() self.assertEqual(self.db.connection(), 1) def test_reuse_connection(self): # Verify the connection pool correctly handles calling connect twice. self.assertEqual(self.db.connection(), 1) self.assertRaises(OperationalError, self.db.connect) self.assertFalse(self.db.connect(reuse_if_open=True)) self.assertEqual(self.db.connection(), 1) self.db.close() self.db.connect() self.assertEqual(self.db.connection(), 1) def test_concurrent_connections(self): db = FakePooledDatabase('testing') barrier = threading.Barrier(6) # 5 workers + main thread. def open_conn(): db.connect() barrier.wait(timeout=2) db.close() # Simulate 5 concurrent connections. threads = [threading.Thread(target=open_conn) for i in range(5)] for thread in threads: thread.start() # Wait for all connections to be opened, then release to run. barrier.wait(timeout=2) for t in threads: t.join() self.assertEqual(db.counter, 5) self.assertEqual( sorted([conn for _, _, conn in db._connections]), [1, 2, 3, 4, 5]) # All 5 are ready to be re-used. self.assertEqual(db._in_use, {}) def test_max_conns(self): for i in range(self.db._max_connections): self.db._state.closed = True # Hack to make it appear closed. self.db.connect() self.assertEqual(self.db.connection(), i + 1) self.db._state.closed = True self.assertRaises(ValueError, self.db.connect) def test_stale_timeout(self): # Create a test database with a very short stale timeout. db = FakePooledDatabase('testing', stale_timeout=.001) self.assertEqual(db.connection(), 1) self.assertTrue(1 in db._in_use) # Sleep long enough for the connection to be considered stale. time.sleep(.001) # When we close, since the conn is stale it won't be returned to # the pool. db.close() self.assertEqual(db._in_use, {}) self.assertEqual(db._connections, []) # A new connection will be returned. self.assertEqual(db.connection(), 2) def test_stale_on_checkout(self): # Create a test database with a very short stale timeout. db = FakePooledDatabase('testing', stale_timeout=1) self.assertEqual(db.connection(), 1) self.assertTrue(1 in db._in_use) # When we close, the conn should not be stale so it won't return to # the pool. db.close() # Sleep long enough for the connection to be considered stale. self.assertEqual(db._in_use, {}) self.assertEqual(len(db._connections), 1) _, hc, conn = db._connections[0] db._connections[0] = (time.time() - 2, hc, conn) # A new connection will be returned, as the original one is stale. # The stale connection (1) will be removed. self.assertEqual(db.connection(), 2) def test_manual_close(self): self.assertEqual(self.db.connection(), 1) self.db.manual_close() # When we manually close a connection that's not yet stale, we add it # back to the queue (because close() calls _close()), then close it # for real, and mark it with a tombstone. The next time it's checked # out, it will simply be removed and skipped over. self.assertEqual(len(self.db._connections), 0) self.assertEqual(self.db._in_use, {}) self.assertEqual(self.db.connection(), 2) self.assertEqual(len(self.db._connections), 0) self.assertEqual(list(self.db._in_use.keys()), [2]) self.db.close() self.assertEqual(self.db.connection(), 2) def test_close_idle(self): db = FakePooledDatabase('testing', counter=3) now = time.time() now = time.time() push_conn(db, now - 10, 3) push_conn(db, now - 5, 2) push_conn(db, now - 1, 1) self.assertEqual(db.connection(), 3) self.assertTrue(3 in db._in_use) db.close_idle() self.assertEqual(len(db._connections), 0) self.assertEqual(len(db._in_use), 1) self.assertTrue(3 in db._in_use) self.assertEqual(db.connection(), 3) db.manual_close() self.assertEqual(db.connection(), 4) def test_close_stale(self): db = FakePooledDatabase('testing', counter=3) now = time.time() # Closing stale uses the last checkout time rather than the creation # time for the connection. db._in_use[1] = PoolConnection(now - 400, 1, now - 300) db._in_use[2] = PoolConnection(now - 200, 2, now - 200) db._in_use[3] = PoolConnection(now - 300, 3, now - 100) db._in_use[4] = PoolConnection(now, 4, now) self.assertEqual(db.close_stale(age=200), 2) self.assertEqual(len(db._in_use), 2) self.assertEqual(sorted(db._in_use), [3, 4]) def test_close_all(self): db = FakePooledDatabase('testing', counter=3) now = time.time() push_conn(db, now - 10, 3) push_conn(db, now - 5, 2) push_conn(db, now - 1, 1) self.assertEqual(db.connection(), 3) self.assertTrue(3 in db._in_use) db.close_all() self.assertEqual(len(db._connections), 0) self.assertEqual(len(db._in_use), 0) self.assertEqual(db.connection(), 4) def test_stale_timeout_cascade(self): now = time.time() db = FakePooledDatabase('testing', stale_timeout=10) conns = [ (now - 20, 1), (now - 15, 2), (now - 5, 3), (now, 4), ] for ts, conn in conns: push_conn(db, ts, conn) self.assertEqual(db.connection(), 3) self.assertEqual(len(db._in_use), 1) self.assertTrue(3 in db._in_use) self.assertEqual(len(db._connections), 1) self.assertEqual(db._connections[0][2], 4) def test_connect_cascade(self): now = time.time() class ClosedPooledDatabase(FakePooledDatabase): def _is_closed(self, conn): return conn in (2, 4) db = ClosedPooledDatabase('testing', stale_timeout=10) conns = [ (now - 15, 1), # Skipped due to being stale. (now - 5, 2), # Will appear closed. (now - 3, 3), (now, 4), # Will appear closed. ] db.counter = 4 # The next connection we create will have id=5. for ts, conn in conns: push_conn(db, ts, conn) # Conn 3 is not stale or closed, so we will get it. self.assertEqual(db.connection(), 3) self.assertEqual(len(db._in_use), 1) self.assertTrue(3 in db._in_use) pool_conn = db._in_use[3] self.assertEqual(pool_conn.timestamp, now - 3) self.assertEqual(pool_conn.connection, 3) # Only conn 4 remains in the idle pool. self.assertEqual(len(db._connections), 1) self.assertEqual(db._connections[0][2], 4) # Since conn 4 is closed, we will open a new conn. db._state.closed = True # Pretend we're in a different thread. db.connect() self.assertEqual(db.connection(), 5) self.assertEqual(sorted(db._in_use.keys()), [3, 5]) self.assertEqual(db._connections, []) def test_db_context(self): self.assertEqual(self.db.connection(), 1) with self.db: self.assertEqual(self.db.connection(), 1) self.assertEqual(self.db.transaction_history, ['O1']) self.assertEqual(self.db.connection(), 1) self.assertEqual(self.db.transaction_history, ['O1', 'X1']) with self.db: self.assertEqual(self.db.connection(), 1) self.assertEqual(len(self.db._connections), 1) self.assertEqual(len(self.db._in_use), 0) def test_db_context_threads(self): barrier = threading.Barrier(6) # 5 workers + main thread. def create_context(): with self.db: barrier.wait(timeout=2) threads = [threading.Thread(target=create_context) for i in range(5)] for thread in threads: thread.start() barrier.wait(timeout=2) for thread in threads: thread.join() self.assertEqual(self.db.counter, 5) self.assertEqual(len(self.db._connections), 5) self.assertEqual(len(self.db._in_use), 0) def test_heap_counter_deterministic_ordering(self): # Verify that connections pushed with the same timestamp are returned # in order. now = time.time() push_conn(self.db, now, 'a') push_conn(self.db, now, 'b') push_conn(self.db, now, 'c') results = [] while self.db._connections: ts, counter, conn = heapq.heappop(self.db._connections) results.append(conn) self.assertEqual(results, ['a', 'b', 'c']) def test_close_conn_removes_from_in_use(self): # _close(conn, close_conn=True) should pop the key from _in_use AND # close the underlying driver conn. self.assertEqual(self.db.connection(), 1) self.assertTrue(1 in self.db._in_use) closed_before = self.db.closed_counter self.db._close(1, close_conn=True) self.assertNotIn(1, self.db._in_use) self.assertEqual(self.db.closed_counter, closed_before + 1) def test_double_close_is_noop(self): # Calling _close on a connection not in _in_use (and close_conn=False) # should be a safe no-op rather than raising or leaking. self.assertEqual(self.db.connection(), 1) self.db.close() # Returns conn 1 to the pool. self.assertNotIn(1, self.db._in_use) closed_before = self.db.closed_counter # Second close should do nothing. self.db._close(1) self.assertEqual(self.db.closed_counter, closed_before) # Pool state unchanged. self.assertEqual(len(self.db._connections), 1) def test_can_reuse_false_closes_connection(self): # When _can_reuse returns False on check-in, the connection should be # closed at the driver level and not returned to the pool. class NotReusablePooledDatabase(FakePooledDatabase): def _can_reuse(self, conn): return False db = NotReusablePooledDatabase('testing') self.assertEqual(db.connection(), 1) closed_before = db.closed_counter db.close() # Connection should have been driver-closed, not pooled. self.assertEqual(db.closed_counter, closed_before + 1) self.assertEqual(len(db._connections), 0) self.assertEqual(db._in_use, {}) # Next connect creates a brand new connection. self.assertEqual(db.connection(), 2) def test_close_raw_swallows_exception(self): called = [] # _close_raw should not propagate exceptions from the driver. class BrokenDriverClose(FakeDatabase): def _close(self, conn): called.append(conn) raise RuntimeError('failed') class BrokenPool(FakePooledDatabase, BrokenDriverClose): pass db = BrokenPool('testing') db._close_raw(1337) self.assertEqual(called, [1337]) def test_close_stale_removes_from_in_use(self): # Verify that close_stale both driver-closes the connection AND # removes it from _in_use (no dangling keys). db = FakePooledDatabase('testing', counter=2) now = time.time() db._in_use[1] = PoolConnection(now - 1000, 1, now - 1000) db._in_use[2] = PoolConnection(now, 2, now) closed_before = db.closed_counter self.assertEqual(db.close_stale(age=500), 1) self.assertNotIn(1, db._in_use) self.assertIn(2, db._in_use) self.assertEqual(db.closed_counter, closed_before + 1) def test_close_all_clears_both_pools(self): # close_all should leave both _connections and _in_use completely # empty, and driver-close every connection. db = FakePooledDatabase('testing', counter=3) now = time.time() push_conn(db, now - 5, 1) push_conn(db, now - 1, 2) # Simulate two in-use connections. db._in_use[3] = PoolConnection(now, 3, now) db._in_use[4] = PoolConnection(now, 4, now) # One more for the "current thread" via normal connect path so # self.close() inside close_all has something to reset. db._state.closed = True db.connect() conn = db.connection() self.assertIn(db.conn_key(conn), db._in_use) closed_before = db.closed_counter db.close_all() self.assertEqual(db._connections, []) self.assertEqual(db._in_use, {}) # 2 idle + 2 manually-added in_use + the current thread's conn = 5. # (close_all calls self.close() which triggers _close for the current # thread's conn, but that goes through the return-to-pool path, not # _close_raw. The subsequent loop over the snapshot handles it.) self.assertGreaterEqual(db.closed_counter, closed_before + 4) def test_connect_timeout_with_condition_variable(self): # Verify that connect() with a timeout raises after the timeout # expires when the pool is exhausted. db = FakePooledDatabase('testing', max_connections=1, timeout=0.15) self.assertEqual(db.connection(), 1) errors = [] def try_connect(): db._state.closed = True # Appear as a new thread. try: db.connect() except MaxConnectionsExceeded: errors.append(True) t = threading.Thread(target=try_connect) start = time.monotonic() t.start() t.join(timeout=2) elapsed = time.monotonic() - start # Should have waited roughly the timeout duration. self.assertEqual(len(errors), 1) self.assertGreaterEqual(elapsed, 0.1) def test_connect_timeout_wakes_on_return(self): # Verify that a waiting thread unblocks promptly when a connection # is returned to the pool (via the Condition variable notify). db = FakePooledDatabase('testing', max_connections=1, timeout=5) self.assertEqual(db.connection(), 1) results = [] def try_connect(): db._state.closed = True try: db.connect() results.append(db.connection()) except MaxConnectionsExceeded: results.append('timeout') t = threading.Thread(target=try_connect) t.start() # Give the thread a moment to start waiting. time.sleep(0.05) # Return conn 1 to the pool — should wake the waiting thread. db.close() t.join(timeout=2) self.assertFalse(t.is_alive(), 'Thread did not wake up.') self.assertEqual(len(results), 1) self.assertEqual(results[0], 1) # Got the recycled connection. def test_connect_timeout_zero_becomes_infinite(self): # A timeout of 0 should be treated as infinite (no immediate failure). db = FakePooledDatabase('testing', max_connections=1, timeout=0) self.assertEqual(db._wait_timeout, float('inf')) def test_close_all_wakes_waiters(self): # Threads blocked in connect() should be woken by close_all() so they # can create fresh connections. db = FakePooledDatabase('testing', max_connections=1, timeout=5) self.assertEqual(db.connection(), 1) results = [] def try_connect(): db._state.closed = True try: db.connect() results.append(db.connection()) except MaxConnectionsExceeded: results.append('timeout') t = threading.Thread(target=try_connect) t.start() time.sleep(0.05) # close_all frees the slot and calls notify_all. db.close_all() t.join(timeout=2) self.assertFalse(t.is_alive(), 'Thread was not woken by close_all.') self.assertEqual(len(results), 1) # After close_all, the thread should have gotten a fresh connection. self.assertEqual(results[0], 2) def test_close_stale_iteration(self): db = FakePooledDatabase('testing', counter=10) now = time.time() for i in range(1, 11): db._in_use[i] = PoolConnection(now - 1000, i, now - 1000) # All 10 should be closed. self.assertEqual(db.close_stale(age=500), 10) self.assertEqual(db._in_use, {}) def test_concurrent_close_stale_and_return(self): # Exercise close_stale running while other threads are actively # returning connections (calling close()). The snapshot-before-mutate # pattern and the RLock should keep everything consistent. db = FakePooledDatabase('testing', max_connections=20) barrier = threading.Barrier(11) # 10 workers + main thread. errors = [] def worker(n): """Check out a connection, wait for all workers to be ready, then return it.""" try: db._state.closed = True db.connect() barrier.wait(timeout=2) # Let all workers connect. barrier.wait(timeout=2) # Let main() finish w/timestamps. # Small stagger so close_stale and close() overlap. time.sleep(0.001 * (n % 3)) db.close() except Exception as exc: errors.append(exc) # Spin up 10 threads that each grab and return a connection. threads = [threading.Thread(target=worker, args=(i,)) for i in range(10)] for t in threads: t.start() # Wait until all threads hold a connection. barrier.wait(timeout=2) # Artificially back-date half the checked_out times so that # close_stale will try to close them while threads are returning. now = time.time() for i, key in enumerate(list(db._in_use)): if i % 2 == 0: pc = db._in_use[key] db._in_use[key] = PoolConnection(pc.timestamp, pc.connection, now - 10000) # Release the barrier so threads start returning connections, and # simultaneously run close_stale from the main thread. barrier.wait(timeout=2) closed = db.close_stale(age=5000) for t in threads: t.join(timeout=2) self.assertEqual(errors, []) for key in db._in_use: for _, _, conn in db._connections: self.assertNotEqual(db.conn_key(conn), key) def test_manual_close_when_already_closed(self): # manual_close on an already-closed database should return False. self.assertFalse(self.db.manual_close()) # Never opened. self.db.connect() self.db.close() self.assertFalse(self.db.manual_close()) # Already closed. def test_close_idle_driver_closes_all(self): # Every idle connection should be driver-closed. db = FakePooledDatabase('testing', counter=5) now = time.time() for i in range(1, 6): push_conn(db, now - i, i) closed_before = db.closed_counter db.close_idle() self.assertEqual(db._connections, []) self.assertEqual(db.closed_counter, closed_before + 5) def test_max_connections_zero_means_unlimited(self): # max_connections=0 (falsy) should mean no limit. db = FakePooledDatabase('testing', max_connections=0) for i in range(50): db._state.closed = True db.connect() self.assertEqual(len(db._in_use), 50) def test_stale_and_closed_all_skipped(self): # If every connection in the pool is either stale or closed, a new one # should be created. class AllClosedDatabase(FakePooledDatabase): def _is_closed(self, conn): return True db = AllClosedDatabase('testing', stale_timeout=10) now = time.time() push_conn(db, now - 20, 1) # Stale. push_conn(db, now, 2) # Closed (per _is_closed override). db.counter = 2 self.assertEqual(db.connection(), 3) self.assertEqual(db._connections, []) self.assertEqual(list(db._in_use.keys()), [3]) def test_init_updates_pool_parameters(self): # The init() method should allow updating pool parameters after # initial construction. db = FakePooledDatabase('testing', max_connections=5, stale_timeout=10, timeout=2) self.assertEqual(db._max_connections, 5) self.assertEqual(db._stale_timeout, 10) self.assertEqual(db._wait_timeout, 2) db.init('testing', max_connections=50, stale_timeout=100, timeout=20) self.assertEqual(db._max_connections, 50) self.assertEqual(db._stale_timeout, 100) self.assertEqual(db._wait_timeout, 20) def test_init_timeout_zero_becomes_infinite(self): db = FakePooledDatabase('testing', timeout=5) self.assertEqual(db._wait_timeout, 5) db.init('testing', timeout=0) self.assertEqual(db._wait_timeout, float('inf')) class TestLivePooledDatabase(ModelTestCase): database = PooledTestDatabase('test_pooled.db') requires = [Register] def tearDown(self): super(TestLivePooledDatabase, self).tearDown() self.database.close_idle() if os.path.exists('test_pooled.db'): os.unlink('test_pooled.db') def test_reuse_connection(self): for i in range(5): Register.create(value=i) conn_id = id(self.database.connection()) self.database.close() for i in range(5, 10): Register.create(value=i) self.assertEqual(id(self.database.connection()), conn_id) self.assertEqual( [x.value for x in Register.select().order_by(Register.id)], list(range(10))) def test_db_context(self): with self.database: Register.create(value=1) with self.database.atomic() as sp: self.assertTrue(isinstance(sp, _savepoint)) Register.create(value=2) sp.rollback() with self.database.atomic() as sp: self.assertTrue(isinstance(sp, _savepoint)) Register.create(value=3) with self.database: values = [r.value for r in Register.select().order_by(Register.id)] self.assertEqual(values, [1, 3]) def test_bad_connection(self): self.database.connection() try: self.database.execute_sql('select 1/0') except Exception as exc: pass self.database.close() self.database.connect() class TestPooledDatabaseIntegration(ModelTestCase): requires = [Register] def setUp(self): params = {} if IS_MYSQL: db_class = PooledMySQLDatabase elif IS_POSTGRESQL: db_class = PooledPostgresqlDatabase elif IS_CRDB: db_class = PooledCockroachDatabase else: db_class = PooledSqliteDatabase params['check_same_thread'] = False self.database = db_loader(BACKEND, db_class=db_class, **params) super(TestPooledDatabaseIntegration, self).setUp() def assertConnections(self, expected): available = len(self.database._connections) in_use = len(self.database._in_use) self.assertEqual(available + in_use, expected, 'expected %s, got: %s available, %s in use' % (expected, available, in_use)) def test_pooled_database_integration(self): # Connection should be open from the setup method. self.assertFalse(self.database.is_closed()) self.assertConnections(1) self.assertTrue(self.database.close()) self.assertTrue(self.database.is_closed()) self.assertConnections(1) barrier = threading.Barrier(5) # 4 workers + main thread. def connect(): self.assertTrue(self.database.is_closed()) self.assertTrue(self.database.connect()) self.assertFalse(self.database.is_closed()) barrier.wait(timeout=2) self.assertTrue(self.database.close()) self.assertTrue(self.database.is_closed()) # Open connections in 4 separate threads. threads = [threading.Thread(target=connect) for _ in range(4)] for t in threads: t.start() # Close connections in all 4 threads. barrier.wait(timeout=2) for t in threads: t.join() # Verify that there are 4 connections available in the pool. self.assertConnections(4) self.assertEqual(len(self.database._connections), 4) # Available. self.assertEqual(len(self.database._in_use), 0) # Verify state of the main thread, just a sanity check. self.assertTrue(self.database.is_closed()) # Opening a connection will pull from the pool. self.assertTrue(self.database.connect()) self.assertFalse(self.database.connect(reuse_if_open=True)) self.assertConnections(4) self.assertEqual(len(self.database._in_use), 1) # Calling close_all() closes everything, including calling thread. self.database.close_all() self.assertConnections(0) self.assertTrue(self.database.is_closed()) def test_pool_with_models(self): self.database.close() barrier = threading.Barrier(5) # 4 workers + main thread. def create_obj(i): with self.database.connection_context(): with self.database.atomic(): Register.create(value=i) barrier.wait(timeout=2) # Create 4 objects, one in each thread. The INSERT will be wrapped in a # transaction, and after COMMIT (but while the conn is still open), we # will wait for the signal that all objects were created. This ensures # that all our connections are open concurrently. threads = [threading.Thread(target=create_obj, args=(i,)) for i in range(4)] for t in threads: t.start() # Signal threads that they can exit now and ensure all exited. self.database.connect() barrier.wait(timeout=2) for t in threads: t.join() # Close connection from main thread as well. self.database.close() self.assertConnections(5) self.assertEqual(len(self.database._in_use), 0) # Cycle through the available connections, running a query on each, and # then manually closing it. for i in range(5): self.assertTrue(self.database.is_closed()) self.assertTrue(self.database.connect()) # Sanity check to verify objects are created. query = Register.select().order_by(Register.value) self.assertEqual([r.value for r in query], [0, 1, 2, 3]) self.database.manual_close() self.assertConnections(4 - i) self.assertConnections(0) self.assertEqual(len(self.database._in_use), 0) ================================================ FILE: tests/postgres.py ================================================ #coding:utf-8 import datetime import functools import json import os import uuid from decimal import Decimal as Dc from types import MethodType from peewee import * from playhouse.postgres_ext import * from playhouse.reflection import Introspector from .base import BaseTestCase from .base import DatabaseTestCase from .base import ModelTestCase from .base import TestModel from .base import db_loader from .base import requires_models from .base import skip_if from .base import skip_unless from .base_models import Register from .base_models import Tweet from .base_models import User from .postgres_helpers import BaseBinaryJsonFieldTestCase from .postgres_helpers import BaseJsonFieldTestCase db = db_loader('postgres', db_class=PostgresqlExtDatabase) class HStoreModel(TestModel): name = CharField() data = HStoreField() D = HStoreModel.data class ArrayModel(TestModel): tags = ArrayField(CharField) ints = ArrayField(IntegerField, dimensions=2) class UUIDList(TestModel): key = CharField() id_list = ArrayField(BinaryUUIDField, convert_values=True, index=False) id_list_native = ArrayField(UUIDField, index=False) class ArrayTSModel(TestModel): key = CharField(max_length=100, primary_key=True) timestamps = ArrayField(TimestampField, convert_values=True) class DecimalArray(TestModel): values = ArrayField(DecimalField, field_kwargs={'decimal_places': 1}) class FTSModel(TestModel): title = CharField() data = TextField() fts_data = TSVectorField() class JsonModel(TestModel): data = JSONField() class JsonModelNull(TestModel): data = JSONField(null=True) class BJson(TestModel): data = BinaryJSONField() class JData(TestModel): d1 = BinaryJSONField() d2 = BinaryJSONField(index=False) class UUIDEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, uuid.UUID): return str(obj) return super(UUIDEncoder, self).default(obj) def dumps(obj): return json.dumps(obj, cls=UUIDEncoder) class CustomJSONDumps(TestModel): jf = JSONField(dumps=dumps) jbf = BinaryJSONField(dumps=dumps) class Normal(TestModel): data = TextField() class Event(TestModel): name = CharField() duration = IntervalField() class TZModel(TestModel): dt = DateTimeTZField() class TestTZField(ModelTestCase): database = db requires = [TZModel] @skip_if(os.environ.get('CI'), 'running in ci mode, skipping') def test_tz_field(self): self.database.set_time_zone('us/eastern') # Our naive datetime is treated as if it were in US/Eastern. dt = datetime.datetime(2019, 1, 1, 12) tz = TZModel.create(dt=dt) self.assertTrue(tz.dt.tzinfo is None) # When we retrieve the row, psycopg will attach the appropriate tzinfo # data. The value is returned as an "aware" datetime in US/Eastern. tz_db = TZModel[tz.id] self.assertTrue(tz_db.dt.tzinfo is not None) self.assertEqual(tz_db.dt.timetuple()[:4], (2019, 1, 1, 12)) self.assertEqual(tz_db.dt.utctimetuple()[:4], (2019, 1, 1, 17)) class _UTC(datetime.tzinfo): def utcoffset(self, dt): return datetime.timedelta(0) def tzname(self, dt): return "UTC" def dst(self, dt): return datetime.timedelta(0) UTC = _UTC() # We can explicitly insert a row with a different timezone, however. # When we read the row back, it is returned in US/Eastern. dt2 = datetime.datetime(2019, 1, 1, 12, tzinfo=UTC) tz2 = TZModel.create(dt=dt2) tz2_db = TZModel[tz2.id] self.assertEqual(tz2_db.dt.timetuple()[:4], (2019, 1, 1, 7)) self.assertEqual(tz2_db.dt.utctimetuple()[:4], (2019, 1, 1, 12)) # Querying using naive datetime, treated as localtime (US/Eastern). tzq1 = TZModel.get(TZModel.dt == dt) self.assertEqual(tzq1.id, tz.id) # Querying using aware datetime, tzinfo is respected. tzq2 = TZModel.get(TZModel.dt == dt2) self.assertEqual(tzq2.id, tz2.id) # Change the connection timezone? self.database.set_time_zone('us/central') tz_db = TZModel[tz.id] self.assertEqual(tz_db.dt.timetuple()[:4], (2019, 1, 1, 11)) self.assertEqual(tz_db.dt.utctimetuple()[:4], (2019, 1, 1, 17)) tz2_db = TZModel[tz2.id] self.assertEqual(tz2_db.dt.timetuple()[:4], (2019, 1, 1, 6)) self.assertEqual(tz2_db.dt.utctimetuple()[:4], (2019, 1, 1, 12)) class TestHStoreField(ModelTestCase): database = db_loader('postgres', db_class=PostgresqlExtDatabase, register_hstore=True) requires = [HStoreModel] def setUp(self): super(TestHStoreField, self).setUp() self.t1 = HStoreModel.create(name='t1', data={'k1': 'v1', 'k2': 'v2'}) self.t2 = HStoreModel.create(name='t2', data={'k2': 'v2', 'k3': 'v3'}) def by_name(self, name): return HStoreModel.get(HStoreModel.name == name).data def test_hstore_storage(self): self.assertEqual(self.by_name('t1'), {'k1': 'v1', 'k2': 'v2'}) self.assertEqual(self.by_name('t2'), {'k2': 'v2', 'k3': 'v3'}) self.t1.data = {'k4': 'v4'} self.t1.save() self.assertEqual(self.by_name('t1'), {'k4': 'v4'}) HStoreModel.create(name='t3', data={}) self.assertEqual(self.by_name('t3'), {}) def query(self, *cols): return (HStoreModel .select(HStoreModel.name, *cols) .order_by(HStoreModel.id)) def test_hstore_selecting(self): query = self.query(D.keys().alias('keys')) self.assertEqual([(x.name, sorted(x.keys)) for x in query], [ ('t1', ['k1', 'k2']), ('t2', ['k2', 'k3'])]) query = self.query(D.values().alias('vals')) self.assertEqual([(x.name, sorted(x.vals)) for x in query], [ ('t1', ['v1', 'v2']), ('t2', ['v2', 'v3'])]) query = self.query(D.items().alias('mtx')) self.assertEqual([(x.name, sorted(x.mtx)) for x in query], [ ('t1', [['k1', 'v1'], ['k2', 'v2']]), ('t2', [['k2', 'v2'], ['k3', 'v3']])]) query = self.query(D.slice('k2', 'k3').alias('kz')) self.assertEqual([(x.name, x.kz) for x in query], [ ('t1', {'k2': 'v2'}), ('t2', {'k2': 'v2', 'k3': 'v3'})]) query = self.query(D.slice('k4').alias('kz')) self.assertEqual([(x.name, x.kz) for x in query], [ ('t1', {}), ('t2', {})]) query = self.query(D.exists('k3').alias('ke')) self.assertEqual([(x.name, x.ke) for x in query], [ ('t1', False), ('t2', True)]) query = self.query(D.defined('k3').alias('ke')) self.assertEqual([(x.name, x.ke) for x in query], [ ('t1', False), ('t2', True)]) query = self.query(D['k1'].alias('k1')) self.assertEqual([(x.name, x.k1) for x in query], [ ('t1', 'v1'), ('t2', None)]) query = self.query().where(D['k1'] == 'v1') self.assertEqual([x.name for x in query], ['t1']) def assertWhere(self, expr, names): query = HStoreModel.select().where(expr) self.assertEqual([x.name for x in query], names) def test_hstore_filtering(self): self.assertWhere(D == {'k1': 'v1', 'k2': 'v2'}, ['t1']) self.assertWhere(D == {'k2': 'v2'}, []) self.assertWhere(D.contains('k3'), ['t2']) self.assertWhere(D.contains(['k2', 'k3']), ['t2']) self.assertWhere(D.contains(['k2']), ['t1', 't2']) # test dict self.assertWhere(D.contains({'k2': 'v2', 'k3': 'v3'}), ['t2']) self.assertWhere(D.contains({'k2': 'v2'}), ['t1', 't2']) self.assertWhere(D.contains({'k2': 'v3'}), []) # test contains any. self.assertWhere(D.contains_any('k3', 'kx'), ['t2']) self.assertWhere(D.contains_any('k2', 'x', 'k3'), ['t1', 't2']) self.assertWhere(D.contains_any('x', 'kx', 'y'), []) def test_hstore_filter_functions(self): self.assertWhere(D.exists('k2') == True, ['t1', 't2']) self.assertWhere(D.exists('k3') == True, ['t2']) self.assertWhere(D.defined('k2') == True, ['t1', 't2']) self.assertWhere(D.defined('k3') == True, ['t2']) def test_hstore_update(self): rc = (HStoreModel .update(data=D.update(k4='v4')) .where(HStoreModel.name == 't1') .execute()) self.assertTrue(rc > 0) self.assertEqual(self.by_name('t1'), {'k1': 'v1', 'k2': 'v2', 'k4': 'v4'}) rc = (HStoreModel .update(data=D.update(k5='v5', k6='v6')) .where(HStoreModel.name == 't2') .execute()) self.assertTrue(rc > 0) self.assertEqual(self.by_name('t2'), {'k2': 'v2', 'k3': 'v3', 'k5': 'v5', 'k6': 'v6'}) HStoreModel.update(data=D.update(k2='vxxx')).execute() self.assertEqual([x.data for x in self.query(D)], [ {'k1': 'v1', 'k2': 'vxxx', 'k4': 'v4'}, {'k2': 'vxxx', 'k3': 'v3', 'k5': 'v5', 'k6': 'v6'}]) (HStoreModel .update(data=D.delete('k4')) .where(HStoreModel.name == 't1') .execute()) self.assertEqual(self.by_name('t1'), {'k1': 'v1', 'k2': 'vxxx'}) HStoreModel.update(data=D.delete('k5')).execute() self.assertEqual([x.data for x in self.query(D)], [ {'k1': 'v1', 'k2': 'vxxx'}, {'k2': 'vxxx', 'k3': 'v3', 'k6': 'v6'} ]) HStoreModel.update(data=D.delete('k1', 'k2')).execute() self.assertEqual([x.data for x in self.query(D)], [ {}, {'k3': 'v3', 'k6': 'v6'}]) class TestArrayField(ModelTestCase): database = db requires = [ArrayModel] def create_sample(self): return ArrayModel.create( tags=['alpha', 'beta', 'gamma', 'delta'], ints=[[1, 2], [3, 4], [5, 6]]) def test_index_expression(self): data = ( (['a', 'b', 'c'], []), (['b', 'c', 'd', 'e'], [])) am_ids = [] for tags, ints in data: am = ArrayModel.create(tags=tags, ints=ints) am_ids.append(am.id) last_tag = fn.array_upper(ArrayModel.tags, 1) query = ArrayModel.select(ArrayModel.tags[last_tag]).tuples() self.assertEqual(sorted([t for t, in query]), ['c', 'e']) q = ArrayModel.select().where(ArrayModel.tags[last_tag] < 'd') self.assertEqual([a.id for a in q], [am_ids[0]]) q = ArrayModel.select().where(ArrayModel.tags[last_tag] > 'd') self.assertEqual([a.id for a in q], [am_ids[1]]) def test_hashable_objectslice(self): ArrayModel.create(tags=[], ints=[[0, 1], [2, 3]]) ArrayModel.create(tags=[], ints=[[4, 5], [6, 7]]) n = (ArrayModel .update({ArrayModel.ints[0][0]: ArrayModel.ints[0][0] + 1}) .execute()) self.assertEqual(n, 2) am1, am2 = ArrayModel.select().order_by(ArrayModel.id) self.assertEqual(am1.ints, [[1, 1], [2, 3]]) self.assertEqual(am2.ints, [[5, 5], [6, 7]]) def test_array_get_set(self): am = self.create_sample() am_db = ArrayModel.get(ArrayModel.id == am.id) self.assertEqual(am_db.tags, ['alpha', 'beta', 'gamma', 'delta']) self.assertEqual(am_db.ints, [[1, 2], [3, 4], [5, 6]]) def test_array_equality(self): am1 = ArrayModel.create(tags=['t1'], ints=[[1, 2]]) am2 = ArrayModel.create(tags=['t2'], ints=[[3, 4]]) obj = ArrayModel.get(ArrayModel.tags == ['t1']) self.assertEqual(obj.id, am1.id) self.assertEqual(obj.tags, ['t1']) obj = ArrayModel.get(ArrayModel.ints == [[3, 4]]) self.assertEqual(obj.id, am2.id) obj = ArrayModel.get(ArrayModel.tags != ['t1']) self.assertEqual(obj.id, am2.id) def test_array_db_value(self): am = ArrayModel.create(tags=('foo', 'bar'), ints=[]) am_db = ArrayModel.get(ArrayModel.id == am.id) self.assertEqual(am_db.tags, ['foo', 'bar']) def test_array_search(self): def assertAM(where, *instances): query = (ArrayModel .select() .where(where) .order_by(ArrayModel.id)) self.assertEqual([x.id for x in query], [x.id for x in instances]) am = self.create_sample() am2 = ArrayModel.create(tags=['alpha', 'beta'], ints=[[1, 1]]) am3 = ArrayModel.create(tags=['delta'], ints=[[3, 4]]) am4 = ArrayModel.create(tags=['中文'], ints=[[3, 4]]) am5 = ArrayModel.create(tags=['中文', '汉语'], ints=[[3, 4]]) AM = ArrayModel T = AM.tags assertAM((Value('beta') == fn.ANY(T)), am, am2) assertAM((Value('delta') == fn.Any(T)), am, am3) assertAM(Value('omega') == fn.Any(T)) # Check the contains operator. assertAM(SQL("tags::text[] @> ARRAY['beta']"), am, am2) # Use the nicer API. assertAM(T.contains('beta'), am, am2) assertAM(T.contains('omega', 'delta')) assertAM(T.contains('汉语'), am5) assertAM(T.contains('alpha', 'delta'), am) assertAM(T.contained_by('alpha', 'beta', 'delta'), am2, am3) assertAM(T.contained_by('alpha', 'beta', 'gamma', 'delta'), am, am2, am3) # Check for any. assertAM(T.contains_any('beta'), am, am2) assertAM(T.contains_any('中文'), am4, am5) assertAM(T.contains_any('omega', 'delta'), am, am3) assertAM(T.contains_any('alpha', 'delta'), am, am2, am3) def test_array_index_slice(self): self.create_sample() AM = ArrayModel I, T = AM.ints, AM.tags row = AM.select(T[1].alias('arrtags')).dicts().get() self.assertEqual(row['arrtags'], 'beta') row = AM.select(T[2:3].alias('foo')).dicts().get() self.assertEqual(row['foo'], ['gamma']) row = AM.select(T[:2].alias('foo')).dicts().get() self.assertEqual(row['foo'], ['alpha', 'beta']) row = AM.select(T[2:].alias('foo')).dicts().get() self.assertEqual(row['foo'], ['gamma', 'delta']) row = AM.select(T[2:4].alias('foo')).dicts().get() self.assertEqual(row['foo'], ['gamma', 'delta']) row = AM.select(I[1][1].alias('ints')).dicts().get() self.assertEqual(row['ints'], 4) row = AM.select(I[1:3][0].alias('ints')).dicts().get() self.assertEqual(row['ints'], [[3], [5]]) @requires_models(DecimalArray) def test_field_kwargs(self): vl1, vl2 = [Dc('3.1'), Dc('1.3')], [Dc('3.14'), Dc('1')] da1, da2 = [DecimalArray.create(values=vl) for vl in (vl1, vl2)] da1_db = DecimalArray.get(DecimalArray.id == da1.id) da2_db = DecimalArray.get(DecimalArray.id == da2.id) self.assertEqual(da1_db.values, [Dc('3.1'), Dc('1.3')]) self.assertEqual(da2_db.values, [Dc('3.1'), Dc('1.0')]) class TestArrayFieldConvertValues(ModelTestCase): database = db requires = [ArrayTSModel] def dt(self, day, hour=0, minute=0, second=0): return datetime.datetime(2018, 1, day, hour, minute, second) def test_value_conversion(self): data = { 'k1': [self.dt(1), self.dt(2), self.dt(3)], 'k2': [], 'k3': [self.dt(4, 5, 6, 7), self.dt(10, 11, 12, 13)], } for key in sorted(data): ArrayTSModel.create(key=key, timestamps=data[key]) for key in sorted(data): am = ArrayTSModel.get(ArrayTSModel.key == key) self.assertEqual(am.timestamps, data[key]) # Perform lookup using timestamp values. ts = ArrayTSModel.get(ArrayTSModel.timestamps.contains(self.dt(3))) self.assertEqual(ts.key, 'k1') ts = ArrayTSModel.get( ArrayTSModel.timestamps.contains(self.dt(4, 5, 6, 7))) self.assertEqual(ts.key, 'k3') self.assertRaises(ArrayTSModel.DoesNotExist, ArrayTSModel.get, ArrayTSModel.timestamps.contains(self.dt(4, 5, 6))) def test_get_with_array_values(self): a1 = ArrayTSModel.create(key='k1', timestamps=[self.dt(1)]) a2 = ArrayTSModel.create(key='k2', timestamps=[self.dt(2), self.dt(3)]) query = (ArrayTSModel .select() .where(ArrayTSModel.timestamps == [self.dt(1)])) a1_db = query.get() self.assertEqual(a1_db.id, a1.id) query = (ArrayTSModel .select() .where(ArrayTSModel.timestamps == [self.dt(2), self.dt(3)])) a2_db = query.get() self.assertEqual(a2_db.id, a2.id) a1_db = ArrayTSModel.get(timestamps=[self.dt(1)]) self.assertEqual(a1_db.id, a1.id) a2_db = ArrayTSModel.get(timestamps=[self.dt(2), self.dt(3)]) self.assertEqual(a2_db.id, a2.id) class TestArrayUUIDField(ModelTestCase): database = db requires = [UUIDList] def test_array_of_uuids(self): u1, u2, u3, u4 = [uuid.uuid4() for _ in range(4)] a = UUIDList.create(key='a', id_list=[u1, u2, u3], id_list_native=[u1, u2, u3]) b = UUIDList.create(key='b', id_list=[u2, u3, u4], id_list_native=[u2, u3, u4]) a_db = UUIDList.get(UUIDList.key == 'a') b_db = UUIDList.get(UUIDList.key == 'b') self.assertEqual(a.id_list, [u1, u2, u3]) self.assertEqual(b.id_list, [u2, u3, u4]) self.assertEqual(a.id_list_native, [u1, u2, u3]) self.assertEqual(b.id_list_native, [u2, u3, u4]) class TestTSVectorField(ModelTestCase): database = db requires = [FTSModel] messages = [ 'A faith is a necessity to a man. Woe to him who believes in nothing.', 'All who call on God in true faith, earnestly from the heart, will ' 'certainly be heard, and will receive what they have asked and desired.', 'Be faithful in small things because it is in them that your strength lies.', 'Faith consists in believing when it is beyond the power of reason to believe.', 'Faith has to do with things that are not seen and hope with things that are not at hand.', ] def setUp(self): super(TestTSVectorField, self).setUp() for idx, message in enumerate(self.messages): FTSModel.create(title=str(idx), data=message, fts_data=fn.to_tsvector(message)) def assertMessages(self, expr, expected): query = FTSModel.select().where(expr).order_by(FTSModel.id) titles = [row.title for row in query] self.assertEqual(list(map(int, titles)), expected) def test_sql(self): query = FTSModel.select().where(Match(FTSModel.data, 'foo bar')) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."title", "t1"."data", "t1"."fts_data" ' 'FROM "fts_model" AS "t1" ' 'WHERE (to_tsvector("t1"."data") @@ to_tsquery(?))'), ['foo bar']) def test_match_function(self): D = FTSModel.data self.assertMessages(Match(D, 'heart'), [1]) self.assertMessages(Match(D, 'god'), [1]) self.assertMessages(Match(D, 'faith'), [0, 1, 2, 3, 4]) self.assertMessages(Match(D, 'thing'), [2, 4]) self.assertMessages(Match(D, 'faith & things'), [2, 4]) self.assertMessages(Match(D, 'god | things'), [1, 2, 4]) self.assertMessages(Match(D, 'god & things'), []) def test_tsvector_field(self): M = FTSModel.fts_data.match self.assertMessages(M('heart'), [1]) self.assertMessages(M('god'), [1]) self.assertMessages(M('faith'), [0, 1, 2, 3, 4]) self.assertMessages(M('thing'), [2, 4]) self.assertMessages(M('faith & things'), [2, 4]) self.assertMessages(M('god | things'), [1, 2, 4]) self.assertMessages(M('god & things'), []) # Using the plain parser we cannot express "OR", but individual term # match works like we expect and multi-term is AND-ed together. self.assertMessages(M('god | things', plain=True), []) self.assertMessages(M('god', plain=True), [1]) self.assertMessages(M('thing', plain=True), [2, 4]) self.assertMessages(M('faith things', plain=True), [2, 4]) def pg12(): with db: return db.server_version >= 120000 class TestJsonField(BaseJsonFieldTestCase, ModelTestCase): M = JsonModel N = Normal database = db requires = [JsonModel, Normal, JsonModelNull] def test_json_null(self): tjn = JsonModelNull.create(data=None) tj = JsonModelNull.create(data={'k1': 'v1'}) results = JsonModelNull.select().order_by(JsonModelNull.id) self.assertEqual( [tj_db.data for tj_db in results], [None, {'k1': 'v1'}]) query = JsonModelNull.select().where( JsonModelNull.data.is_null(True)) self.assertEqual(query.get(), tjn) class TestBinaryJsonField(BaseBinaryJsonFieldTestCase, ModelTestCase): M = BJson N = Normal database = db requires = [BJson, Normal] def test_remove_data(self): BJson.delete().execute() # Clear out db. BJson.create(data={ 'k1': 'v1', 'k2': 'v2', 'k3': {'x1': 'z1', 'x2': 'z2'}, 'k4': [0, 1, 2]}) def assertData(exp_list, expected_data): query = BJson.select(BJson.data.remove(*exp_list)).tuples() data = query[:][0][0] self.assertEqual(data, expected_data) D = BJson.data assertData(['k3'], {'k1': 'v1', 'k2': 'v2', 'k4': [0, 1, 2]}) assertData(['k1', 'k3'], {'k2': 'v2', 'k4': [0, 1, 2]}) assertData(['k1', 'kx', 'ky', 'k3'], {'k2': 'v2', 'k4': [0, 1, 2]}) assertData(['k4', 'k3'], {'k1': 'v1', 'k2': 'v2'}) def test_remove_path(self): BJson.delete().execute() # Clear out db. data = {'k1': {'x1': {'y1': 'z1', 'y2': 'z2'}, 'x2': ['i1', 'i2']}} BJson.create(data=data) def assertData(exp_list, expected_data): curr = BJson.data for exp in exp_list: curr = curr[exp] query = BJson.select(curr.remove()).tuples() data = query[:][0][0] self.assertEqual(data, expected_data) assertData(['k1'], {}) assertData(['k1', 'x1'], {'k1': {'x2': ['i1', 'i2']}}) assertData(['k1', 'x1', 'y1'], {'k1': {'x1': {'y2': 'z2'}, 'x2': ['i1', 'i2']}}) assertData(['k1', 'x1', 'y2'], {'k1': {'x1': {'y1': 'z1'}, 'x2': ['i1', 'i2']}}) assertData(['k1', 'x2', 0], {'k1': {'x1': {'y1': 'z1', 'y2': 'z2'}, 'x2': ['i2']}}) assertData(['k1', 'x2', -1], {'k1': {'x1': {'y1': 'z1', 'y2': 'z2'}, 'x2': ['i1']}}) assertData(['kx'], data) assertData(['k1', 'zz'], data) def test_json_length(self): BJson.delete().execute() # Clear out db. data = {'k1': {'x1': [1, 2, 3], 'x2': [1, 2], 'x3': []}} BJson.create(data=data) def assertLength(exp_list, count): curr = BJson.data for exp in exp_list: curr = curr[exp] query = BJson.select(curr.length()).tuples() data = query[:][0][0] self.assertEqual(data, count) assertLength(('k1', 'x1'), 3) assertLength(('k1', 'x2'), 2) assertLength(('k1', 'x3'), 0) BJson.delete().execute() # Clear out db. BJson.create(data=[0, 1, 2, 3, 4, 5]) assertLength((), 6) def test_json_extract(self): BJson.delete().execute() # Clear out db. data = {'k1': {'x1': {'y1': 'z1', 'y2': 'z2'}, 'x2': ['i1', 'i2']}} BJson.create(data=data) def assertData(node, path, expected_data): query = BJson.select(node.extract(*path)).tuples() data = query[:][0][0] self.assertEqual(data, expected_data) assertData(BJson.data, ('k1', 'x1', 'y1'), 'z1') assertData(BJson.data, ('k1', 'x1'), {'y1': 'z1', 'y2': 'z2'}) assertData(BJson.data, ('k1', 'x2', 0), 'i1') assertData(BJson.data, ('k1', 'x2', -1), 'i2') assertData(BJson.data, ('k1',), {'x1': {'y1': 'z1', 'y2': 'z2'}, 'x2': ['i1', 'i2']}) assertData(BJson.data, ('kx',), None) assertData(BJson.data['k1'], ('x1', 'y1'), 'z1') assertData(BJson.data['k1']['x1'], ('y1',), 'z1') assertData(BJson.data['k1']['x2'], (0,), 'i1') def test_json_contains_in_list(self): m1 = self.M.create(data=[{'k1': 'v1', 'k2': 'v2'}, {'a1': 'b1'}]) m2 = self.M.create(data=[{'k3': 'v3'}, {'k4': 'v4'}]) m3 = self.M.create(data=[{'k5': 'v5', 'k6': 'v6'}, {'k1': 'v1'}]) query = (self.M .select() .where(self.M.data.contains([{'k1': 'v1'}])) .order_by(self.M.id)) self.assertEqual([m.id for m in query], [m1.id, m3.id]) def test_integer_index_weirdness(self): self._create_test_data() def fails(): with self.database.atomic(): expr = BJson.data.contains_any(2, 8, 12) results = list(BJson.select().where( BJson.data.contains_any(2, 8, 12))) # Complains of a missing cast/conversion for the data-type? self.assertRaises(ProgrammingError, fails) class TestJsonFieldRegressions(ModelTestCase): database = db requires = [JData] def test_json_field_concat(self): jd = JData.create( d1={'k1': {'x1': 'y1'}, 'k2': 'v2', 'k3': 'v3'}, d2={'k1': {'x2': 'y2'}, 'k2': 'v2-x', 'k4': 'v4'}) query = JData.select(JData.d1.concat(JData.d2).alias('data')) obj = query.get() self.assertEqual(obj.data, { 'k1': {'x2': 'y2'}, 'k2': 'v2-x', 'k3': 'v3', 'k4': 'v4'}) def test_introspect_bjson_field(self): introspector = Introspector.from_database(self.database) models = introspector.generate_models(table_names=['j_data']) JD = models['j_data'] self.assertEqual(JD._meta.sorted_field_names, ['id', 'd1', 'd2']) self.assertTrue(isinstance(JD.d1, BinaryJSONField)) self.assertTrue(isinstance(JD.d2, BinaryJSONField)) self.assertTrue(JD.d1.index) self.assertEqual(JD.d1.index_type, 'GIN') self.assertFalse(JD.d2.index) class TestJSONFieldCustomDumps(ModelTestCase): database = db requires = [CustomJSONDumps] def test_custom_dumps(self): u1 = uuid.uuid4() u2 = uuid.uuid4() data = {'u1': u1, 'u2': u2, 'u3': [u1, u2]} c = CustomJSONDumps.create(jf=data, jbf=data) c_db = CustomJSONDumps.get_by_id(c.id) self.assertEqual(c_db.jf, { 'u1': str(u1), 'u2': str(u2), 'u3': [str(u1), str(u2)]}) self.assertEqual(c_db.jbf, { 'u1': str(u1), 'u2': str(u2), 'u3': [str(u1), str(u2)]}) class TestIntervalField(ModelTestCase): database = db requires = [Event] def test_interval_field(self): e1 = Event.create(name='hour', duration=datetime.timedelta(hours=1)) e2 = Event.create(name='mix', duration=datetime.timedelta( days=1, hours=2, minutes=3, seconds=4)) events = [(e.name, e.duration) for e in Event.select().order_by(Event.duration)] self.assertEqual(events, [ ('hour', datetime.timedelta(hours=1)), ('mix', datetime.timedelta(days=1, hours=2, minutes=3, seconds=4)) ]) class TestIndexedField(BaseTestCase): def test_indexed_field_ddl(self): class FakeIndexedField(IndexedFieldMixin, CharField): default_index_type = 'GiST' class IndexedModel(TestModel): array_index = ArrayField(CharField) array_noindex= ArrayField(IntegerField, index=False) fake_index = FakeIndexedField() fake_index_with_type = FakeIndexedField(index_type='MAGIC') fake_noindex = FakeIndexedField(index=False) class Meta: database = db create_sql, _ = IndexedModel._schema._create_table(False).query() self.assertEqual(create_sql, ( 'CREATE TABLE "indexed_model" (' '"id" SERIAL NOT NULL PRIMARY KEY, ' '"array_index" VARCHAR(255)[] NOT NULL, ' '"array_noindex" INTEGER[] NOT NULL, ' '"fake_index" VARCHAR(255) NOT NULL, ' '"fake_index_with_type" VARCHAR(255) NOT NULL, ' '"fake_noindex" VARCHAR(255) NOT NULL)')) indexes = [idx.query()[0] for idx in IndexedModel._schema._create_indexes(False)] self.assertEqual(indexes, [ ('CREATE INDEX "indexed_model_array_index" ON "indexed_model" ' 'USING GIN ("array_index")'), ('CREATE INDEX "indexed_model_fake_index" ON "indexed_model" ' 'USING GiST ("fake_index")'), ('CREATE INDEX "indexed_model_fake_index_with_type" ' 'ON "indexed_model" ' 'USING MAGIC ("fake_index_with_type")')]) class IDAlways(TestModel): id = IdentityField(generate_always=True) data = CharField() class IDByDefault(TestModel): id = IdentityField() data = CharField() class TestIdentityField(ModelTestCase): database = db requires = [IDAlways, IDByDefault] def test_identity_field_always(self): iq = IDAlways.insert_many([(d,) for d in ('d1', 'd2', 'd3')]) curs = iq.execute() self.assertEqual(list(curs), [(1,), (2,), (3,)]) # Cannot specify id when generate always is true. with self.assertRaises(ProgrammingError): with self.database.atomic(): IDAlways.create(id=10, data='d10') query = IDAlways.select().order_by(IDAlways.id) self.assertEqual(list(query.tuples()), [ (1, 'd1'), (2, 'd2'), (3, 'd3')]) def test_identity_field_by_default(self): iq = IDByDefault.insert_many([(d,) for d in ('d1', 'd2', 'd3')]) curs = iq.execute() self.assertEqual(list(curs), [(1,), (2,), (3,)]) # Cannot specify id when generate always is true. IDByDefault.create(id=10, data='d10') query = IDByDefault.select().order_by(IDByDefault.id) self.assertEqual(list(query.tuples()), [ (1, 'd1'), (2, 'd2'), (3, 'd3'), (10, 'd10')]) def test_schema(self): sql, params = IDAlways._schema._create_table(False).query() self.assertEqual(sql, ( 'CREATE TABLE "id_always" ("id" INT GENERATED ALWAYS AS IDENTITY ' 'NOT NULL PRIMARY KEY, "data" VARCHAR(255) NOT NULL)')) sql, params = IDByDefault._schema._create_table(False).query() self.assertEqual(sql, ( 'CREATE TABLE "id_by_default" ("id" INT GENERATED BY DEFAULT AS ' 'IDENTITY NOT NULL PRIMARY KEY, "data" VARCHAR(255) NOT NULL)')) class TestServerSide(ModelTestCase): database = db requires = [Register] def setUp(self): super(TestServerSide, self).setUp() with db.atomic(): for i in range(100): Register.create(value=i) def test_server_side_cursor(self): query = Register.select().order_by(Register.value) with self.database.atomic(): with self.assertQueryCount(1): data = [row.value for row in ServerSide(query)] self.assertEqual(data, list(range(100))) ss_query = ServerSide(query.limit(10), array_size=3) self.assertEqual([row.value for row in ss_query], list(range(10))) ss_query = ServerSide(query.where(SQL('1 = 0'))) self.assertEqual(list(ss_query), []) def test_lower_level_apis(self): query = Register.select(Register.value).order_by(Register.value) with self.database.atomic(): ssq = ServerSideQuery(query, array_size=10) curs_wrapper = ssq._execute(self.database) curs = curs_wrapper.cursor self.assertTrue(isinstance(curs, FetchManyCursor)) self.assertEqual(curs.fetchone(), (0,)) self.assertEqual(curs.fetchone(), (1,)) curs.close() def test_close_cursor(self): query = Register.select(Register.value).order_by(Register.value) with self.database.atomic(): ssq = ServerSideQuery(query, array_size=10) accum = [] for i, obj in enumerate(ssq.iterator()): if i == 25: break accum.append(obj.value) self.assertTrue(ssq.close()) self.assertEqual(len(accum), 25) self.assertEqual(accum, list(range(25))) class KX(TestModel): key = CharField(unique=True) value = IntegerField() class TestAutocommitIntegration(ModelTestCase): database = db requires = [KX] def setUp(self): super(TestAutocommitIntegration, self).setUp() with self.database.atomic(): kx1 = KX.create(key='k1', value=1) def force_integrity_error(self): # Force an integrity error, then verify that the current # transaction has been aborted. self.assertRaises(IntegrityError, KX.create, key='k1', value=10) def test_autocommit_default(self): kx2 = KX.create(key='k2', value=2) # Will be committed. self.assertTrue(kx2.id > 0) self.force_integrity_error() self.assertEqual(KX.select().count(), 2) self.assertEqual([(kx.key, kx.value) for kx in KX.select().order_by(KX.key)], [('k1', 1), ('k2', 2)]) def test_autocommit_disabled(self): with self.database.manual_commit(): self.database.begin() kx2 = KX.create(key='k2', value=2) # Not committed. self.assertTrue(kx2.id > 0) # Yes, we have a primary key. self.force_integrity_error() self.database.rollback() self.assertEqual(KX.select().count(), 1) kx1_db = KX.get(KX.key == 'k1') self.assertEqual(kx1_db.value, 1) def test_atomic_block(self): with self.database.atomic() as txn: kx2 = KX.create(key='k2', value=2) self.assertTrue(kx2.id > 0) self.force_integrity_error() txn.rollback(False) self.assertEqual(KX.select().count(), 1) kx1_db = KX.get(KX.key == 'k1') self.assertEqual(kx1_db.value, 1) def test_atomic_block_exception(self): with self.assertRaises(IntegrityError): with self.database.atomic(): KX.create(key='k2', value=2) KX.create(key='k1', value=10) self.assertEqual(KX.select().count(), 1) class TestPostgresIsolationLevel(DatabaseTestCase): database = db_loader('postgres', isolation_level=3) # SERIALIZABLE. def test_isolation_level(self): conn = self.database.connection() self.assertEqual(conn.isolation_level, 3) conn.set_isolation_level(2) self.assertEqual(conn.isolation_level, 2) self.database.close() conn = self.database.connection() self.assertEqual(conn.isolation_level, 3) self.database.close() self.database.set_isolation_level(2) for _ in range(2): conn = self.database.connection() self.assertEqual(conn.isolation_level, 2) self.database.close() def test_isolation_level_str(self): db = db_loader('postgres', isolation_level='SERIALIZABLE') conn = db.connection() self.assertEqual(conn.isolation_level, db._adapter.isolation_levels_inv['SERIALIZABLE']) db.close() db.set_isolation_level('READ COMMITTED') conn = db.connection() self.assertEqual(conn.isolation_level, db._adapter.isolation_levels_inv['READ COMMITTED']) db.close() def test_isolation_level_mixed(self): db = db_loader('postgres', isolation_level='SERIALIZABLE') conn = db.connection() self.assertEqual(conn.isolation_level, db._adapter.isolation_levels_inv['SERIALIZABLE']) db.close() rc = db._adapter.isolation_levels_inv['READ COMMITTED'] db.set_isolation_level(rc) conn = db.connection() self.assertEqual(conn.isolation_level, rc) db.close() @skip_unless(pg12(), 'cte materialization requires pg >= 12') class TestPostgresCTEMaterialization(ModelTestCase): database = db requires = [Register] def test_postgres_cte_materialization(self): Register.insert_many([(i,) for i in (1, 2, 3)]).execute() for materialized in (None, False, True): cte = Register.select().cte('t', materialized=materialized) query = (cte .select_from(cte.c.value) .where(cte.c.value != 2) .order_by(cte.c.value)) self.assertEqual([r.value for r in query], [1, 3]) class TestPostgresLateralJoin(ModelTestCase): database = db test_data = ( ('a', (('a1', 1), ('a2', 2), ('a10', 10))), ('b', (('b3', 3), ('b4', 4), ('b7', 7))), ('c', ())) def create_data(self): ts = lambda d: datetime.datetime(2019, 1, d) with self.database.atomic(): for username, tweets in self.test_data: user = User.create(username=username) for c, d in tweets: Tweet.create(user=user, content=c, timestamp=ts(d)) @requires_models(User, Tweet) def test_lateral_top_n(self): self.create_data() subq = (Tweet .select(Tweet.content, Tweet.timestamp) .where(Tweet.user == User.id) .order_by(Tweet.timestamp.desc()) .limit(2)) query = (User .select(User, subq.c.content) .join(subq, JOIN.LEFT_LATERAL) .order_by(subq.c.timestamp.desc(nulls='last'))) results = [(u.username, u.content) for u in query] self.assertEqual(results, [ ('a', 'a10'), ('b', 'b7'), ('b', 'b4'), ('a', 'a2'), ('c', None)]) query = (Tweet .select(User.username, subq.c.content) .from_(User) .join(subq, JOIN.LEFT_LATERAL) .order_by(User.username, subq.c.timestamp)) results = [(t.username, t.content) for t in query] self.assertEqual(results, [ ('a', 'a2'), ('a', 'a10'), ('b', 'b4'), ('b', 'b7'), ('c', None)]) @requires_models(User, Tweet) def test_lateral_helper(self): self.create_data() subq = (Tweet .select(Tweet.content, Tweet.timestamp) .where(Tweet.user == User.id) .order_by(Tweet.timestamp.desc()) .limit(2) .lateral()) query = (User .select(User, subq.c.content) .join(subq, on=True) .order_by(subq.c.timestamp.desc(nulls='last'))) with self.assertQueryCount(1): results = [(u.username, u.tweet.content) for u in query] self.assertEqual(results, [ ('a', 'a10'), ('b', 'b7'), ('b', 'b4'), ('a', 'a2')]) ================================================ FILE: tests/postgres_helpers.py ================================================ from peewee import Cast class BaseJsonFieldTestCase(object): # Subclasses must define these, as well as specifying requires[]. M = None # Json model. N = None # "Normal" model. def test_json_field(self): data = {'k1': ['a1', 'a2'], 'k2': {'k3': 'v3'}} j = self.M.create(data=data) j_db = self.M.get(j._pk_expr()) self.assertEqual(j_db.data, data) def test_joining_on_json_key(self): values = [ {'foo': 'bar', 'baze': {'nugget': 'alpha'}}, {'foo': 'bar', 'baze': {'nugget': 'beta'}}, {'herp': 'derp', 'baze': {'nugget': 'epsilon'}}, {'herp': 'derp', 'bar': {'nuggie': 'alpha'}}, ] for data in values: self.M.create(data=data) for value in ['alpha', 'beta', 'gamma', 'delta']: self.N.create(data=value) query = (self.M .select() .join(self.N, on=( self.N.data == self.M.data['baze']['nugget'])) .order_by(self.M.id)) results = [jm.data for jm in query] self.assertEqual(results, [ {'foo': 'bar', 'baze': {'nugget': 'alpha'}}, {'foo': 'bar', 'baze': {'nugget': 'beta'}}, ]) def test_json_lookup_methods(self): data = { 'gp1': { 'p1': {'c1': 'foo'}, 'p2': {'c2': 'bar'}}, 'gp2': {}} j = self.M.create(data=data) def assertLookup(lookup, expected): query = (self.M .select(lookup) .where(j._pk_expr()) .dicts()) self.assertEqual(query.get(), expected) expr = self.M.data['gp1']['p1'] assertLookup(expr.alias('p1'), {'p1': '{"c1": "foo"}'}) assertLookup(expr.as_json().alias('p2'), {'p2': {'c1': 'foo'}}) expr = self.M.data['gp1']['p1']['c1'] assertLookup(expr.alias('c1'), {'c1': 'foo'}) assertLookup(expr.as_json().alias('c2'), {'c2': 'foo'}) j.data = [ {'i1': ['foo', 'bar', 'baz']}, ['nugget', 'mickey']] j.save() expr = self.M.data[0]['i1'] assertLookup(expr.alias('i1'), {'i1': '["foo", "bar", "baz"]'}) assertLookup(expr.as_json().alias('i2'), {'i2': ['foo', 'bar', 'baz']}) expr = self.M.data[1][1] assertLookup(expr.alias('l1'), {'l1': 'mickey'}) assertLookup(expr.as_json().alias('l2'), {'l2': 'mickey'}) def test_json_cast(self): self.M.create(data={'foo': {'bar': 3}}) self.M.create(data={'foo': {'bar': 5}}) query = (self.M .select(Cast(self.M.data['foo']['bar'], 'float') * 1.5) .order_by(self.M.id) .tuples()) self.assertEqual(query[:], [(4.5,), (7.5,)]) def test_json_path(self): data = { 'foo': { 'baz': { 'bar': ['i1', 'i2', 'i3'], 'baze': ['j1', 'j2'], }}} j = self.M.create(data=data) def assertPath(path, expected): query = (self.M .select(path) .where(j._pk_expr()) .dicts()) self.assertEqual(query.get(), expected) expr = self.M.data.path('foo', 'baz', 'bar') assertPath(expr.alias('p1'), {'p1': ['i1', 'i2', 'i3']}) assertPath(expr.alias('p2'), {'p2': ['i1', 'i2', 'i3']}) expr = self.M.data.path('foo', 'baz', 'baze', 1) assertPath(expr.alias('p1'), {'p1': 'j2'}) assertPath(expr.alias('p2'), {'p2': 'j2'}) expr = self.M.data['foo'].path('baz', 'bar') assertPath(expr.as_json(False).alias('p1'), {'p1': '["i1", "i2", "i3"]'}) assertPath(expr.alias('p2'), {'p2': ['i1', 'i2', 'i3']}) def test_json_field_sql(self): j = (self.M .select() .where(self.M.data == {'foo': 'bar'})) table = self.M._meta.table_name self.assertSQL(j, ( 'SELECT "t1"."id", "t1"."data" ' 'FROM "%s" AS "t1" WHERE ("t1"."data" = ?)') % table) j = (self.M .select() .where(self.M.data['foo'] == 'bar')) self.assertSQL(j, ( 'SELECT "t1"."id", "t1"."data" ' 'FROM "%s" AS "t1" WHERE ("t1"."data"->>? = ?)') % table) def assertItems(self, where, *items): query = (self.M .select() .where(where) .order_by(self.M.id)) self.assertEqual( [item.id for item in query], [item.id for item in items]) def test_lookup(self): t1 = self.M.create(data={'k1': 'v1', 'k2': {'k3': 'v3'}}) t2 = self.M.create(data={'k1': 'x1', 'k2': {'k3': 'x3'}}) t3 = self.M.create(data={'k1': 'v1', 'j2': {'j3': 'v3'}}) self.assertItems((self.M.data['k2']['k3'] == 'v3'), t1) self.assertItems((self.M.data['k1'] == 'v1'), t1, t3) # Valid key, no matching value. self.assertItems((self.M.data['k2'] == 'v1')) # Non-existent key. self.assertItems((self.M.data['not-here'] == 'v1')) # Non-existent nested key. self.assertItems((self.M.data['not-here']['xxx'] == 'v1')) self.assertItems((self.M.data['k2']['xxx'] == 'v1')) def test_bulk_update(self): m1 = self.M.create(data={'k1': 'v1'}) m2 = self.M.create(data={'k2': 'v2'}) m3 = self.M.create(data=['i1', 'i2']) m1.data['k1'] = 'v1-x' m2.data['k2'] = 'v2-y' m3.data.append('i3') self.M.bulk_update([m1, m2, m3], fields=[self.M.data]) m1_db = self.M.get(self.M.id == m1.id) m2_db = self.M.get(self.M.id == m2.id) m3_db = self.M.get(self.M.id == m3.id) self.assertEqual(m1_db.data, {'k1': 'v1-x'}) self.assertEqual(m2_db.data, {'k2': 'v2-y'}) self.assertEqual(m3_db.data, ['i1', 'i2', 'i3']) def test_json_bulk_update_top_level_list(self): m1 = self.M.create(data=['a', 'b', 'c']) m2 = self.M.create(data=['d', 'e', 'f']) m1.data = ['g', 'h', 'i', {'j': 'kk'}] m2.data = ['j', 'k', 'l'] self.M.bulk_update([m1, m2], fields=[self.M.data]) m1_db = self.M.get(self.M.id == m1.id) m2_db = self.M.get(self.M.id == m2.id) self.assertEqual(m1_db.data, ['g', 'h', 'i', {'j': 'kk'}]) self.assertEqual(m2_db.data, ['j', 'k', 'l']) # Contains additional test-cases suitable for the JSONB data-type. class BaseBinaryJsonFieldTestCase(BaseJsonFieldTestCase): def _create_test_data(self): data = [ {'k1': 'v1', 'k2': 'v2', 'k3': {'k4': ['i1', 'i2'], 'k5': {}}}, ['a1', 'a2', {'a3': 'a4'}], {'a1': 'x1', 'a2': 'x2', 'k4': ['i1', 'i2']}, list(range(10)), list(range(5, 15)), ['k4', 'k1']] self._bjson_objects = [] for json_value in data: self._bjson_objects.append(self.M.create(data=json_value)) def assertObjects(self, expr, *indexes): query = (self.M .select() .where(expr) .order_by(self.M.id)) self.assertEqual( [bjson.data for bjson in query], [self._bjson_objects[index].data for index in indexes]) def test_contained_by(self): self._create_test_data() D = self.M.data item1 = ['a1', 'a2', {'a3': 'a4'}, 'a5'] self.assertObjects(D.contained_by(item1), 1) item2 = {'a1': 'x1', 'a2': 'x2', 'k4': ['i0', 'i1', 'i2'], 'x': 'y'} self.assertObjects(D.contained_by(item2), 2) self.assertObjects(D.contained_by(list(range(10))), 3) self.assertObjects(D.contained_by(list(range(20))), 3, 4) def test_contained_by_nested(self): self._create_test_data() D = self.M.data self.assertObjects(D['k4'].contained_by(['i1', 'i2']), 2) self.assertObjects(D['k3']['k4'].contained_by(['i1', 'i2']), 0) self.assertObjects(D[2].contained_by({'a3': 'a4'}), 1) self.assertObjects(D[0].contained_by(['a1', 'ax']), 1) def test_equality(self): data = {'k1': ['a1', 'a2'], 'k2': {'k3': 'v3'}} j = self.M.create(data=data) j_db = self.M.get(self.M.data == data) self.assertEqual(j.id, j_db.id) def test_subscript_contains(self): self._create_test_data() D = self.M.data # 'k3' is mapped to another dictioary {'k4': [...]}. Therefore, # 'k3' is said to contain 'k4', but *not* ['k4'] or ['k4', 'k5']. self.assertObjects(D['k3'].has_key('k4'), 0) self.assertObjects(D['k3'].contains(['k4'])) self.assertObjects(D['k3'].contains(['k4', 'k5'])) # We can check for the keys this way, though. self.assertObjects(D['k3'].contains_all('k4', 'k5'), 0) self.assertObjects(D['k3'].contains_any('k4', 'kx'), 0) # However, in test object index=2, 'k4' can be said to contain # both 'i1' and ['i1']. self.assertObjects(D['k4'].contains('i1'), 2) self.assertObjects(D['k4'].contains(['i1']), 2) # Interestingly, we can also specify the list of contained values # out-of-order. self.assertObjects(D['k4'].contains(['i2', 'i1']), 2) # We can test whether an object contains another JSON object fragment. self.assertObjects(D['k3'].contains({'k4': ['i1']}), 0) self.assertObjects(D['k3'].contains({'k4': ['i1', 'i2']}), 0) # Check multiple levels of nesting / containment. self.assertObjects(D['k3']['k4'].contains('i2'), 0) self.assertObjects(D['k3']['k4'].contains_all('i1', 'i2'), 0) self.assertObjects(D['k3']['k4'].contains_all('i0', 'i2')) self.assertObjects(D['k4'].contains_all('i1', 'i2'), 2) # Check array indexes. self.assertObjects(D[2].has_key('a3'), 1) self.assertObjects(D[2].contains('a3')) self.assertObjects(D[0].contains('a1'), 1) self.assertObjects(D[0].contains('k1')) def test_contains(self): self._create_test_data() D = self.M.data # Test for keys. 'k4' is both an object key and an array element. self.assertObjects(D.has_key('k4'), 2, 5) self.assertObjects(D.has_key('a1'), 1, 2) self.assertObjects(D.contains('a1'), 1) self.assertObjects(D.has_key('k3'), 0) # We can test for multiple top-level keys/indexes. self.assertObjects(D.contains_all('a1', 'a2'), 1, 2) # If we test for both with .contains(), though, it is treated as # an object match. self.assertObjects(D.contains(['a1', 'a2']), 1) # Check numbers. self.assertObjects(D.contains([2, 5, 6, 7, 8]), 3) self.assertObjects(D.contains([5, 6, 7, 8, 9]), 3, 4) # We can check for partial objects. self.assertObjects(D.contains({'a1': 'x1'}), 2) self.assertObjects(D.contains({'k3': {'k4': []}}), 0) self.assertObjects(D.contains([{'a3': 'a4'}]), 1) # Check for simple keys. self.assertObjects(D.contains(['a1']), 1) self.assertObjects(D.contains('a1'), 1) self.assertObjects(D.contains('k3')) # Contains any. self.assertObjects(D.contains_any('a1', 'k1'), 0, 1, 2, 5) self.assertObjects(D.contains_any('k4', 'xx', 'yy', '2'), 2, 5) self.assertObjects(D.contains_any('i1', 'i2', 'a3')) # Contains all. self.assertObjects(D.contains_all('k1', 'k2', 'k3'), 0) self.assertObjects(D.contains_all('k1', 'k2', 'k3', 'k4')) # Has key. self.assertObjects(D.has_key('a1'), 1, 2) self.assertObjects(D.has_key('k1'), 0, 5) self.assertObjects(D.has_key('k4'), 2, 5) self.assertObjects(D.has_key('a3')) self.assertObjects(D['k3'].has_key('k4'), 0) self.assertObjects(D['k4'].has_key('i2'), 2) def test_contains_contained_by(self): samples = ( {'k1': 'v1', 'k2': 'v2'}, {'k1': 'v10'}, ['i1', 'i2', 'i3', 'test'], 'k1', 123, 1.5, True, False) pks = [] for sample in samples: pks.append(self.M.create(data=sample).id) for i, sample in enumerate(samples): q = self.M.select().where(self.M.data.contains(sample)) self.assertEqual([x.id for x in q], [pks[i]]) q = self.M.select().where(self.M.data.contained_by(sample)) self.assertEqual([x.id for x in q], [pks[i]]) def test_concat_data(self): self.M.delete().execute() self.M.create(data={'k1': {'x1': 'y1'}, 'k2': 'v2', 'k3': [0, 1]}) def assertData(exp, expected_data): query = self.M.select(self.M.data.concat(exp)).tuples() data = query[:][0][0] self.assertEqual(data, expected_data) D = self.M.data assertData({'k2': 'v2-x', 'k1': {'x2': 'y2'}, 'k4': 'v4'}, { 'k1': {'x2': 'y2'}, # NB: not merged/patched!! 'k2': 'v2-x', 'k3': [0, 1], 'k4': 'v4'}) assertData({'k1': 'v1-x', 'k3': [2, 3, 4], 'k4': {'x4': 'y4'}}, { 'k1': 'v1-x', 'k2': 'v2', 'k3': [2, 3, 4], 'k4': {'x4': 'y4'}}) # We can update sub-keys. query = self.M.select(D['k1'].concat({'x2': 'y2', 'x3': 'y3'})) self.assertEqual(query.tuples()[0][0], {'x1': 'y1', 'x2': 'y2', 'x3': 'y3'}) # Concat can be used to extend JSON arrays. query = self.M.select(D['k3'].concat([2, 3])) self.assertEqual(query.tuples()[0][0], [0, 1, 2, 3]) def test_update_data_inplace(self): self.M.delete().execute() b = self.M.create(data={'k1': {'x1': 'y1'}, 'k2': 'v2'}) self.M.update(data=self.M.data.concat({ 'k1': {'x2': 'y2'}, 'k3': 'v3'})).execute() b2 = self.M.get(self.M.id == b.id) self.assertEqual(b2.data, {'k1': {'x2': 'y2'}, 'k2': 'v2', 'k3': 'v3'}) def test_selecting(self): self._create_test_data() query = (self.M .select(self.M.data['k3']['k4'].as_json().alias('k3k4')) .order_by(self.M.id)) k3k4_data = [obj.k3k4 for obj in query] self.assertEqual(k3k4_data, [ ['i1', 'i2'], None, None, None, None, None]) query = (self.M .select( self.M.data[0].as_json(), self.M.data[2].as_json()) .order_by(self.M.id) .tuples()) self.assertEqual(list(query), [ (None, None), ('a1', {'a3': 'a4'}), (None, None), (0, 2), (5, 7), ('k4', None)]) def test_conflict_update(self): b1 = self.M.create(data={'k1': 'v1'}) iq = (self.M .insert(id=b1.id, data={'k1': 'v1-x'}) .on_conflict('update', conflict_target=[self.M.id], update={self.M.data: {'k1': 'v1-z'}})) b1_id_db = iq.execute() self.assertEqual(b1.id, b1_id_db) b1_db = self.M.get(self.M.id == b1.id) self.assertEqual(self.M.data, {'k1': 'v1-z'}) iq = (self.M .insert(id=b1.id, data={'k1': 'v1-y'}) .on_conflict('update', conflict_target=[self.M.id], update={'data': {'k1': 'v1-w'}})) b1_id_db = iq.execute() self.assertEqual(b1.id, b1_id_db) b1_db = self.M.get(self.M.id == b1.id) self.assertEqual(self.M.data, {'k1': 'v1-w'}) self.assertEqual(self.M.select().count(), 1) ================================================ FILE: tests/prefetch_tests.py ================================================ from peewee import * from .base import get_in_memory_db from .base import requires_models from .base import ModelTestCase from .base import TestModel class Person(TestModel): name = TextField() class Relationship(TestModel): from_person = ForeignKeyField(Person, backref='relationships') to_person = ForeignKeyField(Person, backref='related_to') class Note(TestModel): person = ForeignKeyField(Person, backref='notes') content = TextField() class NoteItem(TestModel): note = ForeignKeyField(Note, backref='items') content = TextField() class Like(TestModel): person = ForeignKeyField(Person, backref='likes') note = ForeignKeyField(Note, backref='likes') class Flag(TestModel): note = ForeignKeyField(Note, backref='flags') is_spam = BooleanField() class Category(TestModel): name = TextField() parent = ForeignKeyField('self', backref='children', null=True) class Package(TestModel): barcode = TextField(unique=True) class PackageItem(TestModel): name = TextField() package = ForeignKeyField(Package, backref='items', field=Package.barcode) class TestPrefetch(ModelTestCase): database = get_in_memory_db() requires = [Person, Note, NoteItem, Like, Flag] def create_test_data(self): data = { 'huey': ( ('meow', ('meow-1', 'meow-2', 'meow-3')), ('purr', ()), ('hiss', ('hiss-1', 'hiss-2'))), 'mickey': ( ('woof', ()), ('bark', ('bark-1', 'bark-2'))), 'zaizee': (), } for name, notes in sorted(data.items()): person = Person.create(name=name) for note, items in notes: note = Note.create(person=person, content=note) for item in items: NoteItem.create(note=note, content=item) Flag.create(note=Note.get(Note.content == 'purr'), is_spam=True) Flag.create(note=Note.get(Note.content == 'woof'), is_spam=True) Like.create(note=Note.get(Note.content == 'meow'), person=Person.get(Person.name == 'mickey')) Like.create(note=Note.get(Note.content == 'woof'), person=Person.get(Person.name == 'huey')) def setUp(self): super(TestPrefetch, self).setUp() self.create_test_data() def accumulate_results(self, query, sort_items=False): accum = [] for person in query: notes = [] for note in person.notes: items = [] for item in note.items: items.append(item.content) if sort_items: items.sort() notes.append((note.content, items)) if sort_items: notes.sort() accum.append((person.name, notes)) return accum def test_prefetch_simple(self): for pt in PREFETCH_TYPE.values(): with self.assertQueryCount(3): people = Person.select().order_by(Person.name) query = people.prefetch(Note, NoteItem, prefetch_type=pt) accum = self.accumulate_results(query, sort_items=True) self.assertEqual(accum, [ ('huey', [ ('hiss', ['hiss-1', 'hiss-2']), ('meow', ['meow-1', 'meow-2', 'meow-3']), ('purr', [])]), ('mickey', [ ('bark', ['bark-1', 'bark-2']), ('woof', [])]), ('zaizee', []), ]) def test_prefetch_filter(self): for pt in PREFETCH_TYPE.values(): with self.assertQueryCount(3): people = Person.select().order_by(Person.name) notes = (Note .select() .where(Note.content.not_in(('hiss', 'meow', 'woof'))) .order_by(Note.content.desc())) items = NoteItem.select().where( ~NoteItem.content.endswith('-2')) query = prefetch(people, notes, items, prefetch_type=pt) self.assertEqual(self.accumulate_results(query), [ ('huey', [('purr', [])]), ('mickey', [('bark', ['bark-1'])]), ('zaizee', []), ]) def test_prefetch_reverse(self): for pt in PREFETCH_TYPE.values(): with self.assertQueryCount(2): people = Person.select().order_by(Person.name) notes = Note.select().order_by(Note.content) query = prefetch(notes, people, prefetch_type=pt) accum = [(note.content, note.person.name) for note in query] self.assertEqual(accum, [ ('bark', 'mickey'), ('hiss', 'huey'), ('meow', 'huey'), ('purr', 'huey'), ('woof', 'mickey')]) def test_prefetch_reverse_with_parent_join(self): for pt in PREFETCH_TYPE.values(): with self.assertQueryCount(2): notes = (Note .select(Note, Person) .join(Person) .order_by(Note.content)) items = NoteItem.select().order_by(NoteItem.content.desc()) query = prefetch(notes, items, prefetch_type=pt) accum = [(note.person.name, note.content, [item.content for item in note.items]) for note in query] self.assertEqual(accum, [ ('mickey', 'bark', ['bark-2', 'bark-1']), ('huey', 'hiss', ['hiss-2', 'hiss-1']), ('huey', 'meow', ['meow-3', 'meow-2', 'meow-1']), ('huey', 'purr', []), ('mickey', 'woof', []), ]) def test_prefetch_multi_depth(self): for pt in PREFETCH_TYPE.values(): people = Person.select().order_by(Person.name) notes = Note.select().order_by(Note.content) items = NoteItem.select().order_by(NoteItem.content) flags = Flag.select().order_by(Flag.id) LikePerson = Person.alias('lp') likes = (Like .select(Like, LikePerson.name) .join(LikePerson, on=(Like.person == LikePerson.id))) # Five queries: # - person (outermost query) # - notes for people # - items for notes # - flags for notes # - likes for notes (includes join to person) with self.assertQueryCount(5): query = prefetch(people, notes, items, flags, likes, prefetch_type=pt) accum = [] for person in query: notes = [] for note in person.notes: items = [item.content for item in note.items] likes = [like.person.name for like in note.likes] flags = [flag.is_spam for flag in note.flags] notes.append((note.content, items, likes, flags)) accum.append((person.name, notes)) self.assertEqual(accum, [ ('huey', [ ('hiss', ['hiss-1', 'hiss-2'], [], []), ('meow', ['meow-1', 'meow-2', 'meow-3'], ['mickey'], []), ('purr', [], [], [True])]), ('mickey', [ ('bark', ['bark-1', 'bark-2'], [], []), ('woof', [], ['huey'], [True])]), (u'zaizee', []), ]) def test_prefetch_multi_depth_no_join(self): for pt in PREFETCH_TYPE.values(): LikePerson = Person.alias() people = Person.select().order_by(Person.name) notes = Note.select().order_by(Note.content) items = NoteItem.select().order_by(NoteItem.content) flags = Flag.select().order_by(Flag.id) with self.assertQueryCount(6): query = prefetch(people, notes, items, flags, Like, LikePerson, prefetch_type=pt) accum = [] for person in query: notes = [] for note in person.notes: items = [item.content for item in note.items] likes = [like.person.name for like in note.likes] flags = [flag.is_spam for flag in note.flags] notes.append((note.content, items, likes, flags)) accum.append((person.name, notes)) self.assertEqual(accum, [ ('huey', [ ('hiss', ['hiss-1', 'hiss-2'], [], []), ('meow', ['meow-1', 'meow-2', 'meow-3'], ['mickey'], []), ('purr', [], [], [True])]), ('mickey', [ ('bark', ['bark-1', 'bark-2'], [], []), ('woof', [], ['huey'], [True])]), (u'zaizee', []), ]) def test_prefetch_with_group_by(self): for pt in PREFETCH_TYPE.values(): people = (Person .select(Person, fn.COUNT(Note.id).alias('note_count')) .join(Note, JOIN.LEFT_OUTER) .group_by(Person) .order_by(Person.name)) notes = Note.select().order_by(Note.content) items = NoteItem.select().order_by(NoteItem.content) with self.assertQueryCount(3): query = prefetch(people, notes, items, prefetch_type=pt) self.assertEqual(self.accumulate_results(query), [ ('huey', [ ('hiss', ['hiss-1', 'hiss-2']), ('meow', ['meow-1', 'meow-2', 'meow-3']), ('purr', [])]), ('mickey', [ ('bark', ['bark-1', 'bark-2']), ('woof', [])]), ('zaizee', []), ]) huey, mickey, zaizee = query self.assertEqual(huey.note_count, 3) self.assertEqual(mickey.note_count, 2) self.assertEqual(zaizee.note_count, 0) @requires_models(Category) def test_prefetch_self_join(self): def cc(name, parent=None): return Category.create(name=name, parent=parent) root = cc('root') p1 = cc('p1', root) p2 = cc('p2', root) for p in (p1, p2): for i in range(2): cc('%s-%s' % (p.name, i + 1), p) for pt in PREFETCH_TYPE.values(): Child = Category.alias('child') with self.assertQueryCount(2): query = prefetch(Category.select().order_by(Category.id), Child, prefetch_type=pt) names_and_children = [ (cat.name, [child.name for child in cat.children]) for cat in query] self.assertEqual(names_and_children, [ ('root', ['p1', 'p2']), ('p1', ['p1-1', 'p1-2']), ('p2', ['p2-1', 'p2-2']), ('p1-1', []), ('p1-2', []), ('p2-1', []), ('p2-2', []), ]) @requires_models(Category) def test_prefetch_adjacency_list(self): def cc(name, parent=None): return Category.create(name=name, parent=parent) tree = ('root', ( ('n1', ( ('c11', ()), ('c12', ()))), ('n2', ( ('c21', ()), ('c22', ( ('g221', ()), ('g222', ()))), ('c23', ()), ('c24', ( ('g241', ()), ('g242', ()), ('g243', ()))))))) stack = [(None, tree)] while stack: parent, (name, children) = stack.pop() node = cc(name, parent) for child_tree in children: stack.insert(0, (node, child_tree)) for pt in PREFETCH_TYPE.values(): C = Category.alias('c') G = Category.alias('g') GG = Category.alias('gg') GGG = Category.alias('ggg') query = Category.select().where(Category.name == 'root') with self.assertQueryCount(5): pf = prefetch(query, C, (G, C), (GG, G), (GGG, GG), prefetch_type=pt) def gather(c): children = sorted([gather(ch) for ch in c.children]) return (c.name, tuple(children)) nodes = list(pf) self.assertEqual(len(nodes), 1) pf_tree = gather(nodes[0]) self.assertEqual(tree, pf_tree) def test_prefetch_specific_model(self): # Person -> Note # -> Like (has fks to both person and note) Like.create(note=Note.get(Note.content == 'woof'), person=Person.get(Person.name == 'zaizee')) NoteAlias = Note.alias('na') for pt in PREFETCH_TYPE.values(): with self.assertQueryCount(3): people = Person.select().order_by(Person.name) notes = Note.select().order_by(Note.content) likes = (Like .select(Like, NoteAlias.content) .join(NoteAlias, on=(Like.note == NoteAlias.id)) .order_by(NoteAlias.content)) query = prefetch(people, notes, (likes, Person), prefetch_type=pt) accum = [] for person in query: likes = [] notes = [] for note in person.notes: notes.append(note.content) for like in person.likes: likes.append(like.note.content) accum.append((person.name, notes, likes)) self.assertEqual(accum, [ ('huey', ['hiss', 'meow', 'purr'], ['woof']), ('mickey', ['bark', 'woof'], ['meow']), ('zaizee', [], ['woof']), ]) @requires_models(Relationship) def test_multiple_foreign_keys(self): self.database.pragma('foreign_keys', 0) Person.delete().execute() c, h, z = [Person.create(name=name) for name in ('charlie', 'huey', 'zaizee')] RC = lambda f, t: Relationship.create(from_person=f, to_person=t) r1 = RC(c, h) r2 = RC(c, z) r3 = RC(h, c) r4 = RC(z, c) def assertRelationships(attr, values): self.assertEqual(len(attr),len(values)) for relationship, value in zip(attr, values): self.assertEqual(relationship.__data__, value) for pt in PREFETCH_TYPE.values(): with self.assertQueryCount(2): people = Person.select().order_by(Person.name) relationships = Relationship.select().order_by(Relationship.id) query = prefetch(people, relationships, prefetch_type=pt) cp, hp, zp = list(query) assertRelationships(cp.relationships, [ {'id': r1.id, 'from_person': c.id, 'to_person': h.id}, {'id': r2.id, 'from_person': c.id, 'to_person': z.id}]) assertRelationships(cp.related_to, [ {'id': r3.id, 'from_person': h.id, 'to_person': c.id}, {'id': r4.id, 'from_person': z.id, 'to_person': c.id}]) assertRelationships(hp.relationships, [ {'id': r3.id, 'from_person': h.id, 'to_person': c.id}]) assertRelationships(hp.related_to, [ {'id': r1.id, 'from_person': c.id, 'to_person': h.id}]) assertRelationships(zp.relationships, [ {'id': r4.id, 'from_person': z.id, 'to_person': c.id}]) assertRelationships(zp.related_to, [ {'id': r2.id, 'from_person': c.id, 'to_person': z.id}]) with self.assertQueryCount(2): query = prefetch(relationships, people, prefetch_type=pt) accum = [] for row in query: accum.append((row.from_person.name, row.to_person.name)) self.assertEqual(accum, [ ('charlie', 'huey'), ('charlie', 'zaizee'), ('huey', 'charlie'), ('zaizee', 'charlie')]) m = Person.create(name='mickey') RC(h, m) def assertNames(p, ns): self.assertEqual([r.to_person.name for r in p.relationships], ns) # Use prefetch to go Person -> Relationship <- Person (PA). for pt in PREFETCH_TYPE.values(): with self.assertQueryCount(3): people = (Person .select() .where(Person.name != 'mickey') .order_by(Person.name)) relationships = Relationship.select().order_by(Relationship.id) PA = Person.alias() query = prefetch(people, relationships, PA, prefetch_type=pt) cp, hp, zp = list(query) assertNames(cp, ['huey', 'zaizee']) assertNames(hp, ['charlie', 'mickey']) assertNames(zp, ['charlie']) # User prefetch to go Person -> Relationship+Person (PA). with self.assertQueryCount(2): people = (Person .select() .where(Person.name != 'mickey') .order_by(Person.name)) rels = (Relationship .select(Relationship, PA) .join(PA, on=(Relationship.to_person == PA.id)) .order_by(Relationship.id)) query = prefetch(people, rels, prefetch_type=pt) cp, hp, zp = list(query) assertNames(cp, ['huey', 'zaizee']) assertNames(hp, ['charlie', 'mickey']) assertNames(zp, ['charlie']) def test_prefetch_through_manytomany(self): Like.create(note=Note.get(Note.content == 'meow'), person=Person.get(Person.name == 'zaizee')) Like.create(note=Note.get(Note.content == 'woof'), person=Person.get(Person.name == 'zaizee')) for pt in PREFETCH_TYPE.values(): with self.assertQueryCount(3): people = Person.select().order_by(Person.name) notes = Note.select().order_by(Note.content) likes = Like.select().order_by(Like.id) query = prefetch(people, likes, notes, prefetch_type=pt) accum = [] for person in query: liked_notes = [] for like in person.likes: liked_notes.append(like.note.content) accum.append((person.name, liked_notes)) self.assertEqual(accum, [ ('huey', ['woof']), ('mickey', ['meow']), ('zaizee', ['meow', 'woof']), ]) @requires_models(Package, PackageItem) def test_prefetch_non_pk_fk(self): data = ( ('101', ('a', 'b')), ('102', ('a', 'b')), ('103', ()), ('104', ('a', 'b', 'c', 'd', 'e')), ) for barcode, items in data: Package.create(barcode=barcode) for item in items: PackageItem.create(package=barcode, name=item) for pt in PREFETCH_TYPE.values(): packages = Package.select().order_by(Package.barcode) items = PackageItem.select().order_by(PackageItem.name) with self.assertQueryCount(2): query = prefetch(packages, items, prefetch_type=pt) for package, (barcode, items) in zip(query, data): self.assertEqual(package.barcode, barcode) self.assertEqual([item.name for item in package.items], list(items)) def test_prefetch_mark_dirty_regression(self): for pt in PREFETCH_TYPE.values(): people = Person.select().order_by(Person.name) query = people.prefetch(Note, NoteItem, prefetch_type=pt) for person in query: self.assertEqual(person.dirty_fields, []) for note in person.notes: self.assertEqual(note.dirty_fields, []) for item in note.items: self.assertEqual(item.dirty_fields, []) class X(TestModel): name = TextField() class Z(TestModel): x = ForeignKeyField(X) name = TextField() class A(TestModel): name = TextField() x = ForeignKeyField(X) class B(TestModel): name = TextField() a = ForeignKeyField(A) x = ForeignKeyField(X) class C(TestModel): name = TextField() b = ForeignKeyField(B) x = ForeignKeyField(X, null=True) class C1(TestModel): name = TextField() c = ForeignKeyField(C) class C2(TestModel): name = TextField() c = ForeignKeyField(C) class TestPrefetchMultiRefs(ModelTestCase): database = get_in_memory_db() requires = [X, Z, A, B, C, C1, C2] def test_prefetch_multirefs(self): x1, x2, x3 = [X.create(name=n) for n in ('x1', 'x2', 'x3')] for i, x in enumerate((x1, x2, x3), 1): for j in range(i): Z.create(x=x, name='%s-z%s' % (x.name, j)) xs = {x.name: x for x in X.select()} xs[None] = None data = [ ('a1', 'x1', ['x1-z0'], [ ('a1-b1', 'x1', ['x1-z0'], [ ('a1-b1-c1', 'x1', ['x1-z0'], [], []), ]), ]), ('a2', 'x2', ['x2-z0', 'x2-z1'], [ ('a2-b1', 'x1', ['x1-z0'], [ ('a2-b1-c1', 'x1', ['x1-z0'], [], []), ]), ('a2-b2', 'x2', ['x2-z0', 'x2-z1'], [ ('a2-b2-c1', 'x2', ['x2-z0', 'x2-z1'], [], []), ('a2-b2-c2', 'x1', ['x1-z0'], [], []), ('a2-b2-cx', None, [], [], []), ]), ]), ('a3', 'x3', ['x3-z0', 'x3-z1', 'x3-z2'], [ ('a3-b1', 'x1', ['x1-z0'], [ ('a3-b1-c1', 'x1', ['x1-z0'], [], []), ]), ('a3-b2', 'x2', ['x2-z0', 'x2-z1'], [ ('a3-b2-c1', 'x2', ['x2-z0', 'x2-z1'], [], []), ('a3-b2-c2', 'x2', ['x2-z0', 'x2-z1'], [], []), ('a3-b2-cx1', None, [], [], []), ('a3-b2-cx2', None, [], [], []), ('a3-b2-cx3', None, [], [], []), ]), ('a3-b3', 'x3', ['x3-z0', 'x3-z1', 'x3-z2'], [ ('a3-b3-c1', 'x3', ['x3-z0', 'x3-z1', 'x3-z2'], [], []), ('a3-b3-c2', 'x3', ['x3-z0', 'x3-z1', 'x3-z2'], [], []), ('a3-b3-c3', 'x3', ['x3-z0', 'x3-z1', 'x3-z2'], ['c1-1', 'c1-2', 'c1-3', 'c1-4'], ['c2-1', 'c2-2']), ]), ]), ] for a, ax, azs, bs in data: a = A.create(name=a, x=xs[ax]) for b, bx, bzs, cs in bs: b = B.create(name=b, a=a, x=xs[bx]) for c, cx, czs, c1s, c2s in cs: c = C.create(name=c, b=b, x=xs[cx]) for c1 in c1s: C1.create(name=c1, c=c) for c2 in c2s: C2.create(name=c2, c=c) AX = X.alias('ax') AXZ = Z.alias('axz') BX = X.alias('bx') BXZ = Z.alias('bxz') CX = X.alias('cx') CXZ = Z.alias('cxz') with self.assertQueryCount(11): q = prefetch(A.select().order_by(A.name), *( (AX, A), (AXZ, AX), (B, A), (BX, B), (BXZ, BX), (C, B), (CX, C), (CXZ, CX), (C1, C), (C2, C))) with self.assertQueryCount(0): accum = [] for a in list(q): azs = [z.name for z in a.x.z_set] bs = [] for b in a.b_set: bzs = [z.name for z in b.x.z_set] cs = [] for c in b.c_set: czs = [z.name for z in c.x.z_set] if c.x else [] c1s = [c1.name for c1 in c.c1_set] c2s = [c2.name for c2 in c.c2_set] cs.append((c.name, c.x.name if c.x else None, czs, c1s, c2s)) bs.append((b.name, b.x.name, bzs, cs)) accum.append((a.name, a.x.name, azs, bs)) self.assertEqual(data, accum) ================================================ FILE: tests/pwasyncio.py ================================================ import asyncio import collections import contextvars import glob import itertools import tempfile import os import unittest from unittest.mock import Mock, AsyncMock, MagicMock, patch from peewee import * from playhouse.pwasyncio import * from playhouse.pwasyncio import _State, _ConnectionState, _lazy_cursor_iter from .base import MYSQL_PARAMS from .base import PSQL_PARAMS from .base import IS_MYSQL from .base import IS_POSTGRESQL try: import asyncpg except ImportError: asyncpg = None try: import aiomysql except ImportError: aiomysql = None import aiosqlite SQLITE_RETURNING = aiosqlite.sqlite_version_info >= (3, 35, 0) class TestModel(Model): name = CharField() value = IntegerField(default=0) class User(Model): username = CharField() class Tweet(Model): user = ForeignKeyField(User, backref='tweets') message = TextField() class TestGreenletSpawn(unittest.IsolatedAsyncioTestCase): async def test_simple_function(self): result = await greenlet_spawn(lambda x, y: x + y, 5, 3) self.assertEqual(result, 8) async def test_function_with_await(self): async def async_helper(): await asyncio.sleep(0.01) return 2 def func(): return await_(async_helper()) * 2 self.assertEqual(await greenlet_spawn(func), 4) async def test_multiple_awaits(self): async def fetch_value(val): await asyncio.sleep(0.01) return val def multi(): return sum([await_(fetch_value(i)) for i in [10, 20, 30]]) self.assertEqual(await greenlet_spawn(multi), 60) async def test_exception_propagation(self): with self.assertRaises(ValueError): await greenlet_spawn(lambda: (_ for _ in ()).throw(ValueError('x'))) async def test_exception_in_awaitable(self): async def fail(): raise RuntimeError('async error') with self.assertRaises(RuntimeError): await greenlet_spawn(lambda: await_(fail())) def test_await_outside_greenlet(self): with self.assertRaises(MissingGreenletBridge): await_(Mock()) async def test_contextvars(self): var = contextvars.ContextVar('data', default='x') state = [] def get_var(): state.append(var.get()) async def aget_var(): await greenlet_spawn(get_var) var.set('y') await aget_var() await greenlet_spawn(lambda: await_(aget_var())) await aget_var() self.assertEqual(state, ['y', 'y', 'y']) class TestConnectionState(unittest.IsolatedAsyncioTestCase): async def test_task_isolation(self): cs = _ConnectionState() async def worker(tid): cs._current().conn = tid await asyncio.sleep(0.01) return cs._current().conn results = await asyncio.gather(*[worker(i) for i in range(5)]) self.assertEqual(results, [0, 1, 2, 3, 4]) async def test_state_attributes(self): cs = _ConnectionState() cs.set_connection('c') cs.transactions.append(1) self.assertEqual(cs.conn, 'c') self.assertFalse(cs.closed) self.assertEqual(cs.transactions, [1]) async def test_get_returns_fresh_state(self): s = _ConnectionState()._current() self.assertIsNone(s.conn) self.assertTrue(s.closed) self.assertEqual(s.transactions, []) async def test_reset(self): cs = _ConnectionState() cs.set_connection('x') cs.transactions.append(1) cs.reset() self.assertIsNone(cs.conn) self.assertTrue(cs.closed) self.assertEqual(cs.transactions, []) async def test_set_connection(self): cs = _ConnectionState() m = Mock() cs.set_connection(m) self.assertIs(cs.conn, m) self.assertFalse(cs.closed) async def test_done_callback_orphans_connection(self): cs = _ConnectionState() conn_mock = Mock() async def acquire_and_abandon(): cs.set_connection(conn_mock) return id(asyncio.current_task()) task_id = await asyncio.create_task(acquire_and_abandon()) # After the task completes, the done-callback should have fired. await asyncio.sleep(0) self.assertNotIn(task_id, cs._states) self.assertIn(conn_mock, cs._orphaned_conns) async def test_done_callback_noop_when_closed(self): cs = _ConnectionState() async def open_and_close(): cs.set_connection(Mock()) cs.reset() # Simulate proper close. await asyncio.create_task(open_and_close()) await asyncio.sleep(0) self.assertEqual(cs._orphaned_conns, []) def _make_lazy_cursor(rows, batch_size=2): it = iter(rows) fetch_counts = [] cleanup_called = [] async def fetch_many(count): fetch_counts.append(count) return list(itertools.islice(it, count)) async def cleanup(): cleanup_called.append(True) cursor = CursorAdapter( description=[('id',), ('name',)], fetch_many=fetch_many, cleanup=cleanup, buffer_size=batch_size) return cursor, fetch_counts, cleanup_called class TestCursorAdapter(unittest.IsolatedAsyncioTestCase): def test_eager_fetchone(self): c = CursorAdapter([(1, 'a'), (2, 'b'), (3, 'c')]) self.assertEqual(c.fetchone(), (1, 'a')) self.assertEqual(c.fetchone(), (2, 'b')) self.assertEqual(c.fetchone(), (3, 'c')) self.assertIsNone(c.fetchone()) def test_eager_fetchall(self): rows = [(1,), (2,)] c = CursorAdapter(rows) self.assertIs(c.fetchall(), rows) def test_eager_iter(self): rows = [(1,), (2,), (3,)] self.assertEqual(list(CursorAdapter(rows)), rows) self.assertEqual(CursorAdapter(rows).rowcount, 3) def test_eager_metadata(self): c = CursorAdapter() self.assertEqual(c._rows, []) self.assertEqual(c.rowcount, 0) self.assertEqual(c.description, []) self.assertIsNone(c.fetchone()) self.assertEqual(list(c), []) c = CursorAdapter([(1,)], lastrowid=5, rowcount=1, description=[('id',)]) self.assertEqual(c.lastrowid, 5) self.assertEqual(c.rowcount, 1) self.assertEqual(c.description, [('id',)]) async def test_lazy_fetchone_batches(self): rows = [(i,) for i in range(5)] cursor, counts, _ = _make_lazy_cursor(rows, batch_size=2) collected = [] def drain(): while True: r = cursor.fetchone() if r is None: break collected.append(r) await greenlet_spawn(drain) self.assertEqual(collected, rows) # 2 + 2 + 1 + 0(empty) = 4 calls, each requesting 2 self.assertEqual(len(counts), 4) self.assertTrue(all(c == 2 for c in counts)) async def test_lazy_fetchone_empty(self): cursor, _, _ = _make_lazy_cursor([], batch_size=2) self.assertIsNone(await greenlet_spawn(cursor.fetchone)) # Already exhausted, still returns None. self.assertIsNone(await greenlet_spawn(cursor.fetchone)) async def test_lazy_iter(self): rows = [(i,) for i in range(7)] cursor, counts, _ = _make_lazy_cursor(rows, batch_size=3) self.assertEqual(await greenlet_spawn(list, cursor), rows) # 3 + 3 + 1 + 0 = 4 calls self.assertEqual(len(counts), 4) async def test_lazy_fetchall(self): rows = [(1,), (2,), (3,)] cursor, _, _ = _make_lazy_cursor(rows, batch_size=10) self.assertEqual(await greenlet_spawn(cursor.fetchall), rows) async def test_lazy_buffer_reuse(self): rows = [(i,) for i in range(3)] cursor, counts, _ = _make_lazy_cursor(rows, batch_size=10) await greenlet_spawn(cursor.fetchone) self.assertEqual(len(counts), 1) await greenlet_spawn(cursor.fetchone) await greenlet_spawn(cursor.fetchone) self.assertEqual(len(counts), 1) # still 1 await greenlet_spawn(cursor.fetchone) # second fetch (empty). self.assertEqual(len(counts), 2) async def test_lazy_description(self): cursor, _, _ = _make_lazy_cursor([], batch_size=2) self.assertEqual(cursor.description, [('id',), ('name',)]) async def test_lazy_buffer_size_override(self): rows = [(i,) for i in range(10)] cursor, counts, _ = _make_lazy_cursor(rows, batch_size=5) cursor._buffer_size = 3 await greenlet_spawn(list, cursor) self.assertTrue(all(c == 3 for c in counts)) async def test_aclose_cleanup(self): cursor, _, cleanup = _make_lazy_cursor([], batch_size=2) await cursor.aclose() self.assertEqual(cleanup, [True]) self.assertIsNone(cursor._fetch_many) self.assertIsNone(cursor._cleanup) async def test_aclose_idempotent(self): call_count = [] async def cleanup(): call_count.append(1) cursor, _, _ = _make_lazy_cursor([], batch_size=2) cursor._cleanup = cleanup await cursor.aclose() await cursor.aclose() self.assertEqual(len(call_count), 1) async def test_aclose_noop_for_eager(self): await CursorAdapter([(1,)]).aclose() # must not raise async def test_lazy_cursor_iter(self): rows = [(1,), (2,), (3,)] cursor, _, _ = _make_lazy_cursor(rows, batch_size=10) result = await greenlet_spawn(list, _lazy_cursor_iter(cursor)) self.assertEqual(result, rows) async def test_lazy_cursor_iter_empty(self): cursor, _, _ = _make_lazy_cursor([], batch_size=2) result = await greenlet_spawn(list, _lazy_cursor_iter(cursor)) self.assertEqual(result, []) class TestConnectionWrappers(unittest.IsolatedAsyncioTestCase): async def test_sqlite_execute(self): mock_cursor = AsyncMock() mock_cursor.fetchall.return_value = [(1, 'test')] mock_cursor.lastrowid = 1 mock_cursor.rowcount = 1 mock_cursor.description = [('id',), ('name',)] mock_conn = AsyncMock() mock_conn.execute.return_value = mock_cursor result = await AsyncSqliteConnection(mock_conn).execute( 'SELECT * FROM test') self.assertIsInstance(result, CursorAdapter) self.assertEqual(result.fetchall(), [(1, 'test')]) self.assertEqual(result.lastrowid, 1) mock_cursor.close.assert_awaited_once() async def test_sqlite_execute_iter_returns_lazy(self): mock_cursor = AsyncMock() mock_cursor.description = [('a',), ('b',)] mock_conn = AsyncMock() mock_conn.execute.return_value = mock_cursor conn = AsyncSqliteConnection(mock_conn) cursor = await conn.execute_iter('SELECT a, b FROM t') self.assertIsInstance(cursor, CursorAdapter) self.assertIsNotNone(cursor._fetch_many) self.assertEqual(cursor.description, [('a',), ('b',)]) await cursor.aclose() async def test_sqlite_execute_iter_lock_lifecycle(self): mock_cursor = AsyncMock() mock_cursor.description = [] mock_conn = AsyncMock() mock_conn.execute.return_value = mock_cursor conn = AsyncSqliteConnection(mock_conn) cursor = await conn.execute_iter('SELECT 1') self.assertTrue(conn._lock.locked()) await cursor.aclose() self.assertFalse(conn._lock.locked()) mock_cursor.close.assert_awaited_once() async def test_sqlite_execute_iter_lock_on_failure(self): mock_conn = AsyncMock() mock_conn.execute.side_effect = RuntimeError('fail') conn = AsyncSqliteConnection(mock_conn) with self.assertRaises(RuntimeError): await conn.execute_iter('invalid') self.assertFalse(conn._lock.locked()) async def test_mysql_execute(self): mock_cursor = AsyncMock() mock_cursor.fetchall.return_value = [(1, 'test')] mock_cursor.lastrowid = 1 mock_cursor.rowcount = 1 mock_cursor.description = [('id',), ('name',)] mock_conn = AsyncMock() mock_conn.cursor.return_value = mock_cursor result = await AsyncMySQLConnection(mock_conn).execute( 'SELECT * FROM test') self.assertIsInstance(result, CursorAdapter) self.assertEqual(result.fetchall(), [(1, 'test')]) mock_cursor.close.assert_awaited_once() async def test_mysql_cursor_closed_on_error(self): mock_cursor = AsyncMock() mock_cursor.execute.side_effect = RuntimeError('fail') mock_conn = AsyncMock() mock_conn.cursor.return_value = mock_cursor with self.assertRaises(RuntimeError): await AsyncMySQLConnection(mock_conn).execute('invalid') mock_cursor.close.assert_awaited_once() async def test_mysql_concurrent_serialized(self): order = [] async def tracked(sql, params): order.append(f'start-{sql}') await asyncio.sleep(0.05) order.append(f'end-{sql}') return [] mock_cursor = AsyncMock() mock_cursor.execute = tracked mock_conn = AsyncMock() mock_conn.cursor.return_value = mock_cursor conn = AsyncMySQLConnection(mock_conn) await asyncio.gather(conn.execute('Q1', None), conn.execute('Q2', None)) idx = {e: i for i, e in enumerate(order)} self.assertTrue(idx['end-Q1'] < idx['start-Q2'] or idx['end-Q2'] < idx['start-Q1']) async def test_mysql_execute_iter_uses_ss_cursor(self): import playhouse.pwasyncio as mod mock_cursor = AsyncMock() mock_cursor.description = [('x',)] mock_cursor.execute = AsyncMock() mock_conn = AsyncMock() mock_conn.cursor.return_value = mock_cursor sentinel = object() with patch.object(mod, 'aiomysql') as m: m.SSCursor = sentinel cursor = await AsyncMySQLConnection(mock_conn).execute_iter( 'SELECT 1') mock_conn.cursor.assert_awaited_once_with(sentinel) self.assertIsNotNone(cursor._fetch_many) await cursor.aclose() async def test_mysql_execute_iter_lock_lifecycle(self): import playhouse.pwasyncio as mod mock_cursor = AsyncMock() mock_cursor.description = [('x',)] mock_cursor.execute = AsyncMock() mock_cursor.close = AsyncMock() mock_conn = AsyncMock() mock_conn.cursor.return_value = mock_cursor conn = AsyncMySQLConnection(mock_conn) with patch.object(mod, 'aiomysql', create=True) as m: m.SSCursor = object() cursor = await conn.execute_iter('SELECT 1') self.assertTrue(conn._lock.locked()) await cursor.aclose() self.assertFalse(conn._lock.locked()) mock_cursor.close.assert_awaited_once() async def test_pg_parameter_conversion(self): mock_record = Mock() mock_record.keys.return_value = ['id', 'name'] mock_conn = AsyncMock() mock_conn.fetch.return_value = [mock_record] await AsyncPostgresqlConnection(mock_conn).execute( 'SELECT * FROM t WHERE id = %s AND name = %s', (1, 'x')) sql = mock_conn.fetch.call_args[0][0] self.assertEqual(sql, 'SELECT * FROM t WHERE id = $1 AND name = $2') async def test_pg_concurrent_serialized(self): order = [] async def tracked(sql, params=None): order.append(f'start-{sql}') await asyncio.sleep(0.05) order.append(f'end-{sql}') return [] mock_conn = AsyncMock() mock_conn.fetch = tracked conn = AsyncPostgresqlConnection(mock_conn) await asyncio.gather(conn.execute('Q1', None), conn.execute('Q2', None)) idx = {e: i for i, e in enumerate(order)} self.assertTrue(idx['end-Q1'] < idx['start-Q2'] or idx['end-Q2'] < idx['start-Q1']) async def test_pg_no_params(self): mock_conn = AsyncMock() mock_conn.fetch.return_value = [] await AsyncPostgresqlConnection(mock_conn).execute( 'SELECT * FROM t', None) mock_conn.fetch.assert_called_once_with('SELECT * FROM t') async def test_pg_empty_results(self): mock_conn = AsyncMock() mock_conn.fetch.return_value = [] r = await AsyncPostgresqlConnection(mock_conn).execute( 'SELECT * FROM empty') self.assertEqual(r.fetchall(), []) self.assertEqual(r.description, []) def test_translate_placeholders(self): f = AsyncPostgresqlConnection._translate_placeholders self.assertEqual(f('SELECT 1'), 'SELECT 1') self.assertEqual( f('SELECT * FROM t WHERE a = %s AND b = %s'), 'SELECT * FROM t WHERE a = $1 AND b = $2') self.assertEqual( f('INSERT INTO t VALUES (%s, %s, %s)'), 'INSERT INTO t VALUES ($1, $2, $3)') def _pg_mocks(self, rows=None): rows = rows or [] attr = MagicMock(); attr.name = 'col1' it = iter(rows) mock_cursor = AsyncMock() async def _fetch(count): return list(itertools.islice(it, count)) mock_cursor.fetch = _fetch mock_stmt = AsyncMock() mock_stmt.cursor.return_value = mock_cursor mock_stmt.get_attributes = MagicMock(return_value=[attr]) mock_tr = AsyncMock() mock_conn = MagicMock() mock_conn.transaction.return_value = mock_tr mock_conn.prepare = AsyncMock(return_value=mock_stmt) return mock_conn, mock_tr, mock_stmt, mock_cursor async def test_pg_execute_iter_description(self): mock_conn, _, mock_stmt, _ = self._pg_mocks() a1, a2 = MagicMock(), MagicMock() a1.name = 'id'; a2.name = 'username' mock_stmt.get_attributes.return_value = [a1, a2] conn = AsyncPostgresqlConnection(mock_conn) cursor = await conn.execute_iter('SELECT id, username FROM users') self.assertEqual(cursor.description, [('id',), ('username',)]) await cursor.aclose() async def test_pg_execute_iter_starts_transaction(self): mock_conn, mock_tr, _, _ = self._pg_mocks() conn = AsyncPostgresqlConnection(mock_conn) cursor = await conn.execute_iter('SELECT 1') mock_conn.transaction.assert_called_once() mock_tr.start.assert_awaited_once() await cursor.aclose() async def test_pg_execute_iter_cleanup_rolls_back(self): mock_conn, mock_tr, _, _ = self._pg_mocks() conn = AsyncPostgresqlConnection(mock_conn) cursor = await conn.execute_iter('SELECT 1') self.assertTrue(conn._lock.locked()) await cursor.aclose() mock_tr.rollback.assert_awaited_once() self.assertFalse(conn._lock.locked()) async def test_pg_execute_iter_translates_placeholders(self): mock_conn, _, _, _ = self._pg_mocks() conn = AsyncPostgresqlConnection(mock_conn) cursor = await conn.execute_iter( 'SELECT * FROM t WHERE a = %s AND b = %s', params=(1, 2)) sql = mock_conn.prepare.call_args[0][0] self.assertIn('$1', sql) self.assertNotIn('%s', sql) await cursor.aclose() async def test_pg_execute_iter_lock_on_failure(self): mock_conn, _, _, _ = self._pg_mocks() mock_conn.prepare = AsyncMock(side_effect=RuntimeError('fail')) conn = AsyncPostgresqlConnection(mock_conn) with self.assertRaises(RuntimeError): await conn.execute_iter('invalid') self.assertFalse(conn._lock.locked()) class TestTaskLifecycle(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self): with tempfile.NamedTemporaryFile(delete=False) as f: self.db_path = f.name self.db = AsyncSqliteDatabase(self.db_path) TestModel._meta.set_database(self.db) async with self.db: await self.db.acreate_tables([TestModel]) async def asyncTearDown(self): await self.db.close_pool() if self.db_path and os.path.exists(self.db_path): for fname in glob.glob(self.db_path + '*'): os.unlink(fname) async def test_task_id_behavior(self): async def a1(db): accum = [] async with db: accum.append(db._state._current()) accum.append(await a2(db)) return accum async def a2(db): async with db: return db._state._current() def s1(db): accum = [] with db.connection_context(): accum.append(db._state._current()) accum.append(s2(db)) return accum def s2(db): with db.connection_context(): return db._state._current() async with self.db: ids = await(a1(self.db)) ids.extend(await self.db.run(s1, self.db)) self.assertEqual(len(ids), 4) self.assertEqual(len(set(ids)), 1) async with self.db: ids = await asyncio.create_task(a1(self.db)) ids.extend(await asyncio.create_task(self.db.run(s1, self.db))) self.assertEqual(len(ids), 4) self.assertEqual(len(set(ids)), 2) async def test_task_state_cleanup_after_completion(self): async def task_with_state(): async with self.db: await self.db.run(TestModel.create, name='test', value=1) return id(asyncio.current_task()) await asyncio.create_task(task_with_state()) # Child task properly closed connection (async with db), so close pool # should exit cleanly. await asyncio.wait_for(self.db.close_pool(), timeout=2.0) # Verify the write persisted. async with self.db: self.assertEqual(await self.db.count(TestModel.select()), 1) async def test_concurrent_task_state_isolation(self): async def capture(tid): async with self.db: before = id(self.db._state._current()) await self.db.run(TestModel.create, name=f't{tid}', value=tid) after = id(self.db._state._current()) self.assertEqual(before, after) return before results = await asyncio.gather(*[capture(i) for i in range(5)]) self.assertTrue(all(results)) self.assertEqual(len(set(results)), 5) async def test_connection_returned_when_task_dies(self): async def acquire_and_abandon(): await self.db.aconnect() return # Connection is not closed, callback must handle cleanup. await asyncio.create_task(acquire_and_abandon()) # The done-callback should have moved the connection to orphaned # connections, which are handled either via done callback or during # pool shutdown. await asyncio.wait_for(self.db.close_pool(), timeout=2.0) class IntegrationTests(object): db_path = None models = [TestModel, User, Tweet] def get_database(self): with tempfile.NamedTemporaryFile(delete=False) as f: self.db_path = f.name return AsyncSqliteDatabase(self.db_path) def tearDown(self): if self.db_path and os.path.exists(self.db_path): os.unlink(self.db_path) async def asyncSetUp(self): try: self.db = self.get_database() await self.db.aconnect() await self.db.aclose() except Exception as exc: self.skipTest(f'Cannot connect: {exc}') if isinstance(self.db, AsyncSqliteDatabase): self.driver = 'sqlite' self.support_returning = SQLITE_RETURNING elif isinstance(self.db, AsyncMySQLDatabase): self.driver = 'mysql' self.support_returning = False elif isinstance(self.db, AsyncPostgresqlDatabase): self.driver = 'postgresql' self.support_returning = True else: raise ValueError('Unrecognized driver') for m in self.models: m._meta.set_database(self.db) async with self.db: await self.db.adrop_tables(self.models) await self.db.acreate_tables(self.models) async def asyncTearDown(self): await self.db.aclose() async with self.db: await self.db.adrop_tables(self.models) await self.db.close_pool() async def create_record(self, name='test', value=1): return await self.db.run(TestModel.create, name=name, value=value) async def assertCount(self, expected): count = await self.db.run(TestModel.select().count) self.assertEqual(count, expected) async def assertNames(self, expected): curs = await self.db.list(TestModel.select().order_by(TestModel.name)) self.assertEqual([tm.name for tm in curs], expected) async def seed(self, n=20): def _seed(): with self.db.atomic(): for i in range(n): TestModel.create(name=f'item{i:02d}', value=i * 10) await self.db.run(_seed) async def test_pool_created_on_connect(self): await self.db.aclose() await self.db.close_pool() self.assertIsNone(self.db._pool) await self.db.aconnect() self.assertIsNotNone(self.db._pool) self.assertIsNotNone(self.db._state.conn) self.assertFalse(self.db.is_closed()) await self.db.aclose() self.assertIsNone(self.db._state.conn) self.assertTrue(self.db.is_closed()) async def test_is_closed(self): for i in range(2): await self.db.aconnect() self.assertFalse(self.db.is_closed()) await self.db.aclose() self.assertTrue(self.db.is_closed()) async def test_multiple_close_safe(self): await self.db.aclose() self.assertTrue(self.db.is_closed()) await self.db.aclose() await self.db.aconnect() self.assertFalse(self.db.is_closed()) async def test_reconnect_after_pool_close(self): await self.create_record('first', 1) await self.db.aclose() await self.db.close_pool() self.assertIsNone(self.db._pool) async with self.db: await self.assertCount(1) self.assertIsNotNone(self.db._pool) self.assertTrue(self.db.is_closed()) async def test_connection_reuse_within_task(self): await self.db.aconnect() c1 = self.db._state.conn await self.create_record('a', 1) c2 = self.db._state.conn await self.create_record('b', 2) self.assertIs(c1, c2) self.assertIs(c2, self.db._state.conn) async def test_closing_flag_prevents_connect(self): self.db._closing = True try: with self.assertRaises(InterfaceError): await self.db.aconnect() finally: self.db._closing = False async def test_double_close_pool(self): await self.db.aclose() await self.db.close_pool() await self.db.close_pool() async def test_dead_connection_replaced(self): if self.driver == 'mysql': self.skipTest('closing underlying conn incompatible with aiomysql') return await self.db.aconnect() conn = self.db._state.conn await conn.close() await self.create_record('test', 1) self.assertIsNot(self.db._state.conn, conn) await self.assertCount(1) async def test_context_manager(self): async with self.db: self.assertIsNotNone(self.db._state.conn) self.assertFalse(self.db._state.closed) self.assertFalse(self.db.is_closed()) self.assertIsNone(self.db._state.conn) self.assertTrue(self.db._state.closed) self.assertTrue(self.db.is_closed()) async def test_exception_in_context_manager(self): try: async with self.db: raise RuntimeError('fail') except RuntimeError: pass self.assertTrue(self.db._state.closed) self.assertTrue(self.db.is_closed()) async with self.db: await self.create_record('after_error', 1) self.assertFalse(self.db.is_closed()) await self.assertCount(1) self.assertTrue(self.db.is_closed()) async def test_execute_sql(self): iq, iparams = User.insert(username='x').sql() sq, _= User.select().sql() await self.db.aexecute_sql(iq, iparams) r = await self.db.aexecute_sql(sq) self.assertEqual(r.fetchall()[0][1], 'x') async def test_multiple_tasks_raw_sql(self): iq, _ = User.insert(username='x').sql() sq, _ = User.select(User.username).where(User.username == 'x').sql() async def worker(tid): username = f'u{tid}' await self.db.aconnect() await self.db.aexecute_sql(iq, (username,)) r = await self.db.aexecute_sql(sq, (username,)) row = r.fetchone() self.assertEqual(row[0], username) await self.db.aclose() return row results = await asyncio.gather(*[worker(i) for i in range(3)]) self.assertEqual(sorted(results), [('u0',), ('u1',), ('u2',)]) async def test_list(self): await self.seed(5) query = TestModel.select().order_by(TestModel.value) results = await self.db.list(query) self.assertEqual(len(results), 5) self.assertIsInstance(results[0], TestModel) self.assertEqual([r.value for r in results], [0, 10, 20, 30, 40]) async def test_list_empty(self): self.assertEqual(await self.db.list(TestModel.select()), []) async def test_get(self): rec = await self.create_record('unique', 999) q = TestModel.select().where(TestModel.name == 'unique') fetched = await self.db.get(q) self.assertEqual(fetched.id, rec.id) self.assertEqual(fetched.name, 'unique') self.assertEqual(fetched.value, 999) async def test_get_not_found(self): with self.assertRaises(TestModel.DoesNotExist): q = TestModel.select().where(TestModel.id == 0) await self.db.get(q) async def test_scalar(self): await self.seed(10) query = TestModel.select(fn.MAX(TestModel.value)) self.assertEqual(await self.db.scalar(query), 90) async def test_scalar_no_results(self): query = TestModel.select(fn.COUNT(TestModel.id)) self.assertEqual(await self.db.scalar(query), 0) await self.seed(5) self.assertEqual(await self.db.scalar(query), 5) async def test_count(self): self.assertEqual(await self.db.count(TestModel.select()), 0) await self.seed(5) self.assertEqual(await self.db.count(TestModel.select()), 5) async def test_exists(self): self.assertFalse(await self.db.exists(TestModel.select())) await self.create_record('x', 1) self.assertTrue(await self.db.exists(TestModel.select())) async def test_aexecute(self): q = TestModel.insert_many([(f'item{i}', i) for i in range(10)]) if self.support_returning: q = q.returning(TestModel.name) res = await self.db.aexecute(q) self.assertEqual([t.name for t in res], [f'item{i}' for i in range(10)]) else: await self.db.aexecute(q) await self.assertCount(10) q = (TestModel .update(value=TestModel.value * 10) .where(TestModel.value < 3)) if self.support_returning: q = q.returning(TestModel.name, TestModel.value) res = await self.db.aexecute(q) self.assertEqual(sorted([(t.name, t.value) for t in res]), [('item0', 0), ('item1', 10), ('item2', 20)]) else: res = await self.db.aexecute(q) self.assertEqual(res, 2) q = TestModel.select().where(TestModel.value >= 10) self.assertEqual(await self.db.run(q.count), 2) rows = await self.db.aexecute(q.order_by(TestModel.value)) self.assertEqual([r.name for r in rows], ['item1', 'item2']) q = TestModel.delete().where(TestModel.value >= 10) if self.support_returning: q = q.returning(TestModel.name, TestModel.value) res = await self.db.aexecute(q) self.assertEqual(sorted([(t.name, t.value) for t in res]), [('item1', 10), ('item2', 20)]) else: res = await self.db.aexecute(q) self.assertEqual(res, 2) async def test_run_contextvars(self): var = contextvars.ContextVar('v', default='x') state = [] def do_run(): state.append(var.get()) var.set('y') state.append(var.get()) await self.db.run(do_run) state.append(var.get()) self.assertEqual(state, ['y', 'y', 'y']) async def test_create(self): tm = await self.create_record('test1', 100) self.assertEqual(tm.name, 'test1') self.assertEqual(tm.value, 100) tm = await self.db.run(TestModel.create, name='test2', value=101) self.assertEqual(tm.name, 'test2') self.assertEqual(tm.value, 101) await self.assertCount(2) await self.assertNames(['test1', 'test2']) async def test_select(self): tm = await self.create_record('test1', 100) res = await self.db.list(TestModel.select()) self.assertEqual(len(res), 1) self.assertEqual(res[0].name, 'test1') self.assertEqual(res[0], tm) async def test_filter(self): await self.seed(20) query = TestModel.select().where(TestModel.value > 100) results = await self.db.list(query) self.assertEqual(len(results), 9) async def test_ordering(self): await self.seed(20) query = TestModel.select().order_by(TestModel.value.desc()).limit(5) results = await self.db.list(query) self.assertEqual(results[0].value, 190) self.assertEqual(results[4].value, 150) async def test_create_save_update(self): await self.create_record('test1', 100) def do_update(): r = TestModel.get(TestModel.name == 'test1') r.value = 999; r.save() return TestModel.get(TestModel.name == 'test1').value self.assertEqual(await self.db.run(do_update), 999) uq = TestModel.update(name='test1x').where(TestModel.name == 'test1') res = await self.db.aexecute(uq) #self.assertEqual(res, 1) q = TestModel.select().where(TestModel.name == 'test1x') tm = await self.db.get(q) self.assertEqual(tm.value, 999) async def test_update(self): await self.seed(50) await self.db.aexecute(TestModel .update(value=TestModel.value + 1000) .where(TestModel.value < 250)) query = TestModel.select().where(TestModel.value >= 1000) self.assertEqual(await self.db.count(query), 25) query = TestModel.select(fn.SUM(TestModel.value)) self.assertEqual(await self.db.scalar(query), 37250) async def test_delete(self): await self.seed(20) await self.db.aexecute(TestModel.delete().where(TestModel.value < 50)) await self.assertCount(15) tm = await self.db.get(TestModel.select()) await self.db.run(tm.delete_instance) await self.assertCount(14) async def test_bulk_create(self): recs = [TestModel(name=f'b{i}', value=i) for i in range(100)] await self.db.run(TestModel.bulk_create, recs, batch_size=25) await self.assertCount(100) async def test_bulk_update(self): if self.driver == 'postgresql': self.skipTest('bulk_update incompatible with asyncpg') return accum = [await self.db.run(TestModel.create, name=f'b{i}', value=i) for i in range(5)] for tm in accum: tm.name += '-x' tm.value += 100 await self.db.run(TestModel.bulk_update, accum, fields=[TestModel.name, TestModel.value]) q = await self.db.list(TestModel.select().order_by(TestModel.value)) self.assertEqual([(tm.name, tm.value) for tm in q], [('b0-x', 100), ('b1-x', 101), ('b2-x', 102), ('b3-x', 103), ('b4-x', 104)]) async def test_insert_many(self): def insert(): data = [{'name': f'i{i}', 'value': i} for i in range(100)] TestModel.insert_many(data).execute() await self.db.run(insert) await self.assertCount(100) data = [{'name': f'i{i}', 'value': i} for i in range(100, 200)] await self.db.aexecute(TestModel.insert_many(data)) await self.assertCount(200) data = [(f'i{i}', i) for i in range(200, 300)] iq = (TestModel .insert_many(data, fields=[TestModel.name, TestModel.value])) await self.db.aexecute(iq) await self.assertCount(300) async def test_atomic(self): async with self.db.atomic(): await self.create_record('a', 1) await self.assertCount(1) async with self.db.atomic() as txn: await self.create_record('b', 2) await self.assertCount(2) await self.assertNames(['a', 'b']) await txn.arollback() await self.assertCount(1) await self.create_record('c', 3) await self.create_record('d', 4) await self.assertCount(3) await self.assertNames(['a', 'c', 'd']) async def test_transaction_commit(self): def create_in_tx(): with self.db.atomic(): TestModel.create(name='tx1') TestModel.create(name='tx2') await self.db.run(create_in_tx) await self.assertCount(2) async with self.db.atomic(): await self.db.run(TestModel.create, name='tx1') await self.db.run(TestModel.create, name='tx2') await self.assertCount(4) async def test_transaction_rollback(self): def failing(): with self.db.atomic(): TestModel.create(name='tx1') raise ValueError('fail') with self.assertRaises(ValueError): await self.db.run(failing) await self.assertCount(0) async with self.db.atomic() as txn: await self.create_record('tx2') await self.assertCount(1) await txn.arollback() await self.assertCount(0) async def test_nested_transactions(self): def nested(): with self.db.atomic(): TestModel.create(name='o1', value=1) with self.db.atomic(): TestModel.create(name='i1', value=2) TestModel.create(name='i2', value=3) TestModel.create(name='o2', value=4) await self.db.run(nested) await self.assertCount(4) await self.assertNames(['i1', 'i2', 'o1', 'o2']) async with self.db.atomic(): await self.db.run(TestModel.create, name='o3', value=1) async with self.db.atomic(): await self.db.run(TestModel.create, name='i3', value=2) await self.db.run(TestModel.create, name='i4', value=3) await self.db.run(TestModel.create, name='o4', value=4) await self.assertCount(8) await self.assertNames(['i1', 'i2', 'i3', 'i4', 'o1', 'o2', 'o3', 'o4']) async def test_nested_implicit_rollback(self): def nested(): with self.db.atomic(): TestModel.create(name='o1', value=1) try: with self.db.atomic(): TestModel.create(name='i1', value=2) raise ValueError('fail') except ValueError: pass TestModel.create(name='o2', value=3) await self.db.run(nested) await self.assertCount(2) await self.assertNames(['o1', 'o2']) async with self.db.atomic(): await self.db.run(TestModel.create, name='o3', value=1) try: async with self.db.atomic(): await self.db.run(TestModel.create, name='i3', value=2) raise ValueError('fail') except ValueError: pass await self.assertCount(3) await self.db.run(TestModel.create, name='o4', value=3) await self.assertCount(4) await self.assertNames(['o1', 'o2', 'o3', 'o4']) async def test_nested_explicit_rollback(self): def nested(): with self.db.atomic(): TestModel.create(name='o1') with self.db.atomic() as sp: TestModel.create(name='i1') self.assertEqual(TestModel.select().count(), 2) sp.rollback() self.assertEqual(TestModel.select().count(), 1) TestModel.create(name='o2') await self.db.run(nested) await self.assertCount(2) await self.assertNames(['o1', 'o2']) async with self.db.atomic(): await self.db.run(TestModel.create, name='o3') async with self.db.atomic() as sp: await self.db.run(TestModel.create, name='i2') await self.assertCount(4) await sp.arollback() await self.assertCount(3) await self.db.run(TestModel.create, name='o4') await self.assertCount(4) await self.assertNames(['o1', 'o2', 'o3', 'o4']) async def test_nested_mix(self): async with self.db.atomic(): await self.create_record('t1') async with self.db.atomic(): await self.create_record('t2') async with self.db.atomic(): await self.create_record('t3') try: async with self.db.atomic(): await self.create_record('t4') await self.assertCount(4) raise ValueError('fail') except ValueError: pass async with self.db.atomic() as sp: await self.create_record('t4') await self.assertCount(4) await sp.arollback() await self.assertCount(3) try: async with self.db.atomic(): await self.create_record('t5') await self.assertCount(4) raise ValueError('fail') except ValueError: await self.assertCount(3) await self.assertCount(3) await self.assertNames(['t1', 't2', 't3']) try: async with self.db.atomic(): await self.create_record('t6') async with self.db.atomic(): await self.create_record('t7') async with self.db.atomic(): await self.create_record('t8') await self.assertCount(6) raise ValueError('fail') except ValueError: pass await self.assertCount(3) await self.assertNames(['t1', 't2', 't3']) async def test_acommit_arollback(self): async with self.db.atomic() as txn: await self.create_record('committed', 1) await txn.acommit() await self.create_record('not-committed', 2) await txn.arollback() await self.assertCount(1) await self.assertNames(['committed']) async def test_concurrent_reads_writes(self): await self.seed(10) async def writer(sid): def _write(): for i in range(5): TestModel.create(name=f'w{sid}-{i}', value=sid * 100 + i) async with self.db: await self.db.run(_write) async def reader(): async with self.db: query = TestModel.select() return await self.db.run(lambda: len(list(query))) await asyncio.gather(*[writer(i) for i in range(3)]) reads = await asyncio.gather(*[reader() for _ in range(3)]) self.assertTrue(all(r >= 10 for r in reads)) await self.assertCount(25) async def test_isolated_connections_per_task(self): async def worker(tid): async with self.db: c1 = self.db._state.conn await self.create_record(f't{tid}', tid) return c1 is self.db._state.conn results = await asyncio.gather(*[worker(i) for i in range(5)]) self.assertTrue(all(results)) await self.assertCount(5) async def test_many_concurrent_tasks(self): ntasks = 50 if self.driver == 'sqlite' else 10 async def task(tid): async with self.db: await self.create_record(f't{tid}', tid) await asyncio.gather(*[task(i) for i in range(ntasks)]) await self.assertCount(ntasks) async def test_syntax_error_recovery(self): with self.assertRaises(Exception): await self.db.aexecute_sql('INVALID SQL') await self.create_record('after_error', 1) await self.assertCount(1) async def test_concurrent_errors(self): errors, successes = [], [] async def worker(tid): async with self.db: try: def work(): TestModel.create(name=f't{tid}', value=tid) if tid % 2 == 0: raise ValueError(f'Task {tid} fails') await self.db.run(work) successes.append(tid) except ValueError: errors.append(tid) await asyncio.gather(*[worker(i) for i in range(10)]) self.assertEqual(sorted(errors), [0, 2, 4, 6, 8]) self.assertEqual(sorted(successes), [1, 3, 5, 7, 9]) await self.assertCount(10) async def test_iterate_yields_model_instances(self): await self.seed(20) results = [] query = TestModel.select().order_by(TestModel.value) async for obj in self.db.iterate(query): results.append(obj) self.assertEqual(len(results), 20) self.assertTrue(all(isinstance(r, TestModel) for r in results)) self.assertEqual(results[0].name, 'item00') self.assertEqual(results[0].value, 0) self.assertEqual(results[-1].name, 'item19') self.assertEqual(results[-1].value, 190) async def test_iterate_matches_list(self): await self.seed(20) query = TestModel.select().order_by(TestModel.name) eager = await self.db.list(query) lazy = [obj async for obj in self.db.iterate(query)] self.assertEqual(len(eager), len(lazy)) for e, l in zip(eager, lazy): self.assertEqual(e.name, l.name) self.assertEqual(e.value, l.value) async def test_iterate_dicts(self): await self.seed(5) query = TestModel.select().order_by(TestModel.name) results = [row async for row in self.db.iterate(query.dicts())] self.assertEqual(len(results), 5) self.assertIsInstance(results[0], dict) self.assertEqual(results[0]['name'], 'item00') self.assertEqual(results[-1]['name'], 'item04') async def test_iterate_tuples(self): await self.seed(5) query = TestModel.select(TestModel.name).order_by(TestModel.name) results = [row async for row in self.db.iterate(query.tuples())] self.assertEqual(len(results), 5) self.assertIsInstance(results[0], tuple) self.assertEqual(results[0][0], 'item00') self.assertEqual(results[-1][0], 'item04') async def test_iterate_namedtuples(self): await self.seed(5) query = TestModel.select(TestModel.name).order_by(TestModel.name) results = [row async for row in self.db.iterate(query.namedtuples())] self.assertEqual(len(results), 5) self.assertEqual(results[0].name, 'item00') self.assertEqual(results[0][0], 'item00') self.assertEqual(results[-1].name, 'item04') self.assertEqual(results[-1][0], 'item04') async def test_iterate_with_where(self): await self.seed(20) query = (TestModel.select() .where(TestModel.value >= 150) .order_by(TestModel.value)) results = [row async for row in self.db.iterate(query)] self.assertEqual(len(results), 5) self.assertEqual(results[0].value, 150) self.assertEqual(results[-1].value, 190) async def test_iterate_empty(self): query = TestModel.select().where(TestModel.id == 0) results = [row async for row in self.db.iterate(query)] self.assertEqual(results, []) async def test_iterate_buffer_size(self): await self.seed(20) query = TestModel.select().order_by(TestModel.value) results = [obj async for obj in self.db.iterate(query, buffer_size=3)] self.assertEqual(len(results), 20) self.assertEqual(results[0].value, 0) self.assertEqual(results[-1].value, 190) async def test_iterate_early_break(self): await self.seed(20) count = 0 query = TestModel.select().order_by(TestModel.value) async for obj in self.db.iterate(query): count += 1 if count == 5: break self.assertEqual(count, 5) # Database still usable (lock released). self.assertEqual(await self.db.count(TestModel.select()), 20) async def test_iterate_aggregation(self): await self.seed(20) query = (TestModel .select(fn.AVG(TestModel.value).alias('avg_val')) .dicts()) results = [row async for row in self.db.iterate(query)] self.assertEqual(len(results), 1) self.assertEqual(results[0]['avg_val'], 95.0) async def test_iterate_sequential(self): await self.seed(20) query = (TestModel.select() .where(TestModel.value < 50) .order_by(TestModel.value)) r1 = [obj.value async for obj in self.db.iterate(query)] query = (TestModel.select() .where(TestModel.value >= 150) .order_by(TestModel.value)) r2 = [obj.value async for obj in self.db.iterate(query)] self.assertEqual(r1, [0, 10, 20, 30, 40]) self.assertEqual(r2, [150, 160, 170, 180, 190]) async def test_iterate_break_then_iterate_again(self): await self.seed(20) query = TestModel.select().order_by(TestModel.value) async for obj in self.db.iterate(query): break results = [] async for obj in self.db.iterate(query): results.append(obj.value) self.assertEqual(len(results), 20) async def test_iterate_multi(self): await self.seed(10) async def iterate_multi(): async with self.db: query = TestModel.select().order_by(TestModel.value) return [obj.id async for obj in self.db.iterate(query)] results = await asyncio.gather(*[iterate_multi() for i in range(5)]) self.assertEqual(len(results), 5) self.assertTrue(all(len(r) == 10 for r in results)) async def test_basic_crud(self): rec = await self.create_record('testx', value=2) self.assertEqual(rec.name, 'testx') fetched = await self.db.run(TestModel.get, TestModel.name == 'testx') self.assertEqual(fetched.value, 2) def update(): r = TestModel.get(TestModel.id == rec.id) r.value = 100; r.save() return TestModel.get(TestModel.id == rec.id) self.assertEqual((await self.db.run(update)).value, 100) await self.db.run(rec.delete_instance) await self.assertCount(0) async def test_foreign_keys(self): users = [User(username=f'u{i}') for i in range(3)] await self.db.run(User.bulk_create, users) self.assertEqual(await self.db.run(User.select().count), 3) users = await self.db.list(User.select()) async with self.db.atomic(): for u in users: for i in range(2): await self.db.run( Tweet.create, user=u, message=f'{u.username}-{i}') self.assertEqual(await self.db.run(Tweet.select().count), 6) q = Tweet.select().where(Tweet.message == 'u0-0') tweet = await self.db.get(q) self.assertEqual(await self.db.run(lambda: tweet.user.username), 'u0') q = (Tweet.select(Tweet, User) .join(User) .where(Tweet.message == 'u0-0')) tweet = await self.db.get(q) self.assertEqual(tweet.user.username, 'u0') q = User.select().where(User.username == 'u2') user = await self.db.get(q) tweets = await self.db.list(user.tweets.order_by(Tweet.id)) self.assertEqual([t.message for t in tweets], ['u2-0', 'u2-1']) users_q = User.select().order_by(User.username) tweets_q = Tweet.select().order_by(Tweet.message) await self.db.aprefetch(users_q, tweets_q) self.assertEqual( [(u.username, [t.message for t in u.tweets]) for u in users_q], [('u0', ['u0-0', 'u0-1']), ('u1', ['u1-0', 'u1-1']), ('u2', ['u2-0', 'u2-1'])]) async def test_transactions(self): def ok_tx(): with self.db.atomic(): TestModel.create(name='t1', value=1) TestModel.create(name='t2', value=2) await self.db.run(ok_tx) await self.assertCount(2) def bad_tx(): with self.db.atomic(): TestModel.create(name='t3', value=3) raise ValueError('fail') with self.assertRaises(ValueError): await self.db.run(bad_tx) async with self.db.atomic(): await self.create_record('t4') try: async with self.db.atomic(): await self.create_record('t5') await self.assertCount(4) raise ValueError('fail') except ValueError: pass await self.assertCount(3) await self.assertCount(3) await self.assertNames(['t1', 't2', 't4']) class TestSqliteIntegration(IntegrationTests, unittest.IsolatedAsyncioTestCase): def get_database(self): with tempfile.NamedTemporaryFile(delete=False) as f: self.db_path = f.name return AsyncSqliteDatabase(self.db_path) async def test_pragmas(self): db = AsyncSqliteDatabase(':memory:', pragmas={'user_version': '99'}) conn = await db.aconnect() r = await conn.execute('PRAGMA user_version') self.assertEqual(r.fetchone(), (99,)) await db.close_pool() async def test_custom_functions(self): db = AsyncSqliteDatabase(':memory:') @db.func() def title_case(s): return s.title() async with db: r = await db.aexecute_sql('SELECT title_case(?)', ('test foo',)) self.assertEqual(r.fetchone(), ('Test Foo',)) await db.close_pool() async def test_constraint_violation_recovery(self): await self.db.aexecute_sql( 'CREATE TABLE ut (id INTEGER PRIMARY KEY, v TEXT UNIQUE)') await self.db.aexecute_sql( 'INSERT INTO ut (v) VALUES (?)', ('x',)) with self.assertRaises(IntegrityError): await self.db.aexecute_sql( 'INSERT INTO ut (v) VALUES (?)', ('x',)) await self.db.aexecute_sql( 'INSERT INTO ut (v) VALUES (?)', ('y',)) @unittest.skipIf(not IS_POSTGRESQL, 'skipping postgres test') @unittest.skipUnless(asyncpg, 'asyncpg not installed') class TestPostgresqlIntegration(IntegrationTests, unittest.IsolatedAsyncioTestCase): def get_database(self): return AsyncPostgresqlDatabase('peewee_test', **PSQL_PARAMS) async def test_placeholder_conversion(self): def insert(): return self.db.execute_sql( 'INSERT INTO testmodel (name, value) VALUES (%s, %s)', ('placeholder_test', 999)) await self.db.run(insert) def query(): r = self.db.execute_sql( 'SELECT * FROM testmodel WHERE name = %s', ('placeholder_test',)) return r.fetchone() row = await self.db.run(query) self.assertIsNotNone(row) self.assertEqual(row['name'], 'placeholder_test') self.assertEqual(row['value'], 999) curs = await self.db.aexecute_sql('select %s', ('test',)) self.assertEqual(curs.fetchone()[0], 'test') async def test_iterator_with_transaction(self): async with self.db.atomic() as tx: await self.seed(2) q = TestModel.select().order_by(TestModel.value) results = [obj.value async for obj in self.db.iterate(q)] self.assertEqual(results, [0, 10]) await self.assertCount(2) @unittest.skipIf(not IS_MYSQL, 'skipping mysql test') @unittest.skipUnless(aiomysql, 'aiomysql not installed') class TestMySQLIntegration(IntegrationTests, unittest.IsolatedAsyncioTestCase): def get_database(self): return AsyncMySQLDatabase('peewee_test', **MYSQL_PARAMS) if __name__ == '__main__': unittest.main() ================================================ FILE: tests/pwasyncio_stress.py ================================================ import asyncio import os import random import sys import time import tracemalloc from playhouse.pwasyncio import * def make_db_params(key): params = {} env_vars = [(part, 'PEEWEE_%s_%s' % (key, part.upper())) for part in ('host', 'port', 'user', 'password')] for param, env_var in env_vars: value = os.environ.get(env_var) if value: params[param] = int(value) if param == 'port' else value return params PSQL_PARAMS = make_db_params('PSQL') class User(Model): name = TextField() email = TextField() class Meta: table_name = 'stress_test_users' async def worker_task(db, task_id, num_operations=10): User._meta.set_database(db) try: await db.aconnect() for i in range(num_operations): op = random.choice(['create', 'read', 'update', 'delete', 'transaction']) name = 'User-%s-%s' % (task_id, i) email = 'user%s_%s@test.com' % (task_id, i) if op == 'create': await db.run(User.create, name=name, email=email) elif op == 'read': users = await db.list(User.select().limit(5)) elif op == 'update': users = await db.list(User.select().limit(1)) if users: user = users[0] user.name = 'Updated-%s-%s' % (task_id, i) await db.run(user.save) elif op == 'delete': users = await db.list(User.select().limit(1)) if users: await db.run(users[0].delete_instance) elif op == 'transaction': async with db.atomic(): await db.run(User.create, name='TX-%s-%s' % (task_id, i), email='tx%s_%s@test.com' % (task_id, i)) try: async with db.atomic(): await db.run(User.create, name='Nested-%s-%s' % (task_id, i), email='nested%s_%s@test.com' % (task_id, i)) if random.random() < 0.3: # 30% chance of rollback raise ValueError('Intentional rollback') except ValueError: pass # Small random delay to simulate real work if random.random() < 0.1: await asyncio.sleep(0.001) # Close connection await db.aclose() return "Task %s completed successfully" % task_id except Exception as exc: print(exc) return "Task %s failed: %s" % (task_id, exc) async def stress_test(db, num_tasks=100, ops_per_task=10): print('STRESS TEST: %s tasks x %s ops/task' % (num_tasks, ops_per_task)) print('Database: %s' % db) User._meta.database = db # Setup print('Setting up database...') async with db: await db.acreate_tables([User]) # Track memory tracemalloc.start() initial_memory = tracemalloc.get_traced_memory()[0] # Run stress test print('Spawning %s concurrent tasks...' % num_tasks) start_time = time.time() tasks = [worker_task(db, i, ops_per_task) for i in range(num_tasks)] results = await asyncio.gather(*tasks, return_exceptions=True) elapsed = time.time() - start_time # Check memory final_memory = tracemalloc.get_traced_memory()[0] memory_delta = (final_memory - initial_memory) / 1024 / 1024 # MB tracemalloc.stop() successful = sum(1 for r in results if isinstance(r, str) and 'completed' in r) failed = len(results) - successful # Check final state async with db: total_users = await db.run(User.select().count) # Cleanup dead tasks cleaned = db._state.cleanup_dead_tasks() # Report throughput = (num_tasks * ops_per_task) / elapsed print('RESULTS') print('Duration: %0.2fs' % elapsed) print('Throughput: %0.1f ops/sec' % throughput) print('Successful tasks: %s/%s' % (successful, num_tasks)) print('Failed tasks: %s/%s' % (failed, num_tasks)) print('Total users in DB: %s' % total_users) print('Memory delta: %0.2f MB' % memory_delta) print('Dead tasks cleaned: %s' % cleaned) print('Remaining task states: %s' % len(db._state._state_storage)) print('-' * 60) # Cleanup async with db: await db.adrop_tables([User]) await db.close_pool() return successful == num_tasks async def test_connection_isolation(): print('CONNECTION ISOLATION TEST') db = AsyncPostgresqlDatabase('peewee_test', pool_size=5, **PSQL_PARAMS) User._meta.database = db async with db: await db.acreate_tables([User]) async def task_with_transaction(task_id, delay): """Each task holds a transaction for a specific duration.""" await db.aconnect() async with db.atomic(): # Create a user await db.run(User.create, name='Task-%s' % task_id, email='t%s@test.com' % task_id) # Hold the transaction await asyncio.sleep(delay) # Verify we can still see our own changes users = await db.run(list, User.select().where(User.name == 'Task-%s' % task_id)) assert len(users) == 1, 'Task %s lost its data!' % task_id await db.aclose() return 'Task %s isolated correctly' % task_id # Spawn multiple tasks that will overlap in time tasks = [ task_with_transaction(0, 0.1), task_with_transaction(1, 0.2), task_with_transaction(2, 0.15), task_with_transaction(3, 0.05), ] results = await asyncio.gather(*tasks) print('All tasks completed:') for r in results: print(' - %s' % r) # Cleanup async with db: final_count = await db.run(User.select().count) print('Final user count: %s (expected 4)' % final_count) await db.adrop_tables([User]) await db.close_pool() print('-' * 60) return True async def test_pool_exhaustion(): print('POOL EXHAUSTION TEST') # Create a small pool db = AsyncPostgresqlDatabase('peewee_test', pool_size=3, pool_min_size=1, **PSQL_PARAMS) User._meta.database = db async with db: await db.acreate_tables([User]) async def slow_task(task_id): await db.run(db.connect) await db.run(User.create, name='Slow-%s' % task_id, email='s%s@test.com' % task_id) await asyncio.sleep(0.5) # Hold connection await db.run(db.close) return task_id print('Spawning 10 tasks with pool_size=3...') print('(Tasks should queue and complete successfully)') start = time.time() tasks = [slow_task(i) for i in range(10)] results = await asyncio.gather(*tasks) elapsed = time.time() - start print('All %s tasks completed in %.2fs' % (len(results), elapsed)) print('Expected ~1.5s (3 batches of parallel execution)') # Cleanup async with db: await db.adrop_tables([User]) await db.close_pool() print('-' * 60) return True async def main(): print('ASYNC PEEWEE STRESS TEST SUITE') print('-' * 60) # Test 1: Basic stress test with many tasks db = AsyncPostgresqlDatabase('peewee_test', pool_size=20, **PSQL_PARAMS) success1 = await stress_test(db, num_tasks=100, ops_per_task=20) # Test 2: Even more tasks with smaller pool db2 = AsyncPostgresqlDatabase('peewee_test', pool_size=5, **PSQL_PARAMS) success2 = await stress_test(db2, num_tasks=200, ops_per_task=10) # Test 3: Even smaller pool. db3 = AsyncPostgresqlDatabase('peewee_test', pool_size=3, **PSQL_PARAMS) success3 = await stress_test(db2, num_tasks=100, ops_per_task=20) ## Test 3: Connection isolation success4 = await test_connection_isolation() ## Test 4: Pool exhaustion success5 = await test_pool_exhaustion() # Final report print('=' * 60) print('Stress Test 1 (100 tasks): %s' % 'OK' if success1 else 'FAIL') print('Stress Test 2 (200 tasks): %s' % 'OK' if success2 else 'FAIL') print('Stress Test 3 (100 tasks): %s' % 'OK' if success3 else 'FAIL') print('Isolation Test: %s' % 'OK' if success4 else 'FAIL') print('Pool Exhaustion Test: %s' % 'OK' if success5 else 'FAIL') print('=' * 60) if all([success1, success2, success3, success4, success5]): print('Success') return 0 else: print('Failed') return 1 if __name__ == '__main__': rc = asyncio.run(main()) sys.exit(rc) ================================================ FILE: tests/pwiz_integration.py ================================================ import datetime import os import textwrap import sys from io import StringIO from unittest import mock from peewee import * from pwiz import * from .base import ModelTestCase from .base import TestModel from .base import db_loader from .base import skip_if db = db_loader('sqlite') class User(TestModel): username = CharField(primary_key=True) id = IntegerField(default=0) class Note(TestModel): user = ForeignKeyField(User) text = TextField(index=True) data = IntegerField(default=0) misc = IntegerField(default=0) class Meta: indexes = ( (('user', 'text'), True), (('user', 'data', 'misc'), False), ) class Category(TestModel): name = CharField(unique=True) parent = ForeignKeyField('self', null=True) class OddColumnNames(TestModel): spaces = CharField(column_name='s p aces') symbols = CharField(column_name='w/-nug!') camelCaseName = CharField(column_name='camelCaseName') class Meta: table_name = 'oddColumnNames' class Event(TestModel): data = TextField() status = IntegerField() class capture_output(object): def __enter__(self): self._stdout = sys.stdout sys.stdout = self._buffer = StringIO() return self def __exit__(self, *args): self.data = self._buffer.getvalue() sys.stdout = self._stdout EXPECTED = """ from peewee import * database = SqliteDatabase('peewee_test.db') class UnknownField(object): def __init__(self, *_, **__): pass class BaseModel(Model): class Meta: database = database class Category(BaseModel): name = CharField(unique=True) parent = ForeignKeyField(column_name='parent_id', field='id', model='self', null=True) class Meta: table_name = 'category' class User(BaseModel): id = IntegerField() username = CharField(primary_key=True) class Meta: table_name = 'user' class Note(BaseModel): data = IntegerField() misc = IntegerField() text = TextField(index=True) user = ForeignKeyField(column_name='user_id', field='username', model=User) class Meta: table_name = 'note' indexes = ( (('user', 'data', 'misc'), False), (('user', 'text'), True), ) """.strip() EXPECTED_ORDERED = """ from peewee import * database = SqliteDatabase('peewee_test.db') class UnknownField(object): def __init__(self, *_, **__): pass class BaseModel(Model): class Meta: database = database class User(BaseModel): username = CharField(primary_key=True) id = IntegerField() class Meta: table_name = 'user' class Note(BaseModel): user = ForeignKeyField(column_name='user_id', field='username', model=User) text = TextField(index=True) data = IntegerField() misc = IntegerField() class Meta: table_name = 'note' indexes = ( (('user', 'data', 'misc'), False), (('user', 'text'), True), ) """.strip() class BasePwizTestCase(ModelTestCase): database = db requires = [] def setUp(self): if not self.database.is_closed(): self.database.close() if os.path.exists(self.database.database): os.unlink(self.database.database) super(BasePwizTestCase, self).setUp() self.introspector = Introspector.from_database(self.database) class TestPwiz(BasePwizTestCase): requires = [User, Note, Category] def test_print_models(self): with capture_output() as output: print_models(self.introspector) self.assertEqual(output.data.strip(), EXPECTED) def test_print_header(self): cmdline = '-i -e sqlite %s' % db.database with capture_output() as output: with mock.patch('pwiz.datetime.datetime') as mock_datetime: now = mock_datetime.now.return_value now.strftime.return_value = 'February 03, 2015 15:30PM' print_header(cmdline, self.introspector) self.assertEqual(output.data.strip(), ( '# Code generated by:\n' '# python -m pwiz %s\n' '# Date: February 03, 2015 15:30PM\n' '# Database: %s\n' '# Peewee version: %s') % (cmdline, db.database, peewee_version)) class TestPwizOrdered(BasePwizTestCase): requires = [User, Note] def test_ordered_columns(self): with capture_output() as output: print_models(self.introspector, preserve_order=True) self.assertEqual(output.data.strip(), EXPECTED_ORDERED) class TestPwizUnknownField(BasePwizTestCase): header = ('from peewee import *\n\n' 'database = SqliteDatabase(\'peewee_test.db\')\n\n') unknown = ('class UnknownField(object):\n' ' def __init__(self, *_, **__): pass\n\n') basemodel = ('class BaseModel(Model):\n class Meta:\n' ' database = database\n\n') def setUp(self): super(TestPwizUnknownField, self).setUp() self.database.execute_sql( 'CREATE TABLE "foo" ("id" INTEGER NOT NULL PRIMARY KEY, ' '"unk1", "unk2" BIZBAZ NOT NULL)') def test_unknown_field(self): with capture_output() as output: print_models(self.introspector) self.assertEqual(output.data.strip(), ( self.header + self.unknown + self.basemodel + 'class Foo(BaseModel):\n' ' unk1 = BareField(null=True)\n' ' unk2 = UnknownField() # BIZBAZ\n\n' ' class Meta:\n table_name = \'foo\'')) def test_ignore_unknown(self): with capture_output() as output: print_models(self.introspector, ignore_unknown=True) self.assertEqual(output.data.strip(), ( self.header + self.basemodel + 'class Foo(BaseModel):\n' ' unk1 = BareField(null=True)\n' ' # unk2 - BIZBAZ\n\n' ' class Meta:\n table_name = \'foo\'')) class TestPwizInvalidColumns(BasePwizTestCase): requires = [OddColumnNames] def test_invalid_columns(self): with capture_output() as output: print_models(self.introspector) result = output.data.strip() expected = textwrap.dedent(""" class OddColumnNames(BaseModel): camel_case_name = CharField(column_name='camelCaseName') s_p_aces = CharField(column_name='s p aces') w_nug_ = CharField(column_name='w/-nug!') class Meta: table_name = 'oddColumnNames'""").strip() actual = result[-len(expected):] self.assertEqual(actual, expected) def test_odd_columns_legacy(self): with capture_output() as output: print_models(self.introspector, snake_case=False) result = output.data.strip() expected = textwrap.dedent(""" class Oddcolumnnames(BaseModel): camelcasename = CharField(column_name='camelCaseName') s_p_aces = CharField(column_name='s p aces') w_nug_ = CharField(column_name='w/-nug!') class Meta: table_name = 'oddColumnNames'""").strip() actual = result[-len(expected):] self.assertEqual(actual, expected) class TestPwizIntrospectViews(BasePwizTestCase): requires = [Event] def setUp(self): super(TestPwizIntrospectViews, self).setUp() self.database.execute_sql('CREATE VIEW "events_public" AS ' 'SELECT data FROM event WHERE status = 1') def tearDown(self): self.database.execute_sql('DROP VIEW "events_public"') super(TestPwizIntrospectViews, self).tearDown() def test_introspect_ignore_views(self): # By default views are not included in the output. with capture_output() as output: print_models(self.introspector) self.assertFalse('events_public' in output.data.strip()) def test_introspect_views(self): # Views can be introspected, however. with capture_output() as output: print_models(self.introspector, include_views=True) result = output.data.strip() event_tbl = textwrap.dedent(""" class Event(BaseModel): data = TextField() status = IntegerField() class Meta: table_name = 'event'""").strip() self.assertTrue(event_tbl in result) event_view = textwrap.dedent(""" class EventsPublic(BaseModel): data = TextField(null=True) class Meta: table_name = 'events_public' primary_key = False""").strip() self.assertTrue(event_view in result) ================================================ FILE: tests/pydantic_utils.py ================================================ from __future__ import annotations import datetime import decimal import uuid from typing import List from peewee import * from playhouse.pydantic_utils import to_pydantic from pydantic import BaseModel from .base import ModelDatabaseTestCase from .base import get_in_memory_db from .base import requires_models from .base import TestModel class User(TestModel): name = CharField(verbose_name='Full Name', help_text='Display name') age = IntegerField() active = BooleanField(default=True) bio = TextField(null=True) score = FloatField(null=True, default=0.0) status = CharField( verbose_name='Status', help_text='Record status', choices=[ ('active', 'Active'), ('archived', 'Archived'), ('deleted', 'Deleted')]) created = DateTimeField(default=datetime.datetime.now) class Tweet(TestModel): user = ForeignKeyField(User, backref='tweets') content = TextField() created = DateTimeField(default=datetime.datetime.now) class NullableFK(TestModel): user = ForeignKeyField(User, null=True, backref='nullable_things') label = CharField() class AllTypes(TestModel): f_text = TextField() f_blob = BlobField() f_bool = BooleanField() f_date = DateField() f_datetime = DateTimeField() f_decimal = DecimalField() f_double = DoubleField() f_float = FloatField() f_int = IntegerField() f_smallint = SmallIntegerField() f_time = TimeField() f_uuid = UUIDField() f_char = CharField() class BasePydanticTestCase(ModelDatabaseTestCase): database = get_in_memory_db() class TestPydanticConversion(BasePydanticTestCase): def test_conversion(self): Schema = to_pydantic(User) self.assertTrue(issubclass(Schema, BaseModel)) self.assertEqual(Schema.__name__, 'UserSchema') self.assertEqual(set(Schema.model_fields), { 'name', 'age', 'active', 'bio', 'score', 'status', 'created'}) def test_application(self): Schema = to_pydantic(User) obj = Schema(name='Huey', age=14, status='active') ts = obj.created self.assertEqual(obj.dict(), { 'name': 'Huey', 'age': 14, 'active': True, 'bio': None, 'score': 0.0, 'status': 'active', 'created': ts}) with self.assertRaises(ValueError) as ctx: obj = Schema() self.assertTrue('3 validation errors' in str(ctx.exception)) with self.assertRaises(ValueError) as ctx: obj = Schema(name='Huey', age=14) self.assertTrue('1 validation error' in str(ctx.exception)) with self.assertRaises(ValueError) as ctx: obj = Schema(name='Huey', age=14, status='invalid') self.assertTrue('Input should be' in str(ctx.exception)) def test_autofield(self): Schema = to_pydantic(User) self.assertNotIn('id', Schema.model_fields) Schema = to_pydantic(User, exclude_autofield=False) self.assertIn('id', Schema.model_fields) def test_nullable(self): Schema = to_pydantic(User) self.assertTrue(Schema.model_fields['name'].is_required()) self.assertTrue(Schema.model_fields['age'].is_required()) self.assertFalse(Schema.model_fields['bio'].is_required()) self.assertFalse(Schema.model_fields['score'].is_required()) self.assertIsNone(Schema.model_fields['bio'].default) self.assertEqual(Schema.model_fields['score'].default, 0.0) def test_defaults(self): Schema = to_pydantic(User) obj = Schema(name='Huey', age=14, status='active') self.assertTrue(obj.active) self.assertEqual(obj.score, 0.0) self.assertIsNone(obj.bio) self.assertTrue(isinstance(obj.created, datetime.datetime)) def test_choices(self): Schema = to_pydantic(User, include='status') for choice in ('active', 'archived', 'deleted'): instance = Schema(status=choice) self.assertEqual(instance.status, choice) with self.assertRaises(ValueError): instance = Schema(status='invalid') def test_metadata(self): Schema = to_pydantic(User) self.assertEqual(Schema.model_fields['name'].title, 'Full Name') self.assertEqual(Schema.model_fields['status'].title, 'Status') self.assertIsNone(Schema.model_fields['age'].title) self.assertIn('Display name', Schema.model_fields['name'].description) self.assertIsNone(Schema.model_fields['age'].description) desc = Schema.model_fields['status'].description self.assertIn('Record status', desc) self.assertIn("'active' = Active", desc) self.assertIn("'deleted' = Deleted", desc) jschema = Schema.model_json_schema() self.assertEqual(jschema['properties']['name']['title'], 'Full Name') def test_foreign_key(self): Schema = to_pydantic(Tweet) self.assertEqual(set(Schema.model_fields), {'user_id', 'content', 'created'}) obj = Schema(user_id=1337, content='Test') self.assertEqual(obj.user_id, 1337) self.assertEqual(obj.content, 'Test') with self.assertRaises(ValueError): Schema(user_id='not_an_int', content='test') def test_type_mapping(self): Schema = to_pydantic(AllTypes) valid_data = { 'f_blob': b'\x00\x01', 'f_bool': True, 'f_char': 'world', 'f_date': datetime.date.today(), 'f_datetime': datetime.datetime.now(), 'f_decimal': decimal.Decimal('3.14'), 'f_double': 2.718, 'f_float': 1.5, 'f_int': 42, 'f_smallint': 7, 'f_text': 'hello', 'f_time': datetime.time(12, 14), 'f_uuid': uuid.uuid4(), } instance = Schema(**valid_data) for key, val in valid_data.items(): self.assertEqual(getattr(instance, key), val) def test_include_exclude(self): Schema = to_pydantic(User, exclude={'age', 'bio'}) self.assertNotIn('age', Schema.model_fields) self.assertNotIn('bio', Schema.model_fields) self.assertIn('name', Schema.model_fields) Schema = to_pydantic(User, include={'name', 'status'}) self.assertEqual(set(Schema.model_fields), {'name', 'status'}) Schema = to_pydantic(User, include={'name', 'age'}, exclude={'age'}) self.assertEqual(set(Schema.model_fields), {'name'}) def test_nullable_fields(self): Schema = to_pydantic(User) self.assertTrue(Schema.model_fields['name'].is_required()) self.assertTrue(Schema.model_fields['age'].is_required()) self.assertEqual(Schema.model_fields['bio'].default, None) self.assertFalse(Schema.model_fields['bio'].is_required()) instance = Schema(name='a', age=1, status='active') self.assertEqual(instance.score, 0.0) self.assertIsNone(instance.bio) self.assertEqual(instance.active, True) self.assertIsInstance(instance.created, datetime.datetime) def test_schema_generation(self): for model in (User, Tweet, AllTypes): with self.subTest(model=model.__name__): Schema = to_pydantic(model) schema = Schema.model_json_schema() self.assertIn('properties', schema) Schema = to_pydantic(User) schema = Schema.model_json_schema() bio_schema = schema['properties']['bio'] any_of_types = [s.get('type') for s in bio_schema.get('anyOf', [])] self.assertIn('null', any_of_types) @requires_models(User) def test_validate_model(self): Schema = to_pydantic(User) u = User.create(name='Huey', age=14, status='active') validated = Schema.model_validate(u) self.assertEqual(validated.dict(), { 'name': 'Huey', 'age': 14, 'active': True, 'bio': None, 'score': 0.0, 'status': 'active', 'created': u.created}) us = User(**validated.dict()) self.assertEqual(us.name, 'Huey') self.assertEqual(us.age, 14) self.assertTrue(us.active) self.assertIsNone(us.bio) self.assertEqual(us.score, 0.0) self.assertEqual(us.status, 'active') self.assertEqual(us.created, u.created) self.assertIsNone(us.id) v2 = Schema.model_validate(validated.dict()) self.assertEqual(validated, v2) @requires_models(User, Tweet) def test_validate_model_foreign_key(self): Schema = to_pydantic(Tweet) user = User.create(name='Huey', age=14, status='active') tweet = Tweet.create(user=user, content='hello') validated = Schema.model_validate(tweet) self.assertEqual(validated.dict(), { 'content': 'hello', 'created': tweet.created, 'user_id': user.id}) ts = Tweet(**validated.dict()) self.assertEqual(ts.content, 'hello') self.assertEqual(ts.user_id, user.id) self.assertEqual(ts.user.name, 'Huey') # Triggers query. self.assertIsNone(ts.id) v2 = Schema.model_validate(validated.dict()) self.assertEqual(validated, v2) class TestRelationships(BasePydanticTestCase): def test_nested_schema(self): UserSchema = to_pydantic(User, exclude_autofield=False) TweetResponse = to_pydantic( Tweet, exclude_autofield=False, relationships={Tweet.user: UserSchema}) self.assertEqual(set(TweetResponse.model_fields), {'id', 'user', 'content', 'created'}) instance = TweetResponse( id=1, user={'id': 1, 'name': 'Huey', 'age': 14, 'status': 'active'}, content='hello') self.assertEqual(instance.user.name, 'Huey') self.assertEqual(instance.content, 'hello') with self.assertRaises(ValueError): TweetResponse(id=1, user=42, content='hello') OtherSchema = to_pydantic(Tweet, relationships={}) self.assertIn('user_id', OtherSchema.model_fields) def test_nested_relationship_in_json_schema(self): UserSchema = to_pydantic(User, exclude_autofield=False) TweetResponse = to_pydantic( Tweet, exclude_autofield=False, relationships={Tweet.user: UserSchema}) schema = TweetResponse.model_json_schema() self.assertIn('user', schema['properties']) def test_nullable_fk_relationship(self): UserSchema = to_pydantic(User, exclude_autofield=False) Schema = to_pydantic( NullableFK, exclude_autofield=False, relationships={NullableFK.user: UserSchema}) instance = Schema(id=1, user=None, label='test') self.assertIsNone(instance.user) instance = Schema(id=1, label='test', user={ 'id': 1, 'name': 'Huey', 'age': 14, 'status': 'active'}) self.assertEqual(instance.user.name, 'Huey') def test_metadata_preserved_on_nested_field(self): UserSchema = to_pydantic(User, exclude_autofield=False) Schema = to_pydantic( NullableFK, exclude_autofield=False, relationships={NullableFK.user: UserSchema}) field_info = Schema.model_fields['user'] self.assertFalse(field_info.is_required()) def test_backref(self): TweetFlat = to_pydantic(Tweet, exclude_autofield=False) UserDetail = to_pydantic( User, exclude_autofield=False, relationships={User.tweets: List[TweetFlat]}) self.assertEqual(set(UserDetail.model_fields), { 'id', 'name', 'age', 'active', 'bio', 'score', 'status', 'created', 'tweets'}) instance = UserDetail(id=1, name='Huey', age=14, status='active') self.assertEqual(instance.tweets, []) instance = UserDetail( id=1, name='Huey', age=14, status='active', tweets=[ {'id': 1, 'user_id': 1, 'content': 'hello'}, {'id': 2, 'user_id': 1, 'content': 'world'}, ]) self.assertEqual(len(instance.tweets), 2) self.assertEqual(instance.tweets[0].content, 'hello') with self.assertRaises(ValueError): UserDetail(id=1, name='Huey', age=14, status='active', tweets=[{'bad': 'data'}]) @requires_models(User, Tweet) def test_validate_fk(self): UserSchema = to_pydantic(User, exclude_autofield=False) TweetResponse = to_pydantic( Tweet, exclude_autofield=False, relationships={Tweet.user: UserSchema}) user = User.create(name='Huey', age=14, status='active') Tweet.create(user=user, content='hello') # Re-fetch so rel is not populated. tweet = Tweet.select().get() with self.assertQueryCount(1): result = TweetResponse.model_validate(tweet) self.assertEqual(result.content, 'hello') self.assertEqual(result.user.name, 'Huey') self.assertEqual(result.user.age, 14) tweet = (Tweet .select(Tweet, User) .join(User) .get()) with self.assertQueryCount(0): result = TweetResponse.model_validate(tweet) self.assertEqual(result.content, 'hello') self.assertEqual(result.user.name, 'Huey') self.assertEqual(result.user.age, 14) @requires_models(User, Tweet) def test_validate_backref(self): TweetFlat = to_pydantic(Tweet, exclude_autofield=False, exclude={'user'}) UserDetail = to_pydantic( User, exclude_autofield=False, relationships={User.tweets: List[TweetFlat]}) user = User.create(name='Huey', age=14, status='active') Tweet.create(user=user, content=f't0') Tweet.create(user=user, content=f't1') # Will evaluate user.tweets on demand. with self.assertQueryCount(1): result = UserDetail.model_validate(user) self.assertEqual(result.name, 'Huey') self.assertEqual(sorted([t.content for t in result.tweets]), ['t0', 't1']) # Will use prefetched tweets. user = User.select().prefetch(Tweet.select().order_by(Tweet.id))[0] with self.assertQueryCount(0): result = UserDetail.model_validate(user) self.assertEqual(result.name, 'Huey') self.assertEqual([t.content for t in result.tweets], ['t0', 't1']) ================================================ FILE: tests/queries.py ================================================ from peewee import * from .base import BaseTestCase from .base import DatabaseTestCase from .base import TestModel from .base import get_in_memory_db User = Table('users', ['id', 'username']) Tweet = Table('tweet', ['id', 'user_id', 'content']) Register = Table('register', ['id', 'value']) class TestQueryExecution(DatabaseTestCase): database = get_in_memory_db() def setUp(self): super(TestQueryExecution, self).setUp() User.bind(self.database) Tweet.bind(self.database) Register.bind(self.database) self.execute('CREATE TABLE "users" (id INTEGER NOT NULL PRIMARY KEY, ' 'username TEXT)') self.execute('CREATE TABLE "tweet" (id INTEGER NOT NULL PRIMARY KEY, ' 'user_id INTEGER NOT NULL, content TEXT, FOREIGN KEY ' '(user_id) REFERENCES users (id))') self.execute('CREATE TABLE "register" (' 'id INTEGER NOT NULL PRIMARY KEY, ' 'value REAL)') def tearDown(self): self.execute('DROP TABLE "tweet";') self.execute('DROP TABLE "users";') self.execute('DROP TABLE "register";') super(TestQueryExecution, self).tearDown() def create_user_tweets(self, username, *tweets): user_id = User.insert({User.username: username}).execute() for tweet in tweets: Tweet.insert({ Tweet.user_id: user_id, Tweet.content: tweet}).execute() return user_id def test_selection(self): huey_id = self.create_user_tweets('huey', 'meow', 'purr') query = User.select() self.assertEqual(query[:], [{'id': huey_id, 'username': 'huey'}]) query = (Tweet .select(Tweet.content, User.username) .join(User, on=(Tweet.user_id == User.id)) .order_by(Tweet.id)) self.assertEqual(query[:], [ {'content': 'meow', 'username': 'huey'}, {'content': 'purr', 'username': 'huey'}]) def test_select_peek_first(self): huey_id = self.create_user_tweets('huey', 'meow', 'purr', 'hiss') query = Tweet.select(Tweet.content).order_by(Tweet.id) self.assertEqual(query.peek(n=2), [ {'content': 'meow'}, {'content': 'purr'}]) self.assertEqual(query.first(), {'content': 'meow'}) query = Tweet.select().where(Tweet.id == 0) self.assertIsNone(query.peek(n=2)) self.assertIsNone(query.first()) def test_select_get(self): huey_id = self.create_user_tweets('huey') self.assertEqual(User.select().where(User.username == 'huey').get(), { 'id': huey_id, 'username': 'huey'}) self.assertIsNone(User.select().where(User.username == 'x').get()) def test_select_count(self): huey_id = self.create_user_tweets('huey', 'meow', 'purr') mickey_id = self.create_user_tweets('mickey', 'woof', 'pant', 'whine') self.assertEqual(User.select().count(), 2) self.assertEqual(Tweet.select().count(), 5) query = Tweet.select().where(Tweet.user_id == mickey_id) self.assertEqual(query.count(), 3) query = (Tweet .select() .join(User, on=(Tweet.user_id == User.id)) .where(User.username == 'foo')) self.assertEqual(query.count(), 0) def test_select_exists(self): self.create_user_tweets('huey') self.assertTrue(User.select().where(User.username == 'huey').exists()) self.assertFalse(User.select().where(User.username == 'foo').exists()) def test_scalar(self): values = [1.0, 1.5, 2.0, 5.0, 8.0] (Register .insert([{Register.value: value} for value in values]) .execute()) query = Register.select(fn.AVG(Register.value)) self.assertEqual(query.scalar(), 3.5) query = query.where(Register.value < 5) self.assertEqual(query.scalar(), 1.5) query = (Register .select( fn.SUM(Register.value), fn.COUNT(Register.value), fn.SUM(Register.value) / fn.COUNT(Register.value))) self.assertEqual(query.scalar(as_tuple=True), (17.5, 5, 3.5)) query = query.where(Register.value >= 2) self.assertEqual(query.scalar(as_tuple=True), (15, 3, 5)) def test_scalars(self): values = [1.0, 1.5, 2.0, 5.0, 8.0] (Register .insert([{Register.value: value} for value in values]) .execute()) query = Register.select(Register.value).order_by(Register.value) self.assertEqual(list(query.scalars()), values) query = query.where(Register.value < 5) self.assertEqual(list(query.scalars()), [1.0, 1.5, 2.0]) def test_slicing_select(self): values = [1., 1., 2., 3., 5., 8.] (Register .insert([(v,) for v in values], columns=(Register.value,)) .execute()) query = (Register .select(Register.value) .order_by(Register.value) .tuples()) with self.assertQueryCount(1): self.assertEqual(query[0], (1.,)) self.assertEqual(query[:2], [(1.,), (1.,)]) self.assertEqual(query[1:4], [(1.,), (2.,), (3.,)]) self.assertEqual(query[-1], (8.,)) self.assertEqual(query[-2], (5.,)) self.assertEqual(query[-2:], [(5.,), (8.,)]) self.assertEqual(query[2:-2], [(2.,), (3.,)]) class TestQueryCloning(BaseTestCase): def test_clone_tables(self): self._do_test_clone(User, Tweet) def test_clone_models(self): class User(TestModel): username = TextField() class Meta: table_name = 'users' class Tweet(TestModel): user = ForeignKeyField(User, backref='tweets') content = TextField() self._do_test_clone(User, Tweet) def _do_test_clone(self, User, Tweet): query = Tweet.select(Tweet.id) base_sql = 'SELECT "t1"."id" FROM "tweet" AS "t1"' self.assertSQL(query, base_sql, []) qj = query.join(User, on=(Tweet.user_id == User.id)) self.assertSQL(query, base_sql, []) self.assertSQL(qj, ( 'SELECT "t1"."id" FROM "tweet" AS "t1" ' 'INNER JOIN "users" AS "t2" ON ("t1"."user_id" = "t2"."id")'), []) qw = query.where(Tweet.id > 3) self.assertSQL(query, base_sql, []) self.assertSQL(qw, base_sql + ' WHERE ("t1"."id" > ?)', [3]) qw2 = qw.where(Tweet.id < 6) self.assertSQL(query, base_sql, []) self.assertSQL(qw, base_sql + ' WHERE ("t1"."id" > ?)', [3]) self.assertSQL(qw2, base_sql + (' WHERE (("t1"."id" > ?) ' 'AND ("t1"."id" < ?))'), [3, 6]) qo = query.order_by(Tweet.id) self.assertSQL(query, base_sql, []) self.assertSQL(qo, base_sql + ' ORDER BY "t1"."id"', []) qo2 = qo.order_by(Tweet.content, Tweet.id) self.assertSQL(query, base_sql, []) self.assertSQL(qo, base_sql + ' ORDER BY "t1"."id"', []) self.assertSQL(qo2, base_sql + ' ORDER BY "t1"."content", "t1"."id"', []) qg = query.group_by(Tweet.id) self.assertSQL(query, base_sql, []) self.assertSQL(qg, base_sql + ' GROUP BY "t1"."id"', []) ================================================ FILE: tests/reflection.py ================================================ import datetime import os import re import warnings from peewee import * from playhouse.reflection import * from .base import IS_CRDB from .base import IS_CYSQLITE from .base import IS_SQLITE_OLD from .base import ModelTestCase from .base import TestModel from .base import db from .base import requires_models from .base import requires_sqlite from .base import skip_if from .base_models import Tweet from .base_models import User class ColTypes(TestModel): f1 = BigIntegerField(index=True) f2 = BlobField() f3 = BooleanField() f4 = CharField(max_length=50) f5 = DateField() f6 = DateTimeField() f7 = DecimalField() f8 = DoubleField() f9 = FloatField() f10 = IntegerField(unique=True) f11 = AutoField() f12 = TextField() f13 = TimeField() class Meta: indexes = ( (('f10', 'f11'), True), (('f11', 'f8', 'f13'), False), ) class Nullable(TestModel): nullable_cf = CharField(null=True) nullable_if = IntegerField(null=True) class RelModel(TestModel): col_types = ForeignKeyField(ColTypes, backref='foo') col_types_nullable = ForeignKeyField(ColTypes, null=True) class FKPK(TestModel): col_types = ForeignKeyField(ColTypes, primary_key=True) class Underscores(TestModel): _id = AutoField() _name = CharField() class Category(TestModel): name = CharField(max_length=10) parent = ForeignKeyField('self', null=True) class Nugget(TestModel): category_id = ForeignKeyField(Category, column_name='category_id') category = CharField() class NoPK(TestModel): data = CharField() class Meta: primary_key = False class BaseReflectionTestCase(ModelTestCase): def setUp(self): super(BaseReflectionTestCase, self).setUp() self.introspector = Introspector.from_database(self.database) class TestReflection(BaseReflectionTestCase): requires = [ColTypes, Nullable, RelModel, FKPK, Underscores, Category, Nugget] def test_generate_models(self): models = self.introspector.generate_models() self.assertTrue(set(( 'category', 'col_types', 'fkpk', 'nugget', 'nullable', 'rel_model', 'underscores')).issubset(set(models))) def assertIsInstance(obj, klass): self.assertTrue(isinstance(obj, klass)) category = models['category'] self.assertEqual( sorted(category._meta.fields), ['id', 'name', 'parent']) assertIsInstance(category.id, AutoField) assertIsInstance(category.name, CharField) assertIsInstance(category.parent, ForeignKeyField) self.assertEqual(category.parent.rel_model, category) fkpk = models['fkpk'] self.assertEqual(sorted(fkpk._meta.fields), ['col_types']) assertIsInstance(fkpk.col_types, ForeignKeyField) self.assertEqual(fkpk.col_types.rel_model, models['col_types']) self.assertTrue(fkpk.col_types.primary_key) relmodel = models['rel_model'] self.assertEqual( sorted(relmodel._meta.fields), ['col_types', 'col_types_nullable', 'id']) assertIsInstance(relmodel.col_types, ForeignKeyField) assertIsInstance(relmodel.col_types_nullable, ForeignKeyField) self.assertFalse(relmodel.col_types.null) self.assertTrue(relmodel.col_types_nullable.null) self.assertEqual(relmodel.col_types.rel_model, models['col_types']) self.assertEqual(relmodel.col_types_nullable.rel_model, models['col_types']) @requires_sqlite def test_generate_models_indexes(self): models = self.introspector.generate_models() self.assertEqual(models['fkpk']._meta.indexes, []) self.assertEqual(models['rel_model']._meta.indexes, []) self.assertEqual(models['category']._meta.indexes, []) col_types = models['col_types'] indexed = set(['f1']) unique = set(['f10']) for field in col_types._meta.sorted_fields: self.assertEqual(field.index, field.name in indexed) self.assertEqual(field.unique, field.name in unique) indexes = col_types._meta.indexes self.assertEqual(sorted(indexes), [ (['f10', 'f11'], True), (['f11', 'f8', 'f13'], False), ]) def test_table_subset(self): models = self.introspector.generate_models(table_names=[ 'category', 'col_types', 'foobarbaz']) self.assertEqual(sorted(models.keys()), ['category', 'col_types']) @requires_sqlite def test_sqlite_fk_re(self): user_id_tests = [ 'FOREIGN KEY("user_id") REFERENCES "users"("id")', 'FOREIGN KEY(user_id) REFERENCES users(id)', 'FOREIGN KEY ([user_id]) REFERENCES [users] ([id])', '"user_id" NOT NULL REFERENCES "users" ("id")', 'user_id not null references users (id)', ] fk_pk_tests = [ ('"col_types_id" INTEGER NOT NULL PRIMARY KEY REFERENCES ' '"coltypes" ("f11")'), 'FOREIGN KEY ("col_types_id") REFERENCES "coltypes" ("f11")', ] regex = SqliteMetadata.re_foreign_key for test in user_id_tests: match = re.search(regex, test, re.I) self.assertEqual(match.groups(), ( 'user_id', 'users', 'id', )) for test in fk_pk_tests: match = re.search(regex, test, re.I) self.assertEqual(match.groups(), ( 'col_types_id', 'coltypes', 'f11', )) def test_make_column_name(self): # Tests for is_foreign_key=False. tests = ( ('Column', 'column'), ('Foo_id', 'foo_id'), ('foo_id', 'foo_id'), ('foo_id_id', 'foo_id_id'), ('foo', 'foo'), ('_id', '_id'), ('a123', 'a123'), ('and', 'and_'), ('Class', 'class_'), ('Class_ID', 'class_id'), ('camelCase', 'camel_case'), ('ABCdefGhi', 'ab_cdef_ghi'), ) for col_name, expected in tests: self.assertEqual( self.introspector.make_column_name(col_name), expected) # Tests for is_foreign_key=True. tests = ( ('Foo_id', 'foo'), ('foo_id', 'foo'), ('foo_id_id', 'foo_id'), ('foo', 'foo'), ('_id', '_id'), ('a123', 'a123'), ('and', 'and_'), ('Class', 'class_'), ('Class_ID', 'class_'), ('camelCase', 'camel_case'), ('ABCdefGhi', 'ab_cdef_ghi'), ) for col_name, expected in tests: self.assertEqual( self.introspector.make_column_name(col_name, True), expected) def test_make_model_name(self): tests = ( ('Table', 'Table'), ('table', 'Table'), ('table_baz', 'TableBaz'), ('foo__bar__baz2', 'FooBarBaz2'), ('foo12_3', 'Foo123'), ) for table_name, expected in tests: self.assertEqual( self.introspector.make_model_name(table_name), expected) def test_col_types(self): (columns, primary_keys, foreign_keys, model_names, indexes) = self.introspector.introspect() expected = ( ('col_types', ( ('f1', (BigIntegerField, IntegerField), False), # There do not appear to be separate constants for the blob and # text field types in MySQL's drivers. See GH#1034. ('f2', (BlobField, TextField), False), ('f3', (BooleanField, IntegerField), False), ('f4', CharField, False), ('f5', DateField, False), ('f6', DateTimeField, False), ('f7', DecimalField, False), ('f8', (DoubleField, FloatField), False), ('f9', FloatField, False), ('f10', IntegerField, False), ('f11', AutoField, False), ('f12', TextField, False), ('f13', TimeField, False))), ('rel_model', ( ('col_types_id', ForeignKeyField, False), ('col_types_nullable_id', ForeignKeyField, True))), ('nugget', ( ('category_id', ForeignKeyField, False), ('category', CharField, False))), ('nullable', ( ('nullable_cf', CharField, True), ('nullable_if', IntegerField, True))), ('fkpk', ( ('col_types_id', ForeignKeyField, False),)), ('underscores', ( ('_id', AutoField, False), ('_name', CharField, False))), ('category', ( ('name', CharField, False), ('parent_id', ForeignKeyField, True))), ) for table_name, expected_columns in expected: introspected_columns = columns[table_name] for field_name, field_class, is_null in expected_columns: if not isinstance(field_class, (list, tuple)): field_class = (field_class,) column = introspected_columns[field_name] self.assertTrue(column.field_class in field_class, "%s in %s" % (column.field_class, field_class)) self.assertEqual(column.nullable, is_null) def test_foreign_keys(self): (columns, primary_keys, foreign_keys, model_names, indexes) = self.introspector.introspect() self.assertEqual(foreign_keys['col_types'], []) rel_model = foreign_keys['rel_model'] self.assertEqual(len(rel_model), 2) fkpk = foreign_keys['fkpk'] self.assertEqual(len(fkpk), 1) fkpk_fk = fkpk[0] self.assertEqual(fkpk_fk.table, 'fkpk') self.assertEqual(fkpk_fk.column, 'col_types_id') self.assertEqual(fkpk_fk.dest_table, 'col_types') self.assertEqual(fkpk_fk.dest_column, 'f11') category = foreign_keys['category'] self.assertEqual(len(category), 1) category_fk = category[0] self.assertEqual(category_fk.table, 'category') self.assertEqual(category_fk.column, 'parent_id') self.assertEqual(category_fk.dest_table, 'category') self.assertEqual(category_fk.dest_column, 'id') def test_table_names(self): (columns, primary_keys, foreign_keys, model_names, indexes) = self.introspector.introspect() names = ( ('col_types', 'ColTypes'), ('nullable', 'Nullable'), ('rel_model', 'RelModel'), ('fkpk', 'Fkpk')) for k, v in names: self.assertEqual(model_names[k], v) def test_column_meta(self): (columns, primary_keys, foreign_keys, model_names, indexes) = self.introspector.introspect() rel_model = columns['rel_model'] col_types_id = rel_model['col_types_id'] self.assertEqual(col_types_id.get_field_parameters(), { 'column_name': "'col_types_id'", 'model': 'ColTypes', 'field': "'f11'", }) col_types_nullable_id = rel_model['col_types_nullable_id'] self.assertEqual(col_types_nullable_id.get_field_parameters(), { 'column_name': "'col_types_nullable_id'", 'null': True, 'backref': "'col_types_col_types_nullable_set'", 'model': 'ColTypes', 'field': "'f11'", }) fkpk = columns['fkpk'] self.assertEqual(fkpk['col_types_id'].get_field_parameters(), { 'column_name': "'col_types_id'", 'model': 'ColTypes', 'primary_key': True, 'field': "'f11'"}) category = columns['category'] parent_id = category['parent_id'] self.assertEqual(parent_id.get_field_parameters(), { 'column_name': "'parent_id'", 'null': True, 'model': "'self'", 'field': "'id'", }) nugget = columns['nugget'] category_fk = nugget['category_id'] self.assertEqual(category_fk.name, 'category_id') self.assertEqual(category_fk.get_field_parameters(), { 'field': "'id'", 'model': 'Category', 'column_name': "'category_id'", }) category = nugget['category'] self.assertEqual(category.name, 'category') def test_get_field(self): (columns, primary_keys, foreign_keys, model_names, indexes) = self.introspector.introspect() expected = ( ('col_types', ( ('f1', ('f1 = BigIntegerField(index=True)', 'f1 = IntegerField(index=True)')), ('f2', ('f2 = BlobField()', 'f2 = TextField()')), ('f4', 'f4 = CharField()'), ('f5', 'f5 = DateField()'), ('f6', 'f6 = DateTimeField()'), ('f7', 'f7 = DecimalField()'), ('f10', 'f10 = IntegerField(unique=True)'), ('f11', 'f11 = AutoField()'), ('f12', ('f12 = TextField()', 'f12 = BlobField()')), ('f13', 'f13 = TimeField()'), )), ('nullable', ( ('nullable_cf', 'nullable_cf = ' 'CharField(null=True)'), ('nullable_if', 'nullable_if = IntegerField(null=True)'), )), ('fkpk', ( ('col_types_id', 'col_types = ForeignKeyField(' "column_name='col_types_id', field='f11', model=ColTypes, " 'primary_key=True)'), )), ('nugget', ( ('category_id', 'category_id = ForeignKeyField(' "column_name='category_id', field='id', model=Category)"), ('category', 'category = CharField()'), )), ('rel_model', ( ('col_types_id', 'col_types = ForeignKeyField(' "column_name='col_types_id', field='f11', model=ColTypes)"), ('col_types_nullable_id', 'col_types_nullable = ' "ForeignKeyField(backref='col_types_col_types_nullable_set', " "column_name='col_types_nullable_id', field='f11', " 'model=ColTypes, null=True)'), )), ('underscores', ( ('_id', '_id = AutoField()'), ('_name', '_name = CharField()'), )), ('category', ( ('name', 'name = CharField()'), ('parent_id', 'parent = ForeignKeyField(' "column_name='parent_id', field='id', model='self', " 'null=True)'), )), ) for table, field_data in expected: for field_name, fields in field_data: if not isinstance(fields, tuple): fields = (fields,) actual = columns[table][field_name].get_field() self.assertTrue(actual in fields, '%s not in %s' % (actual, fields)) class TestReflectNoPK(BaseReflectionTestCase): requires = [NoPK] def test_no_pk(self): models = self.introspector.generate_models() NoPK = models['no_pk'] if IS_CRDB: # CockroachDB always includes a "rowid". self.assertEqual(NoPK._meta.sorted_field_names, ['rowid', 'data']) else: self.assertEqual(NoPK._meta.sorted_field_names, ['data']) self.assertTrue(NoPK._meta.primary_key is False) class EventLog(TestModel): data = CharField(constraints=[SQL('DEFAULT \'\'')]) timestamp = DateTimeField(constraints=[SQL('DEFAULT current_timestamp')]) flags = IntegerField(constraints=[SQL('DEFAULT 0')]) misc = TextField(constraints=[SQL('DEFAULT \'foo\'')]) class DefaultVals(TestModel): key = CharField(constraints=[SQL('DEFAULT \'foo\'')]) value = IntegerField(constraints=[SQL('DEFAULT 0')]) class Meta: primary_key = CompositeKey('key', 'value') class TestReflectDefaultValues(BaseReflectionTestCase): requires = [DefaultVals, EventLog] @requires_sqlite def test_default_values(self): models = self.introspector.generate_models() default_vals = models['default_vals'] create_table = ( 'CREATE TABLE IF NOT EXISTS "default_vals" (' '"key" VARCHAR(255) NOT NULL DEFAULT \'foo\', ' '"value" INTEGER NOT NULL DEFAULT 0, ' 'PRIMARY KEY ("key", "value"))') # Re-create table using the introspected schema. self.assertSQL(default_vals._schema._create_table(), create_table, []) default_vals.drop_table() default_vals.create_table() # Verify that the introspected schema has not changed. models = self.introspector.generate_models() default_vals = models['default_vals'] self.assertSQL(default_vals._schema._create_table(), create_table, []) @requires_sqlite def test_default_values_extended(self): models = self.introspector.generate_models() eventlog = models['event_log'] create_table = ( 'CREATE TABLE IF NOT EXISTS "event_log" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"data" VARCHAR(255) NOT NULL DEFAULT \'\', ' '"timestamp" DATETIME NOT NULL DEFAULT current_timestamp, ' '"flags" INTEGER NOT NULL DEFAULT 0, ' '"misc" TEXT NOT NULL DEFAULT \'foo\')') # Re-create table using the introspected schema. self.assertSQL(eventlog._schema._create_table(), create_table, []) eventlog.drop_table() eventlog.create_table() # Verify that the introspected schema has not changed. models = self.introspector.generate_models() eventlog = models['event_log'] self.assertSQL(eventlog._schema._create_table(), create_table, []) class TestReflectionDependencies(BaseReflectionTestCase): requires = [User, Tweet] def test_generate_dependencies(self): models = self.introspector.generate_models(table_names=['tweet']) self.assertEqual(set(models), set(('users', 'tweet'))) IUser = models['users'] ITweet = models['tweet'] self.assertEqual(set(ITweet._meta.fields), set(( 'id', 'user', 'content', 'timestamp'))) self.assertEqual(set(IUser._meta.fields), set(('id', 'username'))) self.assertTrue(ITweet.user.rel_model is IUser) self.assertTrue(ITweet.user.rel_field is IUser.id) def test_ignore_backrefs(self): models = self.introspector.generate_models(table_names=['users']) self.assertEqual(set(models), set(('users',))) class Note(TestModel): content = TextField() timestamp = DateTimeField(default=datetime.datetime.now) status = IntegerField() class TestReflectViews(BaseReflectionTestCase): requires = [Note] def setUp(self): super(TestReflectViews, self).setUp() self.database.execute_sql('CREATE VIEW notes_public AS ' 'SELECT content, timestamp FROM note ' 'WHERE status = 1 ORDER BY timestamp DESC') def tearDown(self): self.database.execute_sql('DROP VIEW notes_public') super(TestReflectViews, self).tearDown() def test_views_ignored_default(self): models = self.introspector.generate_models() self.assertFalse('notes_public' in models) def test_introspect_view(self): models = self.introspector.generate_models(include_views=True) self.assertTrue('notes_public' in models) NotesPublic = models['notes_public'] self.assertEqual(sorted(NotesPublic._meta.fields), ['content', 'timestamp']) self.assertTrue(isinstance(NotesPublic.content, TextField)) self.assertTrue(isinstance(NotesPublic.timestamp, DateTimeField)) @skip_if(IS_SQLITE_OLD) @skip_if(IS_CRDB, 'crdb does not respect order by in view def') def test_introspect_view_integration(self): for i, (ct, st) in enumerate([('n1', 1), ('n2', 2), ('n3', 1)]): Note.create(content=ct, status=st, timestamp=datetime.datetime(2018, 1, 1 + i)) NP = self.introspector.generate_models( table_names=['notes_public'], include_views=True)['notes_public'] self.assertEqual([(np.content, np.timestamp) for np in NP.select()], [ ('n3', datetime.datetime(2018, 1, 3)), ('n1', datetime.datetime(2018, 1, 1))]) class TestCyclicalFK(BaseReflectionTestCase): def setUp(self): super(TestCyclicalFK, self).setUp() warnings.filterwarnings('ignore') @requires_sqlite @skip_if(IS_CYSQLITE, 'cysqlite does not implement cursor at the moment.') def test_cyclical_fk(self): # NOTE: this schema was provided by a user. cursor = self.database.cursor() cursor.executescript( 'CREATE TABLE flow_run_state (id CHAR(36) NOT NULL, ' 'flow_run_id CHAR(36) NOT NULL, ' 'CONSTRAINT pk_flow_run_state PRIMARY KEY (id), ' 'CONSTRAINT fk_flow_run_state__flow_run_id__flow_run ' 'FOREIGN KEY(flow_run_id) REFERENCES flow_run (id) ' 'ON DELETE cascade); ' 'CREATE TABLE flow_run (id CHAR(36) NOT NULL, ' 'state_id CHAR(36) NOT NULL, ' 'CONSTRAINT pk_flow_run PRIMARY KEY (id), ' 'CONSTRAINT fk_flow_run__state_id__flow_run_state ' 'FOREIGN KEY(state_id) REFERENCES flow_run_state (id) ' 'ON DELETE SET NULL);') M = self.introspector.generate_models() FRS = M['flow_run_state'] FR = M['flow_run'] self.assertEqual(sorted(FR._meta.fields), ['id', 'state']) self.assertEqual(sorted(FRS._meta.fields), ['flow_run', 'id']) self.assertTrue(isinstance(FR.id, CharField)) self.assertTrue(isinstance(FR.state, ForeignKeyField)) self.assertTrue(FR.state.rel_model is FRS) self.assertTrue(isinstance(FRS.id, CharField)) self.assertTrue(isinstance(FRS.flow_run, ForeignKeyField)) self.assertTrue(FRS.flow_run.rel_model is FR) class Event(TestModel): key = TextField() timestamp = DateTimeField(index=True) metadata = TextField(default='') class TestInteractiveHelpers(ModelTestCase): requires = [Category, Event] def test_generate_models(self): M = generate_models(self.database) self.assertTrue('category' in M) self.assertTrue('event' in M) def assertFields(m, expected): actual = [(f.name, f.field_type) for f in m._meta.sorted_fields] self.assertEqual(actual, expected) assertFields(M['category'], [('id', 'AUTO'), ('name', 'VARCHAR'), ('parent', 'INT')]) assertFields(M['event'], [ ('id', 'AUTO'), ('key', 'TEXT'), ('timestamp', 'DATETIME'), ('metadata', 'TEXT')]) ================================================ FILE: tests/regressions.py ================================================ import datetime import json import random import threading import time import uuid from peewee import * from playhouse.hybrid import * from playhouse.migrate import migrate from playhouse.migrate import SchemaMigrator from playhouse.shortcuts import ThreadSafeDatabaseMetadata from .base import BaseTestCase from .base import IS_MYSQL from .base import IS_MYSQL_ADVANCED_FEATURES from .base import IS_SQLITE from .base import IS_SQLITE_OLD from .base import ModelTestCase from .base import TestModel from .base import get_in_memory_db from .base import requires_models from .base import requires_mysql from .base import requires_postgresql from .base import skip_if from .base import skip_unless from .base import slow_test from .base_models import Sample from .base_models import Tweet from .base_models import User class ColAlias(TestModel): name = TextField(column_name='pname') class CARef(TestModel): colalias = ForeignKeyField(ColAlias, backref='carefs', column_name='ca', object_id_name='colalias_id') class TestQueryAliasToColumnName(ModelTestCase): requires = [ColAlias, CARef] def setUp(self): super(TestQueryAliasToColumnName, self).setUp() with self.database.atomic(): for name in ('huey', 'mickey'): col_alias = ColAlias.create(name=name) CARef.create(colalias=col_alias) def test_alias_to_column_name(self): # The issue here occurs when we take a field whose name differs from # it's underlying column name, then alias that field to it's column # name. In this case, peewee was *not* respecting the alias and using # the field name instead. query = (ColAlias .select(ColAlias.name.alias('pname')) .order_by(ColAlias.name)) self.assertEqual([c.pname for c in query], ['huey', 'mickey']) # Ensure that when using dicts the logic is preserved. query = query.dicts() self.assertEqual([r['pname'] for r in query], ['huey', 'mickey']) def test_alias_overlap_with_join(self): query = (CARef .select(CARef, ColAlias.name.alias('pname')) .join(ColAlias) .order_by(ColAlias.name)) with self.assertQueryCount(1): self.assertEqual([r.colalias.pname for r in query], ['huey', 'mickey']) # Note: we cannot alias the join to "ca", as this is the object-id # descriptor name. query = (CARef .select(CARef, ColAlias.name.alias('pname')) .join(ColAlias, on=(CARef.colalias == ColAlias.id).alias('ca')) .order_by(ColAlias.name)) with self.assertQueryCount(1): self.assertEqual([r.ca.pname for r in query], ['huey', 'mickey']) def test_cannot_alias_join_to_object_id_name(self): query = CARef.select(CARef, ColAlias.name.alias('pname')) expr = (CARef.colalias == ColAlias.id).alias('colalias_id') self.assertRaises(ValueError, query.join, ColAlias, on=expr) class TestOverrideModelRepr(BaseTestCase): def test_custom_reprs(self): # In 3.5.0, Peewee included a new implementation and semantics for # customizing model reprs. This introduced a regression where model # classes that defined a __repr__() method had this override ignored # silently. This test ensures that it is possible to completely # override the model repr. class Foo(Model): def __repr__(self): return 'FOO: %s' % self.id f = Foo(id=1337) self.assertEqual(repr(f), 'FOO: 1337') class DiA(TestModel): a = TextField(unique=True) class DiB(TestModel): a = ForeignKeyField(DiA) b = TextField() class DiC(TestModel): b = ForeignKeyField(DiB) c = TextField() class DiD(TestModel): c = ForeignKeyField(DiC) d = TextField() class DiBA(TestModel): a = ForeignKeyField(DiA, to_field=DiA.a) b = TextField() class TestDeleteInstanceRegression(ModelTestCase): database = get_in_memory_db() requires = [DiA, DiB, DiC, DiD, DiBA] def test_delete_instance_regression(self): with self.database.atomic(): a1, a2, a3 = [DiA.create(a=a) for a in ('a1', 'a2', 'a3')] for a in (a1, a2, a3): for j in (1, 2): b = DiB.create(a=a, b='%s-b%s' % (a.a, j)) c = DiC.create(b=b, c='%s-c' % (b.b)) d = DiD.create(c=c, d='%s-d' % (c.c)) DiBA.create(a=a, b='%s-b%s' % (a.a, j)) # (a1 (b1 (c (d))), (b2 (c (d)))), (a2 ...), (a3 ...) with self.assertQueryCount(5): a2.delete_instance(recursive=True) self.assertHistory(5, [ ('DELETE FROM "di_d" WHERE ("di_d"."c_id" IN (' 'SELECT "t1"."id" FROM "di_c" AS "t1" WHERE ("t1"."b_id" IN (' 'SELECT "t2"."id" FROM "di_b" AS "t2" WHERE ("t2"."a_id" = ?)' '))))', [2]), ('DELETE FROM "di_c" WHERE ("di_c"."b_id" IN (' 'SELECT "t1"."id" FROM "di_b" AS "t1" WHERE ("t1"."a_id" = ?)' '))', [2]), ('DELETE FROM "di_ba" WHERE ("di_ba"."a_id" = ?)', ['a2']), ('DELETE FROM "di_b" WHERE ("di_b"."a_id" = ?)', [2]), ('DELETE FROM "di_a" WHERE ("di_a"."id" = ?)', [2]) ]) # a1 & a3 exist, plus their relations. self.assertTrue(DiA.select().count(), 2) for rel in (DiB, DiBA, DiC, DiD): self.assertTrue(rel.select().count(), 4) # 2x2 with self.assertQueryCount(5): a1.delete_instance(recursive=True) # Only the objects related to a3 exist still. self.assertTrue(DiA.select().count(), 1) self.assertEqual(DiA.get(DiA.a == 'a3').id, a3.id) self.assertEqual([d.d for d in DiD.select().order_by(DiD.d)], ['a3-b1-c-d', 'a3-b2-c-d']) self.assertEqual([c.c for c in DiC.select().order_by(DiC.c)], ['a3-b1-c', 'a3-b2-c']) self.assertEqual([b.b for b in DiB.select().order_by(DiB.b)], ['a3-b1', 'a3-b2']) self.assertEqual([ba.b for ba in DiBA.select().order_by(DiBA.b)], ['a3-b1', 'a3-b2']) class TestCountUnionRegression(ModelTestCase): @requires_mysql @requires_models(User) def test_count_union(self): with self.database.atomic(): for i in range(5): User.create(username='user-%d' % i) lhs = User.select() rhs = User.select() query = (lhs | rhs) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username" FROM "users" AS "t1" ' 'UNION ' 'SELECT "t2"."id", "t2"."username" FROM "users" AS "t2"'), []) self.assertEqual(query.count(), 5) query = query.limit(3) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username" FROM "users" AS "t1" ' 'UNION ' 'SELECT "t2"."id", "t2"."username" FROM "users" AS "t2" ' 'LIMIT ?'), [3]) self.assertEqual(query.count(), 3) class User2(TestModel): username = TextField() class Category2(TestModel): name = TextField() parent = ForeignKeyField('self', backref='children', null=True) user = ForeignKeyField(User2) class TestGithub1354(ModelTestCase): @requires_models(Category2, User2) def test_get_or_create_self_referential_fk2(self): huey = User2.create(username='huey') parent = Category2.create(name='parent', user=huey) child, created = Category2.get_or_create(parent=parent, name='child', user=huey) child_db = Category2.get(Category2.parent == parent) self.assertEqual(child_db.user.username, 'huey') self.assertEqual(child_db.parent.name, 'parent') self.assertEqual(child_db.name, 'child') class TestInsertFromSQL(ModelTestCase): def setUp(self): super(TestInsertFromSQL, self).setUp() self.database.execute_sql('create table if not exists user_src ' '(name TEXT);') tbl = Table('user_src').bind(self.database) tbl.insert(name='foo').execute() def tearDown(self): super(TestInsertFromSQL, self).tearDown() self.database.execute_sql('drop table if exists user_src') @requires_models(User) def test_insert_from_sql(self): query_src = SQL('SELECT name FROM user_src') User.insert_from(query=query_src, fields=[User.username]).execute() self.assertEqual([u.username for u in User.select()], ['foo']) class TestSubqueryFunctionCall(BaseTestCase): def test_subquery_function_call(self): Sample = Table('sample') SA = Sample.alias('s2') query = (Sample .select(Sample.c.data) .where(~fn.EXISTS( SA.select(SQL('1')).where(SA.c.key == 'foo')))) self.assertSQL(query, ( 'SELECT "t1"."data" FROM "sample" AS "t1" ' 'WHERE NOT EXISTS(' 'SELECT 1 FROM "sample" AS "s2" WHERE ("s2"."key" = ?))'), ['foo']) class A(TestModel): id = IntegerField(primary_key=True) class B(TestModel): id = IntegerField(primary_key=True) class C(TestModel): id = IntegerField(primary_key=True) a = ForeignKeyField(A) b = ForeignKeyField(B) class TestCrossJoin(ModelTestCase): requires = [A, B, C] def setUp(self): super(TestCrossJoin, self).setUp() A.insert_many([(1,), (2,), (3,)], fields=[A.id]).execute() B.insert_many([(1,), (2,)], fields=[B.id]).execute() C.insert_many([ (1, 1, 1), (2, 1, 2), (3, 2, 1)], fields=[C.id, C.a, C.b]).execute() def test_cross_join(self): query = (A .select(A.id.alias('aid'), B.id.alias('bid')) .join(B, JOIN.CROSS) .join(C, JOIN.LEFT_OUTER, on=( (C.a == A.id) & (C.b == B.id))) .where(C.id.is_null()) .order_by(A.id, B.id)) self.assertEqual(list(query.tuples()), [(2, 2), (3, 1), (3, 2)]) def _create_users_tweets(db): data = ( ('huey', ('meow', 'hiss', 'purr')), ('mickey', ('woof', 'bark')), ('zaizee', ())) with db.atomic(): for username, tweets in data: user = User.create(username=username) for tweet in tweets: Tweet.create(user=user, content=tweet) class TestSubqueryInSelect(ModelTestCase): requires = [User, Tweet] def setUp(self): super(TestSubqueryInSelect, self).setUp() _create_users_tweets(self.database) def test_subquery_in_select(self): subq = User.select().where(User.username == 'huey') query = (Tweet .select(Tweet.content, Tweet.user.in_(subq).alias('is_huey')) .order_by(Tweet.content)) self.assertEqual([(r.content, r.is_huey) for r in query], [ ('bark', False), ('hiss', True), ('meow', True), ('purr', True), ('woof', False)]) @requires_postgresql class TestReturningIntegrationRegressions(ModelTestCase): requires = [User, Tweet] def test_returning_integration_subqueries(self): _create_users_tweets(self.database) # We can use a correlated subquery in the RETURNING clause. subq = (Tweet .select(fn.COUNT(Tweet.id).alias('ct')) .where(Tweet.user == User.id)) query = (User .update(username=(User.username + '-x')) .returning(subq.alias('ct'), User.username)) result = query.execute() self.assertEqual(sorted([(r.ct, r.username) for r in result]), [ (0, 'zaizee-x'), (2, 'mickey-x'), (3, 'huey-x')]) # We can use a correlated subquery via UPDATE...FROM, and reference the # FROM table in both the update and the RETURNING clause. subq = (User .select(User.id, fn.COUNT(Tweet.id).alias('ct')) .join(Tweet, JOIN.LEFT_OUTER) .group_by(User.id)) query = (User .update(username=User.username + subq.c.ct) .from_(subq) .where(User.id == subq.c.id) .returning(subq.c.ct, User.username)) result = query.execute() self.assertEqual(sorted([(r.ct, r.username) for r in result]), [ (0, 'zaizee-x0'), (2, 'mickey-x2'), (3, 'huey-x3')]) def test_returning_integration(self): query = (User .insert_many([('huey',), ('mickey',), ('zaizee',)], fields=[User.username]) .returning(User.id, User.username) .objects()) result = query.execute() self.assertEqual([(r.id, r.username) for r in result], [ (1, 'huey'), (2, 'mickey'), (3, 'zaizee')]) query = (User .delete() .where(~User.username.startswith('h')) .returning(User.id, User.username) .objects()) result = query.execute() self.assertEqual(sorted([(r.id, r.username) for r in result]), [ (2, 'mickey'), (3, 'zaizee')]) class TestUpdateIntegrationRegressions(ModelTestCase): requires = [User, Tweet, Sample] def setUp(self): super(TestUpdateIntegrationRegressions, self).setUp() _create_users_tweets(self.database) for i in range(4): Sample.create(counter=i, value=i) @skip_if(IS_MYSQL) def test_update_examples(self): # Do a simple update. res = (User .update(username=(User.username + '-cat')) .where(User.username != 'mickey') .execute()) users = User.select().order_by(User.username) self.assertEqual([u.username for u in users.clone()], ['huey-cat', 'mickey', 'zaizee-cat']) # Do an update using a subquery.. subq = User.select(User.username).where(User.username == 'mickey') res = (User .update(username=(User.username + '-dog')) .where(User.username.in_(subq)) .execute()) self.assertEqual([u.username for u in users.clone()], ['huey-cat', 'mickey-dog', 'zaizee-cat']) # Subquery referring to a different table. subq = User.select().where(User.username == 'mickey-dog') res = (Tweet .update(content=(Tweet.content + '-x')) .where(Tweet.user.in_(subq)) .execute()) self.assertEqual( [t.content for t in Tweet.select().order_by(Tweet.id)], ['meow', 'hiss', 'purr', 'woof-x', 'bark-x']) # Subquery on the right-hand of the assignment. subq = (Tweet .select(fn.COUNT(Tweet.id).cast('text')) .where(Tweet.user == User.id)) res = User.update(username=(User.username + '-' + subq)).execute() self.assertEqual([u.username for u in users.clone()], ['huey-cat-3', 'mickey-dog-2', 'zaizee-cat-0']) def test_update_examples_2(self): SA = Sample.alias() subq = (SA .select(SA.value) .where(SA.value.in_([1.0, 3.0]))) res = (Sample .update(counter=(Sample.counter + Sample.value.cast('int'))) .where(Sample.value.in_(subq)) .execute()) query = (Sample .select(Sample.counter, Sample.value) .order_by(Sample.id) .tuples()) self.assertEqual(list(query.clone()), [(0, 0.), (2, 1.), (2, 2.), (6, 3.)]) subq = (SA .select(SA.counter - SA.value.cast('int')) .where(SA.value == Sample.value)) res = (Sample .update(counter=subq) .where(Sample.value.in_([1., 3.])) .execute()) self.assertEqual(list(query.clone()), [(0, 0.), (1, 1.), (2, 2.), (3, 3.)]) class TestSelectValueConversion(ModelTestCase): requires = [User] @skip_if(IS_SQLITE_OLD or IS_MYSQL) def test_select_value_conversion(self): u1 = User.create(username='u1') cte = User.select(User.id.cast('text')).cte('tmp', columns=('id',)) query = User.select(cte.c.id.alias('id')).with_cte(cte).from_(cte) u1_id, = [user.id for user in query] self.assertEqual(u1_id, u1.id) query2 = User.select(cte.c.id.coerce(False)).with_cte(cte).from_(cte) u1_id, = [user.id for user in query2] self.assertEqual(u1_id, str(u1.id)) class ConflictDetectedException(Exception): pass class BaseVersionedModel(TestModel): version = IntegerField(default=1, index=True) def save_optimistic(self): if not self.id: # This is a new record, so the default logic is to perform an # INSERT. Ideally your model would also have a unique # constraint that made it impossible for two INSERTs to happen # at the same time. return self.save() # Update any data that has changed and bump the version counter. field_data = dict(self.__data__) current_version = field_data.pop('version', 1) self._populate_unsaved_relations(field_data) field_data = self._prune_fields(field_data, self.dirty_fields) if not field_data: raise ValueError('No changes have been made.') ModelClass = type(self) field_data['version'] = ModelClass.version + 1 # Atomic increment. query = ModelClass.update(**field_data).where( (ModelClass.version == current_version) & (ModelClass.id == self.id)) if query.execute() == 0: # No rows were updated, indicating another process has saved # a new version. How you handle this situation is up to you, # but for simplicity I'm just raising an exception. raise ConflictDetectedException() else: # Increment local version to match what is now in the db. self.version += 1 return True class VUser(BaseVersionedModel): username = TextField() class VTweet(BaseVersionedModel): user = ForeignKeyField(VUser, null=True) content = TextField() class TestOptimisticLockingDemo(ModelTestCase): requires = [VUser, VTweet] def test_optimistic_locking(self): vu = VUser(username='u1') vu.save_optimistic() vt = VTweet(user=vu, content='t1') vt.save_optimistic() # Update the "vt" row in the db, which bumps the version counter. vt2 = VTweet.get(VTweet.id == vt.id) vt2.content = 't1-x' vt2.save_optimistic() # Since no data was modified, this returns a ValueError. self.assertRaises(ValueError, vt.save_optimistic) # If we do make an update and attempt to save, a conflict is detected. vt.content = 't1-y' self.assertRaises(ConflictDetectedException, vt.save_optimistic) self.assertEqual(vt.version, 1) vt_db = VTweet.get(VTweet.id == vt.id) self.assertEqual(vt_db.content, 't1-x') self.assertEqual(vt_db.version, 2) self.assertEqual(vt_db.user.username, 'u1') def test_optimistic_locking_populate_fks(self): vt = VTweet(content='t1') vt.save_optimistic() vu = VUser(username='u1') vt.user = vu vu.save_optimistic() vt.save_optimistic() vt_db = VTweet.get(VTweet.content == 't1') self.assertEqual(vt_db.version, 2) self.assertEqual(vt_db.user.username, 'u1') class TS(TestModel): key = CharField(primary_key=True) timestamp = TimestampField(utc=True) class TestZeroTimestamp(ModelTestCase): requires = [TS] def test_zero_timestamp(self): t0 = TS.create(key='t0', timestamp=0) t1 = TS.create(key='t1', timestamp=1) t0_db = TS.get(TS.key == 't0') self.assertEqual(t0_db.timestamp, datetime.datetime(1970, 1, 1)) t1_db = TS.get(TS.key == 't1') self.assertEqual(t1_db.timestamp, datetime.datetime(1970, 1, 1, 0, 0, 1)) class Player(TestModel): name = TextField() class Game(TestModel): name = TextField() player = ForeignKeyField(Player) class Score(TestModel): game = ForeignKeyField(Game) points = IntegerField() class TestJoinSubqueryAggregateViaLeftOuter(ModelTestCase): requires = [Player, Game, Score] def test_join_subquery_aggregate_left_outer(self): with self.database.atomic(): p1, p2 = [Player.create(name=name) for name in ('p1', 'p2')] games = [] for p in (p1, p2): for gnum in (1, 2): g = Game.create(name='%s-g%s' % (p.name, gnum), player=p) games.append(g) score_list = ( (10, 20, 30), (), (100, 110, 100), (50, 50)) for g, plist in zip(games, score_list): for p in plist: Score.create(game=g, points=p) subq = (Game .select(Game.player, fn.SUM(Score.points).alias('ptotal'), fn.AVG(Score.points).alias('pavg')) .join(Score, JOIN.LEFT_OUTER) .group_by(Game.player)) query = (Player .select(Player, subq.c.ptotal, subq.c.pavg) .join(subq, on=(Player.id == subq.c.player_id)) .order_by(Player.name)) with self.assertQueryCount(1): results = [(p.name, p.game.ptotal, p.game.pavg) for p in query] self.assertEqual(results, [('p1', 60, 20), ('p2', 410, 82)]) with self.assertQueryCount(1): obj_query = query.objects() results = [(p.name, p.ptotal, p.pavg) for p in obj_query] self.assertEqual(results, [('p1', 60, 20), ('p2', 410, 82)]) class Project(TestModel): name = TextField() class Task(TestModel): name = TextField() project = ForeignKeyField(Project, backref='tasks') alt = ForeignKeyField(Project, backref='alt_tasks') class TestModelGraphMultiFK(ModelTestCase): requires = [Project, Task] def test_model_graph_multi_fk(self): pa, pb, pc = [Project.create(name=name) for name in 'abc'] t1 = Task.create(name='t1', project=pa, alt=pc) t2 = Task.create(name='t2', project=pb, alt=pb) P1 = Project.alias('p1') P2 = Project.alias('p2') LO = JOIN.LEFT_OUTER # Query using join expression. q1 = (Task .select(Task, P1, P2) .join_from(Task, P1, LO, on=(Task.project == P1.id)) .join_from(Task, P2, LO, on=(Task.alt == P2.id)) .order_by(Task.name)) # Query specifying target field. q2 = (Task .select(Task, P1, P2) .join_from(Task, P1, LO, on=Task.project) .join_from(Task, P2, LO, on=Task.alt) .order_by(Task.name)) # Query specifying with missing target field. q3 = (Task .select(Task, P1, P2) .join_from(Task, P1, LO) .join_from(Task, P2, LO, on=Task.alt) .order_by(Task.name)) for query in (q1, q2, q3): with self.assertQueryCount(1): t1, t2 = list(query) self.assertEqual(t1.project.name, 'a') self.assertEqual(t1.alt.name, 'c') self.assertEqual(t2.project.name, 'b') self.assertEqual(t2.alt.name, 'b') class TestBlobFieldContextRegression(BaseTestCase): def test_blob_field_context_regression(self): class A(Model): f = BlobField() orig = A.f._constructor db = get_in_memory_db() with db.bind_ctx([A]): self.assertTrue(A.f._constructor is db.get_binary_type()) self.assertTrue(A.f._constructor is orig) class Product(TestModel): id = CharField() color = CharField() class Meta: primary_key = CompositeKey('id', 'color') class Sku(TestModel): upc = CharField(primary_key=True) product_id = CharField() color = CharField() class Meta: constraints = [SQL('FOREIGN KEY (product_id, color) REFERENCES ' 'product(id, color)')] @hybrid_property def product(self): if not hasattr(self, '_product'): self._product = Product.get((Product.id == self.product_id) & (Product.color == self.color)) return self._product @product.setter def product(self, obj): self._product = obj self.product_id = obj.id self.color = obj.color @product.expression def product(cls): return (Product.id == cls.product_id) & (Product.color == cls.color) class TestFKCompositePK(ModelTestCase): requires = [Product, Sku] def test_fk_composite_pk_regression(self): Product.insert_many([ (1, 'red'), (1, 'blue'), (2, 'red'), (2, 'green'), (3, 'white')]).execute() Sku.insert_many([ ('1-red', 1, 'red'), ('1-blue', 1, 'blue'), ('2-red', 2, 'red'), ('2-green', 2, 'green'), ('3-white', 3, 'white')]).execute() query = (Product .select(Product, Sku) .join(Sku, on=Sku.product) .where(Product.color == 'red') .order_by(Product.id, Product.color)) with self.assertQueryCount(1): rows = [(p.id, p.color, p.sku.upc) for p in query] self.assertEqual(rows, [ ('1', 'red', '1-red'), ('2', 'red', '2-red')]) query = (Sku .select(Sku, Product) .join(Product, on=Sku.product) .where(Product.color != 'red') .order_by(Sku.upc)) with self.assertQueryCount(1): rows = [(s.upc, s.product_id, s.color, s.product.id, s.product.color) for s in query] self.assertEqual(rows, [ ('1-blue', '1', 'blue', '1', 'blue'), ('2-green', '2', 'green', '2', 'green'), ('3-white', '3', 'white', '3', 'white')]) class RS(TestModel): name = TextField() class RD(TestModel): key = TextField() value = IntegerField() rs = ForeignKeyField(RS, backref='rds') class RKV(TestModel): key = CharField(max_length=10) value = IntegerField() extra = IntegerField() class Meta: primary_key = CompositeKey('key', 'value') class TestRegressionCountDistinct(ModelTestCase): @requires_models(RS, RD) def test_regression_count_distinct(self): rs = RS.create(name='rs') nums = [0, 1, 2, 3, 2, 1, 0] RD.insert_many([('k%s' % i, i, rs) for i in nums]).execute() query = RD.select(RD.key).distinct() self.assertEqual(query.count(), 4) # Try re-selecting using the id/key, which are all distinct. query = query.select(RD.id, RD.key) self.assertEqual(query.count(), 7) # Re-select the key/value, of which there are 4 distinct. query = query.select(RD.key, RD.value) self.assertEqual(query.count(), 4) query = rs.rds.select(RD.key).distinct() self.assertEqual(query.count(), 4) query = rs.rds.select(RD.key, RD.value).distinct() self.assertEqual(query.count(), 4) # Was returning 7! @requires_models(RKV) def test_regression_count_distinct_cpk(self): RKV.insert_many([('k%s' % i, i, i) for i in range(5)]).execute() self.assertEqual(RKV.select().distinct().count(), 5) class TestReselectModelRegression(ModelTestCase): requires = [User] def test_reselect_model_regression(self): u1, u2, u3 = [User.create(username='u%s' % i) for i in '123'] query = User.select(User.username).order_by(User.username.desc()) self.assertEqual(list(query.tuples()), [('u3',), ('u2',), ('u1',)]) query = query.select(User) self.assertEqual(list(query.tuples()), [ (u3.id, 'u3',), (u2.id, 'u2',), (u1.id, 'u1',)]) class TestJoinCorrelatedSubquery(ModelTestCase): requires = [User, Tweet] def test_join_correlated_subquery(self): for i in range(3): user = User.create(username='u%s' % i) for j in range(i + 1): Tweet.create(user=user, content='u%s-%s' % (i, j)) UA = User.alias() subq = (UA .select(UA.username) .where(UA.username.in_(('u0', 'u2')))) query = (Tweet .select(Tweet, User) .join(User, on=( (Tweet.user == User.id) & (User.username.in_(subq)))) .order_by(Tweet.id)) with self.assertQueryCount(1): data = [(t.content, t.user.username) for t in query] self.assertEqual(data, [ ('u0-0', 'u0'), ('u2-0', 'u2'), ('u2-1', 'u2'), ('u2-2', 'u2')]) class RU(TestModel): username = TextField() class Recipe(TestModel): name = TextField() created_by = ForeignKeyField(RU, backref='recipes') changed_by = ForeignKeyField(RU, backref='recipes_modified') class TestMultiFKJoinRegression(ModelTestCase): requires = [RU, Recipe] def test_multi_fk_join_regression(self): u1, u2 = [RU.create(username=u) for u in ('u1', 'u2')] for (n, a, m) in (('r11', u1, u1), ('r12', u1, u2), ('r21', u2, u1)): Recipe.create(name=n, created_by=a, changed_by=m) Change = RU.alias() query = (Recipe .select(Recipe, RU, Change) .join(RU, on=(RU.id == Recipe.created_by).alias('a')) .switch(Recipe) .join(Change, on=(Change.id == Recipe.changed_by).alias('b')) .order_by(Recipe.name)) with self.assertQueryCount(1): data = [(r.name, r.a.username, r.b.username) for r in query] self.assertEqual(data, [ ('r11', 'u1', 'u1'), ('r12', 'u1', 'u2'), ('r21', 'u2', 'u1')]) class TestCompoundExistsRegression(ModelTestCase): requires = [User] def test_compound_regressions_1961(self): UA = User.alias() cq = (User.select(User.id) | UA.select(UA.id)) # Calling .exists() fails with AttributeError, no attribute "columns". self.assertFalse(cq.exists()) self.assertEqual(cq.count(), 0) User.create(username='u1') self.assertTrue(cq.exists()) self.assertEqual(cq.count(), 1) class TestViewFieldMapping(ModelTestCase): requires = [User] def tearDown(self): try: self.execute('drop view user_testview_fm') except Exception as exc: pass super(TestViewFieldMapping, self).tearDown() def test_view_field_mapping(self): user = User.create(username='huey') self.execute('create view user_testview_fm as ' 'select id, username from users') class View(User): class Meta: table_name = 'user_testview_fm' self.assertEqual([(v.id, v.username) for v in View.select()], [(user.id, 'huey')]) class TC(TestModel): ifield = IntegerField() ffield = FloatField() cfield = TextField() tfield = TextField() class TestTypeCoercion(ModelTestCase): requires = [TC] def test_type_coercion(self): t = TC.create(ifield='10', ffield='20.5', cfield=30, tfield=40) t_db = TC.get(TC.id == t.id) self.assertEqual(t_db.ifield, 10) self.assertEqual(t_db.ffield, 20.5) self.assertEqual(t_db.cfield, '30') self.assertEqual(t_db.tfield, '40') class TestLikeColumnValue(ModelTestCase): requires = [User, Tweet] def test_like_column_value(self): # e.g., find all tweets that contain the users own username. u1, u2, u3 = [User.create(username='u%s' % i) for i in (1, 2, 3)] data = ( (u1, ('nada', 'i am u1', 'u1 is my name')), (u2, ('nothing', 'he is u1')), (u3, ('she is u2', 'hey u3 is me', 'xx'))) for user, tweets in data: Tweet.insert_many([(user, tweet) for tweet in tweets], fields=[Tweet.user, Tweet.content]).execute() expressions = ( (Tweet.content ** ('%' + User.username + '%')), Tweet.content.contains(User.username)) for expr in expressions: query = (Tweet .select(Tweet, User) .join(User) .where(expr) .order_by(Tweet.id)) self.assertEqual([(t.user.username, t.content) for t in query], [ ('u1', 'i am u1'), ('u1', 'u1 is my name'), ('u3', 'hey u3 is me')]) class TestUnionParenthesesRegression(ModelTestCase): requires = [User] def test_union_parentheses_regression(self): ua, ub, uc = [User.create(username=u) for u in 'abc'] lhs = User.select(User.id).where(User.username == 'a') rhs = User.select(User.id).where(User.username == 'c') union = lhs.union_all(rhs) self.assertEqual(sorted([u.id for u in union]), [ua.id, uc.id]) query = User.select().where(User.id.in_(union)).order_by(User.id) self.assertEqual([u.username for u in query], ['a', 'c']) class NoPK(TestModel): data = IntegerField() class Meta: primary_key = False class TestNoPKHashRegression(ModelTestCase): requires = [NoPK] def test_no_pk_hash_regression(self): npk = NoPK.create(data=1) npk_db = NoPK.get(NoPK.data == 1) # When a model does not define a primary key, we cannot test equality. self.assertTrue(npk != npk_db) # Their hash is the same, though they are not equal. self.assertEqual(hash(npk), hash(npk_db)) class Site(TestModel): url = TextField() class Page(TestModel): site = ForeignKeyField(Site, backref='pages') title = TextField() class PageItem(TestModel): page = ForeignKeyField(Page, backref='items') content = TextField() class TestModelFilterJoinOrdering(ModelTestCase): requires = [Site, Page, PageItem] def setUp(self): super(TestModelFilterJoinOrdering, self).setUp() with self.database.atomic(): s1, s2 = [Site.create(url=s) for s in ('s1', 's2')] p11, p12, p21 = [Page.create(site=s, title=t) for s, t in ((s1, 'p1-1'), (s1, 'p1-2'), (s2, 'p2-1'))] items = ( (p11, 's1p1i1'), (p11, 's1p1i2'), (p11, 's1p1i3'), (p12, 's1p2i1'), (p21, 's2p1i1')) PageItem.insert_many(items).execute() def test_model_filter_join_ordering(self): q = PageItem.filter(page__site__url='s1').order_by(PageItem.content) self.assertSQL(q, ( 'SELECT "t1"."id", "t1"."page_id", "t1"."content" ' 'FROM "page_item" AS "t1" ' 'INNER JOIN "page" AS "t2" ON ("t1"."page_id" = "t2"."id") ' 'INNER JOIN "site" AS "t3" ON ("t2"."site_id" = "t3"."id") ' 'WHERE ("t3"."url" = ?) ORDER BY "t1"."content"'), ['s1']) def assertQ(q): with self.assertQueryCount(1): self.assertEqual([pi.content for pi in q], ['s1p1i1', 's1p1i2', 's1p1i3', 's1p2i1']) assertQ(q) sid = Site.get(Site.url == 's1').id q = (PageItem .filter(page__site__url='s1', page__site__id=sid) .order_by(PageItem.content)) assertQ(q) q = (PageItem .filter(page__site__id=sid) .filter(page__site__url='s1') .order_by(PageItem.content)) assertQ(q) q = (PageItem .filter(page__site__id=sid) .filter(DQ(page__title='p1-1') | DQ(page__title='p1-2')) .filter(page__site__url='s1') .order_by(PageItem.content)) assertQ(q) class JsonField(TextField): def db_value(self, value): return json.dumps(value) if value is not None else None def python_value(self, value): if value is not None: return json.loads(value) class JM(TestModel): key = TextField() data = JsonField() class TestListValueConversion(ModelTestCase): requires = [JM] def test_list_value_conversion(self): jm = JM.create(key='k1', data=['i0', 'i1']) jm.key = 'k1-x' jm.save() jm_db = JM.get(JM.key == 'k1-x') self.assertEqual(jm_db.data, ['i0', 'i1']) JM.update(data=['i1', 'i2']).execute() jm_db = JM.get(JM.key == 'k1-x') self.assertEqual(jm_db.data, ['i1', 'i2']) jm2 = JM.create(key='k2', data=['i3', 'i4']) jm_db.data = ['i1', 'i2', 'i3'] jm2.data = ['i4', 'i5'] JM.bulk_update([jm_db, jm2], fields=[JM.key, JM.data]) jm = JM.get(JM.key == 'k1-x') self.assertEqual(jm.data, ['i1', 'i2', 'i3']) jm2 = JM.get(JM.key == 'k2') self.assertEqual(jm2.data, ['i4', 'i5']) class TestCountSubqueryEquals(ModelTestCase): requires = [User, Tweet] def test_count_subquery_equals(self): a, b, c = [User.create(username=u) for u in 'abc'] Tweet.insert_many([(a, 'a1'), (b, 'b1')]).execute() subq = (Tweet .select(fn.COUNT(Tweet.id)) .where(Tweet.user == User.id)) query = User.select().where(subq == 0) self.assertEqual([u.username for u in query], ['c']) class BoolModel(TestModel): key = TextField() active = BooleanField() class TestBooleanCompare(ModelTestCase): requires = [BoolModel] def test_boolean_compare(self): b1 = BoolModel.create(key='b1', active=True) b2 = BoolModel.create(key='b2', active=False) expr2key = ( ((BoolModel.active == True), 'b1'), ((BoolModel.active == False), 'b2'), ((BoolModel.active != True), 'b2'), ((BoolModel.active != False), 'b1')) for expr, key in expr2key: q = BoolModel.select().where(expr) self.assertEqual([b.key for b in q], [key]) class CPK(TestModel): name = TextField() class CPKFK(TestModel): key = CharField() cpk = ForeignKeyField(CPK) class Meta: primary_key = CompositeKey('key', 'cpk') class TestCompositePKwithFK(ModelTestCase): requires = [CPK, CPKFK] def test_composite_pk_with_fk(self): c1 = CPK.create(name='c1') c2 = CPK.create(name='c2') CPKFK.create(key='k1', cpk=c1) CPKFK.create(key='k2', cpk=c1) CPKFK.create(key='k3', cpk=c2) query = (CPKFK .select(CPKFK.key, CPK) .join(CPK) .order_by(CPKFK.key, CPK.name)) with self.assertQueryCount(1): self.assertEqual([(r.key, r.cpk.name) for r in query], [('k1', 'c1'), ('k2', 'c1'), ('k3', 'c2')]) class TestChainWhere(ModelTestCase): requires = [User] def test_chain_where(self): for username in 'abcd': User.create(username=username) q = (User.select() .where(User.username != 'a') .where(User.username != 'd') .order_by(User.username)) self.assertEqual([u.username for u in q], ['b', 'c']) q = (User.select() .where(User.username != 'a') .where(User.username != 'd') .where(User.username == 'b')) self.assertEqual([u.username for u in q], ['b']) class BCUser(TestModel): username = CharField(unique=True) class BCTweet(TestModel): user = ForeignKeyField(BCUser, field=BCUser.username) content = TextField() class TestBulkCreateWithFK(ModelTestCase): @requires_models(BCUser, BCTweet) def test_bulk_create_with_fk(self): u1 = BCUser.create(username='u1') u2 = BCUser.create(username='u2') with self.assertQueryCount(1): BCTweet.bulk_create([ BCTweet(user='u1', content='t%s' % i) for i in range(4)]) self.assertEqual(BCTweet.select().where(BCTweet.user == 'u1').count(), 4) self.assertEqual(BCTweet.select().where(BCTweet.user != 'u1').count(), 0) u = BCUser(username='u3') t = BCTweet(user=u, content='tx') with self.assertQueryCount(2): BCUser.bulk_create([u]) BCTweet.bulk_create([t]) with self.assertQueryCount(1): t_db = (BCTweet .select(BCTweet, BCUser) .join(BCUser) .where(BCUser.username == 'u3') .get()) self.assertEqual(t_db.content, 'tx') self.assertEqual(t_db.user.username, 'u3') @requires_postgresql @requires_models(User, Tweet) def test_bulk_create_related_objects(self): u = User(username='u1') t = Tweet(user=u, content='t1') with self.assertQueryCount(2): User.bulk_create([u]) Tweet.bulk_create([t]) with self.assertQueryCount(1): t_db = Tweet.select(Tweet, User).join(User).get() self.assertEqual(t_db.content, 't1') self.assertEqual(t_db.user.username, 'u1') class UUIDReg(TestModel): id = UUIDField(primary_key=True, default=uuid.uuid4) key = TextField() class CharPKKV(TestModel): id = CharField(primary_key=True) key = TextField() value = IntegerField(default=0) class TestBulkUpdateNonIntegerPK(ModelTestCase): @requires_models(UUIDReg) def test_bulk_update_uuid_pk(self): r1 = UUIDReg.create(key='k1') r2 = UUIDReg.create(key='k2') r1.key = 'k1-x' r2.key = 'k2-x' UUIDReg.bulk_update((r1, r2), (UUIDReg.key,)) r1_db, r2_db = UUIDReg.select().order_by(UUIDReg.key) self.assertEqual(r1_db.key, 'k1-x') self.assertEqual(r2_db.key, 'k2-x') @requires_models(CharPKKV) def test_bulk_update_non_integer_pk(self): a, b, c = [CharPKKV.create(id=c, key='k%s' % c) for c in 'abc'] a.key = 'ka-x' a.value = 1 b.value = 2 c.key = 'kc-x' c.value = 3 CharPKKV.bulk_update((a, b, c), (CharPKKV.key, CharPKKV.value)) data = list(CharPKKV.select().order_by(CharPKKV.id).tuples()) self.assertEqual(data, [ ('a', 'ka-x', 1), ('b', 'kb', 2), ('c', 'kc-x', 3)]) class TestSaveClearingPK(ModelTestCase): requires = [User, Tweet] def test_save_clear_pk(self): u = User.create(username='u1') t1 = Tweet.create(content='t1', user=u) orig_id, t1.id = t1.id, None t1.content = 't2' t1.save() self.assertTrue(t1.id is not None) self.assertTrue(t1.id != orig_id) tweets = [t.content for t in u.tweets.order_by(Tweet.id)] self.assertEqual(tweets, ['t1', 't2']) class Bits(TestModel): b1 = BitField(default=1) b1_1 = b1.flag(1) b1_2 = b1.flag(2) b2 = BitField(default=0) b2_1 = b2.flag() b2_2 = b2.flag() class TestBitFieldName(ModelTestCase): requires = [Bits] def assertBits(self, bf, expected): b1_1, b1_2, b2_1, b2_2 = expected self.assertEqual(bf.b1_1, b1_1) self.assertEqual(bf.b1_2, b1_2) self.assertEqual(bf.b2_1, b2_1) self.assertEqual(bf.b2_2, b2_2) def test_bit_field_name(self): bf = Bits.create() self.assertBits(bf, (True, False, False, False)) bf.b1_1 = False bf.b1_2 = True bf.b2_1 = True bf.save() self.assertBits(bf, (False, True, True, False)) bf = Bits.get(Bits.id == bf.id) self.assertBits(bf, (False, True, True, False)) self.assertEqual(bf.b1, 2) self.assertEqual(bf.b2, 1) self.assertEqual(Bits.select().where(Bits.b1_2).count(), 1) self.assertEqual(Bits.select().where(Bits.b2_2).count(), 0) class FKMA(TestModel): name = TextField() class FKMB(TestModel): name = TextField() fkma = ForeignKeyField(FKMA, backref='fkmb_set', null=True) class TestFKMigrationRegression(ModelTestCase): requires = [FKMA, FKMB] def test_fk_migration(self): migrator = SchemaMigrator.from_database(self.database) kw = {'legacy': True} if IS_SQLITE else {} migrate(migrator.drop_column( FKMB._meta.table_name, FKMB.fkma.column_name, **kw)) migrate(migrator.add_column( FKMB._meta.table_name, FKMB.fkma.column_name, FKMB.fkma)) fa = FKMA.create(name='fa') FKMB.create(name='fb', fkma=fa) obj = FKMB.select().first() self.assertEqual(obj.name, 'fb') class ModelTypeField(CharField): def db_value(self, value): if value is not None: return value._meta.name def python_value(self, value): if value is not None: return {'user': User, 'tweet': Tweet}[value] class MTF(TestModel): name = TextField() mtype = ModelTypeField() class TestFieldValueRegression(ModelTestCase): requires = [MTF] def test_field_value_regression(self): u = MTF.create(name='user', mtype=User) u_db = MTF.get() self.assertEqual(u_db.name, 'user') self.assertTrue(u_db.mtype is User) class NLM(TestModel): a = IntegerField() b = IntegerField() class TestRegressionNodeListClone(ModelTestCase): requires = [NLM] def test_node_list_clone_expr(self): expr = (NLM.a + NLM.b) query = NLM.select(expr.alias('expr')).order_by(expr).distinct(expr) self.assertSQL(query, ( 'SELECT DISTINCT ON ("t1"."a" + "t1"."b") ' '("t1"."a" + "t1"."b") AS "expr" ' 'FROM "nlm" AS "t1" ' 'ORDER BY ("t1"."a" + "t1"."b")'), []) class LK(TestModel): key = TextField() class TestLikeEscape(ModelTestCase): requires = [LK] def assertNames(self, expr, expected): query = LK.select().where(expr).order_by(LK.id) self.assertEqual([lk.key for lk in query], expected) def test_like_escape(self): names = ('foo', 'foo%', 'foo%bar', 'foo_bar', 'fooxba', 'fooba') LK.insert_many([(n,) for n in names]).execute() cases = ( (LK.key.contains('bar'), ['foo%bar', 'foo_bar']), (LK.key.contains('%'), ['foo%', 'foo%bar']), (LK.key.contains('_'), ['foo_bar']), (LK.key.contains('o%b'), ['foo%bar']), (LK.key.startswith('foo%'), ['foo%', 'foo%bar']), (LK.key.startswith('foo_'), ['foo_bar']), (LK.key.startswith('bar'), []), (LK.key.endswith('ba'), ['fooxba', 'fooba']), (LK.key.endswith('_bar'), ['foo_bar']), (LK.key.endswith('fo'), []), ) for expr, expected in cases: self.assertNames(expr, expected) def test_like_escape_backslash(self): names = ('foo_bar\\baz', 'bar\\', 'fbar\\baz', 'foo_bar') LK.insert_many([(n,) for n in names]).execute() cases = ( (LK.key.contains('\\'), ['foo_bar\\baz', 'bar\\', 'fbar\\baz']), (LK.key.contains('_bar\\'), ['foo_bar\\baz']), (LK.key.contains('bar\\'), ['foo_bar\\baz', 'bar\\', 'fbar\\baz']), ) for expr, expected in cases: self.assertNames(expr, expected) class FKF_A(TestModel): key = CharField(max_length=16, unique=True) class FKF_B(TestModel): fk_a_1 = ForeignKeyField(FKF_A, field='key') fk_a_2 = IntegerField() class TestQueryWithModelInstanceParam(ModelTestCase): requires = [FKF_A, FKF_B] def test_query_with_model_instance_param(self): a1 = FKF_A.create(key='k1') a2 = FKF_A.create(key='k2') b1 = FKF_B.create(fk_a_1=a1, fk_a_2=a1) b2 = FKF_B.create(fk_a_1=a2, fk_a_2=a2) # Ensure that UPDATE works as expected as well. b1.save() # See also keys.TestFKtoNonPKField test, which replicates much of this. args = (b1.fk_a_1, b1.fk_a_1_id, a1, a1.key) for arg in args: query = FKF_B.select().where(FKF_B.fk_a_1 == arg) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."fk_a_1_id", "t1"."fk_a_2" ' 'FROM "fkf_b" AS "t1" ' 'WHERE ("t1"."fk_a_1_id" = ?)'), ['k1']) b1_db = query.get() self.assertEqual(b1_db.id, b1.id) # When we are handed a model instance and a conversion (an IntegerField # in this case), when the attempted conversion fails we fall back to # using the given model's primary-key. args = (b1.fk_a_2, a1, a1.id) for arg in args: query = FKF_B.select().where(FKF_B.fk_a_2 == arg) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."fk_a_1_id", "t1"."fk_a_2" ' 'FROM "fkf_b" AS "t1" ' 'WHERE ("t1"."fk_a_2" = ?)'), [a1.id]) b1_db = query.get() self.assertEqual(b1_db.id, b1.id) @skip_if(IS_SQLITE_OLD or IS_MYSQL) class TestModelSelectFromSubquery(ModelTestCase): requires = [User] def test_model_select_from_subquery(self): for i in range(5): User.create(username='u%s' % i) UA = User.alias() subquery = (UA.select() .where(UA.username.in_(('u0', 'u2', 'u4')))) cte = (ValuesList([('u0',), ('u4',)], columns=['username']) .cte('user_cte', columns=['username'])) query = (User .select(subquery.c.id, subquery.c.username) .from_(subquery) .join(cte, on=(subquery.c.username == cte.c.username)) .with_cte(cte) .order_by(subquery.c.username.desc())) self.assertEqual([u.username for u in query], ['u4', 'u0']) self.assertTrue(isinstance(query[0], User)) class CharPK(TestModel): id = CharField(primary_key=True) name = CharField(unique=True) def __str__(self): return self.name class CharFK(TestModel): id = IntegerField(primary_key=True) cpk = ForeignKeyField(CharPK, field=CharPK.name) class TestModelConversionRegression(ModelTestCase): requires = [CharPK, CharFK] def test_model_conversion_regression(self): cpks = [CharPK.create(id=str(i), name='u%s' % i) for i in range(3)] query = CharPK.select().where(CharPK.id << cpks) self.assertEqual(sorted([c.id for c in query]), ['0', '1', '2']) query = CharPK.select().where(CharPK.id.in_(list(CharPK.select()))) self.assertEqual(sorted([c.id for c in query]), ['0', '1', '2']) def test_model_conversion_fk_retained(self): cpks = [CharPK.create(id=str(i), name='u%s' % i) for i in range(3)] cfks = [CharFK.create(id=i + 1, cpk='u%s' % i) for i in range(3)] c0, c1, c2 = cpks query = CharFK.select().where(CharFK.cpk << [c0, c2]) self.assertEqual(sorted([f.id for f in query]), [1, 3]) class FKN_A(TestModel): pass class FKN_B(TestModel): a = ForeignKeyField(FKN_A, null=True) class TestSetFKNull(ModelTestCase): requires = [FKN_A, FKN_B] def test_set_fk_null(self): a1 = FKN_A.create() a2 = FKN_A() b1 = FKN_B(a=a1) b2 = FKN_B(a=a2) self.assertTrue(b1.a is a1) self.assertTrue(b2.a is a2) b1.a = b2.a = None self.assertTrue(b1.a is None) self.assertTrue(b2.a is None) class TestWeirdAliases(ModelTestCase): requires = [User] @skip_if(IS_MYSQL) # mysql can't do anything normally. def test_weird_aliases(self): User.create(username='huey') def assertAlias(s, expected): query = User.select(s).dicts() row = query[0] self.assertEqual(list(row)[0], expected) # When we explicitly provide an alias, use that. assertAlias(User.username.alias('"username"'), '"username"') assertAlias(User.username.alias('(username)'), '(username)') assertAlias(User.username.alias('user(name)'), 'user(name)') assertAlias(User.username.alias('(username"'), '(username"') assertAlias(User.username.alias('"username)'), '"username)') assertAlias(fn.LOWER(User.username).alias('user (name)'), 'user (name)') # Here peewee cannot tell that an alias was given, so it will attempt # to clean-up the column name returned by the cursor description. assertAlias(SQL('"t1"."username" AS "user name"'), 'user name') assertAlias(SQL('"t1"."username" AS "user (name)"'), 'user (name') assertAlias(SQL('"t1"."username" AS "(username)"'), 'username') assertAlias(SQL('"t1"."username" AS "x.y.(username)"'), 'username') if IS_SQLITE: assertAlias(SQL('LOWER("t1"."username")'), 'username') class NDF(TestModel): key = CharField(primary_key=True) date = DateTimeField(null=True) class TestBulkUpdateAllNull(ModelTestCase): requires = [NDF] @skip_unless(IS_SQLITE or IS_MYSQL, 'postgres cannot do this properly') def test_bulk_update_all_null(self): n1 = NDF.create(key='n1', date=datetime.datetime(2021, 1, 1)) n2 = NDF.create(key='n2', date=datetime.datetime(2021, 1, 2)) rows = [NDF(key=key, date=None) for key in ('n1', 'n2')] NDF.bulk_update(rows, fields=['date']) query = NDF.select().order_by(NDF.key).tuples() self.assertEqual([r for r in query], [('n1', None), ('n2', None)]) class CQA(TestModel): a = TextField() b = TextField() class TestSelectFromUnion(ModelTestCase): requires = [CQA] def test_select_from_union(self): CQA.insert_many([('a%d' % i, 'b%d' % i) for i in range(10)]).execute() q1 = CQA.select(CQA.a).order_by(CQA.id).limit(3) q2 = CQA.select(CQA.b).order_by(CQA.id).limit(3) wq1 = q1.select_from(SQL('*')) wq2 = q2.select_from(SQL('*')) union = wq1 | wq2 data = [val for val, in union.tuples()] self.assertEqual(sorted(data), ['a0', 'a1', 'a2', 'b0', 'b1', 'b2']) class DF(TestModel): name = TextField() value = IntegerField() class DFC(TestModel): df = ForeignKeyField(DF) name = TextField() value = IntegerField() class DFGC(TestModel): dfc = ForeignKeyField(DFC) name = TextField() value = IntegerField() class TestDjangoFilterRegression(ModelTestCase): requires = [DF, DFC, DFGC] def test_django_filter_regression(self): a, b, c = [DF.create(name=n, value=i) for i, n in enumerate('abc')] ca1 = DFC.create(df=a, name='a1', value=11) ca2 = DFC.create(df=a, name='a2', value=12) cb1 = DFC.create(df=b, name='b1', value=21) gca1_1 = DFGC.create(dfc=ca1, name='a1-1', value=101) gca1_2 = DFGC.create(dfc=ca1, name='a1-2', value=101) gca2_1 = DFGC.create(dfc=ca2, name='a2-1', value=111) def assertNames(q, expected): self.assertEqual(sorted([n.name for n in q]), expected) assertNames(DF.filter(name='a'), ['a']) assertNames(DF.filter(name='a', id=a.id), ['a']) assertNames(DF.filter(name__in=['a', 'c']), ['a', 'c']) assertNames(DF.filter(name__in=['a', 'c'], id=a.id), ['a']) assertNames(DF.filter(dfc_set__name='a1'), ['a']) assertNames(DF.filter(dfc_set__name__in=['a1', 'b1']), ['a', 'b']) assertNames(DF.filter(DQ(dfc_set__name='a1') | DQ(dfc_set__name='b1')), ['a', 'b']) assertNames(DF.filter(dfc_set__dfgc_set__name='a1-1'), ['a']) assertNames(DF.filter( DQ(dfc_set__dfgc_set__name='a1-1') | DQ(dfc_set__dfgc_set__name__in=['x', 'y'])), ['a']) assertNames(DFC.filter(df__name='a'), ['a1', 'a2']) assertNames(DFC.filter(df__name='a', value=11), ['a1']) assertNames(DFC.filter(DQ(df__name='a') | DQ(df__name='b')), ['a1', 'a2', 'b1']) assertNames(DFC.filter( DQ(df__name='a') | DQ(dfgc_set__name='a1-1')).distinct(), ['a1', 'a2']) assertNames(DFGC.filter(dfc__df__name='a'), ['a1-1', 'a1-2', 'a2-1']) assertNames(DFGC.filter(dfc__df__name='a', dfc__name='a2'), ['a2-1']) assertNames(DFGC.filter( DQ(dfc__df__value__lte=0) | DQ(dfc__df__name='a', dfc__name='a1') | DQ(dfc__name='a2')), ['a1-1', 'a1-2', 'a2-1']) assertNames( (DFGC.filter(DQ(dfc__df__value__lte=10) | DQ(dfc__value__lte=101)) .filter(DQ(name__ilike='a1%') | DQ(dfc__value=101))), ['a1-1', 'a1-2']) assertNames(DFGC.filter(dfc__df=a), ['a1-1', 'a1-2', 'a2-1']) assertNames(DFGC.filter(dfc__df=a.id), ['a1-1', 'a1-2', 'a2-1']) q = DFC.select().join(DF) assertNames(q.filter(df=a), ['a1', 'a2']) assertNames(q.filter(df__name='a'), ['a1', 'a2']) DFA = DF.alias() DFCA = DFC.alias() DFGCA = DFGC.alias() q = DFCA.select().join(DFA) assertNames(q.filter(df=a), ['a1', 'a2']) assertNames(q.filter(df__name='a'), ['a1', 'a2']) q = DFGC.select().join(DFC).join(DF) assertNames(q.filter(dfc__df=a), ['a1-1', 'a1-2', 'a2-1']) q = DFGCA.select().join(DFCA).join(DFA) assertNames(q.filter(dfc__df=a), ['a1-1', 'a1-2', 'a2-1']) q = DF.select().join(DFC).join(DFGC) assertNames(q.filter(dfc_set__dfgc_set__name='a1-1'), ['a']) class TestFunctionInfiniteLoop(BaseTestCase): def test_function_infinite_loop(self): self.assertRaises(TypeError, lambda: list(fn.COUNT())) class State(TestModel): name = TextField() class Transition(TestModel): src = ForeignKeyField(State, backref='sources') dest = ForeignKeyField(State, backref='dests') class TestJoinTypePrefetchMultipleFKs(ModelTestCase): requires = [State, Transition] def test_join_prefetch_multiple_fks(self): s1, s2a, s2b, s3 = [State.create(name=s) for s in ('s1', 's2a', 's2b', 's3')] t1 = Transition.create(src=s1, dest=s2a) t2 = Transition.create(src=s1, dest=s2b) t3 = Transition.create(src=s2a, dest=s3) t4 = Transition.create(src=s2b, dest=s3) query = State.select().where(State.name != 's3').order_by(State.name) transitions = (Transition .select(Transition, State) .join(State, on=Transition.dest) .order_by(Transition.id)) with self.assertQueryCount(2): p = prefetch(query, transitions, prefetch_type=PREFETCH_TYPE.JOIN) accum = [] for row in p: accum.append((row.name, row.sources, row.dests, [d.dest.name for d in row.sources], [d.src.name for d in row.dests])) self.assertEqual(accum, [ ('s1', [t1, t2], [], ['s2a', 's2b'], []), ('s2a', [t3], [t1], ['s3'], ['s1']), ('s2b', [t4], [t2], ['s3'], ['s1'])]) @slow_test() class TestThreadSafetyDecorators(ModelTestCase): requires = [User] def test_thread_safety_atomic(self): @self.database.atomic() def get_one(n): time.sleep(n) return User.select().first() def run(n): with self.database.atomic(): get_one(n) User.create(username='u') threads = [threading.Thread(target=run, args=(i,)) for i in (0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.5)] for t in threads: t.start() for t in threads: t.join() class TestQueryCountList(ModelTestCase): requires = [User] def test_iteration_single_query(self): with self.assertQueryCount(1): sq = User.select() for i in range(3): self.assertEqual(list(sq), []) self.assertFalse(bool(sq)) with self.assertQueryCount(1): sq = User.select().tuples() for i in range(3): self.assertEqual(list(sq), []) self.assertFalse(bool(sq)) with self.assertQueryCount(1): self.assertEqual(User.select().count(), 0) class TestSumCaseSubquery(ModelTestCase): requires = [Sample] def test_sum_case_subquery(self): Sample.insert_many([(i, i) for i in range(5)]).execute() subq = Sample.select().where(Sample.counter.in_([1, 3, 5])) case = Case(None, [(Sample.id.in_(subq), Sample.value)], 0) q = Sample.select(fn.SUM(case)) self.assertEqual(q.scalar(), 4.0) class I(TestModel): name = TextField() class S(TestModel): i = ForeignKeyField(I) class P(TestModel): i = ForeignKeyField(I) class PS(TestModel): p = ForeignKeyField(P) s = ForeignKeyField(S) class PP(TestModel): ps = ForeignKeyField(PS) class O(TestModel): ps = ForeignKeyField(PS) s = ForeignKeyField(S) class OX(TestModel): o = ForeignKeyField(O, null=True) class Character(TestModel): name = TextField() class Shape(TestModel): character = ForeignKeyField(Character, null=True) class ShapeDetail(TestModel): shape = ForeignKeyField(Shape) class TestDeleteInstanceDFS(ModelTestCase): @requires_models(Character, Shape, ShapeDetail) def test_delete_instance_dfs_nullable(self): c1, c2 = [Character.create(name=name) for name in ('c1', 'c2')] for c in (c1, c2): s = Shape.create(character=c) ShapeDetail.create(shape=s) # Update nullables. with self.assertQueryCount(2): c1.delete_instance(True) self.assertHistory(2, [ ('UPDATE "shape" SET "character_id" = ? WHERE ' '("shape"."character_id" = ?)', [None, c1.id]), ('DELETE FROM "character" WHERE ("character"."id" = ?)', [c1.id])]) self.assertEqual(Shape.select().count(), 2) # Delete nullables as well. with self.assertQueryCount(3): c2.delete_instance(True, True) self.assertHistory(3, [ ('DELETE FROM "shape_detail" WHERE ' '("shape_detail"."shape_id" IN ' '(SELECT "t1"."id" FROM "shape" AS "t1" WHERE ' '("t1"."character_id" = ?)))', [c2.id]), ('DELETE FROM "shape" WHERE ("shape"."character_id" = ?)', [c2.id]), ('DELETE FROM "character" WHERE ("character"."id" = ?)', [c2.id])]) self.assertEqual(Shape.select().count(), 1) @requires_models(I, S, P, PS, PP, O, OX) def test_delete_instance_dfs(self): i1, i2 = [I.create(name=n) for n in ('i1', 'i2')] for i in (i1, i2): s = S.create(i=i) p = P.create(i=i) ps = PS.create(p=p, s=s) pp = PP.create(ps=ps) o = O.create(ps=ps, s=s) ox = OX.create(o=o) with self.assertQueryCount(9): i1.delete_instance(recursive=True) self.assertHistory(9, [ ('DELETE FROM "pp" WHERE (' '"pp"."ps_id" IN (SELECT "t1"."id" FROM "ps" AS "t1" WHERE (' '"t1"."p_id" IN (SELECT "t2"."id" FROM "p" AS "t2" WHERE (' '"t2"."i_id" = ?)))))', [i1.id]), ('UPDATE "ox" SET "o_id" = ? WHERE (' '"ox"."o_id" IN (SELECT "t1"."id" FROM "o" AS "t1" WHERE (' '"t1"."ps_id" IN (SELECT "t2"."id" FROM "ps" AS "t2" WHERE (' '"t2"."p_id" IN (SELECT "t3"."id" FROM "p" AS "t3" WHERE (' '"t3"."i_id" = ?)))))))', [None, i1.id]), ('DELETE FROM "o" WHERE (' '"o"."ps_id" IN (SELECT "t1"."id" FROM "ps" AS "t1" WHERE (' '"t1"."p_id" IN (SELECT "t2"."id" FROM "p" AS "t2" WHERE (' '"t2"."i_id" = ?)))))', [i1.id]), ('DELETE FROM "o" WHERE (' '"o"."s_id" IN (SELECT "t1"."id" FROM "s" AS "t1" WHERE (' '"t1"."i_id" = ?)))', [i1.id]), ('DELETE FROM "ps" WHERE (' '"ps"."p_id" IN (SELECT "t1"."id" FROM "p" AS "t1" WHERE (' '"t1"."i_id" = ?)))', [i1.id]), ('DELETE FROM "ps" WHERE (' '"ps"."s_id" IN (SELECT "t1"."id" FROM "s" AS "t1" WHERE (' '"t1"."i_id" = ?)))', [i1.id]), ('DELETE FROM "s" WHERE ("s"."i_id" = ?)', [i1.id]), ('DELETE FROM "p" WHERE ("p"."i_id" = ?)', [i1.id]), ('DELETE FROM "i" WHERE ("i"."id" = ?)', [i1.id]), ]) models = [I, S, P, PS, PP, O, OX] counts = {OX: 2} for m in models: self.assertEqual(m.select().count(), counts.get(m, 1)) class IMC(TestModel): a = IntegerField() b = IntegerField(null=True) class TestChunkedInsertMany(ModelTestCase): requires = [IMC] def test_chunked_insert_many(self): data = [(i, i if i % 2 == 0 else None) for i in range(100)] for chunk in chunked(data, 10): IMC.insert_many(chunk).execute() q = IMC.select(IMC.a, IMC.b).order_by(IMC.id).tuples() self.assertEqual(list(q), data) IMC.delete().execute() data = [{'a': i, 'b': i if i % 2 == 0 else None} for i in range(100)] for chunk in chunked(data, 5): IMC.insert_many(chunk).execute() q = IMC.select(IMC.a, IMC.b).order_by(IMC.id).dicts() self.assertEqual(list(q), data) IMC.delete().execute() @slow_test() class TestThreadSafeMetaRegression(ModelTestCase): def test_thread_safe_meta(self): d1 = get_in_memory_db() d2 = get_in_memory_db() class Meta: database = d1 model_metadata_class = ThreadSafeDatabaseMetadata attrs = {'Meta': Meta} for i in range(1, 30): attrs['f%d' % i] = IntegerField() M = type('M', (TestModel,), attrs) sql = ('SELECT "t1"."f1", "t1"."f2", "t1"."f3", "t1"."f4" ' 'FROM "m" AS "t1"') query = M.select(M.f1, M.f2, M.f3, M.f4) def swap_db(): for i in range(100): self.assertEqual(M._meta.database, d1) self.assertSQL(query, sql) with d2.bind_ctx([M]): self.assertEqual(M._meta.database, d2) self.assertSQL(query, sql) self.assertEqual(M._meta.database, d1) self.assertSQL(query, sql) # From a separate thread, swap the database and verify it works # correctly. threads = [threading.Thread(target=swap_db) for i in range(20)] for t in threads: t.start() for t in threads: t.join() # In the main thread the original database has not been altered. self.assertEqual(M._meta.database, d1) self.assertSQL(query, sql) ================================================ FILE: tests/results.py ================================================ import datetime from peewee import * from .base import get_in_memory_db from .base import ModelTestCase from .base_models import * def lange(x, y=None): if y is None: value = range(x) else: value = range(x, y) return list(value) class TestCursorWrapper(ModelTestCase): database = get_in_memory_db() requires = [User] def test_iteration(self): for i in range(10): User.create(username=str(i)) query = User.select() cursor = query.execute() first_five = [] for i, u in enumerate(cursor): first_five.append(int(u.username)) if i == 4: break self.assertEqual(first_five, lange(5)) names = lambda i: [int(obj.username) for obj in i] self.assertEqual(names(query[5:]), lange(5, 10)) self.assertEqual(names(query[2:5]), lange(2, 5)) for i in range(2): self.assertEqual(names(cursor), lange(10)) def test_count(self): for i in range(5): User.create(username=str(i)) with self.assertQueryCount(1): query = User.select() self.assertEqual(len(query), 5) cursor = query.execute() self.assertEqual(len(cursor), 5) with self.assertQueryCount(1): query = query.where(User.username != '0') cursor = query.execute() self.assertEqual(len(cursor), 4) self.assertEqual(len(query), 4) def test_nested_iteration(self): for i in range(4): User.create(username=str(i)) with self.assertQueryCount(1): query = User.select().order_by(User.username) outer = [] inner = [] for o_user in query: outer.append(int(o_user.username)) for i_user in query: inner.append(int(i_user.username)) self.assertEqual(outer, lange(4)) self.assertEqual(inner, lange(4) * 4) def test_iterator_protocol(self): for i in range(3): User.create(username=str(i)) with self.assertQueryCount(1): query = User.select().order_by(User.id) cursor = query.execute() for _ in range(2): for user in cursor: pass it = iter(cursor) for obj in it: pass self.assertRaises(StopIteration, next, it) self.assertEqual([int(u.username) for u in cursor], lange(3)) self.assertEqual(query[0].username, '0') self.assertEqual(query[2].username, '2') self.assertRaises(StopIteration, next, it) def test_iterator(self): for i in range(3): User.create(username=str(i)) with self.assertQueryCount(1): cursor = User.select().order_by(User.id).execute() usernames = [int(u.username) for u in cursor.iterator()] self.assertEqual(usernames, lange(3)) self.assertTrue(cursor.populated) self.assertEqual(cursor.row_cache, []) with self.assertQueryCount(0): self.assertEqual(list(cursor), []) def test_query_iterator(self): for i in range(3): User.create(username=str(i)) with self.assertQueryCount(1): query = User.select().order_by(User.id) usernames = [int(u.username) for u in query.iterator()] self.assertEqual(usernames, lange(3)) with self.assertQueryCount(0): self.assertEqual(list(query), []) def test_row_cache(self): def assertCache(cursor, n): self.assertEqual([int(u.username) for u in cursor.row_cache], lange(n)) for i in range(10): User.create(username=str(i)) with self.assertQueryCount(1): cursor = User.select().order_by(User.id).execute() cursor.fill_cache(5) self.assertFalse(cursor.populated) assertCache(cursor, 5) cursor.fill_cache(5) assertCache(cursor, 5) cursor.fill_cache(6) assertCache(cursor, 6) self.assertFalse(cursor.populated) cursor.fill_cache(11) self.assertTrue(cursor.populated) assertCache(cursor, 10) class TestRowTypes(ModelTestCase): database = get_in_memory_db() requires = [User, Tweet] def make_query(self, *exprs): count = 0 accum = [] for expr in exprs: if isinstance(expr, str): accum.append(Value('v%d' % count).alias(expr)) count += 1 else: accum.append(expr) return User.select(*accum).order_by(User.username) def test_namedtuples(self): User.create(username='u1') query = self.make_query(User.username).namedtuples() self.assertEqual([u.username for u in query], ['u1']) row = query[0] self.assertEqual(repr(row), 'Row(username=\'u1\')') query = (self .make_query(User.username, 'username', 'username') .namedtuples()) row, = list(query) self.assertEqual(row, ('u1', 'v0', 'v1')) self.assertEqual(row.username, 'u1') self.assertEqual(row.username_2, 'v0') self.assertEqual(row.username_3, 'v1') query = (self .make_query('username', User.username) .namedtuples()) row, = list(query) self.assertEqual(row, ('v0', 'u1')) self.assertEqual(row.username, 'v0') self.assertEqual(row.username_2, 'u1') query = (self .make_query('"foo"', '"t1"."foo"()', 'foo ') .namedtuples()) row, = list(query) self.assertEqual(row, ('v0', 'v1', 'v2')) self.assertEqual(row.foo, 'v0') self.assertEqual(row.foo_2, 'v1') self.assertEqual(row.foo_3, 'v2') def test_dicts(self): User.create(username='u1') query = self.make_query(User.username).dicts() self.assertEqual(list(query), [{'username': 'u1'}]) query = (self .make_query(User.username, 'username', 'username') .dicts()) row, = list(query) self.assertEqual(row, { 'username': 'u1', 'username_2': 'v0', 'username_3': 'v1'}) query = (self .make_query('username', User.username) .dicts()) row, = list(query) self.assertEqual(row, { 'username': 'v0', 'username_2': 'u1'}) query = (self .make_query('"foo"', '"t1"."foo"()', 'foo ') .dicts()) row, = list(query) self.assertEqual(row, { '"foo"': 'v0', '"t1"."foo"()': 'v1', 'foo ': 'v2'}) def test_dicts_flat(self): u = User.create(username='u1') for i in range(3): Tweet.create(user=u, content='t%d' % (i + 1)) query = (Tweet .select(Tweet, User.username) .join(User) .order_by(Tweet.id) .dicts()) with self.assertQueryCount(1): results = [(r['id'], r['content'], r['username']) for r in query] self.assertEqual(results, [ (1, 't1', 'u1'), (2, 't2', 'u1'), (3, 't3', 'u1')]) def test_model_objects(self): User.create(username='u1') query = self.make_query(User.username).objects() self.assertEqual([u.username for u in query], ['u1']) query = (self .make_query(User.username, 'username', 'username') .objects()) row, = list(query) self.assertEqual(row.username, 'u1') self.assertEqual(row.username_2, 'v0') self.assertEqual(row.username_3, 'v1') query = (self .make_query('username', User.username) .objects()) row, = list(query) self.assertEqual(row.username, 'v0') self.assertEqual(row.username_2, 'u1') query = (self .make_query('"foo"', '"t1"."foo"()', 'foo ') .objects()) row, = list(query) self.assertEqual(row.foo, 'v0') self.assertEqual(row.foo_2, 'v1') self.assertEqual(row.foo_3, 'v2') def test_model_objects_flat(self): huey = User.create(username='huey') mickey = User.create(username='mickey') for user, tweet in ((huey, 'meow'), (huey, 'purr'), (mickey, 'woof')): Tweet.create(user=user, content=tweet) query = (Tweet .select(Tweet, User.username) .join(User) .order_by(Tweet.id) .objects()) with self.assertQueryCount(1): self.assertEqual([(t.username, t.content) for t in query], [ ('huey', 'meow'), ('huey', 'purr'), ('mickey', 'woof')]) def test_models(self): huey = User.create(username='huey') mickey = User.create(username='mickey') tids = [] for user, tweet in ((huey, 'meow'), (huey, 'purr'), (mickey, 'woof')): tids.append(Tweet.create(user=user, content=tweet).id) query = (Tweet .select(Tweet, User) .join(User) .order_by(Tweet.id)) with self.assertQueryCount(1): accum = [(t.user.id, t.user.username, t.id, t.content) for t in query] self.assertEqual(accum, [ (huey.id, 'huey', tids[0], 'meow'), (huey.id, 'huey', tids[1], 'purr'), (mickey.id, 'mickey', tids[2], 'woof')]) class Reg(TestModel): key = TextField() ts = DateTimeField() class TestSpecifyConverter(ModelTestCase): requires = [Reg] def test_specify_converter(self): D = lambda d: datetime.datetime(2020, 1, d) for i in range(1, 4): Reg.create(key='k%s' % i, ts=D(i)) RA = Reg.alias() subq = RA.select(RA.key, RA.ts, RA.ts.alias('aliased')) ra_a = subq.c.aliased.alias('aliased') q = (Reg .select(Reg.key, subq.c.ts.alias('ts'), ra_a.converter(Reg.ts.python_value)) .join(subq, on=(Reg.key == subq.c.key).alias('rsub')) .order_by(Reg.key)) results = [(r.key, r.ts, r.aliased) for r in q.objects()] self.assertEqual(results, [ ('k1', D(1), D(1)), ('k2', D(2), D(2)), ('k3', D(3), D(3))]) results2 = [(r.key, r.rsub.ts, r.rsub.aliased) for r in q] self.assertEqual(results, [ ('k1', D(1), D(1)), ('k2', D(2), D(2)), ('k3', D(3), D(3))]) ================================================ FILE: tests/returning.py ================================================ import unittest from peewee import * from peewee import __sqlite_version__ from .base import db from .base import skip_unless from .base import IS_SQLITE from .base import ModelTestCase from .base import TestModel class Reg(TestModel): k = CharField() v = IntegerField() x = IntegerField() class Meta: indexes = ( (('k', 'v'), True), ) returning_support = db.returning_clause or (IS_SQLITE and __sqlite_version__ >= (3, 35, 0)) @skip_unless(returning_support, 'database does not support RETURNING') class TestReturningIntegration(ModelTestCase): requires = [Reg] def test_crud(self): iq = Reg.insert_many([('k1', 1, 0), ('k2', 2, 0)]).returning(Reg) self.assertEqual([(r.id is not None, r.k, r.v) for r in iq.execute()], [(True, 'k1', 1), (True, 'k2', 2)]) iq = (Reg .insert_many([('k1', 1, 1), ('k2', 2, 1), ('k3', 3, 0)]) .on_conflict( conflict_target=[Reg.k, Reg.v], preserve=[Reg.x], update={Reg.v: Reg.v + 1}, where=(Reg.k != 'k1')) .returning(Reg)) ic = iq.execute() self.assertEqual([(r.id is not None, r.k, r.v, r.x) for r in ic], [ (True, 'k2', 3, 1), (True, 'k3', 3, 0)]) uq = (Reg .update({Reg.v: Reg.v - 1, Reg.x: Reg.x + 1}) .where(Reg.k != 'k1') .returning(Reg)) self.assertEqual([(r.k, r.v, r.x) for r in uq.execute()], [ ('k2', 2, 2), ('k3', 2, 1)]) dq = Reg.delete().where(Reg.k != 'k1').returning(Reg) self.assertEqual([(r.k, r.v, r.x) for r in dq.execute()], [ ('k2', 2, 2), ('k3', 2, 1)]) def test_returning_expression(self): Rs = (Reg.v + Reg.x).alias('s') iq = (Reg .insert_many([('k1', 1, 10), ('k2', 2, 20)]) .returning(Reg.k, Reg.v, Rs)) self.assertEqual([(r.k, r.v, r.s) for r in iq.execute()], [ ('k1', 1, 11), ('k2', 2, 22)]) uq = (Reg .update({Reg.k: Reg.k + 'x', Reg.v: Reg.v + 1}) .returning(Reg.k, Reg.v, Rs)) self.assertEqual([(r.k, r.v, r.s) for r in uq.execute()], [ ('k1x', 2, 12), ('k2x', 3, 23)]) dq = Reg.delete().returning(Reg.k, Reg.v, Rs) self.assertEqual([(r.k, r.v, r.s) for r in dq.execute()], [ ('k1x', 2, 12), ('k2x', 3, 23)]) def test_returning_types(self): Rs = (Reg.v + Reg.x).alias('s') mapping = ( ((lambda q: q), (lambda r: (r.k, r.v, r.s))), ((lambda q: q.dicts()), (lambda r: (r['k'], r['v'], r['s']))), ((lambda q: q.tuples()), (lambda r: r)), ((lambda q: q.namedtuples()), (lambda r: (r.k, r.v, r.s)))) for qconv, r2t in mapping: iq = (Reg .insert_many([('k1', 1, 10), ('k2', 2, 20)]) .returning(Reg.k, Reg.v, Rs)) self.assertEqual([r2t(r) for r in qconv(iq).execute()], [ ('k1', 1, 11), ('k2', 2, 22)]) uq = (Reg .update({Reg.k: Reg.k + 'x', Reg.v: Reg.v + 1}) .returning(Reg.k, Reg.v, Rs)) self.assertEqual([r2t(r) for r in qconv(uq).execute()], [ ('k1x', 2, 12), ('k2x', 3, 23)]) dq = Reg.delete().returning(Reg.k, Reg.v, Rs) self.assertEqual([r2t(r) for r in qconv(dq).execute()], [ ('k1x', 2, 12), ('k2x', 3, 23)]) ================================================ FILE: tests/schema.py ================================================ import datetime from peewee import * from peewee import NodeList from .base import BaseTestCase from .base import get_in_memory_db from .base import IS_CRDB from .base import IS_SQLITE from .base import ModelDatabaseTestCase from .base import ModelTestCase from .base import TestModel from .base_models import Category from .base_models import Note from .base_models import Person from .base_models import Relationship from .base_models import User class TMUnique(TestModel): data = TextField(unique=True) class TMSequence(TestModel): value = IntegerField(sequence='test_seq') class TMIndexes(TestModel): alpha = IntegerField() beta = IntegerField() gamma = IntegerField() class Meta: indexes = ( (('alpha', 'beta'), True), (('beta', 'gamma'), False)) class TMConstraints(TestModel): data = IntegerField(null=True, constraints=[Check('data < 5')]) value = TextField(collation='NOCASE') added = DateTimeField(constraints=[Default('CURRENT_TIMESTAMP')]) class TMNamedConstraints(TestModel): fk = ForeignKeyField('self', null=True, constraint_name='tmc_fk') k = TextField() v = IntegerField(constraints=[Check('v in (1, 2)')]) class Meta: constraints = [Check('k != \'kx\'', name='chk_k')] class CacheData(TestModel): key = TextField(unique=True) value = TextField() class Meta: schema = 'cache' class Article(TestModel): name = TextField(unique=True) timestamp = TimestampField() status = IntegerField() flags = IntegerField() Article.add_index(Article.timestamp.desc(), Article.status) idx = (Article .index(Article.name, Article.timestamp, Article.flags.bin_and(4)) .where(Article.status == 1)) Article.add_index(idx) Article.add_index(SQL('CREATE INDEX "article_foo" ON "article" ("flags" & 3)')) class TestModelDDL(ModelDatabaseTestCase): database = get_in_memory_db() requires = [Article, CacheData, Category, Note, Person, Relationship, TMUnique, TMSequence, TMIndexes, TMConstraints, TMNamedConstraints, User] def test_database_required(self): class MissingDB(Model): data = TextField() self.assertRaises(ImproperlyConfigured, MissingDB.create_table) def assertCreateTable(self, model_class, expected): sql, params = model_class._schema._create_table(False).query() self.assertEqual(params, []) indexes = [] for create_index in model_class._schema._create_indexes(False): isql, params = create_index.query() self.assertEqual(params, []) indexes.append(isql) self.assertEqual([sql] + indexes, expected) def assertIndexes(self, model_class, expected): indexes = [] for create_index in model_class._schema._create_indexes(False): indexes.append(create_index.query()) self.assertEqual(indexes, expected) def test_model_fk_schema(self): class Base(TestModel): class Meta: database = self.database class User(Base): username = TextField() class Meta: schema = 'foo' class Tweet(Base): user = ForeignKeyField(User) content = TextField() class Meta: schema = 'bar' self.assertCreateTable(User, [ ('CREATE TABLE "foo"."user" ("id" INTEGER NOT NULL PRIMARY KEY, ' '"username" TEXT NOT NULL)')]) self.assertCreateTable(Tweet, [ ('CREATE TABLE "bar"."tweet" ("id" INTEGER NOT NULL PRIMARY KEY, ' '"user_id" INTEGER NOT NULL, "content" TEXT NOT NULL, ' 'FOREIGN KEY ("user_id") REFERENCES "foo"."user" ("id"))'), ('CREATE INDEX "bar"."tweet_user_id" ON "tweet" ("user_id")')]) def test_bigauto_and_fk(self): class CustomDB(SqliteDatabase): field_types = { 'BIGAUTO': 'BIGAUTO', 'BIGINT': 'BIGINT'} db = CustomDB(None) class User(db.Model): id = BigAutoField() class Tweet(db.Model): user = ForeignKeyField(User) self.assertCreateTable(User, [ ('CREATE TABLE "user" ("id" BIGAUTO NOT NULL PRIMARY KEY)')]) self.assertCreateTable(Tweet, [ ('CREATE TABLE "tweet" ("id" INTEGER NOT NULL PRIMARY KEY, ' '"user_id" BIGINT NOT NULL, FOREIGN KEY ("user_id") REFERENCES ' '"user" ("id"))'), ('CREATE INDEX "tweet_user_id" ON "tweet" ("user_id")')]) def test_model_indexes_with_schema(self): # Attach cache database so we can reference "cache." as the schema. self.database.execute_sql("attach database ':memory:' as cache;") self.assertCreateTable(CacheData, [ ('CREATE TABLE "cache"."cache_data" (' '"id" INTEGER NOT NULL PRIMARY KEY, "key" TEXT NOT NULL, ' '"value" TEXT NOT NULL)'), ('CREATE UNIQUE INDEX "cache"."cache_data_key" ON "cache_data" ' '("key")')]) # Actually create the table to verify it works correctly. CacheData.create_table() # Introspect the database and get indexes for the "cache" schema. indexes = self.database.get_indexes('cache_data', 'cache') self.assertEqual(len(indexes), 1) index_metadata = indexes[0] self.assertEqual(index_metadata.name, 'cache_data_key') # Verify the index does not exist in the main schema. self.assertEqual(len(self.database.get_indexes('cache_data')), 0) class TestDatabase(Database): index_schema_prefix = False # When "index_schema_prefix == False", the index name is not prefixed # with the schema, and the schema is referenced via the table name. with CacheData.bind_ctx(TestDatabase(None)): self.assertCreateTable(CacheData, [ ('CREATE TABLE "cache"."cache_data" (' '"id" INTEGER NOT NULL PRIMARY KEY, "key" TEXT NOT NULL, ' '"value" TEXT NOT NULL)'), ('CREATE UNIQUE INDEX "cache_data_key" ON "cache"."cache_data"' ' ("key")')]) def test_model_indexes(self): self.assertIndexes(Article, [ ('CREATE UNIQUE INDEX "article_name" ON "article" ("name")', []), ('CREATE INDEX "article_timestamp_status" ON "article" (' '"timestamp" DESC, "status")', []), ('CREATE INDEX "article_name_timestamp" ON "article" (' '"name", "timestamp", ("flags" & 4)) ' 'WHERE ("status" = 1)', []), ('CREATE INDEX "article_foo" ON "article" ("flags" & 3)', []), ]) def test_model_index_types(self): class Event(TestModel): key = TextField() timestamp = TimestampField(index=True, index_type='BRIN') class Meta: database = self.database self.assertIndexes(Event, [ ('CREATE INDEX "event_timestamp" ON "event" ' 'USING BRIN ("timestamp")', [])]) # Check that we support MySQL-style USING clause. idx, = Event._meta.fields_to_index() self.assertSQL(idx, ( 'CREATE INDEX IF NOT EXISTS "event_timestamp" ' 'USING BRIN ON "event" ("timestamp")'), [], index_using_precedes_table=True) def test_model_indexes_custom_tablename(self): class KV(TestModel): key = TextField() value = TextField() timestamp = TimestampField(index=True) class Meta: database = self.database indexes = ( (('key', 'value'), True), ) table_name = 'kvs' self.assertIndexes(KV, [ ('CREATE INDEX "kvs_timestamp" ON "kvs" ("timestamp")', []), ('CREATE UNIQUE INDEX "kvs_key_value" ON "kvs" ("key", "value")', [])]) def test_model_indexes_computed_columns(self): class FuncIdx(TestModel): a = IntegerField() b = IntegerField() class Meta: database = self.database i = FuncIdx.index(FuncIdx.a, FuncIdx.b, fn.SUM(FuncIdx.a + FuncIdx.b)) FuncIdx.add_index(i) self.assertIndexes(FuncIdx, [ ('CREATE INDEX "func_idx_a_b" ON "func_idx" ' '("a", "b", SUM("a" + "b"))', []), ]) def test_model_indexes_complex_columns(self): class Taxonomy(TestModel): name = CharField() name_class = CharField() class Meta: database = self.database name = NodeList((fn.LOWER(Taxonomy.name), SQL('varchar_pattern_ops'))) index = (Taxonomy .index(name, Taxonomy.name_class) .where(Taxonomy.name_class == 'scientific name')) Taxonomy.add_index(index) self.assertIndexes(Taxonomy, [ ('CREATE INDEX "taxonomy_name_class" ON "taxonomy" (' 'LOWER("name") varchar_pattern_ops, "name_class") ' 'WHERE ("name_class" = ?)', ['scientific name']), ]) def test_legacy_model_table_and_indexes(self): class Base(Model): class Meta: database = self.database class WebHTTPRequest(Base): timestamp = DateTimeField(index=True) data = TextField() self.assertTrue(WebHTTPRequest._meta.legacy_table_names) self.assertCreateTable(WebHTTPRequest, [ ('CREATE TABLE "webhttprequest" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"timestamp" DATETIME NOT NULL, "data" TEXT NOT NULL)'), ('CREATE INDEX "webhttprequest_timestamp" ON "webhttprequest" ' '("timestamp")')]) # Table name is explicit, but legacy table names == false, so we get # the new index name format. class FooBar(Base): data = IntegerField(unique=True) class Meta: legacy_table_names = False table_name = 'foobar_tbl' self.assertFalse(FooBar._meta.legacy_table_names) self.assertCreateTable(FooBar, [ ('CREATE TABLE "foobar_tbl" ("id" INTEGER NOT NULL PRIMARY KEY, ' '"data" INTEGER NOT NULL)'), ('CREATE UNIQUE INDEX "foobar_tbl_data" ON "foobar_tbl" ("data")'), ]) # Table name is explicit and legacy table names == true, so we get # the old index name format. class FooBar2(Base): data = IntegerField(unique=True) class Meta: table_name = 'foobar2_tbl' self.assertTrue(FooBar2._meta.legacy_table_names) self.assertCreateTable(FooBar2, [ ('CREATE TABLE "foobar2_tbl" ("id" INTEGER NOT NULL PRIMARY KEY, ' '"data" INTEGER NOT NULL)'), ('CREATE UNIQUE INDEX "foobar2_data" ON "foobar2_tbl" ("data")')]) def test_without_pk(self): class NoPK(TestModel): data = TextField() class Meta: database = self.database primary_key = False self.assertCreateTable(NoPK, [ ('CREATE TABLE "no_pk" ("data" TEXT NOT NULL)')]) def test_without_rowid(self): class NoRowid(TestModel): key = TextField(primary_key=True) value = TextField() class Meta: database = self.database without_rowid = True self.assertCreateTable(NoRowid, [ ('CREATE TABLE "no_rowid" (' '"key" TEXT NOT NULL PRIMARY KEY, ' '"value" TEXT NOT NULL) WITHOUT ROWID')]) # Subclasses do not inherit "without_rowid" setting. class SubNoRowid(NoRowid): pass self.assertCreateTable(SubNoRowid, [ ('CREATE TABLE "sub_no_rowid" (' '"key" TEXT NOT NULL PRIMARY KEY, ' '"value" TEXT NOT NULL)')]) def test_strict_tables(self): class Strict(TestModel): key = TextField(primary_key=True) value = TextField() class Meta: database = self.database strict_tables = True self.assertCreateTable(Strict, [ ('CREATE TABLE "strict" (' '"key" TEXT NOT NULL PRIMARY KEY, ' '"value" TEXT NOT NULL) STRICT')]) # Subclasses *do* inherit "strict_tables" setting. class SubStrict(Strict): pass self.assertCreateTable(SubStrict, [ ('CREATE TABLE "sub_strict" (' '"key" TEXT NOT NULL PRIMARY KEY, ' '"value" TEXT NOT NULL) STRICT')]) def test_without_rowid_strict(self): class KV(TestModel): key = TextField(primary_key=True) class Meta: database = self.database strict_tables = True without_rowid = True self.assertCreateTable(KV, [ ('CREATE TABLE "kv" ("key" TEXT NOT NULL PRIMARY KEY) ' 'STRICT, WITHOUT ROWID')]) class SKV(KV): pass self.assertCreateTable(SKV, [ ('CREATE TABLE "skv" ("key" TEXT NOT NULL PRIMARY KEY) STRICT')]) def test_table_name(self): class A(TestModel): class Meta: database = self.database table_name = 'A_tbl' class B(TestModel): a = ForeignKeyField(A, backref='bs') class Meta: database = self.database table_name = 'B_tbl' self.assertCreateTable(A, [ 'CREATE TABLE "A_tbl" ("id" INTEGER NOT NULL PRIMARY KEY)']) self.assertCreateTable(B, [ ('CREATE TABLE "B_tbl" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"a_id" INTEGER NOT NULL, ' 'FOREIGN KEY ("a_id") REFERENCES "A_tbl" ("id"))'), 'CREATE INDEX "B_tbl_a_id" ON "B_tbl" ("a_id")']) def test_temporary_table(self): sql, params = User._schema._create_table(temporary=True).query() self.assertEqual(sql, ( 'CREATE TEMPORARY TABLE IF NOT EXISTS "users" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"username" VARCHAR(255) NOT NULL)')) def test_model_temporary_table(self): class TempUser(User): class Meta: temporary = True self.reset_sql_history() TempUser.create_table() TempUser.drop_table() queries = [x.msg for x in self.history] self.assertEqual(queries, [ ('CREATE TEMPORARY TABLE IF NOT EXISTS "temp_user" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"username" VARCHAR(255) NOT NULL)', []), ('DROP TABLE IF EXISTS "temp_user"', [])]) def test_drop_table(self): sql, params = User._schema._drop_table().query() self.assertEqual(sql, 'DROP TABLE IF EXISTS "users"') sql, params = User._schema._drop_table(cascade=True).query() self.assertEqual(sql, 'DROP TABLE IF EXISTS "users" CASCADE') sql, params = User._schema._drop_table(restrict=True).query() self.assertEqual(sql, 'DROP TABLE IF EXISTS "users" RESTRICT') def test_table_constraints(self): class UKV(TestModel): key = TextField() value = TextField() status = IntegerField() class Meta: constraints = [ SQL('CONSTRAINT ukv_kv_uniq UNIQUE (key, value)'), Check('status > 0')] database = self.database table_name = 'ukv' self.assertCreateTable(UKV, [ ('CREATE TABLE "ukv" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"key" TEXT NOT NULL, ' '"value" TEXT NOT NULL, ' '"status" INTEGER NOT NULL, ' 'CONSTRAINT ukv_kv_uniq UNIQUE (key, value), ' 'CHECK (status > 0))')]) def test_table_settings(self): class KVSettings(TestModel): key = TextField(primary_key=True) value = TextField() timestamp = TimestampField() class Meta: database = self.database table_settings = ('PARTITION BY RANGE (timestamp)', 'WITHOUT ROWID') self.assertCreateTable(KVSettings, [ ('CREATE TABLE "kv_settings" (' '"key" TEXT NOT NULL PRIMARY KEY, ' '"value" TEXT NOT NULL, ' '"timestamp" INTEGER NOT NULL) ' 'PARTITION BY RANGE (timestamp) ' 'WITHOUT ROWID')]) def test_table_options(self): class TOpts(TestModel): key = TextField() class Meta: database = self.database options = { 'CHECKSUM': 1, 'COMPRESSION': 'lz4'} self.assertCreateTable(TOpts, [ ('CREATE TABLE "t_opts" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"key" TEXT NOT NULL, ' 'CHECKSUM=1, COMPRESSION=lz4)')]) def test_table_and_index_creation(self): self.assertCreateTable(Person, [ ('CREATE TABLE "person" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"first" VARCHAR(255) NOT NULL, ' '"last" VARCHAR(255) NOT NULL, ' '"dob" DATE NOT NULL)'), 'CREATE INDEX "person_dob" ON "person" ("dob")', ('CREATE UNIQUE INDEX "person_first_last" ON ' '"person" ("first", "last")')]) self.assertCreateTable(Note, [ ('CREATE TABLE "note" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"author_id" INTEGER NOT NULL, ' '"content" TEXT NOT NULL, ' 'FOREIGN KEY ("author_id") REFERENCES "person" ("id"))'), 'CREATE INDEX "note_author_id" ON "note" ("author_id")']) self.assertCreateTable(Category, [ ('CREATE TABLE "category" (' '"name" VARCHAR(20) NOT NULL PRIMARY KEY, ' '"parent_id" VARCHAR(20), ' 'FOREIGN KEY ("parent_id") REFERENCES "category" ("name"))'), 'CREATE INDEX "category_parent_id" ON "category" ("parent_id")']) self.assertCreateTable(Relationship, [ ('CREATE TABLE "relationship" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"from_person_id" INTEGER NOT NULL, ' '"to_person_id" INTEGER NOT NULL, ' 'FOREIGN KEY ("from_person_id") REFERENCES "person" ("id"), ' 'FOREIGN KEY ("to_person_id") REFERENCES "person" ("id"))'), ('CREATE INDEX "relationship_from_person_id" ' 'ON "relationship" ("from_person_id")'), ('CREATE INDEX "relationship_to_person_id" ' 'ON "relationship" ("to_person_id")')]) self.assertCreateTable(TMUnique, [ ('CREATE TABLE "tm_unique" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"data" TEXT NOT NULL)'), 'CREATE UNIQUE INDEX "tm_unique_data" ON "tm_unique" ("data")']) self.assertCreateTable(TMSequence, [ ('CREATE TABLE "tm_sequence" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"value" INTEGER NOT NULL DEFAULT NEXTVAL(\'test_seq\'))')]) self.assertCreateTable(TMIndexes, [ ('CREATE TABLE "tm_indexes" ("id" INTEGER NOT NULL PRIMARY KEY, ' '"alpha" INTEGER NOT NULL, "beta" INTEGER NOT NULL, ' '"gamma" INTEGER NOT NULL)'), ('CREATE UNIQUE INDEX "tm_indexes_alpha_beta" ' 'ON "tm_indexes" ("alpha", "beta")'), ('CREATE INDEX "tm_indexes_beta_gamma" ' 'ON "tm_indexes" ("beta", "gamma")')]) self.assertCreateTable(TMConstraints, [ ('CREATE TABLE "tm_constraints" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"data" INTEGER CHECK (data < 5), ' '"value" TEXT NOT NULL COLLATE NOCASE, ' '"added" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP)')]) self.assertCreateTable(TMNamedConstraints, [ ('CREATE TABLE "tm_named_constraints" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"fk_id" INTEGER, ' '"k" TEXT NOT NULL, ' '"v" INTEGER NOT NULL ' 'CHECK (v in (1, 2)), ' 'CONSTRAINT "tmc_fk" FOREIGN KEY ("fk_id") ' 'REFERENCES "tm_named_constraints" ("id"), ' 'CONSTRAINT "chk_k" CHECK (k != \'kx\'))'), ('CREATE INDEX "tm_named_constraints_fk_id" ' 'ON "tm_named_constraints" ("fk_id")')]) sql, params = (TMNamedConstraints ._schema ._create_foreign_key(TMNamedConstraints.fk) .query()) self.assertEqual(sql, ( 'ALTER TABLE "tm_named_constraints" ADD CONSTRAINT "tmc_fk" ' 'FOREIGN KEY ("fk_id") REFERENCES "tm_named_constraints" ("id")')) def test_index_name_truncation(self): class LongIndex(TestModel): a123456789012345678901234567890 = CharField() b123456789012345678901234567890 = CharField() c123456789012345678901234567890 = CharField() class Meta: database = self.database fields = LongIndex._meta.sorted_fields[1:] self.assertEqual(len(fields), 3) idx = ModelIndex(LongIndex, fields) ctx = LongIndex._schema._create_index(idx) self.assertSQL(ctx, ( 'CREATE INDEX IF NOT EXISTS "' 'long_index_a123456789012345678901234567890_b123456789012_9dd2139' '" ON "long_index" (' '"a123456789012345678901234567890", ' '"b123456789012345678901234567890", ' '"c123456789012345678901234567890")'), []) def test_fk_non_pk_ddl(self): class A(Model): cf = CharField(max_length=100, unique=True) df = DecimalField( max_digits=4, decimal_places=2, auto_round=True, unique=True) class Meta: database = self.database class CF(TestModel): a = ForeignKeyField(A, field='cf') class Meta: database = self.database class DF(TestModel): a = ForeignKeyField(A, field='df') class Meta: database = self.database sql, params = CF._schema._create_table(safe=False).query() self.assertEqual(sql, ( 'CREATE TABLE "cf" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"a_id" VARCHAR(100) NOT NULL, ' 'FOREIGN KEY ("a_id") REFERENCES "a" ("cf"))')) sql, params = DF._schema._create_table(safe=False).query() self.assertEqual(sql, ( 'CREATE TABLE "df" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"a_id" DECIMAL(4, 2) NOT NULL, ' 'FOREIGN KEY ("a_id") REFERENCES "a" ("df"))')) def test_deferred_foreign_key(self): class Language(TestModel): name = CharField() selected_snippet = DeferredForeignKey('Snippet', null=True) class Meta: database = self.database class Snippet(TestModel): code = TextField() language = ForeignKeyField(Language, backref='snippets') class Meta: database = self.database self.assertEqual(Snippet._meta.fields['language'].rel_model, Language) self.assertEqual(Language._meta.fields['selected_snippet'].rel_model, Snippet) sql, params = Snippet._schema._create_table(safe=False).query() self.assertEqual(sql, ( 'CREATE TABLE "snippet" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"code" TEXT NOT NULL, ' '"language_id" INTEGER NOT NULL, ' 'FOREIGN KEY ("language_id") REFERENCES "language" ("id"))')) sql, params = Language._schema._create_table(safe=False).query() self.assertEqual(sql, ( 'CREATE TABLE "language" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"name" VARCHAR(255) NOT NULL, ' '"selected_snippet_id" INTEGER)')) sql, params = (Language ._schema ._create_foreign_key(Language.selected_snippet) .query()) self.assertEqual(sql, ( 'ALTER TABLE "language" ADD CONSTRAINT ' '"fk_language_selected_snippet_id_refs_snippet" ' 'FOREIGN KEY ("selected_snippet_id") REFERENCES "snippet" ("id")')) class SnippetComment(TestModel): snippet_long_foreign_key_identifier = ForeignKeyField(Snippet) comment = TextField() class Meta: database = self.database sql, params = SnippetComment._schema._create_table(safe=True).query() self.assertEqual(sql, ( 'CREATE TABLE IF NOT EXISTS "snippet_comment" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"snippet_long_foreign_key_identifier_id" INTEGER NOT NULL, ' '"comment" TEXT NOT NULL, ' 'FOREIGN KEY ("snippet_long_foreign_key_identifier_id") ' 'REFERENCES "snippet" ("id"))')) sql, params = (SnippetComment._schema ._create_foreign_key( SnippetComment.snippet_long_foreign_key_identifier) .query()) self.assertEqual(sql, ( 'ALTER TABLE "snippet_comment" ADD CONSTRAINT "' 'fk_snippet_comment_snippet_long_foreign_key_identifier_i_2a8b87d"' ' FOREIGN KEY ("snippet_long_foreign_key_identifier_id") ' 'REFERENCES "snippet" ("id")')) def test_deferred_foreign_key_inheritance(self): class Base(TestModel): class Meta: database = self.database class WithTimestamp(Base): timestamp = TimestampField() class Tweet(Base): user = DeferredForeignKey('DUser') content = TextField() class TimestampTweet(Tweet, WithTimestamp): pass class DUser(Base): username = TextField() sql, params = Tweet._schema._create_table(safe=False).query() self.assertEqual(sql, ( 'CREATE TABLE "tweet" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"content" TEXT NOT NULL, ' '"user_id" INTEGER NOT NULL)')) sql, params = TimestampTweet._schema._create_table(safe=False).query() self.assertEqual(sql, ( 'CREATE TABLE "timestamp_tweet" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"timestamp" INTEGER NOT NULL, ' '"content" TEXT NOT NULL, ' '"user_id" INTEGER NOT NULL)')) def test_identity_field(self): class PG10Identity(TestModel): id = IdentityField() data = TextField() class Meta: database = self.database self.assertCreateTable(PG10Identity, [ ('CREATE TABLE "pg10_identity" (' '"id" INT GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, ' '"data" TEXT NOT NULL)'), ]) def test_self_fk_inheritance(self): class BaseCategory(TestModel): parent = ForeignKeyField('self', backref='children') class Meta: database = self.database class CatA1(BaseCategory): name_a1 = TextField() class CatA2(CatA1): name_a2 = TextField() self.assertTrue(CatA1.parent.rel_model is CatA1) self.assertTrue(CatA2.parent.rel_model is CatA2) self.assertCreateTable(CatA1, [ ('CREATE TABLE "cat_a1" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"parent_id" INTEGER NOT NULL, ' '"name_a1" TEXT NOT NULL, ' 'FOREIGN KEY ("parent_id") REFERENCES "cat_a1" ("id"))'), ('CREATE INDEX "cat_a1_parent_id" ON "cat_a1" ("parent_id")')]) self.assertCreateTable(CatA2, [ ('CREATE TABLE "cat_a2" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"parent_id" INTEGER NOT NULL, ' '"name_a1" TEXT NOT NULL, ' '"name_a2" TEXT NOT NULL, ' 'FOREIGN KEY ("parent_id") REFERENCES "cat_a2" ("id"))'), ('CREATE INDEX "cat_a2_parent_id" ON "cat_a2" ("parent_id")')]) def test_field_ddl(self): class Base(self.database.Model): pass class FC(Base): code = FixedCharField(max_length=5) name = CharField() class Dbl(Base): value = DoubleField() label = CharField() class SmInt(Base): value = SmallIntegerField() label = CharField() self.assertSQL(FC._schema._create_table(False), ( 'CREATE TABLE "fc" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"code" CHAR(5) NOT NULL, ' '"name" VARCHAR(255) NOT NULL)'), []) self.assertSQL(Dbl._schema._create_table(False), ( 'CREATE TABLE "dbl" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"value" REAL NOT NULL, ' '"label" VARCHAR(255) NOT NULL)'), []) self.assertSQL(SmInt._schema._create_table(False), ( 'CREATE TABLE "smint" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"value" INTEGER NOT NULL, ' '"label" VARCHAR(255) NOT NULL)'), []) class NoteX(TestModel): content = TextField() timestamp = TimestampField() status = IntegerField() flags = IntegerField() class TestCreateAs(ModelTestCase): requires = [NoteX] test_data = ( # name, timestamp, status, flags. (1, 'n1', datetime.datetime(2019, 1, 1), 1, 1), (2, 'n2', datetime.datetime(2019, 1, 2), 2, 1), (3, 'n3', datetime.datetime(2019, 1, 3), 9, 1), (4, 'nx', datetime.datetime(2019, 1, 1), 9, 0)) def setUp(self): super(TestCreateAs, self).setUp() fields = NoteX._meta.sorted_fields NoteX.insert_many(self.test_data, fields=fields).execute() def tearDown(self): class Note2(TestModel): class Meta: database = self.database self.database.drop_tables([Note2]) super(TestCreateAs, self).tearDown() def test_create_as(self): status = Case(NoteX.status, ( (1, 'published'), (2, 'draft'), (9, 'deleted'))) query = (NoteX .select(NoteX.id, NoteX.content, NoteX.timestamp, status.alias('status')) .where(NoteX.flags == SQL('1'))) query.create_table('note2') class Note2(TestModel): id = IntegerField() content = TextField() timestamp = TimestampField() status = TextField() class Meta: database = self.database query = Note2.select().order_by(Note2.id) self.assertEqual(list(query.tuples()), [ (1, 'n1', datetime.datetime(2019, 1, 1), 'published'), (2, 'n2', datetime.datetime(2019, 1, 2), 'draft'), (3, 'n3', datetime.datetime(2019, 1, 3), 'deleted')]) class TestModelSetTableName(BaseTestCase): def test_set_table_name(self): class Foo(TestModel): pass self.assertEqual(Foo._meta.table_name, 'foo') self.assertEqual(Foo._meta.table.__name__, 'foo') # Writing the attribute directly does not update the cached Table name. Foo._meta.table_name = 'foo2' self.assertEqual(Foo._meta.table.__name__, 'foo') # Use the helper-method. Foo._meta.set_table_name('foo3') self.assertEqual(Foo._meta.table.__name__, 'foo3') class TestTruncateTable(ModelTestCase): requires = [User] def test_truncate_table(self): for i in range(3): User.create(username='u%s' % i) ctx = User._schema._truncate_table() if IS_SQLITE: self.assertSQL(ctx, 'DELETE FROM "users"', []) else: sql, _ = ctx.query() self.assertTrue(sql.startswith('TRUNCATE TABLE ')) User.truncate_table() self.assertEqual(User.select().count(), 0) class TestNamedConstraintsIntegration(ModelTestCase): requires = [TMNamedConstraints] def setUp(self): super(TestNamedConstraintsIntegration, self).setUp() if IS_SQLITE: self.database.pragma('foreign_keys', 'on') def test_named_constraints_integration(self): t = TMNamedConstraints.create(k='k1', v=1) # Sanity test. fails = [ {'fk': t.id - 1, 'k': 'k2', 'v': 1}, # Invalid fk. {'fk': t.id, 'k': 'k3', 'v': 0}, # Invalid val. {'fk': t.id, 'k': 'kx', 'v': 1}] # Invalid key. for f in fails: # MySQL may use OperationalError. with self.assertRaises((IntegrityError, OperationalError)): with self.database.atomic() as tx: TMNamedConstraints.create(**f) self.assertEqual(len(TMNamedConstraints), 1) class TMKV(TestModel): key = CharField() value = IntegerField() extra = IntegerField() class TMKVNew(TestModel): key = CharField() val = IntegerField() class Meta: primary_key = False table_name = 'tmkv_new' class TestCreateTableAsSQL(ModelDatabaseTestCase): database = get_in_memory_db() requires = [TMKV] def test_create_table_as_sql(self): query = (TMKV .select(TMKV.key, TMKV.value.alias('val')) .where(TMKV.extra < 4)) ctx = TMKV._schema._create_table_as('tmkv_new', query) self.assertSQL(ctx, ( 'CREATE TABLE IF NOT EXISTS "tmkv_new" AS ' 'SELECT "t1"."key", "t1"."value" AS "val" FROM "tmkv" AS "t1" ' 'WHERE ("t1"."extra" < ?)'), [4]) ctx = TMKV._schema._create_table_as(('alt', 'tmkv_new'), query) self.assertSQL(ctx, ( 'CREATE TABLE IF NOT EXISTS "alt"."tmkv_new" AS ' 'SELECT "t1"."key", "t1"."value" AS "val" FROM "tmkv" AS "t1" ' 'WHERE ("t1"."extra" < ?)'), [4]) class TestCreateTableAs(ModelTestCase): requires = [TMKV] def tearDown(self): try: TMKVNew.drop_table(safe=True) except: pass super(TestCreateTableAs, self).tearDown() def test_create_table_as(self): TMKV.insert_many([('k%02d' % i, i, i) for i in range(10)]).execute() query = (TMKV .select(TMKV.key, TMKV.value.alias('val')) .where(TMKV.extra < 4)) query.create_table('tmkv_new', safe=True) expected = ['key', 'val'] if IS_CRDB: expected.append('rowid') # CRDB adds this. self.assertEqual( [col.name for col in self.database.get_columns('tmkv_new')], expected) query = TMKVNew.select().order_by(TMKVNew.key) self.assertEqual([(r.key, r.val) for r in query], [('k00', 0), ('k01', 1), ('k02', 2), ('k03', 3)]) ================================================ FILE: tests/shortcuts.py ================================================ import operator from peewee import * from playhouse.shortcuts import * from .base import BaseTestCase from .base import DatabaseTestCase from .base import ModelTestCase from .base import TestModel from .base import db_loader from .base import get_in_memory_db from .base import requires_models from .base import requires_mysql from .base_models import Category class User(TestModel): username = TextField() @property def name_hash(self): return sum(map(ord, self.username)) % 10 class Tweet(TestModel): user = ForeignKeyField(User, backref='tweets') content = TextField() class Tag(TestModel): tag = TextField() class TweetTag(TestModel): tweet = ForeignKeyField(Tweet) tag = ForeignKeyField(Tag) class Meta: primary_key = CompositeKey('tweet', 'tag') class Owner(TestModel): name = TextField() class Label(TestModel): label = TextField() class Gallery(TestModel): name = TextField() labels = ManyToManyField(Label, backref='galleries') owner = ForeignKeyField(Owner, backref='galleries') GalleryLabel = Gallery.labels.through_model class Student(TestModel): name = TextField() StudentCourseProxy = DeferredThroughModel() class Course(TestModel): name = TextField() students = ManyToManyField(Student, through_model=StudentCourseProxy, backref='courses') class StudentCourse(TestModel): student = ForeignKeyField(Student) course = ForeignKeyField(Course) StudentCourseProxy.set_model(StudentCourse) class Host(TestModel): name = TextField() class Service(TestModel): host = ForeignKeyField(Host, backref='services') name = TextField() class Device(TestModel): host = ForeignKeyField(Host, backref='+') name = TextField() class Basket(TestModel): id = IntegerField(primary_key=True) class Item(TestModel): id = IntegerField(primary_key=True) basket = ForeignKeyField(Basket) class NodeTag(TestModel): tag = TextField() class Node(TestModel): name = TextField() tag = ForeignKeyField(NodeTag) parent = ForeignKeyField('self', null=True, backref='children') class TestModelToDict(ModelTestCase): database = get_in_memory_db() requires = [User, Tweet, Tag, TweetTag] def setUp(self): super(TestModelToDict, self).setUp() self.user = User.create(username='peewee') @requires_models(Node, NodeTag) def test_self_referential(self): a, b = [NodeTag.create(tag=tag) for tag in 'ab'] root = Node.create(name='root', tag=a) n1 = Node.create(name='n1', parent=root, tag=a) n2 = Node.create(name='n2', parent=root, tag=b) Parent = Node.alias('parent') ParentTag = NodeTag.alias('parent_tag') def assertSerialization(n, expected): obj = (Node .select(Node, NodeTag, Parent, ParentTag) .join_from(Node, NodeTag, JOIN.LEFT_OUTER) .join_from(Node, Parent, JOIN.LEFT_OUTER) .join_from(Parent, ParentTag, JOIN.LEFT_OUTER) .where(Node.name == n) .first()) self.assertEqual(model_to_dict(obj, recurse=True), expected) assertSerialization('n1', { 'id': n1.id, 'name': 'n1', 'parent': {'id': root.id, 'name': 'root', 'tag': {'id': a.id, 'tag': 'a'}}, 'tag': {'id': a.id, 'tag': 'a'}}) assertSerialization('n2', { 'id': n2.id, 'name': 'n2', 'parent': {'id': root.id, 'name': 'root', 'tag': {'id': a.id, 'tag': 'a'}}, 'tag': {'id': b.id, 'tag': 'b'}}) assertSerialization('root', { 'id': root.id, 'name': 'root', 'parent': None, 'tag': {'id': a.id, 'tag': 'a'}}) def test_simple(self): with self.assertQueryCount(0): self.assertEqual(model_to_dict(self.user), { 'id': self.user.id, 'username': 'peewee'}) def test_simple_recurse(self): tweet = Tweet.create(user=self.user, content='t1') with self.assertQueryCount(0): self.assertEqual(model_to_dict(tweet), { 'id': tweet.id, 'content': tweet.content, 'user': { 'id': self.user.id, 'username': 'peewee'}}) with self.assertQueryCount(0): self.assertEqual(model_to_dict(tweet, recurse=False), { 'id': tweet.id, 'content': tweet.content, 'user': self.user.id}) def test_simple_backref(self): with self.assertQueryCount(1): self.assertEqual(model_to_dict(self.user, backrefs=True), { 'id': self.user.id, 'tweets': [], 'username': 'peewee'}) tweet = Tweet.create(user=self.user, content='t0') # Two queries, one for tweets, one for tweet-tags. with self.assertQueryCount(2): self.assertEqual(model_to_dict(self.user, backrefs=True), { 'id': self.user.id, 'username': 'peewee', 'tweets': [{'id': tweet.id, 'content': 't0', 'tweettag_set': []}]}) def test_recurse_and_backrefs(self): tweet = Tweet.create(user=self.user, content='t0') with self.assertQueryCount(1): self.assertEqual(model_to_dict(tweet, backrefs=True), { 'id': tweet.id, 'content': 't0', 'tweettag_set': [], 'user': {'id': self.user.id, 'username': 'peewee'}}) @requires_models(Category) def test_recursive_fk(self): root = Category.create(name='root') child = Category.create(name='child', parent=root) grandchild = Category.create(name='grandchild', parent=child) with self.assertQueryCount(0): for recurse in (True, False): self.assertEqual(model_to_dict(root, recurse=recurse), { 'name': 'root', 'parent': None}) with self.assertQueryCount(1): self.assertEqual(model_to_dict(root, backrefs=True), { 'name': 'root', 'parent': None, 'children': [{'name': 'child'}]}) with self.assertQueryCount(1): self.assertEqual(model_to_dict(root, backrefs=True), { 'name': 'root', 'parent': None, 'children': [{'name': 'child'}]}) with self.assertQueryCount(1): self.assertEqual(model_to_dict(child, backrefs=True), { 'name': 'child', 'parent': {'name': 'root'}, 'children': [{'name': 'grandchild'}]}) with self.assertQueryCount(0): self.assertEqual(model_to_dict(child, backrefs=False), { 'name': 'child', 'parent': {'name': 'root'}}) def test_manytomany(self): tweet = Tweet.create(user=self.user, content='t0') tag1 = Tag.create(tag='t1') tag2 = Tag.create(tag='t2') Tag.create(tag='tx') TweetTag.create(tweet=tweet, tag=tag1) TweetTag.create(tweet=tweet, tag=tag2) with self.assertQueryCount(4): self.assertEqual(model_to_dict(self.user, backrefs=True), { 'id': self.user.id, 'username': 'peewee', 'tweets': [{ 'id': tweet.id, 'content': 't0', 'tweettag_set': [ {'tag': {'id': tag1.id, 'tag': 't1'}}, {'tag': {'id': tag2.id, 'tag': 't2'}}]}]}) @requires_models(Label, Gallery, GalleryLabel, Owner) def test_manytomany_field(self): data = ( ('charlie', 'family', ('nuggie', 'bearbe')), ('charlie', 'pets', ('huey', 'zaizee', 'beanie')), ('peewee', 'misc', ('nuggie', 'huey'))) for owner_name, gallery, labels in data: owner, _ = Owner.get_or_create(name=owner_name) gallery = Gallery.create(name=gallery, owner=owner) label_objects = [Label.get_or_create(label=l)[0] for l in labels] gallery.labels.add(label_objects) query = (Gallery .select(Gallery, Owner) .join(Owner) .switch(Gallery) .join(GalleryLabel) .join(Label) .where(Label.label == 'nuggie') .order_by(Gallery.id)) rows = [model_to_dict(gallery, backrefs=True, manytomany=True) for gallery in query] self.assertEqual(rows, [ { 'id': 1, 'name': 'family', 'owner': {'id': 1, 'name': 'charlie'}, 'labels': [{'id': 1, 'label': 'nuggie'}, {'id': 2, 'label': 'bearbe'}], }, { 'id': 3, 'name': 'misc', 'owner': {'id': 2, 'name': 'peewee'}, 'labels': [{'id': 1, 'label': 'nuggie'}, {'id': 3, 'label': 'huey'}], }]) @requires_models(Student, Course, StudentCourse) def test_manytomany_deferred(self): data = ( ('s1', ('ca', 'cb', 'cc')), ('s2', ('cb', 'cd')), ('s3', ())) c = {} for student, courses in data: s = Student.create(name=student) for course in courses: if course not in c: c[course] = Course.create(name=course) StudentCourse.create(student=s, course=c[course]) query = Student.select().order_by(Student.name) data = [] for user in query: user_dict = model_to_dict(user, manytomany=True) user_dict['courses'].sort(key=operator.itemgetter('id')) data.append(user_dict) self.assertEqual(data, [ {'id': 1, 'name': 's1', 'courses': [ {'id': 1, 'name': 'ca'}, {'id': 2, 'name': 'cb'}, {'id': 3, 'name': 'cc'}]}, {'id': 2, 'name': 's2', 'courses': [ {'id': 2, 'name': 'cb'}, {'id': 4, 'name': 'cd'}]}, {'id': 3, 'name': 's3', 'courses': []}]) query = Course.select().order_by(Course.name) data = [] for course in query: course_dict = model_to_dict(course, manytomany=True) course_dict['students'].sort(key=operator.itemgetter('id')) data.append(course_dict) self.assertEqual(data, [ {'id': 1, 'name': 'ca', 'students': [ {'id': 1, 'name': 's1'}]}, {'id': 2, 'name': 'cb', 'students': [ {'id': 1, 'name': 's1'}, {'id': 2, 'name': 's2'}]}, {'id': 3, 'name': 'cc', 'students': [ {'id': 1, 'name': 's1'}]}, {'id': 4, 'name': 'cd', 'students': [ {'id': 2, 'name': 's2'}]}]) def test_recurse_max_depth(self): t0, t1, t2 = [Tweet.create(user=self.user, content='t%s' % i) for i in range(3)] tag0, tag1 = [Tag.create(tag=t) for t in ['tag0', 'tag1']] TweetTag.create(tweet=t0, tag=tag0) TweetTag.create(tweet=t0, tag=tag1) TweetTag.create(tweet=t1, tag=tag1) data = model_to_dict(self.user, recurse=True, backrefs=True) self.assertEqual(data, { 'id': self.user.id, 'username': 'peewee', 'tweets': [ {'id': t0.id, 'content': 't0', 'tweettag_set': [ {'tag': {'tag': 'tag0', 'id': tag0.id}}, {'tag': {'tag': 'tag1', 'id': tag1.id}}, ]}, {'id': t1.id, 'content': 't1', 'tweettag_set': [ {'tag': {'tag': 'tag1', 'id': tag1.id}}, ]}, {'id': t2.id, 'content': 't2', 'tweettag_set': []}, ]}) data = model_to_dict(self.user, recurse=True, backrefs=True, max_depth=2) self.assertEqual(data, { 'id': self.user.id, 'username': 'peewee', 'tweets': [ {'id': t0.id, 'content': 't0', 'tweettag_set': [ {'tag': tag0.id}, {'tag': tag1.id}, ]}, {'id': t1.id, 'content': 't1', 'tweettag_set': [ {'tag': tag1.id}, ]}, {'id': t2.id, 'content': 't2', 'tweettag_set': []}, ]}) data = model_to_dict(self.user, recurse=True, backrefs=True, max_depth=1) self.assertEqual(data, { 'id': self.user.id, 'username': 'peewee', 'tweets': [ {'id': t0.id, 'content': 't0'}, {'id': t1.id, 'content': 't1'}, {'id': t2.id, 'content': 't2'}]}) self.assertEqual(model_to_dict(self.user, recurse=True, backrefs=True, max_depth=0), {'id': self.user.id, 'username': 'peewee'}) def test_only(self): username_dict = {'username': 'peewee'} self.assertEqual(model_to_dict(self.user, only=[User.username]), username_dict) self.assertEqual( model_to_dict(self.user, backrefs=True, only=[User.username]), username_dict) tweet = Tweet.create(user=self.user, content='t0') tweet_dict = {'content': 't0', 'user': {'username': 'peewee'}} field_list = [Tweet.content, Tweet.user, User.username] self.assertEqual(model_to_dict(tweet, only=field_list), tweet_dict) self.assertEqual(model_to_dict(tweet, backrefs=True, only=field_list), tweet_dict) tweet_dict['user'] = self.user.id self.assertEqual(model_to_dict(tweet, backrefs=True, recurse=False, only=field_list), tweet_dict) def test_exclude(self): self.assertEqual(model_to_dict(self.user, exclude=[User.id]), {'username': 'peewee'}) # Exclude the foreign key using FK field and backref. self.assertEqual(model_to_dict(self.user, backrefs=True, exclude=[User.id, Tweet.user]), {'username': 'peewee'}) self.assertEqual(model_to_dict(self.user, backrefs=True, exclude=[User.id, User.tweets]), {'username': 'peewee'}) tweet = Tweet.create(user=self.user, content='t0') fields = [Tweet.tweettag_set, Tweet.id, Tweet.user] self.assertEqual(model_to_dict(tweet, backrefs=True, exclude=fields), {'content': 't0'}) fields[-1] = User.id self.assertEqual(model_to_dict(tweet, backrefs=True, exclude=fields), {'content': 't0', 'user': {'username': 'peewee'}}) def test_extra_attrs(self): with self.assertQueryCount(0): extra = ['name_hash'] self.assertEqual(model_to_dict(self.user, extra_attrs=extra), { 'id': self.user.id, 'username': 'peewee', 'name_hash': 5}) with self.assertQueryCount(0): self.assertRaises(AttributeError, model_to_dict, self.user, extra_attrs=['xx']) def test_fields_from_query(self): User.delete().execute() for i in range(3): user = User.create(username='u%d' % i) for x in range(i + 1): Tweet.create(user=user, content='%s-%s' % (user.username, x)) query = (User .select(User.username, fn.COUNT(Tweet.id).alias('ct')) .join(Tweet, JOIN.LEFT_OUTER) .group_by(User.username) .order_by(User.id)) with self.assertQueryCount(1): u0, u1, u2 = list(query) self.assertEqual(model_to_dict(u0, fields_from_query=query), { 'username': 'u0', 'ct': 1}) self.assertEqual(model_to_dict(u2, fields_from_query=query), { 'username': 'u2', 'ct': 3}) query = (Tweet .select(Tweet, User, SQL('1337').alias('magic')) .join(User) .order_by(Tweet.id) .limit(1)) with self.assertQueryCount(1): tweet, = query self.assertEqual(model_to_dict(tweet, fields_from_query=query), { 'id': tweet.id, 'content': 'u0-0', 'magic': 1337, 'user': {'id': tweet.user_id, 'username': 'u0'}}) self.assertEqual(model_to_dict(tweet, fields_from_query=query, exclude=[User.id, Tweet.id]), {'magic': 1337, 'content': 'u0-0', 'user': {'username': 'u0'}}) def test_fields_from_query_alias(self): q = User.select(User.username.alias('name')) res = q[0] self.assertEqual(model_to_dict(res, fields_from_query=q), {'name': 'peewee'}) UA = User.alias() q = UA.select(UA.username.alias('name')) res = q[0] self.assertEqual(model_to_dict(res, fields_from_query=q), {'name': 'peewee'}) def test_only_backref(self): for i in range(3): Tweet.create(user=self.user, content=str(i)) data = model_to_dict(self.user, backrefs=True, only=[ User.username, User.tweets, Tweet.content]) if 'tweets' in data: data['tweets'].sort(key=lambda t: t['content']) self.assertEqual(data, { 'username': 'peewee', 'tweets': [ {'content': '0'}, {'content': '1'}, {'content': '2'}]}) @requires_models(Host, Service, Device) def test_model_to_dict_disabled_backref(self): host = Host.create(name='pi') Device.create(host=host, name='raspberry pi') Service.create(host=host, name='ssh') Service.create(host=host, name='vpn') data = model_to_dict(host, recurse=True, backrefs=True) services = sorted(data.pop('services'), key=operator.itemgetter('id')) self.assertEqual(data, {'id': 1, 'name': 'pi'}) self.assertEqual(services, [ {'id': 1, 'name': 'ssh'}, {'id': 2, 'name': 'vpn'}]) @requires_models(Basket, Item) def test_empty_vs_null_fk(self): b = Basket.create(id=0) i = Item.create(id=0, basket=b) data = model_to_dict(i) self.assertEqual(data, {'id': 0, 'basket': {'id': 0}}) data = model_to_dict(i, recurse=False) self.assertEqual(data, {'id': 0, 'basket': 0}) class TestDictToModel(ModelTestCase): database = get_in_memory_db() requires = [User, Tweet, Tag, TweetTag] def setUp(self): super(TestDictToModel, self).setUp() self.user = User.create(username='peewee') def test_simple(self): data = {'username': 'peewee', 'id': self.user.id} inst = dict_to_model(User, data) self.assertTrue(isinstance(inst, User)) self.assertEqual(inst.username, 'peewee') self.assertEqual(inst.id, self.user.id) def test_update_model_from_dict(self): data = {'content': 'tweet', 'user': {'username': 'zaizee'}} with self.assertQueryCount(0): user = User(id=3, username='orig') tweet = Tweet(id=4, content='orig', user=user) obj = update_model_from_dict(tweet, data) self.assertEqual(obj.id, 4) self.assertEqual(obj.content, 'tweet') self.assertEqual(obj.user.id, 3) self.assertEqual(obj.user.username, 'zaizee') def test_related(self): data = { 'id': 2, 'content': 'tweet-1', 'user': {'id': self.user.id, 'username': 'peewee'}} with self.assertQueryCount(0): inst = dict_to_model(Tweet, data) self.assertTrue(isinstance(inst, Tweet)) self.assertEqual(inst.id, 2) self.assertEqual(inst.content, 'tweet-1') self.assertTrue(isinstance(inst.user, User)) self.assertEqual(inst.user.id, self.user.id) self.assertEqual(inst.user.username, 'peewee') data['user'] = self.user.id with self.assertQueryCount(0): inst = dict_to_model(Tweet, data) with self.assertQueryCount(1): self.assertEqual(inst.user, self.user) def test_backrefs(self): data = { 'id': self.user.id, 'username': 'peewee', 'tweets': [ {'id': 1, 'content': 't1'}, {'id': 2, 'content': 't2'}, ]} with self.assertQueryCount(0): inst = dict_to_model(User, data) self.assertEqual(inst.id, self.user.id) self.assertEqual(inst.username, 'peewee') self.assertTrue(isinstance(inst.tweets, list)) t1, t2 = inst.tweets self.assertEqual(t1.id, 1) self.assertEqual(t1.content, 't1') self.assertEqual(t1.user, self.user) self.assertEqual(t2.id, 2) self.assertEqual(t2.content, 't2') self.assertEqual(t2.user, self.user) def test_unknown_attributes(self): data = { 'id': self.user.id, 'username': 'peewee', 'xx': 'does not exist'} self.assertRaises(AttributeError, dict_to_model, User, data) inst = dict_to_model(User, data, ignore_unknown=True) self.assertEqual(inst.xx, 'does not exist') def test_ignore_id_attribute(self): class Register(Model): key = CharField(primary_key=True) data = {'id': 100, 'key': 'k1'} self.assertRaises(AttributeError, dict_to_model, Register, data) inst = dict_to_model(Register, data, ignore_unknown=True) self.assertEqual(inst.__data__, {'key': 'k1'}) class Base(Model): class Meta: primary_key = False class Register2(Model): key = CharField(primary_key=True) self.assertRaises(AttributeError, dict_to_model, Register2, data) inst = dict_to_model(Register2, data, ignore_unknown=True) self.assertEqual(inst.__data__, {'key': 'k1'}) class ReconnectMySQLDatabase(ReconnectMixin, MySQLDatabase): def cursor(self, named_cursor=None): cursor = super(ReconnectMySQLDatabase, self).cursor(named_cursor) # The first (0th) query fails, as do all queries after the 2nd (1st). if self._query_counter != 1: def _fake_execute(self, *args): raise OperationalError('2006') cursor.execute = _fake_execute self._query_counter += 1 return cursor def close(self): self._close_counter += 1 return super(ReconnectMySQLDatabase, self).close() def _reset_mock(self): self._close_counter = 0 self._query_counter = 0 @requires_mysql class TestReconnectMixin(DatabaseTestCase): database = db_loader('mysql', db_class=ReconnectMySQLDatabase) def test_reconnect_mixin_execute_sql(self): # Verify initial state. self.database._reset_mock() self.assertEqual(self.database._close_counter, 0) sql = 'select 1 + 1' curs = self.database.execute_sql(sql) self.assertEqual(curs.fetchone(), (2,)) self.assertEqual(self.database._close_counter, 1) # Due to how we configured our mock, our queries are now failing and we # can verify a reconnect is occuring *AND* the exception is propagated. self.assertRaises(OperationalError, self.database.execute_sql, sql) self.assertEqual(self.database._close_counter, 2) # We reset the mock counters. The first query we execute will fail. The # second query will succeed (which happens automatically, thanks to the # retry logic). self.database._reset_mock() curs = self.database.execute_sql(sql) self.assertEqual(curs.fetchone(), (2,)) self.assertEqual(self.database._close_counter, 1) def test_reconnect_mixin_begin(self): # Verify initial state. self.database._reset_mock() self.assertEqual(self.database._close_counter, 0) with self.database.atomic(): self.assertTrue(self.database.in_transaction()) self.assertEqual(self.database._close_counter, 1) # Prepare mock for commit call self.database._query_counter = 1 # Due to how we configured our mock, our queries are now failing and we # can verify a reconnect is occuring *AND* the exception is propagated. self.assertRaises(OperationalError, self.database.atomic().__enter__) self.assertEqual(self.database._close_counter, 2) self.assertFalse(self.database.in_transaction()) # We reset the mock counters. The first query we execute will fail. The # second query will succeed (which happens automatically, thanks to the # retry logic). self.database._reset_mock() with self.database.atomic(): self.assertTrue(self.database.in_transaction()) self.assertEqual(self.database._close_counter, 1) # Do not reconnect when nesting transactions self.assertRaises(OperationalError, self.database.atomic().__enter__) self.assertEqual(self.database._close_counter, 1) # Prepare mock for commit call self.database._query_counter = 1 self.assertFalse(self.database.in_transaction()) class MMA(TestModel): key = TextField() value = IntegerField() class MMB(TestModel): key = TextField() class MMC(TestModel): key = TextField() value = IntegerField() misc = TextField(null=True) class TestResolveMultiModelQuery(ModelTestCase): requires = [MMA, MMB, MMC] def test_resolve_multimodel_query(self): MMA.insert_many([('k0', 0), ('k1', 1)]).execute() MMB.insert_many([('k10',), ('k11',)]).execute() MMC.insert_many([('k20', 20, 'a'), ('k21', 21, 'b')]).execute() mma = MMA.select(MMA.key, MMA.value) mmb = MMB.select(MMB.key, Value(99).alias('value')) mmc = MMC.select(MMC.key, MMC.value) query = (mma | mmb | mmc).order_by(SQL('1')) data = [obj for obj in resolve_multimodel_query(query)] expected = [ MMA(key='k0', value=0), MMA(key='k1', value=1), MMB(key='k10', value=99), MMB(key='k11', value=99), MMC(key='k20', value=20), MMC(key='k21', value=21)] self.assertEqual(len(data), len(expected)) for row, exp_row in zip(data, expected): self.assertEqual(row.__class__, exp_row.__class__) self.assertEqual(row.key, exp_row.key) self.assertEqual(row.value, exp_row.value) ts_database = get_in_memory_db() class TSBase(Model): class Meta: database = ts_database model_metadata_class = ThreadSafeDatabaseMetadata class TSReg(TSBase): key = TextField() class TestThreadSafeDatabaseMetadata(BaseTestCase): def setUp(self): super(TestThreadSafeDatabaseMetadata, self).setUp() ts_database.create_tables([TSReg]) def test_threadsafe_database_metadata(self): self.assertTrue(isinstance(TSReg._meta, ThreadSafeDatabaseMetadata)) self.assertEqual(TSReg._meta.database, ts_database) t1 = TSReg.create(key='k1') t1_db = TSReg.get(TSReg.key == 'k1') self.assertEqual(t1.id, t1_db.id) def test_swap_database(self): d1 = get_in_memory_db() d2 = get_in_memory_db() class M(TSBase): pass def swap_db(): self.assertEqual(M._meta.database, ts_database) d1.bind([M]) self.assertEqual(M._meta.database, d1) with d2.bind_ctx([M]): self.assertEqual(M._meta.database, d2) self.assertEqual(M._meta.database, d1) self.assertEqual(M._meta.database, ts_database) # From a separate thread, swap the database and verify it works # correctly. t = threading.Thread(target=swap_db) t.start() ; t.join() # In the main thread the original database has not been altered. self.assertEqual(M._meta.database, ts_database) def test_preserve_original_db(self): outputs = [] d1 = get_in_memory_db() d2 = get_in_memory_db() class M(TSBase): class Meta: database = d1 def swap_db(): self.assertTrue(M._meta.database is d1) with d2.bind_ctx([M]): self.assertTrue(M._meta.database is d2) self.assertTrue(M._meta.database is d1) d2.bind([M]) # Now bind to d2 and leave it bound. self.assertTrue(M._meta.database is d2) # From a separate thread, swap the database and verify it works # correctly. threads = [threading.Thread(target=swap_db) for _ in range(20)] for t in threads: t.start() for t in threads: t.join() # In the main thread the original database has not been altered. self.assertTrue(M._meta.database is d1) class TIW(TestModel): key = CharField() value = IntegerField(default=0) extra = IntegerField(default=lambda: 1) class TestInsertWhere(ModelTestCase): requires = [User, Tweet, TIW] def test_insert_where(self): ua, ub = [User.create(username=n) for n in 'ab'] def _insert_where(user, content): cond = (Tweet.select() .where(Tweet.user == user, Tweet.content == content)) where = ~fn.EXISTS(cond) iq = insert_where(Tweet, { Tweet.user: user, Tweet.content: content}, where=where) return 1 if iq.execute() else 0 self.assertEqual(_insert_where(ua, 't1'), 1) self.assertEqual(_insert_where(ua, 't2'), 1) self.assertEqual(_insert_where(ua, 't1'), 0) self.assertEqual(_insert_where(ua, 't2'), 0) self.assertEqual(_insert_where(ub, 't1'), 1) self.assertEqual(_insert_where(ub, 't2'), 1) def test_insert_where_defaults(self): TIW.create(key='k1', value=1, extra=2) def _insert_where(key): where = ~fn.EXISTS(TIW.select().where(TIW.key == key)) iq = insert_where(TIW, {TIW.key: key}, where) return 1 if iq.execute() else 0 self.assertEqual(_insert_where('k2'), 1) self.assertEqual(_insert_where('k1'), 0) self.assertEqual(_insert_where('k2'), 0) tiw = TIW.get(TIW.key == 'k2') self.assertEqual(tiw.value, 0) self.assertEqual(tiw.extra, 1) ================================================ FILE: tests/signals.py ================================================ from peewee import * from playhouse import signals from .base import get_in_memory_db from .base import ModelTestCase class BaseSignalModel(signals.Model): pass class A(BaseSignalModel): a = TextField(default='') class B(BaseSignalModel): b = TextField(default='') class SubB(B): pass class TestSignals(ModelTestCase): database = get_in_memory_db() requires = [A, B, SubB] def tearDown(self): super(TestSignals, self).tearDown() signals.pre_save._flush() signals.post_save._flush() signals.pre_delete._flush() signals.post_delete._flush() signals.pre_init._flush() def test_pre_save(self): state = [] @signals.pre_save() def pre_save(sender, instance, created): state.append((sender, instance, instance._pk, created)) a = A() self.assertEqual(a.save(), 1) self.assertEqual(state, [(A, a, None, True)]) self.assertEqual(a.save(), 1) self.assertTrue(a.id is not None) self.assertEqual(len(state), 2) self.assertEqual(state[-1], (A, a, a.id, False)) def test_post_save(self): state = [] @signals.post_save() def post_save(sender, instance, created): state.append((sender, instance, instance._pk, created)) a = A() a.save() self.assertTrue(a.id is not None) self.assertEqual(state, [(A, a, a.id, True)]) a.save() self.assertEqual(len(state), 2) self.assertEqual(state[-1], (A, a, a.id, False)) def test_pre_delete(self): state = [] @signals.pre_delete() def pre_delete(sender, instance): state.append((sender, instance, A.select().count())) a = A.create() self.assertEqual(a.delete_instance(), 1) self.assertEqual(state, [(A, a, 1)]) def test_post_delete(self): state = [] @signals.post_delete() def post_delete(sender, instance): state.append((sender, instance, A.select().count())) a = A.create() a.delete_instance() self.assertEqual(state, [(A, a, 0)]) def test_pre_init(self): state = [] A.create(a='a') @signals.pre_init() def pre_init(sender, instance): state.append((sender, instance.a)) A.get() self.assertEqual(state, [(A, 'a')]) def test_sender(self): state = [] @signals.post_save(sender=A) def post_save(sender, instance, created): state.append(instance) m = A.create() self.assertEqual(state, [m]) m2 = B.create() self.assertEqual(state, [m]) def test_connect_disconnect(self): state = [] @signals.post_save(sender=A) def post_save(sender, instance, created): state.append(instance) a = A.create() self.assertEqual(state, [a]) # Signal was registered with a specific sender, so this fails. self.assertRaises(ValueError, signals.post_save.disconnect, post_save) # Disconnect signal, specifying sender. signals.post_save.disconnect(post_save, sender=A) # Signal handler has been unregistered. a2 = A.create() self.assertEqual(state, [a]) # Re-connect without specifying sender. signals.post_save.connect(post_save) a3 = A.create() self.assertEqual(state, [a, a3]) # Signal was not registered with a sender, so this fails. self.assertRaises(ValueError, signals.post_save.disconnect, post_save, sender=A) signals.post_save.disconnect(post_save) def test_function_reuse(self): state = [] @signals.post_save(sender=A) def post_save(sender, instance, created): state.append(instance) # Connect function for sender=B as well. signals.post_save(sender=B)(post_save) a = A.create() b = B.create() self.assertEqual(state, [a, b]) def test_subclass_instance_receive_signals(self): state = [] @signals.post_save(sender=B) def post_save(sender, instance, created): state.append(instance) b = SubB.create() assert b in state def test_disconnect_issue_2687(self): state = [] # Same sender. @signals.post_save(sender=A) def sig1(sender, instance, created): state.append((1, instance.a)) @signals.post_save(sender=A) def sig2(sender, instance, created): state.append((2, instance.a)) A.create(a='a1') self.assertEqual(state, [(1, 'a1'), (2, 'a1')]) signals.post_save.disconnect(name='sig1', sender=A) A.create(a='a2') self.assertEqual(state, [(1, 'a1'), (2, 'a1'), (2, 'a2')]) signals.post_save.disconnect(name='sig2', sender=A) A.create(a='a3') self.assertEqual(state, [(1, 'a1'), (2, 'a1'), (2, 'a2')]) signals.post_save(name='s1')(sig1) signals.post_save(name='s2', sender=A)(sig2) state = state[:0] # Clear state, 2.7 compat. A.create(a='a4') self.assertEqual(state, [(1, 'a4'), (2, 'a4')]) signals.post_save.disconnect(name='s1') A.create(a='a5') self.assertEqual(state, [(1, 'a4'), (2, 'a4'), (2, 'a5')]) signals.post_save.disconnect(name='s2', sender=A) A.create(a='a6') self.assertEqual(state, [(1, 'a4'), (2, 'a4'), (2, 'a5')]) class NoPK(BaseSignalModel): val = IntegerField(index=True) class Meta: primary_key = False class TestSaveNoPrimaryKey(ModelTestCase): database = get_in_memory_db() requires = [NoPK] def test_save_no_pk(self): accum = [0] @signals.pre_save(sender=NoPK) @signals.post_save(sender=NoPK) def save_hook(sender, instance, created): accum[0] += 1 obj = NoPK.create(val=1) self.assertEqual(obj.val, 1) obj_db = NoPK.get(NoPK.val == 1) self.assertEqual(obj_db.val, 1) self.assertEqual(accum[0], 2) ================================================ FILE: tests/sql.py ================================================ import datetime import re from peewee import * from peewee import Expression from peewee import Function from peewee import query_to_string from .base import BaseTestCase from .base import TestModel from .base import db from .base import requires_mysql from .base import requires_sqlite from .base import __sql__ User = Table('users') Tweet = Table('tweets') Person = Table('person', ['id', 'name', 'dob'], primary_key='id') Note = Table('note', ['id', 'person_id', 'content']) class TestSelectQuery(BaseTestCase): def test_select(self): query = (User .select(User.c.id, User.c.username) .where(User.c.username == 'foo')) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username" ' 'FROM "users" AS "t1" ' 'WHERE ("t1"."username" = ?)'), ['foo']) query = (User .select(User.c['id'], User.c['username']) .where(User.c['username'] == 'test')) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username" ' 'FROM "users" AS "t1" ' 'WHERE ("t1"."username" = ?)'), ['test']) def test_select_extend(self): query = User.select(User.c.id, User.c.username) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username" FROM "users" AS "t1"'), []) query = query.select(User.c.username, User.c.is_admin) self.assertSQL(query, ( 'SELECT "t1"."username", "t1"."is_admin" FROM "users" AS "t1"'), []) query = query.select_extend(User.c.is_active, User.c.id) self.assertSQL(query, ( 'SELECT "t1"."username", "t1"."is_admin", "t1"."is_active", ' '"t1"."id" FROM "users" AS "t1"'), []) def test_selected_columns(self): query = (User .select(User.c.id, User.c.username, fn.COUNT(Tweet.c.id)) .join(Tweet, JOIN.LEFT_OUTER, on=(User.c.id == Tweet.c.user_id))) # NOTE: because of operator overloads for equality we have to test by # asserting the attributes of the selected cols. c_id, c_username, c_ct = query.selected_columns self.assertEqual(c_id.name, 'id') self.assertTrue(c_id.source is User) self.assertEqual(c_username.name, 'username') self.assertTrue(c_username.source is User) self.assertTrue(isinstance(c_ct, Function)) self.assertEqual(c_ct.name, 'COUNT') c_tid, = c_ct.arguments self.assertEqual(c_tid.name, 'id') self.assertTrue(c_tid.source is Tweet) query.selected_columns = (User.c.username,) c_username, = query.selected_columns self.assertEqual(c_username.name, 'username') self.assertTrue(c_username.source is User) def test_select_explicit_columns(self): query = (Person .select() .where(Person.dob < datetime.date(1980, 1, 1))) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."name", "t1"."dob" ' 'FROM "person" AS "t1" ' 'WHERE ("t1"."dob" < ?)'), [datetime.date(1980, 1, 1)]) def test_select_in_list_of_values(self): names_vals = [ ['charlie', 'huey'], ('charlie', 'huey'), set(('charlie', 'huey')), frozenset(('charlie', 'huey')), (x for x in ('charlie', 'huey'))] for names in names_vals: query = (Person .select() .where(Person.name.in_(names))) sql, params = Context().sql(query).query() self.assertEqual(sql, ( 'SELECT "t1"."id", "t1"."name", "t1"."dob" ' 'FROM "person" AS "t1" ' 'WHERE ("t1"."name" IN (?, ?))')) self.assertEqual(sorted(params), ['charlie', 'huey']) query = (Person .select() .where(Person.id.in_(range(1, 10, 2)))) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."name", "t1"."dob" ' 'FROM "person" AS "t1" ' 'WHERE ("t1"."id" IN (?, ?, ?, ?, ?))'), [1, 3, 5, 7, 9]) def test_select_subselect_function(self): # For functions whose only argument is a subquery, we do not need to # include additional parentheses -- in fact, some databases will report # a syntax error if we do. exists = fn.EXISTS(Tweet .select(Tweet.c.id) .where(Tweet.c.user_id == User.c.id)) query = User.select(User.c.username, exists.alias('has_tweet')) self.assertSQL(query, ( 'SELECT "t1"."username", EXISTS(' 'SELECT "t2"."id" FROM "tweets" AS "t2" ' 'WHERE ("t2"."user_id" = "t1"."id")) AS "has_tweet" ' 'FROM "users" AS "t1"'), []) # If the function has more than one argument, we need to wrap the # subquery in parentheses. Stat = Table('stat', ['id', 'val']) SA = Stat.alias('sa') subq = SA.select(fn.SUM(SA.val).alias('val_sum')) query = Stat.select(fn.COALESCE(subq, 0)) self.assertSQL(query, ( 'SELECT COALESCE((' 'SELECT SUM("sa"."val") AS "val_sum" FROM "stat" AS "sa"' '), ?) FROM "stat" AS "t1"'), [0]) def test_subquery_in_select_sql(self): subq = User.select(User.c.id).where(User.c.username == 'huey') query = Tweet.select(Tweet.c.content, Tweet.c.user_id.in_(subq).alias('is_huey')) self.assertSQL(query, ( 'SELECT "t1"."content", ("t1"."user_id" IN (' 'SELECT "t2"."id" FROM "users" AS "t2" WHERE ("t2"."username" = ?)' ')) AS "is_huey" FROM "tweets" AS "t1"'), ['huey']) # If we explicitly specify an alias, it will be included. subq = subq.alias('sq') query = Tweet.select(Tweet.c.content, Tweet.c.user_id.in_(subq).alias('is_huey')) self.assertSQL(query, ( 'SELECT "t1"."content", ("t1"."user_id" IN (' 'SELECT "t2"."id" FROM "users" AS "t2" WHERE ("t2"."username" = ?)' ') AS "sq") AS "is_huey" FROM "tweets" AS "t1"'), ['huey']) def test_subquery_in_select_expression_sql(self): Point = Table('point', ('x', 'y')) PA = Point.alias('pa') subq = PA.select(fn.SUM(PA.y).alias('sa')).where(PA.x == Point.x) query = (Point .select(Point.x, Point.y, subq.alias('sy')) .order_by(Point.x, Point.y)) self.assertSQL(query, ( 'SELECT "t1"."x", "t1"."y", (' 'SELECT SUM("pa"."y") AS "sa" FROM "point" AS "pa" ' 'WHERE ("pa"."x" = "t1"."x")) AS "sy" ' 'FROM "point" AS "t1" ' 'ORDER BY "t1"."x", "t1"."y"'), []) def test_star(self): query = User.select(User.__star__) self.assertSQL(query, ('SELECT "t1".* FROM "users" AS "t1"'), []) query = (Tweet .select(Tweet.__star__, User.__star__) .join(User, on=(Tweet.c.user_id == User.c.id))) self.assertSQL(query, ( 'SELECT "t1".*, "t2".* ' 'FROM "tweets" AS "t1" ' 'INNER JOIN "users" AS "t2" ON ("t1"."user_id" = "t2"."id")'), []) query = (Tweet .select(Tweet.__star__, User.c.id) .join(User, on=(Tweet.c.user_id == User.c.id))) self.assertSQL(query, ( 'SELECT "t1".*, "t2"."id" ' 'FROM "tweets" AS "t1" ' 'INNER JOIN "users" AS "t2" ON ("t1"."user_id" = "t2"."id")'), []) def test_from_clause(self): query = (Note .select(Note.content, Person.name) .from_(Note, Person) .where(Note.person_id == Person.id) .order_by(Note.id)) self.assertSQL(query, ( 'SELECT "t1"."content", "t2"."name" ' 'FROM "note" AS "t1", "person" AS "t2" ' 'WHERE ("t1"."person_id" = "t2"."id") ' 'ORDER BY "t1"."id"'), []) def test_from_query(self): inner = Person.select(Person.name) query = (Person .select(Person.name) .from_(inner.alias('i1'))) self.assertSQL(query, ( 'SELECT "t1"."name" ' 'FROM (SELECT "t1"."name" FROM "person" AS "t1") AS "i1"'), []) PA = Person.alias('pa') inner = PA.select(PA.name).alias('i1') query = (Person .select(inner.c.name) .from_(inner) .order_by(inner.c.name)) self.assertSQL(query, ( 'SELECT "i1"."name" ' 'FROM (SELECT "pa"."name" FROM "person" AS "pa") AS "i1" ' 'ORDER BY "i1"."name"'), []) def test_join_explicit_columns(self): query = (Note .select(Note.content) .join(Person, on=(Note.person_id == Person.id)) .where(Person.name == 'charlie') .order_by(Note.id.desc())) self.assertSQL(query, ( 'SELECT "t1"."content" ' 'FROM "note" AS "t1" ' 'INNER JOIN "person" AS "t2" ON ("t1"."person_id" = "t2"."id") ' 'WHERE ("t2"."name" = ?) ' 'ORDER BY "t1"."id" DESC'), ['charlie']) def test_multi_join(self): Like = Table('likes') LikeUser = User.alias('lu') query = (Like .select(Tweet.c.content, User.c.username, LikeUser.c.username) .join(Tweet, on=(Like.c.tweet_id == Tweet.c.id)) .join(User, on=(Tweet.c.user_id == User.c.id)) .join(LikeUser, on=(Like.c.user_id == LikeUser.c.id)) .where(LikeUser.c.username == 'charlie') .order_by(Tweet.c.timestamp)) self.assertSQL(query, ( 'SELECT "t1"."content", "t2"."username", "lu"."username" ' 'FROM "likes" AS "t3" ' 'INNER JOIN "tweets" AS "t1" ON ("t3"."tweet_id" = "t1"."id") ' 'INNER JOIN "users" AS "t2" ON ("t1"."user_id" = "t2"."id") ' 'INNER JOIN "users" AS "lu" ON ("t3"."user_id" = "lu"."id") ' 'WHERE ("lu"."username" = ?) ' 'ORDER BY "t1"."timestamp"'), ['charlie']) def test_correlated_subquery(self): Employee = Table('employee', ['id', 'name', 'salary', 'dept']) EA = Employee.alias('e2') query = (Employee .select(Employee.id, Employee.name) .where(Employee.salary > (EA .select(fn.AVG(EA.salary)) .where(EA.dept == Employee.dept)))) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."name" ' 'FROM "employee" AS "t1" ' 'WHERE ("t1"."salary" > (' 'SELECT AVG("e2"."salary") ' 'FROM "employee" AS "e2" ' 'WHERE ("e2"."dept" = "t1"."dept")))'), []) def test_multiple_where(self): """Ensure multiple calls to WHERE are AND-ed together.""" query = (Person .select(Person.name) .where(Person.dob < datetime.date(1980, 1, 1)) .where(Person.dob > datetime.date(1950, 1, 1))) self.assertSQL(query, ( 'SELECT "t1"."name" ' 'FROM "person" AS "t1" ' 'WHERE (("t1"."dob" < ?) AND ("t1"."dob" > ?))'), [datetime.date(1980, 1, 1), datetime.date(1950, 1, 1)]) def test_orwhere(self): query = (Person .select(Person.name) .orwhere(Person.dob > datetime.date(1980, 1, 1)) .orwhere(Person.dob < datetime.date(1950, 1, 1))) self.assertSQL(query, ( 'SELECT "t1"."name" ' 'FROM "person" AS "t1" ' 'WHERE (("t1"."dob" > ?) OR ("t1"."dob" < ?))'), [datetime.date(1980, 1, 1), datetime.date(1950, 1, 1)]) def test_limit(self): base = User.select(User.c.id) self.assertSQL(base.limit(None), ( 'SELECT "t1"."id" FROM "users" AS "t1"'), []) self.assertSQL(base.limit(10), ( 'SELECT "t1"."id" FROM "users" AS "t1" LIMIT ?'), [10]) self.assertSQL(base.limit(10).offset(3), ( 'SELECT "t1"."id" FROM "users" AS "t1" ' 'LIMIT ? OFFSET ?'), [10, 3]) self.assertSQL(base.limit(0), ( 'SELECT "t1"."id" FROM "users" AS "t1" LIMIT ?'), [0]) self.assertSQL(base.offset(3), ( 'SELECT "t1"."id" FROM "users" AS "t1" OFFSET ?'), [3], limit_max=None) # Some databases do not support offset without corresponding LIMIT: self.assertSQL(base.offset(3), ( 'SELECT "t1"."id" FROM "users" AS "t1" LIMIT ? OFFSET ?'), [-1, 3], limit_max=-1) self.assertSQL(base.limit(0).offset(3), ( 'SELECT "t1"."id" FROM "users" AS "t1" LIMIT ? OFFSET ?'), [0, 3], limit_max=-1) def test_simple_join(self): query = (User .select( User.c.id, User.c.username, fn.COUNT(Tweet.c.id).alias('ct')) .join(Tweet, on=(Tweet.c.user_id == User.c.id)) .group_by(User.c.id, User.c.username)) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username", COUNT("t2"."id") AS "ct" ' 'FROM "users" AS "t1" ' 'INNER JOIN "tweets" AS "t2" ON ("t2"."user_id" = "t1"."id") ' 'GROUP BY "t1"."id", "t1"."username"'), []) def test_subquery(self): inner = (Tweet .select(fn.COUNT(Tweet.c.id).alias('ct')) .where(Tweet.c.user == User.c.id)) query = (User .select(User.c.username, inner.alias('iq')) .order_by(User.c.username)) self.assertSQL(query, ( 'SELECT "t1"."username", ' '(SELECT COUNT("t2"."id") AS "ct" ' 'FROM "tweets" AS "t2" ' 'WHERE ("t2"."user" = "t1"."id")) AS "iq" ' 'FROM "users" AS "t1" ORDER BY "t1"."username"'), []) def test_subquery_in_expr(self): Team = Table('team') Challenge = Table('challenge') subq = Team.select(fn.COUNT(Team.c.id) + 1) query = (Challenge .select((Challenge.c.points / subq).alias('score')) .order_by(SQL('score'))) self.assertSQL(query, ( 'SELECT ("t1"."points" / (' 'SELECT (COUNT("t2"."id") + ?) FROM "team" AS "t2")) AS "score" ' 'FROM "challenge" AS "t1" ORDER BY score'), [1]) def test_user_defined_alias(self): UA = User.alias('alt') query = (User .select(User.c.id, User.c.username, UA.c.nuggz) .join(UA, on=(User.c.id == UA.c.id)) .order_by(UA.c.nuggz)) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username", "alt"."nuggz" ' 'FROM "users" AS "t1" ' 'INNER JOIN "users" AS "alt" ON ("t1"."id" = "alt"."id") ' 'ORDER BY "alt"."nuggz"'), []) def test_simple_cte(self): cte = User.select(User.c.id).cte('user_ids') query = (User .select(User.c.username) .where(User.c.id.in_(cte)) .with_cte(cte)) self.assertSQL(query, ( 'WITH "user_ids" AS (SELECT "t1"."id" FROM "users" AS "t1") ' 'SELECT "t2"."username" FROM "users" AS "t2" ' 'WHERE ("t2"."id" IN "user_ids")'), []) def test_two_ctes(self): c1 = User.select(User.c.id).cte('user_ids') c2 = User.select(User.c.username).cte('user_names') query = (User .select(c1.c.id, c2.c.username) .where((c1.c.id == User.c.id) & (c2.c.username == User.c.username)) .with_cte(c1, c2)) self.assertSQL(query, ( 'WITH "user_ids" AS (SELECT "t1"."id" FROM "users" AS "t1"), ' '"user_names" AS (SELECT "t2"."username" FROM "users" AS "t2") ' 'SELECT "user_ids"."id", "user_names"."username" ' 'FROM "users" AS "t3" ' 'WHERE (("user_ids"."id" = "t3"."id") AND ' '("user_names"."username" = "t3"."username"))'), []) def test_select_from_cte(self): # Use the "select_from()" helper on the CTE object. cte = User.select(User.c.username).cte('user_cte') query = cte.select_from(cte.c.username).order_by(cte.c.username) self.assertSQL(query, ( 'WITH "user_cte" AS (SELECT "t1"."username" FROM "users" AS "t1") ' 'SELECT "user_cte"."username" FROM "user_cte" ' 'ORDER BY "user_cte"."username"'), []) # Test selecting from multiple CTEs, which is done manually. c1 = User.select(User.c.username).where(User.c.is_admin == 1).cte('c1') c2 = User.select(User.c.username).where(User.c.is_staff == 1).cte('c2') query = (Select((c1, c2), (c1.c.username, c2.c.username)) .with_cte(c1, c2)) self.assertSQL(query, ( 'WITH "c1" AS (' 'SELECT "t1"."username" FROM "users" AS "t1" ' 'WHERE ("t1"."is_admin" = ?)), ' '"c2" AS (' 'SELECT "t2"."username" FROM "users" AS "t2" ' 'WHERE ("t2"."is_staff" = ?)) ' 'SELECT "c1"."username", "c2"."username" FROM "c1", "c2"'), [1, 1]) def test_cte_select_from_2(self): cte = (User .select(User.c.username) .where(User.c.username != 'x') .cte('filtered')) query = cte.select_from(cte.c.username) self.assertSQL(query, ( 'WITH "filtered" AS (' 'SELECT "t1"."username" FROM "users" AS "t1" ' 'WHERE ("t1"."username" != ?)) ' 'SELECT "filtered"."username" FROM "filtered"'), ['x']) def test_cte_select_from_with_aggregate(self): cte = (User .select(User.c.username, fn.COUNT(Tweet.c.id).alias('tweet_ct')) .join(Tweet, JOIN.LEFT_OUTER, (Tweet.c.user_id == User.c.id)) .group_by(User.c.username) .cte('user_stats')) query = (cte .select_from(cte.c.username, cte.c.tweet_ct) .where(cte.c.tweet_ct > 0)) self.assertSQL(query, ( 'WITH "user_stats" AS (' 'SELECT "t1"."username", COUNT("t2"."id") AS "tweet_ct" ' 'FROM "users" AS "t1" ' 'LEFT OUTER JOIN "tweets" AS "t2" ' 'ON ("t2"."user_id" = "t1"."id") ' 'GROUP BY "t1"."username") ' 'SELECT "user_stats"."username", "user_stats"."tweet_ct" ' 'FROM "user_stats" ' 'WHERE ("user_stats"."tweet_ct" > ?)'), [0]) def test_two_ctes_with_join(self): cte_a = (User .select(User.c.id, User.c.username) .cte('active_users')) cte_b = (Tweet .select(Tweet.c.user_id, fn.COUNT(Tweet.c.id).alias('ct')) .group_by(Tweet.c.user_id) .cte('tweet_counts')) query = (cte_a .select_from(cte_a.c.username, cte_b.c.ct) .join(cte_b, on=(cte_a.c.id == cte_b.c.user_id)) .with_cte(cte_a, cte_b) .order_by(cte_b.c.ct.desc())) self.assertSQL(query, ( 'WITH "active_users" AS (' 'SELECT "t1"."id", "t1"."username" ' 'FROM "users" AS "t1"), ' '"tweet_counts" AS (' 'SELECT "t2"."user_id", COUNT("t2"."id") AS "ct" ' 'FROM "tweets" AS "t2" ' 'GROUP BY "t2"."user_id") ' 'SELECT "active_users"."username", "tweet_counts"."ct" ' 'FROM "active_users" ' 'INNER JOIN "tweet_counts" ' 'ON ("active_users"."id" = "tweet_counts"."user_id") ' 'ORDER BY "tweet_counts"."ct" DESC'), []) def test_materialize_cte(self): cases = ( (True, 'MATERIALIZED '), (False, 'NOT MATERIALIZED '), (None, '')) for materialized, clause in cases: cte = (User .select(User.c.id) .cte('user_ids', materialized=materialized)) query = cte.select_from(cte.c.id).where(cte.c.id < 10) self.assertSQL(query, ( 'WITH "user_ids" AS %s(' 'SELECT "t1"."id" FROM "users" AS "t1") ' 'SELECT "user_ids"."id" FROM "user_ids" ' 'WHERE ("user_ids"."id" < ?)') % clause, [10]) def test_fibonacci_cte(self): q1 = Select(columns=( Value(1).alias('n'), Value(0).alias('fib_n'), Value(1).alias('next_fib_n'))).cte('fibonacci', recursive=True) n = (q1.c.n + 1).alias('n') rterm = Select(columns=( n, q1.c.next_fib_n, q1.c.fib_n + q1.c.next_fib_n)).from_(q1).where(n < 10) cases = ( (q1.union_all, 'UNION ALL'), (q1.union, 'UNION')) for method, clause in cases: cte = method(rterm) query = cte.select_from(cte.c.n, cte.c.fib_n) self.assertSQL(query, ( 'WITH RECURSIVE "fibonacci" AS (' 'SELECT ? AS "n", ? AS "fib_n", ? AS "next_fib_n" ' '%s ' 'SELECT ("fibonacci"."n" + ?) AS "n", "fibonacci"."next_fib_n", ' '("fibonacci"."fib_n" + "fibonacci"."next_fib_n") ' 'FROM "fibonacci" ' 'WHERE ("n" < ?)) ' 'SELECT "fibonacci"."n", "fibonacci"."fib_n" ' 'FROM "fibonacci"' % clause), [1, 0, 1, 1, 10]) def test_cte_with_count(self): cte = User.select(User.c.id).cte('user_ids') query = (User .select(User.c.username) .join(cte, on=(User.c.id == cte.c.id)) .with_cte(cte)) count = Select([query], [fn.COUNT(SQL('1'))]) self.assertSQL(count, ( 'SELECT COUNT(1) FROM (' 'WITH "user_ids" AS (SELECT "t1"."id" FROM "users" AS "t1") ' 'SELECT "t2"."username" FROM "users" AS "t2" ' 'INNER JOIN "user_ids" ON ("t2"."id" = "user_ids"."id")) ' 'AS "t3"'), []) def test_cte_subquery_in_expression(self): Order = Table('order', ('id', 'description')) Item = Table('item', ('id', 'order_id', 'description')) cte = Order.select(fn.MAX(Order.id).alias('max_id')).cte('max_order') qexpr = (Order .select(Order.id) .join(cte, on=(Order.id == cte.c.max_id)) .with_cte(cte)) query = (Item .select(Item.id, Item.order_id, Item.description) .where(Item.order_id.in_(qexpr))) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."order_id", "t1"."description" ' 'FROM "item" AS "t1" ' 'WHERE ("t1"."order_id" IN (' 'WITH "max_order" AS (' 'SELECT MAX("t2"."id") AS "max_id" FROM "order" AS "t2") ' 'SELECT "t3"."id" ' 'FROM "order" AS "t3" ' 'INNER JOIN "max_order" ' 'ON ("t3"."id" = "max_order"."max_id")))'), []) def test_multi_update_cte(self): data = [(i, 'u%sx' % i) for i in range(1, 3)] vl = ValuesList(data) cte = vl.select().cte('uv', columns=('id', 'username')) subq = cte.select(cte.c.username).where(cte.c.id == User.c.id) query = (User .update(username=subq) .where(User.c.id.in_(cte.select(cte.c.id))) .with_cte(cte)) self.assertSQL(query, ( 'WITH "uv" ("id", "username") AS (' 'SELECT * FROM (VALUES (?, ?), (?, ?)) AS "t1") ' 'UPDATE "users" SET "username" = (' 'SELECT "uv"."username" FROM "uv" ' 'WHERE ("uv"."id" = "users"."id")) ' 'WHERE ("users"."id" IN (SELECT "uv"."id" FROM "uv"))'), [1, 'u1x', 2, 'u2x']) def test_data_modifying_cte_delete(self): Product = Table('products', ('id', 'name', 'timestamp')) Archive = Table('archive', ('id', 'name', 'timestamp')) query = (Product.delete() .where(Product.timestamp < datetime.date(2022, 1, 1)) .returning(Product.id, Product.name, Product.timestamp)) cte = query.cte('moved_rows') src = Select((cte,), (cte.c.id, cte.c.name, cte.c.timestamp)) iq = (Archive .insert(src, (Archive.id, Archive.name, Archive.timestamp)) .with_cte(cte)) self.assertSQL(iq, ( 'WITH "moved_rows" AS (' 'DELETE FROM "products" WHERE ("products"."timestamp" < ?) ' 'RETURNING "products"."id", "products"."name", ' '"products"."timestamp") ' 'INSERT INTO "archive" ("id", "name", "timestamp") ' 'SELECT "moved_rows"."id", "moved_rows"."name", ' '"moved_rows"."timestamp" FROM "moved_rows"'), [datetime.date(2022, 1, 1)]) Part = Table('parts', ('id', 'part', 'sub_part')) base = (Part .select(Part.sub_part, Part.part) .where(Part.part == 'p') .cte('included_parts', recursive=True, columns=('sub_part', 'part'))) PA = Part.alias('p') recursive = (PA .select(PA.sub_part, PA.part) .join(base, on=(PA.part == base.c.sub_part))) cte = base.union_all(recursive) sq = Select((cte,), (cte.c.part,)) query = (Part.delete() .where(Part.part.in_(sq)) .with_cte(cte)) self.assertSQL(query, ( 'WITH RECURSIVE "included_parts" ("sub_part", "part") AS (' 'SELECT "t1"."sub_part", "t1"."part" FROM "parts" AS "t1" ' 'WHERE ("t1"."part" = ?) ' 'UNION ALL ' 'SELECT "p"."sub_part", "p"."part" ' 'FROM "parts" AS "p" ' 'INNER JOIN "included_parts" ' 'ON ("p"."part" = "included_parts"."sub_part")) ' 'DELETE FROM "parts" ' 'WHERE ("parts"."part" IN (' 'SELECT "included_parts"."part" FROM "included_parts"))'), ['p']) def test_data_modifying_cte_update(self): Product = Table('products', ('id', 'name', 'price')) Archive = Table('archive', ('id', 'name', 'price')) query = (Product .update(price=Product.price * 1.05) .returning(Product.id, Product.name, Product.price)) cte = query.cte('t') sq = cte.select_from(cte.c.id, cte.c.name, cte.c.price) self.assertSQL(sq, ( 'WITH "t" AS (' 'UPDATE "products" SET "price" = ("products"."price" * ?) ' 'RETURNING "products"."id", "products"."name", "products"."price")' ' SELECT "t"."id", "t"."name", "t"."price" FROM "t"'), [1.05]) sq = Select((cte,), (cte.c.id, cte.c.price)) uq = (Archive .update(price=sq.c.price) .from_(sq) .where(Archive.id == sq.c.id) .with_cte(cte)) self.assertSQL(uq, ( 'WITH "t" AS (' 'UPDATE "products" SET "price" = ("products"."price" * ?) ' 'RETURNING "products"."id", "products"."name", "products"."price")' ' UPDATE "archive" SET "price" = "t1"."price"' ' FROM (SELECT "t"."id", "t"."price" FROM "t") AS "t1"' ' WHERE ("archive"."id" = "t1"."id")'), [1.05]) def test_data_modifying_cte_insert(self): Product = Table('products', ('id', 'name', 'price')) Archive = Table('archive', ('id', 'name', 'price')) query = (Product .insert({'name': 'p1', 'price': 10}) .returning(Product.id, Product.name, Product.price)) cte = query.cte('t') sq = cte.select_from(cte.c.id, cte.c.name, cte.c.price) self.assertSQL(sq, ( 'WITH "t" AS (' 'INSERT INTO "products" ("name", "price") VALUES (?, ?) ' 'RETURNING "products"."id", "products"."name", "products"."price")' ' SELECT "t"."id", "t"."name", "t"."price" FROM "t"'), ['p1', 10]) sq = Select((cte,), (cte.c.id, cte.c.name, cte.c.price)) iq = (Archive .insert(sq, (sq.c.id, sq.c.name, sq.c.price)) .with_cte(cte)) self.assertSQL(iq, ( 'WITH "t" AS (' 'INSERT INTO "products" ("name", "price") VALUES (?, ?) ' 'RETURNING "products"."id", "products"."name", "products"."price")' ' INSERT INTO "archive" ("id", "name", "price")' ' SELECT "t"."id", "t"."name", "t"."price" FROM "t"'), ['p1', 10]) def test_select_from_subquery(self): subq = (User .select(User.c.username, fn.LENGTH(User.c.username).alias('name_len')) .alias('sub')) query = (User .select(subq.c.username, subq.c.name_len) .from_(subq) .where(subq.c.name_len > 3) .order_by(subq.c.name_len)) self.assertSQL(query, ( 'SELECT "sub"."username", "sub"."name_len" ' 'FROM (' 'SELECT "t1"."username", LENGTH("t1"."username") AS "name_len" ' 'FROM "users" AS "t1") AS "sub" ' 'WHERE ("sub"."name_len" > ?) ' 'ORDER BY "sub"."name_len"'), [3]) def test_complex_select(self): Order = Table('orders', columns=( 'region', 'amount', 'product', 'quantity')) regional_sales = (Order .select( Order.region, fn.SUM(Order.amount).alias('total_sales')) .group_by(Order.region) .cte('regional_sales')) top_regions = (regional_sales .select(regional_sales.c.region) .where(regional_sales.c.total_sales > ( regional_sales.select( fn.SUM(regional_sales.c.total_sales) / 10))) .cte('top_regions')) query = (Order .select( Order.region, Order.product, fn.SUM(Order.quantity).alias('product_units'), fn.SUM(Order.amount).alias('product_sales')) .where( Order.region << top_regions.select(top_regions.c.region)) .group_by(Order.region, Order.product) .with_cte(regional_sales, top_regions)) self.assertSQL(query, ( 'WITH "regional_sales" AS (' 'SELECT "t1"."region", SUM("t1"."amount") AS "total_sales" ' 'FROM "orders" AS "t1" ' 'GROUP BY "t1"."region"' '), ' '"top_regions" AS (' 'SELECT "regional_sales"."region" ' 'FROM "regional_sales" ' 'WHERE ("regional_sales"."total_sales" > ' '(SELECT (SUM("regional_sales"."total_sales") / ?) ' 'FROM "regional_sales"))' ') ' 'SELECT "t2"."region", "t2"."product", ' 'SUM("t2"."quantity") AS "product_units", ' 'SUM("t2"."amount") AS "product_sales" ' 'FROM "orders" AS "t2" ' 'WHERE (' '"t2"."region" IN (' 'SELECT "top_regions"."region" ' 'FROM "top_regions")' ') GROUP BY "t2"."region", "t2"."product"'), [10]) def test_compound_select(self): lhs = User.select(User.c.id).where(User.c.username == 'charlie') rhs = User.select(User.c.username).where(User.c.admin == True) q2 = (lhs | rhs) UA = User.alias('U2') q3 = q2 | UA.select(UA.c.id).where(UA.c.superuser == False) self.assertSQL(q3, ( 'SELECT "t1"."id" ' 'FROM "users" AS "t1" ' 'WHERE ("t1"."username" = ?) ' 'UNION ' 'SELECT "t2"."username" ' 'FROM "users" AS "t2" ' 'WHERE ("t2"."admin" = ?) ' 'UNION ' 'SELECT "U2"."id" ' 'FROM "users" AS "U2" ' 'WHERE ("U2"."superuser" = ?)'), ['charlie', True, False]) def test_compound_operations(self): admin = (User .select(User.c.username, Value('admin').alias('role')) .where(User.c.is_admin == True)) editors = (User .select(User.c.username, Value('editor').alias('role')) .where(User.c.is_editor == True)) union = admin.union(editors) self.assertSQL(union, ( 'SELECT "t1"."username", ? AS "role" ' 'FROM "users" AS "t1" ' 'WHERE ("t1"."is_admin" = ?) ' 'UNION ' 'SELECT "t2"."username", ? AS "role" ' 'FROM "users" AS "t2" ' 'WHERE ("t2"."is_editor" = ?)'), ['admin', 1, 'editor', 1]) xcept = editors.except_(admin) self.assertSQL(xcept, ( 'SELECT "t1"."username", ? AS "role" ' 'FROM "users" AS "t1" ' 'WHERE ("t1"."is_editor" = ?) ' 'EXCEPT ' 'SELECT "t2"."username", ? AS "role" ' 'FROM "users" AS "t2" ' 'WHERE ("t2"."is_admin" = ?)'), ['editor', 1, 'admin', 1]) def test_compound_parentheses_handling(self): admin = (User .select(User.c.username, Value('admin').alias('role')) .where(User.c.is_admin == True) .order_by(User.c.id.desc()) .limit(3)) editors = (User .select(User.c.username, Value('editor').alias('role')) .where(User.c.is_editor == True) .order_by(User.c.id.desc()) .limit(5)) self.assertSQL((admin | editors), ( '(SELECT "t1"."username", ? AS "role" FROM "users" AS "t1" ' 'WHERE ("t1"."is_admin" = ?) ORDER BY "t1"."id" DESC LIMIT ?) ' 'UNION ' '(SELECT "t2"."username", ? AS "role" FROM "users" AS "t2" ' 'WHERE ("t2"."is_editor" = ?) ORDER BY "t2"."id" DESC LIMIT ?)'), ['admin', 1, 3, 'editor', 1, 5], compound_select_parentheses=True) Reg = Table('register', ('value',)) lhs = Reg.select().where(Reg.value < 2) rhs = Reg.select().where(Reg.value > 7) compound = lhs | rhs for csq_setting in (1, 2): self.assertSQL(compound, ( '(SELECT "t1"."value" FROM "register" AS "t1" ' 'WHERE ("t1"."value" < ?)) ' 'UNION ' '(SELECT "t2"."value" FROM "register" AS "t2" ' 'WHERE ("t2"."value" > ?))'), [2, 7], compound_select_parentheses=csq_setting) rhs2 = Reg.select().where(Reg.value == 5) c2 = compound | rhs2 # CSQ = always, we get nested parentheses. self.assertSQL(c2, ( '((SELECT "t1"."value" FROM "register" AS "t1" ' 'WHERE ("t1"."value" < ?)) ' 'UNION ' '(SELECT "t2"."value" FROM "register" AS "t2" ' 'WHERE ("t2"."value" > ?))) ' 'UNION ' '(SELECT "t3"."value" FROM "register" AS "t3" ' 'WHERE ("t3"."value" = ?))'), [2, 7, 5], compound_select_parentheses=1) # Always. # CSQ = unnested, no nesting but all individual queries have parens. self.assertSQL(c2, ( '(SELECT "t1"."value" FROM "register" AS "t1" ' 'WHERE ("t1"."value" < ?)) ' 'UNION ' '(SELECT "t2"."value" FROM "register" AS "t2" ' 'WHERE ("t2"."value" > ?)) ' 'UNION ' '(SELECT "t3"."value" FROM "register" AS "t3" ' 'WHERE ("t3"."value" = ?))'), [2, 7, 5], compound_select_parentheses=2) # Un-nested. def test_compound_select_order_limit(self): A = Table('a', ('col_a',)) B = Table('b', ('col_b',)) C = Table('c', ('col_c',)) q1 = A.select(A.col_a.alias('foo')) q2 = B.select(B.col_b.alias('foo')) q3 = C.select(C.col_c.alias('foo')) qc = (q1 | q2 | q3) qc = qc.order_by(qc.c.foo.desc()).limit(3) self.assertSQL(qc, ( 'SELECT "t1"."col_a" AS "foo" FROM "a" AS "t1" UNION ' 'SELECT "t2"."col_b" AS "foo" FROM "b" AS "t2" UNION ' 'SELECT "t3"."col_c" AS "foo" FROM "c" AS "t3" ' 'ORDER BY "foo" DESC LIMIT ?'), [3]) self.assertSQL(qc, ( '((SELECT "t1"."col_a" AS "foo" FROM "a" AS "t1") UNION ' '(SELECT "t2"."col_b" AS "foo" FROM "b" AS "t2")) UNION ' '(SELECT "t3"."col_c" AS "foo" FROM "c" AS "t3") ' 'ORDER BY "foo" DESC LIMIT ?'), [3], compound_select_parentheses=1) def test_compound_select_as_subquery(self): A = Table('a', ('col_a',)) B = Table('b', ('col_b',)) q1 = A.select(A.col_a.alias('foo')) q2 = B.select(B.col_b.alias('foo')) union = q1 | q2 # Create an outer query and do grouping. outer = (union .select_from(union.c.foo, fn.COUNT(union.c.foo).alias('ct')) .group_by(union.c.foo)) self.assertSQL(outer, ( 'SELECT "t1"."foo", COUNT("t1"."foo") AS "ct" FROM (' 'SELECT "t2"."col_a" AS "foo" FROM "a" AS "t2" UNION ' 'SELECT "t3"."col_b" AS "foo" FROM "b" AS "t3") AS "t1" ' 'GROUP BY "t1"."foo"'), []) def test_union_with_order_and_limit(self): q1 = User.select(User.c.username).where(User.c.id < 5) q2 = User.select(User.c.username).where(User.c.id > 95) combined = (q1 | q2).order_by(SQL('1')).limit(10) self.assertSQL(combined, ( 'SELECT "t1"."username" FROM "users" AS "t1" ' 'WHERE ("t1"."id" < ?) ' 'UNION ' 'SELECT "t2"."username" FROM "users" AS "t2" ' 'WHERE ("t2"."id" > ?) ' 'ORDER BY 1 LIMIT ?'), [5, 95, 10]) def test_intersect(self): q1 = User.select(User.c.username).where(User.c.id < 10) q2 = User.select(User.c.username).where(User.c.id > 5) combined = q1 & q2 self.assertSQL(combined, ( 'SELECT "t1"."username" FROM "users" AS "t1" ' 'WHERE ("t1"."id" < ?) ' 'INTERSECT ' 'SELECT "t2"."username" FROM "users" AS "t2" ' 'WHERE ("t2"."id" > ?)'), [10, 5]) def test_except(self): q1 = User.select(User.c.username) q2 = User.select(User.c.username).where(User.c.id > 5) combined = q1 - q2 self.assertSQL(combined, ( 'SELECT "t1"."username" FROM "users" AS "t1" ' 'EXCEPT ' 'SELECT "t2"."username" FROM "users" AS "t2" ' 'WHERE ("t2"."id" > ?)'), [5]) def test_coalesce(self): Sample = Table('sample', ('counter', 'value')) query = (Sample .select(fn.COALESCE(Sample.value, 0).alias('val')) .where(Sample.counter == 1)) self.assertSQL(query, ( 'SELECT COALESCE("t1"."value", ?) AS "val" ' 'FROM "sample" AS "t1" ' 'WHERE ("t1"."counter" = ?)'), [0, 1]) def test_nullif(self): Sample = Table('sample', ('counter', 'value')) query = (Sample .select(fn.NULLIF(Sample.value, 0).alias('val'))) self.assertSQL(query, ( 'SELECT NULLIF("t1"."value", ?) AS "val" ' 'FROM "sample" AS "t1"'), [0]) def test_join_on_query(self): inner = User.select(User.c.id).alias('j1') query = (Tweet .select(Tweet.c.content) .join(inner, on=(Tweet.c.user_id == inner.c.id))) self.assertSQL(query, ( 'SELECT "t1"."content" FROM "tweets" AS "t1" ' 'INNER JOIN (SELECT "t2"."id" FROM "users" AS "t2") AS "j1" ' 'ON ("t1"."user_id" = "j1"."id")'), []) def test_join_on_misc(self): cond = fn.Magic(Person.id, Note.id).alias('magic') query = Person.select(Person.id).join(Note, on=cond) self.assertSQL(query, ( 'SELECT "t1"."id" FROM "person" AS "t1" ' 'INNER JOIN "note" AS "t2" ' 'ON Magic("t1"."id", "t2"."id") AS "magic"'), []) def test_lateral_subquery_model(self): inner = (Tweet .select(Tweet.c.content) .where(Tweet.c.user_id == User.c.id) .order_by(Tweet.c.timestamp.desc()) .limit(1)) query = (User .select(User.c.username, inner.c.content) .join(inner, JOIN.LEFT_LATERAL, on=True)) self.assertSQL(query, ( 'SELECT "t1"."username", "t2"."content" ' 'FROM "users" AS "t1" ' 'LEFT JOIN LATERAL (' 'SELECT "t3"."content" FROM "tweets" AS "t3" ' 'WHERE ("t3"."user_id" = "t1"."id") ' 'ORDER BY "t3"."timestamp" DESC LIMIT ?) AS "t2" ON ?'), [1, True]) def test_all_clauses(self): count = fn.COUNT(Tweet.c.id).alias('ct') query = (User .select(User.c.username, count) .join(Tweet, JOIN.LEFT_OUTER, on=(User.c.id == Tweet.c.user_id)) .where(User.c.is_admin == 1) .group_by(User.c.username) .having(count > 10) .order_by(count.desc())) self.assertSQL(query, ( 'SELECT "t1"."username", COUNT("t2"."id") AS "ct" ' 'FROM "users" AS "t1" ' 'LEFT OUTER JOIN "tweets" AS "t2" ' 'ON ("t1"."id" = "t2"."user_id") ' 'WHERE ("t1"."is_admin" = ?) ' 'GROUP BY "t1"."username" ' 'HAVING ("ct" > ?) ' 'ORDER BY "ct" DESC'), [1, 10]) def test_order_by_collate(self): query = (User .select(User.c.username) .order_by(User.c.username.asc(collation='binary'))) self.assertSQL(query, ( 'SELECT "t1"."username" FROM "users" AS "t1" ' 'ORDER BY "t1"."username" ASC COLLATE binary'), []) def test_order_by_nulls(self): query = (User .select(User.c.username) .order_by(User.c.ts.desc(nulls='LAST'))) self.assertSQL(query, ( 'SELECT "t1"."username" FROM "users" AS "t1" ' 'ORDER BY "t1"."ts" DESC NULLS LAST'), [], nulls_ordering=True) self.assertSQL(query, ( 'SELECT "t1"."username" FROM "users" AS "t1" ' 'ORDER BY CASE WHEN ("t1"."ts" IS NULL) THEN ? ELSE ? END, ' '"t1"."ts" DESC'), [1, 0], nulls_ordering=False) query = (User .select(User.c.username) .order_by(User.c.ts.desc(nulls='first'))) self.assertSQL(query, ( 'SELECT "t1"."username" FROM "users" AS "t1" ' 'ORDER BY "t1"."ts" DESC NULLS first'), [], nulls_ordering=True) self.assertSQL(query, ( 'SELECT "t1"."username" FROM "users" AS "t1" ' 'ORDER BY CASE WHEN ("t1"."ts" IS NULL) THEN ? ELSE ? END, ' '"t1"."ts" DESC'), [0, 1], nulls_ordering=False) def test_in_value_representation(self): query = (User .select(User.c.id) .where(User.c.username.in_(['foo', 'bar', 'baz']))) self.assertSQL(query, ( 'SELECT "t1"."id" FROM "users" AS "t1" ' 'WHERE ("t1"."username" IN (?, ?, ?))'), ['foo', 'bar', 'baz']) def test_tuple_comparison(self): name_dob = Tuple(Person.name, Person.dob) query = (Person .select(Person.id) .where(name_dob == ('foo', '2017-01-01'))) expected = ('SELECT "t1"."id" FROM "person" AS "t1" ' 'WHERE (("t1"."name", "t1"."dob") = (?, ?))') self.assertSQL(query, expected, ['foo', '2017-01-01']) # Also works specifying rhs values as Tuple(). query = (Person .select(Person.id) .where(name_dob == Tuple('foo', '2017-01-01'))) self.assertSQL(query, expected, ['foo', '2017-01-01']) def test_tuple_comparison_subquery(self): PA = Person.alias('pa') subquery = (PA .select(PA.name, PA.id) .where(PA.name != 'huey')) query = (Person .select(Person.name) .where(Tuple(Person.name, Person.id).in_(subquery))) self.assertSQL(query, ( 'SELECT "t1"."name" FROM "person" AS "t1" ' 'WHERE (("t1"."name", "t1"."id") IN (' 'SELECT "pa"."name", "pa"."id" FROM "person" AS "pa" ' 'WHERE ("pa"."name" != ?)))'), ['huey']) def test_tuple_in_subquery(self): subq = (Tweet .select(Tweet.c.user_id, Tweet.c.content) .where(Tweet.c.content == 'special')) query = (User .select(User.c.id, User.c.username) .where(Tuple(User.c.id, User.c.username).in_(subq))) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."username" ' 'FROM "users" AS "t1" ' 'WHERE (("t1"."id", "t1"."username") IN (' 'SELECT "t2"."user_id", "t2"."content" ' 'FROM "tweets" AS "t2" ' 'WHERE ("t2"."content" = ?)))'), ['special']) def test_empty_in(self): query = User.select(User.c.id).where(User.c.username.in_([])) self.assertSQL(query, ( 'SELECT "t1"."id" FROM "users" AS "t1" ' 'WHERE (0 = 1)'), []) query = User.select(User.c.id).where(User.c.username.not_in([])) self.assertSQL(query, ( 'SELECT "t1"."id" FROM "users" AS "t1" ' 'WHERE (1 = 1)'), []) query = User.select(User.c.id).where(User.c.username.in_(Value([]))) self.assertSQL(query, ( 'SELECT "t1"."id" FROM "users" AS "t1" ' 'WHERE (0 = 1)'), []) def test_add_custom_op(self): def mod(lhs, rhs): return Expression(lhs, '%', rhs) Stat = Table('stats') query = (Stat .select(fn.COUNT(Stat.c.id)) .where(mod(Stat.c.index, 10) == 0)) self.assertSQL(query, ( 'SELECT COUNT("t1"."id") FROM "stats" AS "t1" ' 'WHERE (("t1"."index" % ?) = ?)'), [10, 0]) def test_where_convert_to_is_null(self): Note = Table('notes', ('id', 'content', 'user_id')) query = Note.select().where(Note.user_id == None) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."content", "t1"."user_id" ' 'FROM "notes" AS "t1" WHERE ("t1"."user_id" IS NULL)'), []) def test_like_escape(self): T = Table('tbl', ('key',)) def assertLike(expr, expected): query = T.select().where(expr) sql, params = __sql__(T.select().where(expr)) match_obj = re.search(r'\("t1"."key" (ILIKE[^\)]+)\)', sql) if match_obj is None: raise AssertionError('LIKE expression not found in query.') like, = match_obj.groups() self.assertEqual((like, params), expected) cases = ( (T.key.contains('base'), ('ILIKE ?', ['%base%'])), (T.key.contains('x_y'), ("ILIKE ? ESCAPE ?", ['%x\\_y%', '\\'])), (T.key.contains('__y'), ("ILIKE ? ESCAPE ?", ['%\\_\\_y%', '\\'])), (T.key.contains('%'), ("ILIKE ? ESCAPE ?", ['%\\%%', '\\'])), (T.key.contains('_%'), ("ILIKE ? ESCAPE ?", ['%\\_\\%%', '\\'])), (T.key.startswith('base'), ("ILIKE ?", ['base%'])), (T.key.startswith('x_y'), ("ILIKE ? ESCAPE ?", ['x\\_y%', '\\'])), (T.key.startswith('x%'), ("ILIKE ? ESCAPE ?", ['x\\%%', '\\'])), (T.key.startswith('_%'), ("ILIKE ? ESCAPE ?", ['\\_\\%%', '\\'])), (T.key.endswith('base'), ("ILIKE ?", ['%base'])), (T.key.endswith('x_y'), ("ILIKE ? ESCAPE ?", ['%x\\_y', '\\'])), (T.key.endswith('x%'), ("ILIKE ? ESCAPE ?", ['%x\\%', '\\'])), (T.key.endswith('_%'), ("ILIKE ? ESCAPE ?", ['%\\_\\%', '\\'])), ) for expr, expected in cases: assertLike(expr, expected) def test_like_expr(self): query = User.select(User.c.id).where(User.c.username.like('%foo%')) self.assertSQL(query, ( 'SELECT "t1"."id" FROM "users" AS "t1" ' 'WHERE ("t1"."username" LIKE ?)'), ['%foo%']) query = User.select(User.c.id).where(User.c.username.ilike('%foo%')) self.assertSQL(query, ( 'SELECT "t1"."id" FROM "users" AS "t1" ' 'WHERE ("t1"."username" ILIKE ?)'), ['%foo%']) def test_field_ops(self): query = User.select(User.c.id).where(User.c.username.regexp('[a-z]+')) self.assertSQL(query, ( 'SELECT "t1"."id" FROM "users" AS "t1" ' 'WHERE ("t1"."username" REGEXP ?)'), ['[a-z]+']) query = User.select(User.c.id).where(User.c.username.contains('abc')) self.assertSQL(query, ( 'SELECT "t1"."id" FROM "users" AS "t1" ' 'WHERE ("t1"."username" ILIKE ?)'), ['%abc%']) def test_bitwise_ops(self): query = User.select(User.c.id).where(User.c.id.bin_and(4)) self.assertSQL(query, ( 'SELECT "t1"."id" FROM "users" AS "t1" ' 'WHERE ("t1"."id" & ?)'), [4]) query = User.select(User.c.id).where(User.c.id.bin_or(1)) self.assertSQL(query, ( 'SELECT "t1"."id" FROM "users" AS "t1" ' 'WHERE ("t1"."id" | ?)'), [1]) def test_entity_escaping(self): Tbl = Table('te"st') query = Tbl.select(Tbl.c.id).where(Tbl.c.value > 5) self.assertSQL(query, ( 'SELECT "t1"."id" FROM "te""st" AS "t1" ' 'WHERE ("t1"."value" > ?)'), [5]) self.assertSQL(query, ( 'SELECT `t1`.`id` FROM `te"st` AS `t1` ' 'WHERE (`t1`.`value` > ?)'), [5], quote='``') class TestInsertQuery(BaseTestCase): def test_insert_simple(self): query = User.insert({ User.c.username: 'charlie', User.c.superuser: False, User.c.admin: True}) self.assertSQL(query, ( 'INSERT INTO "users" ("admin", "superuser", "username") ' 'VALUES (?, ?, ?)'), [True, False, 'charlie']) @requires_sqlite def test_replace_sqlite(self): query = User.replace({ User.c.username: 'charlie', User.c.superuser: False}) self.assertSQL(query, ( 'INSERT OR REPLACE INTO "users" ("superuser", "username") ' 'VALUES (?, ?)'), [False, 'charlie']) @requires_mysql def test_replace_mysql(self): query = User.replace({ User.c.username: 'charlie', User.c.superuser: False}) self.assertSQL(query, ( 'REPLACE INTO "users" ("superuser", "username") ' 'VALUES (?, ?)'), [False, 'charlie']) def test_insert_list(self): data = [ {Person.name: 'charlie'}, {Person.name: 'huey'}, {Person.name: 'zaizee'}] query = Person.insert(data) self.assertSQL(query, ( 'INSERT INTO "person" ("name") VALUES (?), (?), (?)'), ['charlie', 'huey', 'zaizee']) def test_insert_list_with_columns(self): data = [(i,) for i in ('charlie', 'huey', 'zaizee')] query = Person.insert(data, columns=[Person.name]) self.assertSQL(query, ( 'INSERT INTO "person" ("name") VALUES (?), (?), (?)'), ['charlie', 'huey', 'zaizee']) # Use column name instead of column instance. query = Person.insert(data, columns=['name']) self.assertSQL(query, ( 'INSERT INTO "person" ("name") VALUES (?), (?), (?)'), ['charlie', 'huey', 'zaizee']) def test_insert_list_infer_columns(self): data = [('p1', '1980-01-01'), ('p2', '1980-02-02')] self.assertSQL(Person.insert(data), ( 'INSERT INTO "person" ("name", "dob") VALUES (?, ?), (?, ?)'), ['p1', '1980-01-01', 'p2', '1980-02-02']) # Cannot infer any columns for User. data = [('u1',), ('u2',)] self.assertRaises(ValueError, User.insert(data).sql) # Note declares columns, but no primary key. So we would have to # include it for this to work. data = [(1, 'p1-n'), (2, 'p2-n')] self.assertRaises(ValueError, Note.insert(data).sql) data = [(1, 1, 'p1-n'), (2, 2, 'p2-n')] self.assertSQL(Note.insert(data), ( 'INSERT INTO "note" ("id", "person_id", "content") ' 'VALUES (?, ?, ?), (?, ?, ?)'), [1, 1, 'p1-n', 2, 2, 'p2-n']) def test_insert_query(self): source = User.select(User.c.username).where(User.c.admin == False) query = Person.insert(source, columns=[Person.name]) self.assertSQL(query, ( 'INSERT INTO "person" ("name") ' 'SELECT "t1"."username" FROM "users" AS "t1" ' 'WHERE ("t1"."admin" = ?)'), [False]) def test_insert_query_cte(self): cte = User.select(User.c.username).cte('foo') source = cte.select(cte.c.username) query = Person.insert(source, columns=[Person.name]).with_cte(cte) self.assertSQL(query, ( 'WITH "foo" AS (SELECT "t1"."username" FROM "users" AS "t1") ' 'INSERT INTO "person" ("name") ' 'SELECT "foo"."username" FROM "foo"'), []) def test_insert_single_value_query(self): query = Person.select(Person.id).where(Person.name == 'huey') insert = Note.insert({ Note.person_id: query, Note.content: 'hello'}) self.assertSQL(insert, ( 'INSERT INTO "note" ("content", "person_id") VALUES (?, ' '(SELECT "t1"."id" FROM "person" AS "t1" ' 'WHERE ("t1"."name" = ?)))'), ['hello', 'huey']) def test_insert_returning(self): query = (Person .insert({ Person.name: 'zaizee', Person.dob: datetime.date(2000, 1, 2)}) .returning(Person.id, Person.name, Person.dob)) self.assertSQL(query, ( 'INSERT INTO "person" ("dob", "name") ' 'VALUES (?, ?) ' 'RETURNING "person"."id", "person"."name", "person"."dob"'), [datetime.date(2000, 1, 2), 'zaizee']) query = query.returning(Person.id, Person.name.alias('new_name')) self.assertSQL(query, ( 'INSERT INTO "person" ("dob", "name") ' 'VALUES (?, ?) ' 'RETURNING "person"."id", "person"."name" AS "new_name"'), [datetime.date(2000, 1, 2), 'zaizee']) def test_insert_returning_expression(self): query = (Person .insert(name='huey') .returning(Person.id, Person.name, fn.LENGTH(Person.name).alias('ulen'))) self.assertSQL(query, ( 'INSERT INTO "person" ("name") VALUES (?) ' 'RETURNING "person"."id", ' '"person"."name", ' 'LENGTH("person"."name") AS "ulen"'), ['huey']) def test_empty(self): class Empty(TestModel): pass query = Empty.insert() if isinstance(db, MySQLDatabase): sql = 'INSERT INTO "empty" () VALUES ()' elif isinstance(db, PostgresqlDatabase): sql = 'INSERT INTO "empty" DEFAULT VALUES RETURNING "empty"."id"' else: sql = 'INSERT INTO "empty" DEFAULT VALUES' self.assertSQL(query, sql, []) class TestUpdateQuery(BaseTestCase): def test_update_query(self): query = (User .update({ User.c.username: 'nuggie', User.c.admin: False, User.c.counter: User.c.counter + 1}) .where(User.c.username == 'nugz')) self.assertSQL(query, ( 'UPDATE "users" SET ' '"admin" = ?, ' '"counter" = ("users"."counter" + ?), ' '"username" = ? ' 'WHERE ("users"."username" = ?)'), [False, 1, 'nuggie', 'nugz']) def test_update_subquery(self): count = fn.COUNT(Tweet.c.id).alias('ct') subquery = (User .select(User.c.id, count) .join(Tweet, on=(Tweet.c.user_id == User.c.id)) .group_by(User.c.id) .having(count > 100)) query = (User .update({ User.c.muted: True, User.c.counter: 0}) .where(User.c.id << subquery)) self.assertSQL(query, ( 'UPDATE "users" SET ' '"counter" = ?, ' '"muted" = ? ' 'WHERE ("users"."id" IN (' 'SELECT "users"."id", COUNT("t1"."id") AS "ct" ' 'FROM "users" AS "users" ' 'INNER JOIN "tweets" AS "t1" ' 'ON ("t1"."user_id" = "users"."id") ' 'GROUP BY "users"."id" ' 'HAVING ("ct" > ?)))'), [0, True, 100]) def test_update_value_subquery(self): subquery = (Tweet .select(fn.MAX(Tweet.c.id)) .where(Tweet.c.user_id == User.c.id)) query = (User .update({User.c.last_tweet_id: subquery}) .where(User.c.last_tweet_id.is_null(True))) self.assertSQL(query, ( 'UPDATE "users" SET ' '"last_tweet_id" = (SELECT MAX("t1"."id") FROM "tweets" AS "t1" ' 'WHERE ("t1"."user_id" = "users"."id")) ' 'WHERE ("users"."last_tweet_id" IS NULL)'), []) def test_update_from(self): data = [(1, 'u1-x'), (2, 'u2-x')] vl = ValuesList(data, columns=('id', 'username'), alias='tmp') query = (User .update(username=vl.c.username) .from_(vl) .where(User.c.id == vl.c.id)) self.assertSQL(query, ( 'UPDATE "users" SET "username" = "tmp"."username" ' 'FROM (VALUES (?, ?), (?, ?)) AS "tmp"("id", "username") ' 'WHERE ("users"."id" = "tmp"."id")'), [1, 'u1-x', 2, 'u2-x']) subq = vl.select(vl.c.id, vl.c.username) query = (User .update({User.c.username: subq.c.username}) .from_(subq) .where(User.c.id == subq.c.id)) self.assertSQL(query, ( 'UPDATE "users" SET "username" = "t1"."username" FROM (' 'SELECT "tmp"."id", "tmp"."username" ' 'FROM (VALUES (?, ?), (?, ?)) AS "tmp"("id", "username")) AS "t1" ' 'WHERE ("users"."id" = "t1"."id")'), [1, 'u1-x', 2, 'u2-x']) def test_update_from_subquery(self): subq = (Tweet .select(Tweet.c.user_id, fn.COUNT(Tweet.c.id).alias('ct')) .group_by(Tweet.c.user_id) .alias('tweet_ct')) query = (User .update({User.c.username: fn.CONCAT( User.c.username, ' (', subq.c.ct, ')')}) .from_(subq) .where(User.c.id == subq.c.user_id)) self.assertSQL(query, ( 'UPDATE "users" SET "username" = CONCAT(' '"users"."username", ?, "tweet_ct"."ct", ?) ' 'FROM (' 'SELECT "t1"."user_id", COUNT("t1"."id") AS "ct" ' 'FROM "tweets" AS "t1" ' 'GROUP BY "t1"."user_id") AS "tweet_ct" ' 'WHERE ("users"."id" = "tweet_ct"."user_id")'), [' (', ')']) def test_update_returning(self): query = (User .update({User.c.is_admin: True}) .where(User.c.username == 'charlie') .returning(User.c.id)) self.assertSQL(query, ( 'UPDATE "users" SET "is_admin" = ? WHERE ("users"."username" = ?) ' 'RETURNING "users"."id"'), [True, 'charlie']) query = query.returning(User.c.is_admin.alias('new_is_admin')) self.assertSQL(query, ( 'UPDATE "users" SET "is_admin" = ? WHERE ("users"."username" = ?) ' 'RETURNING "users"."is_admin" AS "new_is_admin"'), [True, 'charlie']) class TestDeleteQuery(BaseTestCase): def test_delete_query(self): query = (User .delete() .where(User.c.username != 'charlie') .limit(3)) self.assertSQL(query, ( 'DELETE FROM "users" WHERE ("users"."username" != ?) LIMIT ?'), ['charlie', 3]) def test_delete_subquery(self): count = fn.COUNT(Tweet.c.id).alias('ct') subquery = (User .select(User.c.id, count) .join(Tweet, on=(Tweet.c.user_id == User.c.id)) .group_by(User.c.id) .having(count > 100)) query = (User .delete() .where(User.c.id << subquery)) self.assertSQL(query, ( 'DELETE FROM "users" ' 'WHERE ("users"."id" IN (' 'SELECT "users"."id", COUNT("t1"."id") AS "ct" ' 'FROM "users" AS "users" ' 'INNER JOIN "tweets" AS "t1" ON ("t1"."user_id" = "users"."id") ' 'GROUP BY "users"."id" ' 'HAVING ("ct" > ?)))'), [100]) def test_delete_cte(self): cte = (User .select(User.c.id) .where(User.c.admin == True) .cte('u')) query = (User .delete() .where(User.c.id << cte.select(cte.c.id)) .with_cte(cte)) self.assertSQL(query, ( 'WITH "u" AS ' '(SELECT "t1"."id" FROM "users" AS "t1" WHERE ("t1"."admin" = ?)) ' 'DELETE FROM "users" ' 'WHERE ("users"."id" IN (SELECT "u"."id" FROM "u"))'), [True]) def test_delete_returning(self): query = (User .delete() .where(User.c.id > 2) .returning(User.c.username)) self.assertSQL(query, ( 'DELETE FROM "users" ' 'WHERE ("users"."id" > ?) ' 'RETURNING "users"."username"'), [2]) query = query.returning(User.c.id, User.c.username, SQL('1')) self.assertSQL(query, ( 'DELETE FROM "users" ' 'WHERE ("users"."id" > ?) ' 'RETURNING "users"."id", "users"."username", 1'), [2]) query = query.returning(User.c.id.alias('old_id')) self.assertSQL(query, ( 'DELETE FROM "users" ' 'WHERE ("users"."id" > ?) ' 'RETURNING "users"."id" AS "old_id"'), [2]) Register = Table('register', ('id', 'value', 'category')) class TestWindowFunctions(BaseTestCase): def test_partition_unordered(self): partition = [Register.category] query = (Register .select( Register.category, Register.value, fn.AVG(Register.value).over(partition_by=partition)) .order_by(Register.id)) self.assertSQL(query, ( 'SELECT "t1"."category", "t1"."value", AVG("t1"."value") ' 'OVER (PARTITION BY "t1"."category") ' 'FROM "register" AS "t1" ORDER BY "t1"."id"'), []) def test_ordered_unpartitioned(self): query = (Register .select( Register.value, fn.RANK().over(order_by=[Register.value]))) self.assertSQL(query, ( 'SELECT "t1"."value", RANK() OVER (ORDER BY "t1"."value") ' 'FROM "register" AS "t1"'), []) def test_ordered_partitioned(self): query = Register.select( Register.value, fn.SUM(Register.value).over( order_by=Register.id, partition_by=Register.category).alias('rsum')) self.assertSQL(query, ( 'SELECT "t1"."value", SUM("t1"."value") ' 'OVER (PARTITION BY "t1"."category" ORDER BY "t1"."id") AS "rsum" ' 'FROM "register" AS "t1"'), []) def test_empty_over(self): query = (Register .select(Register.value, fn.LAG(Register.value, 1).over()) .order_by(Register.value)) self.assertSQL(query, ( 'SELECT "t1"."value", LAG("t1"."value", ?) OVER () ' 'FROM "register" AS "t1" ' 'ORDER BY "t1"."value"'), [1]) def test_frame(self): query = (Register .select( Register.value, fn.AVG(Register.value).over( partition_by=[Register.category], start=Window.preceding(), end=Window.following(2)))) self.assertSQL(query, ( 'SELECT "t1"."value", AVG("t1"."value") ' 'OVER (PARTITION BY "t1"."category" ' 'ROWS BETWEEN UNBOUNDED PRECEDING AND 2 FOLLOWING) ' 'FROM "register" AS "t1"'), []) query = (Register .select(Register.value, fn.AVG(Register.value).over( partition_by=[Register.category], order_by=[Register.value], start=Window.CURRENT_ROW, end=Window.following()))) self.assertSQL(query, ( 'SELECT "t1"."value", AVG("t1"."value") ' 'OVER (PARTITION BY "t1"."category" ' 'ORDER BY "t1"."value" ' 'ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) ' 'FROM "register" AS "t1"'), []) def test_frame_types(self): def assertFrame(over_kwargs, expected): query = Register.select( Register.value, fn.SUM(Register.value).over(**over_kwargs)) sql, params = __sql__(query) match_obj = re.search(r'OVER \((.*?)\) FROM', sql) self.assertTrue(match_obj is not None) self.assertEqual(match_obj.groups()[0], expected) self.assertEqual(params, []) # No parameters -- empty OVER(). assertFrame({}, ('')) # Explicitly specify RANGE / ROWS frame-types. assertFrame({'frame_type': Window.RANGE}, 'RANGE UNBOUNDED PRECEDING') assertFrame({'frame_type': Window.ROWS}, 'ROWS UNBOUNDED PRECEDING') # Start and end boundaries. assertFrame({'start': Window.preceding(), 'end': Window.following()}, 'ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING') assertFrame({ 'start': Window.preceding(), 'end': Window.following(), 'frame_type': Window.RANGE, }, 'RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING') assertFrame({ 'start': Window.preceding(), 'end': Window.following(), 'frame_type': Window.ROWS, }, 'ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING') # Start boundary. assertFrame({'start': Window.preceding()}, 'ROWS UNBOUNDED PRECEDING') assertFrame({'start': Window.preceding(), 'frame_type': Window.RANGE}, 'RANGE UNBOUNDED PRECEDING') assertFrame({'start': Window.preceding(), 'frame_type': Window.ROWS}, 'ROWS UNBOUNDED PRECEDING') # Ordered or partitioned. assertFrame({'order_by': Register.value}, 'ORDER BY "t1"."value"') assertFrame({'frame_type': Window.RANGE, 'order_by': Register.value}, 'ORDER BY "t1"."value" RANGE UNBOUNDED PRECEDING') assertFrame({'frame_type': Window.ROWS, 'order_by': Register.value}, 'ORDER BY "t1"."value" ROWS UNBOUNDED PRECEDING') assertFrame({'partition_by': Register.category}, 'PARTITION BY "t1"."category"') assertFrame({ 'frame_type': Window.RANGE, 'partition_by': Register.category, }, 'PARTITION BY "t1"."category" RANGE UNBOUNDED PRECEDING') assertFrame({ 'frame_type': Window.ROWS, 'partition_by': Register.category, }, 'PARTITION BY "t1"."category" ROWS UNBOUNDED PRECEDING') # Ordering and boundaries. assertFrame({'order_by': Register.value, 'start': Window.CURRENT_ROW, 'end': Window.following()}, ('ORDER BY "t1"."value" ' 'ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING')) assertFrame({'order_by': Register.value, 'start': Window.CURRENT_ROW, 'end': Window.following(), 'frame_type': Window.RANGE}, ('ORDER BY "t1"."value" ' 'RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING')) assertFrame({'order_by': Register.value, 'start': Window.CURRENT_ROW, 'end': Window.following(), 'frame_type': Window.ROWS}, ('ORDER BY "t1"."value" ' 'ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING')) def test_running_total(self): EventLog = Table('evtlog', ('id', 'timestamp', 'data')) w = fn.SUM(EventLog.timestamp).over(order_by=[EventLog.timestamp]) query = (EventLog .select(EventLog.timestamp, EventLog.data, w.alias('elapsed')) .order_by(EventLog.timestamp)) self.assertSQL(query, ( 'SELECT "t1"."timestamp", "t1"."data", ' 'SUM("t1"."timestamp") OVER (ORDER BY "t1"."timestamp") ' 'AS "elapsed" ' 'FROM "evtlog" AS "t1" ORDER BY "t1"."timestamp"'), []) w = fn.SUM(EventLog.timestamp).over( order_by=[EventLog.timestamp], partition_by=[EventLog.data]) query = (EventLog .select(EventLog.timestamp, EventLog.data, w.alias('elapsed')) .order_by(EventLog.timestamp)) self.assertSQL(query, ( 'SELECT "t1"."timestamp", "t1"."data", ' 'SUM("t1"."timestamp") OVER ' '(PARTITION BY "t1"."data" ORDER BY "t1"."timestamp") AS "elapsed"' ' FROM "evtlog" AS "t1" ORDER BY "t1"."timestamp"'), []) def test_named_window(self): window = Window(partition_by=[Register.category]) query = (Register .select( Register.category, Register.value, fn.AVG(Register.value).over(window)) .window(window)) self.assertSQL(query, ( 'SELECT "t1"."category", "t1"."value", AVG("t1"."value") ' 'OVER "w" ' 'FROM "register" AS "t1" ' 'WINDOW "w" AS (PARTITION BY "t1"."category")'), []) window = Window( partition_by=[Register.category], order_by=[Register.value.desc()]) query = (Register .select( Register.value, fn.RANK().over(window)) .window(window)) self.assertSQL(query, ( 'SELECT "t1"."value", RANK() OVER "w" ' 'FROM "register" AS "t1" ' 'WINDOW "w" AS (' 'PARTITION BY "t1"."category" ' 'ORDER BY "t1"."value" DESC)'), []) def test_multiple_windows(self): w1 = Window(partition_by=[Register.category]).alias('w1') w2 = Window(order_by=[Register.value]).alias('w2') query = (Register .select( Register.value, fn.AVG(Register.value).over(w1), fn.RANK().over(w2)) .window(w1, w2)) self.assertSQL(query, ( 'SELECT "t1"."value", AVG("t1"."value") OVER "w1", ' 'RANK() OVER "w2" ' 'FROM "register" AS "t1" ' 'WINDOW "w1" AS (PARTITION BY "t1"."category"), ' '"w2" AS (ORDER BY "t1"."value")'), []) def test_alias_window(self): w = Window(order_by=Register.value).alias('wx') query = Register.select(Register.value, fn.RANK().over(w)).window(w) # We can re-alias the window and it's updated alias is reflected # correctly in the final query. w.alias('wz') self.assertSQL(query, ( 'SELECT "t1"."value", RANK() OVER "wz" ' 'FROM "register" AS "t1" ' 'WINDOW "wz" AS (ORDER BY "t1"."value")'), []) def test_reuse_window(self): EventLog = Table('evt', ('id', 'timestamp', 'key')) window = Window(partition_by=[EventLog.key], order_by=[EventLog.timestamp]) query = (EventLog .select(EventLog.timestamp, EventLog.key, fn.NTILE(4).over(window).alias('quartile'), fn.NTILE(5).over(window).alias('quintile'), fn.NTILE(100).over(window).alias('percentile')) .order_by(EventLog.timestamp) .window(window)) self.assertSQL(query, ( 'SELECT "t1"."timestamp", "t1"."key", ' 'NTILE(?) OVER "w" AS "quartile", ' 'NTILE(?) OVER "w" AS "quintile", ' 'NTILE(?) OVER "w" AS "percentile" ' 'FROM "evt" AS "t1" ' 'WINDOW "w" AS (' 'PARTITION BY "t1"."key" ORDER BY "t1"."timestamp") ' 'ORDER BY "t1"."timestamp"'), [4, 5, 100]) def test_filter_clause(self): condsum = fn.SUM(Register.value).filter(Register.value > 1).over( order_by=[Register.id], partition_by=[Register.category], start=Window.preceding(1)) query = (Register .select(Register.category, Register.value, condsum) .order_by(Register.category)) self.assertSQL(query, ( 'SELECT "t1"."category", "t1"."value", SUM("t1"."value") FILTER (' 'WHERE ("t1"."value" > ?)) OVER (PARTITION BY "t1"."category" ' 'ORDER BY "t1"."id" ROWS 1 PRECEDING) ' 'FROM "register" AS "t1" ' 'ORDER BY "t1"."category"'), [1]) def test_window_in_orderby(self): Register = Table('register', ['id', 'value']) w = Window(partition_by=[Register.value], order_by=[Register.id]) query = (Register .select() .window(w) .order_by(fn.FIRST_VALUE(Register.id).over(w))) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."value" FROM "register" AS "t1" ' 'WINDOW "w" AS (PARTITION BY "t1"."value" ORDER BY "t1"."id") ' 'ORDER BY FIRST_VALUE("t1"."id") OVER "w"'), []) fv = fn.FIRST_VALUE(Register.id).over( partition_by=[Register.value], order_by=[Register.id]) query = Register.select().order_by(fv) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."value" FROM "register" AS "t1" ' 'ORDER BY FIRST_VALUE("t1"."id") ' 'OVER (PARTITION BY "t1"."value" ORDER BY "t1"."id")'), []) def test_window_extends(self): Tbl = Table('tbl', ('b', 'c')) w1 = Window(partition_by=[Tbl.b], alias='win1') w2 = Window(extends=w1, order_by=[Tbl.c], alias='win2') query = Tbl.select(fn.GROUP_CONCAT(Tbl.c).over(w2)).window(w1, w2) self.assertSQL(query, ( 'SELECT GROUP_CONCAT("t1"."c") OVER "win2" FROM "tbl" AS "t1" ' 'WINDOW "win1" AS (PARTITION BY "t1"."b"), ' '"win2" AS ("win1" ORDER BY "t1"."c")'), []) w1 = Window(partition_by=[Tbl.b], alias='w1') w2 = Window(extends=w1).alias('w2') w3 = Window(extends=w2).alias('w3') w4 = Window(extends=w3, order_by=[Tbl.c]).alias('w4') query = (Tbl .select(fn.GROUP_CONCAT(Tbl.c).over(w4)) .window(w1, w2, w3, w4)) self.assertSQL(query, ( 'SELECT GROUP_CONCAT("t1"."c") OVER "w4" FROM "tbl" AS "t1" ' 'WINDOW "w1" AS (PARTITION BY "t1"."b"), "w2" AS ("w1"), ' '"w3" AS ("w2"), ' '"w4" AS ("w3" ORDER BY "t1"."c")'), []) def test_window_ranged(self): Tbl = Table('tbl', ('a', 'b')) query = (Tbl .select(Tbl.a, fn.SUM(Tbl.b).over( order_by=[Tbl.a.desc()], frame_type=Window.RANGE, start=Window.preceding(1), end=Window.following(2))) .order_by(Tbl.a.asc())) self.assertSQL(query, ( 'SELECT "t1"."a", SUM("t1"."b") OVER (' 'ORDER BY "t1"."a" DESC RANGE BETWEEN 1 PRECEDING AND 2 FOLLOWING)' ' FROM "tbl" AS "t1" ORDER BY "t1"."a" ASC'), []) query = (Tbl .select(Tbl.a, fn.SUM(Tbl.b).over( order_by=[Tbl.a], frame_type=Window.GROUPS, start=Window.preceding(3), end=Window.preceding(1)))) self.assertSQL(query, ( 'SELECT "t1"."a", SUM("t1"."b") OVER (' 'ORDER BY "t1"."a" GROUPS BETWEEN 3 PRECEDING AND 1 PRECEDING) ' 'FROM "tbl" AS "t1"'), []) query = (Tbl .select(Tbl.a, fn.SUM(Tbl.b).over( order_by=[Tbl.a], frame_type=Window.GROUPS, start=Window.following(1), end=Window.following(5)))) self.assertSQL(query, ( 'SELECT "t1"."a", SUM("t1"."b") OVER (' 'ORDER BY "t1"."a" GROUPS BETWEEN 1 FOLLOWING AND 5 FOLLOWING) ' 'FROM "tbl" AS "t1"'), []) def test_window_frametypes(self): Tbl = Table('tbl', ('b', 'c')) fts = (('as_range', Window.RANGE, 'RANGE'), ('as_rows', Window.ROWS, 'ROWS'), ('as_groups', Window.GROUPS, 'GROUPS')) for method, arg, sql in fts: w = getattr(Window(order_by=[Tbl.b + 1]), method)() self.assertSQL(Tbl.select(fn.SUM(Tbl.c).over(w)).window(w), ( 'SELECT SUM("t1"."c") OVER "w" FROM "tbl" AS "t1" ' 'WINDOW "w" AS (ORDER BY ("t1"."b" + ?) ' '%s UNBOUNDED PRECEDING)') % sql, [1]) query = Tbl.select(fn.SUM(Tbl.c) .over(order_by=[Tbl.b + 1], frame_type=arg)) self.assertSQL(query, ( 'SELECT SUM("t1"."c") OVER (ORDER BY ("t1"."b" + ?) ' '%s UNBOUNDED PRECEDING) FROM "tbl" AS "t1"') % sql, [1]) def test_window_frame_exclusion(self): Tbl = Table('tbl', ('b', 'c')) fts = ((Window.CURRENT_ROW, 'CURRENT ROW'), (Window.TIES, 'TIES'), (Window.NO_OTHERS, 'NO OTHERS'), (Window.GROUP, 'GROUP')) for arg, sql in fts: query = Tbl.select(fn.MAX(Tbl.b).over( order_by=[Tbl.c], start=Window.preceding(4), end=Window.following(), frame_type=Window.ROWS, exclude=arg)) self.assertSQL(query, ( 'SELECT MAX("t1"."b") OVER (ORDER BY "t1"."c" ' 'ROWS BETWEEN 4 PRECEDING AND UNBOUNDED FOLLOWING ' 'EXCLUDE %s) FROM "tbl" AS "t1"') % sql, []) def test_filter_window(self): # Example derived from sqlite window test 5.1.3.2. Tbl = Table('tbl', ('a', 'c')) win = Window(partition_by=fn.COALESCE(Tbl.a, ''), frame_type=Window.RANGE, start=Window.CURRENT_ROW, end=Window.following(), exclude=Window.NO_OTHERS) query = (Tbl .select(fn.SUM(Tbl.c).filter(Tbl.c < 5).over(win), fn.RANK().over(win), fn.DENSE_RANK().over(win)) .window(win)) self.assertSQL(query, ( 'SELECT SUM("t1"."c") FILTER (WHERE ("t1"."c" < ?)) OVER "w", ' 'RANK() OVER "w", DENSE_RANK() OVER "w" ' 'FROM "tbl" AS "t1" ' 'WINDOW "w" AS (PARTITION BY COALESCE("t1"."a", ?) ' 'RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING ' 'EXCLUDE NO OTHERS)'), [5, '']) class TestValuesList(BaseTestCase): _data = [(1, 'one'), (2, 'two'), (3, 'three')] def test_values_list(self): vl = ValuesList(self._data) query = vl.select(SQL('*')) self.assertSQL(query, ( 'SELECT * FROM (VALUES (?, ?), (?, ?), (?, ?)) AS "t1"'), [1, 'one', 2, 'two', 3, 'three']) def test_values_list_named_columns(self): vl = ValuesList(self._data).columns('idx', 'name') query = (vl .select(vl.c.idx, vl.c.name) .order_by(vl.c.idx)) self.assertSQL(query, ( 'SELECT "t1"."idx", "t1"."name" ' 'FROM (VALUES (?, ?), (?, ?), (?, ?)) AS "t1"("idx", "name") ' 'ORDER BY "t1"."idx"'), [1, 'one', 2, 'two', 3, 'three']) def test_named_values_list(self): vl = ValuesList(self._data, ['idx', 'name']).alias('vl') query = (vl .select(vl.c.idx, vl.c.name) .order_by(vl.c.idx)) self.assertSQL(query, ( 'SELECT "vl"."idx", "vl"."name" ' 'FROM (VALUES (?, ?), (?, ?), (?, ?)) AS "vl"("idx", "name") ' 'ORDER BY "vl"."idx"'), [1, 'one', 2, 'two', 3, 'three']) def test_docs_examples(self): data = [(1, 'first'), (2, 'second')] vl = ValuesList(data, columns=('idx', 'name')) query = (vl .select(vl.c.idx, vl.c.name) .order_by(vl.c.idx)) self.assertSQL(query, ( 'SELECT "t1"."idx", "t1"."name" ' 'FROM (VALUES (?, ?), (?, ?)) AS "t1"("idx", "name") ' 'ORDER BY "t1"."idx"'), [1, 'first', 2, 'second']) vl = ValuesList([(1, 'first'), (2, 'second')]) vl = vl.columns('idx', 'name').alias('v') query = vl.select(vl.c.idx, vl.c.name) self.assertSQL(query, ( 'SELECT "v"."idx", "v"."name" ' 'FROM (VALUES (?, ?), (?, ?)) AS "v"("idx", "name")'), [1, 'first', 2, 'second']) def test_join_on_valueslist(self): vl = ValuesList([('huey',), ('zaizee',)], columns=['username']) query = (User .select(vl.c.username) .join(vl, on=(User.c.username == vl.c.username)) .order_by(vl.c.username.desc())) self.assertSQL(query, ( 'SELECT "t1"."username" FROM "users" AS "t2" ' 'INNER JOIN (VALUES (?), (?)) AS "t1"("username") ' 'ON ("t2"."username" = "t1"."username") ' 'ORDER BY "t1"."username" DESC'), ['huey', 'zaizee']) class TestCaseFunction(BaseTestCase): def test_case_function(self): NameNum = Table('nn', ('name', 'number')) query = (NameNum .select(NameNum.name, Case(NameNum.number, ( (1, 'one'), (2, 'two')), '?').alias('num_str'))) self.assertSQL(query, ( 'SELECT "t1"."name", CASE "t1"."number" ' 'WHEN ? THEN ? ' 'WHEN ? THEN ? ' 'ELSE ? END AS "num_str" ' 'FROM "nn" AS "t1"'), [1, 'one', 2, 'two', '?']) query = (NameNum .select(NameNum.name, Case(None, ( (NameNum.number == 1, 'one'), (NameNum.number == 2, 'two')), '?'))) self.assertSQL(query, ( 'SELECT "t1"."name", CASE ' 'WHEN ("t1"."number" = ?) THEN ? ' 'WHEN ("t1"."number" = ?) THEN ? ' 'ELSE ? END ' 'FROM "nn" AS "t1"'), [1, 'one', 2, 'two', '?']) def test_multiple_case_expressions(self): Sample = Table('sample', ('id', 'counter', 'value')) case1 = Case(None, [ (Sample.counter < 5, 'low'), (Sample.counter < 10, 'mid')], 'high').alias('tier') case2 = Case(None, [ (Sample.value > 100, True)], False).alias('is_large') query = Sample.select(Sample.counter, case1, case2) self.assertSQL(query, ( 'SELECT "t1"."counter", ' 'CASE WHEN ("t1"."counter" < ?) THEN ? ' 'WHEN ("t1"."counter" < ?) THEN ? ' 'ELSE ? END AS "tier", ' 'CASE WHEN ("t1"."value" > ?) THEN ? ' 'ELSE ? END AS "is_large" ' 'FROM "sample" AS "t1"'), [5, 'low', 10, 'mid', 'high', 100, True, False]) def test_case_subquery(self): Name = Table('n', ('id', 'name',)) case = Case(None, [(Name.id.in_(Name.select(Name.id)), 1)], 0) q = Name.select(fn.SUM(case)) self.assertSQL(q, ( 'SELECT SUM(' 'CASE WHEN ("t1"."id" IN (SELECT "t1"."id" FROM "n" AS "t1")) ' 'THEN ? ELSE ? END) FROM "n" AS "t1"'), [1, 0]) case = Case(None, [ (Name.id < 5, Name.select(fn.SUM(Name.id))), (Name.id > 5, Name.select(fn.COUNT(Name.name)).distinct())], Name.select(fn.MAX(Name.id))) q = Name.select(Name.name, case.alias('magic')) self.assertSQL(q, ( 'SELECT "t1"."name", CASE ' 'WHEN ("t1"."id" < ?) ' 'THEN (SELECT SUM("t1"."id") FROM "n" AS "t1") ' 'WHEN ("t1"."id" > ?) ' 'THEN (SELECT DISTINCT COUNT("t1"."name") FROM "n" AS "t1") ' 'ELSE (SELECT MAX("t1"."id") FROM "n" AS "t1") END AS "magic" ' 'FROM "n" AS "t1"'), [5, 5]) class TestSelectFeatures(BaseTestCase): def test_reselect(self): query = Person.select(Person.name) self.assertSQL(query, 'SELECT "t1"."name" FROM "person" AS "t1"', []) query = query.columns(Person.id, Person.name, Person.dob) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."name", "t1"."dob" ' 'FROM "person" AS "t1"'), []) def test_distinct_on(self): query = (Note .select(Person.name, Note.content) .join(Person, on=(Note.person_id == Person.id)) .order_by(Person.name, Note.content) .distinct(Person.name)) self.assertSQL(query, ( 'SELECT DISTINCT ON ("t1"."name") ' '"t1"."name", "t2"."content" ' 'FROM "note" AS "t2" ' 'INNER JOIN "person" AS "t1" ON ("t2"."person_id" = "t1"."id") ' 'ORDER BY "t1"."name", "t2"."content"'), []) query = (Person .select(Person.name) .distinct(Person.name)) self.assertSQL(query, ( 'SELECT DISTINCT ON ("t1"."name") "t1"."name" ' 'FROM "person" AS "t1"'), []) def test_distinct(self): query = Person.select(Person.name).distinct() self.assertSQL(query, 'SELECT DISTINCT "t1"."name" FROM "person" AS "t1"', []) def test_distinct_count(self): query = Person.select(fn.COUNT(Person.name.distinct())) self.assertSQL(query, ( 'SELECT COUNT(DISTINCT "t1"."name") FROM "person" AS "t1"'), []) def test_filtered_count(self): filtered_count = (fn.COUNT(Person.name) .filter(Person.dob < datetime.date(2000, 1, 1))) query = Person.select(fn.COUNT(Person.name), filtered_count) self.assertSQL(query, ( 'SELECT COUNT("t1"."name"), COUNT("t1"."name") ' 'FILTER (WHERE ("t1"."dob" < ?)) ' 'FROM "person" AS "t1"'), [datetime.date(2000, 1, 1)]) def test_ordered_aggregate(self): agg = fn.array_agg(Person.name).order_by(Person.id.desc()) self.assertSQL(Person.select(agg.alias('names')), ( 'SELECT array_agg("t1"."name" ORDER BY "t1"."id" DESC) AS "names" ' 'FROM "person" AS "t1"'), []) agg = fn.string_agg(Person.name, ',').order_by(Person.dob, Person.id) self.assertSQL(Person.select(agg), ( 'SELECT string_agg("t1"."name", ? ORDER BY "t1"."dob", "t1"."id")' ' FROM "person" AS "t1"'), [',']) agg = (fn.string_agg(Person.name.concat('-x'), ',') .order_by(Person.name.desc(), Person.dob.asc())) self.assertSQL(Person.select(agg), ( 'SELECT string_agg(("t1"."name" || ?), ? ORDER BY "t1"."name" DESC' ', "t1"."dob" ASC) ' 'FROM "person" AS "t1"'), ['-x', ',']) agg = agg.order_by() self.assertSQL(Person.select(agg), ( 'SELECT string_agg(("t1"."name" || ?), ?) ' 'FROM "person" AS "t1"'), ['-x', ',']) def test_for_update(self): query = (Person .select() .where(Person.name == 'charlie') .for_update()) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."name", "t1"."dob" ' 'FROM "person" AS "t1" ' 'WHERE ("t1"."name" = ?) ' 'FOR UPDATE'), ['charlie'], for_update=True) query = query.for_update('FOR SHARE NOWAIT') self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."name", "t1"."dob" ' 'FROM "person" AS "t1" ' 'WHERE ("t1"."name" = ?) ' 'FOR SHARE NOWAIT'), ['charlie'], for_update=True) def test_for_update_nested(self): PA = Person.alias('pa') subq = PA.select(PA.id).where(PA.name == 'charlie').for_update() query = (Person .delete() .where(Person.id.in_(subq))) self.assertSQL(query, ( 'DELETE FROM "person" WHERE ("person"."id" IN (' 'SELECT "pa"."id" FROM "person" AS "pa" ' 'WHERE ("pa"."name" = ?) FOR UPDATE))'), ['charlie'], for_update=True) def test_for_update_options(self): query = (Person .select(Person.id) .where(Person.name == 'huey') .for_update(of=Person, nowait=True)) self.assertSQL(query, ( 'SELECT "t1"."id" FROM "person" AS "t1" WHERE ("t1"."name" = ?) ' 'FOR UPDATE OF "t1" NOWAIT'), ['huey'], for_update=True) # Check default behavior. query = query.for_update() self.assertSQL(query, ( 'SELECT "t1"."id" FROM "person" AS "t1" WHERE ("t1"."name" = ?) ' 'FOR UPDATE'), ['huey'], for_update=True) # Clear flag. query = query.for_update(None) self.assertSQL(query, ( 'SELECT "t1"."id" FROM "person" AS "t1" WHERE ("t1"."name" = ?)'), ['huey']) # Old-style is still supported. query = query.for_update('FOR UPDATE NOWAIT') self.assertSQL(query, ( 'SELECT "t1"."id" FROM "person" AS "t1" WHERE ("t1"."name" = ?) ' 'FOR UPDATE NOWAIT'), ['huey'], for_update=True) # Mix of old and new is OK. query = query.for_update('FOR SHARE NOWAIT', of=Person) self.assertSQL(query, ( 'SELECT "t1"."id" FROM "person" AS "t1" WHERE ("t1"."name" = ?) ' 'FOR SHARE OF "t1" NOWAIT'), ['huey'], for_update=True) def test_parentheses(self): query = (Person .select(fn.MAX( fn.IFNULL(1, 10) * 151, fn.IFNULL(None, 10)))) self.assertSQL(query, ( 'SELECT MAX((IFNULL(?, ?) * ?), IFNULL(?, ?)) ' 'FROM "person" AS "t1"'), [1, 10, 151, None, 10]) query = (Person .select(Person.name) .where(fn.EXISTS( User.select(User.c.id).where( User.c.username == Person.name)))) self.assertSQL(query, ( 'SELECT "t1"."name" FROM "person" AS "t1" ' 'WHERE EXISTS(' 'SELECT "t2"."id" FROM "users" AS "t2" ' 'WHERE ("t2"."username" = "t1"."name"))'), []) class TestExpressionSQL(BaseTestCase): def test_parentheses_functions(self): expr = (User.c.income + 100) expr2 = expr * expr query = User.select(fn.sum(expr), fn.avg(expr2)) self.assertSQL(query, ( 'SELECT sum("t1"."income" + ?), ' 'avg(("t1"."income" + ?) * ("t1"."income" + ?)) ' 'FROM "users" AS "t1"'), [100, 100, 100]) #Person = Table('person', ['id', 'name', 'dob']) class TestOnConflictSqlite(BaseTestCase): database = SqliteDatabase(None) def test_replace(self): query = Person.insert(name='huey').on_conflict('replace') self.assertSQL(query, ( 'INSERT OR REPLACE INTO "person" ("name") VALUES (?)'), ['huey']) def test_ignore(self): query = Person.insert(name='huey').on_conflict('ignore') self.assertSQL(query, ( 'INSERT OR IGNORE INTO "person" ("name") VALUES (?)'), ['huey']) def test_update_not_supported(self): query = Person.insert(name='huey').on_conflict( preserve=(Person.dob,), update={Person.name: Person.name.concat(' (updated)')}) with self.assertRaisesCtx(ValueError): self.database.get_sql_context().parse(query) class TestOnConflictMySQL(BaseTestCase): database = MySQLDatabase(None) def setUp(self): super(TestOnConflictMySQL, self).setUp() self.database.server_version = None def test_replace(self): query = Person.insert(name='huey').on_conflict('replace') self.assertSQL(query, ( 'REPLACE INTO "person" ("name") VALUES (?)'), ['huey']) def test_ignore(self): query = Person.insert(name='huey').on_conflict('ignore') self.assertSQL(query, ( 'INSERT IGNORE INTO "person" ("name") VALUES (?)'), ['huey']) def test_update(self): dob = datetime.date(2010, 1, 1) query = (Person .insert(name='huey', dob=dob) .on_conflict( preserve=(Person.dob,), update={Person.name: Person.name.concat('-x')})) self.assertSQL(query, ( 'INSERT INTO "person" ("dob", "name") VALUES (?, ?) ' 'ON DUPLICATE KEY ' 'UPDATE "dob" = VALUES("dob"), "name" = ("name" || ?)'), [dob, 'huey', '-x']) query = (Person .insert(name='huey', dob=dob) .on_conflict(preserve='dob')) self.assertSQL(query, ( 'INSERT INTO "person" ("dob", "name") VALUES (?, ?) ' 'ON DUPLICATE KEY ' 'UPDATE "dob" = VALUES("dob")'), [dob, 'huey']) def test_update_use_value_mariadb(self): # Verify that we use "VALUE" (not "VALUES") for MariaDB 10.3.3. dob = datetime.date(2010, 1, 1) query = (Person .insert(name='huey', dob=dob) .on_conflict(preserve=(Person.dob,))) self.database.server_version = (10, 3, 3) self.assertSQL(query, ( 'INSERT INTO "person" ("dob", "name") VALUES (?, ?) ' 'ON DUPLICATE KEY ' 'UPDATE "dob" = VALUE("dob")'), [dob, 'huey']) self.database.server_version = (10, 3, 2) self.assertSQL(query, ( 'INSERT INTO "person" ("dob", "name") VALUES (?, ?) ' 'ON DUPLICATE KEY ' 'UPDATE "dob" = VALUES("dob")'), [dob, 'huey']) def test_where_not_supported(self): query = Person.insert(name='huey').on_conflict( preserve=(Person.dob,), where=(Person.name == 'huey')) with self.assertRaisesCtx(ValueError): self.database.get_sql_context().parse(query) class TestOnConflictPostgresql(BaseTestCase): database = PostgresqlDatabase(None) def test_ignore(self): query = Person.insert(name='huey').on_conflict('ignore') self.assertSQL(query, ( 'INSERT INTO "person" ("name") VALUES (?) ' 'ON CONFLICT DO NOTHING'), ['huey']) def test_conflict_target_required(self): query = Person.insert(name='huey').on_conflict(preserve=(Person.dob,)) with self.assertRaisesCtx(ValueError): self.database.get_sql_context().parse(query) def test_conflict_resolution_required(self): query = Person.insert(name='huey').on_conflict(conflict_target='name') with self.assertRaisesCtx(ValueError): self.database.get_sql_context().parse(query) def test_conflict_update_excluded(self): KV = Table('kv', ('key', 'value', 'extra'), _database=self.database) query = (KV.insert(key='k1', value='v1', extra=1) .on_conflict(conflict_target=(KV.key, KV.value), update={KV.extra: EXCLUDED.extra + 2}, where=(EXCLUDED.extra < KV.extra))) self.assertSQL(query, ( 'INSERT INTO "kv" ("extra", "key", "value") VALUES (?, ?, ?) ' 'ON CONFLICT ("key", "value") DO UPDATE ' 'SET "extra" = (EXCLUDED."extra" + ?) ' 'WHERE (EXCLUDED."extra" < "kv"."extra")'), [1, 'k1', 'v1', 2]) def test_conflict_target_or_constraint(self): KV = Table('kv', ('key', 'value', 'extra'), _database=self.database) query = (KV.insert(key='k1', value='v1', extra='e1') .on_conflict(conflict_target=[KV.key, KV.value], preserve=[KV.extra])) self.assertSQL(query, ( 'INSERT INTO "kv" ("extra", "key", "value") VALUES (?, ?, ?) ' 'ON CONFLICT ("key", "value") DO UPDATE ' 'SET "extra" = EXCLUDED."extra"'), ['e1', 'k1', 'v1']) query = (KV.insert(key='k1', value='v1', extra='e1') .on_conflict(conflict_constraint='kv_key_value', preserve=[KV.extra])) self.assertSQL(query, ( 'INSERT INTO "kv" ("extra", "key", "value") VALUES (?, ?, ?) ' 'ON CONFLICT ON CONSTRAINT "kv_key_value" DO UPDATE ' 'SET "extra" = EXCLUDED."extra"'), ['e1', 'k1', 'v1']) query = KV.insert(key='k1', value='v1', extra='e1') self.assertRaises(ValueError, query.on_conflict, conflict_target=[KV.key, KV.value], conflict_constraint='kv_key_value') def test_update(self): dob = datetime.date(2010, 1, 1) query = (Person .insert(name='huey', dob=dob) .on_conflict( conflict_target=(Person.name,), preserve=(Person.dob,), update={Person.name: Person.name.concat('-x')})) self.assertSQL(query, ( 'INSERT INTO "person" ("dob", "name") VALUES (?, ?) ' 'ON CONFLICT ("name") DO ' 'UPDATE SET "dob" = EXCLUDED."dob", ' '"name" = ("person"."name" || ?)'), [dob, 'huey', '-x']) query = (Person .insert(name='huey', dob=dob) .on_conflict( conflict_target='name', preserve='dob')) self.assertSQL(query, ( 'INSERT INTO "person" ("dob", "name") VALUES (?, ?) ' 'ON CONFLICT ("name") DO ' 'UPDATE SET "dob" = EXCLUDED."dob"'), [dob, 'huey']) query = (Person .insert(name='huey') .on_conflict( conflict_target=Person.name, preserve=Person.dob, update={Person.name: Person.name.concat('-x')}, where=(Person.name != 'zaizee'))) self.assertSQL(query, ( 'INSERT INTO "person" ("name") VALUES (?) ' 'ON CONFLICT ("name") DO ' 'UPDATE SET "dob" = EXCLUDED."dob", ' '"name" = ("person"."name" || ?) ' 'WHERE ("person"."name" != ?)'), ['huey', '-x', 'zaizee']) def test_conflict_target_partial_index(self): KVE = Table('kve', ('key', 'value', 'extra')) data = [('k1', 1, 2), ('k2', 2, 3)] columns = [KVE.key, KVE.value, KVE.extra] query = (KVE .insert(data, columns) .on_conflict( conflict_target=(KVE.key, KVE.value), conflict_where=(KVE.extra > 1), preserve=(KVE.extra,), where=(KVE.key != 'kx'))) self.assertSQL(query, ( 'INSERT INTO "kve" ("key", "value", "extra") ' 'VALUES (?, ?, ?), (?, ?, ?) ' 'ON CONFLICT ("key", "value") WHERE ("extra" > ?) ' 'DO UPDATE SET "extra" = EXCLUDED."extra" ' 'WHERE ("kve"."key" != ?)'), ['k1', 1, 2, 'k2', 2, 3, 1, 'kx']) #Person = Table('person', ['id', 'name', 'dob']) #Note = Table('note', ['id', 'person_id', 'content']) class TestIndex(BaseTestCase): def test_simple_index(self): pidx = Index('person_name', Person, (Person.name,), unique=True) self.assertSQL(pidx, ( 'CREATE UNIQUE INDEX "person_name" ON "person" ("name")'), []) pidx = pidx.where(Person.dob > datetime.date(1950, 1, 1)) self.assertSQL(pidx, ( 'CREATE UNIQUE INDEX "person_name" ON "person" ' '("name") WHERE ("dob" > ?)'), [datetime.date(1950, 1, 1)]) def test_advanced_index(self): Article = Table('article') aidx = Index('foo_idx', Article, ( Article.c.status, Article.c.timestamp.desc(), fn.SUBSTR(Article.c.title, 1, 1)), safe=True) self.assertSQL(aidx, ( 'CREATE INDEX IF NOT EXISTS "foo_idx" ON "article" ' '("status", "timestamp" DESC, SUBSTR("title", ?, ?))'), [1, 1]) aidx = aidx.where(Article.c.flags.bin_and(4) == 4) self.assertSQL(aidx, ( 'CREATE INDEX IF NOT EXISTS "foo_idx" ON "article" ' '("status", "timestamp" DESC, SUBSTR("title", ?, ?)) ' 'WHERE (("flags" & ?) = ?)'), [1, 1, 4, 4]) # Check behavior when value-literals are enabled. self.assertSQL(aidx, ( 'CREATE INDEX IF NOT EXISTS "foo_idx" ON "article" ' '("status", "timestamp" DESC, SUBSTR("title", 1, 1)) ' 'WHERE (("flags" & 4) = 4)'), [], value_literals=True) def test_str_cols(self): uidx = Index('users_info', User, ('username DESC', 'id')) self.assertSQL(uidx, ( 'CREATE INDEX "users_info" ON "users" (username DESC, id)'), []) class TestSqlToString(BaseTestCase): def _test_sql_to_string(self, _param): class FakeDB(SqliteDatabase): param = _param db = FakeDB(None) T = Table('tbl', ('id', 'val')).bind(db) query = (T.select() .where((T.val == 'foo') | (T.val == b'bar') | (T.val == True) | (T.val == False) | (T.val == 2) | (T.val == -3.14) | (T.val == datetime.datetime(2018, 1, 1)) | (T.val == datetime.date(2018, 1, 2)) | T.val.is_null() | T.val.is_null(False) | T.val.in_(['aa', 'bb', 'cc']))) self.assertEqual(query_to_string(query), ( 'SELECT "t1"."id", "t1"."val" FROM "tbl" AS "t1" WHERE (((((((((((' '"t1"."val" = \'foo\') OR ' '("t1"."val" = \'bar\')) OR ' '("t1"."val" = 1)) OR ' '("t1"."val" = 0)) OR ' '("t1"."val" = 2)) OR ' '("t1"."val" = -3.14)) OR ' '("t1"."val" = \'2018-01-01 00:00:00\')) OR ' '("t1"."val" = \'2018-01-02\')) OR ' '("t1"."val" IS NULL)) OR ' '("t1"."val" IS NOT NULL)) OR ' '("t1"."val" IN (\'aa\', \'bb\', \'cc\')))')) def test_sql_to_string_qmark(self): self._test_sql_to_string('?') def test_sql_to_string_default(self): self._test_sql_to_string('%s') ================================================ FILE: tests/sqlcipher_ext.py ================================================ import datetime import os from hashlib import sha1 from peewee import DatabaseError from playhouse.sqlcipher_ext import * from playhouse.sqlite_ext import * from .base import ModelTestCase from .base import TestModel PASSPHRASE = 'testing sqlcipher' PRAGMAS = { 'kdf_iter': 10, # Much faster for testing. Totally unsafe. 'cipher_log_level': 'none', } db = SqlCipherDatabase('peewee_test.dbc', passphrase=PASSPHRASE, pragmas=PRAGMAS, rank_functions=True) @db.func('shazam') def shazam(s): return sha1((s or '').encode('utf-8')).hexdigest()[:5] class Thing(TestModel): name = CharField() class FTSNote(FTSModel, TestModel): content = TextField() class Note(TestModel): content = TextField() timestamp = DateTimeField(default=datetime.datetime.now) class CleanUpModelTestCase(ModelTestCase): def tearDown(self): super(CleanUpModelTestCase, self).tearDown() if os.path.exists(self.database.database): os.unlink(self.database.database) class SqlCipherTestCase(CleanUpModelTestCase): database = db requires = [Thing] def test_good_and_bad_passphrases(self): things = ('t1', 't2', 't3') for thing in things: Thing.create(name=thing) # Try to open db with wrong passphrase bad_db = SqlCipherDatabase(db.database, passphrase='wrong passphrase') self.assertRaises(DatabaseError, bad_db.get_tables) # Assert that we can still access the data with the good passphrase. query = Thing.select().order_by(Thing.name) self.assertEqual([t.name for t in query], ['t1', 't2', 't3']) def test_rekey(self): things = ('t1', 't2', 't3') for thing in things: Thing.create(name=thing) self.database.rekey('a new passphrase') db2 = SqlCipherDatabase(db.database, passphrase='a new passphrase', pragmas=PRAGMAS) cursor = db2.execute_sql('select name from thing order by name;') self.assertEqual([name for name, in cursor], ['t1', 't2', 't3']) query = Thing.select().order_by(Thing.name) self.assertEqual([t.name for t in query], ['t1', 't2', 't3']) self.database.close() self.database.connect() query = Thing.select().order_by(Thing.name) self.assertEqual([t.name for t in query], ['t1', 't2', 't3']) # Re-set to the original passphrase. self.database.rekey(PASSPHRASE) def test_empty_passphrase(self): db = SqlCipherDatabase(':memory:') class CM(TestModel): data = TextField() class Meta: database = db db.connect() db.create_tables([CM]) cm = CM.create(data='foo') cm_db = CM.get(CM.data == 'foo') self.assertEqual(cm_db.id, cm.id) self.assertEqual(cm_db.data, 'foo') config_db = SqlCipherDatabase('peewee_test.dbc', pragmas={ 'kdf_iter': 1234, 'cipher_page_size': 8192}, passphrase=PASSPHRASE) class TestSqlCipherConfiguration(CleanUpModelTestCase): database = config_db def test_configuration_via_pragma(self): # Write some data so the database file is created. self.database.execute_sql('create table foo (data TEXT)') self.database.close() self.database.connect() self.assertEqual(int(self.database.pragma('kdf_iter')), 1234) self.assertEqual(int(self.database.pragma('cipher_page_size')), 8192) self.assertTrue('foo' in self.database.get_tables()) class SqlCipherExtTestCase(CleanUpModelTestCase): database = db requires = [Note] def setUp(self): super(SqlCipherExtTestCase, self).setUp() FTSNote._meta.database = db FTSNote.drop_table(True) FTSNote.create_table(tokenize='porter', content=Note.content) def tearDown(self): FTSNote.drop_table(True) super(SqlCipherExtTestCase, self).tearDown() def test_fts(self): strings = [ 'python and peewee for working with databases', 'relational databases are the best', 'sqlite is the best relational database', 'sqlcipher is a cool database extension'] for s in strings: Note.create(content=s) FTSNote.rebuild() query = (FTSNote .select(FTSNote, FTSNote.rank().alias('score')) .where(FTSNote.match('relational databases')) .order_by(SQL('score').desc())) notes = [note.content for note in query] self.assertEqual(notes, [ 'relational databases are the best', 'sqlite is the best relational database']) alt_conn = SqliteDatabase(db.database) self.assertRaises( DatabaseError, alt_conn.execute_sql, 'SELECT * FROM "%s"' % (FTSNote._meta.table_name)) def test_func(self): Note.create(content='hello') Note.create(content='baz') Note.create(content='nug') query = (Note .select(Note.content, fn.shazam(Note.content).alias('shz')) .order_by(Note.id) .dicts()) results = list(query) self.assertEqual(results, [ {'content': 'hello', 'shz': 'aaf4c'}, {'content': 'baz', 'shz': 'bbe96'}, {'content': 'nug', 'shz': '52616'}, ]) ================================================ FILE: tests/sqlite.py ================================================ from decimal import Decimal as D import datetime import os import sys from peewee import * from peewee import sqlite3 from playhouse.sqlite_ext import * from .base import BaseTestCase from .base import IS_SQLITE_37 from .base import IS_SQLITE_9 from .base import ModelTestCase from .base import TestModel from .base import get_in_memory_db from .base import get_sqlite_db from .base import requires_models from .base import skip_if from .base import skip_unless from .base_models import Person from .base_models import Tweet from .base_models import User from .sqlite_helpers import compile_option from .sqlite_helpers import json_installed from .sqlite_helpers import json_patch_installed from .sqlite_helpers import json_text_installed from .sqlite_helpers import jsonb_installed database = SqliteDatabase(':memory:', rank_functions=True, timeout=100) try: from playhouse._sqlite_udf import peewee_rank CYTHON_EXTENSION = True except ImportError: CYTHON_EXTENSION = False class WeightedAverage(object): def __init__(self): self.total = 0. self.count = 0. def step(self, value, weight=None): weight = weight or 1. self.total += weight self.count += (weight * value) def finalize(self): if self.total != 0.: return self.count / self.total return 0. def _cmp(l, r): if l < r: return -1 return 1 if r < l else 0 def collate_reverse(s1, s2): return -_cmp(s1, s2) @database.collation() def collate_case_insensitive(s1, s2): return _cmp(s1.lower(), s2.lower()) def title_case(s): return s.title() @database.func() def rstrip(s, n): return s.rstrip(n) database.register_aggregate(WeightedAverage, 'weighted_avg', 1) database.register_aggregate(WeightedAverage, 'weighted_avg2', 2) database.register_collation(collate_reverse) database.register_function(title_case) class Post(TestModel): message = TextField() class ContentPost(FTSModel, Post): class Meta: options = { 'content': Post, 'tokenize': 'porter'} class ContentPostMessage(FTSModel, TestModel): message = TextField() class Meta: options = {'tokenize': 'porter', 'content': Post.message} class Document(FTSModel, TestModel): message = TextField() class Meta: options = {'tokenize': 'porter'} class MultiColumn(FTSModel, TestModel): c1 = SearchField() c2 = SearchField() c3 = SearchField() c4 = IntegerField() class Meta: options = {'tokenize': 'porter'} class RowIDModel(TestModel): rowid = RowIDField() data = IntegerField() class KeyData(TestModel): key = TextField() data = JSONField() class JBData(TestModel): key = TextField() data = JSONBField() class Values(TestModel): klass = IntegerField() value = FloatField() weight = FloatField() class FTS5Test(FTS5Model): title = SearchField() data = SearchField() misc = SearchField(unindexed=True) class Meta: legacy_table_names = False class FTS5Document(FTS5Model): message = SearchField() class Meta: options = {'tokenize': 'porter'} class DT(TestModel): key = TextField(primary_key=True) d = DateTimeField() iso = ISODateTimeField() @skip_unless(json_installed(), 'requires sqlite json1') class TestJSONField(ModelTestCase): database = database requires = [KeyData] def test_schema(self): self.assertSQL(KeyData._schema._create_table(), ( 'CREATE TABLE IF NOT EXISTS "key_data" (' '"id" INTEGER NOT NULL PRIMARY KEY, ' '"key" TEXT NOT NULL, ' '"data" JSON NOT NULL)'), []) def test_create_read_update(self): test_values = ( 'simple string', '', 1337, 0.0, True, False, ['foo', 'bar', ['baz', 'nug']], {'k1': 'v1', 'k2': {'x1': 'y1', 'x2': 'y2'}}, {'a': 1, 'b': 0.0, 'c': True, 'd': False, 'e': None, 'f': [0, 1], 'g': {'h': 'ijkl'}}, ) # Create a row using the given test value. Verify we can read the value # back from the database, and also that we can query for the row using # the value in the WHERE clause. for i, value in enumerate(test_values): # We can create and re-read values. KeyData.create(key='k%s' % i, data=value) kd_db = KeyData.get(KeyData.key == 'k%s' % i) self.assertEqual(kd_db.data, value) # We can read the data back using the value in the WHERE clause. kd_db = KeyData.get(KeyData.data == value) self.assertEqual(kd_db.key, 'k%s' % i) # Verify we can use values in UPDATE query. kd = KeyData.create(key='kx', data='') for value in test_values: nrows = (KeyData .update(data=value) .where(KeyData.key == 'kx') .execute()) self.assertEqual(nrows, 1) kd_db = KeyData.get(KeyData.key == 'kx') self.assertEqual(kd_db.data, value) def test_json_unicode(self): with self.database.atomic(): KeyData.delete().execute() # Two Chinese characters. unicode_str = b'\xe4\xb8\xad\xe6\x96\x87'.decode('utf8') data = {'foo': unicode_str} kd = KeyData.create(key='k1', data=data) kd_db = KeyData.get(KeyData.key == 'k1') self.assertEqual(kd_db.data, {'foo': unicode_str}) def test_json_to_json(self): kd1 = KeyData.create(key='k1', data={'k1': 'v1', 'k2': 'v2'}) subq = (KeyData .select(KeyData.data) .where(KeyData.key == 'k1')) # Assign value using a subquery. KeyData.create(key='k2', data=subq) kd2_db = KeyData.get(KeyData.key == 'k2') self.assertEqual(kd2_db.data, {'k1': 'v1', 'k2': 'v2'}) def test_json_bulk_update_top_level_list(self): kd1 = KeyData.create(key='k1', data=['a', 'b', 'c']) kd2 = KeyData.create(key='k2', data=['d', 'e', 'f']) kd1.data = ['g', 'h', 'i'] kd2.data = ['j', 'k', 'l'] KeyData.bulk_update([kd1, kd2], fields=[KeyData.data]) kd1_db = KeyData.get(KeyData.key == 'k1') kd2_db = KeyData.get(KeyData.key == 'k2') self.assertEqual(kd1_db.data, ['g', 'h', 'i']) self.assertEqual(kd2_db.data, ['j', 'k', 'l']) def test_json_bulk_update_top_level_dict(self): kd1 = KeyData.create(key='k1', data={'x': 'y1'}) kd2 = KeyData.create(key='k2', data={'x': 'y2'}) kd1.data = {'x': 'z1'} kd2.data = {'X': 'Z2'} KeyData.bulk_update([kd1, kd2], fields=[KeyData.data]) kd1_db = KeyData.get(KeyData.key == 'k1') kd2_db = KeyData.get(KeyData.key == 'k2') self.assertEqual(kd1_db.data, {'x': 'z1'}) self.assertEqual(kd2_db.data, {'X': 'Z2'}) def test_json_multi_ops(self): data = ( ('k1', [0, 1]), ('k2', [1, 2]), ('k3', {'x3': 'y3'}), ('k4', {'x4': 'y4'})) res = KeyData.insert_many(data).execute() if database.returning_clause: self.assertEqual([r for r, in res], [1, 2, 3, 4]) else: self.assertEqual(res, 4) vals = [[1, 2], [2, 3], {'x3': 'y3'}, {'x5': 'y5'}] pw_vals = [Value(v, unpack=False) for v in vals] query = KeyData.select().where(KeyData.data.in_(pw_vals)) self.assertSQL(query, ( 'SELECT "t1"."id", "t1"."key", "t1"."data" ' 'FROM "key_data" AS "t1" ' 'WHERE ("t1"."data" IN (json(?), json(?), json(?), json(?)))'), ['[1, 2]', '[2, 3]', '{"x3": "y3"}', '{"x5": "y5"}']) self.assertEqual(query.count(), 2) self.assertEqual(sorted([k.key for k in query]), ['k2', 'k3']) query = KeyData.select().where(KeyData.data == [1, 2]) self.assertEqual(query.count(), 1) self.assertEqual(query.get().key, 'k2') query = KeyData.select().where(KeyData.data == {'x3': 'y3'}) self.assertEqual(query.count(), 1) self.assertEqual(query.get().key, 'k3') def test_select_json_value(self): data = ( ('k1', {'a': {'b': 'c', 'd': [2, 1, 0]}}), ) KeyData.insert_many(data).execute() kd = (KeyData .select(KeyData.data['a'].alias('a')) .get()) self.assertEqual(kd.a, {'b': 'c', 'd': [2, 1, 0]}) kd = (KeyData .select(KeyData.data['a']['b'].alias('b')) .get()) self.assertEqual(kd.b, 'c') kd = (KeyData .select(KeyData.data['a']['d'].alias('d')) .get()) self.assertEqual(kd.d, [2, 1, 0]) kd = (KeyData .select(KeyData.data['a']['d'][0].alias('d0')) .get()) self.assertEqual(kd.d0, 2) @skip_unless(json_installed(), 'requires sqlite json1') class TestJSONFieldFunctions(ModelTestCase): database = database requires = [KeyData] test_data = [ ('a', {'k1': 'v1', 'x1': {'y1': 'z1'}}), ('b', {'k2': 'v2', 'x2': {'y2': 'z2'}}), ('c', {'k1': 'v1', 'k2': 'v2'}), ('d', {'x1': {'y1': 'z1', 'y2': 'z2'}}), ('e', {'l1': [0, 1, 2], 'l2': [1, [3, 3], 7]}), ] M = KeyData def setUp(self): super(TestJSONFieldFunctions, self).setUp() KeyData = self.M with self.database.atomic(): for key, data in self.test_data: KeyData.create(key=key, data=data) self.Q = KeyData.select().order_by(KeyData.key) def assertRows(self, where, expected): self.assertEqual([kd.key for kd in self.Q.where(where)], expected) def assertData(self, key, expected): KeyData = self.M self.assertEqual(KeyData.get(KeyData.key == key).data, expected) def test_json_group_functions(self): KeyData = self.M with self.database.atomic(): KeyData.delete().execute() for i in range(10): # e.g., {v: 0, v0: {items: []}}, {v: 2, v2: {items: [0, 1]}} KeyData.create(key='k%s' % i, data={'v': i, 'v%s' % i: { 'items': list(range(i))}}) jga_key = fn.json_group_array(KeyData.key) query = (KeyData .select(jga_key) .where(KeyData.data['v'] < 4) .order_by(KeyData.key)) self.assertEqual(json.loads(query.scalar()), ['k0', 'k1', 'k2', 'k3']) # Can specify json.loads as the converter for the function. query = (KeyData .select(jga_key.python_value(json.loads)) .where(KeyData.data['v'] > 6) .order_by(KeyData.key)) self.assertEqual(query.scalar(), ['k7', 'k8', 'k9']) # Aggregating a list of ints? jga_id = fn.json_group_array(KeyData.id) query = (KeyData .select(jga_id) .where(KeyData.data['v'] < 4) .order_by(KeyData.id)) self.assertEqual(json.loads(query.scalar()), [1, 2, 3, 4]) query = (KeyData .select(jga_id.python_value(json.loads)) .where(KeyData.data['v'] > 6) .order_by(KeyData.id)) self.assertEqual(query.scalar(), [8, 9, 10]) # Using json_group_object. jgo_key = fn.json_group_object(KeyData.key, KeyData.data['v']) res = (KeyData .select(jgo_key) .where(KeyData.data['v'] < 4) .scalar()) self.assertEqual(json.loads(res), {'k0': 0, 'k1': 1, 'k2': 2, 'k3': 3}) query = (KeyData .select(jgo_key.python_value(json.loads)) .where(KeyData.data['v'] < 4)) self.assertEqual(query.scalar(), {'k0': 0, 'k1': 1, 'k2': 2, 'k3': 3}) def test_extract(self): KeyData = self.M self.assertRows((KeyData.data['k1'] == 'v1'), ['a', 'c']) self.assertRows((KeyData.data['k2'] == 'v2'), ['b', 'c']) self.assertRows((KeyData.data['x1']['y1'] == 'z1'), ['a', 'd']) self.assertRows((KeyData.data['l1'][1] == 1), ['e']) self.assertRows((KeyData.data['l2'][1][1] == 3), ['e']) @skip_unless(json_text_installed()) def test_extract_text_json(self): KeyData = self.M D = KeyData.data self.assertRows((D.extract('$.k1') == 'v1'), ['a', 'c']) self.assertRows((D.extract_text('$.k1') == 'v1'), ['a', 'c']) self.assertRows((D.extract_json('$.k1') == '"v1"'), ['a', 'c']) self.assertRows((D.extract_text('k2') == 'v2'), ['b', 'c']) self.assertRows((D.extract_json('k2') == '"v2"'), ['b', 'c']) self.assertRows((D.extract_text('$.x1.y1') == 'z1'), ['a', 'd']) self.assertRows((D.extract_json('$.x1.y1') == '"z1"'), ['a', 'd']) self.assertRows((D.extract_text('$.l1[1]') == 1), ['e']) self.assertRows((D.extract_text('$.l2[1][1]') == 3), ['e']) self.assertRows((D.extract_json('x1') == '{"y1":"z1"}'), ['a']) def test_extract_multiple(self): KeyData = self.M query = KeyData.select( KeyData.key, KeyData.data.extract('$.k1', '$.k2').alias('keys')) self.assertEqual(sorted((k.key, k.keys) for k in query), [ ('a', ['v1', None]), ('b', [None, 'v2']), ('c', ['v1', 'v2']), ('d', [None, None]), ('e', [None, None])]) def test_insert(self): KeyData = self.M # Existing values are not overwritten. query = KeyData.update(data=KeyData.data['k1'].insert('v1-x')) self.assertEqual(query.execute(), 5) self.assertData('a', {'k1': 'v1', 'x1': {'y1': 'z1'}}) self.assertData('b', {'k1': 'v1-x', 'k2': 'v2', 'x2': {'y2': 'z2'}}) self.assertData('c', {'k1': 'v1', 'k2': 'v2'}) self.assertData('d', {'k1': 'v1-x', 'x1': {'y1': 'z1', 'y2': 'z2'}}) self.assertData('e', {'k1': 'v1-x', 'l1': [0, 1, 2], 'l2': [1, [3, 3], 7]}) def test_insert_json(self): KeyData = self.M set_json = KeyData.data['k1'].insert([0]) query = KeyData.update(data=set_json) self.assertEqual(query.execute(), 5) self.assertData('a', {'k1': 'v1', 'x1': {'y1': 'z1'}}) self.assertData('b', {'k1': [0], 'k2': 'v2', 'x2': {'y2': 'z2'}}) self.assertData('c', {'k1': 'v1', 'k2': 'v2'}) self.assertData('d', {'k1': [0], 'x1': {'y1': 'z1', 'y2': 'z2'}}) self.assertData('e', {'k1': [0], 'l1': [0, 1, 2], 'l2': [1, [3, 3], 7]}) def test_replace(self): KeyData = self.M # Only existing values are overwritten. query = KeyData.update(data=KeyData.data['k1'].replace('v1-x')) self.assertEqual(query.execute(), 5) self.assertData('a', {'k1': 'v1-x', 'x1': {'y1': 'z1'}}) self.assertData('b', {'k2': 'v2', 'x2': {'y2': 'z2'}}) self.assertData('c', {'k1': 'v1-x', 'k2': 'v2'}) self.assertData('d', {'x1': {'y1': 'z1', 'y2': 'z2'}}) self.assertData('e', {'l1': [0, 1, 2], 'l2': [1, [3, 3], 7]}) def test_replace_json(self): KeyData = self.M set_json = KeyData.data['k1'].replace([0]) query = KeyData.update(data=set_json) self.assertEqual(query.execute(), 5) self.assertData('a', {'k1': [0], 'x1': {'y1': 'z1'}}) self.assertData('b', {'k2': 'v2', 'x2': {'y2': 'z2'}}) self.assertData('c', {'k1': [0], 'k2': 'v2'}) self.assertData('d', {'x1': {'y1': 'z1', 'y2': 'z2'}}) self.assertData('e', {'l1': [0, 1, 2], 'l2': [1, [3, 3], 7]}) def test_set(self): KeyData = self.M query = (KeyData .update({KeyData.data: KeyData.data['k1'].set('v1-x')}) .where(KeyData.data['k1'] == 'v1')) self.assertEqual(query.execute(), 2) self.assertRows((KeyData.data['k1'] == 'v1-x'), ['a', 'c']) self.assertData('a', {'k1': 'v1-x', 'x1': {'y1': 'z1'}}) def test_set_json(self): KeyData = self.M set_json = KeyData.data['x1'].set({'y1': 'z1-x', 'y3': 'z3'}) query = (KeyData .update({KeyData.data: set_json}) .where(KeyData.data['x1']['y1'] == 'z1')) self.assertEqual(query.execute(), 2) self.assertRows((KeyData.data['x1']['y1'] == 'z1-x'), ['a', 'd']) self.assertData('a', {'k1': 'v1', 'x1': {'y1': 'z1-x', 'y3': 'z3'}}) self.assertData('d', {'x1': {'y1': 'z1-x', 'y3': 'z3'}}) def test_append(self): KeyData = self.M for value in ('ix', [], ['c1'], ['c1', 'c2'], {}, {'k1': 'v1'}, {'k1': 'v1', 'k2': 'v2'}, None, 1): KeyData.delete().execute() KeyData.create(key='a0', data=[]) KeyData.create(key='a1', data=['i1']) KeyData.create(key='a2', data=['i1', 'i2']) KeyData.create(key='n0', data={'arr': []}) KeyData.create(key='n1', data={'arr': ['i1']}) KeyData.create(key='n2', data={'arr': ['i1', 'i2']}) query = (KeyData .update(data=KeyData.data.append(value)) .where(KeyData.key.startswith('a'))) self.assertEqual(query.execute(), 3) query = (KeyData .select(KeyData.key, fn.json(KeyData.data)) .where(KeyData.key.startswith('a'))) self.assertEqual(sorted((row.key, row.data) for row in query), [('a0', [value]), ('a1', ['i1', value]), ('a2', ['i1', 'i2', value])]) query = (KeyData .update(data=KeyData.data['arr'].append(value)) .where(KeyData.key.startswith('n'))) self.assertEqual(query.execute(), 3) query = (KeyData .select(KeyData.key, fn.json(KeyData.data)) .where(KeyData.key.startswith('n'))) self.assertEqual(sorted((row.key, row.data) for row in query), [('n0', {'arr': [value]}), ('n1', {'arr': ['i1', value]}), ('n2', {'arr': ['i1', 'i2', value]})]) @skip_unless(json_patch_installed()) def test_update(self): KeyData = self.M merged = KeyData.data.update({'x1': {'y1': 'z1-x', 'y3': 'z3'}}) query = (KeyData .update({KeyData.data: merged}) .where(KeyData.data['x1']['y1'] == 'z1')) self.assertEqual(query.execute(), 2) self.assertRows((KeyData.data['x1']['y1'] == 'z1-x'), ['a', 'd']) self.assertData('a', {'k1': 'v1', 'x1': {'y1': 'z1-x', 'y3': 'z3'}}) self.assertData('d', {'x1': {'y1': 'z1-x', 'y2': 'z2', 'y3': 'z3'}}) @skip_unless(json_patch_installed()) def test_update_with_removal(self): KeyData = self.M m = KeyData.data.update({'k1': None, 'x1': {'y1': None, 'y3': 'z3'}}) query = KeyData.update(data=m).where(KeyData.data['x1']['y1'] == 'z1') self.assertEqual(query.execute(), 2) self.assertRows((KeyData.data['x1']['y3'] == 'z3'), ['a', 'd']) self.assertData('a', {'x1': {'y3': 'z3'}}) self.assertData('d', {'x1': {'y2': 'z2', 'y3': 'z3'}}) @skip_unless(json_patch_installed()) def test_update_nested(self): KeyData = self.M merged = KeyData.data['x1'].update({'y1': 'z1-x', 'y3': 'z3'}) query = (KeyData .update(data=merged) .where(KeyData.data['x1']['y1'] == 'z1')) self.assertEqual(query.execute(), 2) self.assertRows((KeyData.data['x1']['y1'] == 'z1-x'), ['a', 'd']) self.assertData('a', {'k1': 'v1', 'x1': {'y1': 'z1-x', 'y3': 'z3'}}) self.assertData('d', {'x1': {'y1': 'z1-x', 'y2': 'z2', 'y3': 'z3'}}) @skip_unless(json_patch_installed()) def test_updated_nested_with_removal(self): KeyData = self.M merged = KeyData.data['x1'].update({'o1': 'p1', 'y1': None}) nrows = (KeyData .update(data=merged) .where(KeyData.data['x1']['y1'] == 'z1') .execute()) self.assertRows((KeyData.data['x1']['o1'] == 'p1'), ['a', 'd']) self.assertData('a', {'k1': 'v1', 'x1': {'o1': 'p1'}}) self.assertData('d', {'x1': {'o1': 'p1', 'y2': 'z2'}}) def test_remove(self): KeyData = self.M query = (KeyData .update(data=KeyData.data['k1'].remove()) .where(KeyData.data['k1'] == 'v1')) self.assertEqual(query.execute(), 2) self.assertData('a', {'x1': {'y1': 'z1'}}) self.assertData('c', {'k2': 'v2'}) nrows = (KeyData .update(data=KeyData.data['l2'][1][1].remove()) .where(KeyData.key == 'e') .execute()) self.assertData('e', {'l1': [0, 1, 2], 'l2': [1, [3], 7]}) def test_simple_update(self): KeyData = self.M nrows = (KeyData .update(data={'foo': 'bar'}) .where(KeyData.key.in_(['a', 'b'])) .execute()) self.assertData('a', {'foo': 'bar'}) self.assertData('b', {'foo': 'bar'}) def test_children(self): KeyData = self.M children = KeyData.data.children().alias('children') query = (KeyData .select(KeyData.key, children.c.fullkey.alias('fullkey')) .from_(KeyData, children) .where(~children.c.fullkey.contains('k')) .order_by(KeyData.id, SQL('fullkey'))) accum = [(row.key, row.fullkey) for row in query] self.assertEqual(accum, [ ('a', '$.x1'), ('b', '$.x2'), ('d', '$.x1'), ('e', '$.l1'), ('e', '$.l2')]) def test_tree(self): KeyData = self.M tree = KeyData.data.tree().alias('tree') query = (KeyData .select(tree.c.fullkey.alias('fullkey')) .from_(KeyData, tree) .where(KeyData.key == 'd') .order_by(SQL('1')) .tuples()) self.assertEqual([fullkey for fullkey, in query], [ '$', '$.x1', '$.x1.y1', '$.x1.y2']) @skip_unless(jsonb_installed(), 'requires sqlite jsonb support') class TestJSONBFieldFunctions(TestJSONFieldFunctions): requires = [JBData] M = JBData def assertData(self, key, expected): q = JBData.select(fn.json(JBData.data)).where(JBData.key == key) self.assertEqual(q.get().data, expected) def test_extract_multiple(self): # We need to override this, otherwise we end up with jsonb returned. expr = fn.json(JBData.data.extract('$.k1', '$.k2')) query = JBData.select( JBData.key, expr.python_value(json.loads).alias('keys')) self.assertEqual(sorted((k.key, k.keys) for k in query), [ ('a', ['v1', None]), ('b', [None, 'v2']), ('c', ['v1', 'v2']), ('d', [None, None]), ('e', [None, None])]) class TestSqliteExtensions(BaseTestCase): def test_virtual_model(self): class Test(VirtualModel): class Meta: database = database extension_module = 'ext1337' legacy_table_names = False options = {'huey': 'cat', 'mickey': 'dog'} primary_key = False class SubTest(Test): pass self.assertSQL(Test._schema._create_table(), ( 'CREATE VIRTUAL TABLE IF NOT EXISTS "test" ' 'USING ext1337 ' '(huey=cat, mickey=dog)'), []) self.assertSQL(SubTest._schema._create_table(), ( 'CREATE VIRTUAL TABLE IF NOT EXISTS "sub_test" ' 'USING ext1337 ' '(huey=cat, mickey=dog)'), []) self.assertSQL( Test._schema._create_table(huey='kitten', zaizee='cat'), ('CREATE VIRTUAL TABLE IF NOT EXISTS "test" ' 'USING ext1337 (huey=kitten, mickey=dog, zaizee=cat)'), []) def test_autoincrement_field(self): class AutoIncrement(TestModel): id = AutoIncrementField() data = TextField() class Meta: database = database self.assertSQL(AutoIncrement._schema._create_table(), ( 'CREATE TABLE IF NOT EXISTS "auto_increment" ' '("id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, ' '"data" TEXT NOT NULL)'), []) class BaseFTSTestCase(object): messages = ( ('A faith is a necessity to a man. Woe to him who believes in ' 'nothing.'), ('All who call on God in true faith, earnestly from the heart, will ' 'certainly be heard, and will receive what they have asked and ' 'desired.'), ('Be faithful in small things because it is in them that your ' 'strength lies.'), ('Faith consists in believing when it is beyond the power of reason ' 'to believe.'), ('Faith has to do with things that are not seen and hope with things ' 'that are not at hand.')) values = ( ('aaaaa bbbbb ccccc ddddd', 'aaaaa ccccc', 'zzzzz zzzzz', 1), ('bbbbb ccccc ddddd eeeee', 'bbbbb', 'zzzzz', 2), ('ccccc ccccc ddddd fffff', 'ccccc', 'yyyyy', 3), ('ddddd', 'ccccc', 'xxxxx', 4)) def assertMessages(self, query, indexes): self.assertEqual([obj.message for obj in query], [self.messages[idx] for idx in indexes]) class TestFullTextSearch(BaseFTSTestCase, ModelTestCase): database = database requires = [ Post, ContentPost, ContentPostMessage, Document, MultiColumn] @requires_models(Document) def test_fts_insert_or_replace(self): # We can use replace to create a new row. n = Document.replace(docid=100, message='m100').execute() self.assertEqual(n, 100) self.assertEqual(Document.select().count(), 1) # We can use replace to update an existing row. n = Document.replace(docid=100, message='x100').execute() self.assertEqual(n, 100) self.assertEqual(Document.select().count(), 1) # Adds a new row. n = Document.replace(docid=101, message='x101').execute() self.assertEqual(n, 101) self.assertEqual(Document.select().count(), 2) query = Document.select().order_by(Document.message) self.assertEqual(list(query.tuples()), [(100, 'x100'), (101, 'x101')]) @requires_models(Document) def test_fts_manual(self): messages = [Document.create(message=message) for message in self.messages] query = (Document .select() .where(Document.match('believe')) .order_by(Document.docid)) self.assertMessages(query, [0, 3]) query = Document.search('believe') self.assertMessages(query, [3, 0]) # Test peewee's "rank" algorithm, as presented in the SQLite FTS3 docs. query = Document.search('things', with_score=True) self.assertEqual([(row.message, row.score) for row in query], [ (self.messages[4], -2. / 3), (self.messages[2], -1. / 3)]) # Test peewee's bm25 ranking algorithm. query = Document.search_bm25('things', with_score=True) self.assertEqual([(d.message, round(d.score, 2)) for d in query], [ (self.messages[4], -0.45), (self.messages[2], -0.36)]) # Another test of bm25 ranking. query = Document.search_bm25('believe', with_score=True) self.assertEqual([(d.message, round(d.score, 2)) for d in query], [ (self.messages[3], -0.49), (self.messages[0], -0.35)]) query = Document.search_bm25('god faith', with_score=True) self.assertEqual([(d.message, round(d.score, 2)) for d in query], [ (self.messages[1], -0.92)]) query = Document.search_bm25('"it is"', with_score=True) self.assertEqual([(d.message, round(d.score, 2)) for d in query], [ (self.messages[2], -0.36), (self.messages[3], -0.36)]) def test_fts_delete_row(self): posts = [Post.create(message=msg) for msg in self.messages] ContentPost.rebuild() query = (ContentPost .select(ContentPost, ContentPost.rank().alias('score')) .where(ContentPost.match('believe')) .order_by(ContentPost.docid)) self.assertMessages(query, [0, 3]) query = (ContentPost .select(ContentPost.docid) .order_by(ContentPost.docid)) for content_post in query: self.assertEqual(content_post.delete_instance(), 1) for post in posts: self.assertEqual( (ContentPost .delete() .where(ContentPost.message == post.message) .execute()), 1) # None of the deletes were processed since the table is managed. self.assertEqual(ContentPost.select().count(), 5) documents = [Document.create(message=message) for message in self.messages] self.assertEqual(Document.select().count(), 5) for document in documents: self.assertEqual( (Document .delete() .where(Document.message == document.message) .execute()), 1) self.assertEqual(Document.select().count(), 0) def _create_multi_column(self): for c1, c2, c3, c4 in self.values: MultiColumn.create(c1=c1, c2=c2, c3=c3, c4=c4) @requires_models(MultiColumn) def test_fts_multi_column(self): def assertResults(term, expected): results = [(x.c4, round(x.score, 2)) for x in MultiColumn.search(term, with_score=True)] self.assertEqual(results, expected) self._create_multi_column() assertResults('bbbbb', [ (2, -1.5), # 1/2 + 1/1 (1, -0.5)]) # 1/2 # `ccccc` appears four times in `c1`, three times in `c2`. assertResults('ccccc', [ (3, -.83), # 2/4 + 1/3 (1, -.58), # 1/4 + 1/3 (4, -.33), # 1/3 (2, -.25), # 1/4 ]) # `zzzzz` appears three times in c3. assertResults('zzzzz', [(1, -.67), (2, -.33)]) self.assertEqual( [x.score for x in MultiColumn.search('ddddd', with_score=True)], [-.25, -.25, -.25, -.25]) @requires_models(MultiColumn) def test_bm25(self): def assertResults(term, expected): query = MultiColumn.search_bm25(term, [1.0, 0, 0, 0], True) self.assertEqual( [(mc.c4, round(mc.score, 2)) for mc in query], expected) self._create_multi_column() MultiColumn.create(c1='aaaaa fffff', c4=5) assertResults('aaaaa', [(5, -0.39), (1, -0.3)]) assertResults('fffff', [(5, -0.39), (3, -0.3)]) assertResults('eeeee', [(2, -0.97)]) # No column specified, use the first text field. query = MultiColumn.search_bm25('fffff', [1.0, 0, 0, 0], True) self.assertEqual([(mc.c4, round(mc.score, 2)) for mc in query], [ (5, -0.39), (3, -0.3)]) # Use helpers. query = (MultiColumn .select( MultiColumn.c4, MultiColumn.bm25(1.0).alias('score')) .where(MultiColumn.match('aaaaa')) .order_by(SQL('score'))) self.assertEqual([(mc.c4, round(mc.score, 2)) for mc in query], [ (5, -0.39), (1, -0.3)]) def assertAllColumns(term, expected): query = MultiColumn.search_bm25(term, with_score=True) self.assertEqual( [(mc.c4, round(mc.score, 2)) for mc in query], expected) assertAllColumns('aaaaa ddddd', [(1, -1.08)]) assertAllColumns('zzzzz ddddd', [(1, -0.36), (2, -0.34)]) assertAllColumns('ccccc bbbbb ddddd', [(2, -1.39), (1, -0.3)]) @requires_models(Document) def test_bm25_alt_corpus(self): for message in self.messages: Document.create(message=message) query = Document.search_bm25('things', with_score=True) self.assertEqual([(d.message, round(d.score, 2)) for d in query], [ (self.messages[4], -0.45), (self.messages[2], -0.36)]) query = Document.search_bm25('believe', with_score=True) self.assertEqual([(d.message, round(d.score, 2)) for d in query], [ (self.messages[3], -0.49), (self.messages[0], -0.35)]) # Indeterminate order since all are 0.0. All phrases contain the word # faith, so there is no meaningful score. query = Document.search_bm25('faith', with_score=True) self.assertEqual([round(d.score, 2) for d in query], [-0.] * 5) def _test_fts_auto(self, ModelClass): posts = [] for message in self.messages: posts.append(Post.create(message=message)) # Nothing matches, index is not built. pq = ModelClass.select().where(ModelClass.match('faith')) self.assertEqual(list(pq), []) ModelClass.rebuild() ModelClass.optimize() # it will stem faithful -> faith b/c we use the porter tokenizer pq = (ModelClass .select() .where(ModelClass.match('faith')) .order_by(ModelClass.docid)) self.assertMessages(pq, range(len(self.messages))) pq = (ModelClass .select() .where(ModelClass.match('believe')) .order_by(ModelClass.docid)) self.assertMessages(pq, [0, 3]) pq = (ModelClass .select() .where(ModelClass.match('thin*')) .order_by(ModelClass.docid)) self.assertMessages(pq, [2, 4]) pq = (ModelClass .select() .where(ModelClass.match('"it is"')) .order_by(ModelClass.docid)) self.assertMessages(pq, [2, 3]) pq = ModelClass.search('things', with_score=True) self.assertEqual([(x.message, x.score) for x in pq], [ (self.messages[4], -2.0 / 3), (self.messages[2], -1.0 / 3), ]) pq = (ModelClass .select(ModelClass.rank()) .where(ModelClass.match('faithful')) .tuples()) self.assertEqual([x[0] for x in pq], [-.2] * 5) pq = (ModelClass .search('faithful', with_score=True) .dicts()) self.assertEqual([x['score'] for x in pq], [-.2] * 5) def test_fts_auto_model(self): self._test_fts_auto(ContentPost) def test_fts_auto_field(self): self._test_fts_auto(ContentPostMessage) def test_weighting(self): self._create_multi_column() def assertResults(method, term, weights, expected): results = [ (x.c4, round(x.score, 2)) for x in method(term, weights=weights, with_score=True)] self.assertEqual(results, expected) assertResults(MultiColumn.search, 'bbbbb', None, [ (2, -1.5), # 1/2 + 1/1 (1, -0.5), # 1/2 ]) assertResults(MultiColumn.search, 'bbbbb', [1., 5., 0.], [ (2, -5.5), # 1/2 + (5 * 1/1) (1, -0.5), # 1/2 + (5 * 0) ]) assertResults(MultiColumn.search, 'bbbbb', [1., .5, 0.], [ (2, -1.), # 1/2 + (.5 * 1/1) (1, -0.5), # 1/2 + (.5 * 0) ]) assertResults(MultiColumn.search, 'bbbbb', [1., -1., 0.], [ (1, -0.5), # 1/2 + (-1 * 0) (2, 0.5), # 1/2 + (-1 * 1/1) ]) # BM25 assertResults(MultiColumn.search_bm25, 'bbbbb', None, [ (2, -0.85), (1, -0.)]) assertResults(MultiColumn.search_bm25, 'bbbbb', [1., 5., 0.], [ (2, -4.24), (1, -0.)]) assertResults(MultiColumn.search_bm25, 'bbbbb', [1., .5, 0.], [ (2, -0.42), (1, -0.)]) assertResults(MultiColumn.search_bm25, 'bbbbb', [1., -1., 0.], [ (1, -0.), (2, 0.85)]) def test_fts_match_single_column(self): data = ( ('m1c1 aaaa', 'm1c2 bbbb', 'm1c3 cccc'), ('m2c1 dddd', 'm2c2 eeee', 'm2c3 ffff'), ('m3c1 cccc', 'm3c2 bbbb', 'm3c3 aaaa'), ) for c1, c2, c3 in data: MultiColumn.create(c1=c1, c2=c2, c3=c3, c4=0) def assertSearch(field, value, expected): query = (MultiColumn .select() .where(field.match(value)) .order_by(MultiColumn.c1)) self.assertEqual([mc.c1[:2] for mc in query], expected) assertSearch(MultiColumn.c1, 'aaaa', ['m1']) assertSearch(MultiColumn.c1, 'bbbb', []) assertSearch(MultiColumn.c1, 'cccc', ['m3']) assertSearch(MultiColumn.c2, 'bbbb', ['m1', 'm3']) assertSearch(MultiColumn.c2, 'eeee', ['m2']) assertSearch(MultiColumn.c3, 'cccc', ['m1']) assertSearch(MultiColumn.c3, 'aaaa', ['m3']) def test_fts_score_single_column(self): data = ( ('m1c1 aaaa', 'm1c2 bbbb', 'm1c3 cccc'), ('m2c1 dddd', 'm2c2 eeee', 'm2c3 ffff'), ('m3c1 cccc', 'm3c2 bbbb aaaa', 'm3c3 aaaa aaaa'), ) for c1, c2, c3 in data: MultiColumn.create(c1=c1, c2=c2, c3=c3, c4=0) def assertQueryScore(field, search_term, expected, *weights): rank = MultiColumn.bm25(*weights) query = (MultiColumn .select(MultiColumn, rank.alias('score')) .where(field.match(search_term)) .order_by(rank)) results = [(r.c1[:2], round(r.score, 2)) for r in query] self.assertEqual(results, expected) assertQueryScore(MultiColumn.c1, 'aaaa', [('m1', -0.51)]) assertQueryScore(MultiColumn.c1, 'dddd', [('m2', -0.51)]) assertQueryScore(MultiColumn.c2, 'bbbb', [('m1', -0.), ('m3', -0.)]) assertQueryScore(MultiColumn.c2, 'eeee', [('m2', -0.51)]) assertQueryScore(MultiColumn.c3, 'aaaa', [('m3', -0.62)]) assertQueryScore(MultiColumn.c1, 'aaaa', [('m1', -1.02)], 2., 0., 0.) assertQueryScore(MultiColumn.c2, 'bbbb', [('m1', -0.), ('m3', -0.)], 0., 1.0, 0.) assertQueryScore(MultiColumn.c2, 'eeee', [('m2', -1.02)], 0., 2., 0.) assertQueryScore(MultiColumn.c3, 'aaaa', [('m3', -0.31)], 0., 1., 0.5) @skip_unless(compile_option('enable_fts4')) @requires_models(MultiColumn) def test_match_column_queries(self): data = ( ('alpha one', 'apple aspires to ace artsy beta launch'), ('beta two', 'beta boasts better broadcast over apple'), ('gamma three', 'gold gray green gamma ray delta data'), ('delta four', 'delta data indicates downturn for apple beta'), ) MC = MultiColumn for i, (title, message) in enumerate(data): MC.create(c1=title, c2=message, c3='', c4=i) def assertQ(expr, idxscore): q = (MC .select(MC, MC.bm25().alias('score')) .where(expr) .order_by(SQL('score'), MC.c4)) self.assertEqual([(r.c4, round(r.score, 2)) for r in q], idxscore) # Single whitespace does not affect the mapping of col->term. We can # also store the column value in quotes if single-quotes are used. assertQ(MC.match('beta'), [(1, -0.85), (0, -0.), (3, -0.)]) assertQ(MC.match('c1:beta'), [(1, -0.85)]) assertQ(MC.match('c1: beta'), [(1, -0.85)]) assertQ(MC.match('c1: ^bet*'), [(1, -0.85)]) assertQ(MC.match('c1: \'beta\''), [(1, -0.85)]) assertQ(MC.match('"beta"'), [(1, -0.85), (0, -0.), (3, -0.)]) # Alternatively, just specify the column explicitly. assertQ(MC.c1.match('beta'), [(1, -0.85)]) assertQ(MC.c1.match(' beta '), [(1, -0.85)]) assertQ(MC.c1.match('"beta"'), [(1, -0.85)]) assertQ(MC.c1.match('"^bet*"'), [(1, -0.85)]) # apple beta delta gamma # 0 | alpha | X X # 1 | beta | X X # 2 | gamma | X X # 3 | delta | X X X # assertQ(MC.match('delta NOT gamma'), [(3, -0.85)]) assertQ(MC.match('delta NOT c2:gamma'), [(3, -0.85)]) assertQ(MC.match('"delta"'), [(3, -0.85), (2, -0.)]) assertQ(MC.match('c1:delta OR c2:delta'), [(3, -0.85), (2, -0.)]) assertQ(MC.match('"^delta"'), [(3, -1.69)]) assertQ(MC.match('(delta AND c2:apple) OR c1:alpha'), [(3, -0.85), (0, -0.85)]) assertQ(MC.match('(c2:delta AND c2:apple) OR c1:alpha'), [(0, -0.85), (3, -0.)]) assertQ(MC.match('c2:delta c2:apple OR c1:alpha'), [(0, -0.85), (3, -0.)]) assertQ(MC.match('(c2:delta AND c2:apple) OR beta'), [(1, -0.85), (3, -0.), (0, -0.)]) assertQ(MC.match('c2:delta AND (c2:apple OR c1:alpha)'), [(3, -0.)]) # c2 apple (0,1,3) OR (...irrelevant...). assertQ(MC.match('c2:apple OR c1:alpha NOT delta'), [(0, -0.85), (1, -0.), (3, -0.)]) assertQ(MC.match('c2:apple OR (c1:alpha NOT c2:delta)'), [(0, -0.85), (1, -0.), (3, -0.)]) # c2 apple OR c1 alpha (0, 1, 3) AND NOT delta (2, 3) -> (0, 1). assertQ(MC.match('(c2:apple OR c1:alpha) NOT delta'), [(0, -0.85), (1, -0.)]) @skip_unless(CYTHON_EXTENSION, 'requires _sqlite_udf c extension') class TestFullTextSearchCython(TestFullTextSearch): def test_bm25f(self): def assertResults(term, expected): query = MultiColumn.search_bm25f(term, [1.0, 0, 0, 0], True) self.assertEqual( [(mc.c4, round(mc.score, 2)) for mc in query], expected) self._create_multi_column() MultiColumn.create(c1='aaaaa fffff', c4=5) assertResults('aaaaa', [(5, -0.76), (1, -0.62)]) assertResults('fffff', [(5, -0.76), (3, -0.65)]) assertResults('eeeee', [(2, -2.13)]) # No column specified, use the first text field. query = MultiColumn.search_bm25f('aaaaa OR fffff', [1., 3., 0, 0], 1) self.assertEqual([(mc.c4, round(mc.score, 2)) for mc in query], [ (1, -14.18), (5, -12.01), (3, -11.48)]) def test_lucene(self): for message in self.messages: Document.create(message=message) def assertResults(term, expected, sort_cleaned=False): query = Document.search_lucene(term, with_score=True) cleaned = [ (round(doc.score, 3), ' '.join(doc.message.split()[:2])) for doc in query] if sort_cleaned: cleaned = sorted(cleaned) self.assertEqual(cleaned, expected) assertResults('things', [ (-0.166, 'Faith has'), (-0.137, 'Be faithful')]) assertResults('faith', [ (0.036, 'All who'), (0.042, 'Faith has'), (0.047, 'A faith'), (0.049, 'Be faithful'), (0.049, 'Faith consists')], sort_cleaned=True) @skip_unless(FTS5Model.fts5_installed(), 'requires fts5') class TestFTS5(BaseFTSTestCase, ModelTestCase): database = database requires = [FTS5Test] test_corpus = ( ('foo aa bb', 'aa bb cc ' * 10, 1), ('bar bb cc', 'bb cc dd ' * 9, 2), ('baze cc dd', 'cc dd ee ' * 8, 3), ('nug aa dd', 'bb cc ' * 7, 4)) def setUp(self): super(TestFTS5, self).setUp() for title, data, misc in self.test_corpus: FTS5Test.create(title=title, data=data, misc=misc) def test_create_table(self): query = FTS5Test._schema._create_table() self.assertSQL(query, ( 'CREATE VIRTUAL TABLE IF NOT EXISTS "fts5_test" USING fts5 ' '("title", "data", "misc" UNINDEXED)'), []) def test_custom_fts5_command(self): merge_sql = FTS5Test._fts_cmd_sql('merge', rank=4) self.assertSQL(merge_sql, ( 'INSERT INTO "fts5_test" ("fts5_test", "rank") VALUES (?, ?)'), ['merge', 4]) FTS5Test.merge(4) # Runs without error. FTS5Test.insert_many([{'title': 'k%08d' % i, 'data': 'v%08d' % i} for i in range(100)]).execute() FTS5Test.integrity_check(rank=0) FTS5Test.optimize() def test_create_table_options(self): class Test1(FTS5Model): f1 = SearchField() f2 = SearchField(unindexed=True) f3 = SearchField() class Meta: database = self.database options = { 'prefix': (2, 3), 'tokenize': 'porter unicode61', 'content': Post, 'content_rowid': Post.id} query = Test1._schema._create_table() self.assertSQL(query, ( 'CREATE VIRTUAL TABLE IF NOT EXISTS "test1" USING fts5 (' '"f1", "f2" UNINDEXED, "f3", ' 'content="post", content_rowid="id", ' 'prefix=\'2,3\', tokenize="porter unicode61")'), []) def assertResults(self, query, expected, scores=False, alias='score'): if scores: results = [(obj.title, round(getattr(obj, alias), 7)) for obj in query] else: results = [obj.title for obj in query] self.assertEqual(results, expected) def test_search(self): query = FTS5Test.search('bb') self.assertSQL(query, ( 'SELECT "t1"."rowid", "t1"."title", "t1"."data", "t1"."misc" ' 'FROM "fts5_test" AS "t1" ' 'WHERE ("fts5_test" MATCH ?) ORDER BY rank'), ['bb']) self.assertResults(query, ['nug aa dd', 'foo aa bb', 'bar bb cc']) self.assertResults(FTS5Test.search('baze OR dd'), ['baze cc dd', 'bar bb cc', 'nug aa dd']) @requires_models(FTS5Document) def test_fts_manual(self): messages = [FTS5Document.create(message=message) for message in self.messages] query = (FTS5Document .select() .where(FTS5Document.match('believe')) .order_by(FTS5Document.rowid)) self.assertMessages(query, [0, 3]) query = FTS5Document.search('believe') self.assertMessages(query, [3, 0]) # Test SQLite's built-in ranking algorithm (bm25). The results should # be comparable to our user-defined implementation. query = FTS5Document.search('things', with_score=True) self.assertEqual([(d.message, round(d.score, 2)) for d in query], [ (self.messages[4], -0.45), (self.messages[2], -0.37)]) # Another test of bm25 ranking. query = FTS5Document.search_bm25('believe', with_score=True) self.assertEqual([(d.message, round(d.score, 2)) for d in query], [ (self.messages[3], -0.49), (self.messages[0], -0.36)]) query = FTS5Document.search_bm25('god faith', with_score=True) self.assertEqual([(d.message, round(d.score, 2)) for d in query], [ (self.messages[1], -0.93)]) query = FTS5Document.search_bm25('"it is"', with_score=True) self.assertEqual([(d.message, round(d.score, 2)) for d in query], [ (self.messages[2], -0.37), (self.messages[3], -0.37)]) def test_match_column_queries(self): data = ( ('alpha one', 'apple aspires to ace artsy beta launch'), ('beta two', 'beta boasts better broadcast over apple'), ('gamma three', 'gold gray green gamma ray delta data'), ('delta four', 'delta data indicates downturn for apple beta'), ) FT = FTS5Test for i, (title, message) in enumerate(data): FT.create(title=title, data=message, misc=str(i)) def assertQ(expr, idxscore): q = (FT .select(FT, FT.bm25().alias('score')) .where(expr) .order_by(SQL('score'), FT.misc.cast('int'))) self.assertEqual([(int(r.misc), round(r.score, 2)) for r in q], idxscore) # Single whitespace does not affect the mapping of col->term. We can # also store the column value in quotes if single-quotes are used. assertQ(FT.match('beta'), [(1, -0.74), (0, -0.57), (3, -0.57)]) assertQ(FT.match('title: beta'), [(1, -2.08)]) assertQ(FT.match('title: ^bet*'), [(1, -2.08)]) assertQ(FT.match('title: "beta"'), [(1, -2.08)]) assertQ(FT.match('"beta"'), [(1, -0.74), (0, -0.57), (3, -0.57)]) # Alternatively, just specify the column explicitly. assertQ(FT.title.match('beta'), [(1, -2.08)]) assertQ(FT.title.match(' beta '), [(1, -2.08)]) assertQ(FT.title.match('"beta"'), [(1, -2.08)]) assertQ(FT.title.match('^bet*'), [(1, -2.08)]) assertQ(FT.title.match('"^bet*"'), []) # No wildcards in quotes! # apple beta delta gamma # 0 | alpha | X X # 1 | beta | X X # 2 | gamma | X X # 3 | delta | X X X # assertQ(FT.match('delta NOT gamma'), [(3, -1.53)]) assertQ(FT.match('delta NOT data:gamma'), [(3, -1.53)]) assertQ(FT.match('"delta"'), [(3, -1.53), (2, -1.2)]) assertQ(FT.match('title:delta OR data:delta'), [(3, -3.21), (2, -1.2)]) assertQ(FT.match('"^delta"'), [(3, -1.53), (2, -1.2)]) # Different. assertQ(FT.match('^delta'), [(3, -2.57)]) # Different from FTS4. assertQ(FT.match('(delta AND data:apple) OR title:alpha'), [(3, -2.09), (0, -2.02)]) assertQ(FT.match('(data:delta AND data:apple) OR title:alpha'), [(0, -2.02), (3, -1.76)]) assertQ(FT.match('data:delta data:apple OR title:alpha'), [(0, -2.02), (3, -1.76)]) assertQ(FT.match('(data:delta AND data:apple) OR beta'), [(3, -2.33), (1, -0.74), (0, -0.57)]) assertQ(FT.match('data:delta AND (data:apple OR title:alpha)'), [(3, -1.76)]) # data apple (0,1,3) OR (...irrelevant...). assertQ(FT.match('data:apple OR title:alpha NOT delta'), [(0, -2.58), (1, -0.58), (3, -0.57)]) assertQ(FT.match('data:apple OR (title:alpha NOT data:delta)'), [(0, -2.58), (1, -0.58), (3, -0.57)]) # data apple OR title alpha (0, 1, 3) AND NOT delta (2, 3) -> (0, 1). assertQ(FT.match('(data:apple OR title:alpha) NOT delta'), [(0, -2.58), (1, -0.58)]) def test_highlight_function(self): query = (FTS5Test .search('dd') .select(FTS5Test.title.highlight('[', ']').alias('hi'))) accum = [row.hi for row in query] self.assertEqual(accum, ['baze cc [dd]', 'bar bb cc', 'nug aa [dd]']) query = (FTS5Test .search('bb') .select(FTS5Test.data.highlight('[', ']').alias('hi'))) accum = [row.hi[:7] for row in query] self.assertEqual(accum, ['[bb] cc', 'aa [bb]', '[bb] cc']) def test_snippet_function(self): snip = FTS5Test.data.snippet('[', ']', max_tokens=5).alias('snip') query = FTS5Test.search('dd').select(snip) accum = [row.snip for row in query] self.assertEqual(accum, [ 'cc [dd] ee cc [dd]...', 'bb cc [dd] bb cc...', 'bb cc bb cc bb...']) def test_clean_query(self): cases = ( ('test', 'test'), ('"test"', '"test"'), ('"test\u2022"', '"test\u2022"'), ('test\u2022', 'test\u2022'), ('test-', 'test\x1a'), ('"test-"', '"test-"'), ('\\"test-', '\x1a test\x1a'), ('--test--', '\x1a\x1atest\x1a\x1a'), ('-test- "-test-"', '\x1atest\x1a "-test-"'), ) for a, b in cases: self.assertEqual(FTS5Test.clean_query(a), b) class TestUserDefinedCallbacks(ModelTestCase): database = database requires = [Post, Values] def test_custom_agg(self): data = ( (1, 3.4, 1.0), (1, 6.4, 2.3), (1, 4.3, 0.9), (2, 3.4, 1.4), (3, 2.7, 1.1), (3, 2.5, 1.1), ) for klass, value, wt in data: Values.create(klass=klass, value=value, weight=wt) vq = (Values .select( Values.klass, fn.weighted_avg(Values.value).alias('wtavg'), fn.avg(Values.value).alias('avg')) .group_by(Values.klass)) q_data = [(v.klass, v.wtavg, v.avg) for v in vq] self.assertEqual(q_data, [ (1, 4.7, 4.7), (2, 3.4, 3.4), (3, 2.6, 2.6)]) vq = (Values .select( Values.klass, fn.weighted_avg2(Values.value, Values.weight).alias('wtavg'), fn.avg(Values.value).alias('avg')) .group_by(Values.klass)) q_data = [(v.klass, str(v.wtavg)[:4], v.avg) for v in vq] self.assertEqual(q_data, [ (1, '5.23', 4.7), (2, '3.4', 3.4), (3, '2.6', 2.6)]) def test_custom_collation(self): for i in [1, 4, 3, 5, 2]: Post.create(message='p%d' % i) pq = Post.select().order_by(NodeList((Post.message, SQL('collate collate_reverse')))) self.assertEqual([p.message for p in pq], ['p5', 'p4', 'p3', 'p2', 'p1']) def test_collation_decorator(self): posts = [Post.create(message=m) for m in ['aaa', 'Aab', 'ccc', 'Bba', 'BbB']] pq = Post.select().order_by(collate_case_insensitive.collation(Post.message)) self.assertEqual([p.message for p in pq], [ 'aaa', 'Aab', 'Bba', 'BbB', 'ccc']) def test_custom_function(self): p1 = Post.create(message='this is a test') p2 = Post.create(message='another TEST') sq = Post.select().where(fn.title_case(Post.message) == 'This Is A Test') self.assertEqual(list(sq), [p1]) sq = Post.select(fn.title_case(Post.message)).tuples() self.assertEqual([x[0] for x in sq], [ 'This Is A Test', 'Another Test', ]) def test_function_decorator(self): [Post.create(message=m) for m in ['testing', 'chatting ', ' foo']] pq = Post.select(fn.rstrip(Post.message, 'ing')).order_by(Post.id) self.assertEqual([x[0] for x in pq.tuples()], [ 'test', 'chatting ', ' foo']) pq = Post.select(fn.rstrip(Post.message, ' ')).order_by(Post.id) self.assertEqual([x[0] for x in pq.tuples()], [ 'testing', 'chatting', ' foo']) def test_use_across_connections(self): db = get_in_memory_db() @db.func() def rev(s): return s[::-1] db.connect(); db.close(); db.connect() curs = db.execute_sql('select rev(?)', ('hello',)) self.assertEqual(curs.fetchone(), ('olleh',)) class TestRowIDField(ModelTestCase): database = database requires = [RowIDModel] def test_model_meta(self): self.assertEqual(RowIDModel._meta.sorted_field_names, ['rowid', 'data']) self.assertEqual(RowIDModel._meta.primary_key.name, 'rowid') self.assertTrue(RowIDModel._meta.auto_increment) def test_rowid_field(self): r1 = RowIDModel.create(data=10) self.assertEqual(r1.rowid, 1) self.assertEqual(r1.data, 10) r2 = RowIDModel.create(data=20) self.assertEqual(r2.rowid, 2) self.assertEqual(r2.data, 20) query = RowIDModel.select().where(RowIDModel.rowid == 2) self.assertSQL(query, ( 'SELECT "t1"."rowid", "t1"."data" ' 'FROM "row_id_model" AS "t1" ' 'WHERE ("t1"."rowid" = ?)'), [2]) r_db = query.get() self.assertEqual(r_db.rowid, 2) self.assertEqual(r_db.data, 20) r_db2 = query.columns(RowIDModel.rowid, RowIDModel.data).get() self.assertEqual(r_db2.rowid, 2) self.assertEqual(r_db2.data, 20) def test_insert_with_rowid(self): RowIDModel.insert({RowIDModel.rowid: 5, RowIDModel.data: 1}).execute() self.assertEqual(5, RowIDModel.select(RowIDModel.rowid).first().rowid) def test_insert_many_with_rowid_without_field_validation(self): RowIDModel.insert_many([{RowIDModel.rowid: 5, RowIDModel.data: 1}]).execute() self.assertEqual(5, RowIDModel.select(RowIDModel.rowid).first().rowid) def test_insert_many_with_rowid_with_field_validation(self): RowIDModel.insert_many([{RowIDModel.rowid: 5, RowIDModel.data: 1}]).execute() self.assertEqual(5, RowIDModel.select(RowIDModel.rowid).first().rowid) class CalendarMonth(TestModel): name = TextField() value = IntegerField() class CalendarDay(TestModel): month = ForeignKeyField(CalendarMonth, backref='days') value = IntegerField() class TestIntWhereChain(ModelTestCase): database = database requires = [CalendarMonth, CalendarDay] def test_int_where_chain(self): with self.database.atomic(): jan = CalendarMonth.create(name='january', value=1) feb = CalendarMonth.create(name='february', value=2) CalendarDay.insert_many([{'month': jan, 'value': i + 1} for i in range(31)]).execute() CalendarDay.insert_many([{'month': feb, 'value': i + 1} for i in range(28)]).execute() def assertValues(query, expected): self.assertEqual(sorted([d.value for d in query]), list(expected)) q = CalendarDay.select().join(CalendarMonth) jq = q.where(CalendarMonth.name == 'january') jq1 = jq.where(CalendarDay.value >= 25) assertValues(jq1, range(25, 32)) jq2 = jq1.where(CalendarDay.value < 30) assertValues(jq2, range(25, 30)) fq = q.where(CalendarMonth.name == 'february') fq1 = fq.where(CalendarDay.value >= 25) assertValues(fq1, range(25, 29)) fq2 = fq1.where(CalendarDay.value < 30) assertValues(fq2, range(25, 29)) class Datum(TestModel): a = BareField() b = BareField(collation='BINARY') c = BareField(collation='RTRIM') d = BareField(collation='NOCASE') class TestCollatedFieldDefinitions(ModelTestCase): database = get_in_memory_db() requires = [Datum] def test_collated_fields(self): rows = ( (1, 'abc', 'abc', 'abc ', 'abc'), (2, 'abc', 'abc', 'abc', 'ABC'), (3, 'abc', 'abc', 'abc ', 'Abc'), (4, 'abc', 'abc ', 'ABC', 'abc')) for pk, a, b, c, d in rows: Datum.create(id=pk, a=a, b=b, c=c, d=d) def assertC(query, expected): self.assertEqual([r.id for r in query], expected) base = Datum.select().order_by(Datum.id) # Text comparison a=b is performed using binary collating sequence. assertC(base.where(Datum.a == Datum.b), [1, 2, 3]) # Text comparison a=b is performed using the RTRIM collating sequence. assertC(base.where(Datum.a == Datum.b.collate('RTRIM')), [1, 2, 3, 4]) # Text comparison d=a is performed using the NOCASE collating sequence. assertC(base.where(Datum.d == Datum.a), [1, 2, 3, 4]) # Text comparison a=d is performed using the BINARY collating sequence. assertC(base.where(Datum.a == Datum.d), [1, 4]) # Text comparison 'abc'=c is performed using RTRIM collating sequence. assertC(base.where('abc' == Datum.c), [1, 2, 3]) # Text comparison c='abc' is performed using RTRIM collating sequence. assertC(base.where(Datum.c == 'abc'), [1, 2, 3]) # Grouping is performed using the NOCASE collating sequence (Values # 'abc', 'ABC', and 'Abc' are placed in the same group). query = Datum.select(fn.COUNT(Datum.id)).group_by(Datum.d) self.assertEqual(query.scalar(), 4) # Grouping is performed using the BINARY collating sequence. 'abc' and # 'ABC' and 'Abc' form different groups. query = Datum.select(fn.COUNT(Datum.id)).group_by(Datum.d.concat('')) self.assertEqual([r[0] for r in query.tuples()], [1, 1, 2]) # Sorting or column c is performed using the RTRIM collating sequence. assertC(base.order_by(Datum.c, Datum.id), [4, 1, 2, 3]) # Sorting of (c||'') is performed using the BINARY collating sequence. assertC(base.order_by(Datum.c.concat(''), Datum.id), [4, 2, 3, 1]) # Sorting of column c is performed using the NOCASE collating sequence. assertC(base.order_by(Datum.c.collate('NOCASE'), Datum.id), [2, 4, 3, 1]) class TestReadOnly(ModelTestCase): database = get_sqlite_db() @requires_models(User) def test_read_only(self): User.create(username='foo') db_filename = self.database.database db = SqliteDatabase('file:%s?mode=ro' % db_filename, uri=True) cursor = db.execute_sql('select username from users') self.assertEqual(cursor.fetchone(), ('foo',)) self.assertRaises(OperationalError, db.execute_sql, 'insert into users (username) values (?)', ('huey',)) # We cannot create a database if in read-only mode. db = SqliteDatabase('file:xx_not_exists.db?mode=ro', uri=True) self.assertRaises(OperationalError, db.connect) class TDecModel(TestModel): value = TDecimalField(max_digits=24, decimal_places=16, auto_round=True) class TestTDecimalField(ModelTestCase): database = database requires = [TDecModel] def test_tdecimal_field(self): value = D('12345678.0123456789012345') value_ov = D('12345678.012345678901234567890123456789') td1 = TDecModel.create(value=value) td2 = TDecModel.create(value=value_ov) td1_db = TDecModel.get(TDecModel.id == td1.id) self.assertEqual(td1_db.value, value) td2_db = TDecModel.get(TDecModel.id == td2.id) self.assertEqual(td2_db.value, D('12345678.0123456789012346')) class KVR(TestModel): key = TextField(primary_key=True) value = IntegerField() @skip_unless(database.server_version >= (3, 35, 0), 'sqlite returning clause required') class TestSqliteReturning(ModelTestCase): database = database requires = [Person, User, KVR] def test_sqlite_returning(self): iq = (User .insert_many([{'username': 'u%s' % i} for i in range(3)]) .returning(User.id)) self.assertEqual([r.id for r in iq.execute()], [1, 2, 3]) res = (User .insert_many([{'username': 'u%s' % i} for i in (4, 5)]) .returning(User) .execute()) self.assertEqual([(r.id, r.username) for r in res], [(4, 'u4'), (5, 'u5')]) # Simple insert returns the ID. res = User.insert(username='u6').execute() self.assertEqual(res, 6) iq = (User .insert_many([{'username': 'u%s' % i} for i in (7, 8, 9)]) .returning(User) .namedtuples()) curs = iq.execute() self.assertEqual([u.id for u in curs], [7, 8, 9]) def test_sqlite_on_conflict_returning(self): p = Person.create(first='f1', last='l1', dob='1990-01-01') self.assertEqual(p.id, 1) iq = Person.insert_many([ {'first': 'f%s' % i, 'last': 'l%s' %i, 'dob': '1990-01-%02d' % i} for i in range(1, 3)]) iq = iq.on_conflict(conflict_target=[Person.first, Person.last], update={'dob': '2000-01-01'}) p1, p2 = iq.returning(Person).execute() self.assertEqual((p1.first, p1.last), ('f1', 'l1')) self.assertEqual(p1.dob, datetime.date(2000, 1, 1)) self.assertEqual((p2.first, p2.last), ('f2', 'l2')) self.assertEqual(p2.dob, datetime.date(1990, 1, 2)) p3 = Person.insert(first='f3', last='l3', dob='1990-01-03').execute() self.assertEqual(p3, 3) def test_text_pk(self): res = KVR.create(key='k1', value=1) self.assertEqual((res.key, res.value), ('k1', 1)) res = KVR.insert(key='k2', value=2).execute() self.assertEqual(res, 2) #self.assertEqual(res, 'k2') # insert_many() returns the primary-key as usual. iq = (KVR .insert_many([{'key': 'k%s' % i, 'value': i} for i in (3, 4)]) .returning(KVR.key)) self.assertEqual([r.key for r in iq.execute()], ['k3', 'k4']) iq = KVR.insert_many([{'key': 'k%s' % i, 'value': i} for i in (4, 5)]) iq = iq.on_conflict(conflict_target=[KVR.key], update={KVR.value: KVR.value + 10}) res = iq.returning(KVR).execute() self.assertEqual([(r.key, r.value) for r in res], [('k4', 14), ('k5', 5)]) res = (KVR .update(value=KVR.value + 10) .where(KVR.key.in_(['k1', 'k3', 'kx'])) .returning(KVR) .execute()) self.assertEqual([(r.key, r.value) for r in res], [('k1', 11), ('k3', 13)]) res = (KVR.delete() .where(KVR.key.not_in(['k2', 'k3', 'k4'])) .returning(KVR) .execute()) self.assertEqual([(r.key, r.value) for r in res], [('k1', 11), ('k5', 5)]) @skip_unless(database.server_version >= (3, 35, 0), 'sqlite returning clause required') class TestSqliteReturningConfig(ModelTestCase): database = SqliteDatabase(':memory:', returning_clause=True) requires = [KVR, User] def test_pk_set_properly(self): user = User.create(username='u1') self.assertEqual(user.id, 1) kvr = KVR.create(key='k1', value=1) self.assertEqual(kvr.key, 'k1') def test_insert_behavior(self): iq = User.insert({'username': 'u1'}) self.assertEqual(iq.execute(), 1) iq = User.insert_many([{'username': 'u2'}, {'username': 'u3'}]) self.assertEqual(list(iq.execute()), [(2,), (3,)]) # NOTE: sqlite3_changes() does not return the inserted rowcount until # the statement has been consumed. The fact that it returned 2 is a # side-effect of the statement cache and our having consumed the query # in the previous test assertion. So this test is invalid. #iq = User.insert_many([('u4',), ('u5',)]).as_rowcount() #self.assertEqual(iq.execute(), 2) iq = KVR.insert({'key': 'k1', 'value': 1}) self.assertEqual(iq.execute(), 'k1') iq = KVR.insert_many([('k2', 2), ('k3', 3)]) self.assertEqual(list(iq.execute()), [('k2',), ('k3',)]) # See note above. #iq = KVR.insert_many([('k4', 4), ('k5', 5)]).as_rowcount() #self.assertEqual(iq.execute(), 2) def test_insert_on_conflict(self): KVR.create(key='k1', value=1) iq = (KVR.insert({'key': 'k1', 'value': 100}) .on_conflict(conflict_target=[KVR.key], update={KVR.value: KVR.value + 10})) self.assertEqual(iq.execute(), 'k1') self.assertEqual(KVR.get(KVR.key == 'k1').value, 11) KVR.create(key='k2', value=2) iq = (KVR.insert_many([ {'key': 'k1', 'value': 100}, {'key': 'k2', 'value': 200}, {'key': 'k3', 'value': 300}]) .on_conflict(conflict_target=[KVR.key], update={KVR.value: KVR.value + 10})) self.assertEqual(list(iq.execute()), [('k1',), ('k2',), ('k3',)]) self.assertEqual(sorted(KVR.select().tuples()), [('k1', 21), ('k2', 12), ('k3', 300)]) def test_update_delete_rowcounts(self): users = [User.create(username=u) for u in 'abc'] kvrs = [KVR.create(key='k%s' % i, value=i) for i in (1, 2, 3)] uq = User.update(username='c2').where(User.username == 'c') self.assertEqual(uq.execute(), 1) uq = User.update(username=User.username.concat('x')) self.assertEqual(uq.execute(), 3) dq = User.delete().where(User.username.in_(['bx', 'c2x'])) self.assertEqual(dq.execute(), 2) uq = KVR.update(value=KVR.value + 10).where(KVR.key == 'k3') self.assertEqual(uq.execute(), 1) uq = KVR.update(value=KVR.value + 100) self.assertEqual(uq.execute(), 3) dq = KVR.delete().where(KVR.value.in_([102, 113])) self.assertEqual(dq.execute(), 2) def test_update_delete_explicit_returning(self): users = [User.create(username=u) for u in 'abc'] uq = (User.update(username='c2') .where(User.username == 'c') .returning(User.id, User.username)) for _ in range(2): self.assertEqual([u.username for u in uq.execute()], ['c2']) self.assertEqual(list(uq.clone().execute()), []) uq = (User.update(username=User.username.concat('x')) .where(~User.username.endswith('x')) # For idempotency. .returning(User.id, User.username) .tuples()) for _ in range(2): self.assertEqual(sorted(uq.execute()), [(1, 'ax'), (2, 'bx'), (3, 'c2x')]) self.assertEqual(list(uq.clone().execute()), []) dq = User.delete().where(User.username == 'c2x').returning(User) for _ in range(2): # The result is cached to support multiple iterations. self.assertEqual([u.username for u in dq.execute()], ['c2x']) self.assertEqual(list(dq.clone().execute()), []) dq = User.delete().returning(User).tuples() for _ in range(2): # The result is cached to support multiple iterations. self.assertEqual(sorted(dq.execute()), [(1, 'ax'), (2, 'bx')]) self.assertEqual(list(dq.clone().execute()), []) def test_bulk_create_update(self): users = [User(username='u%s' % i) for i in range(5)] with self.assertQueryCount(1): User.bulk_create(users) self.assertEqual(User.select().count(), 5) self.assertEqual(sorted(User.select().tuples()), [ (1, 'u0'), (2, 'u1'), (3, 'u2'), (4, 'u3'), (5, 'u4')]) users[0].username = 'u0x' users[2].username = 'u2x' users[4].username = 'u4x' with self.assertQueryCount(1): n = User.bulk_update(users, ['username']) self.assertEqual(n, 5) self.assertEqual(sorted(User.select().tuples()), [ (1, 'u0x'), (2, 'u1'), (3, 'u2x'), (4, 'u3'), (5, 'u4x')]) @requires_models(User, Tweet) def test_fk_set_correctly(self): # Ensure FK can be set lazily. user = User(username='u1') tweet = Tweet(user=user, content='t1') user.save() tweet.save() @skip_unless(database.server_version >= (3, 20, 0), 'sqlite deterministic requires >= 3.20') @skip_unless(sys.version_info >= (3, 8, 0), 'sqlite deterministic requires Python >= 3.8') class TestDeterministicFunction(ModelTestCase): database = get_in_memory_db() def test_deterministic(self): db = self.database @db.func(deterministic=True) def pylower(s): if s is not None: return s.lower() class Reg(db.Model): key = TextField() class Meta: indexes = [ SQL('create unique index "reg_pylower_key" ' 'on "reg" (pylower("key"))')] db.create_tables([Reg]) Reg.create(key='k1') with self.assertRaises(IntegrityError): with db.atomic(): Reg.create(key='K1') @skip_unless(sys.version_info >= (3, 7, 0), 'isoformat (":") works 3.7+') class TestISODateTimeField(ModelTestCase): database = get_in_memory_db() requires = [DT] def test_aware_datetimes(self): class _UTC(datetime.tzinfo): def utcoffset(self, dt): return datetime.timedelta(0) def tzname(self, dt): return "UTC" def dst(self, dt): return datetime.timedelta(0) UTC = _UTC() d1 = datetime.datetime(2026, 1, 2, 3, 4, 5) d2 = d1.astimezone(UTC) dt = DT.create(key='k1', d=d1, iso=d2) self.assertEqual(dt.d, d1) self.assertEqual(dt.iso, d2) dt = DT['k1'] self.assertEqual(dt.d, d1) self.assertEqual(dt.iso, d2) raw = self.database.execute_sql('select * from dt').fetchone() self.assertEqual(raw, ('k1', str(d1), d2.isoformat())) # # If we have cysqlite, let's run tests on it. # try: from playhouse.cysqlite_ext import CySqliteDatabase except ImportError: pass else: cysqlite_database = CySqliteDatabase('peewee_test.db', timeout=100) cysqlite_database.register_aggregate(WeightedAverage, 'weighted_avg', 1) cysqlite_database.register_aggregate(WeightedAverage, 'weighted_avg2', 2) cysqlite_database.register_collation(collate_reverse) cysqlite_database.register_function(title_case) cysqlite_database.collation()(collate_case_insensitive) cysqlite_database.func()(rstrip) test_cases = [ TestJSONField, TestJSONFieldFunctions, TestJSONBFieldFunctions, TestSqliteExtensions, TestFullTextSearch, TestFTS5, TestUserDefinedCallbacks, TestRowIDField, TestIntWhereChain, TestCollatedFieldDefinitions, TestReadOnly, TestSqliteReturning, TestDeterministicFunction, TestISODateTimeField, # For various reasons these do not work. #TestJsonContains, #TestTDecimalField, #TestSqliteReturningConfig, ] for test_case in test_cases: new_name = test_case.__name__ + 'CySqlite' klass = type(new_name, (test_case,), { 'database': cysqlite_database, }) locals()[new_name] = klass ================================================ FILE: tests/sqlite_changelog.py ================================================ import datetime from peewee import * from playhouse.sqlite_changelog import ChangeLog from playhouse.sqlite_ext import JSONField from .base import ModelTestCase from .base import TestModel from .base import requires_models from .base import skip_unless from .sqlite_helpers import json_installed database = SqliteDatabase(':memory:', pragmas={'foreign_keys': 1}) class Person(TestModel): name = TextField() dob = DateField() class Note(TestModel): person = ForeignKeyField(Person, on_delete='CASCADE') content = TextField() timestamp = TimestampField() status = IntegerField(default=0) class CT1(TestModel): f1 = TextField() f2 = IntegerField(null=True) f3 = FloatField() fi = IntegerField() class CT2(TestModel): data = JSONField() # Diff of json? changelog = ChangeLog(database) CL = changelog.model @skip_unless(json_installed(), 'requires sqlite json1') class TestChangeLog(ModelTestCase): database = database requires = [Person, Note] def setUp(self): super(TestChangeLog, self).setUp() changelog.install(Person) changelog.install(Note, skip_fields=['timestamp']) self.last_index = 0 def assertChanges(self, changes, last_index=None): last_index = last_index or self.last_index query = (CL .select(CL.action, CL.table, CL.changes) .order_by(CL.id) .offset(last_index)) accum = list(query.tuples()) self.last_index += len(accum) self.assertEqual(accum, changes) def test_changelog(self): huey = Person.create(name='huey', dob=datetime.date(2010, 5, 1)) zaizee = Person.create(name='zaizee', dob=datetime.date(2013, 1, 1)) self.assertChanges([ ('INSERT', 'person', {'name': [None, 'huey'], 'dob': [None, '2010-05-01']}), ('INSERT', 'person', {'name': [None, 'zaizee'], 'dob': [None, '2013-01-01']})]) zaizee.dob = datetime.date(2013, 2, 2) zaizee.save() self.assertChanges([ ('UPDATE', 'person', {'dob': ['2013-01-01', '2013-02-02']})]) zaizee.name = 'zaizee-x' zaizee.dob = datetime.date(2013, 3, 3) zaizee.save() huey.save() # No changes. self.assertChanges([ ('UPDATE', 'person', {'name': ['zaizee', 'zaizee-x'], 'dob': ['2013-02-02', '2013-03-03']}), ('UPDATE', 'person', {})]) zaizee.delete_instance() self.assertChanges([ ('DELETE', 'person', {'name': ['zaizee-x', None], 'dob': ['2013-03-03', None]})]) nh1 = Note.create(person=huey, content='huey1', status=1) nh2 = Note.create(person=huey, content='huey2', status=2) self.assertChanges([ ('INSERT', 'note', {'person_id': [None, huey.id], 'content': [None, 'huey1'], 'status': [None, 1]}), ('INSERT', 'note', {'person_id': [None, huey.id], 'content': [None, 'huey2'], 'status': [None, 2]})]) nh1.content = 'huey1-x' nh1.status = 0 nh1.save() mickey = Person.create(name='mickey', dob=datetime.date(2009, 8, 1)) nh2.person = mickey nh2.save() self.assertChanges([ ('UPDATE', 'note', {'content': ['huey1', 'huey1-x'], 'status': [1, 0]}), ('INSERT', 'person', {'name': [None, 'mickey'], 'dob': [None, '2009-08-01']}), ('UPDATE', 'note', {'person_id': [huey.id, mickey.id]})]) mickey.delete_instance() self.assertChanges([ ('DELETE', 'note', {'person_id': [mickey.id, None], 'content': ['huey2', None], 'status': [2, None]}), ('DELETE', 'person', {'name': ['mickey', None], 'dob': ['2009-08-01', None]})]) @requires_models(CT1) def test_changelog_details(self): changelog.install(CT1, skip_fields=['fi'], insert=False, delete=False) c1 = CT1.create(f1='v1', f2=1, f3=1.5, fi=0) self.assertChanges([]) CT1.update(f1='v1-x', f2=2, f3=2.5, fi=1).execute() self.assertChanges([ ('UPDATE', 'ct1', { 'f1': ['v1', 'v1-x'], 'f2': [1, 2], 'f3': [1.5, 2.5]})]) c1.f2 = None c1.save() # Overwrites previously-changed fields. self.assertChanges([('UPDATE', 'ct1', { 'f1': ['v1-x', 'v1'], 'f2': [2, None], 'f3': [2.5, 1.5]})]) c1.delete_instance() self.assertChanges([]) @requires_models(CT2) def test_changelog_jsonfield(self): changelog.install(CT2) ca = CT2.create(data={'k1': 'v1'}) cb = CT2.create(data=['i0', 'i1', 'i2']) cc = CT2.create(data='hello') self.assertChanges([ ('INSERT', 'ct2', {'data': [None, {'k1': 'v1'}]}), ('INSERT', 'ct2', {'data': [None, ['i0', 'i1', 'i2']]}), ('INSERT', 'ct2', {'data': [None, 'hello']})]) ca.data['k1'] = 'v1-x' cb.data.append('i3') cc.data = 'world' ca.save() cb.save() cc.save() self.assertChanges([ ('UPDATE', 'ct2', {'data': [{'k1': 'v1'}, {'k1': 'v1-x'}]}), ('UPDATE', 'ct2', {'data': [['i0', 'i1', 'i2'], ['i0', 'i1', 'i2', 'i3']]}), ('UPDATE', 'ct2', {'data': ['hello', 'world']})]) cc.data = 13.37 cc.save() self.assertChanges([('UPDATE', 'ct2', {'data': ['world', 13.37]})]) ca.delete_instance() self.assertChanges([ ('DELETE', 'ct2', {'data': [{'k1': 'v1-x'}, None]})]) ================================================ FILE: tests/sqlite_helpers.py ================================================ from peewee import sqlite3 def json_installed(): if sqlite3.sqlite_version_info < (3, 9, 0): return False tmp_db = sqlite3.connect(':memory:') try: tmp_db.execute('select json(?)', (1337,)) except: return False finally: tmp_db.close() return True def json_patch_installed(): return sqlite3.sqlite_version_info >= (3, 18, 0) def json_text_installed(): return sqlite3.sqlite_version_info >= (3, 38, 0) def jsonb_installed(): return sqlite3.sqlite_version_info >= (3, 45, 0) def compile_option(p): if not hasattr(compile_option, '_pragma_cache'): conn = sqlite3.connect(':memory:') curs = conn.execute('pragma compile_options') opts = [opt.lower().split('=')[0].strip() for opt, in curs.fetchall()] compile_option._pragma_cache = set(opts) return p in compile_option._pragma_cache ================================================ FILE: tests/sqlite_udf.py ================================================ import datetime import json import random from peewee import * from peewee import sqlite3 from playhouse.sqlite_udf import register_all from .base import IS_SQLITE_9 from .base import ModelTestCase from .base import TestModel from .base import get_sqlite_db from .base import skip_unless try: from playhouse import _sqlite_udf as cython_udf except ImportError: cython_udf = None def requires_cython(method): return skip_unless(cython_udf is not None, 'requires sqlite udf c extension')(method) database = get_sqlite_db() register_all(database) class User(TestModel): username = TextField() class APIResponse(TestModel): url = TextField(default='') data = TextField(default='') timestamp = DateTimeField(default=datetime.datetime.now) class Generic(TestModel): value = IntegerField(default=0) x = Field(null=True) MODELS = [User, APIResponse, Generic] class FixedOffset(datetime.tzinfo): def __init__(self, offset, name, dstoffset=42): if isinstance(offset, int): offset = datetime.timedelta(minutes=offset) if isinstance(dstoffset, int): dstoffset = datetime.timedelta(minutes=dstoffset) self.__offset = offset self.__name = name self.__dstoffset = dstoffset def utcoffset(self, dt): return self.__offset def tzname(self, dt): return self.__name def dst(self, dt): return self.__dstoffset class BaseTestUDF(ModelTestCase): database = database def sql1(self, sql, *params): cursor = self.database.execute_sql(sql, params) return cursor.fetchone()[0] class TestAggregates(BaseTestUDF): requires = [Generic] def _store_values(self, *values): with self.database.atomic(): for value in values: Generic.create(x=value) def mts(self, seconds): return (datetime.datetime(2015, 1, 1) + datetime.timedelta(seconds=seconds)) def test_min_avg_tdiff(self): self.assertEqual(self.sql1('select mintdiff(x) from generic;'), None) self.assertEqual(self.sql1('select avgtdiff(x) from generic;'), None) self._store_values(self.mts(10)) self.assertEqual(self.sql1('select mintdiff(x) from generic;'), None) self.assertEqual(self.sql1('select avgtdiff(x) from generic;'), 0) self._store_values(self.mts(15)) self.assertEqual(self.sql1('select mintdiff(x) from generic;'), 5) self.assertEqual(self.sql1('select avgtdiff(x) from generic;'), 5) self._store_values( self.mts(22), self.mts(52), self.mts(18), self.mts(41), self.mts(2), self.mts(33)) self.assertEqual(self.sql1('select mintdiff(x) from generic;'), 3) self.assertEqual( round(self.sql1('select avgtdiff(x) from generic;'), 1), 7.1) self._store_values(self.mts(22)) self.assertEqual(self.sql1('select mintdiff(x) from generic;'), 0) def test_duration(self): self.assertEqual(self.sql1('select duration(x) from generic;'), None) self._store_values(self.mts(10)) self.assertEqual(self.sql1('select duration(x) from generic;'), 0) self._store_values(self.mts(15)) self.assertEqual(self.sql1('select duration(x) from generic;'), 5) self._store_values( self.mts(22), self.mts(11), self.mts(52), self.mts(18), self.mts(41), self.mts(2), self.mts(33)) self.assertEqual(self.sql1('select duration(x) from generic;'), 50) @requires_cython def test_median(self): self.assertEqual(self.sql1('select median(x) from generic;'), None) self._store_values(1) self.assertEqual(self.sql1('select median(x) from generic;'), 1) self._store_values(3, 6, 6, 6, 7, 7, 7, 7, 12, 12, 17) self.assertEqual(self.sql1('select median(x) from generic;'), 7) Generic.delete().execute() self._store_values(9, 2, 2, 3, 3, 1) self.assertEqual(self.sql1('select median(x) from generic;'), 3) Generic.delete().execute() self._store_values(4, 4, 1, 8, 2, 2, 5, 8, 1) self.assertEqual(self.sql1('select median(x) from generic;'), 4) def test_mode(self): self.assertEqual(self.sql1('select mode(x) from generic;'), None) self._store_values(1) self.assertEqual(self.sql1('select mode(x) from generic;'), 1) self._store_values(4, 5, 6, 1, 3, 4, 1, 4, 9, 3, 4) self.assertEqual(self.sql1('select mode(x) from generic;'), 4) def test_ranges(self): self.assertEqual(self.sql1('select minrange(x) from generic'), None) self.assertEqual(self.sql1('select avgrange(x) from generic'), None) self.assertEqual(self.sql1('select range(x) from generic'), None) self._store_values(1) self.assertEqual(self.sql1('select minrange(x) from generic'), 0) self.assertEqual(self.sql1('select avgrange(x) from generic'), 0) self.assertEqual(self.sql1('select range(x) from generic'), 0) self._store_values(4, 8, 13, 19) self.assertEqual(self.sql1('select minrange(x) from generic'), 3) self.assertEqual(self.sql1('select avgrange(x) from generic'), 4.5) self.assertEqual(self.sql1('select range(x) from generic'), 18) Generic.delete().execute() self._store_values(19, 4, 5, 20, 5, 8) self.assertEqual(self.sql1('select range(x) from generic'), 16) class TestScalarFunctions(BaseTestUDF): requires = MODELS def test_if_then_else(self): for i in range(4): User.create(username='u%d' % (i + 1)) with self.assertQueryCount(1): query = (User .select( User.username, fn.if_then_else( User.username << ['u1', 'u2'], 'one or two', 'other').alias('name_type')) .order_by(User.id)) self.assertEqual([row.name_type for row in query], [ 'one or two', 'one or two', 'other', 'other']) def test_strip_tz(self): dt = datetime.datetime(2015, 1, 1, 12, 0) # 13 hours, 37 minutes. dt_tz = dt.replace(tzinfo=FixedOffset(13 * 60 + 37, 'US/LFK')) api_dt = APIResponse.create(timestamp=dt) api_dt_tz = APIResponse.create(timestamp=dt_tz) # Re-fetch from the database. api_dt_db = APIResponse.get(APIResponse.id == api_dt.id) api_dt_tz_db = APIResponse.get(APIResponse.id == api_dt_tz.id) # Assert the timezone is present, first of all, and that they were # stored in the database. self.assertEqual(api_dt_db.timestamp, dt) query = (APIResponse .select( APIResponse.id, fn.strip_tz(APIResponse.timestamp).alias('ts')) .order_by(APIResponse.id)) ts, ts_tz = query[:] self.assertEqual(ts.ts, dt) self.assertEqual(ts_tz.ts, dt) def test_human_delta(self): values = [0, 1, 30, 300, 3600, 7530, 300000] for value in values: Generic.create(value=value) delta = fn.human_delta(Generic.value).coerce(False) query = (Generic .select( Generic.value, delta.alias('delta')) .order_by(Generic.value)) results = query.tuples()[:] self.assertEqual(results, [ (0, '0 seconds'), (1, '1 second'), (30, '30 seconds'), (300, '5 minutes'), (3600, '1 hour'), (7530, '2 hours, 5 minutes, 30 seconds'), (300000, '3 days, 11 hours, 20 minutes'), ]) def test_file_ext(self): data = ( ('test.py', '.py'), ('test.x.py', '.py'), ('test', ''), ('test.', '.'), ('/foo.bar/test/nug.py', '.py'), ('/foo.bar/test/nug', ''), ) for filename, ext in data: res = self.sql1('SELECT file_ext(?)', filename) self.assertEqual(res, ext) def test_gz(self): random.seed(1) A = ord('A') z = ord('z') with self.database.atomic(): def randstr(l): return ''.join([ chr(random.randint(A, z)) for _ in range(l)]) data = ( 'a', 'a' * 1024, randstr(1024), randstr(4096), randstr(1024 * 64)) for s in data: compressed = self.sql1('select gzip(?)', s) decompressed = self.sql1('select gunzip(?)', compressed) self.assertEqual(decompressed.decode('utf-8'), s) def test_hostname(self): r = json.dumps({'success': True}) data = ( ('https://charlesleifer.com/api/', r), ('https://a.charlesleifer.com/api/foo', r), ('www.nugget.com', r), ('nugz.com', r), ('http://a.b.c.peewee/foo', r), ('https://charlesleifer.com/xx', r), ('https://charlesleifer.com/xx', r), ) with self.database.atomic(): for url, response in data: APIResponse.create(url=url, data=data) with self.assertQueryCount(1): query = (APIResponse .select( fn.hostname(APIResponse.url).alias('host'), fn.COUNT(APIResponse.id).alias('count')) .group_by(fn.hostname(APIResponse.url)) .order_by( fn.COUNT(APIResponse.id).desc(), fn.hostname(APIResponse.url))) results = query.tuples()[:] self.assertEqual(results, [ ('charlesleifer.com', 3), ('', 2), ('a.b.c.peewee', 1), ('a.charlesleifer.com', 1)]) @skip_unless(IS_SQLITE_9, 'requires sqlite >= 3.9') def test_toggle(self): self.assertEqual(self.sql1('select toggle(?)', 'foo'), 1) self.assertEqual(self.sql1('select toggle(?)', 'bar'), 1) self.assertEqual(self.sql1('select toggle(?)', 'foo'), 0) self.assertEqual(self.sql1('select toggle(?)', 'foo'), 1) self.assertEqual(self.sql1('select toggle(?)', 'bar'), 0) self.assertEqual(self.sql1('select clear_toggles()'), None) self.assertEqual(self.sql1('select toggle(?)', 'foo'), 1) def test_setting(self): self.assertEqual(self.sql1('select setting(?, ?)', 'k1', 'v1'), 'v1') self.assertEqual(self.sql1('select setting(?, ?)', 'k2', 'v2'), 'v2') self.assertEqual(self.sql1('select setting(?)', 'k1'), 'v1') self.assertEqual(self.sql1('select setting(?, ?)', 'k2', 'v2-x'), 'v2-x') self.assertEqual(self.sql1('select setting(?)', 'k2'), 'v2-x') self.assertEqual(self.sql1('select setting(?)', 'kx'), None) self.assertEqual(self.sql1('select clear_settings()'), None) self.assertEqual(self.sql1('select setting(?)', 'k1'), None) def test_random_range(self): vals = ((1, 10), (1, 100), (0, 2), (1, 5, 2)) results = [] for params in vals: random.seed(1) results.append(random.randrange(*params)) for params, expected in zip(vals, results): random.seed(1) if len(params) == 3: pstr = '?, ?, ?' else: pstr = '?, ?' self.assertEqual( self.sql1('select randomrange(%s)' % pstr, *params), expected) def test_sqrt(self): self.assertEqual(self.sql1('select sqrt(?)', 4), 2) self.assertEqual(round(self.sql1('select sqrt(?)', 2), 2), 1.41) def test_tonumber(self): data = ( ('123', 123), ('1.23', 1.23), ('1e4', 10000), ('-10', -10), ('x', None), ('13d', None), ) for inp, outp in data: self.assertEqual(self.sql1('select tonumber(?)', inp), outp) @requires_cython def test_leven(self): self.assertEqual( self.sql1('select levenshtein_dist(?, ?)', 'abc', 'ba'), 2) self.assertEqual( self.sql1('select levenshtein_dist(?, ?)', 'abcde', 'eba'), 4) self.assertEqual( self.sql1('select levenshtein_dist(?, ?)', 'abcde', 'abcde'), 0) @requires_cython def test_str_dist(self): self.assertEqual( self.sql1('select str_dist(?, ?)', 'abc', 'ba'), 3) self.assertEqual( self.sql1('select str_dist(?, ?)', 'abcde', 'eba'), 6) self.assertEqual( self.sql1('select str_dist(?, ?)', 'abcde', 'abcde'), 0) def test_substr_count(self): self.assertEqual( self.sql1('select substr_count(?, ?)', 'foo bar baz', 'a'), 2) self.assertEqual( self.sql1('select substr_count(?, ?)', 'foo bor baz', 'o'), 3) self.assertEqual( self.sql1('select substr_count(?, ?)', 'foodooboope', 'oo'), 3) self.assertEqual(self.sql1('select substr_count(?, ?)', 'xx', ''), 0) self.assertEqual(self.sql1('select substr_count(?, ?)', '', ''), 0) def test_strip_chars(self): self.assertEqual( self.sql1('select strip_chars(?, ?)', ' hey foo ', ' '), 'hey foo') ================================================ FILE: tests/sqliteq.py ================================================ import os import sys import threading import time import unittest from functools import partial try: import gevent from gevent.event import Event as GreenEvent except ImportError: gevent = None from peewee import * from playhouse.sqliteq import ResultTimeout from playhouse.sqliteq import SqliteQueueDatabase from playhouse.sqliteq import WriterPaused from .base import BaseTestCase from .base import TestModel from .base import db_loader from .base import get_sqlite_db from .base import skip_if get_db = partial(db_loader, 'sqlite', db_class=SqliteQueueDatabase) db = get_sqlite_db() class User(TestModel): name = TextField(unique=True) class Meta: table_name = 'threaded_db_test_user' class BaseTestQueueDatabase(object): database_config = {} n_rows = 20 n_threads = 20 def setUp(self): super(BaseTestQueueDatabase, self).setUp() User._meta.database = db with db: db.create_tables([User], safe=True) User._meta.database = \ self.database = get_db(**self.database_config) # Sanity check at startup. self.assertEqual(self.database.queue_size(), 0) def tearDown(self): super(BaseTestQueueDatabase, self).tearDown() User._meta.database = db with db: User.drop_table() if not self.database.is_closed(): self.database.close() if not db.is_closed(): db.close() filename = db.database if os.path.exists(filename): os.unlink(filename) def test_query_error(self): self.database.start() curs = self.database.execute_sql('foo bar baz') self.assertRaises(OperationalError, curs.fetchone) self.database.stop() def test_integrity_error(self): self.database.start() u = User.create(name='u') self.assertRaises(IntegrityError, User.create, name='u') def test_query_execution(self): qr = User.select().execute() self.assertEqual(self.database.queue_size(), 0) self.database.start() try: users = list(qr) huey = User.create(name='huey') mickey = User.create(name='mickey') self.assertTrue(huey.id is not None) self.assertTrue(mickey.id is not None) self.assertEqual(self.database.queue_size(), 0) finally: self.database.stop() def create_thread(self, fn, *args): raise NotImplementedError def create_event(self): raise NotImplementedError def test_multiple_threads(self): def create_rows(idx, nrows): for i in range(idx, idx + nrows): User.create(name='u-%s' % i) total = self.n_threads * self.n_rows self.database.start() threads = [self.create_thread(create_rows, i, self.n_rows) for i in range(0, total, self.n_rows)] [t.start() for t in threads] [t.join() for t in threads] self.assertEqual(User.select().count(), total) self.database.stop() def test_pause(self): event_a = self.create_event() event_b = self.create_event() def create_user(name, event, expect_paused): event.wait() if expect_paused: self.assertRaises(WriterPaused, lambda: User.create(name=name)) else: User.create(name=name) self.database.start() t_a = self.create_thread(create_user, 'a', event_a, True) t_a.start() t_b = self.create_thread(create_user, 'b', event_b, False) t_b.start() User.create(name='c') self.assertEqual(User.select().count(), 1) # Pause operations but preserve the writer thread/connection. self.database.pause() event_a.set() self.assertEqual(User.select().count(), 1) t_a.join() self.database.unpause() self.assertEqual(User.select().count(), 1) event_b.set() t_b.join() self.assertEqual(User.select().count(), 2) self.database.stop() def test_restart(self): self.database.start() User.create(name='a') self.database.stop() self.database._results_timeout = 0.0001 self.assertRaises(ResultTimeout, User.create, name='b') self.assertEqual(User.select().count(), 1) self.database.start() # Will execute the pending "b" INSERT. self.database._results_timeout = None User.create(name='c') self.assertEqual(User.select().count(), 3) self.assertEqual(sorted(u.name for u in User.select()), ['a', 'b', 'c']) def test_waiting(self): D = {} def create_user(name): D[name] = User.create(name=name).id threads = [self.create_thread(create_user, name) for name in ('huey', 'charlie', 'zaizee')] [t.start() for t in threads] def get_users(): D['users'] = [(user.id, user.name) for user in User.select()] tg = self.create_thread(get_users) tg.start() threads.append(tg) self.database.start() [t.join() for t in threads] self.database.stop() self.assertEqual(sorted(D), ['charlie', 'huey', 'users', 'zaizee']) def test_next_method(self): self.database.start() User.create(name='mickey') User.create(name='huey') query = iter(User.select().order_by(User.name)) self.assertEqual(next(query).name, 'huey') self.assertEqual(next(query).name, 'mickey') self.assertRaises(StopIteration, lambda: next(query)) self.assertEqual( next(self.database.execute_sql('PRAGMA journal_mode'))[0], 'wal') self.database.stop() class TestThreadedDatabaseThreads(BaseTestQueueDatabase, BaseTestCase): database_config = {'use_gevent': False} def tearDown(self): self.database._results_timeout = None super(TestThreadedDatabaseThreads, self).tearDown() def create_thread(self, fn, *args): t = threading.Thread(target=fn, args=args) t.daemon = True return t def create_event(self): return threading.Event() def test_timeout(self): @self.database.func() def slow(n): time.sleep(n) return 'slept %0.2f' % n self.database.start() # Make the result timeout very small, then call our function which # will cause the query results to time-out. self.database._results_timeout = 0.001 def do_query(): # Prepend a space so that we can force it through the threaded # pipeline, otherwise it would execute normally. cursor = self.database.execute_sql(' select slow(?)', (0.01,)) self.assertEqual(cursor.fetchone()[0], 'slept 0.01') self.assertRaises(ResultTimeout, do_query) self.database.stop() @skip_if(gevent is None, 'gevent not installed') class TestThreadedDatabaseGreenlets(BaseTestQueueDatabase, BaseTestCase): database_config = {'use_gevent': True} n_rows = 10 n_threads = 40 def create_thread(self, fn, *args): return gevent.Greenlet(fn, *args) def create_event(self): return GreenEvent() ================================================ FILE: tests/test_utils.py ================================================ import functools from .base import ModelTestCase from .base import TestModel from peewee import * from playhouse.test_utils import assert_query_count from playhouse.test_utils import count_queries class Data(TestModel): key = CharField() class Meta: order_by = ('key',) class DataItem(TestModel): data = ForeignKeyField(Data, backref='items') value = CharField() class Meta: order_by = ('value',) class TestQueryCounter(ModelTestCase): requires = [DataItem, Data] def test_count(self): with count_queries() as count: Data.create(key='k1') Data.create(key='k2') self.assertEqual(count.count, 2) with count_queries() as count: items = [item.key for item in Data.select().order_by(Data.key)] self.assertEqual(items, ['k1', 'k2']) Data.get(Data.key == 'k1') Data.get(Data.key == 'k2') self.assertEqual(count.count, 3) def test_only_select(self): with count_queries(only_select=True) as count: for i in range(10): Data.create(key=str(i)) items = [item.key for item in Data.select()] Data.get(Data.key == '0') Data.get(Data.key == '9') Data.delete().where( Data.key << ['1', '3', '5', '7', '9']).execute() items = [item.key for item in Data.select().order_by(Data.key)] self.assertEqual(items, ['0', '2', '4', '6', '8']) self.assertEqual(count.count, 4) def test_assert_query_count_decorator(self): @assert_query_count(2) def will_fail_under(): Data.create(key='x') @assert_query_count(2) def will_fail_over(): for i in range(3): Data.create(key=str(i)) @assert_query_count(4) def will_succeed(): for i in range(4): Data.create(key=str(i + 100)) will_succeed() self.assertRaises(AssertionError, will_fail_under) self.assertRaises(AssertionError, will_fail_over) def test_assert_query_count_ctx_mgr(self): with assert_query_count(3): for i in range(3): Data.create(key=str(i)) def will_fail(): with assert_query_count(2): Data.create(key='x') self.assertRaises(AssertionError, will_fail) @assert_query_count(3) def test_only_three(self): for i in range(3): Data.create(key=str(i)) ================================================ FILE: tests/transactions.py ================================================ import threading from peewee import * from .base import DatabaseTestCase from .base import IS_CRDB from .base import IS_CRDB_NESTED_TX from .base import IS_MYSQL from .base import IS_POSTGRESQL from .base import IS_SQLITE from .base import ModelTestCase from .base import db from .base import new_connection from .base import skip_if from .base import skip_unless from .base_models import Register class BaseTransactionTestCase(ModelTestCase): requires = [Register] def assertRegister(self, vals): query = Register.select().order_by(Register.value) self.assertEqual([register.value for register in query], vals) def _save(self, *vals): Register.insert([{Register.value: val} for val in vals]).execute() def requires_nested(fn): return skip_if(IS_CRDB and not IS_CRDB_NESTED_TX, 'nested transaction support is required')(fn) class TestTransaction(BaseTransactionTestCase): def test_simple(self): self.assertFalse(db.in_transaction()) with db.atomic(): self.assertTrue(db.in_transaction()) self._save(1) self.assertFalse(db.in_transaction()) self.assertRegister([1]) # Explicit rollback, implicit commit. with db.atomic() as txn: self._save(2) txn.rollback() self.assertTrue(db.in_transaction()) self._save(3) self.assertFalse(db.in_transaction()) self.assertRegister([1, 3]) # Explicit rollbacks. with db.atomic() as txn: self._save(4) txn.rollback() self._save(5) txn.rollback() self.assertRegister([1, 3]) @requires_nested def test_transactions(self): self.assertFalse(db.in_transaction()) with db.atomic(): self.assertTrue(db.in_transaction()) self._save(1) self.assertRegister([1]) with db.atomic() as txn: self._save(2) txn.rollback() self._save(3) with db.atomic() as sp1: self._save(4) with db.atomic() as sp2: self._save(5) sp2.rollback() with db.atomic() as sp3: self._save(6) with db.atomic() as sp4: self._save(7) with db.atomic() as sp5: self._save(8) self.assertRegister([1, 3, 4, 6, 7, 8]) sp4.rollback() self.assertRegister([1, 3, 4, 6]) self.assertRegister([1, 3, 4, 6]) def test_commit_rollback(self): with db.atomic() as txn: self._save(1) txn.commit() self._save(2) txn.rollback() self.assertRegister([1]) with db.atomic() as txn: self._save(3) txn.rollback() self._save(4) self.assertRegister([1, 4]) @requires_nested def test_commit_rollback_nested(self): with db.atomic() as txn: self.test_commit_rollback() txn.rollback() self.assertRegister([]) with db.atomic(): self.test_commit_rollback() self.assertRegister([1, 4]) def test_nesting_transaction_obj(self): self.assertRegister([]) with db.transaction() as txn: self._save(1) with db.transaction() as txn2: self._save(2) txn2.rollback() # Actually issues a rollback. self.assertRegister([]) self._save(3) self.assertRegister([3]) with db.transaction() as txn: self._save(4) with db.transaction() as txn2: with db.transaction() as txn3: self._save(5) txn3.commit() # Actually commits. self._save(6) txn2.rollback() self.assertRegister([3, 4, 5]) with db.transaction() as txn: self._save(6) try: with db.transaction() as txn2: self._save(7) raise ValueError() except ValueError: pass self.assertRegister([3, 4, 5, 6, 7]) @requires_nested def test_savepoint_commit(self): with db.atomic() as txn: self._save(1) txn.rollback() self._save(2) txn.commit() with db.atomic() as sp: self._save(3) sp.rollback() self._save(4) sp.commit() self.assertRegister([2, 4]) def test_atomic_decorator(self): @db.atomic() def save(i): self._save(i) save(1) self.assertRegister([1]) def text_atomic_exception(self): def will_fail(self): with db.atomic(): self._save(1) self._save(None) self.assertRaises(IntegrityError, will_fail) self.assertRegister([]) def user_error(self): with db.atomic(): self._save(2) raise ValueError self.assertRaises(ValueError, user_error) self.assertRegister([]) def test_manual_commit(self): with db.manual_commit(): db.begin() self._save(1) db.rollback() db.begin() self._save(2) db.commit() with db.manual_commit(): db.begin() self._save(3) db.rollback() db.begin() self._save(4) db.commit() self.assertRegister([2, 4]) def test_mixing_manual_atomic(self): @db.manual_commit() def will_fail(): pass @db.atomic() def also_fails(): pass with db.atomic(): self.assertRaises(ValueError, will_fail) with db.manual_commit(): self.assertRaises(ValueError, also_fails) with db.manual_commit(): with self.assertRaises(ValueError): with db.atomic(): pass with db.atomic(): with self.assertRaises(ValueError): with db.manual_commit(): pass def test_closing_db_in_transaction(self): with db.atomic(): self.assertRaises(OperationalError, db.close) @requires_nested def test_db_context_manager(self): db.close() self.assertTrue(db.is_closed()) with db: self.assertFalse(db.is_closed()) self._save(1) with db: self._save(2) try: with db: self._save(3) raise ValueError('xxx') except ValueError: pass self._save(4) try: with db: self._save(5) with db: self._save(6) raise ValueError('yyy') except ValueError: pass self.assertFalse(db.is_closed()) self.assertTrue(db.is_closed()) self.assertRegister([1, 2, 4]) def test_transaction_concurrency(self): barrier = threading.Barrier(5) accum = [] def run_thread(): barrier.wait(timeout=2) for i in range(10): try: with db.atomic() as tx: for j in range(10): try: with db.atomic() as sp: sp.commit() if j % 2 == 0: raise ValueError() except ValueError: pass if i % 1 == 0: raise ValueError() except ValueError: pass accum.append(True) threads = [threading.Thread(target=run_thread) for _ in range(4)] for t in threads: t.start() barrier.wait(timeout=2) for t in threads: t.join() self.assertEqual(accum, [True, True, True, True]) @requires_nested class TestSession(BaseTransactionTestCase): def test_session(self): self.assertTrue(db.session_start()) self.assertTrue(db.session_start()) self.assertEqual(db.transaction_depth(), 2) self._save(1) self.assertTrue(db.session_commit()) self.assertEqual(db.transaction_depth(), 1) self._save(2) # Now we're in autocommit mode. self.assertTrue(db.session_rollback()) self.assertEqual(db.transaction_depth(), 0) self.assertTrue(db.session_start()) self._save(3) self.assertTrue(db.session_rollback()) self.assertRegister([1]) def test_session_with_closed_db(self): db.close() self.assertTrue(db.session_start()) self.assertFalse(db.is_closed()) self.assertRaises(OperationalError, db.close) self._save(1) self.assertTrue(db.session_rollback()) self.assertRegister([]) def test_session_inside_context_manager(self): with db.atomic(): self.assertTrue(db.session_start()) self._save(1) self.assertTrue(db.session_commit()) self._save(2) self.assertTrue(db.session_rollback()) db.session_start() self._save(3) self.assertRegister([1, 3]) def test_commit_rollback_mix(self): db.session_start() with db.atomic() as txn: # Will be a savepoint. self._save(1) with db.atomic() as t2: self._save(2) with db.atomic() as t3: self._save(3) t2.rollback() txn.commit() self._save(4) txn.rollback() self.assertTrue(db.session_commit()) self.assertRegister([1]) def test_session_rollback(self): db.session_start() self._save(1) with db.atomic() as txn: self._save(2) with db.atomic() as t2: self._save(3) self.assertRegister([1, 2, 3]) self.assertTrue(db.session_rollback()) self.assertRegister([]) db.session_start() self._save(1) with db.transaction() as txn: self._save(2) with db.transaction() as t2: self._save(3) t2.rollback() # Rolls back everything, starts new txn. db.session_commit() self.assertRegister([]) def test_session_commit(self): db.session_start() self._save(1) with db.transaction() as txn: self._save(2) with db.transaction() as t2: self._save(3) t2.commit() # Saves everything, starts new txn. txn.rollback() self.assertTrue(db.session_rollback()) self.assertRegister([1, 2, 3]) @skip_unless(IS_SQLITE, 'requires sqlite for transaction lock type') class TestTransactionLockType(BaseTransactionTestCase): def test_lock_type(self): db2 = new_connection(timeout=0.0001) db2.connect() with self.database.atomic(lock_type='EXCLUSIVE') as txn: with self.assertRaises(OperationalError): with db2.atomic(lock_type='IMMEDIATE') as t2: self._save(1) self._save(2) self.assertRegister([2]) with self.database.atomic('IMMEDIATE') as txn: with self.assertRaises(OperationalError): with db2.atomic('EXCLUSIVE') as t2: self._save(3) self._save(4) self.assertRegister([2, 4]) with self.database.transaction(lock_type='DEFERRED') as txn: self._save(5) # Deferred -> Exclusive after our write. with self.assertRaises(OperationalError): with db2.transaction(lock_type='IMMEDIATE') as t2: self._save(6) self.assertRegister([2, 4, 5]) class TestTransactionIsolationLevel(BaseTransactionTestCase): @skip_unless(IS_POSTGRESQL, 'requires postgresql') def test_isolation_level_pg(self): db2 = new_connection() db2.connect() with db2.atomic(isolation_level='SERIALIZABLE'): with db.atomic(isolation_level='SERIALIZABLE'): self._save(1) self.assertDB2(db2, []) self.assertDB2(db2, []) self.assertDB2(db2, [1]) with db2.atomic(isolation_level='READ COMMITTED'): with db.atomic(): self._save(2) self.assertDB2(db2, [1]) self.assertDB2(db2, [1, 2]) self.assertDB2(db2, [1, 2]) # NB: Read Uncommitted is treated as Read Committed by PG, so we don't # test it here. with db2.atomic(isolation_level='REPEATABLE READ'): with db.atomic(isolation_level='REPEATABLE READ'): self._save(3) self.assertDB2(db2, [1, 2]) self.assertDB2(db2, [1, 2]) self.assertDB2(db2, [1, 2, 3]) @skip_unless(IS_MYSQL, 'requires mysql') def test_isolation_level_mysql(self): db2 = new_connection() db2.connect() with db2.atomic(): with db.atomic(isolation_level='SERIALIZABLE'): self._save(1) self.assertDB2(db2, []) self.assertDB2(db2, []) self.assertDB2(db2, [1]) with db2.atomic(isolation_level='READ COMMITTED'): with db.atomic(): self._save(2) self.assertDB2(db2, [1]) self.assertDB2(db2, [1, 2]) self.assertDB2(db2, [1, 2]) with db2.atomic(isolation_level='READ UNCOMMITTED'): with db.atomic(): self._save(3) self.assertDB2(db2, [1, 2, 3]) self.assertDB2(db2, [1, 2, 3]) self.assertDB2(db2, [1, 2, 3]) with db2.atomic(isolation_level='REPEATABLE READ'): with db.atomic(isolation_level='REPEATABLE READ'): self._save(4) self.assertDB2(db2, [1, 2, 3]) self.assertDB2(db2, [1, 2, 3]) self.assertDB2(db2, [1, 2, 3, 4]) def assertDB2(self, db2, vals): with Register.bind_ctx(db2): q = Register.select().order_by(Register.value) self.assertEqual([r.value for r in q], vals)