Repository: msiemens/tinydb Branch: master Commit: 2283a2b556d5 Files: 59 Total size: 244.1 KB Directory structure: gitextract_xygol1wo/ ├── .coveragerc ├── .github/ │ ├── stale.yml │ └── workflows/ │ ├── ci-workflow.yml │ └── publish-workflow.yml ├── .gitignore ├── .readthedocs.yaml ├── CONTRIBUTING.rst ├── LICENSE ├── MANIFEST.in ├── README.rst ├── SECURITY.md ├── docs/ │ ├── .gitignore │ ├── Makefile │ ├── _templates/ │ │ ├── links.html │ │ └── sidebarlogo.html │ ├── _themes/ │ │ ├── .gitignore │ │ ├── LICENSE │ │ ├── README │ │ ├── flask/ │ │ │ ├── layout.html │ │ │ ├── page.html │ │ │ ├── relations.html │ │ │ ├── static/ │ │ │ │ └── flasky.css_t │ │ │ └── theme.conf │ │ └── flask_theme_support.py │ ├── api.rst │ ├── changelog.rst │ ├── conf.py │ ├── contribute.rst │ ├── extend.rst │ ├── extensions.rst │ ├── getting-started.rst │ ├── index.rst │ ├── intro.rst │ ├── make.bat │ ├── upgrade.rst │ └── usage.rst ├── mypy.ini ├── pyproject.toml ├── pytest.ini ├── tests/ │ ├── __init__.py │ ├── conftest.py │ ├── test_middlewares.py │ ├── test_operations.py │ ├── test_queries.py │ ├── test_storages.py │ ├── test_tables.py │ ├── test_tinydb.py │ └── test_utils.py └── tinydb/ ├── __init__.py ├── database.py ├── middlewares.py ├── mypy_plugin.py ├── operations.py ├── py.typed ├── queries.py ├── storages.py ├── table.py ├── utils.py └── version.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .coveragerc ================================================ [run] branch = True [report] exclude_lines = pragma: no cover raise NotImplementedError.* warnings\.warn.* def __repr__ def __str__ def main() if __name__ == .__main__.: ================================================ FILE: .github/stale.yml ================================================ # Number of days of inactivity before an issue becomes stale daysUntilStale: 30 # Number of days of inactivity before a stale issue is closed daysUntilClose: 7 # Issues with these labels will never be considered stale exemptLabels: - bug - pinned - contributions-welcome # Label to use when marking an issue as stale staleLabel: stale # Comment to post when marking an issue as stale. Set to `false` to disable markComment: > This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Feel free to reopen this if needed. Thank you for your contributions :heart: # Comment to post when closing a stale issue. Set to `false` to disable closeComment: false ================================================ FILE: .github/workflows/ci-workflow.yml ================================================ name: Python CI on: push: {} pull_request: branches: [ master ] jobs: build: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: python-version: - "3.10" - "3.11" - "3.12" - "3.13" - "3.14" os: [ubuntu-latest, macos-latest, windows-latest] include: - python-version: "pypy-3.9" os: ubuntu-latest - python-version: "pypy-3.10" os: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} allow-prereleases: true - name: Set up uv uses: astral-sh/setup-uv@v5 - name: Install dependencies run: | python -m pip install --upgrade pip uv sync --group dev - name: Run test suite run: | uv run py.test -v --cov=tinydb - name: Perform type check run: | uv run pytest --mypy -m mypy tinydb tests if: ${{ contains(matrix.python-version, '3.14') }} - name: Verify dist package format run: | uv build uv run --with twine twine check dist/* if: ${{ contains(matrix.python-version, '3.14') }} - name: Upload coverage result if: ${{ matrix.os != 'windows-latest' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_FLAG_NAME: ${{ matrix.os }}-py${{ matrix.python-version }} COVERALLS_PARALLEL: true run: | uv run coveralls coveralls: name: Indicate completion to coveralls.io needs: build runs-on: ubuntu-latest steps: - name: Install coveralls run: pip3 install --upgrade coveralls - name: Finished env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: coveralls --finish ================================================ FILE: .github/workflows/publish-workflow.yml ================================================ name: Upload Python Package on: push: tags: - v*.*.* jobs: publish: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v2 with: python-version: '3.x' - name: Set up uv uses: astral-sh/setup-uv@v5 - name: Install dependencies run: | python -m pip install --upgrade pip uv sync --group dev - name: Publish package env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.POETRY_PYPI_TOKEN_PYPI }} run: | uv build uv run --with twine twine upload dist/* - name: Create Release uses: actions/create-release@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: tag_name: ${{ github.ref }} release_name: ${{ github.ref }} draft: false prerelease: false ================================================ FILE: .gitignore ================================================ *.py[cod] # C extensions *.so # Packages *.egg *.egg-info dist build eggs parts bin var sdist develop-eggs .installed.cfg lib lib64 __pycache__ # Installer logs pip-log.txt # Unit test / coverage reports .coverage .tox nosetests.xml .pytest_cache/ # Translations *.mo # Mr Developer .mr.developer.cfg .project .pydevproject # Pycharm .idea *.db.yml .DS_Store ================================================ FILE: .readthedocs.yaml ================================================ version: 2 build: os: ubuntu-24.04 tools: python: "3.12" sphinx: configuration: docs/conf.py python: install: - method: pip path: . extra_requirements: - docs formats: all ================================================ FILE: CONTRIBUTING.rst ================================================ Contribution Guidelines ####################### Whether reporting bugs, discussing improvements and new ideas or writing extensions: Contributions to TinyDB are welcome! Here's how to get started: 1. Check for open issues or open a fresh issue to start a discussion around a feature idea or a bug 2. Fork `the repository `_ on GitHub, create a new branch off the `master` branch and start making your changes (known as `GitHub Flow `_) 3. Write a test which shows that the bug was fixed or that the feature works as expected 4. Send a pull request and bug the maintainer until it gets merged and published :) Philosophy of TinyDB ******************** TinyDB aims to be simple and fun to use. Therefore two key values are simplicity and elegance of interfaces and code. These values will contradict each other from time to time. In these cases , try using as little magic as possible. In any case don't forget documenting code that isn't clear at first glance. Code Conventions **************** In general the TinyDB source should always follow `PEP 8 `_. Exceptions are allowed in well justified and documented cases. However we make a small exception concerning docstrings: When using multiline docstrings, keep the opening and closing triple quotes on their own lines and add an empty line after it. .. code-block:: python def some_function(): """ Documentation ... """ # implementation ... Version Numbers *************** TinyDB follows the `SemVer versioning guidelines `_. This implies that backwards incompatible changes in the API will increment the major version. So think twice before making such changes. ================================================ FILE: LICENSE ================================================ Copyright (C) 2013 Markus Siemens Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: MANIFEST.in ================================================ include LICENSE recursive-include tests *.py ================================================ FILE: README.rst ================================================ .. image:: https://raw.githubusercontent.com/msiemens/tinydb/master/artwork/logo.png :height: 150px |Build Status| |Coverage| |Version| Quick Links *********** - `Example Code`_ - `Supported Python Versions`_ - `Documentation `_ - `Changelog `_ - `Extensions `_ - `Contributing`_ Introduction ************ TinyDB is a lightweight document oriented database optimized for your happiness :) It's written in pure Python and has no external dependencies. The target are small apps that would be blown away by a SQL-DB or an external database server. TinyDB is: - **tiny:** The current source code has 1800 lines of code (with about 40% documentation) and 1600 lines tests. - **document oriented:** Like MongoDB_, you can store any document (represented as ``dict``) in TinyDB. - **optimized for your happiness:** TinyDB is designed to be simple and fun to use by providing a simple and clean API. - **written in pure Python:** TinyDB neither needs an external server (as e.g. `PyMongo `_) nor any dependencies from PyPI. - **works on Python 3.8+ and PyPy3:** TinyDB works on all modern versions of Python and PyPy. - **powerfully extensible:** You can easily extend TinyDB by writing new storages or modify the behaviour of storages with Middlewares. - **100% test coverage:** No explanation needed. To dive straight into all the details, head over to the `TinyDB docs `_. You can also discuss everything related to TinyDB like general development, extensions or showcase your TinyDB-based projects on the `discussion forum `_. Supported Python Versions ************************* TinyDB has been tested with Python 3.8 - 3.13 and PyPy3. Project Status ************** This project is in maintenance mode. It has reached a mature, stable state where significant new features or architectural changes are not planned. That said, there will still be releases for bugfixes or features contributed by the community. Read more about what this means in particular `here `_. Example Code ************ .. code-block:: python >>> from tinydb import TinyDB, Query >>> db = TinyDB('/path/to/db.json') >>> db.insert({'int': 1, 'char': 'a'}) >>> db.insert({'int': 1, 'char': 'b'}) Query Language ============== .. code-block:: python >>> User = Query() >>> # Search for a field value >>> db.search(User.name == 'John') [{'name': 'John', 'age': 22}, {'name': 'John', 'age': 37}] >>> # Combine two queries with logical and >>> db.search((User.name == 'John') & (User.age <= 30)) [{'name': 'John', 'age': 22}] >>> # Combine two queries with logical or >>> db.search((User.name == 'John') | (User.name == 'Bob')) [{'name': 'John', 'age': 22}, {'name': 'John', 'age': 37}, {'name': 'Bob', 'age': 42}] >>> # Negate a query with logical not >>> db.search(~(User.name == 'John')) [{'name': 'Megan', 'age': 27}, {'name': 'Bob', 'age': 42}] >>> # Apply transformation to field with `map` >>> db.search((User.age.map(lambda x: x + x) == 44)) >>> [{'name': 'John', 'age': 22}] >>> # More possible comparisons: != < > <= >= >>> # More possible checks: where(...).matches(regex), where(...).test(your_test_func) Tables ====== .. code-block:: python >>> table = db.table('name') >>> table.insert({'value': True}) >>> table.all() [{'value': True}] Using Middlewares ================= .. code-block:: python >>> from tinydb.storages import JSONStorage >>> from tinydb.middlewares import CachingMiddleware >>> db = TinyDB('/path/to/db.json', storage=CachingMiddleware(JSONStorage)) Contributing ************ Whether reporting bugs, discussing improvements and new ideas or writing extensions: Contributions to TinyDB are welcome! Here's how to get started: 1. Check for open issues or open a fresh issue to start a discussion around a feature idea or a bug 2. Fork `the repository `_ on Github, create a new branch off the `master` branch and start making your changes (known as `GitHub Flow `_) 3. Write a test which shows that the bug was fixed or that the feature works as expected 4. Send a pull request and bug the maintainer until it gets merged and published ☺ .. |Build Status| image:: https://img.shields.io/azure-devops/build/msiemens/3e5baa75-12ec-43ac-9728-89823ee8c7e2/2.svg?style=flat-square :target: https://dev.azure.com/msiemens/github/_build?definitionId=2 .. |Coverage| image:: http://img.shields.io/coveralls/msiemens/tinydb.svg?style=flat-square :target: https://coveralls.io/r/msiemens/tinydb .. |Version| image:: http://img.shields.io/pypi/v/tinydb.svg?style=flat-square :target: https://pypi.python.org/pypi/tinydb/ .. _Buzhug: http://buzhug.sourceforge.net/ .. _CodernityDB: https://github.com/perchouli/codernitydb .. _MongoDB: http://mongodb.org/ ================================================ FILE: SECURITY.md ================================================ # Security Policy ## Supported Versions | Version | Supported | | --------------------- | ------------------ | | Latest TinyDB release | :white_check_mark: | | All prior versions | :x: | ## Reporting a Vulnerability **Please do not report security vulnerabilities through public GitHub issues.** If you believe you've found a security vulnerability in TinyDB, please report it by using GitHub's private vulnerability reporting feature. Please include: - A clear description of the vulnerability - A realistic attack scenario demonstrating how untrusted external input leads to the security impact - Steps to reproduce - Your assessment of severity and impact I aim to respond within 7 days and will work with you on a fix and coordinated disclosure on a mutually agreed timeline if the issue is valid. ## Scope: What Constitutes a TinyDB Vulnerability This security policy applies to the TinyDB core library. Third-party extensions and plugins are not covered by this policy. For a report to be considered a valid TinyDB vulnerability, it must demonstrate: 1. **A realistic attack chain** where untrusted external data (user input, network data, file contents, etc.) causes unintended security impact through TinyDB's code 2. **TinyDB as the root cause**, not merely a component downstream of an existing application-level vulnerability ### Explicitly Out of Scope Security reports must demonstrate that TinyDB itself is the source of the vulnerability, not simply present in a vulnerable application. The following are **not** considered TinyDB vulnerabilities: - **Passing malicious callables to TinyDB APIs.** TinyDB accepts callables (for queries, serialization, etc.) by design. If an attacker can inject arbitrary Python callables into your application, you already have an arbitrary code execution vulnerability unrelated to TinyDB. This is an application-level concern. - **Unsafe deserialization in application code.** If your application uses `eval()`, `pickle.loads()`, or similar on untrusted input and passes the result to TinyDB, the vulnerability is in your application's deserialization, not TinyDB. - **Local file access.** TinyDB reads and writes to local files specified by the application developer. If an attacker has filesystem access or can control file paths, this represents a broader system compromise. - **Denial of service via large data.** TinyDB is not designed for adversarial multi-tenant environments. Applications should validate data before storage. _However_, DoS issues **may** be considered in-scope if they are caused by TinyDB internals (e.g., algorithmic complexity or pathological performance triggered by small, valid inputs), rather than by unbounded application data. ================================================ FILE: docs/.gitignore ================================================ _build/ ================================================ FILE: docs/Makefile ================================================ # Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # User-friendly check for sphinx-build ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) endif # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " xml to make Docutils-native XML files" @echo " pseudoxml to make pseudoxml-XML files for display purposes" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/TinyDB.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/TinyDB.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/TinyDB" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/TinyDB" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." latexpdfja: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through platex and dvipdfmx..." $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." xml: $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml @echo @echo "Build finished. The XML files are in $(BUILDDIR)/xml." pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." ================================================ FILE: docs/_templates/links.html ================================================

Useful Links

================================================ FILE: docs/_templates/sidebarlogo.html ================================================ ================================================ FILE: docs/_themes/.gitignore ================================================ *.pyc *.pyo .DS_Store ================================================ FILE: docs/_themes/LICENSE ================================================ Copyright (c) 2010 by Armin Ronacher. Some rights reserved. Redistribution and use in source and binary forms of the theme, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * The names of the contributors may not be used to endorse or promote products derived from this software without specific prior written permission. We kindly ask you to only use these themes in an unmodified manner just for Flask and Flask-related products, not for unrelated projects. If you like the visual style and want to use it for your own projects, please consider making some larger changes to the themes (such as changing font faces, sizes, colors or margins). THIS THEME IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS THEME, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================ FILE: docs/_themes/README ================================================ Flask Sphinx Styles =================== This repository contains sphinx styles for Flask and Flask related projects. To use this style in your Sphinx documentation, follow this guide: 1. put this folder as _themes into your docs folder. Alternatively you can also use git submodules to check out the contents there. 2. add this to your conf.py: sys.path.append(os.path.abspath('_themes')) html_theme_path = ['_themes'] html_theme = 'flask' The following themes exist: - 'flask' - the standard flask documentation theme for large projects - 'flask_small' - small one-page theme. Intended to be used by very small addon libraries for flask. The following options exist for the flask_small theme: [options] index_logo = '' filename of a picture in _static to be used as replacement for the h1 in the index.rst file. index_logo_height = 120px height of the index logo github_fork = '' repository name on github for the "fork me" badge ================================================ FILE: docs/_themes/flask/layout.html ================================================ {%- extends "basic/layout.html" %} {%- block extrahead %} {{ super() }} {% if theme_touch_icon %} {% endif %} {% endblock %} {%- block relbar2 %}{% endblock %} {% block header %} {{ super() }} {% if pagename == 'index' %}
{% endif %} {% endblock %} {%- block footer %}
© Copyright {{ copyright }}. Created using Sphinx.
{% if pagename == 'index' %}
{% endif %} {%- endblock %} ================================================ FILE: docs/_themes/flask/page.html ================================================ {%- extends "basic/page.html" %} {% block body %} {{ super() }} {%- if prev or next and pagename != 'index' %}

{%- if prev %} « {{ prev.title }} {% if next %}|{% endif %} {%- endif %} {%- if next %} {{ next.title }} » {%- endif %}

{%- endif %} {% endblock %} ================================================ FILE: docs/_themes/flask/relations.html ================================================

Navigation

================================================ FILE: docs/_themes/flask/static/flasky.css_t ================================================ /* * flasky.css_t * ~~~~~~~~~~~~ * * :copyright: Copyright 2010 by Armin Ronacher. * :license: Flask Design License, see LICENSE for details. */ {% set page_width = '940px' %} {% set sidebar_width = '220px' %} {% set font_family = "'Open Sans', sans-serif" %} {% set monospace_font_family = "'Source Code Pro', 'Consolas', 'Menlo', 'Deja Vu Sans Mono', 'Bitstream Vera Sans Mono', monospace" %} {% set accent_color = '#2d4e84' %}{# original: #004B6B #} {% set accent_color_alternate = '#2069e1' %}{# original: #6D4100 #} @import url(http://fonts.googleapis.com/css?family=Open+Sans:400,700,400italic|Source+Code+Pro); @import url("basic.css"); /* -- page layout ----------------------------------------------------------- */ html { overflow-y: scroll; } body { font-family: {{ font_family }}; font-size: 17px; background-color: white; color: #000; margin: 0; padding: 0; } div.document { width: {{ page_width }}; margin: 30px auto 0 auto; } div.documentwrapper { float: left; width: 100%; } div.bodywrapper { margin: 0 0 0 {{ sidebar_width }}; } div.sphinxsidebar { width: {{ sidebar_width }}; } hr { border: 1px solid #B1B4B6; } div.body { background-color: #ffffff; color: #3E4349; padding: 0 30px 0 30px; } img.floatingflask { padding: 0 0 10px 10px; float: right; } div.footer { width: {{ page_width }}; margin: 20px auto 30px auto; font-size: 14px; color: #888; text-align: right; } div.footer a { color: #888; } div.related { display: none; } div.sphinxsidebar a { color: #444; text-decoration: none; border-bottom: 1px dotted #999; } div.sphinxsidebar a:hover { border-bottom: 1px solid #999; } div.sphinxsidebar { font-size: 14px; line-height: 1.5; } div.sphinxsidebarwrapper { padding: 18px 10px; } div.sphinxsidebarwrapper p.logo { padding: 0 0 20px 0; margin: 0; text-align: center; } div.sphinxsidebar h3, div.sphinxsidebar h4 { font-family: {{ font_family }}; color: #444; font-size: 24px; font-weight: normal; margin: 0 0 5px 0; padding: 0; } div.sphinxsidebar h4 { font-size: 20px; } div.sphinxsidebar h3 a { color: #444; } div.sphinxsidebar p.logo a, div.sphinxsidebar h3 a, div.sphinxsidebar p.logo a:hover, div.sphinxsidebar h3 a:hover { border: none; } div.sphinxsidebar p { color: #555; margin: 10px 0; } div.sphinxsidebar ul { margin: 10px 0; padding: 0; color: #000; } div.sphinxsidebar input { border: 1px solid #ccc; font-family: {{ font_family }}; font-size: 1em; } /* -- body styles ----------------------------------------------------------- */ a { color: {{ accent_color }}; text-decoration: underline; } a:hover { color: {{ accent_color_alternate }}; text-decoration: underline; } div.body h1, div.body h2, div.body h3, div.body h4, div.body h5, div.body h6 { font-family: {{ font_family }}; font-weight: normal; margin: 30px 0px 10px 0px; padding: 0; } {% if theme_index_logo %} div.indexwrapper h1 { text-indent: -999999px; background: url({{ theme_index_logo }}) no-repeat center center; height: {{ theme_index_logo_height }}; } {% endif %} div.body h1 { margin-top: 0; padding-top: 0; font-size: 240%; } div.body h2 { font-size: 180%; } div.body h3 { font-size: 150%; } div.body h4 { font-size: 130%; } div.body h5 { font-size: 100%; } div.body h6 { font-size: 100%; } a.headerlink { color: #ddd; padding: 0 4px; text-decoration: none; } a.headerlink:hover { color: #444; background: #eaeaea; } div.body p, div.body dd, div.body li { line-height: 1.4em; } div.admonition { background: #fafafa; margin: 20px -30px; padding: 10px 30px; border-top: 1px solid #ccc; border-bottom: 1px solid #ccc; } div.admonition tt.xref, div.admonition a tt { border-bottom: 1px solid #fafafa; } dd div.admonition { margin-left: -60px; padding-left: 60px; } div.admonition p.admonition-title { font-family: {{ font_family }}; font-weight: normal; font-size: 24px; margin: 0 0 10px 0; padding: 0; line-height: 1; } div.admonition p.last { margin-bottom: 0; } div.highlight { background-color: white; } dt:target, .highlight { background: #FAF3E8; } div.note { background-color: #eee; border: 1px solid #ccc; } div.seealso { background-color: #ffc; border: 1px solid #ff6; } div.topic { background-color: #eee; } p.admonition-title { display: inline; } p.admonition-title:after { content: ":"; } pre, tt { font-family: {{ monospace_font_family }}; font-size: 0.9em; } img.screenshot { } tt.descname, tt.descclassname { font-size: 0.95em; } tt.descname { padding-right: 0.08em; } img.screenshot { -moz-box-shadow: 2px 2px 4px #eee; -webkit-box-shadow: 2px 2px 4px #eee; box-shadow: 2px 2px 4px #eee; } table.docutils { border: 1px solid #888; -moz-box-shadow: 2px 2px 4px #eee; -webkit-box-shadow: 2px 2px 4px #eee; box-shadow: 2px 2px 4px #eee; } table.docutils td, table.docutils th { border: 1px solid #888; padding: 0.25em 0.7em; } table.field-list, table.footnote { border: none; -moz-box-shadow: none; -webkit-box-shadow: none; box-shadow: none; } table.footnote { margin: 15px 0; width: 100%; border: 1px solid #eee; background: #fdfdfd; font-size: 0.9em; } table.footnote + table.footnote { margin-top: -15px; border-top: none; } table.field-list th { padding: 0 0.8em 0 0; } table.field-list td { padding: 0; } table.footnote td.label { width: 0px; padding: 0.3em 0 0.3em 0.5em; } table.footnote td { padding: 0.3em 0.5em; } dl { margin: 0; padding: 0; } dl dd { margin-left: 30px; } blockquote { margin: 0 0 0 30px; padding: 0; } ul, ol { margin: 10px 0 10px 30px; padding: 0; } pre { background: #eee; padding: 7px 30px; margin: 15px -30px; line-height: 1.3em; } dl pre, blockquote pre, li pre { margin-left: -60px; padding-left: 60px; } dl dl pre { margin-left: -90px; padding-left: 90px; } tt { background-color: #ecf0f3; color: #222; /* padding: 1px 2px; */ } tt.xref, a tt { background-color: #FBFBFB; border-bottom: 1px solid white; } a.reference { text-decoration: none; border-bottom: 1px dotted {{ accent_color }}; } a.reference:hover { border-bottom: 1px solid {{ accent_color_alternate }}; } a.footnote-reference { text-decoration: none; font-size: 0.7em; vertical-align: top; border-bottom: 1px dotted {{ accent_color }}; } a.footnote-reference:hover { border-bottom: 1px solid {{ accent_color_alternate }}; } a:hover tt { background: #EEE; } @media screen and (max-width: 870px) { div.sphinxsidebar { display: none; } div.document { width: 100%; } div.documentwrapper { margin-left: 0; margin-top: 0; margin-right: 0; margin-bottom: 0; } div.bodywrapper { margin-top: 0; margin-right: 0; margin-bottom: 0; margin-left: 0; } ul { margin-left: 0; } .document { width: auto; } .footer { width: auto; } .bodywrapper { margin: 0; } .footer { width: auto; } .github { display: none; } } @media screen and (max-width: 875px) { body { margin: 0; padding: 20px 30px; } div.documentwrapper { float: none; background: white; } div.sphinxsidebar { display: block; float: none; width: 102.5%; margin: 50px -30px -20px -30px; padding: 10px 20px; background: #333; color: white; } div.sphinxsidebar h3, div.sphinxsidebar h4, div.sphinxsidebar p, div.sphinxsidebar h3 a { color: white; } div.sphinxsidebar a { color: #aaa; } div.sphinxsidebar p.logo { display: none; } div.document { width: 100%; margin: 0; } div.related { display: block; margin: 0; padding: 10px 0 20px 0; } div.related ul, div.related ul li { margin: 0; padding: 0; } div.footer { display: none; } div.bodywrapper { margin: 0; } div.body { min-height: 0; padding: 0; } .rtd_doc_footer { display: none; } .document { width: auto; } .footer { width: auto; } .footer { width: auto; } .github { display: none; } } /* scrollbars */ ::-webkit-scrollbar { width: 6px; height: 6px; } ::-webkit-scrollbar-button:start:decrement, ::-webkit-scrollbar-button:end:increment { display: block; height: 10px; } ::-webkit-scrollbar-button:vertical:increment { background-color: #fff; } ::-webkit-scrollbar-track-piece { background-color: #eee; -webkit-border-radius: 3px; } ::-webkit-scrollbar-thumb:vertical { height: 50px; background-color: #ccc; -webkit-border-radius: 3px; } ::-webkit-scrollbar-thumb:horizontal { width: 50px; background-color: #ccc; -webkit-border-radius: 3px; } /* misc. */ .revsys-inline { display: none!important; } .admonition.warning { background-color: #F5CDCD; border-color: #7B1B1B; } ================================================ FILE: docs/_themes/flask/theme.conf ================================================ [theme] inherit = basic stylesheet = flasky.css pygments_style = flask_theme_support.FlaskyStyle ================================================ FILE: docs/_themes/flask_theme_support.py ================================================ # flasky extensions. flasky pygments style based on tango style from pygments.style import Style from pygments.token import Keyword, Name, Comment, String, Error, \ Number, Operator, Generic, Whitespace, Punctuation, Other, Literal class FlaskyStyle(Style): background_color = "#f8f8f8" default_style = "" styles = { # No corresponding class for the following: # Text: "", # class: '' Whitespace: "underline #f8f8f8", # class: 'w' Error: "#a40000 border:#ef2929", # class: 'err' Other: "#000000", # class 'x' Comment: "italic #8f5902", # class: 'c' Comment.Preproc: "noitalic", # class: 'cp' Keyword: "bold #004461", # class: 'k' Keyword.Constant: "bold #004461", # class: 'kc' Keyword.Declaration: "bold #004461", # class: 'kd' Keyword.Namespace: "bold #004461", # class: 'kn' Keyword.Pseudo: "bold #004461", # class: 'kp' Keyword.Reserved: "bold #004461", # class: 'kr' Keyword.Type: "bold #004461", # class: 'kt' Operator: "#582800", # class: 'o' Operator.Word: "bold #004461", # class: 'ow' - like keywords Punctuation: "bold #000000", # class: 'p' # because special names such as Name.Class, Name.Function, etc. # are not recognized as such later in the parsing, we choose them # to look the same as ordinary variables. Name: "#000000", # class: 'n' Name.Attribute: "#c4a000", # class: 'na' - to be revised Name.Builtin: "#004461", # class: 'nb' Name.Builtin.Pseudo: "#3465a4", # class: 'bp' Name.Class: "#000000", # class: 'nc' - to be revised Name.Constant: "#000000", # class: 'no' - to be revised Name.Decorator: "#888", # class: 'nd' - to be revised Name.Entity: "#ce5c00", # class: 'ni' Name.Exception: "bold #cc0000", # class: 'ne' Name.Function: "#000000", # class: 'nf' Name.Property: "#000000", # class: 'py' Name.Label: "#f57900", # class: 'nl' Name.Namespace: "#000000", # class: 'nn' - to be revised Name.Other: "#000000", # class: 'nx' Name.Tag: "bold #004461", # class: 'nt' - like a keyword Name.Variable: "#000000", # class: 'nv' - to be revised Name.Variable.Class: "#000000", # class: 'vc' - to be revised Name.Variable.Global: "#000000", # class: 'vg' - to be revised Name.Variable.Instance: "#000000", # class: 'vi' - to be revised Number: "#990000", # class: 'm' Literal: "#000000", # class: 'l' Literal.Date: "#000000", # class: 'ld' String: "#4e9a06", # class: 's' String.Backtick: "#4e9a06", # class: 'sb' String.Char: "#4e9a06", # class: 'sc' String.Doc: "italic #8f5902", # class: 'sd' - like a comment String.Double: "#4e9a06", # class: 's2' String.Escape: "#4e9a06", # class: 'se' String.Heredoc: "#4e9a06", # class: 'sh' String.Interpol: "#4e9a06", # class: 'si' String.Other: "#4e9a06", # class: 'sx' String.Regex: "#4e9a06", # class: 'sr' String.Single: "#4e9a06", # class: 's1' String.Symbol: "#4e9a06", # class: 'ss' Generic: "#000000", # class: 'g' Generic.Deleted: "#a40000", # class: 'gd' Generic.Emph: "italic #000000", # class: 'ge' Generic.Error: "#ef2929", # class: 'gr' Generic.Heading: "bold #000080", # class: 'gh' Generic.Inserted: "#00A000", # class: 'gi' Generic.Output: "#888", # class: 'go' Generic.Prompt: "#745334", # class: 'gp' Generic.Strong: "bold #000000", # class: 'gs' Generic.Subheading: "bold #800080", # class: 'gu' Generic.Traceback: "bold #a40000", # class: 'gt' } ================================================ FILE: docs/api.rst ================================================ .. _api_docs: API Documentation ================= ``tinydb.database`` ------------------- .. autoclass:: tinydb.database.TinyDB :members: :private-members: :member-order: bysource .. _table_api: ``tinydb.table`` ---------------- .. autoclass:: tinydb.table.Table :members: :special-members: :exclude-members: __dict__, __weakref__ :member-order: bysource .. autoclass:: tinydb.table.Document :members: :special-members: :exclude-members: __dict__, __weakref__ :member-order: bysource .. py:attribute:: doc_id The document's id ``tinydb.queries`` ------------------ .. autoclass:: tinydb.queries.Query :members: :special-members: :exclude-members: __weakref__ :member-order: bysource .. autoclass:: tinydb.queries.QueryInstance :members: :special-members: :exclude-members: __weakref__ :member-order: bysource ``tinydb.operations`` --------------------- .. automodule:: tinydb.operations :members: :special-members: :exclude-members: __weakref__ :member-order: bysource ``tinydb.storage`` ------------------ .. automodule:: tinydb.storages :members: JSONStorage, MemoryStorage :special-members: :exclude-members: __weakref__ .. class:: Storage The abstract base class for all Storages. A Storage (de)serializes the current state of the database and stores it in some place (memory, file on disk, ...). .. method:: read() Read the last stored state. .. method:: write(data) Write the current state of the database to the storage. .. method:: close() Optional: Close open file handles, etc. ``tinydb.middlewares`` ---------------------- .. automodule:: tinydb.middlewares :members: CachingMiddleware :special-members: :exclude-members: __weakref__ .. class:: Middleware The base class for all Middlewares. Middlewares hook into the read/write process of TinyDB allowing you to extend the behaviour by adding caching, logging, ... If ``read()`` or ``write()`` are not overloaded, they will be forwarded directly to the storage instance. .. attribute:: storage :type: :class:`.Storage` Access to the underlying storage instance. .. method:: read() Read the last stored state. .. method:: write(data) Write the current state of the database to the storage. .. method:: close() Optional: Close open file handles, etc. ``tinydb.utils`` ---------------- .. autoclass:: tinydb.utils.LRUCache :members: :special-members: ================================================ FILE: docs/changelog.rst ================================================ Changelog ========= Version Numbering ^^^^^^^^^^^^^^^^^ TinyDB follows the SemVer versioning guidelines. For more information, see `semver.org `_ .. note:: When new methods are added to the ``Query`` API, this may result in breaking existing code that uses the property syntax to access document fields (e.g. ``Query().some.nested.field``) where the field name is equal to the newly added query method. Thus, breaking changes may occur in feature releases even though they don't change the public API in a backwards-incompatible manner. To prevent this from happening, one can use the dict access syntax (``Query()['some']['nested']['field']``) that will not break even when new methods are added to the ``Query`` API. unreleased ^^^^^^^^^^ - *nothing yet* v4.8.2 (2024-10-12) ^^^^^^^^^^^^^^^^^^^ - Fix: Correctly update query cache when search results have changed (see `issue 560 `_). v4.8.1 (2024-10-07) ^^^^^^^^^^^^^^^^^^^ - Feature: Allow persisting empty tables (see `pull request 518 `_). - Fix: Make replacing ``doc_id`` type work properly (see `issue 545 `_). v4.8.0 (2023-06-12) ^^^^^^^^^^^^^^^^^^^ - Feature: Allow retrieve multiple documents by document ID using ``Table.get(doc_ids=[...])`` (see `pull request 504 `_). v4.7.1 (2023-01-14) ^^^^^^^^^^^^^^^^^^^ - Improvement: Improve typing annotations (see `pull request 477 `_). - Improvement: Fix some typos in the documentation (see `pull request 479 `_ and `pull request 498 `_). v4.7.0 (2022-02-19) ^^^^^^^^^^^^^^^^^^^ - Feature: Allow inserting ``Document`` instances using ``Table.insert_multiple`` (see `pull request 455 `_). - Performance: Only convert document IDs of a table when returning documents. This improves performance the ``Table.count`` and ``Table.get`` operations and also for ``Table.search`` when only returning a few documents (see `pull request 460 `_). - Internal change: Run all ``Table`` tests ``JSONStorage`` in addition to ``MemoryStorage``. v4.6.1 (2022-01-18) ^^^^^^^^^^^^^^^^^^^ - Fix: Make using callables as queries work again (see `issue 454 `__) v4.6.0 (2022-01-17) ^^^^^^^^^^^^^^^^^^^ - Feature: Add `map()` query operation to apply a transformation to a document or field when evaluating a query (see `pull request 445 `_). **Note**: This may break code that queries for a field named ``map`` using the ``Query`` APIs property access syntax - Feature: Add support for `typing-extensions `_ v4 - Documentation: Fix a couple of typos in the documentation (see `pull request 446 `_, `pull request 449 `_ and `pull request 453 `_) v4.5.2 (2021-09-23) ^^^^^^^^^^^^^^^^^^^ - Fix: Make ``Table.delete()``'s argument priorities consistent with other table methods. This means that if you pass both ``cond`` as well as ``doc_ids`` to ``Table.delete()``, the latter will be preferred (see `issue 424 `__) v4.5.1 (2021-07-17) ^^^^^^^^^^^^^^^^^^^ - Fix: Correctly install ``typing-extensions`` on Python 3.7 (see `issue 413 `__) v4.5.0 (2021-06-25) ^^^^^^^^^^^^^^^^^^^ - Feature: Better type hinting/IntelliSense for PyCharm, VS Code and MyPy (see `issue 372 `__). PyCharm and VS Code should work out of the box, for MyPy see :ref:`MyPy Type Checking ` v4.4.0 (2021-02-11) ^^^^^^^^^^^^^^^^^^^ - Feature: Add operation for searching for all documents that match a ``dict`` fragment (see `issue 300 `_) - Fix: Correctly handle queries that use fields that are also Query methods, e.g. ``Query()['test']`` for searching for documents with a ``test`` field (see `issue 373 `_) v4.3.0 (2020-11-14) ^^^^^^^^^^^^^^^^^^^ - Feature: Add operation for updating multiple documents: ``update_multiple`` (see `issue 346 `_) - Improvement: Expose type information for MyPy typechecking (PEP 561) (see `pull request 352 `_) v4.2.0 (2020-10-03) ^^^^^^^^^^^^^^^^^^^ - Feature: Add support for specifying document IDs during insertion (see `issue 303 `_) - Internal change: Use ``OrderedDict.move_to_end()`` in the query cache (see `issue 338 `_) v4.1.1 (2020-05-08) ^^^^^^^^^^^^^^^^^^^ - Fix: Don't install dev-dependencies when installing from PyPI (see `issue 315 `_) v4.1.0 (2020-05-07) ^^^^^^^^^^^^^^^^^^^ - Feature: Add a no-op query ``Query().noop()`` (see `issue 313 `_) - Feature: Add a ``access_mode`` flag to ``JSONStorage`` to allow opening files read-only (see `issue 297 `_) - Fix: Don't drop the first document that's being inserted when inserting data on an existing database (see `issue 314 `_) v4.0.0 (2020-05-02) ^^^^^^^^^^^^^^^^^^^ :ref:`Upgrade Notes ` Breaking Changes ---------------- - Python 2 support has been removed, see `issue 284 `_ for background - API changes: - Removed classes: ``DataProxy``, ``StorageProxy`` - Attributes removed from ``TinyDB`` in favor of customizing ``TinyDB``'s behavior by subclassing it and overloading ``__init__(...)`` and ``table(...)``: - ``DEFAULT_TABLE`` - ``DEFAULT_TABLE_KWARGS`` - ``DEFAULT_STORAGE`` - Arguments removed from ``TinyDB(...)``: - ``default_table``: replace with ``TinyDB.default_table_name = 'name'`` - ``table_class``: replace with ``TinyDB.table_class = Class`` - ``TinyDB.contains(...)``'s ``doc_ids`` parameter has been renamed to ``doc_id`` and now only takes a single document ID - ``TinyDB.purge_tables(...)`` has been renamed to ``TinyDB.drop_tables(...)`` - ``TinyDB.purge_table(...)`` has been renamed to ``TinyDB.drop_table(...)`` - ``TinyDB.write_back(...)`` has been removed - ``TinyDB.process_elements(...)`` has been removed - ``Table.purge()`` has been renamed to ``Table.truncate()`` - Evaluating an empty ``Query()`` without any test operators will now result in an exception, use ``Query().noop()`` (introduced in v4.1.0) instead - ``ujson`` support has been removed, see `issue 263 `_ and `issue 306 `_ for background - The deprecated Element ID API has been removed (e.g. using the ``Element`` class or ``eids`` parameter) in favor the Document API, see `pull request 158 `_ for details on the replacement Improvements ------------ - TinyDB's internal architecture has been reworked to be more simple and streamlined in order to make it easier to customize TinyDB's behavior - With the new architecture, TinyDB performance will improve for many applications Bugfixes -------- - Don't break the tests when ``ujson`` is installed (see `issue 262 `_) - Fix performance when reading data (see `issue 250 `_) - Fix inconsistent purge function names (see `issue 103 `_) v3.15.1 (2019-10-26) ^^^^^^^^^^^^^^^^^^^^ - Internal change: fix missing values handling for ``LRUCache`` v3.15.0 (2019-10-12) ^^^^^^^^^^^^^^^^^^^^ - Feature: allow setting the parameters of TinyDB's default table (see `issue 278 `_) v3.14.2 (2019-09-13) ^^^^^^^^^^^^^^^^^^^^ - Internal change: support correct iteration for ``LRUCache`` objects v3.14.1 (2019-07-03) ^^^^^^^^^^^^^^^^^^^^ - Internal change: fix Query class to permit subclass creation (see `pull request 270 `_) v3.14.0 (2019-06-18) ^^^^^^^^^^^^^^^^^^^^ - Change: support for ``ujson`` is now deprecated (see `issue 263 `_) v3.13.0 (2019-03-16) ^^^^^^^^^^^^^^^^^^^^ - Feature: direct access to a TinyDB instance's storage (see `issue 258 `_) v3.12.2 (2018-12-12) ^^^^^^^^^^^^^^^^^^^^ - Internal change: convert documents to dicts during insertion (see `pull request 256 `_) - Internal change: use tuple literals instead of tuple class/constructor (see `pull request 247 `_) - Infra: ensure YAML tests are run (see `pull request 252 `_) v3.12.1 (2018-11-09) ^^^^^^^^^^^^^^^^^^^^ - Fix: Don't break when searching the same query multiple times (see `pull request 249 `_) - Internal change: allow ``collections.abc.Mutable`` as valid document types (see `pull request 245 `_) v3.12.0 (2018-11-06) ^^^^^^^^^^^^^^^^^^^^ - Feature: Add encoding option to ``JSONStorage`` (see `pull request 238 `_) - Internal change: allow ``collections.abc.Mutable`` as valid document types (see `pull request 245 `_) v3.11.1 (2018-09-13) ^^^^^^^^^^^^^^^^^^^^ - Bugfix: Make path queries (``db.search(where('key))``) work again (see `issue 232 `_) - Improvement: Add custom ``repr`` representations for main classes (see `pull request 229 `_) v3.11.0 (2018-08-20) ^^^^^^^^^^^^^^^^^^^^ - **Drop official support for Python 3.3**. Python 3.3 has reached its official End Of Life as of September 29, 2017. It will probably continue to work, but will not be tested against (`issue 217 `_) - Feature: Allow extending TinyDB with a custom storage proxy class (see `pull request 224 `_) - Bugfix: Return list of document IDs for upsert when creating a new document (see `issue 223 `_) v3.10.0 (2018-07-21) ^^^^^^^^^^^^^^^^^^^^ - Feature: Add support for regex flags (see `pull request 216 `_) v3.9.0 (2018-04-24) ^^^^^^^^^^^^^^^^^^^ - Feature: Allow setting a table class for single table only (see `issue 197 `_) - Internal change: call fsync after flushing ``JSONStorage`` (see `issue 208 `_) v3.8.1 (2018-03-26) ^^^^^^^^^^^^^^^^^^^ - Bugfix: Don't install tests as a package anymore (see `pull request #195 `_) v3.8.0 (2018-03-01) ^^^^^^^^^^^^^^^^^^^ - Feature: Allow disabling the query cache with ``db.table(name, cache_size=0)`` (see `pull request #187 `_) - Feature: Add ``db.write_back(docs)`` for replacing documents (see `pull request #184 `_) v3.7.0 (2017-11-11) ^^^^^^^^^^^^^^^^^^^ - Feature: ``one_of`` for checking if a value is contained in a list (see `issue 164 `_) - Feature: Upsert (insert if document doesn't exist, otherwise update; see https://forum.m-siemens.de/d/30-primary-key-well-sort-of) - Internal change: don't read from storage twice during initialization (see https://forum.m-siemens.de/d/28-reads-the-whole-data-file-twice) v3.6.0 (2017-10-05) ^^^^^^^^^^^^^^^^^^^ - Allow updating all documents using ``db.update(fields)`` (see `issue #157 `_). - Rename elements to documents. Document IDs now available with ``doc.doc_id``, using ``doc.eid`` is now deprecated (see `pull request #158 `_) v3.5.0 (2017-08-30) ^^^^^^^^^^^^^^^^^^^ - Expose the table name via ``table.name`` (see `issue #147 `_). - Allow better subclassing of the ``TinyDB`` class (see `pull request #150 `_). v3.4.1 (2017-08-23) ^^^^^^^^^^^^^^^^^^^ - Expose TinyDB version via ``import tinyb; tinydb.__version__`` (see `issue #148 `_). v3.4.0 (2017-08-08) ^^^^^^^^^^^^^^^^^^^ - Add new update operations: ``add(key, value)``, ``subtract(key, value)``, and ``set(key, value)`` (see `pull request #145 `_). v3.3.1 (2017-06-27) ^^^^^^^^^^^^^^^^^^^ - Use relative imports to allow vendoring TinyDB in other packages (see `pull request #142 `_). v3.3.0 (2017-06-05) ^^^^^^^^^^^^^^^^^^^ - Allow iterating over a database or table yielding all documents (see `pull request #139 `_). v3.2.3 (2017-04-22) ^^^^^^^^^^^^^^^^^^^ - Fix bug with accidental modifications to the query cache when modifying the list of search results (see `issue #132 `_). v3.2.2 (2017-01-16) ^^^^^^^^^^^^^^^^^^^ - Fix the ``Query`` constructor to prevent wrong usage (see `issue #117 `_). v3.2.1 (2016-06-29) ^^^^^^^^^^^^^^^^^^^ - Fix a bug with queries on documents that have a ``path`` key (see `pull request #107 `_). - Don't write to the database file needlessly when opening the database (see `pull request #104 `_). v3.2.0 (2016-04-25) ^^^^^^^^^^^^^^^^^^^ - Add a way to specify the default table name via :ref:`default_table ` (see `pull request #98 `_). - Add ``db.purge_table(name)`` to remove a single table (see `pull request #100 `_). - Along the way: celebrating 100 issues and pull requests! Thanks everyone for every single contribution! - Extend API documentation (see `issue #96 `_). v3.1.3 (2016-02-14) ^^^^^^^^^^^^^^^^^^^ - Fix a bug when using unhashable documents (lists, dicts) with ``Query.any`` or ``Query.all`` queries (see `a forum post by karibul `_). v3.1.2 (2016-01-30) ^^^^^^^^^^^^^^^^^^^ - Fix a bug when using unhashable documents (lists, dicts) with ``Query.any`` or ``Query.all`` queries (see `a forum post by karibul `_). v3.1.1 (2016-01-23) ^^^^^^^^^^^^^^^^^^^ - Inserting a dictionary with data that is not JSON serializable doesn't lead to corrupt files anymore (see `issue #89 `_). - Fix a bug in the LRU cache that may lead to an invalid query cache (see `issue #87 `_). v3.1.0 (2015-12-31) ^^^^^^^^^^^^^^^^^^^ - ``db.update(...)`` and ``db.remove(...)`` now return affected document IDs (see `issue #83 `_). - Inserting an invalid document (i.e. not a ``dict``) now raises an error instead of corrupting the database (see `issue #74 `_). v3.0.0 (2015-11-13) ^^^^^^^^^^^^^^^^^^^ - Overhauled Query model: - ``where('...').contains('...')`` has been renamed to ``where('...').search('...')``. - Support for ORM-like usage: ``User = Query(); db.search(User.name == 'John')``. - ``where('foo')`` is an alias for ``Query().foo``. - ``where('foo').has('bar')`` is replaced by either ``where('foo').bar`` or ``Query().foo.bar``. - In case the key is not a valid Python identifier, array notation can be used: ``where('a.b.c')`` is now ``Query()['a.b.c']``. - Checking for the existence of a key has to be done explicitly: ``where('foo').exists()``. - Migrations from v1 to v2 have been removed. - ``SmartCacheTable`` has been moved to `msiemens/tinydb-smartcache`_. - Serialization has been moved to `msiemens/tinydb-serialization`_. - Empty storages are now expected to return ``None`` instead of raising ``ValueError``. (see `issue #67 `_. .. _msiemens/tinydb-smartcache: https://github.com/msiemens/tinydb-smartcache .. _msiemens/tinydb-serialization: https://github.com/msiemens/tinydb-serialization v2.4.0 (2015-08-14) ^^^^^^^^^^^^^^^^^^^ - Allow custom parameters for custom test functions (see `issue #63 `_ and `pull request #64 `_). v2.3.2 (2015-05-20) ^^^^^^^^^^^^^^^^^^^ - Fix a forgotten debug output in the ``SerializationMiddleware`` (see `issue #55 `_). - Fix an "ignored exception" warning when using the ``CachingMiddleware`` (see `pull request #54 `_) - Fix a problem with symlinks when checking out TinyDB on OSX Yosemite (see `issue #52 `_). v2.3.1 (2015-04-30) ^^^^^^^^^^^^^^^^^^^ - Hopefully fix a problem with using TinyDB as a dependency in a ``setup.py`` script (see `issue #51 `_). v2.3.0 (2015-04-08) ^^^^^^^^^^^^^^^^^^^ - Added support for custom serialization. That way, you can teach TinyDB to store ``datetime`` objects in a JSON file :) (see `issue #48 `_ and `pull request #50 `_) - Fixed a performance regression when searching became slower with every search (see `issue #49 `_) - Internal code has been cleaned up v2.2.2 (2015-02-12) ^^^^^^^^^^^^^^^^^^^ - Fixed a data loss when using ``CachingMiddleware`` together with ``JSONStorage`` (see `issue #47 `_) v2.2.1 (2015-01-09) ^^^^^^^^^^^^^^^^^^^ - Fixed handling of IDs with the JSON backend that converted integers to strings (see `issue #45 `_) v2.2.0 (2014-11-10) ^^^^^^^^^^^^^^^^^^^ - Extended ``any`` and ``all`` queries to take lists as conditions (see `pull request #38 `_) - Fixed an ``decode error`` when installing TinyDB in a non-UTF-8 environment (see `pull request #37 `_) - Fixed some issues with ``CachingMiddleware`` in combination with ``JSONStorage`` (see `pull request #39 `_) v2.1.0 (2014-10-14) ^^^^^^^^^^^^^^^^^^^ - Added ``where(...).contains(regex)`` (see `issue #32 `_) - Fixed a bug that corrupted data after reopening a database (see `issue #34 `_) v2.0.1 (2014-09-22) ^^^^^^^^^^^^^^^^^^^ - Fixed handling of Unicode data in Python 2 (see `issue #28 `_). v2.0.0 (2014-09-05) ^^^^^^^^^^^^^^^^^^^ :ref:`Upgrade Notes ` .. warning:: TinyDB changed the way data is stored. You may need to migrate your databases to the new scheme. Check out the :ref:`Upgrade Notes ` for details. - The syntax ``query in db`` has been removed, use ``db.contains`` instead. - The ``ConcurrencyMiddleware`` has been removed due to a insecure implementation (see `issue #18 `_). Consider :ref:`tinyrecord` instead. - Better support for working with :ref:`Document IDs `. - Added support for `nested comparisons `_. - Added ``all`` and ``any`` `comparisons on lists `_. - Added optional :`_. - The query cache is now a :ref:`fixed size LRU cache `. v1.4.0 (2014-07-22) ^^^^^^^^^^^^^^^^^^^ - Added ``insert_multiple`` function (see `issue #8 `_). v1.3.0 (2014-07-02) ^^^^^^^^^^^^^^^^^^^ - Fixed `bug #7 `_: IDs not unique. - Extended the API: ``db.count(where(...))`` and ``db.contains(where(...))``. - The syntax ``query in db`` is now **deprecated** and replaced by ``db.contains``. v1.2.0 (2014-06-19) ^^^^^^^^^^^^^^^^^^^ - Added ``update`` method (see `issue #6 `_). v1.1.1 (2014-06-14) ^^^^^^^^^^^^^^^^^^^ - Merged `PR #5 `_: Fix minor documentation typos and style issues. v1.1.0 (2014-05-06) ^^^^^^^^^^^^^^^^^^^ - Improved the docs and fixed some typos. - Refactored some internal code. - Fixed a bug with multiple ``TinyDB?`` instances. v1.0.1 (2014-04-26) ^^^^^^^^^^^^^^^^^^^ - Fixed a bug in ``JSONStorage`` that broke the database when removing entries. v1.0.0 (2013-07-20) ^^^^^^^^^^^^^^^^^^^ - First official release – consider TinyDB stable now. ================================================ FILE: docs/conf.py ================================================ # -*- coding: utf-8 -*- # # TinyDB documentation build configuration file, created by # sphinx-quickstart on Sat Jul 13 20:14:55 2013. # # This file is execfile()d with the current directory set to its containing # dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import os import sys from importlib import metadata # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # sys.path.insert(0, os.path.abspath('.')) # -- General configuration ---------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. # needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.coverage', 'sphinx.ext.viewcode', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.extlinks'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. # source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'TinyDB' copyright = u'2021, Markus Siemens' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. try: release = metadata.version('tinydb') except metadata.PackageNotFoundError: print('To build the documentation, The distribution information of TinyDB') print('has to be available. Either install the package into your') print('development environment or run "pip install -e ." to setup the') print('metadata. A virtualenv is recommended!') sys.exit(1) if 'dev' in release: release = release.split('dev')[0] + 'dev' version = '.'.join(release.split('.')[:2]) # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: # today = '' # Else, today_fmt is used as the format for a strftime call. # today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build'] # The reST default role (used for this markup: `text`) to use for all # documents. # default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. # add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). # add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. # show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. # keep_warnings = False # -- Options for HTML output -------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". # html_title = None # A shorter title for the navigation bar. Default is the same as html_title. # html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. # html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. # html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. # html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. # html_use_smartypants = True # Custom sidebar templates, maps document names to template names. html_sidebars = { 'index': ['sidebarlogo.html', 'links.html', 'searchbox.html'], '**': ['sidebarlogo.html', 'localtoc.html', 'links.html', 'searchbox.html'] } # Additional templates that should be rendered to pages, maps page names to # template names. # html_additional_pages = {} # If false, no module index is generated. # html_domain_indices = True # If false, no index is generated. # html_use_index = True # If true, the index is split into individual pages for each letter. # html_split_index = False # If true, links to the reST sources are added to the pages. html_show_sourcelink = False # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. # html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. # html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. # html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). # html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'TinyDBdoc' # -- Options for LaTeX output ------------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # 'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass # [howto/manual]). latex_documents = [ ('index', 'TinyDB.tex', u'TinyDB Documentation', u'Markus Siemens', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. # latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. # latex_use_parts = False # If true, show page references after internal links. # latex_show_pagerefs = False # If true, show URL addresses after external links. # latex_show_urls = False # Documents to append as an appendix to all manuals. # latex_appendices = [] # If false, no module index is generated. # latex_domain_indices = True # -- Options for manual page output ------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'tinydb', u'TinyDB Documentation', [u'Markus Siemens'], 1) ] # If true, show URL addresses after external links. # man_show_urls = False # -- Options for Texinfo output ----------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ('index', 'TinyDB', u'TinyDB Documentation', u'Markus Siemens', 'TinyDB', 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. # texinfo_appendices = [] # If false, no module index is generated. # texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. # texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. # texinfo_no_detailmenu = False extlinks = {'issue': ('https://https://github.com/msiemens/tinydb/issues/%s', 'issue ')} sys.path.append(os.path.abspath('_themes')) html_theme_path = ['_themes'] html_theme = 'flask' todo_include_todos = True ================================================ FILE: docs/contribute.rst ================================================ Contribution Guidelines ####################### Whether reporting bugs, discussing improvements and new ideas or writing extensions: Contributions to TinyDB are welcome! Here's how to get started: 1. Check for open issues or open a fresh issue to start a discussion around a feature idea or a bug 2. Fork `the repository `_ on Github, create a new branch off the `master` branch and start making your changes (known as `GitHub Flow `_) 3. Write a test which shows that the bug was fixed or that the feature works as expected 4. Send a pull request and bug the maintainer until it gets merged and published :) Philosophy of TinyDB ******************** TinyDB aims to be simple and fun to use. Therefore two key values are simplicity and elegance of interfaces and code. These values will contradict each other from time to time. In these cases , try using as little magic as possible. In any case don't forget documenting code that isn't clear at first glance. Code Conventions **************** In general the TinyDB source should always follow `PEP 8 `_. Exceptions are allowed in well justified and documented cases. However we make a small exception concerning docstrings: When using multiline docstrings, keep the opening and closing triple quotes on their own lines and add an empty line after it. .. code-block:: python def some_function(): """ Documentation ... """ # implementation ... Version Numbers *************** TinyDB follows the `SemVer versioning guidelines `_. This implies that backwards incompatible changes in the API will increment the major version. So think twice before making such changes. ================================================ FILE: docs/extend.rst ================================================ How to Extend TinyDB ==================== There are three main ways to extend TinyDB and modify its behaviour: 1. custom storages, 2. custom middlewares, 3. use hooks and overrides, and 4. subclassing ``TinyDB`` and ``Table``. Let's look at them in this order. Write a Custom Storage ---------------------- First, we have support for custom storages. By default TinyDB comes with an in-memory storage and a JSON file storage. But of course you can add your own. Let's look how you could add a `YAML `_ storage using `PyYAML `_: .. code-block:: python import yaml class YAMLStorage(Storage): def __init__(self, filename): # (1) self.filename = filename def read(self): with open(self.filename) as handle: try: data = yaml.safe_load(handle.read()) # (2) return data except yaml.YAMLError: return None # (3) def write(self, data): with open(self.filename, 'w+') as handle: yaml.dump(data, handle) def close(self): # (4) pass There are some things we should look closer at: 1. The constructor will receive all arguments passed to TinyDB when creating the database instance (except ``storage`` which TinyDB itself consumes). In other words calling ``TinyDB('something', storage=YAMLStorage)`` will pass ``'something'`` as an argument to ``YAMLStorage``. If you accept callables or other executable values in your storage constructor (or elsewhere), do not derive them from untrusted or user-controlled input. 2. We use ``yaml.safe_load`` as recommended by the `PyYAML documentation `_ when processing data from a potentially untrusted source. 3. If the storage is uninitialized, TinyDB expects the storage to return ``None`` so it can do any internal initialization that is necessary. 4. If your storage needs any cleanup (like closing file handles) before an instance is destroyed, you can put it in the ``close()`` method. To run these, you'll either have to run ``db.close()`` on your ``TinyDB`` instance or use it as a context manager, like this: .. code-block:: python with TinyDB('db.yml', storage=YAMLStorage) as db: # ... Finally, using the YAML storage is very straight-forward: .. code-block:: python db = TinyDB('db.yml', storage=YAMLStorage) # ... Write Custom Middleware ------------------------- Sometimes you don't want to write a new storage module but rather modify the behaviour of an existing one. As an example we'll build middleware that filters out empty items. Because middleware acts as a wrapper around a storage, they needs a ``read()`` and a ``write(data)`` method. In addition, they can access the underlying storage via ``self.storage``. Before we start implementing we should look at the structure of the data that the middleware receives. Here's what the data that goes through the middleware looks like: .. code-block:: python { '_default': { 1: {'key': 'value'}, 2: {'key': 'value'}, # other items }, # other tables } Thus, we'll need two nested loops: 1. Process every table 2. Process every item Now let's implement that: .. code-block:: python class RemoveEmptyItemsMiddleware(Middleware): def __init__(self, storage_cls): # Any middleware *has* to call the super constructor # with storage_cls super().__init__(storage_cls) # (1) def read(self): data = self.storage.read() for table_name in data: table_data = data[table_name] for doc_id in table_data: item = table_data[doc_id] if item == {}: del table_data[doc_id] return data def write(self, data): for table_name in data: table_data = data[table_name] for doc_id in table_data: item = table_data[doc_id] if item == {}: del table_data[doc_id] self.storage.write(data) def close(self): self.storage.close() Note that the constructor calls the middleware constructor (1) and passes the storage class to the middleware constructor. To wrap storage with this new middleware, we use it like this: .. code-block:: python db = TinyDB(storage=RemoveEmptyItemsMiddleware(SomeStorageClass)) Here ``SomeStorageClass`` should be replaced with the storage you want to use. If you leave it empty, the default storage will be used (which is the ``JSONStorage``). Use hooks and overrides ----------------------- .. _extend_hooks: There are cases when neither creating a custom storage nor using a custom middleware will allow you to adapt TinyDB in the way you need. In this case you can modify TinyDB's behavior by using predefined hooks and override points. For example you can configure the name of the default table by setting ``TinyDB.default_table_name``: .. code-block:: python TinyDB.default_table_name = 'my_table_name' Both :class:`~tinydb.database.TinyDB` and the :class:`~tinydb.table.Table` classes allow modifying their behavior using hooks and overrides. To use ``Table``'s overrides, you can access the class using ``TinyDB.table_class``: .. code-block:: python TinyDB.table_class.default_query_cache_capacity = 100 Read the :ref:`api_docs` for more details on the available hooks and override points. Subclassing ``TinyDB`` and ``Table`` ------------------------------------ Finally, there's the last option to modify TinyDB's behavior. That way you can change how TinyDB itself works more deeply than using the other extension mechanisms. When creating a subclass you can use it by using hooks and overrides to override the default classes that TinyDB uses: .. code-block:: python class MyTable(Table): # Add your method overrides ... TinyDB.table_class = MyTable # Continue using TinyDB as usual TinyDB's source code is documented with extensions in mind, explaining how everything works even for internal methods and classes. Feel free to dig into the source and adapt everything you need for your projects. ================================================ FILE: docs/extensions.rst ================================================ Extensions ========== Here are some extensions that might be useful to you: ``tinydb-rust`` ************** | **Repo:** https://github.com/itsmorninghao/tinydb-rust/ | **Status:** *beta* | **Description:** A drop-in reimplementation of TinyDB that uses Rust for better performance. ``aiotinydb`` ************* | **Repo:** https://github.com/ASMfreaK/aiotinydb | **Status:** *stable* | **Description:** asyncio compatibility shim for TinyDB. Enables usage of TinyDB in asyncio-aware contexts without slow synchronous IO. ``BetterJSONStorage`` ********************* | **Repo:** https://github.com/MrPigss/BetterJSONStorage | **Status:** *stable* | **Description:** BetterJSONStorage is a faster 'Storage Type' for TinyDB. It uses the faster Orjson library for parsing the JSON and BLOSC for compression. ``tinydb-serialization`` ************************ | **Repo:** https://github.com/msiemens/tinydb-serialization | **Status:** *stable* | **Description:** ``tinydb-serialization`` provides serialization for objects that TinyDB otherwise couldn't handle. ``tinydb-smartcache`` ********************* | **Repo:** https://github.com/msiemens/tinydb-smartcache | **Status:** *stable* | **Description:** ``tinydb-smartcache`` provides a smart query cache for TinyDB. It updates the query cache when inserting/removing/updating documents so the cache doesn't get invalidated. It's useful if you perform lots of queries while the data changes only little. .. _tinyrecord: ``tinyrecord`` ************** | **Repo:** https://github.com/eugene-eeo/tinyrecord | **Status:** *stable* | **Description:** Tinyrecord is a library which implements experimental atomic transaction support for the TinyDB NoSQL database. It uses a record-first then execute architecture which allows us to minimize the time that we are within a thread lock. ``tinydb-appengine`` ******************** | **Repo:** https://github.com/imalento/tinydb-appengine | **Status:** *inactive* | **Description:** ``tinydb-appengine`` provides TinyDB storage for App Engine. You can use JSON readonly. ``TinyDBTimestamps`` ******************** | **Repo:** https://github.com/pachacamac/TinyDBTimestamps | **Status:** *inactive* | **Description:** Automatically add create at/ update at timestamps to TinyDB documents. ``tinyindex`` ************* | **Repo:** https://github.com/eugene-eeo/tinyindex | **Status:** *inactive* | **Description:** Document indexing for TinyDB. Basically ensures deterministic (as long as there aren't any changes to the table) yielding of documents. ``tinymongo`` ************* | **Repo:** https://github.com/schapman1974/tinymongo | **Status:** *inactive* | **Description:** A simple wrapper that allows to use TinyDB as a flat file drop-in replacement for MongoDB. ``TinyMP`` ************* | **Repo:** https://github.com/alshapton/TinyMP | **Status:** *inactive* | **Description:** A MessagePack-based storage extension to tinydb using http://msgpack.org ================================================ FILE: docs/getting-started.rst ================================================ :tocdepth: 3 Getting Started =============== Installing TinyDB ----------------- To install TinyDB from PyPI, run:: $ pip install tinydb You can also grab the latest development version from GitHub_. After downloading and unpacking it, you can install it using:: $ pip install . Basic Usage ----------- Let's cover the basics before going more into detail. We'll start by setting up a TinyDB database: >>> from tinydb import TinyDB, Query >>> db = TinyDB('db.json') You now have a TinyDB database that stores its data in ``db.json``. What about inserting some data? TinyDB expects the data to be Python ``dict``\s: >>> db.insert({'type': 'apple', 'count': 7}) >>> db.insert({'type': 'peach', 'count': 3}) .. note:: The ``insert`` method returns the inserted document's ID. Read more about it here: :ref:`document_ids`. Now you can get all documents stored in the database by running: >>> db.all() [{'count': 7, 'type': 'apple'}, {'count': 3, 'type': 'peach'}] You can also iter over stored documents: >>> for item in db: >>> print(item) {'count': 7, 'type': 'apple'} {'count': 3, 'type': 'peach'} Of course you'll also want to search for specific documents. Let's try: >>> Fruit = Query() >>> db.search(Fruit.type == 'peach') [{'count': 3, 'type': 'peach'}] >>> db.search(Fruit.count > 5) [{'count': 7, 'type': 'apple'}] Next we'll update the ``count`` field of the apples: >>> db.update({'count': 10}, Fruit.type == 'apple') >>> db.all() [{'count': 10, 'type': 'apple'}, {'count': 3, 'type': 'peach'}] In the same manner you can also remove documents: >>> db.remove(Fruit.count < 5) >>> db.all() [{'count': 10, 'type': 'apple'}] And of course you can throw away all data to start with an empty database: >>> db.truncate() >>> db.all() [] Recap ***** Before we dive deeper, let's recapitulate the basics: +-------------------------------+---------------------------------------------------------------+ | **Inserting** | +-------------------------------+---------------------------------------------------------------+ | ``db.insert(...)`` | Insert a document | +-------------------------------+---------------------------------------------------------------+ | **Getting data** | +-------------------------------+---------------------------------------------------------------+ | ``db.all()`` | Get all documents | +-------------------------------+---------------------------------------------------------------+ | ``iter(db)`` | Iter over all documents | +-------------------------------+---------------------------------------------------------------+ | ``db.search(query)`` | Get a list of documents matching the query | +-------------------------------+---------------------------------------------------------------+ | **Updating** | +-------------------------------+---------------------------------------------------------------+ | ``db.update(fields, query)`` | Update all documents matching the query to contain ``fields`` | +-------------------------------+---------------------------------------------------------------+ | **Removing** | +-------------------------------+---------------------------------------------------------------+ | ``db.remove(query)`` | Remove all documents matching the query | +-------------------------------+---------------------------------------------------------------+ | ``db.truncate()`` | Remove all documents | +-------------------------------+---------------------------------------------------------------+ | **Querying** | +-------------------------------+---------------------------------------------------------------+ | ``Query()`` | Create a new query object | +-------------------------------+---------------------------------------------------------------+ | ``Query().field == 2`` | Match any document that has a key ``field`` with value | | | ``== 2`` (also possible: ``!=``, ``>``, ``>=``, ``<``, ``<=``)| +-------------------------------+---------------------------------------------------------------+ .. note:: Query comparisons only support literal values on the right-hand side. Field-to-field comparisons like ``Query().a == Query().b`` are not supported. Use a callable predicate like ``db.search(lambda doc: doc.get('a') == doc.get('b'))`` for custom logic. .. note:: Callables passed to query APIs (e.g. ``lambda`` predicates or ``Query().map``) execute in-process and must **never** be derived from untrusted or user-controlled input. .. References .. _GitHub: http://github.com/msiemens/tinydb/ ================================================ FILE: docs/index.rst ================================================ Welcome to TinyDB! ================== Welcome to TinyDB, your tiny, document oriented database optimized for your happiness :) >>> from tinydb import TinyDB, Query >>> db = TinyDB('path/to/db.json') >>> User = Query() >>> db.insert({'name': 'John', 'age': 22}) >>> db.search(User.name == 'John') [{'name': 'John', 'age': 22}] User's Guide ------------ .. toctree:: :maxdepth: 2 intro getting-started usage Extending TinyDB ---------------- .. toctree:: :maxdepth: 2 Extending TinyDB TinyDB Extensions API Reference ------------- .. toctree:: :maxdepth: 2 api Additional Notes ---------------- .. toctree:: :maxdepth: 2 contribute changelog Upgrade Notes ================================================ FILE: docs/intro.rst ================================================ Introduction ============ Great that you've taken time to check out the TinyDB docs! Before we begin looking at TinyDB itself, let's take some time to see whether you should use TinyDB. Why Use TinyDB? --------------- - **tiny:** The current source code has 1800 lines of code (with about 40% documentation) and 1600 lines tests. - **document oriented:** Like MongoDB_, you can store any document (represented as ``dict``) in TinyDB. - **optimized for your happiness:** TinyDB is designed to be simple and fun to use by providing a simple and clean API. - **written in pure Python:** TinyDB neither needs an external server (as e.g. `PyMongo `_) nor any dependencies from PyPI. - **works on Python 3.5+ and PyPy:** TinyDB works on all modern versions of Python and PyPy. - **powerfully extensible:** You can easily extend TinyDB by writing new storages or modify the behaviour of storages with Middlewares. - **100% test coverage:** No explanation needed. In short: If you need a simple database with a clean API that just works without lots of configuration, TinyDB might be the right choice for you. Why **Not** Use TinyDB? ----------------------- - You need **advanced features** like: - access from multiple processes or threads (e.g. when using Flask!), - creating indexes for tables, - an HTTP server, - managing relationships between tables or similar, - `ACID guarantees `_. - You are really concerned about **performance** and need a high speed database. To put it plainly: If you need advanced features or high performance, TinyDB is the wrong database for you – consider using databases like SQLite_, Buzhug_, CodernityDB_ or MongoDB_. .. References .. _Buzhug: https://buzhug.sourceforge.net/ .. _CodernityDB: http://labs.codernity.com/codernitydb/ .. _MongoDB: https://mongodb.org/ .. _SQLite: https://www.sqlite.org/ ================================================ FILE: docs/make.bat ================================================ @ECHO OFF REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set BUILDDIR=_build set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . set I18NSPHINXOPTS=%SPHINXOPTS% . if NOT "%PAPER%" == "" ( set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% ) if "%1" == "" goto help if "%1" == "help" ( :help echo.Please use `make ^` where ^ is one of echo. html to make standalone HTML files echo. dirhtml to make HTML files named index.html in directories echo. singlehtml to make a single large HTML file echo. pickle to make pickle files echo. json to make JSON files echo. htmlhelp to make HTML files and a HTML help project echo. qthelp to make HTML files and a qthelp project echo. devhelp to make HTML files and a Devhelp project echo. epub to make an epub echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter echo. text to make text files echo. man to make manual pages echo. texinfo to make Texinfo files echo. gettext to make PO message catalogs echo. changes to make an overview over all changed/added/deprecated items echo. xml to make Docutils-native XML files echo. pseudoxml to make pseudoxml-XML files for display purposes echo. linkcheck to check all external links for integrity echo. doctest to run all doctests embedded in the documentation if enabled goto end ) if "%1" == "clean" ( for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i del /q /s %BUILDDIR%\* goto end ) %SPHINXBUILD% 2> nul if errorlevel 9009 ( echo. echo.The 'sphinx-build' command was not found. Make sure you have Sphinx echo.installed, then set the SPHINXBUILD environment variable to point echo.to the full path of the 'sphinx-build' executable. Alternatively you echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from echo.http://sphinx-doc.org/ exit /b 1 ) if "%1" == "html" ( %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html if errorlevel 1 exit /b 1 echo. echo.Build finished. The HTML pages are in %BUILDDIR%/html. goto end ) if "%1" == "dirhtml" ( %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml if errorlevel 1 exit /b 1 echo. echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. goto end ) if "%1" == "singlehtml" ( %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml if errorlevel 1 exit /b 1 echo. echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. goto end ) if "%1" == "pickle" ( %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can process the pickle files. goto end ) if "%1" == "json" ( %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can process the JSON files. goto end ) if "%1" == "htmlhelp" ( %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can run HTML Help Workshop with the ^ .hhp project file in %BUILDDIR%/htmlhelp. goto end ) if "%1" == "qthelp" ( %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp if errorlevel 1 exit /b 1 echo. echo.Build finished; now you can run "qcollectiongenerator" with the ^ .qhcp project file in %BUILDDIR%/qthelp, like this: echo.^> qcollectiongenerator %BUILDDIR%\qthelp\TinyDB.qhcp echo.To view the help file: echo.^> assistant -collectionFile %BUILDDIR%\qthelp\TinyDB.ghc goto end ) if "%1" == "devhelp" ( %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp if errorlevel 1 exit /b 1 echo. echo.Build finished. goto end ) if "%1" == "epub" ( %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub if errorlevel 1 exit /b 1 echo. echo.Build finished. The epub file is in %BUILDDIR%/epub. goto end ) if "%1" == "latex" ( %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex if errorlevel 1 exit /b 1 echo. echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. goto end ) if "%1" == "latexpdf" ( %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex cd %BUILDDIR%/latex make all-pdf cd %BUILDDIR%/.. echo. echo.Build finished; the PDF files are in %BUILDDIR%/latex. goto end ) if "%1" == "latexpdfja" ( %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex cd %BUILDDIR%/latex make all-pdf-ja cd %BUILDDIR%/.. echo. echo.Build finished; the PDF files are in %BUILDDIR%/latex. goto end ) if "%1" == "text" ( %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text if errorlevel 1 exit /b 1 echo. echo.Build finished. The text files are in %BUILDDIR%/text. goto end ) if "%1" == "man" ( %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man if errorlevel 1 exit /b 1 echo. echo.Build finished. The manual pages are in %BUILDDIR%/man. goto end ) if "%1" == "texinfo" ( %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo if errorlevel 1 exit /b 1 echo. echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. goto end ) if "%1" == "gettext" ( %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale if errorlevel 1 exit /b 1 echo. echo.Build finished. The message catalogs are in %BUILDDIR%/locale. goto end ) if "%1" == "changes" ( %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes if errorlevel 1 exit /b 1 echo. echo.The overview file is in %BUILDDIR%/changes. goto end ) if "%1" == "linkcheck" ( %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck if errorlevel 1 exit /b 1 echo. echo.Link check complete; look for any errors in the above output ^ or in %BUILDDIR%/linkcheck/output.txt. goto end ) if "%1" == "doctest" ( %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest if errorlevel 1 exit /b 1 echo. echo.Testing of doctests in the sources finished, look at the ^ results in %BUILDDIR%/doctest/output.txt. goto end ) if "%1" == "xml" ( %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml if errorlevel 1 exit /b 1 echo. echo.Build finished. The XML files are in %BUILDDIR%/xml. goto end ) if "%1" == "pseudoxml" ( %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml if errorlevel 1 exit /b 1 echo. echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. goto end ) :end ================================================ FILE: docs/upgrade.rst ================================================ Upgrading to Newer Releases =========================== Version 4.0 ----------- .. _upgrade_v4_0: - API changes: - Replace ``TinyDB.purge_tables(...)`` with ``TinyDB.drop_tables(...)`` - Replace ``TinyDB.purge_table(...)`` with ``TinyDB.drop_table(...)`` - Replace ``Table.purge()`` with ``Table.truncate()`` - Replace ``TinyDB(default_table='name')`` with ``TinyDB.default_table_name = 'name'`` - Replace ``TinyDB(table_class=Class)`` with ``TinyDB.table_class = Class`` - If you were using ``TinyDB.DEFAULT_TABLE``, ``TinyDB.DEFAULT_TABLE_KWARGS``, or ``TinyDB.DEFAULT_STORAGE``: Use the new methods for customizing TinyDB described in :ref:`How to Extend TinyDB ` Version 3.0 ----------- .. _upgrade_v3_0: Breaking API Changes ^^^^^^^^^^^^^^^^^^^^ - Querying (see `Issue #62 `_): - ``where('...').contains('...')`` has been renamed to ``where('...').search('...')``. - ``where('foo').has('bar')`` is replaced by either ``where('foo').bar`` or ``Query().foo.bar``. - In case the key is not a valid Python identifier, array notation can be used: ``where('a.b.c')`` is now ``Query()['a.b.c']``. - Checking for the existence of a key has to be done explicitly: ``where('foo').exists()``. - ``SmartCacheTable`` has been moved to `msiemens/tinydb-smartcache`_. - Serialization has been moved to `msiemens/tinydb-serialization`_. - Empty storages are now expected to return ``None`` instead of raising ``ValueError`` (see `Issue #67 `_). .. _msiemens/tinydb-smartcache: https://github.com/msiemens/tinydb-smartcache .. _msiemens/tinydb-serialization: https://github.com/msiemens/tinydb-serialization .. _upgrade_v2_0: Version 2.0 ----------- Breaking API Changes ^^^^^^^^^^^^^^^^^^^^ - The syntax ``query in db`` is not supported any more. Use ``db.contains(...)`` instead. - The ``ConcurrencyMiddleware`` has been removed due to a insecure implementation (see `Issue #18 `_). Consider :ref:`tinyrecord` instead. Apart from that the API remains compatible to v1.4 and prior. For migration from v1 to v2, check out the `v2.0 documentation `_ ================================================ FILE: docs/usage.rst ================================================ :tocdepth: 3 .. toctree:: :maxdepth: 2 Advanced Usage ============== Remarks on Storage ------------------ Before we dive deeper into the usage of TinyDB, we should stop for a moment and discuss how TinyDB stores data. To convert your data to a format that is writable to disk TinyDB uses the `Python JSON `_ module by default. It's great when only simple data types are involved but it cannot handle more complex data types like custom classes. On Python 2 it also converts strings to Unicode strings upon reading (described `here `_). If that causes problems, you can write :doc:`your own storage `, that uses a more powerful (but also slower) library like `pickle `_ or `PyYAML `_. .. hint:: Opening multiple TinyDB instances on the same data (e.g. with the ``JSONStorage``) may result in unexpected behavior due to query caching. See query_caching_ on how to disable the query cache. Queries ------- With that out of the way, let's start with TinyDB's rich set of queries. There are two main ways to construct queries. The first one resembles the syntax of popular ORM tools: >>> from tinydb import Query >>> User = Query() >>> db.search(User.name == 'John') As you can see, we first create a new Query object and then use it to specify which fields to check. Searching for nested fields is just as easy: >>> db.search(User.birthday.year == 1990) Not all fields can be accessed this way if the field name is not a valid Python identifier. In this case, you can switch to dict access notation: >>> # This would be invalid Python syntax: >>> db.search(User.country-code == 'foo') >>> # Use this instead: >>> db.search(User['country-code'] == 'foo') In addition, you can use arbitrary transform function where a field would be, for example: >>> from unidecode import unidecode >>> db.search(User.name.map(unidecode) == 'Jose') >>> # will match 'José' etc. The second, traditional way of constructing queries is as follows: >>> from tinydb import where >>> db.search(where('field') == 'value') Using ``where('field')`` is a shorthand for the following code: >>> db.search(Query()['field'] == 'value') Accessing nested fields with this syntax can be achieved like this: >>> db.search(where('birthday').year == 1900) >>> db.search(where('birthday')['year'] == 1900) Advanced queries ................ In the :doc:`getting-started` you've learned about the basic comparisons (``==``, ``<``, ``>``, ...). In addition to these TinyDB supports the following queries: >>> # Existence of a field: >>> db.search(User.name.exists()) >>> # Regex: >>> # Full item has to match the regex: >>> db.search(User.name.matches('[aZ]*')) >>> # Case insensitive search for 'John': >>> import re >>> db.search(User.name.matches('John', flags=re.IGNORECASE)) >>> # Any part of the item has to match the regex: >>> db.search(User.name.search('b+')) >>> # Custom test: >>> test_func = lambda s: s == 'John' >>> db.search(User.name.test(test_func)) >>> # Custom test with parameters: >>> def test_func(val, m, n): >>> return m <= val <= n >>> db.search(User.age.test(test_func, 0, 21)) >>> db.search(User.age.test(test_func, 21, 99)) Another case is if you have a ``dict`` where you want to find all documents that match this ``dict``. We call this searching for a fragment: >>> db.search(Query().fragment({'foo': True, 'bar': False})) [{'foo': True, 'bar': False, 'foobar: 'yes!'}] You also can search for documents where a specific field matches the fragment: >>> db.search(Query().field.fragment({'foo': True, 'bar': False})) [{'field': {'foo': True, 'bar': False, 'foobar: 'yes!'}] When a field contains a list, you also can use the ``any`` and ``all`` methods. There are two ways to use them: with lists of values and with nested queries. Let's start with the first one. Assuming we have a user object with a groups list like this: >>> db.insert({'name': 'user1', 'groups': ['user']}) >>> db.insert({'name': 'user2', 'groups': ['admin', 'user']}) >>> db.insert({'name': 'user3', 'groups': ['sudo', 'user']}) Now we can use the following queries: >>> # User's groups include at least one value from ['admin', 'sudo'] >>> db.search(User.groups.any(['admin', 'sudo'])) [{'name': 'user2', 'groups': ['admin', 'user']}, {'name': 'user3', 'groups': ['sudo', 'user']}] >>> >>> # User's groups include all values from ['admin', 'user'] >>> db.search(User.groups.all(['admin', 'user'])) [{'name': 'user2', 'groups': ['admin', 'user']}] In some cases you may want to have more complex ``any``/``all`` queries. This is where nested queries come in as helpful. Let's set up a table like this: >>> Group = Query() >>> Permission = Query() >>> groups = db.table('groups') >>> groups.insert({ 'name': 'user', 'permissions': [{'type': 'read'}]}) >>> groups.insert({ 'name': 'sudo', 'permissions': [{'type': 'read'}, {'type': 'sudo'}]}) >>> groups.insert({ 'name': 'admin', 'permissions': [{'type': 'read'}, {'type': 'write'}, {'type': 'sudo'}]}) Now let's search this table using nested ``any``/``all`` queries: >>> # Group has a permission with type 'read' >>> groups.search(Group.permissions.any(Permission.type == 'read')) [{'name': 'user', 'permissions': [{'type': 'read'}]}, {'name': 'sudo', 'permissions': [{'type': 'read'}, {'type': 'sudo'}]}, {'name': 'admin', 'permissions': [{'type': 'read'}, {'type': 'write'}, {'type': 'sudo'}]}] >>> # Group has ONLY permission 'read' >>> groups.search(Group.permissions.all(Permission.type == 'read')) [{'name': 'user', 'permissions': [{'type': 'read'}]}] As you can see, ``any`` tests if there is *at least one* document matching the query while ``all`` ensures *all* documents match the query. The opposite operation, checking if a single item is contained in a list, is also possible using ``one_of``: >>> db.search(User.name.one_of(['jane', 'john'])) Query modifiers ............... TinyDB also allows you to use logical operations to modify and combine queries: >>> # Negate a query: >>> db.search(~ (User.name == 'John')) >>> # Logical AND: >>> db.search((User.name == 'John') & (User.age <= 30)) >>> # Logical OR: >>> db.search((User.name == 'John') | (User.name == 'Bob')) .. note:: When using ``&`` or ``|``, make sure you wrap the conditions on both sides with parentheses or Python will mess up the comparison. Also, when using negation (``~``) you'll have to wrap the query you want to negate in parentheses. The reason for these requirements is that Python's binary operators that are used for query modifiers have a higher operator precedence than comparison operators. Simply put, ``~ User.name == 'John'`` is parsed by Python as ``(~User.name) == 'John'`` instead of ``~(User.name == 'John')``. See also the Python `docs on operator precedence `_ for details. You can compose queries dynamically by using the no-op query ``Query().noop()``. Comparisons only support literal values on the right-hand side. Field-to-field comparisons like ``Query().a == Query().b`` are not supported. Use a callable predicate like ``db.search(lambda doc: doc.get('a') == doc.get('b'))`` for custom logic. .. note:: Callables passed to query APIs (e.g. predicates, ``Query().map``, ``Query().test``) execute in-process and must **never** be derived from untrusted or user-controlled input. Recap ..... Let's review the query operations we've learned: +-------------------------------------+---------------------------------------------------------------------+ | **Queries** | +-------------------------------------+---------------------------------------------------------------------+ | ``Query().field.exists()`` | Match any document where a field called ``field`` exists | +-------------------------------------+---------------------------------------------------------------------+ | ``Query().field.matches(regex)`` | Match any document with the whole field matching the | | | regular expression | +-------------------------------------+---------------------------------------------------------------------+ | ``Query().field.search(regex)`` | Match any document with a substring of the field matching | | | the regular expression | +-------------------------------------+---------------------------------------------------------------------+ | ``Query().field.test(func, *args)`` | Matches any document for which the function returns | | | ``True`` | +-------------------------------------+---------------------------------------------------------------------+ | ``Query().field.all(query | list)`` | If given a query, matches all documents where all documents | | | in the list ``field`` match the query. | | | If given a list, matches all documents where all documents | | | in the list ``field`` are a member of the given list | +-------------------------------------+---------------------------------------------------------------------+ | ``Query().field.any(query | list)`` | If given a query, matches all documents where at least one | | | document in the list ``field`` match the query. | | | If given a list, matches all documents where at least one | | | documents in the list ``field`` are a member of the given | | | list | +-------------------------------------+---------------------------------------------------------------------+ | ``Query().field.one_of(list)`` | Match if the field is contained in the list | +-------------------------------------+---------------------------------------------------------------------+ | **Logical operations on queries** | +-------------------------------------+---------------------------------------------------------------------+ | ``~ (query)`` | Match documents that don't match the query (logical NOT) | +-------------------------------------+---------------------------------------------------------------------+ | ``(query1) & (query2)`` | Match documents that match both queries (logical AND) | +-------------------------------------+---------------------------------------------------------------------+ | ``(query1) | (query2)`` | Match documents that match at least one of the queries (logical OR) | +-------------------------------------+---------------------------------------------------------------------+ Handling Data ------------- Next, let's look at some more ways to insert, update and retrieve data from your database. Inserting data .............. As already described you can insert a document using ``db.insert(...)``. In case you want to insert multiple documents, you can use ``db.insert_multiple(...)``: >>> db.insert_multiple([ {'name': 'John', 'age': 22}, {'name': 'John', 'age': 37}]) >>> db.insert_multiple({'int': 1, 'value': i} for i in range(2)) Also in some cases it may be useful to specify the document ID yourself when inserting data. You can do that by using the :class:`~tinydb.table.Document` class: >>> db.insert(Document({'name': 'John', 'age': 22}, doc_id=12)) 12 The same is possible when using ``db.insert_multiple(...)``: >>> db.insert_multiple([ Document({'name': 'John', 'age': 22}, doc_id=12), Document({'name': 'Jane', 'age': 24}, doc_id=14), ]) [12, 14] .. note:: Inserting a ``Document`` with an ID that already exists will result in a ``ValueError`` being raised. Updating data ............. Sometimes you want to update all documents in your database. In this case, you can leave out the ``query`` argument: >>> db.update({'foo': 'bar'}) When passing a dict to ``db.update(fields, query)``, it only allows you to update a document by adding or overwriting its values. But sometimes you may need to e.g. remove one field or increment its value. In that case you can pass a function instead of ``fields``: >>> from tinydb.operations import delete >>> db.update(delete('key1'), User.name == 'John') This will remove the key ``key1`` from all matching documents. TinyDB comes with these operations: - ``delete(key)``: delete a key from the document - ``increment(key)``: increment the value of a key - ``decrement(key)``: decrement the value of a key - ``add(key, value)``: add ``value`` to the value of a key (also works for strings) - ``subtract(key, value)``: subtract ``value`` from the value of a key - ``set(key, value)``: set ``key`` to ``value`` Of course you also can write your own operations: >>> def your_operation(your_arguments): ... def transform(doc): ... # do something with the document ... # ... ... return transform ... >>> db.update(your_operation(arguments), query) In order to perform multiple update operations at once, you can use the ``update_multiple`` method like this: >>> db.update_multiple([ ... ({'int': 2}, where('char') == 'a'), ... ({'int': 4}, where('char') == 'b'), ... ]) You also can mix normal updates with update operations: >>> db.update_multiple([ ... ({'int': 2}, where('char') == 'a'), ... ({delete('int'), where('char') == 'b'), ... ]) Data access and modification ---------------------------- Upserting data .............. In some cases you'll need a mix of both ``update`` and ``insert``: ``upsert``. This operation is provided a document and a query. If it finds any documents matching the query, they will be updated with the data from the provided document. On the other hand, if no matching document is found, it inserts the provided document into the table: >>> db.upsert({'name': 'John', 'logged-in': True}, User.name == 'John') This will update all users with the name John to have ``logged-in`` set to ``True``. If no matching user is found, a new document is inserted with both the name set and the ``logged-in`` flag. To use the ID of the document as matching criterion a :class:`~tinydb.table.Document` with ``doc_id`` is passed instead of a query: >>> db.upsert(Document({'name': 'John', 'logged-in': True}, doc_id=12)) Retrieving data ............... There are several ways to retrieve data from your database. For instance you can get the number of stored documents: >>> len(db) 3 .. hint:: This will return the number of documents in the default table (see the notes on the :ref:`default table `). Then of course you can use ``db.search(...)`` as described in the :doc:`getting-started` section. But sometimes you want to get only one matching document. Instead of using >>> try: ... result = db.search(User.name == 'John')[0] ... except IndexError: ... pass you can use ``db.get(...)``: >>> db.get(User.name == 'John') {'name': 'John', 'age': 22} >>> db.get(User.name == 'Bobby') None .. caution:: If multiple documents match the query, probably a random one of them will be returned! Often you don't want to search for documents but only know whether they are stored in the database. In this case ``db.contains(...)`` is your friend: >>> db.contains(User.name == 'John') In a similar manner you can look up the number of documents matching a query: >>> db.count(User.name == 'John') 2 Recap ^^^^^ Let's summarize the ways to handle data: +-------------------------------+---------------------------------------------------------------+ | **Inserting data** | +-------------------------------+---------------------------------------------------------------+ | ``db.insert_multiple(...)`` | Insert multiple documents | +-------------------------------+---------------------------------------------------------------+ | **Updating data** | +-------------------------------+---------------------------------------------------------------+ | ``db.update(operation, ...)`` | Update all matching documents with a special operation | +-------------------------------+---------------------------------------------------------------+ | **Retrieving data** | +-------------------------------+---------------------------------------------------------------+ | ``len(db)`` | Get the number of documents in the database | +-------------------------------+---------------------------------------------------------------+ | ``db.get(query)`` | Get one document matching the query | +-------------------------------+---------------------------------------------------------------+ | ``db.contains(query)`` | Check if the database contains a matching document | +-------------------------------+---------------------------------------------------------------+ | ``db.count(query)`` | Get the number of matching documents | +-------------------------------+---------------------------------------------------------------+ .. note:: This was a new feature in v3.6.0 .. _document_ids: Using Document IDs ------------------ Internally TinyDB associates an ID with every document you insert. It's returned after inserting a document: >>> db.insert({'name': 'John', 'age': 22}) 3 >>> db.insert_multiple([{...}, {...}, {...}]) [4, 5, 6] In addition you can get the ID of already inserted documents using ``document.doc_id``. This works both with ``get`` and ``all``: >>> el = db.get(User.name == 'John') >>> el.doc_id 3 >>> el = db.all()[0] >>> el.doc_id 1 >>> el = db.all()[-1] >>> el.doc_id 12 Different TinyDB methods also work with IDs, namely: ``update``, ``remove``, ``contains`` and ``get``. The first two also return a list of affected IDs. >>> db.update({'value': 2}, doc_ids=[1, 2]) >>> db.contains(doc_id=1) True >>> db.remove(doc_ids=[1, 2]) >>> db.get(doc_id=3) {...} >>> db.get(doc_ids=[1, 2]) [{...}, {...}] Using ``doc_id``/``doc_ids`` instead of ``Query()`` again is slightly faster in operation. Recap ..... Let's sum up the way TinyDB supports working with IDs: +-------------------------------------+------------------------------------------------------------+ | **Getting a document's ID** | +-------------------------------------+------------------------------------------------------------+ | ``db.insert(...)`` | Returns the inserted document's ID | +-------------------------------------+------------------------------------------------------------+ | ``db.insert_multiple(...)`` | Returns the inserted documents' ID | +-------------------------------------+------------------------------------------------------------+ | ``document.doc_id`` | Get the ID of a document fetched from the db | +-------------------------------------+------------------------------------------------------------+ | **Working with IDs** | +-------------------------------------+------------------------------------------------------------+ | ``db.get(doc_id=...)`` | Get the document with the given ID | +-------------------------------------+------------------------------------------------------------+ | ``db.contains(doc_id=...)`` | Check if the db contains a document with the given | | | IDs | +-------------------------------------+------------------------------------------------------------+ | ``db.update({...}, doc_ids=[...])`` | Update all documents with the given IDs | +-------------------------------------+------------------------------------------------------------+ | ``db.remove(doc_ids=[...])`` | Remove all documents with the given IDs | +-------------------------------------+------------------------------------------------------------+ Tables ------ TinyDB supports working with multiple tables. They behave just the same as the ``TinyDB`` class. To create and use a table, use ``db.table(name)``. >>> table = db.table('table_name') >>> table.insert({'value': True}) >>> table.all() [{'value': True}] >>> for row in table: >>> print(row) {'value': True} To remove a table from a database, use: >>> db.drop_table('table_name') If on the other hand you want to remove all tables, use the counterpart: >>> db.drop_tables() Finally, you can get a list with the names of all tables in your database: >>> db.tables() {'_default', 'table_name'} .. _default_table: Default Table ............. TinyDB uses a table named ``_default`` as the default table. All operations on the database object (like ``db.insert(...)``) operate on this table. The name of this table can be modified by setting the ``default_table_name`` class variable to modify the default table name for all instances: >>> #1: for a single instance only >>> db = TinyDB(storage=SomeStorage) >>> db.default_table_name = 'my-default' >>> #2: for all instances >>> TinyDB.default_table_name = 'my-default' .. _query_caching: Query Caching ............. TinyDB caches query result for performance. That way re-running a query won't have to read the data from the storage as long as the database hasn't been modified. You can optimize the query cache size by passing the ``cache_size`` to the ``table(...)`` function: >>> table = db.table('table_name', cache_size=30) .. hint:: You can set ``cache_size`` to ``None`` to make the cache unlimited in size. Also, you can set ``cache_size`` to 0 to disable it. .. hint:: It's not possible to open the same table multiple times with different settings. After the first invocation, all the subsequent calls will return the same table with the same settings as the first one. .. hint:: The TinyDB query cache doesn't check if the underlying storage that the database uses has been modified by an external process. In this case the query cache may return outdated results. To clear the cache and read data from the storage again you can use ``db.clear_cache()``. .. hint:: When using an unlimited cache size and ``test()`` queries, TinyDB will store a reference to the test function. As a result of that behavior long-running applications that use ``lambda`` functions as a test function may experience memory leaks. Storage & Middleware -------------------- Storage Types ............. TinyDB comes with two storage types: JSON and in-memory. By default TinyDB stores its data in JSON files so you have to specify the path where to store it: >>> from tinydb import TinyDB, where >>> db = TinyDB('path/to/db.json') To use the in-memory storage, use: >>> from tinydb.storages import MemoryStorage >>> db = TinyDB(storage=MemoryStorage) .. hint:: All arguments except for the ``storage`` argument are forwarded to the underlying storage. For the JSON storage you can use this to pass additional keyword arguments to Python's `json.dumps(...) `_ method. For example, you can set it to create prettified JSON files like this: >>> db = TinyDB('db.json', sort_keys=True, indent=4, separators=(',', ': ')) .. note:: ``JSONStorage`` forwards ``**kwargs`` to ``json.dumps``. **Never** pass user-controlled values for callable arguments (e.g. ``default`` or ``cls``) as they are executed in-process on every write operation. To modify the default storage for all ``TinyDB`` instances, set the ``default_storage_class`` class variable: >>> TinyDB.default_storage_class = MemoryStorage In case you need to access the storage instance directly, you can use the ``storage`` property of your TinyDB instance. This may be useful to call method directly on the storage or middleware: >>> db = TinyDB(storage=CachingMiddleware(MemoryStorage)) >>> db.storage.flush() Middleware .......... Middleware wraps around existing storage allowing you to customize their behaviour. >>> from tinydb.storages import JSONStorage >>> from tinydb.middlewares import CachingMiddleware >>> db = TinyDB('/path/to/db.json', storage=CachingMiddleware(JSONStorage)) .. hint:: You can nest middleware: >>> db = TinyDB('/path/to/db.json', storage=FirstMiddleware(SecondMiddleware(JSONStorage))) CachingMiddleware ^^^^^^^^^^^^^^^^^ The ``CachingMiddleware`` improves speed by reducing disk I/O. It caches all read operations and writes data to disk after a configured number of write operations. To make sure that all data is safely written when closing the table, use one of these ways: .. code-block:: python # Using a context manager: with database as db: # Your operations .. code-block:: python # Using the close function db.close() .. _mypy_type_checking: MyPy Type Checking ------------------ TinyDB comes with type annotations that MyPy can use to make sure you're using the API correctly. Unfortunately, MyPy doesn't understand all code patterns that TinyDB uses. For that reason TinyDB ships a MyPy plugin that helps correctly type checking code that uses TinyDB. To use it, add it to the plugins list in the `MyPy configuration file `_ (typically located in ``setup.cfg`` or ``mypy.ini``): .. code-block:: ini [mypy] plugins = tinydb.mypy_plugin What's next ----------- Congratulations, you've made through the user guide! Now go and build something awesome or dive deeper into TinyDB with these resources: - Want to learn how to customize TinyDB (storages, middlewares) and what extensions exist? Check out :doc:`extend` and :doc:`extensions`. - Want to study the API in detail? Read :doc:`api`. - Interested in contributing to the TinyDB development guide? Go on to the :doc:`contribute`. ================================================ FILE: mypy.ini ================================================ [mypy] plugins = tinydb/mypy_plugin.py ================================================ FILE: pyproject.toml ================================================ [project] name = "tinydb" version = "4.8.2" description = "TinyDB is a tiny, document oriented database optimized for your happiness :)" authors = [{ name = "Markus Siemens", email = "markus@m-siemens.de" }] requires-python = ">=3.10,<4" readme = "README.rst" license = "MIT" keywords = [ "database", "nosql", ] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Intended Audience :: System Administrators", "License :: OSI Approved :: MIT License", "Topic :: Database", "Topic :: Database :: Database Engines/Servers", "Topic :: Utilities", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Operating System :: OS Independent", "Typing :: Typed", ] [project.urls] Homepage = "https://github.com/msiemens/tinydb" Documentation = "https://tinydb.readthedocs.org/" Changelog = "https://tinydb.readthedocs.io/en/latest/changelog.html" Issues = "https://github.com/msiemens/tinydb/issues" [dependency-groups] dev = [ "pytest~=9.0", "pytest-pycodestyle~=2.3", "pytest-cov~=7.0", "pycodestyle~=2.10", "sphinx~=8.1", "coveralls~=4.0", "pyyaml~=6.0", "pytest-mypy~=1.0 ; platform_python_implementation != 'PyPy'", "types-PyYAML~=6.0", ] [tool.hatch.build.targets.sdist] include = [ "tinydb", "tests", ] [tool.hatch.build.targets.wheel] include = ["tinydb"] [build-system] requires = ["hatchling"] build-backend = "hatchling.build" ================================================ FILE: pytest.ini ================================================ [pytest] addopts=--verbose --cov-append --cov-report term --cov tinydb ================================================ FILE: tests/__init__.py ================================================ ================================================ FILE: tests/conftest.py ================================================ import os.path import tempfile from pathlib import Path import pytest # type: ignore from tinydb.middlewares import CachingMiddleware from tinydb.storages import MemoryStorage from tinydb import TinyDB, JSONStorage @pytest.fixture(params=['memory', 'json']) def db(request, tmp_path: Path): if request.param == 'json': db_ = TinyDB(tmp_path / 'test.db', storage=JSONStorage) else: db_ = TinyDB(storage=MemoryStorage) db_.drop_tables() db_.insert_multiple({'int': 1, 'char': c} for c in 'abc') yield db_ @pytest.fixture def storage(): return CachingMiddleware(MemoryStorage)() ================================================ FILE: tests/test_middlewares.py ================================================ import os from tinydb import TinyDB from tinydb.middlewares import CachingMiddleware from tinydb.storages import MemoryStorage, JSONStorage doc = {'none': [None, None], 'int': 42, 'float': 3.1415899999999999, 'list': ['LITE', 'RES_ACID', 'SUS_DEXT'], 'dict': {'hp': 13, 'sp': 5}, 'bool': [True, False, True, False]} def test_caching(storage): # Write contents storage.write(doc) # Verify contents assert doc == storage.read() def test_caching_read(): db = TinyDB(storage=CachingMiddleware(MemoryStorage)) assert db.all() == [] def test_caching_write_many(storage): storage.WRITE_CACHE_SIZE = 3 # Storage should be still empty assert storage.memory is None # Write contents for x in range(2): storage.write(doc) assert storage.memory is None # Still cached storage.write(doc) # Verify contents: Cache should be emptied and written to storage assert storage.memory def test_caching_flush(storage): # Write contents for _ in range(CachingMiddleware.WRITE_CACHE_SIZE - 1): storage.write(doc) # Not yet flushed... assert storage.memory is None storage.write(doc) # Verify contents: Cache should be emptied and written to storage assert storage.memory def test_caching_flush_manually(storage): # Write contents storage.write(doc) storage.flush() # Verify contents: Cache should be emptied and written to storage assert storage.memory def test_caching_write(storage): # Write contents storage.write(doc) storage.close() # Verify contents: Cache should be emptied and written to storage assert storage.storage.memory def test_nested(): storage = CachingMiddleware(MemoryStorage) storage() # Initialization # Write contents storage.write(doc) # Verify contents assert doc == storage.read() def test_caching_json_write(tmpdir): path = str(tmpdir.join('test.db')) with TinyDB(path, storage=CachingMiddleware(JSONStorage)) as db: db.insert({'key': 'value'}) # Verify database filesize statinfo = os.stat(path) assert statinfo.st_size != 0 # Assert JSON file has been closed assert db._storage._handle.closed del db # Reopen database with TinyDB(path, storage=CachingMiddleware(JSONStorage)) as db: assert db.all() == [{'key': 'value'}] ================================================ FILE: tests/test_operations.py ================================================ from tinydb import where from tinydb.operations import delete, increment, decrement, add, subtract, set def test_delete(db): db.update(delete('int'), where('char') == 'a') assert 'int' not in db.get(where('char') == 'a') def test_add_int(db): db.update(add('int', 5), where('char') == 'a') assert db.get(where('char') == 'a')['int'] == 6 def test_add_str(db): db.update(add('char', 'xyz'), where('char') == 'a') assert db.get(where('char') == 'axyz')['int'] == 1 def test_subtract(db): db.update(subtract('int', 5), where('char') == 'a') assert db.get(where('char') == 'a')['int'] == -4 def test_set(db): db.update(set('char', 'xyz'), where('char') == 'a') assert db.get(where('char') == 'xyz')['int'] == 1 def test_increment(db): db.update(increment('int'), where('char') == 'a') assert db.get(where('char') == 'a')['int'] == 2 def test_decrement(db): db.update(decrement('int'), where('char') == 'a') assert db.get(where('char') == 'a')['int'] == 0 ================================================ FILE: tests/test_queries.py ================================================ import re import pytest from tinydb.queries import Query, where def test_no_path(): with pytest.raises(ValueError): _ = Query() == 2 def test_path_exists(): query = Query()['value'].exists() assert query == where('value').exists() assert query({'value': 1}) assert not query({'something': 1}) assert hash(query) assert hash(query) != hash(where('asd')) query = Query()['value']['val'].exists() assert query == where('value')['val'].exists() assert query({'value': {'val': 2}}) assert not query({'value': 1}) assert not query({'value': {'asd': 1}}) assert not query({'something': 1}) assert hash(query) assert hash(query) != hash(where('asd')) def test_path_and(): query = Query()['value'].exists() & (Query()['value'] == 5) assert query({'value': 5}) assert not query({'value': 10}) assert not query({'something': 1}) assert hash(query) assert hash(query) != hash(where('value')) def test_callable_in_path_with_map(): double = lambda x: x + x query = Query().value.map(double) == 10 assert query({'value': 5}) assert not query({'value': 10}) def test_callable_in_path_with_chain(): rekey = lambda x: {'y': x['a'], 'z': x['b']} query = Query().map(rekey).z == 10 assert query({'a': 5, 'b': 10}) def test_eq(): query = Query().value == 1 assert query({'value': 1}) assert not query({'value': 2}) assert hash(query) query = Query().value == [0, 1] assert query({'value': [0, 1]}) assert not query({'value': [0, 1, 2]}) assert hash(query) def test_ne(): query = Query().value != 1 assert query({'value': 0}) assert query({'value': 2}) assert not query({'value': 1}) assert hash(query) query = Query().value != [0, 1] assert query({'value': [0, 1, 2]}) assert not query({'value': [0, 1]}) assert hash(query) def test_lt(): query = Query().value < 1 assert query({'value': 0}) assert not query({'value': 1}) assert not query({'value': 2}) assert hash(query) def test_le(): query = Query().value <= 1 assert query({'value': 0}) assert query({'value': 1}) assert not query({'value': 2}) assert hash(query) def test_gt(): query = Query().value > 1 assert query({'value': 2}) assert not query({'value': 1}) assert hash(query) def test_ge(): query = Query().value >= 1 assert query({'value': 2}) assert query({'value': 1}) assert not query({'value': 0}) assert hash(query) def test_or(): query = ( (Query().val1 == 1) | (Query().val2 == 2) ) assert query({'val1': 1}) assert query({'val2': 2}) assert query({'val1': 1, 'val2': 2}) assert not query({'val1': '', 'val2': ''}) assert hash(query) def test_and(): query = ( (Query().val1 == 1) & (Query().val2 == 2) ) assert query({'val1': 1, 'val2': 2}) assert not query({'val1': 1}) assert not query({'val2': 2}) assert not query({'val1': '', 'val2': ''}) assert hash(query) def test_not(): query = ~ (Query().val1 == 1) assert query({'val1': 5, 'val2': 2}) assert not query({'val1': 1, 'val2': 2}) assert hash(query) query = ( (~ (Query().val1 == 1)) & (Query().val2 == 2) ) assert query({'val1': '', 'val2': 2}) assert query({'val2': 2}) assert not query({'val1': 1, 'val2': 2}) assert not query({'val1': 1}) assert not query({'val1': '', 'val2': ''}) assert hash(query) def test_has_key(): query = Query().val3.exists() assert query({'val3': 1}) assert not query({'val1': 1, 'val2': 2}) assert hash(query) def test_regex(): query = Query().val.matches(r'\d{2}\.') assert query({'val': '42.'}) assert not query({'val': '44'}) assert not query({'val': 'ab.'}) assert not query({'val': 155}) assert not query({'val': False}) assert not query({'': None}) assert hash(query) query = Query().val.search(r'\d+') assert query({'val': 'ab3'}) assert not query({'val': 'abc'}) assert not query({'val': ''}) assert not query({'val': True}) assert not query({'': None}) assert hash(query) query = Query().val.search(r'JOHN', flags=re.IGNORECASE) assert query({'val': 'john'}) assert query({'val': 'xJohNx'}) assert not query({'val': 'JOH'}) assert not query({'val': 12}) assert not query({'': None}) assert hash(query) def test_custom(): def test(value): return value == 42 query = Query().val.test(test) assert query({'val': 42}) assert not query({'val': 40}) assert not query({'val': '44'}) assert not query({'': None}) assert hash(query) def in_list(value, l): return value in l query = Query().val.test(in_list, tuple([25, 35])) assert not query({'val': 20}) assert query({'val': 25}) assert not query({'val': 30}) assert query({'val': 35}) assert not query({'val': 36}) assert hash(query) def test_custom_with_params(): def test(value, minimum, maximum): return minimum <= value <= maximum query = Query().val.test(test, 1, 10) assert query({'val': 5}) assert not query({'val': 0}) assert not query({'val': 11}) assert not query({'': None}) assert hash(query) def test_any(): query = Query().followers.any(Query().name == 'don') assert query({'followers': [{'name': 'don'}, {'name': 'john'}]}) assert not query({'followers': 1}) assert not query({}) assert hash(query) query = Query().followers.any(Query().num.matches('\\d+')) assert query({'followers': [{'num': '12'}, {'num': 'abc'}]}) assert not query({'followers': [{'num': 'abc'}]}) assert hash(query) query = Query().followers.any(['don', 'jon']) assert query({'followers': ['don', 'greg', 'bill']}) assert not query({'followers': ['greg', 'bill']}) assert not query({}) assert hash(query) query = Query().followers.any([{'name': 'don'}, {'name': 'john'}]) assert query({'followers': [{'name': 'don'}, {'name': 'greg'}]}) assert not query({'followers': [{'name': 'greg'}]}) assert hash(query) def test_all(): query = Query().followers.all(Query().name == 'don') assert query({'followers': [{'name': 'don'}]}) assert not query({'followers': [{'name': 'don'}, {'name': 'john'}]}) assert hash(query) query = Query().followers.all(Query().num.matches('\\d+')) assert query({'followers': [{'num': '123'}, {'num': '456'}]}) assert not query({'followers': [{'num': '123'}, {'num': 'abc'}]}) assert hash(query) query = Query().followers.all(['don', 'john']) assert query({'followers': ['don', 'john', 'greg']}) assert not query({'followers': ['don', 'greg']}) assert not query({}) assert hash(query) query = Query().followers.all([{'name': 'jane'}, {'name': 'john'}]) assert query({'followers': [{'name': 'john'}, {'name': 'jane'}]}) assert query({'followers': [{'name': 'john'}, {'name': 'jane'}, {'name': 'bob'}]}) assert not query({'followers': [{'name': 'john'}, {'name': 'bob'}]}) assert hash(query) def test_has(): query = Query().key1.key2.exists() str(query) # This used to cause a bug... assert query({'key1': {'key2': {'key3': 1}}}) assert query({'key1': {'key2': 1}}) assert not query({'key1': 3}) assert not query({'key1': {'key1': 1}}) assert not query({'key2': {'key1': 1}}) assert hash(query) query = Query().key1.key2 == 1 assert query({'key1': {'key2': 1}}) assert not query({'key1': {'key2': 2}}) assert hash(query) # Nested has: key exists query = Query().key1.key2.key3.exists() assert query({'key1': {'key2': {'key3': 1}}}) # Not a dict assert not query({'key1': 1}) assert not query({'key1': {'key2': 1}}) # Wrong key assert not query({'key1': {'key2': {'key0': 1}}}) assert not query({'key1': {'key0': {'key3': 1}}}) assert not query({'key0': {'key2': {'key3': 1}}}) assert hash(query) # Nested has: check for value query = Query().key1.key2.key3 == 1 assert query({'key1': {'key2': {'key3': 1}}}) assert not query({'key1': {'key2': {'key3': 0}}}) assert hash(query) # Test special methods: regex matches query = Query().key1.value.matches(r'\d+') assert query({'key1': {'value': '123'}}) assert not query({'key2': {'value': '123'}}) assert not query({'key2': {'value': 'abc'}}) assert hash(query) # Test special methods: regex contains query = Query().key1.value.search(r'\d+') assert query({'key1': {'value': 'a2c'}}) assert not query({'key2': {'value': 'a2c'}}) assert not query({'key2': {'value': 'abc'}}) assert hash(query) # Test special methods: nested has and regex matches query = Query().key1.x.y.matches(r'\d+') assert query({'key1': {'x': {'y': '123'}}}) assert not query({'key1': {'x': {'y': 'abc'}}}) assert hash(query) # Test special method: nested has and regex contains query = Query().key1.x.y.search(r'\d+') assert query({'key1': {'x': {'y': 'a2c'}}}) assert not query({'key1': {'x': {'y': 'abc'}}}) assert hash(query) # Test special methods: custom test query = Query().key1.int.test(lambda x: x == 3) assert query({'key1': {'int': 3}}) assert hash(query) def test_one_of(): query = Query().key1.one_of(['value 1', 'value 2']) assert query({'key1': 'value 1'}) assert query({'key1': 'value 2'}) assert not query({'key1': 'value 3'}) def test_hash(): d = { Query().key1 == 2: True, Query().key1.key2.key3.exists(): True, Query().key1.exists() & Query().key2.exists(): True, Query().key1.exists() | Query().key2.exists(): True, } assert (Query().key1 == 2) in d assert (Query().key1.key2.key3.exists()) in d assert (Query()['key1.key2'].key3.exists()) not in d # Commutative property of & and | assert (Query().key1.exists() & Query().key2.exists()) in d assert (Query().key2.exists() & Query().key1.exists()) in d assert (Query().key1.exists() | Query().key2.exists()) in d assert (Query().key2.exists() | Query().key1.exists()) in d def test_orm_usage(): data = {'name': 'John', 'age': {'year': 2000}} User = Query() query1 = User.name == 'John' query2 = User.age.year == 2000 assert query1(data) assert query2(data) def test_repr(): Fruit = Query() assert repr(Fruit) == "Query()" assert repr(Fruit.type == 'peach') == "QueryImpl('==', ('type',), 'peach')" def test_subclass(): # Test that a new query test method in a custom subclass is properly usable class MyQueryClass(Query): def equal_double(self, rhs): return self._generate_test( lambda value: value == rhs * 2, ('equal_double', self._path, rhs) ) query = MyQueryClass().val.equal_double('42') assert query({'val': '4242'}) assert not query({'val': '42'}) assert not query({'': None}) assert hash(query) def test_noop(): query = Query().noop() assert query({'foo': True}) assert query({'foo': None}) assert query({}) def test_equality(): q = Query() assert (q.foo == 2) != 0 assert (q.foo == 'yes') != '' def test_empty_query_error(): with pytest.raises(RuntimeError, match='Empty query was evaluated'): Query()({}) def test_fragment(): query = Query().fragment({'a': 4, 'b': True}) assert query({'a': 4, 'b': True, 'c': 'yes'}) assert not query({'a': 4, 'c': 'yes'}) assert not query({'b': True, 'c': 'yes'}) assert not query({'a': 5, 'b': True, 'c': 'yes'}) assert not query({'a': 4, 'b': 'no', 'c': 'yes'}) def test_fragment_with_path(): query = Query().doc.fragment({'a': 4, 'b': True}) assert query({'doc': {'a': 4, 'b': True, 'c': 'yes'}}) assert not query({'a': 4, 'b': True, 'c': 'yes'}) assert not query({'doc': {'a': 4, 'c': 'yes'}}) def test_get_item(): query = Query()['test'] == 1 assert query({'test': 1}) assert not query({'test': 0}) ================================================ FILE: tests/test_storages.py ================================================ import json import os import random import tempfile import pytest from tinydb import TinyDB, where from tinydb.storages import JSONStorage, MemoryStorage, Storage, touch from tinydb.table import Document random.seed() doc = {'none': [None, None], 'int': 42, 'float': 3.1415899999999999, 'list': ['LITE', 'RES_ACID', 'SUS_DEXT'], 'dict': {'hp': 13, 'sp': 5}, 'bool': [True, False, True, False]} def test_json(tmpdir): # Write contents path = str(tmpdir.join('test.db')) storage = JSONStorage(path) storage.write(doc) # Verify contents assert doc == storage.read() storage.close() def test_json_kwargs(tmpdir): db_file = tmpdir.join('test.db') db = TinyDB(str(db_file), sort_keys=True, indent=4, separators=(',', ': ')) # Write contents db.insert({'b': 1}) db.insert({'a': 1}) assert db_file.read() == '''{ "_default": { "1": { "b": 1 }, "2": { "a": 1 } } }''' db.close() def test_json_readwrite(tmpdir): """ Regression test for issue #1 """ path = str(tmpdir.join('test.db')) # Create TinyDB instance db = TinyDB(path, storage=JSONStorage) item = {'name': 'A very long entry'} item2 = {'name': 'A short one'} def get(s): return db.get(where('name') == s) db.insert(item) assert get('A very long entry') == item db.remove(where('name') == 'A very long entry') assert get('A very long entry') is None db.insert(item2) assert get('A short one') == item2 db.remove(where('name') == 'A short one') assert get('A short one') is None db.close() def test_json_read(tmpdir): r"""Open a database only for reading""" path = str(tmpdir.join('test.db')) with pytest.raises(FileNotFoundError): db = TinyDB(path, storage=JSONStorage, access_mode='r') # Create small database db = TinyDB(path, storage=JSONStorage) db.insert({'b': 1}) db.insert({'a': 1}) db.close() # Access in read mode db = TinyDB(path, storage=JSONStorage, access_mode='r') assert db.get(where('a') == 1) == {'a': 1} # reading is fine with pytest.raises(IOError): db.insert({'c': 1}) # writing is not db.close() def test_create_dirs(): temp_dir = tempfile.gettempdir() while True: dname = os.path.join(temp_dir, str(random.getrandbits(20))) if not os.path.exists(dname): db_dir = dname db_file = os.path.join(db_dir, 'db.json') break with pytest.raises(IOError): JSONStorage(db_file) JSONStorage(db_file, create_dirs=True).close() assert os.path.exists(db_file) # Use create_dirs with already existing directory JSONStorage(db_file, create_dirs=True).close() assert os.path.exists(db_file) os.remove(db_file) os.rmdir(db_dir) def test_json_invalid_directory(): with pytest.raises(IOError): with TinyDB('/this/is/an/invalid/path/db.json', storage=JSONStorage): pass def test_in_memory(): # Write contents storage = MemoryStorage() storage.write(doc) # Verify contents assert doc == storage.read() # Test case for #21 other = MemoryStorage() other.write({}) assert other.read() != storage.read() def test_in_memory_close(): with TinyDB(storage=MemoryStorage) as db: db.insert({}) def test_custom(): # noinspection PyAbstractClass class MyStorage(Storage): pass with pytest.raises(TypeError): MyStorage() def test_read_once(): count = 0 # noinspection PyAbstractClass class MyStorage(Storage): def __init__(self): self.memory = None def read(self): nonlocal count count += 1 return self.memory def write(self, data): self.memory = data with TinyDB(storage=MyStorage) as db: assert count == 0 db.table(db.default_table_name) assert count == 0 db.all() assert count == 1 db.insert({'foo': 'bar'}) assert count == 3 # One for getting the next ID, one for the insert db.all() assert count == 4 def test_custom_with_exception(): class MyStorage(Storage): def read(self): pass def write(self, data): pass def __init__(self): raise ValueError() def close(self): raise RuntimeError() with pytest.raises(ValueError): with TinyDB(storage=MyStorage) as db: pass def test_yaml(tmpdir): """ :type tmpdir: py._path.local.LocalPath """ try: import yaml except ImportError: return pytest.skip('PyYAML not installed') def represent_doc(dumper, data): # Represent `Document` objects as their dict's string representation # which PyYAML understands return dumper.represent_data(dict(data)) yaml.add_representer(Document, represent_doc) class YAMLStorage(Storage): def __init__(self, filename): self.filename = filename touch(filename, False) def read(self): with open(self.filename) as handle: data = yaml.safe_load(handle.read()) return data def write(self, data): with open(self.filename, 'w') as handle: yaml.dump(data, handle) def close(self): pass # Write contents path = str(tmpdir.join('test.db')) db = TinyDB(path, storage=YAMLStorage) db.insert(doc) assert db.all() == [doc] db.update({'name': 'foo'}) assert '!' not in tmpdir.join('test.db').read() assert db.contains(where('name') == 'foo') assert len(db) == 1 def test_encoding(tmpdir): japanese_doc = {"Test": u"こんにちは世界"} path = str(tmpdir.join('test.db')) # cp936 is used for japanese encodings jap_storage = JSONStorage(path, encoding="cp936") jap_storage.write(japanese_doc) try: exception = json.decoder.JSONDecodeError except AttributeError: exception = ValueError with pytest.raises(exception): # cp037 is used for english encodings eng_storage = JSONStorage(path, encoding="cp037") eng_storage.read() jap_storage = JSONStorage(path, encoding="cp936") assert japanese_doc == jap_storage.read() ================================================ FILE: tests/test_tables.py ================================================ import re import pytest from tinydb import where def test_next_id(db): db.truncate() assert db._get_next_id() == 1 assert db._get_next_id() == 2 assert db._get_next_id() == 3 def test_tables_list(db): db.table('table1').insert({'a': 1}) db.table('table2').insert({'a': 1}) assert db.tables() == {'_default', 'table1', 'table2'} def test_one_table(db): table1 = db.table('table1') table1.insert_multiple({'int': 1, 'char': c} for c in 'abc') assert table1.get(where('int') == 1)['char'] == 'a' assert table1.get(where('char') == 'b')['char'] == 'b' def test_multiple_tables(db): table1 = db.table('table1') table2 = db.table('table2') table3 = db.table('table3') table1.insert({'int': 1, 'char': 'a'}) table2.insert({'int': 1, 'char': 'b'}) table3.insert({'int': 1, 'char': 'c'}) assert table1.count(where('char') == 'a') == 1 assert table2.count(where('char') == 'b') == 1 assert table3.count(where('char') == 'c') == 1 db.drop_tables() assert len(table1) == 0 assert len(table2) == 0 assert len(table3) == 0 def test_caching(db): table1 = db.table('table1') table2 = db.table('table1') assert table1 is table2 def test_query_cache(db): query1 = where('int') == 1 assert db.count(query1) == 3 assert query1 in db._query_cache assert db.count(query1) == 3 assert query1 in db._query_cache query2 = where('int') == 0 assert db.count(query2) == 0 assert query2 in db._query_cache assert db.count(query2) == 0 assert query2 in db._query_cache def test_query_cache_with_mutable_callable(db): table = db.table('table') table.insert({'val': 5}) mutable = 5 increase = lambda x: x + mutable assert where('val').is_cacheable() assert not where('val').map(increase).is_cacheable() assert not (where('val').map(increase) == 10).is_cacheable() search = where('val').map(increase) == 10 assert table.count(search) == 1 # now `increase` would yield 15, not 10 mutable = 10 assert table.count(search) == 0 assert len(table._query_cache) == 0 def test_zero_cache_size(db): table = db.table('table3', cache_size=0) query = where('int') == 1 table.insert({'int': 1}) table.insert({'int': 1}) assert table.count(query) == 2 assert table.count(where('int') == 2) == 0 assert len(table._query_cache) == 0 def test_query_cache_size(db): table = db.table('table3', cache_size=1) query = where('int') == 1 table.insert({'int': 1}) table.insert({'int': 1}) assert table.count(query) == 2 assert table.count(where('int') == 2) == 0 assert len(table._query_cache) == 1 def test_lru_cache(db): # Test integration into TinyDB table = db.table('table3', cache_size=2) query = where('int') == 1 table.search(query) table.search(where('int') == 2) table.search(where('int') == 3) assert query not in table._query_cache table.remove(where('int') == 1) assert not table._query_cache.lru table.search(query) assert len(table._query_cache) == 1 table.clear_cache() assert len(table._query_cache) == 0 def test_table_is_iterable(db): table = db.table('table1') table.insert_multiple({'int': i} for i in range(3)) assert [r for r in table] == table.all() def test_table_name(db): name = 'table3' table = db.table(name) assert name == table.name with pytest.raises(AttributeError): table.name = 'foo' def test_table_repr(db): name = 'table4' table = db.table(name) assert re.match( r">", repr(table)) def test_truncate_table(db): db.truncate() assert db._get_next_id() == 1 def test_persist_table(db): db.table("persisted", persist_empty=True) assert "persisted" in db.tables() db.table("nonpersisted", persist_empty=False) assert "nonpersisted" not in db.tables() ================================================ FILE: tests/test_tinydb.py ================================================ import re from collections.abc import Mapping import pytest from tinydb import TinyDB, where, Query from tinydb.middlewares import Middleware, CachingMiddleware from tinydb.storages import MemoryStorage, JSONStorage from tinydb.table import Document def test_drop_tables(db: TinyDB): db.drop_tables() db.insert({}) db.drop_tables() assert len(db) == 0 def test_all(db: TinyDB): db.drop_tables() for i in range(10): db.insert({}) assert len(db.all()) == 10 def test_insert(db: TinyDB): db.drop_tables() db.insert({'int': 1, 'char': 'a'}) assert db.count(where('int') == 1) == 1 db.drop_tables() db.insert({'int': 1, 'char': 'a'}) db.insert({'int': 1, 'char': 'b'}) db.insert({'int': 1, 'char': 'c'}) assert db.count(where('int') == 1) == 3 assert db.count(where('char') == 'a') == 1 def test_insert_ids(db: TinyDB): db.drop_tables() assert db.insert({'int': 1, 'char': 'a'}) == 1 assert db.insert({'int': 1, 'char': 'a'}) == 2 def test_insert_with_doc_id(db: TinyDB): db.drop_tables() assert db.insert({'int': 1, 'char': 'a'}) == 1 assert db.insert(Document({'int': 1, 'char': 'a'}, 12)) == 12 assert db.insert(Document({'int': 1, 'char': 'a'}, 77)) == 77 assert db.insert({'int': 1, 'char': 'a'}) == 78 def test_insert_with_duplicate_doc_id(db: TinyDB): db.drop_tables() assert db.insert({'int': 1, 'char': 'a'}) == 1 with pytest.raises(ValueError): db.insert(Document({'int': 1, 'char': 'a'}, 1)) def test_insert_multiple(db: TinyDB): db.drop_tables() assert not db.contains(where('int') == 1) # Insert multiple from list db.insert_multiple([{'int': 1, 'char': 'a'}, {'int': 1, 'char': 'b'}, {'int': 1, 'char': 'c'}]) assert db.count(where('int') == 1) == 3 assert db.count(where('char') == 'a') == 1 # Insert multiple from generator function def generator(): for j in range(10): yield {'int': j} db.drop_tables() db.insert_multiple(generator()) for i in range(10): assert db.count(where('int') == i) == 1 assert db.count(where('int').exists()) == 10 # Insert multiple from inline generator db.drop_tables() db.insert_multiple({'int': i} for i in range(10)) for i in range(10): assert db.count(where('int') == i) == 1 def test_insert_multiple_with_ids(db: TinyDB): db.drop_tables() # Insert multiple from list assert db.insert_multiple([{'int': 1, 'char': 'a'}, {'int': 1, 'char': 'b'}, {'int': 1, 'char': 'c'}]) == [1, 2, 3] def test_insert_multiple_with_doc_ids(db: TinyDB): db.drop_tables() assert db.insert_multiple([ Document({'int': 1, 'char': 'a'}, 12), Document({'int': 1, 'char': 'b'}, 77) ]) == [12, 77] assert db.get(doc_id=12) == {'int': 1, 'char': 'a'} assert db.get(doc_id=77) == {'int': 1, 'char': 'b'} with pytest.raises(ValueError): db.insert_multiple([Document({'int': 1, 'char': 'a'}, 12)]) def test_insert_invalid_type_raises_error(db: TinyDB): with pytest.raises(ValueError, match='Document is not a Mapping'): # object() as an example of a non-mapping-type db.insert(object()) # type: ignore def test_insert_valid_mapping_type(db: TinyDB): class CustomDocument(Mapping): def __init__(self, data): self.data = data def __getitem__(self, key): return self.data[key] def __iter__(self): return iter(self.data) def __len__(self): return len(self.data) db.drop_tables() db.insert(CustomDocument({'int': 1, 'char': 'a'})) assert db.count(where('int') == 1) == 1 def test_custom_mapping_type_with_json(tmpdir): class CustomDocument(Mapping): def __init__(self, data): self.data = data def __getitem__(self, key): return self.data[key] def __iter__(self): return iter(self.data) def __len__(self): return len(self.data) # Insert db = TinyDB(str(tmpdir.join('test.db'))) db.drop_tables() db.insert(CustomDocument({'int': 1, 'char': 'a'})) assert db.count(where('int') == 1) == 1 # Insert multiple db.insert_multiple([ CustomDocument({'int': 2, 'char': 'a'}), CustomDocument({'int': 3, 'char': 'a'}) ]) assert db.count(where('int') == 1) == 1 assert db.count(where('int') == 2) == 1 assert db.count(where('int') == 3) == 1 # Write back doc_id = db.get(where('int') == 3).doc_id db.update(CustomDocument({'int': 4, 'char': 'a'}), doc_ids=[doc_id]) assert db.count(where('int') == 3) == 0 assert db.count(where('int') == 4) == 1 def test_remove(db: TinyDB): db.remove(where('char') == 'b') assert len(db) == 2 assert db.count(where('int') == 1) == 2 def test_remove_all_fails(db: TinyDB): with pytest.raises(RuntimeError): db.remove() def test_remove_multiple(db: TinyDB): db.remove(where('int') == 1) assert len(db) == 0 def test_remove_ids(db: TinyDB): db.remove(doc_ids=[1, 2]) assert len(db) == 1 def test_remove_returns_ids(db: TinyDB): assert db.remove(where('char') == 'b') == [2] def test_update(db: TinyDB): assert len(db) == 3 db.update({'int': 2}, where('char') == 'a') assert db.count(where('int') == 2) == 1 assert db.count(where('int') == 1) == 2 def test_update_all(db: TinyDB): assert db.count(where('int') == 1) == 3 db.update({'newField': True}) assert db.count(where('newField') == True) == 3 # noqa def test_update_returns_ids(db: TinyDB): db.drop_tables() assert db.insert({'int': 1, 'char': 'a'}) == 1 assert db.insert({'int': 1, 'char': 'a'}) == 2 assert db.update({'char': 'b'}, where('int') == 1) == [1, 2] def test_update_transform(db: TinyDB): def increment(field): def transform(el): el[field] += 1 return transform def delete(field): def transform(el): del el[field] return transform assert db.count(where('int') == 1) == 3 db.update(increment('int'), where('char') == 'a') db.update(delete('char'), where('char') == 'a') assert db.count(where('int') == 2) == 1 assert db.count(where('char') == 'a') == 0 assert db.count(where('int') == 1) == 2 def test_update_ids(db: TinyDB): db.update({'int': 2}, doc_ids=[1, 2]) assert db.count(where('int') == 2) == 2 def test_update_multiple(db: TinyDB): assert len(db) == 3 db.update_multiple([ ({'int': 2}, where('char') == 'a'), ({'int': 4}, where('char') == 'b'), ]) assert db.count(where('int') == 1) == 1 assert db.count(where('int') == 2) == 1 assert db.count(where('int') == 4) == 1 def test_update_multiple_operation(db: TinyDB): def increment(field): def transform(el): el[field] += 1 return transform assert db.count(where('int') == 1) == 3 db.update_multiple([ (increment('int'), where('char') == 'a'), (increment('int'), where('char') == 'b') ]) assert db.count(where('int') == 2) == 2 def test_upsert(db: TinyDB): assert len(db) == 3 # Document existing db.upsert({'int': 5}, where('char') == 'a') assert db.count(where('int') == 5) == 1 # Document missing assert db.upsert({'int': 9, 'char': 'x'}, where('char') == 'x') == [4] assert db.count(where('int') == 9) == 1 def test_upsert_by_id(db: TinyDB): assert len(db) == 3 # Single document existing extant_doc = Document({'char': 'v'}, doc_id=1) assert db.upsert(extant_doc) == [1] doc = db.get(where('char') == 'v') assert isinstance(doc, Document) assert doc is not None assert doc.doc_id == 1 assert len(db) == 3 # Single document missing missing_doc = Document({'int': 5, 'char': 'w'}, doc_id=5) assert db.upsert(missing_doc) == [5] doc = db.get(where('char') == 'w') assert isinstance(doc, Document) assert doc is not None assert doc.doc_id == 5 assert len(db) == 4 # Missing doc_id and condition with pytest.raises(ValueError, match=r"(?=.*\bdoc_id\b)(?=.*\bquery\b)"): db.upsert({'no_Document': 'no_query'}) # Make sure we didn't break anything assert db.insert({'check': '_next_id'}) == 6 def test_search(db: TinyDB): assert not db._query_cache assert len(db.search(where('int') == 1)) == 3 assert len(db._query_cache) == 1 assert len(db.search(where('int') == 1)) == 3 # Query result from cache def test_search_path(db: TinyDB): assert not db._query_cache assert len(db.search(where('int').exists())) == 3 assert len(db._query_cache) == 1 assert len(db.search(where('asd').exists())) == 0 assert len(db.search(where('int').exists())) == 3 # Query result from cache def test_search_no_results_cache(db: TinyDB): assert len(db.search(where('missing').exists())) == 0 assert len(db.search(where('missing').exists())) == 0 def test_get(db: TinyDB): item = db.get(where('char') == 'b') assert isinstance(item, Document) assert item is not None assert item['char'] == 'b' def test_get_ids(db: TinyDB): el = db.all()[0] assert db.get(doc_id=el.doc_id) == el assert db.get(doc_id=float('NaN')) is None # type: ignore def test_get_multiple_ids(db: TinyDB): el = db.all() assert db.get(doc_ids=[x.doc_id for x in el]) == el def test_get_invalid(db: TinyDB): with pytest.raises(RuntimeError): db.get() def test_count(db: TinyDB): assert db.count(where('int') == 1) == 3 assert db.count(where('char') == 'd') == 0 def test_contains(db: TinyDB): assert db.contains(where('int') == 1) assert not db.contains(where('int') == 0) def test_contains_ids(db: TinyDB): assert db.contains(doc_id=1) assert db.contains(doc_id=2) assert not db.contains(doc_id=88) def test_contains_invalid(db: TinyDB): with pytest.raises(RuntimeError): db.contains() def test_get_idempotent(db: TinyDB): u = db.get(where('int') == 1) z = db.get(where('int') == 1) assert u == z def test_multiple_dbs(): """ Regression test for issue #3 """ db1 = TinyDB(storage=MemoryStorage) db2 = TinyDB(storage=MemoryStorage) db1.insert({'int': 1, 'char': 'a'}) db1.insert({'int': 1, 'char': 'b'}) db1.insert({'int': 1, 'value': 5.0}) db2.insert({'color': 'blue', 'animal': 'turtle'}) assert len(db1) == 3 assert len(db2) == 1 def test_storage_closed_once(): class Storage: def __init__(self): self.closed = False def read(self): return {} def write(self, data): pass def close(self): assert not self.closed self.closed = True with TinyDB(storage=Storage) as db: db.close() del db # If db.close() is called during cleanup, the assertion will fail and throw # and exception def test_unique_ids(tmpdir): """ :type tmpdir: py._path.local.LocalPath """ path = str(tmpdir.join('db.json')) # Verify ids are unique when reopening the DB and inserting with TinyDB(path) as _db: _db.insert({'x': 1}) with TinyDB(path) as _db: _db.insert({'x': 1}) with TinyDB(path) as _db: data = _db.all() assert data[0].doc_id != data[1].doc_id # Verify ids stay unique when inserting/removing with TinyDB(path) as _db: _db.drop_tables() _db.insert_multiple({'x': i} for i in range(5)) _db.remove(where('x') == 2) assert len(_db) == 4 ids = [e.doc_id for e in _db.all()] assert len(ids) == len(set(ids)) def test_lastid_after_open(tmpdir): """ Regression test for issue #34 :type tmpdir: py._path.local.LocalPath """ NUM = 100 path = str(tmpdir.join('db.json')) with TinyDB(path) as _db: _db.insert_multiple({'i': i} for i in range(NUM)) with TinyDB(path) as _db: assert _db._get_next_id() - 1 == NUM def test_doc_ids_json(tmpdir): """ Regression test for issue #45 """ path = str(tmpdir.join('db.json')) with TinyDB(path) as _db: _db.drop_tables() assert _db.insert({'int': 1, 'char': 'a'}) == 1 assert _db.insert({'int': 1, 'char': 'a'}) == 2 _db.drop_tables() assert _db.insert_multiple([{'int': 1, 'char': 'a'}, {'int': 1, 'char': 'b'}, {'int': 1, 'char': 'c'}]) == [1, 2, 3] assert _db.contains(doc_id=1) assert _db.contains(doc_id=2) assert not _db.contains(doc_id=88) _db.update({'int': 2}, doc_ids=[1, 2]) assert _db.count(where('int') == 2) == 2 el = _db.all()[0] assert _db.get(doc_id=el.doc_id) == el assert _db.get(doc_id=float('NaN')) is None _db.remove(doc_ids=[1, 2]) assert len(_db) == 1 def test_insert_string(tmpdir): path = str(tmpdir.join('db.json')) with TinyDB(path) as _db: data = [{'int': 1}, {'int': 2}] _db.insert_multiple(data) with pytest.raises(ValueError): _db.insert([1, 2, 3]) # Fails with pytest.raises(ValueError): _db.insert({'bark'}) # Fails assert data == _db.all() _db.insert({'int': 3}) # Does not fail def test_insert_invalid_dict(tmpdir): path = str(tmpdir.join('db.json')) with TinyDB(path) as _db: data = [{'int': 1}, {'int': 2}] _db.insert_multiple(data) with pytest.raises(TypeError): _db.insert({'int': _db}) # Fails assert data == _db.all() _db.insert({'int': 3}) # Does not fail def test_gc(tmpdir): # See https://github.com/msiemens/tinydb/issues/92 path = str(tmpdir.join('db.json')) db = TinyDB(path) table = db.table('foo') table.insert({'something': 'else'}) table.insert({'int': 13}) assert len(table.search(where('int') == 13)) == 1 assert table.all() == [{'something': 'else'}, {'int': 13}] db.close() def test_drop_table(): db = TinyDB(storage=MemoryStorage) default_table_name = db.table(db.default_table_name).name assert [] == list(db.tables()) db.drop_table(default_table_name) db.insert({'a': 1}) assert [default_table_name] == list(db.tables()) db.drop_table(default_table_name) assert [] == list(db.tables()) table_name = 'some-other-table' db = TinyDB(storage=MemoryStorage) db.table(table_name).insert({'a': 1}) assert {table_name} == db.tables() db.drop_table(table_name) assert set() == db.tables() assert table_name not in db._tables db.drop_table('non-existent-table-name') assert set() == db.tables() def test_empty_write(tmpdir): path = str(tmpdir.join('db.json')) class ReadOnlyMiddleware(Middleware): def write(self, data): raise AssertionError('No write for unchanged db') TinyDB(path).close() TinyDB(path, storage=ReadOnlyMiddleware(JSONStorage)).close() def test_query_cache(): db = TinyDB(storage=MemoryStorage) db.insert_multiple([ {'name': 'foo', 'value': 42}, {'name': 'bar', 'value': -1337} ]) query = where('value') > 0 results = db.search(query) assert len(results) == 1 # Modify the db instance to not return any results when # bypassing the query cache db._tables[db.table(db.default_table_name).name]._read_table = lambda: {} # Make sure we got an independent copy of the result list results.extend([1]) assert db.search(query) == [{'name': 'foo', 'value': 42}] def test_tinydb_is_iterable(db: TinyDB): assert [r for r in db] == db.all() def test_repr(tmpdir): path = str(tmpdir.join('db.json')) db = TinyDB(path) db.insert({'a': 1}) assert re.match( r"", repr(db)) def test_delete(tmpdir): path = str(tmpdir.join('db.json')) db = TinyDB(path, ensure_ascii=False) q = Query() db.insert({'network': {'id': '114', 'name': 'ok', 'rpc': 'dac', 'ticker': 'mkay'}}) assert db.search(q.network.id == '114') == [ {'network': {'id': '114', 'name': 'ok', 'rpc': 'dac', 'ticker': 'mkay'}} ] db.remove(q.network.id == '114') assert db.search(q.network.id == '114') == [] def test_insert_multiple_with_single_dict(db: TinyDB): with pytest.raises(ValueError): d = {'first': 'John', 'last': 'smith'} db.insert_multiple(d) # type: ignore db.close() def test_access_storage(): assert isinstance(TinyDB(storage=MemoryStorage).storage, MemoryStorage) assert isinstance(TinyDB(storage=CachingMiddleware(MemoryStorage)).storage, CachingMiddleware) def test_empty_db_len(): db = TinyDB(storage=MemoryStorage) assert len(db) == 0 def test_insert_on_existing_db(tmpdir): path = str(tmpdir.join('db.json')) db = TinyDB(path, ensure_ascii=False) db.insert({'foo': 'bar'}) assert len(db) == 1 db.close() db = TinyDB(path, ensure_ascii=False) db.insert({'foo': 'bar'}) db.insert({'foo': 'bar'}) assert len(db) == 3 def test_storage_access(): db = TinyDB(storage=MemoryStorage) assert isinstance(db.storage, MemoryStorage) def test_lambda_query(): db = TinyDB(storage=MemoryStorage) db.insert({'foo': 'bar'}) query = lambda doc: doc.get('foo') == 'bar' query.is_cacheable = lambda: False assert db.search(query) == [{'foo': 'bar'}] assert not db._query_cache ================================================ FILE: tests/test_utils.py ================================================ import pytest from tinydb.utils import LRUCache, freeze, FrozenDict def test_lru_cache(): cache = LRUCache(capacity=3) cache["a"] = 1 cache["b"] = 2 cache["c"] = 3 _ = cache["a"] # move to front in lru queue cache["d"] = 4 # move oldest item out of lru queue try: _ = cache['f'] except KeyError: pass assert cache.lru == ["c", "a", "d"] def test_lru_cache_set_multiple(): cache = LRUCache(capacity=3) cache["a"] = 1 cache["a"] = 2 cache["a"] = 3 cache["a"] = 4 assert cache.lru == ["a"] def test_lru_cache_set_update(): cache = LRUCache(capacity=3) cache["a"] = 1 cache["a"] = 2 assert cache["a"] == 2 def test_lru_cache_get(): cache = LRUCache(capacity=3) cache["a"] = 1 cache["b"] = 1 cache["c"] = 1 cache.get("a") cache["d"] = 4 assert cache.lru == ["c", "a", "d"] def test_lru_cache_delete(): cache = LRUCache(capacity=3) cache["a"] = 1 cache["b"] = 2 del cache["a"] try: del cache['f'] except KeyError: pass assert cache.lru == ["b"] def test_lru_cache_clear(): cache = LRUCache(capacity=3) cache["a"] = 1 cache["b"] = 2 cache.clear() assert cache.lru == [] def test_lru_cache_unlimited(): cache = LRUCache() for i in range(100): cache[i] = i assert len(cache.lru) == 100 def test_lru_cache_unlimited_explicit(): cache = LRUCache(capacity=None) for i in range(100): cache[i] = i assert len(cache.lru) == 100 def test_lru_cache_iteration_works(): cache = LRUCache() count = 0 for _ in cache: assert False, 'there should be no elements in the cache' assert count == 0 def test_lru_cache_falsy_values_bug(): """ Test for GitHub issue #596: LRU cache should handle falsy values correctly. Bug: `if self.cache.get(key):` treated falsy values as non-existent keys, breaking LRU ordering when updating existing keys with falsy values. """ cache = LRUCache(capacity=3) # Set up cache with falsy value cache["a"] = 0 # Falsy value cache["b"] = 1 cache["c"] = 2 assert cache.lru == ["a", "b", "c"] # Update existing key with falsy value - should move to end cache.set("a", 3) assert cache.lru == ["b", "c", "a"] # Add new item - should evict oldest ("b"), not "a" cache.set("d", 4) assert cache.lru == ["c", "a", "d"] assert "b" not in cache assert cache["a"] == 3 def test_freeze(): frozen = freeze([0, 1, 2, {'a': [1, 2, 3]}, {1, 2}]) assert isinstance(frozen, tuple) assert isinstance(frozen[3], FrozenDict) assert isinstance(frozen[3]['a'], tuple) assert isinstance(frozen[4], frozenset) with pytest.raises(TypeError): frozen[0] = 10 with pytest.raises(TypeError): frozen[3]['a'] = 10 with pytest.raises(TypeError): frozen[3].pop('a') with pytest.raises(TypeError): frozen[3].update({'a': 9}) ================================================ FILE: tinydb/__init__.py ================================================ """ TinyDB is a tiny, document oriented database optimized for your happiness :) TinyDB stores different types of Python data types using a configurable storage mechanism. It comes with a syntax for querying data and storing data in multiple tables. .. codeauthor:: Markus Siemens Usage example: >>> from tinydb import TinyDB, where >>> from tinydb.storages import MemoryStorage >>> db = TinyDB(storage=MemoryStorage) >>> db.insert({'data': 5}) # Insert into '_default' table >>> db.search(where('data') == 5) [{'data': 5, '_id': 1}] >>> # Now let's create a new table >>> tbl = db.table('our_table') >>> for i in range(10): ... tbl.insert({'data': i}) ... >>> len(tbl.search(where('data') < 5)) 5 """ from .queries import Query, where from .storages import Storage, JSONStorage from .database import TinyDB from .version import __version__ __all__ = ('TinyDB', 'Storage', 'JSONStorage', 'Query', 'where') ================================================ FILE: tinydb/database.py ================================================ """ This module contains the main component of TinyDB: the database. """ from typing import Dict, Iterator, Set, Type from . import JSONStorage from .storages import Storage from .table import Table, Document from .utils import with_typehint # The table's base class. This is used to add type hinting from the Table # class to TinyDB. Currently, this supports PyCharm, Pyright/VS Code and MyPy. TableBase: Type[Table] = with_typehint(Table) class TinyDB(TableBase): """ The main class of TinyDB. The ``TinyDB`` class is responsible for creating the storage class instance that will store this database's documents, managing the database tables as well as providing access to the default table. For table management, a simple ``dict`` is used that stores the table class instances accessible using their table name. Default table access is provided by forwarding all unknown method calls and property access operations to the default table by implementing ``__getattr__``. When creating a new instance, all arguments and keyword arguments (except for ``storage``) will be passed to the storage class that is provided. If no storage class is specified, :class:`~tinydb.storages.JSONStorage` will be used. .. admonition:: Customization For customization, the following class variables can be set: - ``table_class`` defines the class that is used to create tables, - ``default_table_name`` defines the name of the default table, and - ``default_storage_class`` will define the class that will be used to create storage instances if no other storage is passed. .. versionadded:: 4.0 .. admonition:: Data Storage Model Data is stored using a storage class that provides persistence for a ``dict`` instance. This ``dict`` contains all tables and their data. The data is modelled like this:: { 'table1': { 0: {document...}, 1: {document...}, }, 'table2': { ... } } Each entry in this ``dict`` uses the table name as its key and a ``dict`` of documents as its value. The document ``dict`` contains document IDs as keys and the documents themselves as values. :param storage: The class of the storage to use. Will be initialized with ``args`` and ``kwargs``. """ #: The class that will be used to create table instances #: #: .. versionadded:: 4.0 table_class = Table #: The name of the default table #: #: .. versionadded:: 4.0 default_table_name = '_default' #: The class that will be used by default to create storage instances #: #: .. versionadded:: 4.0 default_storage_class = JSONStorage def __init__(self, *args, **kwargs) -> None: """ Create a new instance of TinyDB. """ storage = kwargs.pop('storage', self.default_storage_class) # Prepare the storage self._storage: Storage = storage(*args, **kwargs) self._opened = True self._tables: Dict[str, Table] = {} def __repr__(self): args = [ f'tables={list(self.tables())}', f'tables_count={len(self.tables())}', f'default_table_documents_count={self.__len__()}', f'all_tables_documents_count={[f"{table}={len(self.table(table))}" for table in self.tables()]}', ] return '<{} {}>'.format(type(self).__name__, ', '.join(args)) def table(self, name: str, **kwargs) -> Table: """ Get access to a specific table. If the table hasn't been accessed yet, a new table instance will be created using the :attr:`~tinydb.database.TinyDB.table_class` class. Otherwise, the previously created table instance will be returned. All further options besides the name are passed to the table class which by default is :class:`~tinydb.table.Table`. Check its documentation for further parameters you can pass. :param name: The name of the table. :param kwargs: Keyword arguments to pass to the table class constructor """ if name in self._tables: return self._tables[name] table = self.table_class(self.storage, name, **kwargs) self._tables[name] = table return table def tables(self) -> Set[str]: """ Get the names of all tables in the database. :returns: a set of table names """ # TinyDB stores data as a dict of tables like this: # # { # '_default': { # 0: {document...}, # 1: {document...}, # }, # 'table1': { # ... # } # } # # To get a set of table names, we thus construct a set of this main # dict which returns a set of the dict keys which are the table names. # # Storage.read() may return ``None`` if the database file is empty, # so we need to consider this case to and return an empty set in this # case. return set(self.storage.read() or {}) def drop_tables(self) -> None: """ Drop all tables from the database. **CANNOT BE REVERSED!** """ # We drop all tables from this database by writing an empty dict # to the storage thereby returning to the initial state with no tables. self.storage.write({}) # After that we need to remember to empty the ``_tables`` dict, so we'll # create new table instances when a table is accessed again. self._tables.clear() def drop_table(self, name: str) -> None: """ Drop a specific table from the database. **CANNOT BE REVERSED!** :param name: The name of the table to drop. """ # If the table is currently opened, we need to forget the table class # instance if name in self._tables: del self._tables[name] data = self.storage.read() # The database is uninitialized, there's nothing to do if data is None: return # The table does not exist, there's nothing to do if name not in data: return # Remove the table from the data dict del data[name] # Store the updated data back to the storage self.storage.write(data) @property def storage(self) -> Storage: """ Get the storage instance used for this TinyDB instance. :return: This instance's storage :rtype: Storage """ return self._storage def close(self) -> None: """ Close the database. This may be needed if the storage instance used for this database needs to perform cleanup operations like closing file handles. To ensure this method is called, the TinyDB instance can be used as a context manager:: with TinyDB('data.json') as db: db.insert({'foo': 'bar'}) Upon leaving this context, the ``close`` method will be called. """ self._opened = False self.storage.close() def __enter__(self): """ Use the database as a context manager. Using the database as a context manager ensures that the :meth:`~tinydb.database.TinyDB.close` method is called upon leaving the context. :return: The current instance """ return self def __exit__(self, *args): """ Close the storage instance when leaving a context. """ if self._opened: self.close() def __getattr__(self, name): """ Forward all unknown attribute calls to the default table instance. """ return getattr(self.table(self.default_table_name), name) # Here we forward magic methods to the default table instance. These are # not handled by __getattr__ so we need to forward them manually here def __len__(self): """ Get the total number of documents in the default table. >>> db = TinyDB('db.json') >>> len(db) 0 """ return len(self.table(self.default_table_name)) def __iter__(self) -> Iterator[Document]: """ Return an iterator for the default table's documents. """ return iter(self.table(self.default_table_name)) ================================================ FILE: tinydb/middlewares.py ================================================ """ Contains the :class:`base class ` for middlewares and implementations. """ from typing import Optional from tinydb import Storage class Middleware: """ The base class for all Middlewares. Middlewares hook into the read/write process of TinyDB allowing you to extend the behaviour by adding caching, logging, ... Your middleware's ``__init__`` method has to call the parent class constructor so the middleware chain can be configured properly. """ def __init__(self, storage_cls) -> None: self._storage_cls = storage_cls self.storage: Storage = None # type: ignore def __call__(self, *args, **kwargs): """ Create the storage instance and store it as self.storage. Usually a user creates a new TinyDB instance like this:: TinyDB(storage=StorageClass) The storage keyword argument is used by TinyDB this way:: self.storage = storage(*args, **kwargs) As we can see, ``storage(...)`` runs the constructor and returns the new storage instance. Using Middlewares, the user will call:: The 'real' storage class v TinyDB(storage=Middleware(StorageClass)) ^ Already an instance! So, when running ``self.storage = storage(*args, **kwargs)`` Python now will call ``__call__`` and TinyDB will expect the return value to be the storage (or Middleware) instance. Returning the instance is simple, but we also got the underlying (*real*) StorageClass as an __init__ argument that still is not an instance. So, we initialize it in __call__ forwarding any arguments we receive from TinyDB (``TinyDB(arg1, kwarg1=value, storage=...)``). In case of nested Middlewares, calling the instance as if it was a class results in calling ``__call__`` what initializes the next nested Middleware that itself will initialize the next Middleware and so on. """ self.storage = self._storage_cls(*args, **kwargs) return self def __getattr__(self, name): """ Forward all unknown attribute calls to the underlying storage, so we remain as transparent as possible. """ return getattr(self.__dict__['storage'], name) class CachingMiddleware(Middleware): """ Add some caching to TinyDB. This Middleware aims to improve the performance of TinyDB by writing only the last DB state every :attr:`WRITE_CACHE_SIZE` time and reading always from cache. """ #: The number of write operations to cache before writing to disc WRITE_CACHE_SIZE = 1000 def __init__(self, storage_cls): # Initialize the parent constructor super().__init__(storage_cls) # Prepare the cache self.cache = None self._cache_modified_count = 0 def read(self): if self.cache is None: # Empty cache: read from the storage self.cache = self.storage.read() # Return the cached data return self.cache def write(self, data): # Store data in cache self.cache = data self._cache_modified_count += 1 # Check if we need to flush the cache if self._cache_modified_count >= self.WRITE_CACHE_SIZE: self.flush() def flush(self): """ Flush all unwritten data to disk. """ if self._cache_modified_count > 0: # Force-flush the cache by writing the data to the storage self.storage.write(self.cache) self._cache_modified_count = 0 def close(self): # Flush potentially unwritten data self.flush() # Let the storage clean up too self.storage.close() ================================================ FILE: tinydb/mypy_plugin.py ================================================ from typing import TypeVar, Optional, Callable, Dict from mypy.nodes import NameExpr from mypy.options import Options from mypy.plugin import Plugin, DynamicClassDefContext T = TypeVar('T') CB = Optional[Callable[[T], None]] DynamicClassDef = DynamicClassDefContext class TinyDBPlugin(Plugin): def __init__(self, options: Options): super().__init__(options) self.named_placeholders: Dict[str, str] = {} def get_dynamic_class_hook(self, fullname: str) -> CB[DynamicClassDef]: if fullname == 'tinydb.utils.with_typehint': def hook(ctx: DynamicClassDefContext): klass = ctx.call.args[0] assert isinstance(klass, NameExpr) type_name = klass.fullname assert type_name is not None qualified = self.lookup_fully_qualified(type_name) assert qualified is not None ctx.api.add_symbol_table_node(ctx.name, qualified) return hook return None def plugin(_version: str): return TinyDBPlugin ================================================ FILE: tinydb/operations.py ================================================ """ A collection of update operations for TinyDB. They are used for updates like this: >>> db.update(delete('foo'), where('foo') == 2) This would delete the ``foo`` field from all documents where ``foo`` equals 2. """ def delete(field): """ Delete a given field from the document. """ def transform(doc): del doc[field] return transform def add(field, n): """ Add ``n`` to a given field in the document. """ def transform(doc): doc[field] += n return transform def subtract(field, n): """ Subtract ``n`` to a given field in the document. """ def transform(doc): doc[field] -= n return transform def set(field, val): """ Set a given field to ``val``. """ def transform(doc): doc[field] = val return transform def increment(field): """ Increment a given field in the document by 1. """ def transform(doc): doc[field] += 1 return transform def decrement(field): """ Decrement a given field in the document by 1. """ def transform(doc): doc[field] -= 1 return transform ================================================ FILE: tinydb/py.typed ================================================ ================================================ FILE: tinydb/queries.py ================================================ """ Contains the querying interface. Starting with :class:`~tinydb.queries.Query` you can construct complex queries: >>> ((where('f1') == 5) & (where('f2') != 2)) | where('s').matches(r'^\\w+$') (('f1' == 5) and ('f2' != 2)) or ('s' ~= ^\\w+$ ) Queries are executed by using the ``__call__``: >>> q = where('val') == 5 >>> q({'val': 5}) True >>> q({'val': 1}) False """ import re from typing import Mapping, Tuple, Callable, Any, Union, List, Optional, Protocol from .utils import freeze __all__ = ('Query', 'QueryLike', 'where') def is_sequence(obj): return hasattr(obj, '__iter__') class QueryLike(Protocol): """ A typing protocol that acts like a query. Something that we use as a query must have two properties: 1. It must be callable, accepting a `Mapping` object and returning a boolean that indicates whether the value matches the query, and 2. it must have a stable hash that will be used for query caching. In addition, to mark a query as non-cacheable (e.g. if it involves some remote lookup) it needs to have a method called ``is_cacheable`` that returns ``False``. This query protocol is used to make MyPy correctly support the query pattern that TinyDB uses. See also https://mypy.readthedocs.io/en/stable/protocols.html#simple-user-defined-protocols """ def __call__(self, value: Mapping) -> bool: ... def __hash__(self) -> int: ... class QueryInstance: """ A query instance. This is the object on which the actual query operations are performed. The :class:`~tinydb.queries.Query` class acts like a query builder and generates :class:`~tinydb.queries.QueryInstance` objects which will evaluate their query against a given document when called. Query instances can be combined using logical OR and AND and inverted using logical NOT. In order to be usable in a query cache, a query needs to have a stable hash value with the same query always returning the same hash. That way a query instance can be used as a key in a dictionary. """ def __init__(self, test: Callable[[Mapping], bool], hashval: Optional[Tuple]): self._test = test self._hash = hashval def is_cacheable(self) -> bool: return self._hash is not None def __call__(self, value: Mapping) -> bool: """ Evaluate the query to check if it matches a specified value. :param value: The value to check. :return: Whether the value matches this query. """ return self._test(value) def __hash__(self) -> int: # We calculate the query hash by using the ``hashval`` object which # describes this query uniquely, so we can calculate a stable hash # value by simply hashing it return hash(self._hash) def __repr__(self): return 'QueryImpl{}'.format(self._hash) def __eq__(self, other: object): if isinstance(other, QueryInstance): return self._hash == other._hash return False # --- Query modifiers ----------------------------------------------------- def __and__(self, other: 'QueryInstance') -> 'QueryInstance': # We use a frozenset for the hash as the AND operation is commutative # (a & b == b & a) and the frozenset does not consider the order of # elements if self.is_cacheable() and other.is_cacheable(): hashval = ('and', frozenset([self._hash, other._hash])) else: hashval = None return QueryInstance(lambda value: self(value) and other(value), hashval) def __or__(self, other: 'QueryInstance') -> 'QueryInstance': # We use a frozenset for the hash as the OR operation is commutative # (a | b == b | a) and the frozenset does not consider the order of # elements if self.is_cacheable() and other.is_cacheable(): hashval = ('or', frozenset([self._hash, other._hash])) else: hashval = None return QueryInstance(lambda value: self(value) or other(value), hashval) def __invert__(self) -> 'QueryInstance': hashval = ('not', self._hash) if self.is_cacheable() else None return QueryInstance(lambda value: not self(value), hashval) class Query(QueryInstance): """ TinyDB Queries. Allows building queries for TinyDB databases. There are two main ways of using queries: 1) ORM-like usage: >>> User = Query() >>> db.search(User.name == 'John Doe') >>> db.search(User['logged-in'] == True) 2) Classical usage: >>> db.search(where('value') == True) Note that ``where(...)`` is a shorthand for ``Query(...)`` allowing for a more fluent syntax. Besides the methods documented here you can combine queries using the binary AND and OR operators: >>> # Binary AND: >>> db.search((where('field1').exists()) & (where('field2') == 5)) >>> # Binary OR: >>> db.search((where('field1').exists()) | (where('field2') == 5)) Queries are executed by calling the resulting object. They expect to get the document to test as the first argument and return ``True`` or ``False`` depending on whether the documents match the query or not. """ def __init__(self) -> None: # The current path of fields to access when evaluating the object self._path: Tuple[Union[str, Callable], ...] = () # Prevent empty queries to be evaluated def notest(_): raise RuntimeError('Empty query was evaluated') super().__init__( test=notest, hashval=(None,) ) def __repr__(self): return '{}()'.format(type(self).__name__) def __hash__(self): return super().__hash__() def __getattr__(self, item: str): # Generate a new query object with the new query path # We use type(self) to get the class of the current query in case # someone uses a subclass of ``Query`` query = type(self)() # Now we add the accessed item to the query path ... query._path = self._path + (item,) # ... and update the query hash query._hash = ('path', query._path) if self.is_cacheable() else None return query def __getitem__(self, item: str): # A different syntax for ``__getattr__`` # We cannot call ``getattr(item)`` here as it would try to resolve # the name as a method name first, only then call our ``__getattr__`` # method. By calling ``__getattr__`` directly, we make sure that # calling e.g. ``Query()['test']`` will always generate a query for a # document's ``test`` field instead of returning a reference to the # ``Query.test`` method return self.__getattr__(item) def _generate_test( self, test: Callable[[Any], bool], hashval: Tuple, allow_empty_path: bool = False ) -> QueryInstance: """ Generate a query based on a test function that first resolves the query path. :param test: The test the query executes. :param hashval: The hash of the query. :return: A :class:`~tinydb.queries.QueryInstance` object """ if not self._path and not allow_empty_path: raise ValueError('Query has no path') def runner(value): try: # Resolve the path for part in self._path: if isinstance(part, str): value = value[part] else: value = part(value) except (KeyError, TypeError): return False else: # Perform the specified test return test(value) return QueryInstance( lambda value: runner(value), (hashval if self.is_cacheable() else None) ) def __eq__(self, rhs: Any): """ Test a dict value for equality. >>> Query().f1 == 42 :param rhs: The value to compare against """ return self._generate_test( lambda value: value == rhs, ('==', self._path, freeze(rhs)) ) def __ne__(self, rhs: Any): """ Test a dict value for inequality. >>> Query().f1 != 42 :param rhs: The value to compare against """ return self._generate_test( lambda value: value != rhs, ('!=', self._path, freeze(rhs)) ) def __lt__(self, rhs: Any) -> QueryInstance: """ Test a dict value for being lower than another value. >>> Query().f1 < 42 :param rhs: The value to compare against """ return self._generate_test( lambda value: value < rhs, ('<', self._path, rhs) ) def __le__(self, rhs: Any) -> QueryInstance: """ Test a dict value for being lower than or equal to another value. >>> where('f1') <= 42 :param rhs: The value to compare against """ return self._generate_test( lambda value: value <= rhs, ('<=', self._path, rhs) ) def __gt__(self, rhs: Any) -> QueryInstance: """ Test a dict value for being greater than another value. >>> Query().f1 > 42 :param rhs: The value to compare against """ return self._generate_test( lambda value: value > rhs, ('>', self._path, rhs) ) def __ge__(self, rhs: Any) -> QueryInstance: """ Test a dict value for being greater than or equal to another value. >>> Query().f1 >= 42 :param rhs: The value to compare against """ return self._generate_test( lambda value: value >= rhs, ('>=', self._path, rhs) ) def exists(self) -> QueryInstance: """ Test for a dict where a provided key exists. >>> Query().f1.exists() """ return self._generate_test( lambda _: True, ('exists', self._path) ) def matches(self, regex: str, flags: int = 0) -> QueryInstance: """ Run a regex test against a dict value (whole string has to match). >>> Query().f1.matches(r'^\\w+$') :param regex: The regular expression to use for matching :param flags: regex flags to pass to ``re.match`` """ def test(value): if not isinstance(value, str): return False return re.match(regex, value, flags) is not None return self._generate_test(test, ('matches', self._path, regex)) def search(self, regex: str, flags: int = 0) -> QueryInstance: """ Run a regex test against a dict value (only substring string has to match). >>> Query().f1.search(r'^\\w+$') :param regex: The regular expression to use for matching :param flags: regex flags to pass to ``re.match`` """ def test(value): if not isinstance(value, str): return False return re.search(regex, value, flags) is not None return self._generate_test(test, ('search', self._path, regex)) def test(self, func: Callable[[Mapping], bool], *args) -> QueryInstance: """ Run a user-defined test function against a dict value. >>> def test_func(val): ... return val == 42 ... >>> Query().f1.test(test_func) .. warning:: The test function provided needs to be deterministic (returning the same value when provided with the same arguments), otherwise this may mess up the query cache that :class:`~tinydb.table.Table` implements. :param func: The function to call, passing the dict as the first argument :param args: Additional arguments to pass to the test function """ return self._generate_test( lambda value: func(value, *args), ('test', self._path, func, args) ) def any(self, cond: Union[QueryInstance, List[Any]]) -> QueryInstance: """ Check if a condition is met by any document in a list, where a condition can also be a sequence (e.g. list). >>> Query().f1.any(Query().f2 == 1) Matches:: {'f1': [{'f2': 1}, {'f2': 0}]} >>> Query().f1.any([1, 2, 3]) Matches:: {'f1': [1, 2]} {'f1': [3, 4, 5]} :param cond: Either a query that at least one document has to match or a list of which at least one document has to be contained in the tested document. """ if callable(cond): def test(value): return is_sequence(value) and any(cond(e) for e in value) else: def test(value): return is_sequence(value) and any(e in cond for e in value) return self._generate_test( lambda value: test(value), ('any', self._path, freeze(cond)) ) def all(self, cond: Union['QueryInstance', List[Any]]) -> QueryInstance: """ Check if a condition is met by all documents in a list, where a condition can also be a sequence (e.g. list). >>> Query().f1.all(Query().f2 == 1) Matches:: {'f1': [{'f2': 1}, {'f2': 1}]} >>> Query().f1.all([1, 2, 3]) Matches:: {'f1': [1, 2, 3, 4, 5]} :param cond: Either a query that all documents have to match or a list which has to be contained in the tested document. """ if callable(cond): def test(value): return is_sequence(value) and all(cond(e) for e in value) else: def test(value): return is_sequence(value) and all(e in value for e in cond) return self._generate_test( lambda value: test(value), ('all', self._path, freeze(cond)) ) def one_of(self, items: List[Any]) -> QueryInstance: """ Check if the value is contained in a list or generator. >>> Query().f1.one_of(['value 1', 'value 2']) :param items: The list of items to check with """ return self._generate_test( lambda value: value in items, ('one_of', self._path, freeze(items)) ) def fragment(self, document: Mapping) -> QueryInstance: def test(value): for key in document: if key not in value or value[key] != document[key]: return False return True return self._generate_test( lambda value: test(value), ('fragment', freeze(document)), allow_empty_path=True ) def noop(self) -> QueryInstance: """ Always evaluate to ``True``. Useful for having a base value when composing queries dynamically. """ return QueryInstance( lambda value: True, () ) def map(self, fn: Callable[[Any], Any]) -> 'Query': """ Add a function to the query path. Similar to __getattr__ but for arbitrary functions. """ query = type(self)() # Now we add the callable to the query path ... query._path = self._path + (fn,) # ... and kill the hash - callable objects can be mutable, so it's # harmful to cache their results. query._hash = None return query def where(key: str) -> Query: """ A shorthand for ``Query()[key]`` """ return Query()[key] ================================================ FILE: tinydb/storages.py ================================================ """ Contains the :class:`base class ` for storages and implementations. """ import io import json import os import warnings from abc import ABC, abstractmethod from typing import Dict, Any, Optional __all__ = ('Storage', 'JSONStorage', 'MemoryStorage') def touch(path: str, create_dirs: bool): """ Create a file if it doesn't exist yet. :param path: The file to create. :param create_dirs: Whether to create all missing parent directories. """ if create_dirs: base_dir = os.path.dirname(path) # Check if we need to create missing parent directories if not os.path.exists(base_dir): os.makedirs(base_dir) # Create the file by opening it in 'a' mode which creates the file if it # does not exist yet but does not modify its contents with open(path, 'a'): pass class Storage(ABC): """ The abstract base class for all Storages. A Storage (de)serializes the current state of the database and stores it in some place (memory, file on disk, ...). """ # Using ABCMeta as metaclass allows instantiating only storages that have # implemented read and write @abstractmethod def read(self) -> Optional[Dict[str, Dict[str, Any]]]: """ Read the current state. Any kind of deserialization should go here. Return ``None`` here to indicate that the storage is empty. """ raise NotImplementedError('To be overridden!') @abstractmethod def write(self, data: Dict[str, Dict[str, Any]]) -> None: """ Write the current state of the database to the storage. Any kind of serialization should go here. :param data: The current state of the database. """ raise NotImplementedError('To be overridden!') def close(self) -> None: """ Optional: Close open file handles, etc. """ pass class JSONStorage(Storage): """ Store the data in a JSON file. """ def __init__(self, path: str, create_dirs=False, encoding=None, access_mode='r+', **kwargs): """ Create a new instance. Also creates the storage file, if it doesn't exist and the access mode is appropriate for writing. **Note:** Using an access mode other than `r` or `r+` will probably lead to data loss or data corruption! **Note:** **Never** pass untrusted or user-controlled code as ``kwargs`` members like ``cls`` or ``default`` will be called on every write operation. :param path: Where to store the JSON data. :param access_mode: mode in which the file is opened (r, r+) :type access_mode: str """ super().__init__() self._mode = access_mode self.kwargs = kwargs if access_mode not in ('r', 'rb', 'r+', 'rb+'): warnings.warn( 'Using an `access_mode` other than \'r\', \'rb\', \'r+\' ' 'or \'rb+\' can cause data loss or corruption' ) # Create the file if it doesn't exist and creating is allowed by the # access mode if any([character in self._mode for character in ('+', 'w', 'a')]): # any of the writing modes touch(path, create_dirs=create_dirs) # Open the file for reading/writing self._handle = open(path, mode=self._mode, encoding=encoding) def close(self) -> None: self._handle.close() def read(self) -> Optional[Dict[str, Dict[str, Any]]]: # Get the file size by moving the cursor to the file end and reading # its location self._handle.seek(0, os.SEEK_END) size = self._handle.tell() if not size: # File is empty, so we return ``None`` so TinyDB can properly # initialize the database return None else: # Return the cursor to the beginning of the file self._handle.seek(0) # Load the JSON contents of the file return json.load(self._handle) def write(self, data: Dict[str, Dict[str, Any]]): # Move the cursor to the beginning of the file just in case self._handle.seek(0) # Serialize the database state using the user-provided arguments serialized = json.dumps(data, **self.kwargs) # Write the serialized data to the file try: self._handle.write(serialized) except io.UnsupportedOperation: raise IOError('Cannot write to the database. Access mode is "{0}"'.format(self._mode)) # Ensure the file has been written self._handle.flush() os.fsync(self._handle.fileno()) # Remove data that is behind the new cursor in case the file has # gotten shorter self._handle.truncate() class MemoryStorage(Storage): """ Store the data as JSON in memory. """ def __init__(self): """ Create a new instance. """ super().__init__() self.memory = None def read(self) -> Optional[Dict[str, Dict[str, Any]]]: return self.memory def write(self, data: Dict[str, Dict[str, Any]]): self.memory = data ================================================ FILE: tinydb/table.py ================================================ """ This module implements tables, the central place for accessing and manipulating data in TinyDB. """ from typing import ( Callable, Dict, Iterable, Iterator, List, Mapping, NoReturn, Optional, Union, cast, Tuple, overload ) from .queries import QueryLike from .storages import Storage from .utils import LRUCache __all__ = ('Document', 'Table') class Document(dict): """ A document stored in the database. This class provides a way to access both a document's content and its ID using ``doc.doc_id``. """ def __init__(self, value: Mapping, doc_id: int): super().__init__(value) self.doc_id = doc_id class Table: """ Represents a single TinyDB table. It provides methods for accessing and manipulating documents. .. admonition:: Query Cache As an optimization, a query cache is implemented using a :class:`~tinydb.utils.LRUCache`. This class mimics the interface of a normal ``dict``, but starts to remove the least-recently used entries once a threshold is reached. The query cache is updated on every search operation. When writing data, the whole cache is discarded as the query results may have changed. .. admonition:: Customization For customization, the following class variables can be set: - ``document_class`` defines the class that is used to represent documents, - ``document_id_class`` defines the class that is used to represent document IDs, - ``query_cache_class`` defines the class that is used for the query cache - ``default_query_cache_capacity`` defines the default capacity of the query cache .. versionadded:: 4.0 :param storage: The storage instance to use for this table :param name: The table name :param cache_size: Maximum capacity of query cache :param persist_empty: Store new table even with no operations on it """ #: The class used to represent documents #: #: .. versionadded:: 4.0 document_class = Document #: The class used to represent a document ID #: #: .. versionadded:: 4.0 document_id_class = int #: The class used for caching query results #: #: .. versionadded:: 4.0 query_cache_class = LRUCache #: The default capacity of the query cache #: #: .. versionadded:: 4.0 default_query_cache_capacity = 10 def __init__( self, storage: Storage, name: str, cache_size: int = default_query_cache_capacity, persist_empty: bool = False ): """ Create a table instance. """ self._storage = storage self._name = name self._query_cache: LRUCache[QueryLike, List[Document]] \ = self.query_cache_class(capacity=cache_size) self._next_id = None if persist_empty: self._update_table(lambda table: table.clear()) def __repr__(self): args = [ 'name={!r}'.format(self.name), 'total={}'.format(len(self)), 'storage={}'.format(self._storage), ] return '<{} {}>'.format(type(self).__name__, ', '.join(args)) @property def name(self) -> str: """ Get the table name. """ return self._name @property def storage(self) -> Storage: """ Get the table storage instance. """ return self._storage def insert(self, document: Mapping) -> int: """ Insert a new document into the table. :param document: the document to insert :returns: the inserted document's ID """ # Make sure the document implements the ``Mapping`` interface if not isinstance(document, Mapping): raise ValueError('Document is not a Mapping') # First, we get the document ID for the new document if isinstance(document, self.document_class): # For a `Document` object we use the specified ID doc_id = document.doc_id # We also reset the stored next ID so the next insert won't # re-use document IDs by accident when storing an old value self._next_id = None else: # In all other cases we use the next free ID doc_id = self._get_next_id() # Now, we update the table and add the document def updater(table: dict): if doc_id in table: raise ValueError(f'Document with ID {str(doc_id)} ' f'already exists') # By calling ``dict(document)`` we convert the data we got to a # ``dict`` instance even if it was a different class that # implemented the ``Mapping`` interface table[doc_id] = dict(document) # See below for details on ``Table._update`` self._update_table(updater) return doc_id def insert_multiple(self, documents: Iterable[Mapping]) -> List[int]: """ Insert multiple documents into the table. :param documents: an Iterable of documents to insert :returns: a list containing the inserted documents' IDs """ doc_ids = [] def updater(table: dict): for document in documents: # Make sure the document implements the ``Mapping`` interface if not isinstance(document, Mapping): raise ValueError('Document is not a Mapping') if isinstance(document, self.document_class): # Check if document does not override an existing document if document.doc_id in table: raise ValueError( f'Document with ID {str(document.doc_id)} ' f'already exists' ) # Store the doc_id, so we can return all document IDs # later. Then save the document with its doc_id and # skip the rest of the current loop doc_id = document.doc_id doc_ids.append(doc_id) table[doc_id] = dict(document) continue # Generate new document ID for this document # Store the doc_id, so we can return all document IDs # later, then save the document with the new doc_id doc_id = self._get_next_id() doc_ids.append(doc_id) table[doc_id] = dict(document) # See below for details on ``Table._update`` self._update_table(updater) return doc_ids def all(self) -> List[Document]: """ Get all documents stored in the table. :returns: a list with all documents. """ # iter(self) (implemented in Table.__iter__ provides an iterator # that returns all documents in this table. We use it to get a list # of all documents by using the ``list`` constructor to perform the # conversion. return list(iter(self)) def search(self, cond: QueryLike) -> List[Document]: """ Search for all documents matching a 'where' cond. :param cond: the condition to check against :returns: list of matching documents """ # First, we check the query cache to see if it has results for this # query cached_results = self._query_cache.get(cond) if cached_results is not None: return cached_results[:] # Perform the search by applying the query to all documents. # Then, only if the document matches the query, convert it # to the document class and document ID class. docs = [ self.document_class(doc, self.document_id_class(doc_id)) for doc_id, doc in self._read_table().items() if cond(doc) ] # Only cache cacheable queries. # # This weird `getattr` dance is needed to make MyPy happy as # it doesn't know that a query might have a `is_cacheable` method # that is not declared in the `QueryLike` protocol due to it being # optional. # See: https://github.com/python/mypy/issues/1424 # # Note also that by default we expect custom query objects to be # cacheable (which means they need to have a stable hash value). # This is to keep consistency with TinyDB's behavior before # `is_cacheable` was introduced which assumed that all queries # are cacheable. is_cacheable: Callable[[], bool] = getattr(cond, 'is_cacheable', lambda: True) if is_cacheable(): # Update the query cache self._query_cache[cond] = docs[:] return docs @overload def get(self) -> NoReturn: ... @overload def get( self, cond: QueryLike, doc_id: None = ..., doc_ids: None = ... ) -> Optional[Document]: ... @overload def get( self, *, cond: QueryLike, doc_id: None = ..., doc_ids: None = ... ) -> Optional[Document]: ... @overload def get( self, cond: Optional[QueryLike], doc_id: int, doc_ids: Optional[List] = ... ) -> Optional[Document]: ... @overload def get( self, *, cond: Optional[QueryLike] = ..., doc_id: int, doc_ids: Optional[List] = ..., ) -> Optional[Document]: ... @overload def get( self, cond: Optional[QueryLike], doc_id: None, doc_ids: List ) -> List[Document]: ... @overload def get( self, cond: Optional[QueryLike], *, doc_id: None = ..., doc_ids: List ) -> List[Document]: ... @overload def get( self, *, cond: Optional[QueryLike] = ..., doc_id: None = ..., doc_ids: List ) -> List[Document]: ... def get( self, cond: Optional[QueryLike] = None, doc_id: Optional[int] = None, doc_ids: Optional[List] = None ): """ Get exactly one document specified by a query or a document ID. However, if multiple document IDs are given then returns all documents in a list. Returns ``None`` if the document doesn't exist. :param cond: the condition to check against :param doc_id: the document's ID :param doc_ids: the document's IDs(multiple) :returns: the document(s) or ``None`` """ table = self._read_table() if doc_id is not None: # Retrieve a document specified by its ID raw_doc = table.get(str(doc_id), None) if raw_doc is None: return None # Convert the raw data to the document class return self.document_class(raw_doc, doc_id) elif doc_ids is not None: # Filter the table by extracting out all those documents which # have doc id specified in the doc_id list. # Since document IDs will be unique, we make it a set to ensure # constant time lookup doc_ids_set = set(str(doc_id) for doc_id in doc_ids) # Now return the filtered documents in form of list return [ self.document_class(doc, self.document_id_class(doc_id)) for doc_id, doc in table.items() if doc_id in doc_ids_set ] elif cond is not None: # Find a document specified by a query # The trailing underscore in doc_id_ is needed so MyPy # doesn't think that `doc_id_` (which is a string) needs # to have the same type as `doc_id` which is this function's # parameter and is an optional `int`. for doc_id_, doc in self._read_table().items(): if cond(doc): return self.document_class( doc, self.document_id_class(doc_id_) ) return None raise RuntimeError('You have to pass either cond or doc_id or doc_ids') def contains( self, cond: Optional[QueryLike] = None, doc_id: Optional[int] = None ) -> bool: """ Check whether the database contains a document matching a query or an ID. If ``doc_id`` is set, it checks if the db contains the specified ID. :param cond: the condition use :param doc_id: the document ID to look for """ if doc_id is not None: # Documents specified by ID return self.get(doc_id=doc_id) is not None elif cond is not None: # Document specified by condition return self.get(cond) is not None raise RuntimeError('You have to pass either cond or doc_id') def update( self, fields: Union[Mapping, Callable[[Mapping], None]], cond: Optional[QueryLike] = None, doc_ids: Optional[Iterable[int]] = None, ) -> List[int]: """ Update all matching documents to have a given set of fields. :param fields: the fields that the matching documents will have or a method that will update the documents :param cond: which documents to update :param doc_ids: a list of document IDs :returns: a list containing the updated document's ID """ # Define the function that will perform the update if callable(fields): def perform_update(table, doc_id): # Update documents by calling the update function provided by # the user fields(table[doc_id]) else: def perform_update(table, doc_id): # Update documents by setting all fields from the provided data table[doc_id].update(fields) if doc_ids is not None: # Perform the update operation for documents specified by a list # of document IDs updated_ids = list(doc_ids) def updater(table: dict): # Call the processing callback with all document IDs for doc_id in updated_ids: perform_update(table, doc_id) # Perform the update operation (see _update_table for details) self._update_table(updater) return updated_ids elif cond is not None: # Perform the update operation for documents specified by a query # Collect affected doc_ids updated_ids = [] def updater(table: dict): _cond = cast(QueryLike, cond) # We need to convert the keys iterator to a list because # we may remove entries from the ``table`` dict during # iteration and doing this without the list conversion would # result in an exception (RuntimeError: dictionary changed size # during iteration) for doc_id in list(table.keys()): # Pass through all documents to find documents matching the # query. Call the processing callback with the document ID if _cond(table[doc_id]): # Add ID to list of updated documents updated_ids.append(doc_id) # Perform the update (see above) perform_update(table, doc_id) # Perform the update operation (see _update_table for details) self._update_table(updater) return updated_ids else: # Update all documents unconditionally updated_ids = [] def updater(table: dict): # Process all documents for doc_id in list(table.keys()): # Add ID to list of updated documents updated_ids.append(doc_id) # Perform the update (see above) perform_update(table, doc_id) # Perform the update operation (see _update_table for details) self._update_table(updater) return updated_ids def update_multiple( self, updates: Iterable[ Tuple[Union[Mapping, Callable[[Mapping], None]], QueryLike] ], ) -> List[int]: """ Update all matching documents to have a given set of fields. :returns: a list containing the updated document's ID """ # Define the function that will perform the update def perform_update(fields, table, doc_id): if callable(fields): # Update documents by calling the update function provided # by the user fields(table[doc_id]) else: # Update documents by setting all fields from the provided # data table[doc_id].update(fields) # Perform the update operation for documents specified by a query # Collect affected doc_ids updated_ids = [] def updater(table: dict): # We need to convert the keys iterator to a list because # we may remove entries from the ``table`` dict during # iteration and doing this without the list conversion would # result in an exception (RuntimeError: dictionary changed size # during iteration) for doc_id in list(table.keys()): for fields, cond in updates: _cond = cast(QueryLike, cond) # Pass through all documents to find documents matching the # query. Call the processing callback with the document ID if _cond(table[doc_id]): # Add ID to list of updated documents updated_ids.append(doc_id) # Perform the update (see above) perform_update(fields, table, doc_id) # Perform the update operation (see _update_table for details) self._update_table(updater) return updated_ids def upsert(self, document: Mapping, cond: Optional[QueryLike] = None) -> List[int]: """ Update documents, if they exist, insert them otherwise. Note: This will update *all* documents matching the query. Document argument can be a tinydb.table.Document object if you want to specify a doc_id. :param document: the document to insert or the fields to update :param cond: which document to look for, optional if you've passed a Document with a doc_id :returns: a list containing the updated documents' IDs """ # Extract doc_id if isinstance(document, self.document_class) and hasattr(document, 'doc_id'): doc_ids: Optional[List[int]] = [document.doc_id] else: doc_ids = None # Make sure we can actually find a matching document if doc_ids is None and cond is None: raise ValueError("If you don't specify a search query, you must " "specify a doc_id. Hint: use a table.Document " "object.") # Perform the update operation try: updated_docs: Optional[List[int]] = self.update(document, cond, doc_ids) except KeyError: # This happens when a doc_id is specified, but it's missing updated_docs = None # If documents have been updated: return their IDs if updated_docs: return updated_docs # There are no documents that match the specified query -> insert the # data as a new document return [self.insert(document)] def remove( self, cond: Optional[QueryLike] = None, doc_ids: Optional[Iterable[int]] = None, ) -> List[int]: """ Remove all matching documents. :param cond: the condition to check against :param doc_ids: a list of document IDs :returns: a list containing the removed documents' ID """ if doc_ids is not None: # This function returns the list of IDs for the documents that have # been removed. When removing documents identified by a set of # document IDs, it's this list of document IDs we need to return # later. # We convert the document ID iterator into a list, so we can both # use the document IDs to remove the specified documents and # to return the list of affected document IDs removed_ids = list(doc_ids) def updater(table: dict): for doc_id in removed_ids: table.pop(doc_id) # Perform the remove operation self._update_table(updater) return removed_ids if cond is not None: removed_ids = [] # This updater function will be called with the table data # as its first argument. See ``Table._update`` for details on this # operation def updater(table: dict): # We need to convince MyPy (the static type checker) that # the ``cond is not None`` invariant still holds true when # the updater function is called _cond = cast(QueryLike, cond) # We need to convert the keys iterator to a list because # we may remove entries from the ``table`` dict during # iteration and doing this without the list conversion would # result in an exception (RuntimeError: dictionary changed size # during iteration) for doc_id in list(table.keys()): if _cond(table[doc_id]): # Add document ID to list of removed document IDs removed_ids.append(doc_id) # Remove document from the table table.pop(doc_id) # Perform the remove operation self._update_table(updater) return removed_ids raise RuntimeError('Use truncate() to remove all documents') def truncate(self) -> None: """ Truncate the table by removing all documents. """ # Update the table by resetting all data self._update_table(lambda table: table.clear()) # Reset document ID counter self._next_id = None def count(self, cond: QueryLike) -> int: """ Count the documents matching a query. :param cond: the condition use """ return len(self.search(cond)) def clear_cache(self) -> None: """ Clear the query cache. """ self._query_cache.clear() def __len__(self): """ Count the total number of documents in this table. """ return len(self._read_table()) def __iter__(self) -> Iterator[Document]: """ Iterate over all documents stored in the table. :returns: an iterator over all documents. """ # Iterate all documents and their IDs for doc_id, doc in self._read_table().items(): # Convert documents to the document class yield self.document_class(doc, self.document_id_class(doc_id)) def _get_next_id(self): """ Return the ID for a newly inserted document. """ # If we already know the next ID if self._next_id is not None: next_id = self._next_id self._next_id = next_id + 1 return next_id # Determine the next document ID by finding out the max ID value # of the current table documents # Read the table documents table = self._read_table() # If the table is empty, set the initial ID if not table: next_id = 1 self._next_id = next_id + 1 return next_id # Determine the next ID based on the maximum ID that's currently in use max_id = max(self.document_id_class(i) for i in table.keys()) next_id = max_id + 1 # The next ID we will return AFTER this call needs to be larger than # the current next ID we calculated self._next_id = next_id + 1 return next_id def _read_table(self) -> Dict[str, Mapping]: """ Read the table data from the underlying storage. Documents and doc_ids are NOT yet transformed, as we may not want to convert *all* documents when returning only one document for example. """ # Retrieve the tables from the storage tables = self._storage.read() if tables is None: # The database is empty return {} # Retrieve the current table's data try: table = tables[self.name] except KeyError: # The table does not exist yet, so it is empty return {} return table def _update_table(self, updater: Callable[[Dict[int, Mapping]], None]): """ Perform a table update operation. The storage interface used by TinyDB only allows to read/write the complete database data, but not modifying only portions of it. Thus, to only update portions of the table data, we first perform a read operation, perform the update on the table data and then write the updated data back to the storage. As a further optimization, we don't convert the documents into the document class, as the table data will *not* be returned to the user. """ tables = self._storage.read() if tables is None: # The database is empty tables = {} try: raw_table = tables[self.name] except KeyError: # The table does not exist yet, so it is empty raw_table = {} # Convert the document IDs to the document ID class. # This is required as the rest of TinyDB expects the document IDs # to be an instance of ``self.document_id_class`` but the storage # might convert dict keys to strings. table = { self.document_id_class(doc_id): doc for doc_id, doc in raw_table.items() } # Perform the table update operation updater(table) # Convert the document IDs back to strings. # This is required as some storages (most notably the JSON file format) # don't support IDs other than strings. tables[self.name] = { str(doc_id): doc for doc_id, doc in table.items() } # Write the newly updated data back to the storage self._storage.write(tables) # Clear the query cache, as the table contents have changed self.clear_cache() ================================================ FILE: tinydb/utils.py ================================================ """ Utility functions. """ from collections import OrderedDict, abc from typing import List, Iterator, TypeVar, Generic, Union, Optional, Type, \ TYPE_CHECKING K = TypeVar('K') V = TypeVar('V') D = TypeVar('D') T = TypeVar('T') __all__ = ('LRUCache', 'freeze', 'with_typehint') def with_typehint(baseclass: Type[T]): """ Add type hints from a specified class to a base class: >>> class Foo(with_typehint(Bar)): ... pass This would add type hints from class ``Bar`` to class ``Foo``. Note that while PyCharm and Pyright (for VS Code) understand this pattern, MyPy does not. For that reason TinyDB has a MyPy plugin in ``mypy_plugin.py`` that adds support for this pattern. """ if TYPE_CHECKING: # In the case of type checking: pretend that the target class inherits # from the specified base class return baseclass # Otherwise: just inherit from `object` like a regular Python class return object class LRUCache(abc.MutableMapping, Generic[K, V]): """ A least-recently used (LRU) cache with a fixed cache size. This class acts as a dictionary but has a limited size. If the number of entries in the cache exceeds the cache size, the least-recently accessed entry will be discarded. This is implemented using an ``OrderedDict``. On every access the accessed entry is moved to the front by re-inserting it into the ``OrderedDict``. When adding an entry and the cache size is exceeded, the last entry will be discarded. """ def __init__(self, capacity=None) -> None: self.capacity = capacity self.cache: OrderedDict[K, V] = OrderedDict() @property def lru(self) -> List[K]: return list(self.cache.keys()) @property def length(self) -> int: return len(self.cache) def clear(self) -> None: self.cache.clear() def __len__(self) -> int: return self.length def __contains__(self, key: object) -> bool: return key in self.cache def __setitem__(self, key: K, value: V) -> None: self.set(key, value) def __delitem__(self, key: K) -> None: del self.cache[key] def __getitem__(self, key) -> V: value = self.get(key) if value is None: raise KeyError(key) return value def __iter__(self) -> Iterator[K]: return iter(self.cache) def get(self, key: K, default: Optional[D] = None) -> Optional[Union[V, D]]: value = self.cache.get(key) if value is not None: self.cache.move_to_end(key, last=True) return value return default def set(self, key: K, value: V): if key in self.cache: self.cache[key] = value self.cache.move_to_end(key, last=True) else: self.cache[key] = value # Check, if the cache is full and we have to remove old items # If the queue is of unlimited size, self.capacity is NaN and # x > NaN is always False in Python and the cache won't be cleared. if self.capacity is not None and self.length > self.capacity: self.cache.popitem(last=False) class FrozenDict(dict): """ An immutable dictionary. This is used to generate stable hashes for queries that contain dicts. Usually, Python dicts are not hashable because they are mutable. This class removes the mutability and implements the ``__hash__`` method. """ def __hash__(self): # Calculate the has by hashing a tuple of all dict items return hash(tuple(sorted(self.items()))) def _immutable(self, *args, **kws): raise TypeError('object is immutable') # Disable write access to the dict __setitem__ = _immutable __delitem__ = _immutable clear = _immutable setdefault = _immutable # type: ignore popitem = _immutable def update(self, e=None, **f): raise TypeError('object is immutable') def pop(self, k, d=None): raise TypeError('object is immutable') def freeze(obj): """ Freeze an object by making it immutable and thus hashable. """ if isinstance(obj, dict): # Transform dicts into ``FrozenDict``s return FrozenDict((k, freeze(v)) for k, v in obj.items()) elif isinstance(obj, list): # Transform lists into tuples return tuple(freeze(el) for el in obj) elif isinstance(obj, set): # Transform sets into ``frozenset``s return frozenset(obj) else: # Don't handle all other objects return obj ================================================ FILE: tinydb/version.py ================================================ __version__ = '4.8.2'