[ { "path": ".github/workflows/publish-dev.yml", "content": "# This workflows will upload a Python Package using Twine when a release is created\n# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries\n\nname: publish dev\n\non:\n push:\n branches:\n - develop\n\njobs:\n deploy:\n\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v2\n - name: Set up Python\n uses: actions/setup-python@v2\n with:\n python-version: '3.x'\n - name: Install dependencies\n run: |\n python -m pip install --upgrade pip\n pip install setuptools wheel twine\n - name: set dev version\n run: |\n sed -i \"s/^\\(__version__.*\\)'/\\1.dev.${{github.run_number}}'/g\" bulk_update_or_create/__version__.py\n grep dev bulk_update_or_create/__version__.py\n - name: Build and publish\n env:\n TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}\n TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}\n run: |\n python setup.py sdist bdist_wheel\n twine upload dist/*\n" }, { "path": ".github/workflows/publish.yml", "content": "# This workflows will upload a Python Package using Twine when a release is created\n# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries\n\nname: publish\n\non:\n release:\n types: [created]\n\njobs:\n deploy:\n\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v2\n - name: Set up Python\n uses: actions/setup-python@v2\n with:\n python-version: '3.x'\n - name: Install dependencies\n run: |\n python -m pip install --upgrade pip\n pip install setuptools wheel twine\n - name: Build and publish\n env:\n TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}\n TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}\n run: |\n python setup.py sdist bdist_wheel\n twine upload dist/*\n" }, { "path": ".github/workflows/test.yml", "content": "# This workflow will install Python dependencies, run tests and lint with a variety of Python versions\n# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions\n\nname: tests\n\non:\n push:\n branches: [ main ]\n pull_request:\n branches: [ main ]\n\njobs:\n style:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v2\n - name: Set up Python 3.6\n uses: actions/setup-python@v2\n with:\n python-version: 3.6\n - name: Install tox\n run: pip install tox\n - name: Style check\n run : tox -e style\n\n test:\n runs-on: ubuntu-latest\n strategy:\n matrix:\n python-version: [3.6, 3.7, 3.8, 3.9]\n database: [sqlite, mysql, postgresql]\n\n services:\n mysql:\n image: mysql:5\n env:\n MYSQL_ROOT_PASSWORD: root\n ports:\n - 8877:3306\n # needed because the container does not provide a healthcheck\n options: --health-cmd \"mysqladmin ping\" --health-interval 10s --health-timeout 5s --health-retries=5\n postgres:\n image: postgres:10\n env:\n POSTGRES_USER: postgres\n POSTGRES_PASSWORD: postgres\n POSTGRES_DB: postgres\n ports:\n - 8878:5432\n # needed because the postgres container does not provide a healthcheck\n options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5\n\n steps:\n - uses: actions/checkout@v2\n - name: Set up Python ${{ matrix.python-version }}\n uses: actions/setup-python@v2\n with:\n python-version: ${{ matrix.python-version }}\n\n - name: Install tox\n run: pip install tox\n - name: Toxit\n run: tox -e py-${{ matrix.database }} -v\n\n - name: coverage xml\n run: .tox/py-mysql/bin/coverage xml\n if: ${{ matrix.python-version == 3.7 && matrix.database == 'mysql' }}\n - uses: codecov/codecov-action@v1\n with:\n fail_ci_if_error: true\n if: ${{ matrix.python-version == 3.7 && matrix.database == 'mysql' }}\n" }, { "path": ".gitignore", "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\npip-wheel-metadata/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n*.py,cover\n.hypothesis/\n.pytest_cache/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\ndb.sqlite3-journal\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# IPython\nprofile_default/\nipython_config.py\n\n# pyenv\n.python-version\n\n# pipenv\n# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.\n# However, in case of collaboration, if having platform-specific dependencies or dependencies\n# having no cross-platform support, pipenv may install dependencies that don't work, or not\n# install all needed dependencies.\n#Pipfile.lock\n\n# PEP 582; used by e.g. github.com/David-OConnor/pyflow\n__pypackages__/\n\n# Celery stuff\ncelerybeat-schedule\ncelerybeat.pid\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n.dmypy.json\ndmypy.json\n\n# Pyre type checker\n.pyre/\n.vscode\n.DS_Store\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2020 Filipe Pina\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "Makefile", "content": ".PHONY: flake8 test coverage style style_check\n\nstyle:\n\tblack --target-version=py36 \\\n\t --line-length=120 \\\n\t\t --skip-string-normalization \\\n\t\t bulk_update_or_create tests setup.py\n\tflake8 bulk_update_or_create tests\n\nstyle_check: flake8\n\tblack --target-version=py36 \\\n\t --line-length=120 \\\n\t\t --skip-string-normalization \\\n\t\t --check \\\n\t\t bulk_update_or_create tests setup.py\n\nflake8:\n\tflake8 bulk_update_or_create tests\n\nstartmysql:\n\t@docker inspect django-bulk_update_or_create-mysql | grep -q '\"Running\": true' || \\\n\t\tdocker run --name django-bulk_update_or_create-mysql \\\n\t\t -e MYSQL_ROOT_PASSWORD=root \\\n\t\t --rm -p 8877:3306 -d \\\n\t\t\t\t --health-cmd \"mysqladmin ping\" \\\n\t\t\t\t --health-interval 10s \\\n\t\t\t\t --health-timeout 5s \\\n\t\t\t\t --health-retries=5 \\\n\t\t\t\t mysql:5 # TODO: wait for healthy\n\nstartpg:\n\t@docker inspect django-bulk_update_or_create-pg | grep -q '\"Running\": true' || \\\n\t\tdocker run --name django-bulk_update_or_create-pg \\\n\t\t -e POSTGRES_USER=postgres \\\n \t\t -e POSTGRES_PASSWORD=postgres \\\n\t\t\t\t -e POSTGRES_DB=postgres \\\n\t\t --rm -p 8878:5432 -d \\\n\t\t\t\t --health-cmd pg_isready \\\n\t\t\t\t --health-interval 10s \\\n\t\t\t\t --health-timeout 5s \\\n\t\t\t\t --health-retries 5 \\\n\t\t\t\t postgres:10 # TODO: wait for healthy\n\ntest: startmysql\n\tDJANGO_SETTINGS_MODULE=settings_mysql \\\n\t\ttests/manage.py test $${TEST_ARGS:-tests}\n\ntestpg: startpg\n\tDJANGO_SETTINGS_MODULE=settings_postgresql \\\n\t\ttests/manage.py test $${TEST_ARGS:-tests}\n\ntestcmd: startpg startmysql\n\t# default - sqlite\n\tDJANGO_SETTINGS_MODULE=settings tests/manage.py migrate\n\tDJANGO_SETTINGS_MODULE=settings tests/manage.py bulk_it\n\n\t# mysql\n\tDJANGO_SETTINGS_MODULE=settings_mysql tests/manage.py migrate\n\tDJANGO_SETTINGS_MODULE=settings_mysql tests/manage.py bulk_it\n\t\n\t# postgres\n\tDJANGO_SETTINGS_MODULE=settings_postgresql tests/manage.py migrate\n\tDJANGO_SETTINGS_MODULE=settings_postgresql tests/manage.py bulk_it\n\ncoverage:\n\tPYTHONPATH=\"tests\" \\\n\t\tpython -b -W always -m coverage run tests/manage.py test $${TEST_ARGS:-tests}\n\tcoverage report\n" }, { "path": "README.md", "content": "# django-bulk-update-or-create\n\n\n[![tests](https://github.com/fopina/django-bulk-update-or-create/workflows/tests/badge.svg)](https://github.com/fopina/django-bulk-update-or-create/actions?query=workflow%3Atests)\n[![Test coverage status](https://codecov.io/gh/fopina/django-bulk-update-or-create/branch/main/graph/badge.svg)](https://codecov.io/gh/fopina/django-bulk-update-or-create)\n[![Current version on PyPi](https://img.shields.io/pypi/v/django-bulk-update-or-create)](https://pypi.org/project/django-bulk-update-or-create/)\n[![monthly downloads](https://img.shields.io/pypi/dm/django-bulk-update-or-create)](https://pypi.org/project/django-bulk-update-or-create/)\n![PyPI - Python Version](https://img.shields.io/pypi/pyversions/django-bulk-update-or-create)\n![PyPI - Django Version](https://img.shields.io/pypi/djversions/django-bulk-update-or-create)\n\n\nEveryone using Django ORM will eventually find himself doing batch `update_or_create` operations: ingest files from external sources, sync with external APIs, etc.\n\nIf the number of records is big, the slowliness of `QuerySet.update_or_create` will stand out: it is very practical to use but it always does one `SELECT` and then one `INSERT` (if select didn't return anything) or `UPDATE`/`.save` (if it did).\n\nSearching online shows that this does indeed happen to quite a few people though it doesn't seem to be any good solution:\n\n* `bulk_create` is really fast if you know all records are new (and you're not using multi-table inheritance)\n* `bulk_update` does some nice voodoo to update several records with the same `UPDATE` statement (using a huge `WHERE` condition together with `CASE`), but you need to be sure they all exist\n* UPSERTs [(INSERT .. ON DUPLICATE KEY UPDATE](https://dev.mysql.com/doc/refman/8.0/en/insert-on-duplicate.html)) look interesting (TODO on different package) but they will be retricted by `bulk_create` limitations ==> cannot use on models with multi-table inheritance\n\nThis package tries to tackle this introducing `bulk_update_or_create` to model QuerySet/Manager:\n* `update_or_create`: `(1 SELECT + 1 INSERT/UPDATE) * N`\n* `bulk_update_or_create`: `1 BIG_SELECT + 1 BIG_UPDATE + (lte_N) INSERT`\n\nFor a batch of records:\n\n* `SELECT` all from database (based on the `match_field` parameter)\n* Update records in memory\n* Use `bulk_update` for those\n* Use `INSERT`/`.create` on each of the remaining\n\nThe (*SOFTCORE*) [performance test](tests/tests/management/commands/bulk_it.py) looks promising, more than 70% less time (average):\n\n```shell\n$ make testcmd\n# default - sqlite\nDJANGO_SETTINGS_MODULE=settings tests/manage.py bulk_it\nloop of update_or_create - all creates: 3.966486692428589\nloop of update_or_create - all updates: 4.020653247833252\nloop of update_or_create - half half: 3.9968857765197754\nbulk_update_or_create - all creates: 2.949239730834961\nbulk_update_or_create - all updates: 0.15633511543273926\nbulk_update_or_create - half half: 1.4585723876953125\n# mysql\nDJANGO_SETTINGS_MODULE=settings_mysql tests/manage.py bulk_it\nloop of update_or_create - all creates: 5.511938571929932\nloop of update_or_create - all updates: 5.321666955947876\nloop of update_or_create - half half: 5.391834735870361\nbulk_update_or_create - all creates: 1.5671980381011963\nbulk_update_or_create - all updates: 0.14612770080566406\nbulk_update_or_create - half half: 0.7262606620788574\n# postgres\nDJANGO_SETTINGS_MODULE=settings_postgresql tests/manage.py bulk_it\nloop of update_or_create - all creates: 4.3584535121917725\nloop of update_or_create - all updates: 3.6183276176452637\nloop of update_or_create - half half: 4.145816087722778\nbulk_update_or_create - all creates: 1.044851541519165\nbulk_update_or_create - all updates: 0.14954638481140137\nbulk_update_or_create - half half: 0.8407495021820068\n```\n\nInstallation\n============\n\n```\npip install django-bulk-update-or-create\n```\n\n```py\nINSTALLED_APPS = [\n ...\n 'bulk_update_or_create',\n ...\n]\n```\n\nUsage\n=====\n\n* use `BulkUpdateOrCreateQuerySet` as manager of your model(s)\n\n```python\nfrom django.db import models\nfrom bulk_update_or_create import BulkUpdateOrCreateQuerySet\n\n\nclass RandomData(models.Model):\n objects = BulkUpdateOrCreateQuerySet.as_manager()\n\n uuid = models.IntegerField(unique=True)\n data = models.CharField(max_length=200, null=True, blank=True)\n```\n\n* call `bulk_update_or_create`\n\n```python\nitems = [\n RandomData(uuid=1, data='data for 1'),\n RandomData(uuid=2, data='data for 2'),\n]\nRandomData.objects.bulk_update_or_create(items, ['data'], match_field='uuid')\n```\n\n* or use the context manager, if you are updating a big number of items, as it manages a batch queue\n\n```python\nwith RandomData.objects.bulk_update_or_create_context(['data'], match_field='uuid', batch_size=10) as bulkit:\n for i in range(10000):\n bulkit.queue(RandomData(uuid=i, data=i + 20))\n```\n\n`bulk_update_or_create` supports `yield_objects=True` so you can iterate over the created/updated objects. \n`bulk_update_or_create_context` provides the same information to the callback function specified as `status_cb`\n\nDocs\n====\n\nWIP\n\nToDo\n====\n\n* [ ] Docs!\n* [ ] Add option to use `bulk_create` for creates: assert model is not multi-table, if enabled\n* [ ] Fix the collation mess: the keyword arg `case_insensitive_match` should be dropped and collation detected in runtime\n* [x] Add support for multiple `match_field` - probably will need to use `WHERE (K1=X and K2=Y) or (K1=.. and K2\n=..)` instead of `IN` for those, as that SQL standard doesn't seem widely adopted yet\n* [ ] Link to `UPSERT` alternative package once done!\n" }, { "path": "bulk_update_or_create/__init__.py", "content": "from .__version__ import __version__\n\nfrom .query import BulkUpdateOrCreateQuerySet, BulkUpdateOrCreateMixin\n\n__all__ = ['BulkUpdateOrCreateQuerySet', 'BulkUpdateOrCreateMixin']\n\n\ndefault_app_config = 'bulk_update_or_create.apps.BulkUpdateOrCreateConfig'\n" }, { "path": "bulk_update_or_create/__version__.py", "content": "__version__ = '1.0.0'\n" }, { "path": "bulk_update_or_create/apps.py", "content": "from django.apps import AppConfig\n\n\nclass BulkUpdateOrCreateConfig(AppConfig):\n name = 'bulk_update_or_create'\n" }, { "path": "bulk_update_or_create/query.py", "content": "from types import TracebackType\nfrom typing import Any, Callable, Generator, List, Optional, Tuple, Type, Union\n\nfrom django.db import models\nfrom django.db.models import Model, QuerySet\n\n\nclass BulkUpdateOrCreateMixin:\n def bulk_update_or_create_context(\n self,\n update_fields: List[str],\n match_field: str = 'pk',\n batch_size: int = 100,\n case_insensitive_match: bool = False,\n status_cb: Optional[\n Callable[[Tuple[List[Model], List[Model]]], Any]\n ] = None,\n ):\n \"\"\"\n Helper method that returns a context manager (_BulkUpdateOrCreateContextManager) that makes it easier to handle\n a stream of objects with unknown size.\n Call `.queue(obj)` and whenever `batch_size` is reached or the context terminates, this context manager will\n call `bulk_update_or_create` on the queue\n\n :param update_fields: fields that will be updated if record already exists (passed on to bulk_update)\n :param match_field: model field that will match existing records (defaults to \"pk\")\n :param batch_size: number of records to process in each batch (defaults to 100)\n :param case_insensitive_match: set to True if using MySQL with \"ci\" collations (defaults to False)\n :param status_cb: if set to a callable, status_cb is called a tuple of lists with ([created],\n [updated]) objects as they're yielded\n \"\"\"\n return _BulkUpdateOrCreateContextManager(\n self,\n update_fields,\n batch_size=batch_size,\n status_cb=status_cb,\n match_field=match_field,\n case_insensitive_match=case_insensitive_match,\n )\n\n def bulk_update_or_create(\n self,\n objs: List[Model],\n update_fields: List[str],\n match_field: str = 'pk',\n batch_size: int = 100,\n case_insensitive_match: bool = False,\n yield_objects: bool = False,\n ) -> Union[\n Generator[Tuple[List[Model], List[Model]], None, None],\n List[Tuple[List[Model], List[Model]]]\n ]:\n \"\"\"\n\n :param objs: model instances to be updated or created\n :param update_fields: fields that will be updated if record already exists (passed on to bulk_update)\n :param match_field: model fields that will match existing records (defaults to [\"pk\"])\n :param batch_size: number of records to process in each batch (defaults to len(objs))\n :param case_insensitive_match: set to True if using MySQL with \"ci\" collations (defaults to False)\n :param yield_objects: if True, method becomes a generator that will yield a tuple of lists\n with ([created], [updated]) objects. This is one tuple per each `batch`. If this is False,\n a single tuple of lists with ([created], [updated]) will be returned.\n \"\"\"\n\n r = self.__bulk_update_or_create(\n objs,\n update_fields,\n match_field,\n batch_size,\n case_insensitive_match,\n yield_objects,\n )\n if yield_objects:\n return r\n return list(r)\n\n def __bulk_update_or_create_inner_methods(self, match_fields, case_insensitive_match):\n single_match_field = len(match_fields) == 1\n\n def _obj_key_getter_sensitive(obj):\n # use to_python to coerce value same way it's done when fetched from DB\n # https://github.com/fopina/django-bulk-update-or-create/issues/11\n # k = _match_field.to_python(_match_field.value_from_object(obj))\n return tuple(match_field.to_python(match_field.value_from_object(obj)) for match_field in match_fields)\n\n _obj_key_getter = _obj_key_getter_sensitive\n\n if case_insensitive_match:\n\n def _obj_key_getter(obj):\n return tuple(\n map(\n lambda v: v.lower() if hasattr(v, 'lower') else v,\n _obj_key_getter_sensitive(obj),\n )\n )\n\n if single_match_field:\n\n def _obj_filter(obj_map):\n return models.Q(**{f'{match_fields[0].name}__in': obj_map.keys()})\n\n def _obj_key_getter_single(obj):\n return _obj_key_getter(obj)[0]\n\n return _obj_key_getter_single, _obj_filter\n else:\n\n def _obj_filter(obj_map):\n return models.Q(\n *(\n models.Q(**{k.name: obj_key[i] for i, k in enumerate(match_fields)})\n for obj_key in obj_map.keys()\n ),\n _connector=models.Q.OR,\n )\n\n return _obj_key_getter, _obj_filter\n\n def __bulk_update_or_create(\n self,\n objs: List[Model],\n update_fields: List[str],\n match_field: str = 'pk',\n batch_size: Optional[int] = None,\n case_insensitive_match: bool = False,\n yield_objects: bool = False,\n ) -> Union[\n Generator[Tuple[List[Model], List[Model]], None, None],\n None\n ]:\n # validations like bulk_update\n if batch_size is not None and batch_size < 0:\n raise ValueError('Batch size must be a positive integer.')\n if not update_fields:\n raise ValueError('update_fields cannot be empty')\n match_field = (match_field,) if isinstance(match_field, str) else match_field\n _match_fields = [self.model._meta.get_field(name) for name in match_field]\n _update_fields = [self.model._meta.get_field(name) for name in update_fields]\n if any(not f.concrete or f.many_to_many for f in _update_fields):\n raise ValueError('bulk_update_or_create() can only be used with concrete fields.')\n if any(f.primary_key for f in _update_fields):\n raise ValueError('bulk_update_or_create() cannot be used with primary key fields.')\n\n # generators not supported (for now?), as bulk_update doesn't either\n objs = list(objs)\n if not objs:\n return\n\n if batch_size is None:\n batch_size = len(objs)\n\n batches = (objs[i : i + batch_size] for i in range(0, len(objs), batch_size))\n\n _obj_key_getter, _obj_filter = self.__bulk_update_or_create_inner_methods(_match_fields, case_insensitive_match)\n\n for batch in batches:\n obj_map = {_obj_key_getter(obj): obj for obj in batch}\n\n # mass select for bulk_update on existing ones\n to_update = self.filter(_obj_filter(obj_map))\n\n for to_u in to_update:\n obj = obj_map[_obj_key_getter(to_u)]\n for _f in update_fields:\n setattr(to_u, _f, getattr(obj, _f))\n del obj_map[_obj_key_getter(to_u)]\n self.bulk_update(to_update, update_fields)\n\n # .create on the remaining (bulk_create won't work on multi-table inheritance models...)\n created_objs = []\n for obj in obj_map.values():\n obj.save()\n created_objs.append(obj)\n if yield_objects:\n yield created_objs, to_update\n return created_objs, to_update\n\n\nclass BulkUpdateOrCreateQuerySet(BulkUpdateOrCreateMixin, models.QuerySet):\n pass\n\n\nclass _BulkUpdateOrCreateContextManager:\n def __init__(\n self,\n queryset: QuerySet,\n update_fields: List[str],\n batch_size: int = 500,\n status_cb: Optional[\n Callable[[Tuple[List[Model], List[Model]]], Any]\n ] = None,\n **kwargs: Optional[Any]\n ):\n self._queue = []\n self._queryset = queryset\n self._batch_size = batch_size\n assert status_cb is None or callable(status_cb)\n self._cb = status_cb\n self._fields = update_fields\n self._kwargs = kwargs\n\n def queue(self, obj: Model):\n self._queue.append(obj)\n if len(self._queue) >= self._batch_size:\n self.dump_queue()\n\n def queue_obj(self, **kwargs):\n \"\"\"\n proxy method to forward kwargs to self.model instantiation before calling queue()\n \"\"\"\n return self.queue(self._queryset.model(**kwargs))\n\n def dump_queue(self):\n if not self._queue:\n return\n\n r = self._queryset.bulk_update_or_create(\n self._queue,\n self._fields,\n yield_objects=self._cb is not None,\n **self._kwargs,\n )\n if self._cb is not None:\n for st in r:\n self._cb(st)\n\n self._queue = []\n\n def __enter__(self):\n return self\n\n def __exit__(\n self,\n type: Optional[Type[BaseException]],\n value: Optional[BaseException],\n traceback: Optional[TracebackType]\n ):\n self.dump_queue()\n" }, { "path": "setup.cfg", "content": "[metadata]\nname = django-bulk-update-or-create\nversion = attr: bulk_update_or_create.__version__\ndescription = bulk_update_or_create for Django model managers\nlong_description = file: README.md\nlong_description_content_type = text/markdown\nauthor = Filipe Pina\nauthor_email = fopina@gmail.com\nurl = https://github.com/fopina/django-bulk-update-or-create/\ndownload_url = https://pypi.org/project/django-bulk-update-or-create/\nlicense = BSD\nlicense_files = LICENSE\nclassifiers =\n Development Status :: 5 - Production/Stable\n Environment :: Web Environment\n Framework :: Django\n Framework :: Django :: 2.2\n Framework :: Django :: 3.0\n Intended Audience :: Developers\n License :: OSI Approved :: BSD License\n Operating System :: OS Independent\n Programming Language :: Python\n Programming Language :: Python :: 3\n Programming Language :: Python :: 3 :: Only\n Programming Language :: Python :: 3.6\n Programming Language :: Python :: 3.7\n Programming Language :: Python :: 3.8\n Programming Language :: Python :: 3.9\n Topic :: Software Development :: Libraries :: Python Modules\n\n[options]\npython_requires = >=3.6\ninstall_requires =\n Django >= 2.2\npackages = find:\ninclude_package_data = true\nzip_safe = false\n\n[options.packages.find]\nexclude =\n tests\n tests.*\n\n[flake8]\nexclude = conf.py\nignore = E203,W503\nmax-line-length = 120\n\n[coverage:run]\nsource = bulk_update_or_create\n" }, { "path": "setup.py", "content": "#!/usr/bin/env python3\n\nfrom setuptools import setup\n\nsetup()\n" }, { "path": "tests/README.md", "content": "# tests\n\nThis is a django app to run tests on `bulk_update_or_create`.\n\n`manage.py` has been patched to include parent directory in `sys.path` so you can simply run:\n\n```\n./manage.py test\n```\n\n`pytest.ini` added to make it easier to run tests from IDEs (such as VSCode), thanks to [pytest-django](https://github.com/pytest-dev/pytest-django/).\n\n`pytest` needs to be executed inside this directory (where `manage.py` is) and [requirements.txt](requirements.txt) need to be installed:\n\n```\npip install -r requirements.txt\n```\n\nUse `make -f ../Makefile startmysql` to spin up a mysql docker (or set `DJANGO_SETTINGS_MODULE` env var to different settings).\n\n\n## VSCode\n\nTo run/debug the tests in VSCode:\n\n* make sure to open this folder (not parent) as workspace\n * or use multi-project workspaces: open parent and then select \"Add Folder to Workspace\" and add this one\n* select `Python > Configure Tests` and choose `pytest`\n\n:heavy_check_mark:\n" }, { "path": "tests/manage.py", "content": "#!/usr/bin/env python\n\"\"\"Django's command-line utility for administrative tasks.\"\"\"\nimport os\nimport sys\n\nsys.path.insert(0, os.path.join(os.path.dirname(__file__), os.path.pardir))\n\n\ndef main():\n os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')\n try:\n from django.core.management import execute_from_command_line\n except ImportError as exc:\n raise ImportError(\n \"Couldn't import Django. Are you sure it's installed and \"\n \"available on your PYTHONPATH environment variable? Did you \"\n \"forget to activate a virtual environment?\"\n ) from exc\n execute_from_command_line(sys.argv)\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "tests/pytest.ini", "content": "# pytest.ini\n[pytest]\nDJANGO_SETTINGS_MODULE = settings_mysql\nminversion = 6.0\naddopts = -ra -q\ntestpaths =\n tests\npython_files = tests.py test_*.py\n" }, { "path": "tests/requirements.txt", "content": "-e ..\npytest==6.2.4\npytest-django==4.3.0\n" }, { "path": "tests/settings.py", "content": "\"\"\"\nDjango settings for tests project.\n\nGenerated by 'django-admin startproject' using Django 2.2.\n\nFor more information on this file, see\nhttps://docs.djangoproject.com/en/2.2/topics/settings/\n\nFor the full list of settings and their values, see\nhttps://docs.djangoproject.com/en/2.2/ref/settings/\n\"\"\"\n\nimport os\n\n# Build paths inside the project like this: os.path.join(BASE_DIR, ...)\nBASE_DIR = os.path.dirname(os.path.abspath(__file__))\n\n\n# Quick-start development settings - unsuitable for production\n# See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/\n\n# SECURITY WARNING: keep the secret key used in production secret!\nSECRET_KEY = 'nf5e4!s1s+kjxd58j(z1b8il#520m9!-j+*2#1*h0m_hv_-is8'\n\n# SECURITY WARNING: don't run with debug turned on in production!\nDEBUG = True\n\nALLOWED_HOSTS = []\n\n\n# Application definition\n\nINSTALLED_APPS = [\n 'django.contrib.admin',\n 'django.contrib.auth',\n 'django.contrib.contenttypes',\n 'django.contrib.sessions',\n 'django.contrib.messages',\n 'django.contrib.staticfiles',\n 'bulk_update_or_create',\n 'tests',\n]\n\nMIDDLEWARE = [\n 'django.middleware.security.SecurityMiddleware',\n 'django.contrib.sessions.middleware.SessionMiddleware',\n 'django.middleware.common.CommonMiddleware',\n 'django.middleware.csrf.CsrfViewMiddleware',\n 'django.contrib.auth.middleware.AuthenticationMiddleware',\n 'django.contrib.messages.middleware.MessageMiddleware',\n 'django.middleware.clickjacking.XFrameOptionsMiddleware',\n]\n\nROOT_URLCONF = 'urls'\n\nTEMPLATES = [\n {\n 'BACKEND': 'django.template.backends.django.DjangoTemplates',\n 'DIRS': [],\n 'APP_DIRS': True,\n 'OPTIONS': {\n 'context_processors': [\n 'django.template.context_processors.debug',\n 'django.template.context_processors.request',\n 'django.contrib.auth.context_processors.auth',\n 'django.contrib.messages.context_processors.messages',\n ],\n },\n },\n]\n\n\n# Database\n# https://docs.djangoproject.com/en/2.2/ref/settings/#databases\n\nDATABASES = {\n 'default': {\n 'ENGINE': 'django.db.backends.sqlite3',\n 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),\n }\n}\n\n\n# Password validation\n# https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators\n\nAUTH_PASSWORD_VALIDATORS = [\n {\n 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',\n },\n {\n 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',\n },\n {\n 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',\n },\n {\n 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',\n },\n]\n\n\n# Internationalization\n# https://docs.djangoproject.com/en/2.2/topics/i18n/\n\nLANGUAGE_CODE = 'en-us'\n\nTIME_ZONE = 'UTC'\n\nUSE_I18N = True\n\nUSE_L10N = True\n\nUSE_TZ = True\n\n\n# Static files (CSS, JavaScript, Images)\n# https://docs.djangoproject.com/en/2.2/howto/static-files/\n\nSTATIC_URL = '/static/'\n" }, { "path": "tests/settings_mysql.py", "content": "from settings import * # noqa\n\n\nDATABASES = {\n 'default': {\n 'ENGINE': 'django.db.backends.mysql',\n 'NAME': 'mysql',\n 'USER': 'root',\n 'PASSWORD': 'root',\n 'HOST': '127.0.0.1',\n 'PORT': '8877',\n 'TEST': {'CHARSET': 'utf8mb4', 'COLLATION': 'utf8mb4_bin'},\n }\n}\n" }, { "path": "tests/settings_postgresql.py", "content": "from settings import * # noqa\n\nDATABASES = {\n 'default': {\n 'ENGINE': 'django.db.backends.postgresql_psycopg2',\n 'NAME': 'postgres',\n 'USER': 'postgres',\n 'PASSWORD': 'postgres',\n 'HOST': '127.0.0.1',\n 'PORT': '8878',\n }\n}\n" }, { "path": "tests/tests/__init__.py", "content": "" }, { "path": "tests/tests/management/__init__.py", "content": "" }, { "path": "tests/tests/management/commands/__init__.py", "content": "" }, { "path": "tests/tests/management/commands/bulk_it.py", "content": "from time import time\n\nfrom django.core.management.base import BaseCommand\n\nfrom tests.models import RandomData\nfrom contextlib import contextmanager\n\n\n@contextmanager\ndef timing(description: str) -> None:\n start = time()\n yield\n ellapsed_time = time() - start\n\n print(f\"{description}: {ellapsed_time}\")\n\n\nclass Command(BaseCommand):\n help = 'Lock it!'\n\n def _loop(self, n=1000, offset=0, data_offset=0):\n for i in range(n):\n RandomData.objects.update_or_create(\n uuid=i + offset,\n defaults={'data': str(i + offset + data_offset)},\n )\n\n def _bulk(self, n=1000, offset=0, data_offset=0):\n items = [RandomData(uuid=i + offset, data=str(i + offset + data_offset)) for i in range(n)]\n RandomData.objects.bulk_update_or_create(items, ['data'], match_field='uuid')\n\n def _clear(self):\n RandomData.objects.all().delete()\n\n def _check(self, n=1000, min=0, max=999):\n values = sorted([int(x.data) for x in RandomData.objects.all()])\n assert len(values) == n\n assert values[0] == min\n assert values[-1] == max\n\n def handle(self, *args, **options):\n self._clear()\n\n with timing('loop of update_or_create - all creates'):\n self._loop()\n self._check()\n\n with timing('loop of update_or_create - all updates'):\n self._loop(data_offset=1)\n self._check(1000, 1, 1000)\n\n with timing('loop of update_or_create - half half'):\n self._loop(offset=500, data_offset=2)\n self._check(1500, 1, 1501)\n\n self._clear()\n\n with timing('bulk_update_or_create - all creates'):\n self._bulk()\n self._check()\n\n with timing('bulk_update_or_create - all updates'):\n self._bulk(data_offset=1)\n self._check(1000, 1, 1000)\n\n with timing('bulk_update_or_create - half half'):\n self._bulk(offset=500, data_offset=2)\n self._check(1500, 1, 1501)\n" }, { "path": "tests/tests/migrations/0001_initial.py", "content": "# Generated by Django 2.2 on 2020-07-14 10:04\n\nfrom django.db import migrations, models\n\n\nclass Migration(migrations.Migration):\n\n initial = True\n\n dependencies = []\n\n operations = [\n migrations.CreateModel(\n name='RandomData',\n fields=[\n ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),\n ('uuid', models.IntegerField(unique=True)),\n ('value', models.IntegerField(default=0)),\n ('data', models.CharField(blank=True, max_length=200, null=True)),\n ],\n ),\n ]\n" }, { "path": "tests/tests/migrations/__init__.py", "content": "" }, { "path": "tests/tests/models.py", "content": "from django.db import models\nfrom bulk_update_or_create import BulkUpdateOrCreateQuerySet\n\n\nclass RandomData(models.Model):\n objects = BulkUpdateOrCreateQuerySet.as_manager()\n\n uuid = models.IntegerField(unique=True)\n value = models.IntegerField(default=0)\n data = models.CharField(max_length=200, null=True, blank=True)\n\n def __str__(self):\n return f'{self.uuid} - {self.data} - {self.value}'\n" }, { "path": "tests/tests/tests.py", "content": "from django.test import TestCase\nfrom django.core.exceptions import FieldDoesNotExist\n\nfrom tests.models import RandomData\n\n\nclass Test(TestCase):\n def test_all_create(self):\n items = [RandomData(uuid=i, data=i) for i in range(10)]\n # 1 select + 10 creates, all new\n with self.assertNumQueries(11):\n RandomData.objects.bulk_update_or_create(items, ['data'], match_field='uuid')\n self.assertEqual(RandomData.objects.count(), 10)\n self.assertEqual(sorted(int(x.data) for x in RandomData.objects.all()), list(range(10)))\n\n def test_update_some(self):\n self.test_all_create()\n items = [RandomData(uuid=i + 5, data=i + 10) for i in range(10)]\n # 1 select, 1 bulk update, 5 create\n with self.assertNumQueries(7):\n RandomData.objects.bulk_update_or_create(items, ['data'], match_field='uuid')\n self.assertEqual(RandomData.objects.count(), 15)\n self.assertEqual(\n sorted(int(x.data) for x in RandomData.objects.all()),\n list(range(5)) + list(range(10, 20)),\n )\n\n def test_all_update(self):\n self.test_all_create()\n items = [RandomData(uuid=i, data=i + 10) for i in range(10)]\n # 1 select, 1 bulk update\n with self.assertNumQueries(2):\n RandomData.objects.bulk_update_or_create(items, ['data'], match_field='uuid')\n self.assertEqual(RandomData.objects.count(), 10)\n self.assertEqual(\n sorted(int(x.data) for x in RandomData.objects.all()),\n list(range(10, 20)),\n )\n\n def test_update_some_generator(self):\n self.test_all_create()\n items = [RandomData(uuid=i + 5, data=i + 10) for i in range(10)]\n updated_items = RandomData.objects.bulk_update_or_create(\n items, ['data'], match_field='uuid', yield_objects=True\n )\n # not executed yet, just generator\n self.assertEqual(RandomData.objects.count(), 10)\n updated_items = list(updated_items)\n self.assertEqual(RandomData.objects.count(), 15)\n self.assertEqual(\n sorted(int(x.data) for x in RandomData.objects.all()),\n list(range(5)) + list(range(10, 20)),\n )\n # one batch\n self.assertEqual(len(updated_items), 1)\n # tuple with (created, updated)\n self.assertEqual(len(updated_items[0]), 2)\n # 5 were created - 15 to 19\n self.assertEqual(len(updated_items[0][0]), 5)\n self.assertEqual(\n sorted(int(x.data) for x in updated_items[0][0]),\n list(range(15, 20)),\n )\n for x in updated_items[0][0]:\n self.assertIsNotNone(x.pk)\n # 5 were updated - 10 to 14 (from 5 to 9)\n self.assertEqual(len(updated_items[0][1]), 5)\n self.assertEqual(\n sorted(int(x.data) for x in updated_items[0][1]),\n list(range(10, 15)),\n )\n for x in updated_items[0][1]:\n self.assertIsNotNone(x.pk)\n\n def test_errors(self):\n with self.assertRaises(ValueError) as cm:\n RandomData.objects.bulk_update_or_create([None], [])\n self.assertEqual(cm.exception.args, ('update_fields cannot be empty',))\n\n with self.assertRaises(ValueError) as cm:\n RandomData.objects.bulk_update_or_create([None], ['data'], batch_size=-1)\n self.assertEqual(cm.exception.args, ('Batch size must be a positive integer.',))\n\n with self.assertRaises(FieldDoesNotExist) as cm:\n RandomData.objects.bulk_update_or_create([RandomData(uuid=1, data='x')], ['data'], match_field='x')\n self.assertEqual(cm.exception.args, (\"RandomData has no field named 'x'\",))\n with self.assertRaises(FieldDoesNotExist) as cm:\n RandomData.objects.bulk_update_or_create([RandomData(uuid=1, data='x')], ['x'], match_field='uuid')\n self.assertEqual(cm.exception.args, (\"RandomData has no field named 'x'\",))\n\n def test_case_sensitivity(self):\n \"\"\"\n match_fields should always be unique but for test simplicity (no extra model),\n using RandomData.data\n \"\"\"\n RandomData.objects.bulk_update_or_create(\n [\n RandomData(uuid=1, data='x'),\n ],\n ['uuid'],\n match_field='data',\n )\n self.assertEqual(RandomData.objects.count(), 1)\n self.assertEqual(sorted(x.data for x in RandomData.objects.all()), ['x'])\n\n RandomData.objects.bulk_update_or_create(\n [\n RandomData(uuid=2, data='X'),\n ],\n ['uuid'],\n match_field='data',\n case_insensitive_match=True,\n )\n self.assertEqual(RandomData.objects.count(), 1)\n self.assertEqual(sorted(x.data for x in RandomData.objects.all()), ['x'])\n\n RandomData.objects.bulk_update_or_create(\n [\n RandomData(uuid=3, data='X'),\n ],\n ['uuid'],\n match_field='data',\n )\n self.assertEqual(RandomData.objects.count(), 2)\n self.assertEqual(sorted(x.data for x in RandomData.objects.all()), ['X', 'x'])\n\n def test_update_some_with_context_manager(self):\n self.test_all_create()\n with self.assertNumQueries(7):\n with RandomData.objects.bulk_update_or_create_context(\n ['data'], match_field='uuid', batch_size=500\n ) as bulkit:\n for i in range(10):\n bulkit.queue(RandomData(uuid=i + 5, data=i + 10))\n self.assertEqual(RandomData.objects.count(), 15)\n self.assertEqual(\n sorted(int(x.data) for x in RandomData.objects.all()),\n list(range(5)) + list(range(10, 20)),\n )\n\n # smaller batch_size to test more than 1 batch and test status_cb\n cb_calls = []\n\n def _cb(x):\n # nothing created\n self.assertEqual(x[0], [])\n cb_calls.extend(x[1])\n\n # 4 all-update batches = 8 queries\n with self.assertNumQueries(8):\n with RandomData.objects.bulk_update_or_create_context(\n ['data'], match_field='uuid', batch_size=3, status_cb=_cb\n ) as bulkit:\n for i in range(10):\n bulkit.queue(RandomData(uuid=i, data=i + 20))\n self.assertEqual(RandomData.objects.count(), 15)\n self.assertEqual(\n # 20 to 29 ... 15 to 19\n sorted(int(x.data) for x in RandomData.objects.all()),\n list(range(15, 30)),\n )\n self.assertEqual(len(cb_calls), 10)\n for i in range(10):\n self.assertEqual(cb_calls[i].uuid, i)\n self.assertEqual(cb_calls[i].data, i + 20)\n\n def test_context_manager_exact_batch_size(self):\n # test made to hit *empty* queue on context manager __exit__()!\n with self.assertNumQueries(11):\n with RandomData.objects.bulk_update_or_create_context(\n ['data'], match_field='uuid', batch_size=10\n ) as bulkit:\n for i in range(10):\n bulkit.queue(RandomData(uuid=i + 5, data=i + 10))\n self.assertSum(145)\n\n def test_context_manager_queue_kwargs(self):\n with self.assertNumQueries(11):\n with RandomData.objects.bulk_update_or_create_context(\n ['data'], match_field='uuid', batch_size=10\n ) as bulkit:\n for i in range(10):\n bulkit.queue_obj(uuid=i + 5, data=i + 10)\n self.assertSum(145)\n\n def test_empty_objs(self):\n \"\"\"\n test change of behaviour for empty objs to match bulk_update\n https://github.com/fopina/django-bulk-update-or-create/issues/10\n \"\"\"\n with self.assertNumQueries(0):\n RandomData.objects.bulk_update([], fields=['data'])\n with self.assertNumQueries(0):\n RandomData.objects.bulk_update_or_create([], ['data'], match_field='uuid')\n\n def test_keyerror(self):\n \"\"\"\n test for issue https://github.com/fopina/django-bulk-update-or-create/issues/11\n eg: using string values in model IntegerFields cause obj_map lookups to fail on existing objects\n \"\"\"\n\n self.test_all_create()\n self.assertSum(45)\n # this works\n RandomData.objects.bulk_update_or_create(\n [RandomData(uuid=i, data=i + 1) for i in range(10)], ['data'], match_field='uuid'\n )\n self.assertSum(55)\n # but this *DID* not - it does now though!\n RandomData.objects.bulk_update_or_create(\n [RandomData(uuid=str(i), data=i + 2) for i in range(10)], ['data'], match_field='uuid'\n )\n self.assertSum(65)\n\n def assertSum(self, total):\n self.assertEqual(sum(int(x.data) for x in RandomData.objects.all()), total)\n\n def test_multiple_match_fields_update(self):\n items = [RandomData(uuid=i, value=i % 5, data=i) for i in range(10)]\n RandomData.objects.bulk_create(items)\n\n items = [RandomData(uuid=i, value=i % 5, data=i + 10) for i in range(10)]\n # 1 select, 1 bulk update\n with self.assertNumQueries(2):\n RandomData.objects.bulk_update_or_create(items, ['data'], match_field=('uuid', 'value'))\n\n self.assertEqual(RandomData.objects.count(), 10)\n self.assertEqual(\n sorted(int(x.data) for x in RandomData.objects.all()),\n list(range(10, 20)),\n )\n\n def test_multiple_match_fields_update_create(self):\n items = [RandomData(uuid=i, value=i % 5, data=i) for i in range(10)]\n RandomData.objects.bulk_create(items)\n\n items = [RandomData(uuid=i + 5, value=i % 5, data=i + 10) for i in range(10)]\n # 1 select, 1 bulk update, 5 inserts\n with self.assertNumQueries(7):\n RandomData.objects.bulk_update_or_create(items, ['data'], match_field=('uuid', 'value'))\n self.assertEqual(RandomData.objects.count(), 15)\n self.assertEqual(\n list(int(x.data) for x in RandomData.objects.order_by('uuid')),\n [*range(5), *range(10, 15), *range(15, 20)],\n )\n\n def test_multiple_match_fields_update_pk(self):\n items = [RandomData(uuid=i, value=i % 5, data=str(i)) for i in range(10)]\n RandomData.objects.bulk_create(items)\n\n items = [RandomData(uuid=i + 100, value=i % 5, data=str(i)) for i in range(10)]\n # 1 select, 1 bulk update\n with self.assertNumQueries(2):\n RandomData.objects.bulk_update_or_create(items, ['uuid'], match_field=('data', 'value'))\n self.assertEqual(RandomData.objects.count(), 10)\n self.assertEqual(\n list(x.uuid for x in RandomData.objects.order_by('data', 'value')),\n list(range(100, 110)),\n )\n" }, { "path": "tests/urls.py", "content": "\"\"\"tests URL Configuration\n\nThe `urlpatterns` list routes URLs to views. For more information please see:\n https://docs.djangoproject.com/en/2.2/topics/http/urls/\nExamples:\nFunction views\n 1. Add an import: from my_app import views\n 2. Add a URL to urlpatterns: path('', views.home, name='home')\nClass-based views\n 1. Add an import: from other_app.views import Home\n 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')\nIncluding another URLconf\n 1. Import the include() function: from django.urls import include, path\n 2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))\n\"\"\"\nfrom django.contrib import admin\nfrom django.urls import path\n\nurlpatterns = [\n path('admin/', admin.site.urls),\n]\n" }, { "path": "tox.ini", "content": "[tox]\nenvlist =\n flake8\n py{37,38,39}-dj{22,30,32}-{sqlite,postgresql,mysql}\n\n[testenv]\ndeps =\n dj22: Django==2.2.*\n dj30: Django==3.0.*\n dj32: Django==3.2.*\n postgresql: psycopg2-binary\n mysql: mysqlclient\n coverage\nsetenv =\n PYTHONPATH = {toxinidir}\n sqlite: DJANGO_SETTINGS_MODULE = settings\n postgresql: DJANGO_SETTINGS_MODULE = settings_postgresql\n mysql: DJANGO_SETTINGS_MODULE = settings_mysql\nwhitelist_externals = make\npip_pre = True\ncommands = make coverage TEST_ARGS='{posargs:tests}'\n\n[testenv:flake8]\nbasepython = python3\ncommands = make flake8\ndeps = flake8\nskip_install = true\n\n[testenv:style]\nbasepython = python3\ncommands = make style_check\ndeps =\n black>=19.10b0\n flake8\nskip_install = true\n" } ]