Repository: Robpol86/Flask-Celery-Helper
Branch: master
Commit: 92bd3b029544
Files: 15
Total size: 41.0 KB

Directory structure:
gitextract_je8cve6m/

├── .gitignore
├── .travis.yml
├── CONTRIBUTING.md
├── LICENSE
├── README.rst
├── appveyor.yml
├── flask_celery.py
├── setup.py
├── tests/
│   ├── __init__.py
│   ├── conftest.py
│   ├── instances.py
│   ├── test_class.py
│   ├── test_collision.py
│   └── test_timeout.py
└── tox.ini

================================================
FILE CONTENTS
================================================

================================================
FILE: .gitignore
================================================
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class

# C extensions
*.so

# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg

# PyInstaller
#  Usually these files are written by a python script from a template
#  before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec

# Installer logs
pip-log.txt
pip-delete-this-directory.txt

# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
.hypothesis/

# Translations
*.mo
*.pot

# Django stuff:
*.log
local_settings.py

# Flask stuff:
instance/
.webassets-cache

# Scrapy stuff:
.scrapy

# Sphinx documentation
docs/_build/

# PyBuilder
target/

# IPython Notebook
.ipynb_checkpoints

# pyenv
.python-version

# celery beat schedule file
celerybeat-schedule

# dotenv
.env

# virtualenv
venv/
ENV/

# Spyder project settings
.spyderproject

# Rope project settings
.ropeproject

# Robpol86
*.rpm
.idea/
requirements*.txt
.DS_Store


================================================
FILE: .travis.yml
================================================
# Configure.
language: python
python:
  - 3.4
  - 3.3
  - pypy
  - 2.7
  - 2.6
services: [redis-server]
sudo: false

# Environment and matrix.
env:
  - BROKER: sqlite
  - BROKER: mysql
  - BROKER: postgres
  - BROKER: redis
  - BROKER: redis_sock,/tmp/redis.sock
matrix:
  include:
    - python: 3.4
      services: []
      env: TOX_ENV=lint
      before_script: []
      after_success: []

# Run.
install: pip install tox
before_script:
  - if [[ $BROKER == redis_sock* ]]; then echo -e "daemonize yes\nunixsocket /tmp/redis.sock\nport 0" |redis-server -; fi
  - if [ $BROKER == mysql ]; then mysql -u root -e 'CREATE DATABASE flask_celery_helper_test;'; fi
  - if [ $BROKER == mysql ]; then mysql -u root -e 'GRANT ALL PRIVILEGES ON flask_celery_helper_test.* TO "user"@"localhost" IDENTIFIED BY "pass";'; fi
  - if [ $BROKER == postgres ]; then psql -U postgres -c 'CREATE DATABASE flask_celery_helper_test;'; fi
  - if [ $BROKER == postgres ]; then psql -U postgres -c "CREATE USER user1 WITH PASSWORD 'pass';"; fi
  - if [ $BROKER == postgres ]; then psql -U postgres -c 'GRANT ALL PRIVILEGES ON DATABASE flask_celery_helper_test TO user1;'; fi
script: tox -e ${TOX_ENV:-py}
after_success:
  - bash <(curl -s https://codecov.io/bash)

# Deploy.
deploy:
  provider: pypi
  user: Robpol86
  password:
    secure:
      "liwn5bHqjAtW+gRX6r4VgWuc44OUwfGSne4fTxb6G2pnPNW/IneVspQ2bFXeuQDdXzyLoOe\
      bKa8bxjRurUEHedjV9UG9fVZwVsWU981aWOxeEl+6kLkpJ2fE9UVeK7T1O+RzzhkWhHq2/YL\
      4BjBqzOLuBSAGnXZAnwH55Z6HY2g="
  on:
    condition: $TRAVIS_PYTHON_VERSION = 3.4
    tags: true


================================================
FILE: CONTRIBUTING.md
================================================
# Contributing

Everyone that wants to contribute to the project should read this document.

## Getting Started

You may follow these steps if you wish to create a pull request. Fork the repo and clone it on your local machine. Then
in the project's directory:

```bash
virtualenv env  # Create a virtualenv for the project's dependencies.
source env/bin/activate  # Activate the virtualenv.
pip install tox  # Install tox, which runs linting and tests.
tox  # This runs all tests on your local machine. Make sure they pass.
```

If you don't have Python 2.7 or 3.4 installed you can manually run tests on one specific version by running
`tox -e lint,py35` (for Python 3.5) instead.

## Consistency and Style

Keep code style consistent with the rest of the project. Some suggestions:

1. **Write tests for your new features.** `if new_feature else` **Write tests for bug-causing scenarios.**
2. Write docstrings for all classes, functions, methods, modules, etc.
3. Document all function/method arguments and return values.
4. Document all class variables instance variables.
5. Documentation guidelines also apply to tests, though not as strict.
6. Keep code style consistent, such as the kind of quotes to use and spacing.
7. Don't use `except:` or `except Exception:` unless you have a `raise` in the block. Be specific about error handling.
8. Don't use `isinstance()` (it breaks [duck typing](https://en.wikipedia.org/wiki/Duck_typing#In_Python)).

## Thanks

Thanks for fixing bugs or adding features to the project!


================================================
FILE: LICENSE
================================================
The MIT License (MIT)

Copyright (c) 2017 Robpol86

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.


================================================
FILE: README.rst
================================================
===================
Flask-Celery-Helper
===================

Even though the `Flask documentation <http://flask.pocoo.org/docs/patterns/celery/>`_ says Celery extensions are
unnecessary now, I found that I still need an extension to properly use Celery in large Flask applications. Specifically
I need an init_app() method to initialize Celery after I instantiate it.

This extension also comes with a ``single_instance`` method.

* Python 2.6, 2.7, PyPy, 3.3, and 3.4 supported on Linux and OS X.
* Python 2.7, 3.3, and 3.4 supported on Windows (both 32 and 64 bit versions of Python).

.. image:: https://img.shields.io/appveyor/ci/Robpol86/Flask-Celery-Helper/master.svg?style=flat-square&label=AppVeyor%20CI
    :target: https://ci.appveyor.com/project/Robpol86/Flask-Celery-Helper
    :alt: Build Status Windows

.. image:: https://img.shields.io/travis/Robpol86/Flask-Celery-Helper/master.svg?style=flat-square&label=Travis%20CI
    :target: https://travis-ci.org/Robpol86/Flask-Celery-Helper
    :alt: Build Status

.. image:: https://img.shields.io/codecov/c/github/Robpol86/Flask-Celery-Helper/master.svg?style=flat-square&label=Codecov
    :target: https://codecov.io/gh/Robpol86/Flask-Celery-Helper
    :alt: Coverage Status

.. image:: https://img.shields.io/pypi/v/Flask-Celery-Helper.svg?style=flat-square&label=Latest
    :target: https://pypi.python.org/pypi/Flask-Celery-Helper
    :alt: Latest Version

Attribution
===========

Single instance decorator inspired by
`Ryan Roemer <http://loose-bits.com/2010/10/distributed-task-locking-in-celery.html>`_.

Supported Libraries
===================

* `Flask <http://flask.pocoo.org/>`_ 0.12
* `Redis <http://redis.io/>`_ 3.2.6
* `Celery <http://www.celeryproject.org/>`_ 3.1.11

Quickstart
==========

Install:

.. code:: bash

    pip install Flask-Celery-Helper

Examples
========

Basic Example
-------------

.. code:: python

    # example.py
    from flask import Flask
    from flask_celery import Celery

    app = Flask('example')
    app.config['CELERY_BROKER_URL'] = 'redis://localhost'
    app.config['CELERY_RESULT_BACKEND'] = 'redis://localhost'
    celery = Celery(app)

    @celery.task()
    def add_together(a, b):
        return a + b

    if __name__ == '__main__':
        result = add_together.delay(23, 42)
        print(result.get())

Run these two commands in separate terminals:

.. code:: bash

    celery -A example.celery worker
    python example.py

Factory Example
---------------

.. code:: python

    # extensions.py
    from flask_celery import Celery

    celery = Celery()

.. code:: python

    # application.py
    from flask import Flask
    from extensions import celery

    def create_app():
        app = Flask(__name__)
        app.config['CELERY_IMPORTS'] = ('tasks.add_together', )
        app.config['CELERY_BROKER_URL'] = 'redis://localhost'
        app.config['CELERY_RESULT_BACKEND'] = 'redis://localhost'
        celery.init_app(app)
        return app

.. code:: python

    # tasks.py
    from extensions import celery

    @celery.task()
    def add_together(a, b):
        return a + b

.. code:: python

    # manage.py
    from application import create_app

    app = create_app()
    app.run()

Single Instance Example
-----------------------

.. code:: python

    # example.py
    import time
    from flask import Flask
    from flask_celery import Celery, single_instance
    from flask_redis import Redis

    app = Flask('example')
    app.config['REDIS_URL'] = 'redis://localhost'
    app.config['CELERY_BROKER_URL'] = 'redis://localhost'
    app.config['CELERY_RESULT_BACKEND'] = 'redis://localhost'
    celery = Celery(app)
    Redis(app)

    @celery.task(bind=True)
    @single_instance
    def sleep_one_second(a, b):
        time.sleep(1)
        return a + b

    if __name__ == '__main__':
        task1 = sleep_one_second.delay(23, 42)
        time.sleep(0.1)
        task2 = sleep_one_second.delay(20, 40)
        results1 = task1.get(propagate=False)
        results2 = task2.get(propagate=False)
        print(results1)  # 65
        if isinstance(results2, Exception) and str(results2) == 'Failed to acquire lock.':
            print('Another instance is already running.')
        else:
            print(results2)  # Should not happen.

.. changelog-section-start

Changelog
=========

This project adheres to `Semantic Versioning <http://semver.org/>`_.

Unreleased
----------

Changed
    * Supporting Flask 0.12, switching from ``flask.ext.celery`` to ``flask_celery`` import recommendation.

1.1.0 - 2014-12-28
------------------

Added
    * Windows support.
    * ``single_instance`` supported on SQLite/MySQL/PostgreSQL in addition to Redis.

Changed
    * ``CELERY_RESULT_BACKEND`` no longer mandatory.
    * Breaking changes: ``flask.ext.celery.CELERY_LOCK`` moved to ``flask.ext.celery._LockManagerRedis.CELERY_LOCK``.

1.0.0 - 2014-11-01
------------------

Added
    * Support for non-Redis backends.

0.2.2 - 2014-08-11
------------------

Added
    * Python 2.6 and 3.x support.

0.2.1 - 2014-06-18
------------------

Fixed
    * ``single_instance`` arguments with functools.

0.2.0 - 2014-06-18
------------------

Added
    * ``include_args`` argument to ``single_instance``.

0.1.0 - 2014-06-01
------------------

* Initial release.

.. changelog-section-end


================================================
FILE: appveyor.yml
================================================
# Configure.
services:
  - mysql
  - postgresql

# Environment and matrix.
environment:
  PATH: C:\%PYTHON%;C:\%PYTHON%\Scripts;C:\Program Files\MySQL\MySQL Server 5.7\bin;C:\Program Files\PostgreSQL\9.5\bin;%PATH%
  PGPASSWORD: Password12!
  PYTHON: Python34
  matrix:
    - TOX_ENV: lint
      BROKER: sqlite
    - TOX_ENV: py34
      BROKER: sqlite
    - TOX_ENV: py33
      BROKER: sqlite
    - TOX_ENV: py27
      BROKER: sqlite
    - TOX_ENV: py
      PYTHON: Python34-x64
      BROKER: sqlite
    - TOX_ENV: py
      PYTHON: Python33-x64
      BROKER: sqlite
    - TOX_ENV: py
      PYTHON: Python27-x64
      BROKER: sqlite

    - TOX_ENV: lint
      BROKER: mysql
    - TOX_ENV: py34
      BROKER: mysql
    - TOX_ENV: py33
      BROKER: mysql
    - TOX_ENV: py27
      BROKER: mysql
    - TOX_ENV: py
      PYTHON: Python34-x64
      BROKER: mysql
    - TOX_ENV: py
      PYTHON: Python33-x64
      BROKER: mysql
    - TOX_ENV: py
      PYTHON: Python27-x64
      BROKER: mysql

    - TOX_ENV: lint
      BROKER: postgres
    - TOX_ENV: py34
      BROKER: postgres
    - TOX_ENV: py33
      BROKER: postgres
    - TOX_ENV: py27
      BROKER: postgres
    - TOX_ENV: py
      PYTHON: Python34-x64
      BROKER: postgres
    - TOX_ENV: py
      PYTHON: Python33-x64
      BROKER: postgres
    - TOX_ENV: py
      PYTHON: Python27-x64
      BROKER: postgres

    - TOX_ENV: lint
      BROKER: redis
    - TOX_ENV: py34
      BROKER: redis
    - TOX_ENV: py33
      BROKER: redis
    - TOX_ENV: py27
      BROKER: redis
    - TOX_ENV: py
      PYTHON: Python34-x64
      BROKER: redis
    - TOX_ENV: py
      PYTHON: Python33-x64
      BROKER: redis
    - TOX_ENV: py
      PYTHON: Python27-x64
      BROKER: redis

# Run.
build_script: pip install tox
after_build:
  - IF %BROKER% EQU redis cinst redis-64
  - IF %BROKER% EQU redis redis-server --service-install
  - IF %BROKER% EQU redis redis-server --service-start
  - IF %BROKER% EQU mysql mysql -u root -p"Password12!" -e "CREATE DATABASE flask_celery_helper_test;"
  - IF %BROKER% EQU mysql mysql -u root -p"Password12!" -e "GRANT ALL PRIVILEGES ON flask_celery_helper_test.* TO 'user'@'localhost' IDENTIFIED BY 'pass';"
  - IF %BROKER% EQU postgres psql -U postgres -c "CREATE DATABASE flask_celery_helper_test;"
  - IF %BROKER% EQU postgres psql -U postgres -c "CREATE USER user1 WITH PASSWORD 'pass';"
  - IF %BROKER% EQU postgres psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE flask_celery_helper_test TO user1;"
test_script: tox -e %TOX_ENV%
on_success: IF %TOX_ENV% NEQ lint pip install codecov & codecov


================================================
FILE: flask_celery.py
================================================
"""Celery support for Flask without breaking PyCharm inspections.

https://github.com/Robpol86/Flask-Celery-Helper
https://pypi.python.org/pypi/Flask-Celery-Helper
"""

import hashlib
from datetime import datetime, timedelta
from functools import partial, wraps
from logging import getLogger

from celery import _state, Celery as CeleryClass

__author__ = '@Robpol86'
__license__ = 'MIT'
__version__ = '1.1.0'


class OtherInstanceError(Exception):
    """Raised when Celery task is already running, when lock exists and has not timed out."""

    pass


class _LockManager(object):
    """Base class for other lock managers."""

    def __init__(self, celery_self, timeout, include_args, args, kwargs):
        """May raise NotImplementedError if the Celery backend is not supported.

        :param celery_self: From wrapped() within single_instance(). It is the `self` object specified in a binded
            Celery task definition (implicit first argument of the Celery task when @celery.task(bind=True) is used).
        :param int timeout: Lock's timeout value in seconds.
        :param bool include_args: If single instance should take arguments into account.
        :param iter args: The task instance's args.
        :param dict kwargs: The task instance's kwargs.
        """
        self.celery_self = celery_self
        self.timeout = timeout
        self.include_args = include_args
        self.args = args
        self.kwargs = kwargs
        self.log = getLogger('{0}:{1}'.format(self.__class__.__name__, self.task_identifier))

    @property
    def task_identifier(self):
        """Return the unique identifier (string) of a task instance."""
        task_id = self.celery_self.name
        if self.include_args:
            merged_args = str(self.args) + str([(k, self.kwargs[k]) for k in sorted(self.kwargs)])
            task_id += '.args.{0}'.format(hashlib.md5(merged_args.encode('utf-8')).hexdigest())
        return task_id


class _LockManagerRedis(_LockManager):
    """Handle locking/unlocking for Redis backends."""

    CELERY_LOCK = '_celery.single_instance.{task_id}'

    def __init__(self, celery_self, timeout, include_args, args, kwargs):
        super(_LockManagerRedis, self).__init__(celery_self, timeout, include_args, args, kwargs)
        self.lock = None

    def __enter__(self):
        redis_key = self.CELERY_LOCK.format(task_id=self.task_identifier)
        self.lock = self.celery_self.backend.client.lock(redis_key, timeout=self.timeout)
        self.log.debug('Timeout %ds | Redis key %s', self.timeout, redis_key)
        if not self.lock.acquire(blocking=False):
            self.log.debug('Another instance is running.')
            raise OtherInstanceError('Failed to acquire lock, {0} already running.'.format(self.task_identifier))
        else:
            self.log.debug('Got lock, running.')

    def __exit__(self, exc_type, *_):
        if exc_type == OtherInstanceError:
            # Failed to get lock last time, not releasing.
            return
        self.log.debug('Releasing lock.')
        self.lock.release()

    @property
    def is_already_running(self):
        """Return True if lock exists and has not timed out."""
        redis_key = self.CELERY_LOCK.format(task_id=self.task_identifier)
        return self.celery_self.backend.client.exists(redis_key)

    def reset_lock(self):
        """Removed the lock regardless of timeout."""
        redis_key = self.CELERY_LOCK.format(task_id=self.task_identifier)
        self.celery_self.backend.client.delete(redis_key)


class _LockManagerDB(_LockManager):
    """Handle locking/unlocking for SQLite/MySQL/PostgreSQL/etc backends."""

    def __init__(self, celery_self, timeout, include_args, args, kwargs):
        super(_LockManagerDB, self).__init__(celery_self, timeout, include_args, args, kwargs)
        self.save_group = getattr(self.celery_self.backend, '_save_group')
        self.restore_group = getattr(self.celery_self.backend, '_restore_group')
        self.delete_group = getattr(self.celery_self.backend, '_delete_group')

    def __enter__(self):
        self.log.debug('Timeout %ds', self.timeout)
        try:
            self.save_group(self.task_identifier, None)
        except Exception as exc:  # pylint: disable=broad-except
            if 'IntegrityError' not in str(exc) and 'ProgrammingError' not in str(exc):
                raise
            difference = datetime.utcnow() - self.restore_group(self.task_identifier)['date_done']
            if difference < timedelta(seconds=self.timeout):
                self.log.debug('Another instance is running.')
                raise OtherInstanceError('Failed to acquire lock, {0} already running.'.format(self.task_identifier))
            self.log.debug('Timeout expired, stale lock found, releasing lock.')
            self.delete_group(self.task_identifier)
            self.save_group(self.task_identifier, None)
            self.log.debug('Got lock, running.')

    def __exit__(self, exc_type, *_):
        if exc_type == OtherInstanceError:
            # Failed to get lock last time, not releasing.
            return
        self.log.debug('Releasing lock.')
        self.delete_group(self.task_identifier)

    @property
    def is_already_running(self):
        """Return True if lock exists and has not timed out."""
        date_done = (self.restore_group(self.task_identifier) or dict()).get('date_done')
        if not date_done:
            return False
        difference = datetime.utcnow() - date_done
        return difference < timedelta(seconds=self.timeout)

    def reset_lock(self):
        """Removed the lock regardless of timeout."""
        self.delete_group(self.task_identifier)


def _select_manager(backend_name):
    """Select the proper LockManager based on the current backend used by Celery.

    :raise NotImplementedError: If Celery is using an unsupported backend.

    :param str backend_name: Class name of the current Celery backend. Usually value of
        current_app.extensions['celery'].celery.backend.__class__.__name__.

    :return: Class definition object (not instance). One of the _LockManager* classes.
    """
    if backend_name == 'RedisBackend':
        lock_manager = _LockManagerRedis
    elif backend_name == 'DatabaseBackend':
        lock_manager = _LockManagerDB
    else:
        raise NotImplementedError
    return lock_manager


class _CeleryState(object):
    """Remember the configuration for the (celery, app) tuple. Modeled from SQLAlchemy."""

    def __init__(self, celery, app):
        self.celery = celery
        self.app = app


# noinspection PyProtectedMember
class Celery(CeleryClass):
    """Celery extension for Flask applications.

    Involves a hack to allow views and tests importing the celery instance from extensions.py to access the regular
    Celery instance methods. This is done by subclassing celery.Celery and overwriting celery._state._register_app()
    with a lambda/function that does nothing at all.

    That way, on the first super() in this class' __init__(), all of the required instance objects are initialized, but
    the Celery application is not registered. This class will be initialized in extensions.py but at that moment the
    Flask application is not yet available.

    Then, once the Flask application is available, this class' init_app() method will be called, with the Flask
    application as an argument. init_app() will again call celery.Celery.__init__() but this time with the
    celery._state._register_app() restored to its original functionality. in init_app() the actual Celery application is
    initialized like normal.
    """

    def __init__(self, app=None):
        """If app argument provided then initialize celery using application config values.

        If no app argument provided you should do initialization later with init_app method.

        :param app: Flask application instance.
        """
        self.original_register_app = _state._register_app  # Backup Celery app registration function.
        _state._register_app = lambda _: None  # Upon Celery app registration attempt, do nothing.
        super(Celery, self).__init__()
        if app is not None:
            self.init_app(app)

    def init_app(self, app):
        """Actual method to read celery settings from app configuration and initialize the celery instance.

        :param app: Flask application instance.
        """
        _state._register_app = self.original_register_app  # Restore Celery app registration function.
        if not hasattr(app, 'extensions'):
            app.extensions = dict()
        if 'celery' in app.extensions:
            raise ValueError('Already registered extension CELERY.')
        app.extensions['celery'] = _CeleryState(self, app)

        # Instantiate celery and read config.
        super(Celery, self).__init__(app.import_name, broker=app.config['CELERY_BROKER_URL'])

        # Set result backend default.
        if 'CELERY_RESULT_BACKEND' in app.config:
            self._preconf['CELERY_RESULT_BACKEND'] = app.config['CELERY_RESULT_BACKEND']

        self.conf.update(app.config)
        task_base = self.Task

        # Add Flask app context to celery instance.
        class ContextTask(task_base):
            def __call__(self, *_args, **_kwargs):
                with app.app_context():
                    return task_base.__call__(self, *_args, **_kwargs)
        setattr(ContextTask, 'abstract', True)
        setattr(self, 'Task', ContextTask)


def single_instance(func=None, lock_timeout=None, include_args=False):
    """Celery task decorator. Forces the task to have only one running instance at a time.

    Use with binded tasks (@celery.task(bind=True)).

    Modeled after:
    http://loose-bits.com/2010/10/distributed-task-locking-in-celery.html
    http://blogs.it.ox.ac.uk/inapickle/2012/01/05/python-decorators-with-optional-arguments/

    Written by @Robpol86.

    :raise OtherInstanceError: If another instance is already running.

    :param function func: The function to decorate, must be also decorated by @celery.task.
    :param int lock_timeout: Lock timeout in seconds plus five more seconds, in-case the task crashes and fails to
        release the lock. If not specified, the values of the task's soft/hard limits are used. If all else fails,
        timeout will be 5 minutes.
    :param bool include_args: Include the md5 checksum of the arguments passed to the task in the Redis key. This allows
        the same task to run with different arguments, only stopping a task from running if another instance of it is
        running with the same arguments.
    """
    if func is None:
        return partial(single_instance, lock_timeout=lock_timeout, include_args=include_args)

    @wraps(func)
    def wrapped(celery_self, *args, **kwargs):
        """Wrapped Celery task, for single_instance()."""
        # Select the manager and get timeout.
        timeout = (
            lock_timeout or celery_self.soft_time_limit or celery_self.time_limit
            or celery_self.app.conf.get('CELERYD_TASK_SOFT_TIME_LIMIT')
            or celery_self.app.conf.get('CELERYD_TASK_TIME_LIMIT')
            or (60 * 5)
        )
        manager_class = _select_manager(celery_self.backend.__class__.__name__)
        lock_manager = manager_class(celery_self, timeout, include_args, args, kwargs)

        # Lock and execute.
        with lock_manager:
            ret_value = func(*args, **kwargs)
        return ret_value
    return wrapped


================================================
FILE: setup.py
================================================
#!/usr/bin/env python
"""Setup script for the project."""

import codecs
import os
import re

from setuptools import Command, setup

IMPORT = 'flask_celery'
INSTALL_REQUIRES = ['flask', 'celery']
LICENSE = 'MIT'
NAME = 'Flask-Celery-Helper'
VERSION = '1.1.0'


def readme(path='README.rst'):
    """Try to read README.rst or return empty string if failed.

    :param str path: Path to README file.

    :return: File contents.
    :rtype: str
    """
    path = os.path.realpath(os.path.join(os.path.dirname(__file__), path))
    handle = None
    url_prefix = 'https://raw.githubusercontent.com/Robpol86/{name}/v{version}/'.format(name=NAME, version=VERSION)
    try:
        handle = codecs.open(path, encoding='utf-8')
        return handle.read(131072).replace('.. image:: docs', '.. image:: {0}docs'.format(url_prefix))
    except IOError:
        return ''
    finally:
        getattr(handle, 'close', lambda: None)()


class CheckVersion(Command):
    """Make sure version strings and other metadata match here, in module/package, tox, and other places."""

    description = 'verify consistent version/etc strings in project'
    user_options = []

    @classmethod
    def initialize_options(cls):
        """Required by distutils."""
        pass

    @classmethod
    def finalize_options(cls):
        """Required by distutils."""
        pass

    @classmethod
    def run(cls):
        """Check variables."""
        project = __import__(IMPORT, fromlist=[''])
        for expected, var in [('@Robpol86', '__author__'), (LICENSE, '__license__'), (VERSION, '__version__')]:
            if getattr(project, var) != expected:
                raise SystemExit('Mismatch: {0}'.format(var))
        # Check changelog.
        if not re.compile(r'^%s - \d{4}-\d{2}-\d{2}[\r\n]' % VERSION, re.MULTILINE).search(readme()):
            raise SystemExit('Version not found in readme/changelog file.')
        # Check tox.
        if INSTALL_REQUIRES:
            contents = readme('tox.ini')
            section = re.compile(r'[\r\n]+install_requires =[\r\n]+(.+?)[\r\n]+\w', re.DOTALL).findall(contents)
            if not section:
                raise SystemExit('Missing install_requires section in tox.ini.')
            in_tox = re.findall(r'    ([^=]+)==[\w\d.-]+', section[0])
            if INSTALL_REQUIRES != in_tox:
                raise SystemExit('Missing/unordered pinned dependencies in tox.ini.')


if __name__ == '__main__':
    setup(
        author='@Robpol86',
        author_email='robpol86@gmail.com',
        classifiers=[
            'Development Status :: 5 - Production/Stable',
            'Environment :: Web Environment',
            'Environment :: MacOS X',
            'Environment :: Win32 (MS Windows)',
            'Framework :: Flask',
            'Intended Audience :: Developers',
            'License :: OSI Approved :: MIT License',
            'Operating System :: MacOS :: MacOS X',
            'Operating System :: Microsoft :: Windows',
            'Operating System :: POSIX',
            'Operating System :: POSIX :: Linux',
            'Programming Language :: Python :: 2.6',
            'Programming Language :: Python :: 2.7',
            'Programming Language :: Python :: 3.3',
            'Programming Language :: Python :: 3.4',
            'Programming Language :: Python :: Implementation :: PyPy',
            'Topic :: Software Development :: Libraries',
        ],
        cmdclass=dict(check_version=CheckVersion),
        description='Celery support for Flask without breaking PyCharm inspections.',
        install_requires=INSTALL_REQUIRES,
        keywords='flask celery redis',
        license=LICENSE,
        long_description=readme(),
        name=NAME,
        py_modules=[IMPORT],
        url='https://github.com/Robpol86/' + NAME,
        version=VERSION,
        zip_safe=False,
    )


================================================
FILE: tests/__init__.py
================================================
"""Allows importing."""


================================================
FILE: tests/conftest.py
================================================
"""Configure tests."""

import threading
import time

import pytest
from celery.signals import worker_ready

from tests.instances import app, celery

WORKER_READY = list()


class Worker(threading.Thread):
    """Run the Celery worker in a background thread."""

    def run(self):
        """Run the thread."""
        celery_args = ['-C', '-q', '-c', '1', '-P', 'solo', '--without-gossip']
        with app.app_context():
            celery.worker_main(celery_args)


@worker_ready.connect
def on_worker_ready(**_):
    """Called when the Celery worker thread is ready to do work.

    This is to avoid race conditions since everything is in one python process.
    """
    WORKER_READY.append(True)


@pytest.fixture(autouse=True, scope='session')
def celery_worker():
    """Start the Celery worker in a background thread."""
    thread = Worker()
    thread.daemon = True
    thread.start()
    for i in range(10):  # Wait for worker to finish initializing to avoid a race condition I've been experiencing.
        if WORKER_READY:
            break
        time.sleep(1)


================================================
FILE: tests/instances.py
================================================
"""Handle Flask and Celery application global-instances."""

import os

from flask import Flask
from flask_redis import Redis
from flask_sqlalchemy import SQLAlchemy

from flask_celery import Celery, single_instance


def generate_config():
    """Generate a Flask config dict with settings for a specific broker based on an environment variable.

    To be merged into app.config.

    :return: Flask config to be fed into app.config.update().
    :rtype: dict
    """
    config = dict()

    if os.environ.get('BROKER') == 'rabbit':
        config['CELERY_BROKER_URL'] = 'amqp://user:pass@localhost//'
    elif os.environ.get('BROKER') == 'redis':
        config['REDIS_URL'] = 'redis://localhost/1'
        config['CELERY_BROKER_URL'] = config['REDIS_URL']
    elif os.environ.get('BROKER', '').startswith('redis_sock,'):
        config['REDIS_URL'] = 'redis+socket://' + os.environ['BROKER'].split(',', 1)[1]
        config['CELERY_BROKER_URL'] = config['REDIS_URL']
    elif os.environ.get('BROKER') == 'mongo':
        config['CELERY_BROKER_URL'] = 'mongodb://user:pass@localhost/test'
    elif os.environ.get('BROKER') == 'couch':
        config['CELERY_BROKER_URL'] = 'couchdb://user:pass@localhost/test'
    elif os.environ.get('BROKER') == 'beanstalk':
        config['CELERY_BROKER_URL'] = 'beanstalk://user:pass@localhost/test'
    elif os.environ.get('BROKER') == 'iron':
        config['CELERY_BROKER_URL'] = 'ironmq://project:token@/test'
    else:
        if os.environ.get('BROKER') == 'mysql':
            config['SQLALCHEMY_DATABASE_URI'] = 'mysql+pymysql://user:pass@localhost/flask_celery_helper_test'
        elif os.environ.get('BROKER') == 'postgres':
            config['SQLALCHEMY_DATABASE_URI'] = 'postgresql+pg8000://user1:pass@localhost/flask_celery_helper_test'
        else:
            file_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'test_database.sqlite')
            config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + file_path
        config['CELERY_BROKER_URL'] = 'sqla+' + config['SQLALCHEMY_DATABASE_URI']
        config['CELERY_RESULT_BACKEND'] = 'db+' + config['SQLALCHEMY_DATABASE_URI']

    if 'CELERY_BROKER_URL' in config and 'CELERY_RESULT_BACKEND' not in config:
        config['CELERY_RESULT_BACKEND'] = config['CELERY_BROKER_URL']

    return config


def generate_context(config):
    """Create the Flask app context and initializes any extensions such as Celery, Redis, SQLAlchemy, etc.

    :param dict config: Partial Flask config dict from generate_config().

    :return: The Flask app instance.
    """
    flask_app = Flask(__name__)
    flask_app.config.update(config)
    flask_app.config['TESTING'] = True
    flask_app.config['CELERY_ACCEPT_CONTENT'] = ['pickle']

    if 'SQLALCHEMY_DATABASE_URI' in flask_app.config:
        db = SQLAlchemy(flask_app)
        db.engine.execute('DROP TABLE IF EXISTS celery_tasksetmeta;')
    elif 'REDIS_URL' in flask_app.config:
        redis = Redis(flask_app)
        redis.flushdb()

    Celery(flask_app)
    return flask_app


def get_flask_celery_apps():
    """Call generate_context() and generate_config().

    :return: First item is the Flask app instance, second is the Celery app instance.
    :rtype: tuple
    """
    config = generate_config()
    flask_app = generate_context(config=config)
    celery_app = flask_app.extensions['celery'].celery
    return flask_app, celery_app


app, celery = get_flask_celery_apps()


@celery.task(bind=True)
@single_instance
def add(x, y):
    """Celery task: add numbers."""
    return x + y


@celery.task(bind=True)
@single_instance(include_args=True, lock_timeout=20)
def mul(x, y):
    """Celery task: multiply numbers."""
    return x * y


@celery.task(bind=True)
@single_instance()
def sub(x, y):
    """Celery task: subtract numbers."""
    return x - y


@celery.task(bind=True, time_limit=70)
@single_instance
def add2(x, y):
    """Celery task: add numbers."""
    return x + y


@celery.task(bind=True, soft_time_limit=80)
@single_instance
def add3(x, y):
    """Celery task: add numbers."""
    return x + y


================================================
FILE: tests/test_class.py
================================================
"""Test the Celery class."""

import pytest

from flask_celery import Celery
from tests.instances import app


class FakeApp(object):
    """Mock Flask application."""

    config = dict(CELERY_BROKER_URL='redis://localhost', CELERY_RESULT_BACKEND='redis://localhost')
    static_url_path = ''
    import_name = ''

    def register_blueprint(self, _):
        """Mock register_blueprint method."""
        pass


def test_multiple():
    """Test attempted re-initialization of extension."""
    assert 'celery' in app.extensions

    with pytest.raises(ValueError):
        Celery(app)


def test_one_dumb_line():
    """For test coverage."""
    flask_app = FakeApp()
    Celery(flask_app)
    assert 'celery' in flask_app.extensions


================================================
FILE: tests/test_collision.py
================================================
"""Test single-instance collision."""

import pytest

from flask_celery import _select_manager, OtherInstanceError
from tests.instances import celery

PARAMS = [('tests.instances.add', 8), ('tests.instances.mul', 16), ('tests.instances.sub', 0)]


@pytest.mark.parametrize('task_name,expected', PARAMS)
def test_basic(task_name, expected):
    """Test no collision."""
    task = celery.tasks[task_name]
    assert expected == task.apply_async(args=(4, 4)).get()


@pytest.mark.parametrize('task_name,expected', PARAMS)
def test_collision(task_name, expected):
    """Test single-instance collision."""
    manager_class = _select_manager(celery.backend.__class__.__name__)
    manager_instance = list()
    task = celery.tasks[task_name]

    # First run the task and prevent it from removing the lock.
    def new_exit(self, *_):
        manager_instance.append(self)
        return None
    original_exit = manager_class.__exit__
    setattr(manager_class, '__exit__', new_exit)
    assert expected == task.apply_async(args=(4, 4)).get()
    setattr(manager_class, '__exit__', original_exit)
    assert manager_instance[0].is_already_running is True

    # Now run it again.
    with pytest.raises(OtherInstanceError) as e:
        task.apply_async(args=(4, 4)).get()
    if manager_instance[0].include_args:
        assert str(e.value).startswith('Failed to acquire lock, {0}.args.'.format(task_name))
    else:
        assert 'Failed to acquire lock, {0} already running.'.format(task_name) == str(e.value)
    assert manager_instance[0].is_already_running is True

    # Clean up.
    manager_instance[0].reset_lock()
    assert manager_instance[0].is_already_running is False

    # Once more.
    assert expected == task.apply_async(args=(4, 4)).get()


def test_include_args():
    """Test single-instance collision with task arguments taken into account."""
    manager_class = _select_manager(celery.backend.__class__.__name__)
    manager_instance = list()
    task = celery.tasks['tests.instances.mul']

    # First run the tasks and prevent them from removing the locks.
    def new_exit(self, *_):
        """Expected to be run twice."""
        manager_instance.append(self)
        return None
    original_exit = manager_class.__exit__
    setattr(manager_class, '__exit__', new_exit)
    assert 16 == task.apply_async(args=(4, 4)).get()
    assert 20 == task.apply_async(args=(5, 4)).get()
    setattr(manager_class, '__exit__', original_exit)
    assert manager_instance[0].is_already_running is True
    assert manager_instance[1].is_already_running is True

    # Now run them again.
    with pytest.raises(OtherInstanceError) as e:
        task.apply_async(args=(4, 4)).get()
    assert str(e.value).startswith('Failed to acquire lock, tests.instances.mul.args.')
    assert manager_instance[0].is_already_running is True
    with pytest.raises(OtherInstanceError) as e:
        task.apply_async(args=(5, 4)).get()
    assert str(e.value).startswith('Failed to acquire lock, tests.instances.mul.args.')
    assert manager_instance[1].is_already_running is True

    # Clean up.
    manager_instance[0].reset_lock()
    assert manager_instance[0].is_already_running is False
    manager_instance[1].reset_lock()
    assert manager_instance[1].is_already_running is False

    # Once more.
    assert 16 == task.apply_async(args=(4, 4)).get()
    assert 20 == task.apply_async(args=(5, 4)).get()


================================================
FILE: tests/test_timeout.py
================================================
"""Test single-instance lock timeout."""

import time

import pytest

from flask_celery import _select_manager, OtherInstanceError
from tests.instances import celery


@pytest.mark.parametrize('task_name,timeout', [
    ('tests.instances.mul', 20), ('tests.instances.add', 300), ('tests.instances.add2', 70),
    ('tests.instances.add3', 80)
])
def test_instances(task_name, timeout):
    """Test task instances."""
    manager_class = _select_manager(celery.backend.__class__.__name__)
    manager_instance = list()
    task = celery.tasks[task_name]
    original_exit = manager_class.__exit__

    def new_exit(self, *_):
        manager_instance.append(self)
        return original_exit(self, *_)
    setattr(manager_class, '__exit__', new_exit)
    task.apply_async(args=(4, 4)).get()
    setattr(manager_class, '__exit__', original_exit)
    assert timeout == manager_instance[0].timeout


@pytest.mark.parametrize('key,value', [('CELERYD_TASK_TIME_LIMIT', 200), ('CELERYD_TASK_SOFT_TIME_LIMIT', 100)])
def test_settings(key, value):
    """Test different Celery time limit settings."""
    celery.conf.update({key: value})
    manager_class = _select_manager(celery.backend.__class__.__name__)
    manager_instance = list()
    original_exit = manager_class.__exit__

    def new_exit(self, *_):
        manager_instance.append(self)
        return original_exit(self, *_)
    setattr(manager_class, '__exit__', new_exit)
    tasks = [
        ('tests.instances.mul', 20), ('tests.instances.add', value), ('tests.instances.add2', 70),
        ('tests.instances.add3', 80)
    ]

    for task_name, timeout in tasks:
        task = celery.tasks[task_name]
        task.apply_async(args=(4, 4)).get()
        assert timeout == manager_instance.pop().timeout
    setattr(manager_class, '__exit__', original_exit)

    celery.conf.update({key: None})


def test_expired():
    """Test timeout expired task instances."""
    celery.conf.update({'CELERYD_TASK_TIME_LIMIT': 5})
    manager_class = _select_manager(celery.backend.__class__.__name__)
    manager_instance = list()
    task = celery.tasks['tests.instances.add']
    original_exit = manager_class.__exit__

    def new_exit(self, *_):
        manager_instance.append(self)
        return None
    setattr(manager_class, '__exit__', new_exit)

    # Run the task and don't remove the lock after a successful run.
    assert 8 == task.apply_async(args=(4, 4)).get()
    setattr(manager_class, '__exit__', original_exit)

    # Run again, lock is still active so this should fail.
    with pytest.raises(OtherInstanceError):
        task.apply_async(args=(4, 4)).get()

    # Wait 5 seconds (per CELERYD_TASK_TIME_LIMIT), then re-run, should work.
    time.sleep(5)
    assert 8 == task.apply_async(args=(4, 4)).get()
    celery.conf.update({'CELERYD_TASK_TIME_LIMIT': None})


================================================
FILE: tox.ini
================================================
[general]
install_requires =
    flask==0.12
    celery==3.1.11
name = flask_celery

[tox]
envlist = lint,py{34,27}

[testenv]
commands =
    py.test --cov-report term-missing --cov-report xml --cov {[general]name} --cov-config tox.ini {posargs:tests}
deps =
    {[general]install_requires}
    Flask-Redis-Helper==1.0.0
    Flask-SQLAlchemy==2.1
    pg8000==1.10.6
    PyMySQL==0.7.9
    pytest-cov==2.4.0
passenv =
    BROKER
usedevelop = True

[testenv:lint]
commands =
    python setup.py check --strict
    python setup.py check --strict -m
    python setup.py check --strict -s
    python setup.py check_version
    flake8 --application-import-names={[general]name},tests
    pylint --rcfile=tox.ini setup.py {[general]name}
deps =
    {[general]install_requires}
    flake8-docstrings==1.0.3
    flake8-import-order==0.11
    flake8==3.2.1
    pep8-naming==0.4.1
    pylint==1.6.5

[flake8]
exclude = .tox/*,build/*,docs/*,env/*,get-pip.py
import-order-style = smarkets
max-line-length = 120
statistics = True

[pylint]
disable =
    locally-disabled,
    missing-docstring,
    protected-access,
    too-few-public-methods,
ignore = .tox/*,build/*,docs/*,env/*,get-pip.py
max-args = 7
max-line-length = 120
reports = no

[run]
branch = True