pax_global_header00006660000000000000000000000064124501422640014512gustar00rootroot0000000000000052 comment=c58c730f6c7b1fb29350d02684adf7e5b5182a3b Flask-Celery-Helper-1.1.0/000077500000000000000000000000001245014226400152075ustar00rootroot00000000000000Flask-Celery-Helper-1.1.0/.gitignore000066400000000000000000000010611245014226400171750ustar00rootroot00000000000000# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] # C extensions *.so # Distribution / packaging .Python env/ bin/ build/ develop-eggs/ dist/ eggs/ lib/ lib64/ parts/ sdist/ var/ *.egg-info/ .installed.cfg *.egg # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .cache nosetests.xml coverage.xml # Translations *.mo # Mr Developer .mr.developer.cfg .project .pydevproject # Rope .ropeproject # Django stuff: *.log *.pot # Sphinx documentation docs/_build/ # IntelliJ .idea Flask-Celery-Helper-1.1.0/.travis.yml000066400000000000000000000030661245014226400173250ustar00rootroot00000000000000language: python python: - 2.6 - 2.7 - pypy - 3.3 - 3.4 services: - redis-server #- rabbitmq #- mongodb #- couchdb install: - python -c 'import setup; print(setup.REQUIRES_PIP)' |xargs pip install - pip install robpol86-codecov - pip install . before_script: - echo -e "daemonize yes\nunixsocket /tmp/redis.sock\nport 0" |redis-server - - mysql -u root -e 'CREATE DATABASE flask_celery_helper_test;' - mysql -u root -e 'GRANT ALL PRIVILEGES ON flask_celery_helper_test.* TO "user"@"localhost" IDENTIFIED BY "pass";' - psql -U postgres -c 'CREATE DATABASE flask_celery_helper_test;' - psql -U postgres -c "CREATE USER user1 WITH PASSWORD 'pass';" - psql -U postgres -c 'GRANT ALL PRIVILEGES ON DATABASE flask_celery_helper_test TO user1;' script: - python setup.py test && mv .coverage .coverage.sqlite # SQLAlchemy SQLite - BROKER=mysql python setup.py test && mv .coverage .coverage.mysql - BROKER=postgres python setup.py test && mv .coverage .coverage.postgres - BROKER=redis python setup.py test && mv .coverage .coverage.redis - BROKER=redis_sock,/tmp/redis.sock python setup.py test && mv .coverage .coverage.redis_sock #- BROKER=mongo python setup.py test && mv .coverage .coverage.mongo #- BROKER=couch python setup.py test && mv .coverage .coverage.couch #- BROKER=beanstalk python setup.py test && mv .coverage .coverage.beanstalk #- BROKER=iron python setup.py test && mv .coverage .coverage.iron #- BROKER=rabbit python setup.py test && mv .coverage .coverage.rabbit after_success: - coverage combine - codecov Flask-Celery-Helper-1.1.0/LICENSE000066400000000000000000000020621245014226400162140ustar00rootroot00000000000000The MIT License (MIT) Copyright (c) 2014 Robpol86 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.Flask-Celery-Helper-1.1.0/README.rst000066400000000000000000000116011245014226400166750ustar00rootroot00000000000000Flask-Celery-Helper =================== Even though the `Flask documentation `_ says Celery extensions are unnecessary now, I found that I still need an extension to properly use Celery in large Flask applications. Specifically I need an init_app() method to initialize Celery after I instantiate it. This extension also comes with a ``single_instance`` method. * Python 2.6, 2.7, 3.3, and 3.4 supported on Linux and OS X. * Python 2.7, 3.3, and 3.4 supported on Windows (both 32 and 64 bit versions of Python). .. image:: https://img.shields.io/appveyor/ci/Robpol86/Flask-Celery-Helper.svg?style=flat-square :target: https://ci.appveyor.com/project/Robpol86/Flask-Celery-Helper :alt: Build Status Windows .. image:: https://img.shields.io/travis/Robpol86/Flask-Celery-Helper/master.svg?style=flat-square :target: https://travis-ci.org/Robpol86/Flask-Celery-Helper :alt: Build Status .. image:: https://img.shields.io/codecov/c/github/Robpol86/Flask-Celery-Helper/master.svg?style=flat-square :target: https://codecov.io/github/Robpol86/Flask-Celery-Helper :alt: Coverage Status .. image:: https://img.shields.io/pypi/v/Flask-Celery-Helper.svg?style=flat-square :target: https://pypi.python.org/pypi/Flask-Celery-Helper/ :alt: Latest Version .. image:: https://img.shields.io/pypi/dm/Flask-Celery-Helper.svg?style=flat-square :target: https://pypi.python.org/pypi/Flask-Celery-Helper/ :alt: Downloads Attribution ----------- Single instance decorator inspired by `Ryan Roemer `_. Supported Platforms ------------------- * OSX and Linux. * Python 2.6, 2.7, 3.3, 3.4 * `Flask `_ 0.10.1 * `Redis `_ 2.9.1 * `Celery `_ 3.1.11 Quickstart ---------- Install: .. code:: bash pip install Flask-Celery-Helper Example: .. code:: python # example.py from flask import Flask from flask.ext.celery import Celery app = Flask('example') app.config['CELERY_BROKER_URL'] = 'redis://localhost' app.config['CELERY_RESULT_BACKEND'] = 'redis://localhost' celery = Celery(app) @celery.task() def add_together(a, b): return a + b if __name__ == '__main__': result = add_together.delay(23, 42) print(result.get()) Run these two commands in separate terminals: .. code:: bash celery -A example.celery worker python example.py Factory Example --------------- .. code:: python # extensions.py from flask.ext.celery import Celery celery = Celery() .. code:: python # application.py from flask import Flask from extensions import celery def create_app(): app = Flask(__name__) app.config['CELERY_IMPORTS'] = ('tasks.add_together', ) app.config['CELERY_BROKER_URL'] = 'redis://localhost' app.config['CELERY_RESULT_BACKEND'] = 'redis://localhost' celery.init_app(app) return app .. code:: python # tasks.py from extensions import celery @celery.task() def add_together(a, b): return a + b .. code:: python # manage.py from application import create_app app = create_app() app.run() Single Instance Example ----------------------- .. code:: python # example.py import time from flask import Flask from flask.ext.celery import Celery, single_instance from flask.ext.redis import Redis app = Flask('example') app.config['REDIS_URL'] = 'redis://localhost' app.config['CELERY_BROKER_URL'] = 'redis://localhost' app.config['CELERY_RESULT_BACKEND'] = 'redis://localhost' celery = Celery(app) Redis(app) @celery.task(bind=True) @single_instance def sleep_one_second(a, b): time.sleep(1) return a + b if __name__ == '__main__': task1 = sleep_one_second.delay(23, 42) time.sleep(0.1) task2 = sleep_one_second.delay(20, 40) results1 = task1.get(propagate=False) results2 = task2.get(propagate=False) print(results1) # 65 if isinstance(results2, Exception) and str(results2) == 'Failed to acquire lock.': print('Another instance is already running.') else: print(results2) # Should not happen. Changelog --------- 1.1.0 ````` * Added Windows support. * ``CELERY_RESULT_BACKEND`` no longer mandatory. * ``single_instance`` supported on SQLite/MySQL/PostgreSQL in addition to Redis. * Breaking changes: ``flask.ext.celery.CELERY_LOCK`` moved to ``flask.ext.celery._LockManagerRedis.CELERY_LOCK``. 1.0.0 ````` * Support for non-Redis backends. 0.2.2 ````` * Added Python 2.6 and 3.x support. 0.2.1 ````` * Fixed ``single_instance`` arguments with functools. 0.2.0 ````` * Added include_args argument to ``single_instance``. 0.1.0 ````` * Initial release. Flask-Celery-Helper-1.1.0/appveyor.yml000066400000000000000000000044421245014226400176030ustar00rootroot00000000000000environment: CODECOV_TOKEN: secure: SUr0uj2ygHCZaTNHy+OnVmmkR3ZJoE8wz5SaT9uK2xDSoVi4EvFg88v8JmCWtJ5r PYTHONIOENCODING: utf-8 PGPASSWORD: Password12! matrix: - PYTHON: C:\Python27 - PYTHON: C:\Python27-x64 - PYTHON: C:\Python33 - PYTHON: C:\Python33-x64 - PYTHON: C:\Python34 - PYTHON: C:\Python34-x64 init: - IF DEFINED PYTHON set PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% - set PATH=%PATH%;C:\Program Files\MySQL\MySQL Server 5.6\bin;C:\Program Files\PostgreSQL\9.3\bin install: - ps: Start-FileDownload https://raw.github.com/pypa/pip/master/contrib/get-pip.py - python get-pip.py - ps: pip install $(python -c 'import setup; print(setup.REQUIRES_PIP)') - pip install robpol86-codecov services: - mysql - postgresql build_script: - ps: mv .git .. - pip install . - ps: mv ..\.git . after_build: - cinst redis-64 - redis-server --service-install - redis-server --service-start - mysql -u root -p"Password12!" -e "CREATE DATABASE flask_celery_helper_test;" - mysql -u root -p"Password12!" -e "GRANT ALL PRIVILEGES ON flask_celery_helper_test.* TO 'user'@'localhost' IDENTIFIED BY 'pass';" - psql -U postgres -c "CREATE DATABASE flask_celery_helper_test;" - psql -U postgres -c "CREATE USER user1 WITH PASSWORD 'pass';" - psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE flask_celery_helper_test TO user1;" test_script: - python setup.py test && ren .coverage .coverage.sqlite # SQLAlchemy SQLite - cmd /C "set BROKER=mysql&& python setup.py test && ren .coverage .coverage.mysql - cmd /C "set BROKER=postgres&& python setup.py test && ren .coverage .coverage.postgres - cmd /C "set BROKER=redis&& python setup.py test && ren .coverage .coverage.redis #- cmd /C "set BROKER=redis_sock,/tmp/redis.sock&& python setup.py test && ren .coverage .coverage.redis_sock #- cmd /C "set BROKER=mongo&& python setup.py test && ren .coverage .coverage.mongo #- cmd /C "set BROKER=couch&& python setup.py test && ren .coverage .coverage.couch #- cmd /C "set BROKER=beanstalk&& python setup.py test && ren .coverage .coverage.beanstalk #- cmd /C "set BROKER=iron&& python setup.py test && ren .coverage .coverage.iron #- cmd /C "set BROKER=rabbit&& python setup.py test && ren .coverage .coverage.rabbit after_test: - coverage combine - codecov Flask-Celery-Helper-1.1.0/flask_celery.py000066400000000000000000000266461245014226400202420ustar00rootroot00000000000000"""Celery support for Flask without breaking PyCharm inspections. https://github.com/Robpol86/Flask-Celery-Helper https://pypi.python.org/pypi/Flask-Celery-Helper """ from datetime import datetime, timedelta from functools import partial, wraps import hashlib from logging import getLogger from celery import _state, Celery as CeleryClass __author__ = '@Robpol86' __license__ = 'MIT' __version__ = '1.1.0' class OtherInstanceError(Exception): """Raised when Celery task is already running, when lock exists and has not timed out.""" pass class _LockManager(object): """Base class for other lock managers.""" def __init__(self, celery_self, timeout, include_args, args, kwargs): """May raise NotImplementedError if the Celery backend is not supported. Positional arguments: celery_self -- from wrapped() within single_instance(). It is the `self` object specified in a binded Celery task definition (implicit first argument of the Celery task when @celery.task(bind=True) is used). timeout -- lock's timeout value in seconds. include_args -- if single instance should take arguments into account (boolean). args -- the task instance's args. kwargs -- the task instance's kwargs. """ self.celery_self = celery_self self.timeout = timeout self.include_args = include_args self.args = args self.kwargs = kwargs self.log = getLogger('{0}:{1}'.format(self.__class__.__name__, self.task_identifier)) @property def task_identifier(self): """Returns the unique identifier (string) of a task instance.""" task_id = self.celery_self.name if self.include_args: merged_args = str(self.args) + str([(k, self.kwargs[k]) for k in sorted(self.kwargs)]) task_id += '.args.{0}'.format(hashlib.md5(merged_args.encode('utf-8')).hexdigest()) return task_id class _LockManagerRedis(_LockManager): """Handles locking/unlocking for Redis backends.""" CELERY_LOCK = '_celery.single_instance.{task_id}' def __init__(self, celery_self, timeout, include_args, args, kwargs): super(_LockManagerRedis, self).__init__(celery_self, timeout, include_args, args, kwargs) self.lock = None def __enter__(self): redis_key = self.CELERY_LOCK.format(task_id=self.task_identifier) self.lock = self.celery_self.backend.client.lock(redis_key, timeout=self.timeout) self.log.debug('Timeout %ds | Redis key %s', self.timeout, redis_key) if not self.lock.acquire(blocking=False): self.log.debug('Another instance is running.') raise OtherInstanceError('Failed to acquire lock, {0} already running.'.format(self.task_identifier)) else: self.log.debug('Got lock, running.') def __exit__(self, exc_type, *_): if exc_type == OtherInstanceError: # Failed to get lock last time, not releasing. return self.log.debug('Releasing lock.') self.lock.release() @property def is_already_running(self): """Returns True if lock exists and has not timed out.""" redis_key = self.CELERY_LOCK.format(task_id=self.task_identifier) return self.celery_self.backend.client.exists(redis_key) def reset_lock(self): """Removed the lock regardless of timeout.""" redis_key = self.CELERY_LOCK.format(task_id=self.task_identifier) self.celery_self.backend.client.delete(redis_key) class _LockManagerDB(_LockManager): """Handles locking/unlocking for SQLite/MySQL/PostgreSQL/etc backends.""" def __init__(self, celery_self, timeout, include_args, args, kwargs): super(_LockManagerDB, self).__init__(celery_self, timeout, include_args, args, kwargs) self.save_group = getattr(self.celery_self.backend, '_save_group') self.restore_group = getattr(self.celery_self.backend, '_restore_group') self.delete_group = getattr(self.celery_self.backend, '_delete_group') def __enter__(self): self.log.debug('Timeout %ds', self.timeout) try: self.save_group(self.task_identifier, None) except Exception as exc: if 'IntegrityError' not in str(exc) and 'ProgrammingError' not in str(exc): raise difference = datetime.utcnow() - self.restore_group(self.task_identifier)['date_done'] if difference < timedelta(seconds=self.timeout): self.log.debug('Another instance is running.') raise OtherInstanceError('Failed to acquire lock, {0} already running.'.format(self.task_identifier)) self.log.debug('Timeout expired, stale lock found, releasing lock.') self.delete_group(self.task_identifier) self.save_group(self.task_identifier, None) self.log.debug('Got lock, running.') def __exit__(self, exc_type, *_): if exc_type == OtherInstanceError: # Failed to get lock last time, not releasing. return self.log.debug('Releasing lock.') self.delete_group(self.task_identifier) @property def is_already_running(self): """Returns True if lock exists and has not timed out.""" date_done = (self.restore_group(self.task_identifier) or dict()).get('date_done') if not date_done: return False difference = datetime.utcnow() - date_done return difference < timedelta(seconds=self.timeout) def reset_lock(self): """Removed the lock regardless of timeout.""" self.delete_group(self.task_identifier) def _select_manager(backend_name): """Selects the proper LockManager based on the current backend used by Celery. Raises: NotImplementedError if Celery is using an unsupported backend. Positional arguments: backend -- Class name of the current Celery backend. Usually value of: current_app.extensions['celery'].celery.backend.__class__.__name__. Returns: Class definition object (not instance). One of the _LockManager* classes. """ if backend_name == 'RedisBackend': lock_manager = _LockManagerRedis elif backend_name == 'DatabaseBackend': lock_manager = _LockManagerDB else: raise NotImplementedError return lock_manager class _CeleryState(object): """Remembers the configuration for the (celery, app) tuple. Modeled from SQLAlchemy.""" def __init__(self, celery, app): self.celery = celery self.app = app # noinspection PyProtectedMember class Celery(CeleryClass): """Celery extension for Flask applications. Involves a hack to allow views and tests importing the celery instance from extensions.py to access the regular Celery instance methods. This is done by subclassing celery.Celery and overwriting celery._state._register_app() with a lambda/function that does nothing at all. That way, on the first super() in this class' __init__(), all of the required instance objects are initialized, but the Celery application is not registered. This class will be initialized in extensions.py but at that moment the Flask application is not yet available. Then, once the Flask application is available, this class' init_app() method will be called, with the Flask application as an argument. init_app() will again call celery.Celery.__init__() but this time with the celery._state._register_app() restored to its original functionality. in init_app() the actual Celery application is initialized like normal. """ def __init__(self, app=None): """If app argument provided then initialize celery using application config values. If no app argument provided you should do initialization later with init_app method. Keyword arguments: app -- Flask application instance. """ self.original_register_app = _state._register_app # Backup Celery app registration function. _state._register_app = lambda _: None # Upon Celery app registration attempt, do nothing. super(Celery, self).__init__() if app is not None: self.init_app(app) def init_app(self, app): """Actual method to read celery settings from app configuration and initialize the celery instance. Positional arguments: app -- Flask application instance. """ _state._register_app = self.original_register_app # Restore Celery app registration function. if not hasattr(app, 'extensions'): app.extensions = dict() if 'celery' in app.extensions: raise ValueError('Already registered extension CELERY.') app.extensions['celery'] = _CeleryState(self, app) # Instantiate celery and read config. super(Celery, self).__init__(app.import_name, broker=app.config['CELERY_BROKER_URL']) # Set result backend default. if 'CELERY_RESULT_BACKEND' in app.config: self._preconf['CELERY_RESULT_BACKEND'] = app.config['CELERY_RESULT_BACKEND'] self.conf.update(app.config) task_base = self.Task # Add Flask app context to celery instance. class ContextTask(task_base): """Celery instance wrapped within the Flask app context.""" def __call__(self, *_args, **_kwargs): with app.app_context(): return task_base.__call__(self, *_args, **_kwargs) setattr(ContextTask, 'abstract', True) setattr(self, 'Task', ContextTask) def single_instance(func=None, lock_timeout=None, include_args=False): """Celery task decorator. Forces the task to have only one running instance at a time. Use with binded tasks (@celery.task(bind=True)). Modeled after: http://loose-bits.com/2010/10/distributed-task-locking-in-celery.html http://blogs.it.ox.ac.uk/inapickle/2012/01/05/python-decorators-with-optional-arguments/ Written by @Robpol86. Raises: OtherInstanceError -- if another instance is already running. Positional arguments: func -- the function to decorate, must be also decorated by @celery.task. Keyword arguments: lock_timeout -- lock timeout in seconds plus five more seconds, in-case the task crashes and fails to release the lock. If not specified, the values of the task's soft/hard limits are used. If all else fails, timeout will be 5 minutes. include_args -- include the md5 checksum of the arguments passed to the task in the Redis key. This allows the same task to run with different arguments, only stopping a task from running if another instance of it is running with the same arguments. """ if func is None: return partial(single_instance, lock_timeout=lock_timeout, include_args=include_args) @wraps(func) def wrapped(celery_self, *args, **kwargs): """Wrapped Celery task, for single_instance().""" # Select the manager and get timeout. timeout = ( lock_timeout or celery_self.soft_time_limit or celery_self.time_limit or celery_self.app.conf.get('CELERYD_TASK_SOFT_TIME_LIMIT') or celery_self.app.conf.get('CELERYD_TASK_TIME_LIMIT') or (60 * 5) ) manager_class = _select_manager(celery_self.backend.__class__.__name__) lock_manager = manager_class(celery_self, timeout, include_args, args, kwargs) # Lock and execute. with lock_manager: ret_value = func(*args, **kwargs) return ret_value return wrapped Flask-Celery-Helper-1.1.0/setup.py000077500000000000000000000105261245014226400167300ustar00rootroot00000000000000#!/usr/bin/env python import ast import atexit from codecs import open from distutils.spawn import find_executable import os import sys import subprocess import setuptools.command.sdist from setuptools.command.test import test DESCRIPTION = 'Celery support for Flask without breaking PyCharm inspections.' HERE = os.path.abspath(os.path.dirname(__file__)) KEYWORDS = 'flask celery redis' NAME = 'Flask-Celery-Helper' NAME_FILE = 'flask_celery' PACKAGE = False REQUIRES_INSTALL = ['Flask', 'celery'] REQUIRES_TEST = ['pytest-cov', 'Flask-Redis-Helper', 'Flask-SQLAlchemy', 'PyMySQL', 'pg8000'] REQUIRES_PIP = '"' + '" "'.join(set(REQUIRES_INSTALL + REQUIRES_TEST)) + '"' def get_metadata(main_file): """Get metadata about the package/module. Positional arguments: main_file -- python file path within `HERE` which has __author__ and the others defined as global variables. Returns: Dictionary to be passed into setuptools.setup(). """ with open(os.path.join(HERE, 'README.rst'), encoding='utf-8') as f: long_description = f.read(100000) with open(os.path.join(HERE, main_file), encoding='utf-8') as f: lines = [l.strip() for l in f if l.startswith('__')] metadata = ast.literal_eval("{'" + ", '".join([l.replace(' = ', "': ") for l in lines]) + '}') __author__, __license__, __version__ = [metadata[k] for k in ('__author__', '__license__', '__version__')] everything = dict(version=__version__, long_description=long_description, author=__author__, license=__license__) if not all(everything.values()): raise ValueError('Failed to obtain metadata from package/module.') return everything class PyTest(test): description = 'Run all tests.' TEST_ARGS = ['--cov-report', 'term-missing', '--cov', NAME_FILE, 'tests'] def finalize_options(self): test.finalize_options(self) setattr(self, 'test_args', self.TEST_ARGS) setattr(self, 'test_suite', True) def run_tests(self): # Import here, cause outside the eggs aren't loaded. pytest = __import__('pytest') err_no = pytest.main(self.test_args) sys.exit(err_no) class PyTestPdb(PyTest): description = 'Run all tests, drops to ipdb upon unhandled exception.' TEST_ARGS = ['--ipdb', 'tests'] class PyTestCovWeb(PyTest): description = 'Generates HTML report on test coverage.' TEST_ARGS = ['--cov-report', 'html', '--cov', NAME_FILE, 'tests'] def run_tests(self): if find_executable('open'): atexit.register(lambda: subprocess.call(['open', os.path.join(HERE, 'htmlcov', 'index.html')])) PyTest.run_tests(self) class CmdStyle(setuptools.Command): user_options = [] CMD_ARGS = ['flake8', '--max-line-length', '120', '--statistics', NAME_FILE + ('' if PACKAGE else '.py')] def initialize_options(self): pass def finalize_options(self): pass def run(self): subprocess.call(self.CMD_ARGS) class CmdLint(CmdStyle): description = 'Run pylint on entire project.' CMD_ARGS = ['pylint', '--max-line-length', '120', NAME_FILE + ('' if PACKAGE else '.py')] ALL_DATA = dict( name=NAME, description=DESCRIPTION, url='https://github.com/Robpol86/{0}'.format(NAME), author_email='robpol86@gmail.com', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Web Environment', 'Framework :: Flask', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', 'Operating System :: MacOS :: MacOS X', 'Operating System :: Microsoft :: Windows', 'Operating System :: POSIX', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Topic :: Software Development :: Libraries', ], keywords=KEYWORDS, py_modules=[NAME_FILE], zip_safe=False, install_requires=REQUIRES_INSTALL, tests_require=REQUIRES_TEST, cmdclass=dict(test=PyTest, testpdb=PyTestPdb, testcovweb=PyTestCovWeb, style=CmdStyle, lint=CmdLint), # Pass the rest from get_metadata(). **get_metadata(os.path.join(NAME_FILE + ('/__init__.py' if PACKAGE else '.py'))) ) if __name__ == '__main__': setuptools.setup(**ALL_DATA) Flask-Celery-Helper-1.1.0/tests/000077500000000000000000000000001245014226400163515ustar00rootroot00000000000000Flask-Celery-Helper-1.1.0/tests/__init__.py000066400000000000000000000000001245014226400204500ustar00rootroot00000000000000Flask-Celery-Helper-1.1.0/tests/conftest.py000066400000000000000000000020051245014226400205450ustar00rootroot00000000000000"""pytest configuration for all tests in all directories.""" import threading import time from celery.signals import worker_ready import pytest from tests.instances import app, celery WORKER_READY = list() class Worker(threading.Thread): def run(self): celery_args = ['-C', '-q', '-c', '1', '-P', 'solo', '--without-gossip'] with app.app_context(): celery.worker_main(celery_args) @worker_ready.connect def on_worker_ready(**_): """Called when the Celery worker thread is ready to do work. This is to avoid race conditions since everything is in one python process. """ WORKER_READY.append(True) @pytest.fixture(autouse=True, scope='session') def celery_worker(): """Starts the Celery worker in a background thread.""" thread = Worker() thread.daemon = True thread.start() for i in range(10): # Wait for worker to finish initializing to avoid a race condition I've been experiencing. if WORKER_READY: break time.sleep(1) Flask-Celery-Helper-1.1.0/tests/instances.py000066400000000000000000000074141245014226400207200ustar00rootroot00000000000000import os from flask import Flask from flask.ext.celery import Celery, single_instance from flask.ext.redis import Redis from flask.ext.sqlalchemy import SQLAlchemy def generate_config(): """Generates a Flask config dict with settings for a specific broker based on an environment variable. To be merged into app.config. Returns: A dict to be fed into app.config.update(). """ config = dict() if os.environ.get('BROKER') == 'rabbit': config['CELERY_BROKER_URL'] = 'amqp://user:pass@localhost//' elif os.environ.get('BROKER') == 'redis': config['REDIS_URL'] = 'redis://localhost/1' config['CELERY_BROKER_URL'] = config['REDIS_URL'] elif os.environ.get('BROKER', '').startswith('redis_sock,'): config['REDIS_URL'] = 'redis+socket://' + os.environ['BROKER'].split(',', 1)[1] config['CELERY_BROKER_URL'] = config['REDIS_URL'] elif os.environ.get('BROKER') == 'mongo': config['CELERY_BROKER_URL'] = 'mongodb://user:pass@localhost/test' elif os.environ.get('BROKER') == 'couch': config['CELERY_BROKER_URL'] = 'couchdb://user:pass@localhost/test' elif os.environ.get('BROKER') == 'beanstalk': config['CELERY_BROKER_URL'] = 'beanstalk://user:pass@localhost/test' elif os.environ.get('BROKER') == 'iron': config['CELERY_BROKER_URL'] = 'ironmq://project:token@/test' else: if os.environ.get('BROKER') == 'mysql': config['SQLALCHEMY_DATABASE_URI'] = 'mysql+pymysql://user:pass@localhost/flask_celery_helper_test' elif os.environ.get('BROKER') == 'postgres': config['SQLALCHEMY_DATABASE_URI'] = 'postgresql+pg8000://user1:pass@localhost/flask_celery_helper_test' else: file_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'test_database.sqlite') config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + file_path config['CELERY_BROKER_URL'] = 'sqla+' + config['SQLALCHEMY_DATABASE_URI'] config['CELERY_RESULT_BACKEND'] = 'db+' + config['SQLALCHEMY_DATABASE_URI'] if 'CELERY_BROKER_URL' in config and 'CELERY_RESULT_BACKEND' not in config: config['CELERY_RESULT_BACKEND'] = config['CELERY_BROKER_URL'] return config def generate_context(config): """Creates the Flask app context and initializes any extensions such as Celery, Redis, SQLAlchemy, etc. Positional arguments: config -- partial Flask config dict from generate_config(). Returns: The Flask app instance. """ flask_app = Flask(__name__) flask_app.config.update(config) flask_app.config['TESTING'] = True flask_app.config['CELERY_ACCEPT_CONTENT'] = ['pickle'] if 'SQLALCHEMY_DATABASE_URI' in flask_app.config: db = SQLAlchemy(flask_app) db.engine.execute('DROP TABLE IF EXISTS celery_tasksetmeta;') elif 'REDIS_URL' in flask_app.config: redis = Redis(flask_app) redis.flushdb() Celery(flask_app) return flask_app def get_flask_celery_apps(): """Calls generate_context() and generate_config(). Returns: Tuple, first item is the Flask app instance, second is the Celery app instance. """ config = generate_config() flask_app = generate_context(config=config) celery_app = flask_app.extensions['celery'].celery return flask_app, celery_app app, celery = get_flask_celery_apps() @celery.task(bind=True) @single_instance def add(x, y): return x + y @celery.task(bind=True) @single_instance(include_args=True, lock_timeout=20) def mul(x, y): return x * y @celery.task(bind=True) @single_instance() def sub(x, y): return x - y @celery.task(bind=True, time_limit=70) @single_instance def add2(x, y): return x + y @celery.task(bind=True, soft_time_limit=80) @single_instance def add3(x, y): return x + y Flask-Celery-Helper-1.1.0/tests/test_class.py000066400000000000000000000010371245014226400210700ustar00rootroot00000000000000from flask.ext.celery import Celery import pytest from tests.instances import app class FakeApp(object): config = dict(CELERY_BROKER_URL='redis://localhost', CELERY_RESULT_BACKEND='redis://localhost') static_url_path = '' import_name = '' def register_blueprint(self, _): pass def test_multiple(): assert 'celery' in app.extensions with pytest.raises(ValueError): Celery(app) def test_one_dumb_line(): flask_app = FakeApp() Celery(flask_app) assert 'celery' in flask_app.extensions Flask-Celery-Helper-1.1.0/tests/test_collision.py000066400000000000000000000062361245014226400217640ustar00rootroot00000000000000from flask.ext.celery import OtherInstanceError, _select_manager import pytest from tests.instances import celery PARAMS = [('tests.instances.add', 8), ('tests.instances.mul', 16), ('tests.instances.sub', 0)] @pytest.mark.parametrize('task_name,expected', PARAMS) def test_basic(task_name, expected): task = celery.tasks[task_name] assert expected == task.apply_async(args=(4, 4)).get() @pytest.mark.parametrize('task_name,expected', PARAMS) def test_collision(task_name, expected): manager_class = _select_manager(celery.backend.__class__.__name__) manager_instance = list() task = celery.tasks[task_name] # First run the task and prevent it from removing the lock. def new_exit(self, *_): manager_instance.append(self) return None original_exit = manager_class.__exit__ setattr(manager_class, '__exit__', new_exit) assert expected == task.apply_async(args=(4, 4)).get() setattr(manager_class, '__exit__', original_exit) assert manager_instance[0].is_already_running is True # Now run it again. with pytest.raises(OtherInstanceError) as e: task.apply_async(args=(4, 4)).get() if manager_instance[0].include_args: assert str(e.value).startswith('Failed to acquire lock, {0}.args.'.format(task_name)) else: assert 'Failed to acquire lock, {0} already running.'.format(task_name) == str(e.value) assert manager_instance[0].is_already_running is True # Clean up. manager_instance[0].reset_lock() assert manager_instance[0].is_already_running is False # Once more. assert expected == task.apply_async(args=(4, 4)).get() def test_include_args(): manager_class = _select_manager(celery.backend.__class__.__name__) manager_instance = list() task = celery.tasks['tests.instances.mul'] # First run the tasks and prevent them from removing the locks. def new_exit(self, *_): """Expected to be run twice.""" manager_instance.append(self) return None original_exit = manager_class.__exit__ setattr(manager_class, '__exit__', new_exit) assert 16 == task.apply_async(args=(4, 4)).get() assert 20 == task.apply_async(args=(5, 4)).get() setattr(manager_class, '__exit__', original_exit) assert manager_instance[0].is_already_running is True assert manager_instance[1].is_already_running is True # Now run them again. with pytest.raises(OtherInstanceError) as e: task.apply_async(args=(4, 4)).get() assert str(e.value).startswith('Failed to acquire lock, tests.instances.mul.args.') assert manager_instance[0].is_already_running is True with pytest.raises(OtherInstanceError) as e: task.apply_async(args=(5, 4)).get() assert str(e.value).startswith('Failed to acquire lock, tests.instances.mul.args.') assert manager_instance[1].is_already_running is True # Clean up. manager_instance[0].reset_lock() assert manager_instance[0].is_already_running is False manager_instance[1].reset_lock() assert manager_instance[1].is_already_running is False # Once more. assert 16 == task.apply_async(args=(4, 4)).get() assert 20 == task.apply_async(args=(5, 4)).get() Flask-Celery-Helper-1.1.0/tests/test_timeout.py000066400000000000000000000051531245014226400214540ustar00rootroot00000000000000import time from flask.ext.celery import OtherInstanceError, _select_manager import pytest from tests.instances import celery @pytest.mark.parametrize('task_name,timeout', [ ('tests.instances.mul', 20), ('tests.instances.add', 300), ('tests.instances.add2', 70), ('tests.instances.add3', 80) ]) def test_instances(task_name, timeout): manager_class = _select_manager(celery.backend.__class__.__name__) manager_instance = list() task = celery.tasks[task_name] original_exit = manager_class.__exit__ def new_exit(self, *_): manager_instance.append(self) return original_exit(self, *_) setattr(manager_class, '__exit__', new_exit) task.apply_async(args=(4, 4)).get() setattr(manager_class, '__exit__', original_exit) assert timeout == manager_instance[0].timeout @pytest.mark.parametrize('key,value', [('CELERYD_TASK_TIME_LIMIT', 200), ('CELERYD_TASK_SOFT_TIME_LIMIT', 100)]) def test_settings(key, value): celery.conf.update({key: value}) manager_class = _select_manager(celery.backend.__class__.__name__) manager_instance = list() original_exit = manager_class.__exit__ def new_exit(self, *_): manager_instance.append(self) return original_exit(self, *_) setattr(manager_class, '__exit__', new_exit) tasks = [ ('tests.instances.mul', 20), ('tests.instances.add', value), ('tests.instances.add2', 70), ('tests.instances.add3', 80) ] for task_name, timeout in tasks: task = celery.tasks[task_name] task.apply_async(args=(4, 4)).get() assert timeout == manager_instance.pop().timeout setattr(manager_class, '__exit__', original_exit) celery.conf.update({key: None}) def test_expired(): celery.conf.update({'CELERYD_TASK_TIME_LIMIT': 5}) manager_class = _select_manager(celery.backend.__class__.__name__) manager_instance = list() task = celery.tasks['tests.instances.add'] original_exit = manager_class.__exit__ def new_exit(self, *_): manager_instance.append(self) return None setattr(manager_class, '__exit__', new_exit) # Run the task and don't remove the lock after a successful run. assert 8 == task.apply_async(args=(4, 4)).get() setattr(manager_class, '__exit__', original_exit) # Run again, lock is still active so this should fail. with pytest.raises(OtherInstanceError): task.apply_async(args=(4, 4)).get() # Wait 5 seconds (per CELERYD_TASK_TIME_LIMIT), then re-run, should work. time.sleep(5) assert 8 == task.apply_async(args=(4, 4)).get() celery.conf.update({'CELERYD_TASK_TIME_LIMIT': None})