Repository: wakatime/wakaq Branch: main Commit: b4d070fa19c2 Files: 21 Total size: 87.7 KB Directory structure: gitextract_4n0c_lca/ ├── .gitignore ├── AUTHORS ├── CHANGES.md ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── pyproject.toml ├── requirements.txt ├── setup.py └── wakaq/ ├── __about__.py ├── __init__.py ├── cli.py ├── exceptions.py ├── logger.py ├── queue.py ├── scheduler.py ├── serializer.py ├── task.py ├── utils.py └── worker.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ ================================================ FILE: AUTHORS ================================================ WakaQ is written and maintained by Alan Hamlett and various contributors: Development Lead ---------------- - Alan Hamlett ================================================ FILE: CHANGES.md ================================================ # CHANGES ## 4.0.5 (2026-02-03) [commits](https://github.com/wakatime/wakaq/compare/4.0.4...4.0.5) #### Bugfix - Handle BlockingIOError when parent process writing to log file. - Handle missing process when checking ram usage. - Fix required python version 3.14. ## 4.0.4 (2026-02-01) [commits](https://github.com/wakatime/wakaq/compare/4.0.3...4.0.4) #### Bugfix - Fix import path. ## 4.0.3 (2026-02-01) [commits](https://github.com/wakatime/wakaq/compare/4.0.2...4.0.3) #### Bugfix - Handle redis pubsub connection errors and attempt to reconnect. ## 4.0.2 (2026-02-01) [commits](https://github.com/wakatime/wakaq/compare/4.0.1...4.0.2) #### Bugfix - Fix picking child process using most RAM. - Handle corrupted ping messages from child processes when server out of available RAM. ## 4.0.1 (2025-12-02) [commits](https://github.com/wakatime/wakaq/compare/4.0.0...4.0.1) #### Bugfix - Fix accessing constant value. ## 4.0.0 (2025-12-02) [commits](https://github.com/wakatime/wakaq/compare/3.0.7...4.0.0) #### Breaking - Only support Python 3.14 and newer. ## 3.0.7 (2025-05-01) [commits](https://github.com/wakatime/wakaq/compare/3.0.6...3.0.7) #### Bugfix - Ignore SoftTimeout in parent process. - Remove deprecated charset redis config option. ## 3.0.6 (2025-01-04) [commits](https://github.com/wakatime/wakaq/compare/3.0.5...3.0.6) #### Bugfix - Remove noisy debug log. ## 3.0.5 (2024-11-21) [commits](https://github.com/wakatime/wakaq/compare/3.0.4...3.0.5) #### Bugfix - Stop waiting for async tasks after exception raised. [#7](https://github.com/wakatime/wakaq/issues/7) ## 3.0.4 (2024-11-20) [commits](https://github.com/wakatime/wakaq/compare/3.0.3...3.0.4) #### Bugfix - Fix raising asyncio.exceptions.CancelledError when async task hits soft timeout. ## 3.0.3 (2024-11-13) [commits](https://github.com/wakatime/wakaq/compare/3.0.2...3.0.3) #### Bugfix - Fix UnboundLocalError for async task context [#15](https://github.com/wakatime/wakaq/pull/15) ## 3.0.2 (2024-11-13) [commits](https://github.com/wakatime/wakaq/compare/3.0.1...3.0.2) #### Bugfix - Support Python versions older than 3.11. ## 3.0.1 (2024-11-13) [commits](https://github.com/wakatime/wakaq/compare/3.0.0...3.0.1) #### Bugfix - Cancel async tasks when soft_timeout reached. [#14](https://github.com/wakatime/wakaq/pull/14) - Fix setting current async task for logging. [#14](https://github.com/wakatime/wakaq/pull/14) ## 3.0.0 (2024-11-12) [commits](https://github.com/wakatime/wakaq/compare/2.1.25...3.0.0) #### Breaking - Custom task wrapper should no longer use an inner function. #### Feature - Support for async concurrent tasks on the same worker process. [#2](https://github.com/wakatime/wakaq/issues/2) ## 2.1.25 (2024-11-07) [commits](https://github.com/wakatime/wakaq/compare/2.1.24...2.1.25) #### Misc - Synchronous mode for easier debugging. [#10](https://github.com/wakatime/wakaq/pull/10) ## 2.1.24 (2024-08-19) [commits](https://github.com/wakatime/wakaq/compare/2.1.23...2.1.24) #### Bugfix - Fix number of cores not resolving to integer. ## 2.1.23 (2023-12-16) [commits](https://github.com/wakatime/wakaq/compare/2.1.22...2.1.23) #### Bugfix - Fix catching out of memory errors when executing broadcast tasks. ## 2.1.22 (2023-12-16) [commits](https://github.com/wakatime/wakaq/compare/2.1.21...2.1.22) #### Bugfix - Fix logging memory errors and treat BrokenPipeError as mem error. ## 2.1.21 (2023-12-15) [commits](https://github.com/wakatime/wakaq/compare/2.1.20...2.1.21) #### Bugfix - Log memory errors when to debug level. ## 2.1.20 (2023-12-14) [commits](https://github.com/wakatime/wakaq/compare/2.1.19...2.1.20) #### Bugfix - Silently exit child worker(s) when parent process missing. ## 2.1.19 (2023-11-11) [commits](https://github.com/wakatime/wakaq/compare/2.1.18...2.1.19) #### Bugfix - Prevent lost tasks when OOM kills parent worker process, improvement. ## 2.1.18 (2023-11-11) [commits](https://github.com/wakatime/wakaq/compare/2.1.17...2.1.18) #### Bugfix - Prevent lost tasks when OOM kills parent worker process. ## 2.1.17 (2023-11-10) [commits](https://github.com/wakatime/wakaq/compare/2.1.16...2.1.17) #### Bugfix - Fix typo. ## 2.1.16 (2023-11-10) [commits](https://github.com/wakatime/wakaq/compare/2.1.15...2.1.16) #### Bugfix - Fix function decorator callbacks after_worker_started, etc. ## 2.1.15 (2023-11-10) [commits](https://github.com/wakatime/wakaq/compare/2.1.14...2.1.15) #### Bugfix - Unsubscribe from pubsub while worker processes are waiting to exit. ## 2.1.14 (2023-11-09) [commits](https://github.com/wakatime/wakaq/compare/2.1.13...2.1.14) #### Bugfix - Add missing wrapped log.handlers property. ## 2.1.13 (2023-11-09) [commits](https://github.com/wakatime/wakaq/compare/2.1.12...2.1.13) #### Misc - Ignore logging errors. ## 2.1.12 (2023-11-09) [commits](https://github.com/wakatime/wakaq/compare/2.1.11...2.1.12) #### Bugfix - Only refork crashed workers once per wait_timeout. ## 2.1.11 (2023-11-09) [commits](https://github.com/wakatime/wakaq/compare/2.1.10...2.1.11) #### Bugfix - Postpone forking workers at startup if until ram usage below max threshold. ## 2.1.10 (2023-11-09) [commits](https://github.com/wakatime/wakaq/compare/2.1.9...2.1.10) #### Bugfix - Postpone forking missing child workers while using too much RAM. ## 2.1.9 (2023-11-08) [commits](https://github.com/wakatime/wakaq/compare/2.1.8...2.1.9) #### Bugfix - Prevent UnboundLocalError from using task_name var before assignment. ## 2.1.8 (2023-11-08) [commits](https://github.com/wakatime/wakaq/compare/2.1.7...2.1.8) #### Bugfix - Prevent ValueError when unpacking invalid message from child process. ## 2.1.7 (2023-11-08) [commits](https://github.com/wakatime/wakaq/compare/2.1.6...2.1.7) #### Bugfix - Prevent ValueError if no worker processes spawned when checking max mem usage. ## 2.1.6 (2023-10-11) [commits](https://github.com/wakatime/wakaq/compare/2.1.5...2.1.6) #### Misc - Log time task took until exited when killed because max_mem_percent reached. ## 2.1.5 (2023-10-11) [commits](https://github.com/wakatime/wakaq/compare/2.1.4...2.1.5) #### Bugfix - Prevent reset max_mem_reached_at when unrelated child process exits. ## 2.1.4 (2023-10-11) [commits](https://github.com/wakatime/wakaq/compare/2.1.3...2.1.4) #### Misc - Less noisy logs when max_mem_percent reached. - Allow restarting child worker processes more than once per soft timeout. ## 2.1.3 (2023-10-11) [commits](https://github.com/wakatime/wakaq/compare/2.1.2...2.1.3) #### Bugfix - Wait for child worker to finish processing current task when max_mem_percent reached. ## 2.1.2 (2023-10-09) [commits](https://github.com/wakatime/wakaq/compare/2.1.1...2.1.2) #### Misc - Log mem usage and current task when max_mem_percent threshold reached. ## 2.1.1 (2023-10-09) [commits](https://github.com/wakatime/wakaq/compare/2.1.0...2.1.1) #### Bugfix - Fix setting max_mem_percent on WakaQ. ## 2.1.0 (2023-09-22) [commits](https://github.com/wakatime/wakaq/compare/2.0.2...2.1.0) #### Feature - Include number of workers connected when inspecting queues. - Log queue params on startup. #### Misc - Improve docs in readme. ## 2.0.2 (2022-12-09) [commits](https://github.com/wakatime/wakaq/compare/2.0.1...2.0.2) #### Bugfix - Make sure to catch system exceptions to prevent worker infinite loops. ## 2.0.1 (2022-12-09) [commits](https://github.com/wakatime/wakaq/compare/2.0.0...2.0.1) #### Bugfix - Always catch SoftTimeout even when nested in exception context. ## 2.0.0 (2022-11-18) [commits](https://github.com/wakatime/wakaq/compare/1.2.0...2.0.0) #### Feature - Support bytes in task arguments. #### Misc - Tasks always receive datetimes in UTC without tzinfo. ## 1.3.0 (2022-10-05) [commits](https://github.com/wakatime/wakaq/compare/1.2.1...1.3.0) #### Feature - Add username, password, and db redis connection options. [#6](https://github.com/wakatime/wakaq/issues/6) ## 1.2.1 (2022-09-20) [commits](https://github.com/wakatime/wakaq/compare/1.2.0...1.2.1) #### Bugfix - Prevent reading from Redis when no queues defined. #### Misc - Improve error message when app path not WakaQ instance. ## 1.2.0 (2022-09-17) [commits](https://github.com/wakatime/wakaq/compare/1.1.8...1.2.0) #### Feature - Util functions to peek at tasks in queues. ## 1.1.8 (2022-09-15) [commits](https://github.com/wakatime/wakaq/compare/1.1.7...1.1.8) #### Bugfix - Ignore SoftTimeout in child when not processing any task. ## 1.1.7 (2022-09-15) [commits](https://github.com/wakatime/wakaq/compare/1.1.6...1.1.7) #### Bugfix - Allow custom timeouts defined on task decorator. ## 1.1.6 (2022-09-15) [commits](https://github.com/wakatime/wakaq/compare/1.1.5...1.1.6) #### Bugfix - All timeouts should accept timedelta or int seconds. ## 1.1.5 (2022-09-15) [commits](https://github.com/wakatime/wakaq/compare/1.1.4...1.1.5) #### Bugfix - Fix typo. ## 1.1.4 (2022-09-15) [commits](https://github.com/wakatime/wakaq/compare/1.1.3...1.1.4) #### Bugfix - Fix setting task and queue on child from ping. ## 1.1.3 (2022-09-15) [commits](https://github.com/wakatime/wakaq/compare/1.1.2...1.1.3) #### Bugfix - Fix sending task and queue to parent process. ## 1.1.2 (2022-09-14) [commits](https://github.com/wakatime/wakaq/compare/1.1.1...1.1.2) #### Bugfix - Fix getattr. ## 1.1.1 (2022-09-14) [commits](https://github.com/wakatime/wakaq/compare/1.1.0...1.1.1) #### Bugfix - Add missing child timeout class attributes. ## 1.1.0 (2022-09-14) [commits](https://github.com/wakatime/wakaq/compare/1.0.6...1.1.0) #### Feature - Ability to overwrite timeouts per task or queue. ## 1.0.6 (2022-09-08) [commits](https://github.com/wakatime/wakaq/compare/1.0.5...1.0.6) #### Bugfix - Prevent unknown task crashing worker process. ## 1.0.5 (2022-09-08) [commits](https://github.com/wakatime/wakaq/compare/1.0.4...1.0.5) #### Bugfix - Make sure logging has current task set. ## 1.0.4 (2022-09-07) [commits](https://github.com/wakatime/wakaq/compare/1.0.3...1.0.4) #### Bugfix - Fix auto retrying tasks on soft timeout. ## 1.0.3 (2022-09-07) [commits](https://github.com/wakatime/wakaq/compare/1.0.2...1.0.3) #### Bugfix - Ignore SoftTimeout when waiting on BLPOP from Redis list. ## 1.0.2 (2022-09-05) [commits](https://github.com/wakatime/wakaq/compare/1.0.1...1.0.2) #### Bugfix - Ping parent before blocking dequeue in case wait timeout is near soft timeout. ## 1.0.1 (2022-09-05) [commits](https://github.com/wakatime/wakaq/compare/1.0.0...1.0.1) #### Bugfix - All logger vars should be strings. ## 1.0.0 (2022-09-05) [commits](https://github.com/wakatime/wakaq/compare/0.0.11...1.0.0) - First major release. ## 0.0.11 (2022-09-05) [commits](https://github.com/wakatime/wakaq/compare/0.0.10...0.0.11) #### Feature - Add task payload to logger variables. ## 0.0.10 (2022-09-05) [commits](https://github.com/wakatime/wakaq/compare/0.0.9...0.0.10) #### Bugfix - Prevent logging error from crashing parent process. ## 0.0.9 (2022-09-05) [commits](https://github.com/wakatime/wakaq/compare/0.0.8...0.0.9) #### Bugfix - Prevent parent process looping forever while stopping children. ## 0.0.8 (2022-09-05) [commits](https://github.com/wakatime/wakaq/compare/0.0.7...0.0.8) #### Bugfix - Prevent parent process crash leaving zombie child processes. ## 0.0.7 (2022-09-05) [commits](https://github.com/wakatime/wakaq/compare/0.0.6...0.0.7) #### Feature - Ability to retry tasks when they soft timeout. #### Bugfix - Ping parent process at start of task to make sure soft timeout timer is reset. ## 0.0.6 (2022-09-03) [commits](https://github.com/wakatime/wakaq/compare/0.0.5...0.0.6) #### Feature - Implement exclude_queues option. #### Bugfix - Prevent parent process crash if write to child broadcast pipe fails. ## 0.0.5 (2022-09-01) [commits](https://github.com/wakatime/wakaq/compare/0.0.4...0.0.5) #### Bugfix - Run broadcast tasks once per worker instead of randomly. ## 0.0.4 (2022-09-01) [commits](https://github.com/wakatime/wakaq/compare/0.0.3...0.0.4) #### Feature - Allow defining schedules as tuple of cron and task name, without args. ## 0.0.3 (2022-09-01) [commits](https://github.com/wakatime/wakaq/compare/0.0.2...0.0.3) #### Bugfix - Prevent worker process crashing on any exception. #### Feature - Ability to wrap tasks with custom dectorator function. ## 0.0.2 (2022-09-01) [commits](https://github.com/wakatime/wakaq/compare/0.0.1...0.0.2) #### Breaking - Run in foreground by default. - Separate log files and levels for worker and scheduler. - Decorators for after worker started, before task, and after task callbacks. #### Bufix - Keep processing tasks after SoftTimeout. - Scheduler should sleep until scheduled time. ## 0.0.1 (2022-08-30) - Initial release. ================================================ FILE: LICENSE ================================================ BSD 3-Clause License Copyright (c) 2022, WakaTime All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================ FILE: MANIFEST.in ================================================ include README.md LICENSE CHANGES.md requirements.txt recursive-include src/wakaq *.py ================================================ FILE: Makefile ================================================ .PHONY: all test clean build upload all: @echo 'test run the unit tests with the current default python' @echo 'clean remove builds at dist/*' @echo 'build run setup.py dist' @echo 'upload upload all builds in dist folder to pypi' @echo 'release publish the current version to pypi' test: @pytest release: clean build upload clean: @rm -f dist/* build: @python ./setup.py sdist upload: @twine upload ./dist/* ================================================ FILE: README.md ================================================ # ![logo](https://raw.githubusercontent.com/wakatime/wakaq/main/wakatime-logo.png "WakaQ") WakaQ [![wakatime](https://wakatime.com/badge/github/wakatime/wakaq.svg)](https://wakatime.com/badge/github/wakatime/wakaq) Background task queue for Python backed by Redis, a super minimal Celery. Read about the motivation behind this project on [this blog post][blog launch] and the accompanying [Hacker News discussion][hacker news]. WakaQ is currently used in production at [WakaTime.com][wakatime]. WakaQ is also available in [TypeScript][wakaq-ts]. ## Features * Queue priority * Delayed tasks (run tasks after a timedelta eta) * Scheduled periodic tasks * Tasks can be [async][asyncio] or normal synchronous functions * [Broadcast][broadcast] a task to all workers * Task [soft][soft timeout] and [hard][hard timeout] timeout limits * Optionally retry tasks on soft timeout * Combat memory leaks with `max_mem_percent` or `max_tasks_per_worker` * Super minimal Want more features like rate limiting, task deduplication, etc? Too bad, feature PRs are not accepted. Maximal features belong in your app’s worker tasks. ## Installing pip install wakaq ## Using ```python import logging from datetime import timedelta from wakaq import WakaQ, Queue, CronTask # use constants to prevent misspelling queue names Q_HIGH = 'a-high-priority-queue' Q_MED = 'a-medium-priority-queue' Q_LOW = 'a-low-priority-queue' Q_OTHER = 'another-queue' Q_DEFAULT = 'default-lowest-priority-queue' wakaq = WakaQ( # List your queues and their priorities. # Queues can be defined as Queue instances, tuples, or just a str. queues=[ (0, Q_HIGH), (1, Q_MED), (2, Q_LOW), Queue(Q_OTHER, priority=3, max_retries=5, soft_timeout=300, hard_timeout=360), Q_DEFAULT, ], # Number of worker processes. Must be an int or str which evaluates to an # int. The variable "cores" is replaced with the number of processors on # the current machine. concurrency="cores*4", # Number of concurrent asyncio tasks per worker process. Must be an int or # str which evaluates to an int. The variable "cores" is replaced with the # number of processors on the current machine. Default is zero for no limit. async_concurrency=0, # Raise SoftTimeout or asyncio.CancelledError in a task if it runs longer # than 30 seconds. Can also be set per task or queue. If no soft timeout # set, tasks can run forever. soft_timeout=30, # seconds # SIGKILL a task if it runs longer than 1 minute. Can be set per task or queue. hard_timeout=timedelta(minutes=1), # If the task soft timeouts, retry up to 3 times. Max retries comes first # from the task decorator if set, next from the Queue's max_retries, # lastly from the option below. If No max_retries is found, the task # is not retried on a soft timeout. max_retries=3, # Combat memory leaks by reloading a worker (the one using the most RAM), # when the total machine RAM usage is at or greater than 98%. max_mem_percent=98, # Combat memory leaks by reloading a worker after it's processed 5000 tasks. max_tasks_per_worker=5000, # Schedule two tasks, the first runs every minute, the second once every ten minutes. # Scheduled tasks can be passed as CronTask instances or tuples. To run scheduled # tasks you must keep a wakaq scheduler running as a daemon. schedules=[ # Runs mytask on the queue with priority 1. CronTask('* * * * *', 'mytask', queue=Q_MED, args=[2, 2], kwargs={}), # Runs mytask once every 5 minutes. ('*/5 * * * *', 'mytask', [1, 1], {}), # Runs anothertask on the default lowest priority queue. ('*/10 * * * *', 'anothertask'), ], ) # timeouts can be customized per task with a timedelta or integer seconds @wakaq.task(queue=Q_MED, max_retries=7, soft_timeout=420, hard_timeout=480) def mytask(x, y): print(x + y) @wakaq.task def a_cpu_intensive_task(): print("hello world") @wakaq.task async def an_io_intensive_task(): print("hello world") @wakaq.wrap_tasks_with async def custom_task_decorator(fn, args, kwargs): # do something before each task runs, for ex: `with app.app_context():` if inspect.iscoroutinefunction(fn): await fn(*args, **kwargs) else: fn(*args, **kwargs) # do something after each task runs if __name__ == '__main__': # add 1 plus 1 on a worker somewhere mytask.delay(1, 1) # add 1 plus 1 on a worker somewhere, overwriting the task's queue from medium to high mytask.delay(1, 1, queue=Q_HIGH) # print hello world on a worker somewhere, running on the default lowest priority queue anothertask.delay() # print hello world on a worker somewhere, after 10 seconds from now anothertask.delay(eta=timedelta(seconds=10)) # print hello world on a worker concurrently, even if you only have 1 worker process an_io_intensive_task.delay() ``` ## Deploying #### Optimizing See the [WakaQ init params][wakaq init] for a full list of options, like Redis host and Redis socket timeout values. When using in production, make sure to [increase the max open ports][max open ports] allowed for your Redis server process. When using eta tasks a Redis sorted set is used, so eta tasks are automatically deduped based on task name, args, and kwargs. If you want multiple pending eta tasks with the same arguments, just add a throwaway random string to the task’s kwargs for ex: `str(uuid.uuid1())`. #### Running as a Daemon Here’s an example systemd config to run `wakaq-worker` as a daemon: ```systemd [Unit] Description=WakaQ Worker Service [Service] WorkingDirectory=/opt/yourapp ExecStart=/opt/yourapp/venv/bin/python /opt/yourapp/venv/bin/wakaq-worker --app=yourapp.wakaq RemainAfterExit=no Restart=always RestartSec=30s KillSignal=SIGINT LimitNOFILE=99999 [Install] WantedBy=multi-user.target ``` Create a file at `/etc/systemd/system/wakaqworker.service` with the above contents, then run: systemctl daemon-reload && systemctl enable wakaqworker ## Running synchronously in tests or local dev environment In dev and test environments, it’s easier to run tasks synchronously so you don’t need Redis or any worker processes. The recommended way is mocking WakaQ: ```python class WakaQMock: def __init__(self): self.task = TaskMock def wrap_tasks_with(self, fn): return fn class TaskMock(object): fn = None name = None args = () kwargs = {} def __init__(self, *args, **kwargs): if len(args) == 1 and len(kwargs) == 0: self.fn = args[0] self.name = args[0].__name__ else: self.args = args self.kwargs = kwargs def delay(self, *args, **kwargs): kwargs.pop("queue", None) kwargs.pop("eta", None) return self.fn(*args, **kwargs) def broadcast(self, *args, **kwargs): return def __call__(self, *args, **kwargs): if not self.fn: task = TaskMock(args[0]) task.args = self.args task.kwargs = self.kwargs return task else: return self.fn(*args, **kwargs) ``` Then in dev and test environments instead of using `wakaq.WakaQ` use `WakaQMock`. [wakatime]: https://wakatime.com [broadcast]: https://github.com/wakatime/wakaq/blob/58a7e4ce29d9be928b16ffbf5c00c7106aab9360/wakaq/task.py#L65 [soft timeout]: https://github.com/wakatime/wakaq/blob/58a7e4ce29d9be928b16ffbf5c00c7106aab9360/wakaq/exceptions.py#L5 [hard timeout]: https://github.com/wakatime/wakaq/blob/58a7e4ce29d9be928b16ffbf5c00c7106aab9360/wakaq/worker.py#L590 [wakaq init]: https://github.com/wakatime/wakaq/blob/58a7e4ce29d9be928b16ffbf5c00c7106aab9360/wakaq/__init__.py#L47 [max open ports]: https://wakatime.com/blog/47-maximize-your-concurrent-web-server-connections [blog launch]: https://wakatime.com/blog/56-building-a-distributed-task-queue-in-python [hacker news]: https://news.ycombinator.com/item?id=32730038 [wakaq-ts]: https://github.com/wakatime/wakaq-ts [asyncio]: https://docs.python.org/3/library/asyncio.html ================================================ FILE: pyproject.toml ================================================ [tool.black] line-length = 120 [tool.isort] profile = "black" ================================================ FILE: requirements.txt ================================================ click croniter psutil redis ================================================ FILE: setup.py ================================================ from setuptools import setup about = {} with open("wakaq/__about__.py") as f: exec(f.read(), about) install_requires = [x.strip() for x in open("requirements.txt").readlines()] setup( name=about["__title__"], version=about["__version__"], license=about["__license__"], description=about["__description__"], long_description=open("README.md").read(), long_description_content_type="text/markdown", author=about["__author__"], author_email=about["__author_email__"], url=about["__url__"], packages=["wakaq"], package_dir={"wakaq": "wakaq"}, python_requires=">= 3.14", include_package_data=True, platforms="any", install_requires=install_requires, entry_points={ "console_scripts": [ "wakaq-worker = wakaq.cli:worker", "wakaq-scheduler = wakaq.cli:scheduler", "wakaq-info = wakaq.cli:info", "wakaq-purge = wakaq.cli:purge", ], }, classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Natural Language :: English", "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.14", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: System :: Distributed Computing", "Topic :: Software Development :: Object Brokering", "Operating System :: OS Independent", ], ) ================================================ FILE: wakaq/__about__.py ================================================ __title__ = "WakaQ" __description__ = "Background task queue for Python backed by Redis, a minimal Celery." __url__ = "https://github.com/wakatime/wakaq" __version_info__ = ("4", "0", "5") __version__ = ".".join(__version_info__) __author__ = "Alan Hamlett" __author_email__ = "alan.hamlett@gmail.com" __license__ = "BSD" __copyright__ = "Copyright 2022 Alan Hamlett" ================================================ FILE: wakaq/__init__.py ================================================ import calendar import logging import multiprocessing from datetime import datetime, timedelta import redis from .queue import Queue from .scheduler import CronTask from .serializer import serialize from .task import Task from .utils import safe_eval __all__ = [ "WakaQ", "Queue", "CronTask", ] class WakaQ: queues = [] soft_timeout = None hard_timeout = None concurrency = 0 async_concurrency = 0 schedules = [] exclude_queues = [] max_retries = None wait_timeout = None max_mem_percent = None max_tasks_per_worker = None worker_log_file = None scheduler_log_file = None worker_log_level = None scheduler_log_level = None after_worker_started_callback = None before_task_started_callback = None after_task_finished_callback = None wrap_tasks_function = None broadcast_key = "wakaq-broadcast" log_format = "[%(asctime)s] %(levelname)s: %(message)s" task_log_format = "[%(asctime)s] %(levelname)s in %(task)s: %(message)s" def __init__( self, queues=[], schedules=[], host="localhost", port=6379, username=None, password=None, db=0, concurrency=0, async_concurrency=0, exclude_queues=[], max_retries=None, soft_timeout=None, hard_timeout=None, max_mem_percent=None, max_tasks_per_worker=None, worker_log_file=None, scheduler_log_file=None, worker_log_level=None, scheduler_log_level=None, socket_timeout=15, socket_connect_timeout=15, health_check_interval=30, wait_timeout=1, ): self.queues = [Queue.create(x) for x in queues] if len(self.queues) == 0: raise Exception("Missing queues.") lowest_priority = max(self.queues, key=lambda q: q.priority) self.queues = list(map(lambda q: self._default_priority(q, lowest_priority.priority), self.queues)) self.queues.sort(key=lambda q: q.priority) self.queues_by_name = dict([(x.name, x) for x in self.queues]) self.queues_by_key = dict([(x.broker_key, x) for x in self.queues]) self.exclude_queues = self._validate_queue_names(exclude_queues) self.max_retries = int(max_retries or 0) self.broker_keys = [x.broker_key for x in self.queues if x.name not in self.exclude_queues] self.schedules = [CronTask.create(x) for x in schedules] self.concurrency = self._format_concurrency(concurrency) self.async_concurrency = self._format_concurrency(async_concurrency, is_async=True) self.soft_timeout = soft_timeout.total_seconds() if isinstance(soft_timeout, timedelta) else soft_timeout self.hard_timeout = hard_timeout.total_seconds() if isinstance(hard_timeout, timedelta) else hard_timeout self.wait_timeout = wait_timeout.total_seconds() if isinstance(wait_timeout, timedelta) else wait_timeout if self.soft_timeout and self.soft_timeout <= wait_timeout: raise Exception( f"Soft timeout ({self.soft_timeout}) can not be less than or equal to wait timeout ({self.wait_timeout})." ) if self.hard_timeout and self.hard_timeout <= self.wait_timeout: raise Exception( f"Hard timeout ({self.hard_timeout}) can not be less than or equal to wait timeout ({self.wait_timeout})." ) if self.soft_timeout and self.hard_timeout and self.hard_timeout <= self.soft_timeout: raise Exception( f"Hard timeout ({self.hard_timeout}) can not be less than or equal to soft timeout ({self.soft_timeout})." ) if max_mem_percent: self.max_mem_percent = int(max_mem_percent) if self.max_mem_percent < 1 or self.max_mem_percent > 99: raise Exception(f"Max memory percent must be between 1 and 99: {self.max_mem_percent}") else: self.max_mem_percent = None self.max_tasks_per_worker = abs(int(max_tasks_per_worker)) if max_tasks_per_worker else None self.worker_log_file = worker_log_file if isinstance(worker_log_file, str) else None self.scheduler_log_file = scheduler_log_file if isinstance(scheduler_log_file, str) else None self.worker_log_level = worker_log_level if isinstance(worker_log_level, int) else logging.INFO self.scheduler_log_level = scheduler_log_level if isinstance(scheduler_log_level, int) else logging.DEBUG self.tasks = {} self.broker = redis.Redis( host=host, port=port, username=username, password=password, db=db, decode_responses=True, health_check_interval=health_check_interval, socket_timeout=socket_timeout, socket_connect_timeout=socket_connect_timeout, ) def task(self, fn=None, queue=None, max_retries=None, soft_timeout=None, hard_timeout=None): def wrap(f): t = Task( fn=f, wakaq=self, queue=queue, max_retries=max_retries, soft_timeout=soft_timeout, hard_timeout=hard_timeout, ) if t.name in self.tasks: raise Exception(f"Duplicate task name: {t.name}") self.tasks[t.name] = t return t.fn return wrap(fn) if fn else wrap def after_worker_started(self, fn=None): def wrap(f): if not callable(self.after_worker_started_callback): self.after_worker_started_callback = f return f return wrap(fn) if fn else wrap def before_task_started(self, fn=None): def wrap(f): if not callable(self.before_task_started_callback): self.before_task_started_callback = f return f return wrap(fn) if fn else wrap def after_task_finished(self, fn=None): def wrap(f): if not callable(self.after_task_finished_callback): self.after_task_finished_callback = f return f return wrap(fn) if fn else wrap def wrap_tasks_with(self, fn=None): def wrap(f): if not callable(self.wrap_tasks_function): self.wrap_tasks_function = f return f return wrap(fn) if fn else wrap def _validate_queue_names(self, queue_names: list) -> list: try: queue_names = [x for x in queue_names] except: return [] for queue_name in queue_names: if queue_name not in self.queues_by_name: raise Exception(f"Invalid queue: {queue_name}") return queue_names def _enqueue_at_front(self, task_name: str, queue: str, args: list, kwargs: dict): queue = self._queue_or_default(queue) payload = serialize( { "name": task_name, "args": args, "kwargs": kwargs, } ) self.broker.lpush(queue.broker_key, payload) def _enqueue_at_end(self, task_name: str, queue: str, args: list, kwargs: dict, retry=0): queue = self._queue_or_default(queue) payload = serialize( { "name": task_name, "args": args, "kwargs": kwargs, "retry": retry, } ) self.broker.rpush(queue.broker_key, payload) def _enqueue_with_eta(self, task_name: str, queue: str, args: list, kwargs: dict, eta): queue = self._queue_or_default(queue) payload = serialize( { "name": task_name, "args": args, "kwargs": kwargs, } ) if isinstance(eta, timedelta): eta = datetime.utcnow() + eta timestamp = calendar.timegm(eta.utctimetuple()) self.broker.zadd(queue.broker_eta_key, {payload: timestamp}, nx=True) def _broadcast(self, task_name: str, args: list, kwargs: dict): payload = serialize( { "name": task_name, "args": args, "kwargs": kwargs, } ) return self.broker.publish(self.broadcast_key, payload) def _queue_or_default(self, queue_name: str): if queue_name: return Queue.create(queue_name, queues_by_name=self.queues_by_name) # return lowest priority queue by default return self.queues[-1] def _default_priority(self, queue, lowest_priority): if queue.priority < 0: queue.priority = lowest_priority + 1 return queue def _format_concurrency(self, concurrency, is_async=None): if not concurrency: return 0 try: return int(safe_eval(str(concurrency), {"cores": multiprocessing.cpu_count()})) except Exception as e: raise Exception(f"Error parsing {'async_' if is_async else ''}concurrency: {e}") ================================================ FILE: wakaq/cli.py ================================================ import json import click from .scheduler import Scheduler from .utils import ( import_app, inspect, num_pending_eta_tasks_in_queue, num_pending_tasks_in_queue, purge_eta_queue, purge_queue, ) from .worker import Worker @click.command() @click.option("--app", required=True, help="Import path of the WakaQ instance.") def worker(**options): """Run worker(s) to process tasks from queue(s) defined in your app.""" wakaq = import_app(options.pop("app")) worker = Worker(wakaq=wakaq) worker.start() @click.command() @click.option("--app", required=True, help="Import path of the WakaQ instance.") def scheduler(**options): """Run a scheduler to enqueue periodic tasks based on a schedule defined in your app.""" wakaq = import_app(options.pop("app")) scheduler = Scheduler(wakaq=wakaq) scheduler.start() @click.command() @click.option("--app", required=True, help="Import path of the WakaQ instance.") def info(**options): """Inspect and print info about your queues.""" wakaq = import_app(options.pop("app")) click.echo(json.dumps(inspect(wakaq), indent=2, sort_keys=True)) @click.command() @click.option("--app", required=True, help="Import path of the WakaQ instance.") @click.option("--queue", required=True, help="Name of queue to purge.") def purge(**options): """Remove and empty all pending tasks in a queue.""" wakaq = import_app(options.pop("app")) queue_name = options.pop("queue") count = num_pending_tasks_in_queue(wakaq, queue_name=queue_name) purge_queue(wakaq, queue_name=queue_name) count += num_pending_eta_tasks_in_queue(wakaq, queue_name=queue_name) purge_eta_queue(wakaq, queue_name=queue_name) click.echo(f"Purged {count} tasks from {queue_name}") ================================================ FILE: wakaq/exceptions.py ================================================ class WakaQError(Exception): pass class SoftTimeout(WakaQError): pass ================================================ FILE: wakaq/logger.py ================================================ import sys from logging import Formatter as FormatterBase from logging import StreamHandler, captureWarnings, getLogger from logging.handlers import WatchedFileHandler from .serializer import serialize from .utils import current_task logger = getLogger("wakaq") class SafeLogger: def setLevel(self, *args, **kwargs): logger.setLevel(*args, **kwargs) def debug(self, *args, **kwargs): try: logger.debug(*args, **kwargs) except: pass def info(self, *args, **kwargs): try: logger.info(*args, **kwargs) except: pass def warning(self, *args, **kwargs): try: logger.warning(*args, **kwargs) except: pass def error(self, *args, **kwargs): try: logger.error(*args, **kwargs) except: pass def exception(self, *args, **kwargs): try: logger.exception(*args, **kwargs) except: pass def critical(self, *args, **kwargs): try: logger.critical(*args, **kwargs) except: pass def fatal(self, *args, **kwargs): try: logger.fatal(*args, **kwargs) except: pass def log(self, *args, **kwargs): try: logger.log(*args, **kwargs) except: pass @property def handlers(self): return logger.handlers log = SafeLogger() class Formatter(FormatterBase): def __init__(self, wakaq): self.wakaq = wakaq super().__init__(wakaq.log_format) def format(self, record): task = current_task.get() if task is not None: task, payload = task[0], task[1] self._fmt = self.wakaq.task_log_format self._style._fmt = self.wakaq.task_log_format record.__dict__.update(task=task.name) record.__dict__.update(task_args=serialize(payload["args"])) record.__dict__.update(task_kwargs=serialize(payload["kwargs"])) record.__dict__.update(task_retry=serialize(payload.get("retry"))) else: self._fmt = self.wakaq.log_format self._style._fmt = self.wakaq.log_format record.__dict__.setdefault("task", None) record.__dict__.setdefault("task_args", None) record.__dict__.setdefault("task_kwargs", None) record.__dict__.setdefault("task_retry", None) return super().format(record) def setup_logging(wakaq, is_child=None, is_scheduler=None): logger = getLogger("wakaq") for handler in logger.handlers: logger.removeHandler(handler) log_file = wakaq.scheduler_log_file if is_scheduler else wakaq.worker_log_file log_level = wakaq.scheduler_log_level if is_scheduler else wakaq.worker_log_level logger.setLevel(log_level) captureWarnings(True) out = sys.stdout if is_child or not log_file else log_file options = {} if not is_child and log_file: options["encoding"] = "utf8" handler = (StreamHandler if is_child or not log_file else WatchedFileHandler)(out, **options) handler.setLevel(log_level) formatter = Formatter(wakaq) handler.setFormatter(formatter) logger.addHandler(handler) ================================================ FILE: wakaq/queue.py ================================================ import re from datetime import timedelta class Queue: __slots__ = [ "name", "priority", "prefix", "soft_timeout", "hard_timeout", "max_retries", ] def __init__(self, name=None, priority=-1, prefix=None, soft_timeout=None, hard_timeout=None, max_retries=None): self.prefix = re.sub(r"[^a-zA-Z0-9_.-]", "", prefix or "wakaq") self.name = re.sub(r"[^a-zA-Z0-9_.-]", "", name) try: self.priority = int(priority) except: raise Exception(f"Invalid queue priority: {priority}") self.soft_timeout = soft_timeout.total_seconds() if isinstance(soft_timeout, timedelta) else soft_timeout self.hard_timeout = hard_timeout.total_seconds() if isinstance(hard_timeout, timedelta) else hard_timeout if self.soft_timeout and self.hard_timeout and self.hard_timeout <= self.soft_timeout: raise Exception( f"Queue hard timeout ({self.hard_timeout}) can not be less than or equal to soft timeout ({self.soft_timeout})." ) if max_retries: try: self.max_retries = int(max_retries) except: raise Exception(f"Invalid queue max retries: {max_retries}") else: self.max_retries = None @classmethod def create(cls, obj, queues_by_name=None): if isinstance(obj, cls): if queues_by_name is not None and obj.name not in queues_by_name: raise Exception(f"Unknown queue: {obj.name}") return obj elif isinstance(obj, (list, tuple)) and len(obj) == 2: if isinstance(obj[0], int): if queues_by_name is not None and obj[1] not in queues_by_name: raise Exception(f"Unknown queue: {obj[1]}") return cls(priority=obj[0], name=obj[1]) else: if queues_by_name is not None and obj[0] not in queues_by_name: raise Exception(f"Unknown queue: {obj[0]}") return cls(name=obj[0], priority=obj[1]) else: if queues_by_name is not None and obj not in queues_by_name: raise Exception(f"Unknown queue: {obj}") return cls(name=obj) @property def broker_key(self): return f"{self.prefix}:{self.name}" @property def broker_eta_key(self): return f"{self.prefix}:eta:{self.name}" ================================================ FILE: wakaq/scheduler.py ================================================ import time from datetime import datetime, timedelta from croniter import croniter from .logger import log, setup_logging from .serializer import serialize class CronTask: __slots__ = [ "schedule", "task_name", "queue", "args", "kwargs", ] def __init__(self, schedule=None, task_name=None, queue=None, args=None, kwargs=None): if not croniter.is_valid(schedule): log.error(f"Invalid cron schedule (min hour dom month dow): {schedule}") raise Exception(f"Invalid cron schedule (min hour dom month dow): {schedule}") self.schedule = schedule self.task_name = task_name self.queue = queue self.args = args self.kwargs = kwargs @classmethod def create(cls, obj, queues_by_name=None): if isinstance(obj, cls): if queues_by_name is not None and obj.queue and obj.queue not in queues_by_name: log.error(f"Unknown queue: {obj.queue}") raise Exception(f"Unknown queue: {obj.queue}") return obj elif isinstance(obj, (list, tuple)) and len(obj) == 2: return cls(schedule=obj[0], task_name=obj[1]) elif isinstance(obj, (list, tuple)) and len(obj) == 4: return cls(schedule=obj[0], task_name=obj[1], args=obj[2], kwargs=obj[3]) else: log.error(f"Invalid schedule: {obj}") raise Exception(f"Invalid schedule: {obj}") @property def payload(self): return serialize( { "name": self.task_name, "args": self.args if self.args is not None else [], "kwargs": self.kwargs if self.kwargs is not None else {}, } ) class Scheduler: __slots__ = [ "wakaq", "schedules", ] def __init__(self, wakaq=None): self.wakaq = wakaq def start(self): setup_logging(self.wakaq, is_scheduler=True) log.info("starting scheduler") if len(self.wakaq.schedules) == 0: log.error("no scheduled tasks found") raise Exception("No scheduled tasks found.") self.schedules = [] for schedule in self.wakaq.schedules: self.schedules.append(CronTask.create(schedule, queues_by_name=self.wakaq.queues_by_name)) self._run() def _run(self): base = datetime.utcnow() upcoming_tasks = [] while True: for cron_task in upcoming_tasks: task = self.wakaq.tasks[cron_task.task_name] if cron_task.queue: queue = self.wakaq.queues_by_name[cron_task.queue] elif task.queue: queue = task.queue else: queue = self.wakaq.queues[-1] log.debug(f"run scheduled task on queue {queue.name}: {task.name}") self.wakaq.broker.lpush(queue.broker_key, cron_task.payload) upcoming_tasks = [] crons = [(croniter(x.schedule, base).get_next(datetime), x) for x in self.schedules] sleep_until = base + timedelta(days=1) for dt, cron_task in crons: if self._is_same_minute_precision(dt, sleep_until): upcoming_tasks.append(cron_task) elif dt < sleep_until: sleep_until = dt upcoming_tasks = [cron_task] # sleep until the next scheduled task time.sleep((sleep_until - base).total_seconds()) base = sleep_until def _is_same_minute_precision(self, a, b): return a.strftime("%Y%m%d%H%M") == b.strftime("%Y%m%d%H%M") ================================================ FILE: wakaq/serializer.py ================================================ from base64 import b64decode, b64encode from datetime import date, datetime, timedelta, timezone from decimal import Decimal from json import JSONEncoder, dumps, loads class CustomJSONEncoder(JSONEncoder): def __init__(self, *args, allow_nan=False, **kwargs): kwargs["allow_nan"] = allow_nan super().__init__(*args, **kwargs) def default(self, o): if isinstance(o, set): return list(o) if isinstance(o, Decimal): return { "__class__": "Decimal", "value": str(o), } if isinstance(o, datetime): if o.tzinfo is not None: # tasks always receive datetimes in utc without tzinfo o = o.astimezone(timezone.utc) return { "__class__": "datetime", "year": o.year, "month": o.month, "day": o.day, "hour": o.hour, "minute": o.minute, "second": o.second, "microsecond": o.microsecond, "fold": o.fold, } if isinstance(o, date): return { "__class__": "date", "year": o.year, "month": o.month, "day": o.day, } if isinstance(o, timedelta): return { "__class__": "timedelta", "kwargs": { "days": o.days, "seconds": o.seconds, "microseconds": o.microseconds, }, } if isinstance(o, bytes): return { "__class__": "bytes", "value": b64encode(o).decode("ascii"), } return str(o) def object_hook(obj): cls = obj.get("__class__") if not cls: return obj if cls == "Decimal": return Decimal(obj["value"]) if cls == "datetime": return datetime( year=obj["year"], month=obj["month"], day=obj["day"], hour=obj["hour"], minute=obj["minute"], second=obj["second"], microsecond=obj["microsecond"], fold=obj["fold"], ) if cls == "date": return date( year=obj["year"], month=obj["month"], day=obj["day"], ) if cls == "timedelta": return timedelta(**obj["kwargs"]) if cls == "bytes": return b64decode(obj["value"]) return obj def serialize(*args, **kwargs): kwargs["cls"] = CustomJSONEncoder return dumps(*args, **kwargs) def deserialize(*args, **kwargs): return loads(*args, object_hook=object_hook, **kwargs) ================================================ FILE: wakaq/task.py ================================================ import asyncio import threading from datetime import timedelta from .queue import Queue class Task: __slots__ = [ "name", "fn", "wakaq", "queue", "soft_timeout", "hard_timeout", "max_retries", ] def __init__(self, fn=None, wakaq=None, queue=None, soft_timeout=None, hard_timeout=None, max_retries=None): self.fn = fn self.name = fn.__name__ self.wakaq = wakaq if queue: self.queue = Queue.create(queue, queues_by_name=self.wakaq.queues_by_name) else: self.queue = None self.soft_timeout = soft_timeout.total_seconds() if isinstance(soft_timeout, timedelta) else soft_timeout self.hard_timeout = hard_timeout.total_seconds() if isinstance(hard_timeout, timedelta) else hard_timeout if self.soft_timeout and self.hard_timeout and self.hard_timeout <= self.soft_timeout: raise Exception( f"Task hard timeout ({self.hard_timeout}) can not be less than or equal to soft timeout ({self.soft_timeout})." ) self.max_retries = int(max_retries) if max_retries else None self.fn.delay = self._delay self.fn.broadcast = self._broadcast def _delay(self, *args, **kwargs): """Run task in the background.""" queue = kwargs.pop("queue", None) or self.queue eta = kwargs.pop("eta", None) if eta: self.wakaq._enqueue_with_eta(self.name, queue, args, kwargs, eta) else: self.wakaq._enqueue_at_end(self.name, queue, args, kwargs) def get_event_loop(self): try: loop = asyncio.get_event_loop() except RuntimeError: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) if not loop.is_running(): thread = threading.Thread(target=loop.run_forever, daemon=True) thread.start() return loop def _broadcast(self, *args, **kwargs) -> int: """Run task in the background on all workers. Only runs the task once per worker parent daemon, no matter the worker's concurrency. Returns the number of workers the task was sent to. """ return self.wakaq._broadcast(self.name, args, kwargs) ================================================ FILE: wakaq/utils.py ================================================ import ast import calendar import operator import os import sys from datetime import datetime, timedelta from importlib import import_module from typing import Union import psutil from .serializer import deserialize def import_app(app): """Import and return the WakaQ instance from the specified module path.""" cwd = os.getcwd() if cwd not in sys.path: sys.path.insert(0, cwd) try: module_path, class_name = app.rsplit(".", 1) except ValueError: raise Exception( f"Invalid app path: {app}. App must point to a WakaQ instance. For ex: yourapp.background.wakaq" ) module = import_module(module_path) wakaq = getattr(module, class_name) from . import WakaQ if not isinstance(wakaq, WakaQ): raise Exception(f"Invalid app path: {app}. App must point to a WakaQ instance.") return wakaq def inspect(app): """Return the queues and their respective pending task counts, and the number of workers connected.""" queues = {} for queue in app.queues: queues[queue.name] = { "name": queue.name, "priority": queue.priority, "broker_key": queue.broker_key, "broker_eta_key": queue.broker_eta_key, "pending_tasks": num_pending_tasks_in_queue(app, queue), "pending_eta_tasks": num_pending_eta_tasks_in_queue(app, queue), } return { "queues": queues, "workers": num_workers_connected(app), } def pending_tasks_in_queue(app, queue=None, queue_name: str = None, limit: int = 20) -> list: """Retrieve a list of pending tasks from a queue, without removing them from the queue.""" if not queue: if queue_name is None: return [] queue = app.queues_by_name.get(queue_name) if not queue: return [] if not limit: limit = 0 tasks = app.broker.lrange(queue.broker_key, 0, limit - 1) return [deserialize(task) for task in tasks] def pending_eta_tasks_in_queue( app, queue=None, queue_name: str = None, before: Union[datetime, timedelta, int] = None, limit: int = 20, ) -> list: """Retrieve a list of pending eta tasks from a queue, without removing them from the queue.""" if not queue: if queue_name is None: return [] queue = app.queues_by_name.get(queue_name) if not queue: return [] params = [] if before: cmd = "ZRANGEBYSCORE" if isinstance(before, timedelta): before = datetime.utcnow() + before if isinstance(before, datetime): before = calendar.timegm(before.utctimetuple()) params.extend([0, before]) params.append("WITHSCORES") if limit: params.extend(["LIMIT", 0, limit]) else: cmd = "ZRANGE" if not limit: limit = 0 params.extend([0, limit - 1]) params.append("WITHSCORES") tasks = app.broker.execute_command(cmd, queue.broker_eta_key, *params) payloads = [] for n in range(0, len(tasks), 2): payload = deserialize(tasks[n]) payload["eta"] = datetime.utcfromtimestamp(int(tasks[n + 1])) payloads.append(payload) return payloads def num_pending_tasks_in_queue(app, queue=None, queue_name: str = None) -> int: """Count and return the number of pending tasks in a queue.""" if not queue: if queue_name is None: return 0 queue = app.queues_by_name.get(queue_name) if not queue: return 0 return app.broker.llen(queue.broker_key) def num_pending_eta_tasks_in_queue(app, queue=None, queue_name: str = None) -> int: """Count and return the number of pending eta tasks in a queue.""" if not queue: if queue_name is None: return 0 queue = app.queues_by_name.get(queue_name) if not queue: return 0 return app.broker.zcount(queue.broker_eta_key, "-inf", "+inf") def num_workers_connected(app) -> int: """Count and return the number of connected workers.""" return app.broker.pubsub_numsub(app.broadcast_key)[0][1] def purge_queue(app, queue_name: str): """Empty a queue, discarding any pending tasks.""" if queue_name is None: return queue = app.queues_by_name.get(queue_name) if not queue: return app.broker.delete(queue.broker_key) def purge_eta_queue(app, queue_name: str): """Empty a queue of any pending eta tasks.""" if queue_name is None: return queue = app.queues_by_name.get(queue_name) if not queue: return app.broker.delete(queue.broker_eta_key) def kill(pid, signum): try: os.kill(pid, signum) except IOError: pass def read_fd(fd): try: return os.read(fd, 64000).decode("utf8") except OSError: return "" def write_fd_or_raise(fd, s): os.write(fd, s.encode("utf8")) def write_fd(fd, s): try: write_fd_or_raise(fd, s) except: pass def close_fd(fd): try: os.close(fd) except: pass def flush_fh(fh): try: fh.flush() except: pass def mem_usage_percent(): return int(round(psutil.virtual_memory().percent)) _operations = { ast.Add: operator.add, ast.Sub: operator.sub, ast.Mult: operator.mul, ast.Div: operator.truediv, ast.FloorDiv: operator.floordiv, ast.Pow: operator.pow, } def _safe_eval(node, variables, functions): if isinstance(node, ast.Constant) and isinstance(node.value, (int, float)): return node.value elif isinstance(node, ast.Name): try: return variables[node.id] except KeyError: raise Exception(f"Unknown variable: {node.id}") elif isinstance(node, ast.BinOp): try: op = _operations[node.op.__class__] except KeyError: raise Exception(f"Unknown operation: {node.op.__class__}") left = _safe_eval(node.left, variables, functions) right = _safe_eval(node.right, variables, functions) if isinstance(node.op, ast.Pow): assert right < 100 return op(left, right) elif isinstance(node, ast.Call): assert not node.keywords and not node.starargs and not node.kwargs assert isinstance(node.func, ast.Name), "Unsafe function derivation" try: func = functions[node.func.id] except KeyError: raise Exception(f"Unknown function: {node.func.id}") args = [_safe_eval(arg, variables, functions) for arg in node.args] return func(*args) assert False, "Unsafe operation" def safe_eval(expr, variables={}, functions={}): node = ast.parse(expr, "", "eval").body return _safe_eval(node, variables, functions) class Context: __slots__ = ["value"] def __init__(self): self.value = None def get(self): return self.value def set(self, val): self.value = val current_task = Context() def exception_in_chain(e, exception_type): if isinstance(e, exception_type): return True while (e.__cause__ or e.__context__) is not None: if isinstance((e.__cause__ or e.__context__), exception_type): return True e = e.__cause__ or e.__context__ return False def get_timeouts(app, task=None, queue=None): soft_timeout = app.soft_timeout hard_timeout = app.hard_timeout if task and task.soft_timeout: soft_timeout = task.soft_timeout elif queue and queue.soft_timeout: soft_timeout = queue.soft_timeout if task and task.hard_timeout: hard_timeout = task.hard_timeout elif queue and queue.hard_timeout: hard_timeout = queue.hard_timeout return soft_timeout, hard_timeout ================================================ FILE: wakaq/worker.py ================================================ import asyncio import inspect import logging import os import signal import sys import time import traceback import psutil from redis.exceptions import ConnectionError from .exceptions import SoftTimeout, WakaQError from .logger import log, setup_logging from .serializer import deserialize, serialize from .utils import ( close_fd, current_task, exception_in_chain, flush_fh, get_timeouts, kill, mem_usage_percent, read_fd, write_fd, write_fd_or_raise, ) ZRANGEPOP = """ local results = redis.call('ZRANGEBYSCORE', KEYS[1], 0, ARGV[1]) redis.call('ZREMRANGEBYSCORE', KEYS[1], 0, ARGV[1]) return results """ class Child: __slots__ = [ "pid", "stdin", "pingin", "ping_buffer", "log_buffer", "broadcastout", "last_ping", "soft_timeout_reached", "max_mem_reached_at", "done", "soft_timeout", "hard_timeout", "current_task", ] def __init__(self, pid, stdin, pingin, broadcastout): os.set_blocking(stdin, False) os.set_blocking(pingin, False) os.set_blocking(broadcastout, False) self.current_task = None self.pid = pid self.stdin = stdin self.pingin = pingin self.ping_buffer = "" self.log_buffer = b"" self.broadcastout = broadcastout self.soft_timeout_reached = False self.last_ping = time.time() self.done = False self.soft_timeout = None self.hard_timeout = None self.max_mem_reached_at = 0 def close(self): close_fd(self.pingin) close_fd(self.stdin) close_fd(self.broadcastout) def set_timeouts(self, wakaq, task=None, queue=None): self.current_task = task soft_timeout, hard_timeout = get_timeouts(wakaq, task=task, queue=queue) self.soft_timeout = soft_timeout self.hard_timeout = hard_timeout @property def mem_usage_percent(self): try: return psutil.Process(self.pid).memory_percent() except psutil.NoSuchProcess: return 0 class Worker: __slots__ = [ "wakaq", "children", "_stop_processing", "_pubsub", "_pingout", "_broadcastin", "_num_tasks_processed", "_loop", "_active_async_tasks", "_async_task_context", ] def __init__(self, wakaq=None): self.wakaq = wakaq def start(self): setup_logging(self.wakaq) log.info(f"concurrency={self.wakaq.concurrency}") log.info(f"async_concurrency={self.wakaq.async_concurrency}") log.info(f"soft_timeout={self.wakaq.soft_timeout}") log.info(f"hard_timeout={self.wakaq.hard_timeout}") log.info(f"wait_timeout={self.wakaq.wait_timeout}") log.info(f"exclude_queues={self.wakaq.exclude_queues}") log.info(f"max_retries={self.wakaq.max_retries}") log.info(f"max_mem_percent={self.wakaq.max_mem_percent}") log.info(f"max_tasks_per_worker={self.wakaq.max_tasks_per_worker}") log.info(f"worker_log_file={self.wakaq.worker_log_file}") log.info(f"scheduler_log_file={self.wakaq.scheduler_log_file}") log.info(f"worker_log_level={self.wakaq.worker_log_level}") log.info(f"scheduler_log_level={self.wakaq.scheduler_log_level}") log.info(f"starting {self.wakaq.concurrency} workers...") self._run() def _stop(self): self._stop_processing = True for child in self.children: kill(child.pid, signal.SIGTERM) try: if self._pubsub: self._pubsub.unsubscribe() except: log.debug(traceback.format_exc()) def _run(self): self.children = [] self._stop_processing = False pid = None for i in range(self.wakaq.concurrency): # postpone fork if using too much ram if self.wakaq.max_mem_percent: percent_used = mem_usage_percent() if percent_used >= self.wakaq.max_mem_percent: log.info( f"postpone forking workers... mem usage {percent_used}% is more than max_mem_percent threshold ({self.wakaq.max_mem_percent}%)" ) break pid = self._fork() if pid == 0: break if pid != 0: # parent self._parent() def _fork(self) -> int: pingin, pingout = os.pipe() broadcastin, broadcastout = os.pipe() stdin, stdout = os.pipe() pid = os.fork() if pid == 0: # child worker process close_fd(stdin) close_fd(pingin) close_fd(broadcastout) self._child(stdout, pingout, broadcastin) else: # parent process close_fd(stdout) close_fd(pingout) close_fd(broadcastin) self._add_child(pid, stdin, pingin, broadcastout) return pid def _parent(self): signal.signal(signal.SIGCHLD, self._on_child_exited) signal.signal(signal.SIGINT, self._on_exit_parent) signal.signal(signal.SIGTERM, self._on_exit_parent) signal.signal(signal.SIGQUIT, self._on_exit_parent) log.info("finished forking all workers") try: self._setup_pubsub() try: while not self._stop_processing: self._read_child_logs() self._check_max_mem_percent() self._refork_missing_children() self._enqueue_ready_eta_tasks() self._cleanup_children() self._check_child_runtimes() self._listen_for_broadcast_task() except SoftTimeout: pass if len(self.children) > 0: log.info("shutting down...") while len(self.children) > 0: self._cleanup_children() self._check_child_runtimes() time.sleep(0.05) log.info("all workers stopped") except: try: log.error(traceback.format_exc()) except: print(traceback.format_exc()) self._stop() def _child(self, stdout, pingout, broadcastin): os.dup2(stdout, sys.stdout.fileno()) os.dup2(stdout, sys.stderr.fileno()) close_fd(stdout) os.set_blocking(pingout, False) os.set_blocking(broadcastin, False) os.set_blocking(sys.stdout.fileno(), False) os.set_blocking(sys.stderr.fileno(), False) self._pingout = pingout self._broadcastin = broadcastin # reset sigchld signal.signal(signal.SIGCHLD, signal.SIG_DFL) # stop processing and gracefully shutdown signal.signal(signal.SIGTERM, self._on_exit_child) # ignore ctrl-c sent to process group from terminal signal.signal(signal.SIGINT, signal.SIG_IGN) # raise SoftTimeout signal.signal(signal.SIGQUIT, self._on_soft_timeout_child) setup_logging(self.wakaq, is_child=True) try: # redis should eventually detect pid change and reset, but we force it self.wakaq.broker.connection_pool.reset() # cleanup file descriptors opened by parent process self._remove_all_children() self._num_tasks_processed = 0 log.debug("started worker process") if callable(self.wakaq.after_worker_started_callback): self.wakaq.after_worker_started_callback() self._active_async_tasks = set() self._async_task_context = {} self._loop = asyncio.new_event_loop() asyncio.set_event_loop(self._loop) self._loop.run_until_complete(self._event_loop()) except (MemoryError, BlockingIOError, BrokenPipeError): if current_task.get(): raise log.debug(traceback.format_exc()) except Exception as e: if exception_in_chain(e, SoftTimeout): if current_task.get(): raise log.error(traceback.format_exc()) else: log.error(traceback.format_exc()) except: # catch BaseException, SystemExit, KeyboardInterrupt, and GeneratorExit log.error(traceback.format_exc()) finally: if hasattr(self, "_loop"): self._loop.close() flush_fh(sys.stdout) flush_fh(sys.stderr) close_fd(self._broadcastin) close_fd(self._pingout) close_fd(sys.stdout) close_fd(sys.stderr) async def _event_loop(self): while not self._stop_processing and ( not self.wakaq.async_concurrency or len(self._active_async_tasks) < self.wakaq.async_concurrency ): self._send_ping_to_parent() queue_broker_key, payload = await self._blocking_dequeue() if payload is not None: try: task = self.wakaq.tasks[payload["name"]] except KeyError: log.error(f'Task not found: {payload["name"]}') task = None if task is not None: queue = self.wakaq.queues_by_key[queue_broker_key] current_task.set((task, payload)) retry = payload.get("retry") or 0 # make sure parent process is still around (OOM killer may have stopped it without sending child signal) try: self._send_ping_to_parent(task_name=task.name, queue_name=queue.name if queue else None) except: # give task back to queue so it's not lost self.wakaq.broker.lpush(queue_broker_key, serialize(payload)) current_task.set(None) raise async_task = self._loop.create_task(self._execute_task(task, payload, queue=queue)) self._active_async_tasks.add(async_task) self._async_task_context[async_task] = { "task": task, "payload": payload, "queue": queue, "start_time": time.time(), } try: if self._active_async_tasks: done, pending = await asyncio.wait( self._active_async_tasks, timeout=0.01, return_when=asyncio.FIRST_COMPLETED ) for async_task in done: self._active_async_tasks.remove(async_task) context = self._async_task_context.pop(async_task) try: await async_task except (MemoryError, BlockingIOError, BrokenPipeError): current_task.set((context["task"], context["payload"])) raise except asyncio.exceptions.CancelledError: current_task.set((context["task"], context["payload"])) retry += 1 max_retries = context["task"].max_retries if max_retries is None: max_retries = ( queue.max_retries if queue.max_retries is not None else self.wakaq.max_retries ) if retry > max_retries: log.error(traceback.format_exc()) else: log.warning(traceback.format_exc()) self.wakaq._enqueue_at_end( context["task"].name, queue.name, context["payload"]["args"], context["payload"]["kwargs"], retry=retry, ) except Exception as e: current_task.set((context["task"], context["payload"])) if exception_in_chain(e, SoftTimeout): retry += 1 max_retries = context["task"].max_retries if max_retries is None: max_retries = ( queue.max_retries if queue.max_retries is not None else self.wakaq.max_retries ) if retry > max_retries: log.error(traceback.format_exc()) else: log.warning(traceback.format_exc()) self.wakaq._enqueue_at_end( context["task"].name, queue.name, context["payload"]["args"], context["payload"]["kwargs"], retry=retry, ) else: log.error(traceback.format_exc()) for async_task in pending: context = self._async_task_context.get(async_task) if not context: continue soft_timeout, _ = get_timeouts(self.wakaq, task=context["task"], queue=context["queue"]) if not soft_timeout: continue runtime = time.time() - context["start_time"] if runtime - 0.1 > soft_timeout and not async_task.cancelled(): current_task.set((context["task"], context["payload"])) log.debug( f"async task {context['task'].name} runtime {runtime} reached soft timeout, raising asyncio.CancelledError" ) async_task.cancel() current_task.set(None) self._send_ping_to_parent() except (MemoryError, BlockingIOError, BrokenPipeError): raise # catch BaseException, SystemExit, KeyboardInterrupt, and GeneratorExit except: log.error(traceback.format_exc()) flush_fh(sys.stdout) flush_fh(sys.stderr) await self._execute_broadcast_tasks() if self.wakaq.max_tasks_per_worker and self._num_tasks_processed >= self.wakaq.max_tasks_per_worker: log.info(f"restarting worker after {self._num_tasks_processed} tasks") self._stop_processing = True flush_fh(sys.stdout) flush_fh(sys.stderr) def _send_ping_to_parent(self, task_name=None, queue_name=None): msg = task_name or "" if msg: msg = f"{msg}:{queue_name or ''}" write_fd_or_raise(self._pingout, f"{msg}\n") def _add_child(self, pid, stdin, pingin, broadcastout): self.children.append(Child(pid, stdin, pingin, broadcastout)) def _remove_all_children(self): for child in self.children: self._remove_child(child) def _cleanup_children(self): for child in self.children: if child.done: self._remove_child(child) def _remove_child(self, child): child.close() self.children = [c for c in self.children if c.pid != child.pid] def _on_exit_parent(self, signum, frame): log.debug(f"Received signal {signum}") self._stop() def _on_exit_child(self, signum, frame): self._stop_processing = True def _on_soft_timeout_child(self, signum, frame): raise SoftTimeout("SoftTimeout") def _on_child_exited(self, signum, frame): for child in self.children: if child.done: continue try: pid, _ = os.waitpid(child.pid, os.WNOHANG) if pid != 0: # child exited child.done = True except InterruptedError: # child exited while calling os.waitpid child.done = True except ChildProcessError: # child pid no longer valid child.done = True if child.done and child.max_mem_reached_at: after = round(time.time() - child.max_mem_reached_at, 2) log.info(f"Stopped {child.pid} after {after} seconds") def _enqueue_ready_eta_tasks(self): script = self.wakaq.broker.register_script(ZRANGEPOP) for queue in self.wakaq.queues: results = script(keys=[queue.broker_eta_key], args=[int(round(time.time()))]) for payload in results: payload = deserialize(payload) task_name = payload.pop("name") args = payload.pop("args") kwargs = payload.pop("kwargs") self.wakaq._enqueue_at_front(task_name, queue.name, args, kwargs) async def _execute_task(self, task, payload, queue=None): log.debug(f"running with payload {payload}") if callable(self.wakaq.before_task_started_callback): if inspect.iscoroutinefunction(self.wakaq.before_task_started_callback): await self.wakaq.before_task_started_callback() else: self.wakaq.before_task_started_callback() try: if callable(self.wakaq.wrap_tasks_function): if inspect.iscoroutinefunction(task.fn) and not inspect.iscoroutinefunction( self.wakaq.wrap_tasks_function ): raise WakaQError( "Unable to execute sync wrap_tasks_with when task is async. Make your wrap_tasks_with function async." ) if inspect.iscoroutinefunction(self.wakaq.wrap_tasks_function): await self.wakaq.wrap_tasks_function(task.fn, payload["args"], payload["kwargs"]) else: self.wakaq.wrap_tasks_function(task.fn, payload["args"], payload["kwargs"]) else: if inspect.iscoroutinefunction(task.fn): await task.fn(*payload["args"], **payload["kwargs"]) else: task.fn(*payload["args"], **payload["kwargs"]) finally: self._num_tasks_processed += 1 if callable(self.wakaq.after_task_finished_callback): if inspect.iscoroutinefunction(self.wakaq.after_task_finished_callback): await self.wakaq.after_task_finished_callback() else: self.wakaq.after_task_finished_callback() async def _execute_broadcast_tasks(self): payloads = read_fd(self._broadcastin) if payloads == "": return for payload in payloads.splitlines(): payload = deserialize(payload) try: task = self.wakaq.tasks[payload["name"]] except KeyError: log.error(f'Task not found: {payload["name"]}') continue retry = 0 current_task.set((task, payload)) while True: try: self._send_ping_to_parent(task_name=task.name) await self._execute_task(task, payload) current_task.set(None) self._send_ping_to_parent() break except (MemoryError, BlockingIOError, BrokenPipeError): raise except Exception as e: if exception_in_chain(e, SoftTimeout): retry += 1 max_retries = task.max_retries if max_retries is None: max_retries = self.wakaq.max_retries if retry > max_retries: log.error(traceback.format_exc()) break else: log.warning(traceback.format_exc()) else: log.error(traceback.format_exc()) break except: # catch BaseException, SystemExit, KeyboardInterrupt, and GeneratorExit log.error(traceback.format_exc()) break def _read_child_logs(self): for child in self.children: logs = read_fd(child.stdin) if logs: child.log_buffer += logs.encode("utf8") if not child.log_buffer: continue handler = log.handlers[0] stream = handler.stream if stream is None: # filehandle can disappear if we run out of RAM print(child.log_buffer.decode("utf8")) self._stop() return pending = child.log_buffer did_write = False try: fd = stream.fileno() except: try: decoded = pending.decode("utf8") chars_written = stream.write(decoded) if isinstance(chars_written, int) and chars_written > 0: bytes_written = len(decoded[:chars_written].encode("utf8")) pending = pending[bytes_written:] did_write = True else: pending = b"" except BlockingIOError as e: if hasattr(e, "characters_written") and e.characters_written: decoded = pending.decode("utf8") bytes_written = len(decoded[: e.characters_written].encode("utf8")) pending = pending[bytes_written:] did_write = True except BrokenPipeError: pending = b"" else: while pending: try: chars_written = os.write(fd, pending) pending = pending[chars_written:] did_write = True except BlockingIOError: break except BrokenPipeError: pending = b"" break child.log_buffer = pending if did_write: flush_fh(handler) def _check_max_mem_percent(self): if not self.wakaq.max_mem_percent: return max_mem_reached_at = max([c.max_mem_reached_at for c in self.children if not c.done], default=0) task_timeout = self.wakaq.hard_timeout or self.wakaq.soft_timeout or 120 now = time.time() if now - max_mem_reached_at < task_timeout: return if len(self.children) == 0: return percent_used = mem_usage_percent() if percent_used < self.wakaq.max_mem_percent: return log.info(f"Mem usage {percent_used}% is more than max_mem_percent threshold ({self.wakaq.max_mem_percent}%)") self._log_mem_usage_of_all_children() child = self._child_using_most_mem() if child: task = "" if child.current_task: task = f" while processing task {child.current_task.name}" log.info(f"Stopping child process {child.pid}{task}...") child.soft_timeout_reached = True # prevent raising SoftTimeout twice for same child child.max_mem_reached_at = now kill(child.pid, signal.SIGTERM) def _log_mem_usage_of_all_children(self): if self.wakaq.worker_log_level != logging.DEBUG: return for child in self.children: task = "" if child.current_task: task = f" while processing task {child.current_task.name}" try: log.debug(f"Child process {child.pid} using {round(child.mem_usage_percent, 2)}% ram{task}") except: log.warning(f"Unable to get ram usage of child process {child.pid}{task}") log.warning(traceback.format_exc()) def _child_using_most_mem(self): try: return max(self.children, key=lambda c: c.mem_usage_percent) except ValueError: return None def _check_child_runtimes(self): for child in self.children: child.ping_buffer += read_fd(child.pingin) if child.ping_buffer[-1:] == "\n": child.last_ping = time.time() child.soft_timeout_reached = False ping = child.ping_buffer[:-1] child.ping_buffer = "" ping = ping.rsplit("\n", 1)[-1] task, queue = None, None if ping != "": try: task_name, queue_name = ping.split(":", 1) task = self.wakaq.tasks[task_name] queue = self.wakaq.queues_by_name.get(queue_name) except ValueError: log.error(f"Unable to unpack message from child process {child.pid}: {ping}") child.set_timeouts(self.wakaq, task=task, queue=queue) else: soft_timeout = child.soft_timeout or self.wakaq.soft_timeout hard_timeout = child.hard_timeout or self.wakaq.hard_timeout if soft_timeout or hard_timeout: runtime = time.time() - child.last_ping if hard_timeout and runtime > hard_timeout: log.debug(f"child process {child.pid} runtime {runtime} reached hard timeout, sending sigkill") kill(child.pid, signal.SIGKILL) elif not child.soft_timeout_reached and soft_timeout and runtime > soft_timeout: log.debug(f"child process {child.pid} runtime {runtime} reached soft timeout, sending sigquit") child.soft_timeout_reached = True # prevent raising SoftTimeout twice for same child kill(child.pid, signal.SIGQUIT) def _listen_for_broadcast_task(self): if not self._pubsub: self._setup_pubsub() return try: msg = self._pubsub.get_message(ignore_subscribe_messages=True, timeout=self.wakaq.wait_timeout) except (ConnectionError, BrokenPipeError, OSError): log.warning("redis pubsub disconnected, reconnecting...") self._setup_pubsub() return if msg: payload = msg["data"] for child in self.children: if child.done: continue log.debug(f"run broadcast task: {payload}") write_fd(child.broadcastout, f"{payload}\n") break def _setup_pubsub(self): try: if getattr(self, "_pubsub", None): try: self._pubsub.close() except: log.debug(traceback.format_exc()) try: self.wakaq.broker.connection_pool.reset() except: log.debug(traceback.format_exc()) self._pubsub = self.wakaq.broker.pubsub() self._pubsub.subscribe(self.wakaq.broadcast_key) log.info("listening for broadcast tasks") except: self._pubsub = None log.warning("redis pubsub connection failure") log.debug(traceback.format_exc()) async def _blocking_dequeue(self): if len(self.wakaq.broker_keys) == 0: await asyncio.sleep(self.wakaq.wait_timeout) return None, None data = self.wakaq.broker.blpop(self.wakaq.broker_keys, self.wakaq.wait_timeout) if data is None: return None, None return data[0], deserialize(data[1]) def _refork_missing_children(self): if self._stop_processing: return if len(self.children) >= self.wakaq.concurrency: return # postpone fork missing children if using too much ram if self.wakaq.max_mem_percent: percent_used = mem_usage_percent() if percent_used >= self.wakaq.max_mem_percent: log.debug( f"postpone forking missing workers... mem usage {percent_used}% is more than max_mem_percent threshold ({self.wakaq.max_mem_percent}%)" ) return log.debug("restarting a crashed worker") self._fork()