[ { "path": ".coveragerc", "content": "[run]\nsource = celery_prometheus_exporter\n\n[report]\nfail_under = 100\nshow_missing = True\n\n[paths]\nsource = celery_prometheus_exporter\n" }, { "path": ".dockerignore", "content": "*.img" }, { "path": ".gitignore", "content": "*.img\n/dist\n/build\n/*.egg-info\n\n*.pyc\n__pycache__\n.coverage\n.tox/\n.cache/\n" }, { "path": ".travis.yml", "content": "sudo: false\nlanguage: python\n\npython:\n - \"2.7\" \n - \"3.4\"\n - \"3.5\"\n - \"3.6\"\n\ninstall: pip install tox-travis tox\nscript: tox\n" }, { "path": "CONTRIBUTING.md", "content": "The initial release of celery-prometheus-exporter was intended as a minimal\nsolution that would cover what I personally needed at my own projects. That\nbeing said, you might need completely different kinds of metrics being\nexposed. If you do, please feel free to create tickets and pull requests 🙂 As\nsuch, the more details you can provide in your tickets the better.\n\nI will try to look into each issue but please note that I might not be available\nall the time and that timezones exist. Please be patient 😊\n" }, { "path": "Dockerfile-celery3", "content": "FROM python:3.6-alpine\nMAINTAINER Horst Gutmann \n\nRUN mkdir -p /app/requirements\nADD requirements/* /app/requirements/\nWORKDIR /app\n\nENV PYTHONUNBUFFERED 1\nRUN pip install -r requirements/promclient050.txt -r requirements/celery3.txt\nADD celery_prometheus_exporter.py docker-entrypoint.sh /app/\nENTRYPOINT [\"/bin/sh\", \"/app/docker-entrypoint.sh\"]\nCMD []\n\nEXPOSE 8888\n" }, { "path": "Dockerfile-celery4", "content": "FROM python:3.6-alpine\nMAINTAINER Horst Gutmann \n\nRUN mkdir -p /app/requirements\nADD requirements/* /app/requirements/\nWORKDIR /app\n\nENV PYTHONUNBUFFERED 1\nRUN pip install -r requirements/promclient050.txt -r requirements/celery4.txt\nADD celery_prometheus_exporter.py docker-entrypoint.sh /app/\nENTRYPOINT [\"/bin/sh\", \"/app/docker-entrypoint.sh\"]\nCMD []\n\nEXPOSE 8888\n" }, { "path": "LICENSE.txt", "content": "MIT License\n\nCopyright (c) 2016, Horst Gutmann\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "MANIFEST.in", "content": "include README.rst celery_prometheus_exporter.py" }, { "path": "Makefile", "content": "all: celery_exporter-celery3.img celery_exporter-celery4.img\n\ncelery_exporter-celery3.img: celery_prometheus_exporter.py Dockerfile-celery3 requirements/*\n\tdocker build -f Dockerfile-celery3 -t celery_exporter:1-celery3 .\n\tdocker save -o $@ celery_exporter:1-celery3\n\ncelery_exporter-celery4.img: celery_prometheus_exporter.py Dockerfile-celery4 requirements/*\n\tdocker build -f Dockerfile-celery4 -t celery_exporter:1-celery4 .\n\tdocker save -o $@ celery_exporter:1-celery4\n\n.PHONY: clean all\nclean:\n\trm -rf celery_exporter.img *.egg-info build dist\n\npublish: all\n\tdocker tag celery_exporter:1-celery3 zerok/celery_exporter:1-celery3\n\tdocker tag celery_exporter:1-celery3 zerok/celery_exporter:1.3.0-celery3\n\tdocker tag celery_exporter:1-celery4 zerok/celery_exporter:1-celery4\n\tdocker tag celery_exporter:1-celery4 zerok/celery_exporter:1.3.0-celery4\n\tdocker push zerok/celery_exporter:1-celery4\n\tdocker push zerok/celery_exporter:1.3.0-celery4\n\tdocker push zerok/celery_exporter:1-celery3\n\tdocker push zerok/celery_exporter:1.3.0-celery3\n" }, { "path": "README.rst", "content": "==========================\ncelery-prometheus-exporter\n==========================\n\n.. admonition:: info\n\n Sadly, for the last couple of months at the time of writing this\n (Sept 2019) I couldn't find the time to maintain this package\n anymore. I therefore decided to archive it. If you find this code\n useful, please fork it!\n\n A big \"THANK YOU\" goes to everyone who contributed to this project\n over the years!\n\n.. image:: https://img.shields.io/docker/automated/zerok/celery-prometheus-exporter.svg?maxAge=2592000\n :target: https://hub.docker.com/r/zerok/celery-prometheus-exporter/\n\ncelery-prometheus-exporter is a little exporter for Celery related metrics in\norder to get picked up by Prometheus. As with other exporters like\nmongodb\\_exporter or node\\_exporter this has been implemented as a\nstandalone-service to make reuse easier across different frameworks.\n\nSo far it provides access to the following metrics:\n\n* ``celery_tasks`` exposes the number of tasks currently known to the queue\n grouped by ``state`` (RECEIVED, STARTED, ...).\n* ``celery_tasks_by_name`` exposes the number of tasks currently known to the queue\n grouped by ``name`` and ``state``.\n* ``celery_workers`` exposes the number of currently probably alive workers\n* ``celery_task_latency`` exposes a histogram of task latency, i.e. the time until\n tasks are picked up by a worker\n* ``celery_tasks_runtime_seconds`` tracks the number of seconds tasks take\n until completed as histogram\n\n\nHow to use\n==========\n\nThere are multiple ways to install this. The obvious one is using ``pip install\ncelery-prometheus-exporter`` and then using the ``celery-prometheus-exporter``\ncommand::\n\n $ celery-prometheus-exporter\n Starting HTTPD on 0.0.0.0:8888\n\nThis package only depends on Celery directly, so you will have to install\nwhatever other dependencies you will need for it to speak with your broker 🙂\n\nCelery workers have to be configured to send task-related events:\nhttp://docs.celeryproject.org/en/latest/userguide/configuration.html#worker-send-task-events.\n\nRunning ``celery-prometheus-exporter`` with the ``--enable-events`` argument\nwill periodically enable events on the workers. This is useful because it\nallows running celery workers with events disabled, until\n``celery-prometheus-exporter`` is deployed, at which time events get enabled\non the workers.\n\nAlternatively, you can use the bundle Makefile and Dockerfile to generate a\nDocker image.\n\nBy default, the HTTPD will listen at ``0.0.0.0:8888``. If you want the HTTPD\nto listen to another port, use the ``--addr`` option or the environment variable\n``DEFAULT_ADDR``.\n\nBy default, this will expect the broker to be available through\n``redis://redis:6379/0``, although you can change via environment variable\n``BROKER_URL``. If you're using AMQP or something else other than\nRedis, take a look at the Celery documentation and install the additioinal\nrequirements 😊 Also use the ``--broker`` option to specify a different broker\nURL.\n\nIf you need to pass additional options to your broker's transport use the\n``--transport-options`` option. It tries to read a dict from a JSON object.\nE.g. to set your master name when using Redis Sentinel for broker discovery:\n``--transport-options '{\"master_name\": \"mymaster\"}'``\n\nUse ``--tz`` to specify the timezone the Celery app is using. Otherwise the\nsystems local time will be used.\n\nBy default, buckets for histograms are the same as default ones in the prometheus client:\nhttps://github.com/prometheus/client_python#histogram.\nIt means they are intended to cover typical web/rpc requests from milliseconds to seconds,\nso you may want to customize them.\nIt can be done via environment variable ``RUNTIME_HISTOGRAM_BUCKETS`` for tasks runtime and\nvia environment variable ``LATENCY_HISTOGRAM_BUCKETS`` for tasks latency.\nBuckets should be passed as a list of float values separated by a comma.\nE.g. ``\".005, .05, 0.1, 1.0, 2.5\"``.\n\nUse ``--queue-list`` to specify the list of queues that will have its length\nmonitored (Automatic Discovery of queues isn't supported right now, see limitations/\ncaveats. You can use the `QUEUE_LIST` environment variable as well.\n\nIf you then look at the exposed metrics, you should see something like this::\n\n $ http get http://localhost:8888/metrics | grep celery_\n # HELP celery_workers Number of alive workers\n # TYPE celery_workers gauge\n celery_workers 1.0\n # HELP celery_tasks Number of tasks per state\n # TYPE celery_tasks gauge\n celery_tasks{state=\"RECEIVED\"} 3.0\n celery_tasks{state=\"PENDING\"} 0.0\n celery_tasks{state=\"STARTED\"} 1.0\n celery_tasks{state=\"RETRY\"} 2.0\n celery_tasks{state=\"FAILURE\"} 1.0\n celery_tasks{state=\"REVOKED\"} 0.0\n celery_tasks{state=\"SUCCESS\"} 8.0\n # HELP celery_tasks_by_name Number of tasks per state\n # TYPE celery_tasks_by_name gauge\n celery_tasks_by_name{name=\"my_app.tasks.calculate_something\",state=\"RECEIVED\"} 0.0\n celery_tasks_by_name{name=\"my_app.tasks.calculate_something\",state=\"PENDING\"} 0.0\n celery_tasks_by_name{name=\"my_app.tasks.calculate_something\",state=\"STARTED\"} 0.0\n celery_tasks_by_name{name=\"my_app.tasks.calculate_something\",state=\"RETRY\"} 0.0\n celery_tasks_by_name{name=\"my_app.tasks.calculate_something\",state=\"FAILURE\"} 0.0\n celery_tasks_by_name{name=\"my_app.tasks.calculate_something\",state=\"REVOKED\"} 0.0\n celery_tasks_by_name{name=\"my_app.tasks.calculate_something\",state=\"SUCCESS\"} 1.0\n celery_tasks_by_name{name=\"my_app.tasks.fetch_some_data\",state=\"RECEIVED\"} 3.0\n celery_tasks_by_name{name=\"my_app.tasks.fetch_some_data\",state=\"PENDING\"} 0.0\n celery_tasks_by_name{name=\"my_app.tasks.fetch_some_data\",state=\"STARTED\"} 1.0\n celery_tasks_by_name{name=\"my_app.tasks.fetch_some_data\",state=\"RETRY\"} 2.0\n celery_tasks_by_name{name=\"my_app.tasks.fetch_some_data\",state=\"FAILURE\"} 1.0\n celery_tasks_by_name{name=\"my_app.tasks.fetch_some_data\",state=\"REVOKED\"} 0.0\n celery_tasks_by_name{name=\"my_app.tasks.fetch_some_data\",state=\"SUCCESS\"} 7.0\n # HELP celery_task_latency Seconds between a task is received and started.\n # TYPE celery_task_latency histogram\n celery_task_latency_bucket{le=\"0.005\"} 2.0\n celery_task_latency_bucket{le=\"0.01\"} 3.0\n celery_task_latency_bucket{le=\"0.025\"} 4.0\n celery_task_latency_bucket{le=\"0.05\"} 4.0\n celery_task_latency_bucket{le=\"0.075\"} 5.0\n celery_task_latency_bucket{le=\"0.1\"} 5.0\n celery_task_latency_bucket{le=\"0.25\"} 5.0\n celery_task_latency_bucket{le=\"0.5\"} 5.0\n celery_task_latency_bucket{le=\"0.75\"} 5.0\n celery_task_latency_bucket{le=\"1.0\"} 5.0\n celery_task_latency_bucket{le=\"2.5\"} 8.0\n celery_task_latency_bucket{le=\"5.0\"} 11.0\n celery_task_latency_bucket{le=\"7.5\"} 11.0\n celery_task_latency_bucket{le=\"10.0\"} 11.0\n celery_task_latency_bucket{le=\"+Inf\"} 11.0\n celery_task_latency_count 11.0\n celery_task_latency_sum 16.478713035583496\n celery_queue_length{queue_name=\"queue1\"} 35.0\n celery_queue_length{queue_name=\"queue2\"} 0.0\n\nLimitations\n===========\n\n* Among tons of other features celery-prometheus-exporter doesn't support stats\n for multiple queues. As far as I can tell, only the routing key is exposed\n through the events API which might be enough to figure out the final queue,\n though.\n* This has only been tested with Redis so far.\n* At this point, you should specify the queues that will be monitored using an\n environment variable or an arg (`--queue-list`).\n" }, { "path": "celery_prometheus_exporter.py", "content": "from __future__ import print_function\nimport argparse\nimport celery\nimport celery.states\nimport celery.events\nimport collections\nfrom itertools import chain\nimport logging\nimport prometheus_client\nimport signal\nimport sys\nimport threading\nimport time\nimport json\nimport os\nfrom celery.utils.objects import FallbackContext\nimport amqp.exceptions\n\n__VERSION__ = (1, 2, 0, 'final', 0)\n\n\ndef decode_buckets(buckets_list):\n return [float(x) for x in buckets_list.split(',')]\n\n\ndef get_histogram_buckets_from_evn(env_name):\n if env_name in os.environ:\n buckets = decode_buckets(os.environ.get(env_name))\n else:\n if hasattr(prometheus_client.Histogram, 'DEFAULT_BUCKETS'): # pragma: no cover\n buckets = prometheus_client.Histogram.DEFAULT_BUCKETS\n else: # pragma: no cover\n # For prometheus-client < 0.3.0 we cannot easily access\n # the default buckets:\n buckets = (.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, float('inf'))\n return buckets\n\n\nDEFAULT_BROKER = os.environ.get('BROKER_URL', 'redis://redis:6379/0')\nDEFAULT_ADDR = os.environ.get('DEFAULT_ADDR', '0.0.0.0:8888')\nDEFAULT_MAX_TASKS_IN_MEMORY = int(os.environ.get('DEFAULT_MAX_TASKS_IN_MEMORY',\n '10000'))\nRUNTIME_HISTOGRAM_BUCKETS = get_histogram_buckets_from_evn('RUNTIME_HISTOGRAM_BUCKETS')\nLATENCY_HISTOGRAM_BUCKETS = get_histogram_buckets_from_evn('LATENCY_HISTOGRAM_BUCKETS')\nDEFAULT_QUEUE_LIST = os.environ.get('QUEUE_LIST', [])\n\nLOG_FORMAT = '[%(asctime)s] %(name)s:%(levelname)s: %(message)s'\n\nTASKS = prometheus_client.Gauge(\n 'celery_tasks', 'Number of tasks per state', ['state'])\nTASKS_NAME = prometheus_client.Gauge(\n 'celery_tasks_by_name', 'Number of tasks per state and name',\n ['state', 'name'])\nTASKS_RUNTIME = prometheus_client.Histogram(\n 'celery_tasks_runtime_seconds', 'Task runtime (seconds)', ['name'], buckets=RUNTIME_HISTOGRAM_BUCKETS)\nWORKERS = prometheus_client.Gauge(\n 'celery_workers', 'Number of alive workers')\nLATENCY = prometheus_client.Histogram(\n 'celery_task_latency', 'Seconds between a task is received and started.', buckets=LATENCY_HISTOGRAM_BUCKETS)\n\nQUEUE_LENGTH = prometheus_client.Gauge(\n 'celery_queue_length', 'Number of tasks in the queue.',\n ['queue_name']\n)\n\n\nclass MonitorThread(threading.Thread):\n \"\"\"\n MonitorThread is the thread that will collect the data that is later\n exposed from Celery using its eventing system.\n \"\"\"\n\n def __init__(self, app=None, *args, **kwargs):\n self._app = app\n self.log = logging.getLogger('monitor')\n self.log.info('Setting up monitor...')\n max_tasks_in_memory = kwargs.pop('max_tasks_in_memory',\n DEFAULT_MAX_TASKS_IN_MEMORY)\n self._state = self._app.events.State(\n max_tasks_in_memory=max_tasks_in_memory)\n self._known_states = set()\n self._known_states_names = set()\n self._tasks_started = dict()\n super(MonitorThread, self).__init__(*args, **kwargs)\n\n def run(self): # pragma: no cover\n self._monitor()\n\n def _process_event(self, evt):\n # Events might come in in parallel. Celery already has a lock\n # that deals with this exact situation so we'll use that for now.\n with self._state._mutex:\n if celery.events.group_from(evt['type']) == 'task':\n evt_state = evt['type'][5:]\n try:\n # Celery 4\n state = celery.events.state.TASK_EVENT_TO_STATE[evt_state]\n except AttributeError: # pragma: no cover\n # Celery 3\n task = celery.events.state.Task()\n task.event(evt_state)\n state = task.state\n if state == celery.states.STARTED:\n self._observe_latency(evt)\n self._collect_tasks(evt, state)\n\n def _observe_latency(self, evt):\n try:\n prev_evt = self._state.tasks[evt['uuid']]\n except KeyError: # pragma: no cover\n pass\n else:\n # ignore latency if it is a retry\n if prev_evt.state == celery.states.RECEIVED:\n LATENCY.observe(\n evt['local_received'] - prev_evt.local_received)\n\n def _collect_tasks(self, evt, state):\n if state in celery.states.READY_STATES:\n self._incr_ready_task(evt, state)\n else:\n # add event to list of in-progress tasks\n self._state._event(evt)\n self._collect_unready_tasks()\n\n def _incr_ready_task(self, evt, state):\n TASKS.labels(state=state).inc()\n try:\n # remove event from list of in-progress tasks\n event = self._state.tasks.pop(evt['uuid'])\n TASKS_NAME.labels(state=state, name=event.name).inc()\n if 'runtime' in evt:\n TASKS_RUNTIME.labels(name=event.name) \\\n .observe(evt['runtime'])\n except (KeyError, AttributeError): # pragma: no cover\n pass\n\n def _collect_unready_tasks(self):\n # count unready tasks by state\n cnt = collections.Counter(t.state for t in self._state.tasks.values())\n self._known_states.update(cnt.elements())\n for task_state in self._known_states:\n TASKS.labels(state=task_state).set(cnt[task_state])\n\n # count unready tasks by state and name\n cnt = collections.Counter(\n (t.state, t.name) for t in self._state.tasks.values() if t.name)\n self._known_states_names.update(cnt.elements())\n for task_state in self._known_states_names:\n TASKS_NAME.labels(\n state=task_state[0],\n name=task_state[1],\n ).set(cnt[task_state])\n\n def _monitor(self): # pragma: no cover\n while True:\n try:\n self.log.info('Connecting to broker...')\n with self._app.connection() as conn:\n recv = self._app.events.Receiver(conn, handlers={\n '*': self._process_event,\n })\n setup_metrics(self._app)\n recv.capture(limit=None, timeout=None, wakeup=True)\n self.log.info(\"Connected to broker\")\n except Exception:\n self.log.exception(\"Queue connection failed\")\n setup_metrics(self._app)\n time.sleep(5)\n\n\nclass WorkerMonitoringThread(threading.Thread):\n celery_ping_timeout_seconds = 5\n periodicity_seconds = 5\n\n def __init__(self, app=None, *args, **kwargs):\n self._app = app\n self.log = logging.getLogger('workers-monitor')\n super(WorkerMonitoringThread, self).__init__(*args, **kwargs)\n\n def run(self): # pragma: no cover\n while True:\n self.update_workers_count()\n time.sleep(self.periodicity_seconds)\n\n def update_workers_count(self):\n try:\n WORKERS.set(len(self._app.control.ping(\n timeout=self.celery_ping_timeout_seconds)))\n except Exception: # pragma: no cover\n self.log.exception(\"Error while pinging workers\")\n\n\nclass EnableEventsThread(threading.Thread):\n periodicity_seconds = 5\n\n def __init__(self, app=None, *args, **kwargs): # pragma: no cover\n self._app = app\n self.log = logging.getLogger('enable-events')\n super(EnableEventsThread, self).__init__(*args, **kwargs)\n\n def run(self): # pragma: no cover\n while True:\n try:\n self.enable_events()\n except Exception:\n self.log.exception(\"Error while trying to enable events\")\n time.sleep(self.periodicity_seconds)\n\n def enable_events(self):\n self._app.control.enable_events()\n\n\nclass QueueLengthMonitoringThread(threading.Thread):\n periodicity_seconds = 30\n\n def __init__(self, app, queue_list):\n # type: (celery.Celery, [str]) -> None\n self.celery_app = app\n self.queue_list = queue_list\n self.connection = self.celery_app.connection_or_acquire()\n\n if isinstance(self.connection, FallbackContext):\n self.connection = self.connection.fallback()\n\n super(QueueLengthMonitoringThread, self).__init__()\n\n def measure_queues_length(self):\n for queue in self.queue_list:\n try:\n length = self.connection.default_channel.queue_declare(queue=queue, passive=True).message_count\n except (amqp.exceptions.ChannelError,) as e:\n logging.warning(\"Queue Not Found: {}. Setting its value to zero. Error: {}\".format(queue, str(e)))\n length = 0\n\n self.set_queue_length(queue, length)\n\n def set_queue_length(self, queue, length):\n QUEUE_LENGTH.labels(queue).set(length)\n\n def run(self): # pragma: no cover\n while True:\n self.measure_queues_length()\n time.sleep(self.periodicity_seconds)\n\ndef setup_metrics(app):\n \"\"\"\n This initializes the available metrics with default values so that\n even before the first event is received, data can be exposed.\n \"\"\"\n WORKERS.set(0)\n logging.info('Setting up metrics, trying to connect to broker...')\n try:\n registered_tasks = app.control.inspect().registered_tasks().values()\n except Exception: # pragma: no cover\n for metric in TASKS.collect():\n for sample in metric.samples:\n TASKS.labels(**sample[1]).set(0)\n for metric in TASKS_NAME.collect():\n for sample in metric.samples:\n TASKS_NAME.labels(**sample[1]).set(0)\n\n else:\n for state in celery.states.ALL_STATES:\n TASKS.labels(state=state).set(0)\n for task_name in set(chain.from_iterable(registered_tasks)):\n TASKS_NAME.labels(state=state, name=task_name).set(0)\n\n\ndef start_httpd(addr): # pragma: no cover\n \"\"\"\n Starts the exposing HTTPD using the addr provided in a separate\n thread.\n \"\"\"\n host, port = addr.split(':')\n logging.info('Starting HTTPD on {}:{}'.format(host, port))\n prometheus_client.start_http_server(int(port), host)\n\n\ndef shutdown(signum, frame): # pragma: no cover\n \"\"\"\n Shutdown is called if the process receives a TERM signal. This way\n we try to prevent an ugly stacktrace being rendered to the user on\n a normal shutdown.\n \"\"\"\n logging.info(\"Shutting down\")\n sys.exit(0)\n\n\ndef main(): # pragma: no cover\n parser = argparse.ArgumentParser()\n parser.add_argument(\n '--broker', dest='broker', default=DEFAULT_BROKER,\n help=\"URL to the Celery broker. Defaults to {}\".format(DEFAULT_BROKER))\n parser.add_argument(\n '--transport-options', dest='transport_options',\n help=(\"JSON object with additional options passed to the underlying \"\n \"transport.\"))\n parser.add_argument(\n '--addr', dest='addr', default=DEFAULT_ADDR,\n help=\"Address the HTTPD should listen on. Defaults to {}\".format(\n DEFAULT_ADDR))\n parser.add_argument(\n '--enable-events', action='store_true',\n help=\"Periodically enable Celery events\")\n parser.add_argument(\n '--tz', dest='tz',\n help=\"Timezone used by the celery app.\")\n parser.add_argument(\n '--verbose', action='store_true', default=False,\n help=\"Enable verbose logging\")\n parser.add_argument(\n '--max_tasks_in_memory', dest='max_tasks_in_memory',\n default=DEFAULT_MAX_TASKS_IN_MEMORY, type=int,\n help=\"Tasks cache size. Defaults to {}\".format(\n DEFAULT_MAX_TASKS_IN_MEMORY))\n parser.add_argument(\n '--queue-list', dest='queue_list',\n default=DEFAULT_QUEUE_LIST, nargs='+',\n help=\"Queue List. Will be checked for its length.\"\n )\n parser.add_argument(\n '--version', action='version',\n version='.'.join([str(x) for x in __VERSION__]))\n opts = parser.parse_args()\n\n if opts.verbose:\n logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)\n else:\n logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)\n\n signal.signal(signal.SIGINT, shutdown)\n signal.signal(signal.SIGTERM, shutdown)\n\n if opts.tz:\n os.environ['TZ'] = opts.tz\n time.tzset()\n\n logging.info('Setting up celery for {}'.format(opts.broker))\n app = celery.Celery(broker=opts.broker)\n\n if opts.transport_options:\n try:\n transport_options = json.loads(opts.transport_options)\n except ValueError:\n print(\"Error parsing broker transport options from JSON '{}'\"\n .format(opts.transport_options), file=sys.stderr)\n sys.exit(1)\n else:\n app.conf.broker_transport_options = transport_options\n\n setup_metrics(app)\n\n t = MonitorThread(app=app, max_tasks_in_memory=opts.max_tasks_in_memory)\n t.daemon = True\n t.start()\n\n w = WorkerMonitoringThread(app=app)\n w.daemon = True\n w.start()\n\n if opts.queue_list:\n if type(opts.queue_list) == str:\n queue_list = opts.queue_list.split(',')\n else:\n queue_list = opts.queue_list\n\n q = QueueLengthMonitoringThread(app=app, queue_list=queue_list)\n\n q.daemon = True\n q.start()\n\n e = None\n if opts.enable_events:\n e = EnableEventsThread(app=app)\n e.daemon = True\n e.start()\n start_httpd(opts.addr)\n t.join()\n w.join()\n if e is not None:\n e.join()\n\n\nif __name__ == '__main__': # pragma: no cover\n main()\n" }, { "path": "celeryapp.py", "content": "from celery import Celery\nfrom kombu import Queue, Exchange\n\nimport os\nimport time\n\nBROKER_URL = os.getenv(\"BROKER_URL\")\nRESULT_BACKEND_URL = os.getenv(\"RESULT_BACKEND_URL\", None)\n\ncelery_app = Celery(\n broker=BROKER_URL,\n)\n\nif RESULT_BACKEND_URL:\n celery_app.conf.update(backend=RESULT_BACKEND_URL)\n\ncelery_app.conf.update(\n CELERY_DEFAULT_QUEUE=\"queue1\",\n CELERY_QUEUES=(\n Queue('queue1', exchange=Exchange('queue1', type='direct'), routing_key='queue1'),\n Queue('queue2', exchange=Exchange('queue2', type='direct'), routing_key='queue2'),\n Queue('queue3', exchange=Exchange('queue3', type='direct'), routing_key='queue3'),\n ),\n CELERY_ROUTES={\n 'task1': {'queue': 'queue1', 'routing_key': 'queue1'},\n 'task2': {'queue': 'queue2', 'routing_key': 'queue2'},\n 'task3': {'queue': 'queue3', 'routing_key': 'queue3'},\n }\n)\n\n@celery_app.task\ndef task1():\n time.sleep(20)\n\n@celery_app.task\ndef task2():\n time.sleep(20)\n\n@celery_app.task\ndef task3():\n time.sleep(20)\n" }, { "path": "docker-compose.yml", "content": "version: '2'\n\nservices:\n app:\n image: celery-exporter:3\n build:\n context: .\n dockerfile: Dockerfile-celery3\n user: \"65534\"\n volumes:\n - ./:/app\n environment:\n - BROKER_URL=amqp://rabbit\n entrypoint: celery -A celeryapp worker\n\n exporter:\n image: celery-exporter:3\n build:\n context: .\n dockerfile: Dockerfile-celery3\n volumes:\n - ./:/app\n environment:\n - BROKER_URL=amqp://rabbit\n - QUEUE_LIST=queue1,queue2,queue3\n ports:\n - 8888:8888\n\n cache:\n image: redis:alpine\n\n rabbit:\n image: rabbitmq:alpine\n" }, { "path": "docker-entrypoint.sh", "content": "#!/bin/sh\nexec python /app/celery_prometheus_exporter.py $@\n" }, { "path": "requirements/base.txt", "content": "redis==2.10.6\n" }, { "path": "requirements/celery3.txt", "content": "-r base.txt\ncelery==3.1.25\n" }, { "path": "requirements/celery4.txt", "content": "-r base.txt\ncelery==4.2.0\nkombu==4.3.0\n" }, { "path": "requirements/promclient030.txt", "content": "prometheus_client==0.3.0\n" }, { "path": "requirements/promclient050.txt", "content": "prometheus_client==0.5.0\n" }, { "path": "requirements/test.txt", "content": "-r base.txt\npytest\ncoverage" }, { "path": "setup.py", "content": "import io\n\nfrom setuptools import setup\n\n\nlong_description = \"See https://github.com/zerok/celery-prometheus-exporter\"\nwith io.open('README.rst', encoding='utf-8') as fp:\n long_description = fp.read()\n\nsetup(\n name='celery-prometheus-exporter',\n description=\"Simple Prometheus metrics exporter for Celery\",\n long_description=long_description,\n version='1.7.0',\n author='Horst Gutmann',\n license='MIT',\n author_email='horst@zerokspot.com',\n url='https://github.com/zerok/celery-prometheus-exporter',\n classifiers=[\n 'Development Status :: 3 - Alpha',\n 'Environment :: Console',\n 'License :: OSI Approved :: MIT License',\n 'Programming Language :: Python :: 3.5',\n 'Programming Language :: Python :: 3 :: Only',\n ],\n py_modules=[\n 'celery_prometheus_exporter',\n ],\n install_requires=[\n 'celery>=3',\n 'prometheus_client>=0.0.20',\n ],\n entry_points={\n 'console_scripts': [\n 'celery-prometheus-exporter = celery_prometheus_exporter:main',\n ],\n }\n)\n" }, { "path": "test/celery_test_utils.py", "content": "import celery\nimport time\nfrom kombu import Queue, Exchange\n\n\ndef get_celery_app(queue=None):\n app = celery.Celery(broker='memory://', backend='cache+memory://')\n\n if queue:\n app.conf.update(\n CELERY_DEFAULT_QUEUE=queue,\n CELERY_QUEUES=(\n Queue(queue, exchange=Exchange(queue, type='direct'), routing_key=queue),\n ),\n CELERY_ROUTES={\n 'task1': {'queue': queue, 'routing_key': queue},\n }\n )\n\n return app\n\n\nclass SampleTask(celery.Task):\n name = 'sample-task'\n\n def run(self, *args, **kwargs):\n time.sleep(10)\n" }, { "path": "test/test_unit.py", "content": "from time import time\n\nimport os\nimport celery\nimport celery.states\nimport amqp.exceptions\n\nfrom celery.events import Event\nfrom celery.utils import uuid\nfrom prometheus_client import REGISTRY\nfrom unittest import TestCase\ntry:\n from unittest.mock import patch\nexcept ImportError:\n from mock import patch\n\nfrom celery_prometheus_exporter import (\n WorkerMonitoringThread, setup_metrics, MonitorThread, EnableEventsThread,\n TASKS,\n get_histogram_buckets_from_evn,\n QueueLengthMonitoringThread, QUEUE_LENGTH)\n\nfrom celery_test_utils import get_celery_app, SampleTask\n\n\nclass TestBucketLoading(TestCase):\n def tearDown(self):\n if 'TEST_BUCKETS' in os.environ:\n del os.environ['TEST_BUCKETS']\n\n def test_default_buckets(self):\n self.assertIsNotNone(get_histogram_buckets_from_evn('TEST_BUCKETS'))\n\n def test_from_env(self):\n os.environ['TEST_BUCKETS'] = '1,2,3'\n self.assertEqual([1.0, 2.0, 3.0], get_histogram_buckets_from_evn('TEST_BUCKETS'))\n\nclass TestFallbackSetup(TestCase):\n def test_fallback(self):\n TASKS.labels(state='RUNNING').set(0)\n setup_metrics(None)\n\n\nclass TestMockedCelery(TestCase):\n task = 'my_task'\n\n def setUp(self):\n self.app = get_celery_app()\n with patch('celery.task.control.inspect.registered_tasks') as tasks:\n tasks.return_value = {'worker1': [self.task]}\n setup_metrics(self.app) # reset metrics\n\n def test_initial_metric_values(self):\n self._assert_task_states(celery.states.ALL_STATES, 0)\n assert REGISTRY.get_sample_value('celery_workers') == 0\n assert REGISTRY.get_sample_value('celery_task_latency_count') == 0\n assert REGISTRY.get_sample_value('celery_task_latency_sum') == 0\n\n def test_workers_count(self):\n assert REGISTRY.get_sample_value('celery_workers') == 0\n\n with patch.object(self.app.control, 'ping') as mock_ping:\n w = WorkerMonitoringThread(app=self.app)\n\n mock_ping.return_value = []\n w.update_workers_count()\n assert REGISTRY.get_sample_value('celery_workers') == 0\n\n mock_ping.return_value = [0] # 1 worker\n w.update_workers_count()\n assert REGISTRY.get_sample_value('celery_workers') == 1\n\n mock_ping.return_value = [0, 0] # 2 workers\n w.update_workers_count()\n assert REGISTRY.get_sample_value('celery_workers') == 2\n\n mock_ping.return_value = []\n w.update_workers_count()\n assert REGISTRY.get_sample_value('celery_workers') == 0\n\n def test_tasks_events(self):\n task_uuid = uuid()\n hostname = 'myhost'\n local_received = time()\n latency_before_started = 123.45\n runtime = 234.5\n\n m = MonitorThread(app=self.app)\n\n self._assert_task_states(celery.states.ALL_STATES, 0)\n assert REGISTRY.get_sample_value('celery_task_latency_count') == 0\n assert REGISTRY.get_sample_value('celery_task_latency_sum') == 0\n\n m._process_event(Event(\n 'task-received', uuid=task_uuid, name=self.task,\n args='()', kwargs='{}', retries=0, eta=None, hostname=hostname,\n clock=0,\n local_received=local_received))\n self._assert_all_states({celery.states.RECEIVED})\n\n m._process_event(Event(\n 'task-started', uuid=task_uuid, hostname=hostname,\n clock=1, name=self.task,\n local_received=local_received + latency_before_started))\n self._assert_all_states({celery.states.STARTED})\n\n m._process_event(Event(\n 'task-succeeded', uuid=task_uuid, result='42',\n runtime=runtime, hostname=hostname, clock=2,\n local_received=local_received + latency_before_started + runtime))\n self._assert_all_states({celery.states.SUCCESS})\n\n assert REGISTRY.get_sample_value('celery_task_latency_count') == 1\n self.assertAlmostEqual(REGISTRY.get_sample_value(\n 'celery_task_latency_sum'), latency_before_started)\n assert REGISTRY.get_sample_value(\n 'celery_tasks_runtime_seconds_count',\n labels=dict(name=self.task)) == 1\n assert REGISTRY.get_sample_value(\n 'celery_tasks_runtime_seconds_sum',\n labels=dict(name=self.task)) == 234.5\n\n def test_enable_events(self):\n with patch.object(\n self.app.control, 'enable_events') as mock_enable_events:\n e = EnableEventsThread(app=self.app)\n e.enable_events()\n mock_enable_events.assert_called_once_with()\n\n def test_can_measure_queue_length(self):\n celery_app = get_celery_app(queue='realqueue')\n sample_task = SampleTask()\n sample_task.app = celery_app\n monitoring_thread_instance = QueueLengthMonitoringThread(celery_app, queue_list=['realqueue'])\n\n sample_task.delay()\n monitoring_thread_instance.measure_queues_length()\n sample = REGISTRY.get_sample_value('celery_queue_length', {'queue_name':'realqueue'})\n\n self.assertEqual(1.0, sample)\n\n def test_set_zero_on_queue_length_when_an_channel_layer_error_occurs_during_queue_read(self):\n instance = QueueLengthMonitoringThread(app=self.app, queue_list=['noqueue'])\n\n instance.measure_queues_length()\n sample = REGISTRY.get_sample_value('celery_queue_length', {'queue_name':'noqueue'})\n\n self.assertEqual(0.0, sample)\n\n def _assert_task_states(self, states, cnt):\n for state in states:\n assert REGISTRY.get_sample_value(\n 'celery_tasks', labels=dict(state=state)) == cnt\n task_by_name_label = dict(state=state, name=self.task)\n assert REGISTRY.get_sample_value(\n 'celery_tasks_by_name', labels=task_by_name_label) == cnt\n\n def _assert_all_states(self, exclude):\n self._assert_task_states(celery.states.ALL_STATES - exclude, 0)\n self._assert_task_states(exclude, 1)\n\n def _setup_task_with_celery_and_queue_support(self, queue_name, task, celery_app):\n task.app = celery_app\n\n return task\n" }, { "path": "tox.ini", "content": "[tox]\nenvlist = py{27,34,35,36}-celery{3,4}-promclient{030,050}, lint\n\n[testenv]\ndeps =\n -rrequirements/test.txt\n py27: mock\n promclient030: -rrequirements/promclient030.txt\n promclient050: -rrequirements/promclient050.txt\n celery3: -rrequirements/celery3.txt\n celery4: -rrequirements/celery4.txt\ncommands =\n coverage run -m py.test -s -v {toxinidir}/test/\n coverage report\n\n[testenv:lint]\nbasepython = python3\ndeps = flake8>=3.3.0,<4\ncommands = flake8 --max-complexity 15 celery_prometheus_exporter.py test\n" } ]