Repository: Supervisor/superlance Branch: main Commit: ebde11bd8b57 Files: 42 Total size: 157.1 KB Directory structure: gitextract_7woj_tmr/ ├── .github/ │ └── workflows/ │ └── main.yml ├── .gitignore ├── .readthedocs.yaml ├── CHANGES.rst ├── COPYRIGHT.txt ├── LICENSE.txt ├── MANIFEST.in ├── README.rst ├── docs/ │ ├── Makefile │ ├── conf.py │ ├── crashmail.rst │ ├── crashmailbatch.rst │ ├── crashsms.rst │ ├── development.rst │ ├── fatalmailbatch.rst │ ├── httpok.rst │ ├── index.rst │ └── memmon.rst ├── setup.cfg ├── setup.py ├── superlance/ │ ├── __init__.py │ ├── compat.py │ ├── crashmail.py │ ├── crashmailbatch.py │ ├── crashsms.py │ ├── fatalmailbatch.py │ ├── httpok.py │ ├── memmon.py │ ├── process_state_email_monitor.py │ ├── process_state_monitor.py │ ├── tests/ │ │ ├── __init__.py │ │ ├── dummy.py │ │ ├── test_crashmail.py │ │ ├── test_crashmailbatch.py │ │ ├── test_crashsms.py │ │ ├── test_fatalmailbatch.py │ │ ├── test_httpok.py │ │ ├── test_memmon.py │ │ ├── test_process_state_email_monitor.py │ │ └── test_process_state_monitor.py │ └── timeoutconn.py └── tox.ini ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/workflows/main.yml ================================================ name: Run all tests on: [push, pull_request] env: PIP: "env PIP_DISABLE_PIP_VERSION_CHECK=1 PYTHONWARNINGS=ignore:DEPRECATION pip --no-cache-dir" jobs: tests_py2x: runs-on: ubuntu-22.04 container: image: python:2.7 strategy: fail-fast: false matrix: toxenv: [py27] steps: - uses: actions/checkout@v4 - name: Install dependencies run: $PIP install virtualenv tox - name: Run the unit tests run: TOXENV=${{ matrix.toxenv }} tox tests_py34: runs-on: ubuntu-22.04 container: image: ubuntu:20.04 env: LANG: C.UTF-8 steps: - uses: actions/checkout@v4 - name: Install build dependencies run: | apt-get update apt-get install -y build-essential unzip wget \ libncurses5-dev libgdbm-dev libnss3-dev \ libreadline-dev zlib1g-dev - name: Build OpenSSL 1.0.2 (required by Python 3.4) run: | cd $RUNNER_TEMP wget https://github.com/openssl/openssl/releases/download/OpenSSL_1_0_2u/openssl-1.0.2u.tar.gz tar -xf openssl-1.0.2u.tar.gz cd openssl-1.0.2u ./config --prefix=/usr/local/ssl --openssldir=/usr/local/ssl shared zlib-dynamic make make install echo CFLAGS="-I/usr/local/ssl/include $CFLAGS" >> $GITHUB_ENV echo LDFLAGS="-L/usr/local/ssl/lib $LDFLAGS" >> $GITHUB_ENV echo LD_LIBRARY_PATH="/usr/local/ssl/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV ln -s /usr/local/ssl/lib/libssl.so.1.0.0 /usr/lib/libssl.so.1.0.0 ln -s /usr/local/ssl/lib/libcrypto.so.1.0.0 /usr/lib/libcrypto.so.1.0.0 ldconfig - name: Build Python 3.4 run: | cd $RUNNER_TEMP wget -O cpython-3.4.10.zip https://github.com/python/cpython/archive/refs/tags/v3.4.10.zip unzip cpython-3.4.10.zip cd cpython-3.4.10 ./configure --with-ensurepip=install make make install python3.4 --version python3.4 -c 'import ssl' pip3.4 --version ln -s /usr/local/bin/python3.4 /usr/local/bin/python ln -s /usr/local/bin/pip3.4 /usr/local/bin/pip - name: Install Python dependencies run: | $PIP install virtualenv==20.4.7 tox==3.14.0 - name: Run the unit tests run: TOXENV=py34 tox tests_py35: runs-on: ubuntu-22.04 container: image: python:3.5 strategy: fail-fast: false steps: - uses: actions/checkout@v4 - name: Install dependencies run: $PIP install virtualenv tox - name: Run the unit tests run: TOXENV=py35 tox tests_py36: runs-on: ubuntu-22.04 container: image: python:3.6 strategy: fail-fast: false steps: - uses: actions/checkout@v4 - name: Install dependencies run: $PIP install virtualenv tox - name: Run the unit tests run: TOXENV=py36 tox tests_py3x: runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: python-version: [3.7, 3.8, 3.9, "3.10", 3.11, 3.12, 3.13, 3.14] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: $PIP install virtualenv tox - name: Set variable for TOXENV based on Python version id: toxenv run: python -c 'import sys; print("TOXENV=py%d%d" % (sys.version_info.major, sys.version_info.minor))' | tee -a $GITHUB_OUTPUT - name: Run the unit tests run: TOXENV=${{steps.toxenv.outputs.TOXENV}} tox - name: Run the end-to-end tests run: TOXENV=${{steps.toxenv.outputs.TOXENV}} END_TO_END=1 tox docs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: "3.8" - name: Install dependencies run: $PIP install virtualenv tox>=4.0.0 - name: Build the docs run: TOXENV=docs tox ================================================ FILE: .gitignore ================================================ *~ *.egg *.egg-info *.pyc *.pyo *.swp .DS_Store .coverage .eggs/ .tox/ build/ docs/_build/ dist/ env*/ htmlcov/ tmp/ coverage.xml nosetests.xml ================================================ FILE: .readthedocs.yaml ================================================ # .readthedocs.yaml # Read the Docs configuration file # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details # Required version: 2 # Set the version of Python and other tools you might need build: os: ubuntu-22.04 tools: python: "3.11" # Build documentation in the docs/ directory with Sphinx sphinx: configuration: docs/conf.py # We recommend specifying your dependencies to enable reproducible builds: # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html # python: # install: # - requirements: docs/requirements.txt ================================================ FILE: CHANGES.rst ================================================ 2.0.1.dev0 (Next Release) ------------------------- 2.0.0 (2021-12-26) ------------------ - Support for Python 2.6 has been dropped. On Python 2, Superlance now requires Python 2.7. - Support for Python 3.2 and 3.3 has been dropped. On Python 3, Superlance now requires Python 3.4 or later. - Fixed a bug introduced in 0.10 where if the timeout value is shorter than the time to wait between retries, the httpok check never executed. Issue #110. - Fixed a bug where ``crashmailbatch`` and ``fatalmatchbatch`` did not set the intended default subject. Patch by Joe Portela. - Added a new ``--tls`` option to ``crashmailbatch``, ``fatalmailbath``, and ``crashsms`` to use Transport Layer Security (TLS). Patch by Zhe Li. 1.0.0 (2016-10-02) ------------------ - Support for Python 2.5 has been dropped. On Python 2, Superlance now requires Python 2.6 or later. - Support for Python 3 has been added. On Python 3, Superlance requires Python 3.2 or later. - Fixed parsing of ``-n`` and ``--name`` options in ``httpok``. Patch by DenisBY. 0.14 (2016-09-24) ----------------- - Fixed docs build. 0.13 (2016-09-05) ----------------- - ``httpok`` now allows multiple expected status codes to be specified. Patch by valmiRe. - ``httpok`` now has a ``--name`` option like ``memmon``. - All commands now return exit status 0 from ``--help``. 0.12 (2016-09-03) ----------------- - Fixed ``crashmail`` parsing of ``--optionalheader``. Patch by Matt Dziuban. 0.11 (2014-08-15) ----------------- - Added support for ``memmon`` to check against cumulative RSS of a process and all its child processes. Patch by Lukas Graf. - Fixed a bug introduced in 0.9 where the ``-u`` and ``-n`` options in ``memmon`` were parsed incorrectly. Patch by Harald Friessnegger. 0.10 (2014-07-08) ----------------- - Honor timeout in httok checks even on trying the connection. Without it, processes that take make than 60 seconds to accept connections and http_ok with TICK_60 events cause a permanent restart of the process. - ``httpok`` now sends a ``User-Agent`` header of ``httpok``. - Removed ``setuptools`` from the ``requires`` list in ``setup.py`` because it caused installation issues on some systems. 0.9 (2013-09-18) ---------------- - Added license. - Fixed bug in cmd line option validator for ProcessStateEmailMonitor Bug report by Val Jordan - Added ``-u`` option to memmon the only send an email in case the restarted process' uptime (in seconds) is below this limit. This is useful to only get notified if a processes gets restarted too frequently. Patch by Harald Friessnegger. 0.8 (2013-05-26) ---------------- - Superlance will now refuse to install on an unsupported version of Python. - Allow SMTP credentials to be supplied to ProcessStateEmailMonitor Patch by Steven Davidson. - Added ``-n`` option to memmon that adds this name to the email subject to identify which memmon process restarted a process. Useful in case you run multiple supervisors that control different processes with the same name. Patch by Harald Friessnegger. - ProcessStateEmailMonitor now adds Date and Message-ID headers to emails. Patch by Andrei Vereha. 0.7 (2012-08-22) ---------------- - The ``crashmailbatch --toEmail`` option now accepts a comma-separated list of email addresses. 0.6 (2011-08-27) ---------------- - Separated unit tests into their own files - Created ``fatalmailbatch`` plugin - Created ``crashmailbatch`` plugin - Sphinxified documentation. - Fixed ``test_suite`` to use the correct module name in setup.py. - Fixed the tests for ``memmon`` to import the correct module. - Applied patch from Sam Bartlett: processes which are not autostarted have pid "0". This was crashing ``memmon``. - Add ``smtpHost`` command line flag to ``mailbatch`` processors. - Added ``crashsms`` from Juan Batiz-Benet - Converted ``crashmailbatch`` and friends from camel case to pythonic style - Fixed a bug where ``httpok`` would crash with the ``-b`` (in-body) option. Patch by Joaquin Cuenca Abela. - Fixed a bug where ``httpok`` would not handle a URL with a query string correctly. Patch by Joaquin Cuenca Abela. - Fixed a bug where ``httpok`` would not handle process names with a group ("group:process") properly. Patch by Joaquin Cuenca Abela. 0.5 (2009-05-24) ---------------- - Added the ``memmon`` plugin, originally bundled with supervisor and now moved to superlance. 0.4 (2009-02-11) ---------------- - Added ``eager`` and ``not-eager`` options to the ``httpok`` plugin. If ``not-eager`` is set, and no process being monitored is in the ``RUNNING`` state, skip the URL check / mail message. 0.3 (2008-12-10) ---------------- - Added ``gcore`` and ``coredir`` options to the ``httpok`` plugin. 0.2 (2008-11-21) ---------------- - Added the ``crashmail`` plugin. 0.1 (2008-09-18) ---------------- - Initial release ================================================ FILE: COPYRIGHT.txt ================================================ Superlance is Copyright (c) 2008-2013 Agendaless Consulting and Contributors. (http://www.agendaless.com), All Rights Reserved This software is subject to the provisions of the license at http://www.repoze.org/LICENSE.txt . A copy of this license should accompany this distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. ================================================ FILE: LICENSE.txt ================================================ Superlance is licensed under the following license: A copyright notice accompanies this license document that identifies the copyright holders. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions in source code must retain the accompanying copyright notice, this list of conditions, and the following disclaimer. 2. Redistributions in binary form must reproduce the accompanying copyright notice, this list of conditions, and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Names of the copyright holders must not be used to endorse or promote products derived from this software without prior written permission from the copyright holders. 4. If any files are modified, you must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. Disclaimer THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================ FILE: MANIFEST.in ================================================ include CHANGES.rst include COPYRIGHT.txt include LICENSE.txt include README.rst include docs/Makefile recursive-include docs *.py *.rst recursive-exclude docs/_build * ================================================ FILE: README.rst ================================================ superlance README ================= Superlance is a package of plugin utilities for monitoring and controlling processes that run under `supervisor `_. Please see ``docs/index.rst`` for complete documentation. ================================================ FILE: docs/Makefile ================================================ # Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: -rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/superlance.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/superlance.qhc" latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ "run these through (pdf)latex." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." ================================================ FILE: docs/conf.py ================================================ # -*- coding: utf-8 -*- # # superlance documentation build configuration file, created by # sphinx-quickstart on Thu Jun 10 11:55:43 2010. # # This file is execfile()d with the current directory set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import sys, os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.append(os.path.abspath('.')) # -- General configuration ----------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8' # The master toctree document. master_doc = 'index' # General information about the project. project = u'superlance' copyright = u'2010, Chris McDonough, Agendaless Consulting, Inc.' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = '2.0.1.dev0' # The full version, including alpha/beta/rc tags. release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of documents that shouldn't be included in the build. #unused_docs = [] # List of directories, relative to source directory, that shouldn't be searched # for source files. exclude_trees = ['_build'] # The reST default role (used for this markup: `text`) to use for all documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. Major themes that come with # Sphinx are currently 'default' and 'sphinxdoc'. html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". #html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_use_modindex = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = '' # Output file base name for HTML help builder. htmlhelp_basename = 'superlancedoc' # -- Options for LaTeX output -------------------------------------------------- # The paper size ('letter' or 'a4'). #latex_paper_size = 'letter' # The font size ('10pt', '11pt' or '12pt'). #latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ('index', 'superlance.tex', u'superlance Documentation', u'Chris McDonough, Agendaless Consulting, Inc.', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # Additional stuff for the LaTeX preamble. #latex_preamble = '' # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_use_modindex = True ================================================ FILE: docs/crashmail.rst ================================================ :command:`crashmail` Documentation ================================== :command:`crashmail` is a supervisor "event listener", intended to be subscribed to ``PROCESS_STATE_EXITED`` events. When :command:`crashmail` receives that event, and the transition is "unexpected", :command:`crashmail` sends an email notification to a configured address. :command:`crashmail` is incapable of monitoring the process status of processes which are not :command:`supervisord` child processes. :command:`crashmail` is a "console script" installed when you install :mod:`superlance`. Although :command:`crashmail` is an executable program, it isn't useful as a general-purpose script: it must be run as a :command:`supervisor` event listener to do anything useful. Command-Line Syntax ------------------- .. code-block:: sh $ crashmail [-p processname] [-a] [-o string] [-m mail_address] \ [-s sendmail] .. program:: crashmail .. cmdoption:: -p , --program= Send mail when the specified :command:`supervisord` child process transitions unexpectedly to the ``EXITED`` state. This option can be provided more than once to have :command:`crashmail` monitor more than one program. To monitor a process which is part of a :command:`supervisord` group, specify its name as ``group_name:process_name``. .. cmdoption:: -a, --any Send mail when any :command:`supervisord` child process transitions unexpectedly to the ``EXITED`` state. Overrides any ``-p`` parameters passed in the same :command:`crashmail` process invocation. .. cmdoption:: -o , --optionalheader= Specify a parameter used as a prefix in the mail :mailheader:`Subject` header. .. cmdoption:: -s , --sendmail_program= Specify the sendmail command to use to send email. Must be a command which accepts header and message data on stdin and sends mail. Default is ``/usr/sbin/sendmail -t -i``. .. cmdoption:: -m , --email= Specify an email address to which crash notification messages are sent. If no email address is specified, email will not be sent. Configuring :command:`crashmail` Into the Supervisor Config ----------------------------------------------------------- An ``[eventlistener:x]`` section must be placed in :file:`supervisord.conf` in order for :command:`crashmail` to do its work. See the "Events" chapter in the Supervisor manual for more information about event listeners. The following example assumes that :command:`crashmail` is on your system :envvar:`PATH`. .. code-block:: ini [eventlistener:crashmail] command=crashmail -p program1 -p group1:program2 -m dev@example.com events=PROCESS_STATE_EXITED ================================================ FILE: docs/crashmailbatch.rst ================================================ :command:`crashmailbatch` Documentation ======================================= :command:`crashmailbatch` is a supervisor "event listener", intended to be subscribed to ``PROCESS_STATE`` and ``TICK_60`` events. It monitors all processes running under a given supervisord instance. Similar to :command:`crashmail`, :command:`crashmailbatch` sends email alerts when processes die unexpectedly. The difference is that all alerts generated within the configured time interval are batched together to avoid sending too many emails. :command:`crashmailbatch` is a "console script" installed when you install :mod:`superlance`. Although :command:`crashmailbatch` is an executable program, it isn't useful as a general-purpose script: it must be run as a :command:`supervisor` event listener to do anything useful. Command-Line Syntax ------------------- .. code-block:: sh $ crashmailbatch --toEmail= --fromEmail= \ [--interval=] [--subject=] \ [--tickEvent=] [--smtpHost=] \ [--userName=] [--password=] \ [--tls] .. program:: crashmailbatch .. cmdoption:: -t , --toEmail= Specify comma separated email addresses to which crash notification messages are sent. .. cmdoption:: -f , --fromEmail= Specify an email address from which crash notification messages are sent. .. cmdoption:: -i , --interval= Specify the time interval in minutes to use for batching notifcations. Defaults to 1.0 minute. .. cmdoption:: -s , --subject= Override the email subject line. Defaults to "Crash alert from supervisord" .. cmdoption:: -e , --tickEvent= Override the TICK event name. Defaults to "TICK_60" .. cmdoption:: -H --smtpHost Specify STMP server for sending email .. cmdoption:: -u --userName Specify STMP username .. cmdoption:: -p --password Specify STMP password .. cmdoption:: --tls Use Transport Layer Security (TLS) Configuring :command:`crashmailbatch` Into the Supervisor Config ---------------------------------------------------------------- An ``[eventlistener:x]`` section must be placed in :file:`supervisord.conf` in order for :command:`crashmailbatch` to do its work. See the "Events" chapter in the Supervisor manual for more information about event listeners. The following example assumes that :command:`crashmailbatch` is on your system :envvar:`PATH`. .. code-block:: ini [eventlistener:crashmailbatch] command=crashmailbatch --toEmail="alertme@fubar.com" --fromEmail="supervisord@fubar.com" events=PROCESS_STATE,TICK_60 ================================================ FILE: docs/crashsms.rst ================================================ :command:`crashsms` Documentation ================================== :command:`crashsms` is a supervisor "event listener", intended to be subscribed to ``PROCESS_STATE`` events and ``TICK`` events such as ``TICK_60``. It monitors all processes running under a given supervisord instance. Similar to :command:`crashmailbatch`, :command:`crashsms` sends SMS alerts through an email gateway. Messages are formatted to fit in SMS :command:`crashsms` is a "console script" installed when you install :mod:`superlance`. Although :command:`crashsms` is an executable program, it isn't useful as a general-purpose script: it must be run as a :command:`supervisor` event listener to do anything useful. Command-Line Syntax ------------------- .. code-block:: sh $ crashsms --toEmail= --fromEmail= \ [--interval=] [--subject=] \ [--tickEvent=] .. program:: crashsms .. cmdoption:: -t , --toEmail= Specify comma separated email addresses to which crash notification messages are sent. .. cmdoption:: -f , --fromEmail= Specify an email address from which crash notification messages are sent. .. cmdoption:: -i , --interval= Specify the time interval in minutes to use for batching notifcations. Defaults to 1.0 minute. .. cmdoption:: -s , --subject= Set the email subject line. Default is None .. cmdoption:: -e , --tickEvent= Override the TICK event name. Defaults to "TICK_60" Configuring :command:`crashsms` Into the Supervisor Config ----------------------------------------------------------- An ``[eventlistener:x]`` section must be placed in :file:`supervisord.conf` in order for :command:`crashsms` to do its work. See the "Events" chapter in the Supervisor manual for more information about event listeners. The following example assumes that :command:`crashsms` is on your system :envvar:`PATH`. .. code-block:: ini [eventlistener:crashsms] command=crashsms --toEmail="@" --fromEmail="supervisord@fubar.com" events=PROCESS_STATE,TICK_60 ================================================ FILE: docs/development.rst ================================================ Resources and Development ========================= Bug Tracker ----------- Superlance has a bug tracker where you may report any bugs or other errors you find. Please report bugs to the `GitHub issues page `_. Version Control Repository -------------------------- You can also view the `Superlance version control repository `_. Contributing ------------ `Pull requests `_ can be submitted to the Superlance repository on GitHub. In the time since Superlance was created, there are now many `third party plugins for Supervisor `_. Most new plugins should be in their own package rather than added to Superlance. Author Information ------------------ The following people are responsible for creating Superlance. Original Author ~~~~~~~~~~~~~~~ `Chris McDonough `_ is the original author of Superlance. Contributors ~~~~~~~~~~~~ Contributors are tracked on the `GitHub contributions page `_. The list below is included for historical reasons. It records contributors who signed a legal agreement. The legal agreement was `introduced `_ in January 2014 but later `withdrawn `_ in April 2014. This list is being preserved in case it is useful later (e.g. if at some point there was a desire to donate the project to a foundation that required such agreements). - Chris McDonough, 2008-09-18 - Tres Seaver, 2009-02-11 - Roger Hoover, 2010-07-30 - Joaquín Cuenca Abela, 2011-06-23 - Harald Friessnegger, 2012-11-01 - Mikhail Lukyanchenko, 2013-12-23 - Patrick Gerken, 2014-01-27 ================================================ FILE: docs/fatalmailbatch.rst ================================================ :command:`fatalmailbatch` Documentation ======================================= :command:`fatalmailbatch` is a supervisor "event listener", intended to be subscribed to ``PROCESS_STATE`` and ``TICK_60`` events. It monitors all processes running under a given supervisord instance. :command:`fatalmailbatch` sends email alerts when processes fail to start too many times such that supervisord gives up retrying. All of the fatal start events generated within the configured time interval are batched together to avoid sending too many emails. :command:`fatalmailbatch` is a "console script" installed when you install :mod:`superlance`. Although :command:`fatalmailbatch` is an executable program, it isn't useful as a general-purpose script: it must be run as a :command:`supervisor` event listener to do anything useful. Command-Line Syntax ------------------- .. code-block:: sh $ fatalmailbatch --toEmail= --fromEmail= \ [--interval=] [--subject=] .. program:: fatalmailbatch .. cmdoption:: -t , --toEmail= Specify comma separated email addresses to which fatal start notification messages are sent. .. cmdoption:: -f , --fromEmail= Specify an email address from which fatal start notification messages are sent. .. cmdoption:: -i , --interval= Specify the time interval in minutes to use for batching notifcations. Defaults to 1 minute. .. cmdoption:: -s , --subject= Override the email subject line. Defaults to "Fatal start alert from supervisord" Configuring :command:`fatalmailbatch` Into the Supervisor Config ---------------------------------------------------------------- An ``[eventlistener:x]`` section must be placed in :file:`supervisord.conf` in order for :command:`fatalmailbatch` to do its work. See the "Events" chapter in the Supervisor manual for more information about event listeners. The following example assumes that :command:`fatalmailbatch` is on your system :envvar:`PATH`. .. code-block:: ini [eventlistener:fatalmailbatch] command=fatalmailbatch --toEmail="alertme@fubar.com" --fromEmail="supervisord@fubar.com" events=PROCESS_STATE,TICK_60 ================================================ FILE: docs/httpok.rst ================================================ :command:`httpok` Documentation ================================== :command:`httpok` is a supervisor "event listener" which may be subscribed to a concrete ``TICK_5``, ``TICK_60`` or ``TICK_3600`` event. When :command:`httpok` receives a ``TICK_x`` event (``TICK_60`` is recommended, indicating activity every 60 seconds), :command:`httpok` makes an HTTP GET request to a confgured URL. If the request fails or times out, :command:`httpok` will restart the "hung" child process(es). :command:`httpok` can be configured to send an email notification when it restarts a process. :command:`httpok` can only monitor the process status of processes which are :command:`supervisord` child processes. :command:`httpok` is a "console script" installed when you install :mod:`superlance`. Although :command:`httpok` is an executable program, it isn't useful as a general-purpose script: it must be run as a :command:`supervisor` event listener to do anything useful. Command-Line Syntax ------------------- .. code-block:: sh $ httpok [-p processname] [-a] [-g] [-t timeout] [-c status_code] \ [-b inbody] [-m mail_address] [-s sendmail] URL .. program:: httpok .. cmdoption:: -p , --program= Restart the :command:`supervisord` child process named ``process_name`` if it is in the ``RUNNING`` state when the URL returns an unexpected result or times out. This option can be provided more than once to have :command:`httpok` monitor more than one process. To monitor a process which is part of a :command:`supervisord` group, specify its name as ``group_name:process_name``. .. cmdoption:: -a, --any Restart any child of :command:`supervisord` in the ``RUNNING`` state if the URL returns an unexpected result or times out. Overrides any ``-p`` parameters passed in the same :command:`httpok` process invocation. .. cmdoption:: -g , --gcore= Use the specifed program to ``gcore`` the :command:`supervisord` child process. The program should accept two arguments on the command line: a filename and a pid. Defaults to ``/usr/bin/gcore -o``. .. cmdoption:: -d , --coredir= If a core directory is specified, :command:`httpok` will try to use the ``gcore`` program (see ``-g``) to write a core file into this directory for each hung process before restarting it. It will then append any gcore stdout output to the email message, if mail is configured (see the ``-m`` option below). .. cmdoption:: -t , --timeout= The number of seconds that :command:`httpok` should wait for a response to the HTTP request before timing out. If this timeout is exceeded, :command:`httpok` will attempt to restart child processes which are in the ``RUNNING`` state, and specified by ``-p`` or ``-a``. Defaults to 10 seconds. .. cmdoption:: -c , --code= Specify the expected HTTP status code for the configured URL. If this status code is not the status code provided by the response, :command:`httpok` will attempt to restart child processes which are in the ``RUNNING`` state, and specified by ``-p`` or ``-a``. Defaults to 200. .. cmdoption:: -b , --body= Specify a string which should be present in the body resulting from the GET request. If this string is not present in the response, :command:`httpok` will attempt to restart child processes which are in the RUNNING state, and specified by ``-p`` or ``-a``. The default is to ignore the body. .. cmdoption:: -s , --sendmail_program= Specify the sendmail command to use to send email. Must be a command which accepts header and message data on stdin and sends mail. Default is ``/usr/sbin/sendmail -t -i``. .. cmdoption:: -m , --email= Specify an email address to which notification messages are sent. If no email address is specified, email will not be sent. .. cmdoption:: -e, --eager Enable "eager" monitoring: check the URL and emit mail even if no monitored child process is in the ``RUNNING`` state. Enabled by default. .. cmdoption:: -E, --not-eager Disable "eager" monitoring: do not check the URL or emit mail if no monitored process is in the RUNNING state. .. cmdoption:: URL The URL to which to issue a GET request. .. cmdoption:: -n , --name= An optional name that identifies this httpok process. If given, the email subject will start with ``httpok []:`` instead of ``httpok:`` In case you run multiple supervisors on a single host that control different processes with the same name (eg `zopeinstance1`) you can use this option to indicate which project the restarted instance belongs to. Configuring :command:`httpok` Into the Supervisor Config ----------------------------------------------------------- An ``[eventlistener:x]`` section must be placed in :file:`supervisord.conf` in order for :command:`httpok` to do its work. See the "Events" chapter in the Supervisor manual for more information about event listeners. The following example assumes that :command:`httpok` is on your system :envvar:`PATH`. .. code-block:: ini [eventlistener:httpok] command=httpok -p program1 -p group1:program2 http://localhost:8080/tasty events=TICK_60 ================================================ FILE: docs/index.rst ================================================ superlance plugins for supervisor ================================= Superlance is a package of plugin utilities for monitoring and controlling processes that run under `Supervisor `_. It provides these plugins: :command:`httpok` This plugin is meant to be used as a supervisor event listener, subscribed to ``TICK_*`` events. It tests that a given child process which must in the ``RUNNING`` state, is viable via an HTTP ``GET`` request to a configured URL. If the request fails or times out, :command:`httpok` will restart the "hung" child process. :command:`crashmail` This plugin is meant to be used as a supervisor event listener, subscribed to ``PROCESS_STATE_EXITED`` events. It email a user when a process enters the ``EXITED`` state unexpectedly. :command:`memmon` This plugin is meant to be used as a supervisor event listener, subscribed to ``TICK_*`` events. It monitors memory usage for configured child processes, and restarts them when they exceed a configured maximum size. :command:`crashmailbatch` Similar to :command:`crashmail`, :command:`crashmailbatch` sends email alerts when processes die unexpectedly. The difference is that all alerts generated within the configured time interval are batched together to avoid sending too many emails. :command:`fatalmailbatch` This plugin sends email alerts when processes fail to start too many times such that supervisord gives up retrying. All of the fatal start events generated within the configured time interval are batched together to avoid sending too many emails. :command:`crashsms` Similar to :command:`crashmailbatch` except it sends SMS alerts through an email gateway. Messages are formatted to fit in SMS. Contents: .. toctree:: :maxdepth: 2 httpok crashmail memmon crashmailbatch fatalmailbatch crashsms development Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` ================================================ FILE: docs/memmon.rst ================================================ :command:`memmon` Overview ========================== :command:`memmon` is a supervisor "event listener" which may be subscribed to a concrete ``TICK_x`` event. When :command:`memmon` receives a ``TICK_x`` event (``TICK_60`` is recommended, indicating activity every 60 seconds), :command:`memmon` checks that a configurable list of programs (or all programs running under supervisor) are not exceeding a configurable amount of memory (resident segment size, or RSS). If one or more of these processes is consuming more than the amount of memory that :command:`memmon` believes it should, :command:`memmon` will restart the process. :command:`memmon` can be configured to send an email notification when it restarts a process. :command:`memmon` is known to work on Linux and Mac OS X, but has not been tested on other operating systems (it relies on :command:`ps` output and command-line switches). :command:`memmon` is incapable of monitoring the process status of processes which are not :command:`supervisord` child processes. Without the `--cumulative` option, only the RSS of immediate children of the :command:`supervisord` process will be considered. :command:`memmon` is a "console script" installed when you install :mod:`superlance`. Although :command:`memmon` is an executable program, it isn't useful as a general-purpose script: it must be run as a :command:`supervisor` event listener to do anything useful. :command:`memmon` uses Supervisor's XML-RPC interface. Your ``supervisord.conf`` file must have a valid `[unix_http_server] `_ or `[inet_http_server] `_ section, and must have an `[rpcinterface:supervisor] `_ section. If you are able to control your ``supervisord`` instance with ``supervisorctl``, you have already met these requirements. Command-Line Syntax ------------------- .. code-block:: sh $ memmon [-c] [-p processname=byte_size] [-g groupname=byte_size] \ [-a byte_size] [-s sendmail] [-m email_address] \ [-u email_uptime_limit] [-n memmon_name] .. program:: memmon .. cmdoption:: -h, --help Show program help. .. cmdoption:: -c, --cumulative Check against cumulative RSS. When calculating a process' RSS, also consider its child processes. With this option `memmon` will sum up the RSS of the process to be monitored and all its children. .. cmdoption:: -p , --program= A name/size pair, e.g. "foo=1MB". The name represents the supervisor program name that you would like :command:`memmon` to monitor; the size represents the number of bytes (suffix-multiplied using "KB", "MB" or "GB") that should be considered "too much". This option can be provided more than once to have :command:`memmon` monitor more than one program. Programs can be specified using a "namespec", to disambiguate same-named programs in different groups, e.g. ``foo:bar`` represents the program ``bar`` in the ``foo`` group. .. cmdoption:: -g , --groupname= A groupname/size pair, e.g. "group=1MB". The name represents the supervisor group name that you would like :command:`memmon` to monitor; the size represents the number of bytes (suffix-multiplied using "KB", "MB" or "GB") that should be considered "too much". Multiple ``-g`` options can be provided to have :command:`memmon` monitor more than one group. If any process in this group exceeds the maximum, it will be restarted. .. cmdoption:: -a , --any= A size (suffix-multiplied using "KB", "MB" or "GB") that should be considered "too much". If any program running as a child of supervisor exceeds this maximum, it will be restarted. E.g. 100MB. .. cmdoption:: -s , --sendmail= A command that will send mail if passed the email body (including the headers). Defaults to ``/usr/sbin/sendmail -t -i``. .. note:: Specifying this option doesn't cause memmon to send mail by itself: see the ``-m`` / ``--email`` option. .. cmdoption:: -m , --email= An email address to which to send email when a process is restarted. By default, memmon will not send any mail unless an email address is specified. .. cmdoption:: -u , --uptime= Only send an email in case the restarted process' uptime (in seconds) is below this limit. (Useful to only get notified if a processes gets restarted too frequently) Uptime is given in seconds (suffix-multiplied using "m" for minutes, "h" for hours or "d" for days) .. cmdoption:: -n , --name= An optional name that identifies this memmon process. If given, the email subject will start with ``memmon []:`` instead of ``memmon:`` In case you run multiple supervisors on a single host that control different processes with the same name (eg `zopeinstance1`) you can use this option to indicate which project the restarted instance belongs to. Configuring :command:`memmon` Into the Supervisor Config -------------------------------------------------------- An ``[eventlistener:x]`` section must be placed in :file:`supervisord.conf` in order for :command:`memmon` to do its work. See the "Events" chapter in the Supervisor manual for more information about event listeners. If the `[unix_http_server] `_ or `[inet_http_server] `_ has been configured to use authentication, add the environment variables ``SUPERVISOR_USERNAME`` and ``SUPERVISOR_PASSWORD`` in the ``[eventlistener:x]`` section as shown in Example Configuration 5. The following examples assume that :command:`memmon` is on your system :envvar:`PATH`. Example Configuration 1 ####################### This configuration causes :command:`memmon` to restart any process which is a child of :command:`supervisord` consuming more than 200MB of RSS, and will send mail to ``bob@example.com`` when it restarts a process using the default :command:`sendmail` command. .. code-block:: ini [eventlistener:memmon] command=memmon -a 200MB -m bob@example.com events=TICK_60 Example Configuration 2 ####################### This configuration causes :command:`memmon` to restart any process with the supervisor program name ``foo`` consuming more than 200MB of RSS, and will send mail to ``bob@example.com`` when it restarts a process using the default sendmail command. .. code-block:: ini [eventlistener:memmon] command=memmon -p foo=200MB -m bob@example.com events=TICK_60 Example Configuration 3 ####################### This configuration causes :command:`memmon` to restart any process in the process group "bar" consuming more than 200MB of RSS, and will send mail to ``bob@example.com`` when it restarts a process using the default :command:`sendmail` command. .. code-block:: ini [eventlistener:memmon] command=memmon -g bar=200MB -m bob@example.com events=TICK_60 Example Configuration 4 ####################### This configuration causes :command:`memmon` to restart any process meeting the same requirements as in `Example Configuration 2`_ with one difference: The email will only be sent if the process' uptime is less or equal than 2 days (172800 seconds) .. code-block:: ini [eventlistener:memmon] command=memmon -p foo=200MB -m bob@example.com -u 2d events=TICK_60 Example Configuration 5 (Authentication) ######################################## This configuration is the same as the one in `Example Configuration 1`_ with the only difference being that the `[unix_http_server] `_ or `[inet_http_server] `_ has been configured to use authentication. .. code-block:: ini [eventlistener:memmon] command=memmon -a 200MB -m bob@example.com environment=SUPERVISOR_USERNAME="",SUPERVISOR_PASSWORD="" events=TICK_60 ================================================ FILE: setup.cfg ================================================ ;Marking a wheel as universal with "universal = 1" was deprecated ;in Setuptools 75.1.0. Setting "python_tag = py2.py3" should do ;the equivalent on Setuptools 30.3.0 or later. ; ;https://github.com/pypa/setuptools/pull/4617 ;https://github.com/pypa/setuptools/pull/4939 ; [bdist_wheel] python_tag = py2.py3 ================================================ FILE: setup.py ================================================ ############################################################################## # # Copyright (c) 2008-2013 Agendaless Consulting and Contributors. # All Rights Reserved. # # This software is subject to the provisions of the BSD-like license at # http://www.repoze.org/LICENSE.txt. A copy of the license should accompany # this distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL # EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, # THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND # FITNESS FOR A PARTICULAR PURPOSE # ############################################################################## import os import sys py_version = sys.version_info[:2] if py_version < (2, 7): raise RuntimeError('On Python 2, superlance requires Python 2.7 or later') elif (3, 0) < py_version < (3, 4): raise RuntimeError('On Python 3, superlance requires Python 3.4 or later') from setuptools import setup, find_packages here = os.path.abspath(os.path.dirname(__file__)) try: README = open(os.path.join(here, 'README.rst')).read() except (IOError, OSError): README = '' try: CHANGES = open(os.path.join(here, 'CHANGES.rst')).read() except (IOError, OSError): CHANGES = '' setup(name='superlance', version='2.0.1.dev0', license='BSD-derived (http://www.repoze.org/LICENSE.txt)', description='superlance plugins for supervisord', long_description=README + '\n\n' + CHANGES, classifiers=[ "Development Status :: 5 - Production/Stable", 'Environment :: No Input/Output (Daemon)', 'Intended Audience :: System Administrators', 'Natural Language :: English', 'Operating System :: POSIX', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: 3.13', 'Programming Language :: Python :: 3.14', 'Topic :: System :: Boot', 'Topic :: System :: Monitoring', 'Topic :: System :: Systems Administration', ], author='Chris McDonough', author_email='chrism@plope.com', url='https://github.com/Supervisor/superlance', keywords = 'supervisor monitoring', packages = find_packages(), include_package_data=True, zip_safe=False, install_requires=['supervisor',], extras_require={'test': ['pytest'],}, entry_points = """\ [console_scripts] httpok = superlance.httpok:main crashsms = superlance.crashsms:main crashmail = superlance.crashmail:main crashmailbatch = superlance.crashmailbatch:main fatalmailbatch = superlance.fatalmailbatch:main memmon = superlance.memmon:main """ ) ================================================ FILE: superlance/__init__.py ================================================ # superlance package ================================================ FILE: superlance/compat.py ================================================ try: import http.client as httplib except ImportError: import httplib try: from StringIO import StringIO except ImportError: from io import StringIO try: from sys import maxsize as maxint except ImportError: from sys import maxint try: import urllib.parse as urlparse import urllib.parse as urllib except ImportError: import urlparse import urllib try: import xmlrpc.client as xmlrpclib except ImportError: import xmlrpclib ================================================ FILE: superlance/crashmail.py ================================================ #!/usr/bin/env python -u ############################################################################## # # Copyright (c) 2007 Agendaless Consulting and Contributors. # All Rights Reserved. # # This software is subject to the provisions of the BSD-like license at # http://www.repoze.org/LICENSE.txt. A copy of the license should accompany # this distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL # EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, # THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND # FITNESS FOR A PARTICULAR PURPOSE # ############################################################################## # A event listener meant to be subscribed to PROCESS_STATE_CHANGE # events. It will send mail when processes that are children of # supervisord transition unexpectedly to the EXITED state. # A supervisor config snippet that tells supervisor to use this script # as a listener is below. # # [eventlistener:crashmail] # command = # /usr/bin/crashmail # -o hostname -a -m notify-on-crash@domain.com # -s '/usr/sbin/sendmail -t -i -f crash-notifier@domain.com' # events=PROCESS_STATE # # Sendmail is used explicitly here so that we can specify the 'from' address. doc = """\ crashmail.py [-p processname] [-a] [-o string] [-m mail_address] [-s sendmail] URL Options: -p -- specify a supervisor process_name. Send mail when this process transitions to the EXITED state unexpectedly. If this process is part of a group, it can be specified using the 'group_name:process_name' syntax. -a -- Send mail when any child of the supervisord transitions unexpectedly to the EXITED state unexpectedly. Overrides any -p parameters passed in the same crashmail process invocation. -o -- Specify a parameter used as a prefix in the mail subject header. -s -- the sendmail command to use to send email (e.g. "/usr/sbin/sendmail -t -i"). Must be a command which accepts header and message data on stdin and sends mail. Default is "/usr/sbin/sendmail -t -i". -m -- specify an email address. The script will send mail to this address when crashmail detects a process crash. If no email address is specified, email will not be sent. The -p option may be specified more than once, allowing for specification of multiple processes. Specifying -a overrides any selection of -p. A sample invocation: crashmail.py -p program1 -p group1:program2 -m dev@example.com """ import getopt import os import sys from supervisor import childutils def usage(exitstatus=255): print(doc) sys.exit(exitstatus) class CrashMail: def __init__(self, programs, any, email, sendmail, optionalheader): self.programs = programs self.any = any self.email = email self.sendmail = sendmail self.optionalheader = optionalheader self.stdin = sys.stdin self.stdout = sys.stdout self.stderr = sys.stderr def runforever(self, test=False): while 1: # we explicitly use self.stdin, self.stdout, and self.stderr # instead of sys.* so we can unit test this code headers, payload = childutils.listener.wait( self.stdin, self.stdout) if not headers['eventname'] == 'PROCESS_STATE_EXITED': # do nothing with non-TICK events childutils.listener.ok(self.stdout) if test: self.stderr.write('non-exited event\n') self.stderr.flush() break continue pheaders, pdata = childutils.eventdata(payload+'\n') if int(pheaders['expected']): childutils.listener.ok(self.stdout) if test: self.stderr.write('expected exit\n') self.stderr.flush() break continue msg = ('Process %(processname)s in group %(groupname)s exited ' 'unexpectedly (pid %(pid)s) from state %(from_state)s' % pheaders) subject = ' %s crashed at %s' % (pheaders['processname'], childutils.get_asctime()) if self.optionalheader: subject = self.optionalheader + ':' + subject self.stderr.write('unexpected exit, mailing\n') self.stderr.flush() self.mail(self.email, subject, msg) childutils.listener.ok(self.stdout) if test: break def mail(self, email, subject, msg): body = 'To: %s\n' % self.email body += 'Subject: %s\n' % subject body += '\n' body += msg with os.popen(self.sendmail, 'w') as m: m.write(body) self.stderr.write('Mailed:\n\n%s' % body) self.mailed = body def main(argv=sys.argv): short_args = "hp:ao:s:m:" long_args = [ "help", "program=", "any", "optionalheader=", "sendmail_program=", "email=", ] arguments = argv[1:] try: opts, args = getopt.getopt(arguments, short_args, long_args) except: usage() programs = [] any = False sendmail = '/usr/sbin/sendmail -t -i' email = None optionalheader = None for option, value in opts: if option in ('-h', '--help'): usage(exitstatus=0) if option in ('-p', '--program'): programs.append(value) if option in ('-a', '--any'): any = True if option in ('-s', '--sendmail_program'): sendmail = value if option in ('-m', '--email'): email = value if option in ('-o', '--optionalheader'): optionalheader = value if not 'SUPERVISOR_SERVER_URL' in os.environ: sys.stderr.write('crashmail must be run as a supervisor event ' 'listener\n') sys.stderr.flush() return prog = CrashMail(programs, any, email, sendmail, optionalheader) prog.runforever() if __name__ == '__main__': main() ================================================ FILE: superlance/crashmailbatch.py ================================================ #!/usr/bin/env python -u ############################################################################## # # Copyright (c) 2007 Agendaless Consulting and Contributors. # All Rights Reserved. # # This software is subject to the provisions of the BSD-like license at # http://www.repoze.org/LICENSE.txt. A copy of the license should accompany # this distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL # EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, # THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND # FITNESS FOR A PARTICULAR PURPOSE # ############################################################################## # A event listener meant to be subscribed to PROCESS_STATE_CHANGE # events. It will send mail when processes that are children of # supervisord transition unexpectedly to the EXITED state. # A supervisor config snippet that tells supervisor to use this script # as a listener is below. # # [eventlistener:crashmailbatch] # command=python crashmailbatch --toEmail=you@bar.com --fromEmail=me@bar.com # events=PROCESS_STATE,TICK_60 doc = """\ crashmailbatch.py [--interval=] [--toEmail=] [--fromEmail=] [--subject=] [--smtpHost=] Options: --interval - batch cycle length (in minutes). The default is 1.0 minute. This means that all events in each cycle are batched together and sent as a single email --toEmail - the email address(es) to send alerts to - comma separated --fromEmail - the email address to send alerts from --subject - the email subject line --smtpHost - the SMTP server's hostname or address (defaults to 'localhost') A sample invocation: crashmailbatch.py --toEmail="you@bar.com" --fromEmail="me@bar.com" """ from supervisor import childutils from superlance.process_state_email_monitor import ProcessStateEmailMonitor class CrashMailBatch(ProcessStateEmailMonitor): process_state_events = ['PROCESS_STATE_EXITED'] def __init__(self, **kwargs): if kwargs.get('subject') is None: kwargs['subject'] = 'Crash alert from supervisord' ProcessStateEmailMonitor.__init__(self, **kwargs) self.now = kwargs.get('now', None) def get_process_state_change_msg(self, headers, payload): pheaders, pdata = childutils.eventdata(payload+'\n') if int(pheaders['expected']): return None txt = 'Process %(groupname)s:%(processname)s (pid %(pid)s) died \ unexpectedly' % pheaders return '%s -- %s' % (childutils.get_asctime(self.now), txt) def main(): crash = CrashMailBatch.create_from_cmd_line() crash.run() if __name__ == '__main__': main() ================================================ FILE: superlance/crashsms.py ================================================ #!/usr/bin/env python -u ############################################################################## # # Copyright (c) 2007 Agendaless Consulting and Contributors. # All Rights Reserved. # # This software is subject to the provisions of the BSD-like license at # http://www.repoze.org/LICENSE.txt. A copy of the license should accompany # this distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL # EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, # THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND # FITNESS FOR A PARTICULAR PURPOSE # ############################################################################## ############################################################################## # crashsms # author: Juan Batiz-Benet (http://github.com/jbenet) # based on crashmailbatch.py ############################################################################## # A event listener meant to be subscribed to PROCESS_STATE_CHANGE # events. It will send mail when processes that are children of # supervisord transition unexpectedly to the EXITED state. # A supervisor config snippet that tells supervisor to use this script # as a listener is below. # # [eventlistener:crashsms] # command = # python crashsms # -t @ -f me@bar.com -e TICK_5 # events=PROCESS_STATE,TICK_5 doc = """\ crashsms.py [--interval=] [--toEmail=] [--fromEmail=] [--subject=] Options: -i,--interval - batch cycle length (in minutes). The default is 1 minute. This means that all events in each cycle are batched together and sent as a single email -t,--toEmail - the comma separated email addresses to send alerts to. Mobile providers tend to allow sms messages to be sent to their phone numbers via an email address (e.g.: 1234567890@txt.att.net) -f,--fromEmail - the email address to send alerts from -s,--subject - the email subject line -e, --tickEvent - specify which TICK event to use (e.g. TICK_5, TICK_60, TICK_3600) A sample invocation: crashsms.py -t @ -f me@bar.com -e TICK_5 """ from supervisor import childutils from superlance.process_state_email_monitor import ProcessStateEmailMonitor class CrashSMS(ProcessStateEmailMonitor): process_state_events = ['PROCESS_STATE_EXITED'] def __init__(self, **kwargs): ProcessStateEmailMonitor.__init__(self, **kwargs) self.now = kwargs.get('now', None) def get_process_state_change_msg(self, headers, payload): pheaders, pdata = childutils.eventdata(payload+'\n') if int(pheaders['expected']): return None txt = '[%(groupname)s:%(processname)s](%(pid)s) exited unexpectedly' \ % pheaders return '%s %s' % (txt, childutils.get_asctime(self.now)) def main(): crash = CrashSMS.create_from_cmd_line() crash.run() if __name__ == '__main__': main() ================================================ FILE: superlance/fatalmailbatch.py ================================================ #!/usr/bin/env python -u ############################################################################## # # Copyright (c) 2007 Agendaless Consulting and Contributors. # All Rights Reserved. # # This software is subject to the provisions of the BSD-like license at # http://www.repoze.org/LICENSE.txt. A copy of the license should accompany # this distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL # EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, # THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND # FITNESS FOR A PARTICULAR PURPOSE # ############################################################################## # A event listener meant to be subscribed to PROCESS_STATE_CHANGE # events. It will send mail when processes that are children of # supervisord transition unexpectedly to the EXITED state. # A supervisor config snippet that tells supervisor to use this script # as a listener is below. # # [eventlistener:fatalmailbatch] # command=python fatalmailbatch # events=PROCESS_STATE,TICK_60 doc = """\ fatalmailbatch.py [--interval=] [--toEmail=] [--fromEmail=] [--subject=] [--smtpHost=] [--userName=] [--password= 0: try: # build a loop value that is guaranteed to execute at least # once and at most until the timeout is reached and that # has 0 as the last value (to allow raising an exception # in the last iteration) for will_retry in range( (self.timeout - 1) // (self.retry_time or 1), -1, -1): try: headers = {'User-Agent': 'httpok'} conn.request('GET', path, headers=headers) break except socket.error as e: if e.errno == 111 and will_retry: time.sleep(self.retry_time) else: raise res = conn.getresponse() body = res.read() status = res.status msg = 'status contacting %s: %s %s' % (self.url, res.status, res.reason) except Exception as e: body = '' status = None msg = 'error contacting %s:\n\n %s' % (self.url, e) if status not in self.statuses: subject = self.format_subject( '%s: bad status returned' % self.url ) self.act(subject, msg) elif self.inbody and self.inbody not in body: subject = self.format_subject( '%s: bad body returned' % self.url ) self.act(subject, msg) childutils.listener.ok(self.stdout) if test: break def format_subject(self, subject): if self.name is None: return 'httpok: %s' % subject else: return 'httpok [%s]: %s' % (self.name, subject) def act(self, subject, msg): messages = [msg] def write(msg): self.stderr.write('%s\n' % msg) self.stderr.flush() messages.append(msg) try: specs = self.rpc.supervisor.getAllProcessInfo() except Exception as e: write('Exception retrieving process info %s, not acting' % e) return waiting = list(self.programs) if self.any: write('Restarting all running processes') for spec in specs: name = spec['name'] group = spec['group'] self.restart(spec, write) namespec = make_namespec(group, name) if name in waiting: waiting.remove(name) if namespec in waiting: waiting.remove(namespec) else: write('Restarting selected processes %s' % self.programs) for spec in specs: name = spec['name'] group = spec['group'] namespec = make_namespec(group, name) if (name in self.programs) or (namespec in self.programs): self.restart(spec, write) if name in waiting: waiting.remove(name) if namespec in waiting: waiting.remove(namespec) if waiting: write( 'Programs not restarted because they did not exist: %s' % waiting) if self.email: message = '\n'.join(messages) self.mail(self.email, subject, message) def mail(self, email, subject, msg): body = 'To: %s\n' % self.email body += 'Subject: %s\n' % subject body += '\n' body += msg with os.popen(self.sendmail, 'w') as m: m.write(body) self.stderr.write('Mailed:\n\n%s' % body) self.mailed = body def restart(self, spec, write): namespec = make_namespec(spec['group'], spec['name']) if spec['state'] is ProcessStates.RUNNING: if self.coredir and self.gcore: corename = os.path.join(self.coredir, namespec) cmd = self.gcore + ' "%s" %s' % (corename, spec['pid']) with os.popen(cmd) as m: write('gcore output for %s:\n\n %s' % ( namespec, m.read())) write('%s is in RUNNING state, restarting' % namespec) try: self.rpc.supervisor.stopProcess(namespec) except xmlrpclib.Fault as e: write('Failed to stop process %s: %s' % ( namespec, e)) try: self.rpc.supervisor.startProcess(namespec) except xmlrpclib.Fault as e: write('Failed to start process %s: %s' % ( namespec, e)) else: write('%s restarted' % namespec) else: write('%s not in RUNNING state, NOT restarting' % namespec) def main(argv=sys.argv): short_args="hp:at:c:b:s:m:g:d:eEn:" long_args=[ "help", "program=", "any", "timeout=", "code=", "body=", "sendmail_program=", "email=", "gcore=", "coredir=", "eager", "not-eager", "name=", ] arguments = argv[1:] try: opts, args = getopt.getopt(arguments, short_args, long_args) except: usage() # check for -h must be done before positional args check for option, value in opts: if option in ('-h', '--help'): usage(exitstatus=0) if not args: usage() if len(args) > 1: usage() programs = [] any = False sendmail = '/usr/sbin/sendmail -t -i' gcore = '/usr/bin/gcore -o' coredir = None eager = True email = None timeout = 10 retry_time = 10 statuses = [] inbody = None name = None for option, value in opts: if option in ('-p', '--program'): programs.append(value) if option in ('-a', '--any'): any = True if option in ('-s', '--sendmail_program'): sendmail = value if option in ('-m', '--email'): email = value if option in ('-t', '--timeout'): timeout = int(value) if option in ('-c', '--code'): statuses.append(int(value)) if option in ('-b', '--body'): inbody = value if option in ('-g', '--gcore'): gcore = value if option in ('-d', '--coredir'): coredir = value if option in ('-e', '--eager'): eager = True if option in ('-E', '--not-eager'): eager = False if option in ('-n', '--name'): name = value if not statuses: statuses = [200] url = arguments[-1] try: rpc = childutils.getRPCInterface(os.environ) except KeyError as e: if e.args[0] != 'SUPERVISOR_SERVER_URL': raise sys.stderr.write('httpok must be run as a supervisor event ' 'listener\n') sys.stderr.flush() return prog = HTTPOk(rpc, programs, any, url, timeout, statuses, inbody, email, sendmail, coredir, gcore, eager, retry_time, name) prog.runforever() if __name__ == '__main__': main() ================================================ FILE: superlance/memmon.py ================================================ #!/usr/bin/env python ############################################################################## # # Copyright (c) 2007 Agendaless Consulting and Contributors. # All Rights Reserved. # # This software is subject to the provisions of the BSD-like license at # http://www.repoze.org/LICENSE.txt. A copy of the license should accompany # this distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL # EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, # THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND # FITNESS FOR A PARTICULAR PURPOSE # ############################################################################## # A event listener meant to be subscribed to TICK_60 (or TICK_5) # events, which restarts any processes that are children of # supervisord that consume "too much" memory. Performs horrendous # screenscrapes of ps output. Works on Linux and OS X (Tiger/Leopard) # as far as I know. # A supervisor config snippet that tells supervisor to use this script # as a listener is below. # # [eventlistener:memmon] # command=python memmon.py [options] # events=TICK_60 doc = """\ memmon.py [-c] [-p processname=byte_size] [-g groupname=byte_size] [-a byte_size] [-s sendmail] [-m email_address] [-u uptime] [-n memmon_name] Options: -c -- Check against cumulative RSS. When calculating a process' RSS, also consider its child processes. With this option `memmon` will sum up the RSS of the process to be monitored and all its children. -p -- specify a process_name=byte_size pair. Restart the supervisor process named 'process_name' when it uses more than byte_size RSS. If this process is in a group, it can be specified using the 'group_name:process_name' syntax. -g -- specify a group_name=byte_size pair. Restart any process in this group when it uses more than byte_size RSS. -a -- specify a global byte_size. Restart any child of the supervisord under which this runs if it uses more than byte_size RSS. -s -- the sendmail command to use to send email (e.g. "/usr/sbin/sendmail -t -i"). Must be a command which accepts header and message data on stdin and sends mail. Default is "/usr/sbin/sendmail -t -i". -m -- specify an email address. The script will send mail to this address when any process is restarted. If no email address is specified, email will not be sent. -u -- optionally specify the minimum uptime in seconds for the process. if the process uptime is longer than this value, no email is sent (useful to only be notified if processes are restarted too often/early) seconds can be specified as plain integer values or a suffix-multiplied integer (e.g. 1m). Valid suffixes are m (minute), h (hour) and d (day). -n -- optionally specify the name of the memmon process. This name will be used in the email subject to identify which memmon process restarted the process. The -p and -g options may be specified more than once, allowing for specification of multiple groups and processes. Any byte_size can be specified as a plain integer (10000) or a suffix-multiplied integer (e.g. 1GB). Valid suffixes are 'KB', 'MB' and 'GB'. A sample invocation: memmon.py -p program1=200MB -p theprog:thegroup=100MB -g thegroup=100MB -a 1GB -s "/usr/sbin/sendmail -t -i" -m chrism@plope.com -n "Project 1" """ import getopt import os import sys import time from collections import namedtuple from superlance.compat import maxint from superlance.compat import xmlrpclib from supervisor import childutils from supervisor.datatypes import byte_size, SuffixMultiplier def usage(exitstatus=255): print(doc) sys.exit(exitstatus) def shell(cmd): with os.popen(cmd) as f: return f.read() class Memmon: def __init__(self, cumulative, programs, groups, any, sendmail, email, email_uptime_limit, name, rpc=None): self.cumulative = cumulative self.programs = programs self.groups = groups self.any = any self.sendmail = sendmail self.email = email self.email_uptime_limit = email_uptime_limit self.name = name self.rpc = rpc self.stdin = sys.stdin self.stdout = sys.stdout self.stderr = sys.stderr self.pscommand = 'ps -orss= -p %s' self.pstreecommand = 'ps ax -o "pid= ppid= rss="' self.mailed = False # for unit tests def runforever(self, test=False): while 1: # we explicitly use self.stdin, self.stdout, and self.stderr # instead of sys.* so we can unit test this code headers, payload = childutils.listener.wait(self.stdin, self.stdout) if not headers['eventname'].startswith('TICK'): # do nothing with non-TICK events childutils.listener.ok(self.stdout) if test: break continue status = [] if self.programs: keys = sorted(self.programs.keys()) status.append( 'Checking programs %s' % ', '.join( [ '%s=%s' % (k, self.programs[k]) for k in keys ]) ) if self.groups: keys = sorted(self.groups.keys()) status.append( 'Checking groups %s' % ', '.join( [ '%s=%s' % (k, self.groups[k]) for k in keys ]) ) if self.any is not None: status.append('Checking any=%s' % self.any) self.stderr.write('\n'.join(status) + '\n') infos = self.rpc.supervisor.getAllProcessInfo() for info in infos: pid = info['pid'] name = info['name'] group = info['group'] pname = '%s:%s' % (group, name) if not pid: # ps throws an error in this case (for processes # in standby mode, non-auto-started). continue rss = self.calc_rss(pid) if rss is None: # no such pid (deal with race conditions) or # rss couldn't be calculated for other reasons continue for n in name, pname: if n in self.programs: self.stderr.write('RSS of %s is %s\n' % (pname, rss)) if rss > self.programs[name]: self.restart(pname, rss) continue if group in self.groups: self.stderr.write('RSS of %s is %s\n' % (pname, rss)) if rss > self.groups[group]: self.restart(pname, rss) continue if self.any is not None: self.stderr.write('RSS of %s is %s\n' % (pname, rss)) if rss > self.any: self.restart(pname, rss) continue self.stderr.flush() childutils.listener.ok(self.stdout) if test: break def restart(self, name, rss): info = self.rpc.supervisor.getProcessInfo(name) uptime = info['now'] - info['start'] #uptime in seconds self.stderr.write('Restarting %s\n' % name) try: self.rpc.supervisor.stopProcess(name) except xmlrpclib.Fault as e: msg = ('Failed to stop process %s (RSS %s), exiting: %s' % (name, rss, e)) self.stderr.write(str(msg)) if self.email: subject = self.format_subject( 'failed to stop process %s, exiting' % name ) self.mail(self.email, subject, msg) raise try: self.rpc.supervisor.startProcess(name) except xmlrpclib.Fault as e: msg = ('Failed to start process %s after stopping it, ' 'exiting: %s' % (name, e)) self.stderr.write(str(msg)) if self.email: subject = self.format_subject( 'failed to start process %s, exiting' % name ) self.mail(self.email, subject, msg) raise if self.email and uptime <= self.email_uptime_limit: now = time.asctime() timezone = time.strftime('%Z') msg = ( 'memmon.py restarted the process named %s at %s %s because ' 'it was consuming too much memory (%s bytes RSS)' % ( name, now, timezone, rss) ) subject = self.format_subject( 'process %s restarted' % name ) self.mail(self.email, subject, msg) def format_subject(self, subject): if self.name is None: return 'memmon: %s' % subject else: return 'memmon [%s]: %s' % (self.name, subject) def calc_rss(self, pid): ProcInfo = namedtuple('ProcInfo', ['pid', 'ppid', 'rss']) def find_children(parent_pid, procs): children = [] for proc in procs: pid, ppid, rss = proc if ppid == parent_pid: children.append(proc) children.extend(find_children(pid, procs)) return children def cum_rss(pid, procs): parent_proc = [p for p in procs if p.pid == pid][0] children = find_children(pid, procs) tree = [parent_proc] + children total_rss = sum(map(int, [p.rss for p in tree])) return total_rss def get_all_process_infos(data): data = data.strip() procs = [] for line in data.splitlines(): pid, ppid, rss = map(int, line.split()) procs.append(ProcInfo(pid=pid, ppid=ppid, rss=rss)) return procs if self.cumulative: data = shell(self.pstreecommand) procs = get_all_process_infos(data) try: rss = cum_rss(pid, procs) except (ValueError, IndexError): # Could not determine cumulative RSS return None else: data = shell(self.pscommand % pid) if not data: # no such pid (deal with race conditions) return None try: rss = data.lstrip().rstrip() rss = int(rss) except ValueError: # line doesn't contain any data, or rss cant be intified return None rss = rss * 1024 # rss is in KB return rss def mail(self, email, subject, msg): body = 'To: %s\n' % self.email body += 'Subject: %s\n' % subject body += '\n' body += msg with os.popen(self.sendmail, 'w') as m: m.write(body) self.mailed = body def parse_namesize(option, value): try: name, size = value.split('=') except ValueError: print('Unparseable value %r for %r' % (value, option)) usage() size = parse_size(option, size) return name, size def parse_size(option, value): try: size = byte_size(value) except: print('Unparseable byte_size in %r for %r' % (value, option)) usage() return size seconds_size = SuffixMultiplier({'s': 1, 'm': 60, 'h': 60 * 60, 'd': 60 * 60 * 24 }) def parse_seconds(option, value): try: seconds = seconds_size(value) except: print('Unparseable value for time in %r for %s' % (value, option)) usage() return seconds help_request = object() # returned from memmon_from_args to indicate --help def memmon_from_args(arguments): short_args = "hcp:g:a:s:m:n:u:" long_args = [ "help", "cumulative", "program=", "group=", "any=", "sendmail_program=", "email=", "uptime=", "name=", ] if not arguments: return None try: opts, args = getopt.getopt(arguments, short_args, long_args) except: return None cumulative = False programs = {} groups = {} any = None sendmail = '/usr/sbin/sendmail -t -i' email = None uptime_limit = maxint name = None for option, value in opts: if option in ('-h', '--help'): return help_request if option in ('-c', '--cumulative'): cumulative = True if option in ('-p', '--program'): name, size = parse_namesize(option, value) programs[name] = size if option in ('-g', '--group'): name, size = parse_namesize(option, value) groups[name] = size if option in ('-a', '--any'): size = parse_size(option, value) any = size if option in ('-s', '--sendmail_program'): sendmail = value if option in ('-m', '--email'): email = value if option in ('-u', '--uptime'): uptime_limit = parse_seconds(option, value) if option in ('-n', '--name'): name = value memmon = Memmon(cumulative=cumulative, programs=programs, groups=groups, any=any, sendmail=sendmail, email=email, email_uptime_limit=uptime_limit, name=name) return memmon def main(): memmon = memmon_from_args(sys.argv[1:]) if memmon is help_request: # --help usage(exitstatus=0) elif memmon is None: # something went wrong usage() memmon.rpc = childutils.getRPCInterface(os.environ) memmon.runforever() if __name__ == '__main__': main() ================================================ FILE: superlance/process_state_email_monitor.py ================================================ #!/usr/bin/env python -u ############################################################################## # # Copyright (c) 2007 Agendaless Consulting and Contributors. # All Rights Reserved. # # This software is subject to the provisions of the BSD-like license at # http://www.repoze.org/LICENSE.txt. A copy of the license should accompany # this distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL # EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, # THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND # FITNESS FOR A PARTICULAR PURPOSE # ############################################################################## import copy import optparse import os import smtplib import sys from email.mime.text import MIMEText from email.utils import formatdate, make_msgid from superlance.process_state_monitor import ProcessStateMonitor doc = """\ Base class for common functionality when monitoring process state changes and sending email notification """ class ProcessStateEmailMonitor(ProcessStateMonitor): COMMASPACE = ', ' @classmethod def _get_opt_parser(cls): parser = optparse.OptionParser() parser.add_option("-i", "--interval", dest="interval", type="float", default=1.0, help="batch interval in minutes (defaults to 1 minute)") parser.add_option("-t", "--toEmail", dest="to_emails", help="destination email address(es) - comma separated") parser.add_option("-f", "--fromEmail", dest="from_email", help="source email address") parser.add_option("-s", "--subject", dest="subject", help="email subject") parser.add_option("-H", "--smtpHost", dest="smtp_host", default="localhost", help="SMTP server hostname or address") parser.add_option("-e", "--tickEvent", dest="eventname", default="TICK_60", help="TICK event name (defaults to TICK_60)") parser.add_option("-u", "--userName", dest="smtp_user", default="", help="SMTP server user name (defaults to nothing)") parser.add_option("-p", "--password", dest="smtp_password", default="", help="SMTP server password (defaults to nothing)") parser.add_option("--tls", dest="use_tls", action="store_true", default=False, help="Use Transport Layer Security (TLS), default to False") return parser @classmethod def parse_cmd_line_options(cls): parser = cls._get_opt_parser() (options, args) = parser.parse_args() return options @classmethod def validate_cmd_line_options(cls, options): parser = cls._get_opt_parser() if not options.to_emails: parser.print_help() sys.exit(1) if not options.from_email: parser.print_help() sys.exit(1) validated = copy.copy(options) validated.to_emails = [x.strip() for x in options.to_emails.split(",")] return validated @classmethod def get_cmd_line_options(cls): return cls.validate_cmd_line_options(cls.parse_cmd_line_options()) @classmethod def create_from_cmd_line(cls): options = cls.get_cmd_line_options() if not 'SUPERVISOR_SERVER_URL' in os.environ: sys.stderr.write('Must run as a supervisor event listener\n') sys.exit(1) return cls(**options.__dict__) def __init__(self, **kwargs): ProcessStateMonitor.__init__(self, **kwargs) self.from_email = kwargs['from_email'] self.to_emails = kwargs['to_emails'] self.subject = kwargs.get('subject') self.smtp_host = kwargs.get('smtp_host', 'localhost') self.smtp_user = kwargs.get('smtp_user') self.smtp_password = kwargs.get('smtp_password') self.use_tls = kwargs.get('use_tls') self.digest_len = 76 def send_batch_notification(self): email = self.get_batch_email() if email: self.send_email(email) self.log_email(email) def log_email(self, email): email_for_log = copy.copy(email) email_for_log['to'] = self.COMMASPACE.join(email['to']) if len(email_for_log['body']) > self.digest_len: email_for_log['body'] = '%s...' % email_for_log['body'][:self.digest_len] self.write_stderr("Sending notification email:\nTo: %(to)s\n\ From: %(from)s\nSubject: %(subject)s\nBody:\n%(body)s\n" % email_for_log) def get_batch_email(self): if len(self.batchmsgs): return { 'to': self.to_emails, 'from': self.from_email, 'subject': self.subject, 'body': '\n'.join(self.get_batch_msgs()), } return None def send_email(self, email): msg = MIMEText(email['body']) if self.subject: msg['Subject'] = email['subject'] msg['From'] = email['from'] msg['To'] = self.COMMASPACE.join(email['to']) msg['Date'] = formatdate() msg['Message-ID'] = make_msgid() try: self.send_smtp(msg, email['to']) except Exception as e: self.write_stderr("Error sending email: %s\n" % e) def send_smtp(self, mime_msg, to_emails): s = smtplib.SMTP(self.smtp_host) try: if self.smtp_user and self.smtp_password: if self.use_tls: s.starttls() s.login(self.smtp_user, self.smtp_password) s.sendmail(mime_msg['From'], to_emails, mime_msg.as_string()) except: s.quit() raise s.quit() ================================================ FILE: superlance/process_state_monitor.py ================================================ #!/usr/bin/env python -u ############################################################################## # # Copyright (c) 2007 Agendaless Consulting and Contributors. # All Rights Reserved. # # This software is subject to the provisions of the BSD-like license at # http://www.repoze.org/LICENSE.txt. A copy of the license should accompany # this distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL # EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, # THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND # FITNESS FOR A PARTICULAR PURPOSE # ############################################################################## doc = """\ Base class for common functionality when monitoring process state changes """ import sys from supervisor import childutils class ProcessStateMonitor: # In child class, define a list of events to monitor process_state_events = [] def __init__(self, **kwargs): self.interval = kwargs.get('interval', 1.0) self.debug = kwargs.get('debug', False) self.stdin = kwargs.get('stdin', sys.stdin) self.stdout = kwargs.get('stdout', sys.stdout) self.stderr = kwargs.get('stderr', sys.stderr) self.eventname = kwargs.get('eventname', 'TICK_60') self.tickmins = self._get_tick_mins(self.eventname) self.batchmsgs = [] self.batchmins = 0.0 def _get_tick_mins(self, eventname): return float(self._get_tick_secs(eventname))/60.0 def _get_tick_secs(self, eventname): self._validate_tick_name(eventname) return int(eventname.split('_')[1]) def _validate_tick_name(self, eventname): if not eventname.startswith('TICK_'): raise ValueError("Invalid TICK event name: %s" % eventname) def run(self): while 1: hdrs, payload = childutils.listener.wait(self.stdin, self.stdout) self.handle_event(hdrs, payload) childutils.listener.ok(self.stdout) def handle_event(self, headers, payload): if headers['eventname'] in self.process_state_events: self.handle_process_state_change_event(headers, payload) elif headers['eventname'] == self.eventname: self.handle_tick_event(headers, payload) def handle_process_state_change_event(self, headers, payload): msg = self.get_process_state_change_msg(headers, payload) if msg: self.write_stderr('%s\n' % msg) self.batchmsgs.append(msg) """ Override this method in child classes to customize messaging """ def get_process_state_change_msg(self, headers, payload): return None def handle_tick_event(self, headers, payload): self.batchmins += self.tickmins if self.batchmins >= self.interval: self.send_batch_notification() self.clear_batch() """ Override this method in child classes to send notification """ def send_batch_notification(self): pass def get_batch_minutes(self): return self.batchmins def get_batch_msgs(self): return self.batchmsgs def clear_batch(self): self.batchmins = 0.0 self.batchmsgs = [] def write_stderr(self, msg): self.stderr.write(msg) self.stderr.flush() ================================================ FILE: superlance/tests/__init__.py ================================================ ================================================ FILE: superlance/tests/dummy.py ================================================ import time from supervisor.process import ProcessStates _NOW = time.time() class DummyRPCServer: def __init__(self): self.supervisor = DummySupervisorRPCNamespace() self.system = DummySystemRPCNamespace() class DummyResponse: status = 200 reason = 'OK' body = 'OK' def read(self): return self.body class DummySystemRPCNamespace: pass class DummySupervisorRPCNamespace: _restartable = True _restarted = False _shutdown = False _readlog_error = False all_process_info = [ { 'name':'foo', 'group':'foo', 'pid':11, 'state':ProcessStates.RUNNING, 'statename':'RUNNING', 'start':_NOW - 100, 'stop':0, 'spawnerr':'', 'now':_NOW, 'description':'foo description', }, { 'name':'bar', 'group':'bar', 'pid':12, 'state':ProcessStates.FATAL, 'statename':'FATAL', 'start':_NOW - 100, 'stop':_NOW - 50, 'spawnerr':'screwed', 'now':_NOW, 'description':'bar description', }, { 'name':'baz_01', 'group':'baz', 'pid':12, 'state':ProcessStates.STOPPED, 'statename':'STOPPED', 'start':_NOW - 100, 'stop':_NOW - 25, 'spawnerr':'', 'now':_NOW, 'description':'baz description', }, ] def getAllProcessInfo(self): return self.all_process_info def getProcessInfo(self, name): for info in self.all_process_info: if info['name'] == name or name == '%s:%s' %(info['group'], info['name']): return info return None def startProcess(self, name): from supervisor import xmlrpc from superlance.compat import xmlrpclib if name.endswith('SPAWN_ERROR'): raise xmlrpclib.Fault(xmlrpc.Faults.SPAWN_ERROR, 'SPAWN_ERROR') return True def stopProcess(self, name): from supervisor import xmlrpc from superlance.compat import xmlrpclib if name == 'BAD_NAME:BAD_NAME': raise xmlrpclib.Fault(xmlrpc.Faults.BAD_NAME, 'BAD_NAME:BAD_NAME') if name.endswith('FAILED'): raise xmlrpclib.Fault(xmlrpc.Faults.FAILED, 'FAILED') return True ================================================ FILE: superlance/tests/test_crashmail.py ================================================ import unittest from superlance.compat import StringIO class CrashMailTests(unittest.TestCase): def _getTargetClass(self): from superlance.crashmail import CrashMail return CrashMail def _makeOne(self, *opts): return self._getTargetClass()(*opts) def setUp(self): import tempfile self.tempdir = tempfile.mkdtemp() def tearDown(self): import shutil shutil.rmtree(self.tempdir) def _makeOnePopulated(self, programs, any, response=None): import os sendmail = 'cat - > %s' % os.path.join(self.tempdir, 'email.log') email = 'chrism@plope.com' header = '[foo]' prog = self._makeOne(programs, any, email, sendmail, header) prog.stdin = StringIO() prog.stdout = StringIO() prog.stderr = StringIO() return prog def test_runforever_not_process_state_exited(self): programs = {'foo':0, 'bar':0, 'baz_01':0 } any = None prog = self._makeOnePopulated(programs, any) prog.stdin.write('eventname:PROCESS_STATE len:0\n') prog.stdin.seek(0) prog.runforever(test=True) self.assertEqual(prog.stderr.getvalue(), 'non-exited event\n') def test_runforever_expected_exit(self): programs = ['foo'] any = None prog = self._makeOnePopulated(programs, any) payload=('expected:1 processname:foo groupname:bar ' 'from_state:RUNNING pid:1') prog.stdin.write( 'eventname:PROCESS_STATE_EXITED len:%s\n' % len(payload)) prog.stdin.write(payload) prog.stdin.seek(0) prog.runforever(test=True) self.assertEqual(prog.stderr.getvalue(), 'expected exit\n') def test_runforever_unexpected_exit(self): programs = ['foo'] any = None prog = self._makeOnePopulated(programs, any) payload=('expected:0 processname:foo groupname:bar ' 'from_state:RUNNING pid:1') prog.stdin.write( 'eventname:PROCESS_STATE_EXITED len:%s\n' % len(payload)) prog.stdin.write(payload) prog.stdin.seek(0) prog.runforever(test=True) output = prog.stderr.getvalue() lines = output.split('\n') self.assertEqual(lines[0], 'unexpected exit, mailing') self.assertEqual(lines[1], 'Mailed:') self.assertEqual(lines[2], '') self.assertEqual(lines[3], 'To: chrism@plope.com') self.assertTrue('Subject: [foo]: foo crashed at' in lines[4]) self.assertEqual(lines[5], '') self.assertTrue( 'Process foo in group bar exited unexpectedly' in lines[6]) import os f = open(os.path.join(self.tempdir, 'email.log'), 'r') mail = f.read() f.close() self.assertTrue( 'Process foo in group bar exited unexpectedly' in mail) ================================================ FILE: superlance/tests/test_crashmailbatch.py ================================================ import unittest try: # pragma: no cover from unittest.mock import Mock except ImportError: # pragma: no cover from mock import Mock from superlance.compat import StringIO class CrashMailBatchTests(unittest.TestCase): from_email = 'testFrom@blah.com' to_emails = ('testTo@blah.com') subject = 'Test Alert' unexpected_err_msg = 'Process bar:foo (pid 58597) died unexpectedly' def _get_target_class(self): from superlance.crashmailbatch import CrashMailBatch return CrashMailBatch def _make_one_mocked(self, **kwargs): kwargs['stdin'] = StringIO() kwargs['stdout'] = StringIO() kwargs['stderr'] = StringIO() kwargs['from_email'] = kwargs.get('from_email', self.from_email) kwargs['to_emails'] = kwargs.get('to_emails', self.to_emails) kwargs['subject'] = kwargs.get('subject', self.subject) obj = self._get_target_class()(**kwargs) obj.send_email = Mock() return obj def get_process_exited_event(self, pname, gname, expected): headers = { 'ver': '3.0', 'poolserial': '7', 'len': '71', 'server': 'supervisor', 'eventname': 'PROCESS_STATE_EXITED', 'serial': '7', 'pool': 'checkmailbatch', } payload = 'processname:%s groupname:%s from_state:RUNNING expected:%d \ pid:58597' % (pname, gname, expected) return (headers, payload) def test_get_process_state_change_msg_expected(self): crash = self._make_one_mocked() hdrs, payload = self.get_process_exited_event('foo', 'bar', 1) self.assertEqual(None, crash.get_process_state_change_msg(hdrs, payload)) def test_get_process_state_change_msg_unexpected(self): crash = self._make_one_mocked() hdrs, payload = self.get_process_exited_event('foo', 'bar', 0) msg = crash.get_process_state_change_msg(hdrs, payload) self.assertTrue(self.unexpected_err_msg in msg) def test_handle_event_exit_expected(self): crash = self._make_one_mocked() hdrs, payload = self.get_process_exited_event('foo', 'bar', 1) crash.handle_event(hdrs, payload) self.assertEqual([], crash.get_batch_msgs()) self.assertEqual('', crash.stderr.getvalue()) def test_handle_event_exit_unexpected(self): crash = self._make_one_mocked() hdrs, payload = self.get_process_exited_event('foo', 'bar', 0) crash.handle_event(hdrs, payload) msgs = crash.get_batch_msgs() self.assertEqual(1, len(msgs)) self.assertTrue(self.unexpected_err_msg in msgs[0]) self.assertTrue(self.unexpected_err_msg in crash.stderr.getvalue()) def test_sets_default_subject_when_None(self): crash = self._make_one_mocked(subject=None) # see issue #109 self.assertEqual(crash.subject, "Crash alert from supervisord") ================================================ FILE: superlance/tests/test_crashsms.py ================================================ import unittest from .test_crashmailbatch import CrashMailBatchTests class CrashSMSTests(CrashMailBatchTests): subject = None unexpected_err_msg = '[bar:foo](58597) exited unexpectedly' def _get_target_class(self): from superlance.crashsms import CrashSMS return CrashSMS def test_sets_default_subject_when_None(self): crash = self._make_one_mocked(subject=None) self.assertEqual(crash.subject, self.subject) ================================================ FILE: superlance/tests/test_fatalmailbatch.py ================================================ import unittest try: # pragma: no cover from unittest.mock import Mock except ImportError: # pragma: no cover from mock import Mock from superlance.compat import StringIO class FatalMailBatchTests(unittest.TestCase): from_email = 'testFrom@blah.com' to_emails = ('testTo@blah.com') subject = 'Test Alert' unexpected_err_msg = 'Process bar:foo failed to start too many times' def _get_target_class(self): from superlance.fatalmailbatch import FatalMailBatch return FatalMailBatch def _make_one_mocked(self, **kwargs): kwargs['stdin'] = StringIO() kwargs['stdout'] = StringIO() kwargs['stderr'] = StringIO() kwargs['from_email'] = kwargs.get('from_email', self.from_email) kwargs['to_emails'] = kwargs.get('to_emails', self.to_emails) kwargs['subject'] = kwargs.get('subject', self.subject) obj = self._get_target_class()(**kwargs) obj.send_email = Mock() return obj def get_process_fatal_event(self, pname, gname): headers = { 'ver': '3.0', 'poolserial': '7', 'len': '71', 'server': 'supervisor', 'eventname': 'PROCESS_STATE_FATAL', 'serial': '7', 'pool': 'checkmailbatch', } payload = 'processname:%s groupname:%s from_state:BACKOFF' \ % (pname, gname) return (headers, payload) def test_get_process_state_change_msg(self): crash = self._make_one_mocked() hdrs, payload = self.get_process_fatal_event('foo', 'bar') msg = crash.get_process_state_change_msg(hdrs, payload) self.assertTrue(self.unexpected_err_msg in msg) def test_sets_default_subject_when_None(self): crash = self._make_one_mocked(subject=None) # see issue #109 self.assertEqual(crash.subject, "Fatal start alert from supervisord") ================================================ FILE: superlance/tests/test_httpok.py ================================================ import socket import time import unittest from superlance.compat import StringIO from supervisor.process import ProcessStates from superlance.tests.dummy import DummyResponse from superlance.tests.dummy import DummyRPCServer from superlance.tests.dummy import DummySupervisorRPCNamespace _NOW = time.time() _FAIL = [ { 'name':'FAILED', 'group':'foo', 'pid':11, 'state':ProcessStates.RUNNING, 'statename':'RUNNING', 'start':_NOW - 100, 'stop':0, 'spawnerr':'', 'now':_NOW, 'description':'foo description', }, { 'name':'SPAWN_ERROR', 'group':'foo', 'pid':11, 'state':ProcessStates.RUNNING, 'statename':'RUNNING', 'start':_NOW - 100, 'stop':0, 'spawnerr':'', 'now':_NOW, 'description':'foo description', },] def make_connection(response, exc=None): class TestConnection: def __init__(self, hostport): self.hostport = hostport def request(self, method, path, headers): if exc: if exc is True: raise ValueError('foo') else: raise exc.pop() self.method = method self.path = path self.headers = headers def getresponse(self): return response return TestConnection class HTTPOkTests(unittest.TestCase): def _getTargetClass(self): from superlance.httpok import HTTPOk return HTTPOk def _makeOne(self, *args, **kwargs): return self._getTargetClass()(*args, **kwargs) def _makeOnePopulated(self, programs, any=None, statuses=None, inbody=None, eager=True, gcore=None, coredir=None, response=None, exc=None, name=None, timeout=10, retry_time=0): if statuses is None: statuses = [200] if response is None: response = DummyResponse() httpok = self._makeOne( programs=programs, any=any, statuses=statuses, inbody=inbody, eager=eager, coredir=coredir, gcore=gcore, name=name, rpc=DummyRPCServer(), url='http://foo/bar', timeout=timeout, email='chrism@plope.com', sendmail='cat - > /dev/null', retry_time=retry_time, ) httpok.stdin = StringIO() httpok.stdout = StringIO() httpok.stderr = StringIO() httpok.connclass = make_connection(response, exc=exc) return httpok def test_listProcesses_no_programs(self): programs = [] any = None prog = self._makeOnePopulated(programs, any) specs = list(prog.listProcesses()) self.assertEqual(len(specs), 0) def test_listProcesses_w_RUNNING_programs_default_state(self): programs = ['foo'] any = None prog = self._makeOnePopulated(programs, any) specs = list(prog.listProcesses()) self.assertEqual(len(specs), 1) self.assertEqual(specs[0], DummySupervisorRPCNamespace.all_process_info[0]) def test_listProcesses_w_nonRUNNING_programs_default_state(self): programs = ['bar'] any = None prog = self._makeOnePopulated(programs, any) specs = list(prog.listProcesses()) self.assertEqual(len(specs), 1) self.assertEqual(specs[0], DummySupervisorRPCNamespace.all_process_info[1]) def test_listProcesses_w_nonRUNNING_programs_RUNNING_state(self): programs = ['bar'] any = None prog = self._makeOnePopulated(programs, any) specs = list(prog.listProcesses(ProcessStates.RUNNING)) self.assertEqual(len(specs), 0, (prog.programs, specs)) def test_runforever_eager_notatick(self): programs = {'foo':0, 'bar':0, 'baz_01':0 } any = None prog = self._makeOnePopulated(programs, any) prog.stdin.write('eventname:NOTATICK len:0\n') prog.stdin.seek(0) prog.runforever(test=True) self.assertEqual(prog.stderr.getvalue(), '') def test_runforever_doesnt_act_if_status_is_expected(self): statuses = [200, 201] for status in statuses: response = DummyResponse() response.status = status # expected prog = self._makeOnePopulated( programs=['foo'], statuses=statuses, response=response, ) prog.stdin.write('eventname:TICK len:0\n') prog.stdin.seek(0) prog.runforever(test=True) # status is expected so there should be no output self.assertEqual('', prog.stderr.getvalue()) def test_runforever_acts_if_status_is_unexpected(self): statuses = [200, 201] response = DummyResponse() response.status = 500 # unexpected response.reason = 'Internal Server Error' prog = self._makeOnePopulated( programs=['foo'], statuses=[statuses], response=response, ) prog.stdin.write('eventname:TICK len:0\n') prog.stdin.seek(0) prog.runforever(test=True) lines = prog.stderr.getvalue().split('\n') self.assertTrue('Subject: httpok: http://foo/bar: ' 'bad status returned' in lines) self.assertTrue('status contacting http://foo/bar: ' '500 Internal Server Error' in lines) def test_runforever_doesnt_act_if_inbody_is_present(self): response = DummyResponse() response.body = 'It works' prog = self._makeOnePopulated( programs=['foo'], statuses=[response.status], response=response, inbody='works', ) prog.stdin.write('eventname:TICK len:0\n') prog.stdin.seek(0) prog.runforever(test=True) # body is expected so there should be no output self.assertEqual('', prog.stderr.getvalue()) def test_runforever_acts_if_inbody_isnt_present(self): response = DummyResponse() response.body = 'Some kind of error' prog = self._makeOnePopulated( programs=['foo'], statuses=[response.status], response=response, inbody="works", ) prog.stdin.write('eventname:TICK len:0\n') prog.stdin.seek(0) prog.runforever(test=True) lines = prog.stderr.getvalue().split('\n') self.assertTrue('Subject: httpok: http://foo/bar: ' 'bad body returned' in lines) def test_runforever_eager_error_on_request_some(self): programs = ['foo', 'bar', 'baz_01', 'notexisting'] any = None prog = self._makeOnePopulated(programs, any, exc=True) prog.stdin.write('eventname:TICK len:0\n') prog.stdin.seek(0) prog.runforever(test=True) lines = prog.stderr.getvalue().split('\n') self.assertEqual(lines[0], ("Restarting selected processes ['foo', 'bar', " "'baz_01', 'notexisting']") ) self.assertEqual(lines[1], 'foo is in RUNNING state, restarting') self.assertEqual(lines[2], 'foo restarted') self.assertEqual(lines[3], 'bar not in RUNNING state, NOT restarting') self.assertEqual(lines[4], 'baz:baz_01 not in RUNNING state, NOT restarting') self.assertEqual(lines[5], "Programs not restarted because they did not exist: ['notexisting']") mailed = prog.mailed.split('\n') self.assertEqual(len(mailed), 12) self.assertEqual(mailed[0], 'To: chrism@plope.com') self.assertEqual(mailed[1], 'Subject: httpok: http://foo/bar: bad status returned') def test_runforever_eager_error_on_request_any(self): programs = [] any = True prog = self._makeOnePopulated(programs, any, exc=True) prog.stdin.write('eventname:TICK len:0\n') prog.stdin.seek(0) prog.runforever(test=True) lines = prog.stderr.getvalue().split('\n') self.assertEqual(lines[0], 'Restarting all running processes') self.assertEqual(lines[1], 'foo is in RUNNING state, restarting') self.assertEqual(lines[2], 'foo restarted') self.assertEqual(lines[3], 'bar not in RUNNING state, NOT restarting') self.assertEqual(lines[4], 'baz:baz_01 not in RUNNING state, NOT restarting') mailed = prog.mailed.split('\n') self.assertEqual(len(mailed), 11) self.assertEqual(mailed[0], 'To: chrism@plope.com') self.assertEqual(mailed[1], 'Subject: httpok: http://foo/bar: bad status returned') def test_runforever_eager_error_on_process_stop(self): programs = ['FAILED'] any = False prog = self._makeOnePopulated(programs, any, exc=True) prog.rpc.supervisor.all_process_info = _FAIL prog.stdin.write('eventname:TICK len:0\n') prog.stdin.seek(0) prog.runforever(test=True) lines = prog.stderr.getvalue().split('\n') self.assertEqual(lines[0], "Restarting selected processes ['FAILED']") self.assertEqual(lines[1], 'foo:FAILED is in RUNNING state, restarting') self.assertEqual(lines[2], "Failed to stop process foo:FAILED: ") self.assertEqual(lines[3], 'foo:FAILED restarted') mailed = prog.mailed.split('\n') self.assertEqual(len(mailed), 10) self.assertEqual(mailed[0], 'To: chrism@plope.com') self.assertEqual(mailed[1], 'Subject: httpok: http://foo/bar: bad status returned') def test_runforever_eager_error_on_process_start(self): programs = ['SPAWN_ERROR'] any = False prog = self._makeOnePopulated(programs, any, exc=True) prog.rpc.supervisor.all_process_info = _FAIL prog.stdin.write('eventname:TICK len:0\n') prog.stdin.seek(0) prog.runforever(test=True) lines = prog.stderr.getvalue().split('\n') self.assertEqual(lines[0], "Restarting selected processes ['SPAWN_ERROR']") self.assertEqual(lines[1], 'foo:SPAWN_ERROR is in RUNNING state, restarting') self.assertEqual(lines[2], "Failed to start process foo:SPAWN_ERROR: ") mailed = prog.mailed.split('\n') self.assertEqual(len(mailed), 9) self.assertEqual(mailed[0], 'To: chrism@plope.com') self.assertEqual(mailed[1], 'Subject: httpok: http://foo/bar: bad status returned') def test_runforever_eager_gcore(self): programs = ['foo', 'bar', 'baz_01', 'notexisting'] any = None prog = self._makeOnePopulated(programs, any, exc=True, gcore="true", coredir="/tmp") prog.stdin.write('eventname:TICK len:0\n') prog.stdin.seek(0) prog.runforever(test=True) lines = prog.stderr.getvalue().split('\n') self.assertEqual(lines[0], ("Restarting selected processes ['foo', 'bar', " "'baz_01', 'notexisting']") ) self.assertEqual(lines[1], 'gcore output for foo:') self.assertEqual(lines[2], '') self.assertEqual(lines[3], ' ') self.assertEqual(lines[4], 'foo is in RUNNING state, restarting') self.assertEqual(lines[5], 'foo restarted') self.assertEqual(lines[6], 'bar not in RUNNING state, NOT restarting') self.assertEqual(lines[7], 'baz:baz_01 not in RUNNING state, NOT restarting') self.assertEqual(lines[8], "Programs not restarted because they did not exist: ['notexisting']") mailed = prog.mailed.split('\n') self.assertEqual(len(mailed), 15) self.assertEqual(mailed[0], 'To: chrism@plope.com') self.assertEqual(mailed[1], 'Subject: httpok: http://foo/bar: bad status returned') def test_runforever_not_eager_none_running(self): programs = ['bar', 'baz_01'] any = None prog = self._makeOnePopulated(programs, any, exc=True, gcore="true", coredir="/tmp", eager=False) prog.stdin.write('eventname:TICK len:0\n') prog.stdin.seek(0) prog.runforever(test=True) lines = [x for x in prog.stderr.getvalue().split('\n') if x] self.assertEqual(len(lines), 0, lines) self.assertFalse('mailed' in prog.__dict__) def test_runforever_not_eager_running(self): programs = ['foo', 'bar'] any = None prog = self._makeOnePopulated(programs, any, exc=True, eager=False) prog.stdin.write('eventname:TICK len:0\n') prog.stdin.seek(0) prog.runforever(test=True) lines = [x for x in prog.stderr.getvalue().split('\n') if x] self.assertEqual(lines[0], ("Restarting selected processes ['foo', 'bar']") ) self.assertEqual(lines[1], 'foo is in RUNNING state, restarting') self.assertEqual(lines[2], 'foo restarted') self.assertEqual(lines[3], 'bar not in RUNNING state, NOT restarting') mailed = prog.mailed.split('\n') self.assertEqual(len(mailed), 10) self.assertEqual(mailed[0], 'To: chrism@plope.com') self.assertEqual(mailed[1], 'Subject: httpok: http://foo/bar: bad status returned') def test_runforever_honor_timeout_on_connrefused(self): programs = ['foo', 'bar'] any = None error = socket.error() error.errno = 111 prog = self._makeOnePopulated(programs, any, exc=[error], eager=False) prog.stdin.write('eventname:TICK len:0\n') prog.stdin.seek(0) prog.runforever(test=True) self.assertEqual(prog.stderr.getvalue(), '') self.assertEqual(prog.stdout.getvalue(), 'READY\nRESULT 2\nOK') def test_runforever_connrefused_error(self): programs = ['foo', 'bar'] any = None error = socket.error() error.errno = 111 prog = self._makeOnePopulated(programs, any, exc=[error for x in range(100)], eager=False) prog.stdin.write('eventname:TICK len:0\n') prog.stdin.seek(0) prog.runforever(test=True) lines = [x for x in prog.stderr.getvalue().split('\n') if x] self.assertEqual(lines[0], ("Restarting selected processes ['foo', 'bar']") ) self.assertEqual(lines[1], 'foo is in RUNNING state, restarting') self.assertEqual(lines[2], 'foo restarted') self.assertEqual(lines[3], 'bar not in RUNNING state, NOT restarting') mailed = prog.mailed.split('\n') self.assertEqual(len(mailed), 10) self.assertEqual(mailed[0], 'To: chrism@plope.com') self.assertEqual(mailed[1], 'Subject: httpok: http://foo/bar: bad status returned') def test_bug_110(self): error = socket.error() error.errno = 111 prog = self._makeOnePopulated(programs=['foo'], any=None, exc=[error for x in range(100)], eager=False, timeout=1, retry_time=10) prog.stdin.write('eventname:TICK len:0\n') prog.stdin.seek(0) prog.runforever(test=True) lines = [x for x in prog.stderr.getvalue().split('\n') if x] self.assertEqual(lines[0], ("Restarting selected processes ['foo']") ) self.assertEqual(lines[1], 'foo is in RUNNING state, restarting') self.assertEqual(lines[2], 'foo restarted') def test_subject_no_name(self): """set the name to None to check if subject formats to: httpok: %(subject)s """ prog = self._makeOnePopulated( programs=['foo', 'bar'], any=None, eager=False, exc=[ValueError('this causes status to be None')], name=None, ) prog.stdin.write('eventname:TICK len:0\n') prog.stdin.seek(0) prog.runforever(test=True) mailed = prog.mailed.split('\n') self.assertEqual(mailed[1], 'Subject: httpok: http://foo/bar: bad status returned') def test_subject_with_name(self): """set the name to a string to check if subject formats to: httpok [%(name)s]: %(subject)s """ prog = self._makeOnePopulated( programs=['foo', 'bar'], any=None, eager=False, exc=[ValueError('this causes status to be None')], name='thinko', ) prog.stdin.write('eventname:TICK len:0\n') prog.stdin.seek(0) prog.runforever(test=True) mailed = prog.mailed.split('\n') self.assertEqual(mailed[1], 'Subject: httpok [thinko]: http://foo/bar: bad status returned') ================================================ FILE: superlance/tests/test_memmon.py ================================================ # -*- coding: utf-8 -*- import unittest from superlance.compat import StringIO from superlance.compat import maxint from superlance.memmon import ( help_request, memmon_from_args, seconds_size ) from superlance.tests.dummy import DummyRPCServer class MemmonTests(unittest.TestCase): def _getTargetClass(self): from superlance.memmon import Memmon return Memmon def _makeOne(self, *args, **kwargs): return self._getTargetClass()(*args, **kwargs) def _makeOnePopulated(self, programs, groups, any, name=None): memmon = self._makeOne( programs=programs, groups=groups, any=any, name=name, rpc=DummyRPCServer(), cumulative=False, sendmail='cat - > /dev/null', email='chrism@plope.com', email_uptime_limit=2000, ) memmon.stdin = StringIO() memmon.stdout = StringIO() memmon.stderr = StringIO() memmon.pscommand = 'echo 22%s' return memmon def test_runforever_notatick(self): programs = {'foo':0, 'bar':0, 'baz_01':0 } groups = {} any = None memmon = self._makeOnePopulated(programs, groups, any) memmon.stdin.write('eventname:NOTATICK len:0\n') memmon.stdin.seek(0) memmon.runforever(test=True) self.assertEqual(memmon.stderr.getvalue(), '') def test_runforever_tick_programs(self): programs = {'foo':0, 'bar':0, 'baz_01':0 } groups = {} any = None memmon = self._makeOnePopulated(programs, groups, any) memmon.stdin.write('eventname:TICK len:0\n') memmon.stdin.seek(0) memmon.runforever(test=True) lines = memmon.stderr.getvalue().split('\n') self.assertEqual(len(lines), 8) self.assertEqual(lines[0], 'Checking programs bar=0, baz_01=0, foo=0') self.assertEqual(lines[1], 'RSS of foo:foo is 2264064') self.assertEqual(lines[2], 'Restarting foo:foo') self.assertEqual(lines[3], 'RSS of bar:bar is 2265088') self.assertEqual(lines[4], 'Restarting bar:bar') self.assertEqual(lines[5], 'RSS of baz:baz_01 is 2265088') self.assertEqual(lines[6], 'Restarting baz:baz_01') self.assertEqual(lines[7], '') mailed = memmon.mailed.split('\n') self.assertEqual(len(mailed), 4) self.assertEqual(mailed[0], 'To: chrism@plope.com') self.assertEqual(mailed[1], 'Subject: memmon: process baz:baz_01 restarted') self.assertEqual(mailed[2], '') self.assertTrue(mailed[3].startswith('memmon.py restarted')) def test_runforever_tick_groups(self): programs = {} groups = {'foo':0} any = None memmon = self._makeOnePopulated(programs, groups, any) memmon.stdin.write('eventname:TICK len:0\n') memmon.stdin.seek(0) memmon.runforever(test=True) lines = memmon.stderr.getvalue().split('\n') self.assertEqual(len(lines), 4) self.assertEqual(lines[0], 'Checking groups foo=0') self.assertEqual(lines[1], 'RSS of foo:foo is 2264064') self.assertEqual(lines[2], 'Restarting foo:foo') self.assertEqual(lines[3], '') mailed = memmon.mailed.split('\n') self.assertEqual(len(mailed), 4) self.assertEqual(mailed[0], 'To: chrism@plope.com') self.assertEqual(mailed[1], 'Subject: memmon: process foo:foo restarted') self.assertEqual(mailed[2], '') self.assertTrue(mailed[3].startswith('memmon.py restarted')) def test_runforever_tick_any(self): programs = {} groups = {} any = 0 memmon = self._makeOnePopulated(programs, groups, any) memmon.stdin.write('eventname:TICK len:0\n') memmon.stdin.seek(0) memmon.runforever(test=True) lines = memmon.stderr.getvalue().split('\n') self.assertEqual(len(lines), 8) self.assertEqual(lines[0], 'Checking any=0') self.assertEqual(lines[1], 'RSS of foo:foo is 2264064') self.assertEqual(lines[2], 'Restarting foo:foo') self.assertEqual(lines[3], 'RSS of bar:bar is 2265088') self.assertEqual(lines[4], 'Restarting bar:bar') self.assertEqual(lines[5], 'RSS of baz:baz_01 is 2265088') self.assertEqual(lines[6], 'Restarting baz:baz_01') self.assertEqual(lines[7], '') mailed = memmon.mailed.split('\n') self.assertEqual(len(mailed), 4) def test_runforever_tick_programs_and_groups(self): programs = {'baz_01':0} groups = {'foo':0} any = None memmon = self._makeOnePopulated(programs, groups, any) memmon.stdin.write('eventname:TICK len:0\n') memmon.stdin.seek(0) memmon.runforever(test=True) lines = memmon.stderr.getvalue().split('\n') self.assertEqual(len(lines), 7) self.assertEqual(lines[0], 'Checking programs baz_01=0') self.assertEqual(lines[1], 'Checking groups foo=0') self.assertEqual(lines[2], 'RSS of foo:foo is 2264064') self.assertEqual(lines[3], 'Restarting foo:foo') self.assertEqual(lines[4], 'RSS of baz:baz_01 is 2265088') self.assertEqual(lines[5], 'Restarting baz:baz_01') self.assertEqual(lines[6], '') mailed = memmon.mailed.split('\n') self.assertEqual(len(mailed), 4) self.assertEqual(mailed[0], 'To: chrism@plope.com') self.assertEqual(mailed[1], 'Subject: memmon: process baz:baz_01 restarted') self.assertEqual(mailed[2], '') self.assertTrue(mailed[3].startswith('memmon.py restarted')) def test_runforever_tick_programs_norestart(self): programs = {'foo': maxint} groups = {} any = None memmon = self._makeOnePopulated(programs, groups, any) memmon.stdin.write('eventname:TICK len:0\n') memmon.stdin.seek(0) memmon.runforever(test=True) lines = memmon.stderr.getvalue().split('\n') self.assertEqual(len(lines), 3) self.assertEqual(lines[0], 'Checking programs foo=%s' % maxint) self.assertEqual(lines[1], 'RSS of foo:foo is 2264064') self.assertEqual(lines[2], '') self.assertEqual(memmon.mailed, False) def test_stopprocess_fault_tick_programs_norestart(self): programs = {'foo': maxint} groups = {} any = None memmon = self._makeOnePopulated(programs, groups, any) memmon.stdin.write('eventname:TICK len:0\n') memmon.stdin.seek(0) memmon.runforever(test=True) lines = memmon.stderr.getvalue().split('\n') self.assertEqual(len(lines), 3) self.assertEqual(lines[0], 'Checking programs foo=%s' % maxint) self.assertEqual(lines[1], 'RSS of foo:foo is 2264064') self.assertEqual(lines[2], '') self.assertEqual(memmon.mailed, False) def test_stopprocess_fails_to_stop(self): programs = {'BAD_NAME': 0} groups = {} any = None memmon = self._makeOnePopulated(programs, groups, any) memmon.stdin.write('eventname:TICK len:0\n') memmon.stdin.seek(0) from supervisor.process import ProcessStates memmon.rpc.supervisor.all_process_info = [ { 'name':'BAD_NAME', 'group':'BAD_NAME', 'pid':11, 'state':ProcessStates.RUNNING, 'statename':'RUNNING', 'start':0, 'stop':0, 'spawnerr':'', 'now':0, 'description':'BAD_NAME description', } ] from superlance.compat import xmlrpclib self.assertRaises(xmlrpclib.Fault, memmon.runforever, True) lines = memmon.stderr.getvalue().split('\n') self.assertEqual(len(lines), 4) self.assertEqual(lines[0], 'Checking programs BAD_NAME=%s' % 0) self.assertEqual(lines[1], 'RSS of BAD_NAME:BAD_NAME is 2264064') self.assertEqual(lines[2], 'Restarting BAD_NAME:BAD_NAME') self.assertTrue(lines[3].startswith('Failed')) mailed = memmon.mailed.split('\n') self.assertEqual(len(mailed), 4) self.assertEqual(mailed[0], 'To: chrism@plope.com') self.assertEqual(mailed[1], 'Subject: memmon: failed to stop process BAD_NAME:BAD_NAME, exiting') self.assertEqual(mailed[2], '') self.assertTrue(mailed[3].startswith('Failed')) def test_subject_no_name(self): """set the name to None to check if subject formats to: memmon: %(subject)s """ memmon = self._makeOnePopulated( programs={}, groups={}, any=0, name=None, ) memmon.stdin.write('eventname:TICK len:0\n') memmon.stdin.seek(0) memmon.runforever(test=True) mailed = memmon.mailed.split('\n') self.assertEqual(mailed[1], 'Subject: memmon: process baz:baz_01 restarted') def test_subject_with_name(self): """set the name to a string to check if subject formats to: memmon [%(name)s]: %(subject)s """ memmon = self._makeOnePopulated( programs={}, groups={}, any=0, name='thinko', ) memmon.stdin.write('eventname:TICK len:0\n') memmon.stdin.seek(0) memmon.runforever(test=True) mailed = memmon.mailed.split('\n') self.assertEqual(mailed[1], 'Subject: memmon [thinko]: process baz:baz_01 restarted') def test_parse_uptime(self): """test parsing of time parameter for uptime """ self.assertEqual(seconds_size('1'), 1, 'default is seconds') self.assertEqual(seconds_size('1s'), 1, 'seconds suffix is allowed, too') self.assertEqual(seconds_size('2m'), 120) self.assertEqual(seconds_size('3h'), 10800) self.assertEqual(seconds_size('1d'), 86400) self.assertRaises(ValueError, seconds_size, '1y') def test_uptime_short_email(self): """in case an email is provided and the restarted process' uptime is shorter than our uptime_limit we do send an email """ programs = {'foo':0} groups = {} any = None memmon = self._makeOnePopulated(programs, groups, any) memmon.email_uptime_limit = 101 memmon.stdin.write('eventname:TICK len:0\n') memmon.stdin.seek(0) memmon.runforever(test=True) self.assertTrue(memmon.mailed, 'email has been sent') #in case uptime == limit, we send an email too memmon = self._makeOnePopulated(programs, groups, any) memmon.email_uptime_limit = 100 memmon.stdin.write('eventname:TICK len:0\n') memmon.stdin.seek(0) memmon.runforever(test=True) self.assertTrue(memmon.mailed, 'email has been sent') def test_uptime_long_no_email(self): """in case an email is provided and the restarted process' uptime is longer than our uptime_limit we do not send an email """ programs = {'foo':0} groups = {} any = None memmon = self._makeOnePopulated(programs, groups, any) memmon.email_uptime_limit = 99 memmon.stdin.write('eventname:TICK len:0\n') memmon.stdin.seek(0) memmon.runforever(test=True) self.assertFalse(memmon.mailed, 'no email should be sent because uptime is above limit') def test_calc_rss_not_cumulative(self): programs = {} groups = {} any = None memmon = self._makeOnePopulated(programs, groups, any) noop = '_=%s; ' pid = 1 memmon.pscommand = noop + 'echo 16' rss = memmon.calc_rss(pid) self.assertEqual(16 * 1024, rss) memmon.pscommand = noop + 'echo not_an_int' rss = memmon.calc_rss(pid) self.assertEqual( None, rss, 'Failure to parse an integer RSS value from the ps ' 'output should result in calc_rss() returning None.') def test_calc_rss_cumulative(self): """Let calc_rss() do its work on a fake process tree: ├─┬= 99 │ └─┬= 1 │ └─┬= 2 │ ├─── 3 │ └─── 4 (Where the process with PID 1 is the one being monitored) """ programs = {} groups = {} any = None memmon = self._makeOnePopulated(programs, groups, any) memmon.cumulative = True # output of ps ax -o "pid= ppid= rss=" representing the process # tree described above, including extraneous whitespace and # unrelated processes. ps_output = """ 11111 22222 333 1 99 100 2 1 200 3 2 300 4 2 400 11111 22222 333 """ memmon.pstreecommand = 'echo "%s"' % ps_output rss = memmon.calc_rss(1) self.assertEqual( 1000 * 1024, rss, 'Cumulative RSS of the test process and its three children ' 'should add up to 1000 kb.') def test_argparser(self): """test if arguments are parsed correctly """ # help arguments = ['-h', ] memmon = memmon_from_args(arguments) self.assertTrue(memmon is help_request, '-h returns help_request to make main() script print usage' ) #all arguments arguments = ['-c', '-p', 'foo=50MB', '-g', 'bar=10kB', '--any', '250', '-s', 'mutt', '-m', 'me@you.com', '-u', '1d', '-n', 'myproject'] memmon = memmon_from_args(arguments) self.assertEqual(memmon.cumulative, True) self.assertEqual(memmon.programs['foo'], 50 * 1024 * 1024) self.assertEqual(memmon.groups['bar'], 10 * 1024) self.assertEqual(memmon.any, 250) self.assertEqual(memmon.sendmail, 'mutt') self.assertEqual(memmon.email, 'me@you.com') self.assertEqual(memmon.email_uptime_limit, 1 * 24 * 60 * 60) self.assertEqual(memmon.name, 'myproject') #default arguments arguments = ['-m', 'me@you.com'] memmon = memmon_from_args(arguments) self.assertEqual(memmon.cumulative, False) self.assertEqual(memmon.programs, {}) self.assertEqual(memmon.groups, {}) self.assertEqual(memmon.any, None) self.assertTrue('sendmail' in memmon.sendmail, 'not using sendmail as default') self.assertEqual(memmon.email_uptime_limit, maxint) self.assertEqual(memmon.name, None) arguments = ['-p', 'foo=50MB'] memmon = memmon_from_args(arguments) self.assertEqual(memmon.email, None) ================================================ FILE: superlance/tests/test_process_state_email_monitor.py ================================================ import unittest try: # pragma: no cover from unittest.mock import Mock except ImportError: # pragma: no cover from mock import Mock from superlance.compat import StringIO class ProcessStateEmailMonitorTestException(Exception): pass class ProcessStateEmailMonitorTests(unittest.TestCase): from_email = 'testFrom@blah.com' to_emails = ('testTo@blah.com', 'testTo2@blah.com') to_str = 'testTo@blah.com, testTo2@blah.com' subject = 'Test Alert' def _get_target_class(self): from superlance.process_state_email_monitor \ import ProcessStateEmailMonitor return ProcessStateEmailMonitor def _make_one(self, **kwargs): kwargs['stdin'] = StringIO() kwargs['stdout'] = StringIO() kwargs['stderr'] = StringIO() kwargs['from_email'] = kwargs.get('from_email', self.from_email) kwargs['to_emails'] = kwargs.get('to_emails', self.to_emails) kwargs['subject'] = kwargs.get('subject', self.subject) obj = self._get_target_class()(**kwargs) return obj def _make_one_mock_send_email(self, **kwargs): obj = self._make_one(**kwargs) obj.send_email = Mock() return obj def _make_one_mock_send_smtp(self, **kwargs): obj = self._make_one(**kwargs) obj.send_smtp = Mock() return obj def test_validate_cmd_line_options_single_to_email_ok(self): klass = self._get_target_class() options = Mock() options.from_email = 'blah' options.to_emails = 'frog' validated = klass.validate_cmd_line_options(options) self.assertEqual(['frog'], validated.to_emails) def test_validate_cmd_line_options_multi_to_emails_ok(self): klass = self._get_target_class() options = Mock() options.from_email = 'blah' options.to_emails = 'frog, log,dog' validated = klass.validate_cmd_line_options(options) self.assertEqual(['frog', 'log', 'dog'], validated.to_emails) def test_send_email_ok(self): email = { 'body': 'msg1\nmsg2', 'to': self.to_emails, 'from': 'testFrom@blah.com', 'subject': 'Test Alert', } monitor = self._make_one_mock_send_smtp() monitor.send_email(email) # Test that email was sent self.assertEqual(1, monitor.send_smtp.call_count) smtpCallArgs = monitor.send_smtp.call_args[0] mimeMsg = smtpCallArgs[0] self.assertEqual(self.to_str, mimeMsg['To']) self.assertEqual(email['from'], mimeMsg['From']) self.assertEqual(email['subject'], mimeMsg['Subject']) self.assertEqual(email['body'], mimeMsg.get_payload()) def _raiseSTMPException(self, mime, to_emails): raise ProcessStateEmailMonitorTestException('test') def test_send_email_exception(self): email = { 'body': 'msg1\nmsg2', 'to': self.to_emails, 'from': 'testFrom@blah.com', 'subject': 'Test Alert', } monitor = self._make_one_mock_send_smtp() monitor.send_smtp.side_effect = self._raiseSTMPException monitor.send_email(email) # Test that error was logged to stderr self.assertEqual("Error sending email: test\n", monitor.stderr.getvalue()) def test_send_batch_notification(self): test_msgs = ['msg1', 'msg2'] monitor = self._make_one_mock_send_email() monitor.batchmsgs = test_msgs monitor.send_batch_notification() # Test that email was sent expected = { 'body': 'msg1\nmsg2', 'to': self.to_emails, 'from': 'testFrom@blah.com', 'subject': 'Test Alert', } self.assertEqual(1, monitor.send_email.call_count) monitor.send_email.assert_called_with(expected) # Test that email was logged self.assertEqual("""Sending notification email: To: %s From: testFrom@blah.com Subject: Test Alert Body: msg1 msg2 """ % (self.to_str), monitor.stderr.getvalue()) def test_log_email_with_body_digest(self): bodyLen = 80 monitor = self._make_one_mock_send_email() email = { 'to': ['you@fubar.com'], 'from': 'me@fubar.com', 'subject': 'yo yo', 'body': 'a' * bodyLen, } monitor.log_email(email) self.assertEqual("""Sending notification email: To: you@fubar.com From: me@fubar.com Subject: yo yo Body: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa... """, monitor.stderr.getvalue()) self.assertEqual('a' * bodyLen, email['body']) def test_log_email_without_body_digest(self): monitor = self._make_one_mock_send_email() email = { 'to': ['you@fubar.com'], 'from': 'me@fubar.com', 'subject': 'yo yo', 'body': 'a' * 20, } monitor.log_email(email) self.assertEqual("""Sending notification email: To: you@fubar.com From: me@fubar.com Subject: yo yo Body: aaaaaaaaaaaaaaaaaaaa """, monitor.stderr.getvalue()) ================================================ FILE: superlance/tests/test_process_state_monitor.py ================================================ import unittest try: # pragma: no cover from unittest.mock import Mock except ImportError: # pragma: no cover from mock import Mock from superlance.compat import StringIO from superlance.process_state_monitor import ProcessStateMonitor class _TestProcessStateMonitor(ProcessStateMonitor): process_state_events = ['PROCESS_STATE_EXITED'] def get_process_state_change_msg(self, headers, payload): return repr(payload) class ProcessStateMonitorTests(unittest.TestCase): def _get_target_class(self): return _TestProcessStateMonitor def _make_one_mocked(self, **kwargs): kwargs['stdin'] = StringIO() kwargs['stdout'] = StringIO() kwargs['stderr'] = StringIO() obj = self._get_target_class()(**kwargs) obj.send_batch_notification = Mock() return obj def get_process_exited_event(self, pname, gname, expected, eventname='PROCESS_STATE_EXITED'): headers = { 'ver': '3.0', 'poolserial': '7', 'len': '71', 'server': 'supervisor', 'eventname': eventname, 'serial': '7', 'pool': 'checkmailbatch', } payload = 'processname:%s groupname:%s from_state:RUNNING expected:%d \ pid:58597' % (pname, gname, expected) return (headers, payload) def get_tick60_event(self): headers = { 'ver': '3.0', 'poolserial': '5', 'len': '15', 'server': 'supervisor', 'eventname': 'TICK_60', 'serial': '5', 'pool': 'checkmailbatch', } payload = 'when:1279665240' return (headers, payload) def test__get_tick_secs(self): monitor = self._make_one_mocked() self.assertEqual(5, monitor._get_tick_secs('TICK_5')) self.assertEqual(60, monitor._get_tick_secs('TICK_60')) self.assertEqual(3600, monitor._get_tick_secs('TICK_3600')) self.assertRaises(ValueError, monitor._get_tick_secs, 'JUNK_60') def test__get_tick_mins(self): monitor = self._make_one_mocked() self.assertEqual(5.0/60.0, monitor._get_tick_mins('TICK_5')) def test_handle_event_exit(self): monitor = self._make_one_mocked() hdrs, payload = self.get_process_exited_event('foo', 'bar', 0) monitor.handle_event(hdrs, payload) unexpected_err_msg = repr(payload) self.assertEqual([unexpected_err_msg], monitor.get_batch_msgs()) self.assertEqual('%s\n' % unexpected_err_msg, monitor.stderr.getvalue()) def test_handle_event_non_exit(self): monitor = self._make_one_mocked() hdrs, payload = self.get_process_exited_event('foo', 'bar', 0, eventname='PROCESS_STATE_FATAL') monitor.handle_event(hdrs, payload) self.assertEqual([], monitor.get_batch_msgs()) self.assertEqual('', monitor.stderr.getvalue()) def test_handle_event_tick_interval_expired(self): monitor = self._make_one_mocked() #Put msgs in batch hdrs, payload = self.get_process_exited_event('foo', 'bar', 0) monitor.handle_event(hdrs, payload) hdrs, payload = self.get_process_exited_event('bark', 'dog', 0) monitor.handle_event(hdrs, payload) self.assertEqual(2, len(monitor.get_batch_msgs())) #Time expired hdrs, payload = self.get_tick60_event() monitor.handle_event(hdrs, payload) # Test that batch messages are now gone self.assertEqual([], monitor.get_batch_msgs()) # Test that email was sent self.assertEqual(1, monitor.send_batch_notification.call_count) def test_handle_event_tick_interval_not_expired(self): monitor = self._make_one_mocked(interval=3) hdrs, payload = self.get_tick60_event() monitor.handle_event(hdrs, payload) self.assertEqual(1.0, monitor.get_batch_minutes()) monitor.handle_event(hdrs, payload) self.assertEqual(2.0, monitor.get_batch_minutes()) ================================================ FILE: superlance/timeoutconn.py ================================================ from superlance.compat import httplib import socket import ssl class TimeoutHTTPConnection(httplib.HTTPConnection): """A customised HTTPConnection allowing a per-connection timeout, specified at construction.""" timeout = None def connect(self): """Override HTTPConnection.connect to connect to host/port specified in __init__.""" e = "getaddrinfo returns an empty list" for res in socket.getaddrinfo(self.host, self.port, 0, socket.SOCK_STREAM): af, socktype, proto, canonname, sa = res try: self.sock = socket.socket(af, socktype, proto) if self.timeout: # this is the new bit self.sock.settimeout(self.timeout) self.sock.connect(sa) except socket.error: if self.sock: self.sock.close() self.sock = None continue break if not self.sock: raise socket.error(e) class TimeoutHTTPSConnection(httplib.HTTPSConnection): timeout = None def connect(self): "Connect to a host on a given (SSL) port." sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if self.timeout: sock.settimeout(self.timeout) sock.connect((self.host, self.port)) self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file) ================================================ FILE: tox.ini ================================================ [tox] envlist = docs,py27,py34,py35,py36,py37,py38,py39,py310,py311,py312,py313,py314 [testenv] deps = pytest commands = pytest [testenv:py27] deps = {[testenv]deps} mock >= 0.5.0 commands = {[testenv]commands} [testenv:docs] deps = Sphinx readme setuptools >= 18.5 allowlist_externals = make commands = make -C docs html BUILDDIR={envtmpdir} "SPHINXOPTS=-W -E" python setup.py check -m -r -s