Repository: TUT-ARG/sed_eval Branch: master Commit: 0cb1b6d11cee Files: 283 Total size: 4.3 MB Directory structure: gitextract_rwh6jdyn/ ├── .coveragerc ├── .gitignore ├── .travis.yml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.txt ├── README.rst ├── docs/ │ ├── .buildinfo │ ├── .nojekyll │ ├── _modules/ │ │ ├── index.html │ │ └── sed_eval/ │ │ ├── audio_tag.html │ │ ├── io.html │ │ ├── metric.html │ │ ├── scene.html │ │ ├── sound_event.html │ │ ├── test.html │ │ └── util/ │ │ ├── event_list.html │ │ ├── event_matching.html │ │ ├── event_roll.html │ │ └── scene_list.html │ ├── _sources/ │ │ ├── audio_tag.rst.txt │ │ ├── changelog.rst.txt │ │ ├── generated/ │ │ │ ├── sed_eval.audio_tag.AudioTaggingMetrics.evaluate.rst.txt │ │ │ ├── sed_eval.audio_tag.AudioTaggingMetrics.reset.rst.txt │ │ │ ├── sed_eval.audio_tag.AudioTaggingMetrics.result_report_class_wise.rst.txt │ │ │ ├── sed_eval.audio_tag.AudioTaggingMetrics.result_report_class_wise_average.rst.txt │ │ │ ├── sed_eval.audio_tag.AudioTaggingMetrics.result_report_parameters.rst.txt │ │ │ ├── sed_eval.audio_tag.AudioTaggingMetrics.results.rst.txt │ │ │ ├── sed_eval.audio_tag.AudioTaggingMetrics.results_class_wise_average_metrics.rst.txt │ │ │ ├── sed_eval.audio_tag.AudioTaggingMetrics.results_class_wise_metrics.rst.txt │ │ │ ├── sed_eval.audio_tag.AudioTaggingMetrics.results_overall_metrics.rst.txt │ │ │ ├── sed_eval.audio_tag.AudioTaggingMetrics.rst.txt │ │ │ ├── sed_eval.io.load_event_list.rst.txt │ │ │ ├── sed_eval.io.load_file_pair_list.rst.txt │ │ │ ├── sed_eval.io.load_scene_list.rst.txt │ │ │ ├── sed_eval.metric.accuracy.rst.txt │ │ │ ├── sed_eval.metric.accuracy2.rst.txt │ │ │ ├── sed_eval.metric.balanced_accuracy.rst.txt │ │ │ ├── sed_eval.metric.deletion_rate.rst.txt │ │ │ ├── sed_eval.metric.equal_error_rate.rst.txt │ │ │ ├── sed_eval.metric.error_rate.rst.txt │ │ │ ├── sed_eval.metric.f_measure.rst.txt │ │ │ ├── sed_eval.metric.insertion_rate.rst.txt │ │ │ ├── sed_eval.metric.precision.rst.txt │ │ │ ├── sed_eval.metric.recall.rst.txt │ │ │ ├── sed_eval.metric.sensitivity.rst.txt │ │ │ ├── sed_eval.metric.specificity.rst.txt │ │ │ ├── sed_eval.metric.substitution_rate.rst.txt │ │ │ ├── sed_eval.scene.SceneClassificationMetrics.evaluate.rst.txt │ │ │ ├── sed_eval.scene.SceneClassificationMetrics.reset.rst.txt │ │ │ ├── sed_eval.scene.SceneClassificationMetrics.result_report_class_wise.rst.txt │ │ │ ├── sed_eval.scene.SceneClassificationMetrics.result_report_class_wise_average.rst.txt │ │ │ ├── sed_eval.scene.SceneClassificationMetrics.result_report_parameters.rst.txt │ │ │ ├── sed_eval.scene.SceneClassificationMetrics.results.rst.txt │ │ │ ├── sed_eval.scene.SceneClassificationMetrics.results_class_wise_average_metrics.rst.txt │ │ │ ├── sed_eval.scene.SceneClassificationMetrics.results_class_wise_metrics.rst.txt │ │ │ ├── sed_eval.scene.SceneClassificationMetrics.results_overall_metrics.rst.txt │ │ │ ├── sed_eval.scene.SceneClassificationMetrics.rst.txt │ │ │ ├── sed_eval.sound_event.EventBasedMetrics.evaluate.rst.txt │ │ │ ├── sed_eval.sound_event.EventBasedMetrics.reset.rst.txt │ │ │ ├── sed_eval.sound_event.EventBasedMetrics.result_report_class_wise.rst.txt │ │ │ ├── sed_eval.sound_event.EventBasedMetrics.result_report_class_wise_average.rst.txt │ │ │ ├── sed_eval.sound_event.EventBasedMetrics.result_report_overall.rst.txt │ │ │ ├── sed_eval.sound_event.EventBasedMetrics.result_report_parameters.rst.txt │ │ │ ├── sed_eval.sound_event.EventBasedMetrics.results.rst.txt │ │ │ ├── sed_eval.sound_event.EventBasedMetrics.results_class_wise_average_metrics.rst.txt │ │ │ ├── sed_eval.sound_event.EventBasedMetrics.results_class_wise_metrics.rst.txt │ │ │ ├── sed_eval.sound_event.EventBasedMetrics.results_overall_metrics.rst.txt │ │ │ ├── sed_eval.sound_event.EventBasedMetrics.rst.txt │ │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.evaluate.rst.txt │ │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.reset.rst.txt │ │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.result_report_class_wise.rst.txt │ │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.result_report_class_wise_average.rst.txt │ │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.result_report_overall.rst.txt │ │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.result_report_parameters.rst.txt │ │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.results.rst.txt │ │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.results_class_wise_average_metrics.rst.txt │ │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.results_class_wise_metrics.rst.txt │ │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.results_overall_metrics.rst.txt │ │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.rst.txt │ │ │ ├── sed_eval.test.mcnemar.rst.txt │ │ │ ├── sed_eval.util.event_list.filter_event_list.rst.txt │ │ │ ├── sed_eval.util.event_list.max_event_offset.rst.txt │ │ │ ├── sed_eval.util.event_list.unique_event_labels.rst.txt │ │ │ ├── sed_eval.util.event_list.unique_files.rst.txt │ │ │ ├── sed_eval.util.event_matching.bipartite_match.rst.txt │ │ │ ├── sed_eval.util.event_roll.event_list_to_event_roll.rst.txt │ │ │ ├── sed_eval.util.event_roll.match_event_roll_lengths.rst.txt │ │ │ ├── sed_eval.util.event_roll.pad_event_roll.rst.txt │ │ │ └── sed_eval.util.scene_list.unique_scene_labels.rst.txt │ │ ├── glossary.rst.txt │ │ ├── index.rst.txt │ │ ├── install.rst.txt │ │ ├── io.rst.txt │ │ ├── metric.rst.txt │ │ ├── scene.rst.txt │ │ ├── sound_event.rst.txt │ │ ├── test.rst.txt │ │ ├── tutorial.rst.txt │ │ └── util.rst.txt │ ├── _static/ │ │ ├── basic.css │ │ ├── css/ │ │ │ ├── badge_only.css │ │ │ └── theme.css │ │ ├── doctools.js │ │ ├── jquery-3.1.0.js │ │ ├── jquery.js │ │ ├── js/ │ │ │ └── theme.js │ │ ├── pygments.css │ │ ├── searchtools.js │ │ ├── theme_overrides.css │ │ ├── underscore-1.3.1.js │ │ ├── underscore.js │ │ └── websupport.js │ ├── audio_tag.html │ ├── changelog.html │ ├── generated/ │ │ ├── sed_eval.audio_tag.AudioTaggingMetrics.evaluate.html │ │ ├── sed_eval.audio_tag.AudioTaggingMetrics.html │ │ ├── sed_eval.audio_tag.AudioTaggingMetrics.reset.html │ │ ├── sed_eval.audio_tag.AudioTaggingMetrics.result_report_class_wise.html │ │ ├── sed_eval.audio_tag.AudioTaggingMetrics.result_report_class_wise_average.html │ │ ├── sed_eval.audio_tag.AudioTaggingMetrics.result_report_parameters.html │ │ ├── sed_eval.audio_tag.AudioTaggingMetrics.results.html │ │ ├── sed_eval.audio_tag.AudioTaggingMetrics.results_class_wise_average_metrics.html │ │ ├── sed_eval.audio_tag.AudioTaggingMetrics.results_class_wise_metrics.html │ │ ├── sed_eval.audio_tag.AudioTaggingMetrics.results_overall_metrics.html │ │ ├── sed_eval.io.load_event_list.html │ │ ├── sed_eval.io.load_file_pair_list.html │ │ ├── sed_eval.io.load_scene_list.html │ │ ├── sed_eval.metric.accuracy.html │ │ ├── sed_eval.metric.accuracy2.html │ │ ├── sed_eval.metric.balanced_accuracy.html │ │ ├── sed_eval.metric.deletion_rate.html │ │ ├── sed_eval.metric.equal_error_rate.html │ │ ├── sed_eval.metric.error_rate.html │ │ ├── sed_eval.metric.f_measure.html │ │ ├── sed_eval.metric.insertion_rate.html │ │ ├── sed_eval.metric.precision.html │ │ ├── sed_eval.metric.recall.html │ │ ├── sed_eval.metric.sensitivity.html │ │ ├── sed_eval.metric.specificity.html │ │ ├── sed_eval.metric.substitution_rate.html │ │ ├── sed_eval.scene.SceneClassificationMetrics.evaluate.html │ │ ├── sed_eval.scene.SceneClassificationMetrics.html │ │ ├── sed_eval.scene.SceneClassificationMetrics.reset.html │ │ ├── sed_eval.scene.SceneClassificationMetrics.result_report_class_wise.html │ │ ├── sed_eval.scene.SceneClassificationMetrics.result_report_class_wise_average.html │ │ ├── sed_eval.scene.SceneClassificationMetrics.result_report_parameters.html │ │ ├── sed_eval.scene.SceneClassificationMetrics.results.html │ │ ├── sed_eval.scene.SceneClassificationMetrics.results_class_wise_average_metrics.html │ │ ├── sed_eval.scene.SceneClassificationMetrics.results_class_wise_metrics.html │ │ ├── sed_eval.scene.SceneClassificationMetrics.results_overall_metrics.html │ │ ├── sed_eval.sound_event.EventBasedMetrics.evaluate.html │ │ ├── sed_eval.sound_event.EventBasedMetrics.html │ │ ├── sed_eval.sound_event.EventBasedMetrics.reset.html │ │ ├── sed_eval.sound_event.EventBasedMetrics.result_report_class_wise.html │ │ ├── sed_eval.sound_event.EventBasedMetrics.result_report_class_wise_average.html │ │ ├── sed_eval.sound_event.EventBasedMetrics.result_report_overall.html │ │ ├── sed_eval.sound_event.EventBasedMetrics.result_report_parameters.html │ │ ├── sed_eval.sound_event.EventBasedMetrics.results.html │ │ ├── sed_eval.sound_event.EventBasedMetrics.results_class_wise_average_metrics.html │ │ ├── sed_eval.sound_event.EventBasedMetrics.results_class_wise_metrics.html │ │ ├── sed_eval.sound_event.EventBasedMetrics.results_overall_metrics.html │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.evaluate.html │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.html │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.reset.html │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.result_report_class_wise.html │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.result_report_class_wise_average.html │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.result_report_overall.html │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.result_report_parameters.html │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.results.html │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.results_class_wise_average_metrics.html │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.results_class_wise_metrics.html │ │ ├── sed_eval.sound_event.SegmentBasedMetrics.results_overall_metrics.html │ │ ├── sed_eval.test.mcnemar.html │ │ ├── sed_eval.util.event_list.filter_event_list.html │ │ ├── sed_eval.util.event_list.max_event_offset.html │ │ ├── sed_eval.util.event_list.unique_event_labels.html │ │ ├── sed_eval.util.event_list.unique_files.html │ │ ├── sed_eval.util.event_matching.bipartite_match.html │ │ ├── sed_eval.util.event_roll.event_list_to_event_roll.html │ │ ├── sed_eval.util.event_roll.match_event_roll_lengths.html │ │ ├── sed_eval.util.event_roll.pad_event_roll.html │ │ └── sed_eval.util.scene_list.unique_scene_labels.html │ ├── genindex.html │ ├── glossary.html │ ├── index.html │ ├── install.html │ ├── io.html │ ├── metric.html │ ├── objects.inv │ ├── py-modindex.html │ ├── scene.html │ ├── search.html │ ├── searchindex.js │ ├── sound_event.html │ ├── test.html │ ├── tutorial.html │ └── util.html ├── documentation/ │ ├── Makefile │ ├── README.md │ ├── requirements.txt │ └── source/ │ ├── __init__.py │ ├── _static/ │ │ ├── .nojekyll │ │ └── theme_overrides.css │ ├── audio_tag.rst │ ├── changelog.rst │ ├── conf.py │ ├── glossary.rst │ ├── index.rst │ ├── install.rst │ ├── io.rst │ ├── metric.rst │ ├── scene.rst │ ├── sound_event.rst │ ├── test.rst │ ├── tutorial.rst │ └── util.rst ├── evaluators/ │ ├── README.md │ ├── scene_eval.py │ └── sound_event_eval.py ├── requirements.txt ├── sed_eval/ │ ├── __init__.py │ ├── audio_tag.py │ ├── io.py │ ├── metric.py │ ├── scene.py │ ├── sound_event.py │ ├── test.py │ └── util/ │ ├── __init__.py │ ├── event_list.py │ ├── event_matching.py │ ├── event_roll.py │ └── scene_list.py ├── setup.cfg ├── setup.py └── tests/ ├── README.md ├── data/ │ ├── audio_tagging/ │ │ ├── estimated.txt │ │ └── reference.txt │ ├── scene/ │ │ ├── estimated_a.txt │ │ ├── estimated_b.txt │ │ ├── file_list.txt │ │ ├── fold1_estimated.txt │ │ ├── fold1_reference.txt │ │ ├── fold2_estimated.txt │ │ ├── fold2_reference.txt │ │ ├── fold3_estimated.txt │ │ ├── fold3_reference.txt │ │ ├── fold4_estimated.txt │ │ ├── fold4_reference.txt │ │ ├── fold5_estimated.txt │ │ ├── fold5_reference.txt │ │ └── reference.txt │ └── sound_event/ │ ├── audioset1.txt │ ├── audioset1_detected.txt │ ├── binary1.txt │ ├── binary1_detected.txt │ ├── binary2.txt │ ├── binary2_detected.txt │ ├── binary3.txt │ ├── binary3_detected.txt │ ├── binary4.txt │ ├── binary4_detected.txt │ ├── binary5.txt │ ├── binary5_detected.txt │ ├── file_list.txt │ ├── file_list_audioset.txt │ ├── file_list_binary.txt │ ├── mini_detected_a.txt │ ├── mini_detected_b.txt │ ├── mini_reference.txt │ ├── office_snr0_high_v2.txt │ ├── office_snr0_high_v2_detected.txt │ ├── office_snr0_med_v2.txt │ ├── office_snr0_med_v2_detected.txt │ ├── street_fold1_detected.txt │ └── street_fold1_reference.txt ├── test_audio_tag.py ├── test_io.py ├── test_metric.py ├── test_scene.py ├── test_sound_event.py ├── test_test.py └── test_util.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .coveragerc ================================================ [report] omit = */python?.?/* */site-packages/nose/* ================================================ FILE: .gitignore ================================================ *.py[co] # Packages *.egg *.egg-info dist build eggs parts bin var sdist develop-eggs .installed.cfg # Installer logs pip-log.txt # Unit test / coverage reports .coverage .tox #Translations *.mo #Mr Developer .mr.developer.cfg # OS generated files # .DS_Store .DS_Store? ._* .Spotlight-V100 .Trashes ehthumbs.db Thumbs.db *~ # Vim # PyCharm .idea # cover tests/cover/* tests/_dev/* .coverage .coverage.* # Sphinx #docs/generated/ tests/dcase2016/* .nfs* tests/*.zip tests/DCASE2016-baseline-system-python* tests/dev/ documentation/source/generated/ documentation/build/ ================================================ FILE: .travis.yml ================================================ language: - python python: - "2.7" - "3.5" - "3.6" cache: pip: true bundler: true notifications: email: false before_install: - "sudo apt-get install -qq python-scipy" # command to install dependencies install: - "pip install -r requirements.txt" - "pip install python-coveralls" - "pip install coveralls" - "pip install nose" - "pip install scikit-learn" - "pip install 'joblib==0.11' --force-reinstall" - "python setup.py develop" # command to run tests script: - "nosetests --with-coverage --cover-erase --cover-package=sed_eval -v -w tests/" after_success: - "coveralls" - "python setup.py develop --uninstall" after_failure: - "python setup.py develop --uninstall" ================================================ FILE: CODE_OF_CONDUCT.md ================================================ # Contributor Covenant Code of Conduct ## Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. ## Our Standards Examples of behavior that contributes to creating a positive environment include: * Using welcoming and inclusive language * Being respectful of differing viewpoints and experiences * Gracefully accepting constructive criticism * Focusing on what is best for the community * Showing empathy towards other community members Examples of unacceptable behavior by participants include: * The use of sexualized language or imagery and unwelcome sexual attention or advances * Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or electronic address, without explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. ## Scope This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at toni.heittola@tut.fi. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] [homepage]: http://contributor-covenant.org [version]: http://contributor-covenant.org/version/1/4/ ================================================ FILE: CONTRIBUTING.md ================================================ Contributing ============ How to contribute ----------------- The preferred way to contribute to `sed_eval` is to fork the [main repository](https://github.com/TUT-ARG/sed_eval) on GitHub: 1. Fork the [project repository](https://github.com/TUT-ARG/sed_eval): click on the 'Fork' button near the top of the page. This creates a copy of the code under your account on the GitHub server. 2. Clone this copy to your local disk: git clone git@github.com:[YOUR_LOGIN]/sed_eval.git cd sed_eval 3. Create a branch to hold your changes: git checkout -b my-new-feature and start making changes. You should never work in the ``master`` branch directly. 4. Work on this copy on your computer using Git to do the version control. When you're done editing, do: git add [MODIFIED FILES] git commit to record your changes in Git, then push them to GitHub with: git push -u origin my-new-feature Finally, go to the web page of the your fork of the sed_eval repo, and click 'Pull request' to send your changes to the maintainers for review. This will send an email to the committers. More information about this kind of process can be found in [Git documentation](http://git-scm.com/documentation). You should check that your contribution complies with the following rules before submitting a pull request: - All public methods should have informative docstrings - Code should be sufficiently commented - For major new features there should be also an unittest You should check for common programming errors with the following tools: - Check unittests: pip install nose coverage cd tests/ nosetests -v --with-coverage --cover-erase --cover-html --cover-package=sed_eval --nocapture - There should be no major pyflakes warnings, check with: pip install pyflakes pyflakes path/to/module.py - There should be no major PEP8 warnings, check with: pip install pep8 pep8 path/to/module.py Creating an issue ----------------- We use Github issues to track all bugs and feature requests; feel free to open an issue if you have found a bug or wish to see a new feature implemented. It is recommended to check that your issue complies with the following rules before submitting: - Verify that your issue is not being currently addressed by other [issues](https://github.com/TUT-ARG/sed_eval/issues?q=) or [pull requests](https://github.com/TUT-ARG/sed_eval/pulls?q=). - Please ensure all code snippets and error messages are formatted in appropriate code blocks. See [Creating and highlighting code blocks](https://help.github.com/articles/creating-and-highlighting-code-blocks). - Please include your operating system type and version number, as well as your Python, numpy, and dcase_util versions. You can get this information with following code: import platform; print(platform.platform()) import sys; print("Python", sys.version) import numpy; print("NumPy", numpy.__version__) Documentation ------------- You can edit the documentation using any text editor and then generate the HTML output by typing ``make html`` from the docs/ directory. The resulting HTML files will be placed in docs/ and are viewable in a web browser. See the README file in the documentation/ directory for more information. Note ---- This document is based on contribution instructions for [LibROSA](https://github.com/librosa/librosa). ================================================ FILE: LICENSE.txt ================================================ The MIT License (MIT) Copyright (c) 2016 Tampere University of Technology Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.rst ================================================ sed_eval - Evaluation toolbox for Sound Event Detection ======================================================= .. image:: https://travis-ci.org/TUT-ARG/sed_eval.svg?branch=master :target: https://travis-ci.org/TUT-ARG/sed_eval .. image:: https://coveralls.io/repos/github/TUT-ARG/sed_eval/badge.svg?branch=master :target: https://coveralls.io/github/TUT-ARG/sed_eval?branch=master .. image:: https://img.shields.io/pypi/v/sed_eval.svg :target: https://pypi.python.org/pypi/sed_eval .. image:: https://img.shields.io/badge/license-MIT-blue.svg :target: https://opensource.org/licenses/MIT sed_eval is an open source Python toolbox which provides a standardized, and transparent way to evaluate sound event detection systems. Authors - Toni Heittola (toni.heittola@tut.fi, `GitHub `_, ``_) - Annamaria Mesaros (annamaria.mesaros@tut.fi, ``_) Documentation ============= See http://tut-arg.github.io/sed_eval for manual and tutorials. Installation instructions ========================= The latest stable release is available on PyPI, and you can install with pip:: pip install sed_eval Alternatively you can download or clone toolbox and use ``pip`` to handle dependencies:: unzip sed_eval.zip pip install -e sed_eval or:: git clone https://github.com/TUT-ARG/sed_eval.git pip install -e sed_eval Dependencies ------------ The toolbox is tested to work with Python 2.7 and Python 3.6. - numpy >= 1.7.0 - dcase_util >= 0.2.4 Citing ====== If you use sed_eval in a research project, please cite the following paper: Annamaria Mesaros, Toni Heittola, and Tuomas Virtanen, "Metrics for polyphonic sound event detection", Applied Sciences, 6(6):162, 2016 [`HTML `_][`PDF `_] License ======= Code released under `the MIT license `_. ================================================ FILE: docs/.buildinfo ================================================ # Sphinx build info version 1 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. config: 950eb7edc6fd5559b7dbbc2ae8b0df21 tags: 645f666f9bcd5a90fca523b33c5a78b7 ================================================ FILE: docs/.nojekyll ================================================ ================================================ FILE: docs/_modules/index.html ================================================ Overview: module code — sed_eval 0.1 documentation
  • Docs »
  • Overview: module code

© Copyright 2018, Toni Heittola et al..

================================================ FILE: docs/_modules/sed_eval/audio_tag.html ================================================ sed_eval.audio_tag — sed_eval 0.1 documentation

Source code for sed_eval.audio_tag

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Metrics
-------

Main functions:

* :func:`sed_eval.audio_tag.AudioTaggingMetrics.evaluate`: Calculate intermediate values for evaluation and accumulate them.
* :func:`sed_eval.audio_tag.AudioTaggingMetrics.results`: Calculate and return all metrics.
* :func:`sed_eval.audio_tag.AudioTaggingMetrics.results_overall_metrics`: Calculate and return overall metrics (micro-averaged).
* :func:`sed_eval.audio_tag.AudioTaggingMetrics.results_class_wise_metrics`: Calculate and return class-wise metrics.
* :func:`sed_eval.audio_tag.AudioTaggingMetrics.results_class_wise_average_metrics`: Calculate and return class-wise average metrics (macro-averaged).

Function :func:`sed_eval.audio_tag.AudioTaggingMetrics.evaluate` takes as a parameter tag lists,
(use :func:`dcase_util.containers.MetaDataContainer` to read them from a file), and probability lists (use :func:`dcase_util.containers.ProbabilityContainer` to read them from a file).

Usage example:

.. code-block:: python
    :linenos:

    import sed_eval
    import dcase_util

    reference_tag_list = dcase_util.containers.MetaDataContainer([
        {
            'filename': 'test1.wav',
            'tags': 'cat,dog'
        },
        {
            'filename': 'test2.wav',
            'tags': 'dog'
        },
        {
            'filename': 'test3.wav',
            'tags': 'bird,cat'
        },
        {
            'filename': 'test4.wav',
            'tags': 'cat'
        },
        {
            'filename': 'test5.wav',
            'tags': 'bird,speech'
        },
        {
            'filename': 'test6.wav',
            'tags': 'dog,speech'
        },
        {
            'filename': 'test7.wav',
            'tags': 'speech'
        },
    ])

    estimated_tag_probabilities = dcase_util.containers.ProbabilityContainer([
        {
            'filename': 'test1.wav',
            'label': 'bird',
            'probability': 0.2
        },
        {
            'filename': 'test1.wav',
            'label': 'cat',
            'probability': 0.99
        },
        {
            'filename': 'test1.wav',
            'label': 'dog',
            'probability': 0.88
        },
        {
            'filename': 'test1.wav',
            'label': 'speech',
            'probability': 0.01
        },

        {
            'filename': 'test2.wav',
            'label': 'bird',
            'probability': 0.1
        },
        {
            'filename': 'test2.wav',
            'label': 'cat',
            'probability': 0.3
        },
        {
            'filename': 'test2.wav',
            'label': 'dog',
            'probability': 0.8
        },
        {
            'filename': 'test2.wav',
            'label': 'speech',
            'probability': 0.1
        },


        {
            'filename': 'test3.wav',
            'label': 'bird',
            'probability': 0.7
        },
        {
            'filename': 'test3.wav',
            'label': 'cat',
            'probability': 0.6
        },
        {
            'filename': 'test3.wav',
            'label': 'dog',
            'probability': 0.4
        },
        {
            'filename': 'test3.wav',
            'label': 'speech',
            'probability': 0.3
        },

        {
            'filename': 'test4.wav',
            'label': 'bird',
            'probability': 0.323
        },
        {
            'filename': 'test4.wav',
            'label': 'cat',
            'probability': 0.6
        },
        {
            'filename': 'test4.wav',
            'label': 'dog',
            'probability': 0.56
        },
        {
            'filename': 'test4.wav',
            'label': 'speech',
            'probability': 0.4
        },


        {
            'filename': 'test5.wav',
            'label': 'bird',
            'probability': 0.8
        },
        {
            'filename': 'test5.wav',
            'label': 'cat',
            'probability': 0.7
        },
        {
            'filename': 'test5.wav',
            'label': 'dog',
            'probability': 0.45
        },
        {
            'filename': 'test5.wav',
            'label': 'speech',
            'probability': 0.43
        },


        {
            'filename': 'test6.wav',
            'label': 'bird',
            'probability': 0.9
        },
        {
            'filename': 'test6.wav',
            'label': 'cat',
            'probability': 0.53
        },
        {
            'filename': 'test6.wav',
            'label': 'dog',
            'probability': 0.83
        },
        {
            'filename': 'test6.wav',
            'label': 'speech',
            'probability': 0.95
        },


        {
            'filename': 'test7.wav',
            'label': 'bird',
            'probability': 0.2
        },
        {
            'filename': 'test7.wav',
            'label': 'cat',
            'probability': 0.2
        },
        {
            'filename': 'test7.wav',
            'label': 'dog',
            'probability': 0.89
        },
        {
            'filename': 'test7.wav',
            'label': 'speech',
            'probability': 0.45
        },
    ])

    estimated_tag_list = dcase_util.containers.MetaDataContainer()
    for file in estimated_tag_probabilities.unique_files:
        k = estimated_tag_probabilities.filter(filename=file)
        tags = []
        for item in k:
            if item.probability > 0.5:
                tags.append(item.label)

        estimated_tag_list.append(
            {
                'filename': file,
                'tags': tags
            }
        )

    tag_evaluator = sed_eval.audio_tag.AudioTaggingMetrics(
        tags=reference_tag_list.unique_tags
    )

    tag_evaluator.evaluate(
        reference_tag_list=reference_tag_list,
        estimated_tag_list=estimated_tag_list,
        estimated_tag_probabilities=estimated_tag_probabilities
    )
    print(tag_evaluator)


.. autosummary::
    :toctree: generated/

    AudioTaggingMetrics
    AudioTaggingMetrics.evaluate
    AudioTaggingMetrics.results
    AudioTaggingMetrics.results_overall_metrics
    AudioTaggingMetrics.results_class_wise_metrics
    AudioTaggingMetrics.results_class_wise_average_metrics
    AudioTaggingMetrics.result_report_parameters
    AudioTaggingMetrics.result_report_class_wise
    AudioTaggingMetrics.result_report_class_wise_average
    AudioTaggingMetrics.reset

"""

from __future__ import absolute_import
import numpy
from . import metric
import dcase_util



[docs]class AudioTaggingMetrics:

[docs]    def __init__(self, tags=None):
        self.tag_label_list = tags

        self.overall = {
            'Ncorr': 0.0,
            'Nref': 0.0,
            'Nsys': 0.0,
            'Ntp': 0.0,
            'Ntn': 0.0,
            'Nfp': 0.0,
            'Nfn': 0.0,
        }

        self.tag_wise = {}
        self.y_true = {}
        self.y_pred = {}
        self.y_pred_score = {}

        for label in self.tag_label_list:
            self.tag_wise[label] = {
                'Nref': 0.0,
                'Nsys': 0.0,
                'Ntp': 0.0,
                'Ntn': 0.0,
                'Nfp': 0.0,
                'Nfn': 0.0,
            }
            self.y_true[label] = []
            self.y_pred[label] = []
            self.y_pred_score[label] = []

        self.ui = dcase_util.ui.FancyStringifier()


    def __str__(self):
        """Print result reports"""

        output = self.ui.section_header('Audio tagging metrics') + '\n'
        output += self.result_report_parameters() + '\n'
        output += self.result_report_overall() + '\n'
        output += self.result_report_class_wise_average() + '\n'
        output += self.result_report_class_wise() + '\n'

        return output


[docs]    def evaluate(self, reference_tag_list, estimated_tag_list=None, estimated_tag_probabilities=None):
        """Evaluate estimated against reference

        Parameters
        ----------

        reference_tag_list : list of dict or dcase_util.containers.MetaDataContainer
            Reference tag list

        estimated_tag_list : list of dict or dcase_util.containers.MetaDataContainer
            Estimated tag list

        estimated_tag_probabilities : list of dict or dcase_util.containers.ProbabilityContainer
            Estimated tag probabilities

        Returns
        -------
        self

        """

        if estimated_tag_list is None and estimated_tag_probabilities is None:
            raise ValueError("Nothing to evaluate, give at least estimated_tag_list or estimated_tag_probabilities")

        # Make sure reference_tag_list is dcase_util.containers.MetaDataContainer
        if not isinstance(reference_tag_list, dcase_util.containers.MetaDataContainer):
            reference_tag_list = dcase_util.containers.MetaDataContainer(reference_tag_list)

        # Make sure estimated_tag_list is dcase_util.containers.MetaDataContainer
        if estimated_tag_list is not None:
            if not isinstance(estimated_tag_list, dcase_util.containers.MetaDataContainer):
                estimated_tag_list = dcase_util.containers.MetaDataContainer(estimated_tag_list)

        # Make sure estimated_tag_probabilities is dcase_util.containers.ProbabilityContainer
        if estimated_tag_probabilities is not None:
            if not isinstance(estimated_tag_probabilities, dcase_util.containers.ProbabilityContainer):
                estimated_tag_probabilities = dcase_util.containers.ProbabilityContainer(estimated_tag_probabilities)

        y_true = []
        y_pred = []

        # Go though reference and estimated list label by label, and file by file
        for label in self.tag_label_list:
            for filename in reference_tag_list.unique_files:
                reference_item = reference_tag_list.filter(filename=filename)[0]

                # Populate y_true based on reference_item
                if label in reference_item.tags:
                    self.y_true[label].append(1)
                    y_true.append(1)

                else:
                    self.y_true[label].append(0)
                    y_true.append(0)

                if estimated_tag_list is not None:
                    # Evaluate based on estimated tags

                    estimated_item = estimated_tag_list.filter(filename=filename)[0]

                    if not estimated_item:
                        raise ValueError(
                            "Not all reference files estimated, please check [{file}]".format(
                                file=filename
                            )
                        )
                    # Store nref
                    if label in reference_item.tags:
                        self.tag_wise[label]['Nref'] += 1

                    # Populate y_pred based estimated_item
                    if label in estimated_item.tags:
                        self.y_pred[label].append(1)
                        y_pred.append(1)
                        self.tag_wise[label]['Nsys'] += 1

                    else:
                        self.y_pred[label].append(0)
                        y_pred.append(0)

                    # Accumulate intermediate values
                    # True positives (TP)
                    if label in reference_item.tags and label in estimated_item.tags:
                        self.tag_wise[label]['Ntp'] += 1

                    # True negatives (TN)
                    if label not in reference_item.tags and label not in estimated_item.tags:
                        self.tag_wise[label]['Ntn'] += 1

                    # False positives (FP)
                    if label not in reference_item.tags and label in estimated_item.tags:
                        self.tag_wise[label]['Nfp'] += 1

                    # False negatives (FN)
                    if label in reference_item.tags and label not in estimated_item.tags:
                        self.tag_wise[label]['Nfn'] += 1

                if estimated_tag_probabilities is not None:
                    # Evaluate based on per tag probabilities

                    estimated_item = estimated_tag_probabilities.filter(filename=filename, label=label)[0]
                    self.y_pred_score[label].append(float(estimated_item['probability']))

        if estimated_tag_list is not None:
            # Evaluate based on estimated tags

            self.overall['Nref'] += sum(y_true)
            self.overall['Nsys'] += sum(y_pred)

            y_true = numpy.array(y_true)
            y_pred = numpy.array(y_pred)

            self.overall['Ntp'] += sum(y_pred + y_true > 1)
            self.overall['Ntn'] += sum(y_pred + y_true == 0)
            self.overall['Nfp'] += sum(y_pred - y_true > 0)
            self.overall['Nfn'] += sum(y_true - y_pred > 0)

        return self



[docs]    def reset(self):
        """Reset internal state
        """
        self.overall = {
            'Nref': 0.0,
            'Nsys': 0.0,
            'Ntp': 0.0,
            'Ntn': 0.0,
            'Nfp': 0.0,
            'Nfn': 0.0,
        }

        self.tag_wise = {}
        for label in self.tag_label_list:
            self.tag_wise[label] = {
                'Nref': 0.0,
                'Nsys': 0.0,
                'Ntp': 0.0,
                'Ntn': 0.0,
                'Nfp': 0.0,
                'Nfn': 0.0,
            }

        return self


    # Results

[docs]    def results(self):
        """All metrics

        Returns
        -------
        dict
            results in a dictionary format

        """

        results = {
            'overall': self.results_overall_metrics(),
            'class_wise': self.results_class_wise_metrics(),
            'class_wise_average': self.results_class_wise_average_metrics(),
        }

        return results



[docs]    def results_overall_metrics(self):
        """Overall metrics

        Returns
        -------
        dict
            results in a dictionary format

        """

        # F-measure
        if self.overall['Nref'] > 0:
            precision = metric.precision(
                Ntp=self.overall['Ntp'],
                Nsys=self.overall['Nsys']
            )

            recall = metric.recall(
                Ntp=self.overall['Ntp'],
                Nref=self.overall['Nref']
            )

            f_measure = metric.f_measure(
                precision=precision,
                recall=recall
            )

        else:
            precision = None
            recall = None
            f_measure = None

        y_true = []
        y_score = []
        for tag_id, tag_label in enumerate(self.tag_label_list):
            y_true += self.y_true[tag_label]
            y_score += self.y_pred_score[tag_label]

        if y_score:
            eer = metric.equal_error_rate(
                y_true=y_true,
                y_score=y_score
            )

        else:
            eer = None

        return {
                'count': {
                    'Nref': self.overall['Nref'],
                    'Nsys': self.overall['Nsys'],
                },
                'f_measure': {
                    'f_measure': f_measure,
                    'precision': precision,
                    'recall': recall,
                },
                'eer': {
                    'eer': eer
                }

        }



[docs]    def results_class_wise_metrics(self):
        """Class-wise metrics

        Returns
        -------
        dict
            results in a dictionary format

        """

        results = {}
        for tag_id, tag_label in enumerate(self.tag_label_list):
            if tag_label not in results:
                results[tag_label] = {}

            # Counts
            results[tag_label]['count'] = {
                'Nref': self.tag_wise[tag_label]['Nref'],
                'Nsys': self.tag_wise[tag_label]['Nsys'],
                'Ntp': self.tag_wise[tag_label]['Ntp'],
                'Ntn': self.tag_wise[tag_label]['Ntn'],
                'Nfp': self.tag_wise[tag_label]['Nfp'],
                'Nfn': self.tag_wise[tag_label]['Nfn'],
            }

            # Equal error rate
            if self.y_pred_score[tag_label]:
                results[tag_label]['eer'] = {
                    'eer': metric.equal_error_rate(
                        y_true=self.y_true[tag_label],
                        y_score=self.y_pred_score[tag_label]
                    )
                }

            else:
                results[tag_label]['eer'] = {
                    'eer': None
                }

            # F-measure
            if self.tag_wise[tag_label]['Nref'] > 0:

                precision = metric.precision(
                    Ntp=self.tag_wise[tag_label]['Ntp'],
                    Nsys=self.tag_wise[tag_label]['Nsys']
                )

                recall = metric.recall(
                    Ntp=self.tag_wise[tag_label]['Ntp'],
                    Nref=self.tag_wise[tag_label]['Nref']
                )

                f_measure = metric.f_measure(precision=precision, recall=recall)

            else:
                precision = None
                recall = None
                f_measure = None

            results[tag_label]['f_measure'] = {
                'f_measure': f_measure,
                'precision': precision,
                'recall': recall,
            }

        return results



[docs]    def results_class_wise_average_metrics(self):
        """Class-wise averaged metrics

        Returns
        -------
        dict
            results in a dictionary format

        """

        class_wise_results = self.results_class_wise_metrics()

        class_wise_eer = []
        class_wise_fmeasure = []
        class_wise_precision = []
        class_wise_recall = []

        for class_label in class_wise_results:
            if class_wise_results[class_label]['eer']['eer'] is not None:
                class_wise_eer.append(class_wise_results[class_label]['eer']['eer'])

            if class_wise_results[class_label]['f_measure']['f_measure'] is not None:
                class_wise_fmeasure.append(class_wise_results[class_label]['f_measure']['f_measure'])
                class_wise_precision.append(class_wise_results[class_label]['f_measure']['precision'])
                class_wise_recall.append(class_wise_results[class_label]['f_measure']['recall'])

        if class_wise_eer:
            eer = float(numpy.nanmean(class_wise_eer))

        else:
            eer = None

        if class_wise_fmeasure:
            f_measure = float(numpy.nanmean(class_wise_fmeasure))

        else:
            f_measure = None

        if class_wise_precision:
            precision = float(numpy.nanmean(class_wise_precision))

        else:
            precision = None

        if class_wise_recall:
            recall = float(numpy.nanmean(class_wise_recall))

        else:
            recall = None

        return {
            'eer': {
                'eer': eer
            },
            'f_measure': {
                'f_measure': f_measure,
                'precision': precision,
                'recall': recall,
            }
        }


    # Reports

[docs]    def result_report_parameters(self):
        """Report metric parameters

        Returns
        -------
        str
            result report in string format

        """

        output = self.ui.data(field='Tags', value=len(self.tag_label_list)) + '\n'
        output += self.ui.data(field='Evaluated units', value=int(self.overall['Nref'])) + '\n'

        return output


    def result_report_overall(self):
        """Report overall results

        Returns
        -------
        str
            result report in string format

        """

        results = self.results_overall_metrics()

        output = self.ui.section_header('Overall metrics (micro-average)', indent=2) + '\n'

        if 'f_measure' in results and results['f_measure']:

            if results['f_measure']['f_measure'] is not None:
                f_measure = results['f_measure']['f_measure'] * 100

            else:
                f_measure = None

            if results['f_measure']['precision'] is not None:
                precision = results['f_measure']['precision'] * 100

            else:
                precision = None

            if results['f_measure']['recall'] is not None:
                recall = results['f_measure']['recall'] * 100

            else:
                recall = None

            output += self.ui.line('F-measure', indent=2) + '\n'
            output += self.ui.data(field='F-measure (F1)', value=f_measure, unit='%', indent=4) + '\n'
            output += self.ui.data(field='Precision', value=precision, unit='%', indent=4) + '\n'
            output += self.ui.data(field='Recall', value=recall, unit='%', indent=4) + '\n'

        if 'eer' in results and results['eer']:
            if results['eer']['eer'] is not None:
                eer = results['eer']['eer'] * 100

            else:
                eer = None

            output += self.ui.line('Equal error rate', indent=2) + '\n'
            output += self.ui.data(field='Equal error rate (EER)', value=eer, unit='%', indent=4) + '\n'

        return output


[docs]    def result_report_class_wise_average(self):
        """Report class-wise averages

        Returns
        -------
        str
            result report in string format

        """

        results = self.results_class_wise_average_metrics()

        output = self.ui.section_header('Class-wise average metrics (macro-average)', indent=2) + '\n'

        if 'f_measure' in results and results['f_measure']:
            if results['f_measure']['f_measure'] is not None:
                f_measure = results['f_measure']['f_measure'] * 100

            else:
                f_measure = None

            if results['f_measure']['precision'] is not None:
                precision = results['f_measure']['precision'] * 100

            else:
                precision = None

            if results['f_measure']['recall'] is not None:
                recall = results['f_measure']['recall'] * 100

            else:
                recall = None

            output += self.ui.line('F-measure', indent=2) + '\n'
            output += self.ui.data(field='F-measure (F1)', value=f_measure, unit='%', indent=4) + '\n'
            output += self.ui.data(field='Precision', value=precision, unit='%', indent=4) + '\n'
            output += self.ui.data(field='Recall', value=recall, unit='%', indent=4) + '\n'

        if 'eer' in results and results['eer']:
            if results['eer']['eer'] is not None:
                eer = results['eer']['eer'] * 100

            else:
                eer = None

            output += self.ui.line('Equal error rate', indent=2) + '\n'
            output += self.ui.data(field='Equal error rate (EER)', value=eer, unit='%', indent=4) + '\n'

        return output



[docs]    def result_report_class_wise(self):
        """Report class-wise results

        Returns
        -------
        str
            result report in string format

        """

        results = self.results_class_wise_metrics()

        output = self.ui.section_header('Class-wise metrics', indent=2) + '\n'
        output += self.ui.row(
            'Tag', 'Nref', 'Nsys', 'F-score', 'Pre', 'Rec', 'EER',
            widths=[20, 12, 12, 12, 12, 12, 12],
            separators=[True, False, True, False, False, True, False],
            indent=4
        ) + '\n'
        output += self.ui.row('-', '-', '-', '-', '-', '-', '-') + '\n'

        for tag_label in self.tag_label_list:
            if results[tag_label]['f_measure']['f_measure'] is not None:
                f_measure = results[tag_label]['f_measure']['f_measure'] * 100

            else:
                f_measure = None

            if results[tag_label]['f_measure']['precision'] is not None:
                precision = results[tag_label]['f_measure']['precision'] * 100

            else:
                precision = None

            if results[tag_label]['f_measure']['recall'] is not None:
                recall = results[tag_label]['f_measure']['recall'] * 100

            else:
                recall = None

            if results[tag_label]['eer']['eer'] is not None:
                eer = results[tag_label]['eer']['eer'] * 100

            else:
                eer = None

            output += self.ui.row(
                tag_label,
                results[tag_label]['count']['Nref'],
                results[tag_label]['count']['Nsys'],
                f_measure,
                precision,
                recall,
                eer,
                types=['str', 'int', 'int', 'float1_percentage', 'float1', 'float1', 'float1_percentage']
            ) + '\n'

        return output

© Copyright 2018, Toni Heittola et al..

================================================ FILE: docs/_modules/sed_eval/io.html ================================================ sed_eval.io — sed_eval 0.1 documentation

Source code for sed_eval.io

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
I/O
==================
Functions for loading annotations from files in various formats.

.. autosummary::
    :toctree: generated/

    load_event_list
    load_scene_list
    load_file_pair_list

"""

from __future__ import absolute_import
import csv
import dcase_util



[docs]def load_event_list(filename, **kwargs):
    """Load event list from csv formatted text-file

    Supported formats (see more `dcase_util.containers.MetaDataContainer.load()` method):

    - [event onset (float >= 0)][delimiter][event offset (float >= 0)]
    - [event onset (float >= 0)][delimiter][event offset (float >= 0)][delimiter][label]
    - [filename][delimiter][event onset (float >= 0)][delimiter][event offset (float >= 0)][delimiter][event label]
    - [filename][delimiter][scene_label][delimiter][event onset (float >= 0)][delimiter][event offset (float >= 0)][delimiter][event label]
    - [filename]

    Supported delimiters: ``,``, ``;``, ``tab``

    Example of event list file::

        21.64715	23.00552	alert
        36.91184	38.27021	alert
        69.72575	71.09029	alert
        63.53990	64.89827	alert
        84.25553	84.83920	alert
        20.92974	21.82661	clearthroat
        28.39992	29.29679	clearthroat
        80.47837	81.95937	clearthroat
        44.48363	45.96463	clearthroat
        78.13073	79.05953	clearthroat
        15.17031	16.27235	cough
        20.54931	21.65135	cough
        27.79964	28.90168	cough
        75.45959	76.32490	cough
        70.81708	71.91912	cough
        21.23203	22.55902	doorslam
        7.546220	9.014880	doorslam
        34.11303	35.04183	doorslam
        45.86001	47.32867	doorslam


    Parameters
    ----------
    filename : str
        Path to the csv-file

    Returns
    -------
    list of dict
        Event list

    """

    return dcase_util.containers.MetaDataContainer().load(filename=filename, **kwargs)




[docs]def load_scene_list(filename, **kwargs):
    """Load scene list from csv formatted text-file

    Supported formats (see more `dcase_util.containers.MetaDataContainer.load()` method):

    - [filename][delimiter][scene label]
    - [filename][delimiter][segment start (float >= 0)][delimiter][segment stop (float >= 0)][delimiter][scene label]

    Supported delimiters: ``,``, ``;``, ``tab``

    Example of scene list file::

        scenes_stereo/supermarket09.wav	supermarket
        scenes_stereo/tubestation10.wav	tubestation
        scenes_stereo/quietstreet08.wav	quietstreet
        scenes_stereo/restaurant05.wav	restaurant
        scenes_stereo/busystreet05.wav	busystreet
        scenes_stereo/openairmarket04.wav	openairmarket
        scenes_stereo/quietstreet01.wav	quietstreet
        scenes_stereo/supermarket05.wav	supermarket
        scenes_stereo/openairmarket01.wav	openairmarket

    Parameters
    ----------
    filename : str
        Path to the csv-file

    Returns
    -------
    list of dict
        Scene list

    """

    return dcase_util.containers.MetaDataContainer().load(filename=filename, **kwargs)




[docs]def load_file_pair_list(filename):
    """Load file pair list csv formatted text-file

    Format is [reference_file][delimiter][estimated_file]

    Supported delimiters: ``,``, ``;``, ``tab``

    Example of file-list::

        office_snr0_high_v2.txt	office_snr0_high_v2_detected.txt
        office_snr0_med_v2.txt	office_snr0_med_v2_detected.txt

    Parameters
    ----------
    filename : str
        Path to the csv-file

    Returns
    -------
    file_list: list
        File pair dicts in a list

    """

    data = []

    input_file = open(filename, 'rt')

    try:
        dialect = csv.Sniffer().sniff(input_file.readline(), [',', ';', '\t'])

    except csv.Error:
        raise ValueError('Unknown delimiter in file [{file}].'.format(file=filename))

    input_file.seek(0)  

    for row in csv.reader(input_file, dialect):
        if len(row) == 2:
            data.append(
                {
                    'reference_file': row[0],
                    'estimated_file': row[1]
                }
            )

        else:
            raise ValueError('Unknown file pair list format in file [{file}].'.format(file=filename))

    input_file.close()

    return data

© Copyright 2018, Toni Heittola et al..

================================================ FILE: docs/_modules/sed_eval/metric.html ================================================ sed_eval.metric — sed_eval 0.1 documentation

Source code for sed_eval.metric

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""

.. autosummary::
    :toctree: generated/

    f_measure
    precision
    recall
    
    accuracy
    accuracy2
    balanced_accuracy
    sensitivity
    specificity

    error_rate
    substitution_rate
    deletion_rate
    insertion_rate

    equal_error_rate

"""

import numpy


# -- F-measure -- #

[docs]def precision(Ntp, Nsys, eps=numpy.spacing(1)):
    """Precision.

    Wikipedia entry https://en.wikipedia.org/wiki/Precision_and_recall

    Parameters
    ----------
    Ntp : int >=0
        Number of true positives.

    Nsys : int >=0
        Amount of system output.

    eps : float
        eps.
        Default value numpy.spacing(1)

    Returns
    -------
    precision: float
        Precision

    """

    if Nsys == 0:
        return numpy.nan
    else:
        return float(Ntp / float(Nsys))




[docs]def recall(Ntp, Nref, eps=numpy.spacing(1)):
    """Recall.

    Wikipedia entry https://en.wikipedia.org/wiki/Precision_and_recall

    Parameters
    ----------
    Ntp : int >=0
        Number of true positives.

    Nref : int >=0
        Amount of reference.

    eps : float
        eps.
        Default value numpy.spacing(1)

    Returns
    -------
    recall: float
        Recall

    """

    if Nref == 0:
        return numpy.nan
    else:
        return float(Ntp / float(Nref))




[docs]def f_measure(precision, recall, beta=1.0):
    """F-measure from precision and recall scores.

    Wikipedia entry https://en.wikipedia.org/wiki/F1_score

    Parameters
    ----------
    precision : float in (0, 1]
        Precision.

    recall : float in (0, 1]
        Recall.

    beta : float > 0
        Weighting factor for f-measure.
        Default value 1.0

    Returns
    -------
    f_measure: float
        The weighted f-measure

    """

    if precision == 0 and recall == 0:
        return 0.0

    return float((1 + beta**2)*precision*recall/((beta**2)*precision + recall))



# -- Accuracy -- #

[docs]def sensitivity(Ntp, Nfn, eps=numpy.spacing(1)):
    """Sensitivity

    Wikipedia entry https://en.wikipedia.org/wiki/Sensitivity_and_specificity

    Parameters
    ----------
    Ntp : int >=0
        Number of true positives.

    Nfn : int >=0
        Number of false negatives.

    eps : float
        eps.
        Default value numpy.spacing(1)

    Returns
    -------
    sensitivity: float
        Sensitivity

    """

    return float(Ntp / (Ntp + Nfn + eps))




[docs]def specificity(Ntn, Nfp, eps=numpy.spacing(1)):
    """Specificity

    Wikipedia entry https://en.wikipedia.org/wiki/Sensitivity_and_specificity

    Parameters
    ----------
    Ntn : int >= 0
        Number of true negatives.

    Nfp : int >= 0
        Number of false positives.
    
    eps : float
        eps.
        Default value numpy.spacing(1)

    Returns
    -------
    specificity: float
        Specificity

    """

    return float(Ntn / (Ntn + Nfp + eps))




[docs]def balanced_accuracy(sensitivity, specificity, factor=0.5):
    """Balanced accuracy

    Wikipedia entry https://en.wikipedia.org/wiki/Accuracy_and_precision

    Parameters
    ----------
    sensitivity : float in [0, 1]
        sensitivity.

    specificity : float in [0, 1]
        specificity.

    factor : float in [0, 1]
        Balancing factor multiplying true positive rate (sensitivity).
        Default value 0.5

    Returns
    -------
    bacc: float
        Balanced accuracy

    """

    return float(((1-factor) * sensitivity) + (factor * specificity))




[docs]def accuracy(Ntp, Ntn, Nfp, Nfn, eps=numpy.spacing(1)):
    """Accuracy

    Parameters
    ----------
    Ntp : int >= 0
        Number of true positives.

    Ntn : int >= 0
        Number of true negatives.

    Nfp : int >= 0
        Number of false positives.

    Nfn : int >= 0
        Number of false negatives.

    eps : float
        eps.
        Default value numpy.spacing(1)

    Returns
    -------
    acc: float
        Accuracy

    """

    return float((Ntp + Ntn) / (Ntp + Ntn + Nfn + Nfp + eps))



def accuracy_corr(Ncorr, N, eps=numpy.spacing(1)):
    """Accuracy

    Parameters
    ----------
    Ncorr : int >= 0
        Number of correct items.

    N : int >= 0
        Number of all items.

    eps : float
        eps.
        Default value numpy.spacing(1)

    Returns
    -------
    acc: float
        Accuracy.

    """

    return float((Ncorr) / (N + eps))



[docs]def accuracy2(Ntp, Nfp, Nfn, eps=numpy.spacing(1)):
    """Accuracy

    Implements metric described by [1]_.

    .. [1] Dixon, S.
           "On the computer recognition of solo piano music".
           Proceedings of Australasian Computer Music Conference, July 2000, Brisbane, Australia, pp. 31–37.
           http://www.eecs.qmul.ac.uk/~simond/pub/2000/acmc.pdf

    Parameters
    ----------
    Ntp : int >= 0
        Number of true positives.

    Nfp : int >= 0
        Number of false positives.

    Nfn : int >= 0
        Number of false negatives.

    eps : float
        eps.
        Default value numpy.spacing(1)

    Returns
    -------
    acc: float
        Accuracy

    """

    return float((Ntp) / (Ntp + Nfn + Nfp + eps))




[docs]def substitution_rate(Nref, Nsubstitutions, eps=numpy.spacing(1)):
    """Substitution rate

    Parameters
    ----------
    Nref : int >=0
        Number of entries in the reference.

    Nsubstitutions : int >=0
        Number of substitutions.

    eps : float
        eps.
        Default value numpy.spacing(1)

    Returns
    -------
    substitution_rate: float
        Substitution rate

    """

    return float(Nsubstitutions / (Nref + eps))




[docs]def deletion_rate(Nref, Ndeletions, eps=numpy.spacing(1)):
    """Deletion rate

    Parameters
    ----------
    Nref : int >=0
        Number of entries in the reference.

    Ndeletions : int >=0
        Number of deletions.

    eps : float
        eps.
        Default value numpy.spacing(1)

    Returns
    -------
    deletion_rate: float
        Deletion rate
        
    """

    return float(Ndeletions / (Nref + eps))




[docs]def insertion_rate(Nref, Ninsertions, eps=numpy.spacing(1)):
    """Insertion rate

    Parameters
    ----------
    Nref : int >=0
        Number of entries in the reference.

    Ninsertions : int >=0
        Number of insertions.

    eps : float
        eps.
        Default value numpy.spacing(1)

    Returns
    -------
    insertion_rate: float
        Insertion rate

    """

    return float(Ninsertions / (Nref + eps))




[docs]def error_rate(substitution_rate_value=0.0, deletion_rate_value=0.0, insertion_rate_value=0.0):
    """Error rate

    Parameters
    ----------
    substitution_rate_value : float >=0
        Substitution rate.
        Default value 0

    deletion_rate_value : float >=0
        Deletion rate.
        Default value 0

    insertion_rate_value : float >=0
        Insertion rate.
        Default value 0

    Returns
    -------
    error_rate: float
        Error rate

    """

    return float(substitution_rate_value + deletion_rate_value + insertion_rate_value)




[docs]def equal_error_rate(y_true, y_score, eps=numpy.spacing(1)):
    """Equal error rate (EER)

    EER is calculated from the curve of the false negative rate versus the false positive rate.
    Implementation is based on https://github.com/pafoster/dcase2016_task4/blob/master/evaluation_scripts/eer.py

    Parameters
    ----------
    y_true : numpy.array or list
        True binary labels in range {0, 1} or {-1, 1}.

    y_score : numpy.array or list
        Target scores, can either be probability estimates of the positive
        class or confidence values.

    eps : float
        Minimum difference considered equal.
        Default value numpy.spacing(1)

    Returns
    -------
    float

    """

    from sklearn import metrics

    if numpy.any(y_true):
        false_positive_rate, true_positive_rate, thresholds = metrics.roc_curve(
            y_true=y_true,
            y_score=y_score,
            drop_intermediate=True
        )

        points = [(0, 0)] + list(zip(false_positive_rate, true_positive_rate))
        for i, point in enumerate(points):
            if point[0] + eps >= 1 - point[1]:
                break

        point1 = points[i - 1]
        point2 = points[i]

        # Interpolate between point1 and point2
        if abs(point2[0] - point1[0]) < eps:
            eer = point1[0]

        else:
            m = (point2[1] - point1[1]) / (point2[0] - point1[0])
            o = point1[1] - m * point1[0]
            eer = (1 - o) / (1 + m)

    else:
        eer = numpy.nan

    return eer

© Copyright 2018, Toni Heittola et al..

================================================ FILE: docs/_modules/sed_eval/scene.html ================================================ sed_eval.scene — sed_eval 0.1 documentation

Source code for sed_eval.scene

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Metrics
-------

Main functions:

* :func:`sed_eval.scene.SceneClassificationMetrics.evaluate`: Calculate intermediate values for evaluation and accumulate them.
* :func:`sed_eval.scene.SceneClassificationMetrics.results`: Calculate and return all metrics.
* :func:`sed_eval.scene.SceneClassificationMetrics.results_overall_metrics`: Calculate and return overall metrics (micro-averaged).
* :func:`sed_eval.scene.SceneClassificationMetrics.results_class_wise_metrics`: Calculate and return class-wise metrics.
* :func:`sed_eval.scene.SceneClassificationMetrics.results_class_wise_average_metrics`: Calculate and return class-wise average metrics (macro-averaged).

Function :func:`sed_eval.scene.SceneClassificationMetrics.evaluate` takes as a parameter scene lists,
use :func:`sed_eval.io.load_scene_list` to read them from a file.

Usage example to evaluate files:

.. code-block:: python
    :linenos:
    import sed_eval
    import dcase_util

    file_list = [
        {'reference_file': 'fold1_reference.txt', 'estimated_file': 'fold1_estimated.txt'}
    ]

    data = []

    # Get used scene labels and load data in
    all_data = []
    for file_pair in file_list:
        reference_scene_list = sed_eval.io.load_scene_list(
            filename=file_pair['reference_file'],
            csv_header=False,
            file_format=dcase_util.utils.FileFormat.CSV,
            fields=['filename', 'scene_label']
        )
        estimated_scene_list = sed_eval.io.load_scene_list(
            filename=file_pair['estimated_file'],
            csv_header=False,
            file_format=dcase_util.utils.FileFormat.CSV,
            fields=['filename', 'onset', 'offset', 'scene_label']
        )

        data.append(
            {
                'reference_scene_list': reference_scene_list,
                'estimated_scene_list': estimated_scene_list
            }
        )

        all_data += reference_scene_list

    scene_labels = sed_eval.sound_event.util.unique_scene_labels(all_data)

    # Create metrics class
    scene_metrics = sed_eval.scene.SceneClassificationMetrics(
        scene_labels=scene_labels
    )
    for file_pair in data:
        scene_metrics.evaluate(
            reference_scene_list=file_pair['reference_scene_list'],
            estimated_scene_list=file_pair['estimated_scene_list']
        )

    # Get only certain metrics
    overall_metrics_results = scene_metrics.results_overall_metrics()
    print("Accuracy:", overall_metrics_results['accuracy'])

    # Or print all metrics as reports
    print(scene_metrics)

Usage example to evaluate results stored in variables:

.. code-block:: python
    :linenos:

    import sed_eval
    import dcase_util

    reference = dcase_util.containers.MetaDataContainer([
        {
            'scene_label': 'supermarket',
            'file': 'supermarket09.wav'
        },
        {
            'scene_label': 'tubestation',
            'file': 'tubestation10.wav'
        },
        {
            'scene_label': 'quietstreet',
            'file': 'quietstreet08.wav'
        },
        {
            'scene_label': 'office',
            'file': 'office10.wav'
        },
        {
            'scene_label': 'bus',
            'file': 'bus01.wav'
        },
    ])

    estimated = dcase_util.containers.MetaDataContainer([
        {
            'scene_label': 'supermarket',
            'file': 'supermarket09.wav'
        },
        {
            'scene_label': 'bus',
            'file': 'tubestation10.wav'
        },
        {
            'scene_label': 'quietstreet',
            'file': 'quietstreet08.wav'
        },
        {
            'scene_label': 'park',
            'file': 'office10.wav'
        },
        {
            'scene_label': 'car',
            'file': 'bus01.wav'
        },
    ])

    scene_labels = sed_eval.sound_event.util.unique_scene_labels(reference)

    scene_metrics = sed_eval.scene.SceneClassificationMetrics(scene_labels)
    scene_metrics.evaluate(
        reference_scene_list=reference,
        estimated_scene_list=estimated
    )

    print(scene_metrics)

.. autosummary::
    :toctree: generated/

    SceneClassificationMetrics
    SceneClassificationMetrics.evaluate
    SceneClassificationMetrics.results
    SceneClassificationMetrics.results_overall_metrics
    SceneClassificationMetrics.results_class_wise_metrics
    SceneClassificationMetrics.results_class_wise_average_metrics
    SceneClassificationMetrics.result_report_parameters
    SceneClassificationMetrics.result_report_class_wise
    SceneClassificationMetrics.result_report_class_wise_average
    SceneClassificationMetrics.reset

"""

from __future__ import absolute_import
import numpy
import dcase_util
from . import metric



[docs]class SceneClassificationMetrics:

[docs]    def __init__(self, scene_labels=None):
        self.accuracies_per_class = None
        self.scene_label_list = scene_labels

        self.overall = {
            'Ncorr': 0.0,
            'Nref': 0.0,
            'Nsys': 0.0
        }

        self.scene_wise = {}
        for label in self.scene_label_list:
            self.scene_wise[label] = {
                'Ncorr': 0.0,
                'Nref': 0.0,
                'Nsys': 0.0
            }

        self.ui = dcase_util.ui.FancyStringifier()


    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        return self.results()

    def __str__(self):
        """Print result reports"""

        output = self.ui.section_header('Scene classification metrics') + '\n'
        output += self.result_report_parameters() + '\n'
        output += self.result_report_class_wise_average() + '\n'
        output += self.result_report_class_wise() + '\n'

        return output


[docs]    def evaluate(self, reference_scene_list, estimated_scene_list=None, estimated_scene_probabilities=None):
        """Evaluate file pair (reference and estimated)

        Parameters
        ----------

        reference_scene_list : list of dict or dcase_util.containers.MetaDataContainer
            Reference scene list.
            Default value None

        estimated_scene_list : list of dict or dcase_util.containers.MetaDataContainer
            Estimated scene list.
            Default value None

        estimated_scene_probabilities : dcase_util.containers.ProbabilityContainer
            Estimated scene probabilities. Currently not used.
            Default value None

        Returns
        -------
        self

        """

        if estimated_scene_list is None and estimated_scene_probabilities is None:
            raise ValueError("Nothing to evaluate, give at least estimated_scene_list or estimated_scene_probabilities")

        # Make sure reference_scene_list is dcase_util.containers.MetaDataContainer
        if not isinstance(estimated_scene_list, dcase_util.containers.MetaDataContainer):
            reference_scene_list = dcase_util.containers.MetaDataContainer(reference_scene_list)

        # Make sure estimated_scene_list is dcase_util.containers.MetaDataContainer
        if not isinstance(estimated_scene_list, dcase_util.containers.MetaDataContainer):
            estimated_scene_list = dcase_util.containers.MetaDataContainer(estimated_scene_list)

        # Make sure estimated_tag_probabilities is dcase_util.containers.ProbabilityContainer
        if estimated_scene_probabilities is not None:
            if not isinstance(estimated_scene_probabilities, dcase_util.containers.ProbabilityContainer):
                estimated_scene_probabilities = dcase_util.containers.ProbabilityContainer(estimated_scene_probabilities)

        # Translate "file" field to "filename"
        for item in reference_scene_list:
            if 'filename' not in item and 'file' in item:
                item['filename'] = item['file']

        for item in estimated_scene_list:
            if 'filename' not in item and 'file' in item:
                item['filename'] = item['file']

        y_true = []
        y_pred = []

        for estimated_item in estimated_scene_list:
            reference_item_matched = {}
            for reference_item in reference_scene_list:
                if estimated_item['filename'] == reference_item['filename']:
                    reference_item_matched = reference_item
                    break

            if not reference_item_matched:
                raise ValueError(
                    "Cannot find reference_item for estimated item [{item}]".format(item=estimated_item['file'])
                )

            y_true.append(reference_item_matched['scene_label'])
            y_pred.append(estimated_item['scene_label'])

        y_true = numpy.array(y_true)
        y_pred = numpy.array(y_pred)

        Ncorr_overall = 0
        for scene_id, scene_label in enumerate(self.scene_label_list):
            true_id = numpy.where(y_true == scene_label)[0]
            pred_id = numpy.where(y_pred == scene_label)[0]

            Ncorr = 0
            for id in true_id:
                if id in pred_id:
                    Ncorr += 1

            Ncorr_overall += Ncorr
            self.scene_wise[scene_label]['Ncorr'] += Ncorr
            self.scene_wise[scene_label]['Nref'] += true_id.shape[0]
            self.scene_wise[scene_label]['Nsys'] += pred_id.shape[0]

        self.overall['Ncorr'] += Ncorr_overall
        self.overall['Nref'] += y_true.shape[0]
        self.overall['Nsys'] += y_pred.shape[0]

        return self



[docs]    def reset(self):
        """Reset internal state
        """
        self.overall = {
            'Ncorr': 0.0,
            'Nref': 0.0,
            'Nsys': 0.0
        }

        self.scene_wise = {}
        for label in self.scene_label_list:
            self.scene_wise[label] = {
                'Ncorr': 0.0,
                'Nref': 0.0,
                'Nsys': 0.0
            }


    # Reports

[docs]    def result_report_parameters(self):
        """Report metric parameters

        Returns
        -------
        str
            result report in string format

        """

        output = self.ui.data(field='Scene labels', value=len(self.scene_label_list)) + '\n'
        output += self.ui.data(field='Evaluated units', value=int(self.overall['Nref'])) + '\n'

        return output



[docs]    def result_report_class_wise(self):
        """Report class-wise results

        Returns
        -------
        str
            result report in string format

        """

        results = self.results_class_wise_metrics()

        output = self.ui.section_header('Class-wise metrics', indent=2) + '\n'

        output += self.ui.row(
            'Scene label', 'Ncorr', 'Nref', 'Accuracy',
            widths=[20, 12, 12, 12],
            separators=[True, False, True, False],
            indent=4
        ) + '\n'

        output += self.ui.row('-', '-', '-', '-') + '\n'

        for scene_label in self.scene_label_list:
            output += self.ui.row(
                scene_label,
                results[scene_label]['count']['Ncorr'],
                results[scene_label]['count']['Nref'],
                results[scene_label]['accuracy']['accuracy'] * 100,
                types=['str', 'int', 'int', 'float1_percentage']
            ) + '\n'

        return output



[docs]    def result_report_class_wise_average(self):
        """Report class-wise averages

        Returns
        -------
        str
            result report in string format

        """

        results = self.results_class_wise_average_metrics()

        output = self.ui.section_header('Class-wise average metrics (macro-average)', indent=2) + '\n'

        output += self.ui.line('Accuracy', indent=2) + '\n'
        output += self.ui.data(field='Accuracy',
                               value=float(results['accuracy']['accuracy']) * 100, unit='%', indent=4) + '\n'

        return output


    # Results

[docs]    def results(self):
        """All metrics

        Returns
        -------
        dict
            results in a dictionary format

        """

        results = {
            'overall': self.results_overall_metrics(),
            'class_wise': self.results_class_wise_metrics(),
            'class_wise_average': self.results_class_wise_average_metrics()
        }

        return results



[docs]    def results_overall_metrics(self):
        """Overall metrics

        Returns
        -------
        dict
            results in a dictionary format

        """

        if self.overall['Nsys'] != 0:
            accuracy = self.overall['Ncorr'] / float(self.overall['Nsys'])
        else:
            accuracy = None

        return {
            'count': self.overall,
            'accuracy': accuracy
        }



[docs]    def results_class_wise_metrics(self):
        """Class-wise metrics

        Returns
        -------
        dict
            results in a dictionary format

        """

        results = {}
        for scene_id, scene_label in enumerate(self.scene_label_list):
            if scene_label not in results:
                results[scene_label] = {}

            results[scene_label]['count'] = {}
            results[scene_label]['count']['Ncorr'] = self.scene_wise[scene_label]['Ncorr']
            results[scene_label]['count']['Nref'] = self.scene_wise[scene_label]['Nref']
            results[scene_label]['count']['Nsys'] = self.scene_wise[scene_label]['Nsys']
            results[scene_label]['accuracy'] = {
                'accuracy': metric.accuracy_corr(
                    Ncorr=self.scene_wise[scene_label]['Ncorr'],
                    N=self.scene_wise[scene_label]['Nref']
                )
            }

        return results



[docs]    def results_class_wise_average_metrics(self):
        """Class-wise averaged metrics

        Returns
        -------
        dict
            results in a dictionary format

        """

        scene_wise_results = self.results_class_wise_metrics()

        scene_wise_accuracy = []
        for scene_label in scene_wise_results:
            scene_wise_accuracy.append(scene_wise_results[scene_label]['accuracy']['accuracy'])

        return {
            'accuracy': {
                'accuracy': float(numpy.mean(scene_wise_accuracy))
            }
        }


    # Metrics
    def class_wise_accuracy(self, scene_label):
        """Class-wise accuracy

        Returns
        -------
        dict
            results in a dictionary format

        """

        if len(self.accuracies_per_class.shape) == 2:
            return {
                'accuracy': float(numpy.mean(self.accuracies_per_class[:, self.scene_label_list.index(scene_label)]))
            }

        else:
            return {
                'accuracy': float(numpy.mean(self.accuracies_per_class[self.scene_label_list.index(scene_label)]))
            }

© Copyright 2018, Toni Heittola et al..

================================================ FILE: docs/_modules/sed_eval/sound_event.html ================================================ sed_eval.sound_event — sed_eval 0.1 documentation

Source code for sed_eval.sound_event

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Segment-based metrics, main functions:

* :func:`sed_eval.sound_event.SegmentBasedMetrics.evaluate`: Calculate intermediate values for evaluation and accumulate them.
* :func:`sed_eval.sound_event.SegmentBasedMetrics.results`: Calculate and return all metrics.
* :func:`sed_eval.sound_event.SegmentBasedMetrics.results_overall_metrics`: Calculate and return overall metrics (micro-averaged).
* :func:`sed_eval.sound_event.SegmentBasedMetrics.results_class_wise_metrics`: Calculate and return class-wise metrics.
* :func:`sed_eval.sound_event.SegmentBasedMetrics.results_class_wise_average_metrics`: Calculate and return class-wise average metrics (macro-averaged).

Event-based metrics, main functions:

* :func:`sed_eval.sound_event.EventBasedMetrics.evaluate`: Calculate intermediate values for evaluation and accumulate them.
* :func:`sed_eval.sound_event.EventBasedMetrics.results`: Calculate and return all metrics.
* :func:`sed_eval.sound_event.EventBasedMetrics.results_overall_metrics`: Calculate and return overall metrics (micro-averaged).
* :func:`sed_eval.sound_event.EventBasedMetrics.results_class_wise_metrics`: Calculate and return class-wise metrics.
* :func:`sed_eval.sound_event.EventBasedMetrics.results_class_wise_average_metrics`: Calculate and return class-wise average metrics (macro-averaged).

Functions :func:`sed_eval.sound_event.SegmentBasedMetrics.evaluate` and :func:`sed_eval.sound_event.EventBasedMetrics.evaluate`
take as a parameter event lists, use :func:`sed_eval.io.load_event_list` to read them from a file.


Usage example when reading event lists from disk (you can run example in path ``tests/data/sound_event``):

.. code-block:: python
    :linenos:

    import sed_eval
    import dcase_util

    file_list = [
        {
         'reference_file': 'office_snr0_high_v2.txt',
         'estimated_file': 'office_snr0_high_v2_detected.txt'
        },
        {
         'reference_file': 'office_snr0_med_v2.txt',
         'estimated_file': 'office_snr0_med_v2_detected.txt'
        }
    ]

    data = []

    # Get used event labels
    all_data = dcase_util.containers.MetaDataContainer()
    for file_pair in file_list:
        reference_event_list = sed_eval.io.load_event_list(
            filename=file_pair['reference_file']
        )
        estimated_event_list = sed_eval.io.load_event_list(
            filename=file_pair['estimated_file']
        )

        data.append({'reference_event_list': reference_event_list,
                     'estimated_event_list': estimated_event_list})

        all_data += reference_event_list

    event_labels = all_data.unique_event_labels

    # Start evaluating

    # Create metrics classes, define parameters
    segment_based_metrics = sed_eval.sound_event.SegmentBasedMetrics(
        event_label_list=event_labels,
        time_resolution=1.0
    )

    event_based_metrics = sed_eval.sound_event.EventBasedMetrics(
        event_label_list=event_labels,
        t_collar=0.250
    )

    # Go through files
    for file_pair in data:
        segment_based_metrics.evaluate(
            reference_event_list=file_pair['reference_event_list'],
            estimated_event_list=file_pair['estimated_event_list']
        )

        event_based_metrics.evaluate(
            reference_event_list=file_pair['reference_event_list'],
            estimated_event_list=file_pair['estimated_event_list']
        )

    # Get only certain metrics
    overall_segment_based_metrics = segment_based_metrics.results_overall_metrics()
    print("Accuracy:", overall_segment_based_metrics['accuracy']['accuracy'])

    # Or print all metrics as reports
    print(segment_based_metrics)
    print(event_based_metrics)

Usage example to evaluate results stored in variables:

.. code-block:: python
    :linenos:

    import sed_eval
    import dcase_util

    reference_event_list = dcase_util.containers.MetaDataContainer(
        [
            {
                'event_label': 'car',
                'event_onset': 0.0,
                'event_offset': 2.5,
                'file': 'audio/street/b099.wav',
                'scene_label': 'street'
            },
            {
                'event_label': 'car',
                'event_onset': 2.8,
                'event_offset': 4.5,
                'file': 'audio/street/b099.wav',
                'scene_label': 'street'
            },
            {
                'event_label': 'car',
                'event_onset': 6.0,
                'event_offset': 10.0,
                'file': 'audio/street/b099.wav',
                'scene_label': 'street'
            }
        ]
    )

    estimated_event_list = dcase_util.containers.MetaDataContainer(
        [
            {
                'event_label': 'car',
                'event_onset': 1.0,
                'event_offset': 3.5,
                'file': 'audio/street/b099.wav',
                'scene_label': 'street'
            },
            {
                'event_label': 'car',
                'event_onset': 7.0,
                'event_offset': 8.0,
                'file': 'audio/street/b099.wav',
                'scene_label': 'street'
            }
        ]
    )

    segment_based_metrics = sed_eval.sound_event.SegmentBasedMetrics(
        event_label_list=reference_event_list.unique_event_labels,
        time_resolution=1.0
    )
    event_based_metrics = sed_eval.sound_event.EventBasedMetrics(
        event_label_list=reference_event_list.unique_event_labels,
        t_collar=0.250
    )

    for filename in reference_event_list.unique_files:
        reference_event_list_for_current_file = reference_event_list.filter(
            filename=filename
        )

        estimated_event_list_for_current_file = estimated_event_list.filter(
            filename=filename
        )

        segment_based_metrics.evaluate(
            reference_event_list=reference_event_list_for_current_file,
            estimated_event_list=estimated_event_list_for_current_file
        )

        event_based_metrics.evaluate(
            reference_event_list=reference_event_list_for_current_file,
            estimated_event_list=estimated_event_list_for_current_file
        )

    # Get only certain metrics
    overall_segment_based_metrics = segment_based_metrics.results_overall_metrics()
    print("Accuracy:", overall_segment_based_metrics['accuracy']['accuracy'])

    # Or print all metrics as reports
    print(segment_based_metrics)
    print(event_based_metrics)

Segment based metrics
^^^^^^^^^^^^^^^^^^^^^

.. autosummary::
    :toctree: generated/

    SegmentBasedMetrics
    SegmentBasedMetrics.evaluate
    SegmentBasedMetrics.results
    SegmentBasedMetrics.results_overall_metrics
    SegmentBasedMetrics.results_class_wise_metrics
    SegmentBasedMetrics.results_class_wise_average_metrics
    SegmentBasedMetrics.result_report_parameters
    SegmentBasedMetrics.result_report_overall
    SegmentBasedMetrics.result_report_class_wise
    SegmentBasedMetrics.result_report_class_wise_average
    SegmentBasedMetrics.reset

.. autoclass:: SegmentBasedMetrics
   :members:

Event based metrics
^^^^^^^^^^^^^^^^^^^

.. autosummary::
    :toctree: generated/

    EventBasedMetrics
    EventBasedMetrics.evaluate
    EventBasedMetrics.results
    EventBasedMetrics.results_overall_metrics
    EventBasedMetrics.results_class_wise_metrics
    EventBasedMetrics.results_class_wise_average_metrics
    EventBasedMetrics.result_report_parameters
    EventBasedMetrics.result_report_overall
    EventBasedMetrics.result_report_class_wise
    EventBasedMetrics.result_report_class_wise_average
    EventBasedMetrics.reset

.. autoclass:: EventBasedMetrics
   :members:
   :undoc-members:
   :inherited-members:

"""

from __future__ import absolute_import
import numpy
import math
import dcase_util
from . import metric
from . import util


class SoundEventMetrics(object):
    """Base class for sound event detection metrics.

    """
    def __init__(self,
                 empty_system_output_handling=None):
        """Constructor

        Parameters
        ----------

        empty_system_output_handling : str
            Controls how empty system output is handled, i.e. when Nsys = 0. Default behaviour is to show NaN when e.g.
            computing precision (Ntp / Nsys).
            Use 'zero_score' to force these score to zero.
            Default value None

        """

        self.event_label_list = []
        self.ui = dcase_util.ui.FancyStringifier()
        self.empty_system_output_handling = empty_system_output_handling

    # Reports
    def result_report_overall(self):
        """Report overall results

        Returns
        -------
        str
            result report in string format

        """

        results = self.results_overall_metrics()

        output = self.ui.section_header('Overall metrics (micro-average)', indent=2) + '\n'

        if results['f_measure']:
            output += self.ui.line('F-measure', indent=2) + '\n'
            output += self.ui.data(field='F-measure (F1)', value=float(results['f_measure']['f_measure']) * 100,
                                   unit='%', indent=4) + '\n'
            output += self.ui.data(field='Precision', value=float(results['f_measure']['precision']) * 100,
                                   unit='%', indent=4) + '\n'
            output += self.ui.data(field='Recall', value=float(results['f_measure']['recall']) * 100,
                                   unit='%', indent=4) + '\n'

        if results['error_rate']:
            output += self.ui.line('Error rate', indent=2) + '\n'
            output += self.ui.data(field='Error rate (ER)', value=float(results['error_rate']['error_rate']),
                                   indent=4) + '\n'
            output += self.ui.data(field='Substitution rate', value=float(results['error_rate']['substitution_rate']),
                                   indent=4) + '\n'
            output += self.ui.data(field='Deletion rate', value=float(results['error_rate']['deletion_rate']),
                                   indent=4) + '\n'
            output += self.ui.data(field='Insertion rate', value=float(results['error_rate']['insertion_rate']),
                                   indent=4) + '\n'

        if results['accuracy']:
            output += self.ui.line('Accuracy', indent=2) + '\n'
            output += self.ui.data(field='Sensitivity', value=float(results['accuracy']['sensitivity']*100),
                                   unit='%', indent=4) + '\n'
            output += self.ui.data(field='Specificity', value=float(results['accuracy']['specificity'] * 100),
                                   unit='%', indent=4) + '\n'
            output += self.ui.data(field='Balanced accuracy', value=float(results['accuracy']['balanced_accuracy'] * 100),
                                   unit='%', indent=4) + '\n'
            output += self.ui.data(field='Accuracy', value=float(results['accuracy']['accuracy'] * 100),
                                   unit='%', indent=4) + '\n'

        return output

    def result_report_class_wise_average(self):
        """Report class-wise averages

        Returns
        -------
        str
            result report in string format

        """

        results = self.results_class_wise_average_metrics()

        output = self.ui.section_header('Class-wise average metrics (macro-average)', indent=2) + '\n'

        if results['f_measure']:
            output += self.ui.line('F-measure', indent=2) + '\n'
            output += self.ui.data(field='F-measure (F1)', value=float(results['f_measure']['f_measure']) * 100,
                                   unit='%', indent=4) + '\n'
            output += self.ui.data(field='Precision', value=float(results['f_measure']['precision']) * 100,
                                   unit='%', indent=4) + '\n'
            output += self.ui.data(field='Recall', value=float(results['f_measure']['recall']) * 100,
                                   unit='%', indent=4) + '\n'

        if results['error_rate']:
            output += self.ui.line('Error rate', indent=2) + '\n'
            output += self.ui.data(field='Error rate (ER)', value=float(results['error_rate']['error_rate']),
                                   indent=4) + '\n'
            output += self.ui.data(field='Deletion rate', value=float(results['error_rate']['deletion_rate']),
                                   indent=4) + '\n'
            output += self.ui.data(field='Insertion rate', value=float(results['error_rate']['insertion_rate']),
                                   indent=4) + '\n'

        if results['accuracy']:
            output += self.ui.line('Accuracy', indent=2) + '\n'
            output += self.ui.data(field='Sensitivity', value=float(results['accuracy']['sensitivity']*100),
                                   unit='%', indent=4) + '\n'
            output += self.ui.data(field='Specificity', value=float(results['accuracy']['specificity'] * 100),
                                   unit='%', indent=4) + '\n'
            output += self.ui.data(field='Balanced accuracy', value=float(results['accuracy']['balanced_accuracy'] * 100),
                                   unit='%', indent=4) + '\n'
            output += self.ui.data(field='Accuracy', value=float(results['accuracy']['accuracy'] * 100),
                                   unit='%', indent=4) + '\n'

        output += "  \n"

        return output

    def result_report_class_wise(self):
        """Report class-wise results

        Returns
        -------
        str
            result report in string format

        """

        results = self.results_class_wise_metrics()

        accuracy_present = True
        for event_label in self.event_label_list:
            if 'accuracy' not in results[event_label]['accuracy']:
                accuracy_present = False

        output = self.ui.section_header('Class-wise metrics', indent=2) + '\n'

        headers = ['Event label', 'Nref', 'Nsys', 'F', 'Pre', 'Rec', 'ER', 'Del', 'Ins']
        sep = ['-', '-', '-', '-', '-', '-', '-', '-', '-']
        widths = [15, 8, 8, 9, 9, 9, 9, 9, 9]
        separators = [True, False, True, False, False, True, False, False, True]
        if accuracy_present:
            headers += ['Sens', 'Spec', 'Bacc', 'Acc']
            sep += ['-', '-', '-', '-']
            widths += [9, 9, 9, 9]
            separators += [False, False, False, False]

        output += self.ui.row(*headers, widths=widths, indent=4, separators=separators) + '\n'
        output += self.ui.row(*sep) + '\n'
        for event_label in self.event_label_list:
            data = [
                event_label,
                results[event_label]['count']['Nref'],
                results[event_label]['count']['Nsys'],
                results[event_label]['f_measure']['f_measure'] * 100,
                results[event_label]['f_measure']['precision'] * 100,
                results[event_label]['f_measure']['recall'] * 100,
                results[event_label]['error_rate']['error_rate'],
                results[event_label]['error_rate']['deletion_rate'],
                results[event_label]['error_rate']['insertion_rate']
            ]

            types = [
                'str15',
                'int',
                'int',
                'float1_percentage',
                'float1_percentage',
                'float1_percentage',
                'float2',
                'float2',
                'float2',
            ]

            if accuracy_present:
                data += [
                    results[event_label]['accuracy']['sensitivity'] * 100,
                    results[event_label]['accuracy']['specificity'] * 100,
                    results[event_label]['accuracy']['balanced_accuracy'] * 100,
                    results[event_label]['accuracy']['accuracy'] * 100
                ]

                types += [
                    'float1_percentage',
                    'float1_percentage',
                    'float1_percentage',
                    'float1_percentage',
                ]

            output += self.ui.row(*data, types=types) + '\n'

        return output

    # Metrics / overall
    def overall_f_measure(self):
        return {}

    def overall_error_rate(self):
        return {}

    def overall_accuracy(self, factor=0.5):
        return {}
    
    # Metrics / class-wise
    def class_wise_count(self, event_label):
        return {}
    
    def class_wise_f_measure(self, event_label):
        return {}

    def class_wise_error_rate(self, event_label):
        return {}

    def class_wise_accuracy(self, event_label):
        return {}

    # Results
    def results_overall_metrics(self):
        """Overall metrics

        Returns
        -------
        dict
            results in a dictionary format

        """

        return {
            'f_measure': self.overall_f_measure(),
            'error_rate': self.overall_error_rate(),
            'accuracy': self.overall_accuracy()
        }

    def results_class_wise_metrics(self):
        """Class-wise metrics

        Returns
        -------
        dict
            results in a dictionary format

        """

        results = {}
        for event_label in self.event_label_list:
            if event_label not in results:
                results[event_label] = {}

            results[event_label]['f_measure'] = self.class_wise_f_measure(event_label)
            results[event_label]['accuracy'] = self.class_wise_accuracy(event_label)
            results[event_label]['error_rate'] = self.class_wise_error_rate(event_label)
            results[event_label]['count'] = self.class_wise_count(event_label)

        return results

    def results_class_wise_average_metrics(self):
        """Class-wise averaged metrics

        Returns
        -------
        dict
            results in a dictionary format

        """

        event_wise_results = self.results_class_wise_metrics()

        event_wise_f_measure = []
        event_wise_precision = []
        event_wise_recall = []
        
        event_wise_error_rate = []
        event_wise_deletion_rate = []
        event_wise_insertion_rate = []
        
        event_wise_sensitivity = []
        event_wise_specificity = []
        event_wise_balanced_accuracy = []
        event_wise_accuracy = []

        for event_label in event_wise_results:
            # F-measure
            event_wise_f_measure.append(event_wise_results[event_label]['f_measure']['f_measure'])
            event_wise_precision.append(event_wise_results[event_label]['f_measure']['precision'])
            event_wise_recall.append(event_wise_results[event_label]['f_measure']['recall'])
            
            # Error rate
            event_wise_error_rate.append(event_wise_results[event_label]['error_rate']['error_rate'])
            event_wise_deletion_rate.append(event_wise_results[event_label]['error_rate']['deletion_rate'])
            event_wise_insertion_rate.append(event_wise_results[event_label]['error_rate']['insertion_rate'])

            # Accuracy
            if 'sensitivity' in event_wise_results[event_label]['accuracy']:
                event_wise_sensitivity.append(event_wise_results[event_label]['accuracy']['sensitivity'])

            if 'specificity' in event_wise_results[event_label]['accuracy']:
                event_wise_specificity.append(event_wise_results[event_label]['accuracy']['specificity'])

            if 'balanced_accuracy' in event_wise_results[event_label]['accuracy']:
                event_wise_balanced_accuracy.append(event_wise_results[event_label]['accuracy']['balanced_accuracy'])

            if 'accuracy' in event_wise_results[event_label]['accuracy']:
                event_wise_accuracy.append(event_wise_results[event_label]['accuracy']['accuracy'])

        if event_wise_f_measure:
            event_wise_f_measure_dict = {
                'f_measure': float(numpy.nanmean(event_wise_f_measure)),
                'precision': float(numpy.nanmean(event_wise_precision)),
                'recall': float(numpy.nanmean(event_wise_recall))
            }

        else:
            event_wise_f_measure_dict = {}

        if event_wise_error_rate:
            event_wise_error_rate_dict = {
                'error_rate': float(numpy.nanmean(event_wise_error_rate)),
                'deletion_rate': float(numpy.nanmean(event_wise_deletion_rate)),
                'insertion_rate': float(numpy.nanmean(event_wise_insertion_rate))
            }

        else:
            event_wise_error_rate_dict = {}

        if event_wise_accuracy:
            event_wise_accuracy_dict = {
                'sensitivity': float(numpy.nanmean(event_wise_sensitivity)),
                'specificity': float(numpy.nanmean(event_wise_specificity)),
                'balanced_accuracy': float(numpy.nanmean(event_wise_balanced_accuracy)),
                'accuracy': float(numpy.nanmean(event_wise_accuracy))
            }

        else:
            event_wise_accuracy_dict = {}

        return {
            'f_measure': event_wise_f_measure_dict,
            'error_rate': event_wise_error_rate_dict,
            'accuracy': event_wise_accuracy_dict
        }

    def results(self):
        """All metrics

        Returns
        -------
        dict
            results in a dictionary format

        """

        return {
            'overall': self.results_overall_metrics(),
            'class_wise': self.results_class_wise_metrics(),
            'class_wise_average': self.results_class_wise_average_metrics()
        }



[docs]class SegmentBasedMetrics(SoundEventMetrics):

[docs]    def __init__(self,
                 event_label_list,
                 time_resolution=1.0):
        """Constructor

        Parameters
        ----------
        event_label_list : list, numpy.array
            List of unique event labels

        time_resolution : float (0,]
            Segment size used in the evaluation, in seconds.
            Default value 1.0

        """

        SoundEventMetrics.__init__(self)

        if isinstance(event_label_list, numpy.ndarray) and len(event_label_list.shape) == 1:
            # We have numpy array, convert it to list
            event_label_list = event_label_list.tolist()

        if not isinstance(event_label_list, list):
            raise ValueError(
                "event_label_list needs to be list or numpy.array"
            )

        if not isinstance(time_resolution, float) or time_resolution <= 0.0:
            raise ValueError(
                "time_resolution needs to be float > 0"
            )

        self.event_label_list = event_label_list
        self.evaluated_length_seconds = 0.0
        self.evaluated_files = 0

        self.time_resolution = time_resolution

        self.overall = {
            'Ntp': 0.0,
            'Ntn': 0.0,
            'Nfp': 0.0,
            'Nfn': 0.0,
            'Nref': 0.0,
            'Nsys': 0.0,
            'ER': 0.0,
            'S': 0.0,
            'D': 0.0,
            'I': 0.0,
        }

        self.class_wise = {}
        for class_label in self.event_label_list:
            self.class_wise[class_label] = {
                'Ntp': 0.0,
                'Ntn': 0.0,
                'Nfp': 0.0,
                'Nfn': 0.0,
                'Nref': 0.0,
                'Nsys': 0.0,
            }


    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        return self.results()

    def __str__(self):
        """Print result reports"""

        output = self.ui.section_header('Segment based metrics') + '\n'
        output += self.result_report_parameters() + '\n'
        output += self.result_report_overall() + '\n'
        output += self.result_report_class_wise_average() + '\n'
        output += self.result_report_class_wise() + '\n'

        return output


[docs]    def evaluate(self, reference_event_list, estimated_event_list, evaluated_length_seconds=None):
        """Evaluate file pair (reference and estimated)

        Parameters
        ----------

        reference_event_list : list of dict or dcase_util.containers.MetaDataContainer
            Reference event list.

        estimated_event_list : list of dict or dcase_util.containers.MetaDataContainer
            Estimated event list.

        evaluated_length_seconds : float, optional
            Evaluated length. If none given, maximum offset is used.
            Default value None

        Returns
        -------
        self

        """

        # Make sure input is dcase_util.containers.MetaDataContainer
        if not isinstance(reference_event_list, dcase_util.containers.MetaDataContainer):
            reference_event_list = dcase_util.containers.MetaDataContainer(reference_event_list)

        if not isinstance(estimated_event_list, dcase_util.containers.MetaDataContainer):
            estimated_event_list = dcase_util.containers.MetaDataContainer(estimated_event_list)

        # Check that input event list have event only from one file
        reference_files = reference_event_list.unique_files
        if len(reference_files) > 1:
            raise ValueError(
                "reference_event_list contains events from multiple files. Evaluate only file by file."
            )

        estimated_files = estimated_event_list.unique_files
        if len(estimated_files) > 1:
            raise ValueError(
                "estimated_event_list contains events from multiple files. Evaluate only file by file."
            )

        # Evaluate only valid events
        valid_reference_event_list = dcase_util.containers.MetaDataContainer()
        for item in reference_event_list:
            if 'event_onset' in item and 'event_offset' in item and 'event_label' in item:
                valid_reference_event_list.append(item)

            elif 'onset' in item and 'offset' in item and 'event_label' in item:
                valid_reference_event_list.append(item)

        reference_event_list = valid_reference_event_list

        valid_estimated_event_list = dcase_util.containers.MetaDataContainer()
        for item in estimated_event_list:
            if 'event_onset' in item and 'event_offset' in item and 'event_label' in item:
                valid_estimated_event_list.append(item)

            elif 'onset' in item and 'offset' in item and 'event_label' in item:
                valid_estimated_event_list.append(item)

        estimated_event_list = valid_estimated_event_list

        # Convert event list into frame-based representation
        reference_event_roll = util.event_list_to_event_roll(
            source_event_list=reference_event_list,
            event_label_list=self.event_label_list,
            time_resolution=self.time_resolution
        )

        estimated_event_roll = util.event_list_to_event_roll(
            source_event_list=estimated_event_list,
            event_label_list=self.event_label_list,
            time_resolution=self.time_resolution
        )

        if evaluated_length_seconds is None:
            evaluated_length_seconds = max(reference_event_list.max_offset, estimated_event_list.max_offset)
            evaluated_length_segments = int(math.ceil(evaluated_length_seconds * 1 / float(self.time_resolution)))

        else:
            evaluated_length_segments = int(math.ceil(evaluated_length_seconds * 1 / float(self.time_resolution)))

        self.evaluated_length_seconds += evaluated_length_seconds
        self.evaluated_files += 1

        reference_event_roll, estimated_event_roll = util.match_event_roll_lengths(
            reference_event_roll,
            estimated_event_roll,
            evaluated_length_segments
        )

        # Compute segment-based overall metrics
        for segment_id in range(0, reference_event_roll.shape[0]):
            annotated_segment = reference_event_roll[segment_id, :]
            system_segment = estimated_event_roll[segment_id, :]

            Ntp = sum(system_segment + annotated_segment > 1)
            Ntn = sum(system_segment + annotated_segment == 0)
            Nfp = sum(system_segment - annotated_segment > 0)
            Nfn = sum(annotated_segment - system_segment > 0)

            Nref = sum(annotated_segment)
            Nsys = sum(system_segment)

            S = min(Nref, Nsys) - Ntp
            D = max(0, Nref - Nsys)
            I = max(0, Nsys - Nref)

            self.overall['Ntp'] += Ntp
            self.overall['Ntn'] += Ntn
            self.overall['Nfp'] += Nfp
            self.overall['Nfn'] += Nfn
            self.overall['Nref'] += Nref
            self.overall['Nsys'] += Nsys
            self.overall['S'] += S
            self.overall['D'] += D
            self.overall['I'] += I

        # Compute segment-based class-wise metrics
        for class_id, class_label in enumerate(self.event_label_list):
            annotated_segment = reference_event_roll[:, class_id]
            system_segment = estimated_event_roll[:, class_id]

            Ntp = sum(system_segment + annotated_segment > 1)
            Ntn = sum(system_segment + annotated_segment == 0)
            Nfp = sum(system_segment - annotated_segment > 0)
            Nfn = sum(annotated_segment - system_segment > 0)

            Nref = sum(annotated_segment)
            Nsys = sum(system_segment)

            self.class_wise[class_label]['Ntp'] += Ntp
            self.class_wise[class_label]['Ntn'] += Ntn
            self.class_wise[class_label]['Nfp'] += Nfp
            self.class_wise[class_label]['Nfn'] += Nfn
            self.class_wise[class_label]['Nref'] += Nref
            self.class_wise[class_label]['Nsys'] += Nsys

        return self



[docs]    def reset(self):
        """Reset internal state"""

        self.overall = {
            'Ntp': 0.0,
            'Ntn': 0.0,
            'Nfp': 0.0,
            'Nfn': 0.0,
            'Nref': 0.0,
            'Nsys': 0.0,
            'ER': 0.0,
            'S': 0.0,
            'D': 0.0,
            'I': 0.0,
        }

        self.class_wise = {}
        for class_label in self.event_label_list:
            self.class_wise[class_label] = {
                'Ntp': 0.0,
                'Ntn': 0.0,
                'Nfp': 0.0,
                'Nfn': 0.0,
                'Nref': 0.0,
                'Nsys': 0.0,
            }

        return self


    # Metrics

[docs]    def overall_f_measure(self):
        """Overall f-measure metrics (f_measure, precision, and recall)

        Returns
        -------
        dict
            results in a dictionary format

        """
        if self.overall['Nsys'] == 0 and self.empty_system_output_handling == 'zero_score':
            precision = 0

        else:
            precision = metric.precision(
                Ntp=self.overall['Ntp'],
                Nsys=self.overall['Nsys']
            )

        recall = metric.recall(
            Ntp=self.overall['Ntp'],
            Nref=self.overall['Nref']
        )

        f_measure = metric.f_measure(
            precision=precision,
            recall=recall
        )

        return {
            'f_measure': f_measure,
            'precision': precision,
            'recall': recall
        }



[docs]    def overall_error_rate(self):
        """Overall error rate metrics (error_rate, substitution_rate, deletion_rate, and insertion_rate)

        Returns
        -------
        dict
            results in a dictionary format

        """

        substitution_rate = metric.substitution_rate(
            Nref=self.overall['Nref'],
            Nsubstitutions=self.overall['S']
        )

        deletion_rate = metric.deletion_rate(
            Nref=self.overall['Nref'],
            Ndeletions=self.overall['D']
        )

        insertion_rate = metric.insertion_rate(
            Nref=self.overall['Nref'],
            Ninsertions=self.overall['I']
        )

        error_rate = metric.error_rate(
            substitution_rate_value=substitution_rate,
            deletion_rate_value=deletion_rate,
            insertion_rate_value=insertion_rate
        )

        return {
            'error_rate': error_rate,
            'substitution_rate': substitution_rate,
            'deletion_rate': deletion_rate,
            'insertion_rate': insertion_rate
        }



[docs]    def overall_accuracy(self, factor=0.5):
        """Overall accuracy metrics (sensitivity, specificity, accuracy, and balanced_accuracy)

        Parameters
        ----------
        factor : float [0-1]
            Balance factor.
            Default value 0.5

        Returns
        -------
        dict
            results in a dictionary format

        """

        sensitivity = metric.sensitivity(
            Ntp=self.overall['Ntp'],
            Nfn=self.overall['Nfn']
        )

        specificity = metric.specificity(
            Ntn=self.overall['Ntn'],
            Nfp=self.overall['Nfp']
        )

        balanced_accuracy = metric.balanced_accuracy(
            sensitivity=sensitivity,
            specificity=specificity,
            factor=factor
        )

        accuracy = metric.accuracy(
            Ntp=self.overall['Ntp'],
            Ntn=self.overall['Ntn'],
            Nfp=self.overall['Nfp'],
            Nfn=self.overall['Nfn']
        )

        return {
            'accuracy': accuracy,
            'balanced_accuracy': balanced_accuracy,
            'sensitivity': sensitivity,
            'specificity': specificity
        }



[docs]    def class_wise_count(self, event_label):
        """Class-wise counts (Nref and Nsys)

        Returns
        -------
        dict
            results in a dictionary format

        """

        return {
            'Nref': float(self.class_wise[event_label]['Nref']),
            'Nsys': float(self.class_wise[event_label]['Nsys'])
        }



[docs]    def class_wise_f_measure(self, event_label):
        """Class-wise f-measure metrics (f_measure, precision, and recall)

        Returns
        -------
        dict
            results in a dictionary format

        """
        if self.class_wise[event_label]['Nsys'] == 0 and self.empty_system_output_handling == 'zero_score':
            precision = 0

        else:
            precision = metric.precision(
                Ntp=self.class_wise[event_label]['Ntp'],
                Nsys=self.class_wise[event_label]['Nsys']
            )

        recall = metric.recall(
            Ntp=self.class_wise[event_label]['Ntp'],
            Nref=self.class_wise[event_label]['Nref']
        )

        f_measure = metric.f_measure(
            precision=precision,
            recall=recall
        )

        return {
            'f_measure': f_measure,
            'precision': precision,
            'recall': recall
        }



[docs]    def class_wise_error_rate(self, event_label):
        """Class-wise error rate metrics (error_rate, deletion_rate, and insertion_rate)

        Returns
        -------
        dict
            results in a dictionary format

        """

        deletion_rate = metric.deletion_rate(
            Nref=self.class_wise[event_label]['Nref'],
            Ndeletions=self.class_wise[event_label]['Nfn']
        )

        insertion_rate = metric.insertion_rate(
            Nref=self.class_wise[event_label]['Nref'],
            Ninsertions=self.class_wise[event_label]['Nfp']
        )

        error_rate = metric.error_rate(
            deletion_rate_value=deletion_rate,
            insertion_rate_value=insertion_rate
        )

        return {
            'error_rate': error_rate,
            'deletion_rate': deletion_rate,
            'insertion_rate': insertion_rate
        }



[docs]    def class_wise_accuracy(self, event_label, factor=0.5):
        """Class-wise accuracy metrics (sensitivity, specificity, accuracy, and balanced_accuracy)

        Returns
        -------
        dict
            results in a dictionary format

        """

        sensitivity = metric.sensitivity(
            Ntp=self.class_wise[event_label]['Ntp'],
            Nfn=self.class_wise[event_label]['Nfn']
        )

        specificity = metric.specificity(
            Ntn=self.class_wise[event_label]['Ntn'],
            Nfp=self.class_wise[event_label]['Nfp']
        )

        balanced_accuracy = metric.balanced_accuracy(
            sensitivity=sensitivity,
            specificity=specificity,
            factor=factor
        )

        accuracy = metric.accuracy(
            Ntp=self.class_wise[event_label]['Ntp'],
            Ntn=self.class_wise[event_label]['Ntn'],
            Nfp=self.class_wise[event_label]['Nfp'],
            Nfn=self.class_wise[event_label]['Nfn']
        )

        return {
            'accuracy': accuracy,
            'balanced_accuracy': balanced_accuracy,
            'sensitivity': sensitivity,
            'specificity': specificity
        }


    # Reports

[docs]    def result_report_parameters(self):
        """Report metric parameters

        Returns
        -------
        str
            result report in string format

        """

        output = self.ui.data(field='Evaluated length', value=self.evaluated_length_seconds, unit='sec') + '\n'
        output += self.ui.data(field='Evaluated files', value=self.evaluated_files) + '\n'

        if self.time_resolution < 1:
            output += self.ui.data(field='Segment length', value=self.time_resolution * 1000, unit='ms') + '\n'

        else:
            output += self.ui.data(field='Segment length', value=self.time_resolution, unit='sec') + '\n'

        return output




[docs]class EventBasedMetrics(SoundEventMetrics):

[docs]    def __init__(self,
                 event_label_list,
                 evaluate_onset=True,
                 evaluate_offset=True,
                 t_collar=0.200,
                 percentage_of_length=0.5,
                 event_matching_type='optimal',
                 **kwargs):
        """Constructor

        Parameters
        ----------
        event_label_list : list
            List of unique event labels

        evaluate_onset : bool
            Evaluate onset.
            Default value True

        evaluate_offset : bool
            Evaluate offset.
            Default value True

        t_collar : float (0,]
            Time collar used when evaluating validity of the onset and offset, in seconds.
            Default value 0.2

        percentage_of_length : float in [0, 1]
            Second condition, percentage of the length within which the estimated offset has to be in order to be
            consider valid estimation.
            Default value 0.5

        event_matching_type : str
            Event matching type. Set 'optimal' for graph-based matching, or 'greedy' for always select first found match.
            Greedy type of event matching is kept for backward compatibility. Both event matching types produce
            very similar results, however, greedy matching can be sensitive to the order of reference events.
            Use default 'optimal' event matching, if you do not intend to compare your results to old results.
            Default value 'optimal'

        """

        SoundEventMetrics.__init__(self, **kwargs)

        if isinstance(event_label_list, numpy.ndarray) and len(event_label_list.shape) == 1:
            # We have numpy array, convert it to list
            event_label_list = event_label_list.tolist()

        if not isinstance(event_label_list, list):
            raise ValueError(
                "event_label_list needs to be list or numpy.array"
            )

        if not isinstance(t_collar, float) or t_collar <= 0.0:
            raise ValueError(
                "t_collar needs to be float > 0"
            )

        if not isinstance(percentage_of_length, float) or percentage_of_length < 0.0 or percentage_of_length > 1.0:
            raise ValueError(
                "t_collar percentage_of_length to be float in [0, 1]"
            )

        self.event_label_list = event_label_list
        self.evaluated_length = 0.0
        self.evaluated_files = 0
        if not evaluate_onset and not evaluate_offset:
            raise ValueError("Both evaluate_onset and evaluate_offset cannot be set to False")

        self.evaluate_onset = evaluate_onset
        self.evaluate_offset = evaluate_offset

        self.t_collar = t_collar
        self.percentage_of_length = percentage_of_length
        self.event_matching_type = event_matching_type
        self.overall = {
            'Nref': 0.0,
            'Nsys': 0.0,
            'Nsubs': 0.0,
            'Ntp': 0.0,
            'Nfp': 0.0,
            'Nfn': 0.0,
        }
        self.class_wise = {}

        for class_label in self.event_label_list:
            self.class_wise[class_label] = {
                'Nref': 0.0,
                'Nsys': 0.0,
                'Ntp': 0.0,
                'Ntn': 0.0,
                'Nfp': 0.0,
                'Nfn': 0.0,
            }


    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        return self.results()

    def __str__(self):
        """Print result reports"""

        if self.evaluate_onset and self.evaluate_offset:
            title = "Event based metrics (onset-offset)"

        elif self.evaluate_onset and not self.evaluate_offset:
            title = "Event based metrics (onset only)"

        elif not self.evaluate_onset and self.evaluate_offset:
            title = "Event based metrics (offset only)"

        else:
            title = "Event based metrics"

        output = self.ui.section_header(title) + '\n'

        output += self.result_report_parameters() + '\n'
        output += self.result_report_overall() + '\n'
        output += self.result_report_class_wise_average() + '\n'
        output += self.result_report_class_wise() + '\n'

        return output


[docs]    def evaluate(self, reference_event_list, estimated_event_list):
        """Evaluate file pair (reference and estimated)

        Parameters
        ----------

        reference_event_list : event list
            Reference event list

        estimated_event_list : event list
            Estimated event list

        Returns
        -------
        self

        """

        # Make sure input is dcase_util.containers.MetaDataContainer
        if not isinstance(reference_event_list, dcase_util.containers.MetaDataContainer):
            reference_event_list = dcase_util.containers.MetaDataContainer(reference_event_list)

        if not isinstance(estimated_event_list, dcase_util.containers.MetaDataContainer):
            estimated_event_list = dcase_util.containers.MetaDataContainer(estimated_event_list)

        # Check that input event list have event only from one file
        reference_files = reference_event_list.unique_files
        if len(reference_files) > 1:
            raise ValueError(
                "reference_event_list contains events from multiple files. Evaluate only file by file."
            )

        estimated_files = estimated_event_list.unique_files
        if len(estimated_files) > 1:
            raise ValueError(
                "estimated_event_list contains events from multiple files. Evaluate only file by file."
            )

        # Evaluate only valid events
        valid_reference_event_list = dcase_util.containers.MetaDataContainer()
        for item in reference_event_list:
            if 'event_onset' in item and 'event_offset' in item and 'event_label' in item:
                valid_reference_event_list.append(item)

            elif 'onset' in item and 'offset' in item and 'event_label' in item:
                valid_reference_event_list.append(item)

        reference_event_list = valid_reference_event_list

        valid_estimated_event_list = dcase_util.containers.MetaDataContainer()
        for item in estimated_event_list:
            if 'event_onset' in item and 'event_offset' in item and 'event_label' in item:
                valid_estimated_event_list.append(item)

            elif 'onset' in item and 'offset' in item and 'event_label' in item:
                valid_estimated_event_list.append(item)

        estimated_event_list = valid_estimated_event_list

        self.evaluated_length += reference_event_list.max_offset
        self.evaluated_files += 1

        # Overall metrics

        # Total number of detected and reference events
        Nsys = len(estimated_event_list)
        Nref = len(reference_event_list)

        if self.event_matching_type == 'optimal':
            label_hit_matrix = numpy.zeros((len(reference_event_list), len(estimated_event_list)), dtype=bool)
            for j in range(0, len(reference_event_list)):
                for i in range(0, len(estimated_event_list)):
                    label_hit_matrix[j, i] = reference_event_list[j]['event_label'] == estimated_event_list[i]['event_label']

            hit_matrix = label_hit_matrix
            if self.evaluate_onset:
                onset_hit_matrix = numpy.zeros((len(reference_event_list), len(estimated_event_list)), dtype=bool)
                for j in range(0, len(reference_event_list)):
                    for i in range(0, len(estimated_event_list)):
                        onset_hit_matrix[j, i] = self.validate_onset(
                            reference_event=reference_event_list[j],
                            estimated_event=estimated_event_list[i],
                            t_collar=self.t_collar
                        )

                hit_matrix *= onset_hit_matrix

            if self.evaluate_offset:
                offset_hit_matrix = numpy.zeros((len(reference_event_list), len(estimated_event_list)), dtype=bool)
                for j in range(0, len(reference_event_list)):
                    for i in range(0, len(estimated_event_list)):
                        offset_hit_matrix[j, i] = self.validate_offset(
                            reference_event=reference_event_list[j],
                            estimated_event=estimated_event_list[i],
                            t_collar=self.t_collar,
                            percentage_of_length=self.percentage_of_length
                        )

                hit_matrix *= offset_hit_matrix

            hits = numpy.where(hit_matrix)
            G = {}
            for ref_i, est_i in zip(*hits):
                if est_i not in G:
                    G[est_i] = []

                G[est_i].append(ref_i)

            matching = sorted(util.bipartite_match(G).items())

            ref_correct = numpy.zeros(Nref, dtype=bool)
            sys_correct = numpy.zeros(Nsys, dtype=bool)
            for item in matching:
                ref_correct[item[0]] = True
                sys_correct[item[1]] = True

            Ntp = len(matching)
            # Substitutions
            Nsubs = 0

            ref_leftover = numpy.nonzero(numpy.logical_not(ref_correct))[0]
            sys_leftover = numpy.nonzero(numpy.logical_not(sys_correct))[0]
            sys_counted = numpy.zeros(Nsys, dtype=bool)
            for j in ref_leftover:
                for i in sys_leftover:
                    if not sys_counted[i]:
                        if self.evaluate_onset:
                            onset_condition = self.validate_onset(
                                reference_event=reference_event_list[j],
                                estimated_event=estimated_event_list[i],
                                t_collar=self.t_collar
                            )

                        else:
                            onset_condition = True

                        if self.evaluate_offset:
                            offset_condition = self.validate_offset(
                                reference_event=reference_event_list[j],
                                estimated_event=estimated_event_list[i],
                                t_collar=self.t_collar,
                                percentage_of_length=self.percentage_of_length
                            )

                        else:
                            offset_condition = True

                        if onset_condition and offset_condition:
                            sys_counted[i] = True
                            Nsubs += 1
                            break

        elif self.event_matching_type == 'greedy':
            sys_correct = numpy.zeros(Nsys, dtype=bool)
            ref_correct = numpy.zeros(Nref, dtype=bool)

            # Number of correctly detected events
            for j in range(0, len(reference_event_list)):
                for i in range(0, len(estimated_event_list)):
                    if not sys_correct[i]:  # skip already matched events
                        label_condition = reference_event_list[j]['event_label'] == estimated_event_list[i]['event_label']

                        if self.evaluate_onset:
                            onset_condition = self.validate_onset(
                                reference_event=reference_event_list[j],
                                estimated_event=estimated_event_list[i],
                                t_collar=self.t_collar
                            )

                        else:
                            onset_condition = True

                        if self.evaluate_offset:
                            offset_condition = self.validate_offset(
                                reference_event=reference_event_list[j],
                                estimated_event=estimated_event_list[i],
                                t_collar=self.t_collar,
                                percentage_of_length=self.percentage_of_length
                            )

                        else:
                            offset_condition = True

                        if label_condition and onset_condition and offset_condition:
                            ref_correct[j] = True
                            sys_correct[i] = True
                            break

            Ntp = numpy.sum(sys_correct)

            ref_leftover = numpy.nonzero(numpy.logical_not(ref_correct))[0]
            sys_leftover = numpy.nonzero(numpy.logical_not(sys_correct))[0]

            # Substitutions
            Nsubs = 0
            sys_counted = numpy.zeros(Nsys, dtype=bool)
            for j in ref_leftover:
                for i in sys_leftover:
                    if not sys_counted[i]:
                        if self.evaluate_onset:
                            onset_condition = self.validate_onset(
                                reference_event=reference_event_list[j],
                                estimated_event=estimated_event_list[i],
                                t_collar=self.t_collar
                            )

                        else:
                            onset_condition = True

                        if self.evaluate_offset:
                            offset_condition = self.validate_offset(
                                reference_event=reference_event_list[j],
                                estimated_event=estimated_event_list[i],
                                t_collar=self.t_collar,
                                percentage_of_length=self.percentage_of_length
                            )

                        else:
                            offset_condition = True

                        if onset_condition and offset_condition:
                            sys_counted[i] = True
                            Nsubs += 1
                            break

        Nfp = Nsys - Ntp - Nsubs
        Nfn = Nref - Ntp - Nsubs

        self.overall['Nref'] += Nref
        self.overall['Nsys'] += Nsys
        self.overall['Ntp'] += Ntp
        self.overall['Nsubs'] += Nsubs
        self.overall['Nfp'] += Nfp
        self.overall['Nfn'] += Nfn

        # Class-wise metrics
        for class_id, class_label in enumerate(self.event_label_list):
            Nref = 0.0
            Nsys = 0.0
            Ntp = 0.0

            # Count event frequencies in the ground truth
            for i in range(0, len(reference_event_list)):
                if reference_event_list[i]['event_label'] == class_label:
                    Nref += 1

            # Count event frequencies in the system output
            for i in range(0, len(estimated_event_list)):
                if estimated_event_list[i]['event_label'] == class_label:
                    Nsys += 1

            if self.event_matching_type == 'optimal':
                class_reference_event_list = reference_event_list.filter(event_label=class_label)
                class_estimated_event_list = estimated_event_list.filter(event_label=class_label)

                hit_matrix = numpy.ones((len(class_reference_event_list), len(class_estimated_event_list)), dtype=bool)
                if self.evaluate_onset:
                    onset_hit_matrix = numpy.zeros((len(class_reference_event_list), len(class_estimated_event_list)), dtype=bool)
                    for j in range(0, len(class_reference_event_list)):
                        for i in range(0, len(class_estimated_event_list)):
                            onset_hit_matrix[j, i] = self.validate_onset(
                                reference_event=class_reference_event_list[j],
                                estimated_event=class_estimated_event_list[i],
                                t_collar=self.t_collar
                            )

                    hit_matrix *= onset_hit_matrix

                if self.evaluate_offset:
                    offset_hit_matrix = numpy.zeros((len(class_reference_event_list), len(class_estimated_event_list)), dtype=bool)
                    for j in range(0, len(class_reference_event_list)):
                        for i in range(0, len(class_estimated_event_list)):
                            offset_hit_matrix[j, i] = self.validate_offset(
                                reference_event=class_reference_event_list[j],
                                estimated_event=class_estimated_event_list[i],
                                t_collar=self.t_collar,
                                percentage_of_length=self.percentage_of_length
                            )

                    hit_matrix *= offset_hit_matrix

                hits = numpy.where(hit_matrix)
                G = {}
                for ref_i, est_i in zip(*hits):
                    if est_i not in G:
                        G[est_i] = []

                    G[est_i].append(ref_i)

                matching = sorted(util.bipartite_match(G).items())

                ref_correct = numpy.zeros(int(Nref), dtype=bool)
                sys_correct = numpy.zeros(int(Nsys), dtype=bool)

                for item in matching:
                    ref_correct[item[0]] = True
                    sys_correct[item[1]] = True

                Ntp = len(matching)

            elif self.event_matching_type == 'greedy':
                sys_counted = numpy.zeros(len(estimated_event_list), dtype=bool)
                for j in range(0, len(reference_event_list)):
                    if reference_event_list[j]['event_label'] == class_label:
                        for i in range(0, len(estimated_event_list)):
                            if estimated_event_list[i]['event_label'] == class_label and not sys_counted[i]:
                                if self.evaluate_onset:
                                    onset_condition = self.validate_onset(
                                        reference_event=reference_event_list[j],
                                        estimated_event=estimated_event_list[i],
                                        t_collar=self.t_collar
                                    )

                                else:
                                    onset_condition = True

                                if self.evaluate_offset:
                                    offset_condition = self.validate_offset(
                                        reference_event=reference_event_list[j],
                                        estimated_event=estimated_event_list[i],
                                        t_collar=self.t_collar,
                                        percentage_of_length=self.percentage_of_length
                                    )

                                else:
                                    offset_condition = True

                                if onset_condition and offset_condition:
                                    sys_counted[i] = True
                                    Ntp += 1
                                    break

            Nfp = Nsys - Ntp
            Nfn = Nref - Ntp

            self.class_wise[class_label]['Nref'] += Nref
            self.class_wise[class_label]['Nsys'] += Nsys

            self.class_wise[class_label]['Ntp'] += Ntp
            self.class_wise[class_label]['Nfp'] += Nfp
            self.class_wise[class_label]['Nfn'] += Nfn

        return self



[docs]    def reset(self):
        """Reset internal state
        """

        self.overall = {
            'Nref': 0.0,
            'Nsys': 0.0,
            'Nsubs': 0.0,
            'Ntp': 0.0,
            'Nfp': 0.0,
            'Nfn': 0.0,
        }
        self.class_wise = {}

        for class_label in self.event_label_list:
            self.class_wise[class_label] = {
                'Nref': 0.0,
                'Nsys': 0.0,
                'Ntp': 0.0,
                'Ntn': 0.0,
                'Nfp': 0.0,
                'Nfn': 0.0,
            }

        return self


    @staticmethod

[docs]    def validate_onset(reference_event, estimated_event, t_collar=0.200):
        """Validate estimated event based on event onset

        Parameters
        ----------
        reference_event : dict
            Reference event.

        estimated_event: dict
            Estimated event.

        t_collar : float > 0, seconds
            Time collar with which the estimated onset has to be in order to be consider valid estimation.
            Default value 0.2

        Returns
        -------
        bool

        """

        # Detect field naming style used and validate onset
        if 'event_onset' in reference_event and 'event_onset' in estimated_event:
            return math.fabs(reference_event['event_onset'] - estimated_event['event_onset']) <= t_collar

        elif 'onset' in reference_event and 'onset' in estimated_event:
            return math.fabs(reference_event['onset'] - estimated_event['onset']) <= t_collar


    @staticmethod

[docs]    def validate_offset(reference_event, estimated_event, t_collar=0.200, percentage_of_length=0.5):
        """Validate estimated event based on event offset

        Parameters
        ----------
        reference_event : dict
            Reference event.

        estimated_event : dict
            Estimated event.

        t_collar : float > 0, seconds
            First condition, Time collar with which the estimated offset has to be in order to be consider valid estimation.
            Default value 0.2

        percentage_of_length : float in [0, 1]
            Second condition, percentage of the length within which the estimated offset has to be in order to be
            consider valid estimation.
            Default value 0.5

        Returns
        -------
        bool

        """

        # Detect field naming style used and validate onset
        if 'event_offset' in reference_event and 'event_offset' in estimated_event:
            annotated_length = reference_event['event_offset'] - reference_event['event_onset']

            return math.fabs(reference_event['event_offset'] - estimated_event['event_offset']) <= max(t_collar, percentage_of_length * annotated_length)

        elif 'offset' in reference_event and 'offset' in estimated_event:
            annotated_length = reference_event['offset'] - reference_event['onset']

            return math.fabs(reference_event['offset'] - estimated_event['offset']) <= max(t_collar, percentage_of_length * annotated_length)


    # Metrics

[docs]    def overall_f_measure(self):
        """Overall f-measure metrics (f_measure, precision, and recall)

        Returns
        -------
        dict
            results in a dictionary format
        """

        if self.overall['Nsys'] == 0 and self.empty_system_output_handling == 'zero_score':
            precision = 0

        else:
            precision = metric.precision(
                Ntp=self.overall['Ntp'],
                Nsys=self.overall['Nsys']
            )

        recall = metric.recall(
            Ntp=self.overall['Ntp'],
            Nref=self.overall['Nref']
        )

        f_measure = metric.f_measure(
            precision=precision,
            recall=recall
        )

        return {
            'f_measure': f_measure,
            'precision': precision,
            'recall': recall
        }



[docs]    def overall_error_rate(self):
        """Overall error rate metrics (error_rate, substitution_rate, deletion_rate, and insertion_rate)

        Returns
        -------
        dict
            results in a dictionary format

        """

        substitution_rate = metric.substitution_rate(
            Nref=self.overall['Nref'],
            Nsubstitutions=self.overall['Nsubs']
        )

        deletion_rate = metric.deletion_rate(
            Nref=self.overall['Nref'],
            Ndeletions=self.overall['Nfn']
        )

        insertion_rate = metric.insertion_rate(
            Nref=self.overall['Nref'],
            Ninsertions=self.overall['Nfp']
        )

        error_rate = metric.error_rate(
            substitution_rate_value=substitution_rate,
            deletion_rate_value=deletion_rate,
            insertion_rate_value=insertion_rate
        )

        return {
            'error_rate': error_rate,
            'substitution_rate': substitution_rate,
            'deletion_rate': deletion_rate,
            'insertion_rate': insertion_rate
        }



[docs]    def class_wise_count(self, event_label):
        """Class-wise counts (Nref and Nsys)

        Returns
        -------
        dict
            results in a dictionary format

        """

        return {
            'Nref': self.class_wise[event_label]['Nref'],
            'Nsys': self.class_wise[event_label]['Nsys']
        }



[docs]    def class_wise_f_measure(self, event_label):
        """Class-wise f-measure metrics (f_measure, precision, and recall)

        Returns
        -------
        dict
            results in a dictionary format

        """
        if self.class_wise[event_label]['Nsys'] == 0 and self.empty_system_output_handling == 'zero_score':
            precision = 0

        else:
            precision = metric.precision(
                Ntp=self.class_wise[event_label]['Ntp'],
                Nsys=self.class_wise[event_label]['Nsys']
            )

        recall = metric.recall(
            Ntp=self.class_wise[event_label]['Ntp'],
            Nref=self.class_wise[event_label]['Nref']
        )

        f_measure = metric.f_measure(
            precision=precision,
            recall=recall
        )

        return {
            'f_measure': f_measure,
            'precision': precision,
            'recall': recall
        }



[docs]    def class_wise_error_rate(self, event_label):
        """Class-wise error rate metrics (error_rate, deletion_rate, and insertion_rate)

        Returns
        -------
        dict
            results in a dictionary format

        """

        deletion_rate = metric.deletion_rate(
            Nref=self.class_wise[event_label]['Nref'],
            Ndeletions=self.class_wise[event_label]['Nfn']
        )

        insertion_rate = metric.insertion_rate(
            Nref=self.class_wise[event_label]['Nref'],
            Ninsertions=self.class_wise[event_label]['Nfp']
        )

        error_rate = metric.error_rate(
            deletion_rate_value=deletion_rate,
            insertion_rate_value=insertion_rate
        )

        return {
            'error_rate': error_rate,
            'deletion_rate': deletion_rate,
            'insertion_rate': insertion_rate
        }


    # Reports

[docs]    def result_report_parameters(self):
        """Report metric parameters

        Returns
        -------
        str
            result report in string format

        """

        output = self.ui.data(field='Evaluated length', value=self.evaluated_length, unit='sec') + '\n'
        output += self.ui.data(field='Evaluated files', value=self.evaluated_files) + '\n'

        output += self.ui.data(field='Evaluate onset', value=self.evaluate_onset) + '\n'
        output += self.ui.data(field='Evaluate offset', value=self.evaluate_offset) + '\n'

        if self.t_collar < 1:
            output += self.ui.data(field='T collar', value=self.t_collar*1000, unit='ms') + '\n'

        else:
            output += self.ui.data(field='T collar', value=self.t_collar, unit='sec') + '\n'

        output += self.ui.data(field='Offset (length)', value=self.percentage_of_length*100, unit='%') + '\n'

        return output

© Copyright 2018, Toni Heittola et al..

================================================ FILE: docs/_modules/sed_eval/test.html ================================================ sed_eval.test — sed_eval 0.1 documentation

Source code for sed_eval.test

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""

.. autosummary::
    :toctree: generated/

    mcnemar

"""

import numpy



[docs]def mcnemar(reference, estimated_a, estimated_b):
    """McNemar's test

    Wikipedia entry https://en.wikipedia.org/wiki/McNemar%27s_test

    Parameters
    ----------
    reference : list
        Reference value

    estimated_a : list
        System output A

    estimated_b : list
        System output B

    Returns
    -------
    float
        exact P-value

    """

    if len(reference) != len(estimated_a) or len(reference) != len(estimated_b):
        raise ValueError('Input arrays needs to be same length.')

    # Convert lists into numpy.array
    reference = numpy.array(reference)
    estimated_a = numpy.array(estimated_a)
    estimated_b = numpy.array(estimated_b)

    # Intermediate values
    correct_a = estimated_a == reference
    correct_b = estimated_b == reference

    incorrect_a = estimated_a != reference
    incorrect_b = estimated_b != reference

    # Contingency table values
    b = float( numpy.sum( numpy.logical_and(incorrect_a, correct_b) ) )
    c = float( numpy.sum( numpy.logical_and(correct_a, incorrect_b) ) )

    # Continuity corrected version of the McNemar test to approximate the binomial exact-P-value
    # Edwards, A (1948). "Note on the "correction for continuity" in testing the significance of the difference
    # between correlated proportions". Psychometrika. 13: 185–187
    if b + c > 0:
        return (numpy.abs(b - c) - 1)**2 / (b + c)
    else:
        return 0

© Copyright 2018, Toni Heittola et al..

================================================ FILE: docs/_modules/sed_eval/util/event_list.html ================================================ sed_eval.util.event_list — sed_eval 0.1 documentation

Source code for sed_eval.util.event_list

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Event list handling
"""

import dcase_util

__all__ = ['filter_event_list',
           'unique_files',
           'unique_event_labels',
           'max_event_offset']



[docs]def filter_event_list(event_list, scene_label=None, event_label=None, filename=None):
    """Filter event list based on given fields

    Parameters
    ----------
    event_list : list, shape=(n,)
        A list containing event dicts

    scene_label : str
        Scene label

    event_label : str
        Event label

    filename : str
        Filename

    Returns
    -------
    event_list: list, shape=(n,)
        A list containing event dicts

    """

    return dcase_util.containers.MetaDataContainer(event_list).filter(
        filename=filename,
        scene_label=scene_label,
        event_label=event_label
    )




[docs]def unique_files(event_list):
    """Find the unique files

    Parameters
    ----------
    event_list : list or dcase_util.containers.MetaDataContainer
        A list containing event dicts

    Returns
    -------
    list
        Unique filenames in alphabetical order

    """

    if isinstance(event_list, dcase_util.containers.MetaDataContainer):
        return event_list.unique_files

    else:
        files = {}
        for event in event_list:
            if 'file' in event:
                files[event['file']] = event['file']

            elif 'filename' in event:
                files[event['filename']] = event['filename']

        files = list(files.keys())
        files.sort()
        return files




[docs]def unique_event_labels(event_list):
    """Find the unique event labels

    Parameters
    ----------
    event_list : list or dcase_util.containers.MetaDataContainer
        A list containing event dicts

    Returns
    -------
    list
        Unique labels in alphabetical order

    """

    if isinstance(event_list, dcase_util.containers.MetaDataContainer):
        return event_list.unique_event_labels

    else:
        labels = []
        for event in event_list:
            if 'event_label' in event and event['event_label'] not in labels:
                labels.append(event['event_label'])

        labels.sort()
        return labels




[docs]def max_event_offset(event_list):
    """Find the offset (end-time) of last event

    Parameters
    ----------
    event_list : list or dcase_util.containers.MetaDataContainer
        A list containing event dicts

    Returns
    -------
    float > 0
        maximum offset

    """

    if isinstance(event_list, dcase_util.containers.MetaDataContainer):
        return event_list.max_offset

    else:
        max_offset = 0
        for event in event_list:
            if 'event_offset' in event:
                if event['event_offset'] > max_offset:
                    max_offset = event['event_offset']

            elif 'offset' in event:
                if event['offset'] > max_offset:
                    max_offset = event['offset']

        return max_offset

© Copyright 2018, Toni Heittola et al..

================================================ FILE: docs/_modules/sed_eval/util/event_matching.html ================================================ sed_eval.util.event_matching — sed_eval 0.1 documentation

Source code for sed_eval.util.event_matching

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Event matching
"""


[docs]def bipartite_match(graph):
    """
    Find maximum cardinality matching of a bipartite graph (U,V,E).
    Function is borrowed from mir_eval toolbox (https://github.com/craffel/mir_eval).

    The input format is a dictionary mapping members of U to a list of their neighbors in V.
    The output is a dict M mapping members of V to their matches in U.

    Parameters
    ----------
    graph : dictionary : left-vertex -> list of right vertices
        The input bipartite graph.  Each edge need only be specified once.

    Returns
    -------
    matching : dictionary : right-vertex -> left vertex
        A maximal bipartite matching.

    """
    # Implementation is after _bipartite_match function in mir_eval toolbox:
    # Colin Raffel, Brian McFee, Eric J. Humphrey, Justin Salamon, Oriol Nieto, Dawen Liang,
    # and Daniel P. W. Ellis, "mir_eval: A Transparent Implementation of Common MIR Metrics",
    # Proceedings of the 15th International Conference on Music Information Retrieval, 2014.
    #
    # _bipartite_match function:
    # https://github.com/craffel/mir_eval/blob/master/mir_eval/util.py#L547
    #
    # Function is originally adapted from:
    #
    # Hopcroft-Karp bipartite max-cardinality matching and max independent set
    # David Eppstein, UC Irvine, 27 Apr 2002

    # initialize greedy matching (redundant, but faster than full search)
    matching = {}
    for u in graph:
        for v in graph[u]:
            if v not in matching:
                matching[v] = u
                break

    while True:
        # structure residual graph into layers
        # pred[u] gives the neighbor in the previous layer for u in U
        # preds[v] gives a list of neighbors in the previous layer for v in V
        # unmatched gives a list of unmatched vertices in final layer of V,
        # and is also used as a flag value for pred[u] when u is in the first
        # layer

        preds = {}
        unmatched = []
        pred = dict([(u, unmatched) for u in graph])

        for v in matching:
            del pred[matching[v]]

        layer = list(pred)

        # repeatedly extend layering structure by another pair of layers
        while layer and not unmatched:
            new_layer = {}
            for u in layer:
                for v in graph[u]:
                    if v not in preds:
                        new_layer.setdefault(v, []).append(u)

            layer = []
            for v in new_layer:
                preds[v] = new_layer[v]

                if v in matching:
                    layer.append(matching[v])
                    pred[matching[v]] = v

                else:
                    unmatched.append(v)

        # did we finish layering without finding any alternating paths?
        if not unmatched:
            unlayered = {}
            for u in graph:
                for v in graph[u]:
                    if v not in preds:
                        unlayered[v] = None

            return matching

        def recurse(v):
            """Recursively search backward through layers to find alternating
            paths.  recursion returns true if found path, false otherwise
            """

            if v in preds:
                L = preds[v]
                del preds[v]
                for u in L:
                    if u in pred:
                        pu = pred[u]
                        del pred[u]
                        if pu is unmatched or recurse(pu):
                            matching[v] = u
                            return True

            return False

        for v in unmatched:
            recurse(v)

© Copyright 2018, Toni Heittola et al..

================================================ FILE: docs/_modules/sed_eval/util/event_roll.html ================================================ sed_eval.util.event_roll — sed_eval 0.1 documentation

Source code for sed_eval.util.event_roll

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Event roll handling

"""
from __future__ import absolute_import
import math
import numpy
from . import event_list
import dcase_util



[docs]def event_list_to_event_roll(source_event_list, event_label_list=None, time_resolution=0.01):
    """Convert event list into event roll, binary activity matrix

    Parameters
    ----------
    source_event_list : list, shape=(n,)
        A list containing event dicts

    event_label_list : list, shape=(k,) or None
        A list of containing unique labels in alphabetical order
        (Default value = None)
    time_resolution : float > 0
        Time resolution in seconds of the event roll
        (Default value = 0.01)

    Returns
    -------

    event_roll: np.ndarray, shape=(m,k)
        Event roll

    """

    if isinstance(source_event_list, dcase_util.containers.MetaDataContainer):
        max_offset_value = source_event_list.max_offset

        if event_label_list is None:
            event_label_list = source_event_list.unique_event_labels

    elif isinstance(source_event_list, list):
        max_offset_value = event_list.max_event_offset(source_event_list)

        if event_label_list is None:
            event_label_list = event_list.unique_event_labels(source_event_list)

    else:
        raise ValueError('Unknown source_event_list type.')

    # Initialize event roll
    event_roll = numpy.zeros((int(math.ceil(max_offset_value * 1 / time_resolution)), len(event_label_list)))

    # Fill-in event_roll
    for event in source_event_list:
        pos = event_label_list.index(event['event_label'])

        if 'event_onset' in event and 'event_offset' in event:
            event_onset = event['event_onset']
            event_offset = event['event_offset']

        elif 'onset' in event and 'offset' in event:
            event_onset = event['onset']
            event_offset = event['offset']

        onset = int(math.floor(event_onset * 1 / float(time_resolution)))
        offset = int(math.ceil(event_offset * 1 / float(time_resolution)))

        event_roll[onset:offset, pos] = 1

    return event_roll




[docs]def pad_event_roll(event_roll, length):
    """Pad event roll's length to given length

    Parameters
    ----------
    event_roll: np.ndarray, shape=(m,k)
        Event roll

    length : int
        Length to be padded

    Returns
    -------
    event_roll: np.ndarray, shape=(m,k)
        Padded event roll

    """

    if length > event_roll.shape[0]:
        padding = numpy.zeros((length-event_roll.shape[0], event_roll.shape[1]))
        event_roll = numpy.vstack((event_roll, padding))

    return event_roll




[docs]def match_event_roll_lengths(event_roll_a, event_roll_b, length=None):
    """Fix the length of two event rolls

    Parameters
    ----------
    event_roll_a: np.ndarray, shape=(m1,k)
        Event roll A

    event_roll_b: np.ndarray, shape=(m2,k)
        Event roll B

    length: int, optional
        Length of the event roll, if none given, shorter event roll is padded to match longer one.

    Returns
    -------
    event_roll_a: np.ndarray, shape=(max(m1,m2),k)
        Padded event roll A

    event_roll_b: np.ndarray, shape=(max(m1,m2),k)
        Padded event roll B

    """

    # Fix durations of both event_rolls to be equal
    if length is None:
        length = max(event_roll_b.shape[0], event_roll_a.shape[0])

    else:
        length = int(length)

    if length < event_roll_a.shape[0]:
        event_roll_a = event_roll_a[0:length, :]

    else:
        event_roll_a = pad_event_roll(
            event_roll=event_roll_a,
            length=length
        )

    if length < event_roll_b.shape[0]:
        event_roll_b = event_roll_b[0:length, :]

    else:
        event_roll_b = pad_event_roll(
            event_roll=event_roll_b,
            length=length
        )

    return event_roll_a, event_roll_b

© Copyright 2018, Toni Heittola et al..

================================================ FILE: docs/_modules/sed_eval/util/scene_list.html ================================================ sed_eval.util.scene_list — sed_eval 0.1 documentation

Source code for sed_eval.util.scene_list

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Scene list handling
"""

import dcase_util

__all__ = ['unique_scene_labels']



[docs]def unique_scene_labels(scene_list):
    """Find the unique scene labels

    Parameters
    ----------
    scene_list : list, shape=(n,)
        A list containing scene dicts

    Returns
    -------
    labels: list, shape=(n,)
        Unique labels in alphabetical order

    """
    if isinstance(scene_list, dcase_util.containers.MetaDataContainer):
        return scene_list.unique_scene_labels

    else:
        labels = []
        for item in scene_list:
            if 'scene_label' in item and item['scene_label'] not in labels:
                labels.append(item['scene_label'])

        labels.sort()
        return labels

© Copyright 2018, Toni Heittola et al..

================================================ FILE: docs/_sources/audio_tag.rst.txt ================================================ .. _audio_tag: Audio Tagging ============= The goal of audio tagging is to assign tags (one or many) to audio signal. Tags can be considered as sound events without timing information (onset or offset). .. automodule:: sed_eval.audio_tag ================================================ FILE: docs/_sources/changelog.rst.txt ================================================ Changes ======= v0.2.0 ------ - Audio tagging evaluator class added - Equal error rate metric added - Meta data reading is using `dcase_util` library now v0.1.4 ------ - Checks for sound event evaluator to prevent evaluation with event lists containing data from multiple source files - Update documentation v0.1.3 ------ - McNemar's test added - Added utility methods for EventList class v0.1.2 ------ - Minor fixes v0.1.1 ------ - Fixing indexing error in event roll creation (event offsets affected) v0.1.0 ------ - First public release. ================================================ FILE: docs/_sources/generated/sed_eval.audio_tag.AudioTaggingMetrics.evaluate.rst.txt ================================================ sed_eval.audio_tag.AudioTaggingMetrics.evaluate =============================================== .. currentmodule:: sed_eval.audio_tag .. automethod:: AudioTaggingMetrics.evaluate ================================================ FILE: docs/_sources/generated/sed_eval.audio_tag.AudioTaggingMetrics.reset.rst.txt ================================================ sed_eval.audio_tag.AudioTaggingMetrics.reset ============================================ .. currentmodule:: sed_eval.audio_tag .. automethod:: AudioTaggingMetrics.reset ================================================ FILE: docs/_sources/generated/sed_eval.audio_tag.AudioTaggingMetrics.result_report_class_wise.rst.txt ================================================ sed_eval.audio_tag.AudioTaggingMetrics.result_report_class_wise =============================================================== .. currentmodule:: sed_eval.audio_tag .. automethod:: AudioTaggingMetrics.result_report_class_wise ================================================ FILE: docs/_sources/generated/sed_eval.audio_tag.AudioTaggingMetrics.result_report_class_wise_average.rst.txt ================================================ sed_eval.audio_tag.AudioTaggingMetrics.result_report_class_wise_average ======================================================================= .. currentmodule:: sed_eval.audio_tag .. automethod:: AudioTaggingMetrics.result_report_class_wise_average ================================================ FILE: docs/_sources/generated/sed_eval.audio_tag.AudioTaggingMetrics.result_report_parameters.rst.txt ================================================ sed_eval.audio_tag.AudioTaggingMetrics.result_report_parameters =============================================================== .. currentmodule:: sed_eval.audio_tag .. automethod:: AudioTaggingMetrics.result_report_parameters ================================================ FILE: docs/_sources/generated/sed_eval.audio_tag.AudioTaggingMetrics.results.rst.txt ================================================ sed_eval.audio_tag.AudioTaggingMetrics.results ============================================== .. currentmodule:: sed_eval.audio_tag .. automethod:: AudioTaggingMetrics.results ================================================ FILE: docs/_sources/generated/sed_eval.audio_tag.AudioTaggingMetrics.results_class_wise_average_metrics.rst.txt ================================================ sed_eval.audio_tag.AudioTaggingMetrics.results_class_wise_average_metrics ========================================================================= .. currentmodule:: sed_eval.audio_tag .. automethod:: AudioTaggingMetrics.results_class_wise_average_metrics ================================================ FILE: docs/_sources/generated/sed_eval.audio_tag.AudioTaggingMetrics.results_class_wise_metrics.rst.txt ================================================ sed_eval.audio_tag.AudioTaggingMetrics.results_class_wise_metrics ================================================================= .. currentmodule:: sed_eval.audio_tag .. automethod:: AudioTaggingMetrics.results_class_wise_metrics ================================================ FILE: docs/_sources/generated/sed_eval.audio_tag.AudioTaggingMetrics.results_overall_metrics.rst.txt ================================================ sed_eval.audio_tag.AudioTaggingMetrics.results_overall_metrics ============================================================== .. currentmodule:: sed_eval.audio_tag .. automethod:: AudioTaggingMetrics.results_overall_metrics ================================================ FILE: docs/_sources/generated/sed_eval.audio_tag.AudioTaggingMetrics.rst.txt ================================================ sed_eval.audio_tag.AudioTaggingMetrics ====================================== .. currentmodule:: sed_eval.audio_tag .. autoclass:: AudioTaggingMetrics .. automethod:: __init__ .. rubric:: Methods .. autosummary:: ~AudioTaggingMetrics.__init__ ~AudioTaggingMetrics.evaluate ~AudioTaggingMetrics.reset ~AudioTaggingMetrics.result_report_class_wise ~AudioTaggingMetrics.result_report_class_wise_average ~AudioTaggingMetrics.result_report_overall ~AudioTaggingMetrics.result_report_parameters ~AudioTaggingMetrics.results ~AudioTaggingMetrics.results_class_wise_average_metrics ~AudioTaggingMetrics.results_class_wise_metrics ~AudioTaggingMetrics.results_overall_metrics ================================================ FILE: docs/_sources/generated/sed_eval.io.load_event_list.rst.txt ================================================ sed_eval.io.load_event_list =========================== .. currentmodule:: sed_eval.io .. autofunction:: load_event_list ================================================ FILE: docs/_sources/generated/sed_eval.io.load_file_pair_list.rst.txt ================================================ sed_eval.io.load_file_pair_list =============================== .. currentmodule:: sed_eval.io .. autofunction:: load_file_pair_list ================================================ FILE: docs/_sources/generated/sed_eval.io.load_scene_list.rst.txt ================================================ sed_eval.io.load_scene_list =========================== .. currentmodule:: sed_eval.io .. autofunction:: load_scene_list ================================================ FILE: docs/_sources/generated/sed_eval.metric.accuracy.rst.txt ================================================ sed_eval.metric.accuracy ======================== .. currentmodule:: sed_eval.metric .. autofunction:: accuracy ================================================ FILE: docs/_sources/generated/sed_eval.metric.accuracy2.rst.txt ================================================ sed_eval.metric.accuracy2 ========================= .. currentmodule:: sed_eval.metric .. autofunction:: accuracy2 ================================================ FILE: docs/_sources/generated/sed_eval.metric.balanced_accuracy.rst.txt ================================================ sed_eval.metric.balanced_accuracy ================================= .. currentmodule:: sed_eval.metric .. autofunction:: balanced_accuracy ================================================ FILE: docs/_sources/generated/sed_eval.metric.deletion_rate.rst.txt ================================================ sed_eval.metric.deletion_rate ============================= .. currentmodule:: sed_eval.metric .. autofunction:: deletion_rate ================================================ FILE: docs/_sources/generated/sed_eval.metric.equal_error_rate.rst.txt ================================================ sed_eval.metric.equal_error_rate ================================ .. currentmodule:: sed_eval.metric .. autofunction:: equal_error_rate ================================================ FILE: docs/_sources/generated/sed_eval.metric.error_rate.rst.txt ================================================ sed_eval.metric.error_rate ========================== .. currentmodule:: sed_eval.metric .. autofunction:: error_rate ================================================ FILE: docs/_sources/generated/sed_eval.metric.f_measure.rst.txt ================================================ sed_eval.metric.f_measure ========================= .. currentmodule:: sed_eval.metric .. autofunction:: f_measure ================================================ FILE: docs/_sources/generated/sed_eval.metric.insertion_rate.rst.txt ================================================ sed_eval.metric.insertion_rate ============================== .. currentmodule:: sed_eval.metric .. autofunction:: insertion_rate ================================================ FILE: docs/_sources/generated/sed_eval.metric.precision.rst.txt ================================================ sed_eval.metric.precision ========================= .. currentmodule:: sed_eval.metric .. autofunction:: precision ================================================ FILE: docs/_sources/generated/sed_eval.metric.recall.rst.txt ================================================ sed_eval.metric.recall ====================== .. currentmodule:: sed_eval.metric .. autofunction:: recall ================================================ FILE: docs/_sources/generated/sed_eval.metric.sensitivity.rst.txt ================================================ sed_eval.metric.sensitivity =========================== .. currentmodule:: sed_eval.metric .. autofunction:: sensitivity ================================================ FILE: docs/_sources/generated/sed_eval.metric.specificity.rst.txt ================================================ sed_eval.metric.specificity =========================== .. currentmodule:: sed_eval.metric .. autofunction:: specificity ================================================ FILE: docs/_sources/generated/sed_eval.metric.substitution_rate.rst.txt ================================================ sed_eval.metric.substitution_rate ================================= .. currentmodule:: sed_eval.metric .. autofunction:: substitution_rate ================================================ FILE: docs/_sources/generated/sed_eval.scene.SceneClassificationMetrics.evaluate.rst.txt ================================================ sed_eval.scene.SceneClassificationMetrics.evaluate ================================================== .. currentmodule:: sed_eval.scene .. automethod:: SceneClassificationMetrics.evaluate ================================================ FILE: docs/_sources/generated/sed_eval.scene.SceneClassificationMetrics.reset.rst.txt ================================================ sed_eval.scene.SceneClassificationMetrics.reset =============================================== .. currentmodule:: sed_eval.scene .. automethod:: SceneClassificationMetrics.reset ================================================ FILE: docs/_sources/generated/sed_eval.scene.SceneClassificationMetrics.result_report_class_wise.rst.txt ================================================ sed_eval.scene.SceneClassificationMetrics.result_report_class_wise ================================================================== .. currentmodule:: sed_eval.scene .. automethod:: SceneClassificationMetrics.result_report_class_wise ================================================ FILE: docs/_sources/generated/sed_eval.scene.SceneClassificationMetrics.result_report_class_wise_average.rst.txt ================================================ sed_eval.scene.SceneClassificationMetrics.result_report_class_wise_average ========================================================================== .. currentmodule:: sed_eval.scene .. automethod:: SceneClassificationMetrics.result_report_class_wise_average ================================================ FILE: docs/_sources/generated/sed_eval.scene.SceneClassificationMetrics.result_report_parameters.rst.txt ================================================ sed_eval.scene.SceneClassificationMetrics.result_report_parameters ================================================================== .. currentmodule:: sed_eval.scene .. automethod:: SceneClassificationMetrics.result_report_parameters ================================================ FILE: docs/_sources/generated/sed_eval.scene.SceneClassificationMetrics.results.rst.txt ================================================ sed_eval.scene.SceneClassificationMetrics.results ================================================= .. currentmodule:: sed_eval.scene .. automethod:: SceneClassificationMetrics.results ================================================ FILE: docs/_sources/generated/sed_eval.scene.SceneClassificationMetrics.results_class_wise_average_metrics.rst.txt ================================================ sed_eval.scene.SceneClassificationMetrics.results_class_wise_average_metrics ============================================================================ .. currentmodule:: sed_eval.scene .. automethod:: SceneClassificationMetrics.results_class_wise_average_metrics ================================================ FILE: docs/_sources/generated/sed_eval.scene.SceneClassificationMetrics.results_class_wise_metrics.rst.txt ================================================ sed_eval.scene.SceneClassificationMetrics.results_class_wise_metrics ==================================================================== .. currentmodule:: sed_eval.scene .. automethod:: SceneClassificationMetrics.results_class_wise_metrics ================================================ FILE: docs/_sources/generated/sed_eval.scene.SceneClassificationMetrics.results_overall_metrics.rst.txt ================================================ sed_eval.scene.SceneClassificationMetrics.results_overall_metrics ================================================================= .. currentmodule:: sed_eval.scene .. automethod:: SceneClassificationMetrics.results_overall_metrics ================================================ FILE: docs/_sources/generated/sed_eval.scene.SceneClassificationMetrics.rst.txt ================================================ sed_eval.scene.SceneClassificationMetrics ========================================= .. currentmodule:: sed_eval.scene .. autoclass:: SceneClassificationMetrics .. automethod:: __init__ .. rubric:: Methods .. autosummary:: ~SceneClassificationMetrics.__init__ ~SceneClassificationMetrics.class_wise_accuracy ~SceneClassificationMetrics.evaluate ~SceneClassificationMetrics.reset ~SceneClassificationMetrics.result_report_class_wise ~SceneClassificationMetrics.result_report_class_wise_average ~SceneClassificationMetrics.result_report_parameters ~SceneClassificationMetrics.results ~SceneClassificationMetrics.results_class_wise_average_metrics ~SceneClassificationMetrics.results_class_wise_metrics ~SceneClassificationMetrics.results_overall_metrics ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.EventBasedMetrics.evaluate.rst.txt ================================================ sed_eval.sound_event.EventBasedMetrics.evaluate =============================================== .. currentmodule:: sed_eval.sound_event .. automethod:: EventBasedMetrics.evaluate ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.EventBasedMetrics.reset.rst.txt ================================================ sed_eval.sound_event.EventBasedMetrics.reset ============================================ .. currentmodule:: sed_eval.sound_event .. automethod:: EventBasedMetrics.reset ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.EventBasedMetrics.result_report_class_wise.rst.txt ================================================ sed_eval.sound_event.EventBasedMetrics.result_report_class_wise =============================================================== .. currentmodule:: sed_eval.sound_event .. automethod:: EventBasedMetrics.result_report_class_wise ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.EventBasedMetrics.result_report_class_wise_average.rst.txt ================================================ sed_eval.sound_event.EventBasedMetrics.result_report_class_wise_average ======================================================================= .. currentmodule:: sed_eval.sound_event .. automethod:: EventBasedMetrics.result_report_class_wise_average ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.EventBasedMetrics.result_report_overall.rst.txt ================================================ sed_eval.sound_event.EventBasedMetrics.result_report_overall ============================================================ .. currentmodule:: sed_eval.sound_event .. automethod:: EventBasedMetrics.result_report_overall ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.EventBasedMetrics.result_report_parameters.rst.txt ================================================ sed_eval.sound_event.EventBasedMetrics.result_report_parameters =============================================================== .. currentmodule:: sed_eval.sound_event .. automethod:: EventBasedMetrics.result_report_parameters ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.EventBasedMetrics.results.rst.txt ================================================ sed_eval.sound_event.EventBasedMetrics.results ============================================== .. currentmodule:: sed_eval.sound_event .. automethod:: EventBasedMetrics.results ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.EventBasedMetrics.results_class_wise_average_metrics.rst.txt ================================================ sed_eval.sound_event.EventBasedMetrics.results_class_wise_average_metrics ========================================================================= .. currentmodule:: sed_eval.sound_event .. automethod:: EventBasedMetrics.results_class_wise_average_metrics ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.EventBasedMetrics.results_class_wise_metrics.rst.txt ================================================ sed_eval.sound_event.EventBasedMetrics.results_class_wise_metrics ================================================================= .. currentmodule:: sed_eval.sound_event .. automethod:: EventBasedMetrics.results_class_wise_metrics ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.EventBasedMetrics.results_overall_metrics.rst.txt ================================================ sed_eval.sound_event.EventBasedMetrics.results_overall_metrics ============================================================== .. currentmodule:: sed_eval.sound_event .. automethod:: EventBasedMetrics.results_overall_metrics ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.EventBasedMetrics.rst.txt ================================================ sed_eval.sound_event.EventBasedMetrics ====================================== .. currentmodule:: sed_eval.sound_event .. autoclass:: EventBasedMetrics .. automethod:: __init__ .. rubric:: Methods .. autosummary:: ~EventBasedMetrics.__init__ ~EventBasedMetrics.class_wise_accuracy ~EventBasedMetrics.class_wise_count ~EventBasedMetrics.class_wise_error_rate ~EventBasedMetrics.class_wise_f_measure ~EventBasedMetrics.evaluate ~EventBasedMetrics.overall_accuracy ~EventBasedMetrics.overall_error_rate ~EventBasedMetrics.overall_f_measure ~EventBasedMetrics.reset ~EventBasedMetrics.result_report_class_wise ~EventBasedMetrics.result_report_class_wise_average ~EventBasedMetrics.result_report_overall ~EventBasedMetrics.result_report_parameters ~EventBasedMetrics.results ~EventBasedMetrics.results_class_wise_average_metrics ~EventBasedMetrics.results_class_wise_metrics ~EventBasedMetrics.results_overall_metrics ~EventBasedMetrics.validate_offset ~EventBasedMetrics.validate_onset ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.SegmentBasedMetrics.evaluate.rst.txt ================================================ sed_eval.sound_event.SegmentBasedMetrics.evaluate ================================================= .. currentmodule:: sed_eval.sound_event .. automethod:: SegmentBasedMetrics.evaluate ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.SegmentBasedMetrics.reset.rst.txt ================================================ sed_eval.sound_event.SegmentBasedMetrics.reset ============================================== .. currentmodule:: sed_eval.sound_event .. automethod:: SegmentBasedMetrics.reset ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.SegmentBasedMetrics.result_report_class_wise.rst.txt ================================================ sed_eval.sound_event.SegmentBasedMetrics.result_report_class_wise ================================================================= .. currentmodule:: sed_eval.sound_event .. automethod:: SegmentBasedMetrics.result_report_class_wise ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.SegmentBasedMetrics.result_report_class_wise_average.rst.txt ================================================ sed_eval.sound_event.SegmentBasedMetrics.result_report_class_wise_average ========================================================================= .. currentmodule:: sed_eval.sound_event .. automethod:: SegmentBasedMetrics.result_report_class_wise_average ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.SegmentBasedMetrics.result_report_overall.rst.txt ================================================ sed_eval.sound_event.SegmentBasedMetrics.result_report_overall ============================================================== .. currentmodule:: sed_eval.sound_event .. automethod:: SegmentBasedMetrics.result_report_overall ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.SegmentBasedMetrics.result_report_parameters.rst.txt ================================================ sed_eval.sound_event.SegmentBasedMetrics.result_report_parameters ================================================================= .. currentmodule:: sed_eval.sound_event .. automethod:: SegmentBasedMetrics.result_report_parameters ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.SegmentBasedMetrics.results.rst.txt ================================================ sed_eval.sound_event.SegmentBasedMetrics.results ================================================ .. currentmodule:: sed_eval.sound_event .. automethod:: SegmentBasedMetrics.results ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.SegmentBasedMetrics.results_class_wise_average_metrics.rst.txt ================================================ sed_eval.sound_event.SegmentBasedMetrics.results_class_wise_average_metrics =========================================================================== .. currentmodule:: sed_eval.sound_event .. automethod:: SegmentBasedMetrics.results_class_wise_average_metrics ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.SegmentBasedMetrics.results_class_wise_metrics.rst.txt ================================================ sed_eval.sound_event.SegmentBasedMetrics.results_class_wise_metrics =================================================================== .. currentmodule:: sed_eval.sound_event .. automethod:: SegmentBasedMetrics.results_class_wise_metrics ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.SegmentBasedMetrics.results_overall_metrics.rst.txt ================================================ sed_eval.sound_event.SegmentBasedMetrics.results_overall_metrics ================================================================ .. currentmodule:: sed_eval.sound_event .. automethod:: SegmentBasedMetrics.results_overall_metrics ================================================ FILE: docs/_sources/generated/sed_eval.sound_event.SegmentBasedMetrics.rst.txt ================================================ sed_eval.sound_event.SegmentBasedMetrics ======================================== .. currentmodule:: sed_eval.sound_event .. autoclass:: SegmentBasedMetrics .. automethod:: __init__ .. rubric:: Methods .. autosummary:: ~SegmentBasedMetrics.__init__ ~SegmentBasedMetrics.class_wise_accuracy ~SegmentBasedMetrics.class_wise_count ~SegmentBasedMetrics.class_wise_error_rate ~SegmentBasedMetrics.class_wise_f_measure ~SegmentBasedMetrics.evaluate ~SegmentBasedMetrics.overall_accuracy ~SegmentBasedMetrics.overall_error_rate ~SegmentBasedMetrics.overall_f_measure ~SegmentBasedMetrics.reset ~SegmentBasedMetrics.result_report_class_wise ~SegmentBasedMetrics.result_report_class_wise_average ~SegmentBasedMetrics.result_report_overall ~SegmentBasedMetrics.result_report_parameters ~SegmentBasedMetrics.results ~SegmentBasedMetrics.results_class_wise_average_metrics ~SegmentBasedMetrics.results_class_wise_metrics ~SegmentBasedMetrics.results_overall_metrics ================================================ FILE: docs/_sources/generated/sed_eval.test.mcnemar.rst.txt ================================================ sed_eval.test.mcnemar ===================== .. currentmodule:: sed_eval.test .. autofunction:: mcnemar ================================================ FILE: docs/_sources/generated/sed_eval.util.event_list.filter_event_list.rst.txt ================================================ sed_eval.util.event_list.filter_event_list ========================================== .. currentmodule:: sed_eval.util.event_list .. autofunction:: filter_event_list ================================================ FILE: docs/_sources/generated/sed_eval.util.event_list.max_event_offset.rst.txt ================================================ sed_eval.util.event_list.max_event_offset ========================================= .. currentmodule:: sed_eval.util.event_list .. autofunction:: max_event_offset ================================================ FILE: docs/_sources/generated/sed_eval.util.event_list.unique_event_labels.rst.txt ================================================ sed_eval.util.event_list.unique_event_labels ============================================ .. currentmodule:: sed_eval.util.event_list .. autofunction:: unique_event_labels ================================================ FILE: docs/_sources/generated/sed_eval.util.event_list.unique_files.rst.txt ================================================ sed_eval.util.event_list.unique_files ===================================== .. currentmodule:: sed_eval.util.event_list .. autofunction:: unique_files ================================================ FILE: docs/_sources/generated/sed_eval.util.event_matching.bipartite_match.rst.txt ================================================ sed_eval.util.event_matching.bipartite_match ============================================ .. currentmodule:: sed_eval.util.event_matching .. autofunction:: bipartite_match ================================================ FILE: docs/_sources/generated/sed_eval.util.event_roll.event_list_to_event_roll.rst.txt ================================================ sed_eval.util.event_roll.event_list_to_event_roll ================================================= .. currentmodule:: sed_eval.util.event_roll .. autofunction:: event_list_to_event_roll ================================================ FILE: docs/_sources/generated/sed_eval.util.event_roll.match_event_roll_lengths.rst.txt ================================================ sed_eval.util.event_roll.match_event_roll_lengths ================================================= .. currentmodule:: sed_eval.util.event_roll .. autofunction:: match_event_roll_lengths ================================================ FILE: docs/_sources/generated/sed_eval.util.event_roll.pad_event_roll.rst.txt ================================================ sed_eval.util.event_roll.pad_event_roll ======================================= .. currentmodule:: sed_eval.util.event_roll .. autofunction:: pad_event_roll ================================================ FILE: docs/_sources/generated/sed_eval.util.scene_list.unique_scene_labels.rst.txt ================================================ sed_eval.util.scene_list.unique_scene_labels ============================================ .. currentmodule:: sed_eval.util.scene_list .. autofunction:: unique_scene_labels ================================================ FILE: docs/_sources/glossary.rst.txt ================================================ Glossary ======== .. glossary:: acoustic scene Descriptor for surrounding audio environment, for example "outdoor market", "busy street", "office". event label Textual description of sound event, usually one or two words. event offset End of the event instance as a time-stamp (in seconds). event onset Start of the event instance as a time-stamp (in seconds). macro-average Intermediate statistics are aggregated class-wise, class-based metrics are calculated, then average of class based metrics; each class has equal influence on the final metric value. (see :ref:`averaging`) micro-average Intermediate statistics are aggregated over all test data, then metrics are calculated; each instance has equal influence on the final metric value. (see :ref:`averaging`) scene label Textual label used to identify acoustic scene. sound event Audio segment that is attributed to a specific sound source and is perceived as an entity. Marked as having onset and offset and labeled with textual descriptors related to the sound source, for example "dog barking", "car passing by". ================================================ FILE: docs/_sources/index.rst.txt ================================================ ``sed_eval`` - Evaluation toolbox for Sound Event Detection =========================================================== - Toni Heittola (toni.heittola@tut.fi, `GitHub `_, `Home `_) - Annamaria Mesaros (annamaria.mesaros@tut.fi, `Home `_) .. figure:: _static/evaluation_overview.png :target: _static/evaluation_overview.png :align: center :width: 100% ``sed_eval`` is an open source Python toolbox which provides a standardized, and transparent way to evaluate sound event detection systems (see :ref:`sound_event`). In addition to this, it provides tools for evaluating acoustic scene classification systems, as the fields are closely related (see :ref:`scene`). The toolbox can be used in any of the following ways: * By using the included evaluator scripts directly (see :ref:`install` and :ref:`evaluators`). This is suitable if the system to be evaluated is implemented using some other platform than Python. * By importing it and calling it from your own Python code (see :ref:`install` and :ref:`sed_eval_quickstart`) Citing ------ If you use ``sed_eval`` in a research project, please cite the following paper: Annamaria Mesaros, Toni Heittola, and Tuomas Virtanen, "Metrics for polyphonic sound event detection", Applied Sciences, 6(6):162, 2016 [`HTML `_][`PDF `_] .. _installation: Getting started --------------- .. toctree:: :maxdepth: 1 install tutorial API documentation ----------------- .. toctree:: :maxdepth: 1 sound_event scene audio_tag metric test util io Reference --------- .. toctree:: :maxdepth: 1 glossary changelog * :ref:`genindex` ================================================ FILE: docs/_sources/install.rst.txt ================================================ .. _install: Installation instructions ========================= Using pip --------- The latest stable release is available on PyPI, and you can install with pip:: pip install sed_eval Alternatively you can download or clone toolbox and use ``pip`` to handle dependencies:: unzip sed_eval.zip pip install -e sed_eval or:: git clone https://github.com/TUT-ARG/sed_eval.git pip install -e sed_eval Using ``setyp.py`` ------------------ You can install ``sed_eval`` from source by first installing the dependencies:: pip install -r requirements.txt and then running:: python setup.py install To uninstall the toolbox if it was installed with ``setyp.py``: ``python setup.py install --record files.txt`` to get files associated with toolbox ``cat files.txt | xargs rm -rf`` to remove the files recorded by the previous step. You can also install the toolbox in *develop* mode:: python setup.py develop Toolbox can be uninstalled:: python setup.py develop --uninstall Requirements ------------ Following libraries are required: - numpy >= 1.7.0 - dcase_util >= 0.2.4 ================================================ FILE: docs/_sources/io.rst.txt ================================================ .. _io: .. automodule:: sed_eval.io ================================================ FILE: docs/_sources/metric.rst.txt ================================================ .. _metric: Metrics ================== .. automodule:: sed_eval.metric ================================================ FILE: docs/_sources/scene.rst.txt ================================================ .. _scene: Acoustic Scene Classification ============================= The goal of acoustic scene classification is to classify a test recording into one of predefined classes that characterizes the environment in which it was recorded — for example "outdoor market", "busy street", "office". Classification performance is measured using accuracy: the number of correctly classified segments among the total number of test segments. .. automodule:: sed_eval.scene ================================================ FILE: docs/_sources/sound_event.rst.txt ================================================ .. _sound_event: Sound Event Detection ===================== The task of sound event detection involves locating and classifying sounds in audio recordings - estimating onset and offset for distinct sound event instances and providing a textual descriptor for each. The usual approach for this problem is supervised learning with sound event classes defined in advance. Metrics are defined for polyphonic sound event detection, in which the ground truth and system output contain overlapping sound event instances. Two types of metrics are implemented: - **segment-based metrics** - the ground truth and system output are compared in a fixed time grid; sound events are marked as active or inactive in each segment; - **event-based metrics** - the ground truth and system output are compared at event instance level; Intermediate statistics ----------------------- Segment-based ^^^^^^^^^^^^^ - *true positive*: the ground truth and system output both indicate an event to be active in that segment - *false positive*: the ground truth indicates an event to be inactive in that segment, but the system output indicates it as active - *false negative*: the ground truth indicates an event to be active in that segment, but the system output indicates it as inactive. - *true negative*: the ground truth and system output both indicate an event to be inactive. Segment-based metrics implementation allow selecting the desired segment length for evaluation (see ``time_resolution`` parameter). Event-based ^^^^^^^^^^^ - *true positive*: an event in the system output that has a temporal position overlapping with the temporal position of an event with the same label in the ground truth. A *collar* is usually allowed for the onset and offset, or a tolerance with respect to the ground truth event duration. - *false positive*: an event in the system output that has no correspondence to an event with same label in the ground truth within the allowed tolerance; - *false negative*: an event in the ground truth that has no correspondence to an event with same label in the system output within the allowed tolerance. - *true negative*: event-based metrics have no meaningful true negatives. Event-based metrics implementation allow selecting the desired collar size (see ``t_collar`` parameter) and use of onset only or onset and offset conditions for evaluation (see ``evaluate_onset`` and ``evaluate_offset`` parameters). .. _averaging: Averaging ^^^^^^^^^ **Micro-averaging** - intermediate statistics are aggregated over all test data, then metrics are calculated; each instance has equal influence on the final metric value; **Macro-averaging** - intermediate statistics are aggregated class-wise, class-based metrics are calculated, then average of class based metrics; each class has equal influence on the final metric value. Micro and macro averages can result in very different values when classes are highly unbalanced or performance on individual classes is very different. Cross-validation ^^^^^^^^^^^^^^^^ Recommended calculation for a cross-validation setup is to run all train/test folds and perform evaluation at the end (no fold-wise evaluation!). The reason is that folds are most often unbalanced due to the multilabel nature of the problem, and this results in biases when averaging. For more details, consult [1]_. Implemented metrics ------------------- Precision, Recall and F-score ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. math:: P=\frac{TP}{TP+FP},\quad R=\frac{TP}{TP+FN},\quad F=\frac{2 \cdot P \cdot R}{P+R} These can be calculated segment based or event based, micro or macro averaged. Sensitivity and specificity ^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. math:: Sensitivity = \frac{TP}{TP+FN},\quad Specificity = \frac{TN}{TN+FP} Accuracy ^^^^^^^^ .. math:: accuracy = \frac{TP+TN}{TP+TN+FP+FN} .. math:: accuracy2 = \frac{TP}{TP+FP+FN} Balanced accuracy ^^^^^^^^^^^^^^^^^ .. math:: BACC = factor \cdot \frac{TP}{TP+FN} +(1-factor) \cdot \frac{TN}{TN+FP} Specificity and accuracy variants are only calculated as segment-based metrics. Error Rate ^^^^^^^^^^ **Segment-based** **Substitutions** in segment *k* - *S(k)* - the number of ground truth events for which a correct event was not output, yet something else was. One substitution is equivalent to having one false positives and one false negatives in the same segment. There is no need to designate which erroneous event substitutes which. **Insertions** in segment *k* - *I(k)* - events in system output that are not correct (false positives after substitutions are accounted for). **Deletions** in segment *k* - *D(k)* - events in ground truth that are not correct (false positives after substitutions are accounted for). .. math:: &S(k) = min(FN(k),FP(k)) \nonumber \\ &D(k) = max(0,FN(k)-FP(k)) \\ &I(k)= max(0,FP(k)-FN(k)) \nonumber .. math:: ER=\frac{\sum_{k=1}^K{S(k)}+\sum_{k=1}^K{D(k)}+\sum_{k=1}^K{I(k)}}{\sum_{k=1}^K{N(k)}} *N(k)* is the number of events in segment *k* in ground truth. **Event-based** **Substitutions** - events in system output with correct temporal position but incorrect class label **Insertions** - events in system output unaccounted for as correct or substituted **Deletions** - events in ground truth unaccounted for as correct or substituted .. math:: ER=\frac{S + D + I}{N} *N* is the total number of events in ground truth. Code ---- .. automodule:: sed_eval.sound_event Rerefences ---------- .. [1] Forman, G. and Scholz, M. "Apples-to-Apples in Cross-Validation Studies: Pitfalls in Classifier Performance Measurement". SIGKDD Explor. Newsl. 12, 1, November 2010, pp. 49-57. http://kdd.org/exploration_files/v12-1-p49-forman-sigkdd.pdf ================================================ FILE: docs/_sources/test.rst.txt ================================================ .. _test: Tests ===== .. automodule:: sed_eval.test ================================================ FILE: docs/_sources/tutorial.rst.txt ================================================ Tutorial ======== .. _sed_eval: ``sed_eval`` -- Evaluation toolbox for Sound Event Detection ------------------------------------------------------------ The structure of the ``sed_eval`` toolbox is as follows: * For evaluating the **sound event detection system** (SED system later), there are two types of metrics available: segment-based and event-based. For both types, there is a metric class ``SegmentBasedMetrics`` and ``EventBasedMetrics``. A member function ``evaluate()`` is used to go through system output (estimated event list) and ground truth (reference event list) pairs. The ``results()`` function is used to get the metric values in dictionary. There are also functions to return results as a formatted string for convenience (e.g. ``result_report_overall()``), or one can just print class instance. * For evaluating the **acoustic scene classification system**, there is similar evaluation class, ``SceneClassificationMetrics``, as for SED system evaluation. ``sed_eval`` also includes the following additional submodules: * ``io`` which contains convenience functions for loading annotations * ``util`` which includes miscellaneous functions to handle event lists (list of event items), event roll (event activity indicator matrix used in evaluation), and scene list. .. _evaluators: Quickstart: Using the evaluators -------------------------------- The easiest way to evaluate systems with ``sed_eval`` is to use provided evaluators. Evaluators are Python scripts which can be run from the command prompt and utilize ``sed_eval`` to compute metrics according to reference and estimated annotations you provide. To use the evaluators, you must first install ``sed_eval`` and its dependencies (see :ref:`installation`). The evaluator scripts can be found in the ``sed_eval`` repository in the ``evaluators`` folder: https://github.com/TUT-ARG/sed_eval/tree/master/evaluators Currently there are two evaluators available, one for evaluating the sound event detection systems and one for evaluating acoustic scene classification systems. Sound event detection ^^^^^^^^^^^^^^^^^^^^^ To get usage help: ``./sound_event_eval.py --help`` Evaluator takes as argument a csv-formatted file-list. The list contains pairs of filenames, one pair per row: first the filename of the reference event list file and the second the estimated event list file. Format is [reference_file][delimiter][estimated_file], and supported delimiters are ``,``, ``;``, ``tab``. Example of file-list:: office_snr0_high_v2.txt office_snr0_high_v2_detected.txt office_snr0_med_v2.txt office_snr0_med_v2_detected.txt Event list is csv-formatted text-file. Supported formats for the file are: - [event onset (float >= 0)][delimiter][event offset (float >= 0)] - [event onset (float >= 0)][delimiter][event offset (float >= 0)][delimiter][label] - [filename][delimiter][scene_label][delimiter][event onset (float >= 0)][delimiter][event offset (float >= 0)][delimiter][event label] Supported delimiters: ``,``, ``;``, ``tab`` Example of event list file:: 21.64715 23.00552 alert 36.91184 38.27021 alert 69.72575 71.09029 alert 63.53990 64.89827 alert 84.25553 84.83920 alert 20.92974 21.82661 clearthroat 28.39992 29.29679 clearthroat 80.47837 81.95937 clearthroat 44.48363 45.96463 clearthroat 78.13073 79.05953 clearthroat 15.17031 16.27235 cough 20.54931 21.65135 cough 27.79964 28.90168 cough 75.45959 76.32490 cough 70.81708 71.91912 cough 21.23203 22.55902 doorslam 7.546220 9.014880 doorslam 34.11303 35.04183 doorslam 45.86001 47.32867 doorslam To get segment-based and event-based metrics report printed, run: ``./sound_event_eval.py file_list.txt`` To get segment-based and event-based metrics saved in YAML-format, run: ``./sound_event_eval.py file_list.txt -o results.yaml`` Acoustic scene classification ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``./scene_eval.py --help`` Evaluator takes as argument a csv-formatted file-list. The list contains pairs of filenames, one pair per row: first the filename of the reference scene list file and the second the estimated scene list file. Format is [reference_file][delimiter][estimated_file], and supported delimiters are ``,``, ``;``, ``tab``. Example of file-list:: fold1_reference.txt fold1_estimated.txt fold2_reference.txt fold2_estimated.txt fold3_reference.txt fold3_estimated.txt fold4_reference.txt fold4_estimated.txt fold5_reference.txt fold5_estimated.txt Scene list is csv-formatted text-file. Supported formats for the file are: - [filename][delimiter][scene label] - [filename][delimiter][segment start (float >= 0)][delimiter][segment stop (float >= 0)][delimiter][scene label] Supported delimiters: ``,``, ``;``, ``tab`` Example of scene list file:: scenes_stereo/supermarket09.wav supermarket scenes_stereo/tubestation10.wav tubestation scenes_stereo/quietstreet08.wav quietstreet scenes_stereo/restaurant05.wav restaurant scenes_stereo/busystreet05.wav busystreet scenes_stereo/openairmarket04.wav openairmarket scenes_stereo/quietstreet01.wav quietstreet scenes_stereo/supermarket05.wav supermarket scenes_stereo/openairmarket01.wav openairmarket To get metrics printed, run: ``./scene_eval.py file_list.txt`` To get metrics saved in YAML-format, run: ``./scene_eval.py file_list.txt -o results.yaml`` .. _sed_eval_quickstart: Quickstart: Using ``sed_eval`` in Python code --------------------------------------------- After ``sed_eval`` is installed (see :ref:`installation`), it can be imported to your Python code as follows: ``import sed_eval`` Sound event detection ^^^^^^^^^^^^^^^^^^^^^ Usage example when reading event lists from disk (you can run example in path ``tests/data/sound_event``): .. code-block:: python :linenos: import sed_eval import dcase_util file_list = [ { 'reference_file': 'office_snr0_high_v2.txt', 'estimated_file': 'office_snr0_high_v2_detected.txt' }, { 'reference_file': 'office_snr0_med_v2.txt', 'estimated_file': 'office_snr0_med_v2_detected.txt' } ] data = [] # Get used event labels all_data = dcase_util.containers.MetaDataContainer() for file_pair in file_list: reference_event_list = sed_eval.io.load_event_list( filename=file_pair['reference_file'] ) estimated_event_list = sed_eval.io.load_event_list( filename=file_pair['estimated_file'] ) data.append({'reference_event_list': reference_event_list, 'estimated_event_list': estimated_event_list}) all_data += reference_event_list event_labels = all_data.unique_event_labels # Start evaluating # Create metrics classes, define parameters segment_based_metrics = sed_eval.sound_event.SegmentBasedMetrics( event_label_list=event_labels, time_resolution=1.0 ) event_based_metrics = sed_eval.sound_event.EventBasedMetrics( event_label_list=event_labels, t_collar=0.250 ) # Go through files for file_pair in data: segment_based_metrics.evaluate( reference_event_list=file_pair['reference_event_list'], estimated_event_list=file_pair['estimated_event_list'] ) event_based_metrics.evaluate( reference_event_list=file_pair['reference_event_list'], estimated_event_list=file_pair['estimated_event_list'] ) # Get only certain metrics overall_segment_based_metrics = segment_based_metrics.results_overall_metrics() print("Accuracy:", overall_segment_based_metrics['accuracy']['accuracy']) # Or print all metrics as reports print(segment_based_metrics) print(event_based_metrics) Usage example to evaluate results stored in variables: .. code-block:: python :linenos: import sed_eval import dcase_util reference_event_list = dcase_util.containers.MetaDataContainer( [ { 'event_label': 'car', 'event_onset': 0.0, 'event_offset': 2.5, 'file': 'audio/street/b099.wav', 'scene_label': 'street' }, { 'event_label': 'car', 'event_onset': 2.8, 'event_offset': 4.5, 'file': 'audio/street/b099.wav', 'scene_label': 'street' }, { 'event_label': 'car', 'event_onset': 6.0, 'event_offset': 10.0, 'file': 'audio/street/b099.wav', 'scene_label': 'street' } ] ) estimated_event_list = dcase_util.containers.MetaDataContainer( [ { 'event_label': 'car', 'event_onset': 1.0, 'event_offset': 3.5, 'file': 'audio/street/b099.wav', 'scene_label': 'street' }, { 'event_label': 'car', 'event_onset': 7.0, 'event_offset': 8.0, 'file': 'audio/street/b099.wav', 'scene_label': 'street' } ] ) segment_based_metrics = sed_eval.sound_event.SegmentBasedMetrics( event_label_list=reference_event_list.unique_event_labels, time_resolution=1.0 ) event_based_metrics = sed_eval.sound_event.EventBasedMetrics( event_label_list=reference_event_list.unique_event_labels, t_collar=0.250 ) for filename in reference_event_list.unique_files: reference_event_list_for_current_file = reference_event_list.filter( filename=filename ) estimated_event_list_for_current_file = estimated_event_list.filter( filename=filename ) segment_based_metrics.evaluate( reference_event_list=reference_event_list_for_current_file, estimated_event_list=estimated_event_list_for_current_file ) event_based_metrics.evaluate( reference_event_list=reference_event_list_for_current_file, estimated_event_list=estimated_event_list_for_current_file ) # Get only certain metrics overall_segment_based_metrics = segment_based_metrics.results_overall_metrics() print("Accuracy:", overall_segment_based_metrics['accuracy']['accuracy']) # Or print all metrics as reports print(segment_based_metrics) print(event_based_metrics) Acoustic scene classification ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Usage example to evaluate files: .. code-block:: python :linenos: import sed_eval import dcase_util file_list = [ {'reference_file': 'fold1_reference.txt', 'estimated_file': 'fold1_estimated.txt'} ] data = [] # Get used scene labels and load data in all_data = [] for file_pair in file_list: reference_scene_list = sed_eval.io.load_scene_list( filename=file_pair['reference_file'], csv_header=False, file_format=dcase_util.utils.FileFormat.CSV, fields=['filename', 'scene_label'] ) estimated_scene_list = sed_eval.io.load_scene_list( filename=file_pair['estimated_file'], csv_header=False, file_format=dcase_util.utils.FileFormat.CSV, fields=['filename', 'onset', 'offset', 'scene_label'] ) data.append( { 'reference_scene_list': reference_scene_list, 'estimated_scene_list': estimated_scene_list } ) all_data += reference_scene_list scene_labels = sed_eval.sound_event.util.unique_scene_labels(all_data) # Create metrics class scene_metrics = sed_eval.scene.SceneClassificationMetrics( scene_labels=scene_labels ) for file_pair in data: scene_metrics.evaluate( reference_scene_list=file_pair['reference_scene_list'], estimated_scene_list=file_pair['estimated_scene_list'] ) # Get only certain metrics overall_metrics_results = scene_metrics.results_overall_metrics() print("Accuracy:", overall_metrics_results['accuracy']) # Or print all metrics as reports print(scene_metrics) Usage example to evaluate results stored in variables: .. code-block:: python :linenos: import sed_eval import dcase_util reference = dcase_util.containers.MetaDataContainer([ { 'scene_label': 'supermarket', 'file': 'supermarket09.wav' }, { 'scene_label': 'tubestation', 'file': 'tubestation10.wav' }, { 'scene_label': 'quietstreet', 'file': 'quietstreet08.wav' }, { 'scene_label': 'office', 'file': 'office10.wav' }, { 'scene_label': 'bus', 'file': 'bus01.wav' }, ]) estimated = dcase_util.containers.MetaDataContainer([ { 'scene_label': 'supermarket', 'file': 'supermarket09.wav' }, { 'scene_label': 'bus', 'file': 'tubestation10.wav' }, { 'scene_label': 'quietstreet', 'file': 'quietstreet08.wav' }, { 'scene_label': 'park', 'file': 'office10.wav' }, { 'scene_label': 'car', 'file': 'bus01.wav' }, ]) scene_labels = sed_eval.sound_event.util.unique_scene_labels(reference) scene_metrics = sed_eval.scene.SceneClassificationMetrics(scene_labels) scene_metrics.evaluate( reference_scene_list=reference, estimated_scene_list=estimated ) print(scene_metrics) ================================================ FILE: docs/_sources/util.rst.txt ================================================ .. _util: .. automodule:: sed_eval.util ================================================ FILE: docs/_static/basic.css ================================================ /* * basic.css * ~~~~~~~~~ * * Sphinx stylesheet -- basic theme. * * :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ /* -- main layout ----------------------------------------------------------- */ div.clearer { clear: both; } /* -- relbar ---------------------------------------------------------------- */ div.related { width: 100%; font-size: 90%; } div.related h3 { display: none; } div.related ul { margin: 0; padding: 0 0 0 10px; list-style: none; } div.related li { display: inline; } div.related li.right { float: right; margin-right: 5px; } /* -- sidebar --------------------------------------------------------------- */ div.sphinxsidebarwrapper { padding: 10px 5px 0 10px; } div.sphinxsidebar { float: left; width: 230px; margin-left: -100%; font-size: 90%; word-wrap: break-word; overflow-wrap : break-word; } div.sphinxsidebar ul { list-style: none; } div.sphinxsidebar ul ul, div.sphinxsidebar ul.want-points { margin-left: 20px; list-style: square; } div.sphinxsidebar ul ul { margin-top: 0; margin-bottom: 0; } div.sphinxsidebar form { margin-top: 10px; } div.sphinxsidebar input { border: 1px solid #98dbcc; font-family: sans-serif; font-size: 1em; } div.sphinxsidebar #searchbox input[type="text"] { width: 170px; } img { border: 0; max-width: 100%; } /* -- search page ----------------------------------------------------------- */ ul.search { margin: 10px 0 0 20px; padding: 0; } ul.search li { padding: 5px 0 5px 20px; background-image: url(file.png); background-repeat: no-repeat; background-position: 0 7px; } ul.search li a { font-weight: bold; } ul.search li div.context { color: #888; margin: 2px 0 0 30px; text-align: left; } ul.keywordmatches li.goodmatch a { font-weight: bold; } /* -- index page ------------------------------------------------------------ */ table.contentstable { width: 90%; margin-left: auto; margin-right: auto; } table.contentstable p.biglink { line-height: 150%; } a.biglink { font-size: 1.3em; } span.linkdescr { font-style: italic; padding-top: 5px; font-size: 90%; } /* -- general index --------------------------------------------------------- */ table.indextable { width: 100%; } table.indextable td { text-align: left; vertical-align: top; } table.indextable ul { margin-top: 0; margin-bottom: 0; list-style-type: none; } table.indextable > tbody > tr > td > ul { padding-left: 0em; } table.indextable tr.pcap { height: 10px; } table.indextable tr.cap { margin-top: 10px; background-color: #f2f2f2; } img.toggler { margin-right: 3px; margin-top: 3px; cursor: pointer; } div.modindex-jumpbox { border-top: 1px solid #ddd; border-bottom: 1px solid #ddd; margin: 1em 0 1em 0; padding: 0.4em; } div.genindex-jumpbox { border-top: 1px solid #ddd; border-bottom: 1px solid #ddd; margin: 1em 0 1em 0; padding: 0.4em; } /* -- domain module index --------------------------------------------------- */ table.modindextable td { padding: 2px; border-collapse: collapse; } /* -- general body styles --------------------------------------------------- */ div.body p, div.body dd, div.body li, div.body blockquote { -moz-hyphens: auto; -ms-hyphens: auto; -webkit-hyphens: auto; hyphens: auto; } a.headerlink { visibility: hidden; } h1:hover > a.headerlink, h2:hover > a.headerlink, h3:hover > a.headerlink, h4:hover > a.headerlink, h5:hover > a.headerlink, h6:hover > a.headerlink, dt:hover > a.headerlink, caption:hover > a.headerlink, p.caption:hover > a.headerlink, div.code-block-caption:hover > a.headerlink { visibility: visible; } div.body p.caption { text-align: inherit; } div.body td { text-align: left; } .first { margin-top: 0 !important; } p.rubric { margin-top: 30px; font-weight: bold; } img.align-left, .figure.align-left, object.align-left { clear: left; float: left; margin-right: 1em; } img.align-right, .figure.align-right, object.align-right { clear: right; float: right; margin-left: 1em; } img.align-center, .figure.align-center, object.align-center { display: block; margin-left: auto; margin-right: auto; } .align-left { text-align: left; } .align-center { text-align: center; } .align-right { text-align: right; } /* -- sidebars -------------------------------------------------------------- */ div.sidebar { margin: 0 0 0.5em 1em; border: 1px solid #ddb; padding: 7px 7px 0 7px; background-color: #ffe; width: 40%; float: right; } p.sidebar-title { font-weight: bold; } /* -- topics ---------------------------------------------------------------- */ div.topic { border: 1px solid #ccc; padding: 7px 7px 0 7px; margin: 10px 0 10px 0; } p.topic-title { font-size: 1.1em; font-weight: bold; margin-top: 10px; } /* -- admonitions ----------------------------------------------------------- */ div.admonition { margin-top: 10px; margin-bottom: 10px; padding: 7px; } div.admonition dt { font-weight: bold; } div.admonition dl { margin-bottom: 0; } p.admonition-title { margin: 0px 10px 5px 0px; font-weight: bold; } div.body p.centered { text-align: center; margin-top: 25px; } /* -- tables ---------------------------------------------------------------- */ table.docutils { border: 0; border-collapse: collapse; } table caption span.caption-number { font-style: italic; } table caption span.caption-text { } table.docutils td, table.docutils th { padding: 1px 8px 1px 5px; border-top: 0; border-left: 0; border-right: 0; border-bottom: 1px solid #aaa; } table.footnote td, table.footnote th { border: 0 !important; } th { text-align: left; padding-right: 5px; } table.citation { border-left: solid 1px gray; margin-left: 1px; } table.citation td { border-bottom: none; } /* -- figures --------------------------------------------------------------- */ div.figure { margin: 0.5em; padding: 0.5em; } div.figure p.caption { padding: 0.3em; } div.figure p.caption span.caption-number { font-style: italic; } div.figure p.caption span.caption-text { } /* -- field list styles ----------------------------------------------------- */ table.field-list td, table.field-list th { border: 0 !important; } .field-list ul { margin: 0; padding-left: 1em; } .field-list p { margin: 0; } /* -- other body styles ----------------------------------------------------- */ ol.arabic { list-style: decimal; } ol.loweralpha { list-style: lower-alpha; } ol.upperalpha { list-style: upper-alpha; } ol.lowerroman { list-style: lower-roman; } ol.upperroman { list-style: upper-roman; } dl { margin-bottom: 15px; } dd p { margin-top: 0px; } dd ul, dd table { margin-bottom: 10px; } dd { margin-top: 3px; margin-bottom: 10px; margin-left: 30px; } dt:target, .highlighted { background-color: #fbe54e; } dl.glossary dt { font-weight: bold; font-size: 1.1em; } .optional { font-size: 1.3em; } .sig-paren { font-size: larger; } .versionmodified { font-style: italic; } .system-message { background-color: #fda; padding: 5px; border: 3px solid red; } .footnote:target { background-color: #ffa; } .line-block { display: block; margin-top: 1em; margin-bottom: 1em; } .line-block .line-block { margin-top: 0; margin-bottom: 0; margin-left: 1.5em; } .guilabel, .menuselection { font-family: sans-serif; } .accelerator { text-decoration: underline; } .classifier { font-style: oblique; } abbr, acronym { border-bottom: dotted 1px; cursor: help; } /* -- code displays --------------------------------------------------------- */ pre { overflow: auto; overflow-y: hidden; /* fixes display issues on Chrome browsers */ } span.pre { -moz-hyphens: none; -ms-hyphens: none; -webkit-hyphens: none; hyphens: none; } td.linenos pre { padding: 5px 0px; border: 0; background-color: transparent; color: #aaa; } table.highlighttable { margin-left: 0.5em; } table.highlighttable td { padding: 0 0.5em 0 0.5em; } div.code-block-caption { padding: 2px 5px; font-size: small; } div.code-block-caption code { background-color: transparent; } div.code-block-caption + div > div.highlight > pre { margin-top: 0; } div.code-block-caption span.caption-number { padding: 0.1em 0.3em; font-style: italic; } div.code-block-caption span.caption-text { } div.literal-block-wrapper { padding: 1em 1em 0; } div.literal-block-wrapper div.highlight { margin: 0; } code.descname { background-color: transparent; font-weight: bold; font-size: 1.2em; } code.descclassname { background-color: transparent; } code.xref, a code { background-color: transparent; font-weight: bold; } h1 code, h2 code, h3 code, h4 code, h5 code, h6 code { background-color: transparent; } .viewcode-link { float: right; } .viewcode-back { float: right; font-family: sans-serif; } div.viewcode-block:target { margin: -1px -10px; padding: 0 10px; } /* -- math display ---------------------------------------------------------- */ img.math { vertical-align: middle; } div.body div.math p { text-align: center; } span.eqno { float: right; } span.eqno a.headerlink { position: relative; left: 0px; z-index: 1; } div.math:hover a.headerlink { visibility: visible; } /* -- printout stylesheet --------------------------------------------------- */ @media print { div.document, div.documentwrapper, div.bodywrapper { margin: 0 !important; width: 100%; } div.sphinxsidebar, div.related, div.footer, #top-link { display: none; } } ================================================ FILE: docs/_static/css/badge_only.css ================================================ .fa:before{-webkit-font-smoothing:antialiased}.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;content:""}.clearfix:after{clear:both}@font-face{font-family:FontAwesome;font-weight:normal;font-style:normal;src:url("../font/fontawesome_webfont.eot");src:url("../font/fontawesome_webfont.eot?#iefix") format("embedded-opentype"),url("../font/fontawesome_webfont.woff") format("woff"),url("../font/fontawesome_webfont.ttf") format("truetype"),url("../font/fontawesome_webfont.svg#FontAwesome") format("svg")}.fa:before{display:inline-block;font-family:FontAwesome;font-style:normal;font-weight:normal;line-height:1;text-decoration:inherit}a .fa{display:inline-block;text-decoration:inherit}li .fa{display:inline-block}li .fa-large:before,li .fa-large:before{width:1.875em}ul.fas{list-style-type:none;margin-left:2em;text-indent:-0.8em}ul.fas li .fa{width:0.8em}ul.fas li .fa-large:before,ul.fas li .fa-large:before{vertical-align:baseline}.fa-book:before{content:""}.icon-book:before{content:""}.fa-caret-down:before{content:""}.icon-caret-down:before{content:""}.fa-caret-up:before{content:""}.icon-caret-up:before{content:""}.fa-caret-left:before{content:""}.icon-caret-left:before{content:""}.fa-caret-right:before{content:""}.icon-caret-right:before{content:""}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;border-top:solid 10px #343131;font-family:"Lato","proxima-nova","Helvetica Neue",Arial,sans-serif;z-index:400}.rst-versions a{color:#2980B9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27AE60;*zoom:1}.rst-versions .rst-current-version:before,.rst-versions .rst-current-version:after{display:table;content:""}.rst-versions .rst-current-version:after{clear:both}.rst-versions .rst-current-version .fa{color:#fcfcfc}.rst-versions .rst-current-version .fa-book{float:left}.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#E74C3C;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#F1C40F;color:#000}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:gray;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:solid 1px #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px}.rst-versions.rst-badge .icon-book{float:none}.rst-versions.rst-badge .fa-book{float:none}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book{float:left}.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge .rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width: 768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}} /*# sourceMappingURL=badge_only.css.map */ ================================================ FILE: docs/_static/css/theme.css ================================================ *{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}article,aside,details,figcaption,figure,footer,header,hgroup,nav,section{display:block}audio,canvas,video{display:inline-block;*display:inline;*zoom:1}audio:not([controls]){display:none}[hidden]{display:none}*{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}html{font-size:100%;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%}body{margin:0}a:hover,a:active{outline:0}abbr[title]{border-bottom:1px dotted}b,strong{font-weight:bold}blockquote{margin:0}dfn{font-style:italic}ins{background:#ff9;color:#000;text-decoration:none}mark{background:#ff0;color:#000;font-style:italic;font-weight:bold}pre,code,.rst-content tt,.rst-content code,kbd,samp{font-family:monospace,serif;_font-family:"courier new",monospace;font-size:1em}pre{white-space:pre}q{quotes:none}q:before,q:after{content:"";content:none}small{font-size:85%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sup{top:-0.5em}sub{bottom:-0.25em}ul,ol,dl{margin:0;padding:0;list-style:none;list-style-image:none}li{list-style:none}dd{margin:0}img{border:0;-ms-interpolation-mode:bicubic;vertical-align:middle;max-width:100%}svg:not(:root){overflow:hidden}figure{margin:0}form{margin:0}fieldset{border:0;margin:0;padding:0}label{cursor:pointer}legend{border:0;*margin-left:-7px;padding:0;white-space:normal}button,input,select,textarea{font-size:100%;margin:0;vertical-align:baseline;*vertical-align:middle}button,input{line-height:normal}button,input[type="button"],input[type="reset"],input[type="submit"]{cursor:pointer;-webkit-appearance:button;*overflow:visible}button[disabled],input[disabled]{cursor:default}input[type="checkbox"],input[type="radio"]{box-sizing:border-box;padding:0;*width:13px;*height:13px}input[type="search"]{-webkit-appearance:textfield;-moz-box-sizing:content-box;-webkit-box-sizing:content-box;box-sizing:content-box}input[type="search"]::-webkit-search-decoration,input[type="search"]::-webkit-search-cancel-button{-webkit-appearance:none}button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0}textarea{overflow:auto;vertical-align:top;resize:vertical}table{border-collapse:collapse;border-spacing:0}td{vertical-align:top}.chromeframe{margin:0.2em 0;background:#ccc;color:#000;padding:0.2em 0}.ir{display:block;border:0;text-indent:-999em;overflow:hidden;background-color:transparent;background-repeat:no-repeat;text-align:left;direction:ltr;*line-height:0}.ir br{display:none}.hidden{display:none !important;visibility:hidden}.visuallyhidden{border:0;clip:rect(0 0 0 0);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px}.visuallyhidden.focusable:active,.visuallyhidden.focusable:focus{clip:auto;height:auto;margin:0;overflow:visible;position:static;width:auto}.invisible{visibility:hidden}.relative{position:relative}big,small{font-size:100%}@media print{html,body,section{background:none !important}*{box-shadow:none !important;text-shadow:none !important;filter:none !important;-ms-filter:none !important}a,a:visited{text-decoration:underline}.ir a:after,a[href^="javascript:"]:after,a[href^="#"]:after{content:""}pre,blockquote{page-break-inside:avoid}thead{display:table-header-group}tr,img{page-break-inside:avoid}img{max-width:100% !important}@page{margin:0.5cm}p,h2,.rst-content .toctree-wrapper p.caption,h3{orphans:3;widows:3}h2,.rst-content .toctree-wrapper p.caption,h3{page-break-after:avoid}}.fa:before,.wy-menu-vertical li span.toctree-expand:before,.wy-menu-vertical li.on a span.toctree-expand:before,.wy-menu-vertical li.current>a span.toctree-expand:before,.rst-content .admonition-title:before,.rst-content h1 .headerlink:before,.rst-content h2 .headerlink:before,.rst-content h3 .headerlink:before,.rst-content h4 .headerlink:before,.rst-content h5 .headerlink:before,.rst-content h6 .headerlink:before,.rst-content dl dt .headerlink:before,.rst-content p.caption .headerlink:before,.rst-content tt.download span:first-child:before,.rst-content code.download span:first-child:before,.icon:before,.wy-dropdown .caret:before,.wy-inline-validate.wy-inline-validate-success .wy-input-context:before,.wy-inline-validate.wy-inline-validate-danger .wy-input-context:before,.wy-inline-validate.wy-inline-validate-warning .wy-input-context:before,.wy-inline-validate.wy-inline-validate-info .wy-input-context:before,.wy-alert,.rst-content .note,.rst-content .attention,.rst-content .caution,.rst-content .danger,.rst-content .error,.rst-content .hint,.rst-content .important,.rst-content .tip,.rst-content .warning,.rst-content .seealso,.rst-content .admonition-todo,.btn,input[type="text"],input[type="password"],input[type="email"],input[type="url"],input[type="date"],input[type="month"],input[type="time"],input[type="datetime"],input[type="datetime-local"],input[type="week"],input[type="number"],input[type="search"],input[type="tel"],input[type="color"],select,textarea,.wy-menu-vertical li.on a,.wy-menu-vertical li.current>a,.wy-side-nav-search>a,.wy-side-nav-search .wy-dropdown>a,.wy-nav-top a{-webkit-font-smoothing:antialiased}.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;content:""}.clearfix:after{clear:both}/*! * Font Awesome 4.6.3 by @davegandy - http://fontawesome.io - @fontawesome * License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License) */@font-face{font-family:'FontAwesome';src:url("../fonts/fontawesome-webfont.eot?v=4.6.3");src:url("../fonts/fontawesome-webfont.eot?#iefix&v=4.6.3") format("embedded-opentype"),url("../fonts/fontawesome-webfont.woff2?v=4.6.3") format("woff2"),url("../fonts/fontawesome-webfont.woff?v=4.6.3") format("woff"),url("../fonts/fontawesome-webfont.ttf?v=4.6.3") format("truetype"),url("../fonts/fontawesome-webfont.svg?v=4.6.3#fontawesomeregular") format("svg");font-weight:normal;font-style:normal}.fa,.wy-menu-vertical li span.toctree-expand,.wy-menu-vertical li.on a span.toctree-expand,.wy-menu-vertical li.current>a span.toctree-expand,.rst-content .admonition-title,.rst-content h1 .headerlink,.rst-content h2 .headerlink,.rst-content h3 .headerlink,.rst-content h4 .headerlink,.rst-content h5 .headerlink,.rst-content h6 .headerlink,.rst-content dl dt .headerlink,.rst-content p.caption .headerlink,.rst-content tt.download span:first-child,.rst-content code.download span:first-child,.icon{display:inline-block;font:normal normal normal 14px/1 FontAwesome;font-size:inherit;text-rendering:auto;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.fa-lg{font-size:1.33333em;line-height:.75em;vertical-align:-15%}.fa-2x{font-size:2em}.fa-3x{font-size:3em}.fa-4x{font-size:4em}.fa-5x{font-size:5em}.fa-fw{width:1.28571em;text-align:center}.fa-ul{padding-left:0;margin-left:2.14286em;list-style-type:none}.fa-ul>li{position:relative}.fa-li{position:absolute;left:-2.14286em;width:2.14286em;top:.14286em;text-align:center}.fa-li.fa-lg{left:-1.85714em}.fa-border{padding:.2em .25em .15em;border:solid 0.08em #eee;border-radius:.1em}.fa-pull-left{float:left}.fa-pull-right{float:right}.fa.fa-pull-left,.wy-menu-vertical li span.fa-pull-left.toctree-expand,.wy-menu-vertical li.on a span.fa-pull-left.toctree-expand,.wy-menu-vertical li.current>a span.fa-pull-left.toctree-expand,.rst-content .fa-pull-left.admonition-title,.rst-content h1 .fa-pull-left.headerlink,.rst-content h2 .fa-pull-left.headerlink,.rst-content h3 .fa-pull-left.headerlink,.rst-content h4 .fa-pull-left.headerlink,.rst-content h5 .fa-pull-left.headerlink,.rst-content h6 .fa-pull-left.headerlink,.rst-content dl dt .fa-pull-left.headerlink,.rst-content p.caption .fa-pull-left.headerlink,.rst-content tt.download span.fa-pull-left:first-child,.rst-content code.download span.fa-pull-left:first-child,.fa-pull-left.icon{margin-right:.3em}.fa.fa-pull-right,.wy-menu-vertical li span.fa-pull-right.toctree-expand,.wy-menu-vertical li.on a span.fa-pull-right.toctree-expand,.wy-menu-vertical li.current>a span.fa-pull-right.toctree-expand,.rst-content .fa-pull-right.admonition-title,.rst-content h1 .fa-pull-right.headerlink,.rst-content h2 .fa-pull-right.headerlink,.rst-content h3 .fa-pull-right.headerlink,.rst-content h4 .fa-pull-right.headerlink,.rst-content h5 .fa-pull-right.headerlink,.rst-content h6 .fa-pull-right.headerlink,.rst-content dl dt .fa-pull-right.headerlink,.rst-content p.caption .fa-pull-right.headerlink,.rst-content tt.download span.fa-pull-right:first-child,.rst-content code.download span.fa-pull-right:first-child,.fa-pull-right.icon{margin-left:.3em}.pull-right{float:right}.pull-left{float:left}.fa.pull-left,.wy-menu-vertical li span.pull-left.toctree-expand,.wy-menu-vertical li.on a span.pull-left.toctree-expand,.wy-menu-vertical li.current>a span.pull-left.toctree-expand,.rst-content .pull-left.admonition-title,.rst-content h1 .pull-left.headerlink,.rst-content h2 .pull-left.headerlink,.rst-content h3 .pull-left.headerlink,.rst-content h4 .pull-left.headerlink,.rst-content h5 .pull-left.headerlink,.rst-content h6 .pull-left.headerlink,.rst-content dl dt .pull-left.headerlink,.rst-content p.caption .pull-left.headerlink,.rst-content tt.download span.pull-left:first-child,.rst-content code.download span.pull-left:first-child,.pull-left.icon{margin-right:.3em}.fa.pull-right,.wy-menu-vertical li span.pull-right.toctree-expand,.wy-menu-vertical li.on a span.pull-right.toctree-expand,.wy-menu-vertical li.current>a span.pull-right.toctree-expand,.rst-content .pull-right.admonition-title,.rst-content h1 .pull-right.headerlink,.rst-content h2 .pull-right.headerlink,.rst-content h3 .pull-right.headerlink,.rst-content h4 .pull-right.headerlink,.rst-content h5 .pull-right.headerlink,.rst-content h6 .pull-right.headerlink,.rst-content dl dt .pull-right.headerlink,.rst-content p.caption .pull-right.headerlink,.rst-content tt.download span.pull-right:first-child,.rst-content code.download span.pull-right:first-child,.pull-right.icon{margin-left:.3em}.fa-spin{-webkit-animation:fa-spin 2s infinite linear;animation:fa-spin 2s infinite linear}.fa-pulse{-webkit-animation:fa-spin 1s infinite steps(8);animation:fa-spin 1s infinite steps(8)}@-webkit-keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}@keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}.fa-rotate-90{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=1)";-webkit-transform:rotate(90deg);-ms-transform:rotate(90deg);transform:rotate(90deg)}.fa-rotate-180{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2)";-webkit-transform:rotate(180deg);-ms-transform:rotate(180deg);transform:rotate(180deg)}.fa-rotate-270{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=3)";-webkit-transform:rotate(270deg);-ms-transform:rotate(270deg);transform:rotate(270deg)}.fa-flip-horizontal{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1)";-webkit-transform:scale(-1, 1);-ms-transform:scale(-1, 1);transform:scale(-1, 1)}.fa-flip-vertical{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1)";-webkit-transform:scale(1, -1);-ms-transform:scale(1, -1);transform:scale(1, -1)}:root .fa-rotate-90,:root .fa-rotate-180,:root .fa-rotate-270,:root .fa-flip-horizontal,:root .fa-flip-vertical{filter:none}.fa-stack{position:relative;display:inline-block;width:2em;height:2em;line-height:2em;vertical-align:middle}.fa-stack-1x,.fa-stack-2x{position:absolute;left:0;width:100%;text-align:center}.fa-stack-1x{line-height:inherit}.fa-stack-2x{font-size:2em}.fa-inverse{color:#fff}.fa-glass:before{content:""}.fa-music:before{content:""}.fa-search:before,.icon-search:before{content:""}.fa-envelope-o:before{content:""}.fa-heart:before{content:""}.fa-star:before{content:""}.fa-star-o:before{content:""}.fa-user:before{content:""}.fa-film:before{content:""}.fa-th-large:before{content:""}.fa-th:before{content:""}.fa-th-list:before{content:""}.fa-check:before{content:""}.fa-remove:before,.fa-close:before,.fa-times:before{content:""}.fa-search-plus:before{content:""}.fa-search-minus:before{content:""}.fa-power-off:before{content:""}.fa-signal:before{content:""}.fa-gear:before,.fa-cog:before{content:""}.fa-trash-o:before{content:""}.fa-home:before,.icon-home:before{content:""}.fa-file-o:before{content:""}.fa-clock-o:before{content:""}.fa-road:before{content:""}.fa-download:before,.rst-content tt.download span:first-child:before,.rst-content code.download span:first-child:before{content:""}.fa-arrow-circle-o-down:before{content:""}.fa-arrow-circle-o-up:before{content:""}.fa-inbox:before{content:""}.fa-play-circle-o:before{content:""}.fa-rotate-right:before,.fa-repeat:before{content:""}.fa-refresh:before{content:""}.fa-list-alt:before{content:""}.fa-lock:before{content:""}.fa-flag:before{content:""}.fa-headphones:before{content:""}.fa-volume-off:before{content:""}.fa-volume-down:before{content:""}.fa-volume-up:before{content:""}.fa-qrcode:before{content:""}.fa-barcode:before{content:""}.fa-tag:before{content:""}.fa-tags:before{content:""}.fa-book:before,.icon-book:before{content:""}.fa-bookmark:before{content:""}.fa-print:before{content:""}.fa-camera:before{content:""}.fa-font:before{content:""}.fa-bold:before{content:""}.fa-italic:before{content:""}.fa-text-height:before{content:""}.fa-text-width:before{content:""}.fa-align-left:before{content:""}.fa-align-center:before{content:""}.fa-align-right:before{content:""}.fa-align-justify:before{content:""}.fa-list:before{content:""}.fa-dedent:before,.fa-outdent:before{content:""}.fa-indent:before{content:""}.fa-video-camera:before{content:""}.fa-photo:before,.fa-image:before,.fa-picture-o:before{content:""}.fa-pencil:before{content:""}.fa-map-marker:before{content:""}.fa-adjust:before{content:""}.fa-tint:before{content:""}.fa-edit:before,.fa-pencil-square-o:before{content:""}.fa-share-square-o:before{content:""}.fa-check-square-o:before{content:""}.fa-arrows:before{content:""}.fa-step-backward:before{content:""}.fa-fast-backward:before{content:""}.fa-backward:before{content:""}.fa-play:before{content:""}.fa-pause:before{content:""}.fa-stop:before{content:""}.fa-forward:before{content:""}.fa-fast-forward:before{content:""}.fa-step-forward:before{content:""}.fa-eject:before{content:""}.fa-chevron-left:before{content:""}.fa-chevron-right:before{content:""}.fa-plus-circle:before{content:""}.fa-minus-circle:before{content:""}.fa-times-circle:before,.wy-inline-validate.wy-inline-validate-danger .wy-input-context:before{content:""}.fa-check-circle:before,.wy-inline-validate.wy-inline-validate-success .wy-input-context:before{content:""}.fa-question-circle:before{content:""}.fa-info-circle:before{content:""}.fa-crosshairs:before{content:""}.fa-times-circle-o:before{content:""}.fa-check-circle-o:before{content:""}.fa-ban:before{content:""}.fa-arrow-left:before{content:""}.fa-arrow-right:before{content:""}.fa-arrow-up:before{content:""}.fa-arrow-down:before{content:""}.fa-mail-forward:before,.fa-share:before{content:""}.fa-expand:before{content:""}.fa-compress:before{content:""}.fa-plus:before{content:""}.fa-minus:before{content:""}.fa-asterisk:before{content:""}.fa-exclamation-circle:before,.wy-inline-validate.wy-inline-validate-warning .wy-input-context:before,.wy-inline-validate.wy-inline-validate-info .wy-input-context:before,.rst-content .admonition-title:before{content:""}.fa-gift:before{content:""}.fa-leaf:before{content:""}.fa-fire:before,.icon-fire:before{content:""}.fa-eye:before{content:""}.fa-eye-slash:before{content:""}.fa-warning:before,.fa-exclamation-triangle:before{content:""}.fa-plane:before{content:""}.fa-calendar:before{content:""}.fa-random:before{content:""}.fa-comment:before{content:""}.fa-magnet:before{content:""}.fa-chevron-up:before{content:""}.fa-chevron-down:before{content:""}.fa-retweet:before{content:""}.fa-shopping-cart:before{content:""}.fa-folder:before{content:""}.fa-folder-open:before{content:""}.fa-arrows-v:before{content:""}.fa-arrows-h:before{content:""}.fa-bar-chart-o:before,.fa-bar-chart:before{content:""}.fa-twitter-square:before{content:""}.fa-facebook-square:before{content:""}.fa-camera-retro:before{content:""}.fa-key:before{content:""}.fa-gears:before,.fa-cogs:before{content:""}.fa-comments:before{content:""}.fa-thumbs-o-up:before{content:""}.fa-thumbs-o-down:before{content:""}.fa-star-half:before{content:""}.fa-heart-o:before{content:""}.fa-sign-out:before{content:""}.fa-linkedin-square:before{content:""}.fa-thumb-tack:before{content:""}.fa-external-link:before{content:""}.fa-sign-in:before{content:""}.fa-trophy:before{content:""}.fa-github-square:before{content:""}.fa-upload:before{content:""}.fa-lemon-o:before{content:""}.fa-phone:before{content:""}.fa-square-o:before{content:""}.fa-bookmark-o:before{content:""}.fa-phone-square:before{content:""}.fa-twitter:before{content:""}.fa-facebook-f:before,.fa-facebook:before{content:""}.fa-github:before,.icon-github:before{content:""}.fa-unlock:before{content:""}.fa-credit-card:before{content:""}.fa-feed:before,.fa-rss:before{content:""}.fa-hdd-o:before{content:""}.fa-bullhorn:before{content:""}.fa-bell:before{content:""}.fa-certificate:before{content:""}.fa-hand-o-right:before{content:""}.fa-hand-o-left:before{content:""}.fa-hand-o-up:before{content:""}.fa-hand-o-down:before{content:""}.fa-arrow-circle-left:before,.icon-circle-arrow-left:before{content:""}.fa-arrow-circle-right:before,.icon-circle-arrow-right:before{content:""}.fa-arrow-circle-up:before{content:""}.fa-arrow-circle-down:before{content:""}.fa-globe:before{content:""}.fa-wrench:before{content:""}.fa-tasks:before{content:""}.fa-filter:before{content:""}.fa-briefcase:before{content:""}.fa-arrows-alt:before{content:""}.fa-group:before,.fa-users:before{content:""}.fa-chain:before,.fa-link:before,.icon-link:before{content:""}.fa-cloud:before{content:""}.fa-flask:before{content:""}.fa-cut:before,.fa-scissors:before{content:""}.fa-copy:before,.fa-files-o:before{content:""}.fa-paperclip:before{content:""}.fa-save:before,.fa-floppy-o:before{content:""}.fa-square:before{content:""}.fa-navicon:before,.fa-reorder:before,.fa-bars:before{content:""}.fa-list-ul:before{content:""}.fa-list-ol:before{content:""}.fa-strikethrough:before{content:""}.fa-underline:before{content:""}.fa-table:before{content:""}.fa-magic:before{content:""}.fa-truck:before{content:""}.fa-pinterest:before{content:""}.fa-pinterest-square:before{content:""}.fa-google-plus-square:before{content:""}.fa-google-plus:before{content:""}.fa-money:before{content:""}.fa-caret-down:before,.wy-dropdown .caret:before,.icon-caret-down:before{content:""}.fa-caret-up:before{content:""}.fa-caret-left:before{content:""}.fa-caret-right:before{content:""}.fa-columns:before{content:""}.fa-unsorted:before,.fa-sort:before{content:""}.fa-sort-down:before,.fa-sort-desc:before{content:""}.fa-sort-up:before,.fa-sort-asc:before{content:""}.fa-envelope:before{content:""}.fa-linkedin:before{content:""}.fa-rotate-left:before,.fa-undo:before{content:""}.fa-legal:before,.fa-gavel:before{content:""}.fa-dashboard:before,.fa-tachometer:before{content:""}.fa-comment-o:before{content:""}.fa-comments-o:before{content:""}.fa-flash:before,.fa-bolt:before{content:""}.fa-sitemap:before{content:""}.fa-umbrella:before{content:""}.fa-paste:before,.fa-clipboard:before{content:""}.fa-lightbulb-o:before{content:""}.fa-exchange:before{content:""}.fa-cloud-download:before{content:""}.fa-cloud-upload:before{content:""}.fa-user-md:before{content:""}.fa-stethoscope:before{content:""}.fa-suitcase:before{content:""}.fa-bell-o:before{content:""}.fa-coffee:before{content:""}.fa-cutlery:before{content:""}.fa-file-text-o:before{content:""}.fa-building-o:before{content:""}.fa-hospital-o:before{content:""}.fa-ambulance:before{content:""}.fa-medkit:before{content:""}.fa-fighter-jet:before{content:""}.fa-beer:before{content:""}.fa-h-square:before{content:""}.fa-plus-square:before{content:""}.fa-angle-double-left:before{content:""}.fa-angle-double-right:before{content:""}.fa-angle-double-up:before{content:""}.fa-angle-double-down:before{content:""}.fa-angle-left:before{content:""}.fa-angle-right:before{content:""}.fa-angle-up:before{content:""}.fa-angle-down:before{content:""}.fa-desktop:before{content:""}.fa-laptop:before{content:""}.fa-tablet:before{content:""}.fa-mobile-phone:before,.fa-mobile:before{content:""}.fa-circle-o:before{content:""}.fa-quote-left:before{content:""}.fa-quote-right:before{content:""}.fa-spinner:before{content:""}.fa-circle:before{content:""}.fa-mail-reply:before,.fa-reply:before{content:""}.fa-github-alt:before{content:""}.fa-folder-o:before{content:""}.fa-folder-open-o:before{content:""}.fa-smile-o:before{content:""}.fa-frown-o:before{content:""}.fa-meh-o:before{content:""}.fa-gamepad:before{content:""}.fa-keyboard-o:before{content:""}.fa-flag-o:before{content:""}.fa-flag-checkered:before{content:""}.fa-terminal:before{content:""}.fa-code:before{content:""}.fa-mail-reply-all:before,.fa-reply-all:before{content:""}.fa-star-half-empty:before,.fa-star-half-full:before,.fa-star-half-o:before{content:""}.fa-location-arrow:before{content:""}.fa-crop:before{content:""}.fa-code-fork:before{content:""}.fa-unlink:before,.fa-chain-broken:before{content:""}.fa-question:before{content:""}.fa-info:before{content:""}.fa-exclamation:before{content:""}.fa-superscript:before{content:""}.fa-subscript:before{content:""}.fa-eraser:before{content:""}.fa-puzzle-piece:before{content:""}.fa-microphone:before{content:""}.fa-microphone-slash:before{content:""}.fa-shield:before{content:""}.fa-calendar-o:before{content:""}.fa-fire-extinguisher:before{content:""}.fa-rocket:before{content:""}.fa-maxcdn:before{content:""}.fa-chevron-circle-left:before{content:""}.fa-chevron-circle-right:before{content:""}.fa-chevron-circle-up:before{content:""}.fa-chevron-circle-down:before{content:""}.fa-html5:before{content:""}.fa-css3:before{content:""}.fa-anchor:before{content:""}.fa-unlock-alt:before{content:""}.fa-bullseye:before{content:""}.fa-ellipsis-h:before{content:""}.fa-ellipsis-v:before{content:""}.fa-rss-square:before{content:""}.fa-play-circle:before{content:""}.fa-ticket:before{content:""}.fa-minus-square:before{content:""}.fa-minus-square-o:before,.wy-menu-vertical li.on a span.toctree-expand:before,.wy-menu-vertical li.current>a span.toctree-expand:before{content:""}.fa-level-up:before{content:""}.fa-level-down:before{content:""}.fa-check-square:before{content:""}.fa-pencil-square:before{content:""}.fa-external-link-square:before{content:""}.fa-share-square:before{content:""}.fa-compass:before{content:""}.fa-toggle-down:before,.fa-caret-square-o-down:before{content:""}.fa-toggle-up:before,.fa-caret-square-o-up:before{content:""}.fa-toggle-right:before,.fa-caret-square-o-right:before{content:""}.fa-euro:before,.fa-eur:before{content:""}.fa-gbp:before{content:""}.fa-dollar:before,.fa-usd:before{content:""}.fa-rupee:before,.fa-inr:before{content:""}.fa-cny:before,.fa-rmb:before,.fa-yen:before,.fa-jpy:before{content:""}.fa-ruble:before,.fa-rouble:before,.fa-rub:before{content:""}.fa-won:before,.fa-krw:before{content:""}.fa-bitcoin:before,.fa-btc:before{content:""}.fa-file:before{content:""}.fa-file-text:before{content:""}.fa-sort-alpha-asc:before{content:""}.fa-sort-alpha-desc:before{content:""}.fa-sort-amount-asc:before{content:""}.fa-sort-amount-desc:before{content:""}.fa-sort-numeric-asc:before{content:""}.fa-sort-numeric-desc:before{content:""}.fa-thumbs-up:before{content:""}.fa-thumbs-down:before{content:""}.fa-youtube-square:before{content:""}.fa-youtube:before{content:""}.fa-xing:before{content:""}.fa-xing-square:before{content:""}.fa-youtube-play:before{content:""}.fa-dropbox:before{content:""}.fa-stack-overflow:before{content:""}.fa-instagram:before{content:""}.fa-flickr:before{content:""}.fa-adn:before{content:""}.fa-bitbucket:before,.icon-bitbucket:before{content:""}.fa-bitbucket-square:before{content:""}.fa-tumblr:before{content:""}.fa-tumblr-square:before{content:""}.fa-long-arrow-down:before{content:""}.fa-long-arrow-up:before{content:""}.fa-long-arrow-left:before{content:""}.fa-long-arrow-right:before{content:""}.fa-apple:before{content:""}.fa-windows:before{content:""}.fa-android:before{content:""}.fa-linux:before{content:""}.fa-dribbble:before{content:""}.fa-skype:before{content:""}.fa-foursquare:before{content:""}.fa-trello:before{content:""}.fa-female:before{content:""}.fa-male:before{content:""}.fa-gittip:before,.fa-gratipay:before{content:""}.fa-sun-o:before{content:""}.fa-moon-o:before{content:""}.fa-archive:before{content:""}.fa-bug:before{content:""}.fa-vk:before{content:""}.fa-weibo:before{content:""}.fa-renren:before{content:""}.fa-pagelines:before{content:""}.fa-stack-exchange:before{content:""}.fa-arrow-circle-o-right:before{content:""}.fa-arrow-circle-o-left:before{content:""}.fa-toggle-left:before,.fa-caret-square-o-left:before{content:""}.fa-dot-circle-o:before{content:""}.fa-wheelchair:before{content:""}.fa-vimeo-square:before{content:""}.fa-turkish-lira:before,.fa-try:before{content:""}.fa-plus-square-o:before,.wy-menu-vertical li span.toctree-expand:before{content:""}.fa-space-shuttle:before{content:""}.fa-slack:before{content:""}.fa-envelope-square:before{content:""}.fa-wordpress:before{content:""}.fa-openid:before{content:""}.fa-institution:before,.fa-bank:before,.fa-university:before{content:""}.fa-mortar-board:before,.fa-graduation-cap:before{content:""}.fa-yahoo:before{content:""}.fa-google:before{content:""}.fa-reddit:before{content:""}.fa-reddit-square:before{content:""}.fa-stumbleupon-circle:before{content:""}.fa-stumbleupon:before{content:""}.fa-delicious:before{content:""}.fa-digg:before{content:""}.fa-pied-piper-pp:before{content:""}.fa-pied-piper-alt:before{content:""}.fa-drupal:before{content:""}.fa-joomla:before{content:""}.fa-language:before{content:""}.fa-fax:before{content:""}.fa-building:before{content:""}.fa-child:before{content:""}.fa-paw:before{content:""}.fa-spoon:before{content:""}.fa-cube:before{content:""}.fa-cubes:before{content:""}.fa-behance:before{content:""}.fa-behance-square:before{content:""}.fa-steam:before{content:""}.fa-steam-square:before{content:""}.fa-recycle:before{content:""}.fa-automobile:before,.fa-car:before{content:""}.fa-cab:before,.fa-taxi:before{content:""}.fa-tree:before{content:""}.fa-spotify:before{content:""}.fa-deviantart:before{content:""}.fa-soundcloud:before{content:""}.fa-database:before{content:""}.fa-file-pdf-o:before{content:""}.fa-file-word-o:before{content:""}.fa-file-excel-o:before{content:""}.fa-file-powerpoint-o:before{content:""}.fa-file-photo-o:before,.fa-file-picture-o:before,.fa-file-image-o:before{content:""}.fa-file-zip-o:before,.fa-file-archive-o:before{content:""}.fa-file-sound-o:before,.fa-file-audio-o:before{content:""}.fa-file-movie-o:before,.fa-file-video-o:before{content:""}.fa-file-code-o:before{content:""}.fa-vine:before{content:""}.fa-codepen:before{content:""}.fa-jsfiddle:before{content:""}.fa-life-bouy:before,.fa-life-buoy:before,.fa-life-saver:before,.fa-support:before,.fa-life-ring:before{content:""}.fa-circle-o-notch:before{content:""}.fa-ra:before,.fa-resistance:before,.fa-rebel:before{content:""}.fa-ge:before,.fa-empire:before{content:""}.fa-git-square:before{content:""}.fa-git:before{content:""}.fa-y-combinator-square:before,.fa-yc-square:before,.fa-hacker-news:before{content:""}.fa-tencent-weibo:before{content:""}.fa-qq:before{content:""}.fa-wechat:before,.fa-weixin:before{content:""}.fa-send:before,.fa-paper-plane:before{content:""}.fa-send-o:before,.fa-paper-plane-o:before{content:""}.fa-history:before{content:""}.fa-circle-thin:before{content:""}.fa-header:before{content:""}.fa-paragraph:before{content:""}.fa-sliders:before{content:""}.fa-share-alt:before{content:""}.fa-share-alt-square:before{content:""}.fa-bomb:before{content:""}.fa-soccer-ball-o:before,.fa-futbol-o:before{content:""}.fa-tty:before{content:""}.fa-binoculars:before{content:""}.fa-plug:before{content:""}.fa-slideshare:before{content:""}.fa-twitch:before{content:""}.fa-yelp:before{content:""}.fa-newspaper-o:before{content:""}.fa-wifi:before{content:""}.fa-calculator:before{content:""}.fa-paypal:before{content:""}.fa-google-wallet:before{content:""}.fa-cc-visa:before{content:""}.fa-cc-mastercard:before{content:""}.fa-cc-discover:before{content:""}.fa-cc-amex:before{content:""}.fa-cc-paypal:before{content:""}.fa-cc-stripe:before{content:""}.fa-bell-slash:before{content:""}.fa-bell-slash-o:before{content:""}.fa-trash:before{content:""}.fa-copyright:before{content:""}.fa-at:before{content:""}.fa-eyedropper:before{content:""}.fa-paint-brush:before{content:""}.fa-birthday-cake:before{content:""}.fa-area-chart:before{content:""}.fa-pie-chart:before{content:""}.fa-line-chart:before{content:""}.fa-lastfm:before{content:""}.fa-lastfm-square:before{content:""}.fa-toggle-off:before{content:""}.fa-toggle-on:before{content:""}.fa-bicycle:before{content:""}.fa-bus:before{content:""}.fa-ioxhost:before{content:""}.fa-angellist:before{content:""}.fa-cc:before{content:""}.fa-shekel:before,.fa-sheqel:before,.fa-ils:before{content:""}.fa-meanpath:before{content:""}.fa-buysellads:before{content:""}.fa-connectdevelop:before{content:""}.fa-dashcube:before{content:""}.fa-forumbee:before{content:""}.fa-leanpub:before{content:""}.fa-sellsy:before{content:""}.fa-shirtsinbulk:before{content:""}.fa-simplybuilt:before{content:""}.fa-skyatlas:before{content:""}.fa-cart-plus:before{content:""}.fa-cart-arrow-down:before{content:""}.fa-diamond:before{content:""}.fa-ship:before{content:""}.fa-user-secret:before{content:""}.fa-motorcycle:before{content:""}.fa-street-view:before{content:""}.fa-heartbeat:before{content:""}.fa-venus:before{content:""}.fa-mars:before{content:""}.fa-mercury:before{content:""}.fa-intersex:before,.fa-transgender:before{content:""}.fa-transgender-alt:before{content:""}.fa-venus-double:before{content:""}.fa-mars-double:before{content:""}.fa-venus-mars:before{content:""}.fa-mars-stroke:before{content:""}.fa-mars-stroke-v:before{content:""}.fa-mars-stroke-h:before{content:""}.fa-neuter:before{content:""}.fa-genderless:before{content:""}.fa-facebook-official:before{content:""}.fa-pinterest-p:before{content:""}.fa-whatsapp:before{content:""}.fa-server:before{content:""}.fa-user-plus:before{content:""}.fa-user-times:before{content:""}.fa-hotel:before,.fa-bed:before{content:""}.fa-viacoin:before{content:""}.fa-train:before{content:""}.fa-subway:before{content:""}.fa-medium:before{content:""}.fa-yc:before,.fa-y-combinator:before{content:""}.fa-optin-monster:before{content:""}.fa-opencart:before{content:""}.fa-expeditedssl:before{content:""}.fa-battery-4:before,.fa-battery-full:before{content:""}.fa-battery-3:before,.fa-battery-three-quarters:before{content:""}.fa-battery-2:before,.fa-battery-half:before{content:""}.fa-battery-1:before,.fa-battery-quarter:before{content:""}.fa-battery-0:before,.fa-battery-empty:before{content:""}.fa-mouse-pointer:before{content:""}.fa-i-cursor:before{content:""}.fa-object-group:before{content:""}.fa-object-ungroup:before{content:""}.fa-sticky-note:before{content:""}.fa-sticky-note-o:before{content:""}.fa-cc-jcb:before{content:""}.fa-cc-diners-club:before{content:""}.fa-clone:before{content:""}.fa-balance-scale:before{content:""}.fa-hourglass-o:before{content:""}.fa-hourglass-1:before,.fa-hourglass-start:before{content:""}.fa-hourglass-2:before,.fa-hourglass-half:before{content:""}.fa-hourglass-3:before,.fa-hourglass-end:before{content:""}.fa-hourglass:before{content:""}.fa-hand-grab-o:before,.fa-hand-rock-o:before{content:""}.fa-hand-stop-o:before,.fa-hand-paper-o:before{content:""}.fa-hand-scissors-o:before{content:""}.fa-hand-lizard-o:before{content:""}.fa-hand-spock-o:before{content:""}.fa-hand-pointer-o:before{content:""}.fa-hand-peace-o:before{content:""}.fa-trademark:before{content:""}.fa-registered:before{content:""}.fa-creative-commons:before{content:""}.fa-gg:before{content:""}.fa-gg-circle:before{content:""}.fa-tripadvisor:before{content:""}.fa-odnoklassniki:before{content:""}.fa-odnoklassniki-square:before{content:""}.fa-get-pocket:before{content:""}.fa-wikipedia-w:before{content:""}.fa-safari:before{content:""}.fa-chrome:before{content:""}.fa-firefox:before{content:""}.fa-opera:before{content:""}.fa-internet-explorer:before{content:""}.fa-tv:before,.fa-television:before{content:""}.fa-contao:before{content:""}.fa-500px:before{content:""}.fa-amazon:before{content:""}.fa-calendar-plus-o:before{content:""}.fa-calendar-minus-o:before{content:""}.fa-calendar-times-o:before{content:""}.fa-calendar-check-o:before{content:""}.fa-industry:before{content:""}.fa-map-pin:before{content:""}.fa-map-signs:before{content:""}.fa-map-o:before{content:""}.fa-map:before{content:""}.fa-commenting:before{content:""}.fa-commenting-o:before{content:""}.fa-houzz:before{content:""}.fa-vimeo:before{content:""}.fa-black-tie:before{content:""}.fa-fonticons:before{content:""}.fa-reddit-alien:before{content:""}.fa-edge:before{content:""}.fa-credit-card-alt:before{content:""}.fa-codiepie:before{content:""}.fa-modx:before{content:""}.fa-fort-awesome:before{content:""}.fa-usb:before{content:""}.fa-product-hunt:before{content:""}.fa-mixcloud:before{content:""}.fa-scribd:before{content:""}.fa-pause-circle:before{content:""}.fa-pause-circle-o:before{content:""}.fa-stop-circle:before{content:""}.fa-stop-circle-o:before{content:""}.fa-shopping-bag:before{content:""}.fa-shopping-basket:before{content:""}.fa-hashtag:before{content:""}.fa-bluetooth:before{content:""}.fa-bluetooth-b:before{content:""}.fa-percent:before{content:""}.fa-gitlab:before,.icon-gitlab:before{content:""}.fa-wpbeginner:before{content:""}.fa-wpforms:before{content:""}.fa-envira:before{content:""}.fa-universal-access:before{content:""}.fa-wheelchair-alt:before{content:""}.fa-question-circle-o:before{content:""}.fa-blind:before{content:""}.fa-audio-description:before{content:""}.fa-volume-control-phone:before{content:""}.fa-braille:before{content:""}.fa-assistive-listening-systems:before{content:""}.fa-asl-interpreting:before,.fa-american-sign-language-interpreting:before{content:""}.fa-deafness:before,.fa-hard-of-hearing:before,.fa-deaf:before{content:""}.fa-glide:before{content:""}.fa-glide-g:before{content:""}.fa-signing:before,.fa-sign-language:before{content:""}.fa-low-vision:before{content:""}.fa-viadeo:before{content:""}.fa-viadeo-square:before{content:""}.fa-snapchat:before{content:""}.fa-snapchat-ghost:before{content:""}.fa-snapchat-square:before{content:""}.fa-pied-piper:before{content:""}.fa-first-order:before{content:""}.fa-yoast:before{content:""}.fa-themeisle:before{content:""}.fa-google-plus-circle:before,.fa-google-plus-official:before{content:""}.fa-fa:before,.fa-font-awesome:before{content:""}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;margin:0;overflow:visible;clip:auto}.fa,.wy-menu-vertical li span.toctree-expand,.wy-menu-vertical li.on a span.toctree-expand,.wy-menu-vertical li.current>a span.toctree-expand,.rst-content .admonition-title,.rst-content h1 .headerlink,.rst-content h2 .headerlink,.rst-content h3 .headerlink,.rst-content h4 .headerlink,.rst-content h5 .headerlink,.rst-content h6 .headerlink,.rst-content dl dt .headerlink,.rst-content p.caption .headerlink,.rst-content tt.download span:first-child,.rst-content code.download span:first-child,.icon,.wy-dropdown .caret,.wy-inline-validate.wy-inline-validate-success .wy-input-context,.wy-inline-validate.wy-inline-validate-danger .wy-input-context,.wy-inline-validate.wy-inline-validate-warning .wy-input-context,.wy-inline-validate.wy-inline-validate-info .wy-input-context{font-family:inherit}.fa:before,.wy-menu-vertical li span.toctree-expand:before,.wy-menu-vertical li.on a span.toctree-expand:before,.wy-menu-vertical li.current>a span.toctree-expand:before,.rst-content .admonition-title:before,.rst-content h1 .headerlink:before,.rst-content h2 .headerlink:before,.rst-content h3 .headerlink:before,.rst-content h4 .headerlink:before,.rst-content h5 .headerlink:before,.rst-content h6 .headerlink:before,.rst-content dl dt .headerlink:before,.rst-content p.caption .headerlink:before,.rst-content tt.download span:first-child:before,.rst-content code.download span:first-child:before,.icon:before,.wy-dropdown .caret:before,.wy-inline-validate.wy-inline-validate-success .wy-input-context:before,.wy-inline-validate.wy-inline-validate-danger .wy-input-context:before,.wy-inline-validate.wy-inline-validate-warning .wy-input-context:before,.wy-inline-validate.wy-inline-validate-info .wy-input-context:before{font-family:"FontAwesome";display:inline-block;font-style:normal;font-weight:normal;line-height:1;text-decoration:inherit}a .fa,a .wy-menu-vertical li span.toctree-expand,.wy-menu-vertical li a span.toctree-expand,.wy-menu-vertical li.on a span.toctree-expand,.wy-menu-vertical li.current>a span.toctree-expand,a .rst-content .admonition-title,.rst-content a .admonition-title,a .rst-content h1 .headerlink,.rst-content h1 a .headerlink,a .rst-content h2 .headerlink,.rst-content h2 a .headerlink,a .rst-content h3 .headerlink,.rst-content h3 a .headerlink,a .rst-content h4 .headerlink,.rst-content h4 a .headerlink,a .rst-content h5 .headerlink,.rst-content h5 a .headerlink,a .rst-content h6 .headerlink,.rst-content h6 a .headerlink,a .rst-content dl dt .headerlink,.rst-content dl dt a .headerlink,a .rst-content p.caption .headerlink,.rst-content p.caption a .headerlink,a .rst-content tt.download span:first-child,.rst-content tt.download a span:first-child,a .rst-content code.download span:first-child,.rst-content code.download a span:first-child,a .icon{display:inline-block;text-decoration:inherit}.btn .fa,.btn .wy-menu-vertical li span.toctree-expand,.wy-menu-vertical li .btn span.toctree-expand,.btn .wy-menu-vertical li.on a span.toctree-expand,.wy-menu-vertical li.on a .btn span.toctree-expand,.btn .wy-menu-vertical li.current>a span.toctree-expand,.wy-menu-vertical li.current>a .btn span.toctree-expand,.btn .rst-content .admonition-title,.rst-content .btn .admonition-title,.btn .rst-content h1 .headerlink,.rst-content h1 .btn .headerlink,.btn .rst-content h2 .headerlink,.rst-content h2 .btn .headerlink,.btn .rst-content h3 .headerlink,.rst-content h3 .btn .headerlink,.btn .rst-content h4 .headerlink,.rst-content h4 .btn .headerlink,.btn .rst-content h5 .headerlink,.rst-content h5 .btn .headerlink,.btn .rst-content h6 .headerlink,.rst-content h6 .btn .headerlink,.btn .rst-content dl dt .headerlink,.rst-content dl dt .btn .headerlink,.btn .rst-content p.caption .headerlink,.rst-content p.caption .btn .headerlink,.btn .rst-content tt.download span:first-child,.rst-content tt.download .btn span:first-child,.btn .rst-content code.download span:first-child,.rst-content code.download .btn span:first-child,.btn .icon,.nav .fa,.nav .wy-menu-vertical li span.toctree-expand,.wy-menu-vertical li .nav span.toctree-expand,.nav .wy-menu-vertical li.on a span.toctree-expand,.wy-menu-vertical li.on a .nav span.toctree-expand,.nav .wy-menu-vertical li.current>a span.toctree-expand,.wy-menu-vertical li.current>a .nav span.toctree-expand,.nav .rst-content .admonition-title,.rst-content .nav .admonition-title,.nav .rst-content h1 .headerlink,.rst-content h1 .nav .headerlink,.nav .rst-content h2 .headerlink,.rst-content h2 .nav .headerlink,.nav .rst-content h3 .headerlink,.rst-content h3 .nav .headerlink,.nav .rst-content h4 .headerlink,.rst-content h4 .nav .headerlink,.nav .rst-content h5 .headerlink,.rst-content h5 .nav .headerlink,.nav .rst-content h6 .headerlink,.rst-content h6 .nav .headerlink,.nav .rst-content dl dt .headerlink,.rst-content dl dt .nav .headerlink,.nav .rst-content p.caption .headerlink,.rst-content p.caption .nav .headerlink,.nav .rst-content tt.download span:first-child,.rst-content tt.download .nav span:first-child,.nav .rst-content code.download span:first-child,.rst-content code.download .nav span:first-child,.nav .icon{display:inline}.btn .fa.fa-large,.btn .wy-menu-vertical li span.fa-large.toctree-expand,.wy-menu-vertical li .btn span.fa-large.toctree-expand,.btn .rst-content .fa-large.admonition-title,.rst-content .btn .fa-large.admonition-title,.btn .rst-content h1 .fa-large.headerlink,.rst-content h1 .btn .fa-large.headerlink,.btn .rst-content h2 .fa-large.headerlink,.rst-content h2 .btn .fa-large.headerlink,.btn .rst-content h3 .fa-large.headerlink,.rst-content h3 .btn .fa-large.headerlink,.btn .rst-content h4 .fa-large.headerlink,.rst-content h4 .btn .fa-large.headerlink,.btn .rst-content h5 .fa-large.headerlink,.rst-content h5 .btn .fa-large.headerlink,.btn .rst-content h6 .fa-large.headerlink,.rst-content h6 .btn .fa-large.headerlink,.btn .rst-content dl dt .fa-large.headerlink,.rst-content dl dt .btn .fa-large.headerlink,.btn .rst-content p.caption .fa-large.headerlink,.rst-content p.caption .btn .fa-large.headerlink,.btn .rst-content tt.download span.fa-large:first-child,.rst-content tt.download .btn span.fa-large:first-child,.btn .rst-content code.download span.fa-large:first-child,.rst-content code.download .btn span.fa-large:first-child,.btn .fa-large.icon,.nav .fa.fa-large,.nav .wy-menu-vertical li span.fa-large.toctree-expand,.wy-menu-vertical li .nav span.fa-large.toctree-expand,.nav .rst-content .fa-large.admonition-title,.rst-content .nav .fa-large.admonition-title,.nav .rst-content h1 .fa-large.headerlink,.rst-content h1 .nav .fa-large.headerlink,.nav .rst-content h2 .fa-large.headerlink,.rst-content h2 .nav .fa-large.headerlink,.nav .rst-content h3 .fa-large.headerlink,.rst-content h3 .nav .fa-large.headerlink,.nav .rst-content h4 .fa-large.headerlink,.rst-content h4 .nav .fa-large.headerlink,.nav .rst-content h5 .fa-large.headerlink,.rst-content h5 .nav .fa-large.headerlink,.nav .rst-content h6 .fa-large.headerlink,.rst-content h6 .nav .fa-large.headerlink,.nav .rst-content dl dt .fa-large.headerlink,.rst-content dl dt .nav .fa-large.headerlink,.nav .rst-content p.caption .fa-large.headerlink,.rst-content p.caption .nav .fa-large.headerlink,.nav .rst-content tt.download span.fa-large:first-child,.rst-content tt.download .nav span.fa-large:first-child,.nav .rst-content code.download span.fa-large:first-child,.rst-content code.download .nav span.fa-large:first-child,.nav .fa-large.icon{line-height:0.9em}.btn .fa.fa-spin,.btn .wy-menu-vertical li span.fa-spin.toctree-expand,.wy-menu-vertical li .btn span.fa-spin.toctree-expand,.btn .rst-content .fa-spin.admonition-title,.rst-content .btn .fa-spin.admonition-title,.btn .rst-content h1 .fa-spin.headerlink,.rst-content h1 .btn .fa-spin.headerlink,.btn .rst-content h2 .fa-spin.headerlink,.rst-content h2 .btn .fa-spin.headerlink,.btn .rst-content h3 .fa-spin.headerlink,.rst-content h3 .btn .fa-spin.headerlink,.btn .rst-content h4 .fa-spin.headerlink,.rst-content h4 .btn .fa-spin.headerlink,.btn .rst-content h5 .fa-spin.headerlink,.rst-content h5 .btn .fa-spin.headerlink,.btn .rst-content h6 .fa-spin.headerlink,.rst-content h6 .btn .fa-spin.headerlink,.btn .rst-content dl dt .fa-spin.headerlink,.rst-content dl dt .btn .fa-spin.headerlink,.btn .rst-content p.caption .fa-spin.headerlink,.rst-content p.caption .btn .fa-spin.headerlink,.btn .rst-content tt.download span.fa-spin:first-child,.rst-content tt.download .btn span.fa-spin:first-child,.btn .rst-content code.download span.fa-spin:first-child,.rst-content code.download .btn span.fa-spin:first-child,.btn .fa-spin.icon,.nav .fa.fa-spin,.nav .wy-menu-vertical li span.fa-spin.toctree-expand,.wy-menu-vertical li .nav span.fa-spin.toctree-expand,.nav .rst-content .fa-spin.admonition-title,.rst-content .nav .fa-spin.admonition-title,.nav .rst-content h1 .fa-spin.headerlink,.rst-content h1 .nav .fa-spin.headerlink,.nav .rst-content h2 .fa-spin.headerlink,.rst-content h2 .nav .fa-spin.headerlink,.nav .rst-content h3 .fa-spin.headerlink,.rst-content h3 .nav .fa-spin.headerlink,.nav .rst-content h4 .fa-spin.headerlink,.rst-content h4 .nav .fa-spin.headerlink,.nav .rst-content h5 .fa-spin.headerlink,.rst-content h5 .nav .fa-spin.headerlink,.nav .rst-content h6 .fa-spin.headerlink,.rst-content h6 .nav .fa-spin.headerlink,.nav .rst-content dl dt .fa-spin.headerlink,.rst-content dl dt .nav .fa-spin.headerlink,.nav .rst-content p.caption .fa-spin.headerlink,.rst-content p.caption .nav .fa-spin.headerlink,.nav .rst-content tt.download span.fa-spin:first-child,.rst-content tt.download .nav span.fa-spin:first-child,.nav .rst-content code.download span.fa-spin:first-child,.rst-content code.download .nav span.fa-spin:first-child,.nav .fa-spin.icon{display:inline-block}.btn.fa:before,.wy-menu-vertical li span.btn.toctree-expand:before,.rst-content .btn.admonition-title:before,.rst-content h1 .btn.headerlink:before,.rst-content h2 .btn.headerlink:before,.rst-content h3 .btn.headerlink:before,.rst-content h4 .btn.headerlink:before,.rst-content h5 .btn.headerlink:before,.rst-content h6 .btn.headerlink:before,.rst-content dl dt .btn.headerlink:before,.rst-content p.caption .btn.headerlink:before,.rst-content tt.download span.btn:first-child:before,.rst-content code.download span.btn:first-child:before,.btn.icon:before{opacity:0.5;-webkit-transition:opacity 0.05s ease-in;-moz-transition:opacity 0.05s ease-in;transition:opacity 0.05s ease-in}.btn.fa:hover:before,.wy-menu-vertical li span.btn.toctree-expand:hover:before,.rst-content .btn.admonition-title:hover:before,.rst-content h1 .btn.headerlink:hover:before,.rst-content h2 .btn.headerlink:hover:before,.rst-content h3 .btn.headerlink:hover:before,.rst-content h4 .btn.headerlink:hover:before,.rst-content h5 .btn.headerlink:hover:before,.rst-content h6 .btn.headerlink:hover:before,.rst-content dl dt .btn.headerlink:hover:before,.rst-content p.caption .btn.headerlink:hover:before,.rst-content tt.download span.btn:first-child:hover:before,.rst-content code.download span.btn:first-child:hover:before,.btn.icon:hover:before{opacity:1}.btn-mini .fa:before,.btn-mini .wy-menu-vertical li span.toctree-expand:before,.wy-menu-vertical li .btn-mini span.toctree-expand:before,.btn-mini .rst-content .admonition-title:before,.rst-content .btn-mini .admonition-title:before,.btn-mini .rst-content h1 .headerlink:before,.rst-content h1 .btn-mini .headerlink:before,.btn-mini .rst-content h2 .headerlink:before,.rst-content h2 .btn-mini .headerlink:before,.btn-mini .rst-content h3 .headerlink:before,.rst-content h3 .btn-mini .headerlink:before,.btn-mini .rst-content h4 .headerlink:before,.rst-content h4 .btn-mini .headerlink:before,.btn-mini .rst-content h5 .headerlink:before,.rst-content h5 .btn-mini .headerlink:before,.btn-mini .rst-content h6 .headerlink:before,.rst-content h6 .btn-mini .headerlink:before,.btn-mini .rst-content dl dt .headerlink:before,.rst-content dl dt .btn-mini .headerlink:before,.btn-mini .rst-content p.caption .headerlink:before,.rst-content p.caption .btn-mini .headerlink:before,.btn-mini .rst-content tt.download span:first-child:before,.rst-content tt.download .btn-mini span:first-child:before,.btn-mini .rst-content code.download span:first-child:before,.rst-content code.download .btn-mini span:first-child:before,.btn-mini .icon:before{font-size:14px;vertical-align:-15%}.wy-alert,.rst-content .note,.rst-content .attention,.rst-content .caution,.rst-content .danger,.rst-content .error,.rst-content .hint,.rst-content .important,.rst-content .tip,.rst-content .warning,.rst-content .seealso,.rst-content .admonition-todo{padding:12px;line-height:24px;margin-bottom:24px;background:#e7f2fa}.wy-alert-title,.rst-content .admonition-title{color:#fff;font-weight:bold;display:block;color:#fff;background:#6ab0de;margin:-12px;padding:6px 12px;margin-bottom:12px}.wy-alert.wy-alert-danger,.rst-content .wy-alert-danger.note,.rst-content .wy-alert-danger.attention,.rst-content .wy-alert-danger.caution,.rst-content .danger,.rst-content .error,.rst-content .wy-alert-danger.hint,.rst-content .wy-alert-danger.important,.rst-content .wy-alert-danger.tip,.rst-content .wy-alert-danger.warning,.rst-content .wy-alert-danger.seealso,.rst-content .wy-alert-danger.admonition-todo{background:#fdf3f2}.wy-alert.wy-alert-danger .wy-alert-title,.rst-content .wy-alert-danger.note .wy-alert-title,.rst-content .wy-alert-danger.attention .wy-alert-title,.rst-content .wy-alert-danger.caution .wy-alert-title,.rst-content .danger .wy-alert-title,.rst-content .error .wy-alert-title,.rst-content .wy-alert-danger.hint .wy-alert-title,.rst-content .wy-alert-danger.important .wy-alert-title,.rst-content .wy-alert-danger.tip .wy-alert-title,.rst-content .wy-alert-danger.warning .wy-alert-title,.rst-content .wy-alert-danger.seealso .wy-alert-title,.rst-content .wy-alert-danger.admonition-todo .wy-alert-title,.wy-alert.wy-alert-danger .rst-content .admonition-title,.rst-content .wy-alert.wy-alert-danger .admonition-title,.rst-content .wy-alert-danger.note .admonition-title,.rst-content .wy-alert-danger.attention .admonition-title,.rst-content .wy-alert-danger.caution .admonition-title,.rst-content .danger .admonition-title,.rst-content .error .admonition-title,.rst-content .wy-alert-danger.hint .admonition-title,.rst-content .wy-alert-danger.important .admonition-title,.rst-content .wy-alert-danger.tip .admonition-title,.rst-content .wy-alert-danger.warning .admonition-title,.rst-content .wy-alert-danger.seealso .admonition-title,.rst-content .wy-alert-danger.admonition-todo .admonition-title{background:#f29f97}.wy-alert.wy-alert-warning,.rst-content .wy-alert-warning.note,.rst-content .attention,.rst-content .caution,.rst-content .wy-alert-warning.danger,.rst-content .wy-alert-warning.error,.rst-content .wy-alert-warning.hint,.rst-content .wy-alert-warning.important,.rst-content .wy-alert-warning.tip,.rst-content .warning,.rst-content .wy-alert-warning.seealso,.rst-content .admonition-todo{background:#ffedcc}.wy-alert.wy-alert-warning .wy-alert-title,.rst-content .wy-alert-warning.note .wy-alert-title,.rst-content .attention .wy-alert-title,.rst-content .caution .wy-alert-title,.rst-content .wy-alert-warning.danger .wy-alert-title,.rst-content .wy-alert-warning.error .wy-alert-title,.rst-content .wy-alert-warning.hint .wy-alert-title,.rst-content .wy-alert-warning.important .wy-alert-title,.rst-content .wy-alert-warning.tip .wy-alert-title,.rst-content .warning .wy-alert-title,.rst-content .wy-alert-warning.seealso .wy-alert-title,.rst-content .admonition-todo .wy-alert-title,.wy-alert.wy-alert-warning .rst-content .admonition-title,.rst-content .wy-alert.wy-alert-warning .admonition-title,.rst-content .wy-alert-warning.note .admonition-title,.rst-content .attention .admonition-title,.rst-content .caution .admonition-title,.rst-content .wy-alert-warning.danger .admonition-title,.rst-content .wy-alert-warning.error .admonition-title,.rst-content .wy-alert-warning.hint .admonition-title,.rst-content .wy-alert-warning.important .admonition-title,.rst-content .wy-alert-warning.tip .admonition-title,.rst-content .warning .admonition-title,.rst-content .wy-alert-warning.seealso .admonition-title,.rst-content .admonition-todo .admonition-title{background:#f0b37e}.wy-alert.wy-alert-info,.rst-content .note,.rst-content .wy-alert-info.attention,.rst-content .wy-alert-info.caution,.rst-content .wy-alert-info.danger,.rst-content .wy-alert-info.error,.rst-content .wy-alert-info.hint,.rst-content .wy-alert-info.important,.rst-content .wy-alert-info.tip,.rst-content .wy-alert-info.warning,.rst-content .seealso,.rst-content .wy-alert-info.admonition-todo{background:#e7f2fa}.wy-alert.wy-alert-info .wy-alert-title,.rst-content .note .wy-alert-title,.rst-content .wy-alert-info.attention .wy-alert-title,.rst-content .wy-alert-info.caution .wy-alert-title,.rst-content .wy-alert-info.danger .wy-alert-title,.rst-content .wy-alert-info.error .wy-alert-title,.rst-content .wy-alert-info.hint .wy-alert-title,.rst-content .wy-alert-info.important .wy-alert-title,.rst-content .wy-alert-info.tip .wy-alert-title,.rst-content .wy-alert-info.warning .wy-alert-title,.rst-content .seealso .wy-alert-title,.rst-content .wy-alert-info.admonition-todo .wy-alert-title,.wy-alert.wy-alert-info .rst-content .admonition-title,.rst-content .wy-alert.wy-alert-info .admonition-title,.rst-content .note .admonition-title,.rst-content .wy-alert-info.attention .admonition-title,.rst-content .wy-alert-info.caution .admonition-title,.rst-content .wy-alert-info.danger .admonition-title,.rst-content .wy-alert-info.error .admonition-title,.rst-content .wy-alert-info.hint .admonition-title,.rst-content .wy-alert-info.important .admonition-title,.rst-content .wy-alert-info.tip .admonition-title,.rst-content .wy-alert-info.warning .admonition-title,.rst-content .seealso .admonition-title,.rst-content .wy-alert-info.admonition-todo .admonition-title{background:#6ab0de}.wy-alert.wy-alert-success,.rst-content .wy-alert-success.note,.rst-content .wy-alert-success.attention,.rst-content .wy-alert-success.caution,.rst-content .wy-alert-success.danger,.rst-content .wy-alert-success.error,.rst-content .hint,.rst-content .important,.rst-content .tip,.rst-content .wy-alert-success.warning,.rst-content .wy-alert-success.seealso,.rst-content .wy-alert-success.admonition-todo{background:#dbfaf4}.wy-alert.wy-alert-success .wy-alert-title,.rst-content .wy-alert-success.note .wy-alert-title,.rst-content .wy-alert-success.attention .wy-alert-title,.rst-content .wy-alert-success.caution .wy-alert-title,.rst-content .wy-alert-success.danger .wy-alert-title,.rst-content .wy-alert-success.error .wy-alert-title,.rst-content .hint .wy-alert-title,.rst-content .important .wy-alert-title,.rst-content .tip .wy-alert-title,.rst-content .wy-alert-success.warning .wy-alert-title,.rst-content .wy-alert-success.seealso .wy-alert-title,.rst-content .wy-alert-success.admonition-todo .wy-alert-title,.wy-alert.wy-alert-success .rst-content .admonition-title,.rst-content .wy-alert.wy-alert-success .admonition-title,.rst-content .wy-alert-success.note .admonition-title,.rst-content .wy-alert-success.attention .admonition-title,.rst-content .wy-alert-success.caution .admonition-title,.rst-content .wy-alert-success.danger .admonition-title,.rst-content .wy-alert-success.error .admonition-title,.rst-content .hint .admonition-title,.rst-content .important .admonition-title,.rst-content .tip .admonition-title,.rst-content .wy-alert-success.warning .admonition-title,.rst-content .wy-alert-success.seealso .admonition-title,.rst-content .wy-alert-success.admonition-todo .admonition-title{background:#1abc9c}.wy-alert.wy-alert-neutral,.rst-content .wy-alert-neutral.note,.rst-content .wy-alert-neutral.attention,.rst-content .wy-alert-neutral.caution,.rst-content .wy-alert-neutral.danger,.rst-content .wy-alert-neutral.error,.rst-content .wy-alert-neutral.hint,.rst-content .wy-alert-neutral.important,.rst-content .wy-alert-neutral.tip,.rst-content .wy-alert-neutral.warning,.rst-content .wy-alert-neutral.seealso,.rst-content .wy-alert-neutral.admonition-todo{background:#f3f6f6}.wy-alert.wy-alert-neutral .wy-alert-title,.rst-content .wy-alert-neutral.note .wy-alert-title,.rst-content .wy-alert-neutral.attention .wy-alert-title,.rst-content .wy-alert-neutral.caution .wy-alert-title,.rst-content .wy-alert-neutral.danger .wy-alert-title,.rst-content .wy-alert-neutral.error .wy-alert-title,.rst-content .wy-alert-neutral.hint .wy-alert-title,.rst-content .wy-alert-neutral.important .wy-alert-title,.rst-content .wy-alert-neutral.tip .wy-alert-title,.rst-content .wy-alert-neutral.warning .wy-alert-title,.rst-content .wy-alert-neutral.seealso .wy-alert-title,.rst-content .wy-alert-neutral.admonition-todo .wy-alert-title,.wy-alert.wy-alert-neutral .rst-content .admonition-title,.rst-content .wy-alert.wy-alert-neutral .admonition-title,.rst-content .wy-alert-neutral.note .admonition-title,.rst-content .wy-alert-neutral.attention .admonition-title,.rst-content .wy-alert-neutral.caution .admonition-title,.rst-content .wy-alert-neutral.danger .admonition-title,.rst-content .wy-alert-neutral.error .admonition-title,.rst-content .wy-alert-neutral.hint .admonition-title,.rst-content .wy-alert-neutral.important .admonition-title,.rst-content .wy-alert-neutral.tip .admonition-title,.rst-content .wy-alert-neutral.warning .admonition-title,.rst-content .wy-alert-neutral.seealso .admonition-title,.rst-content .wy-alert-neutral.admonition-todo .admonition-title{color:#404040;background:#e1e4e5}.wy-alert.wy-alert-neutral a,.rst-content .wy-alert-neutral.note a,.rst-content .wy-alert-neutral.attention a,.rst-content .wy-alert-neutral.caution a,.rst-content .wy-alert-neutral.danger a,.rst-content .wy-alert-neutral.error a,.rst-content .wy-alert-neutral.hint a,.rst-content .wy-alert-neutral.important a,.rst-content .wy-alert-neutral.tip a,.rst-content .wy-alert-neutral.warning a,.rst-content .wy-alert-neutral.seealso a,.rst-content .wy-alert-neutral.admonition-todo a{color:#2980B9}.wy-alert p:last-child,.rst-content .note p:last-child,.rst-content .attention p:last-child,.rst-content .caution p:last-child,.rst-content .danger p:last-child,.rst-content .error p:last-child,.rst-content .hint p:last-child,.rst-content .important p:last-child,.rst-content .tip p:last-child,.rst-content .warning p:last-child,.rst-content .seealso p:last-child,.rst-content .admonition-todo p:last-child{margin-bottom:0}.wy-tray-container{position:fixed;bottom:0px;left:0;z-index:600}.wy-tray-container li{display:block;width:300px;background:transparent;color:#fff;text-align:center;box-shadow:0 5px 5px 0 rgba(0,0,0,0.1);padding:0 24px;min-width:20%;opacity:0;height:0;line-height:56px;overflow:hidden;-webkit-transition:all 0.3s ease-in;-moz-transition:all 0.3s ease-in;transition:all 0.3s ease-in}.wy-tray-container li.wy-tray-item-success{background:#27AE60}.wy-tray-container li.wy-tray-item-info{background:#2980B9}.wy-tray-container li.wy-tray-item-warning{background:#E67E22}.wy-tray-container li.wy-tray-item-danger{background:#E74C3C}.wy-tray-container li.on{opacity:1;height:56px}@media screen and (max-width: 768px){.wy-tray-container{bottom:auto;top:0;width:100%}.wy-tray-container li{width:100%}}button{font-size:100%;margin:0;vertical-align:baseline;*vertical-align:middle;cursor:pointer;line-height:normal;-webkit-appearance:button;*overflow:visible}button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0}button[disabled]{cursor:default}.btn{display:inline-block;border-radius:2px;line-height:normal;white-space:nowrap;text-align:center;cursor:pointer;font-size:100%;padding:6px 12px 8px 12px;color:#fff;border:1px solid rgba(0,0,0,0.1);background-color:#27AE60;text-decoration:none;font-weight:normal;font-family:"Lato","proxima-nova","Helvetica Neue",Arial,sans-serif;box-shadow:0px 1px 2px -1px rgba(255,255,255,0.5) inset,0px -2px 0px 0px rgba(0,0,0,0.1) inset;outline-none:false;vertical-align:middle;*display:inline;zoom:1;-webkit-user-drag:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;-webkit-transition:all 0.1s linear;-moz-transition:all 0.1s linear;transition:all 0.1s linear}.btn-hover{background:#2e8ece;color:#fff}.btn:hover{background:#2cc36b;color:#fff}.btn:focus{background:#2cc36b;outline:0}.btn:active{box-shadow:0px -1px 0px 0px rgba(0,0,0,0.05) inset,0px 2px 0px 0px rgba(0,0,0,0.1) inset;padding:8px 12px 6px 12px}.btn:visited{color:#fff}.btn:disabled{background-image:none;filter:progid:DXImageTransform.Microsoft.gradient(enabled = false);filter:alpha(opacity=40);opacity:0.4;cursor:not-allowed;box-shadow:none}.btn-disabled{background-image:none;filter:progid:DXImageTransform.Microsoft.gradient(enabled = false);filter:alpha(opacity=40);opacity:0.4;cursor:not-allowed;box-shadow:none}.btn-disabled:hover,.btn-disabled:focus,.btn-disabled:active{background-image:none;filter:progid:DXImageTransform.Microsoft.gradient(enabled = false);filter:alpha(opacity=40);opacity:0.4;cursor:not-allowed;box-shadow:none}.btn::-moz-focus-inner{padding:0;border:0}.btn-small{font-size:80%}.btn-info{background-color:#2980B9 !important}.btn-info:hover{background-color:#2e8ece !important}.btn-neutral{background-color:#f3f6f6 !important;color:#404040 !important}.btn-neutral:hover{background-color:#e5ebeb !important;color:#404040}.btn-neutral:visited{color:#404040 !important}.btn-success{background-color:#27AE60 !important}.btn-success:hover{background-color:#295 !important}.btn-danger{background-color:#E74C3C !important}.btn-danger:hover{background-color:#ea6153 !important}.btn-warning{background-color:#E67E22 !important}.btn-warning:hover{background-color:#e98b39 !important}.btn-invert{background-color:#222}.btn-invert:hover{background-color:#2f2f2f !important}.btn-link{background-color:transparent !important;color:#2980B9;box-shadow:none;border-color:transparent !important}.btn-link:hover{background-color:transparent !important;color:#409ad5 !important;box-shadow:none}.btn-link:active{background-color:transparent !important;color:#409ad5 !important;box-shadow:none}.btn-link:visited{color:#9B59B6}.wy-btn-group .btn,.wy-control .btn{vertical-align:middle}.wy-btn-group{margin-bottom:24px;*zoom:1}.wy-btn-group:before,.wy-btn-group:after{display:table;content:""}.wy-btn-group:after{clear:both}.wy-dropdown{position:relative;display:inline-block}.wy-dropdown-active .wy-dropdown-menu{display:block}.wy-dropdown-menu{position:absolute;left:0;display:none;float:left;top:100%;min-width:100%;background:#fcfcfc;z-index:100;border:solid 1px #cfd7dd;box-shadow:0 2px 2px 0 rgba(0,0,0,0.1);padding:12px}.wy-dropdown-menu>dd>a{display:block;clear:both;color:#404040;white-space:nowrap;font-size:90%;padding:0 12px;cursor:pointer}.wy-dropdown-menu>dd>a:hover{background:#2980B9;color:#fff}.wy-dropdown-menu>dd.divider{border-top:solid 1px #cfd7dd;margin:6px 0}.wy-dropdown-menu>dd.search{padding-bottom:12px}.wy-dropdown-menu>dd.search input[type="search"]{width:100%}.wy-dropdown-menu>dd.call-to-action{background:#e3e3e3;text-transform:uppercase;font-weight:500;font-size:80%}.wy-dropdown-menu>dd.call-to-action:hover{background:#e3e3e3}.wy-dropdown-menu>dd.call-to-action .btn{color:#fff}.wy-dropdown.wy-dropdown-up .wy-dropdown-menu{bottom:100%;top:auto;left:auto;right:0}.wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu{background:#fcfcfc;margin-top:2px}.wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu a{padding:6px 12px}.wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu a:hover{background:#2980B9;color:#fff}.wy-dropdown.wy-dropdown-left .wy-dropdown-menu{right:0;left:auto;text-align:right}.wy-dropdown-arrow:before{content:" ";border-bottom:5px solid #f5f5f5;border-left:5px solid transparent;border-right:5px solid transparent;position:absolute;display:block;top:-4px;left:50%;margin-left:-3px}.wy-dropdown-arrow.wy-dropdown-arrow-left:before{left:11px}.wy-form-stacked select{display:block}.wy-form-aligned input,.wy-form-aligned textarea,.wy-form-aligned select,.wy-form-aligned .wy-help-inline,.wy-form-aligned label{display:inline-block;*display:inline;*zoom:1;vertical-align:middle}.wy-form-aligned .wy-control-group>label{display:inline-block;vertical-align:middle;width:10em;margin:6px 12px 0 0;float:left}.wy-form-aligned .wy-control{float:left}.wy-form-aligned .wy-control label{display:block}.wy-form-aligned .wy-control select{margin-top:6px}fieldset{border:0;margin:0;padding:0}legend{display:block;width:100%;border:0;padding:0;white-space:normal;margin-bottom:24px;font-size:150%;*margin-left:-7px}label{display:block;margin:0 0 .3125em 0;color:#333;font-size:90%}input,select,textarea{font-size:100%;margin:0;vertical-align:baseline;*vertical-align:middle}.wy-control-group{margin-bottom:24px;*zoom:1;max-width:68em;margin-left:auto;margin-right:auto;*zoom:1}.wy-control-group:before,.wy-control-group:after{display:table;content:""}.wy-control-group:after{clear:both}.wy-control-group:before,.wy-control-group:after{display:table;content:""}.wy-control-group:after{clear:both}.wy-control-group.wy-control-group-required>label:after{content:" *";color:#E74C3C}.wy-control-group .wy-form-full,.wy-control-group .wy-form-halves,.wy-control-group .wy-form-thirds{padding-bottom:12px}.wy-control-group .wy-form-full select,.wy-control-group .wy-form-halves select,.wy-control-group .wy-form-thirds select{width:100%}.wy-control-group .wy-form-full input[type="text"],.wy-control-group .wy-form-full input[type="password"],.wy-control-group .wy-form-full input[type="email"],.wy-control-group .wy-form-full input[type="url"],.wy-control-group .wy-form-full input[type="date"],.wy-control-group .wy-form-full input[type="month"],.wy-control-group .wy-form-full input[type="time"],.wy-control-group .wy-form-full input[type="datetime"],.wy-control-group .wy-form-full input[type="datetime-local"],.wy-control-group .wy-form-full input[type="week"],.wy-control-group .wy-form-full input[type="number"],.wy-control-group .wy-form-full input[type="search"],.wy-control-group .wy-form-full input[type="tel"],.wy-control-group .wy-form-full input[type="color"],.wy-control-group .wy-form-halves input[type="text"],.wy-control-group .wy-form-halves input[type="password"],.wy-control-group .wy-form-halves input[type="email"],.wy-control-group .wy-form-halves input[type="url"],.wy-control-group .wy-form-halves input[type="date"],.wy-control-group .wy-form-halves input[type="month"],.wy-control-group .wy-form-halves input[type="time"],.wy-control-group .wy-form-halves input[type="datetime"],.wy-control-group .wy-form-halves input[type="datetime-local"],.wy-control-group .wy-form-halves input[type="week"],.wy-control-group .wy-form-halves input[type="number"],.wy-control-group .wy-form-halves input[type="search"],.wy-control-group .wy-form-halves input[type="tel"],.wy-control-group .wy-form-halves input[type="color"],.wy-control-group .wy-form-thirds input[type="text"],.wy-control-group .wy-form-thirds input[type="password"],.wy-control-group .wy-form-thirds input[type="email"],.wy-control-group .wy-form-thirds input[type="url"],.wy-control-group .wy-form-thirds input[type="date"],.wy-control-group .wy-form-thirds input[type="month"],.wy-control-group .wy-form-thirds input[type="time"],.wy-control-group .wy-form-thirds input[type="datetime"],.wy-control-group .wy-form-thirds input[type="datetime-local"],.wy-control-group .wy-form-thirds input[type="week"],.wy-control-group .wy-form-thirds input[type="number"],.wy-control-group .wy-form-thirds input[type="search"],.wy-control-group .wy-form-thirds input[type="tel"],.wy-control-group .wy-form-thirds input[type="color"]{width:100%}.wy-control-group .wy-form-full{float:left;display:block;margin-right:2.35765%;width:100%;margin-right:0}.wy-control-group .wy-form-full:last-child{margin-right:0}.wy-control-group .wy-form-halves{float:left;display:block;margin-right:2.35765%;width:48.82117%}.wy-control-group .wy-form-halves:last-child{margin-right:0}.wy-control-group .wy-form-halves:nth-of-type(2n){margin-right:0}.wy-control-group .wy-form-halves:nth-of-type(2n+1){clear:left}.wy-control-group .wy-form-thirds{float:left;display:block;margin-right:2.35765%;width:31.76157%}.wy-control-group .wy-form-thirds:last-child{margin-right:0}.wy-control-group .wy-form-thirds:nth-of-type(3n){margin-right:0}.wy-control-group .wy-form-thirds:nth-of-type(3n+1){clear:left}.wy-control-group.wy-control-group-no-input .wy-control{margin:6px 0 0 0;font-size:90%}.wy-control-no-input{display:inline-block;margin:6px 0 0 0;font-size:90%}.wy-control-group.fluid-input input[type="text"],.wy-control-group.fluid-input input[type="password"],.wy-control-group.fluid-input input[type="email"],.wy-control-group.fluid-input input[type="url"],.wy-control-group.fluid-input input[type="date"],.wy-control-group.fluid-input input[type="month"],.wy-control-group.fluid-input input[type="time"],.wy-control-group.fluid-input input[type="datetime"],.wy-control-group.fluid-input input[type="datetime-local"],.wy-control-group.fluid-input input[type="week"],.wy-control-group.fluid-input input[type="number"],.wy-control-group.fluid-input input[type="search"],.wy-control-group.fluid-input input[type="tel"],.wy-control-group.fluid-input input[type="color"]{width:100%}.wy-form-message-inline{display:inline-block;padding-left:0.3em;color:#666;vertical-align:middle;font-size:90%}.wy-form-message{display:block;color:#999;font-size:70%;margin-top:.3125em;font-style:italic}.wy-form-message p{font-size:inherit;font-style:italic;margin-bottom:6px}.wy-form-message p:last-child{margin-bottom:0}input{line-height:normal}input[type="button"],input[type="reset"],input[type="submit"]{-webkit-appearance:button;cursor:pointer;font-family:"Lato","proxima-nova","Helvetica Neue",Arial,sans-serif;*overflow:visible}input[type="text"],input[type="password"],input[type="email"],input[type="url"],input[type="date"],input[type="month"],input[type="time"],input[type="datetime"],input[type="datetime-local"],input[type="week"],input[type="number"],input[type="search"],input[type="tel"],input[type="color"]{-webkit-appearance:none;padding:6px;display:inline-block;border:1px solid #ccc;font-size:80%;font-family:"Lato","proxima-nova","Helvetica Neue",Arial,sans-serif;box-shadow:inset 0 1px 3px #ddd;border-radius:0;-webkit-transition:border 0.3s linear;-moz-transition:border 0.3s linear;transition:border 0.3s linear}input[type="datetime-local"]{padding:.34375em .625em}input[disabled]{cursor:default}input[type="checkbox"],input[type="radio"]{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;padding:0;margin-right:.3125em;*height:13px;*width:13px}input[type="search"]{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}input[type="search"]::-webkit-search-cancel-button,input[type="search"]::-webkit-search-decoration{-webkit-appearance:none}input[type="text"]:focus,input[type="password"]:focus,input[type="email"]:focus,input[type="url"]:focus,input[type="date"]:focus,input[type="month"]:focus,input[type="time"]:focus,input[type="datetime"]:focus,input[type="datetime-local"]:focus,input[type="week"]:focus,input[type="number"]:focus,input[type="search"]:focus,input[type="tel"]:focus,input[type="color"]:focus{outline:0;outline:thin dotted \9;border-color:#333}input.no-focus:focus{border-color:#ccc !important}input[type="file"]:focus,input[type="radio"]:focus,input[type="checkbox"]:focus{outline:thin dotted #333;outline:1px auto #129FEA}input[type="text"][disabled],input[type="password"][disabled],input[type="email"][disabled],input[type="url"][disabled],input[type="date"][disabled],input[type="month"][disabled],input[type="time"][disabled],input[type="datetime"][disabled],input[type="datetime-local"][disabled],input[type="week"][disabled],input[type="number"][disabled],input[type="search"][disabled],input[type="tel"][disabled],input[type="color"][disabled]{cursor:not-allowed;background-color:#fafafa}input:focus:invalid,textarea:focus:invalid,select:focus:invalid{color:#E74C3C;border:1px solid #E74C3C}input:focus:invalid:focus,textarea:focus:invalid:focus,select:focus:invalid:focus{border-color:#E74C3C}input[type="file"]:focus:invalid:focus,input[type="radio"]:focus:invalid:focus,input[type="checkbox"]:focus:invalid:focus{outline-color:#E74C3C}input.wy-input-large{padding:12px;font-size:100%}textarea{overflow:auto;vertical-align:top;width:100%;font-family:"Lato","proxima-nova","Helvetica Neue",Arial,sans-serif}select,textarea{padding:.5em .625em;display:inline-block;border:1px solid #ccc;font-size:80%;box-shadow:inset 0 1px 3px #ddd;-webkit-transition:border 0.3s linear;-moz-transition:border 0.3s linear;transition:border 0.3s linear}select{border:1px solid #ccc;background-color:#fff}select[multiple]{height:auto}select:focus,textarea:focus{outline:0}select[disabled],textarea[disabled],input[readonly],select[readonly],textarea[readonly]{cursor:not-allowed;background-color:#fafafa}input[type="radio"][disabled],input[type="checkbox"][disabled]{cursor:not-allowed}.wy-checkbox,.wy-radio{margin:6px 0;color:#404040;display:block}.wy-checkbox input,.wy-radio input{vertical-align:baseline}.wy-form-message-inline{display:inline-block;*display:inline;*zoom:1;vertical-align:middle}.wy-input-prefix,.wy-input-suffix{white-space:nowrap;padding:6px}.wy-input-prefix .wy-input-context,.wy-input-suffix .wy-input-context{line-height:27px;padding:0 8px;display:inline-block;font-size:80%;background-color:#f3f6f6;border:solid 1px #ccc;color:#999}.wy-input-suffix .wy-input-context{border-left:0}.wy-input-prefix .wy-input-context{border-right:0}.wy-switch{width:36px;height:12px;margin:12px 0;position:relative;border-radius:4px;background:#ccc;cursor:pointer;-webkit-transition:all 0.2s ease-in-out;-moz-transition:all 0.2s ease-in-out;transition:all 0.2s ease-in-out}.wy-switch:before{position:absolute;content:"";display:block;width:18px;height:18px;border-radius:4px;background:#999;left:-3px;top:-3px;-webkit-transition:all 0.2s ease-in-out;-moz-transition:all 0.2s ease-in-out;transition:all 0.2s ease-in-out}.wy-switch:after{content:"false";position:absolute;left:48px;display:block;font-size:12px;color:#ccc}.wy-switch.active{background:#1e8449}.wy-switch.active:before{left:24px;background:#27AE60}.wy-switch.active:after{content:"true"}.wy-switch.disabled,.wy-switch.active.disabled{cursor:not-allowed}.wy-control-group.wy-control-group-error .wy-form-message,.wy-control-group.wy-control-group-error>label{color:#E74C3C}.wy-control-group.wy-control-group-error input[type="text"],.wy-control-group.wy-control-group-error input[type="password"],.wy-control-group.wy-control-group-error input[type="email"],.wy-control-group.wy-control-group-error input[type="url"],.wy-control-group.wy-control-group-error input[type="date"],.wy-control-group.wy-control-group-error input[type="month"],.wy-control-group.wy-control-group-error input[type="time"],.wy-control-group.wy-control-group-error input[type="datetime"],.wy-control-group.wy-control-group-error input[type="datetime-local"],.wy-control-group.wy-control-group-error input[type="week"],.wy-control-group.wy-control-group-error input[type="number"],.wy-control-group.wy-control-group-error input[type="search"],.wy-control-group.wy-control-group-error input[type="tel"],.wy-control-group.wy-control-group-error input[type="color"]{border:solid 1px #E74C3C}.wy-control-group.wy-control-group-error textarea{border:solid 1px #E74C3C}.wy-inline-validate{white-space:nowrap}.wy-inline-validate .wy-input-context{padding:.5em .625em;display:inline-block;font-size:80%}.wy-inline-validate.wy-inline-validate-success .wy-input-context{color:#27AE60}.wy-inline-validate.wy-inline-validate-danger .wy-input-context{color:#E74C3C}.wy-inline-validate.wy-inline-validate-warning .wy-input-context{color:#E67E22}.wy-inline-validate.wy-inline-validate-info .wy-input-context{color:#2980B9}.rotate-90{-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-ms-transform:rotate(90deg);-o-transform:rotate(90deg);transform:rotate(90deg)}.rotate-180{-webkit-transform:rotate(180deg);-moz-transform:rotate(180deg);-ms-transform:rotate(180deg);-o-transform:rotate(180deg);transform:rotate(180deg)}.rotate-270{-webkit-transform:rotate(270deg);-moz-transform:rotate(270deg);-ms-transform:rotate(270deg);-o-transform:rotate(270deg);transform:rotate(270deg)}.mirror{-webkit-transform:scaleX(-1);-moz-transform:scaleX(-1);-ms-transform:scaleX(-1);-o-transform:scaleX(-1);transform:scaleX(-1)}.mirror.rotate-90{-webkit-transform:scaleX(-1) rotate(90deg);-moz-transform:scaleX(-1) rotate(90deg);-ms-transform:scaleX(-1) rotate(90deg);-o-transform:scaleX(-1) rotate(90deg);transform:scaleX(-1) rotate(90deg)}.mirror.rotate-180{-webkit-transform:scaleX(-1) rotate(180deg);-moz-transform:scaleX(-1) rotate(180deg);-ms-transform:scaleX(-1) rotate(180deg);-o-transform:scaleX(-1) rotate(180deg);transform:scaleX(-1) rotate(180deg)}.mirror.rotate-270{-webkit-transform:scaleX(-1) rotate(270deg);-moz-transform:scaleX(-1) rotate(270deg);-ms-transform:scaleX(-1) rotate(270deg);-o-transform:scaleX(-1) rotate(270deg);transform:scaleX(-1) rotate(270deg)}@media only screen and (max-width: 480px){.wy-form button[type="submit"]{margin:0.7em 0 0}.wy-form input[type="text"],.wy-form input[type="password"],.wy-form input[type="email"],.wy-form input[type="url"],.wy-form input[type="date"],.wy-form input[type="month"],.wy-form input[type="time"],.wy-form input[type="datetime"],.wy-form input[type="datetime-local"],.wy-form input[type="week"],.wy-form input[type="number"],.wy-form input[type="search"],.wy-form input[type="tel"],.wy-form input[type="color"]{margin-bottom:0.3em;display:block}.wy-form label{margin-bottom:0.3em;display:block}.wy-form input[type="password"],.wy-form input[type="email"],.wy-form input[type="url"],.wy-form input[type="date"],.wy-form input[type="month"],.wy-form input[type="time"],.wy-form input[type="datetime"],.wy-form input[type="datetime-local"],.wy-form input[type="week"],.wy-form input[type="number"],.wy-form input[type="search"],.wy-form input[type="tel"],.wy-form input[type="color"]{margin-bottom:0}.wy-form-aligned .wy-control-group label{margin-bottom:0.3em;text-align:left;display:block;width:100%}.wy-form-aligned .wy-control{margin:1.5em 0 0 0}.wy-form .wy-help-inline,.wy-form-message-inline,.wy-form-message{display:block;font-size:80%;padding:6px 0}}@media screen and (max-width: 768px){.tablet-hide{display:none}}@media screen and (max-width: 480px){.mobile-hide{display:none}}.float-left{float:left}.float-right{float:right}.full-width{width:100%}.wy-table,.rst-content table.docutils,.rst-content table.field-list{border-collapse:collapse;border-spacing:0;empty-cells:show;margin-bottom:24px}.wy-table caption,.rst-content table.docutils caption,.rst-content table.field-list caption{color:#000;font:italic 85%/1 arial,sans-serif;padding:1em 0;text-align:center}.wy-table td,.rst-content table.docutils td,.rst-content table.field-list td,.wy-table th,.rst-content table.docutils th,.rst-content table.field-list th{font-size:90%;margin:0;overflow:visible;padding:8px 16px}.wy-table td:first-child,.rst-content table.docutils td:first-child,.rst-content table.field-list td:first-child,.wy-table th:first-child,.rst-content table.docutils th:first-child,.rst-content table.field-list th:first-child{border-left-width:0}.wy-table thead,.rst-content table.docutils thead,.rst-content table.field-list thead{color:#000;text-align:left;vertical-align:bottom;white-space:nowrap}.wy-table thead th,.rst-content table.docutils thead th,.rst-content table.field-list thead th{font-weight:bold;border-bottom:solid 2px #e1e4e5}.wy-table td,.rst-content table.docutils td,.rst-content table.field-list td{background-color:transparent;vertical-align:middle}.wy-table td p,.rst-content table.docutils td p,.rst-content table.field-list td p{line-height:18px}.wy-table td p:last-child,.rst-content table.docutils td p:last-child,.rst-content table.field-list td p:last-child{margin-bottom:0}.wy-table .wy-table-cell-min,.rst-content table.docutils .wy-table-cell-min,.rst-content table.field-list .wy-table-cell-min{width:1%;padding-right:0}.wy-table .wy-table-cell-min input[type=checkbox],.rst-content table.docutils .wy-table-cell-min input[type=checkbox],.rst-content table.field-list .wy-table-cell-min input[type=checkbox],.wy-table .wy-table-cell-min input[type=checkbox],.rst-content table.docutils .wy-table-cell-min input[type=checkbox],.rst-content table.field-list .wy-table-cell-min input[type=checkbox]{margin:0}.wy-table-secondary{color:gray;font-size:90%}.wy-table-tertiary{color:gray;font-size:80%}.wy-table-odd td,.wy-table-striped tr:nth-child(2n-1) td,.rst-content table.docutils:not(.field-list) tr:nth-child(2n-1) td{background-color:#f3f6f6}.wy-table-backed{background-color:#f3f6f6}.wy-table-bordered-all,.rst-content table.docutils{border:1px solid #e1e4e5}.wy-table-bordered-all td,.rst-content table.docutils td{border-bottom:1px solid #e1e4e5;border-left:1px solid #e1e4e5}.wy-table-bordered-all tbody>tr:last-child td,.rst-content table.docutils tbody>tr:last-child td{border-bottom-width:0}.wy-table-bordered{border:1px solid #e1e4e5}.wy-table-bordered-rows td{border-bottom:1px solid #e1e4e5}.wy-table-bordered-rows tbody>tr:last-child td{border-bottom-width:0}.wy-table-horizontal tbody>tr:last-child td{border-bottom-width:0}.wy-table-horizontal td,.wy-table-horizontal th{border-width:0 0 1px 0;border-bottom:1px solid #e1e4e5}.wy-table-horizontal tbody>tr:last-child td{border-bottom-width:0}.wy-table-responsive{margin-bottom:24px;max-width:100%;overflow:auto}.wy-table-responsive table{margin-bottom:0 !important}.wy-table-responsive table td,.wy-table-responsive table th{white-space:nowrap}a{color:#2980B9;text-decoration:none;cursor:pointer}a:hover{color:#3091d1}a:visited{color:#9B59B6}html{height:100%;overflow-x:hidden}body{font-family:"Lato","proxima-nova","Helvetica Neue",Arial,sans-serif;font-weight:normal;color:#404040;min-height:100%;overflow-x:hidden;background:#edf0f2}.wy-text-left{text-align:left}.wy-text-center{text-align:center}.wy-text-right{text-align:right}.wy-text-large{font-size:120%}.wy-text-normal{font-size:100%}.wy-text-small,small{font-size:80%}.wy-text-strike{text-decoration:line-through}.wy-text-warning{color:#E67E22 !important}a.wy-text-warning:hover{color:#eb9950 !important}.wy-text-info{color:#2980B9 !important}a.wy-text-info:hover{color:#409ad5 !important}.wy-text-success{color:#27AE60 !important}a.wy-text-success:hover{color:#36d278 !important}.wy-text-danger{color:#E74C3C !important}a.wy-text-danger:hover{color:#ed7669 !important}.wy-text-neutral{color:#404040 !important}a.wy-text-neutral:hover{color:#595959 !important}h1,h2,.rst-content .toctree-wrapper p.caption,h3,h4,h5,h6,legend{margin-top:0;font-weight:700;font-family:"Roboto Slab","ff-tisa-web-pro","Georgia",Arial,sans-serif}p{line-height:24px;margin:0;font-size:16px;margin-bottom:24px}h1{font-size:175%}h2,.rst-content .toctree-wrapper p.caption{font-size:150%}h3{font-size:125%}h4{font-size:115%}h5{font-size:110%}h6{font-size:100%}hr{display:block;height:1px;border:0;border-top:1px solid #e1e4e5;margin:24px 0;padding:0}code,.rst-content tt,.rst-content code{white-space:nowrap;max-width:100%;background:#fff;border:solid 1px #e1e4e5;font-size:75%;padding:0 5px;font-family:Consolas,"Andale Mono WT","Andale Mono","Lucida Console","Lucida Sans Typewriter","DejaVu Sans Mono","Bitstream Vera Sans Mono","Liberation Mono","Nimbus Mono L",Monaco,"Courier New",Courier,monospace;color:#E74C3C;overflow-x:auto}code.code-large,.rst-content tt.code-large{font-size:90%}.wy-plain-list-disc,.rst-content .section ul,.rst-content .toctree-wrapper ul,article ul{list-style:disc;line-height:24px;margin-bottom:24px}.wy-plain-list-disc li,.rst-content .section ul li,.rst-content .toctree-wrapper ul li,article ul li{list-style:disc;margin-left:24px}.wy-plain-list-disc li p:last-child,.rst-content .section ul li p:last-child,.rst-content .toctree-wrapper ul li p:last-child,article ul li p:last-child{margin-bottom:0}.wy-plain-list-disc li ul,.rst-content .section ul li ul,.rst-content .toctree-wrapper ul li ul,article ul li ul{margin-bottom:0}.wy-plain-list-disc li li,.rst-content .section ul li li,.rst-content .toctree-wrapper ul li li,article ul li li{list-style:circle}.wy-plain-list-disc li li li,.rst-content .section ul li li li,.rst-content .toctree-wrapper ul li li li,article ul li li li{list-style:square}.wy-plain-list-disc li ol li,.rst-content .section ul li ol li,.rst-content .toctree-wrapper ul li ol li,article ul li ol li{list-style:decimal}.wy-plain-list-decimal,.rst-content .section ol,.rst-content ol.arabic,article ol{list-style:decimal;line-height:24px;margin-bottom:24px}.wy-plain-list-decimal li,.rst-content .section ol li,.rst-content ol.arabic li,article ol li{list-style:decimal;margin-left:24px}.wy-plain-list-decimal li p:last-child,.rst-content .section ol li p:last-child,.rst-content ol.arabic li p:last-child,article ol li p:last-child{margin-bottom:0}.wy-plain-list-decimal li ul,.rst-content .section ol li ul,.rst-content ol.arabic li ul,article ol li ul{margin-bottom:0}.wy-plain-list-decimal li ul li,.rst-content .section ol li ul li,.rst-content ol.arabic li ul li,article ol li ul li{list-style:disc}.codeblock-example{border:1px solid #e1e4e5;border-bottom:none;padding:24px;padding-top:48px;font-weight:500;background:#fff;position:relative}.codeblock-example:after{content:"Example";position:absolute;top:0px;left:0px;background:#9B59B6;color:#fff;padding:6px 12px}.codeblock-example.prettyprint-example-only{border:1px solid #e1e4e5;margin-bottom:24px}.codeblock,pre.literal-block,.rst-content .literal-block,.rst-content pre.literal-block,div[class^='highlight']{border:1px solid #e1e4e5;padding:0px;overflow-x:auto;background:#fff;margin:1px 0 24px 0}.codeblock div[class^='highlight'],pre.literal-block div[class^='highlight'],.rst-content .literal-block div[class^='highlight'],div[class^='highlight'] div[class^='highlight']{border:none;background:none;margin:0}div[class^='highlight'] td.code{width:100%}.linenodiv pre{border-right:solid 1px #e6e9ea;margin:0;padding:12px 12px;font-family:Consolas,"Andale Mono WT","Andale Mono","Lucida Console","Lucida Sans Typewriter","DejaVu Sans Mono","Bitstream Vera Sans Mono","Liberation Mono","Nimbus Mono L",Monaco,"Courier New",Courier,monospace;font-size:12px;line-height:1.5;color:#d9d9d9}div[class^='highlight'] pre{white-space:pre;margin:0;padding:12px 12px;font-family:Consolas,"Andale Mono WT","Andale Mono","Lucida Console","Lucida Sans Typewriter","DejaVu Sans Mono","Bitstream Vera Sans Mono","Liberation Mono","Nimbus Mono L",Monaco,"Courier New",Courier,monospace;font-size:12px;line-height:1.5;display:block;overflow:auto;color:#404040}@media print{.codeblock,pre.literal-block,.rst-content .literal-block,.rst-content pre.literal-block,div[class^='highlight'],div[class^='highlight'] pre{white-space:pre-wrap}}.hll{background-color:#ffc;margin:0 -12px;padding:0 12px;display:block}.c{color:#998;font-style:italic}.err{color:#a61717;background-color:#e3d2d2}.k{font-weight:bold}.o{font-weight:bold}.cm{color:#998;font-style:italic}.cp{color:#999;font-weight:bold}.c1{color:#998;font-style:italic}.cs{color:#999;font-weight:bold;font-style:italic}.gd{color:#000;background-color:#fdd}.gd .x{color:#000;background-color:#faa}.ge{font-style:italic}.gr{color:#a00}.gh{color:#999}.gi{color:#000;background-color:#dfd}.gi .x{color:#000;background-color:#afa}.go{color:#888}.gp{color:#555}.gs{font-weight:bold}.gu{color:purple;font-weight:bold}.gt{color:#a00}.kc{font-weight:bold}.kd{font-weight:bold}.kn{font-weight:bold}.kp{font-weight:bold}.kr{font-weight:bold}.kt{color:#458;font-weight:bold}.m{color:#099}.s{color:#d14}.n{color:#333}.na{color:teal}.nb{color:#0086b3}.nc{color:#458;font-weight:bold}.no{color:teal}.ni{color:purple}.ne{color:#900;font-weight:bold}.nf{color:#900;font-weight:bold}.nn{color:#555}.nt{color:navy}.nv{color:teal}.ow{font-weight:bold}.w{color:#bbb}.mf{color:#099}.mh{color:#099}.mi{color:#099}.mo{color:#099}.sb{color:#d14}.sc{color:#d14}.sd{color:#d14}.s2{color:#d14}.se{color:#d14}.sh{color:#d14}.si{color:#d14}.sx{color:#d14}.sr{color:#009926}.s1{color:#d14}.ss{color:#990073}.bp{color:#999}.vc{color:teal}.vg{color:teal}.vi{color:teal}.il{color:#099}.gc{color:#999;background-color:#EAF2F5}.wy-breadcrumbs li{display:inline-block}.wy-breadcrumbs li.wy-breadcrumbs-aside{float:right}.wy-breadcrumbs li a{display:inline-block;padding:5px}.wy-breadcrumbs li a:first-child{padding-left:0}.wy-breadcrumbs li code,.wy-breadcrumbs li .rst-content tt,.rst-content .wy-breadcrumbs li tt{padding:5px;border:none;background:none}.wy-breadcrumbs li code.literal,.wy-breadcrumbs li .rst-content tt.literal,.rst-content .wy-breadcrumbs li tt.literal{color:#404040}.wy-breadcrumbs-extra{margin-bottom:0;color:#b3b3b3;font-size:80%;display:inline-block}@media screen and (max-width: 480px){.wy-breadcrumbs-extra{display:none}.wy-breadcrumbs li.wy-breadcrumbs-aside{display:none}}@media print{.wy-breadcrumbs li.wy-breadcrumbs-aside{display:none}}.wy-affix{position:fixed;top:1.618em}.wy-menu a:hover{text-decoration:none}.wy-menu-horiz{*zoom:1}.wy-menu-horiz:before,.wy-menu-horiz:after{display:table;content:""}.wy-menu-horiz:after{clear:both}.wy-menu-horiz ul,.wy-menu-horiz li{display:inline-block}.wy-menu-horiz li:hover{background:rgba(255,255,255,0.1)}.wy-menu-horiz li.divide-left{border-left:solid 1px #404040}.wy-menu-horiz li.divide-right{border-right:solid 1px #404040}.wy-menu-horiz a{height:32px;display:inline-block;line-height:32px;padding:0 16px}.wy-menu-vertical{width:300px}.wy-menu-vertical header,.wy-menu-vertical p.caption{height:32px;display:inline-block;line-height:32px;padding:0 1.618em;margin-bottom:0;display:block;font-weight:bold;text-transform:uppercase;font-size:80%;color:#555;white-space:nowrap}.wy-menu-vertical ul{margin-bottom:0}.wy-menu-vertical li.divide-top{border-top:solid 1px #404040}.wy-menu-vertical li.divide-bottom{border-bottom:solid 1px #404040}.wy-menu-vertical li.current{background:#e3e3e3}.wy-menu-vertical li.current a{color:gray;border-right:solid 1px #c9c9c9;padding:.4045em 2.427em}.wy-menu-vertical li.current a:hover{background:#d6d6d6}.wy-menu-vertical li code,.wy-menu-vertical li .rst-content tt,.rst-content .wy-menu-vertical li tt{border:none;background:inherit;color:inherit;padding-left:0;padding-right:0}.wy-menu-vertical li span.toctree-expand{display:block;float:left;margin-left:-1.2em;font-size:0.8em;line-height:1.6em;color:#4d4d4d}.wy-menu-vertical li.on a,.wy-menu-vertical li.current>a{color:#404040;padding:.4045em 1.618em;font-weight:bold;position:relative;background:#fcfcfc;border:none;border-bottom:solid 1px #c9c9c9;border-top:solid 1px #c9c9c9;padding-left:1.618em -4px}.wy-menu-vertical li.on a:hover,.wy-menu-vertical li.current>a:hover{background:#fcfcfc}.wy-menu-vertical li.on a:hover span.toctree-expand,.wy-menu-vertical li.current>a:hover span.toctree-expand{color:gray}.wy-menu-vertical li.on a span.toctree-expand,.wy-menu-vertical li.current>a span.toctree-expand{display:block;font-size:0.8em;line-height:1.6em;color:#333}.wy-menu-vertical li.toctree-l1.current li.toctree-l2>ul,.wy-menu-vertical li.toctree-l2.current li.toctree-l3>ul{display:none}.wy-menu-vertical li.toctree-l1.current li.toctree-l2.current>ul,.wy-menu-vertical li.toctree-l2.current li.toctree-l3.current>ul{display:block}.wy-menu-vertical li.toctree-l2.current>a{background:#c9c9c9;padding:.4045em 2.427em}.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a{display:block;background:#c9c9c9;padding:.4045em 4.045em}.wy-menu-vertical li.toctree-l2 a:hover span.toctree-expand{color:gray}.wy-menu-vertical li.toctree-l2 span.toctree-expand{color:#a3a3a3}.wy-menu-vertical li.toctree-l3{font-size:0.9em}.wy-menu-vertical li.toctree-l3.current>a{background:#bdbdbd;padding:.4045em 4.045em}.wy-menu-vertical li.toctree-l3.current li.toctree-l4>a{display:block;background:#bdbdbd;padding:.4045em 5.663em;border-top:none;border-bottom:none}.wy-menu-vertical li.toctree-l3 a:hover span.toctree-expand{color:gray}.wy-menu-vertical li.toctree-l3 span.toctree-expand{color:#969696}.wy-menu-vertical li.toctree-l4{font-size:0.9em}.wy-menu-vertical li.current ul{display:block}.wy-menu-vertical li ul{margin-bottom:0;display:none}.wy-menu-vertical .local-toc li ul{display:block}.wy-menu-vertical li ul li a{margin-bottom:0;color:#b3b3b3;font-weight:normal}.wy-menu-vertical a{display:inline-block;line-height:18px;padding:.4045em 1.618em;display:block;position:relative;font-size:90%;color:#b3b3b3}.wy-menu-vertical a:hover{background-color:#4e4a4a;cursor:pointer}.wy-menu-vertical a:hover span.toctree-expand{color:#b3b3b3}.wy-menu-vertical a:active{background-color:#2980B9;cursor:pointer;color:#fff}.wy-menu-vertical a:active span.toctree-expand{color:#fff}.wy-side-nav-search{display:block;width:300px;padding:.809em;margin-bottom:.809em;z-index:200;background-color:#2980B9;text-align:center;padding:.809em;display:block;color:#fcfcfc;margin-bottom:.809em}.wy-side-nav-search input[type=text]{width:100%;border-radius:50px;padding:6px 12px;border-color:#2472a4}.wy-side-nav-search img{display:block;margin:auto auto .809em auto;height:45px;width:45px;background-color:#2980B9;padding:5px;border-radius:100%}.wy-side-nav-search>a,.wy-side-nav-search .wy-dropdown>a{color:#fcfcfc;font-size:100%;font-weight:bold;display:inline-block;padding:4px 6px;margin-bottom:.809em}.wy-side-nav-search>a:hover,.wy-side-nav-search .wy-dropdown>a:hover{background:rgba(255,255,255,0.1)}.wy-side-nav-search>a img.logo,.wy-side-nav-search .wy-dropdown>a img.logo{display:block;margin:0 auto;height:auto;width:auto;border-radius:0;max-width:100%;background:transparent}.wy-side-nav-search>a.icon img.logo,.wy-side-nav-search .wy-dropdown>a.icon img.logo{margin-top:0.85em}.wy-side-nav-search>div.version{margin-top:-.4045em;margin-bottom:.809em;font-weight:normal;color:rgba(255,255,255,0.3)}.wy-nav .wy-menu-vertical header{color:#2980B9}.wy-nav .wy-menu-vertical a{color:#b3b3b3}.wy-nav .wy-menu-vertical a:hover{background-color:#2980B9;color:#fff}[data-menu-wrap]{-webkit-transition:all 0.2s ease-in;-moz-transition:all 0.2s ease-in;transition:all 0.2s ease-in;position:absolute;opacity:1;width:100%;opacity:0}[data-menu-wrap].move-center{left:0;right:auto;opacity:1}[data-menu-wrap].move-left{right:auto;left:-100%;opacity:0}[data-menu-wrap].move-right{right:-100%;left:auto;opacity:0}.wy-body-for-nav{background:left repeat-y #fcfcfc;background-image:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyRpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENTNiAoTWFjaW50b3NoKSIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDoxOERBMTRGRDBFMUUxMUUzODUwMkJCOThDMEVFNURFMCIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDoxOERBMTRGRTBFMUUxMUUzODUwMkJCOThDMEVFNURFMCI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOjE4REExNEZCMEUxRTExRTM4NTAyQkI5OEMwRUU1REUwIiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOjE4REExNEZDMEUxRTExRTM4NTAyQkI5OEMwRUU1REUwIi8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+EwrlwAAAAA5JREFUeNpiMDU0BAgwAAE2AJgB9BnaAAAAAElFTkSuQmCC);background-size:300px 1px}.wy-grid-for-nav{position:absolute;width:100%;height:100%}.wy-nav-side{position:fixed;top:0;bottom:0;left:0;padding-bottom:2em;width:300px;overflow-x:hidden;overflow-y:hidden;min-height:100%;background:#343131;z-index:200}.wy-side-scroll{width:320px;position:relative;overflow-x:hidden;overflow-y:scroll;height:100%}.wy-nav-top{display:none;background:#2980B9;color:#fff;padding:.4045em .809em;position:relative;line-height:50px;text-align:center;font-size:100%;*zoom:1}.wy-nav-top:before,.wy-nav-top:after{display:table;content:""}.wy-nav-top:after{clear:both}.wy-nav-top a{color:#fff;font-weight:bold}.wy-nav-top img{margin-right:12px;height:45px;width:45px;background-color:#2980B9;padding:5px;border-radius:100%}.wy-nav-top i{font-size:30px;float:left;cursor:pointer;padding-top:inherit}.wy-nav-content-wrap{margin-left:300px;background:#fcfcfc;min-height:100%}.wy-nav-content{padding:1.618em 3.236em;height:100%;max-width:800px;margin:auto}.wy-body-mask{position:fixed;width:100%;height:100%;background:rgba(0,0,0,0.2);display:none;z-index:499}.wy-body-mask.on{display:block}footer{color:#999}footer p{margin-bottom:12px}footer span.commit code,footer span.commit .rst-content tt,.rst-content footer span.commit tt{padding:0px;font-family:Consolas,"Andale Mono WT","Andale Mono","Lucida Console","Lucida Sans Typewriter","DejaVu Sans Mono","Bitstream Vera Sans Mono","Liberation Mono","Nimbus Mono L",Monaco,"Courier New",Courier,monospace;font-size:1em;background:none;border:none;color:#999}.rst-footer-buttons{*zoom:1}.rst-footer-buttons:before,.rst-footer-buttons:after{display:table;content:""}.rst-footer-buttons:after{clear:both}.rst-breadcrumbs-buttons{margin-top:12px;*zoom:1}.rst-breadcrumbs-buttons:before,.rst-breadcrumbs-buttons:after{display:table;content:""}.rst-breadcrumbs-buttons:after{clear:both}#search-results .search li{margin-bottom:24px;border-bottom:solid 1px #e1e4e5;padding-bottom:24px}#search-results .search li:first-child{border-top:solid 1px #e1e4e5;padding-top:24px}#search-results .search li a{font-size:120%;margin-bottom:12px;display:inline-block}#search-results .context{color:gray;font-size:90%}@media screen and (max-width: 768px){.wy-body-for-nav{background:#fcfcfc}.wy-nav-top{display:block}.wy-nav-side{left:-300px}.wy-nav-side.shift{width:85%;left:0}.wy-side-scroll{width:auto}.wy-side-nav-search{width:auto}.wy-menu.wy-menu-vertical{width:auto}.wy-nav-content-wrap{margin-left:0}.wy-nav-content-wrap .wy-nav-content{padding:1.618em}.wy-nav-content-wrap.shift{position:fixed;min-width:100%;left:85%;top:0;height:100%;overflow:hidden}}@media screen and (min-width: 1400px){.wy-nav-content-wrap{background:rgba(0,0,0,0.05)}.wy-nav-content{margin:0;background:#fcfcfc}}@media print{.rst-versions,footer,.wy-nav-side{display:none}.wy-nav-content-wrap{margin-left:0}}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;border-top:solid 10px #343131;font-family:"Lato","proxima-nova","Helvetica Neue",Arial,sans-serif;z-index:400}.rst-versions a{color:#2980B9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27AE60;*zoom:1}.rst-versions .rst-current-version:before,.rst-versions .rst-current-version:after{display:table;content:""}.rst-versions .rst-current-version:after{clear:both}.rst-versions .rst-current-version .fa,.rst-versions .rst-current-version .wy-menu-vertical li span.toctree-expand,.wy-menu-vertical li .rst-versions .rst-current-version span.toctree-expand,.rst-versions .rst-current-version .rst-content .admonition-title,.rst-content .rst-versions .rst-current-version .admonition-title,.rst-versions .rst-current-version .rst-content h1 .headerlink,.rst-content h1 .rst-versions .rst-current-version .headerlink,.rst-versions .rst-current-version .rst-content h2 .headerlink,.rst-content h2 .rst-versions .rst-current-version .headerlink,.rst-versions .rst-current-version .rst-content h3 .headerlink,.rst-content h3 .rst-versions .rst-current-version .headerlink,.rst-versions .rst-current-version .rst-content h4 .headerlink,.rst-content h4 .rst-versions .rst-current-version .headerlink,.rst-versions .rst-current-version .rst-content h5 .headerlink,.rst-content h5 .rst-versions .rst-current-version .headerlink,.rst-versions .rst-current-version .rst-content h6 .headerlink,.rst-content h6 .rst-versions .rst-current-version .headerlink,.rst-versions .rst-current-version .rst-content dl dt .headerlink,.rst-content dl dt .rst-versions .rst-current-version .headerlink,.rst-versions .rst-current-version .rst-content p.caption .headerlink,.rst-content p.caption .rst-versions .rst-current-version .headerlink,.rst-versions .rst-current-version .rst-content tt.download span:first-child,.rst-content tt.download .rst-versions .rst-current-version span:first-child,.rst-versions .rst-current-version .rst-content code.download span:first-child,.rst-content code.download .rst-versions .rst-current-version span:first-child,.rst-versions .rst-current-version .icon{color:#fcfcfc}.rst-versions .rst-current-version .fa-book,.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#E74C3C;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#F1C40F;color:#000}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:gray;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:solid 1px #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px}.rst-versions.rst-badge .icon-book{float:none}.rst-versions.rst-badge .fa-book,.rst-versions.rst-badge .icon-book{float:none}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book,.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge .rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width: 768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}}.rst-content img{max-width:100%;height:auto !important}.rst-content .highlight>pre{line-height:normal}.rst-content div.figure{margin-bottom:24px}.rst-content div.figure p.caption{font-style:italic}.rst-content div.figure.align-center{text-align:center}.rst-content .section>img,.rst-content .section>a>img{margin-bottom:24px}.rst-content blockquote{margin-left:24px;line-height:24px;margin-bottom:24px}.rst-content .note .last,.rst-content .attention .last,.rst-content .caution .last,.rst-content .danger .last,.rst-content .error .last,.rst-content .hint .last,.rst-content .important .last,.rst-content .tip .last,.rst-content .warning .last,.rst-content .seealso .last,.rst-content .admonition-todo .last{margin-bottom:0}.rst-content .admonition-title:before{margin-right:4px}.rst-content .admonition table{border-color:rgba(0,0,0,0.1)}.rst-content .admonition table td,.rst-content .admonition table th{background:transparent !important;border-color:rgba(0,0,0,0.1) !important}.rst-content .section ol.loweralpha,.rst-content .section ol.loweralpha li{list-style:lower-alpha}.rst-content .section ol.upperalpha,.rst-content .section ol.upperalpha li{list-style:upper-alpha}.rst-content .section ol p,.rst-content .section ul p{margin-bottom:12px}.rst-content .line-block{margin-left:24px}.rst-content .topic-title{font-weight:bold;margin-bottom:12px}.rst-content .toc-backref{color:#404040}.rst-content .align-right{float:right;margin:0px 0px 24px 24px}.rst-content .align-left{float:left;margin:0px 24px 24px 0px}.rst-content .align-center{margin:auto;display:block}.rst-content h1 .headerlink,.rst-content h2 .headerlink,.rst-content .toctree-wrapper p.caption .headerlink,.rst-content h3 .headerlink,.rst-content h4 .headerlink,.rst-content h5 .headerlink,.rst-content h6 .headerlink,.rst-content dl dt .headerlink,.rst-content p.caption .headerlink{display:none;visibility:hidden;font-size:14px}.rst-content h1 .headerlink:after,.rst-content h2 .headerlink:after,.rst-content .toctree-wrapper p.caption .headerlink:after,.rst-content h3 .headerlink:after,.rst-content h4 .headerlink:after,.rst-content h5 .headerlink:after,.rst-content h6 .headerlink:after,.rst-content dl dt .headerlink:after,.rst-content p.caption .headerlink:after{visibility:visible;content:"";font-family:FontAwesome;display:inline-block}.rst-content h1:hover .headerlink,.rst-content h2:hover .headerlink,.rst-content .toctree-wrapper p.caption:hover .headerlink,.rst-content h3:hover .headerlink,.rst-content h4:hover .headerlink,.rst-content h5:hover .headerlink,.rst-content h6:hover .headerlink,.rst-content dl dt:hover .headerlink,.rst-content p.caption:hover .headerlink{display:inline-block}.rst-content .sidebar{float:right;width:40%;display:block;margin:0 0 24px 24px;padding:24px;background:#f3f6f6;border:solid 1px #e1e4e5}.rst-content .sidebar p,.rst-content .sidebar ul,.rst-content .sidebar dl{font-size:90%}.rst-content .sidebar .last{margin-bottom:0}.rst-content .sidebar .sidebar-title{display:block;font-family:"Roboto Slab","ff-tisa-web-pro","Georgia",Arial,sans-serif;font-weight:bold;background:#e1e4e5;padding:6px 12px;margin:-24px;margin-bottom:24px;font-size:100%}.rst-content .highlighted{background:#F1C40F;display:inline-block;font-weight:bold;padding:0 6px}.rst-content .footnote-reference,.rst-content .citation-reference{vertical-align:super;font-size:90%}.rst-content table.docutils.citation,.rst-content table.docutils.footnote{background:none;border:none;color:#999}.rst-content table.docutils.citation td,.rst-content table.docutils.citation tr,.rst-content table.docutils.footnote td,.rst-content table.docutils.footnote tr{border:none;background-color:transparent !important;white-space:normal}.rst-content table.docutils.citation td.label,.rst-content table.docutils.footnote td.label{padding-left:0;padding-right:0;vertical-align:top}.rst-content table.docutils.citation tt,.rst-content table.docutils.citation code,.rst-content table.docutils.footnote tt,.rst-content table.docutils.footnote code{color:#555}.rst-content table.field-list{border:none}.rst-content table.field-list td{border:none;padding-top:5px}.rst-content table.field-list td>strong{display:inline-block;margin-top:3px}.rst-content table.field-list .field-name{padding-right:10px;text-align:left;white-space:nowrap}.rst-content table.field-list .field-body{text-align:left;padding-left:0}.rst-content tt,.rst-content tt,.rst-content code{color:#000;padding:2px 5px}.rst-content tt big,.rst-content tt em,.rst-content tt big,.rst-content code big,.rst-content tt em,.rst-content code em{font-size:100% !important;line-height:normal}.rst-content tt.literal,.rst-content tt.literal,.rst-content code.literal{color:#E74C3C}.rst-content tt.xref,a .rst-content tt,.rst-content tt.xref,.rst-content code.xref,a .rst-content tt,a .rst-content code{font-weight:bold;color:#404040}.rst-content a tt,.rst-content a tt,.rst-content a code{color:#2980B9}.rst-content dl{margin-bottom:24px}.rst-content dl dt{font-weight:bold}.rst-content dl p,.rst-content dl table,.rst-content dl ul,.rst-content dl ol{margin-bottom:12px !important}.rst-content dl dd{margin:0 0 12px 24px}.rst-content dl:not(.docutils){margin-bottom:24px}.rst-content dl:not(.docutils) dt{display:inline-block;margin:6px 0;font-size:90%;line-height:normal;background:#e7f2fa;color:#2980B9;border-top:solid 3px #6ab0de;padding:6px;position:relative}.rst-content dl:not(.docutils) dt:before{color:#6ab0de}.rst-content dl:not(.docutils) dt .headerlink{color:#404040;font-size:100% !important}.rst-content dl:not(.docutils) dl dt{margin-bottom:6px;border:none;border-left:solid 3px #ccc;background:#f0f0f0;color:#555}.rst-content dl:not(.docutils) dl dt .headerlink{color:#404040;font-size:100% !important}.rst-content dl:not(.docutils) dt:first-child{margin-top:0}.rst-content dl:not(.docutils) tt,.rst-content dl:not(.docutils) tt,.rst-content dl:not(.docutils) code{font-weight:bold}.rst-content dl:not(.docutils) tt.descname,.rst-content dl:not(.docutils) tt.descclassname,.rst-content dl:not(.docutils) tt.descname,.rst-content dl:not(.docutils) code.descname,.rst-content dl:not(.docutils) tt.descclassname,.rst-content dl:not(.docutils) code.descclassname{background-color:transparent;border:none;padding:0;font-size:100% !important}.rst-content dl:not(.docutils) tt.descname,.rst-content dl:not(.docutils) tt.descname,.rst-content dl:not(.docutils) code.descname{font-weight:bold}.rst-content dl:not(.docutils) .optional{display:inline-block;padding:0 4px;color:#000;font-weight:bold}.rst-content dl:not(.docutils) .property{display:inline-block;padding-right:8px}.rst-content .viewcode-link,.rst-content .viewcode-back{display:inline-block;color:#27AE60;font-size:80%;padding-left:24px}.rst-content .viewcode-back{display:block;float:right}.rst-content p.rubric{margin-bottom:12px;font-weight:bold}.rst-content tt.download,.rst-content code.download{background:inherit;padding:inherit;font-weight:normal;font-family:inherit;font-size:inherit;color:inherit;border:inherit;white-space:inherit}.rst-content tt.download span:first-child,.rst-content code.download span:first-child{-webkit-font-smoothing:subpixel-antialiased}.rst-content tt.download span:first-child:before,.rst-content code.download span:first-child:before{margin-right:4px}.rst-content .guilabel{border:1px solid #7fbbe3;background:#e7f2fa;font-size:80%;font-weight:700;border-radius:4px;padding:2.4px 6px;margin:auto 2px}@media screen and (max-width: 480px){.rst-content .sidebar{width:100%}}span[id*='MathJax-Span']{color:#404040}.math{text-align:center}@font-face{font-family:"Inconsolata";font-style:normal;font-weight:400;src:local("Inconsolata"),local("Inconsolata-Regular"),url(../fonts/Inconsolata-Regular.ttf) format("truetype")}@font-face{font-family:"Inconsolata";font-style:normal;font-weight:700;src:local("Inconsolata Bold"),local("Inconsolata-Bold"),url(../fonts/Inconsolata-Bold.ttf) format("truetype")}@font-face{font-family:"Lato";font-style:normal;font-weight:400;src:local("Lato Regular"),local("Lato-Regular"),url(../fonts/Lato-Regular.ttf) format("truetype")}@font-face{font-family:"Lato";font-style:normal;font-weight:700;src:local("Lato Bold"),local("Lato-Bold"),url(../fonts/Lato-Bold.ttf) format("truetype")}@font-face{font-family:"Roboto Slab";font-style:normal;font-weight:400;src:local("Roboto Slab Regular"),local("RobotoSlab-Regular"),url(../fonts/RobotoSlab-Regular.ttf) format("truetype")}@font-face{font-family:"Roboto Slab";font-style:normal;font-weight:700;src:local("Roboto Slab Bold"),local("RobotoSlab-Bold"),url(../fonts/RobotoSlab-Bold.ttf) format("truetype")} /*# sourceMappingURL=theme.css.map */ ================================================ FILE: docs/_static/doctools.js ================================================ /* * doctools.js * ~~~~~~~~~~~ * * Sphinx JavaScript utilities for all documentation. * * :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ /** * select a different prefix for underscore */ $u = _.noConflict(); /** * make the code below compatible with browsers without * an installed firebug like debugger if (!window.console || !console.firebug) { var names = ["log", "debug", "info", "warn", "error", "assert", "dir", "dirxml", "group", "groupEnd", "time", "timeEnd", "count", "trace", "profile", "profileEnd"]; window.console = {}; for (var i = 0; i < names.length; ++i) window.console[names[i]] = function() {}; } */ /** * small helper function to urldecode strings */ jQuery.urldecode = function(x) { return decodeURIComponent(x).replace(/\+/g, ' '); }; /** * small helper function to urlencode strings */ jQuery.urlencode = encodeURIComponent; /** * This function returns the parsed url parameters of the * current request. Multiple values per key are supported, * it will always return arrays of strings for the value parts. */ jQuery.getQueryParameters = function(s) { if (typeof s == 'undefined') s = document.location.search; var parts = s.substr(s.indexOf('?') + 1).split('&'); var result = {}; for (var i = 0; i < parts.length; i++) { var tmp = parts[i].split('=', 2); var key = jQuery.urldecode(tmp[0]); var value = jQuery.urldecode(tmp[1]); if (key in result) result[key].push(value); else result[key] = [value]; } return result; }; /** * highlight a given string on a jquery object by wrapping it in * span elements with the given class name. */ jQuery.fn.highlightText = function(text, className) { function highlight(node) { if (node.nodeType == 3) { var val = node.nodeValue; var pos = val.toLowerCase().indexOf(text); if (pos >= 0 && !jQuery(node.parentNode).hasClass(className)) { var span = document.createElement("span"); span.className = className; span.appendChild(document.createTextNode(val.substr(pos, text.length))); node.parentNode.insertBefore(span, node.parentNode.insertBefore( document.createTextNode(val.substr(pos + text.length)), node.nextSibling)); node.nodeValue = val.substr(0, pos); } } else if (!jQuery(node).is("button, select, textarea")) { jQuery.each(node.childNodes, function() { highlight(this); }); } } return this.each(function() { highlight(this); }); }; /* * backward compatibility for jQuery.browser * This will be supported until firefox bug is fixed. */ if (!jQuery.browser) { jQuery.uaMatch = function(ua) { ua = ua.toLowerCase(); var match = /(chrome)[ \/]([\w.]+)/.exec(ua) || /(webkit)[ \/]([\w.]+)/.exec(ua) || /(opera)(?:.*version|)[ \/]([\w.]+)/.exec(ua) || /(msie) ([\w.]+)/.exec(ua) || ua.indexOf("compatible") < 0 && /(mozilla)(?:.*? rv:([\w.]+)|)/.exec(ua) || []; return { browser: match[ 1 ] || "", version: match[ 2 ] || "0" }; }; jQuery.browser = {}; jQuery.browser[jQuery.uaMatch(navigator.userAgent).browser] = true; } /** * Small JavaScript module for the documentation. */ var Documentation = { init : function() { this.fixFirefoxAnchorBug(); this.highlightSearchWords(); this.initIndexTable(); }, /** * i18n support */ TRANSLATIONS : {}, PLURAL_EXPR : function(n) { return n == 1 ? 0 : 1; }, LOCALE : 'unknown', // gettext and ngettext don't access this so that the functions // can safely bound to a different name (_ = Documentation.gettext) gettext : function(string) { var translated = Documentation.TRANSLATIONS[string]; if (typeof translated == 'undefined') return string; return (typeof translated == 'string') ? translated : translated[0]; }, ngettext : function(singular, plural, n) { var translated = Documentation.TRANSLATIONS[singular]; if (typeof translated == 'undefined') return (n == 1) ? singular : plural; return translated[Documentation.PLURALEXPR(n)]; }, addTranslations : function(catalog) { for (var key in catalog.messages) this.TRANSLATIONS[key] = catalog.messages[key]; this.PLURAL_EXPR = new Function('n', 'return +(' + catalog.plural_expr + ')'); this.LOCALE = catalog.locale; }, /** * add context elements like header anchor links */ addContextElements : function() { $('div[id] > :header:first').each(function() { $('\u00B6'). attr('href', '#' + this.id). attr('title', _('Permalink to this headline')). appendTo(this); }); $('dt[id]').each(function() { $('\u00B6'). attr('href', '#' + this.id). attr('title', _('Permalink to this definition')). appendTo(this); }); }, /** * workaround a firefox stupidity * see: https://bugzilla.mozilla.org/show_bug.cgi?id=645075 */ fixFirefoxAnchorBug : function() { if (document.location.hash) window.setTimeout(function() { document.location.href += ''; }, 10); }, /** * highlight the search words provided in the url in the text */ highlightSearchWords : function() { var params = $.getQueryParameters(); var terms = (params.highlight) ? params.highlight[0].split(/\s+/) : []; if (terms.length) { var body = $('div.body'); if (!body.length) { body = $('body'); } window.setTimeout(function() { $.each(terms, function() { body.highlightText(this.toLowerCase(), 'highlighted'); }); }, 10); $('

' + _('Hide Search Matches') + '

') .appendTo($('#searchbox')); } }, /** * init the domain index toggle buttons */ initIndexTable : function() { var togglers = $('img.toggler').click(function() { var src = $(this).attr('src'); var idnum = $(this).attr('id').substr(7); $('tr.cg-' + idnum).toggle(); if (src.substr(-9) == 'minus.png') $(this).attr('src', src.substr(0, src.length-9) + 'plus.png'); else $(this).attr('src', src.substr(0, src.length-8) + 'minus.png'); }).css('display', ''); if (DOCUMENTATION_OPTIONS.COLLAPSE_INDEX) { togglers.click(); } }, /** * helper function to hide the search marks again */ hideSearchWords : function() { $('#searchbox .highlight-link').fadeOut(300); $('span.highlighted').removeClass('highlighted'); }, /** * make the url absolute */ makeURL : function(relativeURL) { return DOCUMENTATION_OPTIONS.URL_ROOT + '/' + relativeURL; }, /** * get the current relative url */ getCurrentURL : function() { var path = document.location.pathname; var parts = path.split(/\//); $.each(DOCUMENTATION_OPTIONS.URL_ROOT.split(/\//), function() { if (this == '..') parts.pop(); }); var url = parts.join('/'); return path.substring(url.lastIndexOf('/') + 1, path.length - 1); }, initOnKeyListeners: function() { $(document).keyup(function(event) { var activeElementType = document.activeElement.tagName; // don't navigate when in search box or textarea if (activeElementType !== 'TEXTAREA' && activeElementType !== 'INPUT' && activeElementType !== 'SELECT') { switch (event.keyCode) { case 37: // left var prevHref = $('link[rel="prev"]').prop('href'); if (prevHref) { window.location.href = prevHref; return false; } case 39: // right var nextHref = $('link[rel="next"]').prop('href'); if (nextHref) { window.location.href = nextHref; return false; } } } }); } }; // quick alias for translations _ = Documentation.gettext; $(document).ready(function() { Documentation.init(); }); ================================================ FILE: docs/_static/jquery-3.1.0.js ================================================ /*eslint-disable no-unused-vars*/ /*! * jQuery JavaScript Library v3.1.0 * https://jquery.com/ * * Includes Sizzle.js * https://sizzlejs.com/ * * Copyright jQuery Foundation and other contributors * Released under the MIT license * https://jquery.org/license * * Date: 2016-07-07T21:44Z */ ( function( global, factory ) { "use strict"; if ( typeof module === "object" && typeof module.exports === "object" ) { // For CommonJS and CommonJS-like environments where a proper `window` // is present, execute the factory and get jQuery. // For environments that do not have a `window` with a `document` // (such as Node.js), expose a factory as module.exports. // This accentuates the need for the creation of a real `window`. // e.g. var jQuery = require("jquery")(window); // See ticket #14549 for more info. module.exports = global.document ? factory( global, true ) : function( w ) { if ( !w.document ) { throw new Error( "jQuery requires a window with a document" ); } return factory( w ); }; } else { factory( global ); } // Pass this if window is not defined yet } )( typeof window !== "undefined" ? window : this, function( window, noGlobal ) { // Edge <= 12 - 13+, Firefox <=18 - 45+, IE 10 - 11, Safari 5.1 - 9+, iOS 6 - 9.1 // throw exceptions when non-strict code (e.g., ASP.NET 4.5) accesses strict mode // arguments.callee.caller (trac-13335). But as of jQuery 3.0 (2016), strict mode should be common // enough that all such attempts are guarded in a try block. "use strict"; var arr = []; var document = window.document; var getProto = Object.getPrototypeOf; var slice = arr.slice; var concat = arr.concat; var push = arr.push; var indexOf = arr.indexOf; var class2type = {}; var toString = class2type.toString; var hasOwn = class2type.hasOwnProperty; var fnToString = hasOwn.toString; var ObjectFunctionString = fnToString.call( Object ); var support = {}; function DOMEval( code, doc ) { doc = doc || document; var script = doc.createElement( "script" ); script.text = code; doc.head.appendChild( script ).parentNode.removeChild( script ); } /* global Symbol */ // Defining this global in .eslintrc would create a danger of using the global // unguarded in another place, it seems safer to define global only for this module var version = "3.1.0", // Define a local copy of jQuery jQuery = function( selector, context ) { // The jQuery object is actually just the init constructor 'enhanced' // Need init if jQuery is called (just allow error to be thrown if not included) return new jQuery.fn.init( selector, context ); }, // Support: Android <=4.0 only // Make sure we trim BOM and NBSP rtrim = /^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g, // Matches dashed string for camelizing rmsPrefix = /^-ms-/, rdashAlpha = /-([a-z])/g, // Used by jQuery.camelCase as callback to replace() fcamelCase = function( all, letter ) { return letter.toUpperCase(); }; jQuery.fn = jQuery.prototype = { // The current version of jQuery being used jquery: version, constructor: jQuery, // The default length of a jQuery object is 0 length: 0, toArray: function() { return slice.call( this ); }, // Get the Nth element in the matched element set OR // Get the whole matched element set as a clean array get: function( num ) { return num != null ? // Return just the one element from the set ( num < 0 ? this[ num + this.length ] : this[ num ] ) : // Return all the elements in a clean array slice.call( this ); }, // Take an array of elements and push it onto the stack // (returning the new matched element set) pushStack: function( elems ) { // Build a new jQuery matched element set var ret = jQuery.merge( this.constructor(), elems ); // Add the old object onto the stack (as a reference) ret.prevObject = this; // Return the newly-formed element set return ret; }, // Execute a callback for every element in the matched set. each: function( callback ) { return jQuery.each( this, callback ); }, map: function( callback ) { return this.pushStack( jQuery.map( this, function( elem, i ) { return callback.call( elem, i, elem ); } ) ); }, slice: function() { return this.pushStack( slice.apply( this, arguments ) ); }, first: function() { return this.eq( 0 ); }, last: function() { return this.eq( -1 ); }, eq: function( i ) { var len = this.length, j = +i + ( i < 0 ? len : 0 ); return this.pushStack( j >= 0 && j < len ? [ this[ j ] ] : [] ); }, end: function() { return this.prevObject || this.constructor(); }, // For internal use only. // Behaves like an Array's method, not like a jQuery method. push: push, sort: arr.sort, splice: arr.splice }; jQuery.extend = jQuery.fn.extend = function() { var options, name, src, copy, copyIsArray, clone, target = arguments[ 0 ] || {}, i = 1, length = arguments.length, deep = false; // Handle a deep copy situation if ( typeof target === "boolean" ) { deep = target; // Skip the boolean and the target target = arguments[ i ] || {}; i++; } // Handle case when target is a string or something (possible in deep copy) if ( typeof target !== "object" && !jQuery.isFunction( target ) ) { target = {}; } // Extend jQuery itself if only one argument is passed if ( i === length ) { target = this; i--; } for ( ; i < length; i++ ) { // Only deal with non-null/undefined values if ( ( options = arguments[ i ] ) != null ) { // Extend the base object for ( name in options ) { src = target[ name ]; copy = options[ name ]; // Prevent never-ending loop if ( target === copy ) { continue; } // Recurse if we're merging plain objects or arrays if ( deep && copy && ( jQuery.isPlainObject( copy ) || ( copyIsArray = jQuery.isArray( copy ) ) ) ) { if ( copyIsArray ) { copyIsArray = false; clone = src && jQuery.isArray( src ) ? src : []; } else { clone = src && jQuery.isPlainObject( src ) ? src : {}; } // Never move original objects, clone them target[ name ] = jQuery.extend( deep, clone, copy ); // Don't bring in undefined values } else if ( copy !== undefined ) { target[ name ] = copy; } } } } // Return the modified object return target; }; jQuery.extend( { // Unique for each copy of jQuery on the page expando: "jQuery" + ( version + Math.random() ).replace( /\D/g, "" ), // Assume jQuery is ready without the ready module isReady: true, error: function( msg ) { throw new Error( msg ); }, noop: function() {}, isFunction: function( obj ) { return jQuery.type( obj ) === "function"; }, isArray: Array.isArray, isWindow: function( obj ) { return obj != null && obj === obj.window; }, isNumeric: function( obj ) { // As of jQuery 3.0, isNumeric is limited to // strings and numbers (primitives or objects) // that can be coerced to finite numbers (gh-2662) var type = jQuery.type( obj ); return ( type === "number" || type === "string" ) && // parseFloat NaNs numeric-cast false positives ("") // ...but misinterprets leading-number strings, particularly hex literals ("0x...") // subtraction forces infinities to NaN !isNaN( obj - parseFloat( obj ) ); }, isPlainObject: function( obj ) { var proto, Ctor; // Detect obvious negatives // Use toString instead of jQuery.type to catch host objects if ( !obj || toString.call( obj ) !== "[object Object]" ) { return false; } proto = getProto( obj ); // Objects with no prototype (e.g., `Object.create( null )`) are plain if ( !proto ) { return true; } // Objects with prototype are plain iff they were constructed by a global Object function Ctor = hasOwn.call( proto, "constructor" ) && proto.constructor; return typeof Ctor === "function" && fnToString.call( Ctor ) === ObjectFunctionString; }, isEmptyObject: function( obj ) { /* eslint-disable no-unused-vars */ // See https://github.com/eslint/eslint/issues/6125 var name; for ( name in obj ) { return false; } return true; }, type: function( obj ) { if ( obj == null ) { return obj + ""; } // Support: Android <=2.3 only (functionish RegExp) return typeof obj === "object" || typeof obj === "function" ? class2type[ toString.call( obj ) ] || "object" : typeof obj; }, // Evaluates a script in a global context globalEval: function( code ) { DOMEval( code ); }, // Convert dashed to camelCase; used by the css and data modules // Support: IE <=9 - 11, Edge 12 - 13 // Microsoft forgot to hump their vendor prefix (#9572) camelCase: function( string ) { return string.replace( rmsPrefix, "ms-" ).replace( rdashAlpha, fcamelCase ); }, nodeName: function( elem, name ) { return elem.nodeName && elem.nodeName.toLowerCase() === name.toLowerCase(); }, each: function( obj, callback ) { var length, i = 0; if ( isArrayLike( obj ) ) { length = obj.length; for ( ; i < length; i++ ) { if ( callback.call( obj[ i ], i, obj[ i ] ) === false ) { break; } } } else { for ( i in obj ) { if ( callback.call( obj[ i ], i, obj[ i ] ) === false ) { break; } } } return obj; }, // Support: Android <=4.0 only trim: function( text ) { return text == null ? "" : ( text + "" ).replace( rtrim, "" ); }, // results is for internal usage only makeArray: function( arr, results ) { var ret = results || []; if ( arr != null ) { if ( isArrayLike( Object( arr ) ) ) { jQuery.merge( ret, typeof arr === "string" ? [ arr ] : arr ); } else { push.call( ret, arr ); } } return ret; }, inArray: function( elem, arr, i ) { return arr == null ? -1 : indexOf.call( arr, elem, i ); }, // Support: Android <=4.0 only, PhantomJS 1 only // push.apply(_, arraylike) throws on ancient WebKit merge: function( first, second ) { var len = +second.length, j = 0, i = first.length; for ( ; j < len; j++ ) { first[ i++ ] = second[ j ]; } first.length = i; return first; }, grep: function( elems, callback, invert ) { var callbackInverse, matches = [], i = 0, length = elems.length, callbackExpect = !invert; // Go through the array, only saving the items // that pass the validator function for ( ; i < length; i++ ) { callbackInverse = !callback( elems[ i ], i ); if ( callbackInverse !== callbackExpect ) { matches.push( elems[ i ] ); } } return matches; }, // arg is for internal usage only map: function( elems, callback, arg ) { var length, value, i = 0, ret = []; // Go through the array, translating each of the items to their new values if ( isArrayLike( elems ) ) { length = elems.length; for ( ; i < length; i++ ) { value = callback( elems[ i ], i, arg ); if ( value != null ) { ret.push( value ); } } // Go through every key on the object, } else { for ( i in elems ) { value = callback( elems[ i ], i, arg ); if ( value != null ) { ret.push( value ); } } } // Flatten any nested arrays return concat.apply( [], ret ); }, // A global GUID counter for objects guid: 1, // Bind a function to a context, optionally partially applying any // arguments. proxy: function( fn, context ) { var tmp, args, proxy; if ( typeof context === "string" ) { tmp = fn[ context ]; context = fn; fn = tmp; } // Quick check to determine if target is callable, in the spec // this throws a TypeError, but we will just return undefined. if ( !jQuery.isFunction( fn ) ) { return undefined; } // Simulated bind args = slice.call( arguments, 2 ); proxy = function() { return fn.apply( context || this, args.concat( slice.call( arguments ) ) ); }; // Set the guid of unique handler to the same of original handler, so it can be removed proxy.guid = fn.guid = fn.guid || jQuery.guid++; return proxy; }, now: Date.now, // jQuery.support is not used in Core but other projects attach their // properties to it so it needs to exist. support: support } ); if ( typeof Symbol === "function" ) { jQuery.fn[ Symbol.iterator ] = arr[ Symbol.iterator ]; } // Populate the class2type map jQuery.each( "Boolean Number String Function Array Date RegExp Object Error Symbol".split( " " ), function( i, name ) { class2type[ "[object " + name + "]" ] = name.toLowerCase(); } ); function isArrayLike( obj ) { // Support: real iOS 8.2 only (not reproducible in simulator) // `in` check used to prevent JIT error (gh-2145) // hasOwn isn't used here due to false negatives // regarding Nodelist length in IE var length = !!obj && "length" in obj && obj.length, type = jQuery.type( obj ); if ( type === "function" || jQuery.isWindow( obj ) ) { return false; } return type === "array" || length === 0 || typeof length === "number" && length > 0 && ( length - 1 ) in obj; } var Sizzle = /*! * Sizzle CSS Selector Engine v2.3.0 * https://sizzlejs.com/ * * Copyright jQuery Foundation and other contributors * Released under the MIT license * http://jquery.org/license * * Date: 2016-01-04 */ (function( window ) { var i, support, Expr, getText, isXML, tokenize, compile, select, outermostContext, sortInput, hasDuplicate, // Local document vars setDocument, document, docElem, documentIsHTML, rbuggyQSA, rbuggyMatches, matches, contains, // Instance-specific data expando = "sizzle" + 1 * new Date(), preferredDoc = window.document, dirruns = 0, done = 0, classCache = createCache(), tokenCache = createCache(), compilerCache = createCache(), sortOrder = function( a, b ) { if ( a === b ) { hasDuplicate = true; } return 0; }, // Instance methods hasOwn = ({}).hasOwnProperty, arr = [], pop = arr.pop, push_native = arr.push, push = arr.push, slice = arr.slice, // Use a stripped-down indexOf as it's faster than native // https://jsperf.com/thor-indexof-vs-for/5 indexOf = function( list, elem ) { var i = 0, len = list.length; for ( ; i < len; i++ ) { if ( list[i] === elem ) { return i; } } return -1; }, booleans = "checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped", // Regular expressions // http://www.w3.org/TR/css3-selectors/#whitespace whitespace = "[\\x20\\t\\r\\n\\f]", // http://www.w3.org/TR/CSS21/syndata.html#value-def-identifier identifier = "(?:\\\\.|[\\w-]|[^\0-\\xa0])+", // Attribute selectors: http://www.w3.org/TR/selectors/#attribute-selectors attributes = "\\[" + whitespace + "*(" + identifier + ")(?:" + whitespace + // Operator (capture 2) "*([*^$|!~]?=)" + whitespace + // "Attribute values must be CSS identifiers [capture 5] or strings [capture 3 or capture 4]" "*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|(" + identifier + "))|)" + whitespace + "*\\]", pseudos = ":(" + identifier + ")(?:\\((" + // To reduce the number of selectors needing tokenize in the preFilter, prefer arguments: // 1. quoted (capture 3; capture 4 or capture 5) "('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|" + // 2. simple (capture 6) "((?:\\\\.|[^\\\\()[\\]]|" + attributes + ")*)|" + // 3. anything else (capture 2) ".*" + ")\\)|)", // Leading and non-escaped trailing whitespace, capturing some non-whitespace characters preceding the latter rwhitespace = new RegExp( whitespace + "+", "g" ), rtrim = new RegExp( "^" + whitespace + "+|((?:^|[^\\\\])(?:\\\\.)*)" + whitespace + "+$", "g" ), rcomma = new RegExp( "^" + whitespace + "*," + whitespace + "*" ), rcombinators = new RegExp( "^" + whitespace + "*([>+~]|" + whitespace + ")" + whitespace + "*" ), rattributeQuotes = new RegExp( "=" + whitespace + "*([^\\]'\"]*?)" + whitespace + "*\\]", "g" ), rpseudo = new RegExp( pseudos ), ridentifier = new RegExp( "^" + identifier + "$" ), matchExpr = { "ID": new RegExp( "^#(" + identifier + ")" ), "CLASS": new RegExp( "^\\.(" + identifier + ")" ), "TAG": new RegExp( "^(" + identifier + "|[*])" ), "ATTR": new RegExp( "^" + attributes ), "PSEUDO": new RegExp( "^" + pseudos ), "CHILD": new RegExp( "^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\(" + whitespace + "*(even|odd|(([+-]|)(\\d*)n|)" + whitespace + "*(?:([+-]|)" + whitespace + "*(\\d+)|))" + whitespace + "*\\)|)", "i" ), "bool": new RegExp( "^(?:" + booleans + ")$", "i" ), // For use in libraries implementing .is() // We use this for POS matching in `select` "needsContext": new RegExp( "^" + whitespace + "*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\(" + whitespace + "*((?:-\\d)?\\d*)" + whitespace + "*\\)|)(?=[^-]|$)", "i" ) }, rinputs = /^(?:input|select|textarea|button)$/i, rheader = /^h\d$/i, rnative = /^[^{]+\{\s*\[native \w/, // Easily-parseable/retrievable ID or TAG or CLASS selectors rquickExpr = /^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/, rsibling = /[+~]/, // CSS escapes // http://www.w3.org/TR/CSS21/syndata.html#escaped-characters runescape = new RegExp( "\\\\([\\da-f]{1,6}" + whitespace + "?|(" + whitespace + ")|.)", "ig" ), funescape = function( _, escaped, escapedWhitespace ) { var high = "0x" + escaped - 0x10000; // NaN means non-codepoint // Support: Firefox<24 // Workaround erroneous numeric interpretation of +"0x" return high !== high || escapedWhitespace ? escaped : high < 0 ? // BMP codepoint String.fromCharCode( high + 0x10000 ) : // Supplemental Plane codepoint (surrogate pair) String.fromCharCode( high >> 10 | 0xD800, high & 0x3FF | 0xDC00 ); }, // CSS string/identifier serialization // https://drafts.csswg.org/cssom/#common-serializing-idioms rcssescape = /([\0-\x1f\x7f]|^-?\d)|^-$|[^\x80-\uFFFF\w-]/g, fcssescape = function( ch, asCodePoint ) { if ( asCodePoint ) { // U+0000 NULL becomes U+FFFD REPLACEMENT CHARACTER if ( ch === "\0" ) { return "\uFFFD"; } // Control characters and (dependent upon position) numbers get escaped as code points return ch.slice( 0, -1 ) + "\\" + ch.charCodeAt( ch.length - 1 ).toString( 16 ) + " "; } // Other potentially-special ASCII characters get backslash-escaped return "\\" + ch; }, // Used for iframes // See setDocument() // Removing the function wrapper causes a "Permission Denied" // error in IE unloadHandler = function() { setDocument(); }, disabledAncestor = addCombinator( function( elem ) { return elem.disabled === true; }, { dir: "parentNode", next: "legend" } ); // Optimize for push.apply( _, NodeList ) try { push.apply( (arr = slice.call( preferredDoc.childNodes )), preferredDoc.childNodes ); // Support: Android<4.0 // Detect silently failing push.apply arr[ preferredDoc.childNodes.length ].nodeType; } catch ( e ) { push = { apply: arr.length ? // Leverage slice if possible function( target, els ) { push_native.apply( target, slice.call(els) ); } : // Support: IE<9 // Otherwise append directly function( target, els ) { var j = target.length, i = 0; // Can't trust NodeList.length while ( (target[j++] = els[i++]) ) {} target.length = j - 1; } }; } function Sizzle( selector, context, results, seed ) { var m, i, elem, nid, match, groups, newSelector, newContext = context && context.ownerDocument, // nodeType defaults to 9, since context defaults to document nodeType = context ? context.nodeType : 9; results = results || []; // Return early from calls with invalid selector or context if ( typeof selector !== "string" || !selector || nodeType !== 1 && nodeType !== 9 && nodeType !== 11 ) { return results; } // Try to shortcut find operations (as opposed to filters) in HTML documents if ( !seed ) { if ( ( context ? context.ownerDocument || context : preferredDoc ) !== document ) { setDocument( context ); } context = context || document; if ( documentIsHTML ) { // If the selector is sufficiently simple, try using a "get*By*" DOM method // (excepting DocumentFragment context, where the methods don't exist) if ( nodeType !== 11 && (match = rquickExpr.exec( selector )) ) { // ID selector if ( (m = match[1]) ) { // Document context if ( nodeType === 9 ) { if ( (elem = context.getElementById( m )) ) { // Support: IE, Opera, Webkit // TODO: identify versions // getElementById can match elements by name instead of ID if ( elem.id === m ) { results.push( elem ); return results; } } else { return results; } // Element context } else { // Support: IE, Opera, Webkit // TODO: identify versions // getElementById can match elements by name instead of ID if ( newContext && (elem = newContext.getElementById( m )) && contains( context, elem ) && elem.id === m ) { results.push( elem ); return results; } } // Type selector } else if ( match[2] ) { push.apply( results, context.getElementsByTagName( selector ) ); return results; // Class selector } else if ( (m = match[3]) && support.getElementsByClassName && context.getElementsByClassName ) { push.apply( results, context.getElementsByClassName( m ) ); return results; } } // Take advantage of querySelectorAll if ( support.qsa && !compilerCache[ selector + " " ] && (!rbuggyQSA || !rbuggyQSA.test( selector )) ) { if ( nodeType !== 1 ) { newContext = context; newSelector = selector; // qSA looks outside Element context, which is not what we want // Thanks to Andrew Dupont for this workaround technique // Support: IE <=8 // Exclude object elements } else if ( context.nodeName.toLowerCase() !== "object" ) { // Capture the context ID, setting it first if necessary if ( (nid = context.getAttribute( "id" )) ) { nid = nid.replace( rcssescape, fcssescape ); } else { context.setAttribute( "id", (nid = expando) ); } // Prefix every selector in the list groups = tokenize( selector ); i = groups.length; while ( i-- ) { groups[i] = "#" + nid + " " + toSelector( groups[i] ); } newSelector = groups.join( "," ); // Expand context for sibling selectors newContext = rsibling.test( selector ) && testContext( context.parentNode ) || context; } if ( newSelector ) { try { push.apply( results, newContext.querySelectorAll( newSelector ) ); return results; } catch ( qsaError ) { } finally { if ( nid === expando ) { context.removeAttribute( "id" ); } } } } } } // All others return select( selector.replace( rtrim, "$1" ), context, results, seed ); } /** * Create key-value caches of limited size * @returns {function(string, object)} Returns the Object data after storing it on itself with * property name the (space-suffixed) string and (if the cache is larger than Expr.cacheLength) * deleting the oldest entry */ function createCache() { var keys = []; function cache( key, value ) { // Use (key + " ") to avoid collision with native prototype properties (see Issue #157) if ( keys.push( key + " " ) > Expr.cacheLength ) { // Only keep the most recent entries delete cache[ keys.shift() ]; } return (cache[ key + " " ] = value); } return cache; } /** * Mark a function for special use by Sizzle * @param {Function} fn The function to mark */ function markFunction( fn ) { fn[ expando ] = true; return fn; } /** * Support testing using an element * @param {Function} fn Passed the created element and returns a boolean result */ function assert( fn ) { var el = document.createElement("fieldset"); try { return !!fn( el ); } catch (e) { return false; } finally { // Remove from its parent by default if ( el.parentNode ) { el.parentNode.removeChild( el ); } // release memory in IE el = null; } } /** * Adds the same handler for all of the specified attrs * @param {String} attrs Pipe-separated list of attributes * @param {Function} handler The method that will be applied */ function addHandle( attrs, handler ) { var arr = attrs.split("|"), i = arr.length; while ( i-- ) { Expr.attrHandle[ arr[i] ] = handler; } } /** * Checks document order of two siblings * @param {Element} a * @param {Element} b * @returns {Number} Returns less than 0 if a precedes b, greater than 0 if a follows b */ function siblingCheck( a, b ) { var cur = b && a, diff = cur && a.nodeType === 1 && b.nodeType === 1 && a.sourceIndex - b.sourceIndex; // Use IE sourceIndex if available on both nodes if ( diff ) { return diff; } // Check if b follows a if ( cur ) { while ( (cur = cur.nextSibling) ) { if ( cur === b ) { return -1; } } } return a ? 1 : -1; } /** * Returns a function to use in pseudos for input types * @param {String} type */ function createInputPseudo( type ) { return function( elem ) { var name = elem.nodeName.toLowerCase(); return name === "input" && elem.type === type; }; } /** * Returns a function to use in pseudos for buttons * @param {String} type */ function createButtonPseudo( type ) { return function( elem ) { var name = elem.nodeName.toLowerCase(); return (name === "input" || name === "button") && elem.type === type; }; } /** * Returns a function to use in pseudos for :enabled/:disabled * @param {Boolean} disabled true for :disabled; false for :enabled */ function createDisabledPseudo( disabled ) { // Known :disabled false positives: // IE: *[disabled]:not(button, input, select, textarea, optgroup, option, menuitem, fieldset) // not IE: fieldset[disabled] > legend:nth-of-type(n+2) :can-disable return function( elem ) { // Check form elements and option elements for explicit disabling return "label" in elem && elem.disabled === disabled || "form" in elem && elem.disabled === disabled || // Check non-disabled form elements for fieldset[disabled] ancestors "form" in elem && elem.disabled === false && ( // Support: IE6-11+ // Ancestry is covered for us elem.isDisabled === disabled || // Otherwise, assume any non-