[ { "path": ".coveragerc", "content": "[run]\nomit =\n .git/*\n .tox/*\n docs/*\n setup.py\n test/*\n tests/*\n" }, { "path": ".dockerignore", "content": "*\n" }, { "path": ".github/ISSUE_TEMPLATE/bug_report.md", "content": "---\nname: Bug report\nabout: Create a report to help us improve\ntitle: ''\nlabels: ''\nassignees: ''\n\n---\n@confluentinc/devprod-apac-n-frameworks-eng is tagged for visibility\n\n**Describe the bug**\nA clear and concise description of what the bug is.\n\n\n**To Reproduce**\nSteps to reproduce the behavior\n\n**Expected behavior**\nA clear and concise description of what you expected to happen.\n\n**Additional context**\nAdd any other context about the problem here.\n" }, { "path": ".gitignore", "content": "# Duckape\nsystests/.ducktape/\nsystests/results/\nresults\n.ducktape\n.vagrant\n\n# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\ntextcov/\n.tox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n.hypothesis/\n.pytest_cache/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# pyenv\n.python-version\n\n# celery beat schedule file\ncelerybeat-schedule\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n.virtualenvs/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n\n.idea\n/.vagrant/\n.vscode\n" }, { "path": ".readthedocs.yaml", "content": "# .readthedocs.yaml\n# Read the Docs configuration file\n# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details\n\n# Required\nversion: 2\n\nbuild:\n os: ubuntu-20.04\n tools:\n python: \"3.13\"\n\nsphinx:\n configuration: docs/conf.py\n\npython:\n install:\n - requirements: docs/requirements.txt\n - method: setuptools\n path: .\n" }, { "path": ".semaphore/semaphore.yml", "content": "version: v1.0\nname: pr-test-job\nagent:\n machine:\n type: s1-prod-ubuntu24-04-amd64-1\n\nexecution_time_limit:\n hours: 1\n\nglobal_job_config:\n prologue:\n commands:\n - checkout\n\n\nblocks:\n - name: Test\n dependencies: []\n task:\n jobs:\n - name: Test Python 3.8\n commands:\n - sem-version python 3.8\n - pip install tox\n - export PYTESTARGS='--junitxml=test/results-py38.xml'\n - tox -e py38\n - name: Test Python 3.9\n commands:\n - sem-version python 3.9\n - pip install tox\n - export PYTESTARGS='--junitxml=test/results-py39.xml'\n - tox -e py39\n - name: Test Python 3.10\n commands:\n - sem-version python 3.10\n - pip install tox\n - export PYTESTARGS='--junitxml=test/results-py310.xml'\n - tox -e py310\n - name: Test Python 3.11\n commands:\n - sem-version python 3.11\n - pip install tox\n - export PYTESTARGS='--junitxml=test/results-py311.xml'\n - tox -e py311\n - name: Test Python 3.12\n commands:\n - sem-version python 3.12\n - pip install tox\n - export PYTESTARGS='--junitxml=test/results-py312.xml'\n - tox -e py312\n - name: Test Python 3.13\n commands:\n - sem-version python 3.13\n - pip install tox\n - export PYTESTARGS='--junitxml=test/results-py313.xml'\n - tox\n" }, { "path": "CODEOWNERS", "content": "* @confluentinc/cp-test-frameworks-and-readiness\n" }, { "path": "Dockerfile", "content": "# An image of ducktape that can be used to setup a Docker cluster where ducktape is run inside the container.\n\nFROM ubuntu:14.04\n\nRUN apt-get update && \\\n apt-get install -y libffi-dev libssl-dev openssh-server python-dev python-pip python-virtualenv && \\\n virtualenv /opt/ducktape && \\\n . /opt/ducktape/bin/activate && \\\n pip install -U pip setuptools wheel && \\\n pip install bcrypt cryptography==2.2.2 pynacl && \\\n mkdir /var/run/sshd && \\\n mkdir /root/.ssh && \\\n apt-get clean && \\\n rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*\n\nARG DUCKTAPE_VERSION=0.7.3\n\nRUN . /opt/ducktape/bin/activate && \\\n pip install ducktape==$DUCKTAPE_VERSION && \\\n ln -s /opt/ducktape/bin/ducktape /usr/local/bin/ducktape && \\\n deactivate && \\\n /usr/local/bin/ducktape --version\n\nEXPOSE 22\n\nCMD [\"/usr/sbin/sshd\", \"-D\"]\n" }, { "path": "Jenkinsfile.disabled", "content": "python {\n publish = false // Release is done manually to PyPI and not supported yet.\n}\n" }, { "path": "README.md", "content": "[![Documentation Status](https://readthedocs.org/projects/ducktape/badge/?version=latest)](https://ducktape.readthedocs.io/en/latest/?badge=latest)\n\n\nDistributed System Integration & Performance Testing Library\n============================================================\n\nOverview\n--------\n\nDucktape contains tools for running system integration and performance tests. It provides the following features:\n\n* Isolation by default so system tests are as reliable as possible.\n* Utilities for pulling up and tearing down services easily in clusters in different environments\n (e.g. local, custom cluster, Vagrant, K8s, Mesos, Docker, cloud providers, etc.)\n* Easy to write unit tests for distributed systems\n* Trigger special events (e.g. bouncing a service)\n* Collect results (e.g. logs, console output)\n* Report results (e.g. expected conditions met, performance results, etc.)\n\nDocumentation\n-------------\n\nFor detailed documentation on how to install, run, create new tests please refer to: http://ducktape.readthedocs.io/\n\nContribute\n----------\n\n- Source Code: https://github.com/confluentinc/ducktape\n- Issue Tracker: https://github.com/confluentinc/ducktape/issues\n\nLicense\n-------\nThe project is licensed under the Apache 2 license.\n" }, { "path": "Vagrantfile", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# -*- mode: ruby -*-\n# vi: set ft=ruby :\n\nrequire 'socket'\n\n# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!\nVAGRANTFILE_API_VERSION = \"2\"\n\n# General config\nenable_dns = false\nnum_workers = 3\nram_megabytes = 300\nbase_box = \"ubuntu/focal64\"\n\nlocal_config_file = File.join(File.dirname(__FILE__), \"Vagrantfile.local\")\nif File.exists?(local_config_file) then\n eval(File.read(local_config_file), binding, \"Vagrantfile.local\")\nend\n\nVagrant.configure(VAGRANTFILE_API_VERSION) do |config|\n config.hostmanager.enabled = true\n config.hostmanager.manage_host = enable_dns\n config.hostmanager.include_offline = false\n\n ## Provider-specific global configs\n config.vm.provider :virtualbox do |vb,override|\n override.vm.box = base_box\n\n override.hostmanager.ignore_private_ip = false\n\n # Brokers started with the standard script currently set Xms and Xmx to 1G,\n # plus we need some extra head room.\n vb.customize [\"modifyvm\", :id, \"--memory\", ram_megabytes.to_s]\n end\n\n ## Cluster definition\n (1..num_workers).each { |i|\n name = \"ducktape\" + i.to_s\n config.vm.define name do |worker|\n worker.vm.hostname = name\n worker.vm.network :private_network, ip: \"192.168.56.\" + (150 + i).to_s\n end\n }\n\nend\n" }, { "path": "docs/Makefile", "content": "# Minimal makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHINXBUILD = sphinx-build\nSPHINXPROJ = ducktape\nSOURCEDIR = .\nBUILDDIR = _build\n\n# Put it first so that \"make\" without argument is like \"make help\".\nhelp:\n\t@$(SPHINXBUILD) -M help \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)\n\n.PHONY: help Makefile\n\n# Catch-all target: route all unknown targets to Sphinx using the new\n# \"make mode\" option. $(O) is meant as a shortcut for $(SPHINXOPTS).\n%: Makefile\n\t@$(SPHINXBUILD) -M $@ \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)" }, { "path": "docs/README.md", "content": "Ducktape documentation quick start guide\n========================================\n\n\nBuild the documentation\n-----------------------\n\nTo render the pages run::\n```shell\ntox -e docs\n```\n \nThe rendered pages will be in ``docs/_build/html``\n\n\nSpecify documentation format\n----------------------------\n\nDocumentation is built using [sphinx-build](https://www.sphinx-doc.org/en/master/man/sphinx-build.html) command.\nYou can select which builder to use using SPHINX_BUILDER command:\n```shell\nSPHINX_BUILDER=man tox -e docs\n```\nAll available values: https://www.sphinx-doc.org/en/master/man/sphinx-build.html#cmdoption-sphinx-build-M\n\n\nPass options to sphinx-build\n----------------------------\nAny argument after `--` will be passed to the \n[sphinx-build](https://www.sphinx-doc.org/en/master/man/sphinx-build.html) command directly:\n```shell\ntox -e docs -- -E\n```\n\n\n" }, { "path": "docs/_static/theme_overrides.css", "content": "/* override table width restrictions */\n@media screen and (min-width: 767px) {\n\n .wy-table-responsive table td {\n /* !important prevents the common CSS stylesheets from overriding\n this as on RTD they are loaded after this stylesheet */\n white-space: normal !important;\n }\n\n .wy-table-responsive {\n overflow: visible !important;\n }\n}" }, { "path": "docs/api/clusters.rst", "content": "Clusters\n========\n\n.. autoclass:: ducktape.cluster.cluster.Cluster\n :members:\n\n.. autoclass:: ducktape.cluster.vagrant.VagrantCluster\n :members:\n\n.. autoclass:: ducktape.cluster.localhost.LocalhostCluster\n :members:\n\n.. autoclass:: ducktape.cluster.json.JsonCluster\n :members:\n" }, { "path": "docs/api/remoteaccount.rst", "content": "Remote Account\n==============\n\n.. autoclass:: ducktape.cluster.remoteaccount.RemoteAccount\n :members:\n\n.. autoclass:: ducktape.cluster.remoteaccount.LogMonitor\n :members:\n\n.. autoclass:: ducktape.cluster.linux_remoteaccount.LinuxRemoteAccount\n :members:\n\n.. autoclass:: ducktape.cluster.windows_remoteaccount.WindowsRemoteAccount\n :members:\n\n" }, { "path": "docs/api/services.rst", "content": "Services\n========\n\n.. autoclass:: ducktape.services.service.Service\n :members:\n\n.. autoclass:: ducktape.services.background_thread.BackgroundThreadService\n :members:\n\n\n" }, { "path": "docs/api/templates.rst", "content": "Template\n========\n\n.. autoclass:: ducktape.template.TemplateRenderer\n :members:" }, { "path": "docs/api/test.rst", "content": "Test\n====\n\n.. autoclass:: ducktape.tests.test.Test\n :members:\n\n.. autoclass:: ducktape.tests.test.TestContext\n :members:\n" }, { "path": "docs/api.rst", "content": ".. _topics-api:\n\n=======\nAPI Doc\n=======\n\n.. toctree::\n api/test\n api/services\n api/remoteaccount\n api/clusters\n api/templates\n" }, { "path": "docs/changelog.rst", "content": ".. _topics-changelog:\n\n====\nChangelog\n====\n\n0.14.0\n======\nTuesday, March 10th, 2026\n-------------------------\n- Ensure log collection in case of runner client unresponsive issue\n- Enable jvm logging for java based services\n- Graceful shutdown and reporting fix for runner client unresponsive issue\n- Add heterogeneous cluster support\n- Add types to ducktape\n- Call JUnitReporter after each test completes\n- Update dependency requests to v2.32.4 [security]\n\n\n0.13.0\n======\nMonday, June 09th, 2025\n-----------------------\n- Report expected test count in the summary\n- Add python 3.13 support and run pr test job with all supported python versions\n- Upgrade PyYAML and fix style error\n- Add support for historical report in loader\n- Update dependency requests to v2.32.2\n- Update dependency pycryptodome to v3.19.1\n- Removing generated internal project.yml, public project.yml\n\n\n0.12.0\n======\nFriday, October 04th, 2024\n--------------------------\n- Store summary of previous runs when deflaking\n- Runner Client Minor Refactor and Test\n- Adding nodes used in test summary for HTML report\n- Parse args from config files independently\n- add support to python 3.10, 3.11 and 3.12\n\n\n0.11.4\n======\nFriday, August 18th, 2023\n-------------------------\n- Updated `requests` version to 2.31.0\n\n0.11.3\n======\nWednesday, November 30th, 2022\n------------------------------\n- Bugfix: fixed an edge case when BackgroundThread wait() method errors out if start() method has never been called.\n\n0.11.2\n======\nWednesday, November 30th, 2022\n------------------------------\n- Bugfix: fixed an edge case when BackgroundThread wait() method errors out if start() method has never been called.\n\n0.11.1\n======\n- Removed `tox` from requirements. It was not used, but was breaking our builds due to recent pushes to `virtualenv`.\n- Bumped `jinja2` to `3.0.x`\n\n0.11.0\n======\n- Option to fail tests without `@cluster` annotation. Deprecate ``min_cluster_spec()`` method in the ``Test`` class - `#336 `_\n\n0.10.3\n======\nFriday, August 18th, 2023\n-------------------------\n- Updated `requests` version to 2.31.0\n\n0.10.2\n======\n- Removed `tox` from requirements. It was not used, but was breaking our builds due to recent pushes to `virtualenv`.\n\n0.10.1\n======\n- Disable health checks for nodes, effectively disabling `#325 `_. See github issue for details - `#339 `_\n\n0.10.0\n======\n- **DO NOT USE**, this release has a nasty bug - `#339 `_\n- Do not schedule tests on unresponsive nodes - `#325 `_\n\n0.9.4\n=====\nFriday, August 18th, 2023\n-------------------------\n- Updated `requests` version to 2.31.0\n\n0.9.3\n=====\n- Removed `tox` from requirements. It was not used, but was breaking our builds due to recent pushes to `virtualenv`.\n\n0.9.2\n=====\n- Service release, no ducktape changes, simply fixed readthedocs configs.\n\n0.9.1\n=====\n- use a generic network device based on the devices found on the remote machine rather than a hardcoded one - `#314 `_ and `#328 `_\n- clean up process properly after an exception during test runner execution - `#323 `_\n- log ssh errors - `#319 `_\n- update vagrant tests to use ubuntu20 - `#328 `_\n- added command to print the total number of nodes the tests run will require - `#320 `_\n- drop support for python 3.6 and add support for python 3.9 - `#317 `_\n\n0.9.0\n=====\n- Upgrade paramiko version to 2.10.0 - `#312 `_\n- Support SSH timeout - `#311 `_\n\n0.8.18\n======\n- Updated `requests` version to `2.31.0`\n\n0.8.17\n======\n- Removed `tox` from requirements. It was not used, but was breaking our builds due to recent pushes to `virtualenv`.\n\n0.8.x\n=====\n- Support test suites\n- Easier way to rerun failed tests - generate test suite with all the failed tests and also print them in the log so that user can copy them and paste as ducktape command line arguments\n- Python 2 is no longer supported, minimum supported version is 3.6\n- Added `--deflake N` flag - if provided, it will attempt to rerun each failed test up to N times, and if it eventually passes, it will be marked as Flaky - `#299 `_\n- [backport, also in 0.9.1] - use a generic network device based on the devices found on the remote machine rather than a hardcoded one - `#314 `_ and `#328 `_\n- [backport, also in 0.9.1] - clean up process properly after an exception during test runner execution - `#323 `_\n- [backport, also in 0.9.1] - log ssh errors - `#319 `_\n- [backport, also in 0.9.1] - update vagrant tests to use ubuntu20 - `#328 `_\n- [backport, also in 0.9.1] - added command to print the total number of nodes the tests run will require - `#320 `_\n" }, { "path": "docs/conf.py", "content": "# -*- coding: utf-8 -*-\n#\n# ducktape documentation build configuration file, created by\n# sphinx-quickstart on Mon Mar 13 14:06:07 2017.\n#\n# This file is execfile()d with the current directory set to its\n# containing dir.\n#\n# Note that not all possible configuration values are present in this\n# autogenerated file.\n#\n# All configuration values have a default; values that are commented out\n# serve to show the default.\n\n# If extensions (or modules to document with autodoc) are in another directory,\n# add these directories to sys.path here. If the directory is relative to the\n# documentation root, use os.path.abspath to make it absolute, like shown here.\n#\nimport os\nimport sys\nimport sphinx_rtd_theme\nfrom ducktape import __version__\n\nsys.path.insert(0, os.path.abspath(\"..\"))\n\n\n# -- General configuration ------------------------------------------------\n\n# If your documentation needs a minimal Sphinx version, state it here.\n#\n# needs_sphinx = '1.0'\n\n# Add any Sphinx extension module names here, as strings. They can be\n# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom\n# ones.\nextensions = [\"sphinx.ext.viewcode\", \"sphinx.ext.autodoc\", \"sphinxarg.ext\"]\n\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = [\"_templates\"]\n\n# The suffix(es) of source filenames.\n# You can specify multiple suffix as a list of string:\n#\n# source_suffix = ['.rst', '.md']\nsource_suffix = \".rst\"\n\n# The master toctree document.\nmaster_doc = \"index\"\n\n# General information about the project.\nproject = \"Ducktape\"\ncopyright = \"2017, Confluent Inc.\"\nauthor = \"Confluent Inc.\"\n\n# The version info for the project you're documenting, acts as replacement for\n# |version| and |release|, also used in various other places throughout the\n# built documents.\n#\n\n# The short X.Y version.\nversion = __version__\n# The full version, including alpha/beta/rc tags.\nrelease = version\n\n# The language for content autogenerated by Sphinx. Refer to documentation\n# for a list of supported languages.\n#\n# This is also used if you do content translation via gettext catalogs.\n# Usually you set \"language\" from the command line for these cases.\nlanguage = None\n\n# List of patterns, relative to source directory, that match files and\n# directories to ignore when looking for source files.\n# This patterns also effect to html_static_path and html_extra_path\nexclude_patterns = [\"_build\", \"Thumbs.db\", \".DS_Store\"]\n\n# The name of the Pygments (syntax highlighting) style to use.\npygments_style = \"sphinx\"\n\n# If true, `todo` and `todoList` produce output, else they produce nothing.\ntodo_include_todos = False\n\n\n# -- Options for HTML output ----------------------------------------------\n\n# The theme to use for HTML and HTML Help pages. See the documentation for\n# a list of builtin themes.\n#\nhtml_theme = \"sphinx_rtd_theme\"\n\n# Theme options are theme-specific and customize the look and feel of a theme\n# further. For a list of options available for each theme, see the\n# documentation.\n#\n# html_theme_options = {}\n\n# Add any paths that contain custom themes here, relative to this directory.\nhtml_theme_path = [sphinx_rtd_theme.get_html_theme_path()]\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\n# html_static_path = ['_static']\n\n\n# -- Options for HTMLHelp output ------------------------------------------\n\n# Output file base name for HTML help builder.\nhtmlhelp_basename = \"ducktapedoc\"\n\n\n# -- Options for LaTeX output ---------------------------------------------\n\nlatex_elements = {\n # The paper size ('letterpaper' or 'a4paper').\n #\n # 'papersize': 'letterpaper',\n # The font size ('10pt', '11pt' or '12pt').\n #\n # 'pointsize': '10pt',\n # Additional stuff for the LaTeX preamble.\n #\n # 'preamble': '',\n # Latex figure (float) alignment\n #\n # 'figure_align': 'htbp',\n}\n\n# Grouping the document tree into LaTeX files. List of tuples\n# (source start file, target name, title,\n# author, documentclass [howto, manual, or own class]).\nlatex_documents = [\n (master_doc, \"ducktape.tex\", \"Ducktape Documentation\", \"Confluent Inc.\", \"manual\"),\n]\n\n\n# -- Options for manual page output ---------------------------------------\n\n# One entry per manual page. List of tuples\n# (source start file, name, description, authors, manual section).\nman_pages = [(master_doc, \"ducktape\", \"Ducktape Documentation\", [author], 1)]\n\n\n# -- Options for Texinfo output -------------------------------------------\n\n# Grouping the document tree into Texinfo files. List of tuples\n# (source start file, target name, title, author,\n# dir menu entry, description, category)\ntexinfo_documents = [\n (\n master_doc,\n \"ducktape\",\n \"Ducktape Documentation\",\n author,\n \"ducktape\",\n \"One line description of project.\",\n \"Miscellaneous\",\n ),\n]\n\n\n# ---- Options for Autodoc ------------------------------------------------\n\nautodoc_default_flags = [\"show-inheritance\"]\n\n\ndef skip(app, what, name, obj, skip, options):\n if name == \"__init__\":\n return False\n return skip\n\n\ndef setup(app):\n app.connect(\"autodoc-skip-member\", skip)\n" }, { "path": "docs/debug_tests.rst", "content": ".. _topics-debug_tests:\n\n===========\nDebug Tests\n===========\n\nThe test results go in ``results/—``. For results from a particular test, look for ``results/—/test_class_name//`` directory. The ``test_log.debug`` file will contain the log output from the python driver, and logs of services used in the test will be in ``service_name/node_name`` sub-directory.\n\nIf there is not enough information in the logs, you can re-run the test with ``--no-teardown`` argument.\n\n.. code-block:: bash\n\n ducktape dir/tests/my_test.py::TestA.test_a --no-teardown\n\n\nThis will run the test but will not kill any running processes or remove log files when the test finishes running. Then, you can examine the state of a running service or the machine when the service process is running by logging into that machine. Suppose you suspect a particular service being the cause of the test failure. You can find out which machine was allocated to that service by either looking at ``test_log.debug`` or at directory names under ``results/—/test_class_name//service_name/``. It could be useful to add an explicit debug log to ``start_node`` method with a node ID and node’s hostname information for easy debugging:\n\n.. code-block:: python\n\n def start_node(self, node):\n idx = self.idx(node)\n self.logger.info(\"Starting ZK node %d on %s\", idx, node.account.hostname)\n\nThe log statement will look something like this::\n\n [INFO - 2017-03-28 22:07:25,222 - zookeeper - start_node - lineno:50]: Starting ZK node 1 on worker1\n\nIf you are using Vagrant for example, you can then log into that node via:\n\n.. code-block:: bash\n\n vagrant ssh worker1\n\n\n\nUse Logging\n===========\n\nDistributed system tests can be difficult to debug. You want to add a lot of logging for debugging and tracking progress of the test. A good approach would be to log an intention of an operation with some useful information before any operation that can fail. It could be a good idea to use a higher logging level than you would in production so more info is available. For example, make your log levels default to DEBUG instead of INFO. Also, put enough information to a message of ``assert`` to help figure out what went wrong as well as log messages. Consider an example of testing ElasticSearch service:\n\n.. code-block:: python\n\n res = es.search(index=\"test-index\", body={\"query\": {\"match_all\": {}}})\n self.logger.debug(\"result: %s\" % res['hits'])\n assert res['hits']['total'] == 1, \"Expected total 1 hit, but got %d\" % res['hits']['total']\n for hit in res['hits']['hits']:\n assert 'kimchy’ == hit['_source']['author’], \"Expected author kimchy but got %s\" % hit['_source']['author']\n assert 'Elasticsearch: cool.' == hit['_source']['text’], \"Expected text Elasticsearch: cool. but got %s\" % hit['_source']['text’]\n\nFirst, the tests outputs the result of a search, so that if any of the following assertions fail, we can see the whole result in ``test_log.debug``. Assertion messages help to quickly see the difference in expected and retrieved results. \n\n\nFail early\n==========\n\nTry to avoid a situation where a test fails because of an uncaught failure earlier in the test. Suppose we write a ``start_node`` method that does not check if the service starts successfully. The service fails to start, but we get a test failure indication that there was a problem querying the service. It would be much faster to debug the issue if the test failure pointed to the issue with starting the service. So make sure to add checks for operations that may fail, and fail the test earlier than later.\n\n\nFlaky tests\n============\n\nFlaky tests are hard to debug due to their non-determinism, they waste time, and sometimes hide real bugs: developers tend to ignore those failures, and thus could miss real bugs. Flakiness can come from the test itself, the system it is testing, or the environmental issues.\n\nWaiting on Conditions\n^^^^^^^^^^^^^^^^^^^^^\n\nA common cause of a flaky test is asynchronous wait on conditions. A test makes an asynchronous call and does not properly wait for the result of the call to become available before using it::\n\n\tnode.account.kill_process(\"zookeeper\", allow_fail=False)\n\ttime.sleep(2)\n\tassert not self.alive(node), “Expected Zookeeper service to stop” \n\nIn this example, the test terminates a zookeeper service via ``kill_process`` and then uses ``time.sleep`` to wait for it to stop. If terminating the process takes longer, the test will fail. The test may intermittently fail based on how fast a process terminates. Of course, there should be a timeout for termination to ensure that test does not run indefinitely. You could increase sleep time, but that also increases the test run length. A more explicit way to express this condition is to use :meth:`~ ducktape.utils.util.wait_until` with a timeout::\n\n\tnode.account.kill_process(\"zookeeper\", allow_fail=False)\n\twait_until(lambda: not self.alive(node),\n timeout_sec=5,\n err_msg=\"Timed out waiting for zookeeper to stop.\")\n\nThe test will progress as soon as condition is met, and timeout ensures that the test does not run indefinitely if termination never ends.\n\nThink carefully about the condition to check. A common source of issues is incorrect choice of condition of successful service start in ``start_node`` implementation. One way to check that a service starts successfully is to wait for some specific log output. However, make sure that this specific log message is always printed after the things run successfully. If there is still a chance that service may fail to start after the log is printed, this may cause race conditions and flaky tests. Sometimes it could be better to check if the service runs successfully by querying a service or checking some metrics if they are available.\n\n\nTest Order Dependency\n^^^^^^^^^^^^^^^^^^^^^\n\nMake sure that your services properly cleanup the state in ``clean_node`` implementation. Failure to properly clean up the state can cause the next run of the test to fail or fail intermittently if other tests happen to clean same directories for example. One of the benefits of isolation that ducktape assumes is that you can assume you have complete control of the machine. It is ok to delete the entire working space. It is also safe to kill all java processes you can find rather than being more targeted. So, clean up aggressively.\n\nIncorrect Assumptions\n^^^^^^^^^^^^^^^^^^^^^\n\nIt is possible that assumptions about how the system works that we are testing are incorrect. One way to help debug this is to use more detailed comments why certain checks are made.\n\n\nTools for Managing Logs\n=======================\n\nAnalyzing and matching up logs from a distributed service could be time consuming. There are many good tools for working with logs. Examples include http://lnav.org/, http://list.xmodulo.com/multitail.html, and http://glogg.bonnefon.org/.\n\nValidating Ssh Issues\n=======================\n\nDucktape supports running custom validators when an ssh error occurs, allowing you to run your own validation against a host.\nthis is done simply by running ducktape with the `--ssh-checker-function`, followed by the module path to your function, so for instance::\n \n ducktape my-test.py --ssh-checker-function my.module.validator.validate_ssh\n\nthis function will take in the ssh error raised as its first argument, and the remote account object as its second.\n" }, { "path": "docs/index.rst", "content": ".. _topics-index:\n\n============================================================\nDistributed System Integration & Performance Testing Library\n============================================================\nDucktape contains tools for running system integration and performance tests. It provides the following features:\n\n * Write tests for distributed systems in a simple unit test-like style\n * Isolation by default so system tests are as reliable as possible.\n * Utilities for pulling up and tearing down services easily in clusters in different environments (e.g. local, custom cluster, Vagrant, K8s, Mesos, Docker, cloud providers, etc.)\n * Trigger special events (e.g. bouncing a service)\n * Collect results (e.g. logs, console output)\n * Report results (e.g. expected conditions met, performance results, etc.)\n\n.. toctree::\n install\n test_clusters\n run_tests\n new_tests\n new_services\n debug_tests\n api\n misc\n changelog\n\nContribute\n==========\n\n- Source Code: https://github.com/confluentinc/ducktape\n- Issue Tracker: https://github.com/confluentinc/ducktape/issues\n\nLicense\n=======\n\nThe project is licensed under the Apache 2 license.\n" }, { "path": "docs/install.rst", "content": ".. _topics-install:\n\n=======\nInstall\n=======\n\n1. Ducktape requires python 3.7 or later.\n\n2. Install `cryptography`_ (used by `paramiko` which Ducktape depends on), this may have non-python external requirements\n\n.. _cryptography: https://cryptography.io/en/latest/installation\n\n * OSX (if needed)::\n\n brew install openssl\n\n * Ubuntu::\n\n sudo apt-get install build-essential libssl-dev libffi-dev python-dev\n\n * Fedora and RHEL-derivatives::\n\n sudo yum install gcc libffi-devel python-devel openssl-devel\n\n\n3. As a general rule, it's recommended to use an isolation tool such as ``virtualenv``\n\n4. Install Ducktape::\n\n pip install ducktape\n\n.. note::\n\n On OSX you may need to::\n\n C_INCLUDE_PATH=/usr/local/opt/openssl/include LIBRARY_PATH=/usr/local/opt/openssl/lib pip install ducktape\n\n If you are not using a virtualenv and get the error message `failed with error code 1`, you may need to install ducktape to your user directory instead with ::\n\n pip install --user ducktape\n" }, { "path": "docs/make.bat", "content": "@ECHO OFF\r\n\r\npushd %~dp0\r\n\r\nREM Command file for Sphinx documentation\r\n\r\nif \"%SPHINXBUILD%\" == \"\" (\r\n\tset SPHINXBUILD=sphinx-build\r\n)\r\nset SOURCEDIR=.\r\nset BUILDDIR=_build\r\nset SPHINXPROJ=ducktape\r\n\r\nif \"%1\" == \"\" goto help\r\n\r\n%SPHINXBUILD% >NUL 2>NUL\r\nif errorlevel 9009 (\r\n\techo.\r\n\techo.The 'sphinx-build' command was not found. Make sure you have Sphinx\r\n\techo.installed, then set the SPHINXBUILD environment variable to point\r\n\techo.to the full path of the 'sphinx-build' executable. Alternatively you\r\n\techo.may add the Sphinx directory to PATH.\r\n\techo.\r\n\techo.If you don't have Sphinx installed, grab it from\r\n\techo.http://sphinx-doc.org/\r\n\texit /b 1\r\n)\r\n\r\n%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%\r\ngoto end\r\n\r\n:help\r\n%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%\r\n\r\n:end\r\npopd\r\n" }, { "path": "docs/misc.rst", "content": ".. _topics-misc:\n\n====\nMisc\n====\n\nDeveloper Install\n=================\n\nIf you are are a ducktape developer, consider using the develop command instead of install. This allows you to make code changes without constantly reinstalling ducktape (see http://stackoverflow.com/questions/19048732/python-setup-py-develop-vs-install for more information)::\n\n cd ducktape\n python setup.py develop\n\nTo uninstall::\n\n cd ducktape\n python setup.py develop --uninstall\n\n\nUnit Tests\n==========\n\nYou can run the tests with code coverage and style check using `tox `_::\n\n tox\n\nAlternatively, you can activate the virtualenv and run pytest and ruff directly::\n\n source ~/.virtualenvs/ducktape/bin/activate\n pytest tests\n ruff check\n ruff format --check\n\n\nSystem Tests\n============\n\nSystem tests are included under the `systests/` directory. These tests are end to end tests that run across multiple VMs, testing ducktape in an environment similar to how it would be used in practice to test other projects.\n\nThe system tests run against virtual machines managed by `Vagrant `_. With Vagrant installed, start the VMs (3 by default)::\n\n vagrant up\n\nFrom a developer install, running the system tests now looks the same as using ducktape on your own project::\n\n ducktape systests/\n\nYou should see the tests running, and then results and logs will be in the default directory, `results/`. By using a developer install, you can make modifications to the ducktape code and iterate on system tests without having to re-install after each modification.\n\nWhen you're done running tests, you can destroy the VMs::\n\n vagrant destroy\n\n\nWindows\n=======\n\nDucktape support Services that run on Windows, but only in EC2.\n\nWhen a ``Service`` requires a Windows machine, AWS credentials must be configured on the machine running ducktape.\n\nDucktape uses the `boto3`_ Python module to connect to AWS. And ``boto3`` support many different `configuration options`_\n\n.. _boto3: https://aws.amazon.com/sdk-for-python/\n.. _configuration options: https://boto3.readthedocs.io/en/latest/guide/configuration.html#guide-configuration\n\nHere's an example bare minimum configuration using environment variables::\n\n export AWS_ACCESS_KEY_ID=\"ABC123\"\n export AWS_SECRET_ACCESS_KEY=\"secret\"\n export AWS_DEFAULT_REGION=\"us-east-1\"\n\nThe region can be any AWS region, not just ``us-east-1``.\n" }, { "path": "docs/new_services.rst", "content": ".. _topics-new_services:\n\n===================\nCreate New Services\n===================\n\nWriting ducktape services\n=============================\n\n``Service`` refers generally to multiple processes, possibly long-running, which you\nwant to run on the test cluster.\n\nThese can be services you would actually deploy (e.g., Kafka brokers, ZK servers, REST proxy) or processes used during testing (e.g. producer/consumer performance processes). Services that are distributed systems can support a variable number of nodes which allow them to handle a variety of tests.\n\nEach service is implemented as a class and should at least implement the following:\n\n * :meth:`~ducktape.services.service.Service.start_node` - start the service (possibly waiting to ensure it started successfully)\n\n * :meth:`~ducktape.services.service.Service.stop_node` - kill processes on the given node\n\n * :meth:`~ducktape.services.service.Service.clean_node` - remove persistent state leftover from testing, e.g. log files\n\nThese may block to ensure services start or stop properly, but must *not* block for the full lifetime of the service. If you need to run a blocking process (e.g. run a process via SSH and iterate over its output), this should be done in a background thread. For services that exit after completing a fixed operation (e.g. produce N messages to topic foo), you should also implement ``wait``, which will usually just wait for background worker threads to exit. The ``Service`` base class provides a helper method ``run`` which wraps ``start``, ``wait``, and ``stop`` for tests that need to start a service and wait for it to finish. You can also provide additional helper methods for common test functionality. Normal services might provide a ``bounce`` method.\n\nMost of the code you'll write for a service will just be series of SSH commands and tests of output. You should request the number of nodes you'll need using the ``num_nodes`` or ``cluster_spec`` parameter to the Service base class's constructor. Then, in your Service's methods you'll have access to ``self.nodes`` to access the nodes allocated to your service. Each node has an associated :class:`~ducktape.cluster.remoteaccount.RemoteAccount` instance which lets you easily perform remote operations such as running commands via SSH or creating files. By default, these operations try to hide output (but provide it to you if you need to extract some subset of it) and *checks status codes for errors* so any operations that fail cause an obvious failure of the entire test.\n\n.. _service-example-ref:\n\nNew Service Example\n===================\n\nLet’s walk through an example of writing a simple Zookeeper service.\n\n.. code-block:: python\n\n class ZookeeperService(Service):\n PERSISTENT_ROOT = \"/mnt\"\n LOG_FILE = os.path.join(PERSISTENT_ROOT, \"zk.log\")\n DATA_DIR = os.path.join(PERSISTENT_ROOT, \"zookeeper\")\n CONFIG_FILE = os.path.join(PERSISTENT_ROOT, \"zookeeper.properties\")\n\n logs = {\n \"zk_log\": {\n \"path\": LOG_FILE,\n \"collect_default\": True},\n \"zk_data\": {\n \"path\": DATA_DIR,\n \"collect_default\": False}\n }\n\n def __init__(self, context, num_nodes):\n super(ZookeeperService, self).__init__(context, num_nodes)\n\n\n``logs`` is a member of ``Service`` that provides a mechanism for locating and collecting log files produced by the service on its nodes. ``logs`` is a dict with entries that look like ``log_name: {\"path\": log_path, \"collect_default\": boolean}``. In our example, log files will be collected on both successful and failed test runs, while files from the data directory will be collected only on failed test runs. Zookeeper service requests the number of nodes passed to its constructor by passing ``num_nodes`` parameters to the Service base class’s constructor.\n\n.. code-block:: python\n\n def start_node(self, node):\n idx = self.idx(node)\n self.logger.info(\"Starting ZK node %d on %s\", idx, node.account.hostname)\n\n node.account.ssh(\"mkdir -p %s\" % self.DATA_DIR)\n node.account.ssh(\"echo %d > %s/myid\" % (idx, self.DATA_DIR))\n\n prop_file = \"\"\"\\n dataDir=%s\\n clientPort=2181\"\"\" % self.DATA_DIR\n for idx, node in enumerate(self.nodes):\n prop_file += \"\\n server.%d=%s:2888:3888\" % (idx, node.account.hostname)\n self.logger.info(\"zookeeper.properties: %s\" % prop_file)\n node.account.create_file(self.CONFIG_FILE, prop_file)\n\n start_cmd = \"/opt/kafka/bin/zookeeper-server-start.sh %s 1>> %s 2>> %s &\" % \\\n (self.CONFIG_FILE, self.LOG_FILE, self.LOG_FILE)\n\n with node.account.monitor_log(self.LOG_FILE) as monitor:\n node.account.ssh(start_cmd)\n monitor.wait_until(\n \"binding to port\",\n timeout_sec=100,\n backoff_sec=7,\n err_msg=\"Zookeeper service didn't finish startup\"\n )\n self.logger.debug(\"Zookeeper service is successfully started.\")\n\n\nThe ``start_node`` method first creates directories and the config file on the given node, and then invokes the start script to start a Zookeeper service. In this simple example, the config file is created from manually constructed ``prop_file`` string, because it has only a couple of easy to construct lines. More complex config files can be created with templates, as described in :ref:`using-templates-ref`.\n\nA service may take time to start and get to a usable state. Using sleeps to wait for a service to start often leads to a flaky test. The sleep time may be too short, or the service may fail to start altogether. It is useful to verify that the service starts properly before returning from the ``start_node``, and fail the test if the service fails to start. Otherwise, the test will likely fail later, and it would be harder to find the root cause of the failure. One way to check that the service starts successfully is to check whether a service’s process is alive and one additional check that the service is usable such as querying the service or checking some metrics if they are available. Our example checks whether a Zookeeper service is started successfully by searching for a particular output in a log file.\n\nThe :class:`~ducktape.cluster.remoteaccount.RemoteAccount` instance associated with each node provides you with :class:`~ducktape.cluster.remoteaccount.LogMonitor` that let you check or wait for a pattern to appear in the log. Our example waits for 100 seconds for “binding to port” string to appear in the ``self.LOG_FILE`` log file, and raises an exception if it does not.\n\n.. code-block:: python\n\n def pids(self, node):\n try:\n cmd = \"ps ax | grep -i zookeeper | grep java | grep -v grep | awk '{print $1}'\"\n pid_arr = [pid for pid in node.account.ssh_capture(cmd, allow_fail=True, callback=int)]\n return pid_arr\n except (RemoteCommandError, ValueError) as e:\n return []\n\n def alive(self, node):\n return len(self.pids(node)) > 0\n\n def stop_node(self, node):\n idx = self.idx(node)\n self.logger.info(\"Stopping %s node %d on %s\" % (type(self).__name__, idx, node.account.hostname))\n node.account.kill_process(\"zookeeper\", allow_fail=False)\n\n def clean_node(self, node):\n self.logger.info(\"Cleaning Zookeeper node %d on %s\", self.idx(node), node.account.hostname)\n if self.alive(node):\n self.logger.warn(\"%s %s was still alive at cleanup time. Killing forcefully...\" %\n (self.__class__.__name__, node.account))\n node.account.kill_process(\"zookeeper\", clean_shutdown=False, allow_fail=True)\n node.account.ssh(\"rm -rf /mnt/zookeeper /mnt/zookeeper.properties /mnt/zk.log\",\n allow_fail=False)\n\n\nThe ``stop_node`` method uses :meth:`~ducktape.cluster.remoteaccount.RemoteAccount.kill_process` to terminate the service process on the given node. If the remote command to terminate the process fails, :meth:`~ducktape.cluster.remoteaccount.RemoteAccount.kill_process` will raise an ``RemoteCommandError`` exception.\n\nThe ``clean_node`` method forcefully kills the process if it is still alive, and then removes persistent state leftover from testing. Make sure to properly cleanup the state to avoid test order dependency and flaky tests. You can assume complete control of the machine, so it is safe to delete an entire temporary working space and kill all java processes, etc.\n\n.. _using-templates-ref:\n\n\nUsing Templates\n===============\n\nBoth ``Service`` and ``Test`` subclass :class:`~ducktape.template.TemplateRenderer` that lets you render templates directly from strings or from files loaded from *templates/* directory relative to the class. A template contains variables and/or expressions, which are replaced with values when a template is rendered. :class:`~ducktape.template.TemplateRenderer` renders templates using `Jinja2 `_ template engine. A good use-case for templates is a properties file that needs to be passed to a service process. In :ref:`service-example-ref`, the properties file is created by building a string and using it as contents as follows::\n\n prop_file = \"\"\"\\n dataDir=%s\\n clientPort=2181\"\"\" % self.DATA_DIR\n for idx, node in enumerate(self.nodes):\n prop_file += \"\\n server.%d=%s:2888:3888\" % (idx, node.account.hostname)\n node.account.create_file(self.CONFIG_FILE, prop_file)\n\nA template approach is to add a properties file in *templates/* directory relative to the ZookeeperService class:\n\n.. code-block:: rst\n\n dataDir={{ DATA_DIR }}\n clientPort=2181\n {% for node in nodes %}\n server.{{ loop.index }}={{ node.account.hostname }}:2888:3888\n {% endfor %}\n\n\nSuppose we named the file zookeeper.properties. The creation of the config file will look like this:\n\n.. code-block:: python\n\n prop_file = self.render('zookeeper.properties')\n node.account.create_file(self.CONFIG_FILE, prop_file)\n\n" }, { "path": "docs/new_tests.rst", "content": ".. _topics-new_tests:\n\n================\nCreate New Tests\n================\n\nWriting ducktape Tests\n======================\n\nSubclass :class:`~ducktape.tests.test.Test` and implement as many ``test`` methods as you\nwant. The name of each test method must start or end with ``test``,\ne.g. ``test_functionality`` or ``example_test``. Typically, a test will\nstart a few services, collect and/or validate some data, and then finish.\n\nIf the test method finishes with no exceptions, the test is recorded as successful, otherwise it is recorded as a failure.\n\n\nHere is an example of a test that just starts a Zookeeper cluster with 2 nodes, and a\nKafka cluster with 3 nodes::\n\n class StartServicesTest(Test):\n \"\"\"Make sure we can start Kafka and Zookeeper services.\"\"\"\n def __init__(self, test_context):\n super(StartServicesTest, self).__init__(test_context=test_context)\n self.zk = ZookeeperService(test_context, num_nodes=2)\n self.kafka = KafkaService(test_context, num_nodes=3, self.zk)\n\n def test_services_start(self):\n self.zk.start()\n self.kafka.start()\n\nTest Parameters\n===============\n\nUse test decorators to parametrize tests, examples are provided below\n\n.. autofunction:: ducktape.mark.parametrize\n.. autofunction:: ducktape.mark.matrix\n.. autofunction:: ducktape.mark.resource.cluster\n.. autofunction:: ducktape.mark.ignore\n\nLogging\n=======\n\nThe :class:`~ducktape.tests.test.Test` base class sets up logger you can use which is tagged by class name,\nso adding some logging for debugging or to track the progress of tests is easy::\n\n self.logger.debug(\"End-to-end latency %d: %s\", idx, line.strip())\n\nThese types of tests can be difficult to debug, so err toward more rather than\nless logging.\n\n.. note:: Logs are collected a multiple log levels, and only higher log levels are displayed to the console while the test runs. Make sure you log at the appropriate level.\n\nJVM Logging\n-----------\n\nFor Java-based services, ducktape can automatically collect JVM diagnostic logs without requiring any code changes to services or tests. Enable it with the ``--enable-jvm-logs`` flag::\n\n ducktape --enable-jvm-logs \n\nWhen enabled, ducktape wraps the service's ``start_node`` and ``clean_node`` methods to:\n\n- Create a log directory (``/mnt/jvm_logs``) on each worker node before the service starts.\n- Prepend ``JDK_JAVA_OPTIONS`` with the JVM logging flags to every SSH command sent to the node, so the options are inherited by any Java process the service launches.\n- Remove the log directory after ``clean_node`` runs and restore the original SSH methods.\n\nThe following JVM options are injected automatically:\n\n.. list-table::\n :header-rows: 1\n :widths: 40 60\n\n * - Option\n - Purpose\n * - ``-Xlog:disable``\n - Suppress default JVM console output to avoid polluting test logs\n * - ``-Xlog:gc*:file=/gc.log``\n - GC activity with timestamps, uptime, level, and tags\n * - ``-XX:+HeapDumpOnOutOfMemoryError``\n - Generate a heap dump when an OOM error occurs\n * - ``-XX:HeapDumpPath=/heap_dump.hprof``\n - Location for the heap dump file\n * - ``-Xlog:safepoint=info:file=/jvm.log``\n - Safepoint pause events\n * - ``-Xlog:class+load=info:file=/jvm.log``\n - Class loading events\n * - ``-XX:ErrorFile=/hs_err_pid%p.log``\n - Fatal error log (JVM crashes)\n * - ``-XX:NativeMemoryTracking=summary``\n - Native memory usage tracking\n * - ``-Xlog:jit+compilation=info:file=/jvm.log``\n - JIT compilation events\n\nThe following log files are collected from each node:\n\n.. list-table::\n :header-rows: 1\n :widths: 30 20 50\n\n * - File\n - Collected by default\n - Contents\n * - ``gc.log``\n - Yes\n - Garbage collection activity\n * - ``jvm.log``\n - Yes\n - Safepoint, class loading, and JIT compilation events\n * - ``heap_dump.hprof``\n - No (failure only)\n - Heap dump generated on OutOfMemoryError\n\n.. note:: If a service or test injects its own ``-Xlog`` options as part of the command, those options will override the ones injected by JVM logging, since ducktape prepends ``JDK_JAVA_OPTIONS`` before the command. In practice, services should behave as expected.\n\nNew test example\n================\n\nLets expand on the StartServicesTest example. The test starts a Zookeeper cluster with 2 nodes, and a\nKafka cluster with 3 nodes, and then bounces a kafka broker node which is either a special controller node or a non-controller node, depending on the `bounce_controller_broker` test parameter.\n\n.. code-block:: python\n\n class StartServicesTest(Test):\n def __init__(self, test_context):\n super(StartServicesTest, self).__init__(test_context=test_context)\n self.zk = ZookeeperService(test_context, num_nodes=2)\n self.kafka = KafkaService(self.test_context, num_nodes=3, zk=self.zk)\n\n def setUp(self):\n self.zk.start()\n self.kafka.start()\n\n @matrix(bounce_controller_broker=[True, False])\n def test_broker_bounce(self, bounce_controller_broker=False):\n controller_node = self.kafka.controller()\n self.logger.debug(\"Found controller broker %s\", controller_node.account)\n if bounce_controller_broker:\n bounce_node = controller_node\n else:\n bounce_node = self.kafka.nodes[(self.kafka.idx(controller_node) + 1) % self.kafka.num_nodes]\n\n self.logger.debug(\"Will hard kill broker %s\", bounce_node.account)\n self.kafka.signal_node(bounce_node, sig=signal.SIGKILL)\n\n wait_until(lambda: not self.kafka.is_registered(bounce_node),\n timeout_sec=self.kafka.zk_session_timeout + 5,\n err_msg=\"Failed to see timely deregistration of hard-killed broker %s\"\n % bounce_node.account)\n\n self.kafka.start_node(bounce_node)\n\nThis will run two tests, one with ‘bounce_controller_broker’: False and another with 'bounce_controller_broker': True arguments. We moved start of Zookeeper and Kafka services to :meth:`~ducktape.tests.test.Test.setUp`, which is called before every test run.\n\nThe test finds which of Kafka broker nodes is a special controller node via provided ``controller`` method in KafkaService. The ``controller`` method in KafkaService will raise an exception if the controller node is not found. Make sure to check the behavior of methods provided by a service or other helper classes and fail the test as soon as an issue is found. That way, it will be much easier to find the cause of the test failure.\n\nThe test then finds the node to bounce based on `bounce_controller_broker` test parameter and then forcefully terminates the service process on that node via ``signal_node`` method of KafkaService. This method just sends a signal to forcefully kill the process, and does not do any further check. Thus, our test needs to check that the hard killed kafka broker is not part of the Kafka cluster anymore, before restarting the killed broker process. We do this by waiting on ``is_registered`` method provided by KafkaService to return False with a timeout, since de-registering the broker may take some time. Notice the use of ``wait_until`` method instead of a check after ``time.sleep``. This allows the test to continue as soon as de-registration happens.\n\nWe don’t check if the restarted broker is registered, because this is already done in KafkaService ``start_node`` implementation, which will raise an exception if the service is not started successfully on a given node.\n" }, { "path": "docs/requirements.txt", "content": "Sphinx~=8.2.3\nsphinx-argparse~=0.5.2\nsphinx-rtd-theme~=3.0.2\nboto3==1.33.13\npycryptodome==3.23.0\npywinrm==0.4.3\njinja2~=3.1.6\nMarkupSafe~=2.1.5\n" }, { "path": "docs/run_tests.rst", "content": ".. _topics-run_tests:\n\n=========\nRun Tests\n=========\n\nRunning Tests\n=============\n\nducktape discovers and runs tests in the path(s) provided.\nYou can specify a folder with tests (all tests in Python modules named with \"test\\_\" prefix or \"_test\" suffix will be\nrun), a specific test file (with any name) or even a specific class or test method, via absolute or relative paths.\nYou can optionally specify a specific set of parameters for tests with ``@parametrize`` or ``@matrix`` annotations::\n\n ducktape # e.g. ducktape dir/tests\n ducktape # e.g. ducktape dir/tests/my_test.py\n ducktape [::SomeTestClass] # e.g. ducktape dir/tests/my_test.py::TestA\n ducktape [::SomeTestClass[.test_method]] # e.g. ducktape dir/tests/my_test.py::TestA.test_a\n ducktape [::TestClass[.method[@params_json]]] # e.g. ducktape 'dir/tests/my_test.py::TestA.test_a@{\"x\": 100}'\n\n\nExcluding Tests\n===============\n\nPass ``--exclude`` flag to exclude certain test(s) from the run, using the same syntax::\n\n ducktape ./my_tests_dir --exclude ./my_tests_dir/test_a.py ./my_tests_dir/test_b.py::TestB.test_b\n\n\n\nTest Suites\n===========\n\nTest suite is a collection of tests to run, optionally also specifying which tests to exclude. Test suites are specified\nvia YAML file\n\n.. code-block:: yaml\n\n # list all tests that are part of the suite under the test suite name:\n my_test_suite:\n - ./my_tests_dir/ # paths are relative to the test suite file location\n - ./another_tests_dir/test_file.py::TestClass.test_method # same syntax as passing tests directly to ducktape\n - './another_tests_dir/test_file.py::TestClass.parametrized_method@{\"x\": 100}' # params are supported too\n - ./third_tests_dir/prefix_*.py # basic globs are supported (* and ? characters)\n\n # each YAML file can contain one or more test suites:\n another_test_suite:\n # you can optionally specify excluded tests in the suite as well using the following syntax:\n included:\n - ./some_tests_dir/\n excluded:\n - ./some_tests_dir/*_large_test.py\n\n\nRunning Test Suites\n===================\n\nTests suites are run in the same fashion as separate tests.\n\nRun a single test suite::\n\n ducktape ./path/to/test_suite.yml\n\nRun multiple test suites::\n\n ducktape ./path/to/test_suite_1.yml ./test_suite_2.yml\n\nYou can specify both tests and test suites at the same time::\n\n ducktape ./my_test.py ./my_test_suite.yml ./another_test.py::TestClass.test_method\n\nIf the same test method is effectively specified more than once, it will only be executed once.\n\nFor example, if ``test_suite.yml`` lists ``test_a.py`` then running the following command\nwill execute ``test_a.py`` only once::\n\n ducktape test_suite.yml test_a.py\n\nIf you specify a folder, all tests (ie python files) under that folder will be discovered, but test suites will be not.\n\nFor example, if ``test_dir`` contains ``my_test.py`` and ``my_test_suite.yml``, then running::\n\n ducktape ./test_dir\n\nwill execute ``my_test.py`` but skip ``my_test_suite.yml``.\n\nTo execute both ``my_test.py`` and ``my_test_suite.yml`` you need to specify test suite path explicitly::\n\n ducktape ./test_dir/ ./test_dir/my_test_suite.yml\n\n\n\nExclude and Test Suites\n=======================\n\nExclude section in the test suite applies only to that test suite. ``--exclude`` parameter passed to ducktape applies\nto all loaded tests and test suites.\n\nFor example, if ``test_dir`` contains ``test_a.py``, ``test_b.py`` and ``test_c.py``, and ``test_suite.yml`` is:\n\n.. code-block:: yaml\n\n suite_one:\n included:\n - ./test_dir/*.py\n excluded:\n - ./test_dir/test_a.py\n suite_two:\n included:\n - ./test_dir/\n excluded:\n - ./test_dir/test_b.py\n\nThen running::\n\n ducktape test_suite.yml\nruns each of ``test_a.py``, ``test_b.py`` and ``test_c.py`` once\n\n\nBut running::\n\n ducktape test_suite.yml --exclude test_dir/test_a.py\nruns only ``test_b.py`` and ``test_c.py`` once, and skips ``test_a.py``.\n\n\nOptions\n=======\n\nTo see a complete listing of options run::\n\n ducktape --help\n\n.. argparse::\n :module: ducktape.command_line.parse_args\n :func: create_ducktape_parser\n :prog: ducktape\n\nConfiguration File\n==================\n\nYou can configure options in three locations: on the command line (highest priority), in a user configuration file in\n``~/.ducktape/config``, and in a project-specific configuration ``/.ducktape/config`` (lowest priority).\nConfiguration files use the same syntax as command line arguments and may split arguments across multiple lines::\n\n --debug\n --exit-first\n --cluster=ducktape.cluster.json.JsonCluster\n\nOutput\n======\n\nTest results go in ``results/.`` which looks like ``--``. For example: ``results/2015-03-28--002``\n\nducktape does its best to group test results and log files in a sensible way. The output directory is\nstructured like so::\n\n \n session_log.info\n session_log.debug\n report.txt # Summary report of all tests run in this session\n report.html # Open this to see summary report in a browser\n report.css\n\n \n \n test_log.info\n test_log.debug\n report.txt # Report on this single test\n [data.json] # Present if the test returns data\n\n \n \n some_logs\n \n some_logs\n ...\n\n\nTo see an example of the output structure, go `here`_ and click on one of the details links.\n\n.. _here: http://testing.confluent.io/confluent-kafka-system-test-results/\n" }, { "path": "docs/test_clusters.rst", "content": ".. _topics-test_clusters:\n\n===================\nTest Clusters\n===================\n\nDucktape runs on a test cluster with several nodes. Ducktape will take ownership of the nodes and handle starting, stopping, and running services on them.\n\nMany test environments are possible. The nodes may be local nodes, running inside Docker. Or they could be virtual machines running on a public cloud.\n\nCluster Specifications\n======================\n\nA cluster specification-- also called a ClusterSpec-- describes a particular\ncluster configuration. Originally, cluster specifications could only express the\nnumber of nodes of each operating system. Now, with heterogeneous cluster support,\nspecifications can also include node types (e.g., \"small\", \"large\") for more\nfine-grained resource allocation. See `Heterogeneous Clusters`_ for more details.\n\nCluster specifications give us a vocabulary to express what a particular\nservice or test needs to run. For example, a service might require a cluster\nwith three Linux nodes and one Windows node. We could express that with a\nClusterSpec containing three Linux NodeSpec objects and one Windows NodeSpec\nobject.\n\nHeterogeneous Clusters\n======================\n\nDucktape supports heterogeneous clusters where nodes can have different types\n(e.g., \"small\", \"large\", \"arm64\"). This allows tests to request specific node\ntypes while maintaining backward compatibility with existing tests.\n\nUsing Node Types in Tests\n-------------------------\n\nUse the ``@cluster`` decorator with ``node_type`` to request specific node types::\n\n from ducktape.mark.resource import cluster\n\n @cluster(num_nodes=3, node_type=\"large\")\n def test_with_large_nodes(self):\n # This test requires 3 large nodes\n pass\n\n @cluster(num_nodes=5)\n def test_any_nodes(self):\n # This test accepts any 5 nodes (backward compatible)\n pass\n\nCluster Configuration\n---------------------\n\nNode types are defined in your ``cluster.json`` file::\n\n {\n \"nodes\": [\n {\n \"ssh_config\": {\"host\": \"worker1\", ...},\n \"node_type\": \"small\"\n },\n {\n \"ssh_config\": {\"host\": \"worker2\", ...},\n \"node_type\": \"large\"\n }\n ]\n }\n\nBackward Compatibility\n----------------------\n\n- Tests without ``node_type`` will match **any available node**\n- Existing tests and cluster configurations continue to work unchanged\n- Node type is optional in both test annotations and cluster configuration\n" }, { "path": "ducktape/__init__.py", "content": "__version__ = \"0.14.0\"\n" }, { "path": "ducktape/__main__.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.command_line import main\n\nif __name__ == \"__main__\":\n main.main()\n" }, { "path": "ducktape/cluster/__init__.py", "content": "from .json import JsonCluster # NOQA\nfrom .localhost import LocalhostCluster # NOQA\nfrom .vagrant import VagrantCluster # NOQA\n" }, { "path": "ducktape/cluster/cluster.py", "content": "# Copyright 2014 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport collections\nfrom typing import Iterable, List, Union\n\nfrom ducktape.cluster.cluster_node import ClusterNode\nfrom ducktape.cluster.cluster_spec import ClusterSpec\n\n\nclass Cluster(object):\n \"\"\"Interface for a cluster -- a collection of nodes with login credentials.\n This interface doesn't define any mapping of roles/services to nodes. It only interacts with some underlying\n system that can describe available resources and mediates reservations of those resources.\n \"\"\"\n\n def __init__(self):\n self.max_used_nodes = 0\n\n def __len__(self) -> int:\n \"\"\"Size of this cluster object. I.e. number of 'nodes' in the cluster.\"\"\"\n return self.available().size() + self.used().size()\n\n def alloc(self, cluster_spec) -> Union[ClusterNode, List[ClusterNode], \"Cluster\"]:\n \"\"\"\n Allocate some nodes.\n\n :param cluster_spec: A ClusterSpec describing the nodes to be allocated.\n :throws InsufficientResources: If the nodes cannot be allocated.\n :return: Allocated nodes spec\n \"\"\"\n allocated = self.do_alloc(cluster_spec)\n self.max_used_nodes = max(self.max_used_nodes, len(self.used()))\n return allocated\n\n def do_alloc(self, cluster_spec) -> Union[ClusterNode, List[ClusterNode], \"Cluster\"]:\n \"\"\"\n Subclasses should implement actual allocation here.\n\n :param cluster_spec: A ClusterSpec describing the nodes to be allocated.\n :throws InsufficientResources: If the nodes cannot be allocated.\n :return: Allocated nodes spec\n \"\"\"\n raise NotImplementedError\n\n def free(self, nodes: Union[Iterable[ClusterNode], ClusterNode]) -> None:\n \"\"\"Free the given node or list of nodes\"\"\"\n if isinstance(nodes, collections.abc.Iterable):\n for s in nodes:\n self.free_single(s)\n else:\n self.free_single(nodes)\n\n def free_single(self, node: ClusterNode) -> None:\n raise NotImplementedError()\n\n def __eq__(self, other):\n return other is not None and self.__dict__ == other.__dict__\n\n def __hash__(self):\n return hash(tuple(sorted(self.__dict__.items())))\n\n def num_available_nodes(self) -> int:\n return self.available().size()\n\n def available(self) -> ClusterSpec:\n \"\"\"\n Return a ClusterSpec object describing the currently available nodes.\n \"\"\"\n raise NotImplementedError\n\n def used(self) -> ClusterSpec:\n \"\"\"\n Return a ClusterSpec object describing the currently in use nodes.\n \"\"\"\n raise NotImplementedError\n\n def max_used(self) -> int:\n return self.max_used_nodes\n\n def all(self):\n \"\"\"\n Return a ClusterSpec object describing all nodes.\n \"\"\"\n return self.available().clone().add(self.used())\n" }, { "path": "ducktape/cluster/cluster_node.py", "content": "from typing import Optional\n\nfrom ducktape.cluster.remoteaccount import RemoteAccount\n\n\nclass ClusterNode(object):\n def __init__(self, account: RemoteAccount, **kwargs):\n self.account = account\n for k, v in kwargs.items():\n setattr(self, k, v)\n\n @property\n def name(self) -> Optional[str]:\n return self.account.hostname\n\n @property\n def operating_system(self) -> Optional[str]:\n return self.account.operating_system\n" }, { "path": "ducktape/cluster/cluster_spec.py", "content": "# Copyright 2017 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\n\nimport json\nimport typing\n\nfrom ducktape.cluster.node_container import NodeContainer\n\nfrom .consts import LINUX\nfrom .node_spec import NodeSpec\n\n\nclass ClusterSpec(object):\n \"\"\"\n The specification for a ducktape cluster.\n \"\"\"\n\n nodes: typing.Optional[NodeContainer] = None\n\n @staticmethod\n def empty():\n return ClusterSpec([])\n\n @staticmethod\n def simple_linux(num_nodes, node_type=None):\n \"\"\"\n Create a ClusterSpec for Linux nodes, optionally of a specific type.\n\n Examples:\n ClusterSpec.simple_linux(5) # 5 nodes, any type\n ClusterSpec.simple_linux(3, \"large\") # 3 large nodes\n\n :param num_nodes: Number of Linux nodes\n :param node_type: Optional node type label (e.g., \"large\", \"small\")\n \"\"\"\n node_specs = [NodeSpec(LINUX, node_type)] * num_nodes\n return ClusterSpec(node_specs)\n\n @staticmethod\n def from_nodes(nodes):\n \"\"\"\n Create a ClusterSpec describing a list of nodes.\n \"\"\"\n return ClusterSpec([NodeSpec(node.operating_system, getattr(node, \"node_type\", None)) for node in nodes])\n\n def __init__(self, nodes=None):\n \"\"\"\n Initialize the ClusterSpec.\n\n :param nodes: A collection of NodeSpecs, or None to create an empty cluster spec.\n \"\"\"\n self.nodes = NodeContainer(nodes)\n\n def __len__(self):\n return self.size()\n\n def __iter__(self):\n return self.nodes.elements()\n\n def size(self):\n \"\"\"Return the total size of this cluster spec, including all types of nodes.\"\"\"\n return self.nodes.size()\n\n def add(self, other):\n \"\"\"\n Add another ClusterSpec to this one.\n\n :param node_spec: The other cluster spec. This will not be modified.\n :return: This ClusterSpec.\n \"\"\"\n for node_spec in other.nodes:\n self.nodes.add_node(node_spec)\n return self\n\n def clone(self):\n \"\"\"\n Returns a deep copy of this object.\n \"\"\"\n return ClusterSpec(self.nodes.clone())\n\n def __str__(self):\n node_spec_to_num = {}\n for node_spec in self.nodes.elements():\n node_spec_str = str(node_spec)\n node_spec_to_num[node_spec_str] = node_spec_to_num.get(node_spec_str, 0) + 1\n rval = []\n for node_spec_str in sorted(node_spec_to_num.keys()):\n node_spec = json.loads(node_spec_str)\n node_spec[\"num_nodes\"] = node_spec_to_num[node_spec_str]\n rval.append(node_spec)\n return json.dumps(rval, sort_keys=True)\n" }, { "path": "ducktape/cluster/consts.py", "content": "# Copyright 2017 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nLINUX = \"linux\"\n\nWINDOWS = \"windows\"\n\nSUPPORTED_OS_TYPES = [LINUX, WINDOWS]\n" }, { "path": "ducktape/cluster/finite_subcluster.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nimport typing\n\nfrom ducktape.cluster.cluster import Cluster, ClusterNode\nfrom ducktape.cluster.cluster_spec import ClusterSpec\nfrom ducktape.cluster.node_container import NodeContainer\n\n\nclass FiniteSubcluster(Cluster):\n \"\"\"This cluster class gives us a mechanism for allocating finite blocks of nodes from another cluster.\"\"\"\n\n def __init__(self, nodes: typing.Iterable[ClusterNode]):\n super(FiniteSubcluster, self).__init__()\n self.nodes = nodes\n self._available_nodes = NodeContainer(nodes)\n self._in_use_nodes = NodeContainer()\n\n def do_alloc(self, cluster_spec) -> typing.List[ClusterNode]:\n # there cannot be any bad nodes here,\n # since FiniteSubcluster operates on ClusterNode objects,\n # which are not checked for health by NodeContainer.remove_spec\n # however there could be an error, specifically if a test decides to alloc more nodes than are available\n # in a previous ducktape version this exception was raised by remove_spec\n # in this one, for consistency, we let the cluster itself deal with allocation errors\n good_nodes, bad_nodes = self._available_nodes.remove_spec(cluster_spec)\n self._in_use_nodes.add_nodes(good_nodes)\n return good_nodes\n\n def free_single(self, node):\n self._in_use_nodes.remove_node(node)\n self._available_nodes.add_node(node)\n\n def available(self):\n return ClusterSpec.from_nodes(self._available_nodes)\n\n def used(self):\n return ClusterSpec.from_nodes(self._in_use_nodes)\n" }, { "path": "ducktape/cluster/json.py", "content": "# Copyright 2014 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\n\nimport json\nimport os\nimport traceback\n\nfrom ducktape.cluster.cluster_spec import ClusterSpec\nfrom ducktape.cluster.consts import WINDOWS\nfrom ducktape.cluster.linux_remoteaccount import LinuxRemoteAccount\nfrom ducktape.cluster.node_container import InsufficientHealthyNodesError, NodeContainer\nfrom ducktape.cluster.windows_remoteaccount import WindowsRemoteAccount\nfrom ducktape.command_line.defaults import ConsoleDefaults\n\nfrom .cluster import Cluster\nfrom .cluster_node import ClusterNode\nfrom .remoteaccount import RemoteAccountSSHConfig\n\n\ndef make_remote_account(ssh_config, *args, **kwargs):\n \"\"\"Factory function for creating the correct RemoteAccount implementation.\"\"\"\n\n if ssh_config.host and WINDOWS in ssh_config.host:\n return WindowsRemoteAccount(ssh_config, *args, **kwargs)\n else:\n return LinuxRemoteAccount(ssh_config, *args, **kwargs)\n\n\nclass JsonCluster(Cluster):\n \"\"\"An implementation of Cluster that uses static settings specified in a cluster file or json-serializeable dict\"\"\"\n\n def __init__(\n self,\n cluster_json=None,\n *args,\n make_remote_account_func=make_remote_account,\n **kwargs,\n ):\n \"\"\"Initialize JsonCluster\n\n JsonCluster can be initialized from:\n - a json-serializeable dict\n - a \"cluster_file\" containing json\n\n :param cluster_json: a json-serializeable dict containing node information. If ``cluster_json`` is None,\n load from file\n :param cluster_file (optional): Overrides the default location of the json cluster file\n\n Example json with a local Vagrant cluster::\n\n {\n \"nodes\": [\n {\n \"externally_routable_ip\": \"192.168.50.151\",\n\n \"ssh_config\": {\n \"host\": \"worker1\",\n \"hostname\": \"127.0.0.1\",\n \"identityfile\": \"/path/to/private_key\",\n \"password\": null,\n \"port\": 2222,\n \"user\": \"vagrant\"\n }\n },\n {\n \"externally_routable_ip\": \"192.168.50.151\",\n\n \"ssh_config\": {\n \"host\": \"worker2\",\n \"hostname\": \"127.0.0.1\",\n \"identityfile\": \"/path/to/private_key\",\n \"password\": null,\n \"port\": 2223,\n \"user\": \"vagrant\"\n }\n }\n ]\n }\n\n \"\"\"\n super(JsonCluster, self).__init__()\n self._available_accounts: NodeContainer = NodeContainer()\n self._bad_accounts: NodeContainer = NodeContainer()\n self._in_use_nodes: NodeContainer = NodeContainer()\n if cluster_json is None:\n # This is a directly instantiation of JsonCluster rather than from a subclass (e.g. VagrantCluster)\n cluster_file = kwargs.get(\"cluster_file\")\n if cluster_file is None:\n cluster_file = ConsoleDefaults.CLUSTER_FILE\n cluster_json = json.load(open(os.path.abspath(cluster_file)))\n try:\n for ninfo in cluster_json[\"nodes\"]:\n ssh_config_dict = ninfo.get(\"ssh_config\")\n assert ssh_config_dict is not None, (\n \"Cluster json has a node without a ssh_config field: %s\\n Cluster json: %s\" % (ninfo, cluster_json)\n )\n\n ssh_config = RemoteAccountSSHConfig(**ninfo.get(\"ssh_config\", {}))\n\n # Extract node_type from JSON (optional field)\n node_type = ninfo.get(\"node_type\")\n\n remote_account = make_remote_account_func(\n ssh_config,\n ninfo.get(\"externally_routable_ip\"),\n node_type=node_type,\n ssh_exception_checks=kwargs.get(\"ssh_exception_checks\"),\n )\n if remote_account.externally_routable_ip is None:\n remote_account.externally_routable_ip = self._externally_routable_ip(remote_account)\n self._available_accounts.add_node(remote_account)\n except BaseException as e:\n msg = \"JSON cluster definition invalid: %s: %s\" % (\n e,\n traceback.format_exc(limit=16),\n )\n raise ValueError(msg)\n self._id_supplier = 0\n\n def do_alloc(self, cluster_spec):\n try:\n good_nodes, bad_nodes = self._available_accounts.remove_spec(cluster_spec)\n except InsufficientHealthyNodesError as e:\n self._bad_accounts.add_nodes(e.bad_nodes)\n raise e\n\n # even in case of no exceptions, we can still run into bad nodes, so let's track them\n if bad_nodes:\n self._bad_accounts.add_nodes(bad_nodes)\n\n # now let's gather all the good ones and convert them into ClusterNode objects\n allocated_nodes = []\n for account in good_nodes:\n allocated_nodes.append(ClusterNode(account, slot_id=self._id_supplier))\n self._id_supplier += 1\n self._in_use_nodes.add_nodes(allocated_nodes)\n\n return allocated_nodes\n\n def free_single(self, node):\n self._in_use_nodes.remove_node(node)\n self._available_accounts.add_node(node.account)\n node.account.close()\n\n def _externally_routable_ip(self, account):\n return None\n\n def available(self):\n return ClusterSpec.from_nodes(self._available_accounts)\n\n def used(self):\n return ClusterSpec.from_nodes(self._in_use_nodes)\n" }, { "path": "ducktape/cluster/linux_remoteaccount.py", "content": "# Copyright 2014 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom typing import Optional\n\nfrom paramiko import SFTPClient, SSHClient\n\nfrom ducktape.cluster.consts import LINUX\nfrom ducktape.cluster.remoteaccount import RemoteAccount, RemoteAccountError\n\n\nclass LinuxRemoteAccount(RemoteAccount):\n def __init__(self, *args, **kwargs):\n super(LinuxRemoteAccount, self).__init__(*args, **kwargs)\n self._ssh_client: Optional[SSHClient] = None\n self._sftp_client: Optional[SFTPClient] = None\n self.os = LINUX\n\n @property\n def local(self):\n \"\"\"Returns True if this 'remote' account is probably local.\n This is an imperfect heuristic, but should work for simple local testing.\"\"\"\n return self.hostname == \"localhost\" and self.user is None and self.ssh_config is None\n\n def get_network_devices(self):\n \"\"\"\n Utility to get all network devices on a linux account\n \"\"\"\n return [device for device in self.sftp_client.listdir(\"/sys/class/net\")]\n\n def get_external_accessible_network_devices(self):\n \"\"\"\n gets the subset of devices accessible through an external conenction\n \"\"\"\n return [\n device\n for device in self.get_network_devices()\n if device != \"lo\" # do not include local device\n and (device.startswith(\"en\") or device.startswith(\"eth\")) # filter out other devices; \"en\" means ethernet\n # eth0 can also sometimes happen, see https://unix.stackexchange.com/q/134483\n ]\n\n # deprecated, please use the self.externally_routable_ip that is set in your cluster,\n # not explicitly deprecating it as it's used by vagrant cluster\n def fetch_externally_routable_ip(self, is_aws=None):\n if is_aws is not None:\n self.logger.warning(\"fetch_externally_routable_ip: is_aws is a deprecated flag, and does nothing\")\n\n devices = self.get_external_accessible_network_devices()\n\n self.logger.debug(\"found devices: {}\".format(devices))\n\n if not devices:\n raise RemoteAccountError(self, \"Couldn't find any network devices\")\n\n fmt_cmd = (\n \"/sbin/ifconfig {device} | \" \"grep 'inet ' | \" \"tail -n 1 | \" r\"egrep -o '[0-9\\.]+' | \" \"head -n 1 2>&1\"\n )\n\n ips = [\"\".join(self.ssh_capture(fmt_cmd.format(device=device))).strip() for device in devices]\n self.logger.debug(\"found ips: {}\".format(ips))\n self.logger.debug(\"returning the first ip found\")\n return next(iter(ips))\n" }, { "path": "ducktape/cluster/localhost.py", "content": "# Copyright 2014 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.cluster.cluster_spec import ClusterSpec\nfrom ducktape.cluster.node_container import NodeContainer\n\nfrom .cluster import Cluster\nfrom .cluster_node import ClusterNode\nfrom .linux_remoteaccount import LinuxRemoteAccount\nfrom .remoteaccount import RemoteAccountSSHConfig\n\n\nclass LocalhostCluster(Cluster):\n \"\"\"\n A \"cluster\" that runs entirely on localhost using default credentials. This doesn't require any user\n configuration and is equivalent to the old defaults in cluster_config.json. There are no constraints\n on the resources available.\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n super(LocalhostCluster, self).__init__()\n num_nodes = kwargs.get(\"num_nodes\", 1000)\n self._available_nodes = NodeContainer()\n for i in range(num_nodes):\n ssh_config = RemoteAccountSSHConfig(\"localhost%d\" % i, hostname=\"localhost\", port=22)\n self._available_nodes.add_node(\n ClusterNode(\n LinuxRemoteAccount(\n ssh_config,\n ssh_exception_checks=kwargs.get(\"ssh_exception_checks\"),\n )\n )\n )\n self._in_use_nodes = NodeContainer()\n\n def do_alloc(self, cluster_spec):\n # there shouldn't be any bad nodes in localhost cluster\n # since ClusterNode object does not implement `available()` method\n good_nodes, bad_nodes = self._available_nodes.remove_spec(cluster_spec)\n self._in_use_nodes.add_nodes(good_nodes)\n return good_nodes\n\n def free_single(self, node):\n self._in_use_nodes.remove_node(node)\n self._available_nodes.add_node(node)\n node.account.close()\n\n def available(self):\n return ClusterSpec.from_nodes(self._available_nodes)\n\n def used(self):\n return ClusterSpec.from_nodes(self._in_use_nodes)\n" }, { "path": "ducktape/cluster/node_container.py", "content": "# Copyright 2017 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nfrom __future__ import annotations\n\nfrom typing import (\n TYPE_CHECKING,\n Any,\n Dict,\n Iterable,\n Iterator,\n List,\n Optional,\n Tuple,\n Union,\n)\n\nfrom ducktape.cluster.cluster_node import ClusterNode\nfrom ducktape.cluster.remoteaccount import RemoteAccount\n\nif TYPE_CHECKING:\n from ducktape.cluster.cluster_spec import ClusterSpec\n\nNodeType = Union[ClusterNode, RemoteAccount]\n# Key for node grouping: (operating_system, node_type)\nNodeGroupKey = Tuple[Optional[str], Optional[str]]\n\n\nclass NodeNotPresentError(Exception):\n pass\n\n\nclass InsufficientResourcesError(Exception):\n pass\n\n\nclass InsufficientHealthyNodesError(InsufficientResourcesError):\n def __init__(self, bad_nodes: List, *args):\n self.bad_nodes = bad_nodes\n super().__init__(*args)\n\n\ndef _get_node_key(node: NodeType) -> NodeGroupKey:\n \"\"\"Extract the (os, node_type) key from a node.\"\"\"\n os = getattr(node, \"operating_system\", None)\n node_type = getattr(node, \"node_type\", None)\n return (os, node_type)\n\n\nclass NodeContainer(object):\n \"\"\"\n Container for cluster nodes, grouped by (operating_system, node_type).\n\n This enables efficient lookup and allocation of nodes matching specific\n requirements. Nodes with node_type=None are grouped under (os, None) and\n can match any request when no specific type is required.\n \"\"\"\n\n # Key: (os, node_type) tuple, Value: list of nodes\n node_groups: Dict[NodeGroupKey, List[NodeType]]\n\n def __init__(self, nodes: Optional[Iterable[NodeType]] = None) -> None:\n \"\"\"\n Create a NodeContainer with the given nodes.\n\n Node objects should implement at least an operating_system property,\n and optionally a node_type property.\n\n :param nodes: A collection of node objects to add, or None to add nothing.\n \"\"\"\n self.node_groups = {}\n if nodes is not None:\n for node in nodes:\n self.add_node(node)\n\n def size(self) -> int:\n \"\"\"\n Returns the total number of nodes in the container.\n \"\"\"\n return sum([len(val) for val in self.node_groups.values()])\n\n def __len__(self):\n return self.size()\n\n def __iter__(self) -> Iterator[Any]:\n return self.elements()\n\n def elements(self, operating_system: Optional[str] = None, node_type: Optional[str] = None) -> Iterator[NodeType]:\n \"\"\"\n Yield the elements in this container.\n\n :param operating_system: If this is non-None, we will iterate only over elements\n which have this operating system.\n :param node_type: If this is non-None, we will iterate only over elements\n which have this node type.\n \"\"\"\n for (os, nt), node_list in self.node_groups.items():\n # Filter by OS if specified\n if operating_system is not None and os != operating_system:\n continue\n # Filter by node_type if specified\n if node_type is not None and nt != node_type:\n continue\n for node in node_list:\n yield node\n\n def grouped_by_os_and_type(self) -> Dict[Tuple[Optional[str], Optional[str]], int]:\n \"\"\"\n Returns nodes grouped by (operating_system, node_type) with counts.\n\n This is a pure data method that groups nodes without any ordering.\n The caller is responsible for determining processing order.\n\n :return: Dictionary mapping (os, node_type) tuples to counts\n \"\"\"\n result: Dict[Tuple[Optional[str], Optional[str]], int] = {}\n for node in self.elements():\n key = (getattr(node, \"operating_system\", None), getattr(node, \"node_type\", None))\n result[key] = result.get(key, 0) + 1\n return result\n\n def add_node(self, node: Union[ClusterNode, RemoteAccount]) -> None:\n \"\"\"\n Add a node to this collection, grouping by (os, node_type).\n\n :param node: The node to add.\n \"\"\"\n key = _get_node_key(node)\n self.node_groups.setdefault(key, []).append(node)\n\n def add_nodes(self, nodes):\n \"\"\"\n Add a collection of nodes to this collection.\n\n :param nodes: The nodes to add.\n \"\"\"\n for node in nodes:\n self.add_node(node)\n\n def remove_node(self, node):\n \"\"\"\n Removes a node from this collection.\n\n :param node: The node to remove.\n :returns: The node which has been removed.\n :throws NodeNotPresentError: If the node is not in the collection.\n \"\"\"\n key = _get_node_key(node)\n try:\n return self.node_groups.get(key, []).remove(node)\n except ValueError:\n raise NodeNotPresentError\n\n def remove_nodes(self, nodes):\n \"\"\"\n Remove a collection of nodes from this collection.\n\n :param nodes: The nodes to remove.\n \"\"\"\n for node in nodes:\n self.remove_node(node)\n\n def _find_matching_nodes(\n self, required_os: str, required_node_type: Optional[str], num_needed: int\n ) -> Tuple[List[NodeType], List[NodeType], int]:\n \"\"\"\n Find nodes that match the required OS and node_type.\n\n Matching rules:\n - OS must match exactly\n - If required_node_type is None, match nodes of ANY type for this OS\n - If required_node_type is specified, match only nodes with that exact type\n\n :param required_os: The required operating system\n :param required_node_type: The required node type (None means any)\n :param num_needed: Number of nodes needed\n :return: Tuple of (good_nodes, bad_nodes, shortfall) where shortfall is how many more we need\n \"\"\"\n good_nodes: List[NodeType] = []\n bad_nodes: List[NodeType] = []\n\n # Collect candidate keys - keys in node_groups that can satisfy this requirement\n candidate_keys: List[NodeGroupKey] = []\n for os, nt in self.node_groups.keys():\n if os != required_os:\n continue\n # If no specific type required, any node of this OS matches\n # If specific type required, only exact match\n if required_node_type is None or nt == required_node_type:\n candidate_keys.append((os, nt))\n\n # Try to allocate from candidate pools\n for key in candidate_keys:\n if len(good_nodes) >= num_needed:\n break\n\n avail_nodes = self.node_groups.get(key, [])\n while avail_nodes and len(good_nodes) < num_needed:\n node = avail_nodes.pop(0)\n if isinstance(node, RemoteAccount):\n if node.available():\n good_nodes.append(node)\n else:\n bad_nodes.append(node)\n else:\n good_nodes.append(node)\n\n shortfall = max(0, num_needed - len(good_nodes))\n return good_nodes, bad_nodes, shortfall\n\n def remove_spec(self, cluster_spec: ClusterSpec) -> Tuple[List[NodeType], List[NodeType]]:\n \"\"\"\n Remove nodes matching a ClusterSpec from this NodeContainer.\n\n Allocation strategy:\n - Specific node_type requirements are allocated BEFORE any-type (None) requirements\n - For each (os, node_type) in the spec:\n - If node_type is specified, allocate from that exact pool\n - If node_type is None, allocate from any pool matching the OS\n\n :param cluster_spec: The cluster spec. This will not be modified.\n :returns: Tuple of (good_nodes, bad_nodes).\n :raises: InsufficientResourcesError when there aren't enough total nodes\n InsufficientHealthyNodesError when there aren't enough healthy nodes\n \"\"\"\n err = self.attempt_remove_spec(cluster_spec)\n if err:\n raise InsufficientResourcesError(err)\n\n good_nodes: List[NodeType] = []\n bad_nodes: List[NodeType] = []\n msg = \"\"\n\n # Get requirements grouped by (os, node_type) with counts\n grouped_counts = cluster_spec.nodes.grouped_by_os_and_type()\n\n # Sort so specific types (node_type != None) are allocated before any-type (node_type == None)\n # This prevents any-type requests from \"stealing\" nodes needed by specific types\n def allocation_order(item: Tuple[Tuple[str, Optional[str]], int]) -> Tuple[int, str, str]:\n (os, node_type), _ = item\n # Specific types first (0), any-type last (1)\n type_order = 1 if node_type is None else 0\n return (type_order, os or \"\", node_type or \"\")\n\n sorted_requirements = sorted(grouped_counts.items(), key=allocation_order)\n\n for (os, node_type), num_needed in sorted_requirements:\n found_good, found_bad, shortfall = self._find_matching_nodes(os, node_type, num_needed)\n\n good_nodes.extend(found_good)\n bad_nodes.extend(found_bad)\n\n if shortfall > 0:\n type_desc = f\"{os}\" if node_type is None else f\"{os}/{node_type}\"\n msg += f\"{type_desc} nodes requested: {num_needed}. Healthy nodes available: {len(found_good)}. \"\n\n if msg:\n # Return good nodes back to the container\n for node in good_nodes:\n self.add_node(node)\n raise InsufficientHealthyNodesError(bad_nodes, msg)\n\n return good_nodes, bad_nodes\n\n def can_remove_spec(self, cluster_spec: ClusterSpec) -> bool:\n \"\"\"\n Determine if we can remove nodes matching a ClusterSpec from this NodeContainer.\n This container will not be modified.\n\n :param cluster_spec: The cluster spec. This will not be modified.\n :returns: True if we could remove the nodes; false otherwise\n \"\"\"\n msg = self.attempt_remove_spec(cluster_spec)\n return len(msg) == 0\n\n def _count_nodes_by_os(self, target_os: str) -> int:\n \"\"\"\n Count total nodes available for a given OS (regardless of node_type).\n\n :param target_os: The operating system to count nodes for\n :return: Total number of nodes with the given OS\n \"\"\"\n count = 0\n for (os, _), nodes in self.node_groups.items():\n if os == target_os:\n count += len(nodes)\n return count\n\n def _count_nodes_by_os_and_type(self, target_os: str, target_type: str) -> int:\n \"\"\"\n Count nodes available for a specific (os, node_type) combination.\n\n :param target_os: The operating system\n :param target_type: The specific node type (not None)\n :return: Number of nodes matching both OS and type\n \"\"\"\n return len(self.node_groups.get((target_os, target_type), []))\n\n def attempt_remove_spec(self, cluster_spec: ClusterSpec) -> str:\n \"\"\"\n Attempt to remove a cluster_spec from this node container.\n\n Uses holistic per-OS validation to correctly handle mixed typed and untyped\n requirements without double-counting shared capacity.\n\n Validation strategy:\n 1. Check total OS capacity >= total OS demand\n 2. Check each specific type has enough nodes\n 3. Check remaining capacity (after specific types) >= any-type demand\n\n :param cluster_spec: The cluster spec. This will not be modified.\n :returns: An empty string if we can remove the nodes;\n an error string otherwise.\n \"\"\"\n # if cluster_spec is None this means the test cannot be run at all\n # e.g. users didn't specify `@cluster` annotation on it but the session context has a flag to fail\n # on such tests or any other state where the test deems its cluster spec incorrect.\n if cluster_spec is None:\n return \"Invalid or missing cluster spec\"\n # cluster spec may be empty and that's ok, shortcut to returning no error messages\n elif len(cluster_spec) == 0 or cluster_spec.nodes is None:\n return \"\"\n\n msg = \"\"\n\n # Build requirements_by_os: {os -> {node_type -> count}} in a single pass\n requirements_by_os: Dict[str, Dict[Optional[str], int]] = {}\n for node_spec in cluster_spec.nodes.elements():\n os = node_spec.operating_system\n node_type = node_spec.node_type\n requirements_by_os.setdefault(os, {})\n requirements_by_os[os][node_type] = requirements_by_os[os].get(node_type, 0) + 1\n\n # Validate each OS holistically\n for os, type_requirements in requirements_by_os.items():\n total_available = self._count_nodes_by_os(os)\n total_required = sum(type_requirements.values())\n\n # Check 1: Total capacity for this OS\n if total_available < total_required:\n msg += f\"{os} nodes requested: {total_required}. {os} nodes available: {total_available}. \"\n continue # Already failed, no need for detailed checks\n\n # Check 2: Each specific type has enough nodes\n for node_type, count_needed in type_requirements.items():\n if node_type is None:\n continue # Handle any-type separately\n type_available = self._count_nodes_by_os_and_type(os, node_type)\n if type_available < count_needed:\n msg += f\"{os}/{node_type} nodes requested: {count_needed}. {os}/{node_type} nodes available: {type_available}. \"\n\n # Check 3: After reserving specific types, is there capacity for any-type?\n any_type_demand = type_requirements.get(None, 0)\n if any_type_demand > 0:\n specific_demand = sum(c for t, c in type_requirements.items() if t is not None)\n remaining_capacity = total_available - specific_demand\n if remaining_capacity < any_type_demand:\n msg += (\n f\"{os} (any type) nodes requested: {any_type_demand}. \"\n f\"{os} nodes remaining after typed allocations: {remaining_capacity}. \"\n )\n\n return msg\n\n def clone(self) -> \"NodeContainer\":\n \"\"\"\n Returns a deep copy of this object.\n \"\"\"\n container = NodeContainer()\n for key, nodes in self.node_groups.items():\n for node in nodes:\n container.node_groups.setdefault(key, []).append(node)\n return container\n" }, { "path": "ducktape/cluster/node_spec.py", "content": "# Copyright 2017 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\n\nimport json\nfrom typing import Optional\n\nfrom .consts import LINUX, SUPPORTED_OS_TYPES\n\n\nclass NodeSpec(object):\n \"\"\"\n Specification for a single cluster node.\n\n The node_type field is a generic label that can represent size, architecture,\n or any classification scheme defined by the cluster configuration. When None,\n it matches any available node (backward compatible behavior).\n\n :param operating_system: The operating system of the node.\n :param node_type: Node type label (e.g., \"large\", \"small\"). None means \"match any\".\n \"\"\"\n\n def __init__(self, operating_system: str = LINUX, node_type: Optional[str] = None):\n self.operating_system = operating_system\n self.node_type = node_type\n if self.operating_system not in SUPPORTED_OS_TYPES:\n raise RuntimeError(\"Unsupported os type %s\" % self.operating_system)\n\n def matches(self, available_node_spec: \"NodeSpec\") -> bool:\n \"\"\"\n Check if this requirement can be satisfied by an available node.\n\n Matching rules:\n - OS must match exactly\n - If requested node_type is None, match any type\n - If requested node_type is specified, must match exactly\n\n :param available_node_spec: The specification of an available node\n :return: True if this requirement matches the available node\n \"\"\"\n if self.operating_system != available_node_spec.operating_system:\n return False\n if self.node_type is None:\n return True # Requestor doesn't care about type\n return self.node_type == available_node_spec.node_type\n\n def __str__(self):\n d = {\"os\": self.operating_system}\n if self.node_type is not None:\n d[\"node_type\"] = self.node_type\n return json.dumps(d, sort_keys=True)\n\n def __eq__(self, other):\n if not isinstance(other, NodeSpec):\n return False\n return self.operating_system == other.operating_system and self.node_type == other.node_type\n\n def __hash__(self):\n return hash((self.operating_system, self.node_type))\n" }, { "path": "ducktape/cluster/remoteaccount.py", "content": "# Copyright 2014 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\nimport os\nimport shutil\nimport signal\nimport socket\nimport stat\nimport tempfile\nimport warnings\nfrom contextlib import contextmanager\nfrom typing import Callable, List, Optional, Union\n\nfrom paramiko import MissingHostKeyPolicy, SFTPClient, SSHClient, SSHConfig\nfrom paramiko.ssh_exception import NoValidConnectionsError, SSHException\n\nfrom ducktape.errors import DucktapeError\nfrom ducktape.utils.http_utils import HttpMixin\nfrom ducktape.utils.util import wait_until\n\n\ndef check_ssh(method: Callable) -> Callable:\n def wrapper(self, *args, **kwargs):\n try:\n return method(self, *args, **kwargs)\n except (SSHException, NoValidConnectionsError, socket.error) as e:\n if self._custom_ssh_exception_checks:\n self._log(logging.DEBUG, \"caught ssh error\", exc_info=True)\n self._log(logging.DEBUG, \"starting ssh checks:\")\n self._log(\n logging.DEBUG,\n \"\\n\".join(repr(f) for f in self._custom_ssh_exception_checks),\n )\n for func in self._custom_ssh_exception_checks:\n func(e, self)\n raise\n\n return wrapper\n\n\nclass RemoteAccountSSHConfig(object):\n def __init__(\n self,\n host: Optional[str] = None,\n hostname: Optional[str] = None,\n user: Optional[str] = None,\n port: Optional[int] = None,\n password: Optional[str] = None,\n identityfile: Optional[str] = None,\n connecttimeout: Optional[Union[int, float]] = None,\n **kwargs,\n ) -> None:\n \"\"\"Wrapper for ssh configs used by ducktape to connect to remote machines.\n\n The fields in this class are lowercase versions of a small selection of ssh config properties\n (see man page: \"man ssh_config\")\n \"\"\"\n self.host = host\n self.hostname: str = hostname or \"localhost\"\n self.user = user\n self.port = port or 22\n self.port = int(self.port)\n self.password = password\n self.identityfile = identityfile\n # None is default, and it means default TCP timeout will be used.\n self.connecttimeout = int(connecttimeout) if connecttimeout is not None else None\n\n @staticmethod\n def from_string(config_str):\n \"\"\"Construct RemoteAccountSSHConfig object from a string that looks like\n\n Host the-host\n Hostname the-hostname\n Port 22\n User ubuntu\n IdentityFile /path/to/key\n \"\"\"\n config = SSHConfig()\n config.parse(config_str.split(\"\\n\"))\n\n hostnames = config.get_hostnames()\n if \"*\" in hostnames:\n hostnames.remove(\"*\")\n assert len(hostnames) == 1, \"Expected hostnames to have single entry: %s\" % hostnames\n host = hostnames.pop()\n\n config_dict = config.lookup(host)\n if config_dict.get(\"identityfile\") is not None:\n # paramiko.SSHConfig parses this in as a list, but we only want a single string\n config_dict[\"identityfile\"] = config_dict[\"identityfile\"][0]\n\n return RemoteAccountSSHConfig(host, **config_dict)\n\n def to_json(self):\n return self.__dict__\n\n def __repr__(self):\n return str(self.to_json())\n\n def __eq__(self, other):\n return other and other.__dict__ == self.__dict__\n\n def __hash__(self):\n return hash(tuple(sorted(self.__dict__.items())))\n\n\nclass RemoteAccountError(DucktapeError):\n \"\"\"This exception is raised when an attempted action on a remote node fails.\"\"\"\n\n def __init__(self, account, msg):\n self.account_str = str(account)\n self.msg = msg\n\n def __str__(self):\n return \"%s: %s\" % (self.account_str, self.msg)\n\n\nclass RemoteCommandError(RemoteAccountError):\n \"\"\"This exception is raised when a process run by ssh*() returns a non-zero exit status.\"\"\"\n\n def __init__(self, account, cmd, exit_status, msg):\n self.account_str = str(account)\n self.exit_status = exit_status\n self.cmd = cmd\n self.msg = msg\n\n def __str__(self):\n msg = \"%s: Command '%s' returned non-zero exit status %d.\" % (\n self.account_str,\n self.cmd,\n self.exit_status,\n )\n if self.msg:\n msg += \" Remote error message: %s\" % self.msg\n return msg\n\n\nclass RemoteAccount(HttpMixin):\n \"\"\"RemoteAccount is the heart of interaction with cluster nodes,\n and every allocated cluster node has a reference to an instance of RemoteAccount.\n\n It wraps metadata such as ssh configs, and provides methods for file system manipulation and shell commands.\n\n Each operating system has its own RemoteAccount implementation.\n\n The node_type attribute stores the classification label from the cluster\n configuration, enabling type-aware node allocation.\n \"\"\"\n\n def __init__(\n self,\n ssh_config: RemoteAccountSSHConfig,\n externally_routable_ip: Optional[str] = None,\n node_type: Optional[str] = None,\n logger: Optional[logging.Logger] = None,\n ssh_exception_checks: List[Callable] = [],\n ) -> None:\n # Instance of RemoteAccountSSHConfig - use this instead of a dict, because we need the entire object to\n # be hashable\n self.ssh_config = ssh_config\n\n # We don't want to rely on the hostname (e.g. 'worker1') having been added to the driver host's /etc/hosts file.\n # But that means we need to distinguish between the hostname and the value of hostname we use for SSH commands.\n # We try to satisfy all use cases and keep things simple by\n # a) storing the hostname the user probably expects (the \"Host\" value in .ssh/config)\n # b) saving the real value we use for running the SSH command\n self.hostname = ssh_config.host\n self.ssh_hostname = ssh_config.hostname\n\n self.user = ssh_config.user\n self.externally_routable_ip = externally_routable_ip\n self.node_type = node_type # Node type label (e.g., \"large\", \"small\")\n self._logger = logger\n self.os: Optional[str] = None\n self._ssh_client: Optional[SSHClient] = None\n self._sftp_client: Optional[SFTPClient] = None\n self._custom_ssh_exception_checks = ssh_exception_checks\n\n @property\n def operating_system(self) -> Optional[str]:\n return self.os\n\n @property\n def logger(self):\n if self._logger:\n return self._logger\n else:\n return logging.getLogger(__name__)\n\n @logger.setter\n def logger(self, logger):\n self._logger = logger\n\n def _log(self, level, msg, *args, **kwargs):\n msg = \"%s: %s\" % (str(self), msg)\n self.logger.log(level, msg, *args, **kwargs)\n\n @check_ssh\n def _set_ssh_client(self):\n client = SSHClient()\n client.set_missing_host_key_policy(IgnoreMissingHostKeyPolicy())\n\n self._log(logging.DEBUG, \"ssh_config: %s\" % str(self.ssh_config))\n\n client.connect(\n hostname=self.ssh_config.hostname,\n port=self.ssh_config.port,\n username=self.ssh_config.user,\n password=self.ssh_config.password,\n key_filename=self.ssh_config.identityfile,\n look_for_keys=False,\n timeout=self.ssh_config.connecttimeout,\n )\n\n if self._ssh_client:\n self._ssh_client.close()\n self._ssh_client = client\n self._set_sftp_client()\n\n @property\n def ssh_client(self):\n if self._ssh_client and self._ssh_client.get_transport() and self._ssh_client.get_transport().is_active():\n try:\n transport = self._ssh_client.get_transport()\n transport.send_ignore()\n except Exception as e:\n self._log(\n logging.DEBUG,\n \"exception getting ssh_client (creating new client): %s\" % str(e),\n )\n self._set_ssh_client()\n else:\n self._set_ssh_client()\n\n return self._ssh_client\n\n def _set_sftp_client(self):\n if self._sftp_client:\n self._sftp_client.close()\n self._sftp_client = self.ssh_client.open_sftp()\n\n @property\n def sftp_client(self):\n if not self._sftp_client:\n self._set_sftp_client()\n else:\n self.ssh_client # test connection\n\n return self._sftp_client\n\n def close(self) -> None:\n \"\"\"Close/release any outstanding network connections to remote account.\"\"\"\n\n if self._ssh_client:\n self._ssh_client.close()\n self._ssh_client = None\n if self._sftp_client:\n self._sftp_client.close()\n self._sftp_client = None\n\n def __str__(self):\n r = \"\"\n if self.user:\n r += self.user + \"@\"\n r += self.hostname\n return r\n\n def __repr__(self):\n return str(self.__dict__)\n\n def __eq__(self, other):\n return other is not None and self.__dict__ == other.__dict__\n\n def __hash__(self):\n return hash(tuple(sorted(self.__dict__.items())))\n\n def wait_for_http_service(self, port, headers, timeout=20, path=\"/\"):\n \"\"\"Wait until this service node is available/awake.\"\"\"\n url = \"http://%s:%s%s\" % (self.externally_routable_ip, str(port), path)\n\n err_msg = (\n \"Timed out trying to contact service on %s. \" % url\n + \"Either the service failed to start, or there is a problem with the url.\"\n )\n wait_until(\n lambda: self._can_ping_url(url, headers),\n timeout_sec=timeout,\n backoff_sec=0.25,\n err_msg=err_msg,\n )\n\n def _can_ping_url(self, url, headers):\n \"\"\"See if we can successfully issue a GET request to the given url.\"\"\"\n try:\n self.http_request(url, \"GET\", None, headers, timeout=0.75)\n return True\n except Exception:\n return False\n\n def available(self):\n # TODO: https://github.com/confluentinc/ducktape/issues/339\n # try:\n # self.ssh_client\n # except Exception:\n # return False\n # else:\n # return True\n # finally:\n # self.close()\n return True\n\n @check_ssh\n def ssh(self, cmd, allow_fail=False):\n \"\"\"Run the given command on the remote host, and block until the command has finished running.\n\n :param cmd: The remote ssh command\n :param allow_fail: If True, ignore nonzero exit status of the remote command,\n else raise an ``RemoteCommandError``\n\n :return: The exit status of the command.\n :raise RemoteCommandError: If allow_fail is False and the command returns a non-zero exit status\n \"\"\"\n self._log(logging.DEBUG, \"Running ssh command: %s\" % cmd)\n\n client = self.ssh_client\n stdin, stdout, stderr = client.exec_command(cmd)\n\n # Unfortunately we need to read over the channel to ensure that recv_exit_status won't hang. See:\n # http://docs.paramiko.org/en/2.0/api/channel.html#paramiko.channel.Channel.recv_exit_status\n stdout.read()\n exit_status = stdout.channel.recv_exit_status()\n try:\n if exit_status != 0:\n if not allow_fail:\n raise RemoteCommandError(self, cmd, exit_status, stderr.read())\n else:\n self._log(\n logging.DEBUG,\n \"Running ssh command '%s' exited with status %d and message: %s\"\n % (cmd, exit_status, stderr.read()),\n )\n finally:\n stdin.close()\n stdout.close()\n stderr.close()\n\n return exit_status\n\n @check_ssh\n def ssh_capture(\n self,\n cmd,\n allow_fail=False,\n callback=None,\n combine_stderr=True,\n timeout_sec=None,\n ):\n \"\"\"Run the given command asynchronously via ssh, and return an SSHOutputIter object.\n\n Does *not* block\n\n :param cmd: The remote ssh command\n :param allow_fail: If True, ignore nonzero exit status of the remote command,\n else raise an ``RemoteCommandError``\n :param callback: If set, the iterator returns ``callback(line)``\n for each line of output instead of the raw output\n :param combine_stderr: If True, return output from both stderr and stdout of the remote process.\n :param timeout_sec: Set timeout on blocking reads/writes. Default None. For more details see\n http://docs.paramiko.org/en/2.0/api/channel.html#paramiko.channel.Channel.settimeout\n\n :return SSHOutputIter: object which allows iteration through each line of output.\n :raise RemoteCommandError: If ``allow_fail`` is False and the command returns a non-zero exit status\n \"\"\"\n self._log(logging.DEBUG, \"Running ssh command: %s\" % cmd)\n\n client = self.ssh_client\n chan = client.get_transport().open_session(timeout=timeout_sec)\n\n chan.settimeout(timeout_sec)\n chan.exec_command(cmd)\n chan.set_combine_stderr(combine_stderr)\n\n stdin = chan.makefile(\"wb\", -1) # set bufsize to -1\n stdout = chan.makefile(\"r\", -1)\n stderr = chan.makefile_stderr(\"r\", -1)\n\n def output_generator():\n for line in iter(stdout.readline, \"\"):\n if callback is None:\n yield line\n else:\n yield callback(line)\n try:\n exit_status = stdout.channel.recv_exit_status()\n if exit_status != 0:\n if not allow_fail:\n raise RemoteCommandError(self, cmd, exit_status, stderr.read())\n else:\n self._log(\n logging.DEBUG,\n \"Running ssh command '%s' exited with status %d and message: %s\"\n % (cmd, exit_status, stderr.read()),\n )\n finally:\n stdin.close()\n stdout.close()\n stderr.close()\n\n return SSHOutputIter(output_generator, stdout)\n\n @check_ssh\n def ssh_output(self, cmd, allow_fail=False, combine_stderr=True, timeout_sec=None):\n \"\"\"Runs the command via SSH and captures the output, returning it as a string.\n\n :param cmd: The remote ssh command.\n :param allow_fail: If True, ignore nonzero exit status of the remote command,\n else raise an ``RemoteCommandError``\n :param combine_stderr: If True, return output from both stderr and stdout of the remote process.\n :param timeout_sec: Set timeout on blocking reads/writes. Default None. For more details see\n http://docs.paramiko.org/en/2.0/api/channel.html#paramiko.channel.Channel.settimeout\n\n :return: The stdout output from the ssh command.\n :raise RemoteCommandError: If ``allow_fail`` is False and the command returns a non-zero exit status\n \"\"\"\n self._log(logging.DEBUG, \"Running ssh command: %s\" % cmd)\n\n client = self.ssh_client\n chan = client.get_transport().open_session(timeout=timeout_sec)\n\n chan.settimeout(timeout_sec)\n chan.exec_command(cmd)\n chan.set_combine_stderr(combine_stderr)\n\n stdin = chan.makefile(\"wb\", -1) # set bufsize to -1\n stdout = chan.makefile(\"r\", -1)\n stderr = chan.makefile_stderr(\"r\", -1)\n\n try:\n stdoutdata = stdout.read()\n exit_status = stdin.channel.recv_exit_status()\n if exit_status != 0:\n if not allow_fail:\n raise RemoteCommandError(self, cmd, exit_status, stderr.read())\n else:\n self._log(\n logging.DEBUG,\n \"Running ssh command '%s' exited with status %d and message: %s\"\n % (cmd, exit_status, stderr.read()),\n )\n finally:\n stdin.close()\n stdout.close()\n stderr.close()\n self._log(logging.DEBUG, \"Returning ssh command output:\\n%s\" % stdoutdata)\n return stdoutdata\n\n def alive(self, pid):\n \"\"\"Return True if and only if process with given pid is alive.\"\"\"\n try:\n self.ssh(\"kill -0 %s\" % str(pid), allow_fail=False)\n return True\n except Exception:\n return False\n\n def signal(self, pid, sig, allow_fail=False):\n cmd = \"kill -%d %s\" % (int(sig), str(pid))\n self.ssh(cmd, allow_fail=allow_fail)\n\n def kill_process(self, process_grep_str, clean_shutdown=True, allow_fail=False):\n cmd = \"\"\"ps ax | grep -i \"\"\" + process_grep_str + \"\"\" | grep -v grep | awk '{print $1}'\"\"\"\n pids = [pid for pid in self.ssh_capture(cmd, allow_fail=True)]\n\n if clean_shutdown:\n sig = signal.SIGTERM\n else:\n sig = signal.SIGKILL\n\n for pid in pids:\n self.signal(pid, sig, allow_fail=allow_fail)\n\n def java_pids(self, match):\n \"\"\"\n Get all the Java process IDs matching 'match'.\n\n :param match: The AWK expression to match\n \"\"\"\n cmd = \"\"\"jcmd | awk '/%s/ { print $1 }'\"\"\" % match\n return [int(pid) for pid in self.ssh_capture(cmd, allow_fail=True)]\n\n def kill_java_processes(self, match, clean_shutdown=True, allow_fail=False):\n \"\"\"\n Kill all the java processes matching 'match'.\n\n :param match: The AWK expression to match\n :param clean_shutdown: True if we should shut down cleanly with SIGTERM;\n false if we should shut down with SIGKILL.\n :param allow_fail: True if we should throw exceptions if the ssh commands fail.\n \"\"\"\n cmd = \"\"\"jcmd | awk '/%s/ { print $1 }'\"\"\" % match\n pids = [pid for pid in self.ssh_capture(cmd, allow_fail=True)]\n\n if clean_shutdown:\n sig = signal.SIGTERM\n else:\n sig = signal.SIGKILL\n\n for pid in pids:\n self.signal(pid, sig, allow_fail=allow_fail)\n\n def copy_between(self, src, dest, dest_node):\n \"\"\"Copy src to dest on dest_node\n\n :param src: Path to the file or directory we want to copy\n :param dest: The destination path\n :param dest_node: The node to which we want to copy the file/directory\n\n Note that if src is a directory, this will automatically copy recursively.\n\n \"\"\"\n # TODO: if dest is an existing file, what is the behavior?\n\n temp_dir = tempfile.mkdtemp()\n\n try:\n # TODO: deal with very unlikely case that src_name matches temp_dir name?\n # TODO: I think this actually works\n local_dest = self._re_anchor_basename(src, temp_dir)\n\n self.copy_from(src, local_dest)\n\n dest_node.account.copy_to(local_dest, dest)\n\n finally:\n if os.path.isdir(temp_dir):\n shutil.rmtree(temp_dir)\n\n def scp_from(self, src, dest, recursive=False):\n warnings.warn(\"scp_from is now deprecated. Please use copy_from\")\n self.copy_from(src, dest)\n\n def _re_anchor_basename(self, path, directory):\n \"\"\"Anchor the basename of path onto the given directory\n\n Helper for the various copy_* methods.\n\n :param path: Path to a file or directory. Could be on the driver machine or a worker machine.\n :param directory: Path to a directory. Could be on the driver machine or a worker machine.\n\n Example::\n\n path/to/the_basename, another/path/ -> another/path/the_basename\n \"\"\"\n path_basename = path\n\n # trim off path separator from end of path\n # this is necessary because os.path.basename of a path ending in a separator is an empty string\n # For example:\n # os.path.basename(\"the/path/\") == \"\"\n # os.path.basename(\"the/path\") == \"path\"\n if path_basename.endswith(os.path.sep):\n path_basename = path_basename[: -len(os.path.sep)]\n path_basename = os.path.basename(path_basename)\n\n return os.path.join(directory, path_basename)\n\n @check_ssh\n def copy_from(self, src, dest):\n if os.path.isdir(dest):\n # dest is an existing directory, so assuming src looks like path/to/src_name,\n # in this case we'll copy as:\n # path/to/src_name -> dest/src_name\n dest = self._re_anchor_basename(src, dest)\n\n if self.isfile(src):\n self.sftp_client.get(src, dest)\n elif self.isdir(src):\n # we can now assume dest path looks like: path_that_exists/new_directory\n os.mkdir(dest)\n\n # for obj in `ls src`, if it's a file, copy with copy_file_from, elif its a directory, call again\n for obj in self.sftp_client.listdir(src):\n obj_path = os.path.join(src, obj)\n if self.isfile(obj_path) or self.isdir(obj_path):\n self.copy_from(obj_path, dest)\n else:\n # TODO what about uncopyable file types?\n pass\n\n def scp_to(self, src, dest, recursive=False):\n warnings.warn(\"scp_to is now deprecated. Please use copy_to\")\n self.copy_to(src, dest)\n\n @check_ssh\n def copy_to(self, src, dest):\n if self.isdir(dest):\n # dest is an existing directory, so assuming src looks like path/to/src_name,\n # in this case we'll copy as:\n # path/to/src_name -> dest/src_name\n dest = self._re_anchor_basename(src, dest)\n\n if os.path.isfile(src):\n # local to remote\n self.sftp_client.put(src, dest)\n elif os.path.isdir(src):\n # we can now assume dest path looks like: path_that_exists/new_directory\n self.mkdir(dest)\n\n # for obj in `ls src`, if it's a file, copy with copy_file_from, elif its a directory, call again\n for obj in os.listdir(src):\n obj_path = os.path.join(src, obj)\n if os.path.isfile(obj_path) or os.path.isdir(obj_path):\n self.copy_to(obj_path, dest)\n else:\n # TODO what about uncopyable file types?\n pass\n\n @check_ssh\n def islink(self, path):\n try:\n # stat should follow symlinks\n path_stat = self.sftp_client.lstat(path)\n return stat.S_ISLNK(path_stat.st_mode)\n except Exception:\n return False\n\n @check_ssh\n def isdir(self, path):\n try:\n # stat should follow symlinks\n path_stat = self.sftp_client.stat(path)\n return stat.S_ISDIR(path_stat.st_mode)\n except Exception:\n return False\n\n @check_ssh\n def exists(self, path):\n \"\"\"Test that the path exists, but don't follow symlinks.\"\"\"\n try:\n # stat follows symlinks and tries to stat the actual file\n self.sftp_client.lstat(path)\n return True\n except IOError:\n return False\n\n @check_ssh\n def isfile(self, path):\n \"\"\"Imitates semantics of os.path.isfile\n\n :param path: Path to the thing to check\n :return: True if path is a file or a symlink to a file, else False. Note False can mean path does not exist.\n \"\"\"\n try:\n # stat should follow symlinks\n path_stat = self.sftp_client.stat(path)\n return stat.S_ISREG(path_stat.st_mode)\n except Exception:\n return False\n\n def open(self, path, mode=\"r\"):\n return self.sftp_client.open(path, mode)\n\n @check_ssh\n def create_file(self, path, contents):\n \"\"\"Create file at path, with the given contents.\n\n If the path already exists, it will be overwritten.\n \"\"\"\n # TODO: what should semantics be if path exists? what actually happens if it already exists?\n # TODO: what happens if the base part of the path does not exist?\n\n with self.sftp_client.open(path, \"w\") as f:\n f.write(contents)\n\n _DEFAULT_PERMISSIONS = int(\"755\", 8)\n\n @check_ssh\n def mkdir(self, path, mode=_DEFAULT_PERMISSIONS):\n self.sftp_client.mkdir(path, mode)\n\n def mkdirs(self, path, mode=_DEFAULT_PERMISSIONS):\n self.ssh(\"mkdir -p %s && chmod %o %s\" % (path, mode, path))\n\n def remove(self, path, allow_fail=False):\n \"\"\"Remove the given file or directory\"\"\"\n\n if allow_fail:\n cmd = \"rm -rf %s\" % path\n else:\n cmd = \"rm -r %s\" % path\n\n self.ssh(cmd, allow_fail=allow_fail)\n\n @contextmanager\n def monitor_log(self, log):\n \"\"\"\n Context manager that returns an object that helps you wait for events to\n occur in a log. This checks the size of the log at the beginning of the\n block and makes a helper object available with convenience methods for\n checking or waiting for a pattern to appear in the log. This will commonly\n be used to start a process, then wait for a log message indicating the\n process is in a ready state.\n\n See ``LogMonitor`` for more usage information.\n \"\"\"\n try:\n offset = int(self.ssh_output(\"wc -c %s\" % log).split()[0])\n except Exception:\n offset = 0\n yield LogMonitor(self, log, offset)\n\n\nclass SSHOutputIter(object):\n \"\"\"Helper class that wraps around an iterable object to provide has_next() in addition to next()\"\"\"\n\n def __init__(self, iter_obj_func, channel_file=None):\n \"\"\"\n :param iter_obj_func: A generator that returns an iterator over stdout from the remote process\n :param channel_file: A paramiko ``ChannelFile`` object\n \"\"\"\n self.iter_obj_func = iter_obj_func\n self.iter_obj = iter_obj_func()\n self.channel_file = channel_file\n\n # sentinel is used as an indicator that there is currently nothing cached\n # If self.cached is self.sentinel, then next object from ier_obj is not yet cached.\n self.sentinel = object()\n self.cached = self.sentinel\n\n def __iter__(self):\n return self\n\n def next(self):\n if self.cached is self.sentinel:\n return next(self.iter_obj)\n next_obj = self.cached\n self.cached = self.sentinel\n return next_obj\n\n __next__ = next\n\n def has_next(self, timeout_sec=None):\n \"\"\"Return True if next(iter_obj) would return another object within timeout_sec, else False.\n\n If timeout_sec is None, next(iter_obj) may block indefinitely.\n \"\"\"\n assert timeout_sec is None or self.channel_file is not None, \"should have descriptor to enforce timeout\"\n\n prev_timeout = None\n if self.cached is self.sentinel:\n if self.channel_file is not None:\n prev_timeout = self.channel_file.channel.gettimeout()\n\n # when timeout_sec is None, next(iter_obj) will block indefinitely\n self.channel_file.channel.settimeout(timeout_sec)\n try:\n self.cached = next(self.iter_obj, self.sentinel)\n except socket.timeout:\n self.iter_obj = self.iter_obj_func()\n self.cached = self.sentinel\n finally:\n if self.channel_file is not None:\n # restore preexisting timeout\n self.channel_file.channel.settimeout(prev_timeout)\n\n return self.cached is not self.sentinel\n\n\nclass LogMonitor(object):\n \"\"\"\n Helper class returned by monitor_log. Should be used as::\n\n with remote_account.monitor_log(\"/path/to/log\") as monitor:\n remote_account.ssh(\"/command/to/start\")\n monitor.wait_until(\"pattern.*to.*grep.*for\", timeout_sec=5)\n\n to run the command and then wait for the pattern to appear in the log.\n \"\"\"\n\n def __init__(self, acct, log, offset):\n self.acct = acct\n self.log = log\n self.offset = offset\n\n def wait_until(self, pattern, **kwargs):\n \"\"\"\n Wait until the specified pattern is found in the log, after the initial\n offset recorded when the LogMonitor was created. Additional keyword args\n are passed directly to ``ducktape.utils.util.wait_until``\n \"\"\"\n return wait_until(\n lambda: self.acct.ssh(\n \"tail -c +%d %s | grep '%s'\" % (self.offset + 1, self.log, pattern),\n allow_fail=True,\n )\n == 0,\n **kwargs,\n )\n\n\nclass IgnoreMissingHostKeyPolicy(MissingHostKeyPolicy):\n \"\"\"Policy for ignoring missing host keys.\n Many examples show use of AutoAddPolicy, but this clutters up the known_hosts file unnecessarily.\n \"\"\"\n\n def missing_host_key(self, client, hostname, key):\n return\n" }, { "path": "ducktape/cluster/vagrant.py", "content": "# Copyright 2014 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\n\nimport json\nimport os\nimport subprocess\n\nfrom ducktape.json_serializable import DucktapeJSONEncoder\n\nfrom .json import JsonCluster, make_remote_account\nfrom .remoteaccount import RemoteAccountSSHConfig\n\n\nclass VagrantCluster(JsonCluster):\n \"\"\"\n An implementation of Cluster that uses a set of VMs created by Vagrant. Because we need hostnames that can be\n advertised, this assumes that the Vagrant VM's name is a routeable hostname on all the hosts.\n\n - If cluster_file is specified in the constructor's kwargs (i.e. passed via command line argument --cluster-file)\n - If cluster_file exists on the filesystem, read cluster info from the file\n - Otherwise, retrieve cluster info via \"vagrant ssh-config\" from vagrant and write cluster info to cluster_file\n - Otherwise, retrieve cluster info via \"vagrant ssh-config\" from vagrant\n \"\"\"\n\n def __init__(self, *args, make_remote_account_func=make_remote_account, **kwargs) -> None:\n is_read_from_file = False\n self.ssh_exception_checks = kwargs.get(\"ssh_exception_checks\")\n cluster_file = kwargs.get(\"cluster_file\")\n if cluster_file is not None:\n try:\n cluster_json = json.load(open(os.path.abspath(cluster_file)))\n is_read_from_file = True\n except IOError:\n # It is OK if file is not found. Call vagrant ssh-info to read the cluster info.\n pass\n\n if not is_read_from_file:\n cluster_json = {\"nodes\": self._get_nodes_from_vagrant(make_remote_account_func)}\n\n super(VagrantCluster, self).__init__(\n cluster_json,\n *args,\n make_remote_account_func=make_remote_account_func,\n **kwargs,\n )\n\n # If cluster file is specified but the cluster info is not read from it, write the cluster info into the file\n if not is_read_from_file and cluster_file is not None:\n nodes = [\n {\n \"ssh_config\": node_account.ssh_config,\n \"externally_routable_ip\": node_account.externally_routable_ip,\n }\n for node_account in self._available_accounts\n ]\n cluster_json[\"nodes\"] = nodes\n with open(cluster_file, \"w+\") as fd:\n json.dump(\n cluster_json,\n fd,\n cls=DucktapeJSONEncoder,\n indent=2,\n separators=(\",\", \": \"),\n sort_keys=True,\n )\n\n # Release any ssh clients used in querying the nodes for metadata\n for node_account in self._available_accounts:\n node_account.close()\n\n def _get_nodes_from_vagrant(self, make_remote_account_func):\n ssh_config_info, error = self._vagrant_ssh_config()\n\n nodes = []\n node_info_arr = ssh_config_info.split(\"\\n\\n\")\n node_info_arr = [ninfo.strip() for ninfo in node_info_arr if ninfo.strip()]\n\n for ninfo in node_info_arr:\n ssh_config = RemoteAccountSSHConfig.from_string(ninfo)\n\n account = None\n try:\n account = make_remote_account_func(ssh_config, ssh_exception_checks=self.ssh_exception_checks)\n externally_routable_ip = account.fetch_externally_routable_ip()\n finally:\n if account:\n account.close()\n del account\n\n nodes.append(\n {\n \"ssh_config\": ssh_config.to_json(),\n \"externally_routable_ip\": externally_routable_ip,\n }\n )\n\n return nodes\n\n def _vagrant_ssh_config(self):\n ssh_config_info, error = subprocess.Popen(\n \"vagrant ssh-config\",\n shell=True,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n close_fds=True,\n # Force to text mode in py2/3 compatible way\n universal_newlines=True,\n ).communicate()\n return ssh_config_info, error\n" }, { "path": "ducktape/cluster/windows_remoteaccount.py", "content": "# Copyright 2014 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport base64\nimport logging\nimport os\n\nimport boto3\nimport winrm\nfrom botocore.exceptions import ClientError\nfrom Crypto.Cipher import PKCS1_v1_5\nfrom Crypto.PublicKey import RSA\n\nfrom ducktape.cluster.consts import WINDOWS\nfrom ducktape.cluster.remoteaccount import RemoteAccount, RemoteCommandError\n\n\nclass WindowsRemoteAccount(RemoteAccount):\n \"\"\"\n Windows remote accounts are currently only supported in EC2. See ``_setup_winrm()`` for how the WinRM password\n is fetched, which is currently specific to AWS.\n\n The Windows AMI needs to also have an SSH server running to support SSH commands, SCP, and rsync.\n \"\"\"\n\n WINRM_USERNAME = \"Administrator\"\n\n def __init__(self, *args, **kwargs):\n super(WindowsRemoteAccount, self).__init__(*args, **kwargs)\n self.os = WINDOWS\n self._winrm_client = None\n\n @property\n def winrm_client(self):\n # TODO: currently this only works in AWS EC2 provisioned by Vagrant. Add support for other environments.\n\n # check if winrm has already been setup. If yes, return immediately.\n if self._winrm_client:\n return self._winrm_client\n\n # first get the instance ID of this machine from Vagrant's metadata.\n ec2_instance_id_path = os.path.join(os.getcwd(), \".vagrant\", \"machines\", self.ssh_config.host, \"aws\", \"id\")\n instance_id_file = None\n try:\n instance_id_file = open(ec2_instance_id_path, \"r\")\n ec2_instance_id = instance_id_file.read().strip()\n if not ec2_instance_id or ec2_instance_id == \"\":\n raise Exception\n except Exception:\n raise Exception(\"Could not extract EC2 instance ID from local file: %s\" % ec2_instance_id_path)\n finally:\n if instance_id_file:\n instance_id_file.close()\n\n self._log(logging.INFO, \"Found EC2 instance id: %s\" % ec2_instance_id)\n\n # then get the encrypted password.\n client = boto3.client(\"ec2\")\n try:\n response = client.get_password_data(InstanceId=ec2_instance_id)\n except ClientError as ce:\n if \"InvalidInstanceID.NotFound\" in str(ce):\n raise Exception(\n \"The instance id '%s' couldn't be found. Is the correct AWS region configured?\" % ec2_instance_id\n )\n else:\n raise ce\n\n self._log(\n logging.INFO,\n \"Fetched encrypted winrm password and will decrypt with private key: %s\" % self.ssh_config.identityfile,\n )\n\n # then decrypt the password using the private key.\n key_file = None\n try:\n key_file = open(self.ssh_config.identityfile, \"r\")\n key = key_file.read()\n rsa_key = RSA.importKey(key)\n cipher = PKCS1_v1_5.new(rsa_key)\n winrm_password = cipher.decrypt(base64.b64decode(response[\"PasswordData\"]), None)\n self._winrm_client = winrm.Session(\n self.ssh_config.hostname,\n auth=(WindowsRemoteAccount.WINRM_USERNAME, winrm_password),\n )\n finally:\n if key_file:\n key_file.close()\n\n return self._winrm_client\n\n def fetch_externally_routable_ip(self, is_aws=None):\n # EC2 windows machines aren't given an externally routable IP. Use the hostname instead.\n return self.ssh_config.hostname\n\n def run_winrm_command(self, cmd, allow_fail=False):\n self._log(logging.DEBUG, \"Running winrm command: %s\" % cmd)\n result = self.winrm_client.run_cmd(cmd)\n if not allow_fail and result.status_code != 0:\n raise RemoteCommandError(self, cmd, result.status_code, result.std_err)\n return result.status_code\n" }, { "path": "ducktape/command_line/__init__.py", "content": "" }, { "path": "ducktape/command_line/defaults.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os\n\n\nclass ConsoleDefaults(object):\n # Directory for project-specific ducktape configs and runtime data\n DUCKTAPE_DIR = \".ducktape\"\n\n # Store various bookkeeping data here\n METADATA_DIR = os.path.join(DUCKTAPE_DIR, \"metadata\")\n\n # Default path, relative to current project directory, to the project's ducktape config file\n PROJECT_CONFIG_FILE = os.path.join(DUCKTAPE_DIR, \"config\")\n\n # Default path to the user-specific config file\n USER_CONFIG_FILE = os.path.join(\"~\", DUCKTAPE_DIR, \"config\")\n\n # Default cluster implementation\n CLUSTER_TYPE = \"ducktape.cluster.vagrant.VagrantCluster\"\n\n # Default path, relative to current project directory, to the cluster file\n CLUSTER_FILE = os.path.join(DUCKTAPE_DIR, \"cluster.json\")\n\n # Track the last-used session_id here\n SESSION_ID_FILE = os.path.join(METADATA_DIR, \"session_id\")\n\n # Folders with test reports, logs, etc all are created in this directory\n RESULTS_ROOT_DIRECTORY = \"./results\"\n\n SESSION_LOG_FORMATTER = \"[%(levelname)s:%(asctime)s]: %(message)s\"\n TEST_LOG_FORMATTER = \"[%(levelname)-5s - %(asctime)s - %(module)s - %(funcName)s - lineno:%(lineno)s]: %(message)s\"\n\n # Log this to indicate a test is misbehaving to help end user find which test is at fault\n BAD_TEST_MESSAGE = \"BAD_TEST\"\n\n # Ducktape will try to pick a random open port in the range [TEST_DRIVER_PORT_MIN, TEST_DRIVER_PORT_MAX]\n # this range is *inclusive*\n TEST_DRIVER_MIN_PORT = 5556\n TEST_DRIVER_MAX_PORT = 5656\n" }, { "path": "ducktape/command_line/main.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import print_function\n\nimport importlib\nimport json\nimport os\nimport random\nimport sys\nimport traceback\n\nfrom ducktape.command_line.defaults import ConsoleDefaults\nfrom ducktape.command_line.parse_args import parse_args\nfrom ducktape.tests.loader import LoaderException, TestLoader\nfrom ducktape.tests.loggermaker import close_logger\nfrom ducktape.tests.reporter import (\n FailedTestSymbolReporter,\n HTMLSummaryReporter,\n JSONReporter,\n JUnitReporter,\n SimpleFileSummaryReporter,\n SimpleStdoutSummaryReporter,\n)\nfrom ducktape.tests.runner import TestRunner\nfrom ducktape.tests.session import (\n SessionContext,\n SessionLoggerMaker,\n generate_results_dir,\n generate_session_id,\n)\nfrom ducktape.utils import persistence\nfrom ducktape.utils.local_filesystem_utils import mkdir_p\nfrom ducktape.utils.util import load_function\n\n\ndef get_user_defined_globals(globals_str):\n \"\"\"Parse user-defined globals into an immutable dict using globals_str\n\n :param globals_str Either a file, in which case, attempt to open the file and parse the contents as JSON,\n or a JSON string representing a JSON object. The parsed JSON must represent a collection of key-value pairs,\n i.e. a python dict.\n :return dict containing user-defined global variables\n \"\"\"\n if globals_str is None:\n return persistence.make_dict()\n\n from_file = False\n if os.path.isfile(globals_str):\n # The string appears to be a file, so try loading JSON from file\n # This may raise an IOError if the file can't be read or a ValueError if the contents of the file\n # cannot be parsed.\n user_globals = json.loads(open(globals_str, \"r\").read())\n from_file = True\n else:\n try:\n # try parsing directly as json if it doesn't seem to be a file\n user_globals = json.loads(globals_str)\n except ValueError as ve:\n message = str(ve)\n message += \"\\nglobals parameter %s is neither valid JSON nor a valid path to a JSON file.\" % globals_str\n raise ValueError(message)\n\n # Now check that the parsed JSON is a dictionary\n if not isinstance(user_globals, dict):\n if from_file:\n message = \"The JSON contained in file %s must parse to a dict. \" % globals_str\n else:\n message = \"JSON string referred to by globals parameter must parse to a dict. \"\n message += \"I.e. the contents of the JSON must be an object, not an array or primitive. \"\n message += \"Instead found %s, which parsed to %s\" % (\n str(user_globals),\n type(user_globals),\n )\n\n raise ValueError(message)\n\n # create the immutable dict\n return persistence.make_dict(**user_globals)\n\n\ndef setup_results_directory(new_results_dir):\n \"\"\"Make directory in which results will be stored\"\"\"\n if os.path.exists(new_results_dir):\n raise Exception(\"A file or directory at %s already exists. Exiting without overwriting.\" % new_results_dir)\n mkdir_p(new_results_dir)\n\n\ndef update_latest_symlink(results_root, new_results_dir):\n \"\"\"Create or update symlink \"latest\" which points to the new test results directory\"\"\"\n latest_test_dir = os.path.join(results_root, \"latest\")\n if os.path.islink(latest_test_dir):\n os.unlink(latest_test_dir)\n os.symlink(new_results_dir, latest_test_dir)\n\n\ndef main():\n \"\"\"Ducktape entry point. This contains top level logic for ducktape command-line program which does the following:\n\n Discover tests\n Initialize cluster for distributed services\n Run tests\n Report a summary of all results\n \"\"\"\n args_dict = parse_args(sys.argv[1:])\n\n injected_args = None\n if args_dict[\"parameters\"]:\n try:\n injected_args = json.loads(args_dict[\"parameters\"])\n except ValueError as e:\n print(\"parameters are not valid json: \" + str(e))\n sys.exit(1)\n\n args_dict[\"globals\"] = get_user_defined_globals(args_dict.get(\"globals\"))\n\n # Make .ducktape directory where metadata such as the last used session_id is stored\n if not os.path.isdir(ConsoleDefaults.METADATA_DIR):\n os.makedirs(ConsoleDefaults.METADATA_DIR)\n\n # Generate a shared 'global' identifier for this test run and create the directory\n # in which all test results will be stored\n session_id = generate_session_id(ConsoleDefaults.SESSION_ID_FILE)\n results_dir = generate_results_dir(args_dict[\"results_root\"], session_id)\n setup_results_directory(results_dir)\n\n session_context = SessionContext(session_id=session_id, results_dir=results_dir, **args_dict)\n session_logger = SessionLoggerMaker(session_context).logger\n for k, v in args_dict.items():\n session_logger.debug(\"Configuration: %s=%s\", k, v)\n\n # Discover and load tests to be run\n loader = TestLoader(\n session_context,\n session_logger,\n repeat=args_dict[\"repeat\"],\n injected_args=injected_args,\n subset=args_dict[\"subset\"],\n subsets=args_dict[\"subsets\"],\n historical_report=args_dict[\"historical_report\"],\n )\n try:\n tests = loader.load(args_dict[\"test_path\"], excluded_test_symbols=args_dict[\"exclude\"])\n except LoaderException as e:\n print(\"Failed while trying to discover tests: {}\".format(e))\n sys.exit(1)\n\n expected_test_count = len(tests)\n session_logger.info(f\"Discovered {expected_test_count} tests to run\")\n\n if args_dict[\"collect_only\"]:\n print(\"Collected %d tests:\" % expected_test_count)\n for test in tests:\n print(\" \" + str(test))\n sys.exit(0)\n\n if args_dict[\"collect_num_nodes\"]:\n total_nodes = sum(test.expected_num_nodes for test in tests)\n print(total_nodes)\n sys.exit(0)\n\n if args_dict[\"sample\"]:\n print(\"Running a sample of %d tests\" % args_dict[\"sample\"])\n try:\n tests = random.sample(tests, args_dict[\"sample\"])\n except ValueError as e:\n if args_dict[\"sample\"] > len(tests):\n print(\n \"sample size %d greater than number of tests %d; running all tests\"\n % (args_dict[\"sample\"], len(tests))\n )\n else:\n print(\"invalid sample size (%s), running all tests\" % e)\n\n # Initializing the cluster is slow, so do so only if\n # tests are sure to be run\n try:\n (cluster_mod_name, cluster_class_name) = args_dict[\"cluster\"].rsplit(\".\", 1)\n cluster_mod = importlib.import_module(cluster_mod_name)\n cluster_class = getattr(cluster_mod, cluster_class_name)\n\n cluster_kwargs = {\"cluster_file\": args_dict[\"cluster_file\"]}\n checker_function_names = args_dict[\"ssh_checker_function\"]\n if checker_function_names:\n checkers = [load_function(func_path) for func_path in checker_function_names]\n if checkers:\n cluster_kwargs[\"ssh_exception_checks\"] = checkers\n cluster = cluster_class(**cluster_kwargs)\n for ctx in tests:\n # Note that we're attaching a reference to cluster\n # only after test context objects have been instantiated\n ctx.cluster = cluster\n except Exception:\n print(\"Failed to load cluster: \", str(sys.exc_info()[0]))\n print(traceback.format_exc(limit=16))\n sys.exit(1)\n\n # Run the tests\n deflake_num = args_dict[\"deflake\"]\n if deflake_num < 1:\n session_logger.warning(\"specified number of deflake runs specified to be less than 1, running without deflake.\")\n deflake_num = max(1, deflake_num)\n runner = TestRunner(cluster, session_context, session_logger, tests, deflake_num)\n test_results = runner.run_all_tests()\n\n # Report results\n reporters = [\n SimpleStdoutSummaryReporter(test_results),\n SimpleFileSummaryReporter(test_results),\n HTMLSummaryReporter(test_results, expected_test_count),\n JSONReporter(test_results),\n JUnitReporter(test_results),\n FailedTestSymbolReporter(test_results),\n ]\n\n for r in reporters:\n r.report()\n\n update_latest_symlink(args_dict[\"results_root\"], results_dir)\n\n if len(test_results) < expected_test_count:\n session_logger.warning(\n f\"All tests were NOT run. Expected {expected_test_count} tests, only {len(test_results)} were run.\"\n )\n close_logger(session_logger)\n sys.exit(1)\n\n close_logger(session_logger)\n if not test_results.get_aggregate_success():\n # Non-zero exit if at least one test failed\n sys.exit(1)\n" }, { "path": "ducktape/command_line/parse_args.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import print_function\n\nimport argparse\nimport itertools\nimport os\nimport sys\nfrom typing import Dict, List, Optional, Union\n\nfrom ducktape.command_line.defaults import ConsoleDefaults\nfrom ducktape.utils.util import ducktape_version\n\n\ndef create_ducktape_parser() -> argparse.ArgumentParser:\n parser = argparse.ArgumentParser(description=\"Discover and run your tests\")\n parser.add_argument(\n \"test_path\",\n metavar=\"test_path\",\n type=str,\n nargs=\"*\",\n default=[os.getcwd()],\n help=\"One or more test identifiers or test suite paths to execute\",\n )\n parser.add_argument(\n \"--exclude\",\n type=str,\n nargs=\"*\",\n default=None,\n help=\"one or more space-delimited strings indicating which tests to exclude\",\n )\n parser.add_argument(\n \"--collect-only\",\n action=\"store_true\",\n help=\"display collected tests, but do not run.\",\n )\n parser.add_argument(\n \"--collect-num-nodes\",\n action=\"store_true\",\n help=\"display total number of nodes requested by all tests, but do not run anything.\",\n )\n parser.add_argument(\"--debug\", action=\"store_true\", help=\"pipe more verbose test output to stdout.\")\n parser.add_argument(\n \"--config-file\",\n action=\"store\",\n default=ConsoleDefaults.USER_CONFIG_FILE,\n help=\"path to project-specific configuration file.\",\n )\n parser.add_argument(\n \"--compress\",\n action=\"store_true\",\n help=\"compress remote logs before collection.\",\n )\n parser.add_argument(\n \"--cluster\",\n action=\"store\",\n default=ConsoleDefaults.CLUSTER_TYPE,\n help=\"cluster class to use to allocate nodes for tests.\",\n )\n parser.add_argument(\n \"--default-num-nodes\",\n action=\"store\",\n type=int,\n default=None,\n help=\"Global hint for cluster usage. A test without the @cluster annotation will \"\n \"default to this value for expected cluster usage.\",\n )\n parser.add_argument(\n \"--cluster-file\",\n action=\"store\",\n default=None,\n help=\"path to a json file which provides information needed to initialize a json cluster. \"\n \"The file is used to read/write cached cluster info if \"\n \"cluster is ducktape.cluster.vagrant.VagrantCluster.\",\n )\n parser.add_argument(\n \"--results-root\",\n action=\"store\",\n default=ConsoleDefaults.RESULTS_ROOT_DIRECTORY,\n help=\"path to custom root results directory. Running ducktape with this root \"\n \"specified will result in new test results being stored in a subdirectory of \"\n \"this root directory.\",\n )\n parser.add_argument(\"--exit-first\", action=\"store_true\", help=\"exit after first failure\")\n parser.add_argument(\n \"--no-teardown\",\n action=\"store_true\",\n help=\"don't kill running processes or remove log files when a test has finished running. \"\n \"This is primarily useful for test developers who want to interact with running \"\n \"services after a test has run.\",\n )\n parser.add_argument(\"--version\", action=\"store_true\", help=\"display version\")\n parser.add_argument(\n \"--parameters\",\n action=\"store\",\n help=\"inject these arguments into the specified test(s). Specify parameters as a JSON string.\",\n )\n parser.add_argument(\n \"--globals\",\n action=\"store\",\n help=\"user-defined globals go here. \"\n \"This can be a file containing a JSON object, or a string representing a JSON object.\",\n )\n parser.add_argument(\n \"--max-parallel\",\n action=\"store\",\n type=int,\n default=1,\n help=\"Upper bound on number of tests run simultaneously.\",\n )\n parser.add_argument(\n \"--repeat\",\n action=\"store\",\n type=int,\n default=1,\n help=\"Use this flag to repeat all discovered tests the given number of times.\",\n )\n parser.add_argument(\n \"--subsets\",\n action=\"store\",\n type=int,\n default=1,\n help=\"Number of subsets of tests to statically break the tests into to allow for parallel \"\n \"execution without coordination between test runner processes.\",\n )\n parser.add_argument(\n \"--subset\",\n action=\"store\",\n type=int,\n default=0,\n help=\"Which subset of the tests to run, based on the breakdown using the parameter for --subsets\",\n )\n parser.add_argument(\n \"--historical-report\",\n action=\"store\",\n type=str,\n help=\"URL of a JSON report file containing stats from a previous test run. If specified, \"\n \"this will be used when creating subsets of tests to divide evenly by total run time \"\n \"instead of by number of tests.\",\n )\n parser.add_argument(\n \"--skip-nodes-allocation\",\n action=\"store_true\",\n help=\"Use this flag to skip allocating \"\n \"nodes for services. Can be used when running specific tests on a running platform\",\n )\n parser.add_argument(\n \"--sample\",\n action=\"store\",\n type=int,\n help=\"The size of a random test sample to run\",\n )\n parser.add_argument(\n \"--fail-bad-cluster-utilization\",\n action=\"store_true\",\n help=\"Fail a test if the test declared that it needs more nodes than it actually used. \"\n \"E.g. if the test had `@cluster(num_nodes=10)` annotation, \"\n \"but never used more than 5 nodes during its execution.\",\n )\n parser.add_argument(\n \"--fail-greedy-tests\",\n action=\"store_true\",\n help=\"Fail a test if it has no @cluster annotation \"\n \"or if @cluster annotation is empty. \"\n \"You can still specify 0-sized cluster explicitly using either num_nodes=0 \"\n \"or cluster_spec=ClusterSpec.empty()\",\n )\n parser.add_argument(\n \"--test-runner-timeout\",\n action=\"store\",\n type=int,\n default=1800000,\n help=\"Amount of time in milliseconds between test communicating between the test runner\"\n \" before a timeout error occurs. Default is 30 minutes\",\n )\n (\n parser.add_argument(\n \"--ssh-checker-function\",\n action=\"store\",\n type=str,\n nargs=\"+\",\n help=\"Python module path(s) to a function that takes an exception and a remote account\"\n \" that will be called when an ssh error occurs, this can give some \"\n \"validation or better logging when an ssh error occurs. Specify any \"\n \"number of module paths after this flag to be called.\",\n ),\n )\n parser.add_argument(\n \"--deflake\",\n action=\"store\",\n type=int,\n default=1,\n help=\"the number of times a failed test should be ran in total (including its initial run) \"\n \"to determine flakyness. When not present, deflake will not be used, \"\n \"and a test will be marked as either passed or failed. \"\n \"When enabled tests will be marked as flaky if it passes on any of the reruns\",\n )\n parser.add_argument(\n \"--enable-jvm-logs\",\n action=\"store_true\",\n help=\"Enable automatic JVM log collection for Java-based services (Kafka, ZooKeeper, Connect, etc.)\",\n )\n return parser\n\n\ndef get_user_config_file(args: List[str]) -> str:\n \"\"\"Helper function to get specified (or default) user config file.\n :return Filename which is the path to the config file.\n \"\"\"\n parser = create_ducktape_parser()\n config_file = vars(parser.parse_args(args))[\"config_file\"]\n assert config_file is not None\n return os.path.expanduser(config_file)\n\n\ndef config_file_to_args_list(config_file):\n \"\"\"Parse in contents of config file, and return a list of command-line options parseable by the ducktape parser.\n\n Skip whitespace lines and comments (lines prefixed by \"#\")\n \"\"\"\n if config_file is None:\n raise RuntimeError(\"config_file is None\")\n\n # Read in configuration, but ignore empty lines and comments\n config_lines = [\n line for line in open(config_file).readlines() if (len(line.strip()) > 0 and line.lstrip()[0] != \"#\")\n ]\n\n return list(itertools.chain(*[line.split() for line in config_lines]))\n\n\ndef parse_non_default_args(parser: argparse.ArgumentParser, defaults: dict, args: list) -> dict:\n \"\"\"\n Parse and remove default args from a list of args, and return the dict of the parsed args.\n \"\"\"\n parsed_args = vars(parser.parse_args(args))\n\n # remove defaults\n for key, value in defaults.items():\n if parsed_args[key] == value:\n del parsed_args[key]\n\n return parsed_args\n\n\ndef parse_args(\n args: List[str],\n) -> Dict[str, Optional[Union[List[str], bool, str, int]]]:\n \"\"\"Parse in command-line and config file options.\n\n Command line arguments have the highest priority, then user configs specified in ~/.ducktape/config, and finally\n project configs specified in /config.\n \"\"\"\n\n parser = create_ducktape_parser()\n\n if len(args) == 0:\n # Show help if there are no arguments\n parser.print_help()\n sys.exit(0)\n\n # Collect arguments from project config file, user config file, and command line\n # later arguments supersede earlier arguments\n parsed_args_list = []\n\n # First collect all the default values\n defaults = vars(parser.parse_args([]))\n\n project_config_file = ConsoleDefaults.PROJECT_CONFIG_FILE\n # Load all non-default args from project config file.\n if os.path.exists(project_config_file):\n parsed_args_list.append(parse_non_default_args(parser, defaults, config_file_to_args_list(project_config_file)))\n\n # Load all non-default args from user config file.\n user_config_file = get_user_config_file(args)\n if os.path.exists(user_config_file):\n parsed_args_list.append(parse_non_default_args(parser, defaults, config_file_to_args_list(user_config_file)))\n\n # Load all non-default args from the command line.\n parsed_args_list.append(parse_non_default_args(parser, defaults, args))\n\n # Don't need to copy, done with the defaults dict.\n # Start with the default args, and layer on changes.\n parsed_args_dict = defaults\n for parsed_args in parsed_args_list:\n parsed_args_dict.update(parsed_args)\n\n if parsed_args_dict[\"version\"]:\n print(ducktape_version())\n sys.exit(0)\n return parsed_args_dict\n" }, { "path": "ducktape/errors.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nclass DucktapeError(RuntimeError):\n pass\n\n\nclass TimeoutError(DucktapeError):\n pass\n" }, { "path": "ducktape/json_serializable.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nfrom json import JSONEncoder\n\n\nclass DucktapeJSONEncoder(JSONEncoder):\n def default(self, obj):\n if hasattr(obj, \"to_json\"):\n # to_json may return a dict or array or other naturally json serializable object\n # this allows serialization to work correctly on nested items\n return obj.to_json()\n else:\n # Let the base class default method raise the TypeError\n return JSONEncoder.default(self, obj)\n" }, { "path": "ducktape/jvm_logging.py", "content": "# Copyright 2024 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nJVM logging support for Ducktape.\n\nThis module provides automatic JVM log collection for Java-based services without requiring any\ncode changes to services or tests.\n\nPlease Note: We are prepending JVM options to the SSH cmd. If any option is injected again as part of cmd, it will\noverride the option injected from this module.\nFor example, if a test or service injects its own -Xlog options, it may override the GC logging options injected\nby this module. In practice, Services should work as expected.\n\"\"\"\n\nimport os\nimport types\n\n\nclass JVMLogger:\n \"\"\"Handles JVM logging configuration and enablement for services.\"\"\"\n\n def __init__(self, log_dir=\"/mnt/jvm_logs\"):\n \"\"\"\n Initialize JVM logger.\n\n :param log_dir: Directory for JVM logs on worker nodes\n \"\"\"\n self.log_dir = log_dir\n\n def enable_for_service(self, service):\n \"\"\"\n Enable JVM logging for a service instance. Adds JVM log definitions and helper methods to the service.\n Wraps start_node so that when nodes are started, JVM logging is set up and JVM options are injected via SSH,\n and wraps clean_node to clean up JVM logs after node cleanup.\n :param service: Service instance to enable JVM logging for\n \"\"\"\n # Store reference to JVMLogger instance for use in closures\n jvm_logger = self\n\n # Add JVM log definitions\n jvm_logs = {\n \"jvm_gc_log\": {\"path\": os.path.join(jvm_logger.log_dir, \"gc.log\"), \"collect_default\": True},\n \"jvm_stdout_stderr\": {\"path\": os.path.join(jvm_logger.log_dir, \"jvm.log\"), \"collect_default\": True},\n \"jvm_heap_dump\": {\n \"path\": os.path.join(jvm_logger.log_dir, \"heap_dump.hprof\"),\n \"collect_default\": False, # Only on failure\n },\n }\n\n # Initialize logs dict if needed\n if not hasattr(service, \"logs\") or service.logs is None:\n service.logs = {}\n\n # Merge with existing logs\n service.logs.update(jvm_logs)\n\n # Add helper methods\n service.JVM_LOG_DIR = jvm_logger.log_dir\n service.jvm_options = lambda node: jvm_logger._get_jvm_options()\n service.setup_jvm_logging = lambda node: jvm_logger._setup_on_node(node)\n service.clean_jvm_logs = lambda node: jvm_logger._cleanup_on_node(node)\n\n # Wrap start_node to automatically setup JVM logging and wrap SSH\n original_start_node = service.start_node\n\n def wrapped_start_node(self, node, *args, **kwargs):\n jvm_logger._setup_on_node(node) # Setup JVM log directory\n\n # Wrap all SSH methods to inject JDK_JAVA_OPTIONS, wrap once and keep active for entire service lifecycle\n if not hasattr(node.account, \"original_ssh\"):\n original_ssh = node.account.ssh\n original_ssh_capture = node.account.ssh_capture\n original_ssh_output = node.account.ssh_output\n\n node.account.original_ssh = original_ssh\n node.account.original_ssh_capture = original_ssh_capture\n node.account.original_ssh_output = original_ssh_output\n\n jvm_opts = jvm_logger._get_jvm_options()\n # Use env command and append to existing JDK_JAVA_OPTIONS\n env_prefix = f'env JDK_JAVA_OPTIONS=\"${{JDK_JAVA_OPTIONS:-}} {jvm_opts}\" '\n\n def wrapped_ssh(cmd, allow_fail=False):\n return original_ssh(env_prefix + cmd, allow_fail=allow_fail)\n\n def wrapped_ssh_capture(cmd, allow_fail=False, callback=None, combine_stderr=True, timeout_sec=None):\n return original_ssh_capture(\n env_prefix + cmd,\n allow_fail=allow_fail,\n callback=callback,\n combine_stderr=combine_stderr,\n timeout_sec=timeout_sec,\n )\n\n def wrapped_ssh_output(cmd, allow_fail=False, combine_stderr=True, timeout_sec=None):\n return original_ssh_output(\n env_prefix + cmd, allow_fail=allow_fail, combine_stderr=combine_stderr, timeout_sec=timeout_sec\n )\n\n node.account.ssh = wrapped_ssh\n node.account.ssh_capture = wrapped_ssh_capture\n node.account.ssh_output = wrapped_ssh_output\n\n return original_start_node(node, *args, **kwargs)\n\n # Bind the wrapper function to the service object\n service.start_node = types.MethodType(wrapped_start_node, service)\n\n # Wrap clean_node to cleanup JVM logs and restore SSH methods\n original_clean_node = service.clean_node\n\n def wrapped_clean_node(self, node, *args, **kwargs):\n result = original_clean_node(node, *args, **kwargs)\n jvm_logger._cleanup_on_node(node)\n\n # Restore original SSH methods\n if hasattr(node.account, \"original_ssh\"):\n node.account.ssh = node.account.original_ssh\n node.account.ssh_capture = node.account.original_ssh_capture\n node.account.ssh_output = node.account.original_ssh_output\n del node.account.original_ssh\n del node.account.original_ssh_capture\n del node.account.original_ssh_output\n\n return result\n\n # Bind the wrapper function to the service instance\n service.clean_node = types.MethodType(wrapped_clean_node, service)\n\n def _get_jvm_options(self):\n \"\"\"Generate JVM options string for logging.\"\"\"\n gc_log = os.path.join(self.log_dir, \"gc.log\")\n heap_dump = os.path.join(self.log_dir, \"heap_dump.hprof\")\n error_log = os.path.join(self.log_dir, \"hs_err_pid%p.log\")\n jvm_log = os.path.join(self.log_dir, \"jvm.log\")\n\n jvm_logging_opts = [\n \"-Xlog:disable\", # Suppress all default JVM console logging to prevent output pollution\n f\"-Xlog:gc*:file={gc_log}:time,uptime,level,tags\", # GC activity with timestamps\n \"-XX:+HeapDumpOnOutOfMemoryError\", # Generate heap dump on OOM\n f\"-XX:HeapDumpPath={heap_dump}\", # Heap dump file location\n f\"-Xlog:safepoint=info:file={jvm_log}:time,uptime,level,tags\", # Safepoint pause events\n f\"-Xlog:class+load=info:file={jvm_log}:time,uptime,level,tags\", # Class loading events\n f\"-XX:ErrorFile={error_log}\", # Fatal error log location (JVM crashes)\n \"-XX:NativeMemoryTracking=summary\", # Track native memory usage\n f\"-Xlog:jit+compilation=info:file={jvm_log}:time,uptime,level,tags\", # JIT compilation events\n ]\n\n return \" \".join(jvm_logging_opts)\n\n def _setup_on_node(self, node):\n \"\"\"Create JVM log directory on worker node.\"\"\"\n node.account.ssh(f\"mkdir -p {self.log_dir}\")\n node.account.ssh(f\"chmod 755 {self.log_dir}\")\n\n def _cleanup_on_node(self, node):\n \"\"\"Clean JVM logs from worker node.\"\"\"\n node.account.ssh(f\"rm -rf {self.log_dir}\", allow_fail=True)\n" }, { "path": "ducktape/mark/__init__.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ._mark import matrix # NOQA\nfrom ._mark import defaults, env, ignore, ignored, is_env, parametrize, parametrized\n\n__all__ = [\n \"matrix\",\n \"defaults\",\n \"env\",\n \"ignore\",\n \"ignored\",\n \"is_env\",\n \"parametrize\",\n \"parametrized\",\n]\n" }, { "path": "ducktape/mark/_mark.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nimport functools\nimport itertools\nimport os\n\nfrom ducktape.errors import DucktapeError\n\n\nclass Mark(object):\n \"\"\"Common base class for \"marks\" which may be applied to test functions/methods.\"\"\"\n\n @staticmethod\n def mark(fun, mark):\n \"\"\"Attach a tag indicating that fun has been marked with the given mark\n\n Marking fun updates it with two attributes:\n\n - marks: a list of mark objects applied to the function. These may be strings or objects subclassing Mark\n we use a list because in some cases, it is useful to preserve ordering.\n - mark_names: a set of names of marks applied to the function\n \"\"\"\n # Update fun.marks\n if hasattr(fun, \"marks\"):\n fun.marks.append(mark)\n else:\n fun.__dict__[\"marks\"] = [mark]\n\n # Update fun.mark_names\n if hasattr(fun, \"mark_names\"):\n fun.mark_names.add(mark.name)\n else:\n fun.__dict__[\"mark_names\"] = {mark.name}\n\n @staticmethod\n def marked(f, mark):\n if f is None:\n return False\n\n if not hasattr(f, \"mark_names\"):\n return False\n\n return mark.name in f.mark_names\n\n @staticmethod\n def clear_marks(f):\n if not hasattr(f, \"marks\"):\n return\n\n del f.__dict__[\"marks\"]\n del f.__dict__[\"mark_names\"]\n\n @property\n def name(self):\n return \"MARK\"\n\n def apply(self, seed_context, context_list):\n raise NotImplementedError(\"Subclasses should implement apply\")\n\n def __eq__(self, other):\n if not isinstance(self, type(other)):\n return False\n\n return self.name == other.name\n\n\nclass Ignore(Mark):\n \"\"\"Ignore a specific parametrization of test.\"\"\"\n\n def __init__(self, **kwargs):\n # Ignore tests with injected_args matching self.injected_args\n self.injected_args = kwargs\n\n @property\n def name(self):\n return \"IGNORE\"\n\n def apply(self, seed_context, context_list):\n assert len(context_list) > 0, \"ignore annotation is not being applied to any test cases\"\n for ctx in context_list:\n ctx.ignore = ctx.ignore or self.injected_args is None or self.injected_args == ctx.injected_args\n return context_list\n\n def __eq__(self, other):\n return super(Ignore, self).__eq__(other) and self.injected_args == other.injected_args\n\n\nclass IgnoreAll(Ignore):\n \"\"\"This mark signals to ignore all parametrizations of a test.\"\"\"\n\n def __init__(self):\n super(IgnoreAll, self).__init__()\n self.injected_args = None\n\n\nclass Matrix(Mark):\n \"\"\"Parametrize with a matrix of arguments.\n Assume each values in self.injected_args is iterable\n \"\"\"\n\n def __init__(self, **kwargs):\n self.injected_args = kwargs\n for k in self.injected_args:\n try:\n iter(self.injected_args[k])\n except TypeError as te:\n raise DucktapeError(\"Expected all values in @matrix decorator to be iterable: \" + str(te))\n\n @property\n def name(self):\n return \"MATRIX\"\n\n def apply(self, seed_context, context_list):\n for injected_args in cartesian_product_dict(self.injected_args):\n injected_fun = _inject(**injected_args)(seed_context.function)\n context_list.insert(0, seed_context.copy(function=injected_fun, injected_args=injected_args))\n\n return context_list\n\n def __eq__(self, other):\n return super(Matrix, self).__eq__(other) and self.injected_args == other.injected_args\n\n\nclass Defaults(Mark):\n \"\"\"Parametrize with a default matrix of arguments on existing parametrizations.\n Assume each values in self.injected_args is iterable\n \"\"\"\n\n def __init__(self, **kwargs):\n self.injected_args = kwargs\n for k in self.injected_args:\n try:\n iter(self.injected_args[k])\n except TypeError as te:\n raise DucktapeError(\"Expected all values in @defaults decorator to be iterable: \" + str(te))\n\n @property\n def name(self):\n return \"DEFAULTS\"\n\n def apply(self, seed_context, context_list):\n new_context_list = []\n if context_list:\n for ctx in context_list:\n for injected_args in cartesian_product_dict(\n {arg: self.injected_args[arg] for arg in self.injected_args if arg not in ctx.injected_args}\n ):\n injected_args.update(ctx.injected_args)\n injected_fun = _inject(**injected_args)(seed_context.function)\n new_context = seed_context.copy(\n function=injected_fun,\n injected_args=injected_args,\n cluster_use_metadata=ctx.cluster_use_metadata,\n )\n new_context_list.insert(0, new_context)\n else:\n for injected_args in cartesian_product_dict(self.injected_args):\n injected_fun = _inject(**injected_args)(seed_context.function)\n new_context_list.insert(\n 0,\n seed_context.copy(function=injected_fun, injected_args=injected_args),\n )\n\n return new_context_list\n\n def __eq__(self, other):\n return super(Defaults, self).__eq__(other) and self.injected_args == other.injected_args\n\n\nclass Parametrize(Mark):\n \"\"\"Parametrize a test function\"\"\"\n\n def __init__(self, **kwargs):\n self.injected_args = kwargs\n\n @property\n def name(self):\n return \"PARAMETRIZE\"\n\n def apply(self, seed_context, context_list):\n injected_fun = _inject(**self.injected_args)(seed_context.function)\n context_list.insert(\n 0,\n seed_context.copy(function=injected_fun, injected_args=self.injected_args),\n )\n return context_list\n\n def __eq__(self, other):\n return super(Parametrize, self).__eq__(other) and self.injected_args == other.injected_args\n\n\nclass Env(Mark):\n def __init__(self, **kwargs):\n self.injected_args = kwargs\n self.should_ignore = any(os.environ.get(key) != value for key, value in kwargs.items())\n\n @property\n def name(self):\n return \"ENV\"\n\n def apply(self, seed_context, context_list):\n for ctx in context_list:\n ctx.ignore = ctx.ignore or self.should_ignore\n\n return context_list\n\n def __eq__(self, other):\n return super(Env, self).__eq__(other) and self.injected_args == other.injected_args\n\n\nPARAMETRIZED = Parametrize()\nMATRIX = Matrix()\nDEFAULTS = Defaults()\nIGNORE = Ignore()\nENV = Env()\n\n\ndef _is_parametrize_mark(m):\n return m.name == PARAMETRIZED.name or m.name == MATRIX.name or m.name == DEFAULTS.name\n\n\ndef parametrized(f):\n \"\"\"Is this function or object decorated with @parametrize or @matrix?\"\"\"\n return Mark.marked(f, PARAMETRIZED) or Mark.marked(f, MATRIX) or Mark.marked(f, DEFAULTS)\n\n\ndef ignored(f):\n \"\"\"Is this function or object decorated with @ignore?\"\"\"\n return Mark.marked(f, IGNORE)\n\n\ndef is_env(f):\n return Mark.marked(f, ENV)\n\n\ndef cartesian_product_dict(d):\n \"\"\"Return the \"cartesian product\" of this dictionary's values.\n d is assumed to be a dictionary, where each value in the dict is a list of values\n\n Example::\n\n {\n \"x\": [1, 2],\n \"y\": [\"a\", \"b\"]\n }\n\n expand this into a list of dictionaries like so:\n\n [\n {\n \"x\": 1,\n \"y\": \"a\"\n },\n {\n \"x\": 1,\n \"y\": \"b\"\n },\n {\n \"x\": 2,\n \"y\": \"a\"\n },\n {\n \"x\": 2,\n \"y\", \"b\"\n }\n ]\n \"\"\"\n # Establish an ordering of the keys\n key_list = [k for k in d.keys()]\n\n expanded = []\n values_list = [d[k] for k in key_list] # list of lists\n for v in itertools.product(*values_list):\n # Iterate through the cartesian product of the lists of values\n # One dictionary per element in this cartesian product\n new_dict = {}\n for i in range(len(key_list)):\n new_dict[key_list[i]] = v[i]\n expanded.append(new_dict)\n return expanded\n\n\ndef matrix(**kwargs):\n \"\"\"Function decorator used to parametrize with a matrix of values.\n Decorating a function or method with ``@matrix`` marks it with the Matrix mark. When expanded using the\n ``MarkedFunctionExpander``, it yields a list of TestContext objects, one for every possible combination\n of arguments.\n\n Example::\n\n @matrix(x=[1, 2], y=[-1, -2])\n def g(x, y):\n print \"x = %s, y = %s\" % (x, y)\n\n for ctx in MarkedFunctionExpander(..., function=g, ...).expand():\n ctx.function()\n\n # output:\n # x = 1, y = -1\n # x = 1, y = -2\n # x = 2, y = -1\n # x = 2, y = -2\n \"\"\"\n\n def parametrizer(f):\n Mark.mark(f, Matrix(**kwargs))\n return f\n\n return parametrizer\n\n\ndef defaults(**kwargs):\n \"\"\"Function decorator used to parametrize with a default matrix of values.\n Decorating a function or method with ``@defaults`` marks it with the Defaults mark. When expanded using the\n ``MarkedFunctionExpander``, it yields a list of TestContext objects, one for every possible combination\n of defaults combined with ``@matrix`` and ``@parametrize``. If there are overlap between defaults\n and parametrization, defaults will not be applied.\n\n Example::\n\n @defaults(z=[1, 2])\n @matrix(x=[1], y=[1, 2])\n @parametrize(x=3, y=4)\n @parametrize(x=3, y=4, z=999)\n def g(x, y, z):\n print \"x = %s, y = %s\" % (x, y)\n\n for ctx in MarkedFunctionExpander(..., function=g, ...).expand():\n ctx.function()\n\n # output:\n # x = 1, y = 1, z = 1\n # x = 1, y = 1, z = 2\n # x = 1, y = 2, z = 1\n # x = 1, y = 2, z = 2\n # x = 3, y = 4, z = 1\n # x = 3, y = 4, z = 2\n # x = 3, y = 4, z = 999\n \"\"\"\n\n def parametrizer(f):\n Mark.mark(f, Defaults(**kwargs))\n return f\n\n return parametrizer\n\n\ndef parametrize(**kwargs):\n \"\"\"Function decorator used to parametrize its arguments.\n Decorating a function or method with ``@parametrize`` marks it with the Parametrize mark.\n\n Example::\n\n @parametrize(x=1, y=2 z=-1)\n @parametrize(x=3, y=4, z=5)\n def g(x, y, z):\n print \"x = %s, y = %s, z = %s\" % (x, y, z)\n\n for ctx in MarkedFunctionExpander(..., function=g, ...).expand():\n ctx.function()\n\n # output:\n # x = 1, y = 2, z = -1\n # x = 3, y = 4, z = 5\n \"\"\"\n\n def parametrizer(f):\n Mark.mark(f, Parametrize(**kwargs))\n return f\n\n return parametrizer\n\n\ndef ignore(*args, **kwargs):\n \"\"\"\n Test method decorator which signals to the test runner to ignore a given test.\n\n Example::\n\n When no parameters are provided to the @ignore decorator, ignore all parametrizations of the test function\n\n @ignore # Ignore all parametrizations\n @parametrize(x=1, y=0)\n @parametrize(x=2, y=3)\n def the_test(...):\n ...\n\n Example::\n\n If parameters are supplied to the @ignore decorator, only ignore the parametrization with matching parameter(s)\n\n @ignore(x=2, y=3)\n @parametrize(x=1, y=0) # This test will run as usual\n @parametrize(x=2, y=3) # This test will be ignored\n def the_test(...):\n ...\n \"\"\"\n if len(args) == 1 and len(kwargs) == 0:\n # this corresponds to the usage of the decorator with no arguments\n # @ignore\n # def test_function:\n # ...\n Mark.mark(args[0], IgnoreAll())\n return args[0]\n\n # this corresponds to usage of @ignore with arguments\n def ignorer(f):\n Mark.mark(f, Ignore(**kwargs))\n return f\n\n return ignorer\n\n\ndef env(**kwargs):\n def environment(f):\n Mark.mark(f, Env(**kwargs))\n return f\n\n return environment\n\n\ndef _inject(*args, **kwargs):\n \"\"\"Inject variables into the arguments of a function or method.\n This is almost identical to decorating with functools.partial, except we also propagate the wrapped\n function's __name__.\n \"\"\"\n\n def injector(f):\n assert callable(f)\n\n @functools.wraps(f)\n def wrapper(*w_args, **w_kwargs):\n return functools.partial(f, *args, **kwargs)(*w_args, **w_kwargs)\n\n wrapper.args = args\n wrapper.kwargs = kwargs\n wrapper.function = f\n\n return wrapper\n\n return injector\n" }, { "path": "ducktape/mark/consts.py", "content": "CLUSTER_SPEC_KEYWORD = \"cluster_spec\"\nCLUSTER_SIZE_KEYWORD = \"num_nodes\"\nCLUSTER_NODE_TYPE_KEYWORD = \"node_type\"\n" }, { "path": "ducktape/mark/mark_expander.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nfrom ducktape.tests.test_context import TestContext\n\nfrom ._mark import Parametrize, _is_parametrize_mark, parametrized\n\n\nclass MarkedFunctionExpander(object):\n \"\"\"This class helps expand decorated/marked functions into a list of test context objects.\"\"\"\n\n def __init__(\n self,\n session_context=None,\n module=None,\n cls=None,\n function=None,\n file=None,\n cluster=None,\n ):\n self.seed_context = TestContext(\n session_context=session_context,\n module=module,\n cls=cls,\n function=function,\n file=file,\n cluster=cluster,\n )\n\n if parametrized(function):\n self.context_list = []\n else:\n self.context_list = [self.seed_context]\n\n def expand(self, test_parameters=None):\n \"\"\"Inspect self.function for marks, and expand into a list of test context objects useable by the test runner.\"\"\"\n f = self.seed_context.function\n\n # If the user has specified that they want to run tests with specific parameters, apply the parameters first,\n # then subsequently strip any parametrization decorators. Otherwise, everything gets applied normally.\n if test_parameters is not None:\n self.context_list = Parametrize(**test_parameters).apply(self.seed_context, self.context_list)\n\n for m in getattr(f, \"marks\", []):\n if test_parameters is None or not _is_parametrize_mark(m):\n self.context_list = m.apply(self.seed_context, self.context_list)\n\n return self.context_list\n" }, { "path": "ducktape/mark/resource.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport copy\nfrom typing import Callable, List\n\nfrom ducktape.mark._mark import Mark\nfrom ducktape.tests.test_context import TestContext\n\n\nclass ClusterUseMetadata(Mark):\n \"\"\"Provide a hint about how a given test will use the cluster.\"\"\"\n\n def __init__(self, **kwargs) -> None:\n # shallow copy\n self.metadata = copy.copy(kwargs)\n\n @property\n def name(self) -> str:\n return \"RESOURCE_HINT_CLUSTER_USE\"\n\n def apply(self, seed_context: TestContext, context_list: List[TestContext]) -> List[TestContext]:\n assert len(context_list) > 0, \"cluster use annotation is not being applied to any test cases\"\n\n for ctx in context_list:\n if not ctx.cluster_use_metadata:\n # only update if non-None and non-empty\n ctx.cluster_use_metadata = self.metadata\n return context_list\n\n\ndef cluster(**kwargs) -> Callable:\n \"\"\"Test method decorator used to provide hints about how the test will use the given cluster.\n\n If this decorator is not provided, the test will either claim all cluster resources or fail immediately,\n depending on the flags passed to ducktape.\n\n\n :Keywords used by ducktape:\n\n - ``num_nodes`` provide hint about how many nodes the test will consume\n - ``node_type`` provide hint about what type of nodes the test needs (e.g., \"large\", \"small\")\n - ``cluster_spec`` provide hint about how many nodes of each type the test will consume\n\n\n Example::\n\n # basic usage with num_nodes\n @cluster(num_nodes=10)\n def the_test(...):\n ...\n\n # usage with num_nodes and node_type\n @cluster(num_nodes=5, node_type=\"large\")\n def the_test(...):\n ...\n\n # basic usage with cluster_spec\n @cluster(cluster_spec=ClusterSpec.simple_linux(10))\n def the_test(...):\n ...\n\n # parametrized test:\n # both test cases will be marked with cluster_size of 200\n @cluster(num_nodes=200)\n @parametrize(x=1)\n @parametrize(x=2)\n def the_test(x):\n ...\n\n # test case {'x': 1} has cluster size 100, test case {'x': 2} has cluster size 200\n @cluster(num_nodes=100)\n @parametrize(x=1)\n @cluster(num_nodes=200)\n @parametrize(x=2)\n def the_test(x):\n ...\n\n \"\"\"\n\n def cluster_use_metadata_adder(f):\n Mark.mark(f, ClusterUseMetadata(**kwargs))\n return f\n\n return cluster_use_metadata_adder\n" }, { "path": "ducktape/services/__init__.py", "content": "" }, { "path": "ducktape/services/background_thread.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport threading\nimport traceback\n\nfrom ducktape.services.service import Service\n\n\nclass BackgroundThreadService(Service):\n def __init__(self, context, num_nodes=None, cluster_spec=None, *args, **kwargs):\n super(BackgroundThreadService, self).__init__(context, num_nodes, cluster_spec, *args, **kwargs)\n self.worker_threads = {}\n self.worker_errors = {}\n self.errors = \"\"\n self.lock = threading.RLock()\n\n def _protected_worker(self, idx, node):\n \"\"\"Protected worker captures exceptions and makes them available to the main thread.\n\n This gives us the ability to propagate exceptions thrown in background threads, if desired.\n \"\"\"\n try:\n self._worker(idx, node)\n except BaseException:\n with self.lock:\n self.logger.info(\"BackgroundThreadService threw exception: \")\n tb = traceback.format_exc()\n self.logger.info(tb)\n self.worker_errors[threading.currentThread().name] = tb\n if self.errors:\n self.errors += \"\\n\"\n self.errors += \"%s: %s\" % (threading.currentThread().name, tb)\n\n raise\n\n def start_node(self, node):\n idx = self.idx(node)\n\n if idx in self.worker_threads and self.worker_threads[idx].is_alive():\n raise RuntimeError(\"Cannot restart node since previous thread is still alive\")\n\n self.logger.info(\"Running %s node %d on %s\", self.service_id, idx, node.account.hostname)\n worker = threading.Thread(\n name=self.service_id + \"-worker-\" + str(idx),\n target=self._protected_worker,\n args=(idx, node),\n )\n worker.daemon = True\n worker.start()\n self.worker_threads[idx] = worker\n\n def wait(self, timeout_sec=600):\n \"\"\"Wait no more than timeout_sec for all worker threads to finish.\n\n raise TimeoutException if all worker threads do not finish within timeout_sec\n \"\"\"\n super(BackgroundThreadService, self).wait(timeout_sec)\n\n self._propagate_exceptions()\n\n def stop(self):\n alive_workers = [worker for worker in self.worker_threads.values() if worker.is_alive()]\n if len(alive_workers) > 0:\n self.logger.debug(\"Called stop with at least one worker thread is still running: \" + str(alive_workers))\n\n self.logger.debug(\"%s\" % str(self.worker_threads))\n\n super(BackgroundThreadService, self).stop()\n\n self._propagate_exceptions()\n\n def wait_node(self, node, timeout_sec=600):\n idx = self.idx(node)\n worker_thread = self.worker_threads.get(idx)\n # worker thread can be absent if this node has never been started\n if worker_thread:\n worker_thread.join(timeout_sec)\n return not (worker_thread.is_alive())\n else:\n self.logger.debug(f\"Worker thread not found for {self.who_am_i(node)}\")\n return True\n\n def _propagate_exceptions(self):\n \"\"\"\n Propagate exceptions thrown in background threads\n \"\"\"\n with self.lock:\n if len(self.worker_errors) > 0:\n raise Exception(self.errors)\n" }, { "path": "ducktape/services/service.py", "content": "# Copyright 2014 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os\nimport shutil\nimport tempfile\nimport time\nfrom typing import Any, Dict\n\nfrom ducktape.cluster.cluster_spec import ClusterSpec\nfrom ducktape.command_line.defaults import ConsoleDefaults\nfrom ducktape.errors import TimeoutError\nfrom ducktape.template import TemplateRenderer\n\n\nclass ServiceIdFactory:\n def generate_service_id(self, service):\n return \"{service_name}-{service_number}-{service_id}\".format(\n service_name=service.__class__.__name__,\n service_number=service._order,\n service_id=id(service),\n )\n\n\nclass MultiRunServiceIdFactory:\n def __init__(self, run_number=1):\n self.run_number = run_number\n\n def generate_service_id(self, service):\n return \"{run_number}-{service_name}-{service_number}-{service_id}\".format(\n run_number=self.run_number,\n service_name=service.__class__.__name__,\n service_number=service._order,\n service_id=id(service),\n )\n\n\nservice_id_factory = ServiceIdFactory()\n\n\nclass Service(TemplateRenderer):\n \"\"\"Service classes know how to deploy a service onto a set of nodes and then clean up after themselves.\n\n They request the necessary resources from the cluster,\n configure each node, and bring up/tear down the service.\n\n They also expose\n information about the service so that other services or test scripts can\n easily be configured to work with them. Finally, they may be able to collect\n and check logs/output from the service, which can be helpful in writing tests\n or benchmarks.\n\n Services should generally be written to support an arbitrary number of nodes,\n even if instances are independent of each other. They should be able to assume\n that there won't be resource conflicts: the cluster tests are being run on\n should be large enough to use one instance per service instance.\n \"\"\"\n\n # Provides a mechanism for locating and collecting log files produced by the service on its nodes.\n # logs is a dict with entries that look like log_name: {\"path\": log_path, \"collect_default\": boolean}\n #\n # For example, zookeeper service might have self.logs like this:\n # self.logs = {\n # \"zk_log\": {\"path\": \"/mnt/zk.log\",\n # \"collect_default\": True}\n # }\n logs: Dict[str, Dict[str, Any]] = {}\n\n def __init__(self, context, num_nodes=None, cluster_spec=None, *args, **kwargs):\n \"\"\"\n Initialize the Service.\n\n Note: only one of (num_nodes, cluster_spec) may be set.\n\n :param context: An object which has at minimum 'cluster' and 'logger' attributes. In tests, this\n is always a TestContext object.\n :param num_nodes: An integer representing the number of Linux nodes to allocate.\n :param cluster_spec: A ClusterSpec object representing the minimum cluster specification needed.\n \"\"\"\n super(Service, self).__init__(*args, **kwargs)\n # Keep track of significant events in the lifetime of this service\n self._init_time = time.time()\n self._start_time = -1\n self._start_duration_seconds = -1\n self._stop_time = -1\n self._stop_duration_seconds = -1\n self._clean_time = -1\n\n self._initialized = False\n self.service_id_factory = service_id_factory\n self.cluster_spec = Service.setup_cluster_spec(num_nodes=num_nodes, cluster_spec=cluster_spec)\n self.context = context\n\n self.nodes = []\n self.skip_nodes_allocation = kwargs.get(\"skip_nodes_allocation\", False)\n if not self.skip_nodes_allocation:\n self.allocate_nodes()\n\n # Keep track of which nodes nodes were allocated to this service, even after nodes are freed\n # Note: only keep references to representations of the nodes, not the actual node objects themselves\n self._nodes_formerly_allocated = [str(node.account) for node in self.nodes]\n\n # Every time a service instance is created, it registers itself with its\n # context object. This makes it possible for external mechanisms to clean up\n # after the service if something goes wrong.\n #\n # Note: Allocate nodes *before* registering self with the service registry\n self.context.services.append(self)\n\n # Each service instance has its own local scratch directory on the test driver\n self._local_scratch_dir = None\n self._initialized = True\n\n @staticmethod\n def setup_cluster_spec(num_nodes=None, cluster_spec=None):\n if num_nodes is None:\n if cluster_spec is None:\n raise RuntimeError(\"You must set either num_nodes or cluster_spec.\")\n else:\n return cluster_spec\n else:\n if cluster_spec is not None:\n raise RuntimeError(\"You must set only one of (num_nodes, cluster_spec)\")\n return ClusterSpec.simple_linux(num_nodes)\n\n def __repr__(self):\n return \"<%s: %s>\" % (\n self.who_am_i(),\n \"num_nodes: %d, nodes: %s\" % (len(self.nodes), [n.account.hostname for n in self.nodes]),\n )\n\n @property\n def num_nodes(self):\n return len(self.nodes)\n\n @property\n def local_scratch_dir(self):\n \"\"\"This local scratch directory is created/destroyed on the test driver before/after each test is run.\"\"\"\n if not self._local_scratch_dir:\n self._local_scratch_dir = tempfile.mkdtemp()\n return self._local_scratch_dir\n\n @property\n def service_id(self):\n \"\"\"Human-readable identifier (almost certainly) unique within a test run.\"\"\"\n return self.service_id_factory.generate_service_id(self)\n\n @property\n def _order(self):\n \"\"\"Index of this service instance with respect to other services of the same type registered with self.context.\n When used with a test_context, this lets the user know\n\n Example::\n\n suppose the services registered with the same context looks like\n context.services == [Zookeeper, Kafka, Zookeeper, Kafka, MirrorMaker]\n then:\n context.services[0]._order == 0 # \"0th\" Zookeeper instance\n context.services[2]._order == 0 # \"0th\" Kafka instance\n context.services[1]._order == 1 # \"1st\" Zookeeper instance\n context.services[3]._order == 1 # \"1st\" Kafka instance\n context.services[4]._order == 0 # \"0th\" MirrorMaker instance\n\n \"\"\"\n if hasattr(self.context, \"services\"):\n same_services = [id(s) for s in self.context.services if isinstance(self, type(s))]\n\n if self not in self.context.services and not self._initialized:\n # It's possible that _order will be invoked in the constructor *before* self has been registered with\n # the service registry (aka self.context.services).\n return len(same_services)\n\n # Note: index raises ValueError if the item is not in the list\n index = same_services.index(id(self))\n return index\n else:\n return 0\n\n @property\n def logger(self):\n \"\"\"The logger instance for this service.\"\"\"\n return self.context.logger\n\n @property\n def cluster(self):\n \"\"\"The cluster object from which this service instance gets its nodes.\"\"\"\n return self.context.cluster\n\n @property\n def allocated(self):\n \"\"\"Return True iff nodes have been allocated to this service instance.\"\"\"\n return len(self.nodes) > 0\n\n def who_am_i(self, node=None):\n \"\"\"Human-readable identifier useful for log messages.\"\"\"\n if node is None:\n return self.service_id\n else:\n return \"%s node %d on %s\" % (\n self.service_id,\n self.idx(node),\n node.account.hostname,\n )\n\n def allocate_nodes(self):\n \"\"\"Request resources from the cluster.\"\"\"\n if self.allocated:\n raise Exception(\"Requesting nodes for a service that has already been allocated nodes.\")\n\n self.logger.debug(\"Requesting nodes from the cluster: %s\" % self.cluster_spec)\n\n try:\n self.nodes = self.cluster.alloc(self.cluster_spec)\n except RuntimeError as e:\n msg = str(e)\n if hasattr(self.context, \"services\"):\n msg += \" Currently registered services: \" + str(self.context.services)\n raise RuntimeError(msg)\n\n for idx, node in enumerate(self.nodes, 1):\n # Remote accounts utilities should log where this service logs\n if node.account._logger is not None:\n # This log message help test-writer identify which test and/or service didn't clean up after itself\n node.account.logger.critical(ConsoleDefaults.BAD_TEST_MESSAGE)\n raise RuntimeError(\n \"logger was not None on service start. There may be a concurrency issue, \"\n \"or some service which isn't properly cleaning up after itself. \"\n \"Service: %s, node.account: %s\" % (self.__class__.__name__, str(node.account))\n )\n node.account.logger = self.logger\n\n self.logger.debug(\"Successfully allocated %d nodes to %s\" % (len(self.nodes), self.who_am_i()))\n\n def start(self, **kwargs):\n \"\"\"Start the service on all nodes.\"\"\"\n self.logger.info(\"%s: starting service\" % self.who_am_i())\n if self._start_time < 0:\n # Set self._start_time only the first time self.start is invoked\n self._start_time = time.time()\n\n self.logger.debug(self.who_am_i() + \": killing processes and attempting to clean up before starting\")\n for node in self.nodes:\n # Added precaution - kill running processes, clean persistent files (if 'clean'=False flag passed,\n # skip cleaning), try/except for each step, since each of these steps may fail if there\n # are no processes to kill or no files to remove\n\n try:\n self.stop_node(node)\n except Exception:\n pass\n\n try:\n if kwargs.get(\"clean\", True):\n self.clean_node(node)\n else:\n self.logger.debug(\"%s: skip cleaning node\" % self.who_am_i(node))\n except Exception:\n pass\n\n for node in self.nodes:\n self.logger.debug(\"%s: starting node\" % self.who_am_i(node))\n self.start_node(node, **kwargs)\n\n if self._start_duration_seconds < 0:\n self._start_duration_seconds = time.time() - self._start_time\n\n def start_node(self, node, **kwargs):\n \"\"\"Start service process(es) on the given node.\"\"\"\n pass\n\n def wait(self, timeout_sec=600):\n \"\"\"Wait for the service to finish.\n This only makes sense for tasks with a fixed amount of work to do. For services that generate\n output, it is only guaranteed to be available after this call returns.\n \"\"\"\n unfinished_nodes = []\n start = time.time()\n end = start + timeout_sec\n for node in self.nodes:\n now = time.time()\n node_name = self.who_am_i(node)\n if end > now:\n self.logger.debug(\"%s: waiting for node\", node_name)\n if not self.wait_node(node, end - now):\n unfinished_nodes.append(node_name)\n else:\n unfinished_nodes.append(node_name)\n\n if unfinished_nodes:\n raise TimeoutError(\n \"Timed out waiting %s seconds for service nodes to finish. \" % str(timeout_sec)\n + \"These nodes are still alive: \"\n + str(unfinished_nodes)\n )\n\n def wait_node(self, node, timeout_sec=None):\n \"\"\"Wait for the service on the given node to finish.\n Return True if the node finished shutdown, False otherwise.\n \"\"\"\n pass\n\n def stop(self, **kwargs):\n \"\"\"Stop service processes on each node in this service.\n Subclasses must override stop_node.\n \"\"\"\n self._stop_time = time.time() # The last time stop is invoked\n self.logger.info(\"%s: stopping service\" % self.who_am_i())\n for node in self.nodes:\n self.logger.info(\"%s: stopping node\" % self.who_am_i(node))\n self.stop_node(node, **kwargs)\n\n self._stop_duration_seconds = time.time() - self._stop_time\n\n def stop_node(self, node, **kwargs):\n \"\"\"Halt service process(es) on this node.\"\"\"\n pass\n\n def clean(self, **kwargs):\n \"\"\"Clean up persistent state on each node - e.g. logs, config files etc.\n Subclasses must override clean_node.\n \"\"\"\n self._clean_time = time.time()\n self.logger.info(\"%s: cleaning service\" % self.who_am_i())\n for node in self.nodes:\n self.logger.info(\"%s: cleaning node\" % self.who_am_i(node))\n self.clean_node(node, **kwargs)\n\n def clean_node(self, node, **kwargs):\n \"\"\"Clean up persistent state on this node - e.g. service logs, configuration files etc.\"\"\"\n self.logger.warn(\n \"%s: clean_node has not been overriden. \"\n \"This may be fine if the service leaves no persistent state.\" % self.who_am_i()\n )\n\n def free(self):\n \"\"\"Free each node. This 'deallocates' the nodes so the cluster can assign them to other services.\"\"\"\n while self.nodes:\n node = self.nodes.pop()\n self.logger.info(\"%s: freeing node\" % self.who_am_i(node))\n node.account.logger = None\n self.cluster.free(node)\n\n def run(self):\n \"\"\"Helper that executes run(), wait(), and stop() in sequence.\"\"\"\n self.start()\n self.wait()\n self.stop()\n\n def get_node(self, idx):\n \"\"\"ids presented externally are indexed from 1, so we provide a helper method to avoid confusion.\"\"\"\n return self.nodes[idx - 1]\n\n def idx(self, node):\n \"\"\"Return id of the given node. Return -1 if node does not belong to this service.\n\n idx identifies the node within this service instance (not globally).\n \"\"\"\n for idx, n in enumerate(self.nodes, 1):\n if self.get_node(idx) == node:\n return idx\n return -1\n\n def close(self):\n \"\"\"Release resources.\"\"\"\n # Remove local scratch directory\n if self._local_scratch_dir and os.path.exists(self._local_scratch_dir):\n shutil.rmtree(self._local_scratch_dir)\n\n @staticmethod\n def run_parallel(*args):\n \"\"\"Helper to run a set of services in parallel. This is useful if you want\n multiple services of different types to run concurrently, e.g. a\n producer + consumer pair.\n \"\"\"\n for svc in args:\n svc.start()\n for svc in args:\n svc.wait()\n for svc in args:\n svc.stop()\n\n def to_json(self):\n return {\n \"cls_name\": self.__class__.__name__,\n \"module_name\": self.__module__,\n \"lifecycle\": {\n \"init_time\": self._init_time,\n \"start_time\": self._start_time,\n \"start_duration_seconds\": self._start_duration_seconds,\n \"stop_time\": self._stop_time,\n \"stop_duration_seconds\": self._stop_duration_seconds,\n \"clean_time\": self._clean_time,\n },\n \"service_id\": self.service_id,\n \"nodes\": self._nodes_formerly_allocated,\n }\n" }, { "path": "ducktape/services/service_registry.py", "content": "# Copyright 2014 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nfrom collections import OrderedDict\n\nfrom ducktape.jvm_logging import JVMLogger\n\n\nclass ServiceRegistry(object):\n def __init__(self, enable_jvm_logs=False) -> None:\n self._services: OrderedDict = OrderedDict()\n self._nodes: dict = {}\n\n # Initialize JVM logging if enabled\n if enable_jvm_logs:\n self._jvm_logger = JVMLogger()\n else:\n self._jvm_logger = None\n\n def __contains__(self, item):\n return id(item) in self._services\n\n def __iter__(self):\n return iter(self._services.values())\n\n def __repr__(self):\n return str(self._services.values())\n\n def append(self, service):\n # Auto-enable JVM logging for Java-based services\n if self._jvm_logger:\n self._jvm_logger.enable_for_service(service)\n\n self._services[id(service)] = service\n self._nodes[id(service)] = [str(n.account) for n in service.nodes]\n\n def to_json(self):\n return [service.to_json() for service in self._services.values()]\n\n def stop_all(self):\n \"\"\"Stop all currently registered services in the reverse of the order in which they were added.\n\n Note that this does not clean up persistent state or free the nodes back to the cluster.\n \"\"\"\n keyboard_interrupt = None\n for service in reversed(self._services.values()):\n try:\n service.stop()\n except BaseException as e:\n if isinstance(e, KeyboardInterrupt):\n keyboard_interrupt = e\n service.logger.warn(\"Error stopping service %s: %s\", service, e)\n\n if keyboard_interrupt is not None:\n raise keyboard_interrupt\n\n def clean_all(self):\n \"\"\"Clean all services. This should only be called after services are stopped.\"\"\"\n keyboard_interrupt = None\n for service in self._services.values():\n try:\n service.clean()\n except BaseException as e:\n if isinstance(e, KeyboardInterrupt):\n keyboard_interrupt = e\n service.logger.warn(\"Error cleaning service %s: %s\" % (service, e))\n\n if keyboard_interrupt is not None:\n raise keyboard_interrupt\n\n def free_all(self):\n \"\"\"Release nodes back to the cluster.\"\"\"\n keyboard_interrupt = None\n for service in self._services.values():\n try:\n service.free()\n except BaseException as e:\n if isinstance(e, KeyboardInterrupt):\n keyboard_interrupt = e\n service.logger.warn(\"Error freeing service %s: %s\" % (service, e))\n\n if keyboard_interrupt is not None:\n raise keyboard_interrupt\n self._services.clear()\n self._nodes.clear()\n\n def errors(self):\n \"\"\"\n Gets a printable string containing any errors produced by the services.\n \"\"\"\n return \"\\n\\n\".join(\n \"{}: {}\".format(service.who_am_i(), service.error)\n for service in self._services.values()\n if hasattr(service, \"error\") and service.error\n )\n" }, { "path": "ducktape/template.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport inspect\nimport os.path\n\nfrom jinja2 import ChoiceLoader, Environment, FileSystemLoader, PackageLoader, Template\n\nfrom ducktape.utils.util import package_is_installed\n\n\nclass TemplateRenderer(object):\n def _get_ctx(self):\n ctx = {k: getattr(self.__class__, k) for k in dir(self.__class__)}\n ctx.update(self.__dict__)\n return ctx\n\n def render_template(self, template, **kwargs):\n \"\"\"\n Render a template using the context of the current object, optionally with overrides.\n\n :param template: the template to render, a Template or a str\n :param kwargs: optional override parameters\n :return: the rendered template\n \"\"\"\n if not hasattr(template, \"render\"):\n template = Template(template)\n ctx = self._get_ctx()\n return template.render(ctx, **kwargs)\n\n @staticmethod\n def _package_search_path(module_name):\n \"\"\"\n :param module_name: Name of a module\n :return: (package, package_search_path) where package is the package containing the module,\n and package_search_path is a path relative to the package in which to search for templates.\n \"\"\"\n module_parts = module_name.split(\".\")\n package = module_parts[0]\n\n # Construct path relative to package under which \"templates\" would be found\n directory = \"\"\n for d in module_parts[1:-1]:\n directory = os.path.join(directory, d)\n return package, os.path.join(directory, \"templates\")\n\n def render(self, path, **kwargs):\n \"\"\"\n Render a template loaded from a file.\n template files referenced in file f should be in a sibling directory of f called \"templates\".\n\n :param path: path, relative to the search paths, to the template file\n :param kwargs: optional override parameters\n :return: the rendered template\n \"\"\"\n if not hasattr(self, \"template_loader\"):\n class_dir = os.path.dirname(inspect.getfile(self.__class__))\n\n module_name = self.__class__.__module__\n package, package_search_path = self._package_search_path(module_name)\n\n loaders = []\n msg = \"\"\n if os.path.isdir(class_dir):\n # FileSystemLoader overrides PackageLoader if the path containing this directory\n # is a valid directory. FileSystemLoader throws an error from which ChoiceLoader\n # doesn't recover if the directory is invalid\n loaders.append(FileSystemLoader(os.path.join(class_dir, \"templates\")))\n else:\n msg += \"Will not search in %s for template files since it is not a valid directory. \" % class_dir\n\n if package_is_installed(package):\n loaders.append(PackageLoader(package, package_search_path))\n else:\n msg += \"Will not search in package %s for template files because it cannot be imported.\"\n\n if len(loaders) == 0:\n # Expect at least one of FileSystemLoader and PackageLoader to be present\n raise EnvironmentError(msg)\n\n self.template_loader = ChoiceLoader(loaders)\n self.template_env = Environment(loader=self.template_loader, trim_blocks=True, lstrip_blocks=True)\n\n template = self.template_env.get_template(path)\n return self.render_template(template, **kwargs)\n" }, { "path": "ducktape/templates/report/report.css", "content": "body { \n font-family: verdana, arial, helvetica, sans-serif; \n font-size: 80%; \n}\n\ntable { \n font-size: 100%;\n}\n\nh1, h2, h3, h4, h5, h6 {\n color: gray;\n}\n\n#summary_header_row {\n font-weight: bold;\n color: white;\n background-color: #777;\n}\n\n#summary_header_row th {\n border: 1px solid #777;\n padding: 2px; \n}\n\n#summary_report_table {\n width: 80%;\n border-collapse: collapse;\n border: 1px solid #777;\n}\n\n#summary_report_table td {\n border: 1px solid #777;\n padding: 2px;\n}\n\n.pre_stack_trace {\n white-space: pre-wrap; /* Since CSS 2.1 */\n white-space: -moz-pre-wrap; /* Mozilla, since 1999 */\n white-space: -o-pre-wrap; /* Opera 7 */\n word-wrap: break-word; /* Internet Explorer 5.5+ */\n}\n\n.header_row {\n font-weight: bold;\n color: white;\n background-color: #777;\n}\n\n.header_row th {\n border: 1px solid #777;\n padding: 2px; \n}\n\n.report_table {\n width: 80%;\n border-collapse: collapse;\n border: 1px solid #777;\n}\n\n.report_table td {\n border: 1px solid #777;\n padding: 2px;\n}\n\n.pass {\n background-color: #6c6;\n}\n\n.flaky {\n background-color: #dd2;\n}\n\n.fail {\n background-color: #c60; \n}\n\n.ignore {\n background-color: #555;\n}\n\n.testcase { \n margin-left: 2em;\n}\n" }, { "path": "ducktape/templates/report/report.html", "content": "\n\n \n \n \n \n \n \n
\n
\n
\n
\n
\n
\n
\n \n \n\n" }, { "path": "ducktape/tests/__init__.py", "content": "from .test import Test\nfrom .test_context import TestContext\n\n__all__ = [\"Test\", \"TestContext\"]\n" }, { "path": "ducktape/tests/event.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport copy\nimport os\nimport time\n\n\nclass ClientEventFactory(object):\n \"\"\"Used by test runner clients to generate events.\"\"\"\n\n READY = \"READY\" # reply: {test_metadata, cluster, session_context}\n SETTING_UP = \"SETTING_UP\"\n RUNNING = \"RUNNING\"\n TEARING_DOWN = \"TEARING_DOWN\"\n FINISHED = \"FINISHED\"\n LOG = \"LOG\"\n\n # Types of messages available\n TYPES = {READY, SETTING_UP, RUNNING, TEARING_DOWN, FINISHED, LOG}\n\n def __init__(self, test_id, test_index, source_id):\n self.test_id = test_id\n # id of event source\n self.test_index = test_index\n self.source_id = source_id\n self.event_id = 0\n\n def _event(self, event_type, payload=None):\n \"\"\"Create a message object with certain base fields, and augmented by the payload.\n\n :param event_type: type of message this is\n :param payload: a dict containing extra fields for the message. Key names should not conflict with keys\n in the base event.\n \"\"\"\n assert event_type in ClientEventFactory.TYPES, \"Unknown event type\"\n if payload is None:\n payload = {}\n\n event = {\n \"test_id\": self.test_id,\n \"source_id\": self.source_id,\n \"test_index\": self.test_index,\n \"event_id\": self.event_id,\n \"event_type\": event_type,\n \"event_time\": time.time(),\n }\n\n assert len(set(event.keys()).intersection(set(payload.keys()))) == 0, (\n \"Payload and base event should not share keys. base event: %s, payload: %s\" % (str(event), str(payload))\n )\n\n event.update(payload)\n self.event_id += 1\n return event\n\n def copy(self, event):\n \"\"\"Copy constructor: return a copy of the original message, but with a unique message id.\"\"\"\n new_event = copy.copy(event)\n new_event[\"message_id\"] = self.event_id\n self.event_id += 1\n\n return new_event\n\n def running(self):\n return self._event(\n event_type=ClientEventFactory.RUNNING,\n payload={\"pid\": os.getpid(), \"pgroup_id\": os.getpgrp()},\n )\n\n def ready(self):\n return self._event(\n event_type=ClientEventFactory.READY,\n payload={\"pid\": os.getpid(), \"pgroup_id\": os.getpgrp()},\n )\n\n def setting_up(self):\n return self._event(event_type=ClientEventFactory.SETTING_UP)\n\n def finished(self, result):\n return self._event(event_type=ClientEventFactory.FINISHED, payload={\"result\": result})\n\n def log(self, message, level):\n return self._event(\n event_type=ClientEventFactory.LOG,\n payload={\"message\": message, \"log_level\": level},\n )\n\n\nclass EventResponseFactory(object):\n \"\"\"Used by the test runner to create responses to events from client processes.\"\"\"\n\n def _event_response(self, client_event, payload=None):\n if payload is None:\n payload = {}\n\n event_response = {\n \"ack\": True,\n \"source_id\": client_event[\"source_id\"],\n \"event_id\": client_event[\"event_id\"],\n }\n\n assert len(set(event_response.keys()).intersection(set(payload.keys()))) == 0, (\n \"Payload and base event should not share keys. base event: %s, payload: %s\"\n % (str(event_response), str(payload))\n )\n\n event_response.update(payload)\n return event_response\n\n def running(self, client_event):\n return self._event_response(client_event)\n\n def ready(self, client_event, session_context, test_context, cluster):\n payload = {\n \"session_context\": session_context,\n \"test_metadata\": test_context.test_metadata,\n \"cluster\": cluster,\n }\n\n return self._event_response(client_event, payload)\n\n def setting_up(self, client_event):\n return self._event_response(client_event)\n\n def finished(self, client_event):\n return self._event_response(client_event)\n\n def log(self, client_event):\n return self._event_response(client_event)\n" }, { "path": "ducktape/tests/loader.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nimport collections\nimport glob\nimport importlib\nimport inspect\nimport itertools\nimport json\nimport os\nimport re\nimport sys\nfrom logging import Logger\nfrom operator import attrgetter\nfrom typing import Any, Iterable, List, Optional, Set, Tuple, Union\n\nimport requests\nimport yaml\n\nfrom ducktape.mark import parametrized\nfrom ducktape.mark.mark_expander import MarkedFunctionExpander\nfrom ducktape.tests.session import SessionContext\nfrom ducktape.tests.test import Test\nfrom ducktape.tests.test_context import TestContext\n\n\nclass LoaderException(Exception):\n pass\n\n\n# A helper container class\nModuleAndFile = collections.namedtuple(\"ModuleAndFile\", [\"module\", \"file\"])\n\n\nDEFAULT_TEST_FILE_PATTERN = r\"(^test_.*\\.py$)|(^.*_test\\.py$)\"\nDEFAULT_TEST_FUNCTION_PATTERN = \"(^test.*)|(.*test$)\"\n\n# Included for unit tests to be able to add support for loading local file:/// URLs.\n_requests_session = requests.session()\n\n\nclass TestLoader(object):\n \"\"\"Class used to discover and load tests.\"\"\"\n\n def __init__(\n self,\n session_context: SessionContext,\n logger: Logger,\n repeat: int = 1,\n injected_args: None = None,\n cluster: None = None,\n subset: int = 0,\n subsets: int = 1,\n historical_report: None = None,\n ) -> None:\n self.session_context = session_context\n self.cluster = cluster\n assert logger is not None\n self.logger = logger\n\n assert repeat >= 1\n self.repeat = repeat\n\n if subset >= subsets:\n raise ValueError(\"The subset to execute must be in the range [0, subsets-1]\")\n self.subset = subset\n self.subsets = subsets\n\n self.historical_report = historical_report\n\n self.test_file_pattern = DEFAULT_TEST_FILE_PATTERN\n self.test_function_pattern = DEFAULT_TEST_FUNCTION_PATTERN\n\n # A non-None value here means the loader will override the injected_args\n # in any discovered test, whether or not it is parametrized\n self.injected_args = injected_args\n\n def load(self, symbols: List[str], excluded_test_symbols: None = None) -> List[TestContext]:\n \"\"\"\n Discover tests specified by the symbols parameter (iterable of test symbols and/or test suite file paths).\n Skip any tests specified by excluded_test_symbols (iterable of test symbols).\n\n *Test symbol* is a pointer to the test or a group of tests.\n It is specified by the file/folder path or glob, optionally with Class.method after `::` :\n - `test-dir/` - loads all tests under `test-dir` but does NOT load test suites found under `test-dir`\n - `test-dir/prefix_*.py` - loads all files with a specified prefix\n - `/path/to/test/file.py`\n - `test/file.py::TestClass`\n - `test/file.py::TestClass.test_method`\n\n *Test suite* is a yaml file with the following format:\n ```\n # multiple test suites can be included:\n test_suite_name:\n # list included test symbols\n - path/to/test.py\n # optionally test suite can have included and excluded sections:\n # you may also specify a list of other suite's you wish to import\n # that will also be loaded when loading this file by using the\n # import tag.\n import:\n # list of yaml files whose suites will also run:\n - path/to/suite.yml\n another_test_suite:\n included:\n # list of included test symbols:\n - path/to/test-dir/prefix_*.py\n excluded:\n # list of excluded test symbols:\n - path/to/test-dir/prefix_excluded.py\n ```\n Each file found after parsing a symbol is checked to see if it contains a test:\n - Discover modules that 'look like' a test. By default, this means the filename is \"test_*\" or \"*_test.py\"\n - Discover test classes within each test module. A test class is a subclass of Test which is a leaf\n (i.e. it has no subclasses).\n - Discover test methods within each test class. A test method is a method containing 'test' in its name\n\n :param symbols: iterable that contains test symbols and/or test suite file paths.\n :param excluded_test_symbols: iterable that contains test symbols only.\n :return list of test context objects found during discovery. Note: if self.repeat is set to n, each test_context\n will appear in the list n times.\n \"\"\"\n\n test_symbols = []\n test_suites = []\n # symbol can point to a test or a test suite\n for symbol in symbols:\n if symbol.endswith(\".yml\"):\n # if it ends with .yml, its a test suite, read included and excluded paths from the file\n test_suites.append(symbol)\n else:\n # otherwise add it to default suite's included list\n test_symbols.append(symbol)\n\n contexts_from_suites = self._load_test_suite_files(test_suites)\n contexts_from_symbols = self._load_test_contexts(test_symbols)\n all_included = contexts_from_suites.union(contexts_from_symbols)\n\n # excluded_test_symbols apply to both tests from suites and tests from symbols\n global_excluded = self._load_test_contexts(excluded_test_symbols)\n all_test_context_list: Union[List[TestContext], Set[TestContext]] = self._filter_excluded_test_contexts(\n all_included, global_excluded\n )\n\n # make sure no test is loaded twice\n all_test_context_list = self._filter_by_unique_test_id(all_test_context_list)\n\n # Sort to make sure we get a consistent order for when we create subsets\n all_test_context_list = sorted(all_test_context_list, key=attrgetter(\"test_id\"))\n if not all_test_context_list:\n raise LoaderException(\"No tests to run!\")\n self.logger.debug(\"Discovered these tests: \" + str(all_test_context_list))\n # Select the subset of tests.\n if self.historical_report:\n # With timing info, try to pack the subsets reasonably evenly based on timing. To do so, get timing info\n # for each test (using avg as a fallback for missing data), sort in descending order, then start greedily\n # packing tests into bins based on the least full bin at the time.\n raw_results = _requests_session.get(self.historical_report).json()[\"results\"]\n time_results = {r[\"test_id\"]: r[\"run_time_seconds\"] for r in raw_results}\n avg_result_time = sum(time_results.values()) / len(time_results)\n time_results = {tc.test_id: time_results.get(tc.test_id, avg_result_time) for tc in all_test_context_list}\n all_test_context_list = sorted(\n all_test_context_list,\n key=lambda x: time_results[x.test_id],\n reverse=True,\n )\n\n subsets = [[] for _ in range(self.subsets)]\n subsets_accumulated_time = [0] * self.subsets\n\n for tc in all_test_context_list:\n min_subset_idx = min(\n range(len(subsets_accumulated_time)),\n key=lambda i: subsets_accumulated_time[i],\n )\n subsets[min_subset_idx].append(tc.test_id)\n subsets_accumulated_time[min_subset_idx] += time_results[tc.test_id]\n\n subset_test_context_list = subsets[self.subset]\n else:\n # Without timing info, select every nth test instead of blocks of n to avoid groups of tests that are\n # parametrizations of the same test being grouped together since that can lead to a single, parameterized,\n # long-running test causing a very imbalanced workload across different subsets. Imbalance is still\n # possible, but much less likely using this heuristic.\n subset_test_context_list = list(itertools.islice(all_test_context_list, self.subset, None, self.subsets))\n\n self.logger.debug(\"Selected this subset of tests: \" + str(subset_test_context_list))\n return subset_test_context_list * self.repeat\n\n def discover(\n self,\n directory: str,\n module_name: str,\n cls_name: str,\n method_name: str,\n injected_args: None = None,\n ) -> List[TestContext]:\n \"\"\"Discover and unpack parametrized tests tied to the given module/class/method\n\n :return list of test_context objects\n \"\"\"\n self.logger.debug(\n \"Discovering tests at {} - {} - {} - {} - {}\".format(\n directory, module_name, cls_name, method_name, injected_args\n )\n )\n # Check validity of path\n path = os.path.join(directory, module_name)\n if not os.path.exists(path):\n raise LoaderException(\"Path {} does not exist\".format(path))\n\n # Recursively search path for test modules\n module_and_file = self._import_module(path)\n if module_and_file:\n # Find all tests in discovered modules and filter out any that don't match the discovery symbol\n test_context_list = self._expand_module(module_and_file)\n if len(cls_name) > 0:\n test_context_list = [t for t in test_context_list if t.cls_name == cls_name]\n if len(method_name) > 0:\n test_context_list = [t for t in test_context_list if t.function_name == method_name]\n if injected_args is not None:\n if isinstance(injected_args, List):\n\n def condition(t):\n return t.injected_args in injected_args\n\n else:\n\n def condition(t):\n return t.injected_args == injected_args\n\n test_context_list = filter(condition, test_context_list)\n\n listed = list(test_context_list)\n if not listed:\n self.logger.warn(\n \"No tests loaded for {} - {} - {} - {} - {}\".format(\n directory, module_name, cls_name, method_name, injected_args\n )\n )\n return listed\n else:\n return []\n\n def _parse_discovery_symbol(self, discovery_symbol: str, base_dir: None = None) -> Tuple[str, str, str, None]:\n \"\"\"Parse a single 'discovery symbol'\n\n :param discovery_symbol: a symbol used to target test(s).\n Looks like: [::[.method_name]]\n :return tuple of form (directory, module.py, cls_name, function_name)\n\n :raise LoaderException if it can't be parsed\n\n Examples:\n \"path/to/directory\" -> (\"path/to/directory\", \"\", \"\")\n \"path/to/test_file.py\" -> (\"path/to/test_file.py\", \"\", \"\")\n \"path/to/test_file.py::ClassName.method\" -> (\"path/to/test_file.py\", \"ClassName\", \"method\")\n \"\"\"\n\n def divide_by_symbol(ds, symbol):\n if symbol not in ds:\n return ds, \"\"\n return ds.split(symbol, maxsplit=1)\n\n self.logger.debug(\"Trying to parse discovery symbol {}\".format(discovery_symbol))\n if base_dir:\n discovery_symbol = os.path.join(base_dir, discovery_symbol)\n if discovery_symbol.find(\"::\") >= 0:\n path, cls_name = divide_by_symbol(discovery_symbol, \"::\")\n # If the part after :: contains a dot, use it to split into class + method\n cls_name, method_name = divide_by_symbol(cls_name, \".\")\n method_name, injected_args_str = divide_by_symbol(method_name, \"@\")\n\n if injected_args_str:\n if self.injected_args:\n raise LoaderException(\"Cannot use both global and per-method test parameters\")\n try:\n injected_args = json.loads(injected_args_str)\n except Exception as e:\n raise LoaderException(\"Invalid discovery symbol: cannot parse params: \" + injected_args_str) from e\n else:\n injected_args = None\n else:\n # No \"::\" present in symbol\n path, cls_name, method_name, injected_args = discovery_symbol, \"\", \"\", None\n\n return path, cls_name, method_name, injected_args\n\n def _import_module(self, file_path: str) -> Optional[ModuleAndFile]:\n \"\"\"Attempt to import a python module from the file path.\n Assume file_path is an absolute path ending in '.py'\n\n Return the imported module..\n\n :param file_path: file to import module from.\n :return ModuleAndFile object that contains the successfully imported module and\n the file from which it was imported\n \"\"\"\n self.logger.debug(\"Trying to import module at path {}\".format(file_path))\n if file_path[-3:] != \".py\" or not os.path.isabs(file_path):\n raise Exception(\"Expected absolute path ending in '.py' but got \" + file_path)\n\n # Try all possible module imports for given file\n # Strip off '.py' before splitting\n path_pieces = [piece for piece in file_path[:-3].split(\"/\") if len(piece) > 0]\n while len(path_pieces) > 0:\n module_name = \".\".join(path_pieces)\n # Try to import the current file as a module\n self.logger.debug(\"Trying to import module {}\".format(module_name))\n try:\n module_and_file = ModuleAndFile(module=importlib.import_module(module_name), file=file_path)\n self.logger.debug(\"Successfully imported \" + module_name)\n return module_and_file\n except Exception as e:\n # Because of the way we are searching for\n # valid modules in this loop, we expect some of the\n # module names we construct to fail to import.\n #\n # Therefore we check if the failure \"looks normal\", and log\n # expected failures only at debug level.\n #\n # Unexpected errors are aggressively logged, e.g. if the module\n # is valid but itself triggers an ImportError (e.g. typo in an\n # import line), or a SyntaxError.\n expected_error = False\n if isinstance(e, ImportError):\n match = re.search(r\"No module named '?([^\\s\\']+)'?\", str(e))\n\n if match is not None:\n missing_module = match.groups()[0]\n\n if missing_module in module_name:\n expected_error = True\n else:\n # The error is still an expected error if missing_module is a suffix of module_name.\n # This is because the error message may contain only a suffix\n # of the original module_name if leftmost chunk of module_name is a legitimate\n # module name, but the rightmost part doesn't exist.\n #\n # Check this by seeing if it is a \"piecewise suffix\" of module_name - i.e. if the parts\n # delimited by dots match. This is a little bit stricter than just checking for a suffix\n #\n # E.g. \"fancy.cool_module\" is a piecewise suffix of \"my.fancy.cool_module\",\n # but \"module\" is not a piecewise suffix of \"my.fancy.cool_module\"\n missing_module_pieces = missing_module.split(\".\")\n expected_error = missing_module_pieces == path_pieces[-len(missing_module_pieces) :]\n\n if expected_error:\n self.logger.debug(\n \"Failed to import %s. This is likely an artifact of the \"\n \"ducktape module loading process: %s: %s\",\n module_name,\n e.__class__.__name__,\n e,\n )\n else:\n self.logger.error(\n \"Failed to import %s, which may indicate a broken test that cannot be loaded: %s: %s\",\n module_name,\n e.__class__.__name__,\n e,\n )\n finally:\n path_pieces = path_pieces[1:]\n\n self.logger.debug(\"Unable to import %s\" % file_path)\n return None\n\n def _expand_module(self, module_and_file: ModuleAndFile) -> List[TestContext]:\n \"\"\"Return a list of TestContext objects, one object for every 'testable unit' in module\"\"\"\n\n test_context_list = []\n module = module_and_file.module\n file_name = module_and_file.file\n module_objects = module.__dict__.values()\n test_classes = [c for c in module_objects if self._is_test_class(c)]\n\n for cls in test_classes:\n test_context_list.extend(\n self._expand_class(\n TestContext(\n session_context=self.session_context,\n cluster=self.cluster,\n module=module.__name__,\n cls=cls,\n file=file_name,\n )\n )\n )\n\n return test_context_list\n\n def _expand_class(self, t_ctx: TestContext) -> List[TestContext]:\n \"\"\"Return a list of TestContext objects, one object for each method in t_ctx.cls\"\"\"\n test_methods = []\n for f_name in dir(t_ctx.cls):\n f = getattr(t_ctx.cls, f_name)\n if self._is_test_function(f):\n test_methods.append(f)\n\n test_context_list = []\n for f in test_methods:\n t = t_ctx.copy(function=f)\n test_context_list.extend(self._expand_function(t))\n return test_context_list\n\n def _expand_function(self, t_ctx: TestContext) -> List[TestContext]:\n expander = MarkedFunctionExpander(\n t_ctx.session_context,\n t_ctx.module,\n t_ctx.cls,\n t_ctx.function,\n t_ctx.file,\n t_ctx.cluster,\n )\n return expander.expand(self.injected_args)\n\n def _find_test_files(self, path_or_glob):\n \"\"\"\n Return a list of files at the specified path (or glob) that look like test files.\n\n - Globs are not recursive, so ** is not supported.\n - However, if the glob matches a folder (or is not a glob but simply a folder path),\n we will load all tests in that folder and recursively search the sub folders.\n\n :param path_or_glob: path to a test file, folder with test files or a glob that expands to folders and files\n :return: list of absolute paths to test files\n \"\"\"\n test_files = []\n self.logger.debug(\"Looking for test files in {}\".format(path_or_glob))\n # glob is safe to be called on non-glob path - it would just return that same path wrapped in a list\n expanded_glob = glob.glob(path_or_glob)\n self.logger.debug(\"Expanded {} into {}\".format(path_or_glob, expanded_glob))\n\n def maybe_add_test_file(f):\n if self._is_test_file(f):\n test_files.append(f)\n else:\n self.logger.debug(\"Skipping {} because it isn't a test file\".format(f))\n\n for path in expanded_glob:\n if not os.path.exists(path):\n raise LoaderException(\"Path {} does not exist\".format(path))\n self.logger.debug(\"Checking {}\".format(path))\n if os.path.isfile(path):\n maybe_add_test_file(path)\n elif os.path.isdir(path):\n for pwd, dirs, files in os.walk(path):\n if \"__init__.py\" not in files:\n # Not a package - ignore this directory\n continue\n for f in files:\n file_path = os.path.abspath(os.path.join(pwd, f))\n maybe_add_test_file(file_path)\n else:\n raise LoaderException(\"Got a path that we don't understand: \" + path)\n\n return test_files\n\n def _is_test_file(self, file_name):\n \"\"\"By default, a test file looks like test_*.py or *_test.py\"\"\"\n return re.match(self.test_file_pattern, os.path.basename(file_name)) is not None\n\n def _is_test_class(self, obj: Any) -> bool:\n \"\"\"An object is a test class if it's a leafy subclass of Test.\"\"\"\n return inspect.isclass(obj) and issubclass(obj, Test) and len(obj.__subclasses__()) == 0\n\n def _is_test_function(self, function: Any) -> bool:\n \"\"\"A test function looks like a test and is callable (or expandable).\"\"\"\n if function is None:\n return False\n\n if not parametrized(function) and not callable(function):\n return False\n\n return re.match(self.test_function_pattern, function.__name__) is not None\n\n def _load_test_suite_files(self, test_suite_files: List[Any]) -> Set[Any]:\n suites = list()\n\n suites.extend(self._read_test_suite_from_file(test_suite_files))\n\n all_contexts = set()\n for suite in suites:\n all_contexts.update(self._load_test_suite(**suite))\n return all_contexts\n\n def _load_file(self, suite_file_path):\n if not os.path.exists(suite_file_path):\n raise LoaderException(f\"Path {suite_file_path} does not exist\")\n if not os.path.isfile(suite_file_path):\n raise LoaderException(f\"{suite_file_path} is not a file, so it cannot be a test suite\")\n\n with open(suite_file_path) as fp:\n try:\n file_content = yaml.load(fp, Loader=yaml.FullLoader)\n except Exception as e:\n raise LoaderException(\"Failed to load test suite from file: \" + suite_file_path, e)\n\n if not file_content:\n raise LoaderException(\"Test suite file is empty: \" + suite_file_path)\n if not isinstance(file_content, dict):\n raise LoaderException(\"Malformed test suite file: \" + suite_file_path)\n\n for suite_name, suite_content in file_content.items():\n if not suite_content:\n raise LoaderException(\"Empty test suite \" + suite_name + \" in \" + suite_file_path)\n return file_content\n\n def _load_suites(self, file_path, file_content):\n suites = []\n for suite_name, suite_content in file_content.items():\n if not suite_content:\n raise LoaderException(f\"Empty test suite {suite_name} in {file_path}\")\n\n # if test suite is just a list of paths, those are included paths\n # otherwise, expect separate sections for included and excluded\n if isinstance(suite_content, list):\n included_paths = suite_content\n excluded_paths = None\n elif isinstance(suite_content, dict):\n included_paths = suite_content.get(\"included\")\n excluded_paths = suite_content.get(\"excluded\")\n else:\n raise LoaderException(f\"Malformed test suite {suite_name} in {file_path}\")\n suites.append(\n {\n \"name\": suite_name,\n \"included\": included_paths,\n \"excluded\": excluded_paths,\n \"base_dir\": os.path.dirname(file_path),\n }\n )\n return suites\n\n def _read_test_suite_from_file(self, root_suite_file_paths: List[Any]) -> List[Any]:\n root_suite_file_paths = [os.path.abspath(file_path) for file_path in root_suite_file_paths]\n files = {file: self._load_file(file) for file in root_suite_file_paths}\n stack = root_suite_file_paths\n\n # load all files\n while len(stack) != 0:\n curr = stack.pop()\n loaded = files[curr]\n if \"import\" in loaded:\n if isinstance(loaded[\"import\"], str):\n loaded[\"import\"] = [loaded[\"import\"]]\n directory = os.path.dirname(curr)\n # apply path of current file to the files inside\n abs_file_iter = (os.path.abspath(os.path.join(directory, file)) for file in loaded.get(\"import\", []))\n imported = [file for file in abs_file_iter if file not in files]\n for file in imported:\n files[file] = self._load_file(file)\n stack.extend(imported)\n del files[curr][\"import\"]\n\n # load all suites from all loaded files\n suites = []\n for file_name, file_context in files.items():\n suites.extend(self._load_suites(file_name, file_context))\n\n return suites\n\n def _load_test_suite(self, **kwargs):\n name = kwargs[\"name\"]\n included = kwargs[\"included\"]\n excluded = kwargs.get(\"excluded\")\n base_dir = kwargs.get(\"base_dir\")\n excluded_contexts = self._load_test_contexts(excluded, base_dir=base_dir)\n included_contexts = self._load_test_contexts(included, base_dir=base_dir)\n\n self.logger.debug(\"Including tests: \" + str(included_contexts))\n self.logger.debug(\"Excluding tests: \" + str(excluded_contexts))\n\n # filter out any excluded test from the included tests set\n all_test_context_list = self._filter_excluded_test_contexts(included_contexts, excluded_contexts)\n if not all_test_context_list:\n raise LoaderException(\"No tests found in \" + name)\n\n return all_test_context_list\n\n def _load_test_contexts(\n self, test_discovery_symbols: Optional[List[str]], base_dir: None = None\n ) -> Set[TestContext]:\n \"\"\"\n Load all test_context objects found in test_discovery_symbols.\n Each test discovery symbol is a dir or file path, optionally with with a ::Class or ::Class.method specified.\n\n :param test_discovery_symbols: list of test symbols to look into\n :return: List of test_context objects discovered by checking test_discovery_symbols (may be empty if none were\n discovered)\n \"\"\"\n if not test_discovery_symbols:\n return set()\n if not isinstance(test_discovery_symbols, list):\n raise LoaderException(\"Expected test_discovery_symbols to be a list.\")\n all_test_context_list = set()\n for symbol in test_discovery_symbols:\n (\n path_or_glob,\n cls_name,\n method,\n injected_args,\n ) = self._parse_discovery_symbol(symbol, base_dir)\n self.logger.debug(\n \"Parsed symbol into {} - {} - {} - {}\".format(path_or_glob, cls_name, method, injected_args)\n )\n path_or_glob = os.path.abspath(path_or_glob)\n\n # TODO: consider adding a check to ensure glob or dir is not used together with cls_name and method\n test_files = []\n if os.path.isfile(path_or_glob):\n # if it is a single file, just add it directly - https://github.com/confluentinc/ducktape/issues/284\n test_files = [path_or_glob]\n else:\n # otherwise, when dealing with a dir or a glob, apply pattern matching rules\n test_files = self._find_test_files(path_or_glob)\n\n self._add_top_level_dirs_to_sys_path(test_files)\n\n for test_file in test_files:\n directory = os.path.dirname(test_file)\n module_name = os.path.basename(test_file)\n test_context_list_for_file = self.discover(\n directory,\n module_name,\n cls_name,\n method,\n injected_args=injected_args,\n )\n all_test_context_list.update(test_context_list_for_file)\n if len(test_context_list_for_file) == 0:\n self.logger.warn(\"Didn't find any tests in %s \" % test_file)\n\n return all_test_context_list\n\n def _filter_by_unique_test_id(self, contexts: Iterable[TestContext]) -> List[TestContext]:\n contexts_dict = dict()\n for context in contexts:\n if context.test_id not in contexts_dict:\n contexts_dict[context.test_id] = context\n return list(contexts_dict.values())\n\n def _filter_excluded_test_contexts(\n self, included_contexts: Iterable[TestContext], excluded_contexts: Set[Any]\n ) -> Set[TestContext]:\n excluded_test_ids = {ctx.test_id for ctx in excluded_contexts}\n return {ctx for ctx in included_contexts if ctx.test_id not in excluded_test_ids}\n\n def _add_top_level_dirs_to_sys_path(self, test_files: List[str]) -> None:\n seen_dirs = set()\n for path in test_files:\n dir = os.path.dirname(path)\n while os.path.exists(os.path.join(dir, \"__init__.py\")):\n dir = os.path.dirname(dir)\n if dir not in seen_dirs:\n sys.path.append(dir)\n seen_dirs.add(dir)\n" }, { "path": "ducktape/tests/loggermaker.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\n\n\nclass LoggerMaker(object):\n \"\"\"This class helps ensure programmatically configured loggers are configured only once.\"\"\"\n\n def __init__(self, logger_name: str) -> None:\n self.logger_name = logger_name\n\n @property\n def logger(self) -> logging.Logger:\n \"\"\"Read-only logger attribute.\"\"\"\n if not hasattr(self, \"_logger\"):\n self._logger = logging.getLogger(self.logger_name)\n\n if not self.configured:\n self.configure_logger()\n\n return self._logger\n\n @property\n def configured(self) -> bool:\n \"\"\"Return True iff the logger has been configured.\n\n Since logging objects are global in the sense that logging.getLogger(self.logger_name) yields the same\n object, we assume it is already configured if it already has at least 1 handler.\n \"\"\"\n return len(logging.getLogger(self.logger_name).handlers) > 0\n\n def configure_logger(self):\n raise NotImplementedError(\"configure_logger property must be implemented by a subclass\")\n\n\ndef close_logger(logger):\n \"\"\"Filehandles etc are not closed automatically, so close them here\"\"\"\n if logger is not None:\n handlers = logger.handlers[:]\n for handler in handlers:\n handler.close()\n logger.removeHandler(handler)\n" }, { "path": "ducktape/tests/reporter.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import print_function\n\nimport json\nimport os\nimport shutil\nimport sys\nimport xml.etree.ElementTree as ET\nfrom pathlib import Path\n\nimport yaml\n\nfrom ducktape.json_serializable import DucktapeJSONEncoder\nfrom ducktape.tests.status import FAIL, FLAKY, IGNORE, PASS\nfrom ducktape.utils.terminal_size import get_terminal_size\nfrom ducktape.utils.util import ducktape_version\n\nDEFAULT_SEPARATOR_WIDTH = 100\n\n\ndef format_time(t):\n \"\"\"Return human-readable interval of time.\n Assumes t is in units of seconds.\n \"\"\"\n minutes = int(t / 60)\n seconds = t % 60\n\n r = \"\"\n if minutes > 0:\n r += \"%d minute%s \" % (minutes, \"\" if minutes == 1 else \"s\")\n r += \"%.3f seconds\" % seconds\n return r\n\n\nclass SingleResultReporter(object):\n \"\"\"Helper class for creating a view of results from a single test.\"\"\"\n\n def __init__(self, result):\n self.result = result\n self.width = get_terminal_size()[0]\n\n def result_string(self):\n \"\"\"Stringify single result\"\"\"\n result_lines = [\n \"test_id: %s\" % self.result.test_id,\n \"status: %s\" % str(self.result.test_status).upper(),\n \"run time: %s\" % format_time(self.result.run_time_seconds),\n ]\n\n if self.result.test_status == FAIL:\n # Add summary if the test failed\n result_lines.append(\"\\n\")\n result_lines.append(\" \" + self.result.summary)\n\n if self.result.data is not None:\n result_lines.append(json.dumps(self.result.data))\n\n return \"\\n\".join(result_lines)\n\n def report_string(self):\n \"\"\"Get the whole report string.\"\"\"\n return \"\\n\".join([\"=\" * self.width, self.result_string()])\n\n\nclass SingleResultFileReporter(SingleResultReporter):\n def report(self):\n self.width = DEFAULT_SEPARATOR_WIDTH\n report_file = os.path.join(self.result.results_dir, \"report.txt\")\n with open(report_file, \"w\") as fp:\n fp.write(self.report_string())\n\n # write collected data\n if self.result.data is not None:\n data_file = os.path.join(self.result.results_dir, \"data.json\")\n with open(data_file, \"w\") as fp:\n fp.write(json.dumps(self.result.data))\n\n\nclass SummaryReporter(object):\n def __init__(self, results):\n self.results = results\n self.width = get_terminal_size()[0]\n\n def report(self):\n raise NotImplementedError(\"method report must be implemented by subclasses of SummaryReporter\")\n\n\nclass SimpleSummaryReporter(SummaryReporter):\n def header_string(self):\n \"\"\"Header lines of the report\"\"\"\n header_lines = [\n \"=\" * self.width,\n \"SESSION REPORT (ALL TESTS)\",\n \"ducktape version: %s\" % ducktape_version(),\n \"session_id: %s\" % self.results.session_context.session_id,\n \"run time: %s\" % format_time(self.results.run_time_seconds),\n \"tests run: %d\" % len(self.results),\n \"passed: %d\" % self.results.num_passed,\n \"flaky: %d\" % self.results.num_flaky,\n \"failed: %d\" % self.results.num_failed,\n \"ignored: %d\" % self.results.num_ignored,\n \"=\" * self.width,\n ]\n\n return \"\\n\".join(header_lines)\n\n def report_string(self):\n \"\"\"Get the whole report string.\"\"\"\n report_lines = [self.header_string()]\n\n report_lines.extend(\n [SingleResultReporter(result).result_string() + \"\\n\" + \"-\" * self.width for result in self.results]\n )\n\n return \"\\n\".join(report_lines)\n\n\nclass SimpleFileSummaryReporter(SimpleSummaryReporter):\n def report(self):\n self.width = DEFAULT_SEPARATOR_WIDTH\n report_file = os.path.join(self.results.session_context.results_dir, \"report.txt\")\n with open(report_file, \"w\") as fp:\n fp.write(self.report_string())\n\n\nclass SimpleStdoutSummaryReporter(SimpleSummaryReporter):\n def report(self):\n print(self.report_string())\n\n\nclass JSONReporter(object):\n def __init__(self, results):\n self.results = results\n\n def report(self):\n report_file = os.path.abspath(os.path.join(self.results.session_context.results_dir, \"report.json\"))\n with open(report_file, \"w\") as f:\n f.write(\n json.dumps(\n self.results,\n cls=DucktapeJSONEncoder,\n sort_keys=True,\n indent=2,\n separators=(\",\", \": \"),\n )\n )\n\n\nclass JUnitReporter(object):\n def __init__(self, results):\n self.results = results\n\n def report(self):\n report_file = os.path.abspath(os.path.join(self.results.session_context.results_dir, \"report.xml\"))\n testsuites = {}\n\n # First bucket by module_name and argregate counts\n for result in self.results:\n module_name = result.module_name\n testsuites.setdefault(module_name, {})\n # Set default values\n testsuite = testsuites[module_name]\n testsuite.setdefault(\"tests\", 0)\n testsuite.setdefault(\"skipped\", 0)\n testsuite.setdefault(\"failures\", 0)\n testsuite.setdefault(\"errors\", 0)\n testsuite.setdefault(\"testcases\", []).append(result)\n\n # Always increment total number of tests\n testsuite[\"tests\"] += 1\n if result.test_status == FAIL:\n testsuite[\"failures\"] += 1\n elif result.test_status == IGNORE:\n testsuite[\"skipped\"] += 1\n\n total = self.results.num_failed + self.results.num_ignored + self.results.num_passed + self.results.num_flaky\n # Now start building XML document\n root = ET.Element(\n \"testsuites\",\n attrib=dict(\n name=\"ducktape\",\n time=str(self.results.run_time_seconds),\n tests=str(total),\n disabled=\"0\",\n errors=\"0\",\n failures=str(self.results.num_failed),\n ),\n )\n for module_name, testsuite in testsuites.items():\n xml_testsuite = ET.SubElement(\n root,\n \"testsuite\",\n attrib=dict(\n name=module_name,\n tests=str(testsuite[\"tests\"]),\n disabled=\"0\",\n errors=\"0\",\n failures=str(testsuite[\"failures\"]),\n skipped=str(testsuite[\"skipped\"]),\n ),\n )\n for test in testsuite[\"testcases\"]:\n # Since we're already aware of module_name and cls_name, strip that prefix off\n full_name = \"{module_name}.{cls_name}.\".format(module_name=module_name, cls_name=test.cls_name)\n if test.test_id.startswith(full_name):\n name = test.test_id[len(full_name) :]\n else:\n name = test.test_id\n xml_testcase = ET.SubElement(\n xml_testsuite,\n \"testcase\",\n attrib=dict(\n name=name,\n classname=test.cls_name,\n time=str(test.run_time_seconds),\n status=str(test.test_status),\n assertions=\"\",\n ),\n )\n if test.test_status == FAIL:\n xml_failure = ET.SubElement(\n xml_testcase,\n \"failure\",\n attrib=dict(message=test.summary.splitlines()[0]),\n )\n xml_failure.text = test.summary\n elif test.test_status == IGNORE:\n ET.SubElement(xml_testcase, \"skipped\")\n\n with open(report_file, \"w\") as f:\n content = ET.tostring(root)\n if isinstance(content, bytes):\n content = content.decode(\"utf-8\")\n f.write(content)\n\n\nclass HTMLSummaryReporter(SummaryReporter):\n def __init__(self, results, expected_test_count):\n super().__init__(results)\n self.expected_test_count = expected_test_count\n\n def format_test_name(self, result):\n lines = [\n \"Module: \" + result.module_name,\n \"Class: \" + result.cls_name,\n \"Method: \" + result.function_name,\n f\"Nodes (used/allocated): {result.nodes_used}/{result.nodes_allocated}\",\n ]\n\n if result.injected_args is not None:\n lines.append(\"Arguments:\")\n lines.append(\n json.dumps(\n result.injected_args,\n sort_keys=True,\n indent=2,\n separators=(\",\", \": \"),\n )\n )\n\n return \"\\n\".join(lines)\n\n def format_result(self, result):\n test_result = str(result.test_status).lower()\n\n result_json = {\n \"test_name\": self.format_test_name(result),\n \"test_result\": test_result,\n \"description\": result.description,\n \"run_time\": format_time(result.run_time_seconds),\n \"data\": \"\"\n if result.data is None\n else json.dumps(result.data, sort_keys=True, indent=2, separators=(\",\", \": \")),\n \"summary\": result.summary,\n \"test_log\": self.test_results_dir(result),\n }\n return result_json\n\n def test_results_dir(self, result):\n \"\"\"Return *relative path* to test results directory.\n\n Path is relative to the base results_dir. Relative path behaves better if the results directory is copied,\n moved etc.\n \"\"\"\n base_dir = os.path.abspath(result.session_context.results_dir)\n base_dir = os.path.join(base_dir, \"\") # Ensure trailing directory indicator\n\n test_results_dir = os.path.abspath(result.results_dir)\n return test_results_dir[len(base_dir) :] # truncate the \"absolute\" portion\n\n def format_report(self):\n if sys.version_info >= (3, 9):\n import importlib.resources as importlib_resources\n\n template = importlib_resources.files(\"ducktape\").joinpath(\"templates/report/report.html\").read_text(\"utf-8\")\n else:\n import pkg_resources\n\n template = pkg_resources.resource_string(__name__, \"../templates/report/report.html\").decode(\"utf-8\")\n\n num_tests_run = len(self.results)\n num_passes = 0\n failed_result_string = []\n passed_result_string = []\n ignored_result_string = []\n flaky_result_string = []\n\n for result in self.results:\n json_string = json.dumps(self.format_result(result))\n if result.test_status == PASS:\n num_passes += 1\n passed_result_string.append(json_string)\n passed_result_string.append(\",\")\n elif result.test_status == FAIL:\n failed_result_string.append(json_string)\n failed_result_string.append(\",\")\n elif result.test_status == IGNORE:\n ignored_result_string.append(json_string)\n ignored_result_string.append(\",\")\n elif result.test_status == FLAKY:\n flaky_result_string.append(json_string)\n flaky_result_string.append(\",\")\n else:\n raise Exception(\"Unknown test status in report: {}\".format(result.test_status.to_json()))\n\n args = {\n \"ducktape_version\": ducktape_version(),\n \"expected_test_count\": self.expected_test_count,\n \"num_tests_run\": num_tests_run,\n \"num_passes\": self.results.num_passed,\n \"num_flaky\": self.results.num_flaky,\n \"num_failures\": self.results.num_failed,\n \"num_ignored\": self.results.num_ignored,\n \"run_time\": format_time(self.results.run_time_seconds),\n \"session\": self.results.session_context.session_id,\n \"passed_tests\": \"\".join(passed_result_string),\n \"flaky_tests\": \"\".join(flaky_result_string),\n \"failed_tests\": \"\".join(failed_result_string),\n \"ignored_tests\": \"\".join(ignored_result_string),\n \"test_status_names\": \",\".join(f\"'{status}'\" for status in (PASS, FAIL, IGNORE, FLAKY)),\n }\n\n html = template % args\n report_html = os.path.join(self.results.session_context.results_dir, \"report.html\")\n with open(report_html, \"w\") as fp:\n fp.write(html)\n fp.close()\n\n report_css = os.path.join(self.results.session_context.results_dir, \"report.css\")\n\n if sys.version_info >= (3, 9):\n import importlib.resources as importlib_resources\n\n with importlib_resources.as_file(\n importlib_resources.files(\"ducktape\") / \"templates/report/report.css\"\n ) as report_css_origin:\n shutil.copy2(report_css_origin, report_css)\n else:\n import pkg_resources\n\n report_css_origin = pkg_resources.resource_filename(__name__, \"../templates/report/report.css\")\n shutil.copy2(report_css_origin, report_css)\n\n def report(self):\n self.format_report()\n\n\nclass FailedTestSymbolReporter(SummaryReporter):\n def __init__(self, results):\n super().__init__(results)\n self.working_dir = Path().absolute()\n self.separator = \"=\" * self.width\n\n def to_symbol(self, result):\n p = Path(result.file_name).relative_to(self.working_dir)\n line = f\"{p}::{result.cls_name}.{result.function_name}\"\n if result.injected_args:\n injected_args_str = json.dumps(result.injected_args, separators=(\",\", \":\"))\n line += f\"@{injected_args_str}\"\n return line\n\n def dump_test_suite(self, lines):\n print(self.separator)\n print(\"FAILED TEST SUITE\")\n suite = {self.results.session_context.session_id: lines}\n file_path = Path(self.results.session_context.results_dir) / \"rerun-failed.yml\"\n with file_path.open(\"w\") as fp:\n print(f\"Test suite to rerun failed tests: {file_path}\")\n yaml.dump(suite, stream=fp, indent=4)\n\n def print_test_symbols_string(self, lines):\n print(self.separator)\n print(\"FAILED TEST SYMBOLS\")\n print(\"Pass the test symbols below to your ducktape run\")\n # quote the symbol because json parameters will be processed by shell otherwise, making it not copy-pasteable\n print(\" \".join([f\"'{line}'\" for line in lines]))\n\n def report(self):\n symbols = [self.to_symbol(result) for result in self.results if result.test_status == FAIL]\n if not symbols:\n return\n\n self.dump_test_suite(symbols)\n self.print_test_symbols_string(symbols)\n" }, { "path": "ducktape/tests/result.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport json\nimport os\nimport time\nfrom typing import List\n\nfrom ducktape.cluster.cluster import Cluster\nfrom ducktape.cluster.vagrant import VagrantCluster\nfrom ducktape.json_serializable import DucktapeJSONEncoder\nfrom ducktape.tests.reporter import SingleResultFileReporter\nfrom ducktape.tests.session import SessionContext\nfrom ducktape.tests.status import FAIL, FLAKY, IGNORE, PASS\nfrom ducktape.tests.test_context import TestContext\nfrom ducktape.utils.local_filesystem_utils import mkdir_p\nfrom ducktape.utils.util import ducktape_version\n\n\nclass TestResult(object):\n \"\"\"Wrapper class for a single result returned by a single test.\"\"\"\n\n def __init__(\n self,\n test_context,\n test_index,\n session_context,\n test_status=PASS,\n summary=\"\",\n data=None,\n start_time=-1,\n stop_time=-1,\n ):\n \"\"\"\n @param test_context standard test context object\n @param test_status did the test pass or fail, etc?\n @param summary summary information\n @param data data returned by the test, e.g. throughput\n \"\"\"\n self.nodes_allocated = len(test_context.cluster)\n self.nodes_used = test_context.cluster.max_used_nodes\n\n # Collect node_type from cluster metadata if available\n self.node_type = test_context.cluster_use_metadata.get(\"node_type\")\n\n if hasattr(test_context, \"services\"):\n self.services = test_context.services.to_json()\n else:\n self.services = {}\n\n self.test_id = test_context.test_id\n self.module_name = test_context.module_name\n self.cls_name = test_context.cls_name\n self.function_name = test_context.function_name\n self.injected_args = test_context.injected_args\n self.description = test_context.description\n self.results_dir = TestContext.results_dir(test_context, test_index)\n\n self.test_index = test_index\n\n self.session_context = session_context\n self.test_status = test_status\n self.summary = summary\n self.data = data\n self.file_name = test_context.file\n\n self.base_results_dir = session_context.results_dir\n if not self.results_dir.endswith(os.path.sep):\n self.results_dir += os.path.sep\n if not self.base_results_dir.endswith(os.path.sep):\n self.base_results_dir += os.path.sep\n assert self.results_dir.startswith(self.base_results_dir)\n self.relative_results_dir = self.results_dir[len(self.base_results_dir) :]\n\n # For tracking run time\n self.start_time = start_time\n self.stop_time = stop_time\n\n def __repr__(self):\n return \"<%s - test_status:%s, data:%s>\" % (\n self.__class__.__name__,\n self.test_status,\n str(self.data),\n )\n\n @property\n def run_time_seconds(self):\n if self.start_time < 0:\n return -1\n if self.stop_time < 0:\n return time.time() - self.start_time\n\n return self.stop_time - self.start_time\n\n def report(self):\n if not os.path.exists(self.results_dir):\n mkdir_p(self.results_dir)\n\n self.dump_json()\n test_reporter = SingleResultFileReporter(self)\n test_reporter.report()\n\n def dump_json(self):\n \"\"\"Dump this object as json to the given location. By default, dump into self.results_dir/report.json\"\"\"\n with open(os.path.join(self.results_dir, \"report.json\"), \"w\") as fd:\n json.dump(self, fd, cls=DucktapeJSONEncoder, sort_keys=True, indent=2)\n\n def to_json(self):\n result = {\n \"test_id\": self.test_id,\n \"module_name\": self.module_name,\n \"cls_name\": self.cls_name,\n \"function_name\": self.function_name,\n \"injected_args\": self.injected_args,\n \"description\": self.description,\n \"results_dir\": self.results_dir,\n \"relative_results_dir\": self.relative_results_dir,\n \"base_results_dir\": self.base_results_dir,\n \"test_status\": self.test_status,\n \"summary\": self.summary,\n \"data\": self.data,\n \"start_time\": self.start_time,\n \"stop_time\": self.stop_time,\n \"run_time_seconds\": self.run_time_seconds,\n \"nodes_allocated\": self.nodes_allocated,\n \"nodes_used\": self.nodes_used,\n \"services\": self.services,\n }\n\n # Only include node_type if it was specified in the @cluster decorator\n if self.node_type is not None:\n result[\"node_type\"] = self.node_type\n\n return result\n\n\nclass TestResults(object):\n \"\"\"Class used to aggregate individual TestResult objects from many tests.\"\"\"\n\n def __init__(self, session_context: SessionContext, cluster: VagrantCluster, client_status: dict) -> None:\n \"\"\"\n :type session_context: ducktape.tests.session.SessionContext\n \"\"\"\n self._results: List[TestResult] = []\n self.session_context: SessionContext = session_context\n self.cluster: Cluster = cluster\n\n # For tracking total run time\n self.start_time: int = -1\n self.stop_time: int = -1\n self.client_status = client_status\n\n def append(self, obj: TestResult):\n return self._results.append(obj)\n\n def __len__(self):\n return len(self._results)\n\n def __iter__(self):\n return iter(self._results)\n\n @property\n def num_passed(self):\n return len([r for r in self._results if r.test_status == PASS])\n\n @property\n def num_failed(self):\n return len([r for r in self._results if r.test_status == FAIL])\n\n @property\n def num_ignored(self):\n return len([r for r in self._results if r.test_status == IGNORE])\n\n @property\n def num_flaky(self):\n return len([r for r in self._results if r.test_status == FLAKY])\n\n @property\n def run_time_seconds(self):\n if self.start_time < 0:\n return -1\n if self.stop_time < 0:\n self.stop_time = time.time()\n\n return self.stop_time - self.start_time\n\n def get_aggregate_success(self):\n \"\"\"Check cumulative success of all tests run so far\n :rtype: bool\n \"\"\"\n for result in self._results:\n if result.test_status == FAIL:\n return False\n return True\n\n def _stats(self, num_list):\n if len(num_list) == 0:\n return {\"mean\": None, \"min\": None, \"max\": None}\n\n return {\n \"mean\": sum(num_list) / float(len(num_list)),\n \"min\": min(num_list),\n \"max\": max(num_list),\n }\n\n def to_json(self):\n if self.run_time_seconds == 0 or len(self.cluster) == 0:\n # If things go horribly wrong, the test run may be effectively instantaneous\n # Let's handle this case gracefully, and avoid divide-by-zero\n cluster_utilization = 0\n parallelism = 0\n else:\n cluster_utilization = (\n (1.0 / len(self.cluster))\n * (1.0 / self.run_time_seconds)\n * sum([r.nodes_used * r.run_time_seconds for r in self])\n )\n parallelism = sum([r.run_time_seconds for r in self._results]) / self.run_time_seconds\n result = {\n \"ducktape_version\": ducktape_version(),\n \"session_context\": self.session_context,\n \"run_time_seconds\": self.run_time_seconds,\n \"start_time\": self.start_time,\n \"stop_time\": self.stop_time,\n \"run_time_statistics\": self._stats([r.run_time_seconds for r in self]),\n \"cluster_nodes_used\": self._stats([r.nodes_used for r in self]),\n \"cluster_nodes_allocated\": self._stats([r.nodes_allocated for r in self]),\n \"cluster_utilization\": cluster_utilization,\n \"cluster_num_nodes\": len(self.cluster),\n \"num_passed\": self.num_passed,\n \"num_failed\": self.num_failed,\n \"num_ignored\": self.num_ignored,\n \"parallelism\": parallelism,\n \"client_status\": {str(key): value for key, value in self.client_status.items()},\n \"results\": [r for r in self._results],\n }\n if self.num_flaky:\n result[\"num_flaky\"] = self.num_flaky\n return result\n" }, { "path": "ducktape/tests/runner.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nfrom __future__ import annotations\n\nfrom collections import namedtuple, defaultdict\nimport copy\nimport logging\nimport multiprocessing\nimport os\nimport signal\nimport time\nimport traceback\nfrom typing import Dict, List\n\nimport zmq\n\nfrom ducktape.cluster.finite_subcluster import FiniteSubcluster\nfrom ducktape.command_line.defaults import ConsoleDefaults\nfrom ducktape.cluster.vagrant import VagrantCluster\nfrom ducktape.cluster.node_container import InsufficientResourcesError\nfrom ducktape.tests.scheduler import TestScheduler\nfrom ducktape.tests.result import FAIL, TestResult\nfrom ducktape.tests.reporter import (\n SimpleFileSummaryReporter,\n HTMLSummaryReporter,\n JSONReporter,\n JUnitReporter,\n)\nfrom ducktape.utils import persistence\nfrom ducktape.errors import TimeoutError\nfrom ducktape.tests.event import ClientEventFactory, EventResponseFactory\nfrom ducktape.tests.result import TestResults\nfrom ducktape.tests.runner_client import run_client\nfrom ducktape.tests.serde import SerDe\nfrom ducktape.tests.session import SessionContext\nfrom ducktape.tests.test_context import TestContext\nfrom ducktape.utils.terminal_size import get_terminal_size\n\nDEFAULT_MP_JOIN_TIMEOUT = 30\nDEFAULT_TIMEOUT_EXCEPTION_JOIN_TIMEOUT = 120\n\n\nclass Receiver(object):\n def __init__(self, min_port: int, max_port: int) -> None:\n assert min_port <= max_port, \"Expected min_port <= max_port, but instead: min_port: %s, max_port %s\" % (\n min_port,\n max_port,\n )\n self.port = None\n self.min_port = min_port\n self.max_port = max_port\n\n self.serde = SerDe()\n\n self.zmq_context = zmq.Context()\n self.socket = self.zmq_context.socket(zmq.REP)\n\n def start(self):\n \"\"\"Bind to a random port in the range [self.min_port, self.max_port], inclusive\"\"\"\n # note: bind_to_random_port may retry the same port multiple times\n self.port = self.socket.bind_to_random_port(\n addr=\"tcp://*\",\n min_port=self.min_port,\n max_port=self.max_port + 1,\n max_tries=2 * (self.max_port + 1 - self.min_port),\n )\n\n def recv(self, timeout=1800000):\n if timeout is None:\n # use default value of 1800000 or 30 minutes\n timeout = 1800000\n self.socket.RCVTIMEO = timeout\n try:\n message = self.socket.recv()\n except zmq.Again:\n raise TimeoutError(\"runner client unresponsive\")\n return self.serde.deserialize(message)\n\n def send(self, event):\n self.socket.send(self.serde.serialize(event))\n\n def close(self):\n self.socket.setsockopt(zmq.LINGER, 0)\n self.socket.close()\n\n\nTestKey = namedtuple(\"TestKey\", [\"test_id\", \"test_index\"])\n\n\nclass TestRunner(object):\n # When set to True, the test runner will finish running/cleaning the current test, but it will not run any more\n stop_testing = False\n\n def __init__(\n self,\n cluster: VagrantCluster,\n session_context: SessionContext,\n session_logger: logging.Logger,\n tests: List[TestContext],\n deflake_num: int,\n min_port: int = ConsoleDefaults.TEST_DRIVER_MIN_PORT,\n max_port: int = ConsoleDefaults.TEST_DRIVER_MAX_PORT,\n finish_join_timeout: int = DEFAULT_MP_JOIN_TIMEOUT,\n timeout_exception_join_timeout: int = DEFAULT_TIMEOUT_EXCEPTION_JOIN_TIMEOUT,\n ) -> None:\n # Set handler for SIGTERM (aka kill -15)\n # Note: it doesn't work to set a handler for SIGINT (Ctrl-C) in this parent process because the\n # handler is inherited by all forked child processes, and it prevents the default python behavior\n # of translating SIGINT into a KeyboardInterrupt exception\n signal.signal(signal.SIGTERM, self._propagate_sigterm)\n\n # session_logger, message logger,\n self.session_logger = session_logger\n self.cluster = cluster\n self.event_response = EventResponseFactory()\n self.hostname = \"localhost\"\n self.receiver = Receiver(min_port, max_port)\n\n self.deflake_num = deflake_num\n\n self.session_context = session_context\n self.max_parallel = session_context.max_parallel\n self.client_report = defaultdict(dict)\n self.results = TestResults(self.session_context, self.cluster, client_status=self.client_report)\n\n self.exit_first = self.session_context.exit_first\n\n self.main_process_pid = os.getpid()\n self.scheduler = TestScheduler(tests, self.cluster)\n\n self.test_counter = 1\n self.total_tests = len(self.scheduler)\n # This immutable dict tracks test_id -> test_context\n self._test_context = persistence.make_dict(**{t.test_id: t for t in tests})\n self._test_cluster: Dict[TestKey, FiniteSubcluster] = {} # Track subcluster assigned to a particular TestKey\n self._client_procs: Dict[TestKey, multiprocessing.Process] = {} # track client processes running tests\n self.active_tests: Dict[TestKey, bool] = {}\n self.finished_tests: Dict[TestKey, dict] = {}\n self.test_schedule_log: List[TestKey] = []\n self.finish_join_timeout: int = finish_join_timeout\n self.timeout_exception_join_timeout: int = timeout_exception_join_timeout\n\n def _terminate_process(self, process: multiprocessing.Process):\n # use os.kill rather than multiprocessing.terminate for more control\n # This is called after SIGTERM was sent and process didn't exit, so escalate to SIGKILL\n assert process.pid != os.getpid(), \"Signal handler should not reach this point in a client subprocess.\"\n assert process.pid is not None, \"Process has no pid, cannot terminate.\"\n if process.is_alive():\n self._log(logging.WARNING, f\"Process {process.name} did not respond to SIGTERM, escalating to SIGKILL\")\n os.kill(process.pid, signal.SIGKILL)\n\n def _join_test_process(self, process_key, timeout: int = DEFAULT_MP_JOIN_TIMEOUT):\n # waits for process to complete, if it doesn't terminate it\n process: multiprocessing.Process = self._client_procs[process_key]\n start = time.time()\n while time.time() - start <= timeout:\n if not process.is_alive():\n self.client_report[process_key][\"status\"] = \"FINISHED\"\n break\n time.sleep(0.1)\n else:\n # Note: This can lead to some tmp files being uncleaned, otherwise nothing else should be executed by the\n # client after this point.\n self._log(\n logging.ERROR, f\"after waiting {timeout}s, process {process.name} failed to complete. Terminating...\"\n )\n self._terminate_process(process)\n self.client_report[process_key][\"status\"] = \"TERMINATED\"\n process.join()\n self.client_report[process_key][\"exitcode\"] = process.exitcode\n self.client_report[process_key][\"runner_end_time\"] = time.time()\n assert not process.is_alive()\n del self._client_procs[process_key]\n\n def _propagate_sigterm(self, signum, frame):\n \"\"\"Handler SIGTERM and SIGINT by propagating SIGTERM to all client processes.\n\n Note that multiprocessing processes are in the same process group as the main process, so Ctrl-C will\n result in SIGINT being propagated to all client processes automatically. This may result in multiple SIGTERM\n signals getting sent to client processes in quick succession.\n\n However, it is possible that the main process (and not the process group) receives a SIGINT or SIGTERM\n directly. Propagating SIGTERM to client processes is necessary in this case.\n \"\"\"\n if os.getpid() != self.main_process_pid:\n # since we're using the multiprocessing module to create client processes,\n # this signal handler is also attached client processes, so we only want to propagate TERM signals\n # if this process *is* the main runner server process\n return\n\n self.stop_testing = True\n for p in self._client_procs.values():\n self._terminate_process(p)\n\n def who_am_i(self):\n \"\"\"Human-readable name helpful for logging.\"\"\"\n return self.__class__.__name__\n\n @property\n def _ready_to_trigger_more_tests(self):\n \"\"\"Should we pull another test from the scheduler?\"\"\"\n return (\n not self.stop_testing and len(self.active_tests) < self.max_parallel and self.scheduler.peek() is not None\n )\n\n @property\n def _expect_client_requests(self):\n return len(self.active_tests) > 0\n\n def _report_unschedulable(self, unschedulable, err_msg=None):\n if not unschedulable:\n return\n\n self._log(\n logging.ERROR,\n f\"There are {len(unschedulable)} tests which cannot be run due to insufficient cluster resources\",\n )\n for tc in unschedulable:\n if err_msg:\n msg = err_msg\n else:\n msg = (\n f\"Test {tc.test_id} requires more resources than are available in the whole cluster. \"\n f\"{self.cluster.all().nodes.attempt_remove_spec(tc.expected_cluster_spec)}\"\n )\n\n self._log(logging.ERROR, msg)\n\n result = TestResult(\n tc,\n self.test_counter,\n self.session_context,\n test_status=FAIL,\n summary=msg,\n start_time=time.time(),\n stop_time=time.time(),\n )\n self.results.append(result)\n result.report()\n\n self.test_counter += 1\n\n def _check_unschedulable(self):\n self._report_unschedulable(self.scheduler.filter_unschedulable_tests())\n\n def _report_remaining_as_failed(self, reason):\n \"\"\"Mark all remaining tests in the scheduler as failed with the given reason.\"\"\"\n remaining_tests = self.scheduler.drain_remaining_tests()\n if not remaining_tests:\n return\n\n self._log(\n logging.ERROR,\n f\"Marking {len(remaining_tests)} remaining tests as failed: {reason}\",\n )\n for tc in remaining_tests:\n msg = f\"Test not run: {reason}\"\n self._log(logging.ERROR, f\"{tc.test_id}: {msg}\")\n\n result = TestResult(\n tc,\n self.test_counter,\n self.session_context,\n test_status=FAIL,\n summary=msg,\n start_time=time.time(),\n stop_time=time.time(),\n )\n self.results.append(result)\n result.report()\n\n self.test_counter += 1\n\n def _report_active_as_failed(self, reason):\n \"\"\"Mark all currently active/running tests as failed with the given reason.\"\"\"\n active_test_keys = list(self.active_tests.keys())\n if not active_test_keys:\n return\n\n self._log(\n logging.ERROR,\n f\"Marking {len(active_test_keys)} active tests as failed: {reason}\",\n )\n stop_time = time.time()\n for test_key in active_test_keys:\n if hasattr(self, \"_test_cluster\") and test_key in self._test_cluster:\n subcluster = self._test_cluster[test_key]\n # Return nodes to the cluster and remove tracking entry\n self.cluster.free(subcluster.nodes)\n del self._test_cluster[test_key]\n\n tc = self._test_context[test_key.test_id]\n msg = f\"Test timed out: {reason}\"\n self._log(logging.ERROR, f\"{tc.test_id}: {msg}\")\n\n start_time = self.client_report[test_key].get(\"runner_start_time\", stop_time)\n result = TestResult(\n tc,\n test_key.test_index,\n self.session_context,\n test_status=FAIL,\n summary=msg,\n start_time=start_time,\n stop_time=stop_time,\n )\n self.results.append(result)\n result.report()\n\n # Clear active tests since we've reported them all as failed\n self.active_tests.clear()\n\n def run_all_tests(self):\n self.receiver.start()\n self.results.start_time = time.time()\n\n # Report tests which cannot be run\n self._check_unschedulable()\n\n # Run the tests!\n self._log(\n logging.INFO,\n \"starting test run with session id %s...\" % self.session_context.session_id,\n )\n self._log(logging.INFO, \"running %d tests...\" % len(self.scheduler))\n while self._ready_to_trigger_more_tests or self._expect_client_requests:\n try:\n while self._ready_to_trigger_more_tests:\n next_test_context = self.scheduler.peek()\n try:\n self._preallocate_subcluster(next_test_context)\n except InsufficientResourcesError:\n # We were not able to allocate the subcluster for this test,\n # this means not enough nodes passed health check.\n # Don't mark this test as failed just yet, some other test might finish running and\n # free up healthy nodes.\n # However, if some nodes failed, cluster size changed too, so we need to check if\n # there are any tests that can no longer be scheduled.\n self._log(\n logging.INFO,\n f\"Couldn't schedule test context {next_test_context} but we'll keep trying\",\n exc_info=True,\n )\n self._check_unschedulable()\n else:\n # only remove the test from the scheduler once we've successfully allocated a subcluster for it\n self.scheduler.remove(next_test_context)\n self._run_single_test(next_test_context)\n\n if self._expect_client_requests:\n try:\n event = self.receiver.recv(timeout=self.session_context.test_runner_timeout)\n self._handle(event)\n except TimeoutError as e:\n # Handle timeout gracefully - clean up, mark tests as failed, and return results\n err_str = \"Timeout error while receiving message: %s: %s\" % (\n str(type(e)),\n str(e),\n )\n err_str += \"\\n\" + traceback.format_exc(limit=16)\n self._log(logging.ERROR, err_str)\n\n # Send SIGTERM to all client processes immediately to allow graceful cleanup\n # (copy logs, run teardown, etc.)\n for process_key in list(self._client_procs.keys()):\n proc = self._client_procs[process_key]\n if proc.is_alive():\n self._log(logging.INFO, f\"Sending SIGTERM to process {proc.name} for graceful shutdown\")\n os.kill(proc.pid, signal.SIGTERM)\n\n # Wait for processes to shutdown gracefully (in parallel), escalate to SIGKILL if needed\n for process_key in list(self._client_procs.keys()):\n self._join_test_process(process_key, self.timeout_exception_join_timeout)\n self._client_procs = {}\n # Mark active tests as failed with the exception message\n self._report_active_as_failed(str(e))\n\n # Mark all remaining tests as failed with the exception message\n self._report_remaining_as_failed(str(e))\n\n self.receiver.close()\n return self.results\n except Exception as e:\n # For all other exceptions, clean up and re-raise\n err_str = \"Exception receiving message: %s: %s\" % (\n str(type(e)),\n str(e),\n )\n err_str += \"\\n\" + traceback.format_exc(limit=16)\n self._log(logging.ERROR, err_str)\n\n for proc in list(self._client_procs):\n self._join_test_process(proc, self.finish_join_timeout)\n self._client_procs = {}\n raise\n except KeyboardInterrupt:\n # If SIGINT is received, stop triggering new tests, and let the currently running tests finish\n self._log(\n logging.INFO,\n \"Received KeyboardInterrupt. Now waiting for currently running tests to finish...\",\n )\n self.stop_testing = True\n\n # All clients should be cleaned up in their finish block\n if self._client_procs:\n self._log(logging.WARNING, f\"Some clients failed to clean up, waiting 10min to join: {self._client_procs}\")\n for proc in list(self._client_procs):\n self._join_test_process(proc, self.finish_join_timeout)\n\n self.receiver.close()\n\n return self.results\n\n def _run_single_test(self, test_context):\n \"\"\"Start a test runner client in a subprocess\"\"\"\n current_test_counter = self.test_counter\n self.test_counter += 1\n self._log(\n logging.INFO,\n \"Triggering test %d of %d...\" % (current_test_counter, self.total_tests),\n )\n\n # Test is considered \"active\" as soon as we start it up in a subprocess\n test_key = TestKey(test_context.test_id, current_test_counter)\n self.active_tests[test_key] = True\n self.test_schedule_log.append(test_key)\n\n proc = multiprocessing.Process(\n target=run_client,\n args=[\n self.hostname,\n self.receiver.port,\n test_context.test_id,\n current_test_counter,\n TestContext.logger_name(test_context, current_test_counter),\n TestContext.results_dir(test_context, current_test_counter),\n self.session_context.debug,\n self.session_context.fail_bad_cluster_utilization,\n self.deflake_num,\n ],\n )\n\n self._client_procs[test_key] = proc\n proc.start()\n self.client_report[test_key][\"status\"] = \"RUNNING\"\n self.client_report[test_key][\"pid\"] = proc.pid\n self.client_report[test_key][\"name\"] = proc.name\n self.client_report[test_key][\"runner_start_time\"] = time.time()\n\n def _preallocate_subcluster(self, test_context):\n \"\"\"Preallocate the subcluster which will be used to run the test.\n\n Side effect: store association between the test_id and the preallocated subcluster.\n\n :param test_context\n :return None\n \"\"\"\n allocated = self.cluster.alloc(test_context.expected_cluster_spec)\n if len(self.cluster.available()) == 0 and self.max_parallel > 1 and not self._test_cluster:\n self._log(\n logging.WARNING,\n \"Test %s is using entire cluster. It's possible this test has no associated cluster metadata.\"\n % test_context.test_id,\n )\n\n self._test_cluster[TestKey(test_context.test_id, self.test_counter)] = FiniteSubcluster(allocated)\n\n def _handle(self, event):\n self._log(logging.DEBUG, str(event))\n\n if event[\"event_type\"] == ClientEventFactory.READY:\n self._handle_ready(event)\n elif event[\"event_type\"] in [\n ClientEventFactory.RUNNING,\n ClientEventFactory.SETTING_UP,\n ClientEventFactory.TEARING_DOWN,\n ]:\n self._handle_lifecycle(event)\n elif event[\"event_type\"] == ClientEventFactory.FINISHED:\n self._handle_finished(event)\n elif event[\"event_type\"] == ClientEventFactory.LOG:\n self._handle_log(event)\n else:\n raise RuntimeError(\"Received event with unknown event type: \" + str(event))\n\n def _handle_ready(self, event):\n test_key = TestKey(event[\"test_id\"], event[\"test_index\"])\n test_context = self._test_context[event[\"test_id\"]]\n subcluster = self._test_cluster[test_key]\n\n self.receiver.send(self.event_response.ready(event, self.session_context, test_context, subcluster))\n\n def _handle_log(self, event):\n self.receiver.send(self.event_response.log(event))\n self._log(event[\"log_level\"], event[\"message\"])\n\n def _handle_finished(self, event):\n test_key = TestKey(event[\"test_id\"], event[\"test_index\"])\n self.receiver.send(self.event_response.finished(event))\n\n # Idempotency: check if this is a ZMQ retry (test already finished)\n if test_key in self.finished_tests:\n self._log(logging.DEBUG, f\"Received duplicate FINISHED for {test_key} (ZMQ retry), ignoring\")\n return\n\n # Normal case: test is currently active\n if test_key not in self.active_tests:\n # This should never happen - indicates a logic bug\n raise RuntimeError(\n f\"Received FINISHED for test {test_key} which is not in active_tests. \"\n f\"This indicates a bug in the test runner state management.\"\n )\n\n result = event[\"result\"]\n if result.test_status == FAIL and self.exit_first:\n self.stop_testing = True\n\n # Transition this test from running to finished\n del self.active_tests[test_key]\n self.finished_tests[test_key] = event\n self.results.append(result)\n\n # Free nodes used by the test\n subcluster = self._test_cluster[test_key]\n self.cluster.free(subcluster.nodes)\n del self._test_cluster[test_key]\n\n # Join on the finished test process\n self._join_test_process(test_key, timeout=self.finish_join_timeout)\n\n # Report partial result summaries - it is helpful to have partial test reports available if the\n # ducktape process is killed with a SIGKILL partway through\n test_results = copy.copy(self.results) # shallow copy\n reporters = [\n SimpleFileSummaryReporter(test_results),\n HTMLSummaryReporter(test_results, self.total_tests),\n JSONReporter(test_results),\n JUnitReporter(test_results),\n ]\n for r in reporters:\n r.report()\n\n if self._should_print_separator:\n terminal_width, y = get_terminal_size()\n self._log(logging.INFO, \"~\" * int(2 * terminal_width / 3))\n\n @property\n def _should_print_separator(self):\n \"\"\"The separator is the twiddle that goes in between tests on stdout.\n\n This only makes sense to print if tests are run sequentially (aka max_parallel = 1) since\n output from tests is interleaved otherwise.\n\n Also, we don't want to print the separator after the last test output has been received, so\n we check that there's more test output expected.\n \"\"\"\n return self.session_context.max_parallel == 1 and (\n self._expect_client_requests or self._ready_to_trigger_more_tests\n )\n\n def _handle_lifecycle(self, event):\n self.receiver.send(self.event_response._event_response(event))\n\n def _log(self, log_level, msg, *args, **kwargs):\n \"\"\"Log to the service log of the current test.\"\"\"\n self.session_logger.log(log_level, msg, *args, **kwargs)\n" }, { "path": "ducktape/tests/runner_client.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\nimport os\nimport signal\nimport threading\nimport time\nimport traceback\nfrom collections import defaultdict\nfrom typing import List, Mapping, Optional, Tuple\n\nimport zmq\nimport psutil\n\nfrom ducktape.services.service import MultiRunServiceIdFactory, service_id_factory\nfrom ducktape.services.service_registry import ServiceRegistry\nfrom ducktape.tests.event import ClientEventFactory\nfrom ducktape.tests.loader import TestLoader\nfrom ducktape.tests.result import FAIL, IGNORE, PASS, TestResult\nfrom ducktape.tests.serde import SerDe\nfrom ducktape.tests.status import FLAKY, TestStatus\nfrom ducktape.tests.test import Test\nfrom ducktape.tests.test_context import TestContext, test_logger\nfrom ducktape.utils.local_filesystem_utils import mkdir_p\n\n\ndef run_client(*args, **kwargs):\n client = RunnerClient(*args, **kwargs)\n client.ready()\n client.run()\n\n\nclass Sender(object):\n REQUEST_TIMEOUT_MS = 3000\n NUM_RETRIES = 5\n BACKOFF_MULTIPLIER = 2.0 # Exponential backoff multiplier\n\n serde: SerDe\n message_supplier: ClientEventFactory\n server_endpoint: str\n\n zmq_context: zmq.Context\n socket: Optional[zmq.Socket]\n poller: zmq.Poller\n\n def __init__(\n self,\n server_host: str,\n server_port: str,\n message_supplier: ClientEventFactory,\n logger: logging.Logger,\n ):\n self.serde = SerDe()\n self.server_endpoint = \"tcp://%s:%s\" % (str(server_host), str(server_port))\n self.zmq_context = zmq.Context()\n self.socket = None\n self.poller = zmq.Poller()\n\n self.message_supplier = message_supplier\n self.logger = logger\n\n self._init_socket()\n\n def _init_socket(self):\n self.socket = self.zmq_context.socket(zmq.REQ)\n self.socket.connect(self.server_endpoint)\n self.poller.register(self.socket, zmq.POLLIN)\n\n def send(self, event, blocking=True):\n retries_left = Sender.NUM_RETRIES\n current_timeout_ms = Sender.REQUEST_TIMEOUT_MS\n\n while retries_left > 0:\n serialized_event = self.serde.serialize(event)\n self.socket.send(serialized_event)\n retries_left -= 1\n waiting_for_reply = True\n\n while waiting_for_reply:\n sockets = dict(self.poller.poll(current_timeout_ms))\n\n if sockets.get(self.socket) == zmq.POLLIN:\n reply = self.socket.recv()\n if reply:\n return self.serde.deserialize(reply)\n else:\n # send another request...\n break\n else:\n self.close()\n self._init_socket()\n waiting_for_reply = False\n # Apply exponential backoff for next retry\n current_timeout_ms = int(current_timeout_ms * Sender.BACKOFF_MULTIPLIER)\n # Ensure each message we attempt to send has a unique id\n # This copy constructor gives us a duplicate with a new message id\n event = self.message_supplier.copy(event)\n\n raise RuntimeError(\"Unable to receive response from driver\")\n\n def close(self):\n self.socket.setsockopt(zmq.LINGER, 0)\n self.socket.close()\n self.poller.unregister(self.socket)\n\n\nclass RunnerClient(object):\n \"\"\"Run a single test\"\"\"\n\n serde: SerDe\n logger: logging.Logger\n runner_port: int\n message: ClientEventFactory\n sender: Sender\n\n test_id: str\n test_index: int\n id: str\n\n test: Optional[Test]\n test_context: Optional[TestContext]\n all_services: Optional[ServiceRegistry]\n\n # configs\n fail_bad_cluster_utilization: bool\n deflake_num: int\n\n def __init__(\n self,\n server_hostname: str,\n server_port: int,\n test_id: str,\n test_index: int,\n logger_name: str,\n log_dir: str,\n debug: bool,\n fail_bad_cluster_utilization: bool,\n deflake_num: int,\n ):\n signal.signal(signal.SIGTERM, self._sigterm_handler) # register a SIGTERM handler\n\n self.serde = SerDe()\n self.logger = test_logger(logger_name, log_dir, debug)\n self.runner_port = server_port\n\n self.fail_bad_cluster_utilization = fail_bad_cluster_utilization\n self.test_id = test_id\n self.test_index = test_index\n self.id = \"test-runner-%d-%d\" % (os.getpid(), id(self))\n self.message = ClientEventFactory(self.test_id, self.test_index, self.id)\n self.sender = Sender(server_hostname, str(self.runner_port), self.message, self.logger)\n\n self.deflake_num = deflake_num\n\n # Wait to instantiate the test object until running the test\n self.test = None\n self.test_context = None\n self.all_services = None\n self.runner_shutting_down = False\n\n @property\n def deflake_enabled(self) -> bool:\n return self.deflake_num > 1\n\n def ready(self):\n ready_reply = self.sender.send(self.message.ready())\n self.session_context = ready_reply[\"session_context\"]\n self.test_metadata = ready_reply[\"test_metadata\"]\n self.cluster = ready_reply[\"cluster\"]\n\n def send(self, event):\n # Don't attempt to send messages if the runner is shutting down\n # The driver is no longer listening and will timeout anyway\n if self.runner_shutting_down:\n return None\n return self.sender.send(event)\n\n def _kill_all_child_processes(self, send_signal=signal.SIGTERM):\n current_process = psutil.Process()\n # if this client has any children - kill them (for instances background service)\n children = current_process.children(recursive=True)\n for child in children:\n self.logger.warning(f\"process {repr(child)} did not terminate on its own, killing with {send_signal}\")\n child.send_signal(send_signal)\n\n def _sigterm_handler(self, signum, frame):\n \"\"\"Translate SIGTERM to SIGINT on this process\n\n python will treat SIGINT as a Keyboard exception. Exception handling does the rest.\n \"\"\"\n self.runner_shutting_down = True\n self.logger.warning(\"Received SIGTERM, sending SIGINT to self and all child processes\")\n self._kill_all_child_processes(signal.SIGINT)\n os.kill(os.getpid(), signal.SIGINT) # This will send SIGINT to the current process\n\n def _collect_test_context(self, directory, file_name, cls_name, method_name, injected_args):\n loader = TestLoader(\n self.session_context,\n self.logger,\n injected_args=injected_args,\n cluster=self.cluster,\n )\n # TODO: deal with this in a more graceful fashion.\n # In an unlikely even that discover either raises the exception or fails to find exactly one test\n # we should probably continue trying other tests rather than killing this process\n loaded_context_list = loader.discover(directory, file_name, cls_name, method_name)\n\n assert len(loaded_context_list) == 1\n test_context = loaded_context_list[0]\n test_context.cluster = self.cluster\n return test_context\n\n def run(self):\n self.log(logging.INFO, \"Loading test %s\" % str(self.test_metadata))\n self.test_context = self._collect_test_context(**self.test_metadata)\n self.test_context.test_index = self.test_index\n\n self.send(self.message.running())\n if self.test_context.ignore:\n # Skip running this test, but keep track of the fact that we ignored it\n result = TestResult(\n self.test_context,\n self.test_index,\n self.session_context,\n test_status=IGNORE,\n start_time=time.time(),\n stop_time=time.time(),\n )\n result.report()\n # Tell the server we are finished\n self.send(self.message.finished(result=result))\n return\n\n start_time = -1\n stop_time = -1\n test_status = FAIL\n data = None\n self.all_services = ServiceRegistry(enable_jvm_logs=self.session_context.enable_jvm_logs)\n\n summaries = []\n num_runs = 0\n\n try:\n while test_status == FAIL and num_runs < self.deflake_num:\n num_runs += 1\n self.log(logging.INFO, \"on run {}/{}\".format(num_runs, self.deflake_num))\n start_time = time.time()\n test_status, run_summary, data = self._do_run(num_runs)\n if run_summary:\n summaries.append(run_summary)\n\n # dump threads after the test is complete;\n # if any thread is not terminated correctly by the test we'll see it here\n self.dump_threads(f\"Threads after {self.test_id} finished\")\n\n # if run passed, and not on the first run, the test is flaky\n if test_status == PASS and num_runs > 1:\n test_status = FLAKY\n\n msg = str(test_status.to_json())\n if run_summary:\n msg += \": {}\".format(\"\\n\".join(run_summary))\n self.log(logging.INFO, msg)\n\n finally:\n stop_time = time.time()\n summary = self.process_run_summaries(summaries, test_status)\n test_status, summary = self._check_cluster_utilization(test_status, summary)\n # convert summary from list to string\n summary = \"\\n\".join(summary)\n if num_runs > 1:\n # for reporting purposes report all services\n self.test_context.services = self.all_services\n # for flaky tests, we report the start and end time of the successful run, and not the whole run period\n result = TestResult(\n self.test_context,\n self.test_index,\n self.session_context,\n test_status,\n summary,\n data,\n start_time,\n stop_time,\n )\n\n self.log(logging.INFO, \"Data: %s\" % str(result.data))\n\n result.report()\n # Tell the server we are finished\n self._do_safely(\n lambda: self.send(self.message.finished(result=result)),\n f\"Problem sending FINISHED message for {self.test_metadata}:\\n\",\n )\n self._kill_all_child_processes()\n # Release test_context resources only after creating the result and finishing logging activity\n # The Sender object uses the same logger, so we postpone closing until after the finished message is sent\n self.test_context.close()\n self.all_services = None\n self.test_context = None\n self.test = None\n\n def process_run_summaries(self, run_summaries: List[List[str]], test_status: TestStatus) -> List[str]:\n \"\"\"\n Converts individual run summaries (there may be multiple if deflake is enabled)\n into a single run summary\n \"\"\"\n # no summary case, return test passed\n if not run_summaries:\n return [\"Test Passed\"]\n # single run, can just return the summary\n if not self.deflake_enabled:\n return run_summaries[0]\n\n failure_summaries: Mapping[Tuple[str, ...], List[int]] = defaultdict(list)\n # populate run summaries grouping run numbers by stack trace\n for run_num, summary in enumerate(run_summaries):\n # convert to tuple to be serializable (+1 for human readability 1 based indexing)\n failure_summaries[tuple(summary)].append(run_num + 1)\n\n final_summary = []\n\n # handle run summaries for each deflake run:\n sub_summaries = []\n for individual_summary, runs in failure_summaries.items():\n sub_summary = []\n runs_str = \", \".join(str(r) for r in runs)\n run_msg = f\"run{'s' if len(runs) > 1 else ''} {runs_str} summary:\"\n sub_summary.append(run_msg)\n sub_summary.extend(individual_summary)\n sub_summaries.append(sub_summary)\n\n if test_status == FLAKY:\n sub_summaries.append([f\"run {len(run_summaries)}: PASSED\"])\n\n # combine summaries, with a '~~~~~' divider\n for sub_summary in sub_summaries[:-1]:\n final_summary.extend(sub_summary)\n break_line = \"~\" * max(len(line) for line in final_summary) if final_summary else \"\"\n final_summary.append(break_line)\n\n # the pass case could have no summaries, so need to validate that a subsummary exists\n if sub_summaries:\n final_summary.extend(sub_summaries[-1])\n\n return final_summary\n\n def _do_run(self, num_runs):\n test_status = FAIL\n summary = []\n data = None\n sid_factory = MultiRunServiceIdFactory(num_runs) if self.deflake_enabled else service_id_factory\n try:\n # Results from this test, as well as logs will be dumped here\n mkdir_p(TestContext.results_dir(self.test_context, self.test_index))\n # Instantiate test\n self.test = self.test_context.cls(self.test_context)\n\n # Run the test unit\n self.setup_test()\n data = self.run_test()\n test_status = PASS\n\n except BaseException as e:\n # mark the test as failed before doing anything else\n test_status = FAIL\n err_trace = self._exc_msg(e)\n summary.extend(err_trace.split(\"\\n\"))\n\n finally:\n for service in self.test_context.services:\n service.service_id_factory = sid_factory\n self.all_services.append(service)\n\n self.teardown_test(\n teardown_services=not self.session_context.no_teardown,\n test_status=test_status,\n )\n\n if hasattr(self.test_context, \"services\"):\n service_errors = self.test_context.services.errors()\n if service_errors:\n summary.extend([\"\", \"\", service_errors])\n\n # free nodes\n if self.test:\n self.log(logging.DEBUG, \"Freeing nodes...\")\n self._do_safely(self.test.free_nodes, \"Error freeing nodes:\")\n return test_status, summary, data\n\n def _check_cluster_utilization(self, result, summary):\n \"\"\"Checks if the number of nodes used by a test is less than the number of\n nodes requested by the test. If this is the case and we wish to fail\n on bad cluster utilization, the result value is failed. Will also print\n a warning if the test passes and the node utilization doesn't match.\n \"\"\"\n max_used = self.cluster.max_used()\n total = len(self.cluster.all())\n if max_used < total:\n message = \"Test requested %d nodes, used only %d\" % (total, max_used)\n if self.fail_bad_cluster_utilization:\n # only check node utilization on test pass\n if result == PASS or result == FLAKY:\n self.log(logging.INFO, \"FAIL: \" + message)\n\n result = FAIL\n summary.append(message)\n else:\n self.log(logging.WARN, message)\n return result, summary\n\n def setup_test(self):\n \"\"\"start services etc\"\"\"\n self.log(logging.INFO, \"Setting up...\")\n self.test.setup()\n\n def run_test(self):\n \"\"\"Run the test!\n\n We expect test_context.function to be a function or unbound method which takes an\n instantiated test object as its argument.\n \"\"\"\n self.log(logging.INFO, \"Running...\")\n return self.test_context.function(self.test)\n\n def _exc_msg(self, e):\n return repr(e) + \"\\n\" + traceback.format_exc(limit=16)\n\n def _do_safely(self, action, err_msg):\n try:\n action()\n except BaseException as e:\n self.log(logging.WARN, err_msg + \" \" + self._exc_msg(e))\n\n def teardown_test(self, teardown_services=True, test_status=None):\n \"\"\"teardown method which stops services, gathers log data, removes persistent state, and releases cluster nodes.\n\n Catch all exceptions so that every step in the teardown process is tried, but signal that the test runner\n should stop if a keyboard interrupt is caught.\n \"\"\"\n self.log(logging.INFO, \"Tearing down...\")\n if not self.test:\n self.log(logging.WARN, \"%s failed to instantiate\" % self.test_id)\n self.test_context.close()\n return\n\n services = self.test_context.services\n\n if teardown_services:\n self._do_safely(self.test.teardown, \"Error running teardown method:\")\n # stop services\n self._do_safely(services.stop_all, \"Error stopping services:\")\n\n # always collect service logs whether or not we tear down\n # logs are typically removed during \"clean\" phase, so collect logs before cleaning\n self.log(logging.DEBUG, \"Copying logs from services...\")\n self._do_safely(\n lambda: self.test.copy_service_logs(test_status),\n \"Error copying service logs:\",\n )\n\n # clean up stray processes and persistent state\n if teardown_services:\n self.log(logging.DEBUG, \"Cleaning up services...\")\n self._do_safely(services.clean_all, \"Error cleaning services:\")\n\n def log(self, log_level, msg, *args, **kwargs):\n \"\"\"Log to the service log and the test log of the current test.\"\"\"\n\n if self.test_context is None:\n msg = \"%s: %s\" % (self.__class__.__name__, str(msg))\n self.logger.log(log_level, msg, *args, **kwargs)\n else:\n msg = \"%s: %s: %s\" % (\n self.__class__.__name__,\n self.test_context.test_name,\n str(msg),\n )\n self.logger.log(log_level, msg, *args, **kwargs)\n\n self.send(self.message.log(msg, level=log_level))\n\n def dump_threads(self, msg):\n dump = \"\\n\".join([t.name for t in threading.enumerate()])\n self.log(logging.DEBUG, f\"{msg}: {dump}\")\n" }, { "path": "ducktape/tests/scheduler.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nfrom typing import List\n\nfrom ducktape.cluster.vagrant import VagrantCluster\nfrom ducktape.tests.test_context import TestContext\n\n\nclass TestScheduler(object):\n \"\"\"This class tracks tests which are scheduled to run, and provides an ordering based on the current cluster state.\n\n The ordering is \"on-demand\"; calling next returns the largest cluster user which fits in the currently\n available cluster nodes.\n \"\"\"\n\n def __init__(self, test_contexts: List[TestContext], cluster: VagrantCluster) -> None:\n self.cluster = cluster\n\n # Track tests which would never be offered up by the scheduling algorithm due to insufficient\n # cluster resources\n self._test_context_list = test_contexts.copy()\n\n self._sort_test_context_list()\n\n def __len__(self) -> int:\n \"\"\"Number of tests currently in the scheduler\"\"\"\n return len(self._test_context_list)\n\n def __iter__(self):\n return self\n\n def filter_unschedulable_tests(self):\n \"\"\"\n Filter out tests that cannot be scheduled with the current cluster, remove them from\n this scheduler and return them.\n \"\"\"\n all = self.cluster.all()\n unschedulable = []\n for test_context in self._test_context_list:\n if not all.nodes.can_remove_spec(test_context.expected_cluster_spec):\n unschedulable.append(test_context)\n for u in unschedulable:\n self._test_context_list.remove(u)\n return unschedulable\n\n def _sort_test_context_list(self) -> None:\n \"\"\"Replace self.test_context_list with a sorted shallow copy\n\n Sort from largest cluster users to smallest\n \"\"\"\n # sort from the largest cluster users to smallest\n self._test_context_list = sorted(self._test_context_list, key=lambda tc: tc.expected_num_nodes, reverse=True)\n\n def peek(self):\n \"\"\"Locate and return the next object to be scheduled, without removing it internally.\n\n :return test_context for the next test to be scheduled.\n If scheduler is empty, or no test can currently be scheduled, return None.\n \"\"\"\n for tc in self._test_context_list:\n if self.cluster.available().nodes.can_remove_spec(tc.expected_cluster_spec):\n return tc\n\n return None\n\n def remove(self, tc):\n \"\"\"Remove test context object from this scheduler.\n Intended usage is to peek() first, then perform whatever validity checks,\n and if they pass, remove() it from the scheduler.\n \"\"\"\n if tc:\n self._test_context_list.remove(tc)\n\n def drain_remaining_tests(self):\n \"\"\"\n Get all remaining tests in the scheduler and clear the scheduler.\n\n :return List of test contexts that were still in the scheduler\n \"\"\"\n remaining = self._test_context_list.copy()\n self._test_context_list.clear()\n return remaining\n" }, { "path": "ducktape/tests/serde.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport pickle\n\n\nclass SerDe(object):\n def serialize(self, obj):\n if hasattr(obj, \"serialize\"):\n obj.serialize()\n else:\n return pickle.dumps(obj)\n\n def deserialize(self, bytes_obj, obj_cls=None):\n if obj_cls and hasattr(obj_cls, \"deserialize\"):\n return obj_cls.deserialize(bytes_obj)\n else:\n return pickle.loads(bytes_obj)\n" }, { "path": "ducktape/tests/session.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\nimport os\nimport sys\nimport time\n\nfrom ducktape.command_line.defaults import ConsoleDefaults\nfrom ducktape.tests.loggermaker import LoggerMaker\n\n\nclass SessionContext(object):\n \"\"\"Wrapper class for 'global' variables. A call to ducktape generates a single shared SessionContext object\n which helps route logging and reporting, etc.\n \"\"\"\n\n def __init__(self, **kwargs) -> None:\n # session_id, results_dir, cluster, globals):\n self.session_id = kwargs[\"session_id\"]\n self.results_dir = os.path.abspath(kwargs[\"results_dir\"])\n\n self.debug = kwargs.get(\"debug\", False)\n self.compress = kwargs.get(\"compress\", False)\n self.exit_first = kwargs.get(\"exit_first\", False)\n self.no_teardown = kwargs.get(\"no_teardown\", False)\n self.max_parallel = kwargs.get(\"max_parallel\", 1)\n self.default_expected_num_nodes = kwargs.get(\"default_num_nodes\", None)\n self.fail_bad_cluster_utilization = kwargs.get(\"fail_bad_cluster_utilization\")\n self.fail_greedy_tests = kwargs.get(\"fail_greedy_tests\", False)\n self.test_runner_timeout = kwargs.get(\"test_runner_timeout\")\n self.enable_jvm_logs = kwargs.get(\"enable_jvm_logs\", False)\n self._globals = kwargs.get(\"globals\")\n\n @property\n def globals(self):\n \"\"\"None, or an immutable dictionary containing user-defined global variables.\"\"\"\n return self._globals\n\n def to_json(self):\n return self.__dict__\n\n\nclass SessionLoggerMaker(LoggerMaker):\n def __init__(self, session_context: SessionContext) -> None:\n super(SessionLoggerMaker, self).__init__(session_context.session_id + \".session_logger\")\n self.log_dir = session_context.results_dir\n self.debug = session_context.debug\n\n def configure_logger(self) -> None:\n \"\"\"Set up the logger to log to stdout and files. This creates a few files as a side-effect.\"\"\"\n if self.configured:\n return\n\n self._logger.setLevel(logging.DEBUG)\n\n fh_info = logging.FileHandler(os.path.join(self.log_dir, \"session_log.info\"))\n fh_debug = logging.FileHandler(os.path.join(self.log_dir, \"session_log.debug\"))\n fh_info.setLevel(logging.INFO)\n fh_debug.setLevel(logging.DEBUG)\n\n # create console handler with a higher log level\n ch = logging.StreamHandler(sys.stdout)\n ch.setLevel(logging.DEBUG if self.debug else logging.INFO)\n\n # create formatter and add it to the handlers\n formatter = logging.Formatter(ConsoleDefaults.SESSION_LOG_FORMATTER)\n fh_info.setFormatter(formatter)\n fh_debug.setFormatter(formatter)\n ch.setFormatter(formatter)\n\n # add the handlers to the logger\n self._logger.addHandler(fh_info)\n self._logger.addHandler(fh_debug)\n self._logger.addHandler(ch)\n\n\ndef generate_session_id(session_id_file: str) -> str:\n \"\"\"Generate a new session id based on the previous session id found in session_id_file\n :type session_id_file: str Last-used session_id is in this file\n :rtype str New session_id\n \"\"\"\n\n def get_id(day, num):\n return day + \"--%03d\" % num\n\n def split_id(an_id):\n day = an_id[:10]\n num = int(an_id[12:])\n return day, num\n\n def today():\n return time.strftime(\"%Y-%m-%d\")\n\n def next_id(prev_id):\n if prev_id is None:\n prev_day = today()\n prev_num = 0\n else:\n prev_day, prev_num = split_id(prev_id)\n\n if prev_day == today():\n next_day = prev_day\n next_num = prev_num + 1\n else:\n next_day = today()\n next_num = 1\n\n return get_id(next_day, next_num)\n\n if os.path.isfile(session_id_file):\n with open(session_id_file, \"r\") as fp:\n session_id = next_id(fp.read())\n else:\n session_id = next_id(None)\n\n with open(session_id_file, \"w\") as fp:\n fp.write(session_id)\n\n return session_id\n\n\ndef generate_results_dir(results_root: str, session_id: str) -> str:\n \"\"\"Results from a single run of ducktape are assigned a session_id and put together in this directory.\n\n :type session_id: str\n :rtype: str\n \"\"\"\n return os.path.join(os.path.abspath(results_root), session_id)\n" }, { "path": "ducktape/tests/status.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nclass TestStatus(object):\n def __init__(self, status: str) -> None:\n self._status = str(status).lower()\n\n def __eq__(self, other):\n return str(self).lower() == str(other).lower()\n\n def __str__(self):\n return self._status\n\n def to_json(self):\n return str(self).upper()\n\n\nPASS = TestStatus(\"pass\")\nFLAKY = TestStatus(\"flaky\")\nFAIL = TestStatus(\"fail\")\nIGNORE = TestStatus(\"ignore\")\n" }, { "path": "ducktape/tests/test.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os\nimport shutil\nimport tempfile\nfrom contextlib import contextmanager\n\nfrom ducktape.template import TemplateRenderer\nfrom ducktape.tests.status import FAIL\nfrom ducktape.utils.local_filesystem_utils import mkdir_p\n\nfrom .test_context import TestContext\n\n\nclass Test(TemplateRenderer):\n \"\"\"Base class for tests.\"\"\"\n\n def __init__(self, test_context, *args, **kwargs):\n \"\"\"\n :type test_context: ducktape.tests.test.TestContext\n \"\"\"\n super(Test, self).__init__(*args, **kwargs)\n self.test_context = test_context\n\n @property\n def cluster(self):\n return self.test_context.cluster\n\n @property\n def logger(self):\n return self.test_context.logger\n\n def min_cluster_spec(self):\n \"\"\"\n THIS METHOD IS DEPRECATED AND WILL BE REMOVED IN THE SUBSEQUENT RELEASES.\n Nothing in the ducktape framework calls it, it is only provided so that subclasses don't break.\n If you're overriding this method in your subclass, please remove it.\n \"\"\"\n raise NotImplementedError\n\n def min_cluster_size(self):\n \"\"\"\n THIS METHOD IS DEPRECATED AND WILL BE REMOVED IN THE SUBSEQUENT RELEASES.\n Nothing in the ducktape framework calls it, it is only provided so that subclasses don't break.\n If you're overriding this method in your subclass, please remove it.\n \"\"\"\n raise NotImplementedError\n\n def setup(self):\n \"\"\"Override this for custom setup logic.\"\"\"\n\n # for backward compatibility\n self.setUp()\n\n def teardown(self):\n \"\"\"Override this for custom teardown logic.\"\"\"\n\n # for backward compatibility\n self.tearDown()\n\n def setUp(self):\n pass\n\n def tearDown(self):\n pass\n\n def free_nodes(self):\n try:\n self.test_context.services.free_all()\n except BaseException as e:\n if isinstance(e, KeyboardInterrupt):\n raise\n\n def compress_service_logs(self, node, service, node_logs):\n \"\"\"Compress logs on a node corresponding to the given service.\n\n :param node: The node on which to compress the given logs\n :param service: The service to which the node belongs\n :param node_logs: Paths to logs (or log directories) which will be compressed\n :return: a list of paths to compressed logs.\n\n \"\"\"\n compressed_logs = []\n for nlog in node_logs:\n try:\n node.account.ssh(_compress_cmd(nlog))\n if nlog.endswith(os.path.sep):\n nlog = nlog[: -len(os.path.sep)]\n nlog += \".tgz\"\n compressed_logs.append(nlog)\n\n except Exception as e:\n self.test_context.logger.warn(\"Error compressing log %s: service %s: %s\" % (nlog, service, str(e)))\n\n return compressed_logs\n\n def copy_service_logs(self, test_status):\n \"\"\"\n Copy logs from service nodes to the results directory.\n\n If the test passed, only the default set will be collected. If the the test failed, all logs will be collected.\n \"\"\"\n for service in self.test_context.services:\n if not hasattr(service, \"logs\") or len(service.logs) == 0:\n self.test_context.logger.debug(\n \"Won't collect service logs from %s - no logs to collect.\" % service.service_id\n )\n continue\n\n log_dirs = service.logs\n for node in service.nodes:\n # Gather locations of logs to collect\n node_logs = []\n for log_name in log_dirs.keys():\n if test_status == FAIL or self.should_collect_log(log_name, service):\n node_logs.append(log_dirs[log_name][\"path\"])\n\n self.test_context.logger.debug(\n \"Preparing to copy logs from %s: %s\" % (node.account.hostname, node_logs)\n )\n\n if self.test_context.session_context.compress:\n self.test_context.logger.debug(\"Compressing logs...\")\n node_logs = self.compress_service_logs(node, service, node_logs)\n\n if len(node_logs) > 0:\n # Create directory into which service logs will be copied\n dest = os.path.join(\n TestContext.results_dir(self.test_context, self.test_context.test_index),\n service.service_id,\n node.account.hostname,\n )\n if not os.path.isdir(dest):\n mkdir_p(dest)\n\n # Try to copy the service logs\n self.test_context.logger.debug(\"Copying logs...\")\n try:\n for log in node_logs:\n node.account.copy_from(log, dest)\n except Exception as e:\n self.test_context.logger.warn(\n \"Error copying log %(log_name)s from %(source)s to %(dest)s. \\\n service %(service)s: %(message)s\"\n % {\n \"log_name\": log_name,\n \"source\": log_dirs[log_name],\n \"dest\": dest,\n \"service\": service,\n \"message\": e,\n }\n )\n\n def mark_for_collect(self, service, log_name=None):\n if log_name is None:\n # Mark every log for collection\n for log_name in service.logs:\n self.test_context.log_collect[(log_name, service)] = True\n else:\n self.test_context.log_collect[(log_name, service)] = True\n\n def mark_no_collect(self, service, log_name=None):\n self.test_context.log_collect[(log_name, service)] = False\n\n def should_collect_log(self, log_name, service):\n key = (log_name, service)\n default = service.logs[log_name][\"collect_default\"]\n val = self.test_context.log_collect.get(key, default)\n return val\n\n\ndef _compress_cmd(log_path):\n \"\"\"Return bash command which compresses the given path to a tarball.\"\"\"\n compres_cmd = 'cd \"$(dirname %s)\" && ' % log_path\n compres_cmd += 'f=\"$(basename %s)\" && ' % log_path\n compres_cmd += 'if [ -e \"$f\" ]; then tar czf \"$f.tgz\" \"$f\"; fi && '\n compres_cmd += \"rm -rf %s\" % log_path\n\n return compres_cmd\n\n\n@contextmanager\ndef in_dir(path):\n \"\"\"Changes working directory to given path. On exit, restore to original working directory.\"\"\"\n cwd = os.getcwd()\n\n try:\n os.chdir(path)\n yield\n\n finally:\n os.chdir(cwd)\n\n\n@contextmanager\ndef in_temp_dir():\n \"\"\"Creates a temporary directory as the working directory. On exit, it is removed.\"\"\"\n with _new_temp_dir() as tmpdir:\n with in_dir(tmpdir):\n yield tmpdir\n\n\n@contextmanager\ndef _new_temp_dir():\n \"\"\"Create a temporary directory that is removed automatically\"\"\"\n tmpdir = tempfile.mkdtemp()\n\n try:\n yield tmpdir\n\n finally:\n shutil.rmtree(tmpdir)\n" }, { "path": "ducktape/tests/test_context.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport copy\nimport logging\nimport os\nimport re\nimport shutil\nimport sys\nimport tempfile\nfrom typing import TYPE_CHECKING, Dict, Optional, Tuple\n\nfrom ducktape.cluster.cluster_spec import ClusterSpec\nfrom ducktape.command_line.defaults import ConsoleDefaults\nfrom ducktape.mark.consts import CLUSTER_NODE_TYPE_KEYWORD, CLUSTER_SIZE_KEYWORD, CLUSTER_SPEC_KEYWORD\nfrom ducktape.services.service_registry import ServiceRegistry\nfrom ducktape.tests.loggermaker import LoggerMaker, close_logger\nfrom ducktape.tests.session import SessionContext\nfrom ducktape.utils.local_filesystem_utils import mkdir_p\n\nif TYPE_CHECKING:\n from ducktape.services.service import Service\n\n\ndef _escape_pathname(s):\n \"\"\"Remove fishy characters, replace most with dots\"\"\"\n # Remove all whitespace completely\n s = re.sub(r\"\\s+\", \"\", s)\n\n # Replace bad characters with dots\n blacklist = r\"[^\\.\\-=_\\w\\d]+\"\n s = re.sub(blacklist, \".\", s)\n\n # Multiple dots -> single dot (and no leading or trailing dot)\n s = re.sub(r\"[\\.]+\", \".\", s)\n return re.sub(r\"^\\.|\\.$\", \"\", s)\n\n\nclass TestLoggerMaker(LoggerMaker):\n def __init__(self, logger_name, log_dir, debug):\n super(TestLoggerMaker, self).__init__(logger_name)\n self.log_dir = log_dir\n self.debug = debug\n\n def configure_logger(self):\n \"\"\"Set up the logger to log to stdout and files.\n This creates a directory and a few files as a side-effect.\n \"\"\"\n if self.configured:\n return\n\n self._logger.setLevel(logging.DEBUG)\n mkdir_p(self.log_dir)\n\n # Create info and debug level handlers to pipe to log files\n info_fh = logging.FileHandler(os.path.join(self.log_dir, \"test_log.info\"))\n debug_fh = logging.FileHandler(os.path.join(self.log_dir, \"test_log.debug\"))\n\n info_fh.setLevel(logging.INFO)\n debug_fh.setLevel(logging.DEBUG)\n\n formatter = logging.Formatter(ConsoleDefaults.TEST_LOG_FORMATTER)\n info_fh.setFormatter(formatter)\n debug_fh.setFormatter(formatter)\n\n self._logger.addHandler(info_fh)\n self._logger.addHandler(debug_fh)\n\n ch = logging.StreamHandler(sys.stdout)\n ch.setFormatter(formatter)\n if self.debug:\n # If debug flag is set, pipe debug logs to stdout\n ch.setLevel(logging.DEBUG)\n else:\n # default - pipe warning level logging to stdout\n ch.setLevel(logging.WARNING)\n self._logger.addHandler(ch)\n\n\ndef test_logger(logger_name, log_dir, debug):\n \"\"\"Helper method for getting a test logger object\n\n Note that if this method is called multiple times with the same ``logger_name``, it returns the same logger object.\n Note also, that for a fixed ``logger_name``, configuration occurs only the first time this function is called.\n \"\"\"\n return TestLoggerMaker(logger_name, log_dir, debug).logger\n\n\nclass TestContext(object):\n \"\"\"Wrapper class for state variables needed to properly run a single 'test unit'.\"\"\"\n\n def __init__(self, **kwargs) -> None:\n \"\"\"\n :param session_context:\n :param cluster: the cluster object which will be used by this test\n :param module: name of the module containing the test class/method\n :param cls: class object containing the test method\n :param function: the test method\n :param file: file containing this module\n :param injected_args: a dict containing keyword args which will be passed to the test method\n :param cluster_use_metadata: dict containing information about how this test will use cluster resources\n \"\"\"\n\n self.session_context: SessionContext = kwargs[\"session_context\"]\n self.cluster = kwargs.get(\"cluster\")\n self.module = kwargs.get(\"module\")\n self.test_suite_name = kwargs.get(\"test_suite_name\")\n\n if kwargs.get(\"file\") is not None:\n self.file = os.path.abspath(kwargs[\"file\"])\n else:\n self.file = None\n self.cls = kwargs.get(\"cls\")\n self.function = kwargs.get(\"function\")\n self.injected_args = kwargs.get(\"injected_args\")\n self.ignore = kwargs.get(\"ignore\", False)\n\n # cluster_use_metadata is a dict containing information about how this test will use cluster resources\n self.cluster_use_metadata = copy.copy(kwargs.get(\"cluster_use_metadata\", {}))\n\n enable_jvm_logs = self.session_context.enable_jvm_logs if self.session_context else False\n self.services = ServiceRegistry(enable_jvm_logs=enable_jvm_logs)\n self.test_index = None\n\n # dict for toggling service log collection on/off\n self.log_collect: Dict[Tuple[str, Service], bool] = {}\n\n self._logger = None\n self._local_scratch_dir = None\n\n def __repr__(self) -> str:\n return (\n f\"\"\n )\n\n def copy(self, **kwargs) -> \"TestContext\":\n \"\"\"Construct a new TestContext object from another TestContext object\n Note that this is not a true copy, since a fresh ServiceRegistry instance will be created.\n \"\"\"\n ctx_copy = TestContext(**self.__dict__)\n ctx_copy.__dict__.update(**kwargs)\n\n return ctx_copy\n\n @property\n def local_scratch_dir(self):\n \"\"\"This local scratch directory is created/destroyed on the test driver before/after each test is run.\"\"\"\n if not self._local_scratch_dir:\n self._local_scratch_dir = tempfile.mkdtemp()\n return self._local_scratch_dir\n\n @property\n def test_metadata(self):\n return {\n \"directory\": os.path.dirname(self.file),\n \"file_name\": os.path.basename(self.file),\n \"cls_name\": self.cls_name,\n \"method_name\": self.function_name,\n \"injected_args\": self.injected_args,\n }\n\n @staticmethod\n def logger_name(test_context, test_index):\n if test_index is None:\n return test_context.test_id\n else:\n return \"%s-%s\" % (test_context.test_id, str(test_index))\n\n @staticmethod\n def results_dir(test_context, test_index):\n d = test_context.session_context.results_dir\n\n if test_context.cls is not None:\n d = os.path.join(d, test_context.cls.__name__)\n if test_context.function is not None:\n d = os.path.join(d, test_context.function.__name__)\n if test_context.injected_args is not None:\n d = os.path.join(d, test_context.injected_args_name)\n if test_index is not None:\n d = os.path.join(d, str(test_index))\n\n return d\n\n @property\n def expected_num_nodes(self) -> int:\n \"\"\"\n How many nodes of any type we expect this test to consume when run.\n Note that this will be 0 for both unschedulable tests and the tests that legitimately need 0 nodes.\n\n :return: an integer number of nodes.\n \"\"\"\n return self.expected_cluster_spec.size() if self.expected_cluster_spec else 0\n\n @property\n def expected_cluster_spec(self) -> Optional[ClusterSpec]:\n \"\"\"\n The cluster spec we expect this test to consume when run.\n\n :return: A ClusterSpec object or None if the test cannot be run\n (e.g. session context settings disallow tests with no cluster metadata attached).\n \"\"\"\n cluster_spec = self.cluster_use_metadata.get(CLUSTER_SPEC_KEYWORD)\n cluster_size = self.cluster_use_metadata.get(CLUSTER_SIZE_KEYWORD)\n node_type = self.cluster_use_metadata.get(CLUSTER_NODE_TYPE_KEYWORD)\n if cluster_spec is not None:\n return cluster_spec\n elif cluster_size is not None:\n return ClusterSpec.simple_linux(cluster_size, node_type)\n elif not self.cluster:\n return ClusterSpec.empty()\n elif self.session_context.fail_greedy_tests:\n return None\n else:\n return self.cluster.all()\n\n @property\n def globals(self):\n return self.session_context.globals\n\n @property\n def module_name(self) -> str:\n return \"\" if self.module is None else self.module\n\n @property\n def cls_name(self) -> str:\n return \"\" if self.cls is None else self.cls.__name__\n\n @property\n def function_name(self) -> str:\n return \"\" if self.function is None else self.function.__name__\n\n @property\n def description(self):\n \"\"\"Description of the test, needed in particular for reporting.\n If the function has a docstring, return that, otherwise return the class docstring or \"\".\n \"\"\"\n if self.function.__doc__:\n return self.function.__doc__\n elif self.cls.__doc__ is not None:\n return self.cls.__doc__\n else:\n return \"\"\n\n @property\n def injected_args_name(self) -> str:\n if self.injected_args is None:\n return \"\"\n else:\n params = \".\".join([\"%s=%s\" % (k, self.injected_args[k]) for k in self.injected_args])\n return _escape_pathname(params)\n\n @property\n def test_id(self) -> str:\n return self.test_name\n\n @property\n def test_name(self) -> str:\n \"\"\"\n The fully-qualified name of the test. This is similar to test_id, but does not include the session ID. It\n includes the module, class, and method name.\n \"\"\"\n name_components = [\n self.module_name,\n self.cls_name,\n self.function_name,\n self.injected_args_name,\n ]\n\n return \".\".join(filter(lambda x: x is not None and len(x) > 0, name_components))\n\n @property\n def logger(self):\n if self._logger is None:\n self._logger = test_logger(\n TestContext.logger_name(self, self.test_index),\n TestContext.results_dir(self, self.test_index),\n self.session_context.debug,\n )\n return self._logger\n\n def close(self):\n \"\"\"Release resources, etc.\"\"\"\n if hasattr(self, \"services\"):\n for service in self.services:\n service.close()\n # Remove reference to services. This is important to prevent potential memory leaks if users write services\n # which themselves have references to large memory-intensive objects\n del self.services\n\n # Remove local scratch directory\n if self._local_scratch_dir and os.path.exists(self._local_scratch_dir):\n shutil.rmtree(self._local_scratch_dir)\n\n # Release file handles held by logger\n if self._logger:\n close_logger(self._logger)\n" }, { "path": "ducktape/utils/__init__.py", "content": "" }, { "path": "ducktape/utils/http_utils.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom urllib.request import Request, build_opener\n\n\nclass HttpMixin(object):\n def http_request(self, url, method, data=\"\", headers=None, timeout=None):\n if url[0:7].lower() != \"http://\":\n url = \"http://%s\" % url\n\n if hasattr(self, \"logger\") and self.logger is not None:\n self.logger.debug(\"Sending http request. Url: %s, Data: %s, Headers: %s\" % (url, str(data), str(headers)))\n\n req = Request(url, data, headers)\n req.get_method = lambda: method\n # The timeout parameter in urllib2.urlopen has strange behavior, and\n # seems to raise errors when set to a number. Using an opener works however.\n opener = build_opener()\n if timeout is None:\n response = opener.open(req)\n else:\n response = opener.open(req, timeout=timeout)\n\n return response\n" }, { "path": "ducktape/utils/local_filesystem_utils.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport errno\nimport os\n\n\ndef mkdir_p(path: str) -> None:\n \"\"\"mkdir -p functionality.\n :type path: str\n \"\"\"\n try:\n os.makedirs(path)\n except OSError as exc: # Python >2.5\n if exc.errno == errno.EEXIST and os.path.isdir(path):\n pass\n else:\n raise\n" }, { "path": "ducktape/utils/persistence.py", "content": "# Copyright (c) 2009 Jason M Baker\n#\n# Permission is hereby granted, free of charge, to any person obtaining a copy\n# of this software and associated documentation files (the \"Software\"), to deal\n# in the Software without restriction, including without limitation the rights\n# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n# copies of the Software, and to permit persons to whom the Software is\n# furnished to do so, subject to the following conditions:\n#\n# The above copyright notice and this permission notice shall be included in\n# all copies or substantial portions of the Software.\n#\n# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n# THE SOFTWARE.\n\n\"\"\"\nModule contains persistent data structures from pysistence project, patched to work with\npython 3.x.\n\"\"\"\n\n\ndef not_implemented_method(*args, **kwargs):\n raise NotImplementedError(\"Cannot set values in a PDict\")\n\n\nclass PDict(dict):\n \"\"\"\n Persistent dict.\n \"\"\"\n\n __setitem__ = not_implemented_method\n __delitem__ = not_implemented_method\n update = not_implemented_method\n clear = not_implemented_method\n pop = not_implemented_method\n popitem = not_implemented_method\n\n def _as_transient(self):\n return dict(self)\n\n def copy(self):\n return PDict(self)\n\n def without(self, *keys):\n new_dict = self._as_transient()\n for key in keys:\n del new_dict[key]\n return PDict(new_dict)\n\n def using(self, **kwargs):\n new_dict = self._as_transient()\n new_dict.update(kwargs)\n return PDict(new_dict)\n\n def __reduce__(self):\n # Pickling support in python 2.x and 3.x\n return make_dict, (self._as_transient(),)\n\n\nmake_dict = PDict\n" }, { "path": "ducktape/utils/terminal_size.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os\nimport platform\nimport shlex\nimport struct\nimport subprocess\n\n\"\"\"\nAdapted from https://gist.github.com/jtriley/1108174\n\"\"\"\n\n\ndef get_terminal_size():\n \"\"\"getTerminalSize()\n - get width and height of console\n - works on linux,os x,windows,cygwin(windows)\n originally retrieved from:\n http://stackoverflow.com/questions/566746/how-to-get-console-window-width-in-python\n \"\"\"\n current_os = platform.system()\n tuple_xy = None\n if current_os == \"Windows\":\n tuple_xy = _get_terminal_size_windows()\n if tuple_xy is None:\n tuple_xy = _get_terminal_size_tput()\n # needed for window's python in cygwin's xterm!\n if current_os in [\"Linux\", \"Darwin\"] or current_os.startswith(\"CYGWIN\"):\n tuple_xy = _get_terminal_size_linux()\n if tuple_xy is None:\n tuple_xy = (80, 25) # default value\n return tuple_xy\n\n\ndef _get_terminal_size_windows():\n try:\n from ctypes import create_string_buffer, windll\n\n # stdin handle is -10\n # stdout handle is -11\n # stderr handle is -12\n h = windll.kernel32.GetStdHandle(-12)\n csbi = create_string_buffer(22)\n res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi)\n if res:\n (\n bufx,\n bufy,\n curx,\n cury,\n wattr,\n left,\n top,\n right,\n bottom,\n maxx,\n maxy,\n ) = struct.unpack(\"hhhhHhhhhhh\", csbi.raw)\n sizex = right - left + 1\n sizey = bottom - top + 1\n return sizex, sizey\n except Exception:\n pass\n\n\ndef _get_terminal_size_tput():\n # get terminal width\n # src: http://stackoverflow.com/questions/263890/how-do-i-find-the-width-height-of-a-terminal-window\n try:\n cols = int(subprocess.check_call(shlex.split(\"tput cols\")))\n rows = int(subprocess.check_call(shlex.split(\"tput lines\")))\n return (cols, rows)\n except Exception:\n pass\n\n\ndef _get_terminal_size_linux():\n def ioctl_GWINSZ(fd):\n try:\n import fcntl\n import termios\n\n cr = struct.unpack(\"hh\", fcntl.ioctl(fd, termios.TIOCGWINSZ, \"1234\"))\n return cr\n except Exception:\n pass\n\n cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)\n if not cr:\n try:\n fd = os.open(os.ctermid(), os.O_RDONLY)\n cr = ioctl_GWINSZ(fd)\n os.close(fd)\n except Exception:\n pass\n if not cr:\n try:\n cr = (os.environ[\"LINES\"], os.environ[\"COLUMNS\"])\n except Exception:\n return None\n return int(cr[1]), int(cr[0])\n" }, { "path": "ducktape/utils/util.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport importlib\nimport time\nfrom typing import Callable\n\nfrom ducktape import __version__ as __ducktape_version__\nfrom ducktape.errors import TimeoutError\n\n\ndef wait_until(condition, timeout_sec, backoff_sec=0.1, err_msg=\"\", retry_on_exc=False):\n \"\"\"Block until condition evaluates as true or timeout expires, whichever comes first.\n\n :param condition: callable that returns True if the condition is met, False otherwise\n :param timeout_sec: number of seconds to check the condition for before failing\n :param backoff_sec: number of seconds to back off between each failure to meet the condition before checking again\n :param err_msg: a string or callable returning a string that will be included as the exception message if the\n condition is not met\n :param retry_on_exc: if True, will retry if condition raises an exception. If condition raised exception on last\n iteration, that exception will be raised as a cause of TimeoutError.\n If False and condition raises an exception, that exception will be forwarded to the caller\n immediately.\n Defaults to False (original ducktape behavior).\n # TODO: [1.0.0] flip this to True\n :return: silently if condition becomes true within the timeout window, otherwise raise Exception with the given\n error message.\n \"\"\"\n start = time.time()\n stop = start + timeout_sec\n last_exception = None\n while time.time() < stop:\n try:\n if condition():\n return\n else:\n # reset last_exception if last iteration didn't raise any exception, but simply returned False\n last_exception = None\n except BaseException as e:\n # save last raised exception for logging it later\n last_exception = e\n if not retry_on_exc:\n raise e\n finally:\n time.sleep(backoff_sec)\n\n # it is safe to call Exception from None - will be just treated as a normal exception\n raise TimeoutError(err_msg() if callable(err_msg) else err_msg) from last_exception\n\n\ndef package_is_installed(package_name):\n \"\"\"Return true iff package can be successfully imported.\"\"\"\n try:\n importlib.import_module(package_name)\n return True\n except Exception:\n return False\n\n\ndef ducktape_version():\n \"\"\"Return string representation of current ducktape version.\"\"\"\n return __ducktape_version__\n\n\ndef load_function(func_module_path) -> Callable:\n \"\"\"Loads and returns a function from a module path seperated by '.'s\"\"\"\n module, function_name = func_module_path.rsplit(\".\", 1)\n try:\n func = getattr(importlib.import_module(module), function_name)\n if not callable(func):\n raise Exception(\"Function {} from module {} is not callable\".format(function_name, module))\n return func\n except AttributeError:\n raise Exception(\n \"Function could not be loaded from the module path {}, verify that it is '.' seperated\".format(\n func_module_path\n )\n )\n" }, { "path": "requirements-test.txt", "content": "pytest~=6.2.0\n# 4.0 drops py27 support\nmock==4.0.2\nmemory_profiler==0.57\nstatistics==1.0.3.5\nrequests-testadapter==0.3.0\npytest-cov~=3.0\npytest-xdist~=2.5\nruff==0.4.10\n" }, { "path": "requirements.txt", "content": "jinja2~=3.1.6\nboto3==1.33.13\n# jinja2 pulls in MarkupSafe with a > constraint, but we need to constrain it for compatibility\nMarkupSafe~=2.1.5\npyparsing==3.1.4\nzipp==3.20.2\npywinrm==0.4.3\nrequests==2.32.4\nparamiko~=3.4.0\npyzmq==26.4.0\npycryptodome==3.23.0\nmore-itertools==5.0.0\nPyYAML==6.0.2\npsutil==5.7.2\n" }, { "path": "ruff.toml", "content": "\nextend-exclude = [\n \"docs\",\n \".virtualenvs\"\n]\nline-length = 120\n\n[lint]\nselect = [\n \"E4\",\n \"E7\",\n \"E9\",\n \"F\"\n]\nignore = [\n \"E111\",\n \"W292\",\n \"E226\"\n]\n" }, { "path": "service.yml", "content": "name: ducktape\nlang: python\nlang_version: '3.13'\ngit:\n enable: true\nsonarqube:\n enable: true\ngithub:\n enable: true\n repo_name: confluentinc/ducktape\nrenovatebot:\n enable: true\n automerge:\n enable: false\nsemaphore:\n enable: true\n pipeline_enable: false\n branches:\n - master\n - 0.13.x\n - 0.12.x\n - 0.11.x\n - 0.10.x\n - 0.9.x\n - 0.8.x\n - 0.7.x\n" }, { "path": "setup.cfg", "content": "# pytest configuration (can also be defined in in tox.ini or pytest.ini file)\n#\n# To ease possible confusion, prefix ducktape unit tests with 'check' instead of 'test', since\n# many ducktape files, classes, and methods have 'test' somewhere in the name\n[tool:pytest]\npython_files=check_*.py\npython_classes=Check\npython_functions=check_*\n\n# don't search inside any resources directory for unit tests\nnorecursedirs = resources\n" }, { "path": "setup.py", "content": "from setuptools import find_packages, setup\nfrom setuptools.command.test import test as TestCommand\nimport re\nimport sys\n\nversion = \"\"\nwith open(\"ducktape/__init__.py\", \"r\") as fd:\n version = re.search(r'^__version__\\s*=\\s*[\\'\"]([^\\'\"]*)[\\'\"]', fd.read(), re.MULTILINE).group(1)\n\nif not version:\n raise RuntimeError(\"Cannot find version information\")\n\n\nclass PyTest(TestCommand):\n user_options = [(\"pytest-args=\", \"a\", \"Arguments to pass to py.test\")]\n\n def initialize_options(self):\n TestCommand.initialize_options(self)\n self.pytest_args = []\n\n def finalize_options(self):\n TestCommand.finalize_options(self)\n self.test_args = []\n self.test_suite = True\n\n def run_tests(self):\n # import here, cause outside the eggs aren't loaded\n import pytest\n\n errno = pytest.main(self.pytest_args)\n self.run_command(\"flake8\")\n sys.exit(errno)\n\n\ntest_req = open(\"requirements-test.txt\").read()\n\n\nsetup(\n name=\"ducktape\",\n version=version,\n description=\"Distributed system test tools\",\n author=\"Confluent\",\n platforms=[\"any\"],\n entry_points={\n \"console_scripts\": [\"ducktape=ducktape.command_line.main:main\"],\n },\n license=\"apache2.0\",\n url=\"http://github.com/confluentinc/ducktape\",\n packages=find_packages(),\n package_data={\"ducktape\": [\"templates/report/*\"]},\n python_requires=\">= 3.6\",\n install_requires=open(\"requirements.txt\").read(),\n extras_require={\"test\": test_req},\n setup_requires=[\"ruff==0.4.10\"],\n cmdclass={\"test\": PyTest},\n)\n" }, { "path": "systests/__init__.py", "content": "" }, { "path": "systests/cluster/__init__.py", "content": "" }, { "path": "systests/cluster/test_debug.py", "content": "\"\"\"\nThis module contains tests that are useful for developer debugging\nand can contain sleep statements or test that intentionally fail or break things.\nThey're separate from test_remote_account.py for that reason.\n\"\"\"\n\nimport time\n\nfrom ducktape.mark import matrix, parametrize, ignore\nfrom ducktape.mark.resource import cluster\nfrom ducktape.tests.test import Test\nfrom systests.cluster.test_remote_account import GenericService\n\n\nclass FailingTest(Test):\n \"\"\"\n The purpose of this test is to validate reporters. Some of them are intended to fail.\n \"\"\"\n\n def setup(self):\n self.service = GenericService(self.test_context, 1)\n\n @cluster(num_nodes=1)\n @matrix(\n string_param=[\"success-first\", \"fail-second\", \"fail-third\"],\n int_param=[10, 20, -30],\n )\n def matrix_test(self, string_param, int_param):\n assert not string_param.startswith(\"fail\") and int_param > 0\n\n @cluster(num_nodes=1)\n @parametrize(string_param=\"success-first\", int_param=10)\n @parametrize(string_param=\"fail-second\", int_param=-10)\n def parametrized_test(self, string_param, int_param):\n assert not string_param.startswith(\"fail\") and int_param > 0\n\n @cluster(num_nodes=1)\n def failing_test(self):\n assert False\n\n @cluster(num_nodes=1)\n def successful_test(self):\n assert True\n\n\nclass DebugThisTest(Test):\n @cluster(num_nodes=1)\n def one_node_test_sleep_90s(self):\n self.service = GenericService(self.test_context, 1)\n self.logger.warning(\"one_node_test - Sleeping for 90s\")\n time.sleep(90)\n assert True\n\n @cluster(num_nodes=1)\n def one_node_test_sleep_30s(self):\n self.service = GenericService(self.test_context, 1)\n self.logger.warning(\"another_one_node_test - Sleeping for 30s\")\n time.sleep(30)\n assert True\n\n @cluster(num_nodes=1)\n def another_one_node_test_sleep_30s(self):\n self.service = GenericService(self.test_context, 1)\n self.logger.warning(\"yet_another_one_node_test - Sleeping for 30s\")\n time.sleep(30)\n assert True\n\n @cluster(num_nodes=2)\n def two_node_test(self):\n self.service = GenericService(self.test_context, 2)\n assert True\n\n @cluster(num_nodes=2)\n def another_two_node_test(self):\n self.service = GenericService(self.test_context, 2)\n assert True\n\n @ignore\n @cluster(num_nodes=2)\n def a_two_node_ignored_test(self):\n assert False\n\n @cluster(num_nodes=2)\n def yet_another_two_node_test(self):\n self.service = GenericService(self.test_context, 2)\n assert True\n\n @cluster(num_nodes=3)\n def three_node_test(self):\n self.service = GenericService(self.test_context, 3)\n assert True\n\n @cluster(num_nodes=3)\n def three_node_test_sleeping_30s(self):\n self.service = GenericService(self.test_context, 3)\n self.logger.warning(\"Sleeping for 30s\")\n time.sleep(30)\n assert True\n\n @cluster(num_nodes=3)\n def another_three_node_test(self):\n self.service = GenericService(self.test_context, 3)\n assert True\n\n @cluster(num_nodes=2)\n def bad_alloc_test(self):\n # @cluster annotation specifies 2 nodes, but we ask for 3, this will fail\n self.service = GenericService(self.test_context, 3)\n time.sleep(10)\n assert True\n" }, { "path": "systests/cluster/test_no_cluster.py", "content": "from ducktape.mark.resource import cluster\nfrom ducktape.tests.test import Test\n\n\nclass NoClusterTest(Test):\n \"\"\"This test helps validate the behavior for no-cluster tests (ie 0 nodes)\"\"\"\n\n @cluster(num_nodes=0)\n def test_zero_nodes(self):\n self.logger.warn(\"Testing\")\n assert True\n" }, { "path": "systests/cluster/test_remote_account.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nfrom ducktape.cluster.cluster_spec import ClusterSpec\nfrom ducktape.cluster.consts import WINDOWS, LINUX\nfrom ducktape.cluster.node_spec import NodeSpec\nfrom ducktape.services.service import Service\nfrom ducktape.tests.test import Test\nfrom ducktape.errors import TimeoutError\nfrom ducktape.mark.resource import cluster\n\nimport os\nimport pytest\nimport random\nimport shutil\nfrom six import iteritems\nimport tempfile\nfrom threading import Thread\nimport time\nimport logging\n\nfrom ducktape.utils.util import wait_until\n\n\ndef generate_tempdir_name():\n \"\"\"Use this ad-hoc function instead of the tempfile module since we're creating and removing\n this directory with ssh commands.\n \"\"\"\n return \"/tmp/\" + \"t\" + str(int(time.time()))\n\n\nclass RemoteAccountTestService(Service):\n \"\"\"Simple service that allocates one node for performing tests of RemoteAccount functionality\"\"\"\n\n def __init__(self, context):\n super(RemoteAccountTestService, self).__init__(context, num_nodes=1)\n self.temp_dir = generate_tempdir_name()\n self.logs = {\n \"my_log\": {\"path\": self.log_file, \"collect_default\": True},\n \"non_existent_log\": {\n \"path\": os.path.join(self.temp_dir, \"absent.log\"),\n \"collect_default\": True,\n },\n }\n\n @property\n def log_file(self):\n return os.path.join(self.temp_dir, \"test.log\")\n\n def start_node(self, node):\n node.account.ssh(\"mkdir -p \" + self.temp_dir)\n node.account.ssh(\"touch \" + self.log_file)\n\n def stop_node(self, node):\n pass\n\n def clean_node(self, node):\n node.account.ssh(\"rm -rf \" + self.temp_dir)\n\n def write_to_log(self, msg):\n self.nodes[0].account.ssh(\"echo -e -n \" + repr(msg) + \" >> \" + self.log_file)\n\n\nclass GenericService(Service):\n \"\"\"Service which doesn't do anything - just a group of nodes, each of which has a scratch directory.\"\"\"\n\n def __init__(self, context, num_nodes):\n super(GenericService, self).__init__(context, num_nodes)\n self.worker_scratch_dir = \"scratch\"\n for node in self.nodes:\n node.account.mkdirs(self.worker_scratch_dir)\n\n def stop_node(self, node):\n # noop\n pass\n\n def clean_node(self, node):\n node.account.remove(self.worker_scratch_dir, allow_fail=True)\n\n\nclass UnderUtilizedTest(Test):\n def setup(self):\n self.service = GenericService(self.test_context, 1)\n\n @cluster(num_nodes=3)\n def under_utilized_test(self):\n # setup() creates a service instance, which calls alloc() for one node\n assert self.test_context.cluster.max_used() == 1\n assert len(self.test_context.cluster.used()) == 1\n\n self.another_service = GenericService(self.test_context, 1)\n assert len(self.test_context.cluster.used()) == 2\n assert self.test_context.cluster.max_used() == 2\n\n self.service.stop()\n self.service.free()\n assert len(self.test_context.cluster.used()) == 1\n assert self.test_context.cluster.max_used() == 2\n\n\nclass FileSystemTest(Test):\n \"\"\"\n Note that in an attempt to isolate the file system methods, validation should be done with ssh/shell commands.\n \"\"\"\n\n def setup(self):\n self.service = GenericService(self.test_context, 1)\n self.node = self.service.nodes[0]\n self.scratch_dir = self.service.worker_scratch_dir\n\n @cluster(num_nodes=1)\n def create_file_test(self):\n expected_contents = \"hello world\"\n fname = \"myfile.txt\"\n fpath = \"%s/%s\" % (self.scratch_dir, fname)\n\n self.node.account.create_file(fpath, expected_contents)\n\n # validate existence and contents\n self.node.account.ssh(\"test -f %s\" % fpath)\n contents = \"\\n\".join([line for line in self.node.account.ssh_capture(\"cat %s\" % fpath)])\n assert contents == expected_contents\n\n # TODO also check absolute path\n\n @cluster(num_nodes=1)\n def mkdir_test(self):\n dirname = \"%s/mydir\" % self.scratch_dir\n self.node.account.mkdir(dirname)\n\n # TODO - important!! check mode\n self.node.account.ssh(\"test -d %s\" % dirname, allow_fail=False)\n\n # mkdir should not succeed if the base directories do not already exist\n dirname = \"%s/a/b/c/d\" % self.scratch_dir\n with pytest.raises(IOError):\n self.node.account.mkdir(dirname)\n\n # TODO also check absolute path\n\n @cluster(num_nodes=1)\n def mkdirs_nested_test(self):\n dirname = \"%s/a/b/c/d\" % self.scratch_dir\n\n # TODO important!! check mode\n self.node.account.mkdirs(dirname)\n self.node.account.ssh(\"test -d %s\" % dirname, allow_fail=False)\n\n # TODO also check absolute path\n\n @cluster(num_nodes=1)\n def open_test(self):\n \"\"\"Try opening, writing, reading a file.\"\"\"\n fname = \"%s/myfile.txt\" % self.scratch_dir\n expected_contents = b\"hello world\\nhooray!\"\n with self.node.account.open(fname, \"w\") as f:\n f.write(expected_contents)\n\n with self.node.account.open(fname, \"r\") as f:\n contents = f.read()\n assert contents == expected_contents\n\n # Now try opening in append mode\n append = b\"hithere\"\n expected_contents = expected_contents + append\n with self.node.account.open(fname, \"a\") as f:\n f.write(append)\n\n with self.node.account.open(fname, \"r\") as f:\n contents = f.read()\n\n assert contents == expected_contents\n\n @cluster(num_nodes=1)\n def exists_file_test(self):\n \"\"\"\n Create various kinds of files and symlinks, verifying that exists works as expected.\n\n Note that because\n \"\"\"\n\n # create file, test existence with relative and absolute path\n self.node.account.ssh(\"touch %s/hi\" % self.scratch_dir)\n assert self.node.account.exists(\"%s/hi\" % self.scratch_dir)\n # TODO abspath\n\n # create symlink, test existence with relative and absolute path\n self.node.account.ssh(\"ln -s %s/hi %s/hi-link\" % (self.scratch_dir, self.scratch_dir))\n assert self.node.account.exists(\"%s/hi-link\" % self.scratch_dir)\n # TODO abspath\n\n def exists_dir_test(self):\n # check bad path doesn't exist\n assert not self.node.account.exists(\"a/b/c/d\")\n\n # create dir, test existence with relative and absolute path\n dpath = \"%s/mydir\" % self.scratch_dir\n self.node.account.ssh(\"mkdir %s\" % dpath)\n assert self.node.account.exists(dpath)\n # TODO abspath\n\n # create symlink, test existence with relative and absolute path\n self.node.account.ssh(\"ln -s %s %s/mydir-link\" % (dpath, self.scratch_dir))\n assert self.node.account.exists(\"%s/mydir-link\" % self.scratch_dir)\n # # TODO abspath\n\n def remove_test(self):\n \"\"\"Test functionality of remove method\"\"\"\n # remove a non-empty directory\n dpath = \"%s/mydir\" % self.scratch_dir\n self.node.account.ssh(\"mkdir %s\" % dpath)\n self.node.account.ssh(\"touch %s/hi.txt\" % dpath)\n self.node.account.ssh(\"test -d %s\" % dpath)\n self.node.account.remove(dpath)\n self.node.account.ssh(\"test ! -d %s\" % dpath)\n\n # remove a file\n fpath = \"%s/hello.txt\" % self.scratch_dir\n self.node.account.ssh(\"echo 'hello world' > %s\" % fpath)\n self.node.account.remove(fpath)\n\n # remove non-existent path\n with pytest.raises(RuntimeError):\n self.node.account.remove(\"a/b/c/d\")\n\n # remove non-existent path with allow_fail = True should be ok\n self.node.account.remove(\"a/b/c/d\", allow_fail=True)\n\n\n# Representation of a somewhat arbitrary directory structure for testing copy functionality\n# A key which has a string as its value represents a file\n# A key which has a dict as its value represents a subdirectory\nDIR_STRUCTURE = {\n \"d00\": {\n \"another_file\": b\"1\\n2\\n3\\n4\\ncats and dogs\",\n \"d10\": {\"fasdf\": b\"lasdf;asfd\\nahoppoqnbasnb\"},\n \"d11\": {\"f65\": b\"afasdfsafdsadf\"},\n },\n \"a_file\": b\"hello world!\",\n}\n\n\ndef make_dir_structure(base_dir, dir_structure, node=None):\n \"\"\"Make a file tree starting at base_dir with structure specified by dir_structure.\n\n if node is None, make the structure locally, else make it on the given node\n \"\"\"\n for k, v in iteritems(dir_structure):\n if isinstance(v, dict):\n # it's a subdirectory\n subdir_name = k\n subdir_path = os.path.join(base_dir, subdir_name)\n subdir_structure = v\n\n if node:\n node.account.mkdir(subdir_path)\n else:\n os.mkdir(subdir_path)\n\n make_dir_structure(subdir_path, subdir_structure, node)\n else:\n # it's a file\n file_name = k\n file_path = os.path.join(base_dir, file_name)\n file_contents = v\n\n if node:\n with node.account.open(file_path, \"wb\") as f:\n f.write(file_contents)\n else:\n with open(file_path, \"wb\") as f:\n f.write(file_contents)\n\n\ndef verify_dir_structure(base_dir, dir_structure, node=None):\n \"\"\"Verify locally or on the given node whether the file subtree at base_dir matches dir_structure.\"\"\"\n for k, v in iteritems(dir_structure):\n if isinstance(v, dict):\n # it's a subdirectory\n subdir_name = k\n subdir_path = os.path.join(base_dir, subdir_name)\n subdir_structure = v\n\n if node:\n assert node.account.isdir(subdir_path)\n else:\n assert os.path.isdir(subdir_path)\n\n verify_dir_structure(subdir_path, subdir_structure, node)\n else:\n # it's a file\n file_name = k\n file_path = os.path.join(base_dir, file_name)\n expected_file_contents = v\n\n if node:\n with node.account.open(file_path, \"r\") as f:\n contents = f.read()\n else:\n with open(file_path, \"rb\") as f:\n contents = f.read()\n assert expected_file_contents == contents, contents\n\n\nclass CopyToAndFroTest(Test):\n \"\"\"These tests check copy_to, and copy_from functionality.\"\"\"\n\n def setup(self):\n self.service = GenericService(self.test_context, 1)\n self.node = self.service.nodes[0]\n self.remote_scratch_dir = self.service.worker_scratch_dir\n\n self.local_temp_dir = tempfile.mkdtemp()\n\n self.logger.info(\"local_temp_dir: %s\" % self.local_temp_dir)\n self.logger.info(\"node: %s\" % str(self.node.account))\n\n @cluster(num_nodes=1)\n def test_copy_to_dir_with_rename(self):\n # make dir structure locally\n make_dir_structure(self.local_temp_dir, DIR_STRUCTURE)\n dest = os.path.join(self.remote_scratch_dir, \"renamed\")\n self.node.account.copy_to(self.local_temp_dir, dest)\n\n # now validate the directory structure on the remote machine\n verify_dir_structure(dest, DIR_STRUCTURE, node=self.node)\n\n @cluster(num_nodes=1)\n def test_copy_to_dir_as_subtree(self):\n # copy directory \"into\" a directory; this should preserve the original directoryname\n make_dir_structure(self.local_temp_dir, DIR_STRUCTURE)\n self.node.account.copy_to(self.local_temp_dir, self.remote_scratch_dir)\n local_temp_dir_name = self.local_temp_dir\n if local_temp_dir_name.endswith(os.path.sep):\n local_temp_dir_name = local_temp_dir_name[: -len(os.path.sep)]\n\n verify_dir_structure(os.path.join(self.remote_scratch_dir, local_temp_dir_name), DIR_STRUCTURE)\n\n @cluster(num_nodes=1)\n def test_copy_from_dir_with_rename(self):\n # make dir structure remotely\n make_dir_structure(self.remote_scratch_dir, DIR_STRUCTURE, node=self.node)\n dest = os.path.join(self.local_temp_dir, \"renamed\")\n self.node.account.copy_from(self.remote_scratch_dir, dest)\n\n # now validate the directory structure locally\n verify_dir_structure(dest, DIR_STRUCTURE)\n\n @cluster(num_nodes=1)\n def test_copy_from_dir_as_subtree(self):\n # copy directory \"into\" a directory; this should preserve the original directoryname\n make_dir_structure(self.remote_scratch_dir, DIR_STRUCTURE, node=self.node)\n self.node.account.copy_from(self.remote_scratch_dir, self.local_temp_dir)\n\n verify_dir_structure(os.path.join(self.local_temp_dir, \"scratch\"), DIR_STRUCTURE)\n\n def teardown(self):\n # allow_fail in case scratch dir was not successfully created\n if os.path.exists(self.local_temp_dir):\n shutil.rmtree(self.local_temp_dir)\n\n\nclass CopyDirectTest(Test):\n def setup(self):\n self.service = GenericService(self.test_context, 2)\n self.src_node, self.dest_node = self.service.nodes\n self.remote_scratch_dir = self.service.worker_scratch_dir\n\n self.logger.info(\"src_node: %s\" % str(self.src_node.account))\n self.logger.info(\"dest_node: %s\" % str(self.dest_node.account))\n\n @cluster(num_nodes=2)\n def test_copy_file(self):\n \"\"\"Verify that a file can be correctly copied directly between nodes.\n\n This should work with or without the recursive flag.\n \"\"\"\n file_path = os.path.join(self.remote_scratch_dir, \"myfile.txt\")\n expected_contents = b\"123\"\n self.src_node.account.create_file(file_path, expected_contents)\n\n self.src_node.account.copy_between(file_path, file_path, self.dest_node)\n\n assert self.dest_node.account.isfile(file_path)\n with self.dest_node.account.open(file_path, \"r\") as f:\n contents = f.read()\n assert expected_contents == contents\n\n @cluster(num_nodes=2)\n def test_copy_directory(self):\n \"\"\"Verify that a directory can be correctly copied directly between nodes.\"\"\"\n\n make_dir_structure(self.remote_scratch_dir, DIR_STRUCTURE, node=self.src_node)\n self.src_node.account.copy_between(self.remote_scratch_dir, self.remote_scratch_dir, self.dest_node)\n verify_dir_structure(\n os.path.join(self.remote_scratch_dir, \"scratch\"),\n DIR_STRUCTURE,\n node=self.dest_node,\n )\n\n\nclass TestClusterSpec(Test):\n @cluster(cluster_spec=ClusterSpec.simple_linux(2))\n def test_create_two_node_service(self):\n self.service = GenericService(self.test_context, 2)\n for node in self.service.nodes:\n node.account.ssh(\"echo hi\")\n\n @cluster(\n cluster_spec=ClusterSpec.from_nodes(\n [\n NodeSpec(operating_system=WINDOWS),\n NodeSpec(operating_system=LINUX),\n NodeSpec(), # this one is also linux\n ]\n )\n )\n def three_nodes_test(self):\n self.service = GenericService(self.test_context, 3)\n for node in self.service.nodes:\n node.account.ssh(\"echo hi\")\n\n\nclass RemoteAccountTest(Test):\n def __init__(self, test_context):\n super(RemoteAccountTest, self).__init__(test_context)\n self.account_service = RemoteAccountTestService(test_context)\n\n def setup(self):\n self.account_service.start()\n\n @cluster(num_nodes=1)\n def test_flaky(self):\n choices = [\n Exception(\"FLAKE 1\"),\n Exception(\"FLAKE 1\"),\n Exception(\"FLAKE 2\"),\n Exception(\"FLAKE 2\"),\n Exception(\"FLAKE 3\"),\n None,\n ]\n exp = random.choice(choices)\n if exp:\n raise exp\n\n @cluster(num_nodes=1)\n def test_ssh_capture_combine_stderr(self):\n \"\"\"Test that ssh_capture correctly captures stderr and stdout from remote process.\"\"\"\n node = self.account_service.nodes[0]\n\n # swap stdout and stderr in the echo process\n cmd = \"for i in $(seq 1 5); do echo $i 3>&1 1>&2 2>&3; done\"\n\n ssh_output = node.account.ssh_capture(cmd, combine_stderr=True)\n bad_ssh_output = node.account.ssh_capture(cmd, combine_stderr=False) # Same command, but don't capture stderr\n\n lines = [int(line.strip()) for line in ssh_output]\n assert lines == [i for i in range(1, 6)]\n bad_lines = [int(line.strip()) for line in bad_ssh_output]\n assert bad_lines == []\n\n @cluster(num_nodes=1)\n def test_ssh_output_combine_stderr(self):\n \"\"\"Test that ssh_output correctly captures stderr and stdout from remote process.\"\"\"\n node = self.account_service.nodes[0]\n\n # swap stdout and stderr in the echo process\n cmd = \"for i in $(seq 1 5); do echo $i 3>&1 1>&2 2>&3; done\"\n\n ssh_output = node.account.ssh_output(cmd, combine_stderr=True)\n bad_ssh_output = node.account.ssh_output(cmd, combine_stderr=False) # Same command, but don't capture stderr\n\n assert ssh_output == b\"\\n\".join([str(i).encode(\"utf-8\") for i in range(1, 6)]) + b\"\\n\", ssh_output\n assert bad_ssh_output == b\"\", bad_ssh_output\n\n @cluster(num_nodes=1)\n def test_ssh_capture(self):\n \"\"\"Test that ssh_capture correctly captures output from ssh subprocess.\"\"\"\n node = self.account_service.nodes[0]\n cmd = \"for i in $(seq 1 5); do echo $i; done\"\n ssh_output = node.account.ssh_capture(cmd, combine_stderr=False)\n\n lines = [int(line.strip()) for line in ssh_output]\n assert lines == [i for i in range(1, 6)]\n\n @cluster(num_nodes=1)\n def test_ssh_output(self):\n \"\"\"Test that ssh_output correctly captures output from ssh subprocess.\"\"\"\n node = self.account_service.nodes[0]\n cmd = \"for i in $(seq 1 5); do echo $i; done\"\n ssh_output = node.account.ssh_output(cmd, combine_stderr=False)\n\n assert ssh_output == b\"\\n\".join([str(i).encode(\"utf-8\") for i in range(1, 6)]) + b\"\\n\", ssh_output\n\n @cluster(num_nodes=1)\n def test_monitor_log(self):\n \"\"\"Tests log monitoring by writing to a log in the background thread\"\"\"\n\n node = self.account_service.nodes[0]\n\n # Make sure we start the log with some data, including the value we're going to grep for\n self.account_service.write_to_log(\"foo\\nbar\\nbaz\")\n\n # Background thread that simulates a process writing to the log\n self.wrote_log_line = False\n\n def background_logging_thread():\n # This needs to be large enough that we can verify we've actually\n # waited some time for the data to be written, but not too long that\n # the test takes a long time\n time.sleep(3)\n self.wrote_log_line = True\n self.account_service.write_to_log(\"foo\\nbar\\nbaz\")\n\n with node.account.monitor_log(self.account_service.log_file) as monitor:\n logging_thread = Thread(target=background_logging_thread)\n logging_thread.start()\n monitor.wait_until(\"foo\", timeout_sec=10, err_msg=\"Never saw expected log\")\n assert self.wrote_log_line\n\n logging_thread.join(5.0)\n if logging_thread.is_alive():\n raise Exception(\"Timed out waiting for background thread.\")\n\n @cluster(num_nodes=1)\n def test_monitor_log_exception(self):\n \"\"\"Tests log monitoring correctly throws an exception when the regex was not found\"\"\"\n\n node = self.account_service.nodes[0]\n\n # Make sure we start the log with some data, including the value we're going to grep for\n self.account_service.write_to_log(\"foo\\nbar\\nbaz\")\n\n timeout = 3\n try:\n with node.account.monitor_log(self.account_service.log_file) as monitor:\n start = time.time()\n monitor.wait_until(\"foo\", timeout_sec=timeout, err_msg=\"Never saw expected log\")\n assert False, \"Log monitoring should have timed out and thrown an exception\"\n except TimeoutError:\n # expected\n end = time.time()\n assert end - start > timeout, \"Should have waited full timeout period while monitoring the log\"\n\n @cluster(num_nodes=1)\n def test_kill_process(self):\n \"\"\"Tests that kill_process correctly works\"\"\"\n grep_str = '\"nc -l -p 5000\"'\n\n def get_pids():\n pid_cmd = f\"ps ax | grep -i {grep_str} | grep -v grep | awk '{{print $1}}'\"\n\n return list(node.account.ssh_capture(pid_cmd, callback=int))\n\n node = self.account_service.nodes[0]\n\n # Run TCP service using netcat\n node.account.ssh_capture(\"nohup nc -l -p 5000 > /dev/null 2>&1 &\")\n\n wait_until(\n lambda: len(get_pids()) > 0,\n timeout_sec=10,\n err_msg=\"Failed to start process within %d sec\" % 10,\n )\n\n # Kill service.\n node.account.kill_process(grep_str)\n\n wait_until(\n lambda: len(get_pids()) == 0,\n timeout_sec=10,\n err_msg=\"Failed to kill process within %d sec\" % 10,\n )\n\n\nclass TestIterWrapper(Test):\n def setup(self):\n self.line_num = 6\n self.eps = 0.01\n\n self.service = GenericService(self.test_context, num_nodes=1)\n self.node = self.service.nodes[0]\n\n self.temp_file = \"ducktape-test-\" + str(random.randint(0, 100000))\n contents = \"\"\n for i in range(self.line_num):\n contents += \"%d\\n\" % i\n\n self.node.account.create_file(self.temp_file, contents)\n\n def test_iter_wrapper(self):\n \"\"\"Test has_next functionality on the returned iterable item.\"\"\"\n output = self.node.account.ssh_capture(\"cat \" + self.temp_file)\n for i in range(self.line_num):\n assert output.has_next() # with timeout in case of hang\n assert output.next().strip() == str(i)\n start = time.time()\n assert output.has_next() is False\n stop = time.time()\n assert stop - start < self.eps, \"has_next() should return immediately\"\n\n def test_iter_wrapper_timeout(self):\n \"\"\"Test has_next with timeout\"\"\"\n output = self.node.account.ssh_capture(\"tail -F \" + self.temp_file)\n # allow command to be executed before we check output with timeout_sec = 0\n time.sleep(0.5)\n for i in range(self.line_num):\n assert output.has_next(timeout_sec=0)\n assert output.next().strip() == str(i)\n\n timeout = 0.25\n start = time.time()\n # This check will last for the duration of the timeout because the the remote tail -F process\n # remains running, and the output stream is not closed.\n assert output.has_next(timeout_sec=timeout) is False\n stop = time.time()\n assert (stop - start >= timeout) and (stop - start) < timeout + self.eps, (\n \"has_next() should return right after %s second\" % str(timeout)\n )\n\n def teardown(self):\n # tail -F call above will leave stray processes, so clean up\n cmd = \"for p in $(ps ax | grep -v grep | grep \\\"%s\\\" | awk '{print $1}'); do kill $p; done\" % self.temp_file\n self.node.account.ssh(cmd, allow_fail=True)\n\n self.node.account.ssh(\"rm -f \" + self.temp_file, allow_fail=True)\n\n\nclass RemoteAccountCompressedTest(Test):\n def __init__(self, test_context):\n super(RemoteAccountCompressedTest, self).__init__(test_context)\n self.account_service = RemoteAccountTestService(test_context)\n self.test_context.session_context.compress = True\n self.tar_msg = False\n self.tar_error = False\n\n def setup(self):\n self.account_service.start()\n\n @cluster(num_nodes=1)\n def test_log_compression_with_non_existent_files(self):\n \"\"\"Test that log compression with tar works even when a specific log file has not been generated\n (e.g. heap dump)\n \"\"\"\n self.test_context.logger.addFilter(CompressionErrorFilter(self))\n self.copy_service_logs(None)\n\n if not self.tar_msg:\n raise Exception(\"Never saw attempt to compress log\")\n if self.tar_error:\n raise Exception(\"Failure when compressing logs\")\n\n\nclass CompressionErrorFilter(logging.Filter):\n def __init__(self, test):\n super(CompressionErrorFilter, self).__init__()\n self.test = test\n\n def filter(self, record):\n if \"tar czf\" in record.msg:\n self.test.tar_msg = True\n if \"Error\" in record.msg:\n self.test.tar_error = True\n return True\n" }, { "path": "systests/cluster/test_runner_operations.py", "content": "# Copyright 2022 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nfrom ducktape.services.service import Service\nfrom ducktape.tests.test import Test\nfrom ducktape.mark.resource import cluster\nimport time\n\n\nclass SimpleEchoService(Service):\n \"\"\"Simple service that allocates one node for performing tests of RemoteAccount functionality\"\"\"\n\n logs = {\n \"my_log\": {\"path\": \"/tmp/log\", \"collect_default\": True},\n }\n\n def __init__(self, context):\n super(SimpleEchoService, self).__init__(context, num_nodes=1)\n self.count = 0\n\n def echo(self):\n self.nodes[0].account.ssh(\"echo {} >> /tmp/log\".format(self.count))\n self.count += 1\n\n\nclass SimpleRunnerTest(Test):\n def setup(self):\n self.service = SimpleEchoService(self.test_context)\n\n @cluster(num_nodes=1)\n def timeout_test(self):\n \"\"\"\n a simple longer running test to test special run flags agaisnt.\n \"\"\"\n self.service.start()\n\n while self.service.count < 500:\n self.service.echo()\n time.sleep(0.1)\n\n @cluster(num_nodes=1)\n def quick1_test(self):\n \"\"\"\n a simple quick test to test basic execution.\n \"\"\"\n self.service.start()\n\n while self.service.count < 20:\n self.service.echo()\n time.sleep(0.2)\n\n @cluster(num_nodes=1)\n def quick2_test(self):\n \"\"\"\n a simple quick test to test basic execution.\n \"\"\"\n self.service.start()\n\n while self.service.count < 20:\n self.service.echo()\n time.sleep(0.2)\n" }, { "path": "tests/__init__.py", "content": "from ducktape.tests.test import Test\nfrom ducktape.tests.test_context import TestContext\n\n__all__ = [\"Test\", \"TestContext\"]\n" }, { "path": "tests/cluster/__init__.py", "content": "" }, { "path": "tests/cluster/check_cluster.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport collections\n\nfrom ducktape.cluster.cluster_node import ClusterNode\nfrom ducktape.cluster.cluster_spec import ClusterSpec\nfrom ducktape.cluster.node_spec import NodeSpec\nfrom ducktape.cluster.consts import LINUX, WINDOWS\nfrom tests.ducktape_mock import FakeCluster\n\nFakeRemoteAccount = collections.namedtuple(\"FakeRemoteAccount\", [\"operating_system\"])\n\n\nclass CheckCluster(object):\n def setup_method(self, _):\n self.cluster = FakeCluster(0)\n self.cluster._available_nodes.add_node(ClusterNode(FakeRemoteAccount(operating_system=LINUX)))\n self.cluster._available_nodes.add_node(ClusterNode(FakeRemoteAccount(operating_system=LINUX)))\n self.cluster._available_nodes.add_node(ClusterNode(FakeRemoteAccount(operating_system=WINDOWS)))\n self.cluster._available_nodes.add_node(ClusterNode(FakeRemoteAccount(operating_system=WINDOWS)))\n self.cluster._available_nodes.add_node(ClusterNode(FakeRemoteAccount(operating_system=WINDOWS)))\n\n def spec(self, linux_nodes, windows_nodes):\n nodes = []\n for i in range(linux_nodes):\n nodes.append(NodeSpec(LINUX))\n for i in range(windows_nodes):\n nodes.append(NodeSpec(WINDOWS))\n return ClusterSpec(nodes)\n\n def check_enough_capacity(self):\n assert self.cluster.available().nodes.can_remove_spec(self.spec(2, 2))\n assert self.cluster.available().nodes.can_remove_spec(self.spec(2, 3))\n\n def check_not_enough_capacity(self):\n assert not self.cluster.available().nodes.can_remove_spec(self.spec(5, 2))\n assert not self.cluster.available().nodes.can_remove_spec(self.spec(5, 5))\n assert not self.cluster.available().nodes.can_remove_spec(self.spec(3, 3))\n" }, { "path": "tests/cluster/check_cluster_spec.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.cluster.cluster_spec import ClusterSpec\nfrom ducktape.cluster.node_spec import NodeSpec\n\n\nclass CheckClusterSpec(object):\n def check_cluster_spec_sizes(self):\n simple_linux_2 = ClusterSpec.simple_linux(2)\n assert 2 == len(simple_linux_2)\n assert 0 == len(ClusterSpec.empty())\n\n def check_to_string(self):\n empty = ClusterSpec.empty()\n assert \"[]\" == str(empty)\n simple_linux_5 = ClusterSpec.simple_linux(5)\n assert '[{\"num_nodes\": 5, \"os\": \"linux\"}]' == str(simple_linux_5)\n\n def check_simple_linux_with_node_type(self):\n \"\"\"Test simple_linux with node_type parameter.\"\"\"\n spec = ClusterSpec.simple_linux(3, node_type=\"large\")\n assert len(spec) == 3\n for node_spec in spec:\n assert node_spec.operating_system == \"linux\"\n assert node_spec.node_type == \"large\"\n\n def check_simple_linux_without_node_type(self):\n \"\"\"Test simple_linux without node_type (backward compatibility).\"\"\"\n spec = ClusterSpec.simple_linux(2)\n assert len(spec) == 2\n for node_spec in spec:\n assert node_spec.operating_system == \"linux\"\n assert node_spec.node_type is None\n\n def check_grouped_by_os_and_type_empty(self):\n \"\"\"Test grouped_by_os_and_type on empty ClusterSpec via NodeContainer.\"\"\"\n spec = ClusterSpec.empty()\n grouped = spec.nodes.grouped_by_os_and_type()\n assert grouped == {}\n\n def check_grouped_by_os_and_type_single_type(self):\n \"\"\"Test grouped_by_os_and_type with single node type via NodeContainer.\"\"\"\n spec = ClusterSpec.simple_linux(3, node_type=\"small\")\n grouped = spec.nodes.grouped_by_os_and_type()\n assert grouped == {(\"linux\", \"small\"): 3}\n\n def check_grouped_by_os_and_type_mixed(self):\n \"\"\"Test grouped_by_os_and_type with mixed node types via NodeContainer.\"\"\"\n spec = ClusterSpec(\n nodes=[\n NodeSpec(\"linux\", node_type=\"small\"),\n NodeSpec(\"linux\", node_type=\"small\"),\n NodeSpec(\"linux\", node_type=\"large\"),\n NodeSpec(\"linux\", node_type=None),\n NodeSpec(\"windows\", node_type=\"medium\"),\n ]\n )\n grouped = spec.nodes.grouped_by_os_and_type()\n assert grouped == {\n (\"linux\", \"small\"): 2,\n (\"linux\", \"large\"): 1,\n (\"linux\", None): 1,\n (\"windows\", \"medium\"): 1,\n }\n" }, { "path": "tests/cluster/check_finite_subcluster.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.cluster.consts import LINUX\nfrom ducktape.cluster.finite_subcluster import FiniteSubcluster\nfrom ducktape.cluster.node_container import (\n InsufficientResourcesError,\n NodeNotPresentError,\n)\nfrom ducktape.services.service import Service\nimport pickle\nimport pytest\n\n\nclass MockFiniteSubclusterNode:\n @property\n def operating_system(self):\n return LINUX\n\n\nclass CheckFiniteSubcluster(object):\n single_node_cluster_json = {\"nodes\": [{\"hostname\": \"localhost\"}]}\n\n def check_cluster_size(self):\n cluster = FiniteSubcluster([])\n assert len(cluster) == 0\n\n n = 10\n cluster = FiniteSubcluster([MockFiniteSubclusterNode() for _ in range(n)])\n assert len(cluster) == n\n\n def check_pickleable(self):\n cluster = FiniteSubcluster([MockFiniteSubclusterNode() for _ in range(10)])\n pickle.dumps(cluster)\n\n def check_allocate_free(self):\n n = 10\n cluster = FiniteSubcluster([MockFiniteSubclusterNode() for _ in range(n)])\n assert len(cluster) == n\n assert cluster.num_available_nodes() == n\n\n nodes = cluster.alloc(Service.setup_cluster_spec(num_nodes=1))\n assert len(nodes) == 1\n assert len(cluster) == n\n assert cluster.num_available_nodes() == n - 1\n\n nodes2 = cluster.alloc(Service.setup_cluster_spec(num_nodes=2))\n assert len(nodes2) == 2\n assert len(cluster) == n\n assert cluster.num_available_nodes() == n - 3\n\n cluster.free(nodes)\n assert cluster.num_available_nodes() == n - 2\n\n cluster.free(nodes2)\n assert cluster.num_available_nodes() == n\n\n def check_alloc_too_many(self):\n n = 10\n cluster = FiniteSubcluster([MockFiniteSubclusterNode() for _ in range(n)])\n with pytest.raises(InsufficientResourcesError):\n cluster.alloc(Service.setup_cluster_spec(num_nodes=(n + 1)))\n\n def check_free_too_many(self):\n n = 10\n cluster = FiniteSubcluster([MockFiniteSubclusterNode() for _ in range(n)])\n nodes = cluster.alloc(Service.setup_cluster_spec(num_nodes=n))\n with pytest.raises(NodeNotPresentError):\n nodes.append(MockFiniteSubclusterNode())\n cluster.free(nodes)\n" }, { "path": "tests/cluster/check_json.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nfrom ducktape.cluster.json import JsonCluster\nfrom ducktape.cluster.node_container import InsufficientResourcesError\nfrom ducktape.services.service import Service\nimport pickle\nimport pytest\n\nfrom tests.runner.fake_remote_account import create_fake_remote_account\n\n\ndef create_json_cluster(*args, **kwargs):\n return JsonCluster(*args, make_remote_account_func=create_fake_remote_account, **kwargs)\n\n\nclass CheckJsonCluster(object):\n single_node_cluster_json = {\"nodes\": [{\"ssh_config\": {\"host\": \"localhost\"}}]}\n\n def check_invalid_json(self):\n # Missing list of nodes\n with pytest.raises(ValueError):\n create_json_cluster({})\n\n # Missing hostname, which is required\n with pytest.raises(ValueError):\n create_json_cluster({\"nodes\": [{}]})\n\n @staticmethod\n def cluster_hostnames(nodes):\n return set([node.account.hostname for node in nodes])\n\n def check_cluster_size(self):\n cluster = create_json_cluster({\"nodes\": []})\n assert len(cluster) == 0\n\n n = 10\n cluster = create_json_cluster({\"nodes\": [{\"ssh_config\": {\"hostname\": \"localhost%d\" % x}} for x in range(n)]})\n\n assert len(cluster) == n\n\n def check_pickleable(self):\n cluster = create_json_cluster(\n {\n \"nodes\": [\n {\"ssh_config\": {\"host\": \"localhost1\"}},\n {\"ssh_config\": {\"host\": \"localhost2\"}},\n {\"ssh_config\": {\"host\": \"localhost3\"}},\n ]\n }\n )\n\n pickle.dumps(cluster)\n\n def check_allocate_free(self):\n cluster = create_json_cluster(\n {\n \"nodes\": [\n {\"ssh_config\": {\"host\": \"localhost1\"}},\n {\"ssh_config\": {\"host\": \"localhost2\"}},\n {\"ssh_config\": {\"host\": \"localhost3\"}},\n ]\n }\n )\n\n assert len(cluster) == 3\n assert cluster.num_available_nodes() == 3\n\n nodes = cluster.alloc(Service.setup_cluster_spec(num_nodes=1))\n nodes_hostnames = self.cluster_hostnames(nodes)\n assert len(cluster) == 3\n assert cluster.num_available_nodes() == 2\n\n nodes2 = cluster.alloc(Service.setup_cluster_spec(num_nodes=2))\n nodes2_hostnames = self.cluster_hostnames(nodes2)\n assert len(cluster) == 3\n assert cluster.num_available_nodes() == 0\n\n assert nodes_hostnames.isdisjoint(nodes2_hostnames)\n\n cluster.free(nodes)\n assert cluster.num_available_nodes() == 1\n\n cluster.free(nodes2)\n assert cluster.num_available_nodes() == 3\n\n def check_parsing(self):\n \"\"\"Checks that RemoteAccounts are generated correctly from input JSON\"\"\"\n\n node = create_json_cluster({\"nodes\": [{\"ssh_config\": {\"host\": \"hostname\"}}]}).alloc(\n Service.setup_cluster_spec(num_nodes=1)\n )[0]\n\n assert node.account.hostname == \"hostname\"\n assert node.account.user is None\n\n ssh_config = {\n \"host\": \"hostname\",\n \"user\": \"user\",\n \"hostname\": \"localhost\",\n \"port\": 22,\n }\n node = create_json_cluster(\n {\"nodes\": [{\"hostname\": \"hostname\", \"user\": \"user\", \"ssh_config\": ssh_config}]}\n ).alloc(Service.setup_cluster_spec(num_nodes=1))[0]\n\n assert node.account.hostname == \"hostname\"\n assert node.account.user == \"user\"\n\n # check ssh configs\n assert node.account.ssh_config.host == \"hostname\"\n assert node.account.ssh_config.user == \"user\"\n assert node.account.ssh_config.hostname == \"localhost\"\n assert node.account.ssh_config.port == 22\n\n def check_exhausts_supply(self):\n cluster = create_json_cluster(self.single_node_cluster_json)\n with pytest.raises(InsufficientResourcesError):\n cluster.alloc(Service.setup_cluster_spec(num_nodes=2))\n\n def check_node_names(self):\n cluster = create_json_cluster(\n {\n \"nodes\": [\n {\"ssh_config\": {\"host\": \"localhost1\"}},\n {\"ssh_config\": {\"host\": \"localhost2\"}},\n {\"ssh_config\": {\"host\": \"localhost3\"}},\n ]\n }\n )\n hosts = set([\"localhost1\", \"localhost2\", \"localhost3\"])\n nodes = cluster.alloc(cluster.available())\n assert hosts == set(node.name for node in nodes)\n" }, { "path": "tests/cluster/check_localhost.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.cluster.localhost import LocalhostCluster\nfrom ducktape.services.service import Service\n\nimport pickle\n\n\nclass CheckLocalhostCluster(object):\n def setup_method(self, _):\n self.cluster = LocalhostCluster()\n\n def check_size(self):\n len(self.cluster) >= 2**31 - 1\n\n def check_pickleable(self):\n cluster = LocalhostCluster()\n pickle.dumps(cluster)\n\n def check_request_free(self):\n available = self.cluster.num_available_nodes()\n initial_size = len(self.cluster)\n\n # Should be able to allocate arbitrarily many nodes\n nodes = self.cluster.alloc(Service.setup_cluster_spec(num_nodes=100))\n assert len(nodes) == 100\n for i, node in enumerate(nodes):\n assert node.account.hostname == \"localhost%d\" % i\n assert node.account.ssh_hostname == \"localhost\"\n assert node.account.ssh_config.hostname == \"localhost\"\n assert node.account.ssh_config.port == 22\n assert node.account.user is None\n\n assert self.cluster.num_available_nodes() == (available - 100)\n assert len(self.cluster) == initial_size # This shouldn't change\n\n self.cluster.free(nodes)\n\n assert self.cluster.num_available_nodes() == available\n" }, { "path": "tests/cluster/check_node_container.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.cluster.cluster_node import ClusterNode\nfrom ducktape.cluster.cluster_spec import ClusterSpec\nfrom ducktape.cluster.consts import WINDOWS, LINUX\nfrom ducktape.cluster.node_spec import NodeSpec\nfrom ducktape.cluster.node_container import (\n NodeContainer,\n NodeNotPresentError,\n InsufficientResourcesError,\n InsufficientHealthyNodesError,\n)\nimport pytest\n\nfrom ducktape.cluster.remoteaccount import RemoteAccountSSHConfig\nfrom tests.ducktape_mock import MockAccount\nfrom tests.runner.fake_remote_account import FakeRemoteAccount, FakeWindowsRemoteAccount\n\n\ndef fake_account(host, is_available=True, node_type=None):\n return FakeRemoteAccount(\n ssh_config=RemoteAccountSSHConfig(host=host), is_available=is_available, node_type=node_type\n )\n\n\ndef fake_win_account(host, is_available=True, node_type=None):\n return FakeWindowsRemoteAccount(\n ssh_config=RemoteAccountSSHConfig(host=host), is_available=is_available, node_type=node_type\n )\n\n\ndef count_nodes_by_os(container, target_os):\n \"\"\"Helper to count nodes by OS from node_groups.\"\"\"\n count = 0\n for (os, _), nodes in container.node_groups.items():\n if os == target_os:\n count += len(nodes)\n return count\n\n\nclass CheckNodeContainer(object):\n def check_sizes(self):\n empty = NodeContainer()\n assert 0 == empty.size()\n assert 0 == len(empty)\n nodes = [ClusterNode(MockAccount())]\n container = NodeContainer(nodes)\n assert 1 == container.size()\n assert 1 == len(container)\n\n def check_add_and_remove(self):\n nodes = [\n ClusterNode(MockAccount()),\n ClusterNode(MockAccount()),\n ClusterNode(MockAccount()),\n ClusterNode(MockAccount()),\n ClusterNode(MockAccount()),\n ]\n container = NodeContainer([])\n assert 0 == len(container)\n container.add_node(nodes[0])\n container.add_node(nodes[1])\n container.add_node(nodes[2])\n container2 = container.clone()\n i = 0\n for node in container:\n assert nodes[i] == node\n i += 1\n assert 3 == len(container)\n container.remove_node(nodes[0])\n with pytest.raises(NodeNotPresentError):\n container.remove_node(nodes[0])\n assert 2 == len(container)\n assert 3 == len(container2)\n\n def check_remove_single_node_spec(self):\n \"\"\"Check remove_spec() method - verify a simple happy path of removing a single node\"\"\"\n accounts = [\n fake_account(\"host1\"),\n fake_account(\"host2\"),\n fake_win_account(\"w1\"),\n fake_win_account(\"w2\"),\n ]\n container = NodeContainer(accounts)\n one_linux_node_spec = ClusterSpec(nodes=[NodeSpec(LINUX)])\n one_windows_node_spec = ClusterSpec(nodes=[NodeSpec(WINDOWS)])\n\n def _remove_single_node(one_node_spec, os):\n assert container.can_remove_spec(one_node_spec)\n good_nodes, bad_nodes = container.remove_spec(one_node_spec)\n assert good_nodes and len(good_nodes) == 1\n assert good_nodes[0].os == os\n assert not bad_nodes\n\n _remove_single_node(one_windows_node_spec, WINDOWS)\n assert count_nodes_by_os(container, LINUX) == 2\n assert count_nodes_by_os(container, WINDOWS) == 1\n\n _remove_single_node(one_windows_node_spec, WINDOWS)\n assert count_nodes_by_os(container, LINUX) == 2\n assert count_nodes_by_os(container, WINDOWS) == 0\n assert not container.can_remove_spec(one_windows_node_spec)\n with pytest.raises(InsufficientResourcesError):\n container.remove_spec(one_windows_node_spec)\n\n _remove_single_node(one_linux_node_spec, LINUX)\n assert count_nodes_by_os(container, LINUX) == 1\n assert count_nodes_by_os(container, WINDOWS) == 0\n\n _remove_single_node(one_linux_node_spec, LINUX)\n assert count_nodes_by_os(container, LINUX) == 0\n assert count_nodes_by_os(container, WINDOWS) == 0\n assert not container.can_remove_spec(one_linux_node_spec)\n with pytest.raises(InsufficientResourcesError):\n container.remove_spec(one_linux_node_spec)\n\n @pytest.mark.parametrize(\n \"cluster_spec\",\n [\n pytest.param(\n ClusterSpec(nodes=[NodeSpec(LINUX), NodeSpec(WINDOWS), NodeSpec(WINDOWS)]),\n id=\"not enough windows nodes\",\n ),\n pytest.param(\n ClusterSpec(nodes=[NodeSpec(LINUX), NodeSpec(LINUX), NodeSpec(WINDOWS)]),\n id=\"not enough linux nodes\",\n ),\n pytest.param(\n ClusterSpec(\n nodes=[\n NodeSpec(LINUX),\n NodeSpec(LINUX),\n NodeSpec(WINDOWS),\n NodeSpec(WINDOWS),\n ]\n ),\n id=\"not enough nodes\",\n ),\n ],\n )\n def check_not_enough_nodes_to_remove(self, cluster_spec):\n \"\"\"\n Check what happens if there aren't enough resources in this container to match a given spec.\n Various parametrizations check the behavior for when there are enough nodes for one OS but not another,\n or for both.\n \"\"\"\n accounts = [fake_account(\"host1\"), fake_win_account(\"w1\")]\n container = NodeContainer(accounts)\n original_container = container.clone()\n\n assert not container.can_remove_spec(cluster_spec)\n assert len(container.attempt_remove_spec(cluster_spec)) > 0\n\n with pytest.raises(InsufficientResourcesError):\n container.remove_spec(cluster_spec)\n\n # check that container was not modified\n assert container.node_groups == original_container.node_groups\n\n @pytest.mark.parametrize(\n \"accounts\",\n [\n pytest.param(\n [\n fake_account(\"host1\"),\n fake_account(\"host2\"),\n fake_win_account(\"w1\"),\n fake_win_account(\"w2\", is_available=False),\n ],\n id=\"windows not available\",\n ),\n pytest.param(\n [\n fake_account(\"host1\"),\n fake_account(\"host2\", is_available=False),\n fake_win_account(\"w1\"),\n fake_win_account(\"w2\"),\n ],\n id=\"linux not available\",\n ),\n pytest.param(\n [\n fake_account(\"host1\"),\n fake_account(\"host2\", is_available=False),\n fake_win_account(\"w1\"),\n fake_win_account(\"w2\", is_available=False),\n ],\n id=\"neither is available\",\n ),\n ],\n )\n def check_not_enough_healthy_nodes(self, accounts):\n \"\"\"\n When there's not enough healthy nodes in any of the OS-s, we obviously don't want to remove anything.\n Even when there's enough healthy nodes for one of the OS-s, but not enough in another one,\n we don't want to remove any nodes at all.\n Various sets of params check if there aren't enough healthy nodes for one OS but not the other, or both.\n \"\"\"\n container = NodeContainer(accounts)\n original_container = container.clone()\n expected_bad_nodes = [acc for acc in accounts if not acc.is_available]\n spec = ClusterSpec(\n nodes=[\n NodeSpec(LINUX),\n NodeSpec(LINUX),\n NodeSpec(WINDOWS),\n NodeSpec(WINDOWS),\n ]\n )\n assert container.can_remove_spec(spec)\n with pytest.raises(InsufficientHealthyNodesError) as exc_info:\n container.remove_spec(spec)\n assert exc_info.value.bad_nodes == expected_bad_nodes\n # check that no nodes were actually allocated, but unhealthy ones were removed from the cluster\n original_container.remove_nodes(expected_bad_nodes)\n assert container.node_groups == original_container.node_groups\n\n @pytest.mark.parametrize(\n \"accounts\",\n [\n pytest.param(\n [\n fake_account(\"host1\"),\n fake_account(\"host2\"),\n fake_account(\"host3\"),\n fake_win_account(\"w1\", is_available=False),\n fake_win_account(\"w2\"),\n fake_win_account(\"w2\"),\n ],\n id=\"windows not available\",\n ),\n pytest.param(\n [\n fake_account(\"host1\", is_available=False),\n fake_account(\"host2\"),\n fake_account(\"host3\"),\n fake_win_account(\"w1\"),\n fake_win_account(\"w2\"),\n fake_win_account(\"w2\"),\n ],\n id=\"linux not available\",\n ),\n pytest.param(\n [\n fake_account(\"host1\", is_available=False),\n fake_account(\"host2\"),\n fake_account(\"host3\"),\n fake_win_account(\"w1\", is_available=False),\n fake_win_account(\"w2\"),\n fake_win_account(\"w2\"),\n ],\n id=\"neither is available\",\n ),\n pytest.param(\n [\n fake_account(\"host1\"),\n fake_account(\"host2\"),\n fake_account(\"host3\"),\n fake_win_account(\"w1\"),\n fake_win_account(\"w2\"),\n fake_win_account(\"w2\"),\n ],\n id=\"all are available\",\n ),\n ],\n )\n def check_enough_healthy_but_some_bad_nodes_too(self, accounts):\n \"\"\"\n Check that we can successfully allocate all necessary nodes - even if some nodes don't pass health checks,\n we still have enough nodes to match the provided cluster spec.\n\n This test assumes that if we do encounter unhealthy node,\n we encounter it before we can finish allocating healthy ones, otherwise it would just mean testing the happy\n path (which we do in one of the params).\n \"\"\"\n container = NodeContainer(accounts)\n original_container = container.clone()\n expected_bad_nodes = [acc for acc in accounts if not acc.is_available]\n spec = ClusterSpec(\n nodes=[\n NodeSpec(LINUX),\n NodeSpec(LINUX),\n NodeSpec(WINDOWS),\n NodeSpec(WINDOWS),\n ]\n )\n\n assert container.can_remove_spec(spec)\n good_nodes, bad_nodes = container.remove_spec(spec)\n\n # alloc should succeed\n # check that we did catch a bad node if any\n assert bad_nodes == expected_bad_nodes\n # check that container has exactly the right number of nodes left -\n # we removed len(spec) healthy nodes, plus len(expected_bad_nodes) of unhealthy nodes.\n assert len(container) == len(original_container) - len(spec) - len(expected_bad_nodes)\n\n # check that we got 2 windows nodes and two linux nodes in response,\n # don't care which ones in particular\n assert len(good_nodes) == 4\n actual_linux = [node for node in good_nodes if node.os == LINUX]\n assert len(actual_linux) == 2\n actual_win = [node for node in good_nodes if node.os == WINDOWS]\n assert len(actual_win) == 2\n\n def check_empty_cluster_spec(self):\n accounts = [fake_account(\"host1\"), fake_account(\"host2\"), fake_account(\"host3\")]\n container = NodeContainer(accounts)\n spec = ClusterSpec.empty()\n assert not container.attempt_remove_spec(spec)\n assert container.can_remove_spec(spec)\n good, bad = container.remove_spec(spec)\n assert not good\n assert not bad\n\n def check_none_cluster_spec(self):\n accounts = [fake_account(\"host1\"), fake_account(\"host2\"), fake_account(\"host3\")]\n container = NodeContainer(accounts)\n spec = None\n assert container.attempt_remove_spec(spec)\n assert not container.can_remove_spec(spec)\n with pytest.raises(InsufficientResourcesError):\n container.remove_spec(spec)\n\n # ==================== node_type tests ====================\n\n def check_node_groups_by_type(self):\n \"\"\"Check that nodes are grouped by (os, node_type) in node_groups.\"\"\"\n accounts = [\n fake_account(\"host1\", node_type=\"small\"),\n fake_account(\"host2\", node_type=\"small\"),\n fake_account(\"host3\", node_type=\"large\"),\n fake_account(\"host4\"), # no node_type (None)\n fake_win_account(\"w1\", node_type=\"small\"),\n ]\n container = NodeContainer(accounts)\n\n # Check node_groups has correct keys\n assert (LINUX, \"small\") in container.node_groups\n assert (LINUX, \"large\") in container.node_groups\n assert (LINUX, None) in container.node_groups\n assert (WINDOWS, \"small\") in container.node_groups\n\n # Check counts\n assert len(container.node_groups[(LINUX, \"small\")]) == 2\n assert len(container.node_groups[(LINUX, \"large\")]) == 1\n assert len(container.node_groups[(LINUX, None)]) == 1\n assert len(container.node_groups[(WINDOWS, \"small\")]) == 1\n\n def check_remove_spec_with_node_type(self):\n \"\"\"Check remove_spec with node_type specified in the cluster spec.\"\"\"\n accounts = [\n fake_account(\"host1\", node_type=\"small\"),\n fake_account(\"host2\", node_type=\"small\"),\n fake_account(\"host3\", node_type=\"large\"),\n ]\n container = NodeContainer(accounts)\n\n # Request 1 small node\n small_spec = ClusterSpec(nodes=[NodeSpec(LINUX, node_type=\"small\")])\n assert container.can_remove_spec(small_spec)\n good_nodes, bad_nodes = container.remove_spec(small_spec)\n assert len(good_nodes) == 1\n assert good_nodes[0].node_type == \"small\"\n assert not bad_nodes\n\n # Should have 1 small and 1 large left\n assert len(container.node_groups.get((LINUX, \"small\"), [])) == 1\n assert len(container.node_groups.get((LINUX, \"large\"), [])) == 1\n\n def check_remove_spec_with_node_type_not_available(self):\n \"\"\"Check remove_spec fails when requested node_type is not available.\"\"\"\n accounts = [\n fake_account(\"host1\", node_type=\"small\"),\n fake_account(\"host2\", node_type=\"small\"),\n ]\n container = NodeContainer(accounts)\n\n # Request a large node (not available)\n large_spec = ClusterSpec(nodes=[NodeSpec(LINUX, node_type=\"large\")])\n assert not container.can_remove_spec(large_spec)\n with pytest.raises(InsufficientResourcesError):\n container.remove_spec(large_spec)\n\n def check_remove_spec_node_type_none_matches_any(self):\n \"\"\"Check that node_type=None in spec matches any available node.\"\"\"\n accounts = [\n fake_account(\"host1\", node_type=\"small\"),\n fake_account(\"host2\", node_type=\"large\"),\n ]\n container = NodeContainer(accounts)\n\n # Request a node with no specific type - should match any\n any_spec = ClusterSpec(nodes=[NodeSpec(LINUX, node_type=None)])\n assert container.can_remove_spec(any_spec)\n good_nodes, _ = container.remove_spec(any_spec)\n assert len(good_nodes) == 1\n # Should have allocated one of the available nodes\n assert good_nodes[0].node_type in (\"small\", \"large\")\n\n def check_remove_spec_mixed_node_types(self):\n \"\"\"Check remove_spec with mixed node_type requirements.\"\"\"\n accounts = [\n fake_account(\"host1\", node_type=\"small\"),\n fake_account(\"host2\", node_type=\"small\"),\n fake_account(\"host3\", node_type=\"large\"),\n fake_account(\"host4\", node_type=\"large\"),\n ]\n container = NodeContainer(accounts)\n\n # Request 1 small and 1 large\n mixed_spec = ClusterSpec(\n nodes=[\n NodeSpec(LINUX, node_type=\"small\"),\n NodeSpec(LINUX, node_type=\"large\"),\n ]\n )\n assert container.can_remove_spec(mixed_spec)\n good_nodes, _ = container.remove_spec(mixed_spec)\n assert len(good_nodes) == 2\n\n small_nodes = [n for n in good_nodes if n.node_type == \"small\"]\n large_nodes = [n for n in good_nodes if n.node_type == \"large\"]\n assert len(small_nodes) == 1\n assert len(large_nodes) == 1\n\n # ==================== Double-counting fix tests ====================\n\n def check_mixed_typed_and_untyped_double_counting_rejected(self):\n \"\"\"\n Test that mixed typed and untyped requirements correctly fail when\n total capacity is insufficient (the double-counting bug fix).\n\n Scenario:\n - Available: 3 Linux nodes (2 small, 1 large)\n - Request: 2 linux/small + 2 linux/any\n - Expected: FAIL (need 4 nodes, only have 3)\n\n Before the fix, this would incorrectly pass because:\n - linux/small check: 2 available >= 2 needed ✓\n - linux/any check: 3 available >= 2 needed ✓\n But the any-type was double-counting the small nodes!\n \"\"\"\n accounts = [\n fake_account(\"host1\", node_type=\"small\"),\n fake_account(\"host2\", node_type=\"small\"),\n fake_account(\"host3\", node_type=\"large\"),\n ]\n container = NodeContainer(accounts)\n\n # Request 2 small + 2 any = 4 total, but only 3 available\n impossible_spec = ClusterSpec(\n nodes=[\n NodeSpec(LINUX, node_type=\"small\"),\n NodeSpec(LINUX, node_type=\"small\"),\n NodeSpec(LINUX, node_type=None),\n NodeSpec(LINUX, node_type=None),\n ]\n )\n assert not container.can_remove_spec(impossible_spec)\n with pytest.raises(InsufficientResourcesError):\n container.remove_spec(impossible_spec)\n\n def check_mixed_typed_and_untyped_valid_passes(self):\n \"\"\"\n Test that mixed typed and untyped requirements correctly pass when\n there is sufficient capacity.\n\n Scenario:\n - Available: 4 Linux nodes (2 small, 2 large)\n - Request: 2 linux/small + 1 linux/any\n - Expected: PASS (need 3 nodes, have 4)\n \"\"\"\n accounts = [\n fake_account(\"host1\", node_type=\"small\"),\n fake_account(\"host2\", node_type=\"small\"),\n fake_account(\"host3\", node_type=\"large\"),\n fake_account(\"host4\", node_type=\"large\"),\n ]\n container = NodeContainer(accounts)\n\n # Request 2 small + 1 any = 3 total, have 4 available\n valid_spec = ClusterSpec(\n nodes=[\n NodeSpec(LINUX, node_type=\"small\"),\n NodeSpec(LINUX, node_type=\"small\"),\n NodeSpec(LINUX, node_type=None),\n ]\n )\n assert container.can_remove_spec(valid_spec)\n good_nodes, bad_nodes = container.remove_spec(valid_spec)\n assert len(good_nodes) == 3\n assert not bad_nodes\n\n def check_specific_type_shortage_detected(self):\n \"\"\"\n Test that shortage of a specific type is detected even when total\n OS capacity is sufficient.\n\n Scenario:\n - Available: 3 Linux nodes (1 small, 2 large)\n - Request: 2 linux/small + 1 linux/any\n - Expected: FAIL (need 2 small, only have 1)\n \"\"\"\n accounts = [\n fake_account(\"host1\", node_type=\"small\"),\n fake_account(\"host2\", node_type=\"large\"),\n fake_account(\"host3\", node_type=\"large\"),\n ]\n container = NodeContainer(accounts)\n\n # Request 2 small + 1 any - total capacity OK but not enough small\n spec = ClusterSpec(\n nodes=[\n NodeSpec(LINUX, node_type=\"small\"),\n NodeSpec(LINUX, node_type=\"small\"),\n NodeSpec(LINUX, node_type=None),\n ]\n )\n assert not container.can_remove_spec(spec)\n error_msg = container.attempt_remove_spec(spec)\n assert \"linux/small\" in error_msg\n\n def check_holistic_os_capacity_check(self):\n \"\"\"\n Test that total OS capacity is checked before detailed type checks.\n\n Scenario:\n - Available: 2 Linux nodes (1 small, 1 large)\n - Request: 1 linux/small + 1 linux/large + 1 linux/any\n - Expected: FAIL (need 3 total, only have 2)\n \"\"\"\n accounts = [\n fake_account(\"host1\", node_type=\"small\"),\n fake_account(\"host2\", node_type=\"large\"),\n ]\n container = NodeContainer(accounts)\n\n spec = ClusterSpec(\n nodes=[\n NodeSpec(LINUX, node_type=\"small\"),\n NodeSpec(LINUX, node_type=\"large\"),\n NodeSpec(LINUX, node_type=None),\n ]\n )\n assert not container.can_remove_spec(spec)\n error_msg = container.attempt_remove_spec(spec)\n # Should report total Linux shortage\n assert \"linux\" in error_msg.lower()\n\n def check_multi_os_mixed_requirements(self):\n \"\"\"\n Test holistic validation works correctly with multiple OS types.\n\n Scenario:\n - Available: 3 Linux (2 small, 1 large), 2 Windows (1 small, 1 large)\n - Request: 2 linux/small + 1 linux/any + 1 windows/any\n - Expected: FAIL for Linux (need 3, have 3, but 2 small reserved leaves only 1 for any)\n \"\"\"\n accounts = [\n fake_account(\"host1\", node_type=\"small\"),\n fake_account(\"host2\", node_type=\"small\"),\n fake_account(\"host3\", node_type=\"large\"),\n fake_win_account(\"w1\", node_type=\"small\"),\n fake_win_account(\"w2\", node_type=\"large\"),\n ]\n container = NodeContainer(accounts)\n\n # For Linux: 2 small + 2 any = 4 total, but only 3 available\n # For Windows: 1 any = fine\n spec = ClusterSpec(\n nodes=[\n NodeSpec(LINUX, node_type=\"small\"),\n NodeSpec(LINUX, node_type=\"small\"),\n NodeSpec(LINUX, node_type=None),\n NodeSpec(LINUX, node_type=None),\n NodeSpec(WINDOWS, node_type=None),\n ]\n )\n assert not container.can_remove_spec(spec)\n\n def check_only_any_type_requirements(self):\n \"\"\"\n Test that requests with only any-type requirements work correctly.\n \"\"\"\n accounts = [\n fake_account(\"host1\", node_type=\"small\"),\n fake_account(\"host2\", node_type=\"large\"),\n fake_account(\"host3\"), # no type\n ]\n container = NodeContainer(accounts)\n\n # Request 3 any-type - should work\n spec = ClusterSpec(\n nodes=[\n NodeSpec(LINUX, node_type=None),\n NodeSpec(LINUX, node_type=None),\n NodeSpec(LINUX, node_type=None),\n ]\n )\n assert container.can_remove_spec(spec)\n good_nodes, _ = container.remove_spec(spec)\n assert len(good_nodes) == 3\n\n # Request 4 any-type - should fail\n container2 = NodeContainer(accounts)\n spec2 = ClusterSpec(\n nodes=[\n NodeSpec(LINUX, node_type=None),\n NodeSpec(LINUX, node_type=None),\n NodeSpec(LINUX, node_type=None),\n NodeSpec(LINUX, node_type=None),\n ]\n )\n assert not container2.can_remove_spec(spec2)\n\n def check_allocation_order_specific_before_any(self):\n \"\"\"\n Test that specific node_type requirements are allocated before any-type.\n\n This prevents any-type requests from \"stealing\" nodes needed by specific types.\n\n Scenario:\n - Available: 2 Linux nodes (1 small, 1 large)\n - Request: 1 linux/small + 1 linux/any\n - Expected: SUCCESS - small gets the small node, any gets the large node\n\n Without proper ordering, if linux/any is processed first, it might take\n the small node, leaving linux/small with no matching nodes.\n \"\"\"\n accounts = [\n fake_account(\"host1\", node_type=\"small\"),\n fake_account(\"host2\", node_type=\"large\"),\n ]\n container = NodeContainer(accounts)\n\n # Request 1 small + 1 any - should succeed with proper ordering\n spec = ClusterSpec(\n nodes=[\n NodeSpec(LINUX, node_type=\"small\"),\n NodeSpec(LINUX, node_type=None),\n ]\n )\n assert container.can_remove_spec(spec)\n good_nodes, bad_nodes = container.remove_spec(spec)\n assert len(good_nodes) == 2\n assert not bad_nodes\n\n # Verify we got one of each type\n types = [n.node_type for n in good_nodes]\n assert \"small\" in types\n assert \"large\" in types\n\n def check_allocation_order_with_multiple_specific_types(self):\n \"\"\"\n Test allocation order with multiple specific types and any-type.\n\n Scenario:\n - Available: 3 Linux nodes (1 small, 1 medium, 1 large)\n - Request: 1 linux/small + 1 linux/large + 1 linux/any\n - Expected: SUCCESS - specific types get their nodes, any gets medium\n \"\"\"\n accounts = [\n fake_account(\"host1\", node_type=\"small\"),\n fake_account(\"host2\", node_type=\"medium\"),\n fake_account(\"host3\", node_type=\"large\"),\n ]\n container = NodeContainer(accounts)\n\n spec = ClusterSpec(\n nodes=[\n NodeSpec(LINUX, node_type=\"small\"),\n NodeSpec(LINUX, node_type=\"large\"),\n NodeSpec(LINUX, node_type=None),\n ]\n )\n assert container.can_remove_spec(spec)\n good_nodes, _ = container.remove_spec(spec)\n assert len(good_nodes) == 3\n\n # Verify we got all three types\n types = [n.node_type for n in good_nodes]\n assert \"small\" in types\n assert \"medium\" in types\n assert \"large\" in types\n" }, { "path": "tests/cluster/check_remoteaccount.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.errors import TimeoutError\nfrom tests.ducktape_mock import MockAccount\nfrom tests.test_utils import find_available_port\nfrom ducktape.cluster.remoteaccount import RemoteAccount\nfrom ducktape.cluster.remoteaccount import RemoteAccountSSHConfig\nimport pytest\n\nimport logging\nfrom threading import Thread\nfrom http.server import SimpleHTTPRequestHandler\nimport socketserver\nimport threading\nimport time\n\n\nclass DummyException(Exception):\n pass\n\n\ndef raise_error_checker(error, remote_account):\n raise DummyException(\"dummy raise: {}\\nfrom: {}\".format(error, remote_account))\n\n\ndef raise_no_error_checker(error, remote_account):\n pass\n\n\nclass SimpleServer(object):\n \"\"\"Helper class which starts a simple server listening on localhost at the specified port\"\"\"\n\n def __init__(self):\n self.port = find_available_port()\n self.handler = SimpleHTTPRequestHandler\n self.httpd = socketserver.TCPServer((\"\", self.port), self.handler)\n self.close_signal = threading.Event()\n self.server_started = False\n\n def start(self, delay_sec=0.0):\n \"\"\"Run the server after specified delay\"\"\"\n\n def run():\n self.close_signal.wait(delay_sec)\n\n if not self.close_signal.is_set():\n self.server_started = True\n self.httpd.serve_forever()\n\n self.background_thread = Thread(target=run)\n self.background_thread.start()\n\n def stop(self):\n self.close_signal.set()\n\n if self.server_started:\n self.httpd.shutdown()\n self.background_thread.join(timeout=0.5)\n if self.background_thread.is_alive():\n raise Exception(\"SimpleServer failed to stop quickly\")\n\n\nclass CheckRemoteAccount(object):\n def setup(self):\n self.server = SimpleServer()\n self.account = MockAccount()\n\n def check_wait_for_http(self):\n \"\"\"Check waiting without timeout\"\"\"\n self.server.start(delay_sec=0.0)\n self.account.wait_for_http_service(port=self.server.port, headers={}, timeout=10, path=\"/\")\n\n def check_wait_for_http_timeout(self):\n \"\"\"Check waiting with timeout\"\"\"\n\n timeout = 1\n start = time.time()\n self.server.start(delay_sec=5)\n\n try:\n self.account.wait_for_http_service(port=self.server.port, headers={}, timeout=timeout, path=\"/\")\n raise Exception(\"Should have timed out waiting for server to start\")\n except TimeoutError:\n # expected behavior. Now check that we're reasonably close to the expected timeout\n # This is a fairly loose check since there are various internal timeouts that can affect the overall\n # timing\n actual_timeout = time.time() - start\n assert abs(actual_timeout - timeout) / timeout < 1\n\n @pytest.mark.parametrize(\n \"checkers\",\n [\n [raise_error_checker],\n [raise_no_error_checker, raise_error_checker],\n [raise_error_checker, raise_no_error_checker],\n ],\n )\n def check_ssh_checker(self, checkers):\n self.server.start()\n ssh_config = RemoteAccountSSHConfig.from_string(\n \"\"\"\n Host dummy_host.com\n Hostname dummy_host.name.com\n Port 22\n User dummy\n ConnectTimeout 1\n \"\"\"\n )\n self.account = RemoteAccount(ssh_config, ssh_exception_checks=checkers)\n with pytest.raises(DummyException):\n self.account.ssh(\"echo test\")\n\n def teardown(self):\n self.server.stop()\n\n\nclass CheckRemoteAccountEquality(object):\n def check_remote_account_equality(self):\n \"\"\"Different instances of remote account initialized with the same parameters should be equal.\"\"\"\n\n ssh_config = RemoteAccountSSHConfig(host=\"thehost\", hostname=\"localhost\", port=22)\n\n kwargs = {\n \"ssh_config\": ssh_config,\n \"externally_routable_ip\": \"345\",\n \"logger\": logging.getLogger(__name__),\n }\n r1 = RemoteAccount(**kwargs)\n r2 = RemoteAccount(**kwargs)\n\n assert r1 == r2\n" }, { "path": "tests/cluster/check_vagrant.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.cluster.vagrant import VagrantCluster\nfrom ducktape.services.service import Service\nimport json\nimport pickle\nimport os\nimport random\nimport pytest\nfrom ducktape.cluster.remoteaccount import RemoteAccountError\n\nfrom tests.runner.fake_remote_account import create_fake_remote_account\n\nTWO_HOSTS = \"\"\"Host worker1\n HostName 127.0.0.1\n User vagrant\n Port 2222\n UserKnownHostsFile /dev/null\n StrictHostKeyChecking no\n PasswordAuthentication no\n IdentityFile /Users/foo/ducktape.git/.vagrant/machines/worker1/virtualbox/private_key\n IdentitiesOnly yes\n LogLevel FATAL\n\nHost worker2\n HostName 127.0.0.2\n User vagrant\n Port 2200\n UserKnownHostsFile /dev/null\n StrictHostKeyChecking no\n PasswordAuthentication no\n IdentityFile /Users/foo/ducktape.git/.vagrant/machines/worker2/virtualbox/private_key\n IdentitiesOnly yes\n LogLevel FATAL\n\n\"\"\"\n\n\ndef make_vagrant_cluster(*args, **kwargs):\n return VagrantCluster(make_remote_account_func=create_fake_remote_account, *args, **kwargs)\n\n\nclass CheckVagrantCluster(object):\n def setup_method(self, _):\n # We roll our own tempfile name instead of using python tempfile module because\n # in some cases, we want self.cluster_file to be the name of a file which does not yet exist\n self.cluster_file = \"cluster_file_temporary-%d.json\" % random.randint(1, 2**63 - 1)\n if os.path.exists(self.cluster_file):\n os.remove(self.cluster_file)\n\n def teardown_method(self, _):\n if os.path.exists(self.cluster_file):\n os.remove(self.cluster_file)\n\n def _set_monkeypatch_attr(self, monkeypatch):\n monkeypatch.setattr(\n \"ducktape.cluster.vagrant.VagrantCluster._vagrant_ssh_config\",\n lambda vc: (TWO_HOSTS, None),\n )\n monkeypatch.setattr(\n \"ducktape.cluster.linux_remoteaccount.LinuxRemoteAccount.fetch_externally_routable_ip\",\n lambda vc: \"127.0.0.1\",\n )\n\n def check_pickleable(self, monkeypatch):\n self._set_monkeypatch_attr(monkeypatch)\n cluster = VagrantCluster()\n pickle.dumps(cluster)\n\n def check_one_host_parsing(self, monkeypatch):\n \"\"\"check the behavior of VagrantCluster when cluster_file is not specified. VagrantCluster should read\n cluster information from _vagrant_ssh_config().\n \"\"\"\n self._set_monkeypatch_attr(monkeypatch)\n\n cluster = make_vagrant_cluster()\n assert len(cluster) == 2\n assert cluster.num_available_nodes() == 2\n node1, node2 = cluster.alloc(Service.setup_cluster_spec(num_nodes=2))\n\n assert node1.account.hostname == \"worker1\"\n assert node1.account.user == \"vagrant\"\n assert node1.account.ssh_hostname == \"127.0.0.1\"\n\n assert node2.account.hostname == \"worker2\"\n assert node2.account.user == \"vagrant\"\n assert node2.account.ssh_hostname == \"127.0.0.2\"\n\n def check_cluster_file_write(self, monkeypatch):\n \"\"\"check the behavior of VagrantCluster when cluster_file is specified but the file doesn't exist.\n VagrantCluster should read cluster information from _vagrant_ssh_config() and write the information to\n cluster_file.\n \"\"\"\n self._set_monkeypatch_attr(monkeypatch)\n assert not os.path.exists(self.cluster_file)\n\n cluster = make_vagrant_cluster(cluster_file=self.cluster_file)\n cluster_json_expected = {}\n nodes = [\n {\n \"externally_routable_ip\": node_account.externally_routable_ip,\n \"ssh_config\": {\n \"host\": node_account.ssh_config.host,\n \"hostname\": node_account.ssh_config.hostname,\n \"user\": node_account.ssh_config.user,\n \"identityfile\": node_account.ssh_config.identityfile,\n \"password\": node_account.ssh_config.password,\n \"port\": node_account.ssh_config.port,\n \"connecttimeout\": None,\n },\n }\n for node_account in cluster._available_accounts\n ]\n\n cluster_json_expected[\"nodes\"] = nodes\n\n cluster_json_actual = json.load(open(os.path.abspath(self.cluster_file)))\n assert cluster_json_actual == cluster_json_expected\n\n def check_cluster_file_read(self, monkeypatch):\n \"\"\"check the behavior of VagrantCluster when cluster_file is specified and the file exists.\n VagrantCluster should read cluster information from cluster_file.\n \"\"\"\n self._set_monkeypatch_attr(monkeypatch)\n\n # To verify that VagrantCluster reads cluster information from the cluster_file, the\n # content in the file is intentionally made different from that returned by _vagrant_ssh_config().\n nodes_expected = []\n node1_expected = {\n \"externally_routable_ip\": \"127.0.0.3\",\n \"ssh_config\": {\n \"host\": \"worker3\",\n \"hostname\": \"127.0.0.3\",\n \"user\": \"vagrant\",\n \"port\": 2222,\n \"password\": \"password\",\n \"identityfile\": \"/path/to/identfile3\",\n \"connecttimeout\": None,\n },\n }\n nodes_expected.append(node1_expected)\n\n node2_expected = {\n \"externally_routable_ip\": \"127.0.0.2\",\n \"ssh_config\": {\n \"host\": \"worker2\",\n \"hostname\": \"127.0.0.2\",\n \"user\": \"vagrant\",\n \"port\": 2223,\n \"password\": None,\n \"identityfile\": \"/path/to/indentfile2\",\n \"connecttimeout\": 10,\n },\n }\n nodes_expected.append(node2_expected)\n\n cluster_json_expected = {}\n cluster_json_expected[\"nodes\"] = nodes_expected\n json.dump(\n cluster_json_expected,\n open(self.cluster_file, \"w+\"),\n indent=2,\n separators=(\",\", \": \"),\n sort_keys=True,\n )\n\n # Load the cluster from the json file we just created\n cluster = make_vagrant_cluster(cluster_file=self.cluster_file)\n\n assert len(cluster) == 2\n assert cluster.num_available_nodes() == 2\n node2, node3 = cluster.alloc(Service.setup_cluster_spec(num_nodes=2))\n\n assert node3.account.hostname == \"worker2\"\n assert node3.account.user == \"vagrant\"\n assert node3.account.ssh_hostname == \"127.0.0.2\"\n assert node3.account.ssh_config.to_json() == node2_expected[\"ssh_config\"]\n\n assert node2.account.hostname == \"worker3\"\n assert node2.account.user == \"vagrant\"\n assert node2.account.ssh_hostname == \"127.0.0.3\"\n assert node2.account.ssh_config.to_json() == node1_expected[\"ssh_config\"]\n\n def check_no_valid_network_devices(self, monkeypatch):\n \"\"\"\n test to make sure that a remote account error is raised when no network devices are found\n \"\"\"\n monkeypatch.setattr(\n \"ducktape.cluster.vagrant.VagrantCluster._vagrant_ssh_config\",\n lambda vc: (TWO_HOSTS, None),\n )\n monkeypatch.setattr(\n \"ducktape.cluster.linux_remoteaccount.LinuxRemoteAccount.get_network_devices\",\n lambda account: [],\n )\n\n with pytest.raises(RemoteAccountError):\n VagrantCluster()\n" }, { "path": "tests/command_line/__init__.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n" }, { "path": "tests/command_line/check_main.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.command_line.main import get_user_defined_globals\nfrom ducktape.command_line.main import setup_results_directory\nfrom ducktape.command_line.main import update_latest_symlink\n\nimport json\nimport os\nimport os.path\nimport pickle\nimport pytest\nimport tempfile\n\n\nclass CheckSetupResultsDirectory(object):\n def setup_method(self, _):\n self.results_root = tempfile.mkdtemp()\n self.results_dir = os.path.join(self.results_root, \"results_directory\")\n self.latest_symlink = os.path.join(self.results_root, \"latest\")\n\n def validate_directories(self):\n \"\"\"Validate existence of results directory and correct symlink\"\"\"\n assert os.path.exists(self.results_dir)\n assert os.path.exists(self.latest_symlink)\n assert os.path.islink(self.latest_symlink)\n\n # check symlink points to correct location\n assert os.path.realpath(self.latest_symlink) == os.path.realpath(self.results_dir)\n\n def check_creation(self):\n \"\"\"Check results and symlink from scratch\"\"\"\n assert not os.path.exists(self.results_dir)\n setup_results_directory(self.results_dir)\n update_latest_symlink(self.results_root, self.results_dir)\n self.validate_directories()\n\n def check_symlink(self):\n \"\"\"Check \"latest\" symlink behavior\"\"\"\n\n # if symlink already exists\n old_results = os.path.join(self.results_root, \"OLD\")\n os.mkdir(old_results)\n os.symlink(old_results, self.latest_symlink)\n assert os.path.islink(self.latest_symlink) and os.path.exists(self.latest_symlink)\n\n setup_results_directory(self.results_dir)\n update_latest_symlink(self.results_root, self.results_dir)\n self.validate_directories()\n\n # Try again if symlink exists and points to nothing\n os.rmdir(self.results_dir)\n assert os.path.islink(self.latest_symlink) and not os.path.exists(self.latest_symlink)\n setup_results_directory(self.results_dir)\n update_latest_symlink(self.results_root, self.results_dir)\n self.validate_directories()\n\n\nglobals_json = \"\"\"\n{\n \"x\": 200\n}\n\"\"\"\n\ninvalid_globals_json = \"\"\"\n{\n can't parse this!: ?right?\n}\n\"\"\"\n\nvalid_json_not_dict = \"\"\"\n[\n {\n \"x\": 200,\n \"y\": 300\n }\n]\n\"\"\"\n\n\nclass CheckUserDefinedGlobals(object):\n \"\"\"Tests for the helper method which parses in user defined globals option\"\"\"\n\n def check_immutable(self):\n \"\"\"Expect the user defined dict object to be immutable.\"\"\"\n global_dict = get_user_defined_globals(globals_json)\n\n with pytest.raises(NotImplementedError):\n global_dict[\"x\"] = -1\n\n with pytest.raises(NotImplementedError):\n global_dict[\"y\"] = 3\n\n def check_pickleable(self):\n \"\"\"Expect the user defined dict object to be pickleable\"\"\"\n globals_dict = get_user_defined_globals(globals_json)\n\n assert globals_dict # Need to test non-empty dict, to ensure py3 compatibility\n assert pickle.loads(pickle.dumps(globals_dict)) == globals_dict\n\n def check_parseable_json_string(self):\n \"\"\"Check if globals_json is parseable as JSON, we get back a dictionary view of parsed JSON.\"\"\"\n globals_dict = get_user_defined_globals(globals_json)\n assert globals_dict == json.loads(globals_json)\n\n def check_unparseable(self):\n \"\"\"If globals string is not a path to a file, and not parseable as JSON we want to raise a ValueError\"\"\"\n with pytest.raises(ValueError):\n get_user_defined_globals(invalid_globals_json)\n\n def check_parse_from_file(self):\n \"\"\"Validate that, given a filename of a file containing valid JSON, we correctly parse the file contents.\"\"\"\n _, fname = tempfile.mkstemp()\n try:\n with open(fname, \"w\") as fh:\n fh.write(globals_json)\n\n global_dict = get_user_defined_globals(fname)\n assert global_dict == json.loads(globals_json)\n assert global_dict[\"x\"] == 200\n finally:\n os.remove(fname)\n\n def check_bad_parse_from_file(self):\n \"\"\"Validate behavior when given file containing invalid JSON\"\"\"\n _, fname = tempfile.mkstemp()\n try:\n with open(fname, \"w\") as fh:\n # Write invalid JSON\n fh.write(invalid_globals_json)\n\n with pytest.raises(ValueError):\n get_user_defined_globals(fname)\n\n finally:\n os.remove(fname)\n\n def check_non_dict(self):\n \"\"\"Valid JSON which does not parse as a dict should raise a ValueError\"\"\"\n\n # Should be able to parse this as JSON\n json.loads(valid_json_not_dict)\n\n with pytest.raises(ValueError):\n get_user_defined_globals(valid_json_not_dict)\n\n def check_non_dict_from_file(self):\n \"\"\"Validate behavior when given file containing valid JSON which does not parse as a dict\"\"\"\n _, fname = tempfile.mkstemp()\n try:\n with open(fname, \"w\") as fh:\n # Write valid JSON which does not parse as a dict\n fh.write(valid_json_not_dict)\n\n with pytest.raises(ValueError, match=fname):\n get_user_defined_globals(fname)\n\n finally:\n os.remove(fname)\n" }, { "path": "tests/command_line/check_parse_args.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.command_line.parse_args import parse_args\n\nfrom io import StringIO\n\nimport os\nimport re\nimport shutil\nimport sys\nimport tempfile\n\n\nclass Capturing(object):\n \"\"\"This context manager can be used to capture stdout from a function call.\n E.g.\n with Capture() as captured:\n call_function()\n assert captured.output == expected_output\n \"\"\"\n\n def __enter__(self):\n self._stdout = sys.stdout\n sys.stdout = self._stringio = StringIO()\n return self\n\n def __exit__(self, *args):\n self.output = self._stringio.getvalue()\n sys.stdout = self._stdout\n\n\nclass CheckParseArgs(object):\n def check_empty_args(self):\n \"\"\"Check that parsing an empty args list results in printing a usage message, followed by sys.exit(0)\"\"\"\n try:\n with Capturing() as captured:\n parse_args([])\n except SystemExit as e:\n assert e.code == 0\n assert captured.output.find(\"usage\") >= 0\n\n def check_version(self):\n \"\"\"If --version is present, ducktape should print version and exit\"\"\"\n try:\n with Capturing() as captured:\n parse_args([\"--version\"])\n except SystemExit as e:\n assert e.code == 0\n assert re.search(r\"[\\d]+\\.[\\d]+\\.[\\d]+\", captured.output) is not None\n\n def check_empty_test_path(self):\n \"\"\"Check that default test_path is an array consisting of cwd.\"\"\"\n args = [\"--collect-only\"]\n parsed = parse_args(args)\n parsed_paths = [os.path.abspath(p) for p in parsed[\"test_path\"]]\n\n assert parsed_paths == [os.path.abspath(\".\")]\n\n def check_multiple_test_paths(self):\n \"\"\"Check that parser properly handles multiple \"test paths\". It should capture a list of test paths.\"\"\"\n paths = [\"path1\"]\n args = [\"--debug\"] + paths + [\"--collect-only\"]\n parsed = parse_args(args)\n assert parsed[\"test_path\"] == paths\n\n paths = [\"path%d\" % i for i in range(10)]\n args = [\"--cluster-file\", \"my-cluster-file\"] + paths + [\"--debug\", \"--exit-first\"]\n parsed = parse_args(args)\n assert parsed[\"test_path\"] == paths\n\n def check_multiple_exclude(self):\n excluded = [\"excluded1\", \"excluded2\"]\n args = [\"--collect-only\", \"--exclude\"] + excluded + [\"--debug\"]\n parsed = parse_args(args)\n assert parsed[\"exclude\"] == excluded\n\n def check_config_overrides(self, monkeypatch):\n \"\"\"Check that parsed arguments pick up values from config files, and that overrides match precedence.\"\"\"\n\n tmpdir = tempfile.mkdtemp(dir=\"/tmp\")\n # Create tmp file for global config\n project_cfg_filename = os.path.join(tmpdir, \"ducktape-project.cfg\")\n user_cfg_filename = os.path.join(tmpdir, \"ducktape-user.cfg\")\n\n project_cfg = [\n \"--cluster-file CLUSTERFILE-project\",\n \"--results-root RESULTSROOT-project\",\n \"--parameters PARAMETERS-project\",\n ]\n\n # user_cfg options should override project_cfg\n user_cfg = [\"--results-root RESULTSROOT-user\", \"--parameters PARAMETERS-user\"]\n\n try:\n monkeypatch.setattr(\n \"ducktape.command_line.defaults.ConsoleDefaults.PROJECT_CONFIG_FILE\",\n project_cfg_filename,\n )\n monkeypatch.setattr(\n \"ducktape.command_line.defaults.ConsoleDefaults.USER_CONFIG_FILE\",\n user_cfg_filename,\n )\n\n with open(project_cfg_filename, \"w\") as project_f:\n project_f.write(\"\\n\".join(project_cfg))\n\n with open(user_cfg_filename, \"w\") as user_f:\n user_f.write(\"\\n\".join(user_cfg))\n\n # command-line options should override user_cfg and project_cfg\n args_dict = parse_args([\"--parameters\", \"PARAMETERS-commandline\"])\n\n assert args_dict[\"cluster_file\"] == \"CLUSTERFILE-project\"\n assert args_dict[\"results_root\"] == \"RESULTSROOT-user\"\n assert args_dict[\"parameters\"] == \"PARAMETERS-commandline\"\n\n finally:\n shutil.rmtree(tmpdir)\n\n def check_config_file_option(self):\n \"\"\"Check that config file option works\"\"\"\n tmpdir = tempfile.mkdtemp(dir=\"/tmp\")\n user_cfg_filename = os.path.join(tmpdir, \"ducktape-user.cfg\")\n\n user_cfg = [\"--results-root RESULTSROOT-user\", \"--parameters PARAMETERS-user\"]\n\n try:\n with open(user_cfg_filename, \"w\") as user_f:\n user_f.write(\"\\n\".join(user_cfg))\n args_dict = parse_args([\"--config-file\", user_cfg_filename])\n assert args_dict[\"results_root\"] == \"RESULTSROOT-user\"\n assert args_dict[\"parameters\"] == \"PARAMETERS-user\"\n finally:\n shutil.rmtree(tmpdir)\n\n def check_config_overrides_for_n_args(self, monkeypatch):\n \"\"\"Check that parsed arguments pick up values from config files, and that overrides match precedence.\"\"\"\n\n tmpdir = tempfile.mkdtemp(dir=\"/tmp\")\n # Create tmp file for global config\n project_cfg_filename = os.path.join(tmpdir, \"ducktape-project.cfg\")\n user_cfg_filename = os.path.join(tmpdir, \"ducktape-user.cfg\")\n\n project_cfg = [\n \"--exclude\",\n \"test1\",\n \"test2\",\n \"test3\",\n ]\n\n # user_cfg options should override project_cfg\n user_cfg = [\n \"test1\",\n \"test2\",\n \"test3\",\n ]\n\n try:\n monkeypatch.setattr(\n \"ducktape.command_line.defaults.ConsoleDefaults.PROJECT_CONFIG_FILE\",\n project_cfg_filename,\n )\n monkeypatch.setattr(\n \"ducktape.command_line.defaults.ConsoleDefaults.USER_CONFIG_FILE\",\n user_cfg_filename,\n )\n\n with open(project_cfg_filename, \"w\") as project_f:\n project_f.write(\"\\n\".join(project_cfg))\n\n with open(user_cfg_filename, \"w\") as user_f:\n user_f.write(\"\\n\".join(user_cfg))\n\n # command-line options should override user_cfg and project_cfg\n args_dict = parse_args([\"test1\", \"test2\", \"test3\", \"test4\", \"--max-parallel\", \"9000\"])\n\n assert args_dict[\"test_path\"] == [\"test1\", \"test2\", \"test3\", \"test4\"]\n assert args_dict[\"exclude\"] == [\"test1\", \"test2\", \"test3\"]\n assert args_dict[\"max_parallel\"] == 9000\n\n finally:\n shutil.rmtree(tmpdir)\n" }, { "path": "tests/ducktape_mock.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom typing import List, Tuple\nfrom ducktape.cluster.cluster import Cluster\nfrom ducktape.cluster.cluster_spec import ClusterSpec\nfrom ducktape.cluster.consts import LINUX\nfrom ducktape.cluster.node_container import NodeContainer\nfrom ducktape.tests.session import SessionContext\nfrom ducktape.tests.test import Test\nfrom ducktape.tests.test_context import TestContext\nfrom ducktape.cluster.linux_remoteaccount import LinuxRemoteAccount\nfrom ducktape.cluster.remoteaccount import RemoteAccountSSHConfig\nfrom unittest.mock import MagicMock\n\n\nimport os\nimport tempfile\n\n\ndef mock_cluster():\n return MagicMock(\n all=lambda: [MagicMock(spec=ClusterSpec)] * 3,\n max_used=lambda: 3,\n max_used_nodes=3,\n )\n\n\nclass FakeClusterNode(object):\n @property\n def operating_system(self):\n return LINUX\n\n\nclass FakeCluster(Cluster):\n \"\"\"A cluster class with counters, but no actual node objects\"\"\"\n\n def __init__(self, num_nodes):\n super(FakeCluster, self).__init__()\n self._available_nodes = NodeContainer()\n for i in range(0, num_nodes):\n self._available_nodes.add_node(FakeClusterNode())\n self._in_use_nodes = NodeContainer()\n\n def do_alloc(self, cluster_spec):\n good_nodes, bad_nodes = self._available_nodes.remove_spec(cluster_spec)\n self._in_use_nodes.add_nodes(good_nodes)\n return good_nodes\n\n def free_single(self, node):\n self._in_use_nodes.remove_node(node)\n self._available_nodes.add_node(node)\n\n def available(self):\n return ClusterSpec.from_nodes(self._available_nodes)\n\n def used(self):\n return ClusterSpec.from_nodes(self._in_use_nodes)\n\n\ndef session_context(**kwargs):\n \"\"\"Return a SessionContext object\"\"\"\n\n if \"results_dir\" not in kwargs.keys():\n tmp = tempfile.mkdtemp()\n session_dir = os.path.join(tmp, \"test_dir\")\n os.mkdir(session_dir)\n kwargs[\"results_dir\"] = session_dir\n\n return SessionContext(session_id=\"test_session\", **kwargs)\n\n\nclass TestMockTest(Test):\n def mock_test(self):\n pass\n\n\ndef test_context(session_context=session_context(), cluster=mock_cluster()):\n \"\"\"Return a TestContext object\"\"\"\n return TestContext(\n session_context=session_context,\n file=\"tests/ducktape_mock.py\",\n module=__name__,\n cls=TestMockTest,\n function=TestMockTest.mock_test,\n cluster=cluster,\n )\n\n\nclass MockNode(object):\n \"\"\"Mock cluster node\"\"\"\n\n def __init__(self):\n self.account = MockAccount()\n\n\nclass MockAccount(LinuxRemoteAccount):\n \"\"\"Mock node.account object. It's Linux because tests are run in Linux.\"\"\"\n\n def __init__(self, **kwargs):\n ssh_config = RemoteAccountSSHConfig(host=\"localhost\", user=None, hostname=\"localhost\", port=22)\n\n super(MockAccount, self).__init__(ssh_config, externally_routable_ip=\"localhost\", logger=None, **kwargs)\n\n\nclass MockSender(MagicMock):\n send_results: List[Tuple]\n\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n self.send_results = []\n\n def send(self, *args, **kwargs):\n self.send_results.append((args, kwargs))\n" }, { "path": "tests/loader/__init__.py", "content": "" }, { "path": "tests/loader/check_loader.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.tests.loader import TestLoader, LoaderException, _requests_session\n\nimport tests.ducktape_mock\n\nimport os\nimport os.path\nimport pytest\nimport re\nimport requests\nimport tempfile\nimport yaml\n\nfrom mock import Mock\nfrom requests_testadapter import Resp\n\n\nclass LocalFileAdapter(requests.adapters.HTTPAdapter):\n def build_response_from_file(self, request):\n file_path = request.url[7:]\n with open(file_path, \"rb\") as file:\n buff = bytearray(os.path.getsize(file_path))\n file.readinto(buff)\n resp = Resp(buff)\n r = self.build_response(request, resp)\n\n return r\n\n def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None):\n return self.build_response_from_file(request)\n\n\ndef resources_dir():\n return os.path.join(os.path.dirname(os.path.abspath(__file__)), \"resources\")\n\n\ndef discover_dir():\n \"\"\"Return the absolute path to the directory to use with discovery tests.\"\"\"\n return os.path.join(resources_dir(), \"loader_test_directory\")\n\n\ndef sub_dir_a():\n return os.path.join(discover_dir(), \"sub_dir_a\")\n\n\ndef num_tests_in_file(fpath):\n \"\"\"Count expected number of tests in the file.\n Search for NUM_TESTS = N\n\n return N if pattern is present else 0\n \"\"\"\n with open(fpath, \"r\") as fd:\n match = re.search(r\"^NUM_TESTS\\s*=\\s*(\\d+)\", fd.read(), re.MULTILINE)\n\n if not match:\n return 0\n return int(match.group(1))\n\n\ndef num_tests_in_dir(dpath):\n \"\"\"Walk through directory subtree and count up expected number of tests that TestLoader should find.\"\"\"\n assert os.path.exists(dpath)\n assert os.path.isdir(dpath)\n\n num_tests = 0\n for pwd, dirs, files in os.walk(dpath):\n for f in files:\n if not f.endswith(\".py\"):\n continue\n file_path = os.path.abspath(os.path.join(pwd, f))\n num_tests += num_tests_in_file(file_path)\n return num_tests\n\n\ndef invalid_test_suites():\n dpath = os.path.join(discover_dir(), \"invalid_test_suites\")\n params = []\n for pwd, dirs, files in os.walk(dpath):\n for f in files:\n if not f.endswith(\".yml\"):\n continue\n file_path = os.path.abspath(os.path.join(pwd, f))\n params.append(pytest.param(file_path, id=os.path.basename(file_path)))\n return params\n\n\nclass CheckTestLoader(object):\n def setup_method(self, method):\n self.SESSION_CONTEXT = tests.ducktape_mock.session_context()\n # To simplify unit tests, add file:// support to the test loader's functionality for loading previous\n # report.json files\n _requests_session.mount(\"file://\", LocalFileAdapter())\n\n @pytest.mark.parametrize(\"suite_file_path\", invalid_test_suites())\n def check_test_loader_raises_on_invalid_test_suite(self, suite_file_path):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n with pytest.raises(LoaderException):\n loader.load([suite_file_path])\n\n @pytest.mark.parametrize(\n [\"expected_count\", \"input_symbols\", \"excluded_symbols\"],\n [\n pytest.param(\n 6,\n [\n # import a single test suite with an import statement\n os.path.join(discover_dir(), \"test_suite_with_single_import.yml\")\n ],\n None,\n id=\"load dependency path\",\n ),\n pytest.param(\n 2,\n [\n # test that a self import doesn't cause an infinite loop\n os.path.join(discover_dir(), \"test_suite_with_self_import.yml\")\n ],\n None,\n id=\"self load in import\",\n ),\n pytest.param(\n 5,\n [\n # test that a cyclic import doesn't cause an infinite loop\n os.path.join(discover_dir(), \"test_suite_cyclic_b.yml\")\n ],\n None,\n id=\"self load in import\",\n ),\n pytest.param(\n 5,\n [\n # test that a cyclic import starting with second import\n os.path.join(discover_dir(), \"test_suite_cyclic_a.yml\")\n ],\n None,\n id=\"self load in import\",\n ),\n pytest.param(\n 5,\n [\n # test single import statement with parent reference\n os.path.join(discover_dir(), \"test_suites\", \"sub_dir_test_import.yml\")\n ],\n None,\n id=\"self load in import\",\n ),\n pytest.param(\n 8,\n [\n # see test suite files for number of tests in it.\n # decorated test suite includes 2 tests;\n # single includes 4 tests; multiple includes 3 tests;\n # however both single and multiple test suites include one test method of test_by.py,\n # so total -= 1\n os.path.join(discover_dir(), \"test_suite_single.yml\"),\n os.path.join(discover_dir(), \"test_suite_multiple.yml\"),\n os.path.join(discover_dir(), \"test_suite_decorated.yml\"),\n ],\n None,\n id=\"load multiple test suite files\",\n ),\n pytest.param(\n 5,\n [\n # see test suite file for number of tests in it\n os.path.join(discover_dir(), \"test_suites\", \"test_suite_glob.yml\")\n ],\n None,\n id=\"load test suite with globs\",\n ),\n pytest.param(\n 2,\n [\n # test suite that includes sub_dir_a/test_c.py (1 test total):\n os.path.join(discover_dir(), \"test_suites\", \"sub_dir_a_test_c.yml\"),\n # explicitly include test_a.yml (1 test total)\n os.path.join(discover_dir(), \"test_a.py\"),\n ],\n None,\n id=\"load both file and suite\",\n ),\n pytest.param(\n 1,\n [\n # test suite that includes sub_dir_a/test_c.py (1 test total):\n os.path.join(discover_dir(), \"test_suites\", \"sub_dir_a_test_c.yml\"),\n # explicitly include test_a.yml (1 test total)\n os.path.join(discover_dir(), \"test_a.py\"),\n ],\n [\n # explicitly exclude the sub_dir_a/test_c.py (included with test suite):\n os.path.join(sub_dir_a(), \"test_c.py\"),\n ],\n id=\"global exclude overrides test suite include\",\n ),\n pytest.param(\n 4,\n [\n # sub_dir_a contains 4 total tests\n # test suite that includes sub_dir_a/*.py but excludes sub_dir_a/test_d.py:\n os.path.join(discover_dir(), \"test_suites\", \"sub_dir_a_with_exclude.yml\"),\n # explicitly include sub_dir_a/test_d.py to override exclusion from test suite:\n os.path.join(sub_dir_a(), \"test_d.py\"),\n ],\n None,\n id=\"global include overrides test suite exclude\",\n ),\n pytest.param(\n 1,\n [\n # load two test suites and two files that all point to the same actual test\n # and verify that in the end only 1 test has been loaded\n os.path.join(discover_dir(), \"test_suites\", \"sub_dir_a_test_c.yml\"),\n os.path.join(discover_dir(), \"test_suites\", \"sub_dir_a_test_c_via_class.yml\"),\n os.path.join(sub_dir_a(), \"test_c.py\"),\n os.path.join(sub_dir_a(), \"test_c.py::TestC\"),\n ],\n None,\n id=\"same test in test suites and test files\",\n ),\n ],\n )\n def check_test_loader_with_test_suites_and_files(self, expected_count, input_symbols, excluded_symbols):\n \"\"\"\n When both files and test suites are loaded, files (both included and excluded) are\n loaded after and separately from the test suites, so even if a test suite excludes file A,\n it will be included if it's passed directly. And if file A is excluded directly, even if any of\n the test suites includes it, it will still be excluded.\n \"\"\"\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n tests = loader.load(input_symbols, excluded_test_symbols=excluded_symbols)\n assert len(tests) == expected_count\n\n def check_test_loader_with_directory(self):\n \"\"\"Check discovery on a directory.\"\"\"\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n tests = loader.load([discover_dir()])\n assert len(tests) == num_tests_in_dir(discover_dir())\n\n @pytest.mark.parametrize(\n [\"dir_\", \"file_name\"],\n [\n pytest.param(discover_dir(), \"test_a.py\"),\n pytest.param(resources_dir(), \"a.py\"),\n ],\n )\n def check_test_loader_with_file(self, dir_, file_name):\n \"\"\"Check discovery on a file.\"\"\"\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n module_path = os.path.join(dir_, file_name)\n\n tests = loader.load([module_path])\n assert len(tests) == num_tests_in_file(module_path)\n\n def check_test_loader_with_glob(self):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n file_glob = os.path.join(discover_dir(), \"*_a.py\") # should resolve to test_a.py only\n tests = loader.load([file_glob])\n assert len(tests) == 1\n\n def check_test_loader_multiple_files(self):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n file_a = os.path.join(discover_dir(), \"test_a.py\")\n file_b = os.path.join(discover_dir(), \"test_b.py\")\n\n tests = loader.load([file_a, file_b])\n assert len(tests) == num_tests_in_file(file_a) + num_tests_in_file(file_b)\n\n def check_test_loader_include_dir_exclude_file(self):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n excluded_file_a = os.path.join(discover_dir(), \"test_a.py\")\n excluded_file_b = os.path.join(discover_dir(), \"test_b.py\")\n num_excluded = num_tests_in_file(excluded_file_a) + num_tests_in_file(excluded_file_b)\n tests = loader.load([discover_dir()], [excluded_file_a, excluded_file_b])\n assert len(tests) == num_tests_in_dir(discover_dir()) - num_excluded\n\n def check_test_loader_exclude_subdir(self):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n included_dir = discover_dir()\n excluded_dir = sub_dir_a()\n tests = loader.load([included_dir], [excluded_dir])\n assert len(tests) == num_tests_in_dir(included_dir) - num_tests_in_dir(excluded_dir)\n\n def check_test_loader_exclude_subdir_glob(self):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n included_dir = discover_dir()\n excluded_dir = sub_dir_a()\n excluded_dir_glob = os.path.join(sub_dir_a(), \"*.py\")\n tests = loader.load([included_dir], [excluded_dir_glob])\n assert len(tests) == num_tests_in_dir(included_dir) - num_tests_in_dir(excluded_dir)\n\n def check_test_loader_raises_when_nothing_is_included(self):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n file_a = os.path.join(discover_dir(), \"test_a.py\")\n file_b = os.path.join(discover_dir(), \"test_b.py\")\n with pytest.raises(LoaderException):\n loader.load([file_a, file_b], [discover_dir()])\n\n def check_test_loader_raises_on_include_subdir_exclude_parent_dir(self):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n with pytest.raises(LoaderException):\n loader.load([(sub_dir_a())], [(discover_dir())])\n\n def check_test_loader_with_nonexistent_file(self):\n \"\"\"Check discovery on a non-existent path should throw LoaderException\"\"\"\n with pytest.raises(LoaderException):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n loader.load([os.path.join(discover_dir(), \"file_that_does_not_exist.py\")])\n\n def check_test_loader_include_dir_without_tests(self):\n with pytest.raises(LoaderException):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n loader.load([os.path.join(discover_dir(), \"sub_dir_no_tests\")])\n\n def check_test_loader_include_file_without_tests(self):\n with pytest.raises(LoaderException):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n loader.load([os.path.join(discover_dir(), \"sub_dir_no_tests\", \"just_some_file.py\")])\n\n def check_test_loader_allow_exclude_dir_without_tests(self):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n tests = loader.load([discover_dir()], [os.path.join(discover_dir(), \"sub_dir_no_tests\")])\n assert len(tests) == num_tests_in_dir(discover_dir())\n\n def check_test_loader_allow_exclude_file_without_tests(self):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n tests = loader.load(\n [discover_dir()],\n [os.path.join(discover_dir(), \"sub_dir_no_tests\", \"just_some_file.py\")],\n )\n assert len(tests) == num_tests_in_dir(discover_dir())\n\n def check_test_loader_allow_exclude_nonexistent_file(self):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n tests = loader.load(\n [discover_dir()],\n [os.path.join(discover_dir(), \"file_that_does_not_exist.py\")],\n )\n assert len(tests) == num_tests_in_dir(discover_dir())\n\n def check_test_loader_with_class(self):\n \"\"\"Check test discovery with discover class syntax.\"\"\"\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n tests = loader.load([os.path.join(discover_dir(), \"test_b.py::TestBB\")])\n assert len(tests) == 2\n\n # Sanity check, test that it discovers two test class & 3 tests if it searches the whole module\n tests = loader.load([os.path.join(discover_dir(), \"test_b.py\")])\n assert len(tests) == 3\n\n def check_test_loader_include_dir_exclude_class(self):\n \"\"\"Check test discovery with discover class syntax.\"\"\"\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n tests = loader.load([discover_dir()], [os.path.join(discover_dir(), \"test_b.py::TestBB\")])\n # TestBB contains 2 test methods\n assert len(tests) == num_tests_in_dir(discover_dir()) - 2\n\n def check_test_loader_include_class_exclude_method(self):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n included = [os.path.join(discover_dir(), \"test_b.py::TestBB\")]\n excluded = [os.path.join(discover_dir(), \"test_b.py::TestBB.test_bb_one\")]\n tests = loader.load(included, excluded)\n # TestBB contains 2 test methods, but 1 is excluded\n assert len(tests) == 1\n\n def check_test_loader_include_dir_exclude_method(self):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n excluded = [os.path.join(discover_dir(), \"test_b.py::TestBB.test_bb_one\")]\n tests = loader.load([discover_dir()], excluded)\n # excluded 1 method only\n assert len(tests) == num_tests_in_dir(discover_dir()) - 1\n\n def check_test_loader_with_matrix_params(self):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n included = [\n os.path.join(\n discover_dir(),\n 'test_decorated.py::TestMatrix.test_thing@{\"x\": 1,\"y\": \"test \"}',\n )\n ]\n tests = loader.load(included)\n # TestMatrix contains a single parametrized method. Since we only provide a single set of params, we should\n # end up with a single context\n assert len(tests) == 1\n assert tests[0].injected_args == {\"x\": 1, \"y\": \"test \"}\n\n def check_test_loader_with_params_special_chars(self):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n included = [\n os.path.join(\n discover_dir(),\n r\"test_decorated.py::TestParametrizdeSpecial.test_special_characters_params\"\n r'@{\"version\": \"6.1.0\", \"chars\": \"!@#$%^&*()_+::.,/? \\\"{}\\\\\"}',\n )\n ]\n tests = loader.load(included)\n # TestMatrix contains a single parametrized method. Since we only provide a single set of params, we should\n # end up with a single context\n assert len(tests) == 1\n assert tests[0].injected_args == {\n \"version\": \"6.1.0\",\n \"chars\": '!@#$%^&*()_+::.,/? \"{}\\\\',\n }\n\n def check_test_loader_with_multiple_matrix_params(self):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n params = '[{\"x\": 1,\"y\": \"test \"}, {\"x\": 2,\"y\": \"I\\'m\"}]'\n included = [\n os.path.join(\n discover_dir(),\n \"test_decorated.py::TestMatrix.test_thing@{}\".format(params),\n )\n ]\n tests = loader.load(included)\n # TestMatrix contains a single parametrized method.\n # We provide two sets of params, so we should end up with two contexts\n assert len(tests) == 2\n injected_args = map(lambda t: t.injected_args, tests)\n assert {\"x\": 1, \"y\": \"test \"} in injected_args\n assert {\"x\": 2, \"y\": \"I'm\"} in injected_args\n\n def check_test_loader_with_parametrize(self):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n included = [\n os.path.join(\n discover_dir(),\n 'test_decorated.py::TestParametrized.test_thing@{\"x\":1,\"y\":2}',\n )\n ]\n tests = loader.load(included)\n assert len(tests) == 1\n assert tests[0].injected_args == {\"x\": 1, \"y\": 2}\n\n def check_test_loader_with_parametrize_with_objects(self):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n parameters = '{\"d\": {\"a\": \"A\"}, \"lst\": [\"whatever\"]}'\n included = [\n os.path.join(\n discover_dir(),\n \"test_decorated.py::TestObjectParameters.test_thing@{}\".format(parameters),\n )\n ]\n tests = loader.load(included)\n assert len(tests) == 1\n assert tests[0].injected_args == {\"d\": {\"a\": \"A\"}, \"lst\": [\"whatever\"]}\n\n def check_test_loader_with_injected_args(self):\n \"\"\"When the --parameters command-line option is used, the loader behaves a little bit differently:\n\n each test method annotated with @parametrize or @matrix should only expand to a single discovered test,\n and the injected args should be those passed in from command-line.\n \"\"\"\n # parameter values don't have to match any of the pre-defined parameters in the annotation\n # moreover, even parameter keys don't have to match method arguments, though if that's the case\n # the runner will complain, but the loader wouldn't care (this has been ducktape's behavior for a while now)\n injected_args = {\"x\": 100, \"y\": -100}\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock(), injected_args=injected_args)\n\n file = os.path.join(discover_dir(), \"test_decorated.py\")\n tests = loader.load([file])\n assert len(tests) == 6\n\n for t in tests:\n assert t.injected_args == injected_args\n\n def check_test_loader_raises_with_both_injected_args_and_parameters(self):\n \"\"\"One should not use both --parameters command-line flag and parameterized test symbols at the same time.\n Loader will explicitly raise in such cases to avoid confusing behavior.\n \"\"\"\n injected_args = {\"x\": 1, \"y\": \"test \"}\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock(), injected_args=injected_args)\n included = [\n os.path.join(\n discover_dir(),\n 'test_decorated.py::TestMatrix.test_thing@{\"x\": 1,\"y\": \"test \"}',\n )\n ]\n with pytest.raises(LoaderException, match=\"Cannot use both\"):\n loader.load(included)\n\n def check_test_loader_raises_on_params_not_found(self):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n # parameter syntax is valid, but there is no such parameter defined in the test annotation in the code\n included = [\n os.path.join(\n discover_dir(),\n 'test_decorated.py::TestMatrix.test_thing@{\"x\": 1,\"y\": \"missing\"}',\n )\n ]\n with pytest.raises(LoaderException, match=\"No tests to run\"):\n loader.load(included)\n\n @pytest.mark.parametrize(\n \"symbol\",\n [\n # no class\n \"test_decorated.py::.test_thing\"\n # no method\n 'test_decorated.py::TestMatrix@{\"x\": 1, \"y\": \"test \"}'\n # invalid json in params\n 'test_decorated.py::TestMatrix.test_thing@{x: 1,\"y\": \"test \"}'\n ],\n )\n def check_test_loader_raises_on_malformed_test_discovery_symbol(self, symbol):\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n included = [os.path.join(discover_dir(), symbol)]\n with pytest.raises(LoaderException, match=\"Invalid discovery symbol\"):\n loader.load(included)\n\n def check_test_loader_exclude_with_injected_args(self):\n injected_args = {\"x\": 1, \"y\": -1}\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock(), injected_args=injected_args)\n\n included = [os.path.join(discover_dir(), \"test_decorated.py\")]\n excluded = [os.path.join(discover_dir(), \"test_decorated.py::TestStackedMatrix\")]\n tests = loader.load(included, excluded)\n # test_decorated.py contains 5 test methods total\n # we exclude 1 class with 1 method so should be 4\n # exclusion shouldn't care about injected args\n assert len(tests) == 5\n\n for t in tests:\n assert t.injected_args == injected_args\n\n def check_test_loader_exclude_with_params(self):\n \"\"\"\n Checks behavior of exclude flag with parametrized annotations.\n Should exclude only a single permutation of the method\n \"\"\"\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n # included 8 tests\n included = [os.path.join(discover_dir(), \"test_decorated.py::TestMatrix\")]\n # exclude 1 test\n excluded = [\n os.path.join(\n discover_dir(),\n 'test_decorated.py::TestMatrix.test_thing@{\"x\": 1,\"y\": \"test \"}',\n )\n ]\n tests = loader.load(included, excluded)\n assert len(tests) == 7\n\n def check_test_loader_exclude_with_params_multiple(self):\n \"\"\"\n Checks behavior of exclude flag with parametrized annotations.\n Should exclude two permutations of the method\n \"\"\"\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n # include 8 tests\n included = [os.path.join(discover_dir(), \"test_decorated.py::TestMatrix\")]\n # exclude 2 tests\n params = '[{\"x\": 1,\"y\": \"test \"}, {\"x\": 2,\"y\": \"I\\'m\"}]'\n excluded = [\n os.path.join(\n discover_dir(),\n \"test_decorated.py::TestMatrix.test_thing@{}\".format(params),\n )\n ]\n tests = loader.load(included, excluded)\n assert len(tests) == 6\n\n def check_test_loader_with_subsets(self):\n \"\"\"Check that computation of subsets work properly. This validates both that the division of tests is correct\n (i.e. as even a distribution as we can get but uneven in the expected way when necessary) and that the division\n happens after the expansion of tests marked for possible expansion (e.g. matrix, parametrize).\"\"\"\n\n file = os.path.join(discover_dir(), \"test_decorated.py\")\n\n # The test file contains 18 tests. With 4 subsets, first two subset should have an \"extra\"\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock(), subset=0, subsets=4)\n tests = loader.load([file])\n assert len(tests) == 5\n\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock(), subset=1, subsets=4)\n tests = loader.load([file])\n assert len(tests) == 5\n\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock(), subset=2, subsets=4)\n tests = loader.load([file])\n assert len(tests) == 4\n\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock(), subset=3, subsets=4)\n tests = loader.load([file])\n assert len(tests) == 4\n\n def check_test_loader_with_invalid_subsets(self):\n \"\"\"Check that the TestLoader throws an exception if the requests subset is larger than the number of subsets\"\"\"\n with pytest.raises(ValueError):\n TestLoader(self.SESSION_CONTEXT, logger=Mock(), subset=4, subsets=4)\n with pytest.raises(ValueError):\n TestLoader(self.SESSION_CONTEXT, logger=Mock(), subset=5, subsets=4)\n\n def check_test_loader_with_time_based_subsets(self):\n \"\"\"Check that computation of subsets using a report with timing information correctly generates subsets that\n are optimized based on timing rather than number of tests.\n \"\"\"\n\n file = os.path.join(discover_dir(), \"test_b.py\")\n report_url = \"file://\" + os.path.join(resources_dir(), \"report.json\")\n\n # The expected behavior of the current implementation is to add tests to each subset from largest to smallest,\n # using the least full subset each time. The test data with times of (10, 5, 1) should result in the first\n # subset containing 1 test and the second containing 2 (the opposite of the simple count-based strategy)\n\n loader = TestLoader(\n self.SESSION_CONTEXT,\n logger=Mock(),\n subset=0,\n subsets=2,\n historical_report=report_url,\n )\n tests = loader.load([file])\n assert len(tests) == 1\n\n loader = TestLoader(\n self.SESSION_CONTEXT,\n logger=Mock(),\n subset=1,\n subsets=2,\n historical_report=report_url,\n )\n tests = loader.load([file])\n assert len(tests) == 2\n\n def check_loader_with_non_yml_file(self):\n \"\"\"\n test loading a test file as an import\n \"\"\"\n file = os.path.join(discover_dir(), \"test_suite_import_py.yml\")\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n with pytest.raises(LoaderException, match=r\"Failed to load test suite from file: \\S+test_a\\.py\"):\n loader.load([file])\n\n def check_loader_with_non_suite_yml_file(self):\n \"\"\"\n test importing a suite that is malformed\n \"\"\"\n file1 = os.path.join(discover_dir(), \"test_suite_malformed.yml\")\n file2 = os.path.join(discover_dir(), \"test_suite_import_malformed.yml\")\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n with pytest.raises(LoaderException, match=\"No tests found in simple_malformed_suite\"):\n loader.load([file1])\n with pytest.raises(LoaderException, match=\"No tests found in simple_malformed_suite\"):\n loader.load([file2])\n\n def check_test_loader_with_absolute_path(self):\n \"\"\"\n Test loading suites using absolute paths to other imported suites as well as absolute paths\n to tests in the suite\n \"\"\"\n with tempfile.TemporaryDirectory() as td:\n temp_suite1 = os.path.join(td, \"temp_suite1.yml\")\n temp_suite2 = os.path.join(td, \"temp_suite2.yml\")\n with open(temp_suite1, \"w\") as f:\n test_yaml1 = yaml.dump(\n {\n \"import\": str(os.path.join(td, \"temp_suite2.yml\")),\n \"suite\": [os.path.abspath(os.path.join(discover_dir(), \"test_a.py\"))],\n }\n )\n f.write(test_yaml1)\n\n with open(temp_suite2, \"w\") as f:\n test_yaml2 = yaml.dump({\"suite\": [os.path.abspath(os.path.join(discover_dir(), \"test_b.py\"))]})\n f.write(test_yaml2)\n\n loader = TestLoader(self.SESSION_CONTEXT, logger=Mock())\n tests = loader.load([temp_suite1])\n assert len(tests) == 4\n\n\ndef join_parsed_symbol_components(parsed):\n \"\"\"\n Join together a parsed symbol\n\n e.g.\n {\n 'path': 'path/to/dir/test_file.py',\n 'cls': 'ClassName',\n 'method': 'method'\n },\n ->\n 'path/to/dir/test_file.py::ClassName.method'\n \"\"\"\n symbol = os.path.join(parsed[\"path\"])\n\n if parsed[\"cls\"] or parsed[\"method\"]:\n symbol += \"::\"\n symbol += parsed[\"cls\"]\n if parsed[\"method\"]:\n symbol += \".\"\n symbol += parsed[\"method\"]\n\n return symbol\n\n\ndef normalize_ending_slash(dirname):\n if dirname.endswith(os.path.sep):\n dirname = dirname[: -len(os.path.sep)]\n return dirname\n\n\nclass CheckParseSymbol(object):\n def check_parse_discovery_symbol(self):\n \"\"\"Check that \"test discovery symbol\" parsing logic works correctly\"\"\"\n parsed_symbols = [\n {\"path\": \"path/to/dir\", \"cls\": \"\", \"method\": \"\"},\n {\"path\": \"path/to/dir/test_file.py\", \"cls\": \"\", \"method\": \"\"},\n {\"path\": \"path/to/dir/test_file.py\", \"cls\": \"ClassName\", \"method\": \"\"},\n {\n \"path\": \"path/to/dir/test_file.py\",\n \"cls\": \"ClassName\",\n \"method\": \"method\",\n },\n {\"path\": \"path/to/dir\", \"cls\": \"ClassName\", \"method\": \"\"},\n {\"path\": \"test_file.py\", \"cls\": \"\", \"method\": \"\"},\n {\"path\": \"test_file.py\", \"cls\": \"ClassName\", \"method\": \"\"},\n {\"path\": \"test_file.py\", \"cls\": \"ClassName\", \"method\": \"method\"},\n ]\n\n loader = TestLoader(tests.ducktape_mock.session_context(), logger=Mock())\n for parsed in parsed_symbols:\n symbol = join_parsed_symbol_components(parsed)\n\n expected_parsed = (\n normalize_ending_slash(parsed[\"path\"]),\n parsed[\"cls\"],\n parsed[\"method\"],\n )\n\n actually_parsed = loader._parse_discovery_symbol(symbol)\n actually_parsed = (\n normalize_ending_slash(actually_parsed[0]),\n actually_parsed[1],\n actually_parsed[2],\n )\n\n assert actually_parsed == expected_parsed, \"%s did not parse as expected\" % symbol\n" }, { "path": "tests/loader/resources/__init__.py", "content": "" }, { "path": "tests/loader/resources/loader_test_directory/README", "content": "A dummy test directory structure for checking test discovery behavior" }, { "path": "tests/loader/resources/loader_test_directory/__init__.py", "content": "" }, { "path": "tests/loader/resources/loader_test_directory/invalid_test_suites/empty_file.yml", "content": "" }, { "path": "tests/loader/resources/loader_test_directory/invalid_test_suites/malformed_test_suite.yml", "content": "malformed_test_suite:\n included:\n should_have_been_a_list:\n - but\n - is\n - a\n - dict\n" }, { "path": "tests/loader/resources/loader_test_directory/invalid_test_suites/not_yaml.yml", "content": "this is not a yaml file\nNo, really\n" }, { "path": "tests/loader/resources/loader_test_directory/invalid_test_suites/test_suite_refers_to_non_existent_file.yml", "content": "non_existent_file_suit:\n included:\n - file_does_not_exist.py\n" }, { "path": "tests/loader/resources/loader_test_directory/invalid_test_suites/test_suite_with_malformed_params.yml", "content": "non_existent_file_suit:\n included:\n - 'test_decorated.py::TestMatrix.test_thing@[{x: 1,y: \"test \"}]' # this json won't parse because no quotes\n" }, { "path": "tests/loader/resources/loader_test_directory/invalid_test_suites/test_suites_with_no_tests.yml", "content": "empty_test_suite_a:\nempty_test_suite_b:\n excluded:\n - ../sub_dir_a\n" }, { "path": "tests/loader/resources/loader_test_directory/name_does_not_match_pattern.py", "content": "from ducktape.tests.test import Test\n\n\nclass TestNotLoaded(Test):\n \"\"\"Loader should not discover this - module name does not match default pattern.\"\"\"\n\n def test_a(self):\n pass\n" }, { "path": "tests/loader/resources/loader_test_directory/sub_dir_a/__init__.py", "content": "" }, { "path": "tests/loader/resources/loader_test_directory/sub_dir_a/test_c.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.tests.test import Test\n\nNUM_TESTS = 1\n\n\nclass TestC(Test):\n \"\"\"Loader should discover this.\"\"\"\n\n def test(self):\n pass\n\n\nclass TestInvisible(object):\n \"\"\"Loader should not discover this.\"\"\"\n\n def test_invisible(self):\n pass\n" }, { "path": "tests/loader/resources/loader_test_directory/sub_dir_a/test_d.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.tests.test import Test\n\nNUM_TESTS = 3\n\n\nclass TestD(Test):\n \"\"\"Loader should discover this.\"\"\"\n\n def test_d(self):\n pass\n\n def test_dd(self):\n pass\n\n def ddd_test(self):\n pass\n\n\nclass TestInvisible(object):\n \"\"\"Loader should not discover this.\"\"\"\n\n def test_invisible(self):\n pass\n" }, { "path": "tests/loader/resources/loader_test_directory/sub_dir_no_tests/__init__.py", "content": "" }, { "path": "tests/loader/resources/loader_test_directory/sub_dir_no_tests/just_some_file.py", "content": "class JustSomeClass(object):\n pass\n" }, { "path": "tests/loader/resources/loader_test_directory/test_a.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.tests.test import Test\n\nNUM_TESTS = 1\n\n\nclass TestA(Test):\n \"\"\"Loader should discover this.\"\"\"\n\n def test_a(self):\n pass\n\n\nclass TestInvisible(object):\n \"\"\"Loader should not discover this.\"\"\"\n\n def test_invisible(self):\n pass\n" }, { "path": "tests/loader/resources/loader_test_directory/test_b.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.tests.test import Test\n\nNUM_TESTS = 3\n\n\nclass TestB(Test):\n \"\"\"Loader should discover this.\"\"\"\n\n def test_b(self):\n pass\n\n\nclass TestBB(Test):\n \"\"\"Loader should discover this with 2 tests.\"\"\"\n\n test_not_callable = 3\n\n def test_bb_one(self):\n pass\n\n def bb_two_test(self):\n pass\n\n def other_method(self):\n pass\n\n\nclass TestInvisible(object):\n \"\"\"Loader should not discover this.\"\"\"\n\n def test_invisible(self):\n pass\n" }, { "path": "tests/loader/resources/loader_test_directory/test_decorated.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.tests.test import Test\nfrom ducktape.mark import matrix\nfrom ducktape.mark import parametrize\n\nNUM_TESTS = 18\n\n\nclass TestMatrix(Test):\n \"\"\"8 tests here\"\"\"\n\n @matrix(x=[1, 2], y=[\"I'm\", \" a \", \"test \", \"matrix!\"])\n def test_thing(self, x, y):\n pass\n\n\nclass TestStackedMatrix(Test):\n \"\"\"4 tests\"\"\"\n\n @matrix(x=[1, 2], y=[-1, 0])\n def test_thing(self, x, y):\n pass\n\n\nclass TestParametrized(Test):\n @parametrize(x=10)\n def test_single_decorator(self, x=1, y=\"hi\"):\n \"\"\"1 test\"\"\"\n pass\n\n @parametrize(x=1, y=2)\n @parametrize(x=\"abc\", y=[])\n def test_thing(self, x, y):\n \"\"\"2 tests\"\"\"\n pass\n\n\nclass TestParametrizdeSpecial(Test):\n @parametrize(version=\"6.1.0\", chars='!@#$%^&*()_+::.,/? \"{}\\\\')\n def test_special_characters_params(self, version, chars):\n \"\"\"1 tests\"\"\"\n pass\n\n\nclass TestObjectParameters(Test):\n @parametrize(d={\"a\": \"A\"}, lst=[\"whatever\"])\n @parametrize(d={\"z\": \"Z\"}, lst=[\"something\"])\n def test_thing(self, d, lst):\n \"\"\"2 tests\"\"\"\n pass\n" }, { "path": "tests/loader/resources/loader_test_directory/test_suite_cyclic_a.yml", "content": "import:\n - test_suite_cyclic_b.yml # 1 test\n\ntest_suite_cyclic_a:\n - ./sub_dir_a/test_c.py # 1 test\n - ./test_b.py # 3\n# suite total: 4\n\n# total: 5\n" }, { "path": "tests/loader/resources/loader_test_directory/test_suite_cyclic_b.yml", "content": "\nimport:\n - test_suite_cyclic_a.yml # 4 test\n\ntest_suite_cyclic_b:\n - test_a.py # 1 test\n# suite total: 1 test\n\n# total: 5 tests\n" }, { "path": "tests/loader/resources/loader_test_directory/test_suite_decorated.yml", "content": "test_suite_decorated:\n - 'test_decorated.py::TestMatrix.test_thing@[{\"x\": 1,\"y\": \"test \"}, {\"x\": 2, \"y\": \"test \"}]' # 2 tests\n - 'test_decorated.py::TestStackedMatrix.test_thing@{\"x\": 100, \"y\": 100}' # ignored, there are no such params in code\n\n# total: 2 tests\n" }, { "path": "tests/loader/resources/loader_test_directory/test_suite_import_malformed.yml", "content": "import:\n - test_suite_malformed.yml\n\nsimple_suite:\n - test_a.py # 1\n" }, { "path": "tests/loader/resources/loader_test_directory/test_suite_import_py.yml", "content": "import:\n - test_a.py\n# improper import\n\nsimple_suite:\n - test_a.py\n# 1 test\n" }, { "path": "tests/loader/resources/loader_test_directory/test_suite_malformed.yml", "content": "simple_malformed_suite:\n cheese:\n - 'fatty milk'\n - 'salt'\n bread:\n - 'flower'\n - 'water'\n - 'yeast'\n" }, { "path": "tests/loader/resources/loader_test_directory/test_suite_multiple.yml", "content": "test_suite_b:\n - sub_dir_a/test_c.py # 1 test\n - sub_dir_a/test_d.py::TestD.test_d # 1 test\n# suite total: 2 tests\n\ntest_suite_c:\n included:\n - sub_dir_a # +4 tests across all files\n - test_b.py # +3 tests\n excluded:\n - sub_dir_a/test_d.py # -3 tests\n - test_b.py::TestBB # -2 tests\n# suite total: 2 tests\n\n# total: 4\n# - test_c.py (with 1 test method) included in both test suites so total -= 1\n# total: 3\n" }, { "path": "tests/loader/resources/loader_test_directory/test_suite_single.yml", "content": "test_suite_a:\n included:\n - test_a.py # 1 test\n - test_b.py::TestBB.test_bb_one # 1 test\n - sub_dir_a\n # sub_dir_a/test_c.py = 0 (excluded)\n # sub_dir_a/test_d.py = 3 - 1 (excluded) = 2\n excluded:\n - sub_dir_a/test_c.py # 1 test\n - sub_dir_a/test_d.py::TestD.test_dd # 1 test\n\n# total: 4 tests\n" }, { "path": "tests/loader/resources/loader_test_directory/test_suite_with_self_import.yml", "content": "test_suite_d:\n included:\n - sub_dir_a # +4 tests across all files\n - test_b.py # +3 tests\n excluded:\n - sub_dir_a/test_d.py # -3 tests\n - test_b.py::TestBB # -2 tests\n# suite total: 2 tests\nimport:\n - ./test_suite_with_self_import.yml # +0 test\n" }, { "path": "tests/loader/resources/loader_test_directory/test_suite_with_single_import.yml", "content": "test_suite_d:\n included:\n - sub_dir_a # +4 tests across all files\n - test_b.py # +3 tests\n excluded:\n - sub_dir_a/test_d.py # -3 tests\n - test_b.py::TestBB # -2 tests\n# suite total: 6 tests\n\nimport:\n - ./test_suite_single.yml # +4 test\n" }, { "path": "tests/loader/resources/loader_test_directory/test_suites/refers_to_parent_dir.yml", "content": "test_suite_a:\n included:\n - ../test_a.py # 1 test\n# total: 1 tests\n" }, { "path": "tests/loader/resources/loader_test_directory/test_suites/sub_dir_a_test_c.yml", "content": "test_suite:\n - ../sub_dir_a/test_c.py\n" }, { "path": "tests/loader/resources/loader_test_directory/test_suites/sub_dir_a_test_c_via_class.yml", "content": "test_suite:\n - ../sub_dir_a/test_c.py::TestC\n" }, { "path": "tests/loader/resources/loader_test_directory/test_suites/sub_dir_a_with_exclude.yml", "content": "test_suite:\n included:\n - ../sub_dir_a/*.py\n excluded:\n - ../sub_dir_a/test_d.py\n" }, { "path": "tests/loader/resources/loader_test_directory/test_suites/sub_dir_test_import.yml", "content": "import:\n - ../test_suite_cyclic_a.yml # 5 tests\n - .././test_suites/../sub_dir_no_tests/.././test_suite_cyclic_b.yml # 0 tests\n\n# 5 test total\n" }, { "path": "tests/loader/resources/loader_test_directory/test_suites/test_suite_glob.yml", "content": "test_suite_glob:\n included:\n - ../sub_dir_a/*\n - ../test_?.py\n excluded:\n - ../sub_dir_a/*_d.py\n\n# globs expand to:\n# included:\n# sub_dir_a/test_c.py (1 test)\n# sub_dir_a/test_d.py (3 tests) - excluded\n# test_a.py (1 test)\n# test_b.py (3 tests)\n# excluded:\n# sub_dir_a/test_d.py (3 tests)\n# TOTAL:\n# 5 tests\n\n" }, { "path": "tests/loader/resources/report.json", "content": "{\n \"results\": [\n {\n \"test_id\": \"tests.loader.resources.loader_test_directory.test_b.TestB.test_b\",\n \"run_time_seconds\": 10\n },\n {\n \"test_id\": \"tests.loader.resources.loader_test_directory.test_b.TestBB.test_bb_one\",\n \"run_time_seconds\": 5\n },\n {\n \"test_id\": \"tests.loader.resources.loader_test_directory.test_b.TestBB.bb_two_test\",\n \"run_time_seconds\": 1\n }\n ]\n}" }, { "path": "tests/logger/__init__.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n" }, { "path": "tests/logger/check_logger.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\nimport os\nimport psutil\nimport shutil\nimport tempfile\n\nfrom ducktape.tests.loggermaker import LoggerMaker, close_logger\n\n\nclass DummyFileLoggerMaker(LoggerMaker):\n def __init__(self, log_dir, n_handles):\n \"\"\"Create a logger with n_handles file handles, with files in log_dir\"\"\"\n self.log_dir = log_dir\n self.n_handles = n_handles\n\n @property\n def logger_name(self):\n return \"a.b.c\"\n\n def configure_logger(self):\n for i in range(self.n_handles):\n fh = logging.FileHandler(os.path.join(self.log_dir, \"log-\" + str(i)))\n self._logger.addHandler(fh)\n\n\ndef open_files():\n # current process\n p = psutil.Process()\n return p.open_files()\n\n\nclass CheckLogger(object):\n def setup_method(self, _):\n self.temp_dir = tempfile.mkdtemp()\n\n def check_close_logger(self):\n \"\"\"Check that calling close_logger properly cleans up resources.\"\"\"\n initial_open_files = open_files()\n\n n_handles = 100\n log_maker = DummyFileLoggerMaker(self.temp_dir, n_handles)\n # accessing logger attribute lazily triggers configuration of logger\n the_logger = log_maker.logger\n\n assert len(open_files()) == len(initial_open_files) + n_handles\n close_logger(the_logger)\n assert len(open_files()) == len(initial_open_files)\n\n def teardown_method(self, _):\n if os.path.exists(self.temp_dir):\n shutil.rmtree(self.temp_dir)\n" }, { "path": "tests/mark/__init__.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n" }, { "path": "tests/mark/check_cluster_use_metadata.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.cluster import LocalhostCluster\nfrom ducktape.mark import parametrize, matrix, ignore, defaults\nfrom ducktape.mark.mark_expander import MarkedFunctionExpander\nfrom ducktape.mark.resource import cluster\nfrom ducktape.cluster.cluster_spec import ClusterSpec\n\nimport pytest\n\nfrom tests import ducktape_mock\n\n\nclass CheckClusterUseAnnotation(object):\n def check_basic_usage_arbitrary_metadata(self):\n cluster_use_metadata = {\"a\": 2, \"b\": \"hi\", \"num_nodes\": 300}\n\n @cluster(**cluster_use_metadata)\n def function():\n return \"hi\"\n\n assert hasattr(function, \"marks\")\n\n test_context_list = MarkedFunctionExpander(function=function).expand()\n assert len(test_context_list) == 1\n assert test_context_list[0].cluster_use_metadata == cluster_use_metadata\n\n def check_basic_usage_cluster_spec(self):\n num_nodes = 200\n\n @cluster(cluster_spec=ClusterSpec.simple_linux(num_nodes))\n def function():\n return \"hi\"\n\n assert hasattr(function, \"marks\")\n\n test_context_list = MarkedFunctionExpander(function=function).expand()\n assert len(test_context_list) == 1\n assert len(test_context_list[0].expected_cluster_spec.nodes) == num_nodes\n # All nodes should be linux\n for node in test_context_list[0].expected_cluster_spec.nodes.elements():\n assert node.operating_system == \"linux\"\n\n def check_basic_usage_num_nodes(self):\n num_nodes = 200\n\n @cluster(num_nodes=num_nodes)\n def function():\n return \"hi\"\n\n assert hasattr(function, \"marks\")\n\n test_context_list = MarkedFunctionExpander(function=function).expand()\n assert len(test_context_list) == 1\n assert test_context_list[0].expected_num_nodes == num_nodes\n\n @pytest.mark.parametrize(\"fail_greedy_tests\", [True, False])\n @pytest.mark.parametrize(\"has_annotation\", [True, False])\n def check_empty_cluster_annotation(self, fail_greedy_tests, has_annotation):\n @cluster()\n def function_with_annotation():\n return \"hi\"\n\n def function_no_annotation():\n return \"hello\"\n\n assert hasattr(function_with_annotation, \"marks\")\n assert not hasattr(function_no_annotation, \"marks\")\n\n # no annotation and empty annotation behave identically as far as this functionality is concerned\n function = function_with_annotation if has_annotation else function_no_annotation\n\n mock_cluster = LocalhostCluster(num_nodes=1000)\n session_context = ducktape_mock.session_context(fail_greedy_tests=fail_greedy_tests)\n tc_list = MarkedFunctionExpander(\n function=function, cluster=mock_cluster, session_context=session_context\n ).expand()\n\n assert len(tc_list) == 1\n if fail_greedy_tests:\n assert tc_list[0].expected_num_nodes == 0\n assert tc_list[0].expected_cluster_spec is None\n else:\n assert tc_list[0].expected_num_nodes == 1000\n\n @pytest.mark.parametrize(\"fail_greedy_tests\", [True, False])\n def check_zero_nodes_annotation(self, fail_greedy_tests):\n @cluster(num_nodes=0)\n def function():\n return \"hi\"\n\n assert hasattr(function, \"marks\")\n mock_cluster = LocalhostCluster(num_nodes=1000)\n session_context = ducktape_mock.session_context(fail_greedy_tests=fail_greedy_tests)\n tc_list = MarkedFunctionExpander(\n function=function, cluster=mock_cluster, session_context=session_context\n ).expand()\n assert len(tc_list) == 1\n assert tc_list[0].expected_num_nodes == 0\n assert tc_list[0].expected_cluster_spec is not None\n assert len(tc_list[0].expected_cluster_spec) == 0\n\n def check_with_parametrize(self):\n num_nodes = 200\n\n @cluster(num_nodes=num_nodes)\n @parametrize(x=1)\n def f(x, y=2, z=3):\n return x, y, z\n\n test_context_list = MarkedFunctionExpander(function=f).expand()\n assert len(test_context_list) == 1\n assert test_context_list[0].expected_num_nodes == num_nodes\n\n def check_beneath_parametrize(self):\n \"\"\"Annotations such as cluster, which add metadata, but do not create new tests, add the metadata to\n all test cases physically below the annotation.\n\n In the case of a parametrized test, when @cluster has no parametrize annotations below it,\n there are not any test cases to which it will apply, so none of the resulting tests should have\n associated cluster size metadata.\n \"\"\"\n num_nodes = 200\n\n @parametrize(x=1)\n @cluster(num_nodes=num_nodes)\n def f(x, y=2, z=3):\n return x, y, z\n\n with pytest.raises(AssertionError):\n MarkedFunctionExpander(function=f).expand()\n\n def check_with_nodes_default_parametrize_matrix(self):\n default_num_nodes = 5\n parametrize_num_nodes = 3\n matrix_num_nodes = 7\n\n @cluster(num_nodes=default_num_nodes)\n @defaults(y=[5])\n @parametrize(x=100, y=200)\n @cluster(num_nodes=parametrize_num_nodes)\n @parametrize(x=10, y=20)\n @parametrize(x=30, y=40)\n @cluster(num_nodes=matrix_num_nodes)\n @matrix(x=[1, 2])\n def f(x, y):\n return x, y\n\n test_context_list = MarkedFunctionExpander(function=f).expand()\n assert len(test_context_list) == 5\n # {'x': 1, 'y': 5} -> using matrix with matrix_num_nodes\n assert test_context_list[0].expected_num_nodes == matrix_num_nodes\n # {'x': 2, 'y': 5} -> using matrix with matrix_num_nodes\n assert test_context_list[1].expected_num_nodes == matrix_num_nodes\n # {'x': 30, 'y': 40} -> using parametrize with cluster parametrize_num_nodes\n assert test_context_list[2].expected_num_nodes == parametrize_num_nodes\n # {'x': 10, 'y': 20} -> using parametrize with cluster parametrize_num_nodes\n assert test_context_list[3].expected_num_nodes == parametrize_num_nodes\n # {'x': 100, 'y': 200} -> using parametrize with default_num_nodes\n assert test_context_list[4].expected_num_nodes == default_num_nodes\n\n def check_no_override(self):\n \"\"\"cluster use metadata should apply to all test cases physically below it, except for those which already\n have cluster use metadata.\n \"\"\"\n\n num_nodes_1 = 200\n num_nodes_2 = 42\n\n # num_nodes_2 should *not* override num_nodes_1\n @cluster(num_nodes=num_nodes_2)\n @cluster(num_nodes=num_nodes_1)\n def f(x, y=2, z=3):\n return x, y, z\n\n test_context_list = MarkedFunctionExpander(function=f).expand()\n assert len(test_context_list) == 1\n assert test_context_list[0].expected_num_nodes == num_nodes_1\n\n def check_parametrized_with_multiple_cluster_annotations(self):\n num_nodes_1 = 10\n num_nodes_2 = 20\n\n # num_nodes_1 should *not* override num_nodes_2\n @cluster(num_nodes=num_nodes_1)\n @parametrize(x=1)\n @parametrize(x=1.5)\n @cluster(num_nodes=num_nodes_2)\n @parametrize(x=2)\n def f(x, y=2, z=3):\n return x, y, z\n\n test_context_list = MarkedFunctionExpander(function=f).expand()\n assert len(test_context_list) == 3\n assert test_context_list[0].expected_num_nodes == num_nodes_1\n assert test_context_list[1].expected_num_nodes == num_nodes_1\n assert test_context_list[2].expected_num_nodes == num_nodes_2\n\n def check_matrix_with_multiple_cluster_annotations(self):\n num_nodes_1 = 10\n num_nodes_2 = 20\n\n # num_nodes_1 should *not* override num_nodes_2\n @cluster(num_nodes=num_nodes_1)\n @matrix(x=[1])\n @matrix(x=[1.5, 1.6], y=[-15, -16])\n @cluster(num_nodes=num_nodes_2)\n @matrix(x=[2])\n def f(x, y=2, z=3):\n return x, y, z\n\n test_context_list = MarkedFunctionExpander(function=f).expand()\n assert len(test_context_list) == 6\n assert test_context_list[0].expected_num_nodes == num_nodes_1\n assert test_context_list[1].expected_num_nodes == num_nodes_1\n assert test_context_list[2].expected_num_nodes == num_nodes_1\n assert test_context_list[3].expected_num_nodes == num_nodes_1\n assert test_context_list[4].expected_num_nodes == num_nodes_1\n assert test_context_list[5].expected_num_nodes == num_nodes_2\n\n def check_with_ignore(self):\n num_nodes = 200\n\n @cluster(num_nodes=num_nodes)\n @ignore\n def function():\n return \"hi\"\n\n assert hasattr(function, \"marks\")\n\n test_context_list = MarkedFunctionExpander(function=function).expand()\n assert len(test_context_list) == 1\n assert test_context_list[0].expected_num_nodes == num_nodes\n\n # order shouldn't matter here\n @ignore\n @cluster(num_nodes=num_nodes)\n def function():\n return \"hi\"\n\n assert hasattr(function, \"marks\")\n\n test_context_list = MarkedFunctionExpander(function=function).expand()\n assert len(test_context_list) == 1\n assert test_context_list[0].expected_num_nodes == num_nodes\n" }, { "path": "tests/mark/check_env.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.mark.mark_expander import MarkedFunctionExpander\nfrom ducktape.mark import env, is_env\n\nimport os\n\n\nclass CheckEnv(object):\n def check_does_not_raise_exception_when_key_not_exists(self):\n class C(object):\n @env(BLAH=\"8\")\n def function(self):\n return 1\n\n def check_has_env_annotation(self):\n class C(object):\n @env(JAVA_HOME=\"blah\")\n def function(self):\n return 1\n\n assert is_env(C.function)\n\n def check_is_ignored_if_env_not_correct(self):\n class C(object):\n @env(JAVA_HOME=\"blah\")\n def function(self):\n return 1\n\n context_list = MarkedFunctionExpander(function=C.function, cls=C).expand()\n assert context_list[0].ignore\n\n def check_is_not_ignore_if_correct_env(self):\n os.environ[\"test_key\"] = \"test\"\n\n class C(object):\n @env(test_key=\"test\")\n def function(self):\n return 1\n\n context_list = MarkedFunctionExpander(function=C.function, cls=C).expand()\n assert not context_list[0].ignore\n" }, { "path": "tests/mark/check_ignore.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.mark.mark_expander import MarkedFunctionExpander\nfrom ducktape.mark import ignore, ignored, parametrize, matrix\n\nimport pytest\n\n\nclass CheckIgnore(object):\n def check_simple(self):\n @ignore\n def function(x=1, y=2, z=3):\n return x, y, z\n\n assert ignored(function)\n context_list = MarkedFunctionExpander(function=function).expand()\n assert len(context_list) == 1\n assert context_list[0].ignore\n\n def check_simple_method(self):\n class C(object):\n @ignore\n def function(self, x=1, y=2, z=3):\n return x, y, z\n\n assert ignored(C.function)\n context_list = MarkedFunctionExpander(function=C.function, cls=C).expand()\n assert len(context_list) == 1\n assert context_list[0].ignore\n\n def check_ignore_all(self):\n @ignore\n @parametrize(x=100, y=200, z=300)\n @parametrize(x=100, z=300)\n @parametrize(y=200)\n @parametrize()\n def function(x=1, y=2, z=3):\n return x, y, z\n\n assert ignored(function)\n context_list = MarkedFunctionExpander(function=function).expand()\n assert len(context_list) == 4\n for ctx in context_list:\n assert ctx.ignore\n\n def check_ignore_all_method(self):\n \"\"\"Check @ignore() with no arguments used with various parametrizations on a method.\"\"\"\n\n class C(object):\n @ignore\n @parametrize(x=100, y=200, z=300)\n @parametrize(x=100, z=300)\n @parametrize(y=200)\n @matrix(x=[1, 2, 3])\n @parametrize()\n def function(self, x=1, y=2, z=3):\n return x, y, z\n\n assert ignored(C.function)\n context_list = MarkedFunctionExpander(function=C.function, cls=C).expand()\n assert len(context_list) == 7\n for ctx in context_list:\n assert ctx.ignore\n\n def check_ignore_specific(self):\n @ignore(x=100, y=200, z=300)\n @parametrize(x=100, y=200, z=300)\n @parametrize(x=100, z=300)\n @parametrize(y=200)\n @parametrize()\n def function(x=1, y=2, z=3):\n return x, y, z\n\n assert ignored(function)\n context_list = MarkedFunctionExpander(None, function=function).expand()\n assert len(context_list) == 4\n for ctx in context_list:\n if ctx.injected_args == {\"x\": 100, \"y\": 200, \"z\": 300}:\n assert ctx.ignore\n else:\n assert not ctx.ignore\n\n def check_ignore_specific_method(self):\n class C(object):\n @ignore(x=100, y=200, z=300)\n @parametrize(x=100, y=200, z=300)\n @parametrize(x=100, z=300)\n @parametrize(y=200)\n @parametrize()\n def function(self, x=1, y=2, z=3):\n return x, y, z\n\n assert ignored(C.function)\n context_list = MarkedFunctionExpander(function=C.function, cls=C).expand()\n assert len(context_list) == 4\n for ctx in context_list:\n if ctx.injected_args == {\"x\": 100, \"y\": 200, \"z\": 300}:\n assert ctx.ignore\n else:\n assert not ctx.ignore\n\n def check_invalid_specific_ignore(self):\n \"\"\"If there are no test cases to which ignore applies, it should raise an error\n Keeping in mind annotations \"point down\": they only apply to test cases physically below.\n \"\"\"\n\n class C(object):\n @parametrize(x=100, y=200, z=300)\n @parametrize(x=100, z=300)\n @parametrize(y=200)\n @parametrize()\n @ignore(x=100, y=200, z=300)\n def function(self, x=1, y=2, z=3):\n return x, y, z\n\n assert ignored(C.function)\n with pytest.raises(AssertionError):\n MarkedFunctionExpander(function=C.function, cls=C).expand()\n\n def check_invalid_ignore_all(self):\n \"\"\"If there are no test cases to which ignore applies, it should raise an error\"\"\"\n\n class C(object):\n @parametrize(x=100, y=200, z=300)\n @parametrize(x=100, z=300)\n @parametrize(y=200)\n @parametrize()\n @ignore\n def function(self, x=1, y=2, z=3):\n return x, y, z\n\n assert ignored(C.function)\n with pytest.raises(AssertionError):\n MarkedFunctionExpander(function=C.function, cls=C).expand()\n" }, { "path": "tests/mark/check_parametrize.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.mark import parametrize, parametrized, matrix, defaults\nfrom ducktape.mark.mark_expander import MarkedFunctionExpander\n\n\nclass CheckParametrize(object):\n def check_simple(self):\n @parametrize(x=100, z=300)\n def function(x=1, y=2, z=3):\n return x, y, z\n\n assert parametrized(function)\n injected_args_list = [m.injected_args for m in function.marks]\n assert len(injected_args_list) == 1\n assert injected_args_list[0] == {\"x\": 100, \"z\": 300}\n\n context_list = MarkedFunctionExpander(function=function).expand()\n\n all_f = [cxt.function for cxt in context_list]\n assert all_f[0]() == (100, 2, 300)\n\n def check_simple_method(self):\n class C(object):\n @parametrize(x=100, z=300)\n def function(self, x=1, y=2, z=3):\n return x, y, z\n\n assert parametrized(C.function)\n injected_args_list = [m.injected_args for m in C.function.marks]\n assert len(injected_args_list) == 1\n assert injected_args_list[0] == {\"x\": 100, \"z\": 300}\n\n context_list = MarkedFunctionExpander(None, function=C.function).expand()\n all_f = [cxt.function for cxt in context_list]\n assert all_f[0](C()) == (100, 2, 300)\n\n def check_stacked(self):\n @parametrize(x=100, y=200, z=300)\n @parametrize(x=100, z=300)\n @parametrize(y=200)\n @parametrize()\n def function(x=1, y=2, z=3):\n return x, y, z\n\n assert parametrized(function)\n injected_args_list = [m.injected_args for m in function.marks]\n assert len(injected_args_list) == 4\n\n context_list = MarkedFunctionExpander(None, function=function).expand()\n all_f = [cxt.function for cxt in context_list]\n assert all_f[0]() == (100, 200, 300)\n assert all_f[1]() == (100, 2, 300)\n assert all_f[2]() == (1, 200, 3)\n assert all_f[3]() == (1, 2, 3)\n\n def check_stacked_method(self):\n class C(object):\n @parametrize(x=100, y=200, z=300)\n @parametrize(x=100, z=300)\n @parametrize(y=200)\n @parametrize()\n def function(self, x=1, y=2, z=3):\n return x, y, z\n\n assert parametrized(C.function)\n injected_args_list = [m.injected_args for m in C.function.marks]\n assert len(injected_args_list) == 4\n\n context_list = MarkedFunctionExpander(None, function=C.function).expand()\n all_f = [cxt.function for cxt in context_list]\n c = C()\n assert all_f[0](c) == (100, 200, 300)\n assert all_f[1](c) == (100, 2, 300)\n assert all_f[2](c) == (1, 200, 3)\n assert all_f[3](c) == (1, 2, 3)\n\n\nclass CheckMatrix(object):\n def check_simple(self):\n @matrix(x=[1, 2], y=[-1, -2])\n def function(x=1, y=2, z=3):\n return x, y, z\n\n assert parametrized(function)\n injected_args_list = [m.injected_args for m in function.marks]\n assert len(injected_args_list) == 1\n\n context_list = MarkedFunctionExpander(None, function=function).expand()\n assert len(context_list) == 4\n\n for ctx in context_list:\n f = ctx.function\n injected_args = ctx.injected_args\n assert f() == (injected_args[\"x\"], injected_args[\"y\"], 3)\n\n def check_simple_method(self):\n class C(object):\n @matrix(x=[1, 2], y=[-1, -2])\n def function(self, x=1, y=2, z=3):\n return x, y, z\n\n assert parametrized(C.function)\n injected_args_list = [m.injected_args for m in C.function.marks]\n assert len(injected_args_list) == 1\n\n context_list = MarkedFunctionExpander(None, function=C.function).expand()\n assert len(context_list) == 4\n\n c = C()\n for ctx in context_list:\n f = ctx.function\n injected_args = ctx.injected_args\n assert f(c) == (injected_args[\"x\"], injected_args[\"y\"], 3)\n\n def check_stacked(self):\n @matrix(x=[1, 2], y=[0])\n @matrix(x=[-1], z=[-10])\n def function(x=1, y=2, z=3):\n return x, y, z\n\n assert parametrized(function)\n injected_args_list = [m.injected_args for m in function.marks]\n assert len(injected_args_list) == 2\n\n context_list = MarkedFunctionExpander(None, function=function).expand()\n assert len(context_list) == 3\n\n expected_output = {(1, 0, 3), (2, 0, 3), (-1, 2, -10)}\n output = set()\n for c in context_list:\n output.add(c.function())\n\n assert output == expected_output\n\n def check_stacked_method(self):\n class C(object):\n @matrix(x=[1, 2], y=[0])\n @matrix(x=[-1], z=[-10])\n def function(self, x=1, y=2, z=3):\n return x, y, z\n\n assert parametrized(C.function)\n injected_args_list = [m.injected_args for m in C.function.marks]\n assert len(injected_args_list) == 2\n\n context_list = MarkedFunctionExpander(None, function=C.function).expand()\n assert len(context_list) == 3\n\n expected_output = {(1, 0, 3), (2, 0, 3), (-1, 2, -10)}\n output = set()\n for ctx in context_list:\n output.add(ctx.function(C()))\n\n assert output == expected_output\n\n\nclass CheckDefaults(object):\n def check_defaults(self):\n @defaults(z=[1, 2])\n @matrix(x=[1], y=[1, 2])\n @parametrize(x=3, y=4)\n def function(x=1, y=2, z=-1):\n return x, y, z\n\n assert parametrized(function)\n injected_args_list = [m.injected_args for m in function.marks]\n assert len(injected_args_list) == 3\n\n context_list = MarkedFunctionExpander(None, function=function).expand()\n assert len(context_list) == 6\n\n expected_output = {\n (1, 1, 1),\n (1, 1, 2),\n (1, 2, 1),\n (1, 2, 2),\n (3, 4, 1),\n (3, 4, 2),\n }\n output = set()\n for ctx in context_list:\n output.add(ctx.function())\n\n assert output == expected_output\n\n def check_defaults_method(self):\n class C(object):\n @defaults(z=[1, 2])\n @matrix(x=[1], y=[1, 2])\n @parametrize(x=3, y=4)\n def function(self, x=1, y=2, z=-1):\n return x, y, z\n\n assert parametrized(C.function)\n injected_args_list = [m.injected_args for m in C.function.marks]\n assert len(injected_args_list) == 3\n\n context_list = MarkedFunctionExpander(None, function=C.function).expand()\n assert len(context_list) == 6\n\n expected_output = {\n (1, 1, 1),\n (1, 1, 2),\n (1, 2, 1),\n (1, 2, 2),\n (3, 4, 1),\n (3, 4, 2),\n }\n output = set()\n\n c = C()\n for ctx in context_list:\n f = ctx.function\n injected_args = ctx.injected_args\n assert f(c) == (injected_args[\"x\"], injected_args[\"y\"], injected_args[\"z\"])\n output.add(ctx.function(C()))\n\n assert output == expected_output\n\n def check_overlap_param(self):\n @defaults(y=[3, 4], z=[1, 2])\n @parametrize(w=1, x=2, y=3)\n def function(w=10, x=20, y=30, z=40):\n return w, x, y, z\n\n assert parametrized(function)\n injected_args_list = [m.injected_args for m in function.marks]\n assert len(injected_args_list) == 2\n\n context_list = MarkedFunctionExpander(None, function=function).expand()\n assert len(context_list) == 2\n\n expected_output = {(1, 2, 3, 1), (1, 2, 3, 2)}\n output = set()\n for ctx in context_list:\n output.add(ctx.function())\n\n assert output == expected_output\n\n def check_overlap_matrix(self):\n @defaults(y=[3, 4], z=[1, 2])\n @matrix(x=[1, 2], y=[5, 6])\n def function(x=20, y=30, z=40):\n return x, y, z\n\n assert parametrized(function)\n injected_args_list = [m.injected_args for m in function.marks]\n assert len(injected_args_list) == 2\n\n context_list = MarkedFunctionExpander(None, function=function).expand()\n assert len(context_list) == 8\n\n expected_output = {\n (1, 5, 1),\n (1, 5, 2),\n (2, 5, 1),\n (2, 5, 2),\n (1, 6, 1),\n (1, 6, 2),\n (2, 6, 1),\n (2, 6, 2),\n }\n output = set()\n for ctx in context_list:\n output.add(ctx.function())\n\n assert output == expected_output\n\n def check_only_defaults(self):\n @defaults(x=[3], z=[1, 2])\n def function(x=1, y=2, z=-1):\n return x, y, z\n\n assert parametrized(function)\n injected_args_list = [m.injected_args for m in function.marks]\n assert len(injected_args_list) == 1\n\n context_list = MarkedFunctionExpander(None, function=function).expand()\n assert len(context_list) == 2\n\n expected_output = {(3, 2, 1), (3, 2, 2)}\n output = set()\n for ctx in context_list:\n output.add(ctx.function())\n\n assert output == expected_output\n" }, { "path": "tests/mark/resources/__init__.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n" }, { "path": "tests/reporter/check_symbol_reporter.py", "content": "from pathlib import Path\nfrom unittest.mock import Mock\n\nfrom ducktape.tests.reporter import FailedTestSymbolReporter\n\n\ndef check_to_symbol_no_args(tmp_path):\n result = Mock(\n file_name=\"/test_folder/test_file\",\n cls_name=\"TestClass\",\n function_name=\"test_func\",\n injected_args=None,\n )\n reporter = FailedTestSymbolReporter(Mock())\n reporter.working_dir = Path(\"/\")\n assert reporter.to_symbol(result) == \"test_folder/test_file::TestClass.test_func\"\n\n\ndef check_to_symbol_relative_path(tmp_path):\n result = Mock(\n file_name=\"/test_folder/test_file\",\n cls_name=\"TestClass\",\n function_name=\"test_func\",\n injected_args=None,\n )\n reporter = FailedTestSymbolReporter(Mock())\n reporter.working_dir = Path(\"/test_folder\")\n assert reporter.to_symbol(result) == \"test_file::TestClass.test_func\"\n\n\ndef check_to_symbol_with_args():\n result = Mock(\n file_name=\"/test_folder/test_file\",\n cls_name=\"TestClass\",\n function_name=\"test_func\",\n injected_args={\"arg\": \"val\"},\n )\n\n reporter = FailedTestSymbolReporter(Mock())\n reporter.working_dir = Path(\"/\")\n assert reporter.to_symbol(result) == 'test_folder/test_file::TestClass.test_func@{\"arg\":\"val\"}'\n" }, { "path": "tests/runner/__init__.py", "content": "" }, { "path": "tests/runner/check_runner.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom unittest.mock import patch\n\nimport pytest\n\nfrom ducktape.cluster.node_container import NodeContainer, InsufficientResourcesError\nfrom ducktape.tests.runner_client import RunnerClient\nfrom ducktape.tests.status import PASS, FAIL\nfrom ducktape.tests.test_context import TestContext\nfrom ducktape.tests.runner import TestRunner\nfrom ducktape.mark.mark_expander import MarkedFunctionExpander\nfrom ducktape.cluster.localhost import LocalhostCluster\nfrom tests.ducktape_mock import FakeCluster, MockSender\n\nimport tests.ducktape_mock\nfrom tests.runner.resources.test_fails_to_init import FailsToInitTest\nfrom tests.runner.resources.test_fails_to_init_in_setup import FailsToInitInSetupTest\nfrom .resources.test_bad_actor import BadActorTest\nfrom .resources.test_thingy import ClusterTestThingy, TestThingy\nfrom .resources.test_failing_tests import FailingTest\nfrom ducktape.tests.reporter import JUnitReporter\n\nfrom unittest.mock import Mock, MagicMock\nimport os\nimport xml.etree.ElementTree as ET\n\nfrom .resources.test_various_num_nodes import VariousNumNodesTest\n\nTEST_THINGY_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), \"resources/test_thingy.py\"))\nFAILING_TEST_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), \"resources/test_failing_tests.py\"))\nFAILS_TO_INIT_TEST_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), \"resources/test_fails_to_init.py\"))\nFAILS_TO_INIT_IN_SETUP_TEST_FILE = os.path.abspath(\n os.path.join(os.path.dirname(__file__), \"resources/test_fails_to_init_in_setup.py\")\n)\nVARIOUS_NUM_NODES_TEST_FILE = os.path.abspath(\n os.path.join(os.path.dirname(__file__), \"resources/test_various_num_nodes.py\")\n)\nBAD_ACTOR_TEST_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), \"resources/test_bad_actor.py\"))\n\n\nclass CheckRunner(object):\n def check_insufficient_cluster_resources(self):\n \"\"\"The test runner should behave sensibly when the cluster is too small to run a given test.\"\"\"\n mock_cluster = FakeCluster(1)\n session_context = tests.ducktape_mock.session_context()\n\n test_context = TestContext(\n session_context=session_context,\n module=None,\n cls=TestThingy,\n function=TestThingy.test_pi,\n file=TEST_THINGY_FILE,\n cluster=mock_cluster,\n cluster_use_metadata={\"num_nodes\": 1000},\n )\n runner = TestRunner(mock_cluster, session_context, Mock(), [test_context], 1)\n\n # Even though the cluster is too small, the test runner should this handle gracefully without raising an error\n results = runner.run_all_tests()\n assert len(results) == 1\n assert results.num_failed == 1\n assert results.num_passed == 0\n assert results.num_ignored == 0\n\n def _do_expand(self, test_file, test_class, test_methods, cluster=None, session_context=None):\n ctx_list = []\n for f in test_methods:\n ctx_list.extend(\n MarkedFunctionExpander(\n session_context=session_context,\n cls=test_class,\n function=f,\n file=test_file,\n cluster=cluster,\n ).expand()\n )\n return ctx_list\n\n def check_simple_run(self):\n \"\"\"Check expected behavior when running a single test.\"\"\"\n mock_cluster = LocalhostCluster(num_nodes=1000)\n session_context = tests.ducktape_mock.session_context()\n\n test_methods = [\n TestThingy.test_pi,\n TestThingy.test_ignore1,\n TestThingy.test_ignore2,\n TestThingy.test_failure,\n ]\n ctx_list = self._do_expand(\n test_file=TEST_THINGY_FILE,\n test_class=TestThingy,\n test_methods=test_methods,\n cluster=mock_cluster,\n session_context=session_context,\n )\n runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)\n\n results = runner.run_all_tests()\n assert len(results) == 4\n assert results.num_flaky == 0\n assert results.num_failed == 1\n assert results.num_passed == 1\n assert results.num_ignored == 2\n\n result_with_data = [r for r in results if r.data is not None][0]\n assert result_with_data.data == {\"data\": 3.14159}\n\n def check_deflake_run(self):\n \"\"\"Check expected behavior when running a single test.\"\"\"\n mock_cluster = LocalhostCluster(num_nodes=1000)\n session_context = tests.ducktape_mock.session_context()\n\n test_methods = [TestThingy.test_flaky, TestThingy.test_failure]\n ctx_list = self._do_expand(\n test_file=TEST_THINGY_FILE,\n test_class=TestThingy,\n test_methods=test_methods,\n cluster=mock_cluster,\n session_context=session_context,\n )\n runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 2)\n\n results = runner.run_all_tests()\n assert len(results) == 2\n assert results.num_flaky == 1\n assert results.num_failed == 1\n assert results.num_passed == 0\n assert results.num_ignored == 0\n\n def check_runner_report_junit(self):\n \"\"\"Check we can serialize results into a xunit xml format. Also ensures that the XML report\n adheres to the Junit spec using xpath queries\"\"\"\n mock_cluster = LocalhostCluster(num_nodes=1000)\n session_context = tests.ducktape_mock.session_context()\n test_methods = [\n TestThingy.test_pi,\n TestThingy.test_ignore1,\n TestThingy.test_ignore2,\n TestThingy.test_failure,\n ]\n ctx_list = self._do_expand(\n test_file=TEST_THINGY_FILE,\n test_class=TestThingy,\n test_methods=test_methods,\n cluster=mock_cluster,\n session_context=session_context,\n )\n runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)\n\n results = runner.run_all_tests()\n JUnitReporter(results).report()\n xml_report = os.path.join(session_context.results_dir, \"report.xml\")\n assert os.path.exists(xml_report)\n tree = ET.parse(xml_report)\n assert len(tree.findall(\"./testsuite/testcase/failure\")) == 1\n assert len(tree.findall(\"./testsuite/testcase/skipped\")) == 2\n assert len(tree.findall(\"./testsuite/testcase\")) == 4\n\n passed = tree.findall(\"./testsuite/testcase/[@status='pass']\")\n assert len(passed) == 1\n assert passed[0].get(\"classname\") == \"TestThingy\"\n assert passed[0].get(\"name\") == \"test_pi\"\n\n failures = tree.findall(\"./testsuite/testcase/[@status='fail']\")\n assert len(failures) == 1\n assert failures[0].get(\"classname\") == \"TestThingy\"\n assert failures[0].get(\"name\") == \"test_failure\"\n\n ignores = tree.findall(\"./testsuite/testcase/[@status='ignore']\")\n assert len(ignores) == 2\n assert ignores[0].get(\"classname\") == \"TestThingy\"\n assert ignores[1].get(\"classname\") == \"TestThingy\"\n\n assert ignores[0].get(\"name\") == \"test_ignore1\"\n assert ignores[1].get(\"name\") == \"test_ignore2.x=5\"\n\n def check_exit_first(self):\n \"\"\"Confirm that exit_first in session context has desired effect of preventing any tests from running\n after the first test failure.\n \"\"\"\n mock_cluster = LocalhostCluster(num_nodes=1000)\n session_context = tests.ducktape_mock.session_context(**{\"exit_first\": True})\n\n test_methods = [FailingTest.test_fail]\n ctx_list = self._do_expand(\n test_file=FAILING_TEST_FILE,\n test_class=FailingTest,\n test_methods=test_methods,\n cluster=mock_cluster,\n session_context=session_context,\n )\n runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)\n results = runner.run_all_tests()\n assert len(ctx_list) > 1\n assert len(results) == 1\n\n def check_exits_if_failed_to_initialize(self):\n \"\"\"Validate that runner exits correctly when tests failed to initialize.\"\"\"\n mock_cluster = LocalhostCluster(num_nodes=1000)\n session_context = tests.ducktape_mock.session_context()\n\n ctx_list = self._do_expand(\n test_file=FAILS_TO_INIT_TEST_FILE,\n test_class=FailsToInitTest,\n test_methods=[FailsToInitTest.test_nothing],\n cluster=mock_cluster,\n session_context=session_context,\n )\n ctx_list.extend(\n self._do_expand(\n test_file=FAILS_TO_INIT_IN_SETUP_TEST_FILE,\n test_class=FailsToInitInSetupTest,\n test_methods=[FailsToInitInSetupTest.test_nothing],\n cluster=mock_cluster,\n session_context=session_context,\n )\n )\n\n runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)\n results = runner.run_all_tests()\n # These tests fail to initialize, each class has two test methods, so should have 4 results, all failed\n assert len(results) == 4\n assert results.num_flaky == 0\n assert results.num_failed == 4\n assert results.num_passed == 0\n assert results.num_ignored == 0\n\n # mock an error reporting test failure - this should not prevent subsequent tests from execution and mark\n # failed test as failed correctly\n @patch.object(RunnerClient, \"_exc_msg\", side_effect=Exception)\n def check_sends_result_when_error_reporting_exception(self, exc_msg_mock):\n \"\"\"Validates that an error when reporting an exception in the test doesn't prevent subsequent tests\n from executing\"\"\"\n mock_cluster = LocalhostCluster(num_nodes=1000)\n session_context = tests.ducktape_mock.session_context()\n test_methods = [TestThingy.test_failure, TestThingy.test_pi]\n ctx_list = self._do_expand(\n test_file=TEST_THINGY_FILE,\n test_class=TestThingy,\n test_methods=test_methods,\n cluster=mock_cluster,\n session_context=session_context,\n )\n runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)\n\n results = runner.run_all_tests()\n assert len(results) == 2\n assert results.num_flaky == 0\n assert results.num_failed == 1\n assert results.num_passed == 1\n assert results.num_ignored == 0\n\n def check_run_failure_with_bad_cluster_allocation(self):\n \"\"\"Check test should be marked failed if it under-utilizes the cluster resources.\"\"\"\n mock_cluster = LocalhostCluster(num_nodes=1000)\n session_context = tests.ducktape_mock.session_context(\n fail_bad_cluster_utilization=\"fail_bad_cluster_utilization\"\n )\n\n ctx_list = self._do_expand(\n test_file=TEST_THINGY_FILE,\n test_class=ClusterTestThingy,\n test_methods=[ClusterTestThingy.test_bad_num_nodes],\n cluster=mock_cluster,\n session_context=session_context,\n )\n runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)\n\n results = runner.run_all_tests()\n\n assert len(results) == 1\n assert results.num_flaky == 0\n assert results.num_failed == 1\n assert results.num_passed == 0\n assert results.num_ignored == 0\n\n def check_test_failure_with_too_many_nodes_requested(self):\n mock_cluster = LocalhostCluster(num_nodes=1000)\n session_context = tests.ducktape_mock.session_context(debug=True)\n\n ctx_list = self._do_expand(\n test_file=BAD_ACTOR_TEST_FILE,\n test_class=BadActorTest,\n test_methods=[BadActorTest.test_too_many_nodes],\n cluster=mock_cluster,\n session_context=session_context,\n )\n ctx_list.extend(\n self._do_expand(\n test_file=VARIOUS_NUM_NODES_TEST_FILE,\n test_class=VariousNumNodesTest,\n test_methods=[VariousNumNodesTest.test_one_node_a],\n cluster=mock_cluster,\n session_context=session_context,\n )\n )\n runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)\n results = runner.run_all_tests()\n assert results.num_flaky == 0\n assert results.num_failed == 1\n assert results.num_passed == 1\n assert results.num_ignored == 0\n passed = [r for r in results if r.test_status == PASS]\n failed = [r for r in results if r.test_status == FAIL]\n assert passed[0].test_id == \"tests.runner.resources.test_various_num_nodes.VariousNumNodesTest.test_one_node_a\"\n assert failed[0].test_id == \"tests.runner.resources.test_bad_actor.BadActorTest.test_too_many_nodes\"\n\n def check_runner_timeout(self):\n \"\"\"Check process cleanup and error handling when runner times out.\n\n When timeout occurs, the runner should:\n 1. Clean up all client processes\n 2. Mark active tests as failed with 'Test timed out' message\n 3. Mark remaining unrun tests as failed with 'Test not run' message\n 4. Return results gracefully instead of raising\n \"\"\"\n mock_cluster = LocalhostCluster(num_nodes=1000)\n session_context = tests.ducktape_mock.session_context(max_parallel=1000, test_runner_timeout=1)\n\n test_methods = [TestThingy.test_delayed, TestThingy.test_failure]\n ctx_list = self._do_expand(\n test_file=TEST_THINGY_FILE,\n test_class=TestThingy,\n test_methods=test_methods,\n cluster=mock_cluster,\n session_context=session_context,\n )\n runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)\n\n # Should return results gracefully instead of raising\n results = runner.run_all_tests()\n\n # All client processes should be cleaned up\n assert not runner._client_procs\n\n # Both tests should be marked as failed\n assert len(results) == 2\n assert results.num_failed == 2\n assert results.num_passed == 0\n\n # Check that failure summaries contain timeout information\n for result in results:\n assert result.test_status == FAIL\n # Summary should mention either \"timed out\" or \"not run\" with the exception message\n assert \"runner client unresponsive\" in result.summary.lower()\n\n @pytest.mark.parametrize(\"fail_greedy_tests\", [True, False])\n def check_fail_greedy_tests(self, fail_greedy_tests):\n mock_cluster = LocalhostCluster(num_nodes=1000)\n session_context = tests.ducktape_mock.session_context(fail_greedy_tests=fail_greedy_tests)\n\n test_methods = [\n VariousNumNodesTest.test_empty_cluster_annotation,\n VariousNumNodesTest.test_no_cluster_annotation,\n VariousNumNodesTest.test_zero_nodes,\n ]\n ctx_list = self._do_expand(\n test_file=VARIOUS_NUM_NODES_TEST_FILE,\n test_class=VariousNumNodesTest,\n test_methods=test_methods,\n cluster=mock_cluster,\n session_context=session_context,\n )\n runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)\n results = runner.run_all_tests()\n assert results.num_flaky == 0\n assert results.num_failed == (2 if fail_greedy_tests else 0)\n # zero-node test should always pass, whether we fail on greedy or not\n assert results.num_passed == (1 if fail_greedy_tests else 3)\n assert results.num_ignored == 0\n\n def check_cluster_shrink(self):\n \"\"\"\n Check what happens if cluster loses a node while the runner is already running.\n SchedulerTestThingy has two 5-node tests, and one of each for 4, 3, and 2 nodes.\n\n Thus both 5-node tests should pass, first one failing during pre-allocation phase,\n second one shouldn't even attempt to be allocated.\n And all the other tests should pass still.\n \"\"\"\n\n mock_cluster = ShrinkingLocalhostCluster(num_nodes=5)\n session_context = tests.ducktape_mock.session_context(max_parallel=10)\n\n test_methods = [\n VariousNumNodesTest.test_five_nodes_a,\n VariousNumNodesTest.test_five_nodes_b,\n VariousNumNodesTest.test_four_nodes,\n VariousNumNodesTest.test_three_nodes_a,\n VariousNumNodesTest.test_two_nodes_a,\n ]\n\n ctx_list = self._do_expand(\n test_file=VARIOUS_NUM_NODES_TEST_FILE,\n test_class=VariousNumNodesTest,\n test_methods=test_methods,\n cluster=mock_cluster,\n session_context=session_context,\n )\n\n runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)\n results = runner.run_all_tests()\n\n assert len(results) == 5\n assert results.num_flaky == 0\n assert results.num_failed == 2 # both of the 5-node tests should fail\n assert results.num_passed == 3 # 4-node, 3-node and 2-node should all pass\n assert results.num_ignored == 0\n\n def check_cluster_shrink_reschedule(self):\n \"\"\"\n Test that the test that failed to schedule initially due to a node going offline is not lost and is still\n scheduled when more nodes become available.\n\n We start with a 6-node cluster.\n First we run a long-ish 3-node test, leaving 3 nodes available.\n Then when trying to run a second 3-node test, we shrink the cluster, emulating one of the nodes\n going down - this leaves only 2 nodes available, so we cannot run this test.\n\n However, after the first 3-node test finishes running, it will return its 3 nodes back to the cluster,\n so the second 3-node test becomes schedulable again - this is what we test for.\n\n Also two two-node tests should pass too - they should be scheduled before the second 3-node test,\n while two nodes are waiting for the first 3-node test to finish.\n\n It's generally not a good practice to rely on sleep, but I think it's acceptable in this case,\n since we do need to rely on parallelism.\n \"\"\"\n\n mock_cluster = ShrinkingLocalhostCluster(num_nodes=6, shrink_on=2)\n session_context = tests.ducktape_mock.session_context(max_parallel=10)\n\n test_methods = [\n VariousNumNodesTest.test_three_nodes_asleep,\n VariousNumNodesTest.test_three_nodes_b,\n VariousNumNodesTest.test_two_nodes_a,\n VariousNumNodesTest.test_two_nodes_b,\n ]\n\n ctx_list = self._do_expand(\n test_file=VARIOUS_NUM_NODES_TEST_FILE,\n test_class=VariousNumNodesTest,\n test_methods=test_methods,\n cluster=mock_cluster,\n session_context=session_context,\n )\n\n runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)\n results = runner.run_all_tests()\n\n assert len(results) == 4\n assert results.num_flaky == 0\n assert results.num_failed == 0\n assert results.num_passed == 4\n assert results.num_ignored == 0\n\n # normal order on a 6-node cluster would be:\n # - test_three_nodes_asleep, test_three_nodes_b, test_two_nodes_a, test_two_nodes_b\n # however the cluster would shrink to 5 nodes after scheduling the first 3-node test,\n # leaving no space for the second 3-node test to be scheduled, bumping it down the line,\n # while two 2-node tests will be scheduled alongside the\n expected_scheduling_order = [\n \"VariousNumNodesTest.test_three_nodes_asleep\",\n \"VariousNumNodesTest.test_two_nodes_a\",\n \"VariousNumNodesTest.test_two_nodes_b\",\n \"VariousNumNodesTest.test_three_nodes_b\",\n ]\n # We check the actual order the tests were scheduled in, since completion order might be different,\n # with so many fast tests running in parallel.\n actual_scheduling_order = [x.test_id for x in runner.test_schedule_log]\n assert actual_scheduling_order == expected_scheduling_order\n\n def check_cluster_shrink_to_zero(self):\n \"\"\"\n Validates that if the cluster is shrunk to zero nodes size, no tests can run,\n but we still exit gracefully.\n \"\"\"\n\n mock_cluster = ShrinkingLocalhostCluster(num_nodes=1, shrink_on=1)\n session_context = tests.ducktape_mock.session_context(max_parallel=10)\n\n test_methods = [\n VariousNumNodesTest.test_one_node_a,\n VariousNumNodesTest.test_one_node_b,\n ]\n\n ctx_list = self._do_expand(\n test_file=VARIOUS_NUM_NODES_TEST_FILE,\n test_class=VariousNumNodesTest,\n test_methods=test_methods,\n cluster=mock_cluster,\n session_context=session_context,\n )\n\n runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)\n results = runner.run_all_tests()\n\n assert len(results) == 2\n assert results.num_flaky == 0\n assert results.num_failed == 2\n assert results.num_passed == 0\n assert results.num_ignored == 0\n\n def check_runner_client_report(self):\n \"\"\"Validates that an error when reporting an exception in the test doesn't prevent subsequent tests\n from executing\"\"\"\n mock_cluster = tests.ducktape_mock.mock_cluster()\n session_context = tests.ducktape_mock.session_context()\n test_context = tests.ducktape_mock.test_context(session_context=session_context)\n rc = RunnerClient(\n \"localhost\",\n 22,\n test_context.test_id,\n 0,\n \"dummy\",\n \"/tmp/dummy\",\n True,\n False,\n 5,\n )\n rc.sender = MockSender()\n rc.cluster = mock_cluster\n rc.session_context = session_context\n rc.test_metadata = MagicMock()\n with patch(\"ducktape.tests.runner_client.RunnerClient._collect_test_context\") as test_collect_patch:\n test_collect_patch.return_value = test_context\n rc.run()\n # get the last message, get the args send to that message, and get the first arg\n finished_result = rc.sender.send_results[-1][0][0]\n assert finished_result.get(\"event_type\") == \"FINISHED\"\n assert finished_result[\"result\"].summary == \"Test Passed\"\n\n def check_report_remaining_as_failed(self):\n \"\"\"Test that _report_remaining_as_failed marks all unrun tests as failed.\"\"\"\n mock_cluster = LocalhostCluster(num_nodes=1000)\n session_context = tests.ducktape_mock.session_context()\n\n test_methods = [TestThingy.test_pi, TestThingy.test_failure]\n ctx_list = self._do_expand(\n test_file=TEST_THINGY_FILE,\n test_class=TestThingy,\n test_methods=test_methods,\n cluster=mock_cluster,\n session_context=session_context,\n )\n runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)\n\n # Call the method directly\n reason = \"test reason for failure\"\n runner._report_remaining_as_failed(reason)\n\n # All tests should be marked as failed\n assert len(runner.results) == 2\n assert runner.results.num_failed == 2\n for result in runner.results:\n assert result.test_status == FAIL\n assert f\"Test not run: {reason}\" in result.summary\n\n # Scheduler should be empty after draining\n assert len(runner.scheduler) == 0\n\n def check_report_active_as_failed(self):\n \"\"\"Test that _report_active_as_failed marks running tests as failed.\"\"\"\n mock_cluster = LocalhostCluster(num_nodes=1000)\n session_context = tests.ducktape_mock.session_context()\n\n test_methods = [TestThingy.test_pi]\n ctx_list = self._do_expand(\n test_file=TEST_THINGY_FILE,\n test_class=TestThingy,\n test_methods=test_methods,\n cluster=mock_cluster,\n session_context=session_context,\n )\n runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)\n\n # Simulate an active test using the actual test_id from ctx_list\n from ducktape.tests.runner import TestKey\n import time\n\n test_ctx = ctx_list[0]\n test_key = TestKey(test_ctx.test_id, 1)\n runner.active_tests[test_key] = True\n runner.client_report[test_key] = {\"runner_start_time\": time.time() - 10}\n\n # Call the method\n reason = \"test timeout reason\"\n runner._report_active_as_failed(reason)\n\n # Test should be marked as failed\n assert len(runner.results) == 1\n assert runner.results.num_failed == 1\n results_list = [r for r in runner.results]\n assert len(results_list) == 1\n result = results_list[0]\n assert result.test_status == FAIL\n assert f\"Test timed out: {reason}\" in result.summary\n assert result.start_time < result.stop_time\n\n # Active tests should be cleared\n assert len(runner.active_tests) == 0\n\n def check_report_active_as_failed_frees_cluster(self):\n \"\"\"Test that _report_active_as_failed frees cluster resources.\"\"\"\n mock_cluster = LocalhostCluster(num_nodes=1000)\n session_context = tests.ducktape_mock.session_context()\n\n test_methods = [TestThingy.test_pi]\n ctx_list = self._do_expand(\n test_file=TEST_THINGY_FILE,\n test_class=TestThingy,\n test_methods=test_methods,\n cluster=mock_cluster,\n session_context=session_context,\n )\n runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)\n\n # Simulate an active test with allocated cluster\n from ducktape.tests.runner import TestKey\n from ducktape.cluster.finite_subcluster import FiniteSubcluster\n import time\n\n test_ctx = ctx_list[0]\n test_key = TestKey(test_ctx.test_id, 1)\n\n # Allocate cluster nodes for this test\n allocated_nodes = mock_cluster.alloc(test_ctx.expected_cluster_spec)\n runner._test_cluster[test_key] = FiniteSubcluster(allocated_nodes)\n runner.active_tests[test_key] = True\n runner.client_report[test_key] = {\"runner_start_time\": time.time() - 10}\n\n # Check that nodes are allocated\n initial_available = mock_cluster.num_available_nodes()\n assert test_key in runner._test_cluster\n\n # Call the method\n reason = \"test timeout reason\"\n runner._report_active_as_failed(reason)\n\n # Cluster resources should be freed\n assert test_key not in runner._test_cluster\n assert mock_cluster.num_available_nodes() == initial_available + len(allocated_nodes)\n\n # Active tests should be cleared\n assert len(runner.active_tests) == 0\n\n def check_runner_client_shutdown_flag(self):\n \"\"\"Test that runner client respects shutdown flag when sending messages.\"\"\"\n mock_cluster = tests.ducktape_mock.mock_cluster()\n session_context = tests.ducktape_mock.session_context()\n test_context = tests.ducktape_mock.test_context(session_context=session_context)\n\n rc = RunnerClient(\n \"localhost\",\n 22,\n test_context.test_id,\n 0,\n \"dummy\",\n \"/tmp/dummy\",\n True,\n False,\n 5,\n )\n rc.sender = MockSender()\n rc.cluster = mock_cluster\n rc.session_context = session_context\n\n # Set shutdown flag\n rc.runner_shutting_down = True\n\n # Try to send a message - should return None without sending\n from ducktape.tests.event import ClientEventFactory\n\n message_factory = ClientEventFactory(test_context.test_id, 0, \"test-client\")\n result = rc.send(message_factory.ready())\n\n assert result is None\n # No messages should have been sent\n assert len(rc.sender.send_results) == 0\n\n def check_duplicate_finished_message_handling(self):\n \"\"\"Test that duplicate FINISHED messages are handled correctly with three distinct cases.\"\"\"\n from ducktape.tests.runner import TestKey\n from ducktape.tests.result import TestResult\n from ducktape.cluster.finite_subcluster import FiniteSubcluster\n import time\n\n mock_cluster = LocalhostCluster(num_nodes=1000)\n session_context = tests.ducktape_mock.session_context()\n\n test_methods = [TestThingy.test_pi]\n ctx_list = self._do_expand(\n test_file=TEST_THINGY_FILE,\n test_class=TestThingy,\n test_methods=test_methods,\n cluster=mock_cluster,\n session_context=session_context,\n )\n runner = TestRunner(mock_cluster, session_context, Mock(), ctx_list, 1)\n\n # Mock the receiver.send to avoid ZMQ socket issues in tests\n runner.receiver.send = Mock()\n\n # Simulate a test that has been started\n test_ctx = ctx_list[0]\n test_key = TestKey(test_ctx.test_id, 1)\n runner.active_tests[test_key] = True\n\n # Allocate cluster for this test\n allocated_nodes = mock_cluster.alloc(test_ctx.expected_cluster_spec)\n runner._test_cluster[test_key] = FiniteSubcluster(allocated_nodes)\n\n # Mock a client process\n runner._client_procs[test_key] = Mock()\n runner._client_procs[test_key].is_alive.return_value = False\n runner._client_procs[test_key].join.return_value = None\n runner._client_procs[test_key].exitcode = 0\n runner._client_procs[test_key].name = \"MockProcess\"\n\n # Create a FINISHED event\n from ducktape.tests.event import ClientEventFactory\n\n event_factory = ClientEventFactory(test_ctx.test_id, 1, \"test-client\")\n result = TestResult(test_ctx, 1, session_context, PASS, \"Test passed\", None, time.time(), time.time())\n event = event_factory.finished(result=result)\n\n # Case 1: Normal first FINISHED - should process normally\n runner._handle_finished(event)\n assert test_key not in runner.active_tests\n assert test_key in runner.finished_tests\n assert len(runner.results) == 1\n assert test_key not in runner._test_cluster\n assert runner.receiver.send.call_count == 1\n\n # Case 2: Duplicate FINISHED (ZMQ retry) - should be ignored gracefully\n runner._handle_finished(event)\n assert len(runner.results) == 1 # Should not add duplicate\n assert runner.receiver.send.call_count == 2 # ACK still sent\n\n # Case 3: FINISHED for unknown test - should raise RuntimeError\n unknown_event_factory = ClientEventFactory(\"unknown.test.id\", 999, \"test-client\")\n unknown_result = TestResult(test_ctx, 999, session_context, PASS, \"Test passed\", None, time.time(), time.time())\n unknown_event = unknown_event_factory.finished(result=unknown_result)\n\n with pytest.raises(RuntimeError, match=\"not in active_tests\"):\n runner._handle_finished(unknown_event)\n\n def check_timeout_exception_join_timeout_param(self):\n \"\"\"Test that timeout_exception_join_timeout parameter is used correctly.\"\"\"\n mock_cluster = LocalhostCluster(num_nodes=1000)\n session_context = tests.ducktape_mock.session_context()\n\n test_methods = [TestThingy.test_pi]\n ctx_list = self._do_expand(\n test_file=TEST_THINGY_FILE,\n test_class=TestThingy,\n test_methods=test_methods,\n cluster=mock_cluster,\n session_context=session_context,\n )\n\n # Create runner with custom timeout values\n runner = TestRunner(\n mock_cluster,\n session_context,\n Mock(),\n ctx_list,\n 1,\n finish_join_timeout=10,\n timeout_exception_join_timeout=60,\n )\n\n assert runner.finish_join_timeout == 10\n assert runner.timeout_exception_join_timeout == 60\n\n\nclass ShrinkingLocalhostCluster(LocalhostCluster):\n def __init__(self, *args, shrink_on=1, **kwargs):\n super().__init__(*args, **kwargs)\n self.bad_nodes = NodeContainer()\n # which call to shrink on\n self.shrink_on = shrink_on\n self.num_alloc_calls = 0\n\n def do_alloc(self, cluster_spec):\n allocated = super().do_alloc(cluster_spec)\n self.num_alloc_calls += 1\n if self.shrink_on == self.num_alloc_calls:\n bad_node = allocated.pop()\n self._in_use_nodes.remove_node(bad_node)\n self.bad_nodes.add_node(bad_node)\n\n # simplified logic, we know all nodes are of the same OS/type\n # check if we don't have enough nodes any more\n # (which really should be true every time, since the largest test would be scheduled)\n if len(allocated) < len(cluster_spec):\n # return all good nodes back to be available\n for node in allocated:\n self._in_use_nodes.remove_node(node)\n self._available_nodes.add_node(node)\n\n raise InsufficientResourcesError(\"yeah\")\n\n return allocated\n" }, { "path": "tests/runner/check_runner_memory.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.tests.runner import TestRunner\nfrom ducktape.mark.mark_expander import MarkedFunctionExpander\nfrom ducktape.cluster.localhost import LocalhostCluster\n\nfrom .resources.test_memory_leak import MemoryLeakTest\n\nimport math\nfrom memory_profiler import memory_usage\nimport os\nfrom queue import Queue\nimport statistics\nfrom statistics import mean\n\nimport tests.ducktape_mock\n\nfrom mock import Mock\n\n\nN_TEST_CASES = 5\n\n\nMEMORY_LEAK_TEST_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), \"resources/test_memory_leak.py\"))\n\n\nclass InstrumentedTestRunner(TestRunner):\n \"\"\"Identical to TestRunner, except dump memory used by the current process\n before running each test.\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n self.queue = kwargs.get(\"queue\")\n del kwargs[\"queue\"]\n super(InstrumentedTestRunner, self).__init__(*args, **kwargs)\n\n def _run_single_test(self, test_context):\n # write current memory usage to file before running the test\n pid = os.getpid()\n current_memory = memory_usage(pid)[0]\n self.queue.put(current_memory)\n\n super(InstrumentedTestRunner, self)._run_single_test(test_context)\n\n\nclass CheckMemoryUsage(object):\n def setup_method(self, _):\n self.cluster = LocalhostCluster(num_nodes=100)\n self.session_context = tests.ducktape_mock.session_context()\n\n def check_for_inter_test_memory_leak(self):\n \"\"\"Until v0.3.10, ducktape had a serious source of potential memory leaks.\n\n Because test_context objects held a reference to all services for the duration of a test run, the memory\n used by any individual service would not be garbage-collected until well after *all* tests had run.\n\n This memory leak was discovered in Kafka system tests, where many long-running system tests were enough\n to cumulatively use up the memory on the test machine, causing a cascade of test failures due to\n inability to allocate any more memory.\n\n This test provides a regression check against this type of memory leak; it fails without the fix, and passes\n with it.\n \"\"\"\n # Get a list of test_context objects for the test runner\n ctx_list = []\n test_methods = [MemoryLeakTest.test_leak]\n for f in test_methods:\n ctx_list.extend(\n MarkedFunctionExpander(\n session_context=self.session_context,\n cls=MemoryLeakTest,\n function=f,\n file=MEMORY_LEAK_TEST_FILE,\n cluster=self.cluster,\n ).expand()\n )\n assert len(ctx_list) == N_TEST_CASES # Sanity check\n\n q = Queue()\n runner = InstrumentedTestRunner(self.cluster, self.session_context, Mock(), ctx_list, 1, queue=q)\n runner.run_all_tests()\n\n measurements = []\n while not q.empty():\n measurements.append(q.get())\n self.validate_memory_measurements(measurements)\n\n def validate_memory_measurements(self, measurements):\n \"\"\"A rough heuristic to check that stair-case style memory leak is not present.\n\n The idea is that when well-behaved, in this specific test, the maximum memory usage should be near the \"middle\".\n Here we check that the maximum usage is within 5% of the median memory usage.\n\n What is meant by stair-case? When the leak was present in its most blatant form,\n each repetition of MemoryLeak.test_leak run by the test runner adds approximately a fixed amount of memory\n without freeing much, resulting in a memory usage profile that looks like a staircase going up to the right.\n \"\"\"\n median_usage = statistics.median(measurements)\n max_usage = max(measurements)\n\n usage_stats = \"\\nmax: %s,\\nmedian: %s,\\nall: %s\\n\" % (\n max_usage,\n median_usage,\n measurements,\n )\n\n # we want to make sure that max usage doesn't exceed median usage by very much\n relative_diff = (max_usage - median_usage) / median_usage\n slope = self._linear_regression_slope(measurements)\n\n if slope > 0:\n # check max memory usage iff the memory measurements seem to be increasing overall\n assert relative_diff <= 0.05, (\n \"max usage exceeded median usage by too much; there may be a memory leak: %s\" % usage_stats\n )\n\n def _linear_regression_slope(self, arr):\n \"\"\"Return the sign of the slope of the least squares fit line.\"\"\"\n assert len(arr) > 0\n\n x_vals = [i for i in range(len(arr))]\n mean_x = mean(x_vals)\n mean_y = mean(arr)\n\n # mean([x_i * y_i]) - mean_x * mean_y\n # slope = -----------------------------------\n # variance([x_i])\n #\n # where variance is (1/N) * sum([(x_i - mean_x)^2])\n #\n # the denominator in regression formula is always positive, so it's enough to compute the numerator\n\n slope_numerator = mean([i * arr[i] for i in x_vals])\n slope_numerator = slope_numerator - (mean_x * mean_y)\n\n # return the sign\n return math.copysign(slope_numerator, 1)\n" }, { "path": "tests/runner/check_sender_receiver.py", "content": "# Copyright 2021 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.cluster.localhost import LocalhostCluster\nfrom tests.ducktape_mock import test_context, session_context\n\nimport logging\nimport pytest\nfrom ducktape.tests.runner_client import Sender\nfrom ducktape.tests.runner import Receiver\nfrom ducktape.tests.event import ClientEventFactory, EventResponseFactory\nfrom ducktape.errors import TimeoutError\n\nimport multiprocessing as mp\nimport os\n\n\nclass CheckSenderReceiver(object):\n def ready_response(self, client_id, port):\n sender_event_factory = ClientEventFactory(\"test_1\", 0, client_id)\n sender = Sender(\n server_host=\"localhost\",\n server_port=port,\n message_supplier=sender_event_factory,\n logger=logging,\n )\n sender.send(sender_event_factory.ready())\n\n def check_simple_messaging(self):\n s_context = session_context()\n cluster = LocalhostCluster(num_nodes=1000)\n t_context = test_context(s_context, cluster)\n\n client_id = \"test-runner-{}-{}\".format(os.getpid(), id(self))\n receiver_response_factory = EventResponseFactory()\n\n receiver = Receiver(5556, 5656)\n receiver.start()\n port = receiver.port\n\n try:\n p = mp.Process(target=self.ready_response, args=(client_id, port))\n p.start()\n\n event = receiver.recv(timeout=10000)\n assert event[\"event_type\"] == ClientEventFactory.READY\n logging.info(\"replying to client\")\n receiver.send(receiver_response_factory.ready(event, s_context, t_context, cluster))\n finally:\n p.join(timeout=2) # Add timeout to prevent hanging if subprocess has issues\n receiver.close()\n\n def check_timeout(self):\n client_id = \"test-runner-{}-{}\".format(os.getpid(), id(self))\n\n receiver = Receiver(5556, 5656)\n receiver.start()\n port = receiver.port\n\n try:\n p = mp.Process(target=self.ready_response, args=(client_id, port))\n p.start()\n with pytest.raises(TimeoutError):\n receiver.recv(timeout=0)\n finally:\n # Terminate the process instead of waiting for it to timeout\n # (it will keep retrying for up to 15 seconds with default timeouts)\n p.terminate()\n p.join(timeout=1)\n receiver.close()\n\n def check_exponential_backoff(self):\n \"\"\"Test that Sender applies exponential backoff on retries.\"\"\"\n import time\n\n # Save original values\n original_timeout = Sender.REQUEST_TIMEOUT_MS\n original_retries = Sender.NUM_RETRIES\n\n # Override timeout values for faster testing BEFORE creating sender\n Sender.REQUEST_TIMEOUT_MS = 100 # 100ms instead of 3000ms\n Sender.NUM_RETRIES = 3 # 3 retries instead of 5\n\n try:\n client_id = \"test-backoff-client\"\n event_factory = ClientEventFactory(\"test_1\", 0, client_id)\n\n # Create sender that will timeout (no receiver listening on this port)\n sender = Sender(\n server_host=\"localhost\",\n server_port=9999, # Nothing listening here\n message_supplier=event_factory,\n logger=logging,\n )\n\n start_time = time.time()\n try:\n sender.send(event_factory.ready())\n except RuntimeError as e:\n # Expected to fail with \"Unable to receive response from driver\"\n assert \"Unable to receive response from driver\" in str(e)\n\n elapsed = time.time() - start_time\n\n # With 2x backoff: 100ms + 200ms + 400ms = 700ms total\n # Without backoff: 100ms * 3 = 300ms total\n # We expect at least 500ms to prove backoff is working\n # Use 400ms to account for timing variance\n assert elapsed > 0.4, f\"Expected > 0.4s with backoff, got {elapsed:.3f}s\"\n assert elapsed < 2.0, f\"Expected < 2.0s, got {elapsed:.3f}s (test taking too long)\"\n\n sender.close()\n finally:\n # Restore original values\n Sender.REQUEST_TIMEOUT_MS = original_timeout\n Sender.NUM_RETRIES = original_retries\n" }, { "path": "tests/runner/fake_remote_account.py", "content": "from ducktape.cluster.consts import LINUX, WINDOWS\nfrom ducktape.cluster.remoteaccount import RemoteAccount\n\n\nclass FakeRemoteAccount(RemoteAccount):\n def __init__(self, *args, is_available=True, **kwargs):\n super().__init__(*args, **kwargs)\n self.os = LINUX\n self.is_available = is_available\n\n def available(self):\n return self.is_available\n\n def fetch_externally_routable_ip(self, *args, **kwargs):\n return \"fake ip\"\n\n\nclass FakeWindowsRemoteAccount(FakeRemoteAccount):\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n self.os = WINDOWS\n\n\ndef create_fake_remote_account(*args, **kwargs):\n return FakeRemoteAccount(*args, **kwargs)\n" }, { "path": "tests/runner/resources/__init__.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n" }, { "path": "tests/runner/resources/test_bad_actor.py", "content": "from ducktape.mark.resource import cluster\nfrom ducktape.services.service import Service\nfrom ducktape.tests.test import Test\n\n\nclass FakeService(Service):\n pass\n\n\nclass BadActorTest(Test):\n @cluster(num_nodes=2)\n def test_too_many_nodes(self):\n \"\"\"\n This test should fail to allocate and\n should be dealt with gracefully by the framework, marking it as failed\n and moving on.\n \"\"\"\n FakeService(self.test_context, num_nodes=3)\n" }, { "path": "tests/runner/resources/test_failing_tests.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import print_function\n\nfrom ducktape.mark.resource import cluster\nfrom ducktape.tests.test import Test\nfrom ducktape.mark import matrix\n\n\"\"\"All tests in this module fail\"\"\"\n\n\nclass FailingTest(Test):\n def __init__(self, test_context):\n super(FailingTest, self).__init__(test_context)\n\n @cluster(num_nodes=1000)\n @matrix(x=[_ for _ in range(2)])\n def test_fail(self, x):\n print(\"Test %s fails!\" % x)\n raise RuntimeError(\"This test throws an error!\")\n" }, { "path": "tests/runner/resources/test_fails_to_init.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.tests.test import Test\nfrom ducktape.mark import matrix\n\n\"\"\"All tests in this module fail\"\"\"\n\n\nclass FailsToInitTest(Test):\n \"\"\"All tests in this class fail to initialize due to an exception in constructor\"\"\"\n\n def __init__(self, test_context):\n super(FailsToInitTest, self).__init__(test_context)\n x = None\n x.split(\":\")\n\n @matrix(x=[_ for _ in range(2)])\n def test_nothing(self):\n self.logger.info(\"NOTHING\")\n" }, { "path": "tests/runner/resources/test_fails_to_init_in_setup.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.tests.test import Test\nfrom ducktape.mark import matrix\n\n\"\"\"All tests in this module fail\"\"\"\n\n\nclass FailsToInitInSetupTest(Test):\n \"\"\"All tests in this class fail to initialize due to an exception in setUp() method\"\"\"\n\n def __init__(self, test_context):\n super(FailsToInitInSetupTest, self).__init__(test_context)\n\n def setUp(self):\n x = None\n x.split(\":\")\n\n @matrix(x=[_ for _ in range(2)])\n def test_nothing(self):\n self.logger.info(\"NOTHING\")\n" }, { "path": "tests/runner/resources/test_memory_leak.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nfrom ducktape.mark.resource import cluster\nfrom ducktape.tests.test import Test\nfrom ducktape.services.service import Service\nfrom ducktape.mark import matrix\n\n\nMEMORY_EATER_LIST_SIZE = 10000000\nN_TEST_CASES = 5\n\n\nclass MemoryEater(Service):\n \"\"\"Simple service that has a reference to a list with many elements\"\"\"\n\n def __init__(self, context):\n super(MemoryEater, self).__init__(context, 1)\n self.items = []\n\n def start_node(self, node):\n self.items = [x for x in range(MEMORY_EATER_LIST_SIZE)]\n\n def stop_node(self, node):\n pass\n\n def clean_node(self, node):\n pass\n\n @property\n def num_nodes(self):\n return 1\n\n\nclass MemoryLeakTest(Test):\n \"\"\"A group of identical \"memory-hungry\" ducktape tests.\n Each test holds a reference to a service which itself holds a reference to a large (memory intensive) object.\n \"\"\"\n\n def __init__(self, test_context):\n super(MemoryLeakTest, self).__init__(test_context)\n self.memory_eater = MemoryEater(test_context)\n\n @cluster(num_nodes=100)\n @matrix(x=[i for i in range(N_TEST_CASES)])\n def test_leak(self, x):\n self.memory_eater.start()\n" }, { "path": "tests/runner/resources/test_thingy.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport time\nfrom ducktape.tests.test import Test\nfrom ducktape.mark import ignore, parametrize\nfrom ducktape.mark.resource import cluster\n\n\n_flake = False\n\n\nclass TestThingy(Test):\n \"\"\"Fake ducktape test class\"\"\"\n\n @cluster(num_nodes=1000)\n def test_pi(self):\n return {\"data\": 3.14159}\n\n @cluster(num_nodes=1000)\n def test_delayed(self):\n time.sleep(1)\n\n @cluster(num_nodes=1000)\n @ignore\n def test_ignore1(self):\n pass\n\n @cluster(num_nodes=1000)\n @ignore(x=5)\n @parametrize(x=5)\n def test_ignore2(self, x=2):\n pass\n\n @cluster(num_nodes=1000)\n def test_failure(self):\n raise Exception(\"This failed\")\n\n @cluster(num_nodes=1000)\n def test_flaky(self):\n global _flake\n flake, _flake = _flake, not _flake\n assert flake\n\n\nclass ClusterTestThingy(Test):\n \"\"\"Fake ducktape test class\"\"\"\n\n @cluster(num_nodes=10)\n def test_bad_num_nodes(self):\n pass\n" }, { "path": "tests/runner/resources/test_various_num_nodes.py", "content": "import time\n\nfrom ducktape.mark.resource import cluster\nfrom ducktape.tests.test import Test\n\n\nclass VariousNumNodesTest(Test):\n \"\"\"\n Allocates various number of nodes.\n \"\"\"\n\n @cluster(num_nodes=5)\n def test_five_nodes_a(self):\n assert True\n\n @cluster(num_nodes=5)\n def test_five_nodes_b(self):\n assert True\n\n @cluster(num_nodes=4)\n def test_four_nodes(self):\n assert True\n\n @cluster(num_nodes=3)\n def test_three_nodes_asleep(self):\n time.sleep(3)\n assert True\n\n @cluster(num_nodes=3)\n def test_three_nodes_a(self):\n assert True\n\n @cluster(num_nodes=3)\n def test_three_nodes_b(self):\n assert True\n\n @cluster(num_nodes=2)\n def test_two_nodes_a(self):\n assert True\n\n @cluster(num_nodes=2)\n def test_two_nodes_b(self):\n assert True\n\n @cluster(num_nodes=1)\n def test_one_node_a(self):\n assert True\n\n @cluster(num_nodes=1)\n def test_one_node_b(self):\n assert True\n\n def test_no_cluster_annotation(self):\n assert True\n\n @cluster()\n def test_empty_cluster_annotation(self):\n assert True\n\n # this one is valid regardless of\n # whether the greedy tests are allowed or not\n # because it's not greedy, quite the opposite\n @cluster(num_nodes=0)\n def test_zero_nodes(self):\n assert True\n" }, { "path": "tests/scheduler/__init__.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n" }, { "path": "tests/scheduler/check_scheduler.py", "content": "# Copyright 2016 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nimport collections\n\nfrom ducktape.cluster.cluster_spec import ClusterSpec\nfrom tests.ducktape_mock import FakeCluster\nfrom ducktape.tests.scheduler import TestScheduler\nfrom ducktape.services.service import Service\n\nFakeContext = collections.namedtuple(\"FakeContext\", [\"test_id\", \"expected_num_nodes\", \"expected_cluster_spec\"])\n\n\nclass CheckScheduler(object):\n def setup_method(self, _):\n self.cluster = FakeCluster(100)\n self.tc0 = FakeContext(0, expected_num_nodes=10, expected_cluster_spec=ClusterSpec.simple_linux(10))\n self.tc1 = FakeContext(1, expected_num_nodes=50, expected_cluster_spec=ClusterSpec.simple_linux(50))\n self.tc2 = FakeContext(\n 2,\n expected_num_nodes=100,\n expected_cluster_spec=ClusterSpec.simple_linux(100),\n )\n self.tc_list = [\n self.tc0,\n self.tc1,\n self.tc2,\n ]\n\n def check_empty(self):\n \"\"\"Check expected behavior of empty scheduler.\"\"\"\n scheduler = TestScheduler([], self.cluster)\n\n assert len(scheduler) == 0\n assert scheduler.peek() is None\n\n def check_non_empty_cluster_too_small(self):\n \"\"\"Ensure that scheduler does not return tests if the cluster does not have enough available nodes.\"\"\"\n\n scheduler = TestScheduler(self.tc_list, self.cluster)\n assert len(scheduler) == len(self.tc_list)\n assert scheduler.peek() is not None\n\n # alloc all cluster nodes so none are available\n self.cluster.alloc(Service.setup_cluster_spec(num_nodes=len(self.cluster)))\n assert self.cluster.num_available_nodes() == 0\n\n # peeking should not yield an object\n assert scheduler.peek() is None\n\n def check_simple_usage(self):\n \"\"\"Check usage with fully available cluster.\"\"\"\n\n scheduler = TestScheduler(self.tc_list, self.cluster)\n\n c = 2\n while len(scheduler) > 0:\n t = scheduler.peek()\n assert t.test_id == c\n assert len(scheduler) == c + 1\n\n scheduler.remove(t)\n assert len(scheduler) == c\n\n c -= 1\n\n def check_with_changing_cluster_availability(self):\n \"\"\"Modify cluster usage in between calls to next()\"\"\"\n\n scheduler = TestScheduler(self.tc_list, self.cluster)\n\n # start with 100-node cluster (configured in setup_method())\n # allocate 60 nodes; only test_id 0 (which needs 10 nodes) should be available\n nodes = self.cluster.alloc(Service.setup_cluster_spec(num_nodes=60))\n assert self.cluster.num_available_nodes() == 40\n t = scheduler.peek()\n assert t == self.tc0\n scheduler.remove(t)\n assert scheduler.peek() is None\n\n # return 10 nodes, so 50 are available in the cluster\n # next test from the scheduler should be test id 1 (which needs 50 nodes)\n return_nodes = nodes[:10]\n keep_nodes = nodes[10:]\n self.cluster.free(return_nodes)\n assert self.cluster.num_available_nodes() == 50\n t = scheduler.peek()\n assert t == self.tc1\n scheduler.remove(t)\n assert scheduler.peek() is None\n\n # return remaining nodes, so cluster is fully available\n # next test from scheduler should be test id 2 (which needs 100 nodes)\n return_nodes = keep_nodes\n self.cluster.free(return_nodes)\n assert self.cluster.num_available_nodes() == len(self.cluster)\n t = scheduler.peek()\n assert t == self.tc2\n scheduler.remove(t)\n # scheduler should become empty now\n assert len(scheduler) == 0\n assert scheduler.peek() is None\n\n def check_filter_unschedulable_tests(self):\n self.cluster = FakeCluster(49) # only test id 0 can be scheduled on this cluster\n scheduler = TestScheduler(self.tc_list, self.cluster)\n\n unschedulable = scheduler.filter_unschedulable_tests()\n assert set(unschedulable) == {self.tc1, self.tc2}\n # subsequent calls should return empty list and not modify scheduler\n assert not scheduler.filter_unschedulable_tests()\n assert scheduler.peek() == self.tc0\n\n def check_drain_remaining_tests(self):\n \"\"\"Test that drain_remaining_tests returns all tests and empties the scheduler.\"\"\"\n scheduler = TestScheduler(self.tc_list, self.cluster)\n\n # Initially scheduler should have all tests\n assert len(scheduler) == len(self.tc_list)\n\n # Drain all remaining tests\n remaining = scheduler.drain_remaining_tests()\n\n # Should return all tests\n assert len(remaining) == len(self.tc_list)\n assert set(remaining) == set(self.tc_list)\n\n # Scheduler should now be empty\n assert len(scheduler) == 0\n assert scheduler.peek() is None\n\n # Calling again should return empty list\n remaining_again = scheduler.drain_remaining_tests()\n assert len(remaining_again) == 0\n\n def check_drain_remaining_tests_partial(self):\n \"\"\"Test drain_remaining_tests after some tests have been removed.\"\"\"\n scheduler = TestScheduler(self.tc_list, self.cluster)\n\n # Remove one test\n t = scheduler.peek()\n scheduler.remove(t)\n initial_len = len(scheduler)\n\n # Drain remaining tests\n remaining = scheduler.drain_remaining_tests()\n\n # Should return only the remaining tests\n assert len(remaining) == initial_len\n assert t not in remaining\n\n # Scheduler should be empty\n assert len(scheduler) == 0\n" }, { "path": "tests/services/__init__.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n" }, { "path": "tests/services/check_background_thread_service.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.cluster.cluster_spec import ClusterSpec\nfrom ducktape.services.background_thread import BackgroundThreadService\nfrom ducktape.errors import TimeoutError\nfrom tests.ducktape_mock import test_context, MockNode\nimport pytest\nimport time\n\n\nclass DummyService(BackgroundThreadService):\n \"\"\"Single node service that sleeps for self.run_time_sec seconds in a background thread.\"\"\"\n\n def __init__(self, context, run_time_sec, exc=None):\n super(DummyService, self).__init__(context, 1)\n self.running = False\n self.run_time_sec = run_time_sec\n self._exc = exc\n\n def who_am_i(self, node=None):\n return \"DummyService\"\n\n def idx(self, node):\n return 1\n\n def allocate_nodes(self):\n self.nodes = [MockNode()]\n\n def _worker(self, idx, node):\n if self._exc:\n raise self._exc\n\n self.running = True\n\n end = time.time() + self.run_time_sec\n while self.running:\n time.sleep(0.1)\n if time.time() > end:\n self.running = False\n break\n\n def stop_node(self, node):\n self.running = False\n\n\nclass CheckBackgroundThreadService(object):\n def setup_method(self, method):\n self.context = test_context()\n\n def check_service_constructor(self):\n \"\"\"Check that BackgroundThreadService constructor corresponds to the base class's one.\"\"\"\n exp_spec = ClusterSpec.simple_linux(10)\n service = BackgroundThreadService(self.context, cluster_spec=exp_spec)\n assert service.cluster_spec == exp_spec\n\n service = BackgroundThreadService(self.context, num_nodes=20)\n assert service.cluster_spec.size() == 20\n\n with pytest.raises(RuntimeError):\n BackgroundThreadService(self.context, num_nodes=20, cluster_spec=exp_spec)\n\n def check_service_timeout(self):\n \"\"\"Test that wait(timeout_sec) raise a TimeoutError in approximately the expected time.\"\"\"\n self.service = DummyService(self.context, float(\"inf\"))\n self.service.start()\n start = time.time()\n timeout_sec = 0.1\n try:\n self.service.wait(timeout_sec=timeout_sec)\n raise Exception(\"Expected service to timeout.\")\n except TimeoutError:\n end = time.time()\n # Relative difference should be pretty small\n # within 10% should be reasonable\n actual_timeout = end - start\n relative_difference = abs(timeout_sec - actual_timeout) / timeout_sec\n assert relative_difference < 0.1, (\n \"Correctly threw timeout error, but timeout doesn't match closely with expected timeout. \"\n + \"(expected timeout, actual timeout): (%s, %s)\" % (str(timeout_sec), str(actual_timeout))\n )\n\n def check_no_timeout(self):\n \"\"\"Run an instance of DummyService with a short run_time_sec. It should stop without\n timing out.\"\"\"\n\n self.service = DummyService(self.context, run_time_sec=0.1)\n self.service.start()\n self.service.wait(timeout_sec=0.5)\n\n def check_wait_node(self):\n self.service = DummyService(self.context, run_time_sec=float(\"inf\"))\n self.service.start()\n node = self.service.nodes[0]\n assert not self.service.wait_node(node, timeout_sec=0.1)\n self.service.stop_node(node)\n assert self.service.wait_node(node)\n\n def check_wait_node_no_start(self):\n self.service = DummyService(self.context, run_time_sec=float(\"inf\"))\n node = self.service.nodes[0]\n assert self.service.wait_node(node)\n\n def check_background_exception(self):\n self.service = DummyService(self.context, float(\"inf\"), Exception(\"failure\"))\n self.service.start()\n with pytest.raises(Exception):\n self.service.wait(timeout_sec=1)\n with pytest.raises(Exception):\n self.service.stop(timeout_sec=1)\n assert hasattr(self.service, \"errors\")\n" }, { "path": "tests/services/check_jvm_logging.py", "content": "# Copyright 2024 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.services.service import Service\nfrom ducktape.jvm_logging import JVMLogger\nfrom tests.ducktape_mock import test_context, session_context\nfrom ducktape.cluster.localhost import LocalhostCluster\nfrom unittest.mock import Mock\nimport os\n\n\nclass JavaService(Service):\n \"\"\"Mock Java service for testing JVM logging.\"\"\"\n\n def __init__(self, context, num_nodes):\n super(JavaService, self).__init__(context, num_nodes)\n self.start_called = False\n self.clean_called = False\n self.ssh_commands = []\n\n def idx(self, node):\n return 1\n\n def start_node(self, node, **kwargs):\n super(JavaService, self).start_node(node, **kwargs)\n self.start_called = True\n # Simulate a Java command being run\n node.account.ssh(\"java -version\")\n\n def clean_node(self, node, **kwargs):\n super(JavaService, self).clean_node(node, **kwargs)\n self.clean_called = True\n\n\ndef create_mock_node():\n \"\"\"Create a mock node with a mock account that tracks SSH calls.\"\"\"\n node = Mock()\n node.account = Mock()\n node.account.ssh = Mock(return_value=None)\n node.account.hostname = \"mock-host\"\n node.account.externally_routable_ip = \"127.0.0.1\"\n return node\n\n\nclass CheckJVMLogging(object):\n def setup_method(self, _):\n self.cluster = LocalhostCluster()\n self.session_context = session_context()\n self.context = test_context(self.session_context, cluster=self.cluster)\n self.jvm_logger = JVMLogger()\n\n def check_enable_for_service(self):\n \"\"\"Check that JVM logging can be enabled for a service.\"\"\"\n service = JavaService(self.context, 1)\n\n # Enable JVM logging\n self.jvm_logger.enable_for_service(service)\n\n # Verify logs dict was updated\n assert hasattr(service, \"logs\")\n assert \"jvm_gc_log\" in service.logs\n assert \"jvm_stdout_stderr\" in service.logs\n assert \"jvm_heap_dump\" in service.logs\n\n # Verify helper methods were added\n assert hasattr(service, \"JVM_LOG_DIR\")\n assert hasattr(service, \"jvm_options\")\n assert hasattr(service, \"setup_jvm_logging\")\n assert hasattr(service, \"clean_jvm_logs\")\n\n def check_jvm_options_format(self):\n \"\"\"Check that JVM options string is properly formatted.\"\"\"\n jvm_opts = self.jvm_logger._get_jvm_options()\n\n # Should start with -Xlog:disable to prevent console pollution\n assert jvm_opts.startswith(\"-Xlog:disable\")\n\n # Should contain key logging options\n assert \"gc*:file=\" in jvm_opts\n assert \"HeapDumpOnOutOfMemoryError\" in jvm_opts\n assert \"safepoint=info\" in jvm_opts\n assert \"class+load=info\" in jvm_opts\n assert \"jit+compilation=info\" in jvm_opts\n assert \"NativeMemoryTracking=summary\" in jvm_opts\n\n def check_ssh_wrapping(self):\n \"\"\"Check that all SSH methods are wrapped to inject JDK_JAVA_OPTIONS.\"\"\"\n # Create a mock node first\n mock_node = create_mock_node()\n # Add ssh_capture and ssh_output methods to mock\n mock_node.account.ssh_capture = Mock(return_value=iter([]))\n mock_node.account.ssh_output = Mock(return_value=\"\")\n\n # Enable JVM logging for a service\n service = JavaService(self.context, 1)\n self.jvm_logger.enable_for_service(service)\n\n # Manually call the wrapped start_node with our mock node\n # This simulates what happens during service.start()\n service.start_node(mock_node)\n\n # Verify all SSH methods were wrapped\n assert hasattr(mock_node.account, \"original_ssh\")\n assert hasattr(mock_node.account, \"original_ssh_capture\")\n assert hasattr(mock_node.account, \"original_ssh_output\")\n\n # The current methods should be callables (the wrapper functions)\n assert callable(mock_node.account.ssh)\n assert callable(mock_node.account.ssh_capture)\n assert callable(mock_node.account.ssh_output)\n assert service.start_called\n\n def check_ssh_methods_inject_options(self):\n \"\"\"Check that wrapped SSH methods actually inject JDK_JAVA_OPTIONS.\"\"\"\n service = JavaService(self.context, 1)\n node = service.nodes[0]\n\n # Track what commands are actually executed\n executed_commands = []\n\n def track_ssh(cmd, allow_fail=False):\n executed_commands.append((\"ssh\", cmd))\n return 0\n\n def track_ssh_capture(cmd, allow_fail=False, callback=None, combine_stderr=True, timeout_sec=None):\n executed_commands.append((\"ssh_capture\", cmd))\n return iter([])\n\n def track_ssh_output(cmd, allow_fail=False, combine_stderr=True, timeout_sec=None):\n executed_commands.append((\"ssh_output\", cmd))\n return \"\"\n\n # Set the tracking functions as the original methods\n node.account.ssh = track_ssh\n node.account.ssh_capture = track_ssh_capture\n node.account.ssh_output = track_ssh_output\n\n # Enable JVM logging - this wraps start_node\n self.jvm_logger.enable_for_service(service)\n\n # Start node - this wraps the SSH methods\n service.start_node(node)\n\n # Clear the setup commands (from mkdir, etc)\n executed_commands.clear()\n\n # Now execute commands through the wrapped methods\n node.account.ssh(\"java -version\")\n node.account.ssh_capture(\"java -jar app.jar\")\n node.account.ssh_output(\"java -cp test.jar Main\")\n\n # Verify JDK_JAVA_OPTIONS was injected in all commands\n assert len(executed_commands) == 3, f\"Expected 3 commands, got {len(executed_commands)}\"\n for method, cmd in executed_commands:\n assert \"JDK_JAVA_OPTIONS=\" in cmd, f\"{method} didn't inject JDK_JAVA_OPTIONS: {cmd}\"\n assert \"-Xlog:disable\" in cmd, f\"{method} didn't include JVM options: {cmd}\"\n # Verify it uses env command (more portable)\n assert cmd.startswith(\"env JDK_JAVA_OPTIONS=\"), f\"{method} doesn't use env command: {cmd}\"\n # Verify it appends to existing (${JDK_JAVA_OPTIONS:-})\n assert \"${JDK_JAVA_OPTIONS:-}\" in cmd, f\"{method} doesn't preserve existing options: {cmd}\"\n # Verify no extra quotes around the options (shlex.quote would add them)\n assert \"'-Xlog:disable\" not in cmd, f\"{method} has unwanted quotes around options: {cmd}\"\n\n def check_preserves_existing_jvm_options(self):\n \"\"\"Check that existing JDK_JAVA_OPTIONS are preserved and appended to.\"\"\"\n service = JavaService(self.context, 1)\n node = service.nodes[0]\n\n # Track what commands are executed\n executed_commands = []\n\n def track_ssh(cmd, allow_fail=False):\n executed_commands.append(cmd)\n return 0\n\n node.account.ssh = track_ssh\n node.account.ssh_capture = Mock(return_value=iter([]))\n node.account.ssh_output = Mock(return_value=\"\")\n\n # Enable JVM logging\n self.jvm_logger.enable_for_service(service)\n\n # Start node\n service.start_node(node)\n executed_commands.clear()\n\n # Execute a command - the wrapper should preserve any existing JDK_JAVA_OPTIONS\n node.account.ssh(\"java -version\")\n\n # Verify the command structure\n assert len(executed_commands) == 1\n cmd = executed_commands[0]\n\n # Should use env command\n assert cmd.startswith(\"env JDK_JAVA_OPTIONS=\")\n\n # Should preserve existing options with ${JDK_JAVA_OPTIONS:-}\n assert \"${JDK_JAVA_OPTIONS:-}\" in cmd\n\n # Should append our JVM logging options\n assert \"-Xlog:disable\" in cmd\n\n def check_ssh_wrap_idempotent(self):\n \"\"\"Check that SSH wrapping is idempotent (handles restarts).\"\"\"\n mock_node = create_mock_node()\n mock_node.account.ssh_capture = Mock(return_value=iter([]))\n mock_node.account.ssh_output = Mock(return_value=\"\")\n\n service = JavaService(self.context, 1)\n\n # Enable JVM logging\n self.jvm_logger.enable_for_service(service)\n\n # Get the wrapped start_node method\n wrapped_start_node = service.start_node\n\n # Call it twice with the same node\n wrapped_start_node(mock_node)\n\n # Verify wrapping happened\n assert hasattr(mock_node.account, \"original_ssh\")\n assert hasattr(mock_node.account, \"original_ssh_capture\")\n assert hasattr(mock_node.account, \"original_ssh_output\")\n\n # Call again (simulating restart)\n wrapped_start_node(mock_node)\n\n # SSH should still have the original_* attributes (idempotent)\n assert hasattr(mock_node.account, \"original_ssh\")\n assert hasattr(mock_node.account, \"original_ssh_capture\")\n assert hasattr(mock_node.account, \"original_ssh_output\")\n\n def check_clean_node_behavior(self):\n \"\"\"Check that clean_node properly cleans up and restores SSH methods.\"\"\"\n mock_node = create_mock_node()\n mock_node.account.ssh_capture = Mock(return_value=iter([]))\n mock_node.account.ssh_output = Mock(return_value=\"\")\n\n service = JavaService(self.context, 1)\n\n # Enable JVM logging\n self.jvm_logger.enable_for_service(service)\n\n # Get the wrapped methods\n wrapped_start_node = service.start_node\n wrapped_clean_node = service.clean_node\n\n # Start node to wrap SSH methods\n wrapped_start_node(mock_node)\n assert hasattr(mock_node.account, \"original_ssh\")\n assert hasattr(mock_node.account, \"original_ssh_capture\")\n assert hasattr(mock_node.account, \"original_ssh_output\")\n\n # Clean node to restore SSH methods\n wrapped_clean_node(mock_node)\n\n # Verify SSH methods were restored\n assert not hasattr(mock_node.account, \"original_ssh\")\n assert not hasattr(mock_node.account, \"original_ssh_capture\")\n assert not hasattr(mock_node.account, \"original_ssh_output\")\n\n def check_log_paths(self):\n \"\"\"Check that log paths are correctly configured.\"\"\"\n service = JavaService(self.context, 1)\n self.jvm_logger.enable_for_service(service)\n\n log_dir = self.jvm_logger.log_dir\n\n # Verify log paths\n assert service.logs[\"jvm_gc_log\"][\"path\"] == os.path.join(log_dir, \"gc.log\")\n assert service.logs[\"jvm_stdout_stderr\"][\"path\"] == os.path.join(log_dir, \"jvm.log\")\n assert service.logs[\"jvm_heap_dump\"][\"path\"] == os.path.join(log_dir, \"heap_dump.hprof\")\n\n # Verify collection flags\n assert service.logs[\"jvm_gc_log\"][\"collect_default\"] is True\n assert service.logs[\"jvm_stdout_stderr\"][\"collect_default\"] is True\n assert service.logs[\"jvm_heap_dump\"][\"collect_default\"] is False\n\n def check_kwargs_preserved(self):\n \"\"\"Check that start_node and clean_node preserve kwargs after wrapping.\"\"\"\n service = JavaService(self.context, 1)\n\n # Replace the node with a mock node\n mock_node = create_mock_node()\n service.nodes = [mock_node]\n\n self.jvm_logger.enable_for_service(service)\n\n # This should not raise an error even with extra kwargs\n service.start(timeout_sec=30, clean=False)\n assert service.start_called\n\n def check_setup_failure_doesnt_break_wrapping(self):\n \"\"\"Check that if _setup_on_node fails, the error propagates correctly.\"\"\"\n from ducktape.cluster.remoteaccount import RemoteCommandError\n\n mock_node = create_mock_node()\n mock_node.account.ssh_capture = Mock(return_value=iter([]))\n mock_node.account.ssh_output = Mock(return_value=\"\")\n\n # Make mkdir fail\n def failing_ssh(cmd, allow_fail=False):\n if \"mkdir\" in cmd:\n if not allow_fail:\n raise RemoteCommandError(mock_node.account, cmd, 1, \"Permission denied\")\n return 1\n return 0\n\n mock_node.account.ssh = failing_ssh\n\n service = JavaService(self.context, 1)\n self.jvm_logger.enable_for_service(service)\n\n # start_node should fail during setup\n try:\n service.start_node(mock_node)\n assert False, \"Expected RemoteCommandError\"\n except RemoteCommandError as e:\n assert \"Permission denied\" in str(e)\n\n def check_wrapped_ssh_failure_propagates(self):\n \"\"\"Check that SSH failures are properly propagated through the wrapper.\"\"\"\n from ducktape.cluster.remoteaccount import RemoteCommandError\n\n mock_node = create_mock_node()\n\n # Track calls\n ssh_calls = []\n\n def tracking_ssh(cmd, allow_fail=False):\n ssh_calls.append((cmd, allow_fail))\n # Fail on \"test-java\" commands (not the real java from JavaService.start_node)\n if \"test-java\" in cmd and not allow_fail:\n raise RemoteCommandError(mock_node.account, cmd, 127, \"java: command not found\")\n return 0\n\n mock_node.account.ssh = tracking_ssh\n mock_node.account.ssh_capture = Mock(return_value=iter([]))\n mock_node.account.ssh_output = Mock(return_value=\"\")\n\n service = JavaService(self.context, 1)\n self.jvm_logger.enable_for_service(service)\n service.start_node(mock_node)\n\n ssh_calls.clear()\n\n # Command that should fail\n try:\n mock_node.account.ssh(\"test-java -version\")\n assert False, \"Expected RemoteCommandError\"\n except RemoteCommandError as e:\n assert \"java: command not found\" in str(e)\n\n # Command with allow_fail=True should not raise\n result = mock_node.account.ssh(\"test-java -version\", allow_fail=True)\n assert result == 0 # Our mock returns 0 when allow_fail=True\n\n def check_wrapped_ssh_with_allow_fail(self):\n \"\"\"Check that allow_fail parameter works correctly through the wrapper.\"\"\"\n mock_node = create_mock_node()\n\n # Track calls and their allow_fail parameter\n ssh_calls = []\n\n def tracking_ssh(cmd, allow_fail=False):\n ssh_calls.append((cmd, allow_fail))\n return 1 if \"fail\" in cmd else 0\n\n mock_node.account.ssh = tracking_ssh\n mock_node.account.ssh_capture = Mock(return_value=iter([]))\n mock_node.account.ssh_output = Mock(return_value=\"\")\n\n service = JavaService(self.context, 1)\n self.jvm_logger.enable_for_service(service)\n service.start_node(mock_node)\n\n ssh_calls.clear()\n\n # Test with allow_fail=False (default)\n mock_node.account.ssh(\"echo success\")\n assert ssh_calls[-1][1] is False\n\n # Test with allow_fail=True\n mock_node.account.ssh(\"echo fail\", allow_fail=True)\n assert ssh_calls[-1][1] is True\n\n def check_cleanup_failure_still_restores_ssh(self):\n \"\"\"Check that even if cleanup fails, SSH methods are still restored.\"\"\"\n from ducktape.cluster.remoteaccount import RemoteCommandError\n\n mock_node = create_mock_node()\n\n cleanup_called = []\n\n def tracking_ssh(cmd, allow_fail=False):\n if \"rm -rf\" in cmd:\n cleanup_called.append(True)\n if not allow_fail:\n raise RemoteCommandError(mock_node.account, cmd, 1, \"rm failed\")\n return 0\n\n original_ssh = tracking_ssh\n mock_node.account.ssh = original_ssh\n mock_node.account.ssh_capture = Mock(return_value=iter([]))\n mock_node.account.ssh_output = Mock(return_value=\"\")\n\n service = JavaService(self.context, 1)\n self.jvm_logger.enable_for_service(service)\n\n # Start node\n service.start_node(mock_node)\n assert hasattr(mock_node.account, \"original_ssh\")\n\n # Clean node - cleanup uses allow_fail=True, so it shouldn't raise\n service.clean_node(mock_node)\n\n # Verify cleanup was attempted\n assert len(cleanup_called) > 0\n\n # Verify SSH was restored despite cleanup \"failure\"\n assert not hasattr(mock_node.account, \"original_ssh\")\n\n def check_double_cleanup_is_safe(self):\n \"\"\"Check that calling clean_node twice doesn't cause errors.\"\"\"\n mock_node = create_mock_node()\n mock_node.account.ssh_capture = Mock(return_value=iter([]))\n mock_node.account.ssh_output = Mock(return_value=\"\")\n\n service = JavaService(self.context, 1)\n self.jvm_logger.enable_for_service(service)\n\n # Start node\n service.start_node(mock_node)\n assert hasattr(mock_node.account, \"original_ssh\")\n\n # Clean node first time\n service.clean_node(mock_node)\n assert not hasattr(mock_node.account, \"original_ssh\")\n\n # Clean node second time - should not raise\n service.clean_node(mock_node)\n assert not hasattr(mock_node.account, \"original_ssh\")\n" }, { "path": "tests/services/check_service.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.services.service import Service\nfrom tests.ducktape_mock import test_context, session_context\nfrom ducktape.cluster.localhost import LocalhostCluster\n\n\nclass DummyService(Service):\n \"\"\"Simple fake service class.\"\"\"\n\n def __init__(self, context, num_nodes):\n super(DummyService, self).__init__(context, num_nodes)\n self.started_count = 0\n self.cleaned_count = 0\n self.stopped_count = 0\n\n self.started_kwargs = {}\n self.cleaned_kwargs = {}\n self.stopped_kwargs = {}\n\n def idx(self, node):\n return 1\n\n def start_node(self, node, **kwargs):\n super(DummyService, self).start_node(node, **kwargs)\n self.started_count += 1\n self.started_kwargs = kwargs\n\n def clean_node(self, node, **kwargs):\n super(DummyService, self).clean_node(node, **kwargs)\n self.cleaned_count += 1\n self.cleaned_kwargs = kwargs\n\n def stop_node(self, node, **kwargs):\n super(DummyService, self).stop_node(node, **kwargs)\n self.stopped_count += 1\n self.stopped_kwargs = kwargs\n\n\nclass DifferentDummyService(Service):\n \"\"\"Another fake service class.\"\"\"\n\n def __init__(self, context, num_nodes):\n super(DifferentDummyService, self).__init__(context, num_nodes)\n\n def idx(self, node):\n return 1\n\n\nclass CheckAllocateFree(object):\n def setup_method(self, _):\n self.cluster = LocalhostCluster()\n self.session_context = session_context()\n self.context = test_context(self.session_context, cluster=self.cluster)\n\n def check_allocate_free(self):\n \"\"\"Check that allocating and freeing nodes works.\n\n This regression test catches the error with Service.free() introduced in v0.3.3 and fixed in v0.3.4\n \"\"\"\n\n # Node allocation takes place during service instantiation\n initial_cluster_size = len(self.cluster)\n self.service = DummyService(self.context, 10)\n assert self.cluster.num_available_nodes() == initial_cluster_size - 10\n\n self.service.free()\n assert self.cluster.num_available_nodes() == initial_cluster_size\n\n def check_order(self):\n \"\"\"Check expected behavior with service._order method\"\"\"\n self.dummy0 = DummyService(self.context, 4)\n self.diffDummy0 = DifferentDummyService(self.context, 100)\n self.dummy1 = DummyService(self.context, 1)\n self.diffDummy1 = DifferentDummyService(self.context, 2)\n self.diffDummy2 = DifferentDummyService(self.context, 5)\n\n assert self.dummy0._order == 0\n assert self.dummy1._order == 1\n assert self.diffDummy0._order == 0\n assert self.diffDummy1._order == 1\n assert self.diffDummy2._order == 2\n\n\nclass CheckStartStop(object):\n def setup_method(self, _):\n self.cluster = LocalhostCluster()\n self.session_context = session_context()\n self.context = test_context(self.session_context, cluster=self.cluster)\n\n def check_start_stop_clean(self):\n \"\"\"\n Checks that start, stop, and clean invoke the expected per-node calls, and that start also runs stop and\n clean\n \"\"\"\n service = DummyService(self.context, 2)\n\n service.start()\n assert service.started_count == 2\n assert service.stopped_count == 2\n assert service.cleaned_count == 2\n\n service.stop()\n assert service.stopped_count == 4\n\n service.start(clean=False)\n assert service.started_count == 4\n assert service.stopped_count == 6\n assert service.cleaned_count == 2\n\n service.stop()\n assert service.stopped_count == 8\n\n service.clean()\n assert service.cleaned_count == 4\n\n def check_kwargs_support(self):\n \"\"\"Check that start, stop, and clean, and their per-node versions, can accept keyword arguments\"\"\"\n service = DummyService(self.context, 2)\n\n kwargs = {\"foo\": \"bar\"}\n service.start(**kwargs)\n assert service.started_kwargs == kwargs\n service.stop(**kwargs)\n assert service.stopped_kwargs == kwargs\n service.clean(**kwargs)\n assert service.cleaned_kwargs == kwargs\n" }, { "path": "tests/templates/__init__.py", "content": "" }, { "path": "tests/templates/service/__init__.py", "content": "" }, { "path": "tests/templates/service/check_render.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.services.service import Service\nfrom tests.ducktape_mock import test_context\n\n\nclass CheckTemplateRenderingService(object):\n \"\"\"\n Tests rendering of templates, using input from a Service\n \"\"\"\n\n def new_instance(self):\n return TemplateRenderingService()\n\n def check_simple(self):\n self.new_instance().render_simple()\n\n def check_single_variable(self):\n self.new_instance().render_single_variable()\n\n def check_overload(self):\n self.new_instance().render_overload()\n\n def check_class_template(self):\n self.new_instance().render_class_template()\n\n def check_file_template(self):\n self.new_instance().render_file_template()\n\n\nclass TemplateRenderingService(Service):\n NO_VARIABLE = \"fixed content\"\n SIMPLE_VARIABLE = \"Hello {{a_field}}!\"\n OVERLOAD_VARIABLES = \"{{normal}} {{overload}}\"\n\n CLASS_CONSTANT_TEMPLATE = \"{{ CLASS_CONSTANT }}\"\n CLASS_CONSTANT = \"constant\"\n\n def __init__(self):\n super(TemplateRenderingService, self).__init__(test_context(), 1)\n\n def render_simple(self):\n \"\"\"Test that a trivial template works\"\"\"\n assert self.render_template(self.NO_VARIABLE) == self.NO_VARIABLE\n\n def render_single_variable(self):\n \"\"\"Test that fields on the object are available to templates\"\"\"\n self.a_field = \"world\"\n assert self.render_template(self.SIMPLE_VARIABLE) == \"Hello world!\"\n\n def render_overload(self):\n self.normal = \"normal\"\n assert self.render_template(self.OVERLOAD_VARIABLES, overload=\"overloaded\") == \"normal overloaded\"\n\n def render_class_template(self):\n assert self.render_template(self.CLASS_CONSTANT_TEMPLATE) == self.CLASS_CONSTANT\n self.CLASS_CONSTANT = \"instance override\"\n assert self.render_template(self.CLASS_CONSTANT_TEMPLATE) == \"instance override\"\n\n def render_file_template(self):\n self.a_field = \"world\"\n assert \"Sample world\" == self.render(\"sample\")\n" }, { "path": "tests/templates/service/templates/sample", "content": "Sample {{a_field}}" }, { "path": "tests/templates/test/__init__.py", "content": "" }, { "path": "tests/templates/test/check_render.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.tests import Test\nfrom ducktape.tests import TestContext\nfrom ducktape.template import TemplateRenderer\n\nfrom tests.ducktape_mock import session_context\n\nimport os\nimport tempfile\n\n\nclass CheckTemplateRenderingTest(object):\n \"\"\"\n Minimal test to verify template rendering functionality\n \"\"\"\n\n def setup(self):\n dir = tempfile.gettempdir()\n session_ctx = session_context(results_dir=dir)\n test_ctx = TestContext(session_context=session_ctx)\n return TemplateRenderingTest(test_ctx)\n\n def check_string_template(self):\n test = self.setup()\n test.other = \"goodbye\"\n result = test.render_template(\"Hello {{name}} and {{other}}\", name=\"World\")\n assert \"Hello World and goodbye\" == result\n\n def check_file_template(self):\n test = self.setup()\n test.name = \"world\"\n assert \"Sample world\" == test.render(\"sample\")\n\n\nclass CheckPackageSearchPath(object):\n \"\"\"\n Simple check on extracting package and search path based on module name.\n \"\"\"\n\n def check_package_search_path(self):\n package, path = TemplateRenderer._package_search_path(\"a.b.c\")\n # search path should be b/templates since templates is by convention a sibling of c\n assert package == \"a\" and path == os.path.join(\"b\", \"templates\")\n\n package, path = TemplateRenderer._package_search_path(\"hi\")\n assert package == \"hi\" and path == \"templates\"\n\n package, path = TemplateRenderer._package_search_path(\"\")\n assert package == \"\" and path == \"templates\"\n\n def check_get_ctx(self):\n class A(TemplateRenderer):\n x = \"xxx\"\n\n class B(A):\n y = \"yyy\"\n\n b = B()\n b.instance = \"b instance\"\n\n ctx_a = A()._get_ctx()\n assert ctx_a[\"x\"] == \"xxx\"\n assert \"yyy\" not in ctx_a\n assert \"instance\" not in ctx_a\n\n ctx_b = b._get_ctx()\n assert ctx_b[\"x\"] == \"xxx\"\n assert ctx_b[\"y\"] == \"yyy\"\n assert ctx_b[\"instance\"] == \"b instance\"\n\n\nclass TemplateRenderingTest(Test):\n pass\n" }, { "path": "tests/templates/test/templates/sample", "content": "Sample {{name}}" }, { "path": "tests/test_utils.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport socket\n\n\ndef find_available_port(min_port=8000, max_port=9000):\n \"\"\"Return first available port in the range [min_port, max_port], inclusive.\n\n Note that this actually isn't a 100% reliable way of getting a port, but it's probably good enough -- once\n you close a socket you cannot be sure of its availability. This was the source of a bunch of issues in\n the Apache Kafka unit tests.\n \"\"\"\n for p in range(min_port, max_port + 1):\n try:\n s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n s.bind((\"localhost\", p))\n s.close()\n return p\n except socket.error:\n pass\n\n raise Exception(\"No available port found in range [%d, %d]\" % (min_port, max_port))\n" }, { "path": "tests/tests/__init__.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n" }, { "path": "tests/tests/check_session.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.tests.session import generate_results_dir, SessionContext\nfrom ducktape.cluster.localhost import LocalhostCluster\n\nimport os.path\nimport pickle\nimport shutil\nimport tempfile\n\n\nclass CheckGenerateResultsDir(object):\n def setup_method(self, _):\n self.tempdir = tempfile.mkdtemp()\n\n def check_generate_results_root(self):\n \"\"\"Check the generated results directory has the specified path as its root\"\"\"\n results_root = os.path.abspath(tempfile.mkdtemp())\n results_dir = generate_results_dir(results_root, \"my_session_id\")\n assert results_dir.find(results_root) == 0\n\n def check_pickleable(self):\n \"\"\"Check that session_context object is pickleable\n This is necessary so that SessionContext objects can be shared between processes when using python\n multiprocessing module.\n \"\"\"\n\n kwargs = {\n \"session_id\": \"hello-123\",\n \"results_dir\": self.tempdir,\n \"cluster\": LocalhostCluster(),\n \"globals\": {},\n }\n session_context = SessionContext(**kwargs)\n pickle.dumps(session_context)\n\n def teardown_method(self, _):\n if os.path.exists(self.tempdir):\n shutil.rmtree(self.tempdir)\n" }, { "path": "tests/tests/check_test.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\nimport os\nimport random\nimport shutil\nimport sys\nimport tempfile\n\nfrom ducktape.cluster.cluster_spec import ClusterSpec\nfrom ducktape.tests.test import Test, _compress_cmd, in_dir, in_temp_dir\nfrom ducktape.tests.test_context import _escape_pathname, TestContext\nfrom tests import ducktape_mock\n\n\nclass DummyTest(Test):\n \"\"\"class description\"\"\"\n\n def test_class_description(self):\n pass\n\n def test_function_description(self):\n \"\"\"function description\"\"\"\n pass\n\n\nclass DummyTestNoDescription(Test):\n def test_this(self):\n pass\n\n\nclass CheckLifecycle(object):\n def check_test_context_double_close(self):\n context = TestContext(\n session_context=ducktape_mock.session_context(),\n cls=DummyTest,\n function=DummyTest.test_function_description,\n )\n context.close()\n context.close()\n assert not hasattr(context, \"services\")\n\n def check_cluster_property(self):\n exp_cluster = ClusterSpec.simple_linux(5)\n tc = TestContext(\n session_context=ducktape_mock.session_context(),\n cluster=exp_cluster,\n cls=DummyTest,\n function=DummyTest.test_function_description,\n )\n test_obj = tc.cls(tc)\n assert test_obj.cluster == exp_cluster\n\n\nclass CheckEscapePathname(object):\n def check_illegal_path(self):\n path = \"\\\\/.a=2, b=x/y/z\"\n assert _escape_pathname(path) == \"a=2.b=x.y.z\"\n\n def check_negative(self):\n # it's better if negative numbers are preserved\n path = \"x= -2, y=-50\"\n assert _escape_pathname(path) == \"x=-2.y=-50\"\n\n def check_many_dots(self):\n path = \"..a.....b.c...d.\"\n assert _escape_pathname(path) == \"a.b.c.d\"\n\n\nclass CheckDescription(object):\n \"\"\"Check that pulling a description from a test works as expected.\"\"\"\n\n def check_from_function(self):\n \"\"\"If the function has a docstring, the description should come from the function\"\"\"\n context = TestContext(\n session_context=ducktape_mock.session_context(),\n cls=DummyTest,\n function=DummyTest.test_function_description,\n )\n assert context.description == \"function description\"\n\n def check_from_class(self):\n \"\"\"If the test method has no docstring, description should come from the class docstring\"\"\"\n context = TestContext(\n session_context=ducktape_mock.session_context(),\n cls=DummyTest,\n function=DummyTest.test_class_description,\n )\n assert context.description == \"class description\"\n\n def check_no_description(self):\n \"\"\"If nobody has a docstring, there shouldn't be an error, and description should be empty string\"\"\"\n context = TestContext(\n session_context=ducktape_mock.session_context(),\n cls=DummyTestNoDescription,\n function=DummyTestNoDescription.test_this,\n )\n assert context.description == \"\"\n\n\nclass CheckCompressCmd(object):\n \"\"\"Check expected behavior of compress command used before collecting service logs\"\"\"\n\n def setup_method(self, _):\n self.tempdir = tempfile.mkdtemp()\n\n def _make_random_file(self, dir, num_chars=10000):\n \"\"\"Populate filename with random characters.\"\"\"\n filename = os.path.join(dir, \"f-%d\" % random.randint(1, 2**63 - 1))\n content = \"\".join([random.choice(\"0123456789abcdefghijklmnopqrstuvwxyz\\n\") for _ in range(num_chars)])\n with open(filename, \"w\") as f:\n f.writelines(content)\n return filename\n\n def _make_files(self, dir, num_files=10):\n \"\"\"Populate dir with several files with random characters.\"\"\"\n for i in range(num_files):\n self._make_random_file(dir)\n\n def _validate_compressed(self, uncompressed_path):\n if uncompressed_path.endswith(os.path.sep):\n uncompressed_path = uncompressed_path[: -len(os.path.sep)]\n\n compressed_path = uncompressed_path + \".tgz\"\n\n # verify original file is replaced by filename.tgz\n assert os.path.exists(compressed_path)\n assert not os.path.exists(uncompressed_path)\n\n # verify that uncompressing gets us back the original\n with in_dir(self.tempdir):\n os.system(\"tar xzf %s\" % (compressed_path))\n assert os.path.exists(uncompressed_path)\n\n def check_compress_service_logs_swallow_error(self):\n \"\"\"Try compressing a non-existent service log, and check that it logs a message without throwing an error.\"\"\"\n from tests.ducktape_mock import session_context\n\n tc = TestContext(\n session_context=session_context(),\n module=sys.modules[DummyTestNoDescription.__module__],\n cls=DummyTestNoDescription,\n function=DummyTestNoDescription.test_this,\n )\n\n tc._logger = logging.getLogger(__name__)\n temp_log_file = tempfile.NamedTemporaryFile(delete=False).name\n\n try:\n tmp_log_handler = logging.FileHandler(temp_log_file)\n tc._logger.addHandler(tmp_log_handler)\n\n test_obj = tc.cls(tc)\n\n # Expect an error to be triggered but swallowed\n test_obj.compress_service_logs(node=None, service=None, node_logs=[\"hi\"])\n\n tmp_log_handler.close()\n with open(temp_log_file, \"r\") as f:\n s = f.read()\n assert s.find(\"Error compressing log hi\") >= 0\n finally:\n if os.path.exists(temp_log_file):\n os.remove(temp_log_file)\n\n def check_abs_path_file(self):\n \"\"\"Check compress command on an absolute path to a file\"\"\"\n with in_temp_dir() as dir1:\n filename = self._make_random_file(self.tempdir)\n abspath_filename = os.path.abspath(filename)\n\n # since we're using absolute path to file, we should be able to run the compress command from anywhere\n with in_temp_dir() as dir2:\n assert dir1 != dir2\n\n os.system(_compress_cmd(abspath_filename))\n self._validate_compressed(abspath_filename)\n\n def check_relative_path_file(self):\n \"\"\"Check compress command on a relative path to a file\"\"\"\n with in_temp_dir():\n filename = self._make_random_file(self.tempdir)\n with in_dir(self.tempdir):\n filename = os.path.basename(filename)\n assert len(filename.split(os.path.sep)) == 1\n\n # compress it!\n os.system(_compress_cmd(filename))\n self._validate_compressed(filename)\n\n def check_abs_path_dir(self):\n \"\"\"Validate compress command with absolute path to a directory\"\"\"\n dirname = tempfile.mkdtemp(dir=self.tempdir)\n dirname = os.path.abspath(dirname)\n self._make_files(dirname)\n\n # compress it!\n if not dirname.endswith(os.path.sep):\n # extra check - ensure all of this works with trailing '/'\n dirname += os.path.sep\n\n os.system(_compress_cmd(dirname))\n self._validate_compressed(dirname)\n\n def check_relative_path_dir(self):\n \"\"\"Validate tarball compression of a directory\"\"\"\n dirname = tempfile.mkdtemp(dir=self.tempdir)\n self._make_files(dirname)\n\n dirname = os.path.basename(dirname)\n assert len(dirname.split(os.path.sep)) == 1\n\n # compress it!\n with in_dir(self.tempdir):\n os.system(_compress_cmd(dirname))\n self._validate_compressed(dirname)\n\n def teardown_method(self, _):\n if os.path.exists(self.tempdir):\n shutil.rmtree(self.tempdir)\n" }, { "path": "tests/tests/check_test_context.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom ducktape.tests.test import Test\nfrom ducktape.services.service import Service\nfrom ducktape.mark import parametrize\nfrom ducktape.mark.mark_expander import MarkedFunctionExpander\n\nfrom tests.ducktape_mock import session_context\n\nfrom mock import MagicMock\n\n\nclass CheckTestContext(object):\n def check_copy_constructor(self):\n \"\"\"Regression test against a bug introduced in 0.3.7\n The TestContext copy constructor was copying the ServiceRegistry object by reference.\n As a result, services registering themselves with one test context would be registered with the copied\n context as well, resulting in the length of the service registry to grow additively from test to test.\n\n This problem cropped up in particular with parametrized tests.\n \"\"\"\n expander = MarkedFunctionExpander(\n session_context=session_context(),\n cls=DummyTest,\n function=DummyTest.test_me,\n cluster=MagicMock(),\n )\n ctx_list = expander.expand()\n\n for ctx in ctx_list:\n # Constructing an instance of the test class causes a service to be registered with the test context\n ctx.cls(ctx)\n\n # Ensure that each context.services object is a unique reference\n assert len(set(id(ctx.services) for ctx in ctx_list)) == len(ctx_list)\n\n\nclass DummyTest(Test):\n def __init__(self, test_context):\n super(DummyTest, self).__init__(test_context)\n self.service = DummyService(test_context)\n\n @parametrize(x=1)\n @parametrize(x=2)\n def test_me(self):\n pass\n\n\nclass DummyService(Service):\n def __init__(self, context):\n super(DummyService, self).__init__(context, 1)\n" }, { "path": "tests/utils/__init__.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n" }, { "path": "tests/utils/check_util.py", "content": "# Copyright 2015 Confluent Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nimport pytest\n\nfrom ducktape.errors import TimeoutError\nfrom ducktape.utils.util import wait_until\nimport time\n\n\nclass CheckUtils(object):\n def check_wait_until(self):\n \"\"\"Check normal wait until behavior\"\"\"\n start = time.time()\n\n wait_until(lambda: time.time() > start + 0.5, timeout_sec=2, backoff_sec=0.1)\n\n def check_wait_until_timeout(self):\n \"\"\"Check that timeout throws exception\"\"\"\n start = time.time()\n\n with pytest.raises(TimeoutError, match=\"Hello world\"):\n wait_until(\n lambda: time.time() > start + 5,\n timeout_sec=0.5,\n backoff_sec=0.1,\n err_msg=\"Hello world\",\n )\n\n def check_wait_until_timeout_callable_msg(self):\n \"\"\"Check that timeout throws exception and the error message is generated via a callable\"\"\"\n start = time.time()\n\n with pytest.raises(TimeoutError, match=\"Hello world\"):\n wait_until(\n lambda: time.time() > start + 5,\n timeout_sec=0.5,\n backoff_sec=0.1,\n err_msg=lambda: \"Hello world\",\n )\n\n def check_wait_until_with_exception(self):\n def condition_that_raises():\n raise Exception(\"OG\")\n\n with pytest.raises(TimeoutError) as exc_info:\n wait_until(\n condition_that_raises,\n timeout_sec=0.5,\n backoff_sec=0.1,\n err_msg=\"Hello world\",\n retry_on_exc=True,\n )\n exc_chain = exc_info.getrepr(chain=True).chain\n # 2 exceptions in the chain - OG and Hello world\n assert len(exc_chain) == 2\n\n # each element of a chain is a tuple of traceback, \"crash\" (which is the short message)\n # and optionally descr, which is None for the bottom of the chain\n # and \"The exception above is ...\" for all the others\n # We're interested in crash, since that one is a one-liner with the actual exception message.\n og_message = str(exc_chain[0][1])\n hello_message = str(exc_chain[1][1])\n assert \"OG\" in og_message\n assert \"Hello world\" in hello_message\n\n def check_wait_until_with_exception_on_first_step_only_but_still_fails(self):\n start = time.time()\n\n def condition_that_raises_before_3():\n if time.time() < start + 0.3:\n raise Exception(\"OG\")\n else:\n return False\n\n with pytest.raises(TimeoutError) as exc_info:\n wait_until(\n condition_that_raises_before_3,\n timeout_sec=0.5,\n backoff_sec=0.1,\n err_msg=\"Hello world\",\n retry_on_exc=True,\n )\n exc_chain = exc_info.getrepr(chain=True).chain\n assert len(exc_chain) == 1\n hello_message = str(exc_chain[0][1])\n assert \"Hello world\" in hello_message\n\n def check_wait_until_exception_which_succeeds_eventually(self):\n start = time.time()\n\n def condition_that_raises_before_3_but_then_succeeds():\n if time.time() < start + 0.3:\n raise Exception(\"OG\")\n else:\n return True\n\n wait_until(\n condition_that_raises_before_3_but_then_succeeds,\n timeout_sec=0.5,\n backoff_sec=0.1,\n err_msg=\"Hello world\",\n retry_on_exc=True,\n )\n\n def check_wait_until_breaks_early_on_exception(self):\n def condition_that_raises():\n raise Exception(\"OG\")\n\n with pytest.raises(Exception, match=\"OG\") as exc_info:\n wait_until(\n condition_that_raises,\n timeout_sec=0.5,\n backoff_sec=0.1,\n err_msg=\"Hello world\",\n )\n assert \"Hello world\" not in str(exc_info)\n" }, { "path": "tox.ini", "content": "[tox]\nenvlist = py38, py39, py310, py311, py312, py313, cover, style, docs\n\n[testenv]\n# Consolidate all deps here instead of separately in test/style/cover so we\n# have a single env to work with, which makes debugging easier (like which env?).\n# Not as clean but easier to work with during development, which is better.\ndeps =\n -r requirements.txt\n -r requirements-test.txt\ninstall_command =\n pip install -U {packages}\nrecreate = False\nskipsdist = True\nusedevelop = True\nsetenv =\n PIP_PROCESS_DEPENDENCY_LINKS=1\n PIP_DEFAULT_TIMEOUT=60\n ARCHFLAGS=-Wno-error=unused-command-line-argument-hard-error-in-future\nenvdir = {package_root}/.virtualenvs/ducktape_{envname}\ncommands =\n pytest {env:PYTESTARGS:} {posargs}\n\n[testenv:py38]\nenvdir = {package_root}/.virtualenvs/ducktape-py38\n\n[testenv:py39]\nenvdir = {package_root}/.virtualenvs/ducktape-py39\n\n[testenv:py310]\nenvdir = {package_root}/.virtualenvs/ducktape-py310\n\n[testenv:py311]\nenvdir = {package_root}/.virtualenvs/ducktape-py311\n\n[testenv:py312]\nenvdir = {package_root}/.virtualenvs/ducktape-py312\n\n[testenv:py313]\nenvdir = {package_root}/.virtualenvs/ducktape-py313\n\n[testenv:style]\nbasepython = python3.13\nenvdir = {package_root}/.virtualenvs/ducktape\ncommands =\n ruff format --check\n ruff check\n\n[testenv:ruff]\nbasepython = python3.13\nenvdir = {package_root}/.virtualenvs/ducktape\ncommands =\n ruff {posargs}\n\n[testenv:cover]\nbasepython = python3.13\nenvdir = {package_root}/.virtualenvs/ducktape\ncommands =\n pytest {env:PYTESTARGS:} --cov ducktape --cov-report=xml --cov-report=html --cov-report=term --cov-report=annotate:textcov \\\n --cov-fail-under=70\n\n[testenv:docs]\nbasepython = python3.13\ndeps =\n -r {toxinidir}/docs/requirements.txt\nchangedir = {toxinidir}/docs\ncommands = sphinx-build -M {env:SPHINX_BUILDER:html} . _build {posargs}\n" } ]