[ { "path": ".coveragerc", "content": "[run]\nsource=.\nomit=./env/*\n" }, { "path": ".dockerignore", "content": "dist\nenv\nvenv\n*.egg-info\nlogs\n.vscode\n.pytest_cache\n" }, { "path": ".github/dependabot.yml", "content": "version: 2\nupdates:\n - package-ecosystem: \"github-actions\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"\n open-pull-requests-limit: 5\n" }, { "path": ".github/workflows/package-verification.yml", "content": "# This workflow will install Python dependencies, run tests and lint with a variety of Python versions\n# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python\n\nname: Run Package Verifications\n\non:\n push:\n branches: [ \"master\" ]\n paths-ignore:\n - \"*.md\"\n\n pull_request:\n branches: [ \"master\" ]\n paths-ignore:\n - \"*.md\"\n\njobs:\n lint:\n runs-on: ubuntu-latest\n strategy:\n fail-fast: false\n matrix:\n python-version: [\"3.8\", \"3.9\", \"3.10\", \"3.11\", \"3.12\", \"3.13\", \"3.14\"]\n\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n - name: Set up Python ${{ matrix.python-version }}\n uses: actions/setup-python@v6\n with:\n python-version: ${{ matrix.python-version }}\n - name: Install dependencies\n run: |\n python -m pip install --upgrade pip\n python -m pip install flake8 flake8-pyproject ruff\n - name: Lint with flake8\n run: |\n flake8 .\n - name: Lint with ruff\n run: |\n ruff check .\n\n build-check:\n runs-on: ubuntu-latest\n needs: lint\n steps:\n - name: Checkout\n uses: actions/checkout@v6\n \n - name: Set up Python 3.12\n uses: actions/setup-python@v6\n with:\n python-version: \"3.12\"\n \n - name: Install build tools\n run: |\n python -m pip install --upgrade pip\n python -m pip install build twine\n\n - name: Build package\n run: |\n python -m build\n\n - name: Check package metadata\n run: |\n twine check dist/*\n\n test:\n runs-on: ubuntu-latest\n needs: build-check\n strategy:\n fail-fast: false\n matrix:\n include:\n - platform: \"std2-all\"\n python: \"3.7\"\n postgres: \"17\"\n - platform: \"std2-all\"\n python: \"3.8.0\"\n postgres: \"17\"\n - platform: \"std2-all\"\n python: \"3.8\"\n postgres: \"17\"\n - platform: \"std2-all\"\n python: \"3.9\"\n postgres: \"17\"\n - platform: \"std2-all\"\n python: \"3.10\"\n postgres: \"17\"\n - platform: \"std2-all\"\n python: \"3.11\"\n postgres: \"17\"\n - platform: \"std2-all\"\n python: \"3.12\"\n postgres: \"17\"\n - platform: \"std2-all\"\n python: \"3.13\"\n postgres: \"17\"\n - platform: \"std2-all\"\n python: \"3.14\"\n postgres: \"17\"\n - platform: \"std\"\n python: \"3\"\n postgres: \"10\"\n - platform: \"std\"\n python: \"3\"\n postgres: \"11\"\n - platform: \"std\"\n python: \"3\"\n postgres: \"12\"\n - platform: \"std\"\n python: \"3\"\n postgres: \"13\"\n - platform: \"std\"\n python: \"3\"\n postgres: \"14\"\n - platform: \"std\"\n python: \"3\"\n postgres: \"15\"\n - platform: \"std\"\n python: \"3\"\n postgres: \"16\"\n - platform: \"std-all\"\n python: \"3\"\n postgres: \"17\"\n - platform: \"std-all\"\n python: \"3\"\n postgres: \"18\"\n - platform: \"ubuntu_24_04\"\n python: \"3\"\n postgres: \"17\"\n - platform: \"altlinux_10\"\n python: \"3\"\n postgres: \"17\"\n - platform: \"altlinux_11\"\n python: \"3\"\n postgres: \"17\"\n - platform: \"astralinux_1_7\"\n python: \"3\"\n postgres: \"17\"\n\n env:\n BASE_SIGN: \"${{ matrix.platform }}-py${{ matrix.python }}-pg${{ matrix.postgres }}\"\n\n steps:\n - name: Prepare variables\n run: |\n echo \"RUN_CFG__NOW=$(date +'%Y%m%d_%H%M%S')\" >> $GITHUB_ENV\n echo \"RUN_CFG__LOGS_DIR=logs-${{ env.BASE_SIGN }}\" >> $GITHUB_ENV\n echo \"RUN_CFG__DOCKER_IMAGE_NAME=tests-${{ env.BASE_SIGN }}\" >> $GITHUB_ENV\n echo \"---------- [$GITHUB_ENV]\"\n cat $GITHUB_ENV\n - name: Checkout\n uses: actions/checkout@v6\n - name: Prepare logs folder on the host\n run: mkdir -p \"${{ env.RUN_CFG__LOGS_DIR }}\"\n - name: Adjust logs folder permission\n run: chmod -R 777 \"${{ env.RUN_CFG__LOGS_DIR }}\"\n - name: Build local image ${{ matrix.alpine }}\n run: docker build --build-arg PG_VERSION=\"${{ matrix.postgres }}\" --build-arg PYTHON_VERSION=\"${{ matrix.python }}\" -t \"${{ env.RUN_CFG__DOCKER_IMAGE_NAME }}\" -f Dockerfile--${{ matrix.platform }}.tmpl .\n - name: Run\n run: docker run $(bash <(curl -s https://codecov.io/env)) -t -v ${{ github.workspace }}/${{ env.RUN_CFG__LOGS_DIR }}:/home/test/testgres/logs \"${{ env.RUN_CFG__DOCKER_IMAGE_NAME }}\"\n - name: Upload Logs\n uses: actions/upload-artifact@v7\n if: always() # IT IS IMPORTANT!\n with:\n name: testgres--test_logs--${{ env.RUN_CFG__NOW }}-${{ env.BASE_SIGN }}-id${{ github.run_id }}\n path: \"${{ env.RUN_CFG__LOGS_DIR }}/\"\n" }, { "path": ".github/workflows/python-publish.yml", "content": "# This workflow will upload a Python Package to PyPI when a release is created\n# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries\n\n# This workflow uses actions that are not certified by GitHub.\n# They are provided by a third-party and are governed by\n# separate terms of service, privacy policy, and support\n# documentation.\n\nname: Upload Python Package\n\non:\n release:\n types: [published]\n\npermissions:\n contents: read\n\njobs:\n release-build:\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v6\n\n - uses: actions/setup-python@v6\n with:\n python-version: \"3.x\"\n\n - name: Build release distributions\n run: |\n # NOTE: put your own distribution build steps here.\n python -m pip install build\n python -m build\n\n - name: Upload distributions\n uses: actions/upload-artifact@v7\n with:\n name: release-dists\n path: dist/\n\n pypi-publish:\n runs-on: ubuntu-latest\n needs:\n - release-build\n permissions:\n # IMPORTANT: this permission is mandatory for trusted publishing\n id-token: write\n\n # Dedicated environments with protections for publishing are strongly recommended.\n # For more information, see: https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#deployment-protection-rules\n environment:\n name: pypi\n # OPTIONAL: uncomment and update to include your PyPI project URL in the deployment status:\n # url: https://pypi.org/p/YOURPROJECT\n #\n # ALTERNATIVE: if your GitHub Release name is the PyPI project version string\n # ALTERNATIVE: exactly, uncomment the following line instead:\n # url: https://pypi.org/project/YOURPROJECT/${{ github.event.release.name }}\n\n steps:\n - name: Retrieve release distributions\n uses: actions/download-artifact@v8\n with:\n name: release-dists\n path: dist/\n\n - name: Publish release distributions to PyPI\n uses: pypa/gh-action-pypi-publish@release/v1\n with:\n packages-dir: dist/\n" }, { "path": ".gitignore", "content": "*.pyc\n*.egg\n*.egg-info/\n.eggs/\ndist/\nbuild/\ndocs/build/\nlogs/\n\nenv/\nvenv/\n\n.coverage\ncoverage.xml\n\nDockerfile\n\n*~\n*.swp\ntags\n" }, { "path": ".style.yapf", "content": "[style]\nbased_on_style = pep8\nspaces_before_comment = 4\nsplit_before_logical_operator = false\ncolumn_limit=80\n" }, { "path": "Dockerfile--altlinux_10.tmpl", "content": "ARG PG_VERSION\nARG PYTHON_VERSION \n\n# --------------------------------------------- base1\nFROM alt:p10 AS base1\n\nRUN apt-get update\nRUN apt-get install -y sudo curl ca-certificates\nRUN apt-get update\nRUN apt-get install -y openssh-server openssh-clients\nRUN apt-get install -y time\n\n# RUN apt-get install -y mc\n\nRUN apt-get install -y libsqlite3-devel\n\nEXPOSE 22\n\nRUN ssh-keygen -A\n\n# --------------------------------------------- postgres\nFROM base1 AS base1_with_dev_tools\n\nRUN apt-get update\n\nRUN apt-get install -y git\nRUN apt-get install -y gcc\nRUN apt-get install -y make\n\nRUN apt-get install -y meson\nRUN apt-get install -y flex\nRUN apt-get install -y bison\n\nRUN apt-get install -y pkg-config\nRUN apt-get install -y libssl-devel\nRUN apt-get install -y libicu-devel\nRUN apt-get install -y libzstd-devel\nRUN apt-get install -y zlib-devel\nRUN apt-get install -y liblz4-devel\nRUN apt-get install -y libzstd-devel\nRUN apt-get install -y libxml2-devel\n\n# --------------------------------------------- postgres\nFROM base1_with_dev_tools AS base1_with_pg-17\n\nRUN git clone https://github.com/postgres/postgres.git -b REL_17_STABLE /pg/postgres/source\n\nWORKDIR /pg/postgres/source\n\nRUN ./configure --prefix=/pg/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml\nRUN make -j 4 install\nRUN make -j 4 -C contrib install\n\n# SETUP PG_CONFIG\n# When pg_config symlink in /usr/local/bin it returns a real (right) result of --bindir\nRUN ln -s /pg/postgres/install/bin/pg_config -t /usr/local/bin\n\n# SETUP PG CLIENT LIBRARY\n# libpq.so.5 is enough\nRUN ln -s /pg/postgres/install/lib/libpq.so.5.17 /usr/lib64/libpq.so.5\n\n# --------------------------------------------- base2_with_python-3\nFROM base1_with_pg-${PG_VERSION} AS base2_with_python-3\nRUN apt-get install -y python3\nRUN apt-get install -y python3-dev\nRUN apt-get install -y python3-modules-sqlite3\n\nENV PYTHON_BINARY=python3\n\n# --------------------------------------------- final\nFROM base2_with_python-${PYTHON_VERSION} AS final\n\nRUN adduser test -G wheel\n\n# It enables execution of \"sudo service ssh start\" without password\nRUN echo \"test ALL=(ALL:ALL) NOPASSWD: ALL\" >> /etc/sudoers\n\nADD --chown=test:test . /home/test/testgres\nWORKDIR /home/test/testgres\nRUN mkdir /home/test/testgres/logs\nRUN chown -R test:test /home/test/testgres/logs\n\nENV LANG=C.UTF-8\n\nUSER test\n\nRUN chmod 700 ~/\nRUN mkdir -p ~/.ssh\n\n#\n# Altlinux 10 and 11 too slowly create a new SSH connection (x6).\n#\n\nENTRYPOINT sh -c \" \\\nset -eux; \\\necho HELLO FROM ENTRYPOINT; \\\necho HOME DIR IS [`realpath ~/`]; \\\nls -la .; \\\nsudo /usr/sbin/sshd; \\\nsudo chmod 777 /home/test/testgres/logs; \\\nls -la . | grep logs; \\\nssh-keyscan -H localhost >> ~/.ssh/known_hosts; \\\nssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \\\nssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \\\ncat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \\\nchmod 600 ~/.ssh/authorized_keys; \\\nls -la ~/.ssh/; \\\nTEST_FILTER=\\\"\\\" bash ./run_tests.sh;\"\n" }, { "path": "Dockerfile--altlinux_11.tmpl", "content": "ARG PG_VERSION\nARG PYTHON_VERSION \n\n# --------------------------------------------- base1\nFROM alt:p11 AS base1\n\nRUN apt-get update\nRUN apt-get install -y sudo curl ca-certificates\nRUN apt-get update\nRUN apt-get install -y openssh-server openssh-clients\nRUN apt-get install -y time\n\n# pgrep (for testgres.os_ops)\nRUN apt-get install -y procps\n\n# RUN apt-get install -y mc\n\nRUN apt-get install -y libsqlite3-devel\n\nEXPOSE 22\n\nRUN ssh-keygen -A\n\n# --------------------------------------------- postgres\nFROM base1 AS base1_with_dev_tools\n\nRUN apt-get update\n\nRUN apt-get install -y git\nRUN apt-get install -y gcc\nRUN apt-get install -y make\n\nRUN apt-get install -y meson\nRUN apt-get install -y flex\nRUN apt-get install -y bison\n\nRUN apt-get install -y pkg-config\nRUN apt-get install -y libssl-devel\nRUN apt-get install -y libicu-devel\nRUN apt-get install -y libzstd-devel\nRUN apt-get install -y zlib-devel\nRUN apt-get install -y liblz4-devel\nRUN apt-get install -y libzstd-devel\nRUN apt-get install -y libxml2-devel\n\n# --------------------------------------------- postgres\nFROM base1_with_dev_tools AS base1_with_pg-17\n\nRUN git clone https://github.com/postgres/postgres.git -b REL_17_STABLE /pg/postgres/source\n\nWORKDIR /pg/postgres/source\n\nRUN ./configure --prefix=/pg/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml\nRUN make -j 4 install\nRUN make -j 4 -C contrib install\n\n# SETUP PG_CONFIG\n# When pg_config symlink in /usr/local/bin it returns a real (right) result of --bindir\nRUN ln -s /pg/postgres/install/bin/pg_config -t /usr/local/bin\n\n# SETUP PG CLIENT LIBRARY\n# libpq.so.5 is enough\nRUN ln -s /pg/postgres/install/lib/libpq.so.5.17 /usr/lib64/libpq.so.5\n\n# --------------------------------------------- base2_with_python-3\nFROM base1_with_pg-${PG_VERSION} AS base2_with_python-3\nRUN apt-get install -y python3\nRUN apt-get install -y python3-dev\nRUN apt-get install -y python3-modules-sqlite3\n\nENV PYTHON_BINARY=python3\n\n# --------------------------------------------- final\nFROM base2_with_python-${PYTHON_VERSION} AS final\n\nRUN adduser test -G wheel\n\n# It enables execution of \"sudo service ssh start\" without password\nRUN echo \"test ALL=(ALL:ALL) NOPASSWD: ALL\" >> /etc/sudoers\n\nADD --chown=test:test . /home/test/testgres\nWORKDIR /home/test/testgres\nRUN mkdir /home/test/testgres/logs\nRUN chown -R test:test /home/test/testgres/logs\n\nENV LANG=C.UTF-8\n\nUSER test\n\nRUN chmod 700 ~/\nRUN mkdir -p ~/.ssh\n\n#\n# Altlinux 10 and 11 too slowly create a new SSH connection (x6).\n#\n\nENTRYPOINT sh -c \" \\\nset -eux; \\\necho HELLO FROM ENTRYPOINT; \\\necho HOME DIR IS [`realpath ~/`]; \\\nls -la .; \\\nsudo /usr/sbin/sshd; \\\nsudo chmod 777 /home/test/testgres/logs; \\\nls -la . | grep logs; \\\nssh-keyscan -H localhost >> ~/.ssh/known_hosts; \\\nssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \\\nssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \\\ncat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \\\nchmod 600 ~/.ssh/authorized_keys; \\\nls -la ~/.ssh/; \\\nTEST_FILTER=\\\"\\\" bash ./run_tests.sh;\"\n" }, { "path": "Dockerfile--astralinux_1_7.tmpl", "content": "ARG PG_VERSION\nARG PYTHON_VERSION \n\n# --------------------------------------------- base1\nFROM packpack/packpack:astra-1.7 AS base1\n\nRUN apt update\nRUN apt install -y sudo curl ca-certificates\nRUN apt update\nRUN apt install -y openssh-server\nRUN apt install -y time\n# RUN apt install -y netcat-traditional\n\nRUN apt install -y git\n\n# --------------------------------------------- postgres\nFROM base1 AS base1_with_dev_tools\n\nRUN apt-get update\n\nRUN apt-get install -y git\nRUN apt-get install -y gcc\nRUN apt-get install -y make\n\nRUN apt-get install -y meson\nRUN apt-get install -y flex\nRUN apt-get install -y bison\n\nRUN apt-get install -y pkg-config\nRUN apt-get install -y libssl-dev\nRUN apt-get install -y libicu-dev\nRUN apt-get install -y libzstd-dev\nRUN apt-get install -y zlib1g-dev\nRUN apt-get install -y liblz4-dev\nRUN apt-get install -y libzstd-dev\nRUN apt-get install -y libxml2-dev\n\n# --------------------------------------------- postgres\nFROM base1_with_dev_tools AS base1_with_pg-17\n\nRUN curl -fsSL https://ftp.postgresql.org/pub/source/v17.7/postgresql-17.7.tar.bz2 -o postgresql.tar.bz2 \\\n && mkdir -p /pg/postgres/source \\\n && tar -xjf postgresql.tar.bz2 -C /pg/postgres/source --strip-components=1 \\\n && rm postgresql.tar.bz2\n\nWORKDIR /pg/postgres/source\n\nRUN ./configure --prefix=/pg/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml\nRUN make -j 4 install\nRUN make -j 4 -C contrib install\n\n# SETUP PG_CONFIG\n# When pg_config symlink in /usr/local/bin it returns a real (right) result of --bindir\nRUN ln -s /pg/postgres/install/bin/pg_config -t /usr/local/bin\n\n# SETUP PG CLIENT LIBRARY\n# libpq.so.5 is enough\nRUN ln -s /pg/postgres/install/lib/libpq.so.5.17 /usr/lib/libpq.so.5\n\n# --------------------------------------------- base2_with_python-3\nFROM base1_with_pg-${PG_VERSION} AS base2_with_python-3\nRUN apt install -y python3 python3-dev python3-venv\n\nENV PYTHON_BINARY=python3\n\n# --------------------------------------------- final\nFROM base2_with_python-${PYTHON_VERSION} AS final\n\nEXPOSE 22\n\nRUN ssh-keygen -A\n\nRUN useradd -m test\n\n# It enables execution of \"sudo service ssh start\" without password\n# MY OLD:\n# RUN sh -c \"echo test ALL=NOPASSWD:ALL\" >> /etc/sudoers\n# AI:\nRUN echo \"test ALL=(ALL) NOPASSWD:ALL\" >> /etc/sudoers\n\n# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD\nRUN sh -c \"echo \"test:*\" | chpasswd -e\"\nRUN sed -i 's/UsePAM yes/UsePAM no/' /etc/ssh/sshd_config\n\nADD --chown=test:test . /home/test/testgres\nWORKDIR /home/test/testgres\nRUN mkdir /home/test/testgres/logs\nRUN chown -R test:test /home/test/testgres/logs\n\nENV LANG=C.UTF-8\n\nUSER test\n\nRUN chmod 700 ~/\nRUN mkdir -p ~/.ssh\nRUN chmod 700 ~/.ssh\n\nENTRYPOINT sh -c \" \\\nset -eux; \\\necho HELLO FROM ENTRYPOINT; \\\necho HOME DIR IS [`realpath ~/`]; \\\nls -la .; \\\nsudo service ssh start; \\\nsudo chmod 777 /home/test/testgres/logs; \\\nls -la . | grep logs; \\\nssh-keyscan -H localhost >> ~/.ssh/known_hosts; \\\nssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \\\nssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \\\ncat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \\\nchmod 600 ~/.ssh/authorized_keys; \\\nls -la ~/.ssh/; \\\nTEST_FILTER=\\\"\\\" bash ./run_tests.sh;\"\n" }, { "path": "Dockerfile--std-all.tmpl", "content": "ARG PG_VERSION\nARG PYTHON_VERSION \n\n# --------------------------------------------- base1\nFROM postgres:${PG_VERSION}-alpine AS base1\n\n# --------------------------------------------- base2_with_python-3\nFROM base1 AS base2_with_python-3\nRUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers\nENV PYTHON_BINARY=python3\n\n# --------------------------------------------- final\nFROM base2_with_python-${PYTHON_VERSION} AS final\n\n#RUN apk add --no-cache mc\nRUN apk add --no-cache git\n\n# Full version of \"ps\" command\nRUN apk add --no-cache procps\n\nRUN apk add --no-cache openssh\nRUN apk add --no-cache sudo\n\nENV LANG=C.UTF-8\n\nRUN addgroup -S sudo\nRUN adduser -D test\nRUN addgroup test sudo\n\nEXPOSE 22\nRUN ssh-keygen -A\n\nADD --chown=test:test . /home/test/testgres\nWORKDIR /home/test/testgres\nRUN mkdir /home/test/testgres/logs\nRUN chown -R test:test /home/test/testgres/logs\n\n# It allows to use sudo without password\nRUN echo \"test ALL=(ALL:ALL) NOPASSWD:ALL\" >> /etc/sudoers\n\n# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD\nRUN echo \"test:*\" | chpasswd -e\n\nUSER test\n\n# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD\nRUN chmod 700 ~/\n\nRUN mkdir -p ~/.ssh\n#RUN chmod 700 ~/.ssh\n\n#ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh\n\nENTRYPOINT sh -c \" \\\nset -eux; \\\necho HELLO FROM ENTRYPOINT; \\\necho HOME DIR IS [`realpath ~/`]; \\\nls -la .; \\\nssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \\\ncat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \\\nchmod 600 ~/.ssh/authorized_keys; \\\nls -la ~/.ssh/; \\\nsudo /usr/sbin/sshd; \\\nsudo chmod 777 /home/test/testgres/logs; \\\nls -la . | grep logs; \\\nssh-keyscan -H localhost >> ~/.ssh/known_hosts; \\\nssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \\\nTEST_FILTER=\\\"\\\" bash run_tests.sh;\"\n" }, { "path": "Dockerfile--std.tmpl", "content": "ARG PG_VERSION\nARG PYTHON_VERSION \n\n# --------------------------------------------- base1\nFROM postgres:${PG_VERSION}-alpine AS base1\n\n# --------------------------------------------- base2_with_python-3\nFROM base1 AS base2_with_python-3\nRUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers\nENV PYTHON_BINARY=python3\n\n# --------------------------------------------- final\nFROM base2_with_python-${PYTHON_VERSION} AS final\n\nRUN apk add --no-cache git\n\nRUN apk add --no-cache sudo\n\nENV LANG=C.UTF-8\n\nRUN addgroup -S sudo\nRUN adduser -D test\nRUN addgroup test sudo\n\nADD --chown=test:test . /home/test/testgres\nWORKDIR /home/test/testgres\nRUN mkdir /home/test/testgres/logs\nRUN chown -R test:test /home/test/testgres/logs\n\n# It allows to use sudo without password\nRUN echo \"test ALL=(ALL:ALL) NOPASSWD:ALL\" >> /etc/sudoers\n\nUSER test\nENTRYPOINT sh -c \" \\\nset -eux; \\\necho HELLO FROM ENTRYPOINT; \\\necho HOME DIR IS [`realpath ~/`]; \\\nls -la .; \\\nsudo chmod 777 /home/test/testgres/logs; \\\nls -la . | grep logs; \\\nbash run_tests.sh;\"\n" }, { "path": "Dockerfile--std2-all.tmpl", "content": "ARG PG_VERSION\nARG PYTHON_VERSION \n\n# --------------------------------------------- base1\nFROM postgres:${PG_VERSION}-alpine AS base1\n\n# --------------------------------------------- base2_with_python-3\nFROM base1 AS base2_with_python-3\nENV PYTHON_BINARY=python3\nRUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers\n\n# For pyenv\nRUN apk add patch\nRUN apk add git\nRUN apk add xz-dev\nRUN apk add zip\nRUN apk add zlib-dev\nRUN apk add libffi-dev\nRUN apk add readline-dev\nRUN apk add openssl openssl-dev\nRUN apk add sqlite-dev\nRUN apk add bzip2-dev\n\n# --------------------------------------------- base3_with_python-3.7\nFROM base2_with_python-3 AS base3_with_python-3.7\nENV PYTHON_VERSION=3.7\n\n# --------------------------------------------- base3_with_python-3.8.0\nFROM base2_with_python-3 AS base3_with_python-3.8.0\nENV PYTHON_VERSION=3.8.0\n\n# --------------------------------------------- base3_with_python-3.8\nFROM base2_with_python-3 AS base3_with_python-3.8\nENV PYTHON_VERSION=3.8\n\n# --------------------------------------------- base3_with_python-3.9\nFROM base2_with_python-3 AS base3_with_python-3.9\nENV PYTHON_VERSION=3.9\n\n# --------------------------------------------- base3_with_python-3.10\nFROM base2_with_python-3 AS base3_with_python-3.10\nENV PYTHON_VERSION=3.10\n\n# --------------------------------------------- base3_with_python-3.11\nFROM base2_with_python-3 AS base3_with_python-3.11\nENV PYTHON_VERSION=3.11\n\n# --------------------------------------------- base3_with_python-3.12\nFROM base2_with_python-3 AS base3_with_python-3.12\nENV PYTHON_VERSION=3.12\n\n# --------------------------------------------- base3_with_python-3.13\nFROM base2_with_python-3 AS base3_with_python-3.13\nENV PYTHON_VERSION=3.13\n\n# --------------------------------------------- base3_with_python-3.14\nFROM base2_with_python-3 AS base3_with_python-3.14\nENV PYTHON_VERSION=3.14\n\n# --------------------------------------------- final\nFROM base3_with_python-${PYTHON_VERSION} AS final\n\n#RUN apk add --no-cache mc\n\n# Full version of \"ps\" command\nRUN apk add --no-cache procps\n\nRUN apk add --no-cache openssh\nRUN apk add --no-cache sudo\n\nENV LANG=C.UTF-8\n\nRUN addgroup -S sudo\nRUN adduser -D test\nRUN addgroup test sudo\n\nEXPOSE 22\nRUN ssh-keygen -A\n\n# It allows to use sudo without password\nRUN echo \"test ALL=(ALL:ALL) NOPASSWD:ALL\" >> /etc/sudoers\n\n# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD\nRUN echo \"test:*\" | chpasswd -e\n\nUSER test\n\nRUN curl https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash\nRUN ~/.pyenv/bin/pyenv install ${PYTHON_VERSION}\n\nADD --chown=test:test . /home/test/testgres\nWORKDIR /home/test/testgres\nRUN mkdir /home/test/testgres/logs\nRUN chown -R test:test /home/test/testgres/logs\n\n# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD\nRUN chmod 700 ~/\n\nRUN mkdir -p ~/.ssh\n\nENTRYPOINT sh -c \" \\\nset -eux; \\\necho HELLO FROM ENTRYPOINT; \\\necho HOME DIR IS [`realpath ~/`]; \\\nls -la .; \\\nssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \\\ncat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \\\nchmod 600 ~/.ssh/authorized_keys; \\\nls -la ~/.ssh/; \\\nsudo /usr/sbin/sshd; \\\nsudo chmod 777 /home/test/testgres/logs; \\\nls -la . | grep logs; \\\nssh-keyscan -H localhost >> ~/.ssh/known_hosts; \\\nssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \\\nexport PATH=\\\"~/.pyenv/bin:$PATH\\\"; \\\nTEST_FILTER=\\\"\\\" bash run_tests2.sh;\"\n" }, { "path": "Dockerfile--ubuntu_24_04.tmpl", "content": "ARG PG_VERSION\nARG PYTHON_VERSION \n\n# --------------------------------------------- base1\nFROM ubuntu:24.04 AS base1\nARG PG_VERSION\n\nRUN apt update\nRUN apt install -y sudo curl ca-certificates\nRUN apt update\nRUN apt install -y openssh-server\nRUN apt install -y time\nRUN apt install -y netcat-traditional\n\nRUN apt install -y git\n\nRUN apt update\nRUN apt install -y postgresql-common\n\nRUN bash /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y\n\nRUN install -d /usr/share/postgresql-common/pgdg\nRUN curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc\n\nRUN apt update\nRUN apt install -y postgresql-${PG_VERSION}\n\n# --------------------------------------------- base2_with_python-3\nFROM base1 AS base2_with_python-3\nRUN apt install -y python3 python3-dev python3-venv libpq-dev build-essential\nENV PYTHON_BINARY=python3\n\n# --------------------------------------------- final\nFROM base2_with_python-${PYTHON_VERSION} AS final\n\nEXPOSE 22\n\nRUN ssh-keygen -A\n\nRUN adduser test\nRUN chown postgres:postgres /var/run/postgresql\nRUN chmod 775 /var/run/postgresql\nRUN usermod -aG postgres test\n\n# It enables execution of \"sudo service ssh start\" without password\n# RUN echo \"test ALL=NOPASSWD:/usr/sbin/service ssh start\" >> /etc/sudoers\n# RUN echo \"test ALL=NOPASSWD:ALL\" >> /etc/sudoers\nRUN echo \"test ALL=(ALL) NOPASSWD:ALL\" >> /etc/sudoers\n\nADD --chown=test:test . /home/test/testgres\nWORKDIR /home/test/testgres\nRUN mkdir /home/test/testgres/logs\nRUN chown -R test:test /home/test/testgres/logs\n\nENV LANG=C.UTF-8\n\nUSER test\n\nRUN chmod 700 ~/\nRUN mkdir -p ~/.ssh\n\nENTRYPOINT sh -c \" \\\n#set -eux; \\\necho HELLO FROM ENTRYPOINT; \\\necho HOME DIR IS [`realpath ~/`]; \\\nls -la .; \\\nservice ssh enable; \\\nsudo service ssh start; \\\nsudo chmod 777 /home/test/testgres/logs; \\\nls -la . | grep logs; \\\nssh-keyscan -H localhost >> ~/.ssh/known_hosts; \\\nssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \\\nssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \\\ncat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \\\nchmod 600 ~/.ssh/authorized_keys; \\\nls -la ~/.ssh/; \\\nTEST_FILTER=\\\"\\\" bash ./run_tests.sh;\"\n" }, { "path": "LICENSE", "content": "testgres is released under the PostgreSQL License, a liberal Open Source license, similar to the BSD or MIT licenses.\r\n\r\nCopyright (c) 2016-2026, Postgres Professional\r\n\r\nPermission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this paragraph and the following two paragraphs appear in all copies.\r\n\r\nIN NO EVENT SHALL POSTGRES PROFESSIONAL BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF POSTGRES PROFESSIONAL HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n\r\nPOSTGRES PROFESSIONAL SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN \"AS IS\" BASIS, AND POSTGRES PROFESSIONAL HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\r\n" }, { "path": "README.md", "content": "[![CI Status](https://img.shields.io/github/actions/workflow/status/postgrespro/testgres/.github/workflows/package-verification.yml?label=CI)](https://github.com/postgrespro/testgres/actions/workflows/package-verification.yml)\n[![codecov](https://codecov.io/gh/postgrespro/testgres/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/testgres)\n[![PyPI package version](https://badge.fury.io/py/testgres.svg)](https://badge.fury.io/py/testgres)\n[![PyPI python versions](https://img.shields.io/pypi/pyversions/testgres)](https://pypi.org/project/testgres)\n[![PyPI downloads](https://img.shields.io/pypi/dm/testgres)](https://pypi.org/project/testgres)\n\n[Documentation](https://postgrespro.github.io/testgres/)\n\n# testgres\n\nUtility for orchestrating temporary PostgreSQL clusters in Python tests. Supports Python 3.7.3 and newer.\n\n## Installation\n\nInstall `testgres` from PyPI:\n\n```sh\npip install testgres\n```\n\nUse a dedicated virtual environment for isolated test dependencies.\n\n## Usage\n\n### Environment\n\n> Note: by default `testgres` invokes `initdb`, `pg_ctl`, and `psql` binaries found in `PATH`.\n\nSpecify a custom PostgreSQL installation in one of the following ways:\n\n- Set the `PG_CONFIG` environment variable to point to the `pg_config` executable.\n- Set the `PG_BIN` environment variable to point to the directory with PostgreSQL binaries.\n\nExample:\n\n```sh\nexport PG_BIN=$HOME/pg_16/bin\npython my_tests.py\n```\n\n### Examples\n\nCreate a temporary node, run queries, and let `testgres` clean up automatically:\n\n```python\n# create a node with a random name, port, and data directory\nwith testgres.get_new_node() as node:\n\n # run initdb\n node.init()\n\n # start PostgreSQL\n node.start()\n\n # execute a query in the default database\n print(node.execute('select 1'))\n\n# the node is stopped and its files are removed automatically\n```\n\n### Query helpers\n\n`testgres` provides four helpers for executing queries against the node:\n\n| Command | Description |\n|---------|-------------|\n| `node.psql(query, ...)` | Runs the query via `psql` and returns a tuple `(returncode, stdout, stderr)`. |\n| `node.safe_psql(query, ...)` | Same as `psql()` but returns only `stdout` and raises if the command fails. |\n| `node.execute(query, ...)` | Connects via `psycopg2` or `pg8000` (whichever is available) and returns a list of tuples. |\n| `node.connect(dbname, ...)` | Returns a `NodeConnection` wrapper for executing multiple statements within a transaction. |\n\nExample of transactional usage:\n\n```python\nwith node.connect() as con:\n con.begin('serializable')\n print(con.execute('select %s', 1))\n con.rollback()\n```\n\n### Logging\n\nBy default `cleanup()` removes all temporary files (data directories, logs, and so on) created by the API. Call `configure_testgres(node_cleanup_full=False)` before starting nodes if you want to keep logs for inspection.\n\n> Note: context managers (the `with` statement) call `stop()` and `cleanup()` automatically.\n\n`testgres` integrates with the standard [Python logging](https://docs.python.org/3/library/logging.html) module, so you can aggregate logs from multiple nodes:\n\n```python\nimport logging\n\n# write everything to /tmp/testgres.log\nlogging.basicConfig(filename='/tmp/testgres.log')\n\n# enable logging and create two nodes\ntestgres.configure_testgres(use_python_logging=True)\nnode1 = testgres.get_new_node().init().start()\nnode2 = testgres.get_new_node().init().start()\n\nnode1.execute('select 1')\nnode2.execute('select 2')\n\n# disable logging\ntestgres.configure_testgres(use_python_logging=False)\n```\n\nSee `tests/test_simple.py` for a complete logging example.\n\n### Backup and replication\n\nCreating backups and spawning replicas is straightforward:\n\n```python\nwith testgres.get_new_node('master') as master:\n master.init().start()\n\n with master.backup() as backup:\n replica = backup.spawn_replica('replica').start()\n replica.catchup()\n\n print(replica.execute('postgres', 'select 1'))\n```\n\n### Benchmarks\n\nUse `pgbench` through `testgres` to run quick benchmarks:\n\n```python\nwith testgres.get_new_node('master') as master:\n master.init().start()\n\n result = master.pgbench_init(scale=2).pgbench_run(time=10)\n print(result)\n```\n\n### Custom configuration\n\n`testgres` ships with sensible defaults. Adjust them as needed with `default_conf()` and `append_conf()`:\n\n```python\nextra_conf = \"shared_preload_libraries = 'postgres_fdw'\"\n\nwith testgres.get_new_node().init() as master:\n master.default_conf(fsync=True, allow_streaming=True)\n master.append_conf('postgresql.conf', extra_conf)\n```\n\n`default_conf()` is called by `init()` and rewrites the configuration file. Apply `append_conf()` afterwards to keep custom lines.\n\n### Remote mode\n\nYou can provision nodes on a remote host (Linux only) by wiring `RemoteOperations` into the configuration:\n\n```python\nfrom testgres import ConnectionParams, RemoteOperations, TestgresConfig, get_remote_node\n\nconn_params = ConnectionParams(\n host='example.com',\n username='postgres',\n ssh_key='/path/to/ssh/key'\n)\nos_ops = RemoteOperations(conn_params)\n\nTestgresConfig.set_os_ops(os_ops=os_ops)\n\ndef test_basic_query():\n with get_remote_node(conn_params=conn_params) as node:\n node.init().start()\n assert node.execute('SELECT 1') == [(1,)]\n```\n\n### Pytest integration\n\nUse fixtures to create and clean up nodes automatically when testing with `pytest`:\n\n```python\nimport pytest\nimport testgres\n\n@pytest.fixture\ndef pg_node():\n node = testgres.get_new_node().init().start()\n try:\n yield node\n finally:\n node.stop()\n node.cleanup()\n\ndef test_simple(pg_node):\n assert pg_node.execute('select 1')[0][0] == 1\n```\n\nThis pattern keeps tests concise and ensures that every node is stopped and removed even if the test fails.\n\n### Scaling tips\n\n- Run tests in parallel with `pytest -n auto` (requires `pytest-xdist`). Ensure each node uses a distinct port by setting `PGPORT` in the fixture or by passing the `port` argument to `get_new_node()`.\n- Always call `node.cleanup()` after each test, or rely on context managers/fixtures that do it for you, to avoid leftover data directories.\n- Prefer `node.safe_psql()` for lightweight assertions that should fail fast; use `node.execute()` when you need structured Python results.\n\n## Authors\n\n[Ildar Musin](https://github.com/zilder) \n[Dmitry Ivanov](https://github.com/funbringer) \n[Ildus Kurbangaliev](https://github.com/ildus) \n[Yury Zhuravlev](https://github.com/stalkerg)\n" }, { "path": "docs/Makefile", "content": "# Minimal makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHINXBUILD = sphinx-build\nSPHINXPROJ = testgres\nSOURCEDIR = source\nBUILDDIR = build\n\n# Put it first so that \"make\" without argument is like \"make help\".\nhelp:\n\t@$(SPHINXBUILD) -M help \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)\n\n.PHONY: help Makefile\n\n# Catch-all target: route all unknown targets to Sphinx using the new\n# \"make mode\" option. $(O) is meant as a shortcut for $(SPHINXOPTS).\n%: Makefile\n\t@pip install --force-reinstall ..\n\t@$(SPHINXBUILD) -M $@ \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)" }, { "path": "docs/README.md", "content": "# Building the documentation\n\nMake sure you have `Sphinx` package installed:\n\n```\npip install Sphinx\n```\n\nThen just run\n\n```\nmake html\n```\n\nDocumentation will be built in `build/html` directory. Other output formats are also available; run `make` without arguments to see the options.\n" }, { "path": "docs/source/conf.py", "content": "# -*- coding: utf-8 -*-\n#\n# Configuration file for the Sphinx documentation builder.\n#\n# This file does only contain a selection of the most common options. For a\n# full list see the documentation:\n# http://www.sphinx-doc.org/en/stable/config\n\n# -- Path setup --------------------------------------------------------------\n\n# If extensions (or modules to document with autodoc) are in another directory,\n# add these directories to sys.path here. If the directory is relative to the\n# documentation root, use os.path.abspath to make it absolute, like shown here.\n#\nimport os\nimport sys\nimport testgres\n\nassert testgres.__path__ is not None\nassert len(testgres.__path__) == 1\nassert type(testgres.__path__[0]) is str\np = os.path.dirname(testgres.__path__[0])\nassert type(p) is str\nsys.path.insert(0, os.path.abspath(p))\n\n# -- Project information -----------------------------------------------------\n\nproject = u'testgres'\npackage_name = u'testgres'\ncopyright = u'2016-2026, Postgres Professional'\nauthor = u'Postgres Professional'\n\n# The full version, including alpha/beta/rc tags\nrelease = testgres.__version__\n\n# The short X.Y version\nversion = '.'.join(release.split('.')[:2])\n\n# -- General configuration ---------------------------------------------------\n\n# If your documentation needs a minimal Sphinx version, state it here.\n#\n# needs_sphinx = '1.0'\n\n# Add any Sphinx extension module names here, as strings. They can be\n# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom\n# ones.\nextensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon']\n\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = ['_templates']\n\n# The suffix(es) of source filenames.\n# You can specify multiple suffix as a list of string:\n#\n# source_suffix = ['.rst', '.md']\nsource_suffix = '.rst'\n\n# The master toctree document.\nmaster_doc = 'index'\n\n# The language for content autogenerated by Sphinx. Refer to documentation\n# for a list of supported languages.\n#\n# This is also used if you do content translation via gettext catalogs.\n# Usually you set \"language\" from the command line for these cases.\nlanguage = \"en\"\n\n# List of patterns, relative to source directory, that match files and\n# directories to ignore when looking for source files.\n# This pattern also affects html_static_path and html_extra_path .\nexclude_patterns = []\n\n# The name of the Pygments (syntax highlighting) style to use.\npygments_style = 'sphinx'\n\n# -- Options for HTML output -------------------------------------------------\n\n# The theme to use for HTML and HTML Help pages. See the documentation for\n# a list of builtin themes.\n#\nhtml_theme = 'alabaster'\n\n# Theme options are theme-specific and customize the look and feel of a theme\n# further. For a list of options available for each theme, see the\n# documentation.\n#\n# html_theme_options = {}\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\nhtml_static_path = []\n\n# Custom sidebar templates, must be a dictionary that maps document names\n# to template names.\n#\n# The default sidebars (for documents that don't match any pattern) are\n# defined by theme itself. Builtin themes are using these templates by\n# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',\n# 'searchbox.html']``.\n#\n# html_sidebars = {}\n\n# -- Options for HTMLHelp output ---------------------------------------------\n\n# Output file base name for HTML help builder.\nhtmlhelp_basename = 'testgresdoc'\n\n# -- Options for LaTeX output ------------------------------------------------\n\nlatex_elements = {\n # The paper size ('letterpaper' or 'a4paper').\n #\n # 'papersize': 'letterpaper',\n\n # The font size ('10pt', '11pt' or '12pt').\n #\n # 'pointsize': '10pt',\n\n # Additional stuff for the LaTeX preamble.\n #\n # 'preamble': '',\n\n # Latex figure (float) alignment\n #\n # 'figure_align': 'htbp',\n}\n\n# Grouping the document tree into LaTeX files. List of tuples\n# (source start file, target name, title,\n# author, documentclass [howto, manual, or own class]).\nlatex_documents = [\n (master_doc, 'testgres.tex', u'testgres Documentation',\n u'Postgres Professional', 'manual'),\n]\n\n# -- Options for manual page output ------------------------------------------\n\n# One entry per manual page. List of tuples\n# (source start file, name, description, authors, manual section).\nman_pages = [(master_doc, 'testgres', u'testgres Documentation', [author], 1)]\n\n# -- Options for Texinfo output ----------------------------------------------\n\n# Grouping the document tree into Texinfo files. List of tuples\n# (source start file, target name, title, author,\n# dir menu entry, description, category)\ntexinfo_documents = [\n (master_doc, 'testgres', u'testgres Documentation', author, 'testgres',\n 'One line description of project.', 'Miscellaneous'),\n]\n\n# -- Extension configuration -------------------------------------------------\n" }, { "path": "docs/source/index.rst", "content": "\nTestgres documentation\n======================\n\nUtility for orchestrating temporary PostgreSQL clusters in Python tests. Supports Python 3.7.17 and newer.\n\nInstallation\n============\n\nTo install testgres, run:\n\n.. code-block:: bash\n\n pip install testgres\n\nWe encourage you to use ``virtualenv`` for your testing environment.\n\nUsage\n=====\n\nEnvironment\n-----------\n\nNote: by default ``testgres`` runs ``initdb``, ``pg_ctl``, and ``psql`` found in ``PATH``.\n\nThere are several ways to specify a custom PostgreSQL installation:\n\n- export ``PG_CONFIG`` environment variable pointing to the ``pg_config`` executable;\n- export ``PG_BIN`` environment variable pointing to the directory with executable files.\n\nExample:\n\n.. code-block:: bash\n\n export PG_BIN=$HOME/pg_16/bin\n python my_tests.py\n\nExamples\n--------\n\nCreate a temporary node, run queries, and let ``testgres`` clean up automatically:\n\n.. code-block:: python\n\n # create a node with a random name, port, and data directory\n with testgres.get_new_node() as node:\n\n # run initdb\n node.init()\n\n # start PostgreSQL\n node.start()\n\n # execute a query in the default database\n print(node.execute('select 1'))\n\n # the node is stopped and its files are removed automatically\n\nQuery helpers\n-------------\n\n``testgres`` provides four helpers for executing queries against the node:\n\n========================== =======================================================\nCommand Description\n========================== =======================================================\n``node.psql(query, ...)`` Runs the query via ``psql`` and returns ``(code, out, err)``.\n``node.safe_psql(...)`` Returns only ``stdout`` and raises if the command fails.\n``node.execute(...)`` Uses ``psycopg2``/``pg8000`` and returns a list of tuples.\n``node.connect(...)`` Returns a ``NodeConnection`` for transactional usage.\n========================== =======================================================\n\nExample:\n\n.. code-block:: python\n\n with node.connect() as con:\n con.begin('serializable')\n print(con.execute('select %s', 1))\n con.rollback()\n\nLogging\n-------\n\nBy default ``cleanup()`` removes all temporary files (data directories, logs, and so on) created by the API. Call ``configure_testgres(node_cleanup_full=False)`` before starting nodes if you want to keep logs for inspection.\n\nNote: context managers (the ``with`` statement) call ``stop()`` and ``cleanup()`` automatically.\n\n``testgres`` integrates with the standard `Python logging `_ module, so you can aggregate logs from multiple nodes:\n\n.. code-block:: python\n\n import logging\n\n logging.basicConfig(filename='/tmp/testgres.log')\n\n testgres.configure_testgres(use_python_logging=True)\n node1 = testgres.get_new_node().init().start()\n node2 = testgres.get_new_node().init().start()\n\n node1.execute('select 1')\n node2.execute('select 2')\n\n testgres.configure_testgres(use_python_logging=False)\n\nBackup & replication\n--------------------\n\nIt's quite easy to create a backup and start a new replica:\n\n.. code-block:: python\n\n with testgres.get_new_node('master') as master:\n master.init().start()\n\n # create a backup\n with master.backup() as backup:\n\n # create and start a new replica\n replica = backup.spawn_replica('replica').start()\n\n replica.catchup()\n\n print(replica.execute('postgres', 'select 1'))\n\nBenchmarks\n----------\n\nUse ``pgbench`` through ``testgres`` to run quick benchmarks:\n\n.. code-block:: python\n\n with testgres.get_new_node('master') as master:\n master.init().start()\n\n result = master.pgbench_init(scale=2).pgbench_run(time=10)\n print(result)\n\nCustom configuration\n--------------------\n\n``testgres`` ships with sensible defaults. Adjust them as needed with ``default_conf()`` and ``append_conf()``:\n\n.. code-block:: python\n\n extra_conf = \"shared_preload_libraries = 'postgres_fdw'\"\n\n with testgres.get_new_node().init() as master:\n master.default_conf(fsync=True, allow_streaming=True)\n master.append_conf('postgresql.conf', extra_conf)\n\n``default_conf()`` is called by ``init()`` and rewrites the configuration file. Apply ``append_conf()`` afterwards to keep custom lines.\n\nRemote mode\n-----------\n\nProvision nodes on a remote host (Linux only) by wiring ``RemoteOperations`` into the configuration:\n\n.. code-block:: python\n\n from testgres import ConnectionParams, RemoteOperations, TestgresConfig, get_remote_node\n\n conn_params = ConnectionParams(\n host='example.com',\n username='postgres',\n ssh_key='/path/to/ssh/key'\n )\n os_ops = RemoteOperations(conn_params)\n\n TestgresConfig.set_os_ops(os_ops=os_ops)\n\n def test_basic_query():\n with get_remote_node(conn_params=conn_params) as node:\n node.init().start()\n assert node.execute('SELECT 1') == [(1,)]\n\nPytest integration\n------------------\n\nUse fixtures to create and clean up nodes automatically when testing with ``pytest``:\n\n.. code-block:: python\n\n import pytest\n import testgres\n\n @pytest.fixture\n def pg_node():\n node = testgres.get_new_node().init().start()\n try:\n yield node\n finally:\n node.stop()\n node.cleanup()\n\n def test_simple(pg_node):\n assert pg_node.execute('select 1')[0][0] == 1\n\nScaling tips\n------------\n\n* Run tests in parallel with ``pytest -n auto`` (requires ``pytest-xdist``). Set unique ports by passing ``port`` to ``get_new_node()`` or exporting ``PGPORT`` in the fixture.\n* Always call ``node.cleanup()`` after each test, or rely on context managers/fixtures that do it for you, to avoid leftover data directories.\n* Prefer ``safe_psql()`` for quick assertions, and ``execute()`` when you need Python data structures.\n\nModules\n=======\n\n.. toctree::\n :maxdepth: 2\n\n testgres\n\n\n.. Indices and tables\n.. ==================\n\n.. * :ref:`genindex`\n.. * :ref:`modindex`\n.. * :ref:`search`\n" }, { "path": "docs/source/testgres.rst", "content": "testgres package\n================\n\ntestgres.api\n------------\n\n.. automodule:: testgres.api\n :members:\n :undoc-members:\n :show-inheritance:\n\ntestgres.backup\n---------------\n\n.. automodule:: testgres.backup\n :members:\n :undoc-members:\n :show-inheritance:\n\ntestgres.config\n---------------\n\n.. automodule:: testgres.config\n :members:\n :undoc-members:\n :show-inheritance:\n :member-order: groupwise\n\ntestgres.connection\n-------------------\n\n.. automodule:: testgres.connection\n :members:\n :undoc-members:\n :show-inheritance:\n\ntestgres.enums\n--------------\n\n.. automodule:: testgres.enums\n :members:\n :undoc-members:\n :show-inheritance:\n\ntestgres.exceptions\n-------------------\n\n.. automodule:: testgres.exceptions\n :members:\n :undoc-members:\n :show-inheritance:\n\ntestgres.node\n-------------\n\n.. autoclass:: testgres.node.PostgresNode\n :members:\n\n .. automethod:: __init__\n\n.. autoclass:: testgres.node.ProcessProxy\n :members:\n\ntestgres.standby\n----------------\n\n.. automodule:: testgres.standby\n :members:\n :undoc-members:\n :show-inheritance:\n\ntestgres.pubsub\n---------------\n\n.. automodule:: testgres.pubsub\n\n.. autoclass:: testgres.node.Publication\n :members:\n\n .. automethod:: __init__\n\n.. autoclass:: testgres.node.Subscription\n :members:\n\n .. automethod:: __init__\n" }, { "path": "pyproject.toml", "content": "[build-system]\nrequires = [\"setuptools>=61.0\"]\nbuild-backend = \"setuptools.build_meta\"\n\n[tool.setuptools.package-dir]\n\"testgres\" = \"src\"\n\n[tool.setuptools.dynamic]\nversion = {attr = \"testgres.__version__\"}\n\n[tool.flake8]\nextend-ignore = [\"E501\"]\nexclude = [\".git\", \"__pycache__\", \"env\", \"venv\"]\n\n# Pytest settings\n[tool.pytest.ini_options]\n\ntestpaths = [\"tests\"]\nlog_file_level = \"NOTSET\"\nlog_file_format = \"%(levelname)8s [%(asctime)s] %(message)s\"\nlog_file_date_format = \"%Y-%m-%d %H:%M:%S\"\n\n[project]\nname = \"testgres\"\ndynamic = [\"version\"]\n\ndescription = \"Testing utility for PostgreSQL and its extensions\"\nreadme = \"README.md\"\n\n# [2026-01-05]\n# This old format is used to ensure compatibility with Python 3.7.\nlicense = {text = \"PostgreSQL\"}\n\nauthors = [\n {name = \"Postgres Professional\", email = \"testgres@postgrespro.ru\"},\n]\n\nkeywords = [\n 'test',\n 'testing',\n 'postgresql',\n]\n\nrequires-python = \">=3.7.3\"\n\nclassifiers = [\n \"Intended Audience :: Developers\",\n \"Operating System :: Unix\",\n \"Programming Language :: Python :: 3 :: Only\",\n \"Programming Language :: Python :: 3.7\",\n \"Programming Language :: Python :: 3.8\",\n \"Programming Language :: Python :: 3.9\",\n \"Programming Language :: Python :: 3.10\",\n \"Programming Language :: Python :: 3.11\",\n \"Programming Language :: Python :: 3.12\",\n \"Programming Language :: Python :: 3.13\",\n \"Programming Language :: Python :: 3.14\",\n \"Topic :: Software Development :: Libraries\",\n \"Topic :: Software Development :: Testing\",\n]\n\ndependencies = [\n \"pg8000\",\n \"port-for>=0.4\",\n \"six>=1.9.0\",\n \"psutil\",\n \"packaging\",\n \"testgres.os_ops>=2.1.0,<3.0.0\",\n]\n\n[project.urls]\n\"HomePage\" = \"https://github.com/postgrespro/testgres\"\n" }, { "path": "run_tests.sh", "content": "#!/usr/bin/env bash\n\nset -eux\n\nif [ -z ${TEST_FILTER+x} ]; \\\nthen export TEST_FILTER=\"TestTestgresLocal or (TestTestgresCommon and (not remote))\"; \\\nfi\n\n# fail early\necho check that pg_config is in PATH\ncommand -v pg_config\n\n# prepare python environment\nVENV_PATH=\"/tmp/testgres_venv\"\nrm -rf $VENV_PATH\n${PYTHON_BINARY} -m venv \"${VENV_PATH}\"\nexport VIRTUAL_ENV_DISABLE_PROMPT=1\nsource \"${VENV_PATH}/bin/activate\"\npip install --upgrade pip setuptools wheel\npip install -r tests/requirements.txt\n\n# remove existing coverage file\nexport COVERAGE_FILE=.coverage\nrm -f $COVERAGE_FILE\n\npip install coverage\n\n# run tests (PATH)\ntime coverage run -a -m pytest -l -vvv -n 4 -k \"${TEST_FILTER}\"\n\n# run tests (PG_BIN)\nPG_BIN=$(pg_config --bindir) \\\ntime coverage run -a -m pytest -l -vvv -n 4 -k \"${TEST_FILTER}\"\n\n# run tests (PG_CONFIG)\nPG_CONFIG=$(pg_config --bindir)/pg_config \\\ntime coverage run -a -m pytest -l -vvv -n 4 -k \"${TEST_FILTER}\"\n\n# test pg8000\npip uninstall -y psycopg2\npip install pg8000\nPG_CONFIG=$(pg_config --bindir)/pg_config \\\ntime coverage run -a -m pytest -l -vvv -n 4 -k \"${TEST_FILTER}\"\n\n# show coverage\ncoverage report\n\npip uninstall -y coverage\n\n# build documentation\npip install Sphinx\n\ncd docs\nmake html\ncd ..\n\npip uninstall -y Sphinx\n\n# attempt to fix codecov\nset +eux\n\n# send coverage stats to Codecov\nbash <(curl -s https://codecov.io/bash)\n" }, { "path": "run_tests2.sh", "content": "#!/usr/bin/env bash\n\nset -eux\n\neval \"$(pyenv init -)\"\neval \"$(pyenv virtualenv-init -)\"\n\npyenv virtualenv --force ${PYTHON_VERSION} cur\npyenv activate cur\n\n./run_tests.sh\n" }, { "path": "src/__init__.py", "content": "from .api import get_new_node, get_remote_node\nfrom .backup import NodeBackup\n\nfrom .config import \\\n TestgresConfig, \\\n configure_testgres, \\\n scoped_config, \\\n push_config, \\\n pop_config\n\nfrom .connection import \\\n NodeConnection, \\\n DatabaseError, \\\n InternalError, \\\n ProgrammingError, \\\n OperationalError\n\nfrom .exceptions import \\\n TestgresException, \\\n ExecUtilException, \\\n QueryException, \\\n QueryTimeoutException, \\\n TimeoutException, \\\n CatchUpException, \\\n StartNodeException, \\\n InitNodeException, \\\n BackupException, \\\n InvalidOperationException\n\nfrom .enums import \\\n XLogMethod, \\\n IsolationLevel, \\\n NodeStatus, \\\n ProcessType, \\\n DumpFormat\n\nfrom .node import PostgresNode\nfrom .node import PortManager\nfrom .node_app import NodeApp\n\nfrom .utils import \\\n reserve_port, \\\n release_port, \\\n bound_ports, \\\n get_bin_path, \\\n get_pg_config, \\\n get_pg_version\n\nfrom .standby import \\\n First, \\\n Any\n\nfrom .config import testgres_config\n\nfrom testgres.operations.os_ops import OsOperations, ConnectionParams\nfrom testgres.operations.local_ops import LocalOperations\nfrom testgres.operations.remote_ops import RemoteOperations\n\n__version__ = \"1.13.7\"\n\n__all__ = [\n \"get_new_node\",\n \"get_remote_node\",\n \"NodeBackup\", \"testgres_config\",\n \"TestgresConfig\", \"configure_testgres\", \"scoped_config\", \"push_config\", \"pop_config\",\n \"NodeConnection\", \"DatabaseError\", \"InternalError\", \"ProgrammingError\", \"OperationalError\",\n \"TestgresException\", \"ExecUtilException\", \"QueryException\",\n QueryTimeoutException.__name__,\n \"TimeoutException\", \"CatchUpException\", \"StartNodeException\", \"InitNodeException\", \"BackupException\", \"InvalidOperationException\",\n \"XLogMethod\", \"IsolationLevel\", \"NodeStatus\", \"ProcessType\", \"DumpFormat\",\n NodeApp.__name__,\n PostgresNode.__name__,\n PortManager.__name__,\n \"reserve_port\", \"release_port\", \"bound_ports\", \"get_bin_path\", \"get_pg_config\", \"get_pg_version\",\n \"First\", \"Any\",\n \"OsOperations\", \"LocalOperations\", \"RemoteOperations\", \"ConnectionParams\"\n]\n" }, { "path": "src/api.py", "content": "# coding: utf-8\n\"\"\"\nTesting framework for PostgreSQL and its extensions\n\nThis module was created under influence of Postgres TAP test feature\n(PostgresNode.pm module). It can manage Postgres clusters: initialize,\nedit configuration files, start/stop cluster, execute queries. The\ntypical flow may look like:\n\n>>> with get_new_node() as node:\n... node.init().start()\n... result = node.safe_psql('postgres', 'select 1')\n... print(result.decode('utf-8').strip())\n... node.stop()\nPostgresNode(name='...', port=..., base_dir='...')\n1\nPostgresNode(name='...', port=..., base_dir='...')\n\n Or:\n\n>>> with get_new_node() as master:\n... master.init().start()\n... with master.backup() as backup:\n... with backup.spawn_replica() as replica:\n... replica = replica.start()\n... master.execute('postgres', 'create table test (val int4)')\n... master.execute('postgres', 'insert into test values (0), (1), (2)')\n... replica.catchup() # wait until changes are visible\n... print(replica.execute('postgres', 'select count(*) from test'))\nPostgresNode(name='...', port=..., base_dir='...')\n[(3,)]\n\"\"\"\nfrom .node import PostgresNode\n\n\ndef get_new_node(name=None, base_dir=None, **kwargs):\n \"\"\"\n Simply a wrapper around :class:`.PostgresNode` constructor.\n See :meth:`.PostgresNode.__init__` for details.\n \"\"\"\n # NOTE: leave explicit 'name' and 'base_dir' for compatibility\n return PostgresNode(name=name, base_dir=base_dir, **kwargs)\n\n\ndef get_remote_node(name=None, conn_params=None):\n \"\"\"\n Simply a wrapper around :class:`.PostgresNode` constructor for remote node.\n See :meth:`.PostgresNode.__init__` for details.\n For remote connection you can add the next parameter:\n conn_params = ConnectionParams(host='127.0.0.1', ssh_key=None, username=default_username())\n \"\"\"\n return get_new_node(name=name, conn_params=conn_params)\n" }, { "path": "src/backup.py", "content": "# coding: utf-8\n\nfrom six import raise_from\n\nfrom .enums import XLogMethod\n\nfrom .consts import \\\n DATA_DIR, \\\n TMP_NODE, \\\n TMP_BACKUP, \\\n PG_CONF_FILE, \\\n BACKUP_LOG_FILE\n\nfrom .exceptions import BackupException\n\nfrom testgres.operations.os_ops import OsOperations\n\nfrom .utils import \\\n get_bin_path2, \\\n execute_utility2, \\\n clean_on_error\n\n\nclass NodeBackup(object):\n \"\"\"\n Smart object responsible for backups\n \"\"\"\n @property\n def log_file(self):\n assert self.os_ops is not None\n assert isinstance(self.os_ops, OsOperations)\n return self.os_ops.build_path(self.base_dir, BACKUP_LOG_FILE)\n\n def __init__(self,\n node,\n base_dir=None,\n username=None,\n xlog_method=XLogMethod.fetch,\n options=None):\n \"\"\"\n Create a new backup.\n\n Args:\n node: :class:`.PostgresNode` we're going to backup.\n base_dir: where should we store it?\n username: database user name.\n xlog_method: none | fetch | stream (see docs)\n \"\"\"\n assert node.os_ops is not None\n assert isinstance(node.os_ops, OsOperations)\n\n if not options:\n options = []\n self.os_ops = node.os_ops\n if not node.status():\n raise BackupException('Node must be running')\n\n # Check arguments\n if not isinstance(xlog_method, XLogMethod):\n try:\n xlog_method = XLogMethod(xlog_method)\n except ValueError:\n msg = 'Invalid xlog_method \"{}\"'.format(xlog_method)\n raise BackupException(msg)\n\n # Set default arguments\n username = username or self.os_ops.get_user()\n base_dir = base_dir or self.os_ops.mkdtemp(prefix=TMP_BACKUP)\n\n # public\n self.original_node = node\n self.base_dir = base_dir\n self.username = username\n\n # private\n self._available = True\n\n data_dir = self.os_ops.build_path(self.base_dir, DATA_DIR)\n\n _params = [\n get_bin_path2(self.os_ops, \"pg_basebackup\"),\n \"-p\", str(node.port),\n \"-h\", node.host,\n \"-U\", username,\n \"-D\", data_dir,\n \"-X\", xlog_method.value\n ] # yapf: disable\n _params += options\n execute_utility2(self.os_ops, _params, self.log_file)\n\n def __enter__(self):\n return self\n\n def __exit__(self, type, value, traceback):\n self.cleanup()\n\n def _prepare_dir(self, destroy):\n \"\"\"\n Provide a data directory for a copy of node.\n\n Args:\n destroy: should we convert this backup into a node?\n\n Returns:\n Path to data directory.\n \"\"\"\n\n if not self._available:\n raise BackupException('Backup is exhausted')\n\n # Do we want to use this backup several times?\n available = not destroy\n\n if available:\n assert self.os_ops is not None\n assert isinstance(self.os_ops, OsOperations)\n\n dest_base_dir = self.os_ops.mkdtemp(prefix=TMP_NODE)\n\n data1 = self.os_ops.build_path(self.base_dir, DATA_DIR)\n data2 = self.os_ops.build_path(dest_base_dir, DATA_DIR)\n\n try:\n # Copy backup to new data dir\n self.os_ops.copytree(data1, data2)\n except Exception as e:\n raise_from(BackupException('Failed to copy files'), e)\n else:\n dest_base_dir = self.base_dir\n\n # Is this backup exhausted?\n self._available = available\n\n # Return path to new node\n return dest_base_dir\n\n def spawn_primary(self, name=None, destroy=True):\n \"\"\"\n Create a primary node from a backup.\n\n Args:\n name: primary's application name.\n destroy: should we convert this backup into a node?\n\n Returns:\n New instance of :class:`.PostgresNode`.\n \"\"\"\n\n # Prepare a data directory for this node\n base_dir = self._prepare_dir(destroy)\n\n # Build a new PostgresNode\n assert self.original_node is not None\n\n if (hasattr(self.original_node, \"clone_with_new_name_and_base_dir\")):\n node = self.original_node.clone_with_new_name_and_base_dir(name=name, base_dir=base_dir)\n else:\n # For backward compatibility\n NodeClass = self.original_node.__class__\n node = NodeClass(name=name, base_dir=base_dir, conn_params=self.original_node.os_ops.conn_params)\n\n assert node is not None\n assert type(node) is self.original_node.__class__\n\n with clean_on_error(node) as node:\n # Set a new port\n node.append_conf(filename=PG_CONF_FILE, line='\\n')\n node.append_conf(filename=PG_CONF_FILE, port=node.port)\n\n return node\n\n def spawn_replica(self, name=None, destroy=True, slot=None):\n \"\"\"\n Create a replica of the original node from a backup.\n\n Args:\n name: replica's application name.\n slot: create a replication slot with the specified name.\n destroy: should we convert this backup into a node?\n\n Returns:\n New instance of :class:`.PostgresNode`.\n \"\"\"\n\n # Build a new PostgresNode\n node = self.spawn_primary(name=name, destroy=destroy)\n assert node is not None\n\n try:\n # Assign it a master and a recovery file (private magic)\n node._assign_master(self.original_node)\n node._create_recovery_conf(username=self.username, slot=slot)\n except: # noqa: E722\n # TODO: Pass 'final=True' ?\n node.cleanup(release_resources=True)\n raise\n\n return node\n\n def cleanup(self):\n \"\"\"\n Remove all files that belong to this backup.\n No-op if it's been converted to a PostgresNode (destroy=True).\n \"\"\"\n\n if self._available:\n self._available = False\n self.os_ops.rmdirs(self.base_dir, ignore_errors=True)\n" }, { "path": "src/cache.py", "content": "# coding: utf-8\n\nfrom six import raise_from\n\nfrom .config import testgres_config\n\nfrom .consts import XLOG_CONTROL_FILE\n\nfrom .defaults import generate_system_id\n\nfrom .exceptions import \\\n InitNodeException, \\\n ExecUtilException\n\nfrom .utils import \\\n get_bin_path2, \\\n execute_utility2\n\nfrom testgres.operations.local_ops import LocalOperations\nfrom testgres.operations.os_ops import OsOperations\n\n\ndef cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = None, bin_path=None, cached=True):\n \"\"\"\n Perform initdb or use cached node files.\n \"\"\"\n\n assert os_ops is None or isinstance(os_ops, OsOperations)\n\n if os_ops is None:\n os_ops = LocalOperations.get_single_instance()\n\n assert isinstance(os_ops, OsOperations)\n\n def make_utility_path(name):\n assert name is not None\n assert type(name) is str\n\n if bin_path:\n return os_ops.build_path(bin_path, name)\n\n return get_bin_path2(os_ops, name)\n\n def call_initdb(initdb_dir, log=logfile):\n try:\n initdb_path = make_utility_path(\"initdb\")\n _params = [initdb_path, \"-D\", initdb_dir, \"-N\"]\n execute_utility2(os_ops, _params + (params or []), log)\n except ExecUtilException as e:\n raise_from(InitNodeException(\"Failed to run initdb\"), e)\n\n if params or not testgres_config.cache_initdb or not cached:\n call_initdb(data_dir, logfile)\n else:\n # Fetch cached initdb dir\n cached_data_dir = testgres_config.cached_initdb_dir\n\n # Initialize cached initdb\n\n if not os_ops.path_exists(cached_data_dir) or \\\n not os_ops.listdir(cached_data_dir):\n call_initdb(cached_data_dir)\n\n try:\n # Copy cached initdb to current data dir\n os_ops.copytree(cached_data_dir, data_dir)\n\n # Assign this node a unique system id if asked to\n if testgres_config.cached_initdb_unique:\n # XXX: write new unique system id to control file\n # Some users might rely upon unique system ids, but\n # our initdb caching mechanism breaks this contract.\n pg_control = os_ops.build_path(data_dir, XLOG_CONTROL_FILE)\n system_id = generate_system_id()\n cur_pg_control = os_ops.read(pg_control, binary=True)\n new_pg_control = system_id + cur_pg_control[len(system_id):]\n os_ops.write(pg_control, new_pg_control, truncate=True, binary=True, read_and_write=True)\n\n # XXX: build new WAL segment with our system id\n _params = [make_utility_path(\"pg_resetwal\"), \"-D\", data_dir, \"-f\"]\n execute_utility2(os_ops, _params, logfile)\n\n except ExecUtilException as e:\n msg = \"Failed to reset WAL for system id\"\n raise_from(InitNodeException(msg), e)\n\n except Exception as e:\n raise_from(InitNodeException(\"Failed to spawn a node\"), e)\n" }, { "path": "src/config.py", "content": "# coding: utf-8\n\nimport atexit\nimport copy\nimport logging\nimport os\nimport tempfile\n\nfrom contextlib import contextmanager\n\nfrom .consts import TMP_CACHE\nfrom testgres.operations.os_ops import OsOperations\nfrom testgres.operations.local_ops import LocalOperations\n\nlog_level = os.getenv('LOGGING_LEVEL', 'WARNING').upper()\nlog_format = os.getenv('LOGGING_FORMAT', '%(asctime)s - %(levelname)s - %(message)s')\nlogging.basicConfig(level=log_level, format=log_format)\n\n\nclass GlobalConfig(object):\n \"\"\"\n Global configuration object which allows user to override default settings.\n \"\"\"\n # NOTE: attributes must not be callable or begin with __.\n\n cache_initdb = True\n \"\"\" shall we use cached initdb instance? \"\"\"\n\n cached_initdb_unique = False\n \"\"\" shall we give new node a unique system id? \"\"\"\n\n cache_pg_config = True\n \"\"\" shall we cache pg_config results? \"\"\"\n\n use_python_logging = False\n \"\"\" enable python logging subsystem (see logger.py). \"\"\"\n\n error_log_lines = 20\n \"\"\" N of log lines to be shown in exceptions (0=inf). \"\"\"\n\n node_cleanup_full = True\n \"\"\" shall we remove EVERYTHING (including logs)? \"\"\"\n\n node_cleanup_on_good_exit = True\n \"\"\" remove base_dir on nominal __exit__(). \"\"\"\n\n node_cleanup_on_bad_exit = False\n \"\"\" remove base_dir on __exit__() via exception. \"\"\"\n\n _cached_initdb_dir = None\n \"\"\" underlying class attribute for cached_initdb_dir property \"\"\"\n\n os_ops = LocalOperations.get_single_instance()\n \"\"\" OsOperation object that allows work on remote host \"\"\"\n\n @property\n def cached_initdb_dir(self):\n \"\"\" path to a temp directory for cached initdb. \"\"\"\n return self._cached_initdb_dir\n\n @cached_initdb_dir.setter\n def cached_initdb_dir(self, value):\n self._cached_initdb_dir = value\n\n if value:\n cached_initdb_dirs.add(value)\n return testgres_config.cached_initdb_dir\n\n @property\n def temp_dir(self):\n \"\"\" path to temp dir containing nodes with default 'base_dir'. \"\"\"\n return tempfile.tempdir\n\n @temp_dir.setter\n def temp_dir(self, value):\n tempfile.tempdir = value\n\n def __init__(self, **options):\n self.update(options)\n\n def __setitem__(self, key, value):\n setattr(self, key, value)\n\n def __getitem__(self, key):\n return getattr(self, key)\n\n def __setattr__(self, name, value):\n if name not in self.keys():\n raise TypeError('Unknown option {}'.format(name))\n\n super(GlobalConfig, self).__setattr__(name, value)\n\n def keys(self):\n \"\"\"\n Return a list of all available settings.\n \"\"\"\n\n keys = []\n\n for key in dir(GlobalConfig):\n if not key.startswith('__') and not callable(self[key]):\n keys.append(key)\n\n return keys\n\n def items(self):\n \"\"\"\n Return setting-value pairs.\n \"\"\"\n\n return ((key, self[key]) for key in self.keys())\n\n def update(self, config):\n \"\"\"\n Extract setting-value pairs from 'config' and\n assign those values to corresponding settings\n of this GlobalConfig object.\n \"\"\"\n\n for key, value in config.items():\n self[key] = value\n\n return self\n\n def copy(self):\n \"\"\"\n Return a copy of this object.\n \"\"\"\n\n return copy.copy(self)\n\n @staticmethod\n def set_os_ops(os_ops: OsOperations):\n testgres_config.os_ops = os_ops\n testgres_config.cached_initdb_dir = os_ops.mkdtemp(prefix=TMP_CACHE)\n\n\n# cached dirs to be removed\ncached_initdb_dirs = set()\n\n# default config object\ntestgres_config = GlobalConfig()\n\n# NOTE: for compatibility\nTestgresConfig = testgres_config\n\n# stack of GlobalConfigs\nconfig_stack = []\n\n\n@atexit.register\ndef _rm_cached_initdb_dirs():\n for d in cached_initdb_dirs:\n testgres_config.os_ops.rmdirs(d, ignore_errors=True)\n\n\ndef push_config(**options):\n \"\"\"\n Permanently set custom GlobalConfig options and\n put previous settings on top of the config stack.\n \"\"\"\n\n # push current config to stack\n config_stack.append(testgres_config.copy())\n\n return testgres_config.update(options)\n\n\ndef pop_config():\n \"\"\"\n Set previous GlobalConfig options from stack.\n \"\"\"\n\n if len(config_stack) == 0:\n raise IndexError('Reached initial config')\n\n # restore popped config\n return testgres_config.update(config_stack.pop())\n\n\n@contextmanager\ndef scoped_config(**options):\n \"\"\"\n Temporarily set custom GlobalConfig options for this context.\n Previous options are pushed to the config stack.\n\n Example:\n >>> from .api import get_new_node\n >>> with scoped_config(cache_initdb=False):\n ... # create a new node with fresh initdb\n ... with get_new_node().init().start() as node:\n ... print(node.execute('select 1'))\n [(1,)]\n \"\"\"\n\n try:\n # set a new config with options\n config = push_config(**options)\n\n # return it\n yield config\n finally:\n # restore previous config\n pop_config()\n\n\ndef configure_testgres(**options):\n \"\"\"\n Adjust current global options.\n Look at the GlobalConfig to learn about existing settings.\n \"\"\"\n\n testgres_config.update(options)\n\n\n# NOTE: assign initial cached dir for initdb\ntestgres_config.cached_initdb_dir = testgres_config.os_ops.mkdtemp(prefix=TMP_CACHE)\n" }, { "path": "src/connection.py", "content": "# coding: utf-8\nimport logging\n\n# we support both pg8000 and psycopg2\ntry:\n import psycopg2 as pglib\nexcept ImportError:\n try:\n import pg8000 as pglib\n except ImportError:\n raise ImportError(\"You must have psycopg2 or pg8000 modules installed\")\n\nfrom .enums import IsolationLevel\n\nfrom .defaults import \\\n default_dbname, \\\n default_username\n\nfrom .exceptions import QueryException\n\n# export some exceptions\nDatabaseError = pglib.DatabaseError\nInternalError = pglib.InternalError\nProgrammingError = pglib.ProgrammingError\nOperationalError = pglib.OperationalError\n\n\nclass NodeConnection(object):\n \"\"\"\n Transaction wrapper returned by Node\n \"\"\"\n def __init__(self,\n node,\n dbname=None,\n username=None,\n password=None,\n autocommit=False):\n\n # Set default arguments\n dbname = dbname or default_dbname()\n username = username or default_username()\n\n self._node = node\n\n self._connection = pglib.connect(\n database=dbname,\n user=username,\n password=password,\n host=node.host,\n port=node.port\n )\n\n self._connection.autocommit = autocommit\n self._cursor = self.connection.cursor()\n\n @property\n def node(self):\n return self._node\n\n @property\n def connection(self):\n return self._connection\n\n @property\n def pid(self):\n return self.execute(\"select pg_catalog.pg_backend_pid()\")[0][0]\n\n @property\n def cursor(self):\n return self._cursor\n\n def __enter__(self):\n return self\n\n def __exit__(self, type, value, traceback):\n self.close()\n\n def begin(self, isolation_level=IsolationLevel.ReadCommitted):\n # Check if level isn't an IsolationLevel\n if not isinstance(isolation_level, IsolationLevel):\n # Get name of isolation level\n level_str = str(isolation_level).lower()\n\n # Validate level string\n try:\n isolation_level = IsolationLevel(level_str)\n except ValueError:\n error = 'Invalid isolation level \"{}\"'\n raise QueryException(error.format(level_str))\n\n # Set isolation level\n cmd = 'SET TRANSACTION ISOLATION LEVEL {}'\n self.cursor.execute(cmd.format(isolation_level.value))\n\n return self\n\n def commit(self):\n self.connection.commit()\n\n return self\n\n def rollback(self):\n self.connection.rollback()\n\n return self\n\n def execute(self, query, *args):\n self.cursor.execute(query, args)\n try:\n # pg8000 might return tuples\n res = [tuple(t) for t in self.cursor.fetchall()]\n return res\n except ProgrammingError:\n return None\n except Exception as e:\n logging.error(\"Error executing query: {}\\n {}\".format(repr(e), query))\n return None\n\n def close(self):\n self.cursor.close()\n self.connection.close()\n" }, { "path": "src/consts.py", "content": "# coding: utf-8\n\n# names for dirs in base_dir\nDATA_DIR = \"data\"\nLOGS_DIR = \"logs\"\n\n# prefixes for temp dirs\nTMP_NODE = 'tgsn_'\nTMP_DUMP = 'tgsd_'\nTMP_CACHE = 'tgsc_'\nTMP_BACKUP = 'tgsb_'\n\n# path to control file\nXLOG_CONTROL_FILE = \"global/pg_control\"\n\n# names for config files\nRECOVERY_CONF_FILE = \"recovery.conf\"\nPG_AUTO_CONF_FILE = \"postgresql.auto.conf\"\nPG_CONF_FILE = \"postgresql.conf\"\nPG_PID_FILE = 'postmaster.pid'\nHBA_CONF_FILE = \"pg_hba.conf\"\n\n# names for log files\nPG_LOG_FILE = \"postgresql.log\"\nUTILS_LOG_FILE = \"utils.log\"\nBACKUP_LOG_FILE = \"backup.log\"\n\n# defaults for node settings\nMAX_LOGICAL_REPLICATION_WORKERS = 5\nMAX_REPLICATION_SLOTS = 10\nMAX_WORKER_PROCESSES = 10\nWAL_KEEP_SEGMENTS = 20\nWAL_KEEP_SIZE = 320\nMAX_WAL_SENDERS = 10\n\n# logical replication settings\nLOGICAL_REPL_MAX_CATCHUP_ATTEMPTS = 60\n\nPG_CTL__STATUS__OK = 0\nPG_CTL__STATUS__NODE_IS_STOPPED = 3\nPG_CTL__STATUS__BAD_DATADIR = 4\n" }, { "path": "src/decorators.py", "content": "import six\nimport functools\n\n\ndef positional_args_hack(*special_cases):\n \"\"\"\n Convert positional args described by\n 'special_cases' into named args.\n\n Example:\n @positional_args_hack(['abc'], ['def', 'abc'])\n def some_api_func(...)\n\n This is useful for compatibility.\n \"\"\"\n\n cases = dict()\n\n for case in special_cases:\n k = len(case)\n assert k not in six.iterkeys(cases), 'len must be unique'\n cases[k] = case\n\n def decorator(function):\n @functools.wraps(function)\n def wrapper(*args, **kwargs):\n k = len(args)\n\n if k in six.iterkeys(cases):\n case = cases[k]\n\n for i in range(0, k):\n arg_name = case[i]\n arg_val = args[i]\n\n # transform into named\n kwargs[arg_name] = arg_val\n\n # get rid of them\n args = []\n\n return function(*args, **kwargs)\n\n return wrapper\n\n return decorator\n\n\ndef method_decorator(decorator):\n \"\"\"\n Convert a function decorator into a method decorator.\n \"\"\"\n def _dec(func):\n def _wrapper(self, *args, **kwargs):\n @decorator\n def bound_func(*args2, **kwargs2):\n return func.__get__(self, type(self))(*args2, **kwargs2)\n\n # 'bound_func' is a closure and can see 'self'\n return bound_func(*args, **kwargs)\n\n # preserve docs\n functools.update_wrapper(_wrapper, func)\n\n return _wrapper\n\n # preserve docs\n functools.update_wrapper(_dec, decorator)\n\n # change name for easier debugging\n _dec.__name__ = 'method_decorator({})'.format(decorator.__name__)\n\n return _dec\n" }, { "path": "src/defaults.py", "content": "import datetime\nimport struct\nimport uuid\n\nfrom .config import testgres_config as tconf\n\n\ndef default_dbname():\n \"\"\"\n Return default DB name.\n \"\"\"\n\n return 'postgres'\n\n\ndef default_username():\n \"\"\"\n Return default username (current user).\n \"\"\"\n return tconf.os_ops.get_user()\n\n\ndef generate_app_name():\n \"\"\"\n Generate a new application name for node.\n \"\"\"\n\n return 'testgres-{}'.format(str(uuid.uuid4()))\n\n\ndef generate_system_id():\n \"\"\"\n Generate a new 64-bit unique system identifier for node.\n \"\"\"\n\n date1 = datetime.datetime.utcfromtimestamp(0)\n date2 = datetime.datetime.utcnow()\n\n secs = int((date2 - date1).total_seconds())\n usecs = date2.microsecond\n\n # see pg_resetwal.c : GuessControlValues()\n system_id = 0\n system_id |= (secs << 32)\n system_id |= (usecs << 12)\n system_id |= (tconf.os_ops.get_pid() & 0xFFF)\n\n # pack ULL in native byte order\n return struct.pack('=Q', system_id)\n" }, { "path": "src/enums.py", "content": "from enum import Enum, IntEnum\nfrom six import iteritems\nfrom psutil import NoSuchProcess\n\n\nclass XLogMethod(Enum):\n \"\"\"\n Available WAL methods for :class:`.NodeBackup`\n \"\"\"\n\n none = 'none'\n fetch = 'fetch'\n stream = 'stream'\n\n\nclass IsolationLevel(Enum):\n \"\"\"\n Transaction isolation level for :class:`.NodeConnection`\n \"\"\"\n\n ReadUncommitted = 'read uncommitted'\n ReadCommitted = 'read committed'\n RepeatableRead = 'repeatable read'\n Serializable = 'serializable'\n\n\nclass NodeStatus(IntEnum):\n \"\"\"\n Status of a PostgresNode\n \"\"\"\n\n Running, Stopped, Uninitialized = range(3)\n\n # for Python 3.x\n def __bool__(self):\n return self == NodeStatus.Running\n\n # for Python 2.x\n __nonzero__ = __bool__\n\n\nclass ProcessType(Enum):\n \"\"\"\n Types of processes\n \"\"\"\n\n AutovacuumLauncher = 'autovacuum launcher'\n BackgroundWriter = 'background writer'\n Checkpointer = 'checkpointer'\n LogicalReplicationLauncher = 'logical replication launcher'\n Startup = 'startup'\n StatsCollector = 'stats collector'\n WalReceiver = 'wal receiver'\n WalSender = 'wal sender'\n WalWriter = 'wal writer'\n\n # special value\n Unknown = 'unknown'\n\n @staticmethod\n def from_process(process):\n # legacy names for older releases of PG\n alternative_names = {\n ProcessType.LogicalReplicationLauncher: [\n 'logical replication worker'\n ],\n ProcessType.BackgroundWriter: [\n 'writer'\n ],\n } # yapf: disable\n\n try:\n cmdline = ''.join(process.cmdline())\n except (FileNotFoundError, ProcessLookupError, NoSuchProcess):\n return ProcessType.Unknown\n\n # we deliberately cut special words and spaces\n cmdline = cmdline.replace('postgres:', '', 1) \\\n .replace('bgworker:', '', 1) \\\n .replace(' ', '')\n\n for ptype in ProcessType:\n if cmdline.startswith(ptype.value.replace(' ', '')):\n return ptype\n\n for ptype, names in iteritems(alternative_names):\n for name in names:\n if cmdline.startswith(name.replace(' ', '')):\n return ptype\n\n # default\n return ProcessType.Unknown\n\n\nclass DumpFormat(Enum):\n \"\"\"\n Available dump formats\n \"\"\"\n\n Plain = 'plain'\n Custom = 'custom'\n Directory = 'directory'\n Tar = 'tar'\n" }, { "path": "src/exceptions.py", "content": "# coding: utf-8\n\nimport six\nimport typing\n\nfrom testgres.operations.exceptions import TestgresException\nfrom testgres.operations.exceptions import ExecUtilException\nfrom testgres.operations.exceptions import InvalidOperationException\n\n\nclass PortForException(TestgresException):\n _message: typing.Optional[str]\n\n def __init__(\n self,\n message: typing.Optional[str] = None,\n ):\n assert message is None or type(message) is str\n super().__init__(message)\n self._message = message\n return\n\n @property\n def message(self) -> str:\n assert self._message is None or type(self._message) is str\n if self._message is None:\n return \"\"\n return self._message\n\n def __repr__(self) -> str:\n args = []\n\n if self._message is not None:\n args.append((\"message\", self._message))\n\n result = \"{}(\".format(type(self).__name__)\n sep = \"\"\n for a in args:\n result += sep + a[0] + \"=\" + repr(a[1])\n sep = \", \"\n continue\n result += \")\"\n return result\n\n\n@six.python_2_unicode_compatible\nclass QueryException(TestgresException):\n _description: typing.Optional[str]\n _query: typing.Optional[str]\n\n def __init__(\n self,\n message: typing.Optional[str] = None,\n query: typing.Optional[str] = None\n ):\n assert message is None or type(message) is str\n assert query is None or type(query) is str\n\n super().__init__(message)\n\n self._description = message\n self._query = query\n return\n\n @property\n def message(self) -> str:\n assert self._description is None or type(self._description) is str\n assert self._query is None or type(self._query) is str\n\n msg = []\n\n if self._description:\n msg.append(self._description)\n\n if self._query:\n msg.append(u'Query: {}'.format(self._query))\n\n r = six.text_type('\\n').join(msg)\n assert type(r) is str\n return r\n\n @property\n def description(self) -> typing.Optional[str]:\n assert self._description is None or type(self._description) is str\n return self._description\n\n @property\n def query(self) -> typing.Optional[str]:\n assert self._query is None or type(self._query) is str\n return self._query\n\n def __repr__(self) -> str:\n args = []\n\n if self._description is not None:\n args.append((\"message\", self._description))\n\n if self._query is not None:\n args.append((\"query\", self._query))\n\n result = \"{}(\".format(type(self).__name__)\n sep = \"\"\n for a in args:\n result += sep + a[0] + \"=\" + repr(a[1])\n sep = \", \"\n continue\n result += \")\"\n return result\n\n\nclass QueryTimeoutException(QueryException):\n def __init__(\n self,\n message: typing.Optional[str] = None,\n query: typing.Optional[str] = None\n ):\n assert message is None or type(message) is str\n assert query is None or type(query) is str\n\n super().__init__(message, query)\n return\n\n\n# [2026-01-10] To backward compatibility.\nTimeoutException = QueryTimeoutException\n\n\n# [2026-01-10] It inherits TestgresException now, not QueryException\nclass CatchUpException(TestgresException):\n _message: typing.Optional[str]\n\n def __init__(\n self,\n message: typing.Optional[str] = None,\n ):\n assert message is None or type(message) is str\n super().__init__(message)\n self._message = message\n return\n\n @property\n def message(self) -> str:\n assert self._message is None or type(self._message) is str\n if self._message is None:\n return \"\"\n return self._message\n\n def __repr__(self) -> str:\n args = []\n\n if self._message is not None:\n args.append((\"message\", self._message))\n\n result = \"{}(\".format(type(self).__name__)\n sep = \"\"\n for a in args:\n result += sep + a[0] + \"=\" + repr(a[1])\n sep = \", \"\n continue\n result += \")\"\n return result\n\n\n@six.python_2_unicode_compatible\nclass StartNodeException(TestgresException):\n _description: typing.Optional[str]\n _files: typing.Optional[typing.Iterable]\n\n def __init__(\n self,\n message: typing.Optional[str] = None,\n files: typing.Optional[typing.Iterable] = None\n ):\n assert message is None or type(message) is str\n assert files is None or isinstance(files, typing.Iterable)\n\n super().__init__(message)\n\n self._description = message\n self._files = files\n return\n\n @property\n def message(self) -> str:\n assert self._description is None or type(self._description) is str\n assert self._files is None or isinstance(self._files, typing.Iterable)\n\n msg = []\n\n if self._description:\n msg.append(self._description)\n\n for f, lines in self._files or []:\n assert type(f) is str\n assert type(lines) in [str, bytes]\n msg.append(u'{}\\n----\\n{}\\n'.format(f, lines))\n\n return six.text_type('\\n').join(msg)\n\n @property\n def description(self) -> typing.Optional[str]:\n assert self._description is None or type(self._description) is str\n return self._description\n\n @property\n def files(self) -> typing.Optional[typing.Iterable]:\n assert self._files is None or isinstance(self._files, typing.Iterable)\n return self._files\n\n def __repr__(self) -> str:\n args = []\n\n if self._description is not None:\n args.append((\"message\", self._description))\n\n if self._files is not None:\n args.append((\"files\", self._files))\n\n result = \"{}(\".format(type(self).__name__)\n sep = \"\"\n for a in args:\n result += sep + a[0] + \"=\" + repr(a[1])\n sep = \", \"\n continue\n result += \")\"\n return result\n\n\nclass InitNodeException(TestgresException):\n _message: typing.Optional[str]\n\n def __init__(\n self,\n message: typing.Optional[str] = None,\n ):\n assert message is None or type(message) is str\n super().__init__(message)\n self._message = message\n return\n\n @property\n def message(self) -> str:\n assert self._message is None or type(self._message) is str\n if self._message is None:\n return \"\"\n return self._message\n\n def __repr__(self) -> str:\n args = []\n\n if self._message is not None:\n args.append((\"message\", self._message))\n\n result = \"{}(\".format(type(self).__name__)\n sep = \"\"\n for a in args:\n result += sep + a[0] + \"=\" + repr(a[1])\n sep = \", \"\n continue\n result += \")\"\n return result\n\n\nclass BackupException(TestgresException):\n _message: typing.Optional[str]\n\n def __init__(\n self,\n message: typing.Optional[str] = None,\n ):\n assert message is None or type(message) is str\n super().__init__(message)\n self._message = message\n return\n\n @property\n def message(self) -> str:\n assert self._message is None or type(self._message) is str\n if self._message is None:\n return \"\"\n return self._message\n\n def __repr__(self) -> str:\n args = []\n\n if self._message is not None:\n args.append((\"message\", self._message))\n\n result = \"{}(\".format(type(self).__name__)\n sep = \"\"\n for a in args:\n result += sep + a[0] + \"=\" + repr(a[1])\n sep = \", \"\n continue\n result += \")\"\n return result\n\n\nassert ExecUtilException.__name__ == \"ExecUtilException\"\nassert InvalidOperationException.__name__ == \"InvalidOperationException\"\n" }, { "path": "src/impl/internal_utils.py", "content": "import logging\n\n\ndef send_log(level: int, msg: str) -> None:\n assert type(level) is int\n assert type(msg) is str\n\n return logging.log(level, \"[testgres] \" + msg)\n\n\ndef send_log_info(msg: str) -> None:\n assert type(msg) is str\n\n return send_log(logging.INFO, msg)\n\n\ndef send_log_debug(msg: str) -> None:\n assert type(msg) is str\n\n return send_log(logging.DEBUG, msg)\n" }, { "path": "src/impl/platforms/internal_platform_utils.py", "content": "from __future__ import annotations\n\nimport enum\nimport typing\n\nfrom testgres.operations.os_ops import OsOperations\n\n\nclass InternalPlatformUtils:\n class FindPostmasterResultCode(enum.Enum):\n ok = 0\n not_found = 1,\n not_implemented = 2\n many_processes = 3\n has_problems = 4\n\n class FindPostmasterResult:\n code: InternalPlatformUtils.FindPostmasterResultCode\n pid: typing.Optional[int]\n\n def __init__(\n self,\n code: InternalPlatformUtils.FindPostmasterResultCode,\n pid: typing.Optional[int]\n ):\n assert type(code) is InternalPlatformUtils.FindPostmasterResultCode\n assert pid is None or type(pid) is int\n self.code = code\n self.pid = pid\n return\n\n @staticmethod\n def create_ok(pid: int) -> InternalPlatformUtils.FindPostmasterResult:\n assert type(pid) is int\n return __class__(InternalPlatformUtils.FindPostmasterResultCode.ok, pid)\n\n @staticmethod\n def create_not_found() -> InternalPlatformUtils.FindPostmasterResult:\n return __class__(InternalPlatformUtils.FindPostmasterResultCode.not_found, None)\n\n @staticmethod\n def create_not_implemented() -> InternalPlatformUtils.FindPostmasterResult:\n return __class__(InternalPlatformUtils.FindPostmasterResultCode.not_implemented, None)\n\n @staticmethod\n def create_many_processes() -> InternalPlatformUtils.FindPostmasterResult:\n return __class__(InternalPlatformUtils.FindPostmasterResultCode.many_processes, None)\n\n @staticmethod\n def create_has_problems() -> InternalPlatformUtils.FindPostmasterResult:\n return __class__(InternalPlatformUtils.FindPostmasterResultCode.has_problems, None)\n\n def FindPostmaster(\n self,\n os_ops: OsOperations,\n bin_dir: str,\n data_dir: str\n ) -> FindPostmasterResult:\n assert isinstance(os_ops, OsOperations)\n assert type(bin_dir) is str\n assert type(data_dir) is str\n raise NotImplementedError(\"InternalPlatformUtils::FindPostmaster is not implemented.\")\n" }, { "path": "src/impl/platforms/internal_platform_utils_factory.py", "content": "from .internal_platform_utils import InternalPlatformUtils\n\nfrom testgres.operations.os_ops import OsOperations\n\n\ndef create_internal_platform_utils(\n os_ops: OsOperations\n) -> InternalPlatformUtils:\n assert isinstance(os_ops, OsOperations)\n\n platform_name = os_ops.get_platform()\n assert type(platform_name) is str\n\n if platform_name == \"linux\":\n from .linux import internal_platform_utils as x\n return x.InternalPlatformUtils()\n\n if platform_name == \"win32\":\n from .win32 import internal_platform_utils as x\n return x.InternalPlatformUtils()\n\n # not implemented\n return InternalPlatformUtils()\n" }, { "path": "src/impl/platforms/linux/internal_platform_utils.py", "content": "from __future__ import annotations\n\nfrom .. import internal_platform_utils as base\nfrom ... import internal_utils\n\nfrom testgres.operations.os_ops import OsOperations\nfrom testgres.operations.exceptions import ExecUtilException\n\nimport re\nimport shlex\n\n\nclass InternalPlatformUtils(base.InternalPlatformUtils):\n C_BASH_EXE = \"/bin/bash\"\n\n sm_exec_env = {\n \"LANG\": \"en_US.UTF-8\",\n \"LC_ALL\": \"en_US.UTF-8\",\n }\n\n # --------------------------------------------------------------------\n def FindPostmaster(\n self,\n os_ops: OsOperations,\n bin_dir: str,\n data_dir: str\n ) -> InternalPlatformUtils.FindPostmasterResult:\n assert isinstance(os_ops, OsOperations)\n assert type(bin_dir) is str\n assert type(data_dir) is str\n assert type(__class__.C_BASH_EXE) is str\n assert type(__class__.sm_exec_env) is dict\n assert len(__class__.C_BASH_EXE) > 0\n assert len(bin_dir) > 0\n assert len(data_dir) > 0\n\n pg_path_e = re.escape(os_ops.build_path(bin_dir, \"postgres\"))\n data_dir_e = re.escape(data_dir)\n\n assert type(pg_path_e) is str\n assert type(data_dir_e) is str\n\n regexp = r\"^\\s*[0-9]+\\s+\" + pg_path_e + r\"(\\s+.*)?\\s+\\-[D]\\s+\" + data_dir_e + r\"(\\s+.*)?\"\n\n cmd = [\n __class__.C_BASH_EXE,\n \"-c\",\n \"ps -ewwo \\\"pid=,args=\\\" | grep -E \" + shlex.quote(regexp),\n ]\n\n exit_status, output_b, error_b = os_ops.exec_command(\n cmd=cmd,\n ignore_errors=True,\n verbose=True,\n exec_env=__class__.sm_exec_env,\n )\n\n assert type(output_b) is bytes\n assert type(error_b) is bytes\n\n output = output_b.decode(\"utf-8\")\n error = error_b.decode(\"utf-8\")\n\n assert type(output) is str\n assert type(error) is str\n\n if exit_status == 1:\n return __class__.FindPostmasterResult.create_not_found()\n\n if exit_status != 0:\n errMsg = f\"test command returned an unexpected exit code: {exit_status}\"\n raise ExecUtilException(\n message=errMsg,\n command=cmd,\n exit_code=exit_status,\n out=output,\n error=error,\n )\n\n lines = output.splitlines()\n assert type(lines) is list\n\n if len(lines) == 0:\n return __class__.FindPostmasterResult.create_not_found()\n\n if len(lines) > 1:\n msgs = []\n msgs.append(\"Many processes like a postmaster are found: {}.\".format(len(lines)))\n\n for i in range(len(lines)):\n assert type(lines[i]) is str\n lines.append(\"[{}] '{}'\".format(i, lines[i]))\n continue\n\n internal_utils.send_log_debug(\"\\n\".join(lines))\n return __class__.FindPostmasterResult.create_many_processes()\n\n def is_space_or_tab(ch) -> bool:\n assert type(ch) is str\n return ch == \" \" or ch == \"\\t\"\n\n line = lines[0]\n start = 0\n while start < len(line) and is_space_or_tab(line[start]):\n start += 1\n\n pos = start\n while pos < len(line) and line[pos].isnumeric():\n pos += 1\n\n if pos == start:\n return __class__.FindPostmasterResult.create_has_problems()\n\n if pos != len(line) and not line[pos].isspace():\n return __class__.FindPostmasterResult.create_has_problems()\n\n pid = int(line[start:pos])\n assert type(pid) is int\n\n return __class__.FindPostmasterResult.create_ok(pid)\n" }, { "path": "src/impl/platforms/win32/internal_platform_utils.py", "content": "from __future__ import annotations\n\nfrom .. import internal_platform_utils as base\nfrom testgres.operations.os_ops import OsOperations\n\n\nclass InternalPlatformUtils(base.InternalPlatformUtils):\n def FindPostmaster(\n self,\n os_ops: OsOperations,\n bin_dir: str,\n data_dir: str\n ) -> InternalPlatformUtils.FindPostmasterResult:\n assert isinstance(os_ops, OsOperations)\n assert type(bin_dir) is str\n assert type(data_dir) is str\n return __class__.FindPostmasterResult.create_not_implemented()\n" }, { "path": "src/impl/port_manager__generic.py", "content": "from testgres.operations.os_ops import OsOperations\n\nfrom ..port_manager import PortManager\nfrom ..exceptions import PortForException\n\nimport threading\nimport random\nimport typing\nimport logging\n\n\nclass PortManager__Generic(PortManager):\n _C_MIN_PORT_NUMBER = 1024\n _C_MAX_PORT_NUMBER = 65535\n\n _os_ops: OsOperations\n _guard: object\n # TODO: is there better to use bitmap fot _available_ports?\n _available_ports: typing.Set[int]\n _reserved_ports: typing.Set[int]\n\n def __init__(self, os_ops: OsOperations):\n assert __class__._C_MIN_PORT_NUMBER <= __class__._C_MAX_PORT_NUMBER\n\n assert os_ops is not None\n assert isinstance(os_ops, OsOperations)\n self._os_ops = os_ops\n self._guard = threading.Lock()\n\n self._available_ports = set(\n range(__class__._C_MIN_PORT_NUMBER, __class__._C_MAX_PORT_NUMBER + 1)\n )\n assert len(self._available_ports) == (\n (__class__._C_MAX_PORT_NUMBER - __class__._C_MIN_PORT_NUMBER) + 1\n )\n self._reserved_ports = set()\n return\n\n def reserve_port(self) -> int:\n assert self._guard is not None\n assert type(self._available_ports) is set\n assert type(self._reserved_ports) is set\n\n with self._guard:\n t = tuple(self._available_ports)\n assert len(t) == len(self._available_ports)\n sampled_ports = random.sample(t, min(len(t), 100))\n t = None\n\n for port in sampled_ports:\n assert type(port) is int\n assert port not in self._reserved_ports\n assert port in self._available_ports\n\n assert port >= __class__._C_MIN_PORT_NUMBER\n assert port <= __class__._C_MAX_PORT_NUMBER\n\n if not self._os_ops.is_port_free(port):\n continue\n\n self._reserved_ports.add(port)\n self._available_ports.discard(port)\n assert port in self._reserved_ports\n assert port not in self._available_ports\n __class__.helper__send_debug_msg(\"Port {} is reserved.\", port)\n return port\n\n raise PortForException(\"Can't select a port.\")\n\n def release_port(self, number: int) -> None:\n assert type(number) is int\n assert number >= __class__._C_MIN_PORT_NUMBER\n assert number <= __class__._C_MAX_PORT_NUMBER\n\n assert self._guard is not None\n assert type(self._reserved_ports) is set\n\n with self._guard:\n assert number in self._reserved_ports\n assert number not in self._available_ports\n self._available_ports.add(number)\n self._reserved_ports.discard(number)\n assert number not in self._reserved_ports\n assert number in self._available_ports\n __class__.helper__send_debug_msg(\"Port {} is released.\", number)\n return\n\n @staticmethod\n def helper__send_debug_msg(msg_template: str, *args) -> None:\n assert msg_template is not None\n assert args is not None\n assert type(msg_template) is str\n assert type(args) is tuple\n assert msg_template != \"\"\n s = \"[port manager] \"\n s += msg_template.format(*args)\n logging.debug(s)\n" }, { "path": "src/impl/port_manager__this_host.py", "content": "from ..port_manager import PortManager\n\nfrom .. import utils\n\nimport threading\n\n\nclass PortManager__ThisHost(PortManager):\n sm_single_instance: PortManager = None\n sm_single_instance_guard = threading.Lock()\n\n @staticmethod\n def get_single_instance() -> PortManager:\n assert __class__ == PortManager__ThisHost\n assert __class__.sm_single_instance_guard is not None\n\n if __class__.sm_single_instance is not None:\n assert type(__class__.sm_single_instance) is __class__\n return __class__.sm_single_instance\n\n with __class__.sm_single_instance_guard:\n if __class__.sm_single_instance is None:\n __class__.sm_single_instance = __class__()\n assert __class__.sm_single_instance is not None\n assert type(__class__.sm_single_instance) is __class__\n return __class__.sm_single_instance\n\n def reserve_port(self) -> int:\n return utils.reserve_port()\n\n def release_port(self, number: int) -> None:\n assert type(number) is int\n return utils.release_port(number)\n" }, { "path": "src/logger.py", "content": "# coding: utf-8\n\nimport logging\nimport select\nimport threading\nimport time\n\n\nclass TestgresLogger(threading.Thread):\n \"\"\"\n Helper class to implement reading from log files.\n \"\"\"\n def __init__(self, node_name, log_file_name):\n threading.Thread.__init__(self)\n\n self._node_name = node_name\n self._log_file_name = log_file_name\n self._stop_event = threading.Event()\n self._logger = logging.getLogger(node_name)\n self._logger.setLevel(logging.INFO)\n\n def run(self):\n # open log file for reading\n with open(self._log_file_name, 'r') as fd:\n # work until we're asked to stop\n while not self._stop_event.is_set():\n sleep_time = 0.1\n new_lines = False\n\n # do we have new lines?\n if fd in select.select([fd], [], [], 0)[0]:\n for line in fd.readlines():\n line = line.strip()\n if line:\n new_lines = True\n extra = {'node': self._node_name}\n self._logger.info(line, extra=extra)\n\n if not new_lines:\n time.sleep(sleep_time)\n\n # don't forget to clear event\n self._stop_event.clear()\n\n def stop(self, wait=True):\n self._stop_event.set()\n\n if wait:\n self.join()\n" }, { "path": "src/node.py", "content": "# coding: utf-8\nfrom __future__ import annotations\n\nimport logging\nimport os\nimport signal\nimport subprocess\n\nimport time\nimport typing\n\ntry:\n from collections.abc import Iterable\nexcept ImportError:\n from collections import Iterable\n\n# we support both pg8000 and psycopg2\ntry:\n import psycopg2 as pglib\nexcept ImportError:\n try:\n import pg8000 as pglib\n except ImportError:\n raise ImportError(\"You must have psycopg2 or pg8000 modules installed\")\n\nfrom six import raise_from, iteritems, text_type\n\nfrom .enums import \\\n NodeStatus, \\\n ProcessType, \\\n DumpFormat\n\nfrom .cache import cached_initdb\n\nfrom .config import testgres_config\n\nfrom .connection import NodeConnection\n\nfrom .consts import \\\n DATA_DIR, \\\n LOGS_DIR, \\\n TMP_NODE, \\\n TMP_DUMP, \\\n PG_CONF_FILE, \\\n PG_AUTO_CONF_FILE, \\\n HBA_CONF_FILE, \\\n RECOVERY_CONF_FILE, \\\n PG_LOG_FILE, \\\n UTILS_LOG_FILE\n\nfrom .consts import \\\n MAX_LOGICAL_REPLICATION_WORKERS, \\\n MAX_REPLICATION_SLOTS, \\\n MAX_WORKER_PROCESSES, \\\n MAX_WAL_SENDERS, \\\n WAL_KEEP_SEGMENTS, \\\n WAL_KEEP_SIZE\n\nfrom .decorators import \\\n method_decorator, \\\n positional_args_hack\n\nfrom .defaults import \\\n default_dbname, \\\n generate_app_name\n\nfrom .exceptions import \\\n CatchUpException, \\\n ExecUtilException, \\\n QueryException, \\\n QueryTimeoutException, \\\n StartNodeException, \\\n TimeoutException, \\\n InitNodeException, \\\n TestgresException, \\\n BackupException, \\\n InvalidOperationException\n\nfrom .port_manager import PortManager\nfrom .impl.port_manager__this_host import PortManager__ThisHost\nfrom .impl.port_manager__generic import PortManager__Generic\n\nfrom .logger import TestgresLogger\n\nfrom .pubsub import Publication, Subscription\n\nfrom .standby import First\n\nfrom . import utils\n\nfrom .utils import \\\n PgVer, \\\n eprint, \\\n get_bin_path2, \\\n get_pg_version2, \\\n execute_utility2, \\\n options_string, \\\n clean_on_error\n\nfrom .raise_error import RaiseError\n\nfrom .backup import NodeBackup\n\nfrom testgres.operations.os_ops import ConnectionParams\nfrom testgres.operations.os_ops import OsOperations\nfrom testgres.operations.local_ops import LocalOperations\n\nInternalError = pglib.InternalError\nProgrammingError = pglib.ProgrammingError\nOperationalError = pglib.OperationalError\n\n\nassert TimeoutException == QueryTimeoutException\n\n\nclass ProcessProxy(object):\n \"\"\"\n Wrapper for psutil.Process\n\n Attributes:\n process: wrapped psutill.Process object\n ptype: instance of ProcessType\n \"\"\"\n\n _process: typing.Any\n _ptype: ProcessType\n\n def __init__(self, process, ptype: typing.Optional[ProcessType] = None):\n assert process is not None\n assert ptype is None or type(ptype) is ProcessType\n self._process = process\n\n if ptype is not None:\n self._ptype = ptype\n else:\n self._ptype = ProcessType.from_process(process)\n assert type(self._ptype) is ProcessType\n return\n\n def __getattr__(self, name):\n return getattr(self.process, name)\n\n def __repr__(self):\n return '{}(ptype={}, process={})'.format(\n self.__class__.__name__,\n str(self.ptype),\n repr(self.process))\n\n @property\n def process(self) -> typing.Any:\n assert self._process is not None\n return self._process\n\n @property\n def ptype(self) -> ProcessType:\n assert type(self._ptype) is ProcessType\n return self._ptype\n\n\nclass PostgresNode(object):\n # a max number of node start attempts\n _C_MAX_START_ATEMPTS = 5\n\n _C_PM_PID__IS_NOT_DETECTED = -1\n\n _name: typing.Optional[str]\n _port: typing.Optional[int]\n _should_free_port: bool\n _os_ops: OsOperations\n _port_manager: typing.Optional[PortManager]\n _manually_started_pm_pid: typing.Optional[int]\n\n def __init__(self,\n name=None,\n base_dir=None,\n port: typing.Optional[int] = None,\n conn_params: typing.Optional[ConnectionParams] = None,\n bin_dir=None,\n prefix=None,\n os_ops: typing.Optional[OsOperations] = None,\n port_manager: typing.Optional[PortManager] = None):\n \"\"\"\n PostgresNode constructor.\n\n Args:\n name: node's application name.\n port: port to accept connections.\n base_dir: path to node's data directory.\n bin_dir: path to node's binary directory.\n os_ops: None or correct OS operation object.\n port_manager: None or correct port manager object.\n \"\"\"\n assert port is None or type(port) is int\n assert os_ops is None or isinstance(os_ops, OsOperations)\n assert port_manager is None or isinstance(port_manager, PortManager)\n\n if conn_params is not None:\n assert type(conn_params) is ConnectionParams\n\n raise InvalidOperationException(\"conn_params is deprecated, please use os_ops parameter instead.\")\n\n # private\n if os_ops is None:\n self._os_ops = __class__._get_os_ops()\n else:\n assert isinstance(os_ops, OsOperations)\n self._os_ops = os_ops\n pass\n\n assert self._os_ops is not None\n assert isinstance(self._os_ops, OsOperations)\n\n self._pg_version = PgVer(get_pg_version2(self._os_ops, bin_dir))\n self._base_dir = base_dir\n self._bin_dir = bin_dir\n self._prefix = prefix\n self._logger = None\n self._master = None\n\n # basic\n self._name = name or generate_app_name()\n\n if port is not None:\n assert type(port) is int\n assert port_manager is None\n self._port = port\n self._should_free_port = False\n self._port_manager = None\n else:\n if port_manager is None:\n self._port_manager = __class__._get_port_manager(self._os_ops)\n elif os_ops is None:\n raise InvalidOperationException(\"When port_manager is not None you have to define os_ops, too.\")\n else:\n assert isinstance(port_manager, PortManager)\n assert self._os_ops is os_ops\n self._port_manager = port_manager\n\n assert self._port_manager is not None\n assert isinstance(self._port_manager, PortManager)\n\n self._port = self._port_manager.reserve_port() # raises\n assert type(self._port) is int\n self._should_free_port = True\n\n assert type(self._port) is int\n\n # defaults for __exit__()\n self.cleanup_on_good_exit = testgres_config.node_cleanup_on_good_exit\n self.cleanup_on_bad_exit = testgres_config.node_cleanup_on_bad_exit\n self.shutdown_max_attempts = 3\n\n # NOTE: for compatibility\n self.utils_log_name = self.utils_log_file\n self.pg_log_name = self.pg_log_file\n\n # Node state\n self._manually_started_pm_pid = None\n\n def __enter__(self):\n return self\n\n def __exit__(self, type, value, traceback):\n # NOTE: Ctrl+C does not count!\n got_exception = type is not None and type is not KeyboardInterrupt\n\n c1 = self.cleanup_on_good_exit and not got_exception\n c2 = self.cleanup_on_bad_exit and got_exception\n\n attempts = self.shutdown_max_attempts\n\n if c1 or c2:\n self.cleanup(attempts)\n else:\n self._try_shutdown(attempts)\n\n self._release_resources()\n\n def __repr__(self):\n return \"{}(name='{}', port={}, base_dir='{}')\".format(\n self.__class__.__name__,\n self.name,\n str(self._port) if self._port is not None else \"None\",\n self.base_dir\n )\n\n @staticmethod\n def _get_os_ops() -> OsOperations:\n if testgres_config.os_ops:\n return testgres_config.os_ops\n\n return LocalOperations.get_single_instance()\n\n @staticmethod\n def _get_port_manager(os_ops: OsOperations) -> PortManager:\n assert os_ops is not None\n assert isinstance(os_ops, OsOperations)\n\n if os_ops is LocalOperations.get_single_instance():\n assert utils._old_port_manager is not None\n assert type(utils._old_port_manager) is PortManager__Generic\n assert utils._old_port_manager._os_ops is os_ops\n return PortManager__ThisHost.get_single_instance()\n\n # TODO: Throw the exception \"Please define a port manager.\" ?\n return PortManager__Generic(os_ops)\n\n def clone_with_new_name_and_base_dir(self, name: str, base_dir: str):\n assert name is None or type(name) is str\n assert base_dir is None or type(base_dir) is str\n\n assert __class__ == PostgresNode\n\n if self._port_manager is None:\n raise InvalidOperationException(\"PostgresNode without PortManager can't be cloned.\")\n\n assert self._port_manager is not None\n assert isinstance(self._port_manager, PortManager)\n assert self._os_ops is not None\n assert isinstance(self._os_ops, OsOperations)\n\n node = PostgresNode(\n name=name,\n base_dir=base_dir,\n bin_dir=self._bin_dir,\n prefix=self._prefix,\n os_ops=self._os_ops,\n port_manager=self._port_manager)\n\n return node\n\n @property\n def os_ops(self) -> OsOperations:\n assert self._os_ops is not None\n assert isinstance(self._os_ops, OsOperations)\n return self._os_ops\n\n @property\n def port_manager(self) -> typing.Optional[PortManager]:\n assert self._port_manager is None or isinstance(self._port_manager, PortManager)\n return self._port_manager\n\n @property\n def name(self) -> str:\n if self._name is None:\n raise InvalidOperationException(\"PostgresNode name is not defined.\")\n assert type(self._name) is str\n return self._name\n\n @property\n def host(self) -> str:\n assert self._os_ops is not None\n assert isinstance(self._os_ops, OsOperations)\n return self._os_ops.host\n\n @property\n def port(self) -> int:\n if self._port is None:\n raise InvalidOperationException(\"PostgresNode port is not defined.\")\n\n assert type(self._port) is int\n return self._port\n\n @property\n def ssh_key(self) -> typing.Optional[str]:\n assert self._os_ops is not None\n assert isinstance(self._os_ops, OsOperations)\n return self._os_ops.ssh_key\n\n @property\n def pid(self) -> int:\n \"\"\"\n Return postmaster's PID if node is running, else 0.\n \"\"\"\n\n x = self._get_node_state()\n assert type(x) is utils.PostgresNodeState\n\n if x.pid is None:\n assert x.node_status != NodeStatus.Running\n return 0\n\n assert x.node_status == NodeStatus.Running\n assert type(x.pid) is int\n return x.pid\n\n @property\n def is_started(self) -> bool:\n if self._manually_started_pm_pid is None:\n return False\n\n assert type(self._manually_started_pm_pid) is int\n return True\n\n @property\n def auxiliary_pids(self) -> typing.Dict[ProcessType, typing.List[int]]:\n \"\"\"\n Returns a dict of { ProcessType : PID }.\n \"\"\"\n\n result = {}\n\n for process in self.auxiliary_processes:\n assert type(process) is ProcessProxy\n if process.ptype not in result:\n result[process.ptype] = []\n\n result[process.ptype].append(process.pid)\n\n return result\n\n @property\n def auxiliary_processes(self) -> typing.List[ProcessProxy]:\n \"\"\"\n Returns a list of auxiliary processes.\n Each process is represented by :class:`.ProcessProxy` object.\n \"\"\"\n def is_aux(process: ProcessProxy) -> bool:\n assert type(process) is ProcessProxy\n return process.ptype != ProcessType.Unknown\n\n return list(filter(is_aux, self.child_processes))\n\n @property\n def child_processes(self) -> typing.List[ProcessProxy]:\n \"\"\"\n Returns a list of all child processes.\n Each process is represented by :class:`.ProcessProxy` object.\n \"\"\"\n\n # get a list of postmaster's children\n x = self._get_node_state()\n assert type(x) is utils.PostgresNodeState\n if x.pid is None:\n assert x.node_status != NodeStatus.Running\n RaiseError.node_err__cant_enumerate_child_processes(\n x.node_status\n )\n\n assert x.node_status == NodeStatus.Running\n assert type(x.pid) is int\n return self._get_child_processes(x.pid)\n\n def _get_child_processes(self, pid: int) -> typing.List[ProcessProxy]:\n assert type(pid) is int\n assert isinstance(self._os_ops, OsOperations)\n\n # get a list of postmaster's children\n children = self._os_ops.get_process_children(pid)\n\n return [ProcessProxy(p) for p in children]\n\n @property\n def source_walsender(self):\n \"\"\"\n Returns master's walsender feeding this replica.\n \"\"\"\n\n sql = \"\"\"\n select pid\n from pg_catalog.pg_stat_replication\n where application_name = %s\n \"\"\"\n\n if self.master is None:\n raise TestgresException(\"Node doesn't have a master\")\n\n assert type(self.master) is PostgresNode\n\n # master should be on the same host\n assert self.master.host == self.host\n\n with self.master.connect() as con:\n for row in con.execute(sql, self.name):\n for child in self.master.auxiliary_processes:\n if child.pid == int(row[0]):\n return child\n\n msg = \"Master doesn't send WAL to {}\".format(self.name)\n raise TestgresException(msg)\n\n @property\n def master(self):\n return self._master\n\n @property\n def base_dir(self):\n if not self._base_dir:\n self._base_dir = self.os_ops.mkdtemp(prefix=self._prefix or TMP_NODE)\n\n # NOTE: it's safe to create a new dir\n if not self.os_ops.path_exists(self._base_dir):\n self.os_ops.makedirs(self._base_dir)\n\n return self._base_dir\n\n @property\n def bin_dir(self):\n if not self._bin_dir:\n self._bin_dir = os.path.dirname(get_bin_path2(self.os_ops, \"pg_config\"))\n return self._bin_dir\n\n @property\n def logs_dir(self):\n assert self._os_ops is not None\n assert isinstance(self._os_ops, OsOperations)\n\n path = self._os_ops.build_path(self.base_dir, LOGS_DIR)\n assert type(path) is str\n\n # NOTE: it's safe to create a new dir\n if not self.os_ops.path_exists(path):\n self.os_ops.makedirs(path)\n\n return path\n\n @property\n def data_dir(self):\n assert self._os_ops is not None\n assert isinstance(self._os_ops, OsOperations)\n\n # NOTE: we can't run initdb without user's args\n path = self._os_ops.build_path(self.base_dir, DATA_DIR)\n assert type(path) is str\n return path\n\n @property\n def utils_log_file(self):\n assert self._os_ops is not None\n assert isinstance(self._os_ops, OsOperations)\n\n path = self._os_ops.build_path(self.logs_dir, UTILS_LOG_FILE)\n assert type(path) is str\n return path\n\n @property\n def pg_log_file(self):\n assert self._os_ops is not None\n assert isinstance(self._os_ops, OsOperations)\n\n path = self._os_ops.build_path(self.logs_dir, PG_LOG_FILE)\n assert type(path) is str\n return path\n\n @property\n def version(self):\n \"\"\"\n Return PostgreSQL version for this node.\n\n Returns:\n Instance of :class:`distutils.version.LooseVersion`.\n \"\"\"\n return self._pg_version\n\n def _try_shutdown(self, max_attempts, with_force=False):\n assert type(max_attempts) is int\n assert type(with_force) is bool\n assert max_attempts > 0\n\n attempts = 0\n\n # try stopping server N times\n while attempts < max_attempts:\n attempts += 1\n try:\n self.stop()\n except ExecUtilException:\n continue # one more time\n except Exception:\n eprint('cannot stop node {}'.format(self.name))\n break\n\n return # OK\n\n # If force stopping is enabled and PID is valid\n if not with_force:\n return\n\n node_pid = self.pid\n assert node_pid is not None\n assert type(node_pid) is int\n\n if node_pid == 0:\n return\n\n # TODO: [2025-02-28] It is really the old ugly code. We have to rewrite it!\n\n ps_command = ['ps', '-o', 'pid=', '-p', str(node_pid)]\n\n ps_output = self.os_ops.exec_command(cmd=ps_command, shell=True, ignore_errors=True).decode('utf-8')\n assert type(ps_output) is str\n\n if ps_output == \"\":\n return\n\n if ps_output != str(node_pid):\n __class__._throw_bugcheck__unexpected_result_of_ps(\n ps_output,\n ps_command)\n\n try:\n eprint('Force stopping node {0} with PID {1}'.format(self.name, node_pid))\n self.os_ops.kill(node_pid, signal.SIGKILL)\n except Exception:\n # The node has already stopped\n pass\n\n # Check that node stopped - print only column pid without headers\n ps_output = self.os_ops.exec_command(cmd=ps_command, shell=True, ignore_errors=True).decode('utf-8')\n assert type(ps_output) is str\n\n if ps_output == \"\":\n eprint('Node {0} has been stopped successfully.'.format(self.name))\n return\n\n if ps_output == str(node_pid):\n eprint('Failed to stop node {0}.'.format(self.name))\n return\n\n __class__._throw_bugcheck__unexpected_result_of_ps(\n ps_output,\n ps_command)\n\n @staticmethod\n def _throw_bugcheck__unexpected_result_of_ps(result, cmd):\n assert type(result) is str\n assert type(cmd) is list\n errLines = []\n errLines.append(\"[BUG CHECK] Unexpected result of command ps:\")\n errLines.append(result)\n errLines.append(\"-----\")\n errLines.append(\"Command line is {0}\".format(cmd))\n raise RuntimeError(\"\\n\".join(errLines))\n\n def _assign_master(self, master):\n \"\"\"NOTE: this is a private method!\"\"\"\n\n # now this node has a master\n self._master = master\n\n def _create_recovery_conf(self, username, slot=None):\n \"\"\"NOTE: this is a private method!\"\"\"\n\n # fetch master of this node\n master = self.master\n assert master is not None\n\n conninfo = {\n \"application_name\": self.name,\n \"port\": master.port,\n \"user\": username\n } # yapf: disable\n\n # host is tricky\n try:\n import ipaddress\n ipaddress.ip_address(master.host)\n conninfo[\"hostaddr\"] = master.host\n except ValueError:\n conninfo[\"host\"] = master.host\n\n line = (\n \"primary_conninfo='{}'\\n\"\n ).format(options_string(**conninfo)) # yapf: disable\n # Since 12 recovery.conf had disappeared\n if self.version >= PgVer('12'):\n assert self._os_ops is not None\n assert isinstance(self._os_ops, OsOperations)\n\n signal_name = self._os_ops.build_path(self.data_dir, \"standby.signal\")\n assert type(signal_name) is str\n self.os_ops.touch(signal_name)\n else:\n line += \"standby_mode=on\\n\"\n\n if slot:\n # Connect to master for some additional actions\n with master.connect(username=username) as con:\n # check if slot already exists\n res = con.execute(\n \"\"\"\n select exists (\n select from pg_catalog.pg_replication_slots\n where slot_name = %s\n )\n \"\"\", slot)\n\n if res[0][0]:\n raise TestgresException(\n \"Slot '{}' already exists\".format(slot))\n\n # TODO: we should drop this slot after replica's cleanup()\n con.execute(\n \"\"\"\n select pg_catalog.pg_create_physical_replication_slot(%s)\n \"\"\", slot)\n\n line += \"primary_slot_name={}\\n\".format(slot)\n\n if self.version >= PgVer('12'):\n self.append_conf(line=line)\n else:\n self.append_conf(filename=RECOVERY_CONF_FILE, line=line)\n\n def _maybe_start_logger(self):\n if testgres_config.use_python_logging:\n # spawn new logger if it doesn't exist or is stopped\n if not self._logger or not self._logger.is_alive():\n self._logger = TestgresLogger(self.name, self.pg_log_file)\n self._logger.start()\n\n def _maybe_stop_logger(self):\n if self._logger:\n self._logger.stop()\n\n def _collect_special_files(self) -> typing.List[typing.Tuple[str, bytes]]:\n result = []\n\n # list of important files + last N lines\n assert self._os_ops is not None\n assert isinstance(self._os_ops, OsOperations)\n\n files = [\n (self._os_ops.build_path(self.data_dir, PG_CONF_FILE), 0),\n (self._os_ops.build_path(self.data_dir, PG_AUTO_CONF_FILE), 0),\n (self._os_ops.build_path(self.data_dir, RECOVERY_CONF_FILE), 0),\n (self._os_ops.build_path(self.data_dir, HBA_CONF_FILE), 0),\n (self.pg_log_file, testgres_config.error_log_lines)\n ] # yapf: disable\n\n for f, num_lines in files:\n # skip missing files\n if not self.os_ops.path_exists(f):\n continue\n\n file_lines = self.os_ops.readlines(f, num_lines, binary=True, encoding=None)\n lines = b''.join(file_lines)\n\n # fill list\n result.append((f, lines))\n\n return result\n\n def init(self, initdb_params=None, cached=True, **kwargs):\n \"\"\"\n Perform initdb for this node.\n\n Args:\n initdb_params: parameters for initdb (list).\n fsync: should this node use fsync to keep data safe?\n unix_sockets: should we enable UNIX sockets?\n allow_streaming: should this node add a hba entry for replication?\n\n Returns:\n This instance of :class:`.PostgresNode`\n \"\"\"\n\n # initialize this PostgreSQL node\n assert self._os_ops is not None\n assert isinstance(self._os_ops, OsOperations)\n\n cached_initdb(\n data_dir=self.data_dir,\n logfile=self.utils_log_file,\n os_ops=self._os_ops,\n params=initdb_params,\n bin_path=self.bin_dir,\n cached=False)\n\n # initialize default config files\n self.default_conf(**kwargs)\n\n return self\n\n def default_conf(self,\n fsync=False,\n unix_sockets=True,\n allow_streaming=True,\n allow_logical=False,\n log_statement='all'):\n \"\"\"\n Apply default settings to this node.\n\n Args:\n fsync: should this node use fsync to keep data safe?\n unix_sockets: should we enable UNIX sockets?\n allow_streaming: should this node add a hba entry for replication?\n allow_logical: can this node be used as a logical replication publisher?\n log_statement: one of ('all', 'off', 'mod', 'ddl').\n\n Returns:\n This instance of :class:`.PostgresNode`.\n \"\"\"\n\n assert self._os_ops is not None\n assert isinstance(self._os_ops, OsOperations)\n\n postgres_conf = self._os_ops.build_path(self.data_dir, PG_CONF_FILE)\n hba_conf = self._os_ops.build_path(self.data_dir, HBA_CONF_FILE)\n\n # filter lines in hba file\n # get rid of comments and blank lines\n hba_conf_file = self.os_ops.readlines(hba_conf)\n lines = [\n s for s in hba_conf_file\n if len(s.strip()) > 0 and not s.startswith('#')\n ]\n\n # write filtered lines\n self.os_ops.write(hba_conf, lines, truncate=True)\n\n # replication-related settings\n if allow_streaming:\n # get auth method for host or local users\n def get_auth_method(t):\n return next((s.split()[-1]\n for s in lines if s.startswith(t)), 'trust')\n\n # get auth methods\n auth_local = get_auth_method('local')\n auth_host = get_auth_method('host')\n subnet_base = \".\".join(self.os_ops.host.split('.')[:-1] + ['0'])\n\n new_lines = [\n u\"local\\treplication\\tall\\t\\t\\t{}\\n\".format(auth_local),\n u\"host\\treplication\\tall\\t127.0.0.1/32\\t{}\\n\".format(auth_host),\n u\"host\\treplication\\tall\\t::1/128\\t\\t{}\\n\".format(auth_host),\n u\"host\\treplication\\tall\\t{}/24\\t\\t{}\\n\".format(subnet_base, auth_host),\n u\"host\\tall\\tall\\t{}/24\\t\\t{}\\n\".format(subnet_base, auth_host),\n u\"host\\tall\\tall\\tall\\t{}\\n\".format(auth_host),\n u\"host\\treplication\\tall\\tall\\t{}\\n\".format(auth_host)\n ] # yapf: disable\n\n # write missing lines\n self.os_ops.write(hba_conf, new_lines)\n\n # overwrite config file\n self.os_ops.write(postgres_conf, '', truncate=True)\n\n self.append_conf(fsync=fsync,\n max_worker_processes=MAX_WORKER_PROCESSES,\n log_statement=log_statement,\n listen_addresses=self.host,\n port=self.port) # yapf:disable\n\n # common replication settings\n if allow_streaming or allow_logical:\n self.append_conf(max_replication_slots=MAX_REPLICATION_SLOTS,\n max_wal_senders=MAX_WAL_SENDERS) # yapf: disable\n\n # binary replication\n if allow_streaming:\n # select a proper wal_level for PostgreSQL\n wal_level = 'replica' if self._pg_version >= PgVer('9.6') else 'hot_standby'\n\n if self._pg_version < PgVer('13'):\n self.append_conf(hot_standby=True,\n wal_keep_segments=WAL_KEEP_SEGMENTS,\n wal_level=wal_level) # yapf: disable\n else:\n self.append_conf(hot_standby=True,\n wal_keep_size=WAL_KEEP_SIZE,\n wal_level=wal_level) # yapf: disable\n\n # logical replication\n if allow_logical:\n if self._pg_version < PgVer('10'):\n raise InitNodeException(\"Logical replication is only \"\n \"available on PostgreSQL 10 and newer\")\n\n self.append_conf(\n max_logical_replication_workers=MAX_LOGICAL_REPLICATION_WORKERS,\n wal_level='logical')\n\n # disable UNIX sockets if asked to\n if not unix_sockets:\n self.append_conf(unix_socket_directories='')\n\n return self\n\n @method_decorator(positional_args_hack(['filename', 'line']))\n def append_conf(self, line='', filename=PG_CONF_FILE, **kwargs):\n \"\"\"\n Append line to a config file.\n\n Args:\n line: string to be appended to config.\n filename: config file (postgresql.conf by default).\n **kwargs: named config options.\n\n Returns:\n This instance of :class:`.PostgresNode`.\n\n Examples:\n >>> append_conf(fsync=False)\n >>> append_conf('log_connections = yes')\n >>> append_conf(random_page_cost=1.5, fsync=True, ...)\n >>> append_conf('postgresql.conf', 'synchronous_commit = off')\n \"\"\"\n\n lines = [line]\n\n for option, value in iteritems(kwargs):\n if isinstance(value, bool):\n value = 'on' if value else 'off'\n elif not str(value).replace('.', '', 1).isdigit():\n value = \"'{}'\".format(value)\n if value == '*':\n lines.append(\"{} = '*'\".format(option))\n else:\n # format a new config line\n lines.append('{} = {}'.format(option, value))\n\n config_name = self._os_ops.build_path(self.data_dir, filename)\n conf_text = ''\n for line in lines:\n conf_text += text_type(line) + '\\n'\n self.os_ops.write(config_name, conf_text)\n\n return self\n\n def status(self):\n \"\"\"\n Check this node's status.\n\n Returns:\n An instance of :class:`.NodeStatus`.\n \"\"\"\n x = self._get_node_state()\n assert type(x) is utils.PostgresNodeState\n return x.node_status\n\n def _get_node_state(self) -> utils.PostgresNodeState:\n return utils.get_pg_node_state(\n self._os_ops,\n self.bin_dir,\n self.data_dir,\n self.utils_log_file\n )\n\n def get_control_data(self):\n \"\"\"\n Return contents of pg_control file.\n \"\"\"\n\n # this one is tricky (blame PG 9.4)\n _params = [self._get_bin_path(\"pg_controldata\")]\n _params += [\"-D\"] if self._pg_version >= PgVer('9.5') else []\n _params += [self.data_dir]\n\n data = execute_utility2(self.os_ops, _params, self.utils_log_file)\n\n out_dict = {}\n\n for line in data.splitlines():\n key, _, value = line.partition(':')\n out_dict[key.strip()] = value.strip()\n\n return out_dict\n\n def slow_start(self, replica=False, dbname='template1', username=None, max_attempts=0, exec_env=None):\n \"\"\"\n Starts the PostgreSQL instance and then polls the instance\n until it reaches the expected state (primary or replica). The state is checked\n using the pg_is_in_recovery() function.\n\n Args:\n dbname:\n username:\n replica: If True, waits for the instance to be in recovery (i.e., replica mode).\n If False, waits for the instance to be in primary mode. Default is False.\n max_attempts:\n \"\"\"\n assert exec_env is None or type(exec_env) is dict\n\n self.start(exec_env=exec_env)\n\n try:\n if replica:\n query = 'SELECT pg_is_in_recovery()'\n else:\n query = 'SELECT not pg_is_in_recovery()'\n\n # Call poll_query_until until the expected value is returned\n suppressed_exceptions = {\n InternalError,\n QueryException,\n ProgrammingError,\n OperationalError\n }\n\n self.poll_query_until(\n query=query,\n dbname=dbname,\n username=username or self.os_ops.username,\n suppress=suppressed_exceptions,\n max_attempts=max_attempts,\n )\n except: # noqa: E722\n self.stop()\n raise\n return\n\n def start(\n self,\n params: typing.Optional[typing.List[str]] = None,\n wait: bool = True,\n exec_env: typing.Optional[typing.Dict] = None,\n ) -> PostgresNode:\n \"\"\"\n Starts the PostgreSQL node using pg_ctl and set flag 'is_started'.\n By default, it waits for the operation to complete before returning.\n Optionally, it can return immediately without waiting for the start operation\n to complete by setting the `wait` parameter to False.\n\n Args:\n params: additional arguments for pg_ctl.\n wait: wait until operation completes.\n\n Returns:\n This instance of :class:`.PostgresNode`.\n \"\"\"\n assert params is None or type(params) is list\n assert type(wait) is bool\n assert exec_env is None or type(exec_env) is dict\n\n self._start(params, wait, exec_env)\n\n if not wait:\n # Postmaster process is starting in background\n self._manually_started_pm_pid = __class__._C_PM_PID__IS_NOT_DETECTED\n else:\n self._manually_started_pm_pid = self._get_node_state().pid\n if self._manually_started_pm_pid is None:\n self._raise_cannot_start_node(None, \"Cannot detect postmaster pid.\")\n\n assert type(self._manually_started_pm_pid) is int\n return self\n\n def start2(\n self,\n params: typing.Optional[typing.List[str]] = None,\n wait: bool = True,\n exec_env: typing.Optional[typing.Dict] = None,\n ) -> None:\n \"\"\"\n Starts the PostgreSQL node using pg_ctl.\n By default, it waits for the operation to complete before returning.\n Optionally, it can return immediately without waiting for the start operation\n to complete by setting the `wait` parameter to False.\n\n Args:\n params: additional arguments for pg_ctl.\n wait: wait until operation completes.\n\n Returns:\n None.\n \"\"\"\n assert params is None or type(params) is list\n assert type(wait) is bool\n assert exec_env is None or type(exec_env) is dict\n\n self._start(params, wait, exec_env)\n return\n\n def _start(\n self,\n params: typing.Optional[typing.List[str]] = None,\n wait: bool = True,\n exec_env: typing.Optional[typing.Dict] = None,\n ) -> None:\n assert params is None or type(params) is list\n assert type(wait) is bool\n assert exec_env is None or type(exec_env) is dict\n\n assert __class__._C_MAX_START_ATEMPTS > 1\n\n if self._port is None:\n raise InvalidOperationException(\"Can't start PostgresNode. Port is not defined.\")\n\n assert type(self._port) is int\n\n _params = [\n self._get_bin_path(\"pg_ctl\"),\n \"start\",\n \"-D\", self.data_dir,\n \"-l\", self.pg_log_file,\n \"-w\" if wait else '-W', # --wait or --no-wait\n ]\n\n if params is not None:\n assert type(params) is list\n _params += params\n\n def LOCAL__start_node():\n # 'error' will be None on Windows\n _, _, error = execute_utility2(self.os_ops, _params, self.utils_log_file, verbose=True, exec_env=exec_env)\n assert error is None or type(error) is str\n if error and 'does not exist' in error:\n raise Exception(error)\n\n def LOCAL__raise_cannot_start_node__std(from_exception):\n assert isinstance(from_exception, Exception)\n self._raise_cannot_start_node(from_exception, 'Cannot start node')\n\n if not self._should_free_port:\n try:\n LOCAL__start_node()\n except Exception as e:\n LOCAL__raise_cannot_start_node__std(e)\n else:\n assert self._should_free_port\n assert self._port_manager is not None\n assert isinstance(self._port_manager, PortManager)\n assert __class__._C_MAX_START_ATEMPTS > 1\n\n log_reader = PostgresNodeLogReader(self, from_beginnig=False)\n\n nAttempt = 0\n timeout = 1\n while True:\n assert nAttempt >= 0\n assert nAttempt < __class__._C_MAX_START_ATEMPTS\n nAttempt += 1\n try:\n LOCAL__start_node()\n except Exception as e:\n assert nAttempt > 0\n assert nAttempt <= __class__._C_MAX_START_ATEMPTS\n if nAttempt == __class__._C_MAX_START_ATEMPTS:\n self._raise_cannot_start_node(e, \"Cannot start node after multiple attempts.\")\n\n is_it_port_conflict = PostgresNodeUtils.delect_port_conflict(log_reader)\n\n if not is_it_port_conflict:\n LOCAL__raise_cannot_start_node__std(e)\n\n logging.warning(\n \"Detected a conflict with using the port {0}. Trying another port after a {1}-second sleep...\".format(self._port, timeout)\n )\n time.sleep(timeout)\n timeout = min(2 * timeout, 5)\n cur_port = self._port\n new_port = self._port_manager.reserve_port() # can raise\n try:\n options = {'port': new_port}\n self.set_auto_conf(options)\n except: # noqa: E722\n self._port_manager.release_port(new_port)\n raise\n self._port = new_port\n self._port_manager.release_port(cur_port)\n continue\n break\n self._maybe_start_logger()\n return\n\n def _raise_cannot_start_node(\n self,\n from_exception: typing.Optional[Exception],\n msg: str\n ):\n assert from_exception is None or isinstance(from_exception, Exception)\n assert type(msg) is str\n files = self._collect_special_files()\n raise_from(StartNodeException(msg, files), from_exception)\n\n def stop(self, params=[], wait=True):\n \"\"\"\n Stops the PostgreSQL node using pg_ctl if the node has been started.\n\n Args:\n params: A list of additional arguments for pg_ctl. Defaults to None.\n wait: If True, waits until the operation is complete. Defaults to True.\n\n Returns:\n This instance of :class:`.PostgresNode`.\n \"\"\"\n _params = [\n self._get_bin_path(\"pg_ctl\"),\n \"-D\", self.data_dir,\n \"-w\" if wait else '-W', # --wait or --no-wait\n \"stop\"\n ] + params # yapf: disable\n\n execute_utility2(self.os_ops, _params, self.utils_log_file)\n\n self._manually_started_pm_pid = None\n\n self._maybe_stop_logger()\n return self\n\n def kill(self, someone=None):\n \"\"\"\n Kills the PostgreSQL node or a specified auxiliary process if the node is running.\n\n Args:\n someone: A key to the auxiliary process in the auxiliary_pids dictionary.\n If None, the main PostgreSQL node process will be killed. Defaults to None.\n \"\"\"\n x = self._get_node_state()\n assert type(x) is utils.PostgresNodeState\n\n if x.node_status != NodeStatus.Running:\n RaiseError.node_err__cant_kill(x.node_status)\n assert False\n\n assert x.node_status == NodeStatus.Running\n assert type(x.pid) is int\n sig = signal.SIGKILL if os.name != 'nt' else signal.SIGBREAK\n if someone is None:\n self._os_ops.kill(x.pid, sig)\n self._manually_started_pm_pid = None\n else:\n childs = self._get_child_processes(x.pid)\n for c in childs:\n assert type(c) is ProcessProxy\n if c.ptype == someone:\n self._os_ops.kill(c.process.pid, sig)\n continue\n return\n\n def restart(self, params=[]):\n \"\"\"\n Restart this node using pg_ctl.\n\n Args:\n params: additional arguments for pg_ctl.\n\n Returns:\n This instance of :class:`.PostgresNode`.\n \"\"\"\n\n _params = [\n self._get_bin_path(\"pg_ctl\"),\n \"-D\", self.data_dir,\n \"-l\", self.pg_log_file,\n \"-w\", # wait\n \"restart\"\n ] + params # yapf: disable\n\n try:\n error_code, out, error = execute_utility2(self.os_ops, _params, self.utils_log_file, verbose=True)\n if error and 'could not start server' in error:\n raise ExecUtilException\n except ExecUtilException as e:\n msg = 'Cannot restart node'\n files = self._collect_special_files()\n raise_from(StartNodeException(msg, files), e)\n\n self._maybe_start_logger()\n\n return self\n\n def reload(self, params=[]):\n \"\"\"\n Asynchronously reload config files using pg_ctl.\n\n Args:\n params: additional arguments for pg_ctl.\n\n Returns:\n This instance of :class:`.PostgresNode`.\n \"\"\"\n\n _params = [\n self._get_bin_path(\"pg_ctl\"),\n \"-D\", self.data_dir,\n \"reload\"\n ] + params # yapf: disable\n\n execute_utility2(self.os_ops, _params, self.utils_log_file)\n\n return self\n\n def promote(self, dbname=None, username=None):\n \"\"\"\n Promote standby instance to master using pg_ctl. For PostgreSQL versions\n below 10 some additional actions required to ensure that instance\n became writable and hence `dbname` and `username` parameters may be\n needed.\n\n Returns:\n This instance of :class:`.PostgresNode`.\n \"\"\"\n\n _params = [\n self._get_bin_path(\"pg_ctl\"),\n \"-D\", self.data_dir,\n \"-w\", # wait\n \"promote\"\n ] # yapf: disable\n\n execute_utility2(self.os_ops, _params, self.utils_log_file)\n\n # for versions below 10 `promote` is asynchronous so we need to wait\n # until it actually becomes writable\n if self._pg_version < PgVer('10'):\n check_query = \"SELECT pg_is_in_recovery()\"\n\n self.poll_query_until(query=check_query,\n expected=False,\n dbname=dbname,\n username=username,\n max_attempts=0) # infinite\n\n # node becomes master itself\n self._master = None\n\n return self\n\n def pg_ctl(self, params):\n \"\"\"\n Invoke pg_ctl with params.\n\n Args:\n params: arguments for pg_ctl.\n\n Returns:\n Stdout + stderr of pg_ctl.\n \"\"\"\n\n _params = [\n self._get_bin_path(\"pg_ctl\"),\n \"-D\", self.data_dir,\n \"-w\" # wait\n ] + params # yapf: disable\n\n return execute_utility2(self.os_ops, _params, self.utils_log_file)\n\n def release_resources(self):\n \"\"\"\n Release resorces owned by this node.\n \"\"\"\n return self._release_resources()\n\n def free_port(self):\n \"\"\"\n Reclaim port owned by this node.\n NOTE: this method does not release manually defined port but reset it.\n \"\"\"\n return self._free_port()\n\n def cleanup(self, max_attempts=3, full=False, release_resources=False):\n \"\"\"\n Stop node if needed and remove its data/logs directory.\n NOTE: take a look at TestgresConfig.node_cleanup_full.\n\n Args:\n max_attempts: how many times should we try to stop()?\n full: clean full base dir\n\n Returns:\n This instance of :class:`.PostgresNode`.\n \"\"\"\n\n self._try_shutdown(max_attempts)\n\n # choose directory to be removed\n if testgres_config.node_cleanup_full or full:\n rm_dir = self.base_dir # everything\n else:\n rm_dir = self.data_dir # just data, save logs\n\n self.os_ops.rmdirs(rm_dir, ignore_errors=False)\n\n if release_resources:\n self._release_resources()\n\n return self\n\n @method_decorator(positional_args_hack(['dbname', 'query']))\n def psql(self,\n query=None,\n filename=None,\n dbname=None,\n username=None,\n input=None,\n host: typing.Optional[str] = None,\n port: typing.Optional[int] = None,\n **variables):\n \"\"\"\n Execute a query using psql.\n\n Args:\n query: query to be executed.\n filename: file with a query.\n dbname: database name to connect to.\n username: database user name.\n input: raw input to be passed.\n host: an explicit host of server.\n port: an explicit port of server.\n **variables: vars to be set before execution.\n\n Returns:\n A tuple of (code, stdout, stderr).\n\n Examples:\n >>> psql('select 1')\n >>> psql('postgres', 'select 2')\n >>> psql(query='select 3', ON_ERROR_STOP=1)\n \"\"\"\n\n assert host is None or type(host) is str\n assert port is None or type(port) is int\n assert type(variables) is dict\n\n return self._psql(\n ignore_errors=True,\n query=query,\n filename=filename,\n dbname=dbname,\n username=username,\n input=input,\n host=host,\n port=port,\n **variables\n )\n\n def _psql(\n self,\n ignore_errors,\n query=None,\n filename=None,\n dbname=None,\n username=None,\n input=None,\n host: typing.Optional[str] = None,\n port: typing.Optional[int] = None,\n **variables):\n assert host is None or type(host) is str\n assert port is None or type(port) is int\n assert type(variables) is dict\n\n #\n # We do not support encoding. It may be added later. Ok?\n #\n if input is None:\n pass\n elif type(input) is bytes:\n pass\n else:\n raise Exception(\"Input data must be None or bytes.\")\n\n if host is None:\n host = self.host\n\n if port is None:\n port = self.port\n\n assert host is not None\n assert port is not None\n assert type(host) is str\n assert type(port) is int\n\n psql_params = [\n self._get_bin_path(\"psql\"),\n \"-p\", str(port),\n \"-h\", host,\n \"-U\", username or self.os_ops.username,\n \"-d\", dbname or default_dbname(),\n \"-X\", # no .psqlrc\n \"-A\", # unaligned output\n \"-t\", # print rows only\n \"-q\" # run quietly\n ] # yapf: disable\n\n # set variables before execution\n for key, value in iteritems(variables):\n psql_params.extend([\"--set\", '{}={}'.format(key, value)])\n\n # select query source\n if query:\n psql_params.extend((\"-c\", query))\n elif filename:\n psql_params.extend((\"-f\", filename))\n else:\n raise QueryException('Query or filename must be provided')\n\n return self.os_ops.exec_command(\n psql_params,\n verbose=True,\n input=input,\n stderr=subprocess.PIPE,\n stdout=subprocess.PIPE,\n ignore_errors=ignore_errors)\n\n @method_decorator(positional_args_hack(['dbname', 'query']))\n def safe_psql(self, query=None, expect_error=False, **kwargs):\n \"\"\"\n Execute a query using psql.\n\n Args:\n query: query to be executed.\n filename: file with a query.\n dbname: database name to connect to.\n username: database user name.\n input: raw input to be passed.\n expect_error: if True - fail if we didn't get ret\n if False - fail if we got ret\n\n **kwargs are passed to psql().\n\n Returns:\n psql's output as str.\n \"\"\"\n assert type(kwargs) is dict\n assert \"ignore_errors\" not in kwargs.keys()\n assert \"expect_error\" not in kwargs.keys()\n\n # force this setting\n kwargs['ON_ERROR_STOP'] = 1\n try:\n ret, out, err = self._psql(ignore_errors=False, query=query, **kwargs)\n except ExecUtilException as e:\n if not expect_error:\n raise QueryException(e.message, query)\n\n if type(e.error) is bytes:\n return e.error.decode(\"utf-8\") # throw\n\n # [2024-12-09] This situation is not expected\n assert False\n return e.error\n\n if expect_error:\n raise InvalidOperationException(\"Exception was expected, but query finished successfully: `{}`.\".format(query))\n\n return out\n\n def dump(self,\n filename=None,\n dbname=None,\n username=None,\n format=DumpFormat.Plain,\n options=None):\n \"\"\"\n Dump database into a file using pg_dump.\n NOTE: the file is not removed automatically.\n\n Args:\n filename: database dump taken by pg_dump.\n dbname: database name to connect to.\n username: database user name.\n format: format argument plain/custom/directory/tar.\n options: additional options for pg_dump (list).\n\n Returns:\n Path to a file containing dump.\n \"\"\"\n\n # Check arguments\n if not isinstance(format, DumpFormat):\n try:\n format = DumpFormat(format)\n except ValueError:\n msg = 'Invalid format \"{}\"'.format(format)\n raise BackupException(msg)\n\n # Generate tmpfile or tmpdir\n def tmpfile():\n if format == DumpFormat.Directory:\n fname = self.os_ops.mkdtemp(prefix=TMP_DUMP)\n else:\n fname = self.os_ops.mkstemp(prefix=TMP_DUMP)\n return fname\n\n filename = filename or tmpfile()\n\n _params = [\n self._get_bin_path(\"pg_dump\"),\n \"-p\", str(self.port),\n \"-h\", self.host,\n \"-f\", filename,\n \"-U\", username or self.os_ops.username,\n \"-d\", dbname or default_dbname(),\n \"-F\", format.value\n ] # yapf: disable\n\n # Add additional options if provided\n if options:\n _params.extend(options)\n\n execute_utility2(self.os_ops, _params, self.utils_log_file)\n\n return filename\n\n def restore(self, filename, dbname=None, username=None):\n \"\"\"\n Restore database from pg_dump's file.\n\n Args:\n filename: database dump taken by pg_dump in custom/directory/tar formats.\n dbname: database name to connect to.\n username: database user name.\n \"\"\"\n\n # Set default arguments\n dbname = dbname or default_dbname()\n username = username or self.os_ops.username\n\n _params = [\n self._get_bin_path(\"pg_restore\"),\n \"-p\", str(self.port),\n \"-h\", self.host,\n \"-U\", username,\n \"-d\", dbname,\n filename\n ] # yapf: disable\n\n # try pg_restore if dump is binary format, and psql if not\n try:\n execute_utility2(self.os_ops, _params, self.utils_log_name)\n except ExecUtilException:\n self.psql(filename=filename, dbname=dbname, username=username)\n\n @method_decorator(positional_args_hack(['dbname', 'query']))\n def poll_query_until(self,\n query,\n dbname=None,\n username=None,\n max_attempts=0,\n sleep_time: typing.Union[int, float] = 1,\n expected=True,\n commit=True,\n suppress=None):\n \"\"\"\n Run a query once per second until it returns 'expected'.\n Query should return a single value (1 row, 1 column).\n\n Args:\n query: query to be executed.\n dbname: database name to connect to.\n username: database user name.\n max_attempts: how many times should we try? 0 == infinite\n sleep_time: how much should we sleep after a failure?\n expected: what should be returned to break the cycle?\n commit: should (possible) changes be committed?\n suppress: a collection of exceptions to be suppressed.\n\n Examples:\n >>> poll_query_until('select true')\n >>> poll_query_until('postgres', \"select now() > '01.01.2018'\")\n >>> poll_query_until('select false', expected=True, max_attempts=4)\n >>> poll_query_until('select 1', suppress={testgres.OperationalError})\n \"\"\"\n\n # sanity checks\n assert type(max_attempts) is int\n assert max_attempts >= 0\n assert type(sleep_time) in [int, float]\n assert sleep_time > 0\n attempts = 0\n while max_attempts == 0 or attempts < max_attempts:\n try:\n res = self.execute(dbname=dbname,\n query=query,\n username=username,\n commit=commit)\n\n if expected is None and res is None:\n return # done\n\n if res is None:\n raise QueryException('Query returned None', query)\n\n # result set is not empty\n if len(res):\n if len(res[0]) == 0:\n raise QueryException('Query returned 0 columns', query)\n if res[0][0] == expected:\n return # done\n # empty result set is considered as None\n elif expected is None:\n return # done\n\n except tuple(suppress or []):\n logging.info(f\"Trying execute, attempt {attempts + 1}.\\nQuery: {query}\")\n pass # we're suppressing them\n\n time.sleep(sleep_time)\n attempts += 1\n\n raise QueryTimeoutException('Query timeout', query)\n\n @method_decorator(positional_args_hack(['dbname', 'query']))\n def execute(self,\n query,\n dbname=None,\n username=None,\n password=None,\n commit=True):\n \"\"\"\n Execute a query and return all rows as list.\n\n Args:\n query: query to be executed.\n dbname: database name to connect to.\n username: database user name.\n password: user's password.\n commit: should we commit this query?\n\n Returns:\n A list of tuples representing rows.\n \"\"\"\n\n with self.connect(dbname=dbname,\n username=username,\n password=password,\n autocommit=commit) as node_con: # yapf: disable\n\n res = node_con.execute(query)\n\n return res\n\n def backup(self, **kwargs):\n \"\"\"\n Perform pg_basebackup.\n\n Args:\n username: database user name.\n xlog_method: a method for collecting the logs ('fetch' | 'stream').\n base_dir: the base directory for data files and logs\n\n Returns:\n A smart object of type NodeBackup.\n \"\"\"\n\n return NodeBackup(node=self, **kwargs)\n\n def replicate(self, name=None, slot=None, **kwargs):\n \"\"\"\n Create a binary replica of this node.\n\n Args:\n name: replica's application name.\n slot: create a replication slot with the specified name.\n username: database user name.\n xlog_method: a method for collecting the logs ('fetch' | 'stream').\n base_dir: the base directory for data files and logs\n \"\"\"\n\n # transform backup into a replica\n with clean_on_error(self.backup(**kwargs)) as backup:\n return backup.spawn_replica(name=name, destroy=True, slot=slot)\n\n def set_synchronous_standbys(self, standbys):\n \"\"\"\n Set standby synchronization options. This corresponds to\n `synchronous_standby_names `_\n option. Note that :meth:`~.PostgresNode.reload` or\n :meth:`~.PostgresNode.restart` is needed for changes to take place.\n\n Args:\n standbys: either :class:`.First` or :class:`.Any` object specifying\n synchronization parameters or just a plain list of\n :class:`.PostgresNode`s replicas which would be equivalent\n to passing ``First(1, )``. For PostgreSQL 9.5 and below\n it is only possible to specify a plain list of standbys as\n `FIRST` and `ANY` keywords aren't supported.\n\n Example::\n\n from testgres import get_new_node, First\n\n master = get_new_node().init().start()\n with master.replicate().start() as standby:\n master.append_conf(\"synchronous_commit = remote_apply\")\n master.set_synchronous_standbys(First(1, [standby]))\n master.restart()\n\n \"\"\"\n if self._pg_version >= PgVer('9.6'):\n if isinstance(standbys, Iterable):\n standbys = First(1, standbys)\n else:\n if isinstance(standbys, Iterable):\n standbys = u\", \".join(u\"\\\"{}\\\"\".format(r.name)\n for r in standbys)\n else:\n raise TestgresException(\"Feature isn't supported in \"\n \"Postgres 9.5 and below\")\n\n self.append_conf(\"synchronous_standby_names = '{}'\".format(standbys))\n\n def catchup(self, dbname=None, username=None):\n \"\"\"\n Wait until async replica catches up with its master.\n \"\"\"\n\n if not self.master:\n raise TestgresException(\"Node doesn't have a master\")\n\n if self._pg_version >= PgVer('10'):\n poll_lsn = \"select pg_catalog.pg_current_wal_lsn()::text\"\n wait_lsn = \"select pg_catalog.pg_last_wal_replay_lsn() >= '{}'::pg_lsn\"\n else:\n poll_lsn = \"select pg_catalog.pg_current_xlog_location()::text\"\n wait_lsn = \"select pg_catalog.pg_last_xlog_replay_location() >= '{}'::pg_lsn\"\n\n try:\n # fetch latest LSN\n lsn = self.master.execute(query=poll_lsn,\n dbname=dbname,\n username=username)[0][0] # yapf: disable\n\n # wait until this LSN reaches replica\n self.poll_query_until(query=wait_lsn.format(lsn),\n dbname=dbname,\n username=username,\n max_attempts=0) # infinite\n except Exception as e:\n raise_from(CatchUpException(\"Failed to catch up.\"), e)\n\n def publish(self, name, **kwargs):\n \"\"\"\n Create publication for logical replication\n\n Args:\n pubname: publication name\n tables: tables names list\n dbname: database name where objects or interest are located\n username: replication username\n \"\"\"\n return Publication(name=name, node=self, **kwargs)\n\n def subscribe(self,\n publication,\n name,\n dbname=None,\n username=None,\n **params):\n \"\"\"\n Create subscription for logical replication\n\n Args:\n name: subscription name\n publication: publication object obtained from publish()\n dbname: database name\n username: replication username\n params: subscription parameters (see documentation on `CREATE SUBSCRIPTION\n `_\n for details)\n \"\"\"\n # yapf: disable\n return Subscription(name=name, node=self, publication=publication,\n dbname=dbname, username=username, **params)\n # yapf: enable\n\n def pgbench(self,\n dbname=None,\n username=None,\n stdout=None,\n stderr=None,\n options=None):\n \"\"\"\n Spawn a pgbench process.\n\n Args:\n dbname: database name to connect to.\n username: database user name.\n stdout: stdout file to be used by Popen.\n stderr: stderr file to be used by Popen.\n options: additional options for pgbench (list).\n\n Returns:\n Process created by subprocess.Popen.\n \"\"\"\n if options is None:\n options = []\n\n dbname = dbname or default_dbname()\n\n _params = [\n self._get_bin_path(\"pgbench\"),\n \"-p\", str(self.port),\n \"-h\", self.host,\n \"-U\", username or self.os_ops.username\n ] + options # yapf: disable\n\n # should be the last one\n _params.append(dbname)\n\n proc = self.os_ops.exec_command(_params, stdout=stdout, stderr=stderr, wait_exit=True, get_process=True)\n\n return proc\n\n def pgbench_with_wait(self,\n dbname=None,\n username=None,\n stdout=None,\n stderr=None,\n options=None):\n \"\"\"\n Do pgbench command and wait.\n\n Args:\n dbname: database name to connect to.\n username: database user name.\n stdout: stdout file to be used by Popen.\n stderr: stderr file to be used by Popen.\n options: additional options for pgbench (list).\n \"\"\"\n if options is None:\n options = []\n\n with self.pgbench(dbname, username, stdout, stderr, options) as pgbench:\n pgbench.wait()\n return\n\n def pgbench_init(self, **kwargs):\n \"\"\"\n Small wrapper for pgbench_run().\n Sets initialize=True.\n\n Returns:\n This instance of :class:`.PostgresNode`.\n \"\"\"\n\n self.pgbench_run(initialize=True, **kwargs)\n\n return self\n\n def pgbench_run(self, dbname=None, username=None, options=[], **kwargs):\n \"\"\"\n Run pgbench with some options.\n This event is logged (see self.utils_log_file).\n\n Args:\n dbname: database name to connect to.\n username: database user name.\n options: additional options for pgbench (list).\n\n **kwargs: named options for pgbench.\n Run pgbench --help to learn more.\n\n Returns:\n Stdout produced by pgbench.\n\n Examples:\n >>> pgbench_run(initialize=True, scale=2)\n >>> pgbench_run(time=10)\n \"\"\"\n\n dbname = dbname or default_dbname()\n\n _params = [\n self._get_bin_path(\"pgbench\"),\n \"-p\", str(self.port),\n \"-h\", self.host,\n \"-U\", username or self.os_ops.username\n ] + options # yapf: disable\n\n for key, value in iteritems(kwargs):\n # rename keys for pgbench\n key = key.replace('_', '-')\n\n # append option\n if not isinstance(value, bool):\n _params.append('--{}={}'.format(key, value))\n else:\n assert value is True # just in case\n _params.append('--{}'.format(key))\n\n # should be the last one\n _params.append(dbname)\n\n return execute_utility2(self.os_ops, _params, self.utils_log_file)\n\n def connect(self,\n dbname=None,\n username=None,\n password=None,\n autocommit=False):\n \"\"\"\n Connect to a database.\n\n Args:\n dbname: database name to connect to.\n username: database user name.\n password: user's password.\n autocommit: commit each statement automatically. Also it should be\n set to `True` for statements requiring to be run outside\n a transaction? such as `VACUUM` or `CREATE DATABASE`.\n\n Returns:\n An instance of :class:`.NodeConnection`.\n \"\"\"\n\n return NodeConnection(node=self,\n dbname=dbname,\n username=username,\n password=password,\n autocommit=autocommit) # yapf: disable\n\n def table_checksum(\n self,\n table: str,\n dbname: str = \"postgres\"\n ) -> int:\n assert type(table) is str\n assert type(dbname) is str\n\n cn = self.connect(dbname=dbname)\n assert type(cn) is NodeConnection\n\n try:\n sum = __class__._table_checksum__use_cn(cn, table)\n assert type(sum) is int\n finally:\n assert type(cn) is NodeConnection\n cn.close()\n\n assert type(sum) is int\n return sum\n\n sm_pgbench_tables = [\n 'pgbench_branches',\n 'pgbench_tellers',\n 'pgbench_accounts',\n 'pgbench_history'\n ]\n\n def pgbench_table_checksums(\n self,\n dbname: str = \"postgres\",\n pgbench_tables: typing.Iterable[str] = sm_pgbench_tables\n ) -> typing.Set[typing.Tuple[str, int]]:\n assert type(dbname) is str\n\n r1 = self._tables_checksum(dbname, pgbench_tables)\n assert type(r1) is list\n\n r2 = set(r1)\n assert type(r2) is set\n return r2\n\n def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}):\n \"\"\"\n Update or remove configuration options in the specified configuration file,\n updates the options specified in the options dictionary, removes any options\n specified in the rm_options set, and writes the updated configuration back to\n the file.\n\n Args:\n options (dict): A dictionary containing the options to update or add,\n with the option names as keys and their values as values.\n config (str, optional): The name of the configuration file to update.\n Defaults to 'postgresql.auto.conf'.\n rm_options (set, optional): A set containing the names of the options to remove.\n Defaults to an empty set.\n \"\"\"\n assert self._os_ops is not None\n assert isinstance(self._os_ops, OsOperations)\n\n # parse postgresql.auto.conf\n path = self.os_ops.build_path(self.data_dir, config)\n\n lines = self.os_ops.readlines(path)\n current_options = {}\n current_directives = []\n for line in lines:\n\n # ignore comments\n if line.startswith('#'):\n continue\n\n if line.strip() == '':\n continue\n\n if line.startswith('include'):\n current_directives.append(line)\n continue\n\n name, var = line.partition('=')[::2]\n name = name.strip()\n\n # Remove options specified in rm_options list\n if name in rm_options:\n continue\n\n current_options[name] = var\n\n for option in options:\n assert type(option) is str\n assert option != \"\"\n assert option.strip() == option\n\n value = options[option]\n valueType = type(value)\n\n if valueType is str:\n value = __class__._escape_config_value(value)\n elif valueType is bool:\n value = \"on\" if value else \"off\"\n\n current_options[option] = value\n\n auto_conf = ''\n for option in current_options:\n auto_conf += option + \" = \" + str(current_options[option]) + \"\\n\"\n\n for directive in current_directives:\n auto_conf += directive + \"\\n\"\n\n self.os_ops.write(path, auto_conf, truncate=True)\n\n def upgrade_from(self, old_node, options=None, expect_error=False):\n \"\"\"\n Upgrade this node from an old node using pg_upgrade.\n\n Args:\n old_node: An instance of PostgresNode representing the old node.\n \"\"\"\n assert isinstance(self._os_ops, OsOperations)\n if not self._os_ops.path_exists(old_node.data_dir):\n raise Exception(\"Old node must be initialized\")\n\n if not self._os_ops.path_exists(self.data_dir):\n self.init()\n\n if not options:\n options = []\n\n pg_upgrade_binary = self._get_bin_path(\"pg_upgrade\")\n\n if not self._os_ops.path_exists(pg_upgrade_binary):\n raise Exception(\"pg_upgrade does not exist in the new node's binary path\")\n\n upgrade_command = [\n pg_upgrade_binary,\n \"--old-bindir\", old_node.bin_dir,\n \"--new-bindir\", self.bin_dir,\n \"--old-datadir\", old_node.data_dir,\n \"--new-datadir\", self.data_dir,\n \"--old-port\", str(old_node.port),\n \"--new-port\", str(self.port)\n ]\n upgrade_command += options\n\n return self.os_ops.exec_command(upgrade_command, expect_error=expect_error)\n\n def _release_resources(self):\n self._free_port()\n\n def _free_port(self):\n assert type(self._should_free_port) is bool\n\n if not self._should_free_port:\n self._port = None\n else:\n assert type(self._port) is int\n\n assert self._port_manager is not None\n assert isinstance(self._port_manager, PortManager)\n\n port = self._port\n self._should_free_port = False\n self._port = None\n self._port_manager.release_port(port)\n\n def _get_bin_path(self, filename):\n assert self._os_ops is not None\n assert isinstance(self._os_ops, OsOperations)\n\n if self.bin_dir:\n bin_path = self._os_ops.build_path(self.bin_dir, filename)\n else:\n bin_path = get_bin_path2(self.os_ops, filename)\n return bin_path\n\n @staticmethod\n def _escape_config_value(value):\n assert type(value) is str\n\n result = \"'\"\n\n for ch in value:\n if ch == \"'\":\n result += \"\\\\'\"\n elif ch == \"\\n\":\n result += \"\\\\n\"\n elif ch == \"\\r\":\n result += \"\\\\r\"\n elif ch == \"\\t\":\n result += \"\\\\t\"\n elif ch == \"\\b\":\n result += \"\\\\b\"\n elif ch == \"\\\\\":\n result += \"\\\\\\\\\"\n else:\n result += ch\n\n result += \"'\"\n return result\n\n def _tables_checksum(\n self,\n dbname: str,\n tables: typing.Iterable[str],\n ) -> typing.List[typing.Tuple[str, int]]:\n assert isinstance(tables, typing.Iterable)\n assert type(dbname) is str\n\n result = []\n\n cn = self.connect(dbname=dbname)\n assert type(cn) is NodeConnection\n\n try:\n cn.begin()\n\n for table in tables:\n assert type(table) is str\n sum = __class__._table_checksum__use_cn(cn, table)\n assert type(sum) is int\n result.append((table, sum))\n\n cn.commit()\n finally:\n assert type(cn) is NodeConnection\n cn.close()\n\n assert type(result) is list\n return result\n\n @staticmethod\n def _table_checksum__use_cn(\n cn: NodeConnection,\n table: str,\n ) -> int:\n assert type(cn) is NodeConnection\n assert type(table) is str\n\n sum = 0\n\n cursor = cn.connection.cursor()\n assert cursor is not None\n\n try:\n cursor.execute(\"SELECT SUM(hashtext(t::text)) FROM {} as t\".format(\n __class__._delim_sql_ident(table)\n ))\n\n row = cursor.fetchone()\n assert row is not None\n assert type(row) in [list, tuple]\n assert len(row) == 1\n v = row[0]\n sum += int(v if v is not None else 0)\n finally:\n cursor.close()\n\n assert type(sum) is int\n return sum\n\n @staticmethod\n def _delim_sql_ident(name: str) -> str:\n assert isinstance(name, str)\n\n result = '\"'\n\n for ch in name:\n if ch == '\"':\n result = result + '\"\"'\n else:\n result = result + ch\n\n result = result + '\"'\n\n return result\n\n\nclass PostgresNodeLogReader:\n class LogInfo:\n position: int\n\n def __init__(self, position: int):\n self.position = position\n\n # --------------------------------------------------------------------\n class LogDataBlock:\n _file_name: str\n _position: int\n _data: str\n\n def __init__(\n self,\n file_name: str,\n position: int,\n data: str\n ):\n assert type(file_name) is str\n assert type(position) is int\n assert type(data) is str\n assert file_name != \"\"\n assert position >= 0\n self._file_name = file_name\n self._position = position\n self._data = data\n\n @property\n def file_name(self) -> str:\n assert type(self._file_name) is str\n assert self._file_name != \"\"\n return self._file_name\n\n @property\n def position(self) -> int:\n assert type(self._position) is int\n assert self._position >= 0\n return self._position\n\n @property\n def data(self) -> str:\n assert type(self._data) is str\n return self._data\n\n # --------------------------------------------------------------------\n _node: PostgresNode\n _logs: typing.Dict[str, LogInfo]\n\n # --------------------------------------------------------------------\n def __init__(self, node: PostgresNode, from_beginnig: bool):\n assert node is not None\n assert isinstance(node, PostgresNode)\n assert type(from_beginnig) is bool\n\n self._node = node\n\n if from_beginnig:\n self._logs = dict()\n else:\n self._logs = self._collect_logs()\n\n assert type(self._logs) is dict\n return\n\n def read(self) -> typing.List[LogDataBlock]:\n assert self._node is not None\n assert isinstance(self._node, PostgresNode)\n\n cur_logs: typing.Dict[str, __class__.LogInfo] = self._collect_logs()\n assert cur_logs is not None\n assert type(cur_logs) is dict\n\n assert type(self._logs) is dict\n\n result = list()\n\n for file_name, cur_log_info in cur_logs.items():\n assert type(file_name) is str\n assert type(cur_log_info) is __class__.LogInfo\n\n read_pos = 0\n\n if file_name in self._logs.keys():\n prev_log_info = self._logs[file_name]\n assert type(prev_log_info) is __class__.LogInfo\n read_pos = prev_log_info.position # the previous size\n\n file_content_b = self._node.os_ops.read_binary(file_name, read_pos)\n assert type(file_content_b) is bytes\n\n #\n # A POTENTIAL PROBLEM: file_content_b may contain an incompleted UTF-8 symbol.\n #\n file_content_s = file_content_b.decode()\n assert type(file_content_s) is str\n\n next_read_pos = read_pos + len(file_content_b)\n\n # It is a research/paranoja check.\n # When we will process partial UTF-8 symbol, it must be adjusted.\n assert cur_log_info.position <= next_read_pos\n\n cur_log_info.position = next_read_pos\n\n block = __class__.LogDataBlock(\n file_name,\n read_pos,\n file_content_s\n )\n\n result.append(block)\n\n # A new check point\n self._logs = cur_logs\n\n return result\n\n def _collect_logs(self) -> typing.Dict[str, LogInfo]:\n assert self._node is not None\n assert isinstance(self._node, PostgresNode)\n\n files = [\n self._node.pg_log_file\n ] # yapf: disable\n\n result = dict()\n\n for f in files:\n assert type(f) is str\n\n # skip missing files\n if not self._node.os_ops.path_exists(f):\n continue\n\n file_size = self._node.os_ops.get_file_size(f)\n assert type(file_size) is int\n assert file_size >= 0\n\n result[f] = __class__.LogInfo(file_size)\n\n return result\n\n\nclass PostgresNodeUtils:\n @staticmethod\n def delect_port_conflict(log_reader: PostgresNodeLogReader) -> bool:\n assert type(log_reader) is PostgresNodeLogReader\n\n blocks = log_reader.read()\n assert type(blocks) is list\n\n for block in blocks:\n assert type(block) is PostgresNodeLogReader.LogDataBlock\n\n if 'Is another postmaster already running on port' in block.data:\n return True\n\n return False\n" }, { "path": "src/node_app.py", "content": "from .node import OsOperations\nfrom .node import LocalOperations\nfrom .node import PostgresNode\nfrom .node import PortManager\n\nimport os\nimport platform\nimport tempfile\nimport typing\n\n\nT_DICT_STR_STR = typing.Dict[str, str]\nT_LIST_STR = typing.List[str]\n\n\nclass NodeApp:\n _test_path: str\n _os_ops: OsOperations\n _port_manager: PortManager\n _nodes_to_cleanup: typing.List[PostgresNode]\n\n def __init__(\n self,\n test_path: typing.Optional[str] = None,\n nodes_to_cleanup: typing.Optional[list] = None,\n os_ops: typing.Optional[OsOperations] = None,\n port_manager: typing.Optional[PortManager] = None,\n ):\n assert test_path is None or type(test_path) is str\n assert os_ops is None or isinstance(os_ops, OsOperations)\n assert port_manager is None or isinstance(port_manager, PortManager)\n\n if os_ops is None:\n os_ops = LocalOperations.get_single_instance()\n\n assert isinstance(os_ops, OsOperations)\n self._os_ops = os_ops\n self._port_manager = port_manager\n\n if test_path is None:\n self._test_path = os_ops.cwd()\n elif os.path.isabs(test_path):\n self._test_path = test_path\n else:\n self._test_path = os_ops.build_path(os_ops.cwd(), test_path)\n\n if nodes_to_cleanup is None:\n self._nodes_to_cleanup = []\n else:\n self._nodes_to_cleanup = nodes_to_cleanup\n\n @property\n def test_path(self) -> str:\n assert type(self._test_path) is str\n return self._test_path\n\n @property\n def os_ops(self) -> OsOperations:\n assert isinstance(self._os_ops, OsOperations)\n return self._os_ops\n\n @property\n def port_manager(self) -> PortManager:\n assert self._port_manager is None or isinstance(self._port_manager, PortManager)\n return self._port_manager\n\n @property\n def nodes_to_cleanup(self) -> typing.List[PostgresNode]:\n assert type(self._nodes_to_cleanup) is list\n return self._nodes_to_cleanup\n\n def make_empty(\n self,\n base_dir: str,\n port: typing.Optional[int] = None,\n bin_dir: typing.Optional[str] = None\n ) -> PostgresNode:\n assert type(base_dir) is str\n assert port is None or type(port) is int\n assert bin_dir is None or type(bin_dir) is str\n\n assert isinstance(self._os_ops, OsOperations)\n assert type(self._test_path) is str\n\n if base_dir is None:\n raise ValueError(\"Argument 'base_dir' is not defined.\")\n\n if base_dir == \"\":\n raise ValueError(\"Argument 'base_dir' is empty.\")\n\n real_base_dir = self._os_ops.build_path(self._test_path, base_dir)\n self._os_ops.rmdirs(real_base_dir, ignore_errors=True)\n self._os_ops.makedirs(real_base_dir)\n\n port_manager: typing.Optional[PortManager] = None\n\n if port is None:\n port_manager = self._port_manager\n\n node = PostgresNode(\n base_dir=real_base_dir,\n port=port,\n bin_dir=bin_dir,\n os_ops=self._os_ops,\n port_manager=port_manager\n )\n\n try:\n assert type(self._nodes_to_cleanup) is list\n self._nodes_to_cleanup.append(node)\n except: # noqa: E722\n node.cleanup(release_resources=True)\n raise\n\n return node\n\n def make_simple(\n self,\n base_dir: str,\n port: typing.Optional[int] = None,\n set_replication: bool = False,\n ptrack_enable: bool = False,\n initdb_params: typing.Optional[T_LIST_STR] = None,\n pg_options: typing.Optional[T_DICT_STR_STR] = None,\n checksum: bool = True,\n bin_dir: typing.Optional[str] = None\n ) -> PostgresNode:\n assert type(base_dir) is str\n assert port is None or type(port) is int\n assert type(set_replication) is bool\n assert type(ptrack_enable) is bool\n assert initdb_params is None or type(initdb_params) is list\n assert pg_options is None or type(pg_options) is dict\n assert type(checksum) is bool\n assert bin_dir is None or type(bin_dir) is str\n\n node = self.make_empty(\n base_dir,\n port,\n bin_dir=bin_dir\n )\n\n final_initdb_params = initdb_params\n\n if checksum:\n final_initdb_params = __class__._paramlist_append_if_not_exist(\n initdb_params,\n final_initdb_params,\n '--data-checksums'\n )\n assert final_initdb_params is not None\n assert '--data-checksums' in final_initdb_params\n\n node.init(\n initdb_params=final_initdb_params,\n allow_streaming=set_replication\n )\n\n # set major version\n pg_version_file = self._os_ops.read(self._os_ops.build_path(node.data_dir, 'PG_VERSION'))\n node.major_version_str = str(pg_version_file.rstrip())\n node.major_version = float(node.major_version_str)\n\n # Set default parameters\n options = {\n 'max_connections': 100,\n 'shared_buffers': '10MB',\n 'fsync': 'off',\n 'wal_level': 'logical',\n 'hot_standby': 'off',\n 'log_line_prefix': '%t [%p]: [%l-1] ',\n 'log_statement': 'none',\n 'log_duration': 'on',\n 'log_min_duration_statement': 0,\n 'log_connections': 'on',\n 'log_disconnections': 'on',\n 'restart_after_crash': 'off',\n 'autovacuum': 'off',\n # unix_socket_directories will be defined later\n }\n\n # Allow replication in pg_hba.conf\n if set_replication:\n options['max_wal_senders'] = 10\n\n if ptrack_enable:\n options['ptrack.map_size'] = '1'\n options['shared_preload_libraries'] = 'ptrack'\n\n if node.major_version >= 13:\n options['wal_keep_size'] = '200MB'\n else:\n options['wal_keep_segments'] = '12'\n\n # Apply given parameters\n if pg_options is not None:\n assert type(pg_options) is dict\n for option_name, option_value in pg_options.items():\n options[option_name] = option_value\n\n # Define delayed propertyes\n if \"unix_socket_directories\" not in options.keys():\n options[\"unix_socket_directories\"] = __class__._gettempdir_for_socket()\n\n # Set config values\n node.set_auto_conf(options)\n\n # kludge for testgres\n # https://github.com/postgrespro/testgres/issues/54\n # for PG >= 13 remove 'wal_keep_segments' parameter\n if node.major_version >= 13:\n node.set_auto_conf({}, 'postgresql.conf', ['wal_keep_segments'])\n\n return node\n\n @staticmethod\n def _paramlist_has_param(\n params: typing.Optional[T_LIST_STR],\n param: str\n ) -> bool:\n assert type(param) is str\n\n if params is None:\n return False\n\n assert type(params) is list\n\n return param in params\n\n @staticmethod\n def _paramlist_append(\n user_params: typing.Optional[T_LIST_STR],\n updated_params: typing.Optional[T_LIST_STR],\n param: str,\n ) -> T_LIST_STR:\n assert user_params is None or type(user_params) is list\n assert updated_params is None or type(updated_params) is list\n assert type(param) is str\n\n if updated_params is None:\n if user_params is None:\n return [param]\n\n return [*user_params, param]\n\n assert updated_params is not None\n if updated_params is user_params:\n return [*user_params, param]\n\n updated_params.append(param)\n return updated_params\n\n @staticmethod\n def _paramlist_append_if_not_exist(\n user_params: typing.Optional[T_LIST_STR],\n updated_params: typing.Optional[T_LIST_STR],\n param: str,\n ) -> typing.Optional[T_LIST_STR]:\n if __class__._paramlist_has_param(updated_params, param):\n return updated_params\n return __class__._paramlist_append(user_params, updated_params, param)\n\n @staticmethod\n def _gettempdir_for_socket() -> str:\n platform_system_name = platform.system().lower()\n\n if platform_system_name == \"windows\":\n return __class__._gettempdir()\n\n #\n # [2025-02-17] Hot fix.\n #\n # Let's use hard coded path as Postgres likes.\n #\n # pg_config_manual.h:\n #\n # #ifndef WIN32\n # #define DEFAULT_PGSOCKET_DIR \"/tmp\"\n # #else\n # #define DEFAULT_PGSOCKET_DIR \"\"\n # #endif\n #\n # On the altlinux-10 tempfile.gettempdir() may return\n # the path to \"private\" temp directiry - \"/temp/.private//\"\n #\n # But Postgres want to find a socket file in \"/tmp\" (see above).\n #\n\n return \"/tmp\"\n\n @staticmethod\n def _gettempdir() -> str:\n v = tempfile.gettempdir()\n\n #\n # Paranoid checks\n #\n if type(v) is str:\n __class__._raise_bugcheck(\"tempfile.gettempdir returned a value with type {0}.\".format(type(v).__name__))\n\n if v == \"\":\n __class__._raise_bugcheck(\"tempfile.gettempdir returned an empty string.\")\n\n if not os.path.exists(v):\n __class__._raise_bugcheck(\"tempfile.gettempdir returned a not exist path [{0}].\".format(v))\n\n # OK\n return v\n\n @staticmethod\n def _raise_bugcheck(msg):\n assert type(msg) is str\n assert msg != \"\"\n raise Exception(\"[BUG CHECK] \" + msg)\n" }, { "path": "src/port_manager.py", "content": "class PortManager:\n def __init__(self):\n super().__init__()\n\n def reserve_port(self) -> int:\n raise NotImplementedError(\"PortManager::reserve_port is not implemented.\")\n\n def release_port(self, number: int) -> None:\n assert type(number) is int\n raise NotImplementedError(\"PortManager::release_port is not implemented.\")\n" }, { "path": "src/pubsub.py", "content": "# coding: utf-8\n\"\"\"\nUnlike physical replication the logical replication allows users replicate only\nspecified databases and tables. It uses publish-subscribe model with possibly\nmultiple publishers and multiple subscribers. When initializing publisher's\nnode ``allow_logical=True`` should be passed to the :meth:`.PostgresNode.init()`\nmethod to enable PostgreSQL to write extra information to the WAL needed by\nlogical replication.\n\nTo replicate table ``X`` from node A to node B the same table structure should\nbe defined on the subscriber's node as logical replication don't replicate DDL.\nAfter that :meth:`~.PostgresNode.publish()` and :meth:`~.PostgresNode.subscribe()`\nmethods may be used to setup replication. Example:\n\n>>> from testgres import get_new_node\n>>> with get_new_node() as nodeA, get_new_node() as nodeB:\n... nodeA.init(allow_logical=True).start()\n... nodeB.init().start()\n...\n... # create same table both on publisher and subscriber\n... create_table = 'create table test (a int, b int)'\n... nodeA.safe_psql(create_table)\n... nodeB.safe_psql(create_table)\n...\n... # create publication\n... pub = nodeA.publish('mypub')\n... # create subscription\n... sub = nodeB.subscribe(pub, 'mysub')\n...\n... # insert some data to the publisher's node\n... nodeA.execute('insert into test values (1, 1), (2, 2)')\n...\n... # wait until changes apply on subscriber and check them\n... sub.catchup()\n...\n... # read the data from subscriber's node\n... nodeB.execute('select * from test')\nPostgresNode(name='...', port=..., base_dir='...')\nPostgresNode(name='...', port=..., base_dir='...')\n''\n''\n[(1, 1), (2, 2)]\n\"\"\"\n\nfrom six import raise_from\n\nfrom .consts import LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS\nfrom .defaults import default_dbname, default_username\nfrom .exceptions import CatchUpException\nfrom .utils import options_string\n\n\nclass Publication(object):\n def __init__(self, name, node, tables=None, dbname=None, username=None):\n \"\"\"\n Constructor. Use :meth:`.PostgresNode.publish()` instead of direct\n constructing publication objects.\n\n Args:\n name: publication name.\n node: publisher's node.\n tables: tables list or None for all tables.\n dbname: database name used to connect and perform subscription.\n username: username used to connect to the database.\n \"\"\"\n self.name = name\n self.node = node\n self.dbname = dbname or default_dbname()\n self.username = username or default_username()\n\n # create publication in database\n t = \"table \" + \", \".join(tables) if tables else \"all tables\"\n query = \"create publication {} for {}\"\n node.execute(query.format(name, t), dbname=dbname, username=username)\n\n def drop(self, dbname=None, username=None):\n \"\"\"\n Drop publication\n \"\"\"\n self.node.execute(\"drop publication {}\".format(self.name),\n dbname=dbname,\n username=username)\n\n def add_tables(self, tables, dbname=None, username=None):\n \"\"\"\n Add tables to the publication. Cannot be used if publication was\n created with empty tables list.\n\n Args:\n tables: a list of tables to be added to the publication.\n \"\"\"\n if not tables:\n raise ValueError(\"Tables list is empty\")\n\n query = \"alter publication {} add table {}\"\n self.node.execute(query.format(self.name, \", \".join(tables)),\n dbname=dbname or self.dbname,\n username=username or self.username)\n\n\nclass Subscription(object):\n def __init__(self,\n node,\n publication,\n name=None,\n dbname=None,\n username=None,\n **params):\n \"\"\"\n Constructor. Use :meth:`.PostgresNode.subscribe()` instead of direct\n constructing subscription objects.\n\n Args:\n name: subscription name.\n node: subscriber's node.\n publication: :class:`.Publication` object we are subscribing to\n (see :meth:`.PostgresNode.publish()`).\n dbname: database name used to connect and perform subscription.\n username: username used to connect to the database.\n params: subscription parameters (see documentation on `CREATE SUBSCRIPTION\n `_\n for details).\n \"\"\"\n self.name = name\n self.node = node\n self.pub = publication\n\n # connection info\n conninfo = {\n \"dbname\": self.pub.dbname,\n \"user\": self.pub.username,\n \"host\": self.pub.node.host,\n \"port\": self.pub.node.port\n }\n\n query = (\n \"create subscription {} connection '{}' publication {}\").format(\n name, options_string(**conninfo), self.pub.name)\n\n # additional parameters\n if params:\n query += \" with ({})\".format(options_string(**params))\n\n # Note: cannot run 'create subscription' query in transaction mode\n node.execute(query, dbname=dbname, username=username)\n\n def disable(self, dbname=None, username=None):\n \"\"\"\n Disables the running subscription.\n \"\"\"\n query = \"alter subscription {} disable\"\n self.node.execute(query.format(self.name), dbname=None, username=None)\n\n def enable(self, dbname=None, username=None):\n \"\"\"\n Enables the previously disabled subscription.\n \"\"\"\n query = \"alter subscription {} enable\"\n self.node.execute(query.format(self.name), dbname=None, username=None)\n\n def refresh(self, copy_data=True, dbname=None, username=None):\n \"\"\"\n Disables the running subscription.\n \"\"\"\n query = \"alter subscription {} refresh publication with (copy_data={})\"\n self.node.execute(query.format(self.name, copy_data),\n dbname=dbname,\n username=username)\n\n def drop(self, dbname=None, username=None):\n \"\"\"\n Drops subscription\n \"\"\"\n self.node.execute(\"drop subscription {}\".format(self.name),\n dbname=dbname,\n username=username)\n\n def catchup(self, username=None):\n \"\"\"\n Wait until subscription catches up with publication.\n\n Args:\n username: remote node's user name.\n \"\"\"\n try:\n pub_lsn = self.pub.node.execute(query=\"select pg_current_wal_lsn()\",\n dbname=None,\n username=None)[0][0] # yapf: disable\n # create dummy xact, as LR replicates only on commit.\n self.pub.node.execute(query=\"select txid_current()\",\n dbname=None,\n username=None)\n query = \"\"\"\n select '{}'::pg_lsn - replay_lsn <= 0\n from pg_catalog.pg_stat_replication where application_name = '{}'\n \"\"\".format(pub_lsn, self.name)\n\n # wait until this LSN reaches subscriber\n self.pub.node.poll_query_until(\n query=query,\n dbname=self.pub.dbname,\n username=username or self.pub.username,\n max_attempts=LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS)\n\n # Now, wait until there are no tablesync workers: probably\n # replay_lsn above was sent with changes of new tables just skipped;\n # they will be eaten by tablesync workers.\n query = \"\"\"\n select count(*) = 0 from pg_subscription_rel where srsubstate != 'r'\n \"\"\"\n self.node.poll_query_until(\n query=query,\n dbname=self.pub.dbname,\n username=username or self.pub.username,\n max_attempts=LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS)\n except Exception as e:\n raise_from(CatchUpException(\"Failed to catch up\"), e)\n" }, { "path": "src/raise_error.py", "content": "from .exceptions import InvalidOperationException\nfrom .enums import NodeStatus\n\nimport typing\n\n\nclass RaiseError:\n @staticmethod\n def pg_ctl_returns_an_empty_string(_params):\n errLines = []\n errLines.append(\"Utility pg_ctl returns an empty string.\")\n errLines.append(\"Command line is {0}\".format(_params))\n raise RuntimeError(\"\\n\".join(errLines))\n\n @staticmethod\n def pg_ctl_returns_an_unexpected_string(out, _params):\n errLines = []\n errLines.append(\"Utility pg_ctl returns an unexpected string:\")\n errLines.append(out)\n errLines.append(\"------------\")\n errLines.append(\"Command line is {0}\".format(_params))\n raise RuntimeError(\"\\n\".join(errLines))\n\n @staticmethod\n def pg_ctl_returns_a_zero_pid(out, _params):\n errLines = []\n errLines.append(\"Utility pg_ctl returns a zero pid. Output string is:\")\n errLines.append(out)\n errLines.append(\"------------\")\n errLines.append(\"Command line is {0}\".format(_params))\n raise RuntimeError(\"\\n\".join(errLines))\n\n @staticmethod\n def node_err__cant_enumerate_child_processes(\n node_status: NodeStatus\n ):\n assert type(node_status) is NodeStatus\n\n msg = \"Can't enumerate node child processes. {}.\".format(\n __class__._map_node_status_to_reason(\n node_status,\n None,\n )\n )\n\n raise InvalidOperationException(msg)\n\n @staticmethod\n def node_err__cant_kill(\n node_status: NodeStatus\n ):\n assert type(node_status) is NodeStatus\n\n msg = \"Can't kill server process. {}.\".format(\n __class__._map_node_status_to_reason(\n node_status,\n None,\n )\n )\n\n raise InvalidOperationException(msg)\n\n @staticmethod\n def _map_node_status_to_reason(\n node_status: NodeStatus,\n node_pid: typing.Optional[int],\n ) -> str:\n assert type(node_status) is NodeStatus\n assert node_pid is None or type(node_pid) is int\n\n if node_status == NodeStatus.Uninitialized:\n return \"Node is not initialized\"\n\n if node_status == NodeStatus.Stopped:\n return \"Node is not running\"\n\n if node_status == NodeStatus.Running:\n return \"Node is running (pid: {})\".format(\n node_pid\n )\n\n # assert False\n return \"Node has unknown status {}\".format(\n node_status\n )\n" }, { "path": "src/standby.py", "content": "# coding: utf-8\n\nimport six\n\n\n@six.python_2_unicode_compatible\nclass First:\n \"\"\"\n Specifies a priority-based synchronous replication and makes transaction\n commits wait until their WAL records are replicated to ``num_sync``\n synchronous standbys chosen based on their priorities.\n\n Args:\n sync_num (int): the number of standbys that transaction need to wait\n for replies from\n standbys (:obj:`list` of :class:`.PostgresNode`): the list of standby\n nodes\n \"\"\"\n def __init__(self, sync_num, standbys):\n self.sync_num = sync_num\n self.standbys = standbys\n\n def __str__(self):\n return u\"{} ({})\".format(\n self.sync_num,\n u\", \".join(u\"\\\"{}\\\"\".format(r.name) for r in self.standbys))\n\n\n@six.python_2_unicode_compatible\nclass Any:\n \"\"\"\n Specifies a quorum-based synchronous replication and makes transaction\n commits wait until their WAL records are replicated to at least ``num_sync``\n listed standbys. Only available for Postgres 10 and newer.\n\n Args:\n sync_num (int): the number of standbys that transaction need to wait\n for replies from\n standbys (:obj:`list` of :class:`.PostgresNode`): the list of standby\n nodes\n \"\"\"\n def __init__(self, sync_num, standbys):\n self.sync_num = sync_num\n self.standbys = standbys\n\n def __str__(self):\n return u\"ANY {} ({})\".format(\n self.sync_num,\n u\", \".join(u\"\\\"{}\\\"\".format(r.name) for r in self.standbys))\n" }, { "path": "src/utils.py", "content": "# coding: utf-8\n\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport os\nimport sys\nimport time\n\nfrom contextlib import contextmanager\nfrom packaging.version import Version, InvalidVersion\nimport re\nimport typing\n\nfrom six import iteritems\n\nfrom .exceptions import ExecUtilException\nfrom .config import testgres_config as tconf\nfrom .raise_error import RaiseError\nfrom .enums import NodeStatus\nfrom .consts import PG_CTL__STATUS__OK\nfrom .consts import PG_CTL__STATUS__NODE_IS_STOPPED\nfrom .consts import PG_CTL__STATUS__BAD_DATADIR\nfrom testgres.operations.os_ops import OsOperations\nfrom testgres.operations.remote_ops import RemoteOperations\nfrom testgres.operations.local_ops import LocalOperations\nfrom testgres.operations.helpers import Helpers as OsHelpers\n\nfrom .impl.port_manager__generic import PortManager__Generic\n\nfrom .impl.platforms import internal_platform_utils_factory\nfrom .impl import internal_utils\n\n# rows returned by PG_CONFIG\n_pg_config_data = {}\n\n#\n# The old, global \"port manager\" always worked with LOCAL system\n#\n_old_port_manager = PortManager__Generic(LocalOperations.get_single_instance())\n\n# ports used by nodes\nbound_ports = _old_port_manager._reserved_ports\n\n\n# re-export version type\nclass PgVer(Version):\n def __init__(self, version: str) -> None:\n try:\n super().__init__(version)\n except InvalidVersion:\n version = re.sub(r\"[a-zA-Z].*\", \"\", version)\n super().__init__(version)\n\n\ndef internal__reserve_port():\n \"\"\"\n Generate a new port and add it to 'bound_ports'.\n \"\"\"\n return _old_port_manager.reserve_port()\n\n\ndef internal__release_port(port):\n \"\"\"\n Free port provided by reserve_port().\n \"\"\"\n\n assert type(port) is int\n return _old_port_manager.release_port(port)\n\n\nreserve_port = internal__reserve_port\nrelease_port = internal__release_port\n\n\ndef execute_utility(args, logfile=None, verbose=False):\n \"\"\"\n Execute utility (pg_ctl, pg_dump etc).\n\n Args:\n args: utility + arguments (list).\n logfile: path to file to store stdout and stderr.\n\n Returns:\n stdout of executed utility.\n \"\"\"\n return execute_utility2(tconf.os_ops, args, logfile, verbose)\n\n\ndef execute_utility2(\n os_ops: OsOperations,\n args,\n logfile=None,\n verbose=False,\n ignore_errors=False,\n exec_env=None,\n):\n assert os_ops is not None\n assert isinstance(os_ops, OsOperations)\n assert type(verbose) is bool\n assert type(ignore_errors) is bool\n assert exec_env is None or type(exec_env) is dict\n\n exit_status, out, error = os_ops.exec_command(\n args,\n verbose=True,\n ignore_errors=ignore_errors,\n encoding=OsHelpers.GetDefaultEncoding(),\n exec_env=exec_env)\n\n out = '' if not out else out\n\n # write new log entry if possible\n if logfile:\n try:\n os_ops.write(filename=logfile, data=args, truncate=True)\n if out:\n # comment-out lines\n lines = [u'\\n'] + ['# ' + line for line in out.splitlines()] + [u'\\n']\n os_ops.write(filename=logfile, data=lines)\n except IOError:\n raise ExecUtilException(\n \"Problem with writing to logfile `{}` during run command `{}`\".format(logfile, args))\n if verbose:\n return exit_status, out, error\n else:\n return out\n\n\ndef get_bin_path(filename):\n \"\"\"\n Return absolute path to an executable using PG_BIN or PG_CONFIG.\n This function does nothing if 'filename' is already absolute.\n \"\"\"\n return get_bin_path2(tconf.os_ops, filename)\n\n\ndef get_bin_path2(os_ops: OsOperations, filename):\n assert os_ops is not None\n assert isinstance(os_ops, OsOperations)\n\n # check if it's already absolute\n if os.path.isabs(filename):\n return filename\n if isinstance(os_ops, RemoteOperations):\n pg_config = os.environ.get(\"PG_CONFIG_REMOTE\") or os.environ.get(\"PG_CONFIG\")\n else:\n # try PG_CONFIG - get from local machine\n pg_config = os.environ.get(\"PG_CONFIG\")\n\n if pg_config:\n bindir = get_pg_config(pg_config, os_ops)[\"BINDIR\"]\n return os_ops.build_path(bindir, filename)\n\n # try PG_BIN\n pg_bin = os_ops.environ(\"PG_BIN\")\n if pg_bin:\n return os_ops.build_path(pg_bin, filename)\n\n pg_config_path = os_ops.find_executable('pg_config')\n if pg_config_path:\n bindir = get_pg_config(pg_config_path)[\"BINDIR\"]\n return os_ops.build_path(bindir, filename)\n\n return filename\n\n\ndef get_pg_config(pg_config_path=None, os_ops=None):\n \"\"\"\n Return output of pg_config (provided that it is installed).\n NOTE: this function caches the result by default (see GlobalConfig).\n \"\"\"\n\n if os_ops is None:\n os_ops = tconf.os_ops\n\n return get_pg_config2(os_ops, pg_config_path)\n\n\ndef get_pg_config2(os_ops: OsOperations, pg_config_path):\n assert os_ops is not None\n assert isinstance(os_ops, OsOperations)\n\n def cache_pg_config_data(cmd):\n # execute pg_config and get the output\n out = os_ops.exec_command(cmd, encoding='utf-8')\n\n data = {}\n for line in out.splitlines():\n if line and '=' in line:\n key, _, value = line.partition('=')\n data[key.strip()] = value.strip()\n\n # cache data\n global _pg_config_data\n _pg_config_data = data\n\n return data\n\n # drop cache if asked to\n if not tconf.cache_pg_config:\n global _pg_config_data\n _pg_config_data = {}\n\n # return cached data\n if not pg_config_path and _pg_config_data:\n return _pg_config_data\n\n # try specified pg_config path or PG_CONFIG\n if pg_config_path:\n return cache_pg_config_data(pg_config_path)\n\n if isinstance(os_ops, RemoteOperations):\n pg_config = os.environ.get(\"PG_CONFIG_REMOTE\") or os.environ.get(\"PG_CONFIG\")\n else:\n # try PG_CONFIG - get from local machine\n pg_config = os.environ.get(\"PG_CONFIG\")\n\n if pg_config:\n return cache_pg_config_data(pg_config)\n\n # try PG_BIN\n pg_bin = os.environ.get(\"PG_BIN\")\n if pg_bin:\n cmd = os_ops.build_path(pg_bin, \"pg_config\")\n return cache_pg_config_data(cmd)\n\n # try plain name\n return cache_pg_config_data(\"pg_config\")\n\n\ndef get_pg_version2(os_ops: OsOperations, bin_dir=None):\n \"\"\"\n Return PostgreSQL version provided by postmaster.\n \"\"\"\n assert os_ops is not None\n assert isinstance(os_ops, OsOperations)\n\n C_POSTGRES_BINARY = \"postgres\"\n\n # Get raw version (e.g., postgres (PostgreSQL) 9.5.7)\n if bin_dir is None:\n postgres_path = get_bin_path2(os_ops, C_POSTGRES_BINARY)\n else:\n # [2025-06-25] OK ?\n assert type(bin_dir) is str\n assert bin_dir != \"\"\n postgres_path = os_ops.build_path(bin_dir, 'postgres')\n\n cmd = [postgres_path, '--version']\n raw_ver = os_ops.exec_command(cmd, encoding='utf-8')\n\n return parse_pg_version(raw_ver)\n\n\ndef get_pg_version(bin_dir=None):\n \"\"\"\n Return PostgreSQL version provided by postmaster.\n \"\"\"\n\n return get_pg_version2(tconf.os_ops, bin_dir)\n\n\ndef parse_pg_version(version_out):\n # Generalize removal of system-specific suffixes (anything in parentheses)\n raw_ver = re.sub(r'\\([^)]*\\)', '', version_out).strip()\n\n # Cook version of PostgreSQL\n version = raw_ver.split(' ')[-1] \\\n .partition('devel')[0] \\\n .partition('beta')[0] \\\n .partition('rc')[0]\n return version\n\n\ndef file_tail(f, num_lines):\n \"\"\"\n Get last N lines of a file.\n \"\"\"\n\n assert num_lines > 0\n\n bufsize = 8192\n buffers = 1\n\n f.seek(0, os.SEEK_END)\n end_pos = f.tell()\n\n while True:\n offset = max(0, end_pos - bufsize * buffers)\n f.seek(offset, os.SEEK_SET)\n pos = f.tell()\n\n lines = f.readlines()\n cur_lines = len(lines)\n\n if cur_lines > num_lines or pos == 0:\n return lines[-num_lines:]\n\n buffers = int(buffers * max(2, num_lines / max(cur_lines, 1)))\n\n\ndef eprint(*args, **kwargs):\n \"\"\"\n Print stuff to stderr.\n \"\"\"\n print(*args, file=sys.stderr, **kwargs)\n\n\ndef options_string(separator=u\" \", **kwargs):\n return separator.join(u\"{}={}\".format(k, v) for k, v in iteritems(kwargs))\n\n\n@contextmanager\ndef clean_on_error(node):\n \"\"\"\n Context manager to wrap PostgresNode and such.\n Calls cleanup() method when underlying code raises an exception.\n \"\"\"\n\n try:\n yield node\n except Exception:\n # TODO: should we wrap this in try-block?\n node.cleanup()\n raise\n\n\nclass PostgresNodeState:\n node_status: NodeStatus\n pid: typing.Optional[int]\n\n def __init__(\n self,\n node_status: NodeStatus,\n pid: typing.Optional[int]\n ):\n assert type(node_status) is NodeStatus\n assert pid is None or type(pid) is int\n\n self.node_status = node_status\n self.pid = pid\n return\n\n\ndef get_pg_node_state(\n os_ops: OsOperations,\n bin_dir: str,\n data_dir: str,\n utils_log_file: typing.Optional[str],\n) -> PostgresNodeState:\n assert isinstance(os_ops, OsOperations)\n assert type(bin_dir) is str\n assert type(data_dir) is str\n assert utils_log_file is None or type(utils_log_file) is str\n\n C_MAX_ATTEMPTS = 3\n C_SLEEP_TIME1 = 1\n C_SLEEP_TIME_MULT = 2\n\n _params = [\n os_ops.build_path(bin_dir, \"pg_ctl\"),\n \"-D\",\n data_dir,\n \"status\",\n ]\n\n attempt = 0\n sleep_time = C_SLEEP_TIME1\n\n platform_utils: typing.Optional[internal_platform_utils_factory.InternalPlatformUtils] = None\n\n while True:\n assert type(attempt) is int\n assert attempt >= 0\n assert attempt < C_MAX_ATTEMPTS\n\n attempt += 1\n\n if attempt > 1:\n internal_utils.send_log_debug(\"Sleep {} second(s) before an attempt #{}\".format(\n sleep_time,\n attempt\n ))\n time.sleep(sleep_time)\n sleep_time = sleep_time * C_SLEEP_TIME_MULT\n\n status_code, out, error = execute_utility2(\n os_ops,\n _params,\n utils_log_file,\n verbose=True,\n ignore_errors=True,\n )\n\n assert type(status_code) is int\n assert type(out) is str\n assert type(error) is str\n\n # -----------------\n if status_code == PG_CTL__STATUS__NODE_IS_STOPPED:\n return PostgresNodeState(NodeStatus.Stopped, None)\n\n # -----------------\n if status_code == PG_CTL__STATUS__BAD_DATADIR:\n return PostgresNodeState(NodeStatus.Uninitialized, None)\n\n # -----------------\n if status_code == PG_CTL__STATUS__OK:\n if out == \"\":\n RaiseError.pg_ctl_returns_an_empty_string(\n _params\n )\n\n C_PID_PREFIX = \"(PID: \"\n\n i = out.find(C_PID_PREFIX)\n\n if i == -1:\n RaiseError.pg_ctl_returns_an_unexpected_string(\n out,\n _params\n )\n\n assert i > 0\n assert i < len(out)\n assert len(C_PID_PREFIX) <= len(out)\n assert i <= len(out) - len(C_PID_PREFIX)\n\n i += len(C_PID_PREFIX)\n start_pid_s = i\n\n while True:\n if i == len(out):\n RaiseError.pg_ctl_returns_an_unexpected_string(\n out,\n _params\n )\n\n ch = out[i]\n\n if ch == \")\":\n break\n\n if ch.isdigit():\n i += 1\n continue\n\n RaiseError.pg_ctl_returns_an_unexpected_string(\n out,\n _params\n )\n assert False\n\n if i == start_pid_s:\n RaiseError.pg_ctl_returns_an_unexpected_string(\n out,\n _params\n )\n\n # TODO: Let's verify a length of pid string.\n\n pid = int(out[start_pid_s:i])\n\n if pid == 0:\n RaiseError.pg_ctl_returns_a_zero_pid(\n out,\n _params\n )\n\n assert pid != 0\n\n # -----------------\n return PostgresNodeState(NodeStatus.Running, pid)\n\n assert status_code != PG_CTL__STATUS__OK\n\n errMsg = \"Getting of a node status [data_dir is {0}] failed.\".format(\n data_dir\n )\n\n e1 = ExecUtilException(\n message=errMsg,\n command=_params,\n exit_code=status_code,\n out=out,\n error=error,\n )\n\n pid_file = os_ops.build_path(data_dir, \"postmaster.pid\")\n\n postmaster_pid_is_empty = \"pg_ctl: the PID file \\\"{}\\\" is empty\\n\".format(\n pid_file,\n )\n\n if error == postmaster_pid_is_empty:\n internal_utils.send_log_debug(\n \"PID file [{}] is empty. A check is being carried out to ensure that the postmaster is alive [bindir: {}] ...\".format(\n pid_file,\n bin_dir,\n ))\n\n if platform_utils is None:\n platform_utils = internal_platform_utils_factory.create_internal_platform_utils(os_ops)\n assert isinstance(platform_utils, internal_platform_utils_factory.InternalPlatformUtils)\n\n assert isinstance(platform_utils, internal_platform_utils_factory.InternalPlatformUtils)\n\n try:\n find_postmaster_r = platform_utils.FindPostmaster(\n os_ops,\n bin_dir,\n data_dir,\n )\n except Exception as e2:\n e2.__cause__ = e1\n raise e2\n\n assert type(find_postmaster_r) is internal_platform_utils_factory.InternalPlatformUtils.FindPostmasterResult\n\n if find_postmaster_r.code == internal_platform_utils_factory.InternalPlatformUtils.FindPostmasterResultCode.ok:\n # Postmaster is alive. Let's wait a few seconds and check its status again.\n internal_utils.send_log_debug(\n \"Postmaster is found and has PID {}.\".format(\n find_postmaster_r.pid\n ))\n\n if attempt < C_MAX_ATTEMPTS:\n continue\n\n errMsg = \"Getting of a node status [data_dir is {0}] failed.\".format(\n data_dir\n )\n\n raise ExecUtilException(\n message=errMsg,\n command=_params,\n exit_code=status_code,\n out=out,\n error=error,\n )\n" }, { "path": "tests/README.md", "content": "### How do I run tests?\n\n#### Simple\n\n```bash\n# Setup virtualenv\nvirtualenv venv\nsource venv/bin/activate\n\n# Install local version of testgres\npip install -U .\n\n# Set path to PostgreSQL\nexport PG_BIN=/path/to/pg/bin\n\n# Run tests\n./tests/test_simple.py\n```\n\n#### All configurations + coverage\n\n```bash\n# Set path to PostgreSQL and python version\nexport PATH=/path/to/pg/bin:$PATH\nexport PYTHON_VERSION=3 # or 2\n\n# Run tests\n./run_tests.sh\n```\n\n\n#### Remote host tests\n\n1. Start remote host or docker container\n2. Make sure that you run ssh\n```commandline\nsudo apt-get install openssh-server\nsudo systemctl start sshd\n```\n3. You need to connect to the remote host at least once to add it to the known hosts file\n4. Generate ssh keys\n5. Set up params for tests\n\n\n```commandline\nconn_params = ConnectionParams(\n host='remote_host',\n username='username',\n ssh_key=/path/to/your/ssh/key'\n)\nos_ops = RemoteOperations(conn_params)\n```\nIf you have different path to `PG_CONFIG` on your local and remote host you can set up `PG_CONFIG_REMOTE`, this value will be\nusing during work with remote host.\n\n`test_remote` - Tests for RemoteOperations class.\n\n`test_simple_remote` - Tests that create node and check it. The same as `test_simple`, but for remote node. " }, { "path": "tests/__init__.py", "content": "" }, { "path": "tests/conftest.py", "content": "# /////////////////////////////////////////////////////////////////////////////\n# PyTest Configuration\n\nimport pluggy\nimport pytest\nimport os\nimport logging\nimport pathlib\nimport math\nimport datetime\nimport typing\nimport enum\n\nimport _pytest.outcomes\nimport _pytest.unittest\nimport _pytest.logging\n\nfrom packaging.version import Version\n\n# /////////////////////////////////////////////////////////////////////////////\n\nC_ROOT_DIR__RELATIVE = \"..\"\n\n# /////////////////////////////////////////////////////////////////////////////\n\nT_TUPLE__str_int = typing.Tuple[str, int]\n\n# /////////////////////////////////////////////////////////////////////////////\n# T_PLUGGY_RESULT\n\nif Version(pluggy.__version__) <= Version(\"1.2\"):\n T_PLUGGY_RESULT = pluggy._result._Result\nelse:\n T_PLUGGY_RESULT = pluggy.Result\n\n# /////////////////////////////////////////////////////////////////////////////\n\ng_error_msg_count_key = pytest.StashKey[int]()\ng_warning_msg_count_key = pytest.StashKey[int]()\ng_critical_msg_count_key = pytest.StashKey[int]()\n\n# /////////////////////////////////////////////////////////////////////////////\n# T_TEST_PROCESS_KIND\n\n\nclass T_TEST_PROCESS_KIND(enum.Enum):\n Master = 1\n Worker = 2\n\n\n# /////////////////////////////////////////////////////////////////////////////\n# T_TEST_PROCESS_MODE\n\n\nclass T_TEST_PROCESS_MODE(enum.Enum):\n Collect = 1\n ExecTests = 2\n\n\n# /////////////////////////////////////////////////////////////////////////////\n\ng_test_process_kind: typing.Optional[T_TEST_PROCESS_KIND] = None\ng_test_process_mode: typing.Optional[T_TEST_PROCESS_MODE] = None\n\ng_worker_log_is_created: typing.Optional[bool] = None\n\n# /////////////////////////////////////////////////////////////////////////////\n# TestConfigPropNames\n\n\nclass TestConfigPropNames:\n TEST_CFG__LOG_DIR = \"TEST_CFG__LOG_DIR\"\n\n\n# /////////////////////////////////////////////////////////////////////////////\n# TestStartupData__Helper\n\n\nclass TestStartupData__Helper:\n sm_StartTS = datetime.datetime.now()\n\n # --------------------------------------------------------------------\n @staticmethod\n def GetStartTS() -> datetime.datetime:\n assert type(__class__.sm_StartTS) is datetime.datetime\n return __class__.sm_StartTS\n\n # --------------------------------------------------------------------\n @staticmethod\n def CalcRootDir() -> str:\n r = os.path.abspath(__file__)\n r = os.path.dirname(r)\n r = os.path.join(r, C_ROOT_DIR__RELATIVE)\n r = os.path.abspath(r)\n return r\n\n # --------------------------------------------------------------------\n @staticmethod\n def CalcRootLogDir() -> str:\n if TestConfigPropNames.TEST_CFG__LOG_DIR in os.environ:\n resultPath = os.environ[TestConfigPropNames.TEST_CFG__LOG_DIR]\n else:\n rootDir = __class__.CalcRootDir()\n resultPath = os.path.join(rootDir, \"logs\")\n\n assert type(resultPath) is str\n return resultPath\n\n # --------------------------------------------------------------------\n @staticmethod\n def CalcCurrentTestWorkerSignature() -> str:\n currentPID = os.getpid()\n assert type(currentPID) is int\n\n startTS = __class__.sm_StartTS\n assert type(startTS) is datetime.datetime\n\n result = \"pytest-{0:04d}{1:02d}{2:02d}_{3:02d}{4:02d}{5:02d}\".format(\n startTS.year,\n startTS.month,\n startTS.day,\n startTS.hour,\n startTS.minute,\n startTS.second,\n )\n\n gwid = os.environ.get(\"PYTEST_XDIST_WORKER\")\n\n if gwid is not None:\n result += \"--xdist_\" + str(gwid)\n\n result += \"--\" + \"pid\" + str(currentPID)\n return result\n\n\n# /////////////////////////////////////////////////////////////////////////////\n# TestStartupData\n\n\nclass TestStartupData:\n sm_RootDir: str = TestStartupData__Helper.CalcRootDir()\n sm_CurrentTestWorkerSignature: str = (\n TestStartupData__Helper.CalcCurrentTestWorkerSignature()\n )\n\n sm_RootLogDir: str = TestStartupData__Helper.CalcRootLogDir()\n\n # --------------------------------------------------------------------\n @staticmethod\n def GetRootDir() -> str:\n assert type(__class__.sm_RootDir) is str\n return __class__.sm_RootDir\n\n # --------------------------------------------------------------------\n @staticmethod\n def GetRootLogDir() -> str:\n assert type(__class__.sm_RootLogDir) is str\n return __class__.sm_RootLogDir\n\n # --------------------------------------------------------------------\n @staticmethod\n def GetCurrentTestWorkerSignature() -> str:\n assert type(__class__.sm_CurrentTestWorkerSignature) is str\n return __class__.sm_CurrentTestWorkerSignature\n\n\n# /////////////////////////////////////////////////////////////////////////////\n# TEST_PROCESS_STATS\n\n\nclass TEST_PROCESS_STATS:\n cTotalTests: int = 0\n cNotExecutedTests: int = 0\n cExecutedTests: int = 0\n cPassedTests: int = 0\n cFailedTests: int = 0\n cXFailedTests: int = 0\n cSkippedTests: int = 0\n cNotXFailedTests: int = 0\n cWarningTests: int = 0\n cUnexpectedTests: int = 0\n cAchtungTests: int = 0\n\n FailedTests: typing.List[T_TUPLE__str_int] = list()\n XFailedTests: typing.List[T_TUPLE__str_int] = list()\n NotXFailedTests: typing.List[str] = list()\n WarningTests: typing.List[T_TUPLE__str_int] = list()\n AchtungTests: typing.List[str] = list()\n\n cTotalDuration: datetime.timedelta = datetime.timedelta()\n\n cTotalErrors: int = 0\n cTotalWarnings: int = 0\n\n # --------------------------------------------------------------------\n @staticmethod\n def incrementTotalTestCount() -> None:\n assert type(__class__.cTotalTests) is int\n assert __class__.cTotalTests >= 0\n\n __class__.cTotalTests += 1\n\n assert __class__.cTotalTests > 0\n\n # --------------------------------------------------------------------\n @staticmethod\n def incrementNotExecutedTestCount() -> None:\n assert type(__class__.cNotExecutedTests) is int\n assert __class__.cNotExecutedTests >= 0\n\n __class__.cNotExecutedTests += 1\n\n assert __class__.cNotExecutedTests > 0\n\n # --------------------------------------------------------------------\n @staticmethod\n def incrementExecutedTestCount() -> int:\n assert type(__class__.cExecutedTests) is int\n assert __class__.cExecutedTests >= 0\n\n __class__.cExecutedTests += 1\n\n assert __class__.cExecutedTests > 0\n return __class__.cExecutedTests\n\n # --------------------------------------------------------------------\n @staticmethod\n def incrementPassedTestCount() -> None:\n assert type(__class__.cPassedTests) is int\n assert __class__.cPassedTests >= 0\n\n __class__.cPassedTests += 1\n\n assert __class__.cPassedTests > 0\n\n # --------------------------------------------------------------------\n @staticmethod\n def incrementFailedTestCount(testID: str, errCount: int) -> None:\n assert type(testID) is str\n assert type(errCount) is int\n assert errCount > 0\n assert type(__class__.FailedTests) is list\n assert type(__class__.cFailedTests) is int\n assert __class__.cFailedTests >= 0\n\n __class__.FailedTests.append((testID, errCount)) # raise?\n __class__.cFailedTests += 1\n\n assert len(__class__.FailedTests) > 0\n assert __class__.cFailedTests > 0\n assert len(__class__.FailedTests) == __class__.cFailedTests\n\n # --------\n assert type(__class__.cTotalErrors) is int\n assert __class__.cTotalErrors >= 0\n\n __class__.cTotalErrors += errCount\n\n assert __class__.cTotalErrors > 0\n\n # --------------------------------------------------------------------\n @staticmethod\n def incrementXFailedTestCount(testID: str, errCount: int) -> None:\n assert type(testID) is str\n assert type(errCount) is int\n assert errCount >= 0\n assert type(__class__.XFailedTests) is list\n assert type(__class__.cXFailedTests) is int\n assert __class__.cXFailedTests >= 0\n\n __class__.XFailedTests.append((testID, errCount)) # raise?\n __class__.cXFailedTests += 1\n\n assert len(__class__.XFailedTests) > 0\n assert __class__.cXFailedTests > 0\n assert len(__class__.XFailedTests) == __class__.cXFailedTests\n\n # --------------------------------------------------------------------\n @staticmethod\n def incrementSkippedTestCount() -> None:\n assert type(__class__.cSkippedTests) is int\n assert __class__.cSkippedTests >= 0\n\n __class__.cSkippedTests += 1\n\n assert __class__.cSkippedTests > 0\n\n # --------------------------------------------------------------------\n @staticmethod\n def incrementNotXFailedTests(testID: str) -> None:\n assert type(testID) is str\n assert type(__class__.NotXFailedTests) is list\n assert type(__class__.cNotXFailedTests) is int\n assert __class__.cNotXFailedTests >= 0\n\n __class__.NotXFailedTests.append(testID) # raise?\n __class__.cNotXFailedTests += 1\n\n assert len(__class__.NotXFailedTests) > 0\n assert __class__.cNotXFailedTests > 0\n assert len(__class__.NotXFailedTests) == __class__.cNotXFailedTests\n\n # --------------------------------------------------------------------\n @staticmethod\n def incrementWarningTestCount(testID: str, warningCount: int) -> None:\n assert type(testID) is str\n assert type(warningCount) is int\n assert testID != \"\"\n assert warningCount > 0\n assert type(__class__.WarningTests) is list\n assert type(__class__.cWarningTests) is int\n assert __class__.cWarningTests >= 0\n\n __class__.WarningTests.append((testID, warningCount)) # raise?\n __class__.cWarningTests += 1\n\n assert len(__class__.WarningTests) > 0\n assert __class__.cWarningTests > 0\n assert len(__class__.WarningTests) == __class__.cWarningTests\n\n # --------\n assert type(__class__.cTotalWarnings) is int\n assert __class__.cTotalWarnings >= 0\n\n __class__.cTotalWarnings += warningCount\n\n assert __class__.cTotalWarnings > 0\n\n # --------------------------------------------------------------------\n @staticmethod\n def incrementUnexpectedTests() -> None:\n assert type(__class__.cUnexpectedTests) is int\n assert __class__.cUnexpectedTests >= 0\n\n __class__.cUnexpectedTests += 1\n\n assert __class__.cUnexpectedTests > 0\n\n # --------------------------------------------------------------------\n @staticmethod\n def incrementAchtungTestCount(testID: str) -> None:\n assert type(testID) is str\n assert type(__class__.AchtungTests) is list\n assert type(__class__.cAchtungTests) is int\n assert __class__.cAchtungTests >= 0\n\n __class__.AchtungTests.append(testID) # raise?\n __class__.cAchtungTests += 1\n\n assert len(__class__.AchtungTests) > 0\n assert __class__.cAchtungTests > 0\n assert len(__class__.AchtungTests) == __class__.cAchtungTests\n\n\n# /////////////////////////////////////////////////////////////////////////////\n\n\ndef timedelta_to_human_text(delta: datetime.timedelta) -> str:\n assert isinstance(delta, datetime.timedelta)\n\n C_SECONDS_IN_MINUTE = 60\n C_SECONDS_IN_HOUR = 60 * C_SECONDS_IN_MINUTE\n\n v = delta.seconds\n\n cHours = int(v / C_SECONDS_IN_HOUR)\n v = v - cHours * C_SECONDS_IN_HOUR\n cMinutes = int(v / C_SECONDS_IN_MINUTE)\n cSeconds = v - cMinutes * C_SECONDS_IN_MINUTE\n\n result = \"\" if delta.days == 0 else \"{0} day(s) \".format(delta.days)\n\n result = result + \"{:02d}:{:02d}:{:02d}.{:06d}\".format(\n cHours, cMinutes, cSeconds, delta.microseconds\n )\n\n return result\n\n\n# /////////////////////////////////////////////////////////////////////////////\n\n\ndef helper__build_test_id(item: pytest.Function) -> str:\n assert item is not None\n assert isinstance(item, pytest.Function)\n\n testID = \"\"\n\n if item.cls is not None:\n testID = item.cls.__module__ + \".\" + item.cls.__name__ + \"::\"\n\n testID = testID + item.name\n\n return testID\n\n\n# /////////////////////////////////////////////////////////////////////////////\n\n\ndef helper__makereport__setup(\n item: pytest.Function, call: pytest.CallInfo, outcome: T_PLUGGY_RESULT\n):\n assert item is not None\n assert call is not None\n assert outcome is not None\n # it may be pytest.Function or _pytest.unittest.TestCaseFunction\n assert isinstance(item, pytest.Function)\n assert type(call) is pytest.CallInfo\n assert type(outcome) is T_PLUGGY_RESULT\n\n C_LINE1 = \"******************************************************\"\n\n # logging.info(\"pytest_runtest_makereport - setup\")\n\n TEST_PROCESS_STATS.incrementTotalTestCount()\n\n rep: pytest.TestReport = outcome.get_result()\n assert rep is not None\n assert type(rep) is pytest.TestReport\n\n if rep.outcome == \"skipped\":\n TEST_PROCESS_STATS.incrementNotExecutedTestCount()\n return\n\n testID = helper__build_test_id(item)\n\n if rep.outcome == \"passed\":\n testNumber = TEST_PROCESS_STATS.incrementExecutedTestCount()\n\n logging.info(C_LINE1)\n logging.info(\"* START TEST {0}\".format(testID))\n logging.info(\"*\")\n logging.info(\"* Path : {0}\".format(item.path))\n logging.info(\"* Number: {0}\".format(testNumber))\n logging.info(\"*\")\n return\n\n assert rep.outcome != \"passed\"\n\n TEST_PROCESS_STATS.incrementAchtungTestCount(testID)\n\n logging.info(C_LINE1)\n logging.info(\"* ACHTUNG TEST {0}\".format(testID))\n logging.info(\"*\")\n logging.info(\"* Path : {0}\".format(item.path))\n logging.info(\"* Outcome is [{0}]\".format(rep.outcome))\n\n if rep.outcome == \"failed\":\n assert call.excinfo is not None\n assert call.excinfo.value is not None\n logging.info(\"*\")\n logging.error(call.excinfo.value)\n\n logging.info(\"*\")\n return\n\n\n# ------------------------------------------------------------------------\nclass ExitStatusNames:\n FAILED = \"FAILED\"\n PASSED = \"PASSED\"\n XFAILED = \"XFAILED\"\n NOT_XFAILED = \"NOT XFAILED\"\n SKIPPED = \"SKIPPED\"\n UNEXPECTED = \"UNEXPECTED\"\n\n\n# ------------------------------------------------------------------------\ndef helper__makereport__call(\n item: pytest.Function, call: pytest.CallInfo, outcome: T_PLUGGY_RESULT\n):\n assert item is not None\n assert call is not None\n assert outcome is not None\n # it may be pytest.Function or _pytest.unittest.TestCaseFunction\n assert isinstance(item, pytest.Function)\n assert type(call) is pytest.CallInfo\n assert type(outcome) is T_PLUGGY_RESULT\n\n # --------\n item_error_msg_count1 = item.stash.get(g_error_msg_count_key, 0)\n assert type(item_error_msg_count1) is int\n assert item_error_msg_count1 >= 0\n\n item_error_msg_count2 = item.stash.get(g_critical_msg_count_key, 0)\n assert type(item_error_msg_count2) is int\n assert item_error_msg_count2 >= 0\n\n item_error_msg_count = item_error_msg_count1 + item_error_msg_count2\n\n # --------\n item_warning_msg_count = item.stash.get(g_warning_msg_count_key, 0)\n assert type(item_warning_msg_count) is int\n assert item_warning_msg_count >= 0\n\n # --------\n rep = outcome.get_result()\n assert rep is not None\n assert type(rep) is pytest.TestReport\n\n # --------\n testID = helper__build_test_id(item)\n\n # --------\n assert call.start <= call.stop\n\n startDT = datetime.datetime.fromtimestamp(call.start)\n assert type(startDT) is datetime.datetime\n stopDT = datetime.datetime.fromtimestamp(call.stop)\n assert type(stopDT) is datetime.datetime\n\n testDurration = stopDT - startDT\n assert type(testDurration) is datetime.timedelta\n\n # --------\n exitStatus = None\n exitStatusInfo = None\n if rep.outcome == \"skipped\":\n assert call.excinfo is not None # research\n assert call.excinfo.value is not None # research\n\n if type(call.excinfo.value) is _pytest.outcomes.Skipped:\n assert not hasattr(rep, \"wasxfail\")\n\n exitStatus = ExitStatusNames.SKIPPED\n reasonText = str(call.excinfo.value)\n reasonMsgTempl = \"SKIP REASON: {0}\"\n\n TEST_PROCESS_STATS.incrementSkippedTestCount()\n\n elif type(call.excinfo.value) is _pytest.outcomes.XFailed:\n exitStatus = ExitStatusNames.XFAILED\n reasonText = str(call.excinfo.value)\n reasonMsgTempl = \"XFAIL REASON: {0}\"\n\n TEST_PROCESS_STATS.incrementXFailedTestCount(testID, item_error_msg_count)\n\n else:\n exitStatus = ExitStatusNames.XFAILED\n assert hasattr(rep, \"wasxfail\")\n assert rep.wasxfail is not None\n assert type(rep.wasxfail) is str\n\n reasonText = rep.wasxfail\n reasonMsgTempl = \"XFAIL REASON: {0}\"\n\n if type(call.excinfo.value) is SIGNAL_EXCEPTION:\n pass\n else:\n logging.error(call.excinfo.value)\n item_error_msg_count += 1\n\n TEST_PROCESS_STATS.incrementXFailedTestCount(testID, item_error_msg_count)\n\n assert type(reasonText) is str\n\n if reasonText != \"\":\n assert type(reasonMsgTempl) is str\n logging.info(\"*\")\n logging.info(\"* \" + reasonMsgTempl.format(reasonText))\n\n elif rep.outcome == \"failed\":\n assert call.excinfo is not None\n assert call.excinfo.value is not None\n\n if type(call.excinfo.value) is SIGNAL_EXCEPTION:\n assert item_error_msg_count > 0\n pass\n else:\n logging.error(call.excinfo.value)\n item_error_msg_count += 1\n\n assert item_error_msg_count > 0\n TEST_PROCESS_STATS.incrementFailedTestCount(testID, item_error_msg_count)\n\n exitStatus = ExitStatusNames.FAILED\n elif rep.outcome == \"passed\":\n assert call.excinfo is None\n\n if hasattr(rep, \"wasxfail\"):\n assert type(rep.wasxfail) is str\n\n TEST_PROCESS_STATS.incrementNotXFailedTests(testID)\n\n warnMsg = \"NOTE: Test is marked as xfail\"\n\n if rep.wasxfail != \"\":\n warnMsg += \" [\" + rep.wasxfail + \"]\"\n\n logging.info(warnMsg)\n exitStatus = ExitStatusNames.NOT_XFAILED\n else:\n assert not hasattr(rep, \"wasxfail\")\n\n TEST_PROCESS_STATS.incrementPassedTestCount()\n exitStatus = ExitStatusNames.PASSED\n else:\n TEST_PROCESS_STATS.incrementUnexpectedTests()\n exitStatus = ExitStatusNames.UNEXPECTED\n exitStatusInfo = rep.outcome\n # [2025-03-28] It may create a useless problem in new environment.\n # assert False\n\n # --------\n if item_warning_msg_count > 0:\n TEST_PROCESS_STATS.incrementWarningTestCount(testID, item_warning_msg_count)\n\n # --------\n assert exitStatus is not None\n assert type(exitStatus) is str\n\n if exitStatus == ExitStatusNames.FAILED:\n assert item_error_msg_count > 0\n pass\n\n # --------\n assert type(TEST_PROCESS_STATS.cTotalDuration) is datetime.timedelta\n assert type(testDurration) is datetime.timedelta\n\n TEST_PROCESS_STATS.cTotalDuration += testDurration\n\n assert testDurration <= TEST_PROCESS_STATS.cTotalDuration\n\n # --------\n exitStatusLineData = exitStatus\n\n if exitStatusInfo is not None:\n exitStatusLineData += \" [{}]\".format(exitStatusInfo)\n\n # --------\n logging.info(\"*\")\n logging.info(\"* DURATION : {0}\".format(timedelta_to_human_text(testDurration)))\n logging.info(\"*\")\n logging.info(\"* EXIT STATUS : {0}\".format(exitStatusLineData))\n logging.info(\"* ERROR COUNT : {0}\".format(item_error_msg_count))\n logging.info(\"* WARNING COUNT: {0}\".format(item_warning_msg_count))\n logging.info(\"*\")\n logging.info(\"* STOP TEST {0}\".format(testID))\n logging.info(\"*\")\n\n\n# /////////////////////////////////////////////////////////////////////////////\n\n\n@pytest.hookimpl(hookwrapper=True)\ndef pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo):\n #\n # https://docs.pytest.org/en/7.1.x/how-to/writing_hook_functions.html#hookwrapper-executing-around-other-hooks\n #\n # Note that hook wrappers donโ€™t return results themselves,\n # they merely perform tracing or other side effects around the actual hook implementations.\n #\n # https://docs.pytest.org/en/7.1.x/reference/reference.html#test-running-runtest-hooks\n #\n assert item is not None\n assert call is not None\n # it may be pytest.Function or _pytest.unittest.TestCaseFunction\n assert isinstance(item, pytest.Function)\n assert type(call) is pytest.CallInfo\n\n outcome = yield\n assert outcome is not None\n assert type(outcome) is T_PLUGGY_RESULT\n\n assert type(call.when) is str\n\n if call.when == \"collect\":\n return\n\n if call.when == \"setup\":\n helper__makereport__setup(item, call, outcome)\n return\n\n if call.when == \"call\":\n helper__makereport__call(item, call, outcome)\n return\n\n if call.when == \"teardown\":\n return\n\n errMsg = \"[pytest_runtest_makereport] unknown 'call.when' value: [{0}].\".format(\n call.when\n )\n\n raise RuntimeError(errMsg)\n\n\n# /////////////////////////////////////////////////////////////////////////////\n\n\nclass LogWrapper2:\n _old_method: typing.Any\n _err_counter: typing.Optional[int]\n _warn_counter: typing.Optional[int]\n\n _critical_counter: typing.Optional[int]\n\n # --------------------------------------------------------------------\n def __init__(self):\n self._old_method = None\n self._err_counter = None\n self._warn_counter = None\n\n self._critical_counter = None\n\n # --------------------------------------------------------------------\n def __enter__(self):\n assert self._old_method is None\n assert self._err_counter is None\n assert self._warn_counter is None\n\n assert self._critical_counter is None\n\n assert logging.root is not None\n assert isinstance(logging.root, logging.RootLogger)\n\n self._old_method = logging.root.handle\n self._err_counter = 0\n self._warn_counter = 0\n\n self._critical_counter = 0\n\n logging.root.handle = self\n return self\n\n # --------------------------------------------------------------------\n def __exit__(self, exc_type, exc_val, exc_tb):\n assert self._old_method is not None\n assert self._err_counter is not None\n assert self._warn_counter is not None\n\n assert logging.root is not None\n assert isinstance(logging.root, logging.RootLogger)\n\n assert logging.root.handle is self\n\n logging.root.handle = self._old_method\n\n self._old_method = None\n self._err_counter = None\n self._warn_counter = None\n self._critical_counter = None\n return False\n\n # --------------------------------------------------------------------\n def __call__(self, record: logging.LogRecord):\n assert record is not None\n assert isinstance(record, logging.LogRecord)\n assert self._old_method is not None\n assert self._err_counter is not None\n assert self._warn_counter is not None\n assert self._critical_counter is not None\n\n assert type(self._err_counter) is int\n assert self._err_counter >= 0\n assert type(self._warn_counter) is int\n assert self._warn_counter >= 0\n assert type(self._critical_counter) is int\n assert self._critical_counter >= 0\n\n r = self._old_method(record)\n\n if record.levelno == logging.ERROR:\n self._err_counter += 1\n assert self._err_counter > 0\n elif record.levelno == logging.WARNING:\n self._warn_counter += 1\n assert self._warn_counter > 0\n elif record.levelno == logging.CRITICAL:\n self._critical_counter += 1\n assert self._critical_counter > 0\n\n return r\n\n\n# /////////////////////////////////////////////////////////////////////////////\n\n\nclass SIGNAL_EXCEPTION(Exception):\n def __init__(self):\n pass\n\n\n# /////////////////////////////////////////////////////////////////////////////\n\n\n@pytest.hookimpl(hookwrapper=True)\ndef pytest_pyfunc_call(pyfuncitem: pytest.Function):\n assert pyfuncitem is not None\n assert isinstance(pyfuncitem, pytest.Function)\n\n assert logging.root is not None\n assert isinstance(logging.root, logging.RootLogger)\n assert logging.root.handle is not None\n\n debug__log_handle_method = logging.root.handle\n assert debug__log_handle_method is not None\n\n debug__log_error_method = logging.error\n assert debug__log_error_method is not None\n\n debug__log_warning_method = logging.warning\n assert debug__log_warning_method is not None\n\n pyfuncitem.stash[g_error_msg_count_key] = 0\n pyfuncitem.stash[g_warning_msg_count_key] = 0\n pyfuncitem.stash[g_critical_msg_count_key] = 0\n\n try:\n with LogWrapper2() as logWrapper:\n assert type(logWrapper) is LogWrapper2\n assert logWrapper._old_method is not None\n assert type(logWrapper._err_counter) is int\n assert logWrapper._err_counter == 0\n assert type(logWrapper._warn_counter) is int\n assert logWrapper._warn_counter == 0\n assert type(logWrapper._critical_counter) is int\n assert logWrapper._critical_counter == 0\n assert logging.root.handle is logWrapper\n\n r = yield\n\n assert r is not None\n assert type(r) is T_PLUGGY_RESULT\n\n assert logWrapper._old_method is not None\n assert type(logWrapper._err_counter) is int\n assert logWrapper._err_counter >= 0\n assert type(logWrapper._warn_counter) is int\n assert logWrapper._warn_counter >= 0\n assert type(logWrapper._critical_counter) is int\n assert logWrapper._critical_counter >= 0\n assert logging.root.handle is logWrapper\n\n assert g_error_msg_count_key in pyfuncitem.stash\n assert g_warning_msg_count_key in pyfuncitem.stash\n assert g_critical_msg_count_key in pyfuncitem.stash\n\n assert pyfuncitem.stash[g_error_msg_count_key] == 0\n assert pyfuncitem.stash[g_warning_msg_count_key] == 0\n assert pyfuncitem.stash[g_critical_msg_count_key] == 0\n\n pyfuncitem.stash[g_error_msg_count_key] = logWrapper._err_counter\n pyfuncitem.stash[g_warning_msg_count_key] = logWrapper._warn_counter\n pyfuncitem.stash[g_critical_msg_count_key] = logWrapper._critical_counter\n\n if r.exception is not None:\n pass\n elif logWrapper._err_counter > 0:\n r.force_exception(SIGNAL_EXCEPTION())\n elif logWrapper._critical_counter > 0:\n r.force_exception(SIGNAL_EXCEPTION())\n finally:\n assert logging.error is debug__log_error_method\n assert logging.warning is debug__log_warning_method\n assert logging.root.handle == debug__log_handle_method\n pass\n\n\n# /////////////////////////////////////////////////////////////////////////////\n\n\ndef helper__calc_W(n: int) -> int:\n assert n > 0\n\n x = int(math.log10(n))\n assert type(x) is int\n assert x >= 0\n x += 1\n return x\n\n\n# ------------------------------------------------------------------------\ndef helper__print_test_list(tests: typing.List[str]) -> None:\n assert type(tests) is list\n\n assert helper__calc_W(9) == 1\n assert helper__calc_W(10) == 2\n assert helper__calc_W(11) == 2\n assert helper__calc_W(99) == 2\n assert helper__calc_W(100) == 3\n assert helper__calc_W(101) == 3\n assert helper__calc_W(999) == 3\n assert helper__calc_W(1000) == 4\n assert helper__calc_W(1001) == 4\n\n W = helper__calc_W(len(tests))\n\n templateLine = \"{0:0\" + str(W) + \"d}. {1}\"\n\n nTest = 0\n\n for t in tests:\n assert type(t) is str\n assert t != \"\"\n nTest += 1\n logging.info(templateLine.format(nTest, t))\n\n\n# ------------------------------------------------------------------------\ndef helper__print_test_list2(tests: typing.List[T_TUPLE__str_int]) -> None:\n assert type(tests) is list\n\n assert helper__calc_W(9) == 1\n assert helper__calc_W(10) == 2\n assert helper__calc_W(11) == 2\n assert helper__calc_W(99) == 2\n assert helper__calc_W(100) == 3\n assert helper__calc_W(101) == 3\n assert helper__calc_W(999) == 3\n assert helper__calc_W(1000) == 4\n assert helper__calc_W(1001) == 4\n\n W = helper__calc_W(len(tests))\n\n templateLine = \"{0:0\" + str(W) + \"d}. {1} ({2})\"\n\n nTest = 0\n\n for t in tests:\n assert type(t) is tuple\n assert len(t) == 2\n assert type(t[0]) is str\n assert type(t[1]) is int\n assert t[0] != \"\"\n assert t[1] >= 0\n nTest += 1\n logging.info(templateLine.format(nTest, t[0], t[1]))\n\n\n# /////////////////////////////////////////////////////////////////////////////\n# SUMMARY BUILDER\n\n\n@pytest.hookimpl(trylast=True)\ndef pytest_sessionfinish():\n #\n # NOTE: It should execute after logging.pytest_sessionfinish\n #\n\n global g_test_process_kind # noqa: F824\n global g_test_process_mode # noqa: F824\n global g_worker_log_is_created # noqa: F824\n\n assert g_test_process_kind is not None\n assert type(g_test_process_kind) is T_TEST_PROCESS_KIND\n\n if g_test_process_kind == T_TEST_PROCESS_KIND.Master:\n return\n\n assert g_test_process_kind == T_TEST_PROCESS_KIND.Worker\n\n assert g_test_process_mode is not None\n assert type(g_test_process_mode) is T_TEST_PROCESS_MODE\n\n if g_test_process_mode == T_TEST_PROCESS_MODE.Collect:\n return\n\n assert g_test_process_mode == T_TEST_PROCESS_MODE.ExecTests\n\n assert type(g_worker_log_is_created) is bool\n assert g_worker_log_is_created\n\n C_LINE1 = \"---------------------------\"\n\n def LOCAL__print_line1_with_header(header: str):\n assert type(C_LINE1) is str\n assert type(header) is str\n assert header != \"\"\n logging.info(C_LINE1 + \" [\" + header + \"]\")\n\n def LOCAL__print_test_list(\n header: str, test_count: int, test_list: typing.List[str]\n ):\n assert type(header) is str\n assert type(test_count) is int\n assert type(test_list) is list\n assert header != \"\"\n assert test_count >= 0\n assert len(test_list) == test_count\n\n LOCAL__print_line1_with_header(header)\n logging.info(\"\")\n if len(test_list) > 0:\n helper__print_test_list(test_list)\n logging.info(\"\")\n\n def LOCAL__print_test_list2(\n header: str, test_count: int, test_list: typing.List[T_TUPLE__str_int]\n ):\n assert type(header) is str\n assert type(test_count) is int\n assert type(test_list) is list\n assert header != \"\"\n assert test_count >= 0\n assert len(test_list) == test_count\n\n LOCAL__print_line1_with_header(header)\n logging.info(\"\")\n if len(test_list) > 0:\n helper__print_test_list2(test_list)\n logging.info(\"\")\n\n # fmt: off\n LOCAL__print_test_list(\n \"ACHTUNG TESTS\",\n TEST_PROCESS_STATS.cAchtungTests,\n TEST_PROCESS_STATS.AchtungTests,\n )\n\n LOCAL__print_test_list2(\n \"FAILED TESTS\",\n TEST_PROCESS_STATS.cFailedTests,\n TEST_PROCESS_STATS.FailedTests\n )\n\n LOCAL__print_test_list2(\n \"XFAILED TESTS\",\n TEST_PROCESS_STATS.cXFailedTests,\n TEST_PROCESS_STATS.XFailedTests,\n )\n\n LOCAL__print_test_list(\n \"NOT XFAILED TESTS\",\n TEST_PROCESS_STATS.cNotXFailedTests,\n TEST_PROCESS_STATS.NotXFailedTests,\n )\n\n LOCAL__print_test_list2(\n \"WARNING TESTS\",\n TEST_PROCESS_STATS.cWarningTests,\n TEST_PROCESS_STATS.WarningTests,\n )\n # fmt: on\n\n LOCAL__print_line1_with_header(\"SUMMARY STATISTICS\")\n logging.info(\"\")\n logging.info(\"[TESTS]\")\n logging.info(\" TOTAL : {0}\".format(TEST_PROCESS_STATS.cTotalTests))\n logging.info(\" EXECUTED : {0}\".format(TEST_PROCESS_STATS.cExecutedTests))\n logging.info(\" NOT EXECUTED : {0}\".format(TEST_PROCESS_STATS.cNotExecutedTests))\n logging.info(\" ACHTUNG : {0}\".format(TEST_PROCESS_STATS.cAchtungTests))\n logging.info(\"\")\n logging.info(\" PASSED : {0}\".format(TEST_PROCESS_STATS.cPassedTests))\n logging.info(\" FAILED : {0}\".format(TEST_PROCESS_STATS.cFailedTests))\n logging.info(\" XFAILED : {0}\".format(TEST_PROCESS_STATS.cXFailedTests))\n logging.info(\" NOT XFAILED : {0}\".format(TEST_PROCESS_STATS.cNotXFailedTests))\n logging.info(\" SKIPPED : {0}\".format(TEST_PROCESS_STATS.cSkippedTests))\n logging.info(\" WITH WARNINGS: {0}\".format(TEST_PROCESS_STATS.cWarningTests))\n logging.info(\" UNEXPECTED : {0}\".format(TEST_PROCESS_STATS.cUnexpectedTests))\n logging.info(\"\")\n\n assert type(TEST_PROCESS_STATS.cTotalDuration) is datetime.timedelta\n\n LOCAL__print_line1_with_header(\"TIME\")\n logging.info(\"\")\n logging.info(\n \" TOTAL DURATION: {0}\".format(\n timedelta_to_human_text(TEST_PROCESS_STATS.cTotalDuration)\n )\n )\n logging.info(\"\")\n\n LOCAL__print_line1_with_header(\"TOTAL INFORMATION\")\n logging.info(\"\")\n logging.info(\" TOTAL ERROR COUNT : {0}\".format(TEST_PROCESS_STATS.cTotalErrors))\n logging.info(\" TOTAL WARNING COUNT: {0}\".format(TEST_PROCESS_STATS.cTotalWarnings))\n logging.info(\"\")\n\n\n# /////////////////////////////////////////////////////////////////////////////\n\n\ndef helper__detect_test_process_kind(config: pytest.Config) -> T_TEST_PROCESS_KIND:\n assert isinstance(config, pytest.Config)\n\n #\n # xdist' master process registers DSession plugin.\n #\n p = config.pluginmanager.get_plugin(\"dsession\")\n\n if p is not None:\n return T_TEST_PROCESS_KIND.Master\n\n return T_TEST_PROCESS_KIND.Worker\n\n\n# ------------------------------------------------------------------------\ndef helper__detect_test_process_mode(config: pytest.Config) -> T_TEST_PROCESS_MODE:\n assert isinstance(config, pytest.Config)\n\n if config.getvalue(\"collectonly\"):\n return T_TEST_PROCESS_MODE.Collect\n\n return T_TEST_PROCESS_MODE.ExecTests\n\n\n# ------------------------------------------------------------------------\n@pytest.hookimpl(trylast=True)\ndef helper__pytest_configure__logging(config: pytest.Config) -> None:\n assert isinstance(config, pytest.Config)\n\n log_name = TestStartupData.GetCurrentTestWorkerSignature()\n log_name += \".log\"\n\n log_dir = TestStartupData.GetRootLogDir()\n\n pathlib.Path(log_dir).mkdir(exist_ok=True)\n\n logging_plugin = config.pluginmanager.get_plugin(\"logging-plugin\")\n\n assert logging_plugin is not None\n assert isinstance(logging_plugin, _pytest.logging.LoggingPlugin)\n\n log_file_path = os.path.join(log_dir, log_name)\n assert log_file_path is not None\n assert type(log_file_path) is str\n\n logging_plugin.set_log_path(log_file_path)\n return\n\n\n# ------------------------------------------------------------------------\n@pytest.hookimpl(trylast=True)\ndef pytest_configure(config: pytest.Config) -> None:\n assert isinstance(config, pytest.Config)\n\n global g_test_process_kind\n global g_test_process_mode\n global g_worker_log_is_created\n\n assert g_test_process_kind is None\n assert g_test_process_mode is None\n assert g_worker_log_is_created is None\n\n g_test_process_mode = helper__detect_test_process_mode(config)\n g_test_process_kind = helper__detect_test_process_kind(config)\n\n assert type(g_test_process_kind) is T_TEST_PROCESS_KIND\n assert type(g_test_process_mode) is T_TEST_PROCESS_MODE\n\n if g_test_process_kind == T_TEST_PROCESS_KIND.Master:\n pass\n else:\n assert g_test_process_kind == T_TEST_PROCESS_KIND.Worker\n\n if g_test_process_mode == T_TEST_PROCESS_MODE.Collect:\n g_worker_log_is_created = False\n else:\n assert g_test_process_mode == T_TEST_PROCESS_MODE.ExecTests\n helper__pytest_configure__logging(config)\n g_worker_log_is_created = True\n\n return\n\n\n# /////////////////////////////////////////////////////////////////////////////\n" }, { "path": "tests/helpers/__init__.py", "content": "" }, { "path": "tests/helpers/global_data.py", "content": "from testgres.operations.os_ops import OsOperations\nfrom testgres.operations.os_ops import ConnectionParams\nfrom testgres.operations.local_ops import LocalOperations\nfrom testgres.operations.remote_ops import RemoteOperations\n\nfrom src.node import PortManager\nfrom src.node import PortManager__ThisHost\nfrom src.node import PortManager__Generic\n\nimport os\n\n\nclass OsOpsDescr:\n sign: str\n os_ops: OsOperations\n\n def __init__(self, sign: str, os_ops: OsOperations):\n assert type(sign) is str\n assert isinstance(os_ops, OsOperations)\n self.sign = sign\n self.os_ops = os_ops\n\n\nclass OsOpsDescrs:\n sm_remote_conn_params = ConnectionParams(\n host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1',\n username=os.getenv('USER'),\n ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY'))\n\n sm_remote_os_ops = RemoteOperations(sm_remote_conn_params)\n\n sm_remote_os_ops_descr = OsOpsDescr(\"remote_ops\", sm_remote_os_ops)\n\n sm_local_os_ops = LocalOperations.get_single_instance()\n\n sm_local_os_ops_descr = OsOpsDescr(\"local_ops\", sm_local_os_ops)\n\n\nclass PortManagers:\n sm_remote_port_manager = PortManager__Generic(OsOpsDescrs.sm_remote_os_ops)\n\n sm_local_port_manager = PortManager__ThisHost.get_single_instance()\n\n sm_local2_port_manager = PortManager__Generic(OsOpsDescrs.sm_local_os_ops)\n\n\nclass PostgresNodeService:\n sign: str\n os_ops: OsOperations\n port_manager: PortManager\n\n def __init__(self, sign: str, os_ops: OsOperations, port_manager: PortManager):\n assert type(sign) is str\n assert isinstance(os_ops, OsOperations)\n assert isinstance(port_manager, PortManager)\n self.sign = sign\n self.os_ops = os_ops\n self.port_manager = port_manager\n\n\nclass PostgresNodeServices:\n sm_remote = PostgresNodeService(\n \"remote\",\n OsOpsDescrs.sm_remote_os_ops,\n PortManagers.sm_remote_port_manager\n )\n\n sm_local = PostgresNodeService(\n \"local\",\n OsOpsDescrs.sm_local_os_ops,\n PortManagers.sm_local_port_manager\n )\n\n sm_local2 = PostgresNodeService(\n \"local2\",\n OsOpsDescrs.sm_local_os_ops,\n PortManagers.sm_local2_port_manager\n )\n\n sm_locals_and_remotes = [\n sm_local,\n sm_local2,\n sm_remote,\n ]\n" }, { "path": "tests/helpers/pg_node_utils.py", "content": "from src import PostgresNode\nfrom src import PortManager\nfrom src import OsOperations\nfrom src import NodeStatus\nfrom src.node import PostgresNodeLogReader\n\nfrom tests.helpers.utils import Utils as HelperUtils\nfrom tests.helpers.utils import T_WAIT_TIME\n\nfrom tests.helpers.global_data import PostgresNodeService\n\nimport typing\n\n\nclass PostgresNodeUtils:\n class PostgresNodeUtilsException(Exception):\n pass\n\n class PortConflictNodeException(PostgresNodeUtilsException):\n _data_dir: str\n _port: int\n\n def __init__(self, data_dir: str, port: int):\n assert type(data_dir) is str\n assert type(port) is int\n\n super().__init__()\n\n self._data_dir = data_dir\n self._port = port\n return\n\n @property\n def data_dir(self) -> str:\n assert type(self._data_dir) is str\n return self._data_dir\n\n @property\n def port(self) -> int:\n assert type(self._port) is int\n return self._port\n\n @property\n def message(self) -> str:\n assert type(self._data_dir) is str\n assert type(self._port) is int\n\n r = \"PostgresNode [data:{}][port: {}] conflicts with port of another instance.\".format(\n self._data_dir,\n self._port,\n )\n assert type(r) is str\n return r\n\n def __str__(self) -> str:\n r = self.message\n assert type(r) is str\n return r\n\n def __repr__(self) -> str:\n # It must be overrided!\n assert type(self) is __class__\n r = \"{}({}, {})\".format(\n __class__.__name__,\n repr(self._data_dir),\n repr(self._port),\n )\n assert type(r) is str\n return r\n\n # --------------------------------------------------------------------\n class StartNodeException(PostgresNodeUtilsException):\n _data_dir: str\n _files: typing.Optional[typing.Iterable]\n\n def __init__(\n self,\n data_dir: str,\n files: typing.Optional[typing.Iterable] = None\n ):\n assert type(data_dir) is str\n assert files is None or isinstance(files, typing.Iterable)\n\n super().__init__()\n\n self._data_dir = data_dir\n self._files = files\n return\n\n @property\n def message(self) -> str:\n assert self._data_dir is None or type(self._data_dir) is str\n assert self._files is None or isinstance(self._files, typing.Iterable)\n\n msg_parts = []\n\n msg_parts.append(\"PostgresNode [data_dir: {}] is not started.\".format(\n self._data_dir\n ))\n\n for f, lines in self._files or []:\n assert type(f) is str\n assert type(lines) in [str, bytes]\n msg_parts.append(u'{}\\n----\\n{}\\n'.format(f, lines))\n\n return \"\\n\".join(msg_parts)\n\n @property\n def data_dir(self) -> typing.Optional[str]:\n assert type(self._data_dir) is str\n return self._data_dir\n\n @property\n def files(self) -> typing.Optional[typing.Iterable]:\n assert self._files is None or isinstance(self._files, typing.Iterable)\n return self._files\n\n def __repr__(self) -> str:\n assert type(self._data_dir) is str\n assert self._files is None or isinstance(self._files, typing.Iterable)\n\n r = \"{}({}, {})\".format(\n __class__.__name__,\n repr(self._data_dir),\n repr(self._files),\n )\n assert type(r) is str\n return r\n\n # --------------------------------------------------------------------\n @staticmethod\n def get_node(\n node_svc: PostgresNodeService,\n name: typing.Optional[str] = None,\n port: typing.Optional[int] = None,\n port_manager: typing.Optional[PortManager] = None\n ) -> PostgresNode:\n assert isinstance(node_svc, PostgresNodeService)\n assert isinstance(node_svc.os_ops, OsOperations)\n assert isinstance(node_svc.port_manager, PortManager)\n\n if port_manager is None:\n port_manager = node_svc.port_manager\n\n return PostgresNode(\n name,\n port=port,\n os_ops=node_svc.os_ops,\n port_manager=port_manager if port is None else None\n )\n\n # --------------------------------------------------------------------\n @staticmethod\n def wait_for_running_state(\n node: PostgresNode,\n node_log_reader: PostgresNodeLogReader,\n timeout: T_WAIT_TIME,\n ):\n assert type(node) is PostgresNode\n assert type(node_log_reader) is PostgresNodeLogReader\n assert type(timeout) in [int, float]\n assert node_log_reader._node is node\n assert timeout > 0\n\n for _ in HelperUtils.WaitUntil(\n timeout=timeout\n ):\n s = node.status()\n\n if s == NodeStatus.Running:\n return\n\n assert s == NodeStatus.Stopped\n\n blocks = node_log_reader.read()\n assert type(blocks) is list\n\n for block in blocks:\n assert type(block) is PostgresNodeLogReader.LogDataBlock\n\n if 'Is another postmaster already running on port' in block.data:\n raise __class__.PortConflictNodeException(node.data_dir, node.port)\n\n if 'database system is shut down' in block.data:\n raise __class__.StartNodeException(\n node.data_dir,\n node._collect_special_files(),\n )\n continue\n" }, { "path": "tests/helpers/run_conditions.py", "content": "# coding: utf-8\nimport pytest\nimport platform\n\n\nclass RunConditions:\n # It is not a test kit!\n __test__ = False\n\n @staticmethod\n def skip_if_windows():\n if platform.system().lower() == \"windows\":\n pytest.skip(\"This test does not support Windows.\")\n" }, { "path": "tests/helpers/utils.py", "content": "import typing\nimport time\nimport logging\n\n\nT_WAIT_TIME = typing.Union[int, float]\n\n\nclass Utils:\n @staticmethod\n def PrintAndSleep(wait: T_WAIT_TIME):\n assert type(wait) in [int, float]\n logging.info(\"Wait for {} second(s)\".format(wait))\n time.sleep(wait)\n return\n\n @staticmethod\n def WaitUntil(\n error_message: str = \"Did not complete\",\n timeout: T_WAIT_TIME = 30,\n interval: T_WAIT_TIME = 1,\n notification_interval: T_WAIT_TIME = 5,\n ):\n \"\"\"\n Loop until the timeout is reached. If the timeout is reached, raise an\n exception with the given error message.\n\n Source of idea: pgbouncer\n \"\"\"\n assert type(timeout) in [int, float]\n assert type(interval) in [int, float]\n assert type(notification_interval) in [int, float]\n assert timeout >= 0\n assert interval >= 0\n assert notification_interval >= 0\n\n start_ts = time.monotonic()\n end_ts = start_ts + timeout\n last_printed_progress = start_ts\n last_iteration_ts = start_ts\n\n yield\n attempt = 1\n\n while end_ts > time.monotonic():\n if (timeout > 5 and time.monotonic() - last_printed_progress) > notification_interval:\n last_printed_progress = time.monotonic()\n\n m = \"{} in {} seconds and {} attempts - will retry\".format(\n error_message,\n time.monotonic() - start_ts,\n attempt,\n )\n logging.info(m)\n\n interval_remaining = last_iteration_ts + interval - time.monotonic()\n if interval_remaining > 0:\n time.sleep(interval_remaining)\n\n last_iteration_ts = time.monotonic()\n yield\n attempt += 1\n continue\n\n raise TimeoutError(error_message + \" in time\")\n" }, { "path": "tests/requirements.txt", "content": "psutil\npytest\npytest-env\npytest-xdist\npsycopg2\nsix\ntestgres.os_ops>=2.1.0,<3.0.0\n" }, { "path": "tests/test_config.py", "content": "from src import TestgresConfig\nfrom src import configure_testgres\nfrom src import scoped_config\nfrom src import pop_config\n\nimport src as testgres\n\nimport pytest\n\n\nclass TestConfig:\n def test_config_stack(self):\n # no such option\n with pytest.raises(expected_exception=TypeError):\n configure_testgres(dummy=True)\n\n # we have only 1 config in stack\n with pytest.raises(expected_exception=IndexError):\n pop_config()\n\n d0 = TestgresConfig.cached_initdb_dir\n d1 = 'dummy_abc'\n d2 = 'dummy_def'\n\n with scoped_config(cached_initdb_dir=d1) as c1:\n assert (c1.cached_initdb_dir == d1)\n\n with scoped_config(cached_initdb_dir=d2) as c2:\n stack_size = len(testgres.config.config_stack)\n\n # try to break a stack\n with pytest.raises(expected_exception=TypeError):\n with scoped_config(dummy=True):\n pass\n\n assert (c2.cached_initdb_dir == d2)\n assert (len(testgres.config.config_stack) == stack_size)\n\n assert (c1.cached_initdb_dir == d1)\n\n assert (TestgresConfig.cached_initdb_dir == d0)\n" }, { "path": "tests/test_conftest.py--devel", "content": "import pytest\nimport logging\n\n\nclass TestConfest:\n def test_failed(self):\n raise Exception(\"TEST EXCEPTION!\")\n\n def test_ok(self):\n pass\n\n @pytest.mark.skip()\n def test_mark_skip__no_reason(self):\n pass\n\n @pytest.mark.xfail()\n def test_mark_xfail__no_reason(self):\n raise Exception(\"XFAIL EXCEPTION\")\n\n @pytest.mark.xfail()\n def test_mark_xfail__no_reason___no_error(self):\n pass\n\n @pytest.mark.skip(reason=\"reason\")\n def test_mark_skip__with_reason(self):\n pass\n\n @pytest.mark.xfail(reason=\"reason\")\n def test_mark_xfail__with_reason(self):\n raise Exception(\"XFAIL EXCEPTION\")\n\n @pytest.mark.xfail(reason=\"reason\")\n def test_mark_xfail__with_reason___no_error(self):\n pass\n\n def test_exc_skip__no_reason(self):\n pytest.skip()\n\n def test_exc_xfail__no_reason(self):\n pytest.xfail()\n\n def test_exc_skip__with_reason(self):\n pytest.skip(reason=\"SKIP REASON\")\n\n def test_exc_xfail__with_reason(self):\n pytest.xfail(reason=\"XFAIL EXCEPTION\")\n\n def test_log_error(self):\n logging.error(\"IT IS A LOG ERROR!\")\n\n def test_log_error_and_exc(self):\n logging.error(\"IT IS A LOG ERROR!\")\n\n raise Exception(\"TEST EXCEPTION!\")\n\n def test_log_error_and_warning(self):\n logging.error(\"IT IS A LOG ERROR!\")\n logging.warning(\"IT IS A LOG WARNING!\")\n logging.error(\"IT IS THE SECOND LOG ERROR!\")\n logging.warning(\"IT IS THE SECOND LOG WARNING!\")\n\n @pytest.mark.xfail()\n def test_log_error_and_xfail_mark_without_reason(self):\n logging.error(\"IT IS A LOG ERROR!\")\n\n @pytest.mark.xfail(reason=\"It is a reason message\")\n def test_log_error_and_xfail_mark_with_reason(self):\n logging.error(\"IT IS A LOG ERROR!\")\n\n @pytest.mark.xfail()\n def test_two_log_error_and_xfail_mark_without_reason(self):\n logging.error(\"IT IS THE FIRST LOG ERROR!\")\n logging.info(\"----------\")\n logging.error(\"IT IS THE SECOND LOG ERROR!\")\n\n @pytest.mark.xfail(reason=\"It is a reason message\")\n def test_two_log_error_and_xfail_mark_with_reason(self):\n logging.error(\"IT IS THE FIRST LOG ERROR!\")\n logging.info(\"----------\")\n logging.error(\"IT IS THE SECOND LOG ERROR!\")\n" }, { "path": "tests/test_os_ops_common.py", "content": "# coding: utf-8\nfrom .helpers.global_data import OsOpsDescr\nfrom .helpers.global_data import OsOpsDescrs\nfrom .helpers.global_data import OsOperations\nfrom .helpers.run_conditions import RunConditions\n\nimport os\nimport sys\n\nimport pytest\nimport re\nimport tempfile\nimport logging\nimport socket\nimport threading\nimport typing\nimport uuid\nimport subprocess\nimport psutil\nimport time\nimport signal as os_signal\n\nfrom src import InvalidOperationException\nfrom src import ExecUtilException\n\nfrom concurrent.futures import ThreadPoolExecutor\nfrom concurrent.futures import Future as ThreadFuture\n\n\nclass TestOsOpsCommon:\n sm_os_ops_descrs: typing.List[OsOpsDescr] = [\n OsOpsDescrs.sm_local_os_ops_descr,\n OsOpsDescrs.sm_remote_os_ops_descr\n ]\n\n @pytest.fixture(\n params=[descr.os_ops for descr in sm_os_ops_descrs],\n ids=[descr.sign for descr in sm_os_ops_descrs]\n )\n def os_ops(self, request: pytest.FixtureRequest) -> OsOperations:\n assert isinstance(request, pytest.FixtureRequest)\n assert isinstance(request.param, OsOperations)\n return request.param\n\n def test_get_platform(self, os_ops: OsOperations):\n assert isinstance(os_ops, OsOperations)\n p = os_ops.get_platform()\n assert p is not None\n assert type(p) is str\n assert p == sys.platform\n\n def test_get_platform__is_known(self, os_ops: OsOperations):\n assert isinstance(os_ops, OsOperations)\n p = os_ops.get_platform()\n assert p is not None\n assert type(p) is str\n assert p in {\"win32\", \"linux\"}\n\n def test_create_clone(self, os_ops: OsOperations):\n assert isinstance(os_ops, OsOperations)\n clone = os_ops.create_clone()\n assert clone is not None\n assert clone is not os_ops\n assert type(clone) is type(os_ops)\n\n def test_exec_command_success(self, os_ops: OsOperations):\n \"\"\"\n Test exec_command for successful command execution.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n RunConditions.skip_if_windows()\n\n cmd = [\"sh\", \"-c\", \"python3 --version\"]\n\n response = os_ops.exec_command(cmd)\n\n assert b'Python 3.' in response\n\n def test_exec_command_failure(self, os_ops: OsOperations):\n \"\"\"\n Test exec_command for command execution failure.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n RunConditions.skip_if_windows()\n\n cmd = [\"sh\", \"-c\", \"nonexistent_command\"]\n\n while True:\n try:\n os_ops.exec_command(cmd)\n except ExecUtilException as e:\n assert type(e.exit_code) is int\n assert e.exit_code == 127\n\n assert type(e.message) is str\n assert type(e.error) is bytes\n\n assert e.message.startswith(\"Utility exited with non-zero code (127). Error:\")\n assert \"nonexistent_command\" in e.message\n assert \"not found\" in e.message\n assert b\"nonexistent_command\" in e.error\n assert b\"not found\" in e.error\n break\n raise Exception(\"We wait an exception!\")\n\n def test_exec_command_failure__expect_error(self, os_ops: OsOperations):\n \"\"\"\n Test exec_command for command execution failure.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n RunConditions.skip_if_windows()\n\n cmd = [\"sh\", \"-c\", \"nonexistent_command\"]\n\n exit_status, result, error = os_ops.exec_command(cmd, verbose=True, expect_error=True)\n\n assert exit_status == 127\n assert result == b''\n assert type(error) is bytes\n assert b\"nonexistent_command\" in error\n assert b\"not found\" in error\n\n def test_exec_command_with_exec_env(self, os_ops: OsOperations):\n assert isinstance(os_ops, OsOperations)\n\n RunConditions.skip_if_windows()\n\n C_ENV_NAME = \"TESTGRES_TEST__EXEC_ENV_20250414\"\n\n cmd = [\"sh\", \"-c\", \"echo ${}\".format(C_ENV_NAME)]\n\n exec_env = {C_ENV_NAME: \"Hello!\"}\n\n response = os_ops.exec_command(cmd, exec_env=exec_env)\n assert response is not None\n assert type(response) is bytes\n assert response == b'Hello!\\n'\n\n response = os_ops.exec_command(cmd)\n assert response is not None\n assert type(response) is bytes\n assert response == b'\\n'\n\n def test_exec_command_with_exec_env__2(self, os_ops: OsOperations):\n assert isinstance(os_ops, OsOperations)\n\n RunConditions.skip_if_windows()\n\n C_ENV_NAME = \"TESTGRES_TEST__EXEC_ENV_20250414\"\n\n tmp_file_content = \"echo ${{{}}}\".format(C_ENV_NAME)\n\n logging.info(\"content is [{}]\".format(tmp_file_content))\n\n tmp_file = os_ops.mkstemp()\n assert type(tmp_file) is str\n assert tmp_file != \"\"\n\n logging.info(\"file is [{}]\".format(tmp_file))\n assert os_ops.path_exists(tmp_file)\n\n os_ops.write(tmp_file, tmp_file_content)\n\n cmd = [\"sh\", tmp_file]\n\n exec_env = {C_ENV_NAME: \"Hello!\"}\n\n response = os_ops.exec_command(cmd, exec_env=exec_env)\n assert response is not None\n assert type(response) is bytes\n assert response == b'Hello!\\n'\n\n response = os_ops.exec_command(cmd)\n assert response is not None\n assert type(response) is bytes\n assert response == b'\\n'\n\n os_ops.remove_file(tmp_file)\n assert not os_ops.path_exists(tmp_file)\n return\n\n def test_exec_command_with_cwd(self, os_ops: OsOperations):\n assert isinstance(os_ops, OsOperations)\n\n RunConditions.skip_if_windows()\n\n cmd = [\"pwd\"]\n\n response = os_ops.exec_command(cmd, cwd=\"/tmp\")\n assert response is not None\n assert type(response) is bytes\n assert response == b'/tmp\\n'\n\n response = os_ops.exec_command(cmd)\n assert response is not None\n assert type(response) is bytes\n assert response != b'/tmp\\n'\n\n def test_exec_command__test_unset(self, os_ops: OsOperations):\n assert isinstance(os_ops, OsOperations)\n\n RunConditions.skip_if_windows()\n\n C_ENV_NAME = \"LANG\"\n\n cmd = [\"sh\", \"-c\", \"echo ${}\".format(C_ENV_NAME)]\n\n response1 = os_ops.exec_command(cmd)\n assert response1 is not None\n assert type(response1) is bytes\n\n if response1 == b'\\n':\n logging.warning(\"Environment variable {} is not defined.\".format(C_ENV_NAME))\n return\n\n exec_env = {C_ENV_NAME: None}\n response2 = os_ops.exec_command(cmd, exec_env=exec_env)\n assert response2 is not None\n assert type(response2) is bytes\n assert response2 == b'\\n'\n\n response3 = os_ops.exec_command(cmd)\n assert response3 is not None\n assert type(response3) is bytes\n assert response3 == response1\n\n def test_exec_command__test_unset_dummy_var(self, os_ops: OsOperations):\n assert isinstance(os_ops, OsOperations)\n\n RunConditions.skip_if_windows()\n\n C_ENV_NAME = \"TESTGRES_TEST__DUMMY_VAR_20250414\"\n\n cmd = [\"sh\", \"-c\", \"echo ${}\".format(C_ENV_NAME)]\n\n exec_env = {C_ENV_NAME: None}\n response2 = os_ops.exec_command(cmd, exec_env=exec_env)\n assert response2 is not None\n assert type(response2) is bytes\n assert response2 == b'\\n'\n\n def test_is_executable_true(self, os_ops: OsOperations):\n \"\"\"\n Test is_executable for an existing executable.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n RunConditions.skip_if_windows()\n\n response = os_ops.is_executable(\"/bin/sh\")\n\n assert response is True\n\n def test_is_executable_false(self, os_ops: OsOperations):\n \"\"\"\n Test is_executable for a non-executable.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n response = os_ops.is_executable(__file__)\n\n assert response is False\n\n def test_makedirs_and_rmdirs_success(self, os_ops: OsOperations):\n \"\"\"\n Test makedirs and rmdirs for successful directory creation and removal.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n RunConditions.skip_if_windows()\n\n cmd = \"pwd\"\n pwd = os_ops.exec_command(cmd, wait_exit=True, encoding='utf-8').strip()\n\n path = \"{}/test_dir\".format(pwd)\n\n # Test makedirs\n os_ops.makedirs(path)\n assert os.path.exists(path)\n assert os_ops.path_exists(path)\n\n # Test rmdirs\n os_ops.rmdirs(path)\n assert not os.path.exists(path)\n assert not os_ops.path_exists(path)\n\n def test_makedirs_failure(self, os_ops: OsOperations):\n \"\"\"\n Test makedirs for failure.\n \"\"\"\n # Try to create a directory in a read-only location\n assert isinstance(os_ops, OsOperations)\n\n RunConditions.skip_if_windows()\n\n path = \"/root/test_dir\"\n\n # Test makedirs\n with pytest.raises(Exception):\n os_ops.makedirs(path)\n\n def test_listdir(self, os_ops: OsOperations):\n \"\"\"\n Test listdir for listing directory contents.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n RunConditions.skip_if_windows()\n\n path = \"/etc\"\n files = os_ops.listdir(path)\n assert isinstance(files, list)\n for f in files:\n assert f is not None\n assert type(f) is str\n\n def test_path_exists_true__directory(self, os_ops: OsOperations):\n \"\"\"\n Test path_exists for an existing directory.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n RunConditions.skip_if_windows()\n\n assert os_ops.path_exists(\"/etc\") is True\n\n def test_path_exists_true__file(self, os_ops: OsOperations):\n \"\"\"\n Test path_exists for an existing file.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n RunConditions.skip_if_windows()\n\n assert os_ops.path_exists(__file__) is True\n\n def test_path_exists_false__directory(self, os_ops: OsOperations):\n \"\"\"\n Test path_exists for a non-existing directory.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n RunConditions.skip_if_windows()\n\n assert os_ops.path_exists(\"/nonexistent_path\") is False\n\n def test_path_exists_false__file(self, os_ops: OsOperations):\n \"\"\"\n Test path_exists for a non-existing file.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n RunConditions.skip_if_windows()\n\n assert os_ops.path_exists(\"/etc/nonexistent_path.txt\") is False\n\n def test_mkdtemp__default(self, os_ops: OsOperations):\n assert isinstance(os_ops, OsOperations)\n\n path = os_ops.mkdtemp()\n logging.info(\"Path is [{0}].\".format(path))\n assert os.path.exists(path)\n os.rmdir(path)\n assert not os.path.exists(path)\n\n def test_mkdtemp__custom(self, os_ops: OsOperations):\n assert isinstance(os_ops, OsOperations)\n\n C_TEMPLATE = \"abcdef\"\n path = os_ops.mkdtemp(C_TEMPLATE)\n logging.info(\"Path is [{0}].\".format(path))\n assert os.path.exists(path)\n assert C_TEMPLATE in os.path.basename(path)\n os.rmdir(path)\n assert not os.path.exists(path)\n\n def test_rmdirs(self, os_ops: OsOperations):\n assert isinstance(os_ops, OsOperations)\n\n path = os_ops.mkdtemp()\n assert os.path.exists(path)\n\n assert os_ops.rmdirs(path, ignore_errors=False) is True\n assert not os.path.exists(path)\n\n def test_rmdirs__01_with_subfolder(self, os_ops: OsOperations):\n assert isinstance(os_ops, OsOperations)\n\n # folder with subfolder\n path = os_ops.mkdtemp()\n assert os.path.exists(path)\n\n dir1 = os.path.join(path, \"dir1\")\n assert not os.path.exists(dir1)\n\n os_ops.makedirs(dir1)\n assert os.path.exists(dir1)\n\n assert os_ops.rmdirs(path, ignore_errors=False) is True\n assert not os.path.exists(path)\n assert not os.path.exists(dir1)\n\n def test_rmdirs__02_with_file(self, os_ops: OsOperations):\n assert isinstance(os_ops, OsOperations)\n\n # folder with file\n path = os_ops.mkdtemp()\n assert os.path.exists(path)\n\n file1 = os.path.join(path, \"file1.txt\")\n assert not os.path.exists(file1)\n\n os_ops.touch(file1)\n assert os.path.exists(file1)\n\n assert os_ops.rmdirs(path, ignore_errors=False) is True\n assert not os.path.exists(path)\n assert not os.path.exists(file1)\n\n def test_rmdirs__03_with_subfolder_and_file(self, os_ops: OsOperations):\n assert isinstance(os_ops, OsOperations)\n\n # folder with subfolder and file\n path = os_ops.mkdtemp()\n assert os.path.exists(path)\n\n dir1 = os.path.join(path, \"dir1\")\n assert not os.path.exists(dir1)\n\n os_ops.makedirs(dir1)\n assert os.path.exists(dir1)\n\n file1 = os.path.join(dir1, \"file1.txt\")\n assert not os.path.exists(file1)\n\n os_ops.touch(file1)\n assert os.path.exists(file1)\n\n assert os_ops.rmdirs(path, ignore_errors=False) is True\n assert not os.path.exists(path)\n assert not os.path.exists(dir1)\n assert not os.path.exists(file1)\n\n def test_write_text_file(self, os_ops: OsOperations):\n \"\"\"\n Test write for writing data to a text file.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n RunConditions.skip_if_windows()\n\n filename = os_ops.mkstemp()\n data = \"Hello, world!\"\n\n os_ops.write(filename, data, truncate=True)\n os_ops.write(filename, data)\n\n response = os_ops.read(filename)\n\n assert response == data + data\n\n os_ops.remove_file(filename)\n\n def test_write_binary_file(self, os_ops: OsOperations):\n \"\"\"\n Test write for writing data to a binary file.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n RunConditions.skip_if_windows()\n\n filename = \"/tmp/test_file.bin\"\n data = b\"\\x00\\x01\\x02\\x03\"\n\n os_ops.write(filename, data, binary=True, truncate=True)\n\n response = os_ops.read(filename, binary=True)\n\n assert response == data\n\n def test_read_text_file(self, os_ops: OsOperations):\n \"\"\"\n Test read for reading data from a text file.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n RunConditions.skip_if_windows()\n\n filename = \"/etc/hosts\"\n\n response = os_ops.read(filename)\n\n assert isinstance(response, str)\n\n def test_read_binary_file(self, os_ops: OsOperations):\n \"\"\"\n Test read for reading data from a binary file.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n RunConditions.skip_if_windows()\n\n filename = \"/usr/bin/python3\"\n\n response = os_ops.read(filename, binary=True)\n\n assert isinstance(response, bytes)\n\n def test_read__text(self, os_ops: OsOperations):\n \"\"\"\n Test OsOperations::read for text data.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n filename = __file__ # current file\n\n with open(filename, 'r') as file: # open in a text mode\n response0 = file.read()\n\n assert type(response0) is str\n\n response1 = os_ops.read(filename)\n assert type(response1) is str\n assert response1 == response0\n\n response2 = os_ops.read(filename, encoding=None, binary=False)\n assert type(response2) is str\n assert response2 == response0\n\n response3 = os_ops.read(filename, encoding=\"\")\n assert type(response3) is str\n assert response3 == response0\n\n response4 = os_ops.read(filename, encoding=\"UTF-8\")\n assert type(response4) is str\n assert response4 == response0\n\n def test_read__binary(self, os_ops: OsOperations):\n \"\"\"\n Test OsOperations::read for binary data.\n \"\"\"\n filename = __file__ # current file\n\n with open(filename, 'rb') as file: # open in a binary mode\n response0 = file.read()\n\n assert type(response0) is bytes\n\n response1 = os_ops.read(filename, binary=True)\n assert type(response1) is bytes\n assert response1 == response0\n\n def test_read__binary_and_encoding(self, os_ops: OsOperations):\n \"\"\"\n Test OsOperations::read for binary data and encoding.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n filename = __file__ # current file\n\n with pytest.raises(\n InvalidOperationException,\n match=re.escape(\"Enconding is not allowed for read binary operation\")):\n os_ops.read(filename, encoding=\"\", binary=True)\n\n def test_read_binary__spec(self, os_ops: OsOperations):\n \"\"\"\n Test OsOperations::read_binary.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n filename = __file__ # currnt file\n\n with open(filename, 'rb') as file: # open in a binary mode\n response0 = file.read()\n\n assert type(response0) is bytes\n\n response1 = os_ops.read_binary(filename, 0)\n assert type(response1) is bytes\n assert response1 == response0\n\n response2 = os_ops.read_binary(filename, 1)\n assert type(response2) is bytes\n assert len(response2) < len(response1)\n assert len(response2) + 1 == len(response1)\n assert response2 == response1[1:]\n\n response3 = os_ops.read_binary(filename, len(response1))\n assert type(response3) is bytes\n assert len(response3) == 0\n\n response4 = os_ops.read_binary(filename, len(response2))\n assert type(response4) is bytes\n assert len(response4) == 1\n assert response4[0] == response1[len(response1) - 1]\n\n response5 = os_ops.read_binary(filename, len(response1) + 1)\n assert type(response5) is bytes\n assert len(response5) == 0\n\n def test_read_binary__spec__negative_offset(self, os_ops: OsOperations):\n \"\"\"\n Test OsOperations::read_binary with negative offset.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n with pytest.raises(\n ValueError,\n match=re.escape(\"Negative 'offset' is not supported.\")):\n os_ops.read_binary(__file__, -1)\n\n def test_get_file_size(self, os_ops: OsOperations):\n \"\"\"\n Test OsOperations::get_file_size.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n filename = __file__ # current file\n\n sz0 = os.path.getsize(filename)\n assert type(sz0) is int\n\n sz1 = os_ops.get_file_size(filename)\n assert type(sz1) is int\n assert sz1 == sz0\n\n def test_isfile_true(self, os_ops: OsOperations):\n \"\"\"\n Test isfile for an existing file.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n filename = __file__\n\n response = os_ops.isfile(filename)\n\n assert response is True\n\n def test_isfile_false__not_exist(self, os_ops: OsOperations):\n \"\"\"\n Test isfile for a non-existing file.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n filename = os.path.join(os.path.dirname(__file__), \"nonexistent_file.txt\")\n\n response = os_ops.isfile(filename)\n\n assert response is False\n\n def test_isfile_false__directory(self, os_ops: OsOperations):\n \"\"\"\n Test isfile for a firectory.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n name = os.path.dirname(__file__)\n\n assert os_ops.isdir(name)\n\n response = os_ops.isfile(name)\n\n assert response is False\n\n def test_isdir_true(self, os_ops: OsOperations):\n \"\"\"\n Test isdir for an existing directory.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n name = os.path.dirname(__file__)\n\n response = os_ops.isdir(name)\n\n assert response is True\n\n def test_isdir_false__not_exist(self, os_ops: OsOperations):\n \"\"\"\n Test isdir for a non-existing directory.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n name = os.path.join(os.path.dirname(__file__), \"it_is_nonexistent_directory\")\n\n response = os_ops.isdir(name)\n\n assert response is False\n\n def test_isdir_false__file(self, os_ops: OsOperations):\n \"\"\"\n Test isdir for a file.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n name = __file__\n\n assert os_ops.isfile(name)\n\n response = os_ops.isdir(name)\n\n assert response is False\n\n def test_cwd(self, os_ops: OsOperations):\n \"\"\"\n Test cwd.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n v = os_ops.cwd()\n\n assert v is not None\n assert type(v) is str\n assert v != \"\"\n\n class tagWriteData001:\n def __init__(self, sign, source, cp_rw, cp_truncate, cp_binary, cp_data, result):\n self.sign = sign\n self.source = source\n self.call_param__rw = cp_rw\n self.call_param__truncate = cp_truncate\n self.call_param__binary = cp_binary\n self.call_param__data = cp_data\n self.result = result\n\n sm_write_data001 = [\n tagWriteData001(\"A001\", \"1234567890\", False, False, False, \"ABC\", \"1234567890ABC\"),\n tagWriteData001(\"A002\", b\"1234567890\", False, False, True, b\"ABC\", b\"1234567890ABC\"),\n\n tagWriteData001(\"B001\", \"1234567890\", False, True, False, \"ABC\", \"ABC\"),\n tagWriteData001(\"B002\", \"1234567890\", False, True, False, \"ABC1234567890\", \"ABC1234567890\"),\n tagWriteData001(\"B003\", b\"1234567890\", False, True, True, b\"ABC\", b\"ABC\"),\n tagWriteData001(\"B004\", b\"1234567890\", False, True, True, b\"ABC1234567890\", b\"ABC1234567890\"),\n\n tagWriteData001(\"C001\", \"1234567890\", True, False, False, \"ABC\", \"1234567890ABC\"),\n tagWriteData001(\"C002\", b\"1234567890\", True, False, True, b\"ABC\", b\"1234567890ABC\"),\n\n tagWriteData001(\"D001\", \"1234567890\", True, True, False, \"ABC\", \"ABC\"),\n tagWriteData001(\"D002\", \"1234567890\", True, True, False, \"ABC1234567890\", \"ABC1234567890\"),\n tagWriteData001(\"D003\", b\"1234567890\", True, True, True, b\"ABC\", b\"ABC\"),\n tagWriteData001(\"D004\", b\"1234567890\", True, True, True, b\"ABC1234567890\", b\"ABC1234567890\"),\n\n tagWriteData001(\"E001\", \"\\0001234567890\\000\", False, False, False, \"\\000ABC\\000\", \"\\0001234567890\\000\\000ABC\\000\"),\n tagWriteData001(\"E002\", b\"\\0001234567890\\000\", False, False, True, b\"\\000ABC\\000\", b\"\\0001234567890\\000\\000ABC\\000\"),\n\n tagWriteData001(\"F001\", \"a\\nb\\n\", False, False, False, [\"c\", \"d\"], \"a\\nb\\nc\\nd\\n\"),\n tagWriteData001(\"F002\", b\"a\\nb\\n\", False, False, True, [b\"c\", b\"d\"], b\"a\\nb\\nc\\nd\\n\"),\n\n tagWriteData001(\"G001\", \"a\\nb\\n\", False, False, False, [\"c\\n\\n\", \"d\\n\"], \"a\\nb\\nc\\nd\\n\"),\n tagWriteData001(\"G002\", b\"a\\nb\\n\", False, False, True, [b\"c\\n\\n\", b\"d\\n\"], b\"a\\nb\\nc\\nd\\n\"),\n ]\n\n @pytest.fixture(\n params=sm_write_data001,\n ids=[x.sign for x in sm_write_data001],\n )\n def write_data001(self, request):\n assert isinstance(request, pytest.FixtureRequest)\n assert type(request.param) is __class__.tagWriteData001\n return request.param\n\n def test_write(self, write_data001: tagWriteData001, os_ops: OsOperations):\n assert type(write_data001) is __class__.tagWriteData001\n assert isinstance(os_ops, OsOperations)\n\n mode = \"w+b\" if write_data001.call_param__binary else \"w+\"\n\n with tempfile.NamedTemporaryFile(mode=mode, delete=True) as tmp_file:\n tmp_file.write(write_data001.source)\n tmp_file.flush()\n\n os_ops.write(\n tmp_file.name,\n write_data001.call_param__data,\n read_and_write=write_data001.call_param__rw,\n truncate=write_data001.call_param__truncate,\n binary=write_data001.call_param__binary)\n\n tmp_file.seek(0)\n\n s = tmp_file.read()\n\n assert s == write_data001.result\n\n def test_touch(self, os_ops: OsOperations):\n \"\"\"\n Test touch for creating a new file or updating access and modification times of an existing file.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n filename = os_ops.mkstemp()\n\n # TODO: this test does not check the result of 'touch' command!\n\n os_ops.touch(filename)\n\n assert os_ops.isfile(filename)\n\n os_ops.remove_file(filename)\n\n def test_is_port_free__true(self, os_ops: OsOperations):\n assert isinstance(os_ops, OsOperations)\n\n C_LIMIT = 128\n\n ports = set(range(1024, 65535))\n assert type(ports) is set\n\n ok_count = 0\n no_count = 0\n\n for port in ports:\n with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:\n try:\n s.bind((\"\", port))\n except OSError:\n continue\n\n r = os_ops.is_port_free(port)\n\n if r:\n ok_count += 1\n logging.info(\"OK. Port {} is free.\".format(port))\n else:\n no_count += 1\n logging.warning(\"NO. Port {} is not free.\".format(port))\n\n if ok_count == C_LIMIT:\n return\n\n if no_count == C_LIMIT:\n raise RuntimeError(\"To many false positive test attempts.\")\n\n if ok_count == 0:\n raise RuntimeError(\"No one free port was found.\")\n\n def test_is_port_free__false(self, os_ops: OsOperations):\n assert isinstance(os_ops, OsOperations)\n\n C_LIMIT = 10\n\n ports = set(range(1024, 65535))\n assert type(ports) is set\n\n def LOCAL_server(s: socket.socket):\n assert s is not None\n assert type(s) is socket.socket\n\n try:\n while True:\n r = s.accept()\n\n if r is None:\n break\n except Exception as e:\n assert e is not None\n pass\n\n ok_count = 0\n no_count = 0\n\n for port in ports:\n with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:\n try:\n s.bind((\"\", port))\n except OSError:\n continue\n\n th = threading.Thread(target=LOCAL_server, args=[s])\n\n s.listen(10)\n\n assert type(th) is threading.Thread\n th.start()\n\n try:\n r = os_ops.is_port_free(port)\n finally:\n s.shutdown(2)\n th.join()\n\n if not r:\n ok_count += 1\n logging.info(\"OK. Port {} is not free.\".format(port))\n else:\n no_count += 1\n logging.warning(\"NO. Port {} does not accept connection.\".format(port))\n\n if ok_count == C_LIMIT:\n return\n\n if no_count == C_LIMIT:\n raise RuntimeError(\"To many false positive test attempts.\")\n\n if ok_count == 0:\n raise RuntimeError(\"No one free port was found.\")\n\n def test_get_tmpdir(self, os_ops: OsOperations):\n assert isinstance(os_ops, OsOperations)\n\n dir = os_ops.get_tempdir()\n assert type(dir) is str\n assert os_ops.path_exists(dir)\n assert os.path.exists(dir)\n\n file_path = os.path.join(dir, \"testgres--\" + uuid.uuid4().hex + \".tmp\")\n\n os_ops.write(file_path, \"1234\", binary=False)\n\n assert os_ops.path_exists(file_path)\n assert os.path.exists(file_path)\n\n d = os_ops.read(file_path, binary=False)\n\n assert d == \"1234\"\n\n os_ops.remove_file(file_path)\n\n assert not os_ops.path_exists(file_path)\n assert not os.path.exists(file_path)\n\n def test_get_tmpdir__compare_with_py_info(self, os_ops: OsOperations):\n assert isinstance(os_ops, OsOperations)\n\n actual_dir = os_ops.get_tempdir()\n assert actual_dir is not None\n assert type(actual_dir) is str\n\n # --------\n cmd = [sys.executable, \"-c\", \"import tempfile;print(tempfile.gettempdir());\"]\n\n expected_dir_b = os_ops.exec_command(cmd)\n assert type(expected_dir_b) is bytes\n expected_dir = expected_dir_b.decode()\n assert type(expected_dir) is str\n assert actual_dir + \"\\n\" == expected_dir\n return\n\n class tagData_OS_OPS__NUMS:\n os_ops_descr: OsOpsDescr\n nums: int\n\n def __init__(self, os_ops_descr: OsOpsDescr, nums: int):\n assert isinstance(os_ops_descr, OsOpsDescr)\n assert type(nums) is int\n\n self.os_ops_descr = os_ops_descr\n self.nums = nums\n\n sm_test_exclusive_creation__mt__data = [\n tagData_OS_OPS__NUMS(OsOpsDescrs.sm_local_os_ops_descr, 100000),\n tagData_OS_OPS__NUMS(OsOpsDescrs.sm_remote_os_ops_descr, 120),\n ]\n\n @pytest.fixture(\n params=sm_test_exclusive_creation__mt__data,\n ids=[x.os_ops_descr.sign for x in sm_test_exclusive_creation__mt__data]\n )\n def data001(self, request: pytest.FixtureRequest) -> tagData_OS_OPS__NUMS:\n assert isinstance(request, pytest.FixtureRequest)\n return request.param\n\n def test_mkdir__mt(self, data001: tagData_OS_OPS__NUMS):\n assert type(data001) is __class__.tagData_OS_OPS__NUMS\n\n N_WORKERS = 4\n N_NUMBERS = data001.nums\n assert type(N_NUMBERS) is int\n\n os_ops = data001.os_ops_descr.os_ops\n assert isinstance(os_ops, OsOperations)\n\n lock_dir_prefix = \"test_mkdir_mt--\" + uuid.uuid4().hex\n\n lock_dir = os_ops.mkdtemp(prefix=lock_dir_prefix)\n\n logging.info(\"A lock file [{}] is creating ...\".format(lock_dir))\n\n assert os.path.exists(lock_dir)\n\n def MAKE_PATH(lock_dir: str, num: int) -> str:\n assert type(lock_dir) is str\n assert type(num) is int\n return os.path.join(lock_dir, str(num) + \".lock\")\n\n def LOCAL_WORKER(os_ops: OsOperations,\n workerID: int,\n lock_dir: str,\n cNumbers: int,\n reservedNumbers: typing.Set[int]) -> None:\n assert isinstance(os_ops, OsOperations)\n assert type(workerID) is int\n assert type(lock_dir) is str\n assert type(cNumbers) is int\n assert type(reservedNumbers) is set\n assert cNumbers > 0\n assert len(reservedNumbers) == 0\n\n assert os.path.exists(lock_dir)\n\n def LOG_INFO(template: str, *args) -> None:\n assert type(template) is str\n assert type(args) is tuple\n\n msg = template.format(*args)\n assert type(msg) is str\n\n logging.info(\"[Worker #{}] {}\".format(workerID, msg))\n return\n\n LOG_INFO(\"HELLO! I am here!\")\n\n for num in range(cNumbers):\n assert num not in reservedNumbers\n\n file_path = MAKE_PATH(lock_dir, num)\n\n try:\n os_ops.makedir(file_path)\n except Exception as e:\n LOG_INFO(\n \"Can't reserve {}. Error ({}): {}\",\n num,\n type(e).__name__,\n str(e)\n )\n continue\n\n LOG_INFO(\"Number {} is reserved!\", num)\n assert os_ops.path_exists(file_path)\n reservedNumbers.add(num)\n continue\n\n n_total = cNumbers\n n_ok = len(reservedNumbers)\n assert n_ok <= n_total\n\n LOG_INFO(\"Finish! OK: {}. FAILED: {}.\", n_ok, n_total - n_ok)\n return\n\n # -----------------------\n logging.info(\"Worker are creating ...\")\n\n threadPool = ThreadPoolExecutor(\n max_workers=N_WORKERS,\n thread_name_prefix=\"ex_creator\"\n )\n\n class tadWorkerData:\n future: ThreadFuture\n reservedNumbers: typing.Set[int]\n\n workerDatas: typing.List[tadWorkerData] = list()\n\n nErrors = 0\n\n try:\n for n in range(N_WORKERS):\n logging.info(\"worker #{} is creating ...\".format(n))\n\n workerDatas.append(tadWorkerData())\n\n workerDatas[n].reservedNumbers = set()\n\n workerDatas[n].future = threadPool.submit(\n LOCAL_WORKER,\n os_ops,\n n,\n lock_dir,\n N_NUMBERS,\n workerDatas[n].reservedNumbers\n )\n\n assert workerDatas[n].future is not None\n\n logging.info(\"OK. All the workers were created!\")\n except Exception as e:\n nErrors += 1\n logging.error(\"A problem is detected ({}): {}\".format(type(e).__name__, str(e)))\n\n logging.info(\"Will wait for stop of all the workers...\")\n\n nWorkers = 0\n\n assert type(workerDatas) is list\n\n for i in range(len(workerDatas)):\n worker = workerDatas[i].future\n\n if worker is None:\n continue\n\n nWorkers += 1\n\n assert isinstance(worker, ThreadFuture)\n\n try:\n logging.info(\"Wait for worker #{}\".format(i))\n worker.result()\n except Exception as e:\n nErrors += 1\n logging.error(\"Worker #{} finished with error ({}): {}\".format(\n i,\n type(e).__name__,\n str(e),\n ))\n continue\n\n assert nWorkers == N_WORKERS\n\n if nErrors != 0:\n raise RuntimeError(\"Some problems were detected. Please examine the log messages.\")\n\n logging.info(\"OK. Let's check worker results!\")\n\n reservedNumbers: typing.Dict[int, int] = dict()\n\n for i in range(N_WORKERS):\n logging.info(\"Worker #{} is checked ...\".format(i))\n\n workerNumbers = workerDatas[i].reservedNumbers\n assert type(workerNumbers) is set\n\n for n in workerNumbers:\n if n < 0 or n >= N_NUMBERS:\n nErrors += 1\n logging.error(\"Unexpected number {}\".format(n))\n continue\n\n if n in reservedNumbers.keys():\n nErrors += 1\n logging.error(\"Number {} was already reserved by worker #{}\".format(\n n,\n reservedNumbers[n]\n ))\n else:\n reservedNumbers[n] = i\n\n file_path = MAKE_PATH(lock_dir, n)\n if not os_ops.path_exists(file_path):\n nErrors += 1\n logging.error(\"File {} is not found!\".format(file_path))\n continue\n\n continue\n\n logging.info(\"OK. Let's check reservedNumbers!\")\n\n for n in range(N_NUMBERS):\n if n not in reservedNumbers.keys():\n nErrors += 1\n logging.error(\"Number {} is not reserved!\".format(n))\n continue\n\n file_path = MAKE_PATH(lock_dir, n)\n if not os_ops.path_exists(file_path):\n nErrors += 1\n logging.error(\"File {} is not found!\".format(file_path))\n continue\n\n # OK!\n continue\n\n logging.info(\"Verification is finished! Total error count is {}.\".format(nErrors))\n\n if nErrors == 0:\n logging.info(\"Root lock-directory [{}] will be deleted.\".format(\n lock_dir\n ))\n\n for n in range(N_NUMBERS):\n file_path = MAKE_PATH(lock_dir, n)\n try:\n os_ops.rmdir(file_path)\n except Exception as e:\n nErrors += 1\n logging.error(\"Cannot delete directory [{}]. Error ({}): {}\".format(\n file_path,\n type(e).__name__,\n str(e)\n ))\n continue\n\n if os_ops.path_exists(file_path):\n nErrors += 1\n logging.error(\"Directory {} is not deleted!\".format(file_path))\n continue\n\n if nErrors == 0:\n try:\n os_ops.rmdir(lock_dir)\n except Exception as e:\n nErrors += 1\n logging.error(\"Cannot delete directory [{}]. Error ({}): {}\".format(\n lock_dir,\n type(e).__name__,\n str(e)\n ))\n\n logging.info(\"Test is finished! Total error count is {}.\".format(nErrors))\n return\n\n T_KILL_SIGNAL_DESCR = typing.Tuple[\n str,\n typing.Union[int, os_signal.Signals],\n str\n ]\n\n sm_kill_signal_ids: typing.List[T_KILL_SIGNAL_DESCR] = [\n (\"SIGINT\", os_signal.SIGINT, \"2\"),\n # (\"SIGQUIT\", os_signal.SIGQUIT, \"3\"), # it creates coredump\n (\"SIGKILL\", os_signal.SIGKILL, \"9\"),\n (\"SIGTERM\", os_signal.SIGTERM, \"15\"),\n (\"2\", 2, \"2\"),\n # (\"3\", 3, \"3\"), # it creates coredump\n (\"9\", 9, \"9\"),\n (\"15\", 15, \"15\"),\n ]\n\n @pytest.fixture(\n params=sm_kill_signal_ids,\n ids=[\"signal: {}\".format(x[0]) for x in sm_kill_signal_ids],\n )\n def kill_signal_id(self, request: pytest.FixtureRequest) -> T_KILL_SIGNAL_DESCR:\n assert isinstance(request, pytest.FixtureRequest)\n assert type(request.param) is tuple\n return request.param\n\n def test_kill_signal(\n self,\n kill_signal_id: T_KILL_SIGNAL_DESCR,\n ):\n assert type(kill_signal_id) is tuple\n assert \"{}\".format(kill_signal_id[1]) == kill_signal_id[2]\n assert \"{}\".format(int(kill_signal_id[1])) == kill_signal_id[2]\n\n def test_kill(\n self,\n os_ops: OsOperations,\n kill_signal_id: T_KILL_SIGNAL_DESCR,\n ):\n \"\"\"\n Test listdir for listing directory contents.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n assert type(kill_signal_id) is tuple\n\n cmd = [\n sys.executable,\n \"-c\",\n \"import time; print('ENTER');time.sleep(300);print('EXIT')\"\n ]\n\n logging.info(\"Local test process is creating ...\")\n proc = subprocess.Popen(\n cmd,\n text=True,\n )\n\n assert proc is not None\n assert type(proc) is subprocess.Popen\n proc_pid = proc.pid\n assert type(proc_pid) is int\n logging.info(\"Test process pid is {}\".format(proc_pid))\n\n logging.info(\"Get this test process ...\")\n p1 = psutil.Process(proc_pid)\n assert p1 is not None\n del p1\n\n logging.info(\"Kill this test process ...\")\n os_ops.kill(proc_pid, kill_signal_id[1])\n\n logging.info(\"Wait for finish ...\")\n proc.wait()\n\n logging.info(\"Try to get this test process ...\")\n\n attempt = 0\n while True:\n if attempt == 20:\n raise RuntimeError(\"Process did not die.\")\n\n attempt += 1\n\n if attempt > 1:\n logging.info(\"Sleep 1 seconds...\")\n time.sleep(1)\n\n try:\n psutil.Process(proc_pid)\n except psutil.ZombieProcess as e:\n logging.info(\"Exception {}: {}\".format(\n type(e).__name__,\n str(e),\n ))\n except psutil.NoSuchProcess:\n logging.info(\"OK. Process died.\")\n break\n\n logging.info(\"Process is alive!\")\n continue\n\n return\n\n def test_kill__unk_pid(\n self,\n os_ops: OsOperations,\n kill_signal_id: T_KILL_SIGNAL_DESCR,\n ):\n \"\"\"\n Test listdir for listing directory contents.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n assert type(kill_signal_id) is tuple\n\n cmd = [\n sys.executable,\n \"-c\",\n \"import sys; print(\\\"a\\\", file=sys.stdout); print(\\\"b\\\", file=sys.stderr)\"\n ]\n\n logging.info(\"Local test process is creating ...\")\n proc = subprocess.Popen(\n cmd,\n text=True,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n )\n\n assert proc is not None\n assert type(proc) is subprocess.Popen\n proc_pid = proc.pid\n assert type(proc_pid) is int\n logging.info(\"Test process pid is {}\".format(proc_pid))\n\n logging.info(\"Wait for finish ...\")\n pout, perr = proc.communicate()\n logging.info(\"STDOUT: {}\".format(pout))\n logging.info(\"STDERR: {}\".format(pout))\n assert type(pout) is str\n assert type(perr) is str\n assert pout == \"a\\n\"\n assert perr == \"b\\n\"\n assert type(proc.returncode) is int\n assert proc.returncode == 0\n\n logging.info(\"Try to get this test process ...\")\n\n attempt = 0\n while True:\n if attempt == 20:\n raise RuntimeError(\"Process did not die.\")\n\n attempt += 1\n\n if attempt > 1:\n logging.info(\"Sleep 1 seconds...\")\n time.sleep(1)\n\n try:\n psutil.Process(proc_pid)\n except psutil.ZombieProcess as e:\n logging.info(\"Exception {}: {}\".format(\n type(e).__name__,\n str(e),\n ))\n except psutil.NoSuchProcess:\n logging.info(\"OK. Process died.\")\n break\n\n logging.info(\"Process is alive!\")\n continue\n\n # --------------------\n with pytest.raises(expected_exception=Exception) as x:\n os_ops.kill(proc_pid, kill_signal_id[1])\n\n assert x is not None\n assert isinstance(x.value, Exception)\n assert not isinstance(x.value, AssertionError)\n\n logging.info(\"Our error is [{}]\".format(str(x.value)))\n logging.info(\"Our exception has type [{}]\".format(type(x.value).__name__))\n\n if type(os_ops).__name__ == \"LocalOsOperations\":\n assert type(x.value) is ProcessLookupError\n assert \"No such process\" in str(x.value)\n elif type(os_ops).__name__ == \"RemoteOsOperations\":\n assert type(x.value) is ExecUtilException\n assert \"No such process\" in str(x.value)\n else:\n RuntimeError(\"Unknown os_ops type: {}\".format(type(os_ops).__name__))\n\n return\n" }, { "path": "tests/test_os_ops_local.py", "content": "# coding: utf-8\nfrom .helpers.global_data import OsOpsDescrs\nfrom .helpers.global_data import OsOperations\n\nimport os\n\nimport pytest\nimport re\n\n\nclass TestOsOpsLocal:\n @pytest.fixture\n def os_ops(self):\n return OsOpsDescrs.sm_local_os_ops\n\n def test_read__unknown_file(self, os_ops: OsOperations):\n \"\"\"\n Test LocalOperations::read with unknown file.\n \"\"\"\n\n with pytest.raises(FileNotFoundError, match=re.escape(\"[Errno 2] No such file or directory: '/dummy'\")):\n os_ops.read(\"/dummy\")\n\n def test_read_binary__spec__unk_file(self, os_ops: OsOperations):\n \"\"\"\n Test LocalOperations::read_binary with unknown file.\n \"\"\"\n\n with pytest.raises(\n FileNotFoundError,\n match=re.escape(\"[Errno 2] No such file or directory: '/dummy'\")):\n os_ops.read_binary(\"/dummy\", 0)\n\n def test_get_file_size__unk_file(self, os_ops: OsOperations):\n \"\"\"\n Test LocalOperations::get_file_size.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n with pytest.raises(FileNotFoundError, match=re.escape(\"[Errno 2] No such file or directory: '/dummy'\")):\n os_ops.get_file_size(\"/dummy\")\n\n def test_cwd(self, os_ops: OsOperations):\n \"\"\"\n Test cwd.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n v = os_ops.cwd()\n\n assert v is not None\n assert type(v) is str\n\n expectedValue = os.getcwd()\n assert expectedValue is not None\n assert type(expectedValue) is str\n assert expectedValue != \"\" # research\n\n # Comp result\n assert v == expectedValue\n" }, { "path": "tests/test_os_ops_remote.py", "content": "# coding: utf-8\n\nfrom .helpers.global_data import OsOpsDescrs\nfrom .helpers.global_data import OsOperations\n\nfrom src import ExecUtilException\n\nimport os\nimport pytest\n\n\nclass TestOsOpsRemote:\n @pytest.fixture\n def os_ops(self):\n return OsOpsDescrs.sm_remote_os_ops\n\n def test_rmdirs__try_to_delete_nonexist_path(self, os_ops: OsOperations):\n assert isinstance(os_ops, OsOperations)\n\n path = \"/root/test_dir\"\n\n assert os_ops.rmdirs(path, ignore_errors=False) is True\n\n def test_rmdirs__try_to_delete_file(self, os_ops: OsOperations):\n assert isinstance(os_ops, OsOperations)\n\n path = os_ops.mkstemp()\n assert type(path) is str\n assert os.path.exists(path)\n\n with pytest.raises(ExecUtilException) as x:\n os_ops.rmdirs(path, ignore_errors=False)\n\n assert os.path.exists(path)\n assert type(x.value) is ExecUtilException\n assert type(x.value.description) is str\n assert x.value.description == \"Utility exited with non-zero code (20). Error: `cannot remove '\" + path + \"': it is not a directory`\"\n assert x.value.message.startswith(x.value.description)\n assert type(x.value.error) is str\n assert x.value.error.strip() == \"cannot remove '\" + path + \"': it is not a directory\"\n assert type(x.value.exit_code) is int\n assert x.value.exit_code == 20\n\n def test_read__unknown_file(self, os_ops: OsOperations):\n \"\"\"\n Test RemoteOperations::read with unknown file.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n with pytest.raises(ExecUtilException) as x:\n os_ops.read(\"/dummy\")\n\n assert \"Utility exited with non-zero code (1).\" in str(x.value)\n assert \"No such file or directory\" in str(x.value)\n assert \"/dummy\" in str(x.value)\n\n def test_read_binary__spec__unk_file(self, os_ops: OsOperations):\n \"\"\"\n Test RemoteOperations::read_binary with unknown file.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n with pytest.raises(ExecUtilException) as x:\n os_ops.read_binary(\"/dummy\", 0)\n\n assert \"Utility exited with non-zero code (1).\" in str(x.value)\n assert \"No such file or directory\" in str(x.value)\n assert \"/dummy\" in str(x.value)\n\n def test_get_file_size__unk_file(self, os_ops: OsOperations):\n \"\"\"\n Test RemoteOperations::get_file_size.\n \"\"\"\n assert isinstance(os_ops, OsOperations)\n\n with pytest.raises(ExecUtilException) as x:\n os_ops.get_file_size(\"/dummy\")\n\n assert \"Utility exited with non-zero code (1).\" in str(x.value)\n assert \"No such file or directory\" in str(x.value)\n assert \"/dummy\" in str(x.value)\n" }, { "path": "tests/test_raise_error.py", "content": "from src import InvalidOperationException\nfrom src import NodeStatus\nfrom src.raise_error import RaiseError\n\nimport pytest\nimport typing\n\n\nclass TestRaiseError:\n class tagTestData001:\n node_status: NodeStatus\n expected_msg: str\n\n def __init__(\n self,\n node_status: NodeStatus,\n expected_msg: str,\n ):\n assert type(node_status) is NodeStatus\n assert type(expected_msg) is str\n self.node_status = node_status\n self.expected_msg = expected_msg\n return\n\n @property\n def sign(self) -> str:\n assert type(self.node_status) is NodeStatus\n\n msg = \"status: {}\".format(self.node_status)\n return msg\n\n sm_Data001: typing.List[tagTestData001] = [\n tagTestData001(\n NodeStatus.Uninitialized,\n \"Can't enumerate node child processes. Node is not initialized.\",\n ),\n tagTestData001(\n NodeStatus.Stopped,\n \"Can't enumerate node child processes. Node is not running.\",\n ),\n ]\n\n @pytest.fixture(\n params=sm_Data001,\n ids=[x.sign for x in sm_Data001],\n )\n def data001(self, request: pytest.FixtureRequest) -> tagTestData001:\n assert isinstance(request, pytest.FixtureRequest)\n assert type(request.param).__name__ == \"tagTestData001\"\n return request.param\n\n def test_001__node_err__cant_enumerate_child_processes(\n self,\n data001: tagTestData001,\n ):\n assert type(data001) is __class__.tagTestData001\n\n with pytest.raises(expected_exception=InvalidOperationException) as x:\n RaiseError.node_err__cant_enumerate_child_processes(\n data001.node_status\n )\n\n assert x is not None\n assert str(x.value) == data001.expected_msg\n return\n\n sm_Data002: typing.List[tagTestData001] = [\n tagTestData001(\n NodeStatus.Uninitialized,\n \"Can't kill server process. Node is not initialized.\",\n ),\n tagTestData001(\n NodeStatus.Stopped,\n \"Can't kill server process. Node is not running.\",\n ),\n ]\n\n @pytest.fixture(\n params=sm_Data002,\n ids=[x.sign for x in sm_Data002],\n )\n def data002(self, request: pytest.FixtureRequest) -> tagTestData001:\n assert isinstance(request, pytest.FixtureRequest)\n assert type(request.param).__name__ == \"tagTestData001\"\n return request.param\n\n def test_002__node_err__cant_kill(\n self,\n data002: tagTestData001,\n ):\n assert type(data002) is __class__.tagTestData001\n\n with pytest.raises(expected_exception=InvalidOperationException) as x:\n RaiseError.node_err__cant_kill(\n data002.node_status\n )\n\n assert x is not None\n assert str(x.value) == data002.expected_msg\n return\n" }, { "path": "tests/test_testgres_common.py", "content": "from __future__ import annotations\n\nfrom .helpers.global_data import PostgresNodeService\nfrom .helpers.global_data import PostgresNodeServices\nfrom .helpers.global_data import OsOperations\nfrom .helpers.global_data import PortManager\n\nfrom src import __version__ as testgres_version\nfrom src.node import PgVer\nfrom src.node import PostgresNode\nfrom src.node import NodeConnection\nfrom src.node import PostgresNodeLogReader\nfrom src.node import PostgresNodeUtils\nfrom src.node import ProcessProxy\nfrom src.utils import get_pg_version2\nfrom src.utils import file_tail\nfrom src.utils import get_bin_path2\nfrom src.utils import execute_utility2\nfrom src import ProcessType\nfrom src import NodeStatus\nfrom src import IsolationLevel\nfrom src import NodeApp\nfrom src import enums\n\n# New name prevents to collect test-functions in TestgresException and fixes\n# the problem with pytest warning.\nfrom src import TestgresException as testgres_TestgresException\n\nfrom src import InitNodeException\nfrom src import StartNodeException\nfrom src import QueryException\nfrom src import ExecUtilException\nfrom src import QueryTimeoutException\nfrom src import InvalidOperationException\nfrom src import BackupException\nfrom src import ProgrammingError\nfrom src import scoped_config\nfrom src import First, Any\n\nfrom contextlib import contextmanager\n\nimport pytest\nimport six\nimport logging\nimport time\nimport tempfile\nimport uuid\nimport os\nimport re\nimport subprocess\nimport typing\nimport types\nimport psutil\n\nfrom packaging.version import Version\n\n\n@contextmanager\ndef removing(os_ops: OsOperations, f):\n assert isinstance(os_ops, OsOperations)\n\n try:\n yield f\n finally:\n if os_ops.isfile(f):\n os_ops.remove_file(f)\n\n elif os_ops.isdir(f):\n os_ops.rmdirs(f, ignore_errors=True)\n\n\nclass TestTestgresCommon:\n sm_node_svcs: typing.List[PostgresNodeService] = [\n PostgresNodeServices.sm_local,\n PostgresNodeServices.sm_local2,\n PostgresNodeServices.sm_remote,\n ]\n\n @pytest.fixture(\n params=sm_node_svcs,\n ids=[descr.sign for descr in sm_node_svcs]\n )\n def node_svc(self, request: pytest.FixtureRequest) -> PostgresNodeService:\n assert isinstance(request, pytest.FixtureRequest)\n assert isinstance(request.param, PostgresNodeService)\n assert isinstance(request.param.os_ops, OsOperations)\n assert isinstance(request.param.port_manager, PortManager)\n return request.param\n\n def test_testgres_version(self):\n assert type(testgres_version) is str\n\n v = Version(testgres_version)\n\n # Author: Mark G.\n assert v.major == 1\n assert v.minor == 13\n assert v.micro == 7\n\n assert str(v) == testgres_version\n return\n\n def test_version_management(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n a = PgVer('10.0')\n b = PgVer('10')\n c = PgVer('9.6.5')\n d = PgVer('15.0')\n e = PgVer('15rc1')\n f = PgVer('15beta4')\n h = PgVer('15.3biha')\n i = PgVer('15.3')\n g = PgVer('15.3.1bihabeta1')\n k = PgVer('15.3.1')\n\n assert (a == b)\n assert (b > c)\n assert (a > c)\n assert (d > e)\n assert (e > f)\n assert (d > f)\n assert (h > f)\n assert (h == i)\n assert (g == k)\n assert (g > h)\n\n version = get_pg_version2(node_svc.os_ops)\n\n with __class__.helper__get_node(node_svc) as node:\n assert (isinstance(version, six.string_types))\n assert (isinstance(node.version, PgVer))\n assert (node.version == PgVer(version))\n\n def test_node_repr(self, node_svc: PostgresNodeService):\n with __class__.helper__get_node(node_svc).init() as node:\n pattern = r\"PostgresNode\\(name='.+', port=.+, base_dir='.+'\\)\"\n assert re.match(pattern, str(node)) is not None\n\n def test_custom_init(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n with __class__.helper__get_node(node_svc) as node:\n # enable page checksums\n node.init(initdb_params=['-k']).start()\n\n with __class__.helper__get_node(node_svc) as node:\n node.init(\n allow_streaming=True,\n initdb_params=['--auth-local=reject', '--auth-host=reject'])\n\n hba_file = os.path.join(node.data_dir, 'pg_hba.conf')\n lines = node.os_ops.readlines(hba_file)\n\n # check number of lines\n assert (len(lines) >= 6)\n\n # there should be no trust entries at all\n assert not (any('trust' in s for s in lines))\n\n def test_double_init(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n with __class__.helper__get_node(node_svc).init() as node:\n # can't initialize node more than once\n with pytest.raises(expected_exception=InitNodeException):\n node.init()\n\n def test_init_after_cleanup(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n with __class__.helper__get_node(node_svc) as node:\n node.init().start().execute('select 1')\n node.cleanup()\n node.init().start().execute('select 1')\n\n def test_init_unique_system_id(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n # this function exists in PostgreSQL 9.6+\n current_version = get_pg_version2(node_svc.os_ops)\n\n __class__.helper__skip_test_if_util_not_exist(node_svc.os_ops, \"pg_resetwal\")\n __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, '9.6')\n\n query = 'select system_identifier from pg_control_system()'\n\n with scoped_config(cache_initdb=False):\n with __class__.helper__get_node(node_svc).init().start() as node0:\n id0 = node0.execute(query)[0]\n\n with scoped_config(cache_initdb=True,\n cached_initdb_unique=True) as config:\n assert (config.cache_initdb)\n assert (config.cached_initdb_unique)\n\n # spawn two nodes; ids must be different\n with __class__.helper__get_node(node_svc).init().start() as node1, \\\n __class__.helper__get_node(node_svc).init().start() as node2:\n id1 = node1.execute(query)[0]\n id2 = node2.execute(query)[0]\n\n # ids must increase\n assert (id1 > id0)\n assert (id2 > id1)\n\n def test_node_exit(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n with pytest.raises(expected_exception=QueryException):\n with __class__.helper__get_node(node_svc).init() as node:\n base_dir = node.base_dir\n node.safe_psql('select 1')\n\n # we should save the DB for \"debugging\"\n assert (node_svc.os_ops.path_exists(base_dir))\n node_svc.os_ops.rmdirs(base_dir, ignore_errors=True)\n\n with __class__.helper__get_node(node_svc).init() as node:\n base_dir = node.base_dir\n\n # should have been removed by default\n assert not (node_svc.os_ops.path_exists(base_dir))\n\n def test_double_start(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n with __class__.helper__get_node(node_svc) as node:\n node.init()\n assert not node.is_started\n assert node.status() == NodeStatus.Stopped\n node.start()\n assert node.is_started\n assert node.status() == NodeStatus.Running\n\n with pytest.raises(expected_exception=StartNodeException) as x:\n # can't start node more than once\n node.start()\n\n assert x is not None\n assert type(x.value) is StartNodeException\n assert type(x.value.description) is str\n assert type(x.value.message) is str\n\n assert x.value.description == \"Cannot start node\"\n assert x.value.message.startswith(x.value.description)\n\n assert node.is_started\n assert node.status() == NodeStatus.Running\n\n return\n\n def test_start__manually_stop__start_again(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n with __class__.helper__get_node(node_svc) as node:\n node.init()\n assert not node.is_started\n\n logging.info(\"Start node\")\n node.start()\n assert node.is_started\n assert node.status() == NodeStatus.Running\n\n logging.info(\"Stop node manually via pg_ctl\")\n stop_cmd = [\n node.os_ops.build_path(node.bin_dir, \"pg_ctl\"),\n \"stop\",\n \"-D\",\n node.data_dir,\n ]\n\n execute_utility2(\n node.os_ops,\n stop_cmd,\n node.utils_log_file\n )\n\n assert node.is_started\n assert node.status() == NodeStatus.Stopped\n\n logging.info(\"Start node again\")\n node.start()\n assert node.is_started\n assert node.status() == NodeStatus.Running\n\n assert not node.is_started\n assert node.status() == NodeStatus.Uninitialized\n return\n\n def test_uninitialized_start(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n with __class__.helper__get_node(node_svc) as node:\n # node is not initialized yet\n assert node.status() == NodeStatus.Uninitialized\n\n with pytest.raises(expected_exception=StartNodeException):\n node.start()\n\n assert node.status() == NodeStatus.Uninitialized\n return\n\n def test_start2(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n with __class__.helper__get_node(node_svc) as node:\n node.init()\n assert not node.is_started\n assert node.status() == NodeStatus.Stopped\n node.start2()\n assert not node.is_started\n assert node.status() == NodeStatus.Running\n\n with pytest.raises(expected_exception=StartNodeException) as x:\n # can't start node more than once\n node.start2()\n\n assert x is not None\n assert type(x.value) is StartNodeException\n assert type(x.value.description) is str\n assert type(x.value.message) is str\n\n assert x.value.description == \"Cannot start node\"\n assert x.value.message.startswith(x.value.description)\n\n assert not node.is_started\n assert node.status() == NodeStatus.Running\n\n return\n\n def test_restart(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n with __class__.helper__get_node(node_svc) as node:\n node.init().start()\n\n # restart, ok\n res = node.execute('select 1')\n assert (res == [(1,)])\n node.restart()\n res = node.execute('select 2')\n assert (res == [(2,)])\n\n assert node.status() == NodeStatus.Running\n\n # restart, fail\n with pytest.raises(expected_exception=StartNodeException):\n node.append_conf('pg_hba.conf', 'DUMMY')\n node.restart()\n\n assert node.status() == NodeStatus.Stopped\n return\n\n def test_double_stop(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n with __class__.helper__get_node(node_svc) as node:\n node.init()\n assert not node.is_started\n node.start()\n assert node.is_started\n node.stop()\n assert not node.is_started\n\n with pytest.raises(expected_exception=Exception) as x:\n # can't start node more than once\n node.stop()\n\n assert x is not None\n assert \"Is server running?\" in str(x.value)\n\n assert not node.is_started\n\n return\n\n def test_reload(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n with __class__.helper__get_node(node_svc) as node:\n node.init().start()\n\n # change client_min_messages and save old value\n cmm_old = node.execute('show client_min_messages')\n node.append_conf(client_min_messages='DEBUG1')\n\n # reload config\n node.reload()\n\n # check new value\n cmm_new = node.execute('show client_min_messages')\n assert ('debug1' == cmm_new[0][0].lower())\n assert (cmm_old != cmm_new)\n\n def test_pg_ctl(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n with __class__.helper__get_node(node_svc) as node:\n node.init().start()\n\n status = node.pg_ctl(['status'])\n assert ('PID' in status)\n\n def test_status(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n assert (NodeStatus.Running)\n assert not (NodeStatus.Stopped)\n assert not (NodeStatus.Uninitialized)\n\n # check statuses after each operation\n with __class__.helper__get_node(node_svc) as node:\n assert (node.pid == 0)\n assert (node.status() == NodeStatus.Uninitialized)\n\n node.init()\n\n assert (node.pid == 0)\n assert (node.status() == NodeStatus.Stopped)\n\n node.start()\n\n assert (node.pid != 0)\n assert (node.status() == NodeStatus.Running)\n\n node.stop()\n\n assert (node.pid == 0)\n assert (node.status() == NodeStatus.Stopped)\n\n node.cleanup()\n\n assert (node.pid == 0)\n assert (node.status() == NodeStatus.Uninitialized)\n\n def test_status__empty_postmaster_pid(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n assert (NodeStatus.Running)\n assert not (NodeStatus.Stopped)\n assert not (NodeStatus.Uninitialized)\n\n # check statuses after each operation\n with __class__.helper__get_node(node_svc) as node:\n assert (node.pid == 0)\n assert (node.status() == NodeStatus.Uninitialized)\n\n node.init()\n\n postmaster_pid_file = node.os_ops.build_path(node.data_dir, \"postmaster.pid\")\n\n node.os_ops.write(\n postmaster_pid_file,\n \"\"\n )\n\n with pytest.raises(expected_exception=ExecUtilException) as x:\n node.status()\n\n expected_msg = \"pg_ctl: the PID file \\\"{}\\\" is empty\\n\".format(\n postmaster_pid_file\n )\n\n assert expected_msg == x.value.error\n return\n\n def test_status__force_clean_postmaster_pid(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n assert (NodeStatus.Running)\n assert not (NodeStatus.Stopped)\n assert not (NodeStatus.Uninitialized)\n\n # check statuses after each operation\n with __class__.helper__get_node(node_svc) as node:\n assert (node.pid == 0)\n assert (node.status() == NodeStatus.Uninitialized)\n\n node.init()\n node.start()\n\n assert node.status() == NodeStatus.Running\n logging.info(\"Postmaster PID is {}.\".format(node.pid))\n\n postmaster_pid_file = node.os_ops.build_path(node.data_dir, \"postmaster.pid\")\n\n logging.info(\"Clean postmaster pid file [{}].\".format(\n postmaster_pid_file\n ))\n\n node.os_ops.write(\n postmaster_pid_file,\n \"\",\n truncate=True,\n )\n\n x = node.os_ops.read(\n postmaster_pid_file,\n encoding=\"utf-8\",\n binary=False\n )\n assert x == \"\"\n\n with pytest.raises(expected_exception=ExecUtilException) as x:\n node.status()\n\n expected_msg = \"pg_ctl: the PID file \\\"{}\\\" is empty\\n\".format(\n postmaster_pid_file\n )\n\n assert expected_msg == x.value.error\n return\n\n def test_kill__is_not_initialized(\n self,\n node_svc: PostgresNodeService\n ):\n assert isinstance(node_svc, PostgresNodeService)\n\n with __class__.helper__get_node(node_svc) as node:\n assert isinstance(node, PostgresNode)\n assert (node.pid == 0)\n assert (node.status() == NodeStatus.Uninitialized)\n\n with pytest.raises(expected_exception=InvalidOperationException) as x:\n node.kill()\n\n assert x is not None\n assert str(x.value) == \"Can't kill server process. Node is not initialized.\"\n return\n\n def test_kill__is_not_running(\n self,\n node_svc: PostgresNodeService\n ):\n assert isinstance(node_svc, PostgresNodeService)\n\n with __class__.helper__get_node(node_svc) as node:\n assert isinstance(node, PostgresNode)\n assert (node.pid == 0)\n assert (node.status() == NodeStatus.Uninitialized)\n\n node.init()\n\n try:\n with pytest.raises(expected_exception=InvalidOperationException) as x:\n node.kill()\n\n assert x is not None\n assert str(x.value) == \"Can't kill server process. Node is not running.\"\n finally:\n try:\n node.cleanup(release_resources=True)\n except Exception as e:\n logging.error(\"Exception ({}): {}\".format(\n type(e).__name__,\n e,\n ))\n return\n\n def test_kill__ok(\n self,\n node_svc: PostgresNodeService\n ):\n assert isinstance(node_svc, PostgresNodeService)\n\n with __class__.helper__get_node(node_svc) as node:\n assert isinstance(node, PostgresNode)\n assert (node.pid == 0)\n assert (node.status() == NodeStatus.Uninitialized)\n\n node.init()\n assert not node.is_started\n node.slow_start()\n assert node.is_started\n node.kill()\n assert not node.is_started\n\n attempt = 0\n\n while True:\n if attempt == 60:\n raise RuntimeError(\"Node is not stopped.\")\n\n attempt += 1\n\n if attempt > 1:\n time.sleep(1)\n\n s = node.status()\n\n logging.info(\"Node status is {}\".format(s.name))\n\n if s == NodeStatus.Running:\n continue\n\n assert s == NodeStatus.Stopped\n break\n return\n\n def test_kill_backgroud_writer__ok(\n self,\n node_svc: PostgresNodeService\n ):\n assert isinstance(node_svc, PostgresNodeService)\n\n with __class__.helper__get_node(node_svc) as node:\n assert isinstance(node, PostgresNode)\n assert (node.pid == 0)\n assert (node.status() == NodeStatus.Uninitialized)\n\n node.init()\n assert not node.is_started\n node.slow_start()\n assert node.is_started\n node_pid = node.pid\n assert type(node_pid) is int\n aux_pids = node.auxiliary_pids\n assert type(aux_pids) is dict\n assert ProcessType.BackgroundWriter in aux_pids\n bw_pids = aux_pids[ProcessType.BackgroundWriter]\n assert type(bw_pids) is list\n assert len(bw_pids) == 1\n bw_pid = bw_pids[0]\n assert type(bw_pid) is int\n node.kill(ProcessType.BackgroundWriter)\n assert node.is_started\n\n attempt = 0\n\n while True:\n if attempt == 60:\n raise RuntimeError(\"Node is not stopped.\")\n\n attempt += 1\n\n if attempt > 1:\n time.sleep(1)\n\n try:\n psutil.Process(bw_pid)\n except psutil.NoSuchProcess:\n logging.info(\"Process is not found\")\n break\n\n logging.info(\"Process is still alive.\")\n continue\n\n assert node.is_started\n assert node.pid == node_pid\n return\n\n def test_child_processes__is_not_initialized(\n self,\n node_svc: PostgresNodeService\n ):\n assert isinstance(node_svc, PostgresNodeService)\n\n with __class__.helper__get_node(node_svc) as node:\n assert isinstance(node, PostgresNode)\n assert (node.pid == 0)\n assert (node.status() == NodeStatus.Uninitialized)\n\n with pytest.raises(expected_exception=InvalidOperationException) as x:\n node.child_processes\n\n assert x is not None\n assert str(x.value) == \"Can't enumerate node child processes. Node is not initialized.\"\n return\n\n def test_child_processes__is_not_running(\n self,\n node_svc: PostgresNodeService\n ):\n assert isinstance(node_svc, PostgresNodeService)\n\n with __class__.helper__get_node(node_svc) as node:\n assert isinstance(node, PostgresNode)\n assert (node.pid == 0)\n assert (node.status() == NodeStatus.Uninitialized)\n\n node.init()\n\n try:\n with pytest.raises(expected_exception=InvalidOperationException) as x:\n node.child_processes\n\n assert x is not None\n assert str(x.value) == \"Can't enumerate node child processes. Node is not running.\"\n finally:\n try:\n node.cleanup(release_resources=True)\n except Exception as e:\n logging.error(\"Exception ({}): {}\".format(\n type(e).__name__,\n e,\n ))\n return\n\n def test_child_processes__ok(\n self,\n node_svc: PostgresNodeService\n ):\n assert isinstance(node_svc, PostgresNodeService)\n\n with __class__.helper__get_node(node_svc) as node:\n assert isinstance(node, PostgresNode)\n assert (node.pid == 0)\n assert (node.status() == NodeStatus.Uninitialized)\n\n node.init()\n\n try:\n node.slow_start()\n\n children = node.child_processes\n assert children is not None\n assert type(children) is list\n\n logging.info(\"Children count is {}\".format(len(children)))\n logging.info(\"\")\n\n def LOCAL__safe_call_cmdline(p: ProcessProxy) -> str:\n assert type(p) is ProcessProxy\n try:\n return p.cmdline()\n except Exception as e:\n return \"Exception ({}): {}\".format(\n type(e).__name__,\n e,\n )\n\n for i in range(len(children)):\n logging.info(\"------ check child [{}]\".format(i))\n child = children[i]\n\n try:\n assert child is not None\n assert type(child) is ProcessProxy\n assert hasattr(child, \"process\")\n assert hasattr(child, \"ptype\")\n assert hasattr(child, \"pid\")\n assert hasattr(child, \"cmdline\")\n assert child.process is not None\n assert child.ptype is not None\n assert child.pid is not None\n assert type(child.ptype) is ProcessType\n assert type(child.pid) is int\n assert type(child.cmdline) is types.MethodType\n\n logging.info(\"ptype is {}\".format(child.ptype))\n logging.info(\"pid is {}\".format(child.pid))\n logging.info(\"cmdline is [{}]\".format(LOCAL__safe_call_cmdline(child)))\n except Exception as e:\n logging.error(\"Exception ({}): {}\".format(\n type(e).__name__,\n e,\n ))\n continue\n finally:\n try:\n node.cleanup(release_resources=True)\n except Exception as e:\n logging.error(\"Exception ({}): {}\".format(\n type(e).__name__,\n e,\n ))\n return\n\n def test_child_pids(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n master_processes = [\n ProcessType.AutovacuumLauncher,\n ProcessType.BackgroundWriter,\n ProcessType.Checkpointer,\n ProcessType.StatsCollector,\n ProcessType.WalSender,\n ProcessType.WalWriter,\n ]\n\n postgresVersion = get_pg_version2(node_svc.os_ops)\n\n if __class__.helper__pg_version_ge(postgresVersion, '10'):\n master_processes.append(ProcessType.LogicalReplicationLauncher)\n\n if __class__.helper__pg_version_ge(postgresVersion, '14'):\n master_processes.remove(ProcessType.StatsCollector)\n\n repl_processes = [\n ProcessType.Startup,\n ProcessType.WalReceiver,\n ]\n\n def LOCAL__test_auxiliary_pids(\n node: PostgresNode,\n expectedTypes: typing.List[ProcessType]\n ) -> typing.List[ProcessType]:\n # returns list of the absence processes\n assert node is not None\n assert type(node) is PostgresNode\n assert expectedTypes is not None\n assert type(expectedTypes) is list\n\n pids = node.auxiliary_pids\n assert pids is not None\n assert type(pids) is dict\n\n result: typing.List[ProcessType] = list()\n for ptype in expectedTypes:\n if ptype not in pids:\n result.append(ptype)\n return result\n\n def LOCAL__check_auxiliary_pids__multiple_attempts(\n node: PostgresNode,\n expectedTypes: typing.List[ProcessType],\n ):\n assert node is not None\n assert type(node) is PostgresNode\n assert expectedTypes is not None\n assert type(expectedTypes) is list\n\n nAttempt = 0\n\n while True:\n nAttempt += 1\n\n logging.info(\"Test pids of [{0}] node. Attempt #{1}.\".format(\n node.name,\n nAttempt\n ))\n\n if nAttempt > 1:\n time.sleep(1)\n\n absenceList = LOCAL__test_auxiliary_pids(node, expectedTypes)\n assert absenceList is not None\n assert type(absenceList) is list\n if len(absenceList) == 0:\n logging.info(\"Bingo!\")\n break\n\n if nAttempt == 5:\n raise Exception(\"Node {0} does not have the following processes: {1}.\".format(\n node.name,\n absenceList,\n ))\n\n logging.info(\"These processes are not found: {0}.\".format(absenceList))\n continue\n return\n\n with __class__.helper__get_node(node_svc).init().start() as master:\n\n # master node doesn't have a source walsender!\n with pytest.raises(expected_exception=testgres_TestgresException):\n master.source_walsender\n\n with master.connect() as con:\n assert (con.pid > 0)\n\n with master.replicate().start() as replica:\n assert type(replica) is PostgresNode\n\n # test __str__ method\n str(master.child_processes[0])\n\n LOCAL__check_auxiliary_pids__multiple_attempts(\n master,\n master_processes)\n\n LOCAL__check_auxiliary_pids__multiple_attempts(\n replica,\n repl_processes)\n\n master_pids = master.auxiliary_pids\n\n # there should be exactly 1 source walsender for replica\n assert (len(master_pids[ProcessType.WalSender]) == 1)\n pid1 = master_pids[ProcessType.WalSender][0]\n pid2 = replica.source_walsender.pid\n assert (pid1 == pid2)\n\n replica.stop()\n\n # there should be no walsender after we've stopped replica\n with pytest.raises(expected_exception=testgres_TestgresException):\n replica.source_walsender\n\n def test_exceptions(self):\n str(StartNodeException('msg', [('file', 'lines')]))\n str(ExecUtilException('msg', 'cmd', 1, 'out'))\n str(QueryException('msg', 'query'))\n\n def test_auto_name(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n with __class__.helper__get_node(node_svc).init(allow_streaming=True).start() as m:\n with m.replicate().start() as r:\n # check that nodes are running\n assert (m.status())\n assert (r.status())\n\n # check their names\n assert (m.name != r.name)\n assert ('testgres' in m.name)\n assert ('testgres' in r.name)\n\n def test_file_tail(self):\n s1 = \"the quick brown fox jumped over that lazy dog\\n\"\n s2 = \"abc\\n\"\n s3 = \"def\\n\"\n\n with tempfile.NamedTemporaryFile(mode='r+', delete=True) as f:\n sz = 0\n while sz < 3 * 8192:\n sz += len(s1)\n f.write(s1)\n f.write(s2)\n f.write(s3)\n\n f.seek(0)\n lines = file_tail(f, 3)\n assert (lines[0] == s1)\n assert (lines[1] == s2)\n assert (lines[2] == s3)\n\n f.seek(0)\n lines = file_tail(f, 1)\n assert (lines[0] == s3)\n\n def test_isolation_levels(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n with __class__.helper__get_node(node_svc).init().start() as node:\n with node.connect() as con:\n # string levels\n con.begin('Read Uncommitted').commit()\n con.begin('Read Committed').commit()\n con.begin('Repeatable Read').commit()\n con.begin('Serializable').commit()\n\n # enum levels\n con.begin(IsolationLevel.ReadUncommitted).commit()\n con.begin(IsolationLevel.ReadCommitted).commit()\n con.begin(IsolationLevel.RepeatableRead).commit()\n con.begin(IsolationLevel.Serializable).commit()\n\n # check wrong level\n with pytest.raises(expected_exception=QueryException):\n con.begin('Garbage').commit()\n\n def test_users(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n with __class__.helper__get_node(node_svc).init().start() as node:\n node.psql('create role test_user login')\n value = node.safe_psql('select 1', username='test_user')\n value = __class__.helper__rm_carriage_returns(value)\n assert (value == b'1\\n')\n\n def test_poll_query_until(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n with __class__.helper__get_node(node_svc) as node:\n node.init().start()\n\n get_time = 'select extract(epoch from now())'\n check_time = 'select extract(epoch from now()) - {} >= 5'\n\n start_time = node.execute(get_time)[0][0]\n node.poll_query_until(query=check_time.format(start_time))\n end_time = node.execute(get_time)[0][0]\n\n assert (end_time - start_time >= 5)\n\n # check 0 columns\n with pytest.raises(expected_exception=QueryException):\n node.poll_query_until(\n query='select from pg_catalog.pg_class limit 1')\n\n # check None, fail\n with pytest.raises(expected_exception=QueryException):\n node.poll_query_until(query='create table abc (val int)')\n\n # check None, ok\n node.poll_query_until(query='create table def()',\n expected=None) # returns nothing\n\n # check 0 rows equivalent to expected=None\n node.poll_query_until(\n query='select * from pg_catalog.pg_class where true = false',\n expected=None)\n\n # check arbitrary expected value, fail\n with pytest.raises(expected_exception=QueryTimeoutException):\n node.poll_query_until(query='select 3',\n expected=1,\n max_attempts=3,\n sleep_time=0.01)\n\n # check arbitrary expected value, ok\n node.poll_query_until(query='select 2', expected=2)\n\n # check timeout\n with pytest.raises(expected_exception=QueryTimeoutException):\n node.poll_query_until(query='select 1 > 2',\n max_attempts=3,\n sleep_time=0.01)\n\n # check ProgrammingError, fail\n with pytest.raises(expected_exception=ProgrammingError):\n node.poll_query_until(query='dummy1')\n\n # check ProgrammingError, ok\n with pytest.raises(expected_exception=(QueryTimeoutException)):\n node.poll_query_until(query='dummy2',\n max_attempts=3,\n sleep_time=0.01,\n suppress={ProgrammingError})\n\n # check 1 arg, ok\n node.poll_query_until('select true')\n\n def test_logging(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n C_MAX_ATTEMPTS = 50\n # This name is used for testgres logging, too.\n C_NODE_NAME = \"testgres_tests.\" + __class__.__name__ + \"test_logging-master-\" + uuid.uuid4().hex\n\n logging.info(\"Node name is [{0}]\".format(C_NODE_NAME))\n\n with tempfile.NamedTemporaryFile('w', delete=True) as logfile:\n formatter = logging.Formatter(fmt=\"%(node)-5s: %(message)s\")\n handler = logging.FileHandler(filename=logfile.name)\n handler.formatter = formatter\n logger = logging.getLogger(C_NODE_NAME)\n assert logger is not None\n assert len(logger.handlers) == 0\n\n try:\n # It disables to log on the root level\n logger.propagate = False\n logger.addHandler(handler)\n\n with scoped_config(use_python_logging=True):\n with __class__.helper__get_node(node_svc, name=C_NODE_NAME) as master:\n logging.info(\"Master node is initilizing\")\n master.init()\n\n logging.info(\"Master node is starting\")\n master.start()\n\n logging.info(\"Dummy query is executed a few times\")\n for _ in range(20):\n master.execute('select 1')\n time.sleep(0.01)\n\n # let logging worker do the job\n time.sleep(0.1)\n\n logging.info(\"Master node log file is checking\")\n nAttempt = 0\n\n while True:\n assert nAttempt <= C_MAX_ATTEMPTS\n if nAttempt == C_MAX_ATTEMPTS:\n raise Exception(\"Test failed!\")\n\n # let logging worker do the job\n time.sleep(0.1)\n\n nAttempt += 1\n\n logging.info(\"Attempt {0}\".format(nAttempt))\n\n # check that master's port is found\n with open(logfile.name, 'r') as log:\n lines = log.readlines()\n\n assert lines is not None\n assert type(lines) is list\n\n def LOCAL__test_lines(lines: typing.Iterable[str]) -> bool:\n assert isinstance(lines, typing.Iterable)\n for s in lines:\n assert type(s) is str\n if C_NODE_NAME in s:\n logging.info(\"OK. We found the node_name in a line \\\"{0}\\\"\".format(s))\n return True\n return False\n\n if LOCAL__test_lines(lines):\n break\n\n logging.info(\"Master node log file does not have an expected information.\")\n continue\n\n # test logger after stop/start/restart\n logging.info(\"Master node is stopping...\")\n master.stop()\n logging.info(\"Master node is staring again...\")\n master.start()\n logging.info(\"Master node is restaring...\")\n master.restart()\n assert (master._logger.is_alive())\n finally:\n # It is a hack code to logging cleanup\n with logging._lock:\n assert logging.Logger.manager is not None\n assert C_NODE_NAME in logging.Logger.manager.loggerDict.keys()\n logging.Logger.manager.loggerDict.pop(C_NODE_NAME, None)\n assert C_NODE_NAME not in logging.Logger.manager.loggerDict.keys()\n assert handler not in logging._handlers.values()\n # GO HOME!\n return\n\n def test_psql(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n with __class__.helper__get_node(node_svc).init().start() as node:\n\n # check returned values (1 arg)\n res = node.psql('select 1')\n assert (__class__.helper__rm_carriage_returns(res) == (0, b'1\\n', b''))\n\n # check returned values (2 args)\n res = node.psql('postgres', 'select 2')\n assert (__class__.helper__rm_carriage_returns(res) == (0, b'2\\n', b''))\n\n # check returned values (named)\n res = node.psql(query='select 3', dbname='postgres')\n assert (__class__.helper__rm_carriage_returns(res) == (0, b'3\\n', b''))\n\n # check returned values (1 arg)\n res = node.safe_psql('select 4')\n assert (__class__.helper__rm_carriage_returns(res) == b'4\\n')\n\n # check returned values (2 args)\n res = node.safe_psql('postgres', 'select 5')\n assert (__class__.helper__rm_carriage_returns(res) == b'5\\n')\n\n # check returned values (named)\n res = node.safe_psql(query='select 6', dbname='postgres')\n assert (__class__.helper__rm_carriage_returns(res) == b'6\\n')\n\n # check feeding input\n node.safe_psql('create table horns (w int)')\n node.safe_psql('copy horns from stdin (format csv)',\n input=b\"1\\n2\\n3\\n\\\\.\\n\")\n _sum = node.safe_psql('select sum(w) from horns')\n assert (__class__.helper__rm_carriage_returns(_sum) == b'6\\n')\n\n # check psql's default args, fails\n with pytest.raises(expected_exception=QueryException):\n r = node.psql() # raises!\n logging.error(\"node.psql returns [{}]\".format(r))\n\n node.stop()\n\n # check psql on stopped node, fails\n with pytest.raises(expected_exception=QueryException):\n # [2025-04-03] This call does not raise exception! I do not know why.\n r = node.safe_psql('select 1') # raises!\n logging.error(\"node.safe_psql returns [{}]\".format(r))\n\n def test_psql__another_port(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n with __class__.helper__get_node(node_svc).init() as node1:\n with __class__.helper__get_node(node_svc).init() as node2:\n node1.start()\n node2.start()\n assert node1.port != node2.port\n assert node1.host == node2.host\n\n node1.stop()\n\n logging.info(\"test table in node2 is creating ...\")\n node2.safe_psql(\n dbname=\"postgres\",\n query=\"create table test (id integer);\"\n )\n\n logging.info(\"try to find test table through node1.psql ...\")\n res = node1.psql(\n dbname=\"postgres\",\n query=\"select count(*) from pg_class where relname='test'\",\n host=node2.host,\n port=node2.port,\n )\n assert (__class__.helper__rm_carriage_returns(res) == (0, b'1\\n', b''))\n\n def test_psql__another_bad_host(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n with __class__.helper__get_node(node_svc).init() as node:\n logging.info(\"try to execute node1.psql ...\")\n res = node.psql(\n dbname=\"postgres\",\n query=\"select count(*) from pg_class where relname='test'\",\n host=\"DUMMY_HOST_NAME\",\n port=node.port,\n )\n\n res2 = __class__.helper__rm_carriage_returns(res)\n\n assert res2[0] != 0\n assert b\"DUMMY_HOST_NAME\" in res[2]\n\n def test_safe_psql__another_port(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n with __class__.helper__get_node(node_svc).init() as node1:\n with __class__.helper__get_node(node_svc).init() as node2:\n node1.start()\n node2.start()\n assert node1.port != node2.port\n assert node1.host == node2.host\n\n node1.stop()\n\n logging.info(\"test table in node2 is creating ...\")\n node2.safe_psql(\n dbname=\"postgres\",\n query=\"create table test (id integer);\"\n )\n\n logging.info(\"try to find test table through node1.psql ...\")\n res = node1.safe_psql(\n dbname=\"postgres\",\n query=\"select count(*) from pg_class where relname='test'\",\n host=node2.host,\n port=node2.port,\n )\n assert (__class__.helper__rm_carriage_returns(res) == b'1\\n')\n\n def test_safe_psql__another_bad_host(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n with __class__.helper__get_node(node_svc).init() as node:\n logging.info(\"try to execute node1.psql ...\")\n\n with pytest.raises(expected_exception=Exception) as x:\n node.safe_psql(\n dbname=\"postgres\",\n query=\"select count(*) from pg_class where relname='test'\",\n host=\"DUMMY_HOST_NAME\",\n port=node.port,\n )\n\n assert \"DUMMY_HOST_NAME\" in str(x.value)\n\n def test_safe_psql__expect_error(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n with __class__.helper__get_node(node_svc).init().start() as node:\n err = node.safe_psql('select_or_not_select 1', expect_error=True)\n assert (type(err) is str)\n assert ('select_or_not_select' in err)\n assert ('ERROR: syntax error at or near \"select_or_not_select\"' in err)\n\n # ---------\n with pytest.raises(\n expected_exception=InvalidOperationException,\n match=\"^\" + re.escape(\"Exception was expected, but query finished successfully: `select 1;`.\") + \"$\"\n ):\n node.safe_psql(\"select 1;\", expect_error=True)\n\n # ---------\n res = node.safe_psql(\"select 1;\", expect_error=False)\n assert (__class__.helper__rm_carriage_returns(res) == b'1\\n')\n\n def test_transactions(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n with __class__.helper__get_node(node_svc).init().start() as node:\n\n with node.connect() as con:\n con.begin()\n con.execute('create table test(val int)')\n con.execute('insert into test values (1)')\n con.commit()\n\n con.begin()\n con.execute('insert into test values (2)')\n res = con.execute('select * from test order by val asc')\n assert (res == [(1, ), (2, )])\n con.rollback()\n\n con.begin()\n res = con.execute('select * from test')\n assert (res == [(1, )])\n con.rollback()\n\n con.begin()\n con.execute('drop table test')\n con.commit()\n\n def test_control_data(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n with __class__.helper__get_node(node_svc) as node:\n\n # node is not initialized yet\n with pytest.raises(expected_exception=ExecUtilException):\n node.get_control_data()\n\n node.init()\n data = node.get_control_data()\n\n # check returned dict\n assert data is not None\n assert (any('pg_control' in s for s in data.keys()))\n\n def test_backup_simple(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n with __class__.helper__get_node(node_svc) as master:\n\n # enable streaming for backups\n master.init(allow_streaming=True)\n\n # node must be running\n with pytest.raises(expected_exception=BackupException):\n master.backup()\n\n # it's time to start node\n master.start()\n\n # fill node with some data\n master.psql('create table test as select generate_series(1, 4) i')\n\n with master.backup(xlog_method='stream') as backup:\n with backup.spawn_primary().start() as slave:\n res = slave.execute('select * from test order by i asc')\n assert (res == [(1, ), (2, ), (3, ), (4, )])\n\n def test_backup_multiple(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n with __class__.helper__get_node(node_svc) as node:\n node.init(allow_streaming=True).start()\n\n with node.backup(xlog_method='fetch') as backup1, \\\n node.backup(xlog_method='fetch') as backup2:\n assert (backup1.base_dir != backup2.base_dir)\n\n with node.backup(xlog_method='fetch') as backup:\n with backup.spawn_primary('node1', destroy=False) as node1, \\\n backup.spawn_primary('node2', destroy=False) as node2:\n assert (node1.base_dir != node2.base_dir)\n\n def test_backup_exhaust(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n with __class__.helper__get_node(node_svc) as node:\n node.init(allow_streaming=True).start()\n\n with node.backup(xlog_method='fetch') as backup:\n # exhaust backup by creating new node\n with backup.spawn_primary():\n pass\n\n # now let's try to create one more node\n with pytest.raises(expected_exception=BackupException):\n backup.spawn_primary()\n\n def test_backup_wrong_xlog_method(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n with __class__.helper__get_node(node_svc) as node:\n node.init(allow_streaming=True).start()\n\n with pytest.raises(\n expected_exception=BackupException,\n match=\"^\" + re.escape('Invalid xlog_method \"wrong\"') + \"$\"\n ):\n node.backup(xlog_method='wrong')\n\n def test_pg_ctl_wait_option(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n C_MAX_ATTEMPT = 5\n\n nAttempt = 0\n\n while True:\n if nAttempt == C_MAX_ATTEMPT:\n raise Exception(\"PostgresSQL did not start.\")\n\n nAttempt += 1\n logging.info(\"------------------------ attempt #{}\".format(\n nAttempt\n ))\n\n if nAttempt > 1:\n logging.info(\"Sleep 3 seconds\")\n time.sleep(3)\n\n port = node_svc.port_manager.reserve_port()\n assert type(port) is int\n ok = False\n try:\n with __class__.helper__get_node(node_svc, port=port) as node:\n if self.impl__test_pg_ctl_wait_option(node_svc, node):\n ok = True\n finally:\n node_svc.port_manager.release_port(port)\n\n if ok:\n break\n\n continue\n\n logging.info(\"OK. Test is passed. Number of attempts is {}\".format(\n nAttempt\n ))\n return\n\n def impl__test_pg_ctl_wait_option(\n self,\n node_svc: PostgresNodeService,\n node: PostgresNode\n ) -> bool:\n assert isinstance(node_svc, PostgresNodeService)\n assert isinstance(node, PostgresNode)\n assert node.status() == NodeStatus.Uninitialized\n\n C_MAX_ATTEMPTS = 50\n\n logging.info(\"init node\")\n node.init()\n assert node.status() == NodeStatus.Stopped\n logging.info(\"node is inited\")\n\n node_log_reader = PostgresNodeLogReader(node, from_beginnig=True)\n\n logging.info(\"start node\")\n\n try:\n node.start(wait=False)\n except StartNodeException as e:\n logging.info(\"Exception ({}): {}\".format(\n type(e).__name__,\n e,\n ))\n return False\n logging.info(\"node is started\")\n\n nAttempt = 0\n while True:\n if PostgresNodeUtils.delect_port_conflict(node_log_reader):\n logging.info(\"Node port {} conflicted with another PostgreSQL instance.\".format(\n node.port\n ))\n return False\n\n if nAttempt == C_MAX_ATTEMPTS:\n #\n # [2025-03-11]\n # We have an unexpected problem with this test in CI\n # Let's get an additional information about this test failure.\n #\n logging.error(\"Node was not stopped.\")\n if not node.os_ops.path_exists(node.pg_log_file):\n logging.warning(\"Node log does not exist.\")\n else:\n logging.info(\"Let's read node log file [{0}]\".format(node.pg_log_file))\n logFileData = node.os_ops.read(node.pg_log_file, binary=False)\n logging.info(\"Node log file content:\\n{0}\".format(logFileData))\n\n raise Exception(\"Could not stop node.\")\n\n nAttempt += 1\n\n if nAttempt > 1:\n logging.info(\"Wait 1 second.\")\n time.sleep(1)\n logging.info(\"\")\n\n logging.info(\"Try to stop node. Attempt #{0}.\".format(nAttempt))\n\n try:\n node.stop(wait=False)\n break\n except ExecUtilException as e:\n # it's ok to get this exception here since node\n # could be not started yet\n logging.info(\"Node is not stopped. Exception ({0}): {1}\".format(type(e).__name__, e))\n continue\n\n logging.info(\"OK. Stop command was executed. Let's wait while our node will stop really.\")\n nAttempt = 0\n while True:\n if nAttempt == C_MAX_ATTEMPTS:\n raise Exception(\"Could not stop node.\")\n\n nAttempt += 1\n if nAttempt > 1:\n logging.info(\"Wait 1 second.\")\n time.sleep(1)\n logging.info(\"\")\n\n logging.info(\"Attempt #{0}.\".format(nAttempt))\n s1 = node.status()\n\n if s1 == NodeStatus.Running:\n continue\n\n if s1 == NodeStatus.Stopped:\n break\n\n raise Exception(\"Unexpected node status: {0}.\".format(s1))\n\n logging.info(\"OK. Node is stopped.\")\n return True\n\n def test_replicate(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n with __class__.helper__get_node(node_svc) as node:\n node.init(allow_streaming=True).start()\n\n with node.replicate().start() as replica:\n res = replica.execute('select 1')\n assert (res == [(1, )])\n\n node.execute('create table test (val int)', commit=True)\n\n replica.catchup()\n\n res = node.execute('select * from test')\n assert (res == [])\n\n def test_synchronous_replication(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n current_version = get_pg_version2(node_svc.os_ops)\n\n __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, \"9.6\")\n\n with __class__.helper__get_node(node_svc) as master:\n old_version = not __class__.helper__pg_version_ge(current_version, '9.6')\n\n master.init(allow_streaming=True).start()\n\n if not old_version:\n master.append_conf('synchronous_commit = remote_apply')\n\n # create standby\n with master.replicate() as standby1, master.replicate() as standby2:\n standby1.start()\n standby2.start()\n\n # check formatting\n assert (\n '1 (\"{}\", \"{}\")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2)))\n ) # yapf: disable\n assert (\n 'ANY 1 (\"{}\", \"{}\")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2)))\n ) # yapf: disable\n\n # set synchronous_standby_names\n master.set_synchronous_standbys(First(2, [standby1, standby2]))\n master.restart()\n\n # the following part of the test is only applicable to newer\n # versions of PostgresQL\n if not old_version:\n master.safe_psql('create table abc(a int)')\n\n # Create a large transaction that will take some time to apply\n # on standby to check that it applies synchronously\n # (If set synchronous_commit to 'on' or other lower level then\n # standby most likely won't catchup so fast and test will fail)\n master.safe_psql(\n 'insert into abc select generate_series(1, 1000000)')\n res = standby1.safe_psql('select count(*) from abc')\n assert (__class__.helper__rm_carriage_returns(res) == b'1000000\\n')\n\n def test_logical_replication(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n current_version = get_pg_version2(node_svc.os_ops)\n\n __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, \"10\")\n\n with __class__.helper__get_node(node_svc) as node1, __class__.helper__get_node(node_svc) as node2:\n node1.init(allow_logical=True)\n node1.start()\n node2.init().start()\n\n create_table = 'create table test (a int, b int)'\n node1.safe_psql(create_table)\n node2.safe_psql(create_table)\n\n # create publication / create subscription\n pub = node1.publish('mypub')\n sub = node2.subscribe(pub, 'mysub')\n\n node1.safe_psql('insert into test values (1, 1), (2, 2)')\n\n # wait until changes apply on subscriber and check them\n sub.catchup()\n res = node2.execute('select * from test')\n assert (res == [(1, 1), (2, 2)])\n\n # disable and put some new data\n sub.disable()\n node1.safe_psql('insert into test values (3, 3)')\n\n # enable and ensure that data successfully transferred\n sub.enable()\n sub.catchup()\n res = node2.execute('select * from test')\n assert (res == [(1, 1), (2, 2), (3, 3)])\n\n # Add new tables. Since we added \"all tables\" to publication\n # (default behaviour of publish() method) we don't need\n # to explicitly perform pub.add_tables()\n create_table = 'create table test2 (c char)'\n node1.safe_psql(create_table)\n node2.safe_psql(create_table)\n sub.refresh()\n\n # put new data\n node1.safe_psql('insert into test2 values (\\'a\\'), (\\'b\\')')\n sub.catchup()\n res = node2.execute('select * from test2')\n assert (res == [('a', ), ('b', )])\n\n # drop subscription\n sub.drop()\n pub.drop()\n\n # create new publication and subscription for specific table\n # (omitting copying data as it's already done)\n pub = node1.publish('newpub', tables=['test'])\n sub = node2.subscribe(pub, 'newsub', copy_data=False)\n\n node1.safe_psql('insert into test values (4, 4)')\n sub.catchup()\n res = node2.execute('select * from test')\n assert (res == [(1, 1), (2, 2), (3, 3), (4, 4)])\n\n # explicitly add table\n with pytest.raises(expected_exception=ValueError):\n pub.add_tables([]) # fail\n pub.add_tables(['test2'])\n node1.safe_psql('insert into test2 values (\\'c\\')')\n sub.catchup()\n res = node2.execute('select * from test2')\n assert (res == [('a', ), ('b', )])\n\n def test_logical_catchup(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n \"\"\" Runs catchup for 100 times to be sure that it is consistent \"\"\"\n\n current_version = get_pg_version2(node_svc.os_ops)\n\n __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, \"10\")\n\n with __class__.helper__get_node(node_svc) as node1, __class__.helper__get_node(node_svc) as node2:\n node1.init(allow_logical=True)\n node1.start()\n node2.init().start()\n\n create_table = 'create table test (key int primary key, val int); '\n node1.safe_psql(create_table)\n node1.safe_psql('alter table test replica identity default')\n node2.safe_psql(create_table)\n\n # create publication / create subscription\n sub = node2.subscribe(node1.publish('mypub'), 'mysub')\n\n for i in range(0, 100):\n node1.execute('insert into test values ({0}, {0})'.format(i))\n sub.catchup()\n res = node2.execute('select * from test')\n assert (res == [(i, i, )])\n node1.execute('delete from test')\n\n def test_logical_replication_fail(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n current_version = get_pg_version2(node_svc.os_ops)\n\n __class__.helper__skip_test_if_pg_version_is_ge(current_version, \"10\")\n\n with __class__.helper__get_node(node_svc) as node:\n with pytest.raises(expected_exception=InitNodeException):\n node.init(allow_logical=True)\n\n def test_replication_slots(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n with __class__.helper__get_node(node_svc) as node:\n node.init(allow_streaming=True).start()\n\n with node.replicate(slot='slot1').start() as replica:\n replica.execute('select 1')\n\n # cannot create new slot with the same name\n with pytest.raises(expected_exception=testgres_TestgresException):\n node.replicate(slot='slot1')\n\n def test_incorrect_catchup(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n with __class__.helper__get_node(node_svc) as node:\n node.init(allow_streaming=True).start()\n\n # node has no master, can't catch up\n with pytest.raises(expected_exception=testgres_TestgresException):\n node.catchup()\n\n def test_promotion(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n with __class__.helper__get_node(node_svc) as master:\n master.init().start()\n master.safe_psql('create table abc(id serial)')\n\n with master.replicate().start() as replica:\n master.stop()\n replica.promote()\n\n # make standby becomes writable master\n replica.safe_psql('insert into abc values (1)')\n res = replica.safe_psql('select * from abc')\n assert (__class__.helper__rm_carriage_returns(res) == b'1\\n')\n\n @pytest.fixture(\n params=[\n enums.DumpFormat.Plain,\n enums.DumpFormat.Custom,\n enums.DumpFormat.Directory,\n enums.DumpFormat.Tar\n ]\n )\n def dump_fmt(self, request: pytest.FixtureRequest) -> enums.DumpFormat:\n assert type(request.param) is enums.DumpFormat\n return request.param\n\n def test_dump(self, node_svc: PostgresNodeService, dump_fmt: enums.DumpFormat):\n assert isinstance(node_svc, PostgresNodeService)\n assert type(dump_fmt) is enums.DumpFormat\n query_create = 'create table test as select generate_series(1, 2) as val'\n query_select = 'select * from test order by val asc'\n\n with __class__.helper__get_node(node_svc).init().start() as node1:\n node1.execute(query_create)\n with removing(node_svc.os_ops, node1.dump(format=dump_fmt)) as dump:\n with __class__.helper__get_node(node_svc).init().start() as node3:\n if dump_fmt == enums.DumpFormat.Directory:\n assert (os.path.isdir(dump))\n else:\n assert (os.path.isfile(dump))\n # restore dump\n node3.restore(filename=dump)\n res = node3.execute(query_select)\n assert (res == [(1, ), (2, )])\n\n def test_dump_with_options(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n query_create = 'create table test_options as select generate_series(1, 5) as val'\n\n with __class__.helper__get_node(node_svc).init().start() as node1:\n node1.execute(query_create)\n\n # Test dump with --schema-only option\n with removing(node_svc.os_ops, node1.dump(options=['--schema-only'])) as dump:\n with __class__.helper__get_node(node_svc).init().start() as node2:\n assert (os.path.isfile(dump))\n # restore schema-only dump\n node2.restore(filename=dump)\n\n # Check that table exists but has no data\n res = node2.execute(\"SELECT COUNT(*) FROM test_options\")\n assert (res == [(0,)]) # Table exists but empty\n\n # Verify table structure exists\n res = node2.execute(\"\"\"\n SELECT COUNT(*) FROM information_schema.tables\n WHERE table_name = 'test_options'\n \"\"\")\n assert (res == [(1,)]) # Table structure exists\n\n def test_pgbench(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n __class__.helper__skip_test_if_util_not_exist(node_svc.os_ops, \"pgbench\")\n\n with __class__.helper__get_node(node_svc).init().start() as node:\n # initialize pgbench DB and run benchmarks\n node.pgbench_init(\n scale=2,\n foreign_keys=True,\n options=['-q']\n ).pgbench_run(time=2)\n\n # run TPC-B benchmark\n proc = node.pgbench(stdout=subprocess.PIPE,\n stderr=subprocess.STDOUT,\n options=['-T3'])\n out = proc.communicate()[0]\n assert (b'tps = ' in out)\n\n def test_unix_sockets(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n with __class__.helper__get_node(node_svc) as node:\n node.init(unix_sockets=False, allow_streaming=True)\n node.start()\n\n res_exec = node.execute('select 1')\n assert (res_exec == [(1,)])\n res_psql = node.safe_psql('select 1')\n assert (res_psql == b'1\\n')\n\n with node.replicate() as r:\n assert type(r) is PostgresNode\n r.start()\n res_exec = r.execute('select 1')\n assert (res_exec == [(1,)])\n res_psql = r.safe_psql('select 1')\n assert (res_psql == b'1\\n')\n\n def test_the_same_port(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n with __class__.helper__get_node(node_svc) as node:\n node.init().start()\n assert (node._should_free_port)\n assert (type(node.port) is int)\n node_port_copy = node.port\n r = node.safe_psql(\"SELECT 1;\")\n assert (__class__.helper__rm_carriage_returns(r) == b'1\\n')\n\n with __class__.helper__get_node(node_svc, port=node.port) as node2:\n assert (type(node2.port) is int)\n assert (node2.port == node.port)\n assert (not node2._should_free_port)\n assert (node2.status() == NodeStatus.Uninitialized)\n\n node2.init()\n\n with pytest.raises(\n expected_exception=StartNodeException,\n match=re.escape(\"Cannot start node\")\n ):\n node2.start()\n\n assert (node2.status() == NodeStatus.Stopped)\n\n # node is still working\n assert (node.port == node_port_copy)\n assert (node._should_free_port)\n r = node.safe_psql(\"SELECT 3;\")\n assert (__class__.helper__rm_carriage_returns(r) == b'3\\n')\n\n class tagPortManagerProxy(PortManager):\n m_PrevPortManager: PortManager\n\n m_DummyPortNumber: int\n m_DummyPortMaxUsage: int\n\n m_DummyPortCurrentUsage: int\n m_DummyPortTotalUsage: int\n\n def __init__(self, prevPortManager: PortManager, dummyPortNumber: int, dummyPortMaxUsage: int):\n assert isinstance(prevPortManager, PortManager)\n assert type(dummyPortNumber) is int\n assert type(dummyPortMaxUsage) is int\n assert dummyPortNumber >= 0\n assert dummyPortMaxUsage >= 0\n\n super().__init__()\n\n self.m_PrevPortManager = prevPortManager\n\n self.m_DummyPortNumber = dummyPortNumber\n self.m_DummyPortMaxUsage = dummyPortMaxUsage\n\n self.m_DummyPortCurrentUsage = 0\n self.m_DummyPortTotalUsage = 0\n\n def __enter__(self):\n return self\n\n def __exit__(self, type, value, traceback):\n assert self.m_DummyPortCurrentUsage == 0\n\n assert self.m_PrevPortManager is not None\n\n def reserve_port(self) -> int:\n assert type(self.m_DummyPortMaxUsage) is int\n assert type(self.m_DummyPortTotalUsage) is int\n assert type(self.m_DummyPortCurrentUsage) is int\n assert self.m_DummyPortTotalUsage >= 0\n assert self.m_DummyPortCurrentUsage >= 0\n\n assert self.m_DummyPortTotalUsage <= self.m_DummyPortMaxUsage\n assert self.m_DummyPortCurrentUsage <= self.m_DummyPortTotalUsage\n\n assert self.m_PrevPortManager is not None\n assert isinstance(self.m_PrevPortManager, PortManager)\n\n if self.m_DummyPortTotalUsage == self.m_DummyPortMaxUsage:\n return self.m_PrevPortManager.reserve_port()\n\n self.m_DummyPortTotalUsage += 1\n self.m_DummyPortCurrentUsage += 1\n return self.m_DummyPortNumber\n\n def release_port(self, number: int) -> None:\n assert type(number) is int\n\n assert type(self.m_DummyPortMaxUsage) is int\n assert type(self.m_DummyPortTotalUsage) is int\n assert type(self.m_DummyPortCurrentUsage) is int\n assert self.m_DummyPortTotalUsage >= 0\n assert self.m_DummyPortCurrentUsage >= 0\n\n assert self.m_DummyPortTotalUsage <= self.m_DummyPortMaxUsage\n assert self.m_DummyPortCurrentUsage <= self.m_DummyPortTotalUsage\n\n assert self.m_PrevPortManager is not None\n assert isinstance(self.m_PrevPortManager, PortManager)\n\n if self.m_DummyPortCurrentUsage > 0 and number == self.m_DummyPortNumber:\n assert self.m_DummyPortTotalUsage > 0\n self.m_DummyPortCurrentUsage -= 1\n return\n\n return self.m_PrevPortManager.release_port(number)\n\n def test_port_rereserve_during_node_start(self, node_svc: PostgresNodeService):\n assert type(node_svc) is PostgresNodeService\n assert PostgresNode._C_MAX_START_ATEMPTS == 5\n\n C_COUNT_OF_BAD_PORT_USAGE = 3\n\n with __class__.helper__get_node(node_svc) as node1:\n node1.init().start()\n assert node1._should_free_port\n assert type(node1.port) is int\n node1_port_copy = node1.port\n assert __class__.helper__rm_carriage_returns(node1.safe_psql(\"SELECT 1;\")) == b'1\\n'\n\n with __class__.tagPortManagerProxy(node_svc.port_manager, node1.port, C_COUNT_OF_BAD_PORT_USAGE) as proxy:\n assert proxy.m_DummyPortNumber == node1.port\n with __class__.helper__get_node(node_svc, port_manager=proxy) as node2:\n assert node2._should_free_port\n assert node2.port == node1.port\n\n node2.init().start()\n\n assert node2.port != node1.port\n assert node2._should_free_port\n assert proxy.m_DummyPortCurrentUsage == 0\n assert proxy.m_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE\n assert node2.is_started\n r = node2.safe_psql(\"SELECT 2;\")\n assert __class__.helper__rm_carriage_returns(r) == b'2\\n'\n\n # node1 is still working\n assert node1.port == node1_port_copy\n assert node1._should_free_port\n r = node1.safe_psql(\"SELECT 3;\")\n assert __class__.helper__rm_carriage_returns(r) == b'3\\n'\n\n def test_port_conflict(self, node_svc: PostgresNodeService):\n assert type(node_svc) is PostgresNodeService\n assert PostgresNode._C_MAX_START_ATEMPTS > 1\n\n C_COUNT_OF_BAD_PORT_USAGE = PostgresNode._C_MAX_START_ATEMPTS\n\n with __class__.helper__get_node(node_svc) as node1:\n node1.init().start()\n assert node1._should_free_port\n assert type(node1.port) is int\n node1_port_copy = node1.port\n assert __class__.helper__rm_carriage_returns(node1.safe_psql(\"SELECT 1;\")) == b'1\\n'\n\n with __class__.tagPortManagerProxy(node_svc.port_manager, node1.port, C_COUNT_OF_BAD_PORT_USAGE) as proxy:\n assert proxy.m_DummyPortNumber == node1.port\n with __class__.helper__get_node(node_svc, port_manager=proxy) as node2:\n assert node2._should_free_port\n assert node2.port == node1.port\n\n node2.init()\n assert node2.status() == NodeStatus.Stopped\n\n with pytest.raises(\n expected_exception=StartNodeException,\n match=re.escape(\"Cannot start node after multiple attempts.\")\n ):\n node2.start()\n\n assert node2.port == node1.port\n assert node2._should_free_port\n assert proxy.m_DummyPortCurrentUsage == 1\n assert proxy.m_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE\n assert not node2.is_started\n assert node2.status() == NodeStatus.Stopped\n\n # node2 must release our dummyPort (node1.port)\n assert (proxy.m_DummyPortCurrentUsage == 0)\n\n # node1 is still working\n assert node1.port == node1_port_copy\n assert node1._should_free_port\n r = node1.safe_psql(\"SELECT 3;\")\n assert __class__.helper__rm_carriage_returns(r) == b'3\\n'\n\n def test_try_to_get_port_after_free_manual_port(self, node_svc: PostgresNodeService):\n assert type(node_svc) is PostgresNodeService\n\n assert node_svc.port_manager is not None\n assert isinstance(node_svc.port_manager, PortManager)\n\n with __class__.helper__get_node(node_svc) as node1:\n assert node1 is not None\n assert type(node1) is PostgresNode\n assert node1.port is not None\n assert type(node1.port) is int\n with __class__.helper__get_node(node_svc, port=node1.port, port_manager=None) as node2:\n assert node2 is not None\n assert type(node1) is PostgresNode\n assert node2 is not node1\n assert node2.port is not None\n assert type(node2.port) is int\n assert node2.port == node1.port\n\n logging.info(\"Release node2 port\")\n node2.free_port()\n\n logging.info(\"try to get node2.port...\")\n with pytest.raises(\n InvalidOperationException,\n match=\"^\" + re.escape(\"PostgresNode port is not defined.\") + \"$\"\n ):\n p = node2.port\n assert p is None\n\n def test_try_to_start_node_after_free_manual_port(self, node_svc: PostgresNodeService):\n assert type(node_svc) is PostgresNodeService\n\n assert node_svc.port_manager is not None\n assert isinstance(node_svc.port_manager, PortManager)\n\n with __class__.helper__get_node(node_svc) as node1:\n assert node1 is not None\n assert type(node1) is PostgresNode\n assert node1.port is not None\n assert type(node1.port) is int\n with __class__.helper__get_node(node_svc, port=node1.port, port_manager=None) as node2:\n assert node2 is not None\n assert type(node1) is PostgresNode\n assert node2 is not node1\n assert node2.port is not None\n assert type(node2.port) is int\n assert node2.port == node1.port\n\n logging.info(\"Release node2 port\")\n node2.free_port()\n\n logging.info(\"node2 is trying to start...\")\n with pytest.raises(\n InvalidOperationException,\n match=\"^\" + re.escape(\"Can't start PostgresNode. Port is not defined.\") + \"$\"\n ):\n node2.start()\n\n def test_node__os_ops(self, node_svc: PostgresNodeService):\n assert type(node_svc) is PostgresNodeService\n\n assert node_svc.os_ops is not None\n assert isinstance(node_svc.os_ops, OsOperations)\n\n with PostgresNode(name=\"node\", os_ops=node_svc.os_ops, port_manager=node_svc.port_manager) as node:\n # retest\n assert node_svc.os_ops is not None\n assert isinstance(node_svc.os_ops, OsOperations)\n\n assert node.os_ops is node_svc.os_ops\n # one more time\n assert node.os_ops is node_svc.os_ops\n\n def test_node__port_manager(self, node_svc: PostgresNodeService):\n assert type(node_svc) is PostgresNodeService\n\n assert node_svc.port_manager is not None\n assert isinstance(node_svc.port_manager, PortManager)\n\n with PostgresNode(name=\"node\", os_ops=node_svc.os_ops, port_manager=node_svc.port_manager) as node:\n # retest\n assert node_svc.port_manager is not None\n assert isinstance(node_svc.port_manager, PortManager)\n\n assert node.port_manager is node_svc.port_manager\n # one more time\n assert node.port_manager is node_svc.port_manager\n\n def test_node__port_manager_and_explicit_port(self, node_svc: PostgresNodeService):\n assert type(node_svc) is PostgresNodeService\n\n assert isinstance(node_svc.os_ops, OsOperations)\n assert node_svc.port_manager is not None\n assert isinstance(node_svc.port_manager, PortManager)\n\n port = node_svc.port_manager.reserve_port()\n assert type(port) is int\n\n try:\n with PostgresNode(name=\"node\", port=port, os_ops=node_svc.os_ops) as node:\n # retest\n assert isinstance(node_svc.os_ops, OsOperations)\n assert node_svc.port_manager is not None\n assert isinstance(node_svc.port_manager, PortManager)\n\n assert node.port_manager is None\n assert node.os_ops is node_svc.os_ops\n\n # one more time\n assert node.port_manager is None\n assert node.os_ops is node_svc.os_ops\n finally:\n node_svc.port_manager.release_port(port)\n\n def test_node__no_port_manager(self, node_svc: PostgresNodeService):\n assert type(node_svc) is PostgresNodeService\n\n assert isinstance(node_svc.os_ops, OsOperations)\n assert node_svc.port_manager is not None\n assert isinstance(node_svc.port_manager, PortManager)\n\n port = node_svc.port_manager.reserve_port()\n assert type(port) is int\n\n try:\n with PostgresNode(name=\"node\", port=port, os_ops=node_svc.os_ops, port_manager=None) as node:\n # retest\n assert isinstance(node_svc.os_ops, OsOperations)\n assert node_svc.port_manager is not None\n assert isinstance(node_svc.port_manager, PortManager)\n\n assert node.port_manager is None\n assert node.os_ops is node_svc.os_ops\n\n # one more time\n assert node.port_manager is None\n assert node.os_ops is node_svc.os_ops\n finally:\n node_svc.port_manager.release_port(port)\n\n class tagTableChecksumTestData:\n record_count: int\n\n def __init__(\n self,\n record_count: int,\n ):\n assert type(record_count) is int\n self.record_count = record_count\n return\n\n sm_TableCheckSumTestDatas = [\n tagTableChecksumTestData(0),\n tagTableChecksumTestData(1),\n tagTableChecksumTestData(2),\n tagTableChecksumTestData(3),\n tagTableChecksumTestData(987),\n tagTableChecksumTestData(999),\n tagTableChecksumTestData(1000),\n tagTableChecksumTestData(1001),\n tagTableChecksumTestData(1999),\n tagTableChecksumTestData(19999),\n tagTableChecksumTestData(199999),\n tagTableChecksumTestData(1999999),\n ]\n\n @pytest.fixture(\n params=sm_TableCheckSumTestDatas,\n ids=[x.record_count for x in sm_TableCheckSumTestDatas],\n )\n def table_checksum_test_data(\n self,\n request: pytest.FixtureRequest\n ) -> tagTableChecksumTestData:\n assert isinstance(request, pytest.FixtureRequest)\n assert type(request.param).__name__ == \"tagTableChecksumTestData\"\n return request.param\n\n def test_node__table_checksum(\n self,\n node_svc: PostgresNodeService,\n table_checksum_test_data: tagTableChecksumTestData,\n ):\n assert type(node_svc) is PostgresNodeService\n assert type(table_checksum_test_data) is __class__.tagTableChecksumTestData\n assert node_svc.port_manager is not None\n assert isinstance(node_svc.port_manager, PortManager)\n\n with __class__.helper__get_node(node_svc) as node:\n assert node is not None\n assert type(node) is PostgresNode\n assert node.port is not None\n assert type(node.port) is int\n assert type(table_checksum_test_data.record_count) is int\n assert table_checksum_test_data.record_count >= 0\n\n node.init()\n node.slow_start()\n\n C_DB = \"postgres\"\n\n with node.connect(dbname=C_DB) as cn:\n assert type(cn) is NodeConnection\n\n cn.execute(\"create table t (id integer, data varchar(32));\")\n cn.commit()\n\n if table_checksum_test_data.record_count > 0:\n cn.execute(\"insert into t (id, data) select x, x from generate_series(1, {}) x\".format(\n table_checksum_test_data.record_count\n ))\n cn.commit()\n\n with cn.connection.cursor() as cursor:\n assert cursor is not None\n cursor.execute(\"SELECT hashtext(t::text) FROM \\\"t\\\" as t;\")\n\n checksum1 = 0\n record_count = 0\n while True:\n row = cursor.fetchone()\n if row is None:\n break\n assert type(row) in [list, tuple]\n assert len(row) == 1\n record_count += 1\n checksum1 += int(row[0])\n pass\n\n assert record_count == table_checksum_test_data.record_count\n\n checksum2 = node.table_checksum(\"t\", C_DB)\n assert type(checksum2) is int\n\n assert checksum1 == checksum2\n pass\n return\n\n def test_node__pgbench_table_checksums__one_table(\n self,\n node_svc: PostgresNodeService,\n table_checksum_test_data: tagTableChecksumTestData,\n ):\n assert type(node_svc) is PostgresNodeService\n assert type(table_checksum_test_data) is __class__.tagTableChecksumTestData\n assert node_svc.port_manager is not None\n assert isinstance(node_svc.port_manager, PortManager)\n\n with __class__.helper__get_node(node_svc) as node:\n assert node is not None\n assert type(node) is PostgresNode\n assert node.port is not None\n assert type(node.port) is int\n assert type(table_checksum_test_data.record_count) is int\n assert table_checksum_test_data.record_count >= 0\n\n node.init()\n node.slow_start()\n\n C_DB = \"postgres\"\n\n with node.connect(dbname=C_DB) as cn:\n assert type(cn) is NodeConnection\n\n cn.execute(\"create table t (id integer, data varchar(32));\")\n cn.commit()\n\n if table_checksum_test_data.record_count > 0:\n cn.execute(\"insert into t (id, data) select x, x from generate_series(1, {}) x\".format(\n table_checksum_test_data.record_count\n ))\n cn.commit()\n\n with cn.connection.cursor() as cursor:\n assert cursor is not None\n cursor.execute(\"SELECT hashtext(t::text) FROM \\\"t\\\" as t;\")\n\n checksum1 = 0\n record_count = 0\n while True:\n row = cursor.fetchone()\n if row is None:\n break\n assert type(row) in [list, tuple]\n assert len(row) == 1\n record_count += 1\n checksum1 += int(row[0])\n pass\n\n assert record_count == table_checksum_test_data.record_count\n\n actual_result = node.pgbench_table_checksums(C_DB, [\"t\"])\n assert type(actual_result) is set\n actual1 = actual_result.pop()\n assert type(actual1) is tuple\n assert len(actual1) == 2\n assert type(actual1[0]) is str\n assert type(actual1[1]) is int\n\n assert checksum1 == actual1[1]\n pass\n return\n\n def test_node__pgbench_table_checksums__pbckp_2278(self, node_svc: PostgresNodeService):\n assert type(node_svc) is PostgresNodeService\n\n assert node_svc.port_manager is not None\n assert isinstance(node_svc.port_manager, PortManager)\n\n with __class__.helper__get_node(node_svc) as node:\n assert node is not None\n assert type(node) is PostgresNode\n assert node.port is not None\n assert type(node.port) is int\n\n node.init()\n node.slow_start()\n\n logging.info(\"init pgbench database\")\n node.pgbench_init(scale=20)\n\n nPass = 0\n while nPass < 3:\n nPass += 1\n logging.info(\"------------------- pass: {}\".format(nPass))\n\n if not __class__.helper__call_and_check_pgbench_table_checksums(node):\n raise RuntimeError(\"pgbench_table_checksums created a problem. Please, check a test log.\")\n continue\n\n return\n\n @staticmethod\n def helper__call_and_check_pgbench_table_checksums(\n node: PostgresNode\n ) -> bool:\n assert node is not None\n assert type(node) is PostgresNode\n assert node.status() == NodeStatus.Running\n\n # We will check\n # 1) the structure of result\n # 2) the release of cursor locks\n\n logging.info(\"run pgbench_table_checksums\")\n full_checksums = node.pgbench_table_checksums()\n assert full_checksums is not None\n assert type(full_checksums) is set\n assert len(full_checksums) == 4\n\n expectedTables: typing.Dict[str, bool] = {\n 'pgbench_branches': False,\n 'pgbench_tellers': False,\n 'pgbench_accounts': False,\n 'pgbench_history': False,\n }\n\n ok = True\n\n for tcs in full_checksums:\n assert type(tcs) is tuple\n assert len(tcs) == 2\n assert type(tcs[0]) is str\n assert type(tcs[1]) is int\n\n tableName = tcs[0]\n if tableName not in expectedTables:\n ok = False\n logging.error(\"pgbench_table_checksums returns unknown table [{}].\".format(\n tableName\n ))\n continue\n\n if expectedTables[tableName]:\n ok = False\n logging.error(\"pgbench_table_checksums returns table [{}] more than one time.\".format(\n tableName\n ))\n continue\n\n expectedTables[tableName] = True\n continue\n\n C_SQL = \"\"\"select x.granted, x.mode\nfrom pg_locks x join pg_class c on x.relation=c.oid\nwhere c.relname=%s;\"\"\"\n\n cn = node.connect(dbname=\"postgres\")\n assert type(cn) is NodeConnection\n\n try:\n for tcs in full_checksums:\n tableName = tcs[0]\n recs = cn.execute(C_SQL, tableName)\n assert type(recs) is list\n if len(recs) == 0:\n logging.info(\"Table [{}] does not have a lock. It is ok.\".format(\n tableName,\n ))\n else:\n ok = False\n assert len(recs) == 1\n rec = recs[0]\n assert type(rec) is tuple\n assert len(rec) == 2\n logging.error(\"Table [{}] has a lock [granted: {}][mode: {}].\".format(\n tableName,\n rec[0],\n rec[1],\n ))\n continue\n finally:\n try:\n cn.close()\n except Exception as e:\n logging.error(\"Can't close connection. Exception ({}): {}\".format(\n type(e).__name__,\n e,\n ))\n return ok\n\n class tag_rmdirs_protector:\n _os_ops: OsOperations\n _cwd: str\n _old_rmdirs: any\n _cwd: str\n\n def __init__(self, os_ops: OsOperations):\n self._os_ops = os_ops\n self._cwd = os.path.abspath(os_ops.cwd())\n self._old_rmdirs = os_ops.rmdirs\n\n def __enter__(self):\n assert self._os_ops.rmdirs == self._old_rmdirs\n self._os_ops.rmdirs = self.proxy__rmdirs\n return self\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n assert self._os_ops.rmdirs == self.proxy__rmdirs\n self._os_ops.rmdirs = self._old_rmdirs\n return False\n\n def proxy__rmdirs(self, path, ignore_errors=True):\n raise Exception(\"Call of rmdirs is not expected!\")\n\n def test_node_app__make_empty__base_dir_is_None(self, node_svc: PostgresNodeService):\n assert type(node_svc) is PostgresNodeService\n\n assert isinstance(node_svc.os_ops, OsOperations)\n assert node_svc.port_manager is not None\n assert isinstance(node_svc.port_manager, PortManager)\n\n tmp_dir = node_svc.os_ops.mkdtemp()\n assert tmp_dir is not None\n assert type(tmp_dir) is str\n logging.info(\"temp directory is [{}]\".format(tmp_dir))\n\n # -----------\n os_ops = node_svc.os_ops.create_clone()\n assert os_ops is not node_svc.os_ops\n\n # -----------\n with __class__.tag_rmdirs_protector(os_ops):\n node_app = NodeApp(test_path=tmp_dir, os_ops=os_ops)\n assert node_app.os_ops is os_ops\n\n with pytest.raises(expected_exception=BaseException) as x:\n node_app.make_empty(base_dir=None)\n\n if type(x.value) is AssertionError:\n pass\n else:\n assert type(x.value) is ValueError\n assert str(x.value) == \"Argument 'base_dir' is not defined.\"\n\n # -----------\n logging.info(\"temp directory [{}] is deleting\".format(tmp_dir))\n node_svc.os_ops.rmdir(tmp_dir)\n\n def test_node_app__make_empty__base_dir_is_Empty(self, node_svc: PostgresNodeService):\n assert type(node_svc) is PostgresNodeService\n\n assert isinstance(node_svc.os_ops, OsOperations)\n assert node_svc.port_manager is not None\n assert isinstance(node_svc.port_manager, PortManager)\n\n tmp_dir = node_svc.os_ops.mkdtemp()\n assert tmp_dir is not None\n assert type(tmp_dir) is str\n logging.info(\"temp directory is [{}]\".format(tmp_dir))\n\n # -----------\n os_ops = node_svc.os_ops.create_clone()\n assert os_ops is not node_svc.os_ops\n\n # -----------\n with __class__.tag_rmdirs_protector(os_ops):\n node_app = NodeApp(test_path=tmp_dir, os_ops=os_ops)\n assert node_app.os_ops is os_ops\n\n with pytest.raises(expected_exception=ValueError) as x:\n node_app.make_empty(base_dir=\"\")\n\n assert str(x.value) == \"Argument 'base_dir' is empty.\"\n\n # -----------\n logging.info(\"temp directory [{}] is deleting\".format(tmp_dir))\n node_svc.os_ops.rmdir(tmp_dir)\n\n def test_node_app__make_empty(self, node_svc: PostgresNodeService):\n assert type(node_svc) is PostgresNodeService\n\n assert isinstance(node_svc.os_ops, OsOperations)\n assert node_svc.port_manager is not None\n assert isinstance(node_svc.port_manager, PortManager)\n\n tmp_dir = node_svc.os_ops.mkdtemp()\n assert tmp_dir is not None\n assert type(tmp_dir) is str\n logging.info(\"temp directory is [{}]\".format(tmp_dir))\n\n # -----------\n node_app = NodeApp(\n test_path=tmp_dir,\n os_ops=node_svc.os_ops,\n port_manager=node_svc.port_manager\n )\n\n assert node_app.os_ops is node_svc.os_ops\n assert node_app.port_manager is node_svc.port_manager\n assert type(node_app.nodes_to_cleanup) is list\n assert len(node_app.nodes_to_cleanup) == 0\n\n node: typing.Optional[PostgresNode] = None\n try:\n node = node_app.make_simple(\"node\")\n assert node is not None\n assert isinstance(node, PostgresNode)\n assert node.os_ops is node_svc.os_ops\n assert node.port_manager is node_svc.port_manager\n\n assert type(node_app.nodes_to_cleanup) is list\n assert len(node_app.nodes_to_cleanup) == 1\n assert node_app.nodes_to_cleanup[0] is node\n\n node.slow_start()\n finally:\n if node is not None:\n node.stop()\n node.release_resources()\n\n if node is not None:\n node.cleanup(release_resources=True)\n\n # -----------\n logging.info(\"temp directory [{}] is deleting\".format(tmp_dir))\n node_svc.os_ops.rmdir(tmp_dir)\n\n def test_node_app__make_simple__checksum(self, node_svc: PostgresNodeService):\n assert type(node_svc) is PostgresNodeService\n\n assert isinstance(node_svc.os_ops, OsOperations)\n assert node_svc.port_manager is not None\n assert isinstance(node_svc.port_manager, PortManager)\n\n tmp_dir = node_svc.os_ops.mkdtemp()\n assert tmp_dir is not None\n assert type(tmp_dir) is str\n\n logging.info(\"temp directory is [{}]\".format(tmp_dir))\n node_app = NodeApp(test_path=tmp_dir, os_ops=node_svc.os_ops)\n\n C_NODE = \"node\"\n\n # -----------\n def LOCAL__test(checksum: bool, initdb_params: typing.Optional[list]):\n initdb_params0 = initdb_params\n initdb_params0_copy = initdb_params0.copy() if initdb_params0 is not None else None\n\n with node_app.make_simple(C_NODE, checksum=checksum, initdb_params=initdb_params):\n assert initdb_params is initdb_params0\n if initdb_params0 is not None:\n assert initdb_params0 == initdb_params0_copy\n\n assert initdb_params is initdb_params0\n if initdb_params0 is not None:\n assert initdb_params0 == initdb_params0_copy\n\n # -----------\n LOCAL__test(checksum=False, initdb_params=None)\n LOCAL__test(checksum=True, initdb_params=None)\n\n # -----------\n params = []\n LOCAL__test(checksum=False, initdb_params=params)\n LOCAL__test(checksum=True, initdb_params=params)\n\n # -----------\n params = [\"--no-sync\"]\n LOCAL__test(checksum=False, initdb_params=params)\n LOCAL__test(checksum=True, initdb_params=params)\n\n # -----------\n params = [\"--data-checksums\"]\n LOCAL__test(checksum=False, initdb_params=params)\n LOCAL__test(checksum=True, initdb_params=params)\n\n # -----------\n logging.info(\"temp directory [{}] is deleting\".format(tmp_dir))\n node_svc.os_ops.rmdir(tmp_dir)\n\n def test_node_app__make_empty_with_explicit_port(self, node_svc: PostgresNodeService):\n assert type(node_svc) is PostgresNodeService\n\n assert isinstance(node_svc.os_ops, OsOperations)\n assert node_svc.port_manager is not None\n assert isinstance(node_svc.port_manager, PortManager)\n\n C_MAX_ATTEMPTS = 5\n\n tmp_dir = node_svc.os_ops.mkdtemp()\n assert tmp_dir is not None\n assert type(tmp_dir) is str\n logging.info(\"temp directory is [{}]\".format(tmp_dir))\n\n # -----------\n node_app = NodeApp(\n test_path=tmp_dir,\n os_ops=node_svc.os_ops,\n port_manager=node_svc.port_manager\n )\n\n assert node_app.os_ops is node_svc.os_ops\n assert node_app.port_manager is node_svc.port_manager\n assert type(node_app.nodes_to_cleanup) is list\n assert len(node_app.nodes_to_cleanup) == 0\n\n attempt = 0\n ports = []\n try:\n while True:\n if attempt == C_MAX_ATTEMPTS:\n raise RuntimeError(\"Node did not start.\")\n\n attempt += 1\n\n logging.info(\"------------- attempt #{}\".format(\n attempt\n ))\n\n port = node_app.port_manager.reserve_port()\n assert type(port) is int\n assert port is not ports\n\n try:\n ports.append(port)\n except: # noqa: E722\n node_app.port_manager.release_port(port)\n raise\n\n assert len(ports) == attempt\n node_name = \"node_{}\".format(attempt)\n\n logging.info(\"Node [{}] is creating...\".format(node_name))\n node = node_app.make_simple(node_name, port=port)\n assert node is not None\n assert isinstance(node, PostgresNode)\n assert node.os_ops is node_svc.os_ops\n assert node.port_manager is None # <---------\n assert node.port == port\n assert node._should_free_port == False # noqa: E712\n\n assert type(node_app.nodes_to_cleanup) is list\n assert len(node_app.nodes_to_cleanup) == attempt\n assert node_app.nodes_to_cleanup[-1] is node\n\n assert node.status() == NodeStatus.Stopped\n logging.info(\"Node is created\")\n\n logging.info(\"Try to start a node...\")\n try:\n node.slow_start()\n except StartNodeException as e:\n logging.info(\"Exception ({}): {}\".format(\n type(e).__name__,\n e\n ))\n assert node.status() == NodeStatus.Stopped\n continue\n\n assert node.status() == NodeStatus.Running\n logging.info(\"Node is started\")\n\n logging.info(\"Stop node\")\n node.stop()\n assert node.status() == NodeStatus.Stopped\n logging.info(\"Node is stopped\")\n\n logging.info(\"OK. Go home.\")\n assert node is not None\n assert isinstance(node, PostgresNode)\n assert node._port is not None\n assert node._port == port\n assert not node._should_free_port\n break\n finally:\n while len(ports) > 0:\n node_app.port_manager.release_port(ports.pop())\n\n # -----------\n logging.info(\"temp directory [{}] is deleting\".format(tmp_dir))\n node_svc.os_ops.rmdirs(tmp_dir)\n return\n\n @staticmethod\n def helper__get_node(\n node_svc: PostgresNodeService,\n name: typing.Optional[str] = None,\n port: typing.Optional[int] = None,\n port_manager: typing.Optional[PortManager] = None\n ) -> PostgresNode:\n assert isinstance(node_svc, PostgresNodeService)\n assert isinstance(node_svc.os_ops, OsOperations)\n assert isinstance(node_svc.port_manager, PortManager)\n\n if port_manager is None:\n port_manager = node_svc.port_manager\n\n return PostgresNode(\n name,\n port=port,\n os_ops=node_svc.os_ops,\n port_manager=port_manager if port is None else None\n )\n\n @staticmethod\n def helper__skip_test_if_pg_version_is_not_ge(ver1: str, ver2: str):\n assert type(ver1) is str\n assert type(ver2) is str\n if not __class__.helper__pg_version_ge(ver1, ver2):\n pytest.skip('requires {0}+'.format(ver2))\n\n @staticmethod\n def helper__skip_test_if_pg_version_is_ge(ver1: str, ver2: str):\n assert type(ver1) is str\n assert type(ver2) is str\n if __class__.helper__pg_version_ge(ver1, ver2):\n pytest.skip('requires <{0}'.format(ver2))\n\n @staticmethod\n def helper__pg_version_ge(ver1: str, ver2: str) -> bool:\n assert type(ver1) is str\n assert type(ver2) is str\n v1 = PgVer(ver1)\n v2 = PgVer(ver2)\n return v1 >= v2\n\n @staticmethod\n def helper__rm_carriage_returns(out):\n \"\"\"\n In Windows we have additional '\\r' symbols in output.\n Let's get rid of them.\n \"\"\"\n if isinstance(out, (int, float, complex)):\n return out\n\n if isinstance(out, tuple):\n return tuple(__class__.helper__rm_carriage_returns(item) for item in out)\n\n if isinstance(out, bytes):\n return out.replace(b'\\r', b'')\n\n assert type(out) is str\n return out.replace('\\r', '')\n\n @staticmethod\n def helper__skip_test_if_util_not_exist(os_ops: OsOperations, name: str):\n assert isinstance(os_ops, OsOperations)\n assert type(name) is str\n if not __class__.helper__util_exists(os_ops, name):\n pytest.skip('might be missing')\n\n @staticmethod\n def helper__util_exists(os_ops: OsOperations, util):\n assert isinstance(os_ops, OsOperations)\n\n def good_properties(f):\n return (os_ops.path_exists(f) and # noqa: W504\n os_ops.isfile(f) and # noqa: W504\n os_ops.is_executable(f)) # yapf: disable\n\n # try to resolve it\n if good_properties(get_bin_path2(os_ops, util)):\n return True\n\n # check if util is in PATH\n for path in os_ops.environ(\"PATH\").split(os.pathsep):\n if good_properties(os.path.join(path, util)):\n return True\n" }, { "path": "tests/test_testgres_local.py", "content": "# coding: utf-8\nimport os\nimport re\nimport subprocess\nimport pytest\nimport psutil\nimport platform\nimport logging\n\nimport src as testgres\n\nfrom src import StartNodeException\nfrom src import ExecUtilException\nfrom src import NodeApp\nfrom src import NodeStatus\nfrom src import scoped_config\nfrom src import get_new_node\nfrom src import get_bin_path\nfrom src import get_pg_config\nfrom src import get_pg_version\n\n# NOTE: those are ugly imports\nfrom src.utils import bound_ports\nfrom src.utils import PgVer\nfrom src.node import ProcessProxy\n\n\ndef pg_version_ge(version):\n cur_ver = PgVer(get_pg_version())\n min_ver = PgVer(version)\n return cur_ver >= min_ver\n\n\ndef util_exists(util):\n def good_properties(f):\n return (os.path.exists(f) and # noqa: W504\n os.path.isfile(f) and # noqa: W504\n os.access(f, os.X_OK)) # yapf: disable\n\n # try to resolve it\n if good_properties(get_bin_path(util)):\n return True\n\n # check if util is in PATH\n for path in os.environ[\"PATH\"].split(os.pathsep):\n if good_properties(os.path.join(path, util)):\n return True\n\n\ndef rm_carriage_returns(out):\n \"\"\"\n In Windows we have additional '\\r' symbols in output.\n Let's get rid of them.\n \"\"\"\n if os.name == 'nt':\n if isinstance(out, (int, float, complex)):\n return out\n elif isinstance(out, tuple):\n return tuple(rm_carriage_returns(item) for item in out)\n elif isinstance(out, bytes):\n return out.replace(b'\\r', b'')\n else:\n return out.replace('\\r', '')\n else:\n return out\n\n\nclass TestTestgresLocal:\n def test_pg_config(self):\n # check same instances\n a = get_pg_config()\n b = get_pg_config()\n assert (id(a) == id(b))\n\n # save right before config change\n c1 = get_pg_config()\n\n # modify setting for this scope\n with scoped_config(cache_pg_config=False) as config:\n # sanity check for value\n assert not (config.cache_pg_config)\n\n # save right after config change\n c2 = get_pg_config()\n\n # check different instances after config change\n assert (id(c1) != id(c2))\n\n # check different instances\n a = get_pg_config()\n b = get_pg_config()\n assert (id(a) != id(b))\n\n def test_ports_management(self):\n assert bound_ports is not None\n assert type(bound_ports) is set\n\n if len(bound_ports) != 0:\n logging.warning(\"bound_ports is not empty: {0}\".format(bound_ports))\n\n stage0__bound_ports = bound_ports.copy()\n\n with get_new_node() as node:\n assert bound_ports is not None\n assert type(bound_ports) is set\n\n assert node.port is not None\n assert type(node.port) is int\n\n logging.info(\"node port is {0}\".format(node.port))\n\n assert node.port in bound_ports\n assert node.port not in stage0__bound_ports\n\n assert stage0__bound_ports <= bound_ports\n assert len(stage0__bound_ports) + 1 == len(bound_ports)\n\n stage1__bound_ports = stage0__bound_ports.copy()\n stage1__bound_ports.add(node.port)\n\n assert stage1__bound_ports == bound_ports\n\n # check that port has been freed successfully\n assert bound_ports is not None\n assert type(bound_ports) is set\n assert bound_ports == stage0__bound_ports\n\n def test_child_process_dies(self):\n # test for FileNotFound exception during child_processes() function\n cmd = [\"timeout\", \"60\"] if os.name == 'nt' else [\"sleep\", \"60\"]\n\n nAttempt = 0\n\n while True:\n if nAttempt == 5:\n raise Exception(\"Max attempt number is exceed.\")\n\n nAttempt += 1\n\n logging.info(\"Attempt #{0}\".format(nAttempt))\n\n with subprocess.Popen(cmd, shell=True) as process: # shell=True might be needed on Windows\n r = process.poll()\n\n if r is not None:\n logging.warning(\"process.pool() returns an unexpected result: {0}.\".format(r))\n continue\n\n assert r is None\n # collect list of processes currently running\n children = psutil.Process(os.getpid()).children()\n # kill a process, so received children dictionary becomes invalid\n process.kill()\n process.wait()\n # try to handle children list -- missing processes will have ptype \"ProcessType.Unknown\"\n [ProcessProxy(p) for p in children]\n break\n\n def test_upgrade_node(self):\n old_bin_dir = os.path.dirname(get_bin_path(\"pg_config\"))\n new_bin_dir = os.path.dirname(get_bin_path(\"pg_config\"))\n with get_new_node(prefix='node_old', bin_dir=old_bin_dir) as node_old:\n node_old.init()\n node_old.start()\n node_old.stop()\n with get_new_node(prefix='node_new', bin_dir=new_bin_dir) as node_new:\n node_new.init(cached=False)\n res = node_new.upgrade_from(old_node=node_old)\n node_new.start()\n assert (b'Upgrade Complete' in res)\n\n class tagPortManagerProxy:\n sm_prev_testgres_reserve_port = None\n sm_prev_testgres_release_port = None\n\n sm_DummyPortNumber = None\n sm_DummyPortMaxUsage = None\n\n sm_DummyPortCurrentUsage = None\n sm_DummyPortTotalUsage = None\n\n def __init__(self, dummyPortNumber, dummyPortMaxUsage):\n assert type(dummyPortNumber) is int\n assert type(dummyPortMaxUsage) is int\n assert dummyPortNumber >= 0\n assert dummyPortMaxUsage >= 0\n\n assert __class__.sm_prev_testgres_reserve_port is None\n assert __class__.sm_prev_testgres_release_port is None\n assert testgres.utils.reserve_port == testgres.utils.internal__reserve_port\n assert testgres.utils.release_port == testgres.utils.internal__release_port\n\n __class__.sm_prev_testgres_reserve_port = testgres.utils.reserve_port\n __class__.sm_prev_testgres_release_port = testgres.utils.release_port\n\n testgres.utils.reserve_port = __class__._proxy__reserve_port\n testgres.utils.release_port = __class__._proxy__release_port\n\n assert testgres.utils.reserve_port == __class__._proxy__reserve_port\n assert testgres.utils.release_port == __class__._proxy__release_port\n\n __class__.sm_DummyPortNumber = dummyPortNumber\n __class__.sm_DummyPortMaxUsage = dummyPortMaxUsage\n\n __class__.sm_DummyPortCurrentUsage = 0\n __class__.sm_DummyPortTotalUsage = 0\n\n def __enter__(self):\n return self\n\n def __exit__(self, type, value, traceback):\n assert __class__.sm_DummyPortCurrentUsage == 0\n\n assert __class__.sm_prev_testgres_reserve_port is not None\n assert __class__.sm_prev_testgres_release_port is not None\n\n assert testgres.utils.reserve_port == __class__._proxy__reserve_port\n assert testgres.utils.release_port == __class__._proxy__release_port\n\n testgres.utils.reserve_port = __class__.sm_prev_testgres_reserve_port\n testgres.utils.release_port = __class__.sm_prev_testgres_release_port\n\n __class__.sm_prev_testgres_reserve_port = None\n __class__.sm_prev_testgres_release_port = None\n\n @staticmethod\n def _proxy__reserve_port():\n assert type(__class__.sm_DummyPortMaxUsage) is int\n assert type(__class__.sm_DummyPortTotalUsage) is int\n assert type(__class__.sm_DummyPortCurrentUsage) is int\n assert __class__.sm_DummyPortTotalUsage >= 0\n assert __class__.sm_DummyPortCurrentUsage >= 0\n\n assert __class__.sm_DummyPortTotalUsage <= __class__.sm_DummyPortMaxUsage\n assert __class__.sm_DummyPortCurrentUsage <= __class__.sm_DummyPortTotalUsage\n\n assert __class__.sm_prev_testgres_reserve_port is not None\n\n if __class__.sm_DummyPortTotalUsage == __class__.sm_DummyPortMaxUsage:\n return __class__.sm_prev_testgres_reserve_port()\n\n __class__.sm_DummyPortTotalUsage += 1\n __class__.sm_DummyPortCurrentUsage += 1\n return __class__.sm_DummyPortNumber\n\n @staticmethod\n def _proxy__release_port(dummyPortNumber):\n assert type(dummyPortNumber) is int\n\n assert type(__class__.sm_DummyPortMaxUsage) is int\n assert type(__class__.sm_DummyPortTotalUsage) is int\n assert type(__class__.sm_DummyPortCurrentUsage) is int\n assert __class__.sm_DummyPortTotalUsage >= 0\n assert __class__.sm_DummyPortCurrentUsage >= 0\n\n assert __class__.sm_DummyPortTotalUsage <= __class__.sm_DummyPortMaxUsage\n assert __class__.sm_DummyPortCurrentUsage <= __class__.sm_DummyPortTotalUsage\n\n assert __class__.sm_prev_testgres_release_port is not None\n\n if __class__.sm_DummyPortCurrentUsage > 0 and dummyPortNumber == __class__.sm_DummyPortNumber:\n assert __class__.sm_DummyPortTotalUsage > 0\n __class__.sm_DummyPortCurrentUsage -= 1\n return\n\n return __class__.sm_prev_testgres_release_port(dummyPortNumber)\n\n def test_port_rereserve_during_node_start(self):\n assert testgres.PostgresNode._C_MAX_START_ATEMPTS == 5\n\n C_COUNT_OF_BAD_PORT_USAGE = 3\n\n with get_new_node() as node1:\n node1.init().start()\n assert (node1._should_free_port)\n assert (type(node1.port) is int)\n node1_port_copy = node1.port\n assert (rm_carriage_returns(node1.safe_psql(\"SELECT 1;\")) == b'1\\n')\n\n with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE):\n assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port\n with get_new_node() as node2:\n assert (node2._should_free_port)\n assert (node2.port == node1.port)\n\n node2.init().start()\n\n assert (node2.port != node1.port)\n assert (node2._should_free_port)\n assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 0)\n assert (__class__.tagPortManagerProxy.sm_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE)\n assert (node2.is_started)\n\n assert (rm_carriage_returns(node2.safe_psql(\"SELECT 2;\")) == b'2\\n')\n\n # node1 is still working\n assert (node1.port == node1_port_copy)\n assert (node1._should_free_port)\n assert (rm_carriage_returns(node1.safe_psql(\"SELECT 3;\")) == b'3\\n')\n\n def test_port_conflict(self):\n assert testgres.PostgresNode._C_MAX_START_ATEMPTS > 1\n\n C_COUNT_OF_BAD_PORT_USAGE = testgres.PostgresNode._C_MAX_START_ATEMPTS\n\n with get_new_node() as node1:\n node1.init().start()\n assert (node1._should_free_port)\n assert (type(node1.port) is int)\n node1_port_copy = node1.port\n assert (rm_carriage_returns(node1.safe_psql(\"SELECT 1;\")) == b'1\\n')\n\n with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE):\n assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port\n with get_new_node() as node2:\n assert (node2._should_free_port)\n assert (node2.port == node1.port)\n\n node2.init()\n assert (node2.status() == NodeStatus.Stopped)\n with pytest.raises(\n expected_exception=StartNodeException,\n match=re.escape(\"Cannot start node after multiple attempts.\")\n ):\n node2.start()\n\n assert (node2.port == node1.port)\n assert (node2._should_free_port)\n assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 1)\n assert (__class__.tagPortManagerProxy.sm_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE)\n assert (not node2.is_started)\n assert (node2.status() == NodeStatus.Stopped)\n\n # node2 must release our dummyPort (node1.port)\n assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 0)\n\n # node1 is still working\n assert (node1.port == node1_port_copy)\n assert (node1._should_free_port)\n assert (node1.status() == NodeStatus.Running)\n assert (rm_carriage_returns(node1.safe_psql(\"SELECT 3;\")) == b'3\\n')\n\n def test_simple_with_bin_dir(self):\n with get_new_node() as node:\n node.init().start()\n bin_dir = node.bin_dir\n\n app = NodeApp()\n with app.make_simple(base_dir=node.base_dir, bin_dir=bin_dir) as correct_bin_dir:\n correct_bin_dir.slow_start()\n correct_bin_dir.safe_psql(\"SELECT 1;\")\n correct_bin_dir.stop()\n\n while True:\n try:\n app.make_simple(base_dir=node.base_dir, bin_dir=\"wrong/path\")\n except FileNotFoundError:\n break # Expected error\n except ExecUtilException:\n break # Expected error\n\n raise RuntimeError(\"Error was expected.\") # We should not reach this\n\n return\n\n def test_set_auto_conf(self):\n # elements contain [property id, value, storage value]\n testData = [\n [\"archive_command\",\n \"cp '%p' \\\"/mnt/server/archivedir/%f\\\"\",\n \"'cp \\\\'%p\\\\' \\\"/mnt/server/archivedir/%f\\\"\"],\n [\"log_line_prefix\",\n \"'\\n\\r\\t\\b\\\\\\\"\",\n \"'\\\\\\'\\\\n\\\\r\\\\t\\\\b\\\\\\\\\\\"\"],\n [\"log_connections\",\n True,\n \"on\"],\n [\"log_disconnections\",\n False,\n \"off\"],\n [\"autovacuum_max_workers\",\n 3,\n \"3\"]\n ]\n if pg_version_ge('12'):\n testData.append([\"restore_command\",\n 'cp \"/mnt/server/archivedir/%f\" \\'%p\\'',\n \"'cp \\\"/mnt/server/archivedir/%f\\\" \\\\'%p\\\\''\"])\n\n with get_new_node() as node:\n node.init().start()\n\n options = {}\n\n for x in testData:\n options[x[0]] = x[1]\n\n node.set_auto_conf(options)\n node.stop()\n node.slow_start()\n\n auto_conf_path = f\"{node.data_dir}/postgresql.auto.conf\"\n with open(auto_conf_path, \"r\") as f:\n content = f.read()\n\n for x in testData:\n assert x[0] + \" = \" + x[2] in content\n\n @staticmethod\n def helper__skip_test_if_util_not_exist(name: str):\n assert type(name) is str\n\n if platform.system().lower() == \"windows\":\n name2 = name + \".exe\"\n else:\n name2 = name\n\n if not util_exists(name2):\n pytest.skip('might be missing')\n" }, { "path": "tests/test_testgres_remote.py", "content": "# coding: utf-8\nimport os\n\nimport pytest\nimport logging\nimport typing\n\nfrom .helpers.global_data import PostgresNodeService\nfrom .helpers.global_data import PostgresNodeServices\n\nimport src as testgres\n\nfrom src.exceptions import InitNodeException\nfrom src.exceptions import ExecUtilException\n\nfrom src.config import scoped_config\nfrom src.config import testgres_config\n\nfrom src import get_bin_path\nfrom src import get_pg_config\n\n# NOTE: those are ugly imports\n\n\ndef util_exists(util):\n def good_properties(f):\n return (testgres_config.os_ops.path_exists(f) and # noqa: W504\n testgres_config.os_ops.isfile(f) and # noqa: W504\n testgres_config.os_ops.is_executable(f)) # yapf: disable\n\n # try to resolve it\n if good_properties(get_bin_path(util)):\n return True\n\n # check if util is in PATH\n for path in testgres_config.os_ops.environ(\"PATH\").split(testgres_config.os_ops.pathsep):\n if good_properties(os.path.join(path, util)):\n return True\n\n\nclass TestTestgresRemote:\n @pytest.fixture(autouse=True, scope=\"class\")\n def implicit_fixture(self):\n cur_os_ops = PostgresNodeServices.sm_remote.os_ops\n assert cur_os_ops is not None\n\n prev_ops = testgres_config.os_ops\n assert prev_ops is not None\n testgres_config.set_os_ops(os_ops=cur_os_ops)\n assert testgres_config.os_ops is cur_os_ops\n yield\n assert testgres_config.os_ops is cur_os_ops\n testgres_config.set_os_ops(os_ops=prev_ops)\n assert testgres_config.os_ops is prev_ops\n\n def test_init__LANG_ะก(self):\n # PBCKP-1744\n prev_LANG = os.environ.get(\"LANG\")\n\n try:\n os.environ[\"LANG\"] = \"C\"\n\n with __class__.helper__get_node() as node:\n node.init().start()\n finally:\n __class__.helper__restore_envvar(\"LANG\", prev_LANG)\n\n def test_init__unk_LANG_and_LC_CTYPE(self):\n # PBCKP-1744\n prev_LANG = os.environ.get(\"LANG\")\n prev_LANGUAGE = os.environ.get(\"LANGUAGE\")\n prev_LC_CTYPE = os.environ.get(\"LC_CTYPE\")\n prev_LC_COLLATE = os.environ.get(\"LC_COLLATE\")\n\n try:\n # TODO: Pass unkData through test parameter.\n unkDatas = [\n (\"UNKNOWN_LANG\", \"UNKNOWN_CTYPE\"),\n (\"\\\"UNKNOWN_LANG\\\"\", \"\\\"UNKNOWN_CTYPE\\\"\"),\n (\"\\\\UNKNOWN_LANG\\\\\", \"\\\\UNKNOWN_CTYPE\\\\\"),\n (\"\\\"UNKNOWN_LANG\", \"UNKNOWN_CTYPE\\\"\"),\n (\"\\\\UNKNOWN_LANG\", \"UNKNOWN_CTYPE\\\\\"),\n (\"\\\\\", \"\\\\\"),\n (\"\\\"\", \"\\\"\"),\n ]\n\n errorIsDetected = False\n\n for unkData in unkDatas:\n logging.info(\"----------------------\")\n logging.info(\"Unk LANG is [{0}]\".format(unkData[0]))\n logging.info(\"Unk LC_CTYPE is [{0}]\".format(unkData[1]))\n\n os.environ[\"LANG\"] = unkData[0]\n os.environ.pop(\"LANGUAGE\", None)\n os.environ[\"LC_CTYPE\"] = unkData[1]\n os.environ.pop(\"LC_COLLATE\", None)\n\n assert os.environ.get(\"LANG\") == unkData[0]\n assert \"LANGUAGE\" not in os.environ.keys()\n assert os.environ.get(\"LC_CTYPE\") == unkData[1]\n assert \"LC_COLLATE\" not in os.environ.keys()\n\n assert os.getenv('LANG') == unkData[0]\n assert os.getenv('LANGUAGE') is None\n assert os.getenv('LC_CTYPE') == unkData[1]\n assert os.getenv('LC_COLLATE') is None\n\n exc: typing.Optional[BaseException] = None\n with __class__.helper__get_node() as node:\n try:\n node.init() # IT RAISES!\n except InitNodeException as e:\n exc = e.__cause__\n assert exc is not None\n assert isinstance(exc, ExecUtilException)\n\n if exc is None:\n logging.warning(\"We expected an error!\")\n continue\n\n errorIsDetected = True\n\n assert isinstance(exc, ExecUtilException)\n\n errMsg = str(exc)\n logging.info(\"Error message is {0}: {1}\".format(type(exc).__name__, errMsg))\n\n assert \"warning: setlocale: LC_CTYPE: cannot change locale (\" + unkData[1] + \")\" in errMsg\n assert \"initdb: error: invalid locale settings; check LANG and LC_* environment variables\" in errMsg\n continue\n\n if not errorIsDetected:\n pytest.xfail(\"All the bad data are processed without errors!\")\n\n finally:\n __class__.helper__restore_envvar(\"LANG\", prev_LANG)\n __class__.helper__restore_envvar(\"LANGUAGE\", prev_LANGUAGE)\n __class__.helper__restore_envvar(\"LC_CTYPE\", prev_LC_CTYPE)\n __class__.helper__restore_envvar(\"LC_COLLATE\", prev_LC_COLLATE)\n\n def test_pg_config(self):\n # check same instances\n a = get_pg_config()\n b = get_pg_config()\n assert (id(a) == id(b))\n\n # save right before config change\n c1 = get_pg_config()\n\n # modify setting for this scope\n with scoped_config(cache_pg_config=False) as config:\n # sanity check for value\n assert not (config.cache_pg_config)\n\n # save right after config change\n c2 = get_pg_config()\n\n # check different instances after config change\n assert (id(c1) != id(c2))\n\n # check different instances\n a = get_pg_config()\n b = get_pg_config()\n assert (id(a) != id(b))\n\n @staticmethod\n def helper__get_node(name=None):\n svc = PostgresNodeServices.sm_remote\n\n assert isinstance(svc, PostgresNodeService)\n assert isinstance(svc.os_ops, testgres.OsOperations)\n assert isinstance(svc.port_manager, testgres.PortManager)\n\n return testgres.PostgresNode(\n name,\n os_ops=svc.os_ops,\n port_manager=svc.port_manager)\n\n @staticmethod\n def helper__restore_envvar(name, prev_value):\n if prev_value is None:\n os.environ.pop(name, None)\n else:\n os.environ[name] = prev_value\n\n @staticmethod\n def helper__skip_test_if_util_not_exist(name: str):\n assert type(name) is str\n if not util_exists(name):\n pytest.skip('might be missing')\n" }, { "path": "tests/test_utils.py", "content": "from .helpers.global_data import OsOpsDescr\nfrom .helpers.global_data import OsOpsDescrs\nfrom .helpers.global_data import OsOperations\n\nfrom src.utils import parse_pg_version\nfrom src.utils import get_pg_config2\nfrom src import scoped_config\n\nimport pytest\nimport typing\n\n\nclass TestUtils:\n sm_os_ops_descrs: typing.List[OsOpsDescr] = [\n OsOpsDescrs.sm_local_os_ops_descr,\n OsOpsDescrs.sm_remote_os_ops_descr\n ]\n\n @pytest.fixture(\n params=[descr.os_ops for descr in sm_os_ops_descrs],\n ids=[descr.sign for descr in sm_os_ops_descrs]\n )\n def os_ops(self, request: pytest.FixtureRequest) -> OsOperations:\n assert isinstance(request, pytest.FixtureRequest)\n assert isinstance(request.param, OsOperations)\n return request.param\n\n def test_parse_pg_version(self):\n # Linux Mint\n assert parse_pg_version(\"postgres (PostgreSQL) 15.5 (Ubuntu 15.5-1.pgdg22.04+1)\") == \"15.5\"\n # Linux Ubuntu\n assert parse_pg_version(\"postgres (PostgreSQL) 12.17\") == \"12.17\"\n # Windows\n assert parse_pg_version(\"postgres (PostgreSQL) 11.4\") == \"11.4\"\n # Macos\n assert parse_pg_version(\"postgres (PostgreSQL) 14.9 (Homebrew)\") == \"14.9\"\n\n def test_get_pg_config2(self, os_ops: OsOperations):\n assert isinstance(os_ops, OsOperations)\n\n # check same instances\n a = get_pg_config2(os_ops, None)\n b = get_pg_config2(os_ops, None)\n assert (id(a) == id(b))\n\n # save right before config change\n c1 = get_pg_config2(os_ops, None)\n\n # modify setting for this scope\n with scoped_config(cache_pg_config=False) as config:\n # sanity check for value\n assert not (config.cache_pg_config)\n\n # save right after config change\n c2 = get_pg_config2(os_ops, None)\n\n # check different instances after config change\n assert (id(c1) != id(c2))\n\n # check different instances\n a = get_pg_config2(os_ops, None)\n b = get_pg_config2(os_ops, None)\n assert (id(a) != id(b))\n" }, { "path": "tests/units/__init__.py", "content": "" }, { "path": "tests/units/exceptions/BackupException/__init__.py", "content": "" }, { "path": "tests/units/exceptions/BackupException/test_set001__constructor.py", "content": "from src.exceptions import BackupException\nfrom src.exceptions import TestgresException as testgres__TestgresException\n\n\nclass TestSet001_Constructor:\n def test_001__default(self):\n e = BackupException()\n assert type(e) is BackupException\n assert isinstance(e, testgres__TestgresException)\n assert e.source is None\n assert e.message == \"\"\n assert str(e) == \"\"\n assert repr(e) == \"BackupException()\"\n return\n\n def test_002__message(self):\n e = BackupException(message=\"abc\\n123\")\n assert type(e) is BackupException\n assert isinstance(e, testgres__TestgresException)\n assert e.source is None\n assert e.message == \"abc\\n123\"\n assert str(e) == \"abc\\n123\"\n assert repr(e) == \"BackupException(message='abc\\\\n123')\"\n return\n" }, { "path": "tests/units/exceptions/CatchUpException/__init__.py", "content": "" }, { "path": "tests/units/exceptions/CatchUpException/test_set001__constructor.py", "content": "from src.exceptions import CatchUpException\nfrom src.exceptions import TestgresException as testgres__TestgresException\n\n\nclass TestSet001_Constructor:\n def test_001__default(self):\n e = CatchUpException()\n assert type(e) is CatchUpException\n assert isinstance(e, testgres__TestgresException)\n assert e.source is None\n assert e.message == \"\"\n assert str(e) == \"\"\n assert repr(e) == \"CatchUpException()\"\n return\n\n def test_002__message(self):\n e = CatchUpException(message=\"abc\\n123\")\n assert type(e) is CatchUpException\n assert isinstance(e, testgres__TestgresException)\n assert e.source is None\n assert e.message == \"abc\\n123\"\n assert str(e) == \"abc\\n123\"\n assert repr(e) == \"CatchUpException(message='abc\\\\n123')\"\n return\n" }, { "path": "tests/units/exceptions/InitNodeException/__init__.py", "content": "" }, { "path": "tests/units/exceptions/InitNodeException/test_set001__constructor.py", "content": "from src.exceptions import InitNodeException\nfrom src.exceptions import TestgresException as testgres__TestgresException\n\n\nclass TestSet001_Constructor:\n def test_001__default(self):\n e = InitNodeException()\n assert type(e) is InitNodeException\n assert isinstance(e, testgres__TestgresException)\n assert e.source is None\n assert e.message == \"\"\n assert str(e) == \"\"\n assert repr(e) == \"InitNodeException()\"\n return\n\n def test_002__message(self):\n e = InitNodeException(message=\"abc\\n123\")\n assert type(e) is InitNodeException\n assert isinstance(e, testgres__TestgresException)\n assert e.source is None\n assert e.message == \"abc\\n123\"\n assert str(e) == \"abc\\n123\"\n assert repr(e) == \"InitNodeException(message='abc\\\\n123')\"\n return\n" }, { "path": "tests/units/exceptions/PortForException/__init__.py", "content": "" }, { "path": "tests/units/exceptions/PortForException/test_set001__constructor.py", "content": "from src.exceptions import PortForException\nfrom src.exceptions import TestgresException as testgres__TestgresException\n\n\nclass TestSet001_Constructor:\n def test_001__default(self):\n e = PortForException()\n assert type(e) is PortForException\n assert isinstance(e, testgres__TestgresException)\n assert e.source is None\n assert e.message == \"\"\n assert str(e) == \"\"\n assert repr(e) == \"PortForException()\"\n return\n\n def test_002__message(self):\n e = PortForException(message=\"abc\\n123\")\n assert type(e) is PortForException\n assert isinstance(e, testgres__TestgresException)\n assert e.source is None\n assert e.message == \"abc\\n123\"\n assert str(e) == \"abc\\n123\"\n assert repr(e) == \"PortForException(message='abc\\\\n123')\"\n return\n" }, { "path": "tests/units/exceptions/QueryException/__init__.py", "content": "" }, { "path": "tests/units/exceptions/QueryException/test_set001__constructor.py", "content": "from src.exceptions import QueryException\nfrom src.exceptions import TestgresException as testgres__TestgresException\n\n\nclass TestSet001_Constructor:\n def test_001__default(self):\n e = QueryException()\n assert type(e) is QueryException\n assert isinstance(e, testgres__TestgresException)\n assert e.source is None\n assert e.message == \"\"\n assert e.description is None\n assert e.query is None\n assert str(e) == \"\"\n assert repr(e) == \"QueryException()\"\n return\n\n def test_002__message(self):\n e = QueryException(message=\"abc\\n123\")\n assert type(e) is QueryException\n assert isinstance(e, testgres__TestgresException)\n assert e.source is None\n assert e.message == \"abc\\n123\"\n assert e.description == \"abc\\n123\"\n assert e.query is None\n assert str(e) == \"abc\\n123\"\n assert repr(e) == \"QueryException(message='abc\\\\n123')\"\n return\n\n def test_003__query(self):\n e = QueryException(query=\"cba\\n321\")\n assert type(e) is QueryException\n assert isinstance(e, testgres__TestgresException)\n assert e.source is None\n assert e.message == \"Query: cba\\n321\"\n assert e.description is None\n assert e.query == \"cba\\n321\"\n assert str(e) == \"Query: cba\\n321\"\n assert repr(e) == \"QueryException(query='cba\\\\n321')\"\n return\n\n def test_004__all(self):\n e = QueryException(message=\"mmm\", query=\"cba\\n321\")\n assert type(e) is QueryException\n assert isinstance(e, testgres__TestgresException)\n assert e.source is None\n assert e.message == \"mmm\\nQuery: cba\\n321\"\n assert e.description == \"mmm\"\n assert e.query == \"cba\\n321\"\n assert str(e) == \"mmm\\nQuery: cba\\n321\"\n assert repr(e) == \"QueryException(message='mmm', query='cba\\\\n321')\"\n return\n" }, { "path": "tests/units/exceptions/QueryTimeoutException/__init__.py", "content": "" }, { "path": "tests/units/exceptions/QueryTimeoutException/test_set001__constructor.py", "content": "from src.exceptions import QueryTimeoutException\nfrom src.exceptions import QueryException\nfrom src.exceptions import TestgresException as testgres__TestgresException\n\n\nclass TestSet001_Constructor:\n def test_001__default(self):\n e = QueryTimeoutException()\n assert type(e) is QueryTimeoutException\n assert isinstance(e, QueryException)\n assert isinstance(e, testgres__TestgresException)\n assert e.source is None\n assert e.message == \"\"\n assert e.description is None\n assert e.query is None\n assert str(e) == \"\"\n assert repr(e) == \"QueryTimeoutException()\"\n return\n\n def test_002__message(self):\n e = QueryTimeoutException(message=\"abc\\n123\")\n assert type(e) is QueryTimeoutException\n assert isinstance(e, QueryException)\n assert isinstance(e, testgres__TestgresException)\n assert e.source is None\n assert e.message == \"abc\\n123\"\n assert e.description == \"abc\\n123\"\n assert e.query is None\n assert str(e) == \"abc\\n123\"\n assert repr(e) == \"QueryTimeoutException(message='abc\\\\n123')\"\n return\n\n def test_003__query(self):\n e = QueryTimeoutException(query=\"cba\\n321\")\n assert type(e) is QueryTimeoutException\n assert isinstance(e, QueryException)\n assert isinstance(e, testgres__TestgresException)\n assert e.source is None\n assert e.message == \"Query: cba\\n321\"\n assert e.description is None\n assert e.query == \"cba\\n321\"\n assert str(e) == \"Query: cba\\n321\"\n assert repr(e) == \"QueryTimeoutException(query='cba\\\\n321')\"\n return\n\n def test_004__all(self):\n e = QueryTimeoutException(message=\"mmm\", query=\"cba\\n321\")\n assert type(e) is QueryTimeoutException\n assert isinstance(e, QueryException)\n assert isinstance(e, testgres__TestgresException)\n assert e.source is None\n assert e.message == \"mmm\\nQuery: cba\\n321\"\n assert e.description == \"mmm\"\n assert e.query == \"cba\\n321\"\n assert str(e) == \"mmm\\nQuery: cba\\n321\"\n assert repr(e) == \"QueryTimeoutException(message='mmm', query='cba\\\\n321')\"\n return\n" }, { "path": "tests/units/exceptions/StartNodeException/__init__.py", "content": "" }, { "path": "tests/units/exceptions/StartNodeException/test_set001__constructor.py", "content": "from src.exceptions import StartNodeException\nfrom src.exceptions import TestgresException as testgres__TestgresException\n\n\nclass TestSet001_Constructor:\n def test_001__default(self):\n e = StartNodeException()\n assert type(e) is StartNodeException\n assert isinstance(e, testgres__TestgresException)\n assert e.source is None\n assert e.message == \"\"\n assert e.description is None\n assert e.files is None\n assert str(e) == \"\"\n assert repr(e) == \"StartNodeException()\"\n return\n\n def test_002__message(self):\n e = StartNodeException(message=\"abc\\n123\")\n assert type(e) is StartNodeException\n assert isinstance(e, testgres__TestgresException)\n assert e.source is None\n assert e.message == \"abc\\n123\"\n assert e.description == \"abc\\n123\"\n assert e.files is None\n assert str(e) == \"abc\\n123\"\n assert repr(e) == \"StartNodeException(message='abc\\\\n123')\"\n return\n\n def test_003__files(self):\n e = StartNodeException(files=[(\"f\\n1\", b'line1\\nline2')])\n assert type(e) is StartNodeException\n assert isinstance(e, testgres__TestgresException)\n assert e.source is None\n assert e.message == \"f\\n1\\n----\\nb'line1\\\\nline2'\\n\"\n assert e.description is None\n assert e.files == [(\"f\\n1\", b'line1\\nline2')]\n assert str(e) == \"f\\n1\\n----\\nb'line1\\\\nline2'\\n\"\n assert repr(e) == \"StartNodeException(files=[('f\\\\n1', b'line1\\\\nline2')])\"\n return\n\n def test_004__all(self):\n e = StartNodeException(message=\"mmm\", files=[(\"f\\n1\", b'line1\\nline2')])\n assert type(e) is StartNodeException\n assert isinstance(e, testgres__TestgresException)\n assert e.source is None\n assert e.message == \"mmm\\nf\\n1\\n----\\nb'line1\\\\nline2'\\n\"\n assert e.description == \"mmm\"\n assert e.files == [(\"f\\n1\", b'line1\\nline2')]\n assert str(e) == \"mmm\\nf\\n1\\n----\\nb'line1\\\\nline2'\\n\"\n assert repr(e) == \"StartNodeException(message='mmm', files=[('f\\\\n1', b'line1\\\\nline2')])\"\n return\n" }, { "path": "tests/units/exceptions/TimeoutException/__init__.py", "content": "" }, { "path": "tests/units/exceptions/TimeoutException/test_set001.py", "content": "from src.exceptions import QueryTimeoutException\nfrom src.exceptions import TimeoutException\nfrom src.exceptions import QueryException\n\n\nclass TestSet001:\n def test_001__default(self):\n # It is an alias\n assert TimeoutException == QueryTimeoutException\n assert issubclass(TimeoutException, QueryException)\n return\n" }, { "path": "tests/units/exceptions/__init__.py", "content": "" }, { "path": "tests/units/impl/__init__.py", "content": "" }, { "path": "tests/units/impl/platforms/__init__.py", "content": "" }, { "path": "tests/units/impl/platforms/internal_platform_utils/InternalPlatformUtils/__init__.py", "content": "" }, { "path": "tests/units/impl/platforms/internal_platform_utils/__init__.py", "content": "" }, { "path": "tests/units/node/PostgresNode/__init__.py", "content": "" }, { "path": "tests/units/node/PostgresNode/test_setM001__start.py", "content": "from __future__ import annotations\n\nfrom tests.helpers.global_data import PostgresNodeService\nfrom tests.helpers.global_data import PostgresNodeServices\nfrom tests.helpers.global_data import OsOperations\nfrom tests.helpers.global_data import PortManager\nfrom tests.helpers.utils import Utils as HelperUtils\nfrom tests.helpers.pg_node_utils import PostgresNodeUtils as PostgresNodeTestUtils\n\nfrom src import PostgresNode\nfrom src import NodeStatus\nfrom src import NodeConnection\n\nfrom src.node import PostgresNodeLogReader\n\nimport pytest\nimport typing\nimport logging\n\n\nclass TestSet001__start:\n @pytest.fixture(\n params=PostgresNodeServices.sm_locals_and_remotes,\n ids=[descr.sign for descr in PostgresNodeServices.sm_locals_and_remotes]\n )\n def node_svc(self, request: pytest.FixtureRequest) -> PostgresNodeService:\n assert isinstance(request, pytest.FixtureRequest)\n assert isinstance(request.param, PostgresNodeService)\n assert isinstance(request.param.os_ops, OsOperations)\n assert isinstance(request.param.port_manager, PortManager)\n return request.param\n\n class tagData001:\n wait: typing.Optional[bool]\n\n def __init__(self, wait: typing.Optional[bool]):\n assert wait is None or type(wait) is bool\n self.wait = wait\n return\n\n sm_Data001: typing.List[tagData001] = [\n tagData001(None),\n tagData001(True)\n ]\n\n @pytest.fixture(\n params=sm_Data001,\n ids=[\"wait={}\".format(x.wait) for x in sm_Data001]\n )\n def data001(self, request: pytest.FixtureRequest) -> tagData001:\n assert isinstance(request, pytest.FixtureRequest)\n assert type(request.param).__name__ == \"tagData001\"\n return request.param\n\n def test_001__wait_true(\n self,\n node_svc: PostgresNodeService,\n data001: tagData001\n ):\n assert isinstance(node_svc, PostgresNodeService)\n assert type(data001) is __class__.tagData001\n assert data001.wait is None or type(data001.wait) is bool\n\n with PostgresNodeTestUtils.get_node(node_svc) as node:\n assert type(node) is PostgresNode\n node.init()\n assert not node.is_started\n assert node.status() == NodeStatus.Stopped\n\n kwargs = {}\n\n if data001.wait is not None:\n assert data001.wait == True # noqa: E712\n kwargs[\"wait\"] = data001.wait\n\n node.start(**kwargs)\n assert node.is_started\n assert node.status() == NodeStatus.Running\n\n # Internals\n assert type(node._manually_started_pm_pid) is int\n assert node._manually_started_pm_pid != 0\n assert node._manually_started_pm_pid != node._C_PM_PID__IS_NOT_DETECTED\n assert node._manually_started_pm_pid == node.pid\n return\n\n def test_002__wait_false(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n C_MAX_ATTEMPTS = 3\n\n attempt = 0\n\n while True:\n assert type(attempt) is int\n assert attempt >= 0\n assert attempt <= C_MAX_ATTEMPTS\n\n if attempt == C_MAX_ATTEMPTS:\n raise RuntimeError(\"Node is not started\")\n\n attempt += 1\n\n logging.info(\"------------- attempt #{}\".format(attempt))\n\n if attempt > 1:\n HelperUtils.PrintAndSleep(5)\n\n with PostgresNodeTestUtils.get_node(node_svc) as node:\n assert type(node) is PostgresNode\n node.init()\n assert not node.is_started\n assert node.status() == NodeStatus.Stopped\n\n node_log_reader = PostgresNodeLogReader(node, from_beginnig=False)\n node.start(wait=False)\n assert node.is_started\n assert node.status() in [NodeStatus.Stopped, NodeStatus.Running]\n\n # Internals\n assert type(node._manually_started_pm_pid) is int\n assert node._manually_started_pm_pid == node._C_PM_PID__IS_NOT_DETECTED\n\n logging.info(\"Wait for running state ...\")\n\n try:\n PostgresNodeTestUtils.wait_for_running_state(\n node=node,\n node_log_reader=node_log_reader,\n timeout=60,\n )\n except PostgresNodeTestUtils.PortConflictNodeException as e:\n logging.warning(\"Exception {}: {}\".format(\n type(e).__name__,\n e,\n ))\n continue\n\n logging.info(\"Node is running.\")\n assert node.status() == NodeStatus.Running\n return\n\n def test_003__exec_env(\n self,\n node_svc: PostgresNodeService,\n ):\n assert isinstance(node_svc, PostgresNodeService)\n\n with PostgresNodeTestUtils.get_node(node_svc) as node:\n assert type(node) is PostgresNode\n node.init()\n assert not node.is_started\n assert node.status() == NodeStatus.Stopped\n\n C_ENV_NAME = \"MYTESTVAR\"\n C_ENV_VALUE = \"abcdefg\"\n\n envs = {\n C_ENV_NAME: C_ENV_VALUE\n }\n\n node.start(exec_env=envs)\n assert node.is_started\n assert node.status() == NodeStatus.Running\n\n with node.connect(dbname=\"postgres\") as cn:\n assert type(cn) is NodeConnection\n\n cn.execute(\"CREATE TEMP TABLE cmd_out(content text);\")\n cn.commit()\n cn.execute(\"COPY cmd_out FROM PROGRAM 'bash -c \\'\\'echo ${}\\'\\'';\".format(\n C_ENV_NAME,\n ))\n cn.commit()\n recs = cn.execute(\"select content from cmd_out;\")\n assert type(recs) is list\n assert len(recs) == 1\n assert type(recs[0]) is tuple\n rec = recs[0]\n assert len(rec) == 1\n assert rec[0] == C_ENV_VALUE\n logging.info(\"Env has value [{}]. It is OK!\".find(rec[0]))\n return\n\n def test_004__params_is_None(\n self,\n node_svc: PostgresNodeService,\n ):\n assert isinstance(node_svc, PostgresNodeService)\n\n with PostgresNodeTestUtils.get_node(node_svc) as node:\n assert type(node) is PostgresNode\n node.init()\n assert not node.is_started\n assert node.status() == NodeStatus.Stopped\n\n node.start(params=None)\n assert node.is_started\n assert node.status() == NodeStatus.Running\n return\n\n def test_005__params_is_empty(\n self,\n node_svc: PostgresNodeService,\n ):\n assert isinstance(node_svc, PostgresNodeService)\n\n with PostgresNodeTestUtils.get_node(node_svc) as node:\n assert type(node) is PostgresNode\n node.init()\n assert not node.is_started\n assert node.status() == NodeStatus.Stopped\n\n node.start(params=[])\n assert node.is_started\n assert node.status() == NodeStatus.Running\n return\n" }, { "path": "tests/units/node/PostgresNode/test_setM002__start2.py", "content": "from __future__ import annotations\n\nfrom tests.helpers.global_data import PostgresNodeService\nfrom tests.helpers.global_data import PostgresNodeServices\nfrom tests.helpers.global_data import OsOperations\nfrom tests.helpers.global_data import PortManager\nfrom tests.helpers.utils import Utils as HelperUtils\nfrom tests.helpers.pg_node_utils import PostgresNodeUtils as PostgresNodeTestUtils\n\nfrom src import PostgresNode\nfrom src import NodeStatus\nfrom src import NodeConnection\n\nfrom src.node import PostgresNodeLogReader\n\nimport pytest\nimport typing\nimport logging\n\n\nclass TestSet002__start2:\n @pytest.fixture(\n params=PostgresNodeServices.sm_locals_and_remotes,\n ids=[descr.sign for descr in PostgresNodeServices.sm_locals_and_remotes]\n )\n def node_svc(self, request: pytest.FixtureRequest) -> PostgresNodeService:\n assert isinstance(request, pytest.FixtureRequest)\n assert isinstance(request.param, PostgresNodeService)\n assert isinstance(request.param.os_ops, OsOperations)\n assert isinstance(request.param.port_manager, PortManager)\n return request.param\n\n class tagData001:\n wait: typing.Optional[bool]\n\n def __init__(self, wait: typing.Optional[bool]):\n assert wait is None or type(wait) is bool\n self.wait = wait\n return\n\n sm_Data001: typing.List[tagData001] = [\n tagData001(None),\n tagData001(True)\n ]\n\n @pytest.fixture(\n params=sm_Data001,\n ids=[\"wait={}\".format(x.wait) for x in sm_Data001]\n )\n def data001(self, request: pytest.FixtureRequest) -> tagData001:\n assert isinstance(request, pytest.FixtureRequest)\n assert type(request.param).__name__ == \"tagData001\"\n return request.param\n\n def test_001__wait_true(\n self,\n node_svc: PostgresNodeService,\n data001: tagData001\n ):\n assert isinstance(node_svc, PostgresNodeService)\n assert type(data001) is __class__.tagData001\n assert data001.wait is None or type(data001.wait) is bool\n\n with PostgresNodeTestUtils.get_node(node_svc) as node:\n assert type(node) is PostgresNode\n node.init()\n assert not node.is_started\n assert node.status() == NodeStatus.Stopped\n\n kwargs = {}\n\n if data001.wait is not None:\n assert data001.wait == True # noqa: E712\n kwargs[\"wait\"] = data001.wait\n\n node.start2(**kwargs)\n assert not node.is_started\n assert node.status() == NodeStatus.Running\n\n # Internals\n assert node._manually_started_pm_pid is None\n return\n\n def test_002__wait_false(self, node_svc: PostgresNodeService):\n assert isinstance(node_svc, PostgresNodeService)\n\n C_MAX_ATTEMPTS = 3\n\n attempt = 0\n\n while True:\n assert type(attempt) is int\n assert attempt >= 0\n assert attempt <= C_MAX_ATTEMPTS\n\n if attempt == C_MAX_ATTEMPTS:\n raise RuntimeError(\"Node is not started\")\n\n logging.info(\"------------- attempt #{}\".format(attempt))\n\n if attempt > 1:\n HelperUtils.PrintAndSleep(5)\n\n with PostgresNodeTestUtils.get_node(node_svc) as node:\n assert type(node) is PostgresNode\n node.init()\n assert not node.is_started\n assert node.status() == NodeStatus.Stopped\n\n node_log_reader = PostgresNodeLogReader(node, from_beginnig=False)\n node.start2(wait=False)\n assert not node.is_started\n assert node.status() in [NodeStatus.Stopped, NodeStatus.Running]\n\n # Internals\n assert node._manually_started_pm_pid is None\n\n logging.info(\"Wait for running state ...\")\n\n try:\n PostgresNodeTestUtils.wait_for_running_state(\n node=node,\n node_log_reader=node_log_reader,\n timeout=60,\n )\n except PostgresNodeTestUtils.PortConflictNodeException as e:\n logging.warning(\"Exception {}: {}\".format(\n type(e).__name__,\n e,\n ))\n continue\n\n logging.info(\"Node is running.\")\n assert node.status() == NodeStatus.Running\n return\n\n def test_003__exec_env(\n self,\n node_svc: PostgresNodeService,\n ):\n assert isinstance(node_svc, PostgresNodeService)\n\n with PostgresNodeTestUtils.get_node(node_svc) as node:\n assert type(node) is PostgresNode\n node.init()\n assert not node.is_started\n assert node.status() == NodeStatus.Stopped\n\n C_ENV_NAME = \"MYTESTVAR\"\n C_ENV_VALUE = \"abcdefg\"\n\n envs = {\n C_ENV_NAME: C_ENV_VALUE\n }\n\n node.start2(exec_env=envs)\n assert not node.is_started\n assert node.status() == NodeStatus.Running\n\n with node.connect(dbname=\"postgres\") as cn:\n assert type(cn) is NodeConnection\n\n cn.execute(\"CREATE TEMP TABLE cmd_out(content text);\")\n cn.commit()\n cn.execute(\"COPY cmd_out FROM PROGRAM 'bash -c \\'\\'echo ${}\\'\\'';\".format(\n C_ENV_NAME,\n ))\n cn.commit()\n recs = cn.execute(\"select content from cmd_out;\")\n assert type(recs) is list\n assert len(recs) == 1\n assert type(recs[0]) is tuple\n rec = recs[0]\n assert len(rec) == 1\n assert rec[0] == C_ENV_VALUE\n logging.info(\"Env has value [{}]. It is OK!\".find(rec[0]))\n return\n\n def test_004__params_is_None(\n self,\n node_svc: PostgresNodeService,\n ):\n assert isinstance(node_svc, PostgresNodeService)\n\n with PostgresNodeTestUtils.get_node(node_svc) as node:\n assert type(node) is PostgresNode\n node.init()\n assert not node.is_started\n assert node.status() == NodeStatus.Stopped\n\n node.start(params=None)\n assert node.is_started\n assert node.status() == NodeStatus.Running\n return\n\n def test_005__params_is_empty(\n self,\n node_svc: PostgresNodeService,\n ):\n assert isinstance(node_svc, PostgresNodeService)\n\n with PostgresNodeTestUtils.get_node(node_svc) as node:\n assert type(node) is PostgresNode\n node.init()\n assert not node.is_started\n assert node.status() == NodeStatus.Stopped\n\n node.start(params=[])\n assert node.is_started\n assert node.status() == NodeStatus.Running\n return\n" }, { "path": "tests/units/node/__init__.py", "content": "" } ]