Repository: postgrespro/testgres Branch: master Commit: 722ef1bb3fa8 Files: 95 Total size: 456.7 KB Directory structure: gitextract_6f31v0iv/ ├── .coveragerc ├── .dockerignore ├── .github/ │ ├── dependabot.yml │ └── workflows/ │ ├── package-verification.yml │ └── python-publish.yml ├── .gitignore ├── .style.yapf ├── Dockerfile--altlinux_10.tmpl ├── Dockerfile--altlinux_11.tmpl ├── Dockerfile--astralinux_1_7.tmpl ├── Dockerfile--std-all.tmpl ├── Dockerfile--std.tmpl ├── Dockerfile--std2-all.tmpl ├── Dockerfile--ubuntu_24_04.tmpl ├── LICENSE ├── README.md ├── docs/ │ ├── Makefile │ ├── README.md │ └── source/ │ ├── conf.py │ ├── index.rst │ └── testgres.rst ├── pyproject.toml ├── run_tests.sh ├── run_tests2.sh ├── src/ │ ├── __init__.py │ ├── api.py │ ├── backup.py │ ├── cache.py │ ├── config.py │ ├── connection.py │ ├── consts.py │ ├── decorators.py │ ├── defaults.py │ ├── enums.py │ ├── exceptions.py │ ├── impl/ │ │ ├── internal_utils.py │ │ ├── platforms/ │ │ │ ├── internal_platform_utils.py │ │ │ ├── internal_platform_utils_factory.py │ │ │ ├── linux/ │ │ │ │ └── internal_platform_utils.py │ │ │ └── win32/ │ │ │ └── internal_platform_utils.py │ │ ├── port_manager__generic.py │ │ └── port_manager__this_host.py │ ├── logger.py │ ├── node.py │ ├── node_app.py │ ├── port_manager.py │ ├── pubsub.py │ ├── raise_error.py │ ├── standby.py │ └── utils.py └── tests/ ├── README.md ├── __init__.py ├── conftest.py ├── helpers/ │ ├── __init__.py │ ├── global_data.py │ ├── pg_node_utils.py │ ├── run_conditions.py │ └── utils.py ├── requirements.txt ├── test_config.py ├── test_conftest.py--devel ├── test_os_ops_common.py ├── test_os_ops_local.py ├── test_os_ops_remote.py ├── test_raise_error.py ├── test_testgres_common.py ├── test_testgres_local.py ├── test_testgres_remote.py ├── test_utils.py └── units/ ├── __init__.py ├── exceptions/ │ ├── BackupException/ │ │ ├── __init__.py │ │ └── test_set001__constructor.py │ ├── CatchUpException/ │ │ ├── __init__.py │ │ └── test_set001__constructor.py │ ├── InitNodeException/ │ │ ├── __init__.py │ │ └── test_set001__constructor.py │ ├── PortForException/ │ │ ├── __init__.py │ │ └── test_set001__constructor.py │ ├── QueryException/ │ │ ├── __init__.py │ │ └── test_set001__constructor.py │ ├── QueryTimeoutException/ │ │ ├── __init__.py │ │ └── test_set001__constructor.py │ ├── StartNodeException/ │ │ ├── __init__.py │ │ └── test_set001__constructor.py │ ├── TimeoutException/ │ │ ├── __init__.py │ │ └── test_set001.py │ └── __init__.py ├── impl/ │ ├── __init__.py │ └── platforms/ │ ├── __init__.py │ └── internal_platform_utils/ │ ├── InternalPlatformUtils/ │ │ └── __init__.py │ └── __init__.py └── node/ ├── PostgresNode/ │ ├── __init__.py │ ├── test_setM001__start.py │ └── test_setM002__start2.py └── __init__.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .coveragerc ================================================ [run] source=. omit=./env/* ================================================ FILE: .dockerignore ================================================ dist env venv *.egg-info logs .vscode .pytest_cache ================================================ FILE: .github/dependabot.yml ================================================ version: 2 updates: - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" open-pull-requests-limit: 5 ================================================ FILE: .github/workflows/package-verification.yml ================================================ # This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python name: Run Package Verifications on: push: branches: [ "master" ] paths-ignore: - "*.md" pull_request: branches: [ "master" ] paths-ignore: - "*.md" jobs: lint: runs-on: ubuntu-latest strategy: fail-fast: false matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - name: Checkout uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip python -m pip install flake8 flake8-pyproject ruff - name: Lint with flake8 run: | flake8 . - name: Lint with ruff run: | ruff check . build-check: runs-on: ubuntu-latest needs: lint steps: - name: Checkout uses: actions/checkout@v6 - name: Set up Python 3.12 uses: actions/setup-python@v6 with: python-version: "3.12" - name: Install build tools run: | python -m pip install --upgrade pip python -m pip install build twine - name: Build package run: | python -m build - name: Check package metadata run: | twine check dist/* test: runs-on: ubuntu-latest needs: build-check strategy: fail-fast: false matrix: include: - platform: "std2-all" python: "3.7" postgres: "17" - platform: "std2-all" python: "3.8.0" postgres: "17" - platform: "std2-all" python: "3.8" postgres: "17" - platform: "std2-all" python: "3.9" postgres: "17" - platform: "std2-all" python: "3.10" postgres: "17" - platform: "std2-all" python: "3.11" postgres: "17" - platform: "std2-all" python: "3.12" postgres: "17" - platform: "std2-all" python: "3.13" postgres: "17" - platform: "std2-all" python: "3.14" postgres: "17" - platform: "std" python: "3" postgres: "10" - platform: "std" python: "3" postgres: "11" - platform: "std" python: "3" postgres: "12" - platform: "std" python: "3" postgres: "13" - platform: "std" python: "3" postgres: "14" - platform: "std" python: "3" postgres: "15" - platform: "std" python: "3" postgres: "16" - platform: "std-all" python: "3" postgres: "17" - platform: "std-all" python: "3" postgres: "18" - platform: "ubuntu_24_04" python: "3" postgres: "17" - platform: "altlinux_10" python: "3" postgres: "17" - platform: "altlinux_11" python: "3" postgres: "17" - platform: "astralinux_1_7" python: "3" postgres: "17" env: BASE_SIGN: "${{ matrix.platform }}-py${{ matrix.python }}-pg${{ matrix.postgres }}" steps: - name: Prepare variables run: | echo "RUN_CFG__NOW=$(date +'%Y%m%d_%H%M%S')" >> $GITHUB_ENV echo "RUN_CFG__LOGS_DIR=logs-${{ env.BASE_SIGN }}" >> $GITHUB_ENV echo "RUN_CFG__DOCKER_IMAGE_NAME=tests-${{ env.BASE_SIGN }}" >> $GITHUB_ENV echo "---------- [$GITHUB_ENV]" cat $GITHUB_ENV - name: Checkout uses: actions/checkout@v6 - name: Prepare logs folder on the host run: mkdir -p "${{ env.RUN_CFG__LOGS_DIR }}" - name: Adjust logs folder permission run: chmod -R 777 "${{ env.RUN_CFG__LOGS_DIR }}" - name: Build local image ${{ matrix.alpine }} run: docker build --build-arg PG_VERSION="${{ matrix.postgres }}" --build-arg PYTHON_VERSION="${{ matrix.python }}" -t "${{ env.RUN_CFG__DOCKER_IMAGE_NAME }}" -f Dockerfile--${{ matrix.platform }}.tmpl . - name: Run run: docker run $(bash <(curl -s https://codecov.io/env)) -t -v ${{ github.workspace }}/${{ env.RUN_CFG__LOGS_DIR }}:/home/test/testgres/logs "${{ env.RUN_CFG__DOCKER_IMAGE_NAME }}" - name: Upload Logs uses: actions/upload-artifact@v7 if: always() # IT IS IMPORTANT! with: name: testgres--test_logs--${{ env.RUN_CFG__NOW }}-${{ env.BASE_SIGN }}-id${{ github.run_id }} path: "${{ env.RUN_CFG__LOGS_DIR }}/" ================================================ FILE: .github/workflows/python-publish.yml ================================================ # This workflow will upload a Python Package to PyPI when a release is created # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries # This workflow uses actions that are not certified by GitHub. # They are provided by a third-party and are governed by # separate terms of service, privacy policy, and support # documentation. name: Upload Python Package on: release: types: [published] permissions: contents: read jobs: release-build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - uses: actions/setup-python@v6 with: python-version: "3.x" - name: Build release distributions run: | # NOTE: put your own distribution build steps here. python -m pip install build python -m build - name: Upload distributions uses: actions/upload-artifact@v7 with: name: release-dists path: dist/ pypi-publish: runs-on: ubuntu-latest needs: - release-build permissions: # IMPORTANT: this permission is mandatory for trusted publishing id-token: write # Dedicated environments with protections for publishing are strongly recommended. # For more information, see: https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#deployment-protection-rules environment: name: pypi # OPTIONAL: uncomment and update to include your PyPI project URL in the deployment status: # url: https://pypi.org/p/YOURPROJECT # # ALTERNATIVE: if your GitHub Release name is the PyPI project version string # ALTERNATIVE: exactly, uncomment the following line instead: # url: https://pypi.org/project/YOURPROJECT/${{ github.event.release.name }} steps: - name: Retrieve release distributions uses: actions/download-artifact@v8 with: name: release-dists path: dist/ - name: Publish release distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: dist/ ================================================ FILE: .gitignore ================================================ *.pyc *.egg *.egg-info/ .eggs/ dist/ build/ docs/build/ logs/ env/ venv/ .coverage coverage.xml Dockerfile *~ *.swp tags ================================================ FILE: .style.yapf ================================================ [style] based_on_style = pep8 spaces_before_comment = 4 split_before_logical_operator = false column_limit=80 ================================================ FILE: Dockerfile--altlinux_10.tmpl ================================================ ARG PG_VERSION ARG PYTHON_VERSION # --------------------------------------------- base1 FROM alt:p10 AS base1 RUN apt-get update RUN apt-get install -y sudo curl ca-certificates RUN apt-get update RUN apt-get install -y openssh-server openssh-clients RUN apt-get install -y time # RUN apt-get install -y mc RUN apt-get install -y libsqlite3-devel EXPOSE 22 RUN ssh-keygen -A # --------------------------------------------- postgres FROM base1 AS base1_with_dev_tools RUN apt-get update RUN apt-get install -y git RUN apt-get install -y gcc RUN apt-get install -y make RUN apt-get install -y meson RUN apt-get install -y flex RUN apt-get install -y bison RUN apt-get install -y pkg-config RUN apt-get install -y libssl-devel RUN apt-get install -y libicu-devel RUN apt-get install -y libzstd-devel RUN apt-get install -y zlib-devel RUN apt-get install -y liblz4-devel RUN apt-get install -y libzstd-devel RUN apt-get install -y libxml2-devel # --------------------------------------------- postgres FROM base1_with_dev_tools AS base1_with_pg-17 RUN git clone https://github.com/postgres/postgres.git -b REL_17_STABLE /pg/postgres/source WORKDIR /pg/postgres/source RUN ./configure --prefix=/pg/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml RUN make -j 4 install RUN make -j 4 -C contrib install # SETUP PG_CONFIG # When pg_config symlink in /usr/local/bin it returns a real (right) result of --bindir RUN ln -s /pg/postgres/install/bin/pg_config -t /usr/local/bin # SETUP PG CLIENT LIBRARY # libpq.so.5 is enough RUN ln -s /pg/postgres/install/lib/libpq.so.5.17 /usr/lib64/libpq.so.5 # --------------------------------------------- base2_with_python-3 FROM base1_with_pg-${PG_VERSION} AS base2_with_python-3 RUN apt-get install -y python3 RUN apt-get install -y python3-dev RUN apt-get install -y python3-modules-sqlite3 ENV PYTHON_BINARY=python3 # --------------------------------------------- final FROM base2_with_python-${PYTHON_VERSION} AS final RUN adduser test -G wheel # It enables execution of "sudo service ssh start" without password RUN echo "test ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers ADD --chown=test:test . /home/test/testgres WORKDIR /home/test/testgres RUN mkdir /home/test/testgres/logs RUN chown -R test:test /home/test/testgres/logs ENV LANG=C.UTF-8 USER test RUN chmod 700 ~/ RUN mkdir -p ~/.ssh # # Altlinux 10 and 11 too slowly create a new SSH connection (x6). # ENTRYPOINT sh -c " \ set -eux; \ echo HELLO FROM ENTRYPOINT; \ echo HOME DIR IS [`realpath ~/`]; \ ls -la .; \ sudo /usr/sbin/sshd; \ sudo chmod 777 /home/test/testgres/logs; \ ls -la . | grep logs; \ ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ chmod 600 ~/.ssh/authorized_keys; \ ls -la ~/.ssh/; \ TEST_FILTER=\"\" bash ./run_tests.sh;" ================================================ FILE: Dockerfile--altlinux_11.tmpl ================================================ ARG PG_VERSION ARG PYTHON_VERSION # --------------------------------------------- base1 FROM alt:p11 AS base1 RUN apt-get update RUN apt-get install -y sudo curl ca-certificates RUN apt-get update RUN apt-get install -y openssh-server openssh-clients RUN apt-get install -y time # pgrep (for testgres.os_ops) RUN apt-get install -y procps # RUN apt-get install -y mc RUN apt-get install -y libsqlite3-devel EXPOSE 22 RUN ssh-keygen -A # --------------------------------------------- postgres FROM base1 AS base1_with_dev_tools RUN apt-get update RUN apt-get install -y git RUN apt-get install -y gcc RUN apt-get install -y make RUN apt-get install -y meson RUN apt-get install -y flex RUN apt-get install -y bison RUN apt-get install -y pkg-config RUN apt-get install -y libssl-devel RUN apt-get install -y libicu-devel RUN apt-get install -y libzstd-devel RUN apt-get install -y zlib-devel RUN apt-get install -y liblz4-devel RUN apt-get install -y libzstd-devel RUN apt-get install -y libxml2-devel # --------------------------------------------- postgres FROM base1_with_dev_tools AS base1_with_pg-17 RUN git clone https://github.com/postgres/postgres.git -b REL_17_STABLE /pg/postgres/source WORKDIR /pg/postgres/source RUN ./configure --prefix=/pg/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml RUN make -j 4 install RUN make -j 4 -C contrib install # SETUP PG_CONFIG # When pg_config symlink in /usr/local/bin it returns a real (right) result of --bindir RUN ln -s /pg/postgres/install/bin/pg_config -t /usr/local/bin # SETUP PG CLIENT LIBRARY # libpq.so.5 is enough RUN ln -s /pg/postgres/install/lib/libpq.so.5.17 /usr/lib64/libpq.so.5 # --------------------------------------------- base2_with_python-3 FROM base1_with_pg-${PG_VERSION} AS base2_with_python-3 RUN apt-get install -y python3 RUN apt-get install -y python3-dev RUN apt-get install -y python3-modules-sqlite3 ENV PYTHON_BINARY=python3 # --------------------------------------------- final FROM base2_with_python-${PYTHON_VERSION} AS final RUN adduser test -G wheel # It enables execution of "sudo service ssh start" without password RUN echo "test ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers ADD --chown=test:test . /home/test/testgres WORKDIR /home/test/testgres RUN mkdir /home/test/testgres/logs RUN chown -R test:test /home/test/testgres/logs ENV LANG=C.UTF-8 USER test RUN chmod 700 ~/ RUN mkdir -p ~/.ssh # # Altlinux 10 and 11 too slowly create a new SSH connection (x6). # ENTRYPOINT sh -c " \ set -eux; \ echo HELLO FROM ENTRYPOINT; \ echo HOME DIR IS [`realpath ~/`]; \ ls -la .; \ sudo /usr/sbin/sshd; \ sudo chmod 777 /home/test/testgres/logs; \ ls -la . | grep logs; \ ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ chmod 600 ~/.ssh/authorized_keys; \ ls -la ~/.ssh/; \ TEST_FILTER=\"\" bash ./run_tests.sh;" ================================================ FILE: Dockerfile--astralinux_1_7.tmpl ================================================ ARG PG_VERSION ARG PYTHON_VERSION # --------------------------------------------- base1 FROM packpack/packpack:astra-1.7 AS base1 RUN apt update RUN apt install -y sudo curl ca-certificates RUN apt update RUN apt install -y openssh-server RUN apt install -y time # RUN apt install -y netcat-traditional RUN apt install -y git # --------------------------------------------- postgres FROM base1 AS base1_with_dev_tools RUN apt-get update RUN apt-get install -y git RUN apt-get install -y gcc RUN apt-get install -y make RUN apt-get install -y meson RUN apt-get install -y flex RUN apt-get install -y bison RUN apt-get install -y pkg-config RUN apt-get install -y libssl-dev RUN apt-get install -y libicu-dev RUN apt-get install -y libzstd-dev RUN apt-get install -y zlib1g-dev RUN apt-get install -y liblz4-dev RUN apt-get install -y libzstd-dev RUN apt-get install -y libxml2-dev # --------------------------------------------- postgres FROM base1_with_dev_tools AS base1_with_pg-17 RUN curl -fsSL https://ftp.postgresql.org/pub/source/v17.7/postgresql-17.7.tar.bz2 -o postgresql.tar.bz2 \ && mkdir -p /pg/postgres/source \ && tar -xjf postgresql.tar.bz2 -C /pg/postgres/source --strip-components=1 \ && rm postgresql.tar.bz2 WORKDIR /pg/postgres/source RUN ./configure --prefix=/pg/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml RUN make -j 4 install RUN make -j 4 -C contrib install # SETUP PG_CONFIG # When pg_config symlink in /usr/local/bin it returns a real (right) result of --bindir RUN ln -s /pg/postgres/install/bin/pg_config -t /usr/local/bin # SETUP PG CLIENT LIBRARY # libpq.so.5 is enough RUN ln -s /pg/postgres/install/lib/libpq.so.5.17 /usr/lib/libpq.so.5 # --------------------------------------------- base2_with_python-3 FROM base1_with_pg-${PG_VERSION} AS base2_with_python-3 RUN apt install -y python3 python3-dev python3-venv ENV PYTHON_BINARY=python3 # --------------------------------------------- final FROM base2_with_python-${PYTHON_VERSION} AS final EXPOSE 22 RUN ssh-keygen -A RUN useradd -m test # It enables execution of "sudo service ssh start" without password # MY OLD: # RUN sh -c "echo test ALL=NOPASSWD:ALL" >> /etc/sudoers # AI: RUN echo "test ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers # THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD RUN sh -c "echo "test:*" | chpasswd -e" RUN sed -i 's/UsePAM yes/UsePAM no/' /etc/ssh/sshd_config ADD --chown=test:test . /home/test/testgres WORKDIR /home/test/testgres RUN mkdir /home/test/testgres/logs RUN chown -R test:test /home/test/testgres/logs ENV LANG=C.UTF-8 USER test RUN chmod 700 ~/ RUN mkdir -p ~/.ssh RUN chmod 700 ~/.ssh ENTRYPOINT sh -c " \ set -eux; \ echo HELLO FROM ENTRYPOINT; \ echo HOME DIR IS [`realpath ~/`]; \ ls -la .; \ sudo service ssh start; \ sudo chmod 777 /home/test/testgres/logs; \ ls -la . | grep logs; \ ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ chmod 600 ~/.ssh/authorized_keys; \ ls -la ~/.ssh/; \ TEST_FILTER=\"\" bash ./run_tests.sh;" ================================================ FILE: Dockerfile--std-all.tmpl ================================================ ARG PG_VERSION ARG PYTHON_VERSION # --------------------------------------------- base1 FROM postgres:${PG_VERSION}-alpine AS base1 # --------------------------------------------- base2_with_python-3 FROM base1 AS base2_with_python-3 RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers ENV PYTHON_BINARY=python3 # --------------------------------------------- final FROM base2_with_python-${PYTHON_VERSION} AS final #RUN apk add --no-cache mc RUN apk add --no-cache git # Full version of "ps" command RUN apk add --no-cache procps RUN apk add --no-cache openssh RUN apk add --no-cache sudo ENV LANG=C.UTF-8 RUN addgroup -S sudo RUN adduser -D test RUN addgroup test sudo EXPOSE 22 RUN ssh-keygen -A ADD --chown=test:test . /home/test/testgres WORKDIR /home/test/testgres RUN mkdir /home/test/testgres/logs RUN chown -R test:test /home/test/testgres/logs # It allows to use sudo without password RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers # THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD RUN echo "test:*" | chpasswd -e USER test # THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD RUN chmod 700 ~/ RUN mkdir -p ~/.ssh #RUN chmod 700 ~/.ssh #ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh ENTRYPOINT sh -c " \ set -eux; \ echo HELLO FROM ENTRYPOINT; \ echo HOME DIR IS [`realpath ~/`]; \ ls -la .; \ ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ chmod 600 ~/.ssh/authorized_keys; \ ls -la ~/.ssh/; \ sudo /usr/sbin/sshd; \ sudo chmod 777 /home/test/testgres/logs; \ ls -la . | grep logs; \ ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ TEST_FILTER=\"\" bash run_tests.sh;" ================================================ FILE: Dockerfile--std.tmpl ================================================ ARG PG_VERSION ARG PYTHON_VERSION # --------------------------------------------- base1 FROM postgres:${PG_VERSION}-alpine AS base1 # --------------------------------------------- base2_with_python-3 FROM base1 AS base2_with_python-3 RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers ENV PYTHON_BINARY=python3 # --------------------------------------------- final FROM base2_with_python-${PYTHON_VERSION} AS final RUN apk add --no-cache git RUN apk add --no-cache sudo ENV LANG=C.UTF-8 RUN addgroup -S sudo RUN adduser -D test RUN addgroup test sudo ADD --chown=test:test . /home/test/testgres WORKDIR /home/test/testgres RUN mkdir /home/test/testgres/logs RUN chown -R test:test /home/test/testgres/logs # It allows to use sudo without password RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers USER test ENTRYPOINT sh -c " \ set -eux; \ echo HELLO FROM ENTRYPOINT; \ echo HOME DIR IS [`realpath ~/`]; \ ls -la .; \ sudo chmod 777 /home/test/testgres/logs; \ ls -la . | grep logs; \ bash run_tests.sh;" ================================================ FILE: Dockerfile--std2-all.tmpl ================================================ ARG PG_VERSION ARG PYTHON_VERSION # --------------------------------------------- base1 FROM postgres:${PG_VERSION}-alpine AS base1 # --------------------------------------------- base2_with_python-3 FROM base1 AS base2_with_python-3 ENV PYTHON_BINARY=python3 RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers # For pyenv RUN apk add patch RUN apk add git RUN apk add xz-dev RUN apk add zip RUN apk add zlib-dev RUN apk add libffi-dev RUN apk add readline-dev RUN apk add openssl openssl-dev RUN apk add sqlite-dev RUN apk add bzip2-dev # --------------------------------------------- base3_with_python-3.7 FROM base2_with_python-3 AS base3_with_python-3.7 ENV PYTHON_VERSION=3.7 # --------------------------------------------- base3_with_python-3.8.0 FROM base2_with_python-3 AS base3_with_python-3.8.0 ENV PYTHON_VERSION=3.8.0 # --------------------------------------------- base3_with_python-3.8 FROM base2_with_python-3 AS base3_with_python-3.8 ENV PYTHON_VERSION=3.8 # --------------------------------------------- base3_with_python-3.9 FROM base2_with_python-3 AS base3_with_python-3.9 ENV PYTHON_VERSION=3.9 # --------------------------------------------- base3_with_python-3.10 FROM base2_with_python-3 AS base3_with_python-3.10 ENV PYTHON_VERSION=3.10 # --------------------------------------------- base3_with_python-3.11 FROM base2_with_python-3 AS base3_with_python-3.11 ENV PYTHON_VERSION=3.11 # --------------------------------------------- base3_with_python-3.12 FROM base2_with_python-3 AS base3_with_python-3.12 ENV PYTHON_VERSION=3.12 # --------------------------------------------- base3_with_python-3.13 FROM base2_with_python-3 AS base3_with_python-3.13 ENV PYTHON_VERSION=3.13 # --------------------------------------------- base3_with_python-3.14 FROM base2_with_python-3 AS base3_with_python-3.14 ENV PYTHON_VERSION=3.14 # --------------------------------------------- final FROM base3_with_python-${PYTHON_VERSION} AS final #RUN apk add --no-cache mc # Full version of "ps" command RUN apk add --no-cache procps RUN apk add --no-cache openssh RUN apk add --no-cache sudo ENV LANG=C.UTF-8 RUN addgroup -S sudo RUN adduser -D test RUN addgroup test sudo EXPOSE 22 RUN ssh-keygen -A # It allows to use sudo without password RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers # THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD RUN echo "test:*" | chpasswd -e USER test RUN curl https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash RUN ~/.pyenv/bin/pyenv install ${PYTHON_VERSION} ADD --chown=test:test . /home/test/testgres WORKDIR /home/test/testgres RUN mkdir /home/test/testgres/logs RUN chown -R test:test /home/test/testgres/logs # THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD RUN chmod 700 ~/ RUN mkdir -p ~/.ssh ENTRYPOINT sh -c " \ set -eux; \ echo HELLO FROM ENTRYPOINT; \ echo HOME DIR IS [`realpath ~/`]; \ ls -la .; \ ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ chmod 600 ~/.ssh/authorized_keys; \ ls -la ~/.ssh/; \ sudo /usr/sbin/sshd; \ sudo chmod 777 /home/test/testgres/logs; \ ls -la . | grep logs; \ ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ export PATH=\"~/.pyenv/bin:$PATH\"; \ TEST_FILTER=\"\" bash run_tests2.sh;" ================================================ FILE: Dockerfile--ubuntu_24_04.tmpl ================================================ ARG PG_VERSION ARG PYTHON_VERSION # --------------------------------------------- base1 FROM ubuntu:24.04 AS base1 ARG PG_VERSION RUN apt update RUN apt install -y sudo curl ca-certificates RUN apt update RUN apt install -y openssh-server RUN apt install -y time RUN apt install -y netcat-traditional RUN apt install -y git RUN apt update RUN apt install -y postgresql-common RUN bash /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y RUN install -d /usr/share/postgresql-common/pgdg RUN curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc RUN apt update RUN apt install -y postgresql-${PG_VERSION} # --------------------------------------------- base2_with_python-3 FROM base1 AS base2_with_python-3 RUN apt install -y python3 python3-dev python3-venv libpq-dev build-essential ENV PYTHON_BINARY=python3 # --------------------------------------------- final FROM base2_with_python-${PYTHON_VERSION} AS final EXPOSE 22 RUN ssh-keygen -A RUN adduser test RUN chown postgres:postgres /var/run/postgresql RUN chmod 775 /var/run/postgresql RUN usermod -aG postgres test # It enables execution of "sudo service ssh start" without password # RUN echo "test ALL=NOPASSWD:/usr/sbin/service ssh start" >> /etc/sudoers # RUN echo "test ALL=NOPASSWD:ALL" >> /etc/sudoers RUN echo "test ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers ADD --chown=test:test . /home/test/testgres WORKDIR /home/test/testgres RUN mkdir /home/test/testgres/logs RUN chown -R test:test /home/test/testgres/logs ENV LANG=C.UTF-8 USER test RUN chmod 700 ~/ RUN mkdir -p ~/.ssh ENTRYPOINT sh -c " \ #set -eux; \ echo HELLO FROM ENTRYPOINT; \ echo HOME DIR IS [`realpath ~/`]; \ ls -la .; \ service ssh enable; \ sudo service ssh start; \ sudo chmod 777 /home/test/testgres/logs; \ ls -la . | grep logs; \ ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ chmod 600 ~/.ssh/authorized_keys; \ ls -la ~/.ssh/; \ TEST_FILTER=\"\" bash ./run_tests.sh;" ================================================ FILE: LICENSE ================================================ testgres is released under the PostgreSQL License, a liberal Open Source license, similar to the BSD or MIT licenses. Copyright (c) 2016-2026, Postgres Professional Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this paragraph and the following two paragraphs appear in all copies. IN NO EVENT SHALL POSTGRES PROFESSIONAL BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF POSTGRES PROFESSIONAL HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. POSTGRES PROFESSIONAL SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND POSTGRES PROFESSIONAL HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. ================================================ FILE: README.md ================================================ [![CI Status](https://img.shields.io/github/actions/workflow/status/postgrespro/testgres/.github/workflows/package-verification.yml?label=CI)](https://github.com/postgrespro/testgres/actions/workflows/package-verification.yml) [![codecov](https://codecov.io/gh/postgrespro/testgres/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/testgres) [![PyPI package version](https://badge.fury.io/py/testgres.svg)](https://badge.fury.io/py/testgres) [![PyPI python versions](https://img.shields.io/pypi/pyversions/testgres)](https://pypi.org/project/testgres) [![PyPI downloads](https://img.shields.io/pypi/dm/testgres)](https://pypi.org/project/testgres) [Documentation](https://postgrespro.github.io/testgres/) # testgres Utility for orchestrating temporary PostgreSQL clusters in Python tests. Supports Python 3.7.3 and newer. ## Installation Install `testgres` from PyPI: ```sh pip install testgres ``` Use a dedicated virtual environment for isolated test dependencies. ## Usage ### Environment > Note: by default `testgres` invokes `initdb`, `pg_ctl`, and `psql` binaries found in `PATH`. Specify a custom PostgreSQL installation in one of the following ways: - Set the `PG_CONFIG` environment variable to point to the `pg_config` executable. - Set the `PG_BIN` environment variable to point to the directory with PostgreSQL binaries. Example: ```sh export PG_BIN=$HOME/pg_16/bin python my_tests.py ``` ### Examples Create a temporary node, run queries, and let `testgres` clean up automatically: ```python # create a node with a random name, port, and data directory with testgres.get_new_node() as node: # run initdb node.init() # start PostgreSQL node.start() # execute a query in the default database print(node.execute('select 1')) # the node is stopped and its files are removed automatically ``` ### Query helpers `testgres` provides four helpers for executing queries against the node: | Command | Description | |---------|-------------| | `node.psql(query, ...)` | Runs the query via `psql` and returns a tuple `(returncode, stdout, stderr)`. | | `node.safe_psql(query, ...)` | Same as `psql()` but returns only `stdout` and raises if the command fails. | | `node.execute(query, ...)` | Connects via `psycopg2` or `pg8000` (whichever is available) and returns a list of tuples. | | `node.connect(dbname, ...)` | Returns a `NodeConnection` wrapper for executing multiple statements within a transaction. | Example of transactional usage: ```python with node.connect() as con: con.begin('serializable') print(con.execute('select %s', 1)) con.rollback() ``` ### Logging By default `cleanup()` removes all temporary files (data directories, logs, and so on) created by the API. Call `configure_testgres(node_cleanup_full=False)` before starting nodes if you want to keep logs for inspection. > Note: context managers (the `with` statement) call `stop()` and `cleanup()` automatically. `testgres` integrates with the standard [Python logging](https://docs.python.org/3/library/logging.html) module, so you can aggregate logs from multiple nodes: ```python import logging # write everything to /tmp/testgres.log logging.basicConfig(filename='/tmp/testgres.log') # enable logging and create two nodes testgres.configure_testgres(use_python_logging=True) node1 = testgres.get_new_node().init().start() node2 = testgres.get_new_node().init().start() node1.execute('select 1') node2.execute('select 2') # disable logging testgres.configure_testgres(use_python_logging=False) ``` See `tests/test_simple.py` for a complete logging example. ### Backup and replication Creating backups and spawning replicas is straightforward: ```python with testgres.get_new_node('master') as master: master.init().start() with master.backup() as backup: replica = backup.spawn_replica('replica').start() replica.catchup() print(replica.execute('postgres', 'select 1')) ``` ### Benchmarks Use `pgbench` through `testgres` to run quick benchmarks: ```python with testgres.get_new_node('master') as master: master.init().start() result = master.pgbench_init(scale=2).pgbench_run(time=10) print(result) ``` ### Custom configuration `testgres` ships with sensible defaults. Adjust them as needed with `default_conf()` and `append_conf()`: ```python extra_conf = "shared_preload_libraries = 'postgres_fdw'" with testgres.get_new_node().init() as master: master.default_conf(fsync=True, allow_streaming=True) master.append_conf('postgresql.conf', extra_conf) ``` `default_conf()` is called by `init()` and rewrites the configuration file. Apply `append_conf()` afterwards to keep custom lines. ### Remote mode You can provision nodes on a remote host (Linux only) by wiring `RemoteOperations` into the configuration: ```python from testgres import ConnectionParams, RemoteOperations, TestgresConfig, get_remote_node conn_params = ConnectionParams( host='example.com', username='postgres', ssh_key='/path/to/ssh/key' ) os_ops = RemoteOperations(conn_params) TestgresConfig.set_os_ops(os_ops=os_ops) def test_basic_query(): with get_remote_node(conn_params=conn_params) as node: node.init().start() assert node.execute('SELECT 1') == [(1,)] ``` ### Pytest integration Use fixtures to create and clean up nodes automatically when testing with `pytest`: ```python import pytest import testgres @pytest.fixture def pg_node(): node = testgres.get_new_node().init().start() try: yield node finally: node.stop() node.cleanup() def test_simple(pg_node): assert pg_node.execute('select 1')[0][0] == 1 ``` This pattern keeps tests concise and ensures that every node is stopped and removed even if the test fails. ### Scaling tips - Run tests in parallel with `pytest -n auto` (requires `pytest-xdist`). Ensure each node uses a distinct port by setting `PGPORT` in the fixture or by passing the `port` argument to `get_new_node()`. - Always call `node.cleanup()` after each test, or rely on context managers/fixtures that do it for you, to avoid leftover data directories. - Prefer `node.safe_psql()` for lightweight assertions that should fail fast; use `node.execute()` when you need structured Python results. ## Authors [Ildar Musin](https://github.com/zilder) [Dmitry Ivanov](https://github.com/funbringer) [Ildus Kurbangaliev](https://github.com/ildus) [Yury Zhuravlev](https://github.com/stalkerg) ================================================ FILE: docs/Makefile ================================================ # Minimal makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build SPHINXPROJ = testgres SOURCEDIR = source BUILDDIR = build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @pip install --force-reinstall .. @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) ================================================ FILE: docs/README.md ================================================ # Building the documentation Make sure you have `Sphinx` package installed: ``` pip install Sphinx ``` Then just run ``` make html ``` Documentation will be built in `build/html` directory. Other output formats are also available; run `make` without arguments to see the options. ================================================ FILE: docs/source/conf.py ================================================ # -*- coding: utf-8 -*- # # Configuration file for the Sphinx documentation builder. # # This file does only contain a selection of the most common options. For a # full list see the documentation: # http://www.sphinx-doc.org/en/stable/config # -- Path setup -------------------------------------------------------------- # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # import os import sys import testgres assert testgres.__path__ is not None assert len(testgres.__path__) == 1 assert type(testgres.__path__[0]) is str p = os.path.dirname(testgres.__path__[0]) assert type(p) is str sys.path.insert(0, os.path.abspath(p)) # -- Project information ----------------------------------------------------- project = u'testgres' package_name = u'testgres' copyright = u'2016-2026, Postgres Professional' author = u'Postgres Professional' # The full version, including alpha/beta/rc tags release = testgres.__version__ # The short X.Y version version = '.'.join(release.split('.')[:2]) # -- General configuration --------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. # # needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] source_suffix = '.rst' # The master toctree document. master_doc = 'index' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path . exclude_patterns = [] # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = 'alabaster' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # # html_theme_options = {} # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = [] # Custom sidebar templates, must be a dictionary that maps document names # to template names. # # The default sidebars (for documents that don't match any pattern) are # defined by theme itself. Builtin themes are using these templates by # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', # 'searchbox.html']``. # # html_sidebars = {} # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. htmlhelp_basename = 'testgresdoc' # -- Options for LaTeX output ------------------------------------------------ latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # # 'preamble': '', # Latex figure (float) alignment # # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, 'testgres.tex', u'testgres Documentation', u'Postgres Professional', 'manual'), ] # -- Options for manual page output ------------------------------------------ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [(master_doc, 'testgres', u'testgres Documentation', [author], 1)] # -- Options for Texinfo output ---------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ (master_doc, 'testgres', u'testgres Documentation', author, 'testgres', 'One line description of project.', 'Miscellaneous'), ] # -- Extension configuration ------------------------------------------------- ================================================ FILE: docs/source/index.rst ================================================ Testgres documentation ====================== Utility for orchestrating temporary PostgreSQL clusters in Python tests. Supports Python 3.7.17 and newer. Installation ============ To install testgres, run: .. code-block:: bash pip install testgres We encourage you to use ``virtualenv`` for your testing environment. Usage ===== Environment ----------- Note: by default ``testgres`` runs ``initdb``, ``pg_ctl``, and ``psql`` found in ``PATH``. There are several ways to specify a custom PostgreSQL installation: - export ``PG_CONFIG`` environment variable pointing to the ``pg_config`` executable; - export ``PG_BIN`` environment variable pointing to the directory with executable files. Example: .. code-block:: bash export PG_BIN=$HOME/pg_16/bin python my_tests.py Examples -------- Create a temporary node, run queries, and let ``testgres`` clean up automatically: .. code-block:: python # create a node with a random name, port, and data directory with testgres.get_new_node() as node: # run initdb node.init() # start PostgreSQL node.start() # execute a query in the default database print(node.execute('select 1')) # the node is stopped and its files are removed automatically Query helpers ------------- ``testgres`` provides four helpers for executing queries against the node: ========================== ======================================================= Command Description ========================== ======================================================= ``node.psql(query, ...)`` Runs the query via ``psql`` and returns ``(code, out, err)``. ``node.safe_psql(...)`` Returns only ``stdout`` and raises if the command fails. ``node.execute(...)`` Uses ``psycopg2``/``pg8000`` and returns a list of tuples. ``node.connect(...)`` Returns a ``NodeConnection`` for transactional usage. ========================== ======================================================= Example: .. code-block:: python with node.connect() as con: con.begin('serializable') print(con.execute('select %s', 1)) con.rollback() Logging ------- By default ``cleanup()`` removes all temporary files (data directories, logs, and so on) created by the API. Call ``configure_testgres(node_cleanup_full=False)`` before starting nodes if you want to keep logs for inspection. Note: context managers (the ``with`` statement) call ``stop()`` and ``cleanup()`` automatically. ``testgres`` integrates with the standard `Python logging `_ module, so you can aggregate logs from multiple nodes: .. code-block:: python import logging logging.basicConfig(filename='/tmp/testgres.log') testgres.configure_testgres(use_python_logging=True) node1 = testgres.get_new_node().init().start() node2 = testgres.get_new_node().init().start() node1.execute('select 1') node2.execute('select 2') testgres.configure_testgres(use_python_logging=False) Backup & replication -------------------- It's quite easy to create a backup and start a new replica: .. code-block:: python with testgres.get_new_node('master') as master: master.init().start() # create a backup with master.backup() as backup: # create and start a new replica replica = backup.spawn_replica('replica').start() replica.catchup() print(replica.execute('postgres', 'select 1')) Benchmarks ---------- Use ``pgbench`` through ``testgres`` to run quick benchmarks: .. code-block:: python with testgres.get_new_node('master') as master: master.init().start() result = master.pgbench_init(scale=2).pgbench_run(time=10) print(result) Custom configuration -------------------- ``testgres`` ships with sensible defaults. Adjust them as needed with ``default_conf()`` and ``append_conf()``: .. code-block:: python extra_conf = "shared_preload_libraries = 'postgres_fdw'" with testgres.get_new_node().init() as master: master.default_conf(fsync=True, allow_streaming=True) master.append_conf('postgresql.conf', extra_conf) ``default_conf()`` is called by ``init()`` and rewrites the configuration file. Apply ``append_conf()`` afterwards to keep custom lines. Remote mode ----------- Provision nodes on a remote host (Linux only) by wiring ``RemoteOperations`` into the configuration: .. code-block:: python from testgres import ConnectionParams, RemoteOperations, TestgresConfig, get_remote_node conn_params = ConnectionParams( host='example.com', username='postgres', ssh_key='/path/to/ssh/key' ) os_ops = RemoteOperations(conn_params) TestgresConfig.set_os_ops(os_ops=os_ops) def test_basic_query(): with get_remote_node(conn_params=conn_params) as node: node.init().start() assert node.execute('SELECT 1') == [(1,)] Pytest integration ------------------ Use fixtures to create and clean up nodes automatically when testing with ``pytest``: .. code-block:: python import pytest import testgres @pytest.fixture def pg_node(): node = testgres.get_new_node().init().start() try: yield node finally: node.stop() node.cleanup() def test_simple(pg_node): assert pg_node.execute('select 1')[0][0] == 1 Scaling tips ------------ * Run tests in parallel with ``pytest -n auto`` (requires ``pytest-xdist``). Set unique ports by passing ``port`` to ``get_new_node()`` or exporting ``PGPORT`` in the fixture. * Always call ``node.cleanup()`` after each test, or rely on context managers/fixtures that do it for you, to avoid leftover data directories. * Prefer ``safe_psql()`` for quick assertions, and ``execute()`` when you need Python data structures. Modules ======= .. toctree:: :maxdepth: 2 testgres .. Indices and tables .. ================== .. * :ref:`genindex` .. * :ref:`modindex` .. * :ref:`search` ================================================ FILE: docs/source/testgres.rst ================================================ testgres package ================ testgres.api ------------ .. automodule:: testgres.api :members: :undoc-members: :show-inheritance: testgres.backup --------------- .. automodule:: testgres.backup :members: :undoc-members: :show-inheritance: testgres.config --------------- .. automodule:: testgres.config :members: :undoc-members: :show-inheritance: :member-order: groupwise testgres.connection ------------------- .. automodule:: testgres.connection :members: :undoc-members: :show-inheritance: testgres.enums -------------- .. automodule:: testgres.enums :members: :undoc-members: :show-inheritance: testgres.exceptions ------------------- .. automodule:: testgres.exceptions :members: :undoc-members: :show-inheritance: testgres.node ------------- .. autoclass:: testgres.node.PostgresNode :members: .. automethod:: __init__ .. autoclass:: testgres.node.ProcessProxy :members: testgres.standby ---------------- .. automodule:: testgres.standby :members: :undoc-members: :show-inheritance: testgres.pubsub --------------- .. automodule:: testgres.pubsub .. autoclass:: testgres.node.Publication :members: .. automethod:: __init__ .. autoclass:: testgres.node.Subscription :members: .. automethod:: __init__ ================================================ FILE: pyproject.toml ================================================ [build-system] requires = ["setuptools>=61.0"] build-backend = "setuptools.build_meta" [tool.setuptools.package-dir] "testgres" = "src" [tool.setuptools.dynamic] version = {attr = "testgres.__version__"} [tool.flake8] extend-ignore = ["E501"] exclude = [".git", "__pycache__", "env", "venv"] # Pytest settings [tool.pytest.ini_options] testpaths = ["tests"] log_file_level = "NOTSET" log_file_format = "%(levelname)8s [%(asctime)s] %(message)s" log_file_date_format = "%Y-%m-%d %H:%M:%S" [project] name = "testgres" dynamic = ["version"] description = "Testing utility for PostgreSQL and its extensions" readme = "README.md" # [2026-01-05] # This old format is used to ensure compatibility with Python 3.7. license = {text = "PostgreSQL"} authors = [ {name = "Postgres Professional", email = "testgres@postgrespro.ru"}, ] keywords = [ 'test', 'testing', 'postgresql', ] requires-python = ">=3.7.3" classifiers = [ "Intended Audience :: Developers", "Operating System :: Unix", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Topic :: Software Development :: Libraries", "Topic :: Software Development :: Testing", ] dependencies = [ "pg8000", "port-for>=0.4", "six>=1.9.0", "psutil", "packaging", "testgres.os_ops>=2.1.0,<3.0.0", ] [project.urls] "HomePage" = "https://github.com/postgrespro/testgres" ================================================ FILE: run_tests.sh ================================================ #!/usr/bin/env bash set -eux if [ -z ${TEST_FILTER+x} ]; \ then export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote))"; \ fi # fail early echo check that pg_config is in PATH command -v pg_config # prepare python environment VENV_PATH="/tmp/testgres_venv" rm -rf $VENV_PATH ${PYTHON_BINARY} -m venv "${VENV_PATH}" export VIRTUAL_ENV_DISABLE_PROMPT=1 source "${VENV_PATH}/bin/activate" pip install --upgrade pip setuptools wheel pip install -r tests/requirements.txt # remove existing coverage file export COVERAGE_FILE=.coverage rm -f $COVERAGE_FILE pip install coverage # run tests (PATH) time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" # run tests (PG_BIN) PG_BIN=$(pg_config --bindir) \ time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" # run tests (PG_CONFIG) PG_CONFIG=$(pg_config --bindir)/pg_config \ time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" # test pg8000 pip uninstall -y psycopg2 pip install pg8000 PG_CONFIG=$(pg_config --bindir)/pg_config \ time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" # show coverage coverage report pip uninstall -y coverage # build documentation pip install Sphinx cd docs make html cd .. pip uninstall -y Sphinx # attempt to fix codecov set +eux # send coverage stats to Codecov bash <(curl -s https://codecov.io/bash) ================================================ FILE: run_tests2.sh ================================================ #!/usr/bin/env bash set -eux eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" pyenv virtualenv --force ${PYTHON_VERSION} cur pyenv activate cur ./run_tests.sh ================================================ FILE: src/__init__.py ================================================ from .api import get_new_node, get_remote_node from .backup import NodeBackup from .config import \ TestgresConfig, \ configure_testgres, \ scoped_config, \ push_config, \ pop_config from .connection import \ NodeConnection, \ DatabaseError, \ InternalError, \ ProgrammingError, \ OperationalError from .exceptions import \ TestgresException, \ ExecUtilException, \ QueryException, \ QueryTimeoutException, \ TimeoutException, \ CatchUpException, \ StartNodeException, \ InitNodeException, \ BackupException, \ InvalidOperationException from .enums import \ XLogMethod, \ IsolationLevel, \ NodeStatus, \ ProcessType, \ DumpFormat from .node import PostgresNode from .node import PortManager from .node_app import NodeApp from .utils import \ reserve_port, \ release_port, \ bound_ports, \ get_bin_path, \ get_pg_config, \ get_pg_version from .standby import \ First, \ Any from .config import testgres_config from testgres.operations.os_ops import OsOperations, ConnectionParams from testgres.operations.local_ops import LocalOperations from testgres.operations.remote_ops import RemoteOperations __version__ = "1.13.7" __all__ = [ "get_new_node", "get_remote_node", "NodeBackup", "testgres_config", "TestgresConfig", "configure_testgres", "scoped_config", "push_config", "pop_config", "NodeConnection", "DatabaseError", "InternalError", "ProgrammingError", "OperationalError", "TestgresException", "ExecUtilException", "QueryException", QueryTimeoutException.__name__, "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException", "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat", NodeApp.__name__, PostgresNode.__name__, PortManager.__name__, "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", "First", "Any", "OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams" ] ================================================ FILE: src/api.py ================================================ # coding: utf-8 """ Testing framework for PostgreSQL and its extensions This module was created under influence of Postgres TAP test feature (PostgresNode.pm module). It can manage Postgres clusters: initialize, edit configuration files, start/stop cluster, execute queries. The typical flow may look like: >>> with get_new_node() as node: ... node.init().start() ... result = node.safe_psql('postgres', 'select 1') ... print(result.decode('utf-8').strip()) ... node.stop() PostgresNode(name='...', port=..., base_dir='...') 1 PostgresNode(name='...', port=..., base_dir='...') Or: >>> with get_new_node() as master: ... master.init().start() ... with master.backup() as backup: ... with backup.spawn_replica() as replica: ... replica = replica.start() ... master.execute('postgres', 'create table test (val int4)') ... master.execute('postgres', 'insert into test values (0), (1), (2)') ... replica.catchup() # wait until changes are visible ... print(replica.execute('postgres', 'select count(*) from test')) PostgresNode(name='...', port=..., base_dir='...') [(3,)] """ from .node import PostgresNode def get_new_node(name=None, base_dir=None, **kwargs): """ Simply a wrapper around :class:`.PostgresNode` constructor. See :meth:`.PostgresNode.__init__` for details. """ # NOTE: leave explicit 'name' and 'base_dir' for compatibility return PostgresNode(name=name, base_dir=base_dir, **kwargs) def get_remote_node(name=None, conn_params=None): """ Simply a wrapper around :class:`.PostgresNode` constructor for remote node. See :meth:`.PostgresNode.__init__` for details. For remote connection you can add the next parameter: conn_params = ConnectionParams(host='127.0.0.1', ssh_key=None, username=default_username()) """ return get_new_node(name=name, conn_params=conn_params) ================================================ FILE: src/backup.py ================================================ # coding: utf-8 from six import raise_from from .enums import XLogMethod from .consts import \ DATA_DIR, \ TMP_NODE, \ TMP_BACKUP, \ PG_CONF_FILE, \ BACKUP_LOG_FILE from .exceptions import BackupException from testgres.operations.os_ops import OsOperations from .utils import \ get_bin_path2, \ execute_utility2, \ clean_on_error class NodeBackup(object): """ Smart object responsible for backups """ @property def log_file(self): assert self.os_ops is not None assert isinstance(self.os_ops, OsOperations) return self.os_ops.build_path(self.base_dir, BACKUP_LOG_FILE) def __init__(self, node, base_dir=None, username=None, xlog_method=XLogMethod.fetch, options=None): """ Create a new backup. Args: node: :class:`.PostgresNode` we're going to backup. base_dir: where should we store it? username: database user name. xlog_method: none | fetch | stream (see docs) """ assert node.os_ops is not None assert isinstance(node.os_ops, OsOperations) if not options: options = [] self.os_ops = node.os_ops if not node.status(): raise BackupException('Node must be running') # Check arguments if not isinstance(xlog_method, XLogMethod): try: xlog_method = XLogMethod(xlog_method) except ValueError: msg = 'Invalid xlog_method "{}"'.format(xlog_method) raise BackupException(msg) # Set default arguments username = username or self.os_ops.get_user() base_dir = base_dir or self.os_ops.mkdtemp(prefix=TMP_BACKUP) # public self.original_node = node self.base_dir = base_dir self.username = username # private self._available = True data_dir = self.os_ops.build_path(self.base_dir, DATA_DIR) _params = [ get_bin_path2(self.os_ops, "pg_basebackup"), "-p", str(node.port), "-h", node.host, "-U", username, "-D", data_dir, "-X", xlog_method.value ] # yapf: disable _params += options execute_utility2(self.os_ops, _params, self.log_file) def __enter__(self): return self def __exit__(self, type, value, traceback): self.cleanup() def _prepare_dir(self, destroy): """ Provide a data directory for a copy of node. Args: destroy: should we convert this backup into a node? Returns: Path to data directory. """ if not self._available: raise BackupException('Backup is exhausted') # Do we want to use this backup several times? available = not destroy if available: assert self.os_ops is not None assert isinstance(self.os_ops, OsOperations) dest_base_dir = self.os_ops.mkdtemp(prefix=TMP_NODE) data1 = self.os_ops.build_path(self.base_dir, DATA_DIR) data2 = self.os_ops.build_path(dest_base_dir, DATA_DIR) try: # Copy backup to new data dir self.os_ops.copytree(data1, data2) except Exception as e: raise_from(BackupException('Failed to copy files'), e) else: dest_base_dir = self.base_dir # Is this backup exhausted? self._available = available # Return path to new node return dest_base_dir def spawn_primary(self, name=None, destroy=True): """ Create a primary node from a backup. Args: name: primary's application name. destroy: should we convert this backup into a node? Returns: New instance of :class:`.PostgresNode`. """ # Prepare a data directory for this node base_dir = self._prepare_dir(destroy) # Build a new PostgresNode assert self.original_node is not None if (hasattr(self.original_node, "clone_with_new_name_and_base_dir")): node = self.original_node.clone_with_new_name_and_base_dir(name=name, base_dir=base_dir) else: # For backward compatibility NodeClass = self.original_node.__class__ node = NodeClass(name=name, base_dir=base_dir, conn_params=self.original_node.os_ops.conn_params) assert node is not None assert type(node) is self.original_node.__class__ with clean_on_error(node) as node: # Set a new port node.append_conf(filename=PG_CONF_FILE, line='\n') node.append_conf(filename=PG_CONF_FILE, port=node.port) return node def spawn_replica(self, name=None, destroy=True, slot=None): """ Create a replica of the original node from a backup. Args: name: replica's application name. slot: create a replication slot with the specified name. destroy: should we convert this backup into a node? Returns: New instance of :class:`.PostgresNode`. """ # Build a new PostgresNode node = self.spawn_primary(name=name, destroy=destroy) assert node is not None try: # Assign it a master and a recovery file (private magic) node._assign_master(self.original_node) node._create_recovery_conf(username=self.username, slot=slot) except: # noqa: E722 # TODO: Pass 'final=True' ? node.cleanup(release_resources=True) raise return node def cleanup(self): """ Remove all files that belong to this backup. No-op if it's been converted to a PostgresNode (destroy=True). """ if self._available: self._available = False self.os_ops.rmdirs(self.base_dir, ignore_errors=True) ================================================ FILE: src/cache.py ================================================ # coding: utf-8 from six import raise_from from .config import testgres_config from .consts import XLOG_CONTROL_FILE from .defaults import generate_system_id from .exceptions import \ InitNodeException, \ ExecUtilException from .utils import \ get_bin_path2, \ execute_utility2 from testgres.operations.local_ops import LocalOperations from testgres.operations.os_ops import OsOperations def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = None, bin_path=None, cached=True): """ Perform initdb or use cached node files. """ assert os_ops is None or isinstance(os_ops, OsOperations) if os_ops is None: os_ops = LocalOperations.get_single_instance() assert isinstance(os_ops, OsOperations) def make_utility_path(name): assert name is not None assert type(name) is str if bin_path: return os_ops.build_path(bin_path, name) return get_bin_path2(os_ops, name) def call_initdb(initdb_dir, log=logfile): try: initdb_path = make_utility_path("initdb") _params = [initdb_path, "-D", initdb_dir, "-N"] execute_utility2(os_ops, _params + (params or []), log) except ExecUtilException as e: raise_from(InitNodeException("Failed to run initdb"), e) if params or not testgres_config.cache_initdb or not cached: call_initdb(data_dir, logfile) else: # Fetch cached initdb dir cached_data_dir = testgres_config.cached_initdb_dir # Initialize cached initdb if not os_ops.path_exists(cached_data_dir) or \ not os_ops.listdir(cached_data_dir): call_initdb(cached_data_dir) try: # Copy cached initdb to current data dir os_ops.copytree(cached_data_dir, data_dir) # Assign this node a unique system id if asked to if testgres_config.cached_initdb_unique: # XXX: write new unique system id to control file # Some users might rely upon unique system ids, but # our initdb caching mechanism breaks this contract. pg_control = os_ops.build_path(data_dir, XLOG_CONTROL_FILE) system_id = generate_system_id() cur_pg_control = os_ops.read(pg_control, binary=True) new_pg_control = system_id + cur_pg_control[len(system_id):] os_ops.write(pg_control, new_pg_control, truncate=True, binary=True, read_and_write=True) # XXX: build new WAL segment with our system id _params = [make_utility_path("pg_resetwal"), "-D", data_dir, "-f"] execute_utility2(os_ops, _params, logfile) except ExecUtilException as e: msg = "Failed to reset WAL for system id" raise_from(InitNodeException(msg), e) except Exception as e: raise_from(InitNodeException("Failed to spawn a node"), e) ================================================ FILE: src/config.py ================================================ # coding: utf-8 import atexit import copy import logging import os import tempfile from contextlib import contextmanager from .consts import TMP_CACHE from testgres.operations.os_ops import OsOperations from testgres.operations.local_ops import LocalOperations log_level = os.getenv('LOGGING_LEVEL', 'WARNING').upper() log_format = os.getenv('LOGGING_FORMAT', '%(asctime)s - %(levelname)s - %(message)s') logging.basicConfig(level=log_level, format=log_format) class GlobalConfig(object): """ Global configuration object which allows user to override default settings. """ # NOTE: attributes must not be callable or begin with __. cache_initdb = True """ shall we use cached initdb instance? """ cached_initdb_unique = False """ shall we give new node a unique system id? """ cache_pg_config = True """ shall we cache pg_config results? """ use_python_logging = False """ enable python logging subsystem (see logger.py). """ error_log_lines = 20 """ N of log lines to be shown in exceptions (0=inf). """ node_cleanup_full = True """ shall we remove EVERYTHING (including logs)? """ node_cleanup_on_good_exit = True """ remove base_dir on nominal __exit__(). """ node_cleanup_on_bad_exit = False """ remove base_dir on __exit__() via exception. """ _cached_initdb_dir = None """ underlying class attribute for cached_initdb_dir property """ os_ops = LocalOperations.get_single_instance() """ OsOperation object that allows work on remote host """ @property def cached_initdb_dir(self): """ path to a temp directory for cached initdb. """ return self._cached_initdb_dir @cached_initdb_dir.setter def cached_initdb_dir(self, value): self._cached_initdb_dir = value if value: cached_initdb_dirs.add(value) return testgres_config.cached_initdb_dir @property def temp_dir(self): """ path to temp dir containing nodes with default 'base_dir'. """ return tempfile.tempdir @temp_dir.setter def temp_dir(self, value): tempfile.tempdir = value def __init__(self, **options): self.update(options) def __setitem__(self, key, value): setattr(self, key, value) def __getitem__(self, key): return getattr(self, key) def __setattr__(self, name, value): if name not in self.keys(): raise TypeError('Unknown option {}'.format(name)) super(GlobalConfig, self).__setattr__(name, value) def keys(self): """ Return a list of all available settings. """ keys = [] for key in dir(GlobalConfig): if not key.startswith('__') and not callable(self[key]): keys.append(key) return keys def items(self): """ Return setting-value pairs. """ return ((key, self[key]) for key in self.keys()) def update(self, config): """ Extract setting-value pairs from 'config' and assign those values to corresponding settings of this GlobalConfig object. """ for key, value in config.items(): self[key] = value return self def copy(self): """ Return a copy of this object. """ return copy.copy(self) @staticmethod def set_os_ops(os_ops: OsOperations): testgres_config.os_ops = os_ops testgres_config.cached_initdb_dir = os_ops.mkdtemp(prefix=TMP_CACHE) # cached dirs to be removed cached_initdb_dirs = set() # default config object testgres_config = GlobalConfig() # NOTE: for compatibility TestgresConfig = testgres_config # stack of GlobalConfigs config_stack = [] @atexit.register def _rm_cached_initdb_dirs(): for d in cached_initdb_dirs: testgres_config.os_ops.rmdirs(d, ignore_errors=True) def push_config(**options): """ Permanently set custom GlobalConfig options and put previous settings on top of the config stack. """ # push current config to stack config_stack.append(testgres_config.copy()) return testgres_config.update(options) def pop_config(): """ Set previous GlobalConfig options from stack. """ if len(config_stack) == 0: raise IndexError('Reached initial config') # restore popped config return testgres_config.update(config_stack.pop()) @contextmanager def scoped_config(**options): """ Temporarily set custom GlobalConfig options for this context. Previous options are pushed to the config stack. Example: >>> from .api import get_new_node >>> with scoped_config(cache_initdb=False): ... # create a new node with fresh initdb ... with get_new_node().init().start() as node: ... print(node.execute('select 1')) [(1,)] """ try: # set a new config with options config = push_config(**options) # return it yield config finally: # restore previous config pop_config() def configure_testgres(**options): """ Adjust current global options. Look at the GlobalConfig to learn about existing settings. """ testgres_config.update(options) # NOTE: assign initial cached dir for initdb testgres_config.cached_initdb_dir = testgres_config.os_ops.mkdtemp(prefix=TMP_CACHE) ================================================ FILE: src/connection.py ================================================ # coding: utf-8 import logging # we support both pg8000 and psycopg2 try: import psycopg2 as pglib except ImportError: try: import pg8000 as pglib except ImportError: raise ImportError("You must have psycopg2 or pg8000 modules installed") from .enums import IsolationLevel from .defaults import \ default_dbname, \ default_username from .exceptions import QueryException # export some exceptions DatabaseError = pglib.DatabaseError InternalError = pglib.InternalError ProgrammingError = pglib.ProgrammingError OperationalError = pglib.OperationalError class NodeConnection(object): """ Transaction wrapper returned by Node """ def __init__(self, node, dbname=None, username=None, password=None, autocommit=False): # Set default arguments dbname = dbname or default_dbname() username = username or default_username() self._node = node self._connection = pglib.connect( database=dbname, user=username, password=password, host=node.host, port=node.port ) self._connection.autocommit = autocommit self._cursor = self.connection.cursor() @property def node(self): return self._node @property def connection(self): return self._connection @property def pid(self): return self.execute("select pg_catalog.pg_backend_pid()")[0][0] @property def cursor(self): return self._cursor def __enter__(self): return self def __exit__(self, type, value, traceback): self.close() def begin(self, isolation_level=IsolationLevel.ReadCommitted): # Check if level isn't an IsolationLevel if not isinstance(isolation_level, IsolationLevel): # Get name of isolation level level_str = str(isolation_level).lower() # Validate level string try: isolation_level = IsolationLevel(level_str) except ValueError: error = 'Invalid isolation level "{}"' raise QueryException(error.format(level_str)) # Set isolation level cmd = 'SET TRANSACTION ISOLATION LEVEL {}' self.cursor.execute(cmd.format(isolation_level.value)) return self def commit(self): self.connection.commit() return self def rollback(self): self.connection.rollback() return self def execute(self, query, *args): self.cursor.execute(query, args) try: # pg8000 might return tuples res = [tuple(t) for t in self.cursor.fetchall()] return res except ProgrammingError: return None except Exception as e: logging.error("Error executing query: {}\n {}".format(repr(e), query)) return None def close(self): self.cursor.close() self.connection.close() ================================================ FILE: src/consts.py ================================================ # coding: utf-8 # names for dirs in base_dir DATA_DIR = "data" LOGS_DIR = "logs" # prefixes for temp dirs TMP_NODE = 'tgsn_' TMP_DUMP = 'tgsd_' TMP_CACHE = 'tgsc_' TMP_BACKUP = 'tgsb_' # path to control file XLOG_CONTROL_FILE = "global/pg_control" # names for config files RECOVERY_CONF_FILE = "recovery.conf" PG_AUTO_CONF_FILE = "postgresql.auto.conf" PG_CONF_FILE = "postgresql.conf" PG_PID_FILE = 'postmaster.pid' HBA_CONF_FILE = "pg_hba.conf" # names for log files PG_LOG_FILE = "postgresql.log" UTILS_LOG_FILE = "utils.log" BACKUP_LOG_FILE = "backup.log" # defaults for node settings MAX_LOGICAL_REPLICATION_WORKERS = 5 MAX_REPLICATION_SLOTS = 10 MAX_WORKER_PROCESSES = 10 WAL_KEEP_SEGMENTS = 20 WAL_KEEP_SIZE = 320 MAX_WAL_SENDERS = 10 # logical replication settings LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS = 60 PG_CTL__STATUS__OK = 0 PG_CTL__STATUS__NODE_IS_STOPPED = 3 PG_CTL__STATUS__BAD_DATADIR = 4 ================================================ FILE: src/decorators.py ================================================ import six import functools def positional_args_hack(*special_cases): """ Convert positional args described by 'special_cases' into named args. Example: @positional_args_hack(['abc'], ['def', 'abc']) def some_api_func(...) This is useful for compatibility. """ cases = dict() for case in special_cases: k = len(case) assert k not in six.iterkeys(cases), 'len must be unique' cases[k] = case def decorator(function): @functools.wraps(function) def wrapper(*args, **kwargs): k = len(args) if k in six.iterkeys(cases): case = cases[k] for i in range(0, k): arg_name = case[i] arg_val = args[i] # transform into named kwargs[arg_name] = arg_val # get rid of them args = [] return function(*args, **kwargs) return wrapper return decorator def method_decorator(decorator): """ Convert a function decorator into a method decorator. """ def _dec(func): def _wrapper(self, *args, **kwargs): @decorator def bound_func(*args2, **kwargs2): return func.__get__(self, type(self))(*args2, **kwargs2) # 'bound_func' is a closure and can see 'self' return bound_func(*args, **kwargs) # preserve docs functools.update_wrapper(_wrapper, func) return _wrapper # preserve docs functools.update_wrapper(_dec, decorator) # change name for easier debugging _dec.__name__ = 'method_decorator({})'.format(decorator.__name__) return _dec ================================================ FILE: src/defaults.py ================================================ import datetime import struct import uuid from .config import testgres_config as tconf def default_dbname(): """ Return default DB name. """ return 'postgres' def default_username(): """ Return default username (current user). """ return tconf.os_ops.get_user() def generate_app_name(): """ Generate a new application name for node. """ return 'testgres-{}'.format(str(uuid.uuid4())) def generate_system_id(): """ Generate a new 64-bit unique system identifier for node. """ date1 = datetime.datetime.utcfromtimestamp(0) date2 = datetime.datetime.utcnow() secs = int((date2 - date1).total_seconds()) usecs = date2.microsecond # see pg_resetwal.c : GuessControlValues() system_id = 0 system_id |= (secs << 32) system_id |= (usecs << 12) system_id |= (tconf.os_ops.get_pid() & 0xFFF) # pack ULL in native byte order return struct.pack('=Q', system_id) ================================================ FILE: src/enums.py ================================================ from enum import Enum, IntEnum from six import iteritems from psutil import NoSuchProcess class XLogMethod(Enum): """ Available WAL methods for :class:`.NodeBackup` """ none = 'none' fetch = 'fetch' stream = 'stream' class IsolationLevel(Enum): """ Transaction isolation level for :class:`.NodeConnection` """ ReadUncommitted = 'read uncommitted' ReadCommitted = 'read committed' RepeatableRead = 'repeatable read' Serializable = 'serializable' class NodeStatus(IntEnum): """ Status of a PostgresNode """ Running, Stopped, Uninitialized = range(3) # for Python 3.x def __bool__(self): return self == NodeStatus.Running # for Python 2.x __nonzero__ = __bool__ class ProcessType(Enum): """ Types of processes """ AutovacuumLauncher = 'autovacuum launcher' BackgroundWriter = 'background writer' Checkpointer = 'checkpointer' LogicalReplicationLauncher = 'logical replication launcher' Startup = 'startup' StatsCollector = 'stats collector' WalReceiver = 'wal receiver' WalSender = 'wal sender' WalWriter = 'wal writer' # special value Unknown = 'unknown' @staticmethod def from_process(process): # legacy names for older releases of PG alternative_names = { ProcessType.LogicalReplicationLauncher: [ 'logical replication worker' ], ProcessType.BackgroundWriter: [ 'writer' ], } # yapf: disable try: cmdline = ''.join(process.cmdline()) except (FileNotFoundError, ProcessLookupError, NoSuchProcess): return ProcessType.Unknown # we deliberately cut special words and spaces cmdline = cmdline.replace('postgres:', '', 1) \ .replace('bgworker:', '', 1) \ .replace(' ', '') for ptype in ProcessType: if cmdline.startswith(ptype.value.replace(' ', '')): return ptype for ptype, names in iteritems(alternative_names): for name in names: if cmdline.startswith(name.replace(' ', '')): return ptype # default return ProcessType.Unknown class DumpFormat(Enum): """ Available dump formats """ Plain = 'plain' Custom = 'custom' Directory = 'directory' Tar = 'tar' ================================================ FILE: src/exceptions.py ================================================ # coding: utf-8 import six import typing from testgres.operations.exceptions import TestgresException from testgres.operations.exceptions import ExecUtilException from testgres.operations.exceptions import InvalidOperationException class PortForException(TestgresException): _message: typing.Optional[str] def __init__( self, message: typing.Optional[str] = None, ): assert message is None or type(message) is str super().__init__(message) self._message = message return @property def message(self) -> str: assert self._message is None or type(self._message) is str if self._message is None: return "" return self._message def __repr__(self) -> str: args = [] if self._message is not None: args.append(("message", self._message)) result = "{}(".format(type(self).__name__) sep = "" for a in args: result += sep + a[0] + "=" + repr(a[1]) sep = ", " continue result += ")" return result @six.python_2_unicode_compatible class QueryException(TestgresException): _description: typing.Optional[str] _query: typing.Optional[str] def __init__( self, message: typing.Optional[str] = None, query: typing.Optional[str] = None ): assert message is None or type(message) is str assert query is None or type(query) is str super().__init__(message) self._description = message self._query = query return @property def message(self) -> str: assert self._description is None or type(self._description) is str assert self._query is None or type(self._query) is str msg = [] if self._description: msg.append(self._description) if self._query: msg.append(u'Query: {}'.format(self._query)) r = six.text_type('\n').join(msg) assert type(r) is str return r @property def description(self) -> typing.Optional[str]: assert self._description is None or type(self._description) is str return self._description @property def query(self) -> typing.Optional[str]: assert self._query is None or type(self._query) is str return self._query def __repr__(self) -> str: args = [] if self._description is not None: args.append(("message", self._description)) if self._query is not None: args.append(("query", self._query)) result = "{}(".format(type(self).__name__) sep = "" for a in args: result += sep + a[0] + "=" + repr(a[1]) sep = ", " continue result += ")" return result class QueryTimeoutException(QueryException): def __init__( self, message: typing.Optional[str] = None, query: typing.Optional[str] = None ): assert message is None or type(message) is str assert query is None or type(query) is str super().__init__(message, query) return # [2026-01-10] To backward compatibility. TimeoutException = QueryTimeoutException # [2026-01-10] It inherits TestgresException now, not QueryException class CatchUpException(TestgresException): _message: typing.Optional[str] def __init__( self, message: typing.Optional[str] = None, ): assert message is None or type(message) is str super().__init__(message) self._message = message return @property def message(self) -> str: assert self._message is None or type(self._message) is str if self._message is None: return "" return self._message def __repr__(self) -> str: args = [] if self._message is not None: args.append(("message", self._message)) result = "{}(".format(type(self).__name__) sep = "" for a in args: result += sep + a[0] + "=" + repr(a[1]) sep = ", " continue result += ")" return result @six.python_2_unicode_compatible class StartNodeException(TestgresException): _description: typing.Optional[str] _files: typing.Optional[typing.Iterable] def __init__( self, message: typing.Optional[str] = None, files: typing.Optional[typing.Iterable] = None ): assert message is None or type(message) is str assert files is None or isinstance(files, typing.Iterable) super().__init__(message) self._description = message self._files = files return @property def message(self) -> str: assert self._description is None or type(self._description) is str assert self._files is None or isinstance(self._files, typing.Iterable) msg = [] if self._description: msg.append(self._description) for f, lines in self._files or []: assert type(f) is str assert type(lines) in [str, bytes] msg.append(u'{}\n----\n{}\n'.format(f, lines)) return six.text_type('\n').join(msg) @property def description(self) -> typing.Optional[str]: assert self._description is None or type(self._description) is str return self._description @property def files(self) -> typing.Optional[typing.Iterable]: assert self._files is None or isinstance(self._files, typing.Iterable) return self._files def __repr__(self) -> str: args = [] if self._description is not None: args.append(("message", self._description)) if self._files is not None: args.append(("files", self._files)) result = "{}(".format(type(self).__name__) sep = "" for a in args: result += sep + a[0] + "=" + repr(a[1]) sep = ", " continue result += ")" return result class InitNodeException(TestgresException): _message: typing.Optional[str] def __init__( self, message: typing.Optional[str] = None, ): assert message is None or type(message) is str super().__init__(message) self._message = message return @property def message(self) -> str: assert self._message is None or type(self._message) is str if self._message is None: return "" return self._message def __repr__(self) -> str: args = [] if self._message is not None: args.append(("message", self._message)) result = "{}(".format(type(self).__name__) sep = "" for a in args: result += sep + a[0] + "=" + repr(a[1]) sep = ", " continue result += ")" return result class BackupException(TestgresException): _message: typing.Optional[str] def __init__( self, message: typing.Optional[str] = None, ): assert message is None or type(message) is str super().__init__(message) self._message = message return @property def message(self) -> str: assert self._message is None or type(self._message) is str if self._message is None: return "" return self._message def __repr__(self) -> str: args = [] if self._message is not None: args.append(("message", self._message)) result = "{}(".format(type(self).__name__) sep = "" for a in args: result += sep + a[0] + "=" + repr(a[1]) sep = ", " continue result += ")" return result assert ExecUtilException.__name__ == "ExecUtilException" assert InvalidOperationException.__name__ == "InvalidOperationException" ================================================ FILE: src/impl/internal_utils.py ================================================ import logging def send_log(level: int, msg: str) -> None: assert type(level) is int assert type(msg) is str return logging.log(level, "[testgres] " + msg) def send_log_info(msg: str) -> None: assert type(msg) is str return send_log(logging.INFO, msg) def send_log_debug(msg: str) -> None: assert type(msg) is str return send_log(logging.DEBUG, msg) ================================================ FILE: src/impl/platforms/internal_platform_utils.py ================================================ from __future__ import annotations import enum import typing from testgres.operations.os_ops import OsOperations class InternalPlatformUtils: class FindPostmasterResultCode(enum.Enum): ok = 0 not_found = 1, not_implemented = 2 many_processes = 3 has_problems = 4 class FindPostmasterResult: code: InternalPlatformUtils.FindPostmasterResultCode pid: typing.Optional[int] def __init__( self, code: InternalPlatformUtils.FindPostmasterResultCode, pid: typing.Optional[int] ): assert type(code) is InternalPlatformUtils.FindPostmasterResultCode assert pid is None or type(pid) is int self.code = code self.pid = pid return @staticmethod def create_ok(pid: int) -> InternalPlatformUtils.FindPostmasterResult: assert type(pid) is int return __class__(InternalPlatformUtils.FindPostmasterResultCode.ok, pid) @staticmethod def create_not_found() -> InternalPlatformUtils.FindPostmasterResult: return __class__(InternalPlatformUtils.FindPostmasterResultCode.not_found, None) @staticmethod def create_not_implemented() -> InternalPlatformUtils.FindPostmasterResult: return __class__(InternalPlatformUtils.FindPostmasterResultCode.not_implemented, None) @staticmethod def create_many_processes() -> InternalPlatformUtils.FindPostmasterResult: return __class__(InternalPlatformUtils.FindPostmasterResultCode.many_processes, None) @staticmethod def create_has_problems() -> InternalPlatformUtils.FindPostmasterResult: return __class__(InternalPlatformUtils.FindPostmasterResultCode.has_problems, None) def FindPostmaster( self, os_ops: OsOperations, bin_dir: str, data_dir: str ) -> FindPostmasterResult: assert isinstance(os_ops, OsOperations) assert type(bin_dir) is str assert type(data_dir) is str raise NotImplementedError("InternalPlatformUtils::FindPostmaster is not implemented.") ================================================ FILE: src/impl/platforms/internal_platform_utils_factory.py ================================================ from .internal_platform_utils import InternalPlatformUtils from testgres.operations.os_ops import OsOperations def create_internal_platform_utils( os_ops: OsOperations ) -> InternalPlatformUtils: assert isinstance(os_ops, OsOperations) platform_name = os_ops.get_platform() assert type(platform_name) is str if platform_name == "linux": from .linux import internal_platform_utils as x return x.InternalPlatformUtils() if platform_name == "win32": from .win32 import internal_platform_utils as x return x.InternalPlatformUtils() # not implemented return InternalPlatformUtils() ================================================ FILE: src/impl/platforms/linux/internal_platform_utils.py ================================================ from __future__ import annotations from .. import internal_platform_utils as base from ... import internal_utils from testgres.operations.os_ops import OsOperations from testgres.operations.exceptions import ExecUtilException import re import shlex class InternalPlatformUtils(base.InternalPlatformUtils): C_BASH_EXE = "/bin/bash" sm_exec_env = { "LANG": "en_US.UTF-8", "LC_ALL": "en_US.UTF-8", } # -------------------------------------------------------------------- def FindPostmaster( self, os_ops: OsOperations, bin_dir: str, data_dir: str ) -> InternalPlatformUtils.FindPostmasterResult: assert isinstance(os_ops, OsOperations) assert type(bin_dir) is str assert type(data_dir) is str assert type(__class__.C_BASH_EXE) is str assert type(__class__.sm_exec_env) is dict assert len(__class__.C_BASH_EXE) > 0 assert len(bin_dir) > 0 assert len(data_dir) > 0 pg_path_e = re.escape(os_ops.build_path(bin_dir, "postgres")) data_dir_e = re.escape(data_dir) assert type(pg_path_e) is str assert type(data_dir_e) is str regexp = r"^\s*[0-9]+\s+" + pg_path_e + r"(\s+.*)?\s+\-[D]\s+" + data_dir_e + r"(\s+.*)?" cmd = [ __class__.C_BASH_EXE, "-c", "ps -ewwo \"pid=,args=\" | grep -E " + shlex.quote(regexp), ] exit_status, output_b, error_b = os_ops.exec_command( cmd=cmd, ignore_errors=True, verbose=True, exec_env=__class__.sm_exec_env, ) assert type(output_b) is bytes assert type(error_b) is bytes output = output_b.decode("utf-8") error = error_b.decode("utf-8") assert type(output) is str assert type(error) is str if exit_status == 1: return __class__.FindPostmasterResult.create_not_found() if exit_status != 0: errMsg = f"test command returned an unexpected exit code: {exit_status}" raise ExecUtilException( message=errMsg, command=cmd, exit_code=exit_status, out=output, error=error, ) lines = output.splitlines() assert type(lines) is list if len(lines) == 0: return __class__.FindPostmasterResult.create_not_found() if len(lines) > 1: msgs = [] msgs.append("Many processes like a postmaster are found: {}.".format(len(lines))) for i in range(len(lines)): assert type(lines[i]) is str lines.append("[{}] '{}'".format(i, lines[i])) continue internal_utils.send_log_debug("\n".join(lines)) return __class__.FindPostmasterResult.create_many_processes() def is_space_or_tab(ch) -> bool: assert type(ch) is str return ch == " " or ch == "\t" line = lines[0] start = 0 while start < len(line) and is_space_or_tab(line[start]): start += 1 pos = start while pos < len(line) and line[pos].isnumeric(): pos += 1 if pos == start: return __class__.FindPostmasterResult.create_has_problems() if pos != len(line) and not line[pos].isspace(): return __class__.FindPostmasterResult.create_has_problems() pid = int(line[start:pos]) assert type(pid) is int return __class__.FindPostmasterResult.create_ok(pid) ================================================ FILE: src/impl/platforms/win32/internal_platform_utils.py ================================================ from __future__ import annotations from .. import internal_platform_utils as base from testgres.operations.os_ops import OsOperations class InternalPlatformUtils(base.InternalPlatformUtils): def FindPostmaster( self, os_ops: OsOperations, bin_dir: str, data_dir: str ) -> InternalPlatformUtils.FindPostmasterResult: assert isinstance(os_ops, OsOperations) assert type(bin_dir) is str assert type(data_dir) is str return __class__.FindPostmasterResult.create_not_implemented() ================================================ FILE: src/impl/port_manager__generic.py ================================================ from testgres.operations.os_ops import OsOperations from ..port_manager import PortManager from ..exceptions import PortForException import threading import random import typing import logging class PortManager__Generic(PortManager): _C_MIN_PORT_NUMBER = 1024 _C_MAX_PORT_NUMBER = 65535 _os_ops: OsOperations _guard: object # TODO: is there better to use bitmap fot _available_ports? _available_ports: typing.Set[int] _reserved_ports: typing.Set[int] def __init__(self, os_ops: OsOperations): assert __class__._C_MIN_PORT_NUMBER <= __class__._C_MAX_PORT_NUMBER assert os_ops is not None assert isinstance(os_ops, OsOperations) self._os_ops = os_ops self._guard = threading.Lock() self._available_ports = set( range(__class__._C_MIN_PORT_NUMBER, __class__._C_MAX_PORT_NUMBER + 1) ) assert len(self._available_ports) == ( (__class__._C_MAX_PORT_NUMBER - __class__._C_MIN_PORT_NUMBER) + 1 ) self._reserved_ports = set() return def reserve_port(self) -> int: assert self._guard is not None assert type(self._available_ports) is set assert type(self._reserved_ports) is set with self._guard: t = tuple(self._available_ports) assert len(t) == len(self._available_ports) sampled_ports = random.sample(t, min(len(t), 100)) t = None for port in sampled_ports: assert type(port) is int assert port not in self._reserved_ports assert port in self._available_ports assert port >= __class__._C_MIN_PORT_NUMBER assert port <= __class__._C_MAX_PORT_NUMBER if not self._os_ops.is_port_free(port): continue self._reserved_ports.add(port) self._available_ports.discard(port) assert port in self._reserved_ports assert port not in self._available_ports __class__.helper__send_debug_msg("Port {} is reserved.", port) return port raise PortForException("Can't select a port.") def release_port(self, number: int) -> None: assert type(number) is int assert number >= __class__._C_MIN_PORT_NUMBER assert number <= __class__._C_MAX_PORT_NUMBER assert self._guard is not None assert type(self._reserved_ports) is set with self._guard: assert number in self._reserved_ports assert number not in self._available_ports self._available_ports.add(number) self._reserved_ports.discard(number) assert number not in self._reserved_ports assert number in self._available_ports __class__.helper__send_debug_msg("Port {} is released.", number) return @staticmethod def helper__send_debug_msg(msg_template: str, *args) -> None: assert msg_template is not None assert args is not None assert type(msg_template) is str assert type(args) is tuple assert msg_template != "" s = "[port manager] " s += msg_template.format(*args) logging.debug(s) ================================================ FILE: src/impl/port_manager__this_host.py ================================================ from ..port_manager import PortManager from .. import utils import threading class PortManager__ThisHost(PortManager): sm_single_instance: PortManager = None sm_single_instance_guard = threading.Lock() @staticmethod def get_single_instance() -> PortManager: assert __class__ == PortManager__ThisHost assert __class__.sm_single_instance_guard is not None if __class__.sm_single_instance is not None: assert type(__class__.sm_single_instance) is __class__ return __class__.sm_single_instance with __class__.sm_single_instance_guard: if __class__.sm_single_instance is None: __class__.sm_single_instance = __class__() assert __class__.sm_single_instance is not None assert type(__class__.sm_single_instance) is __class__ return __class__.sm_single_instance def reserve_port(self) -> int: return utils.reserve_port() def release_port(self, number: int) -> None: assert type(number) is int return utils.release_port(number) ================================================ FILE: src/logger.py ================================================ # coding: utf-8 import logging import select import threading import time class TestgresLogger(threading.Thread): """ Helper class to implement reading from log files. """ def __init__(self, node_name, log_file_name): threading.Thread.__init__(self) self._node_name = node_name self._log_file_name = log_file_name self._stop_event = threading.Event() self._logger = logging.getLogger(node_name) self._logger.setLevel(logging.INFO) def run(self): # open log file for reading with open(self._log_file_name, 'r') as fd: # work until we're asked to stop while not self._stop_event.is_set(): sleep_time = 0.1 new_lines = False # do we have new lines? if fd in select.select([fd], [], [], 0)[0]: for line in fd.readlines(): line = line.strip() if line: new_lines = True extra = {'node': self._node_name} self._logger.info(line, extra=extra) if not new_lines: time.sleep(sleep_time) # don't forget to clear event self._stop_event.clear() def stop(self, wait=True): self._stop_event.set() if wait: self.join() ================================================ FILE: src/node.py ================================================ # coding: utf-8 from __future__ import annotations import logging import os import signal import subprocess import time import typing try: from collections.abc import Iterable except ImportError: from collections import Iterable # we support both pg8000 and psycopg2 try: import psycopg2 as pglib except ImportError: try: import pg8000 as pglib except ImportError: raise ImportError("You must have psycopg2 or pg8000 modules installed") from six import raise_from, iteritems, text_type from .enums import \ NodeStatus, \ ProcessType, \ DumpFormat from .cache import cached_initdb from .config import testgres_config from .connection import NodeConnection from .consts import \ DATA_DIR, \ LOGS_DIR, \ TMP_NODE, \ TMP_DUMP, \ PG_CONF_FILE, \ PG_AUTO_CONF_FILE, \ HBA_CONF_FILE, \ RECOVERY_CONF_FILE, \ PG_LOG_FILE, \ UTILS_LOG_FILE from .consts import \ MAX_LOGICAL_REPLICATION_WORKERS, \ MAX_REPLICATION_SLOTS, \ MAX_WORKER_PROCESSES, \ MAX_WAL_SENDERS, \ WAL_KEEP_SEGMENTS, \ WAL_KEEP_SIZE from .decorators import \ method_decorator, \ positional_args_hack from .defaults import \ default_dbname, \ generate_app_name from .exceptions import \ CatchUpException, \ ExecUtilException, \ QueryException, \ QueryTimeoutException, \ StartNodeException, \ TimeoutException, \ InitNodeException, \ TestgresException, \ BackupException, \ InvalidOperationException from .port_manager import PortManager from .impl.port_manager__this_host import PortManager__ThisHost from .impl.port_manager__generic import PortManager__Generic from .logger import TestgresLogger from .pubsub import Publication, Subscription from .standby import First from . import utils from .utils import \ PgVer, \ eprint, \ get_bin_path2, \ get_pg_version2, \ execute_utility2, \ options_string, \ clean_on_error from .raise_error import RaiseError from .backup import NodeBackup from testgres.operations.os_ops import ConnectionParams from testgres.operations.os_ops import OsOperations from testgres.operations.local_ops import LocalOperations InternalError = pglib.InternalError ProgrammingError = pglib.ProgrammingError OperationalError = pglib.OperationalError assert TimeoutException == QueryTimeoutException class ProcessProxy(object): """ Wrapper for psutil.Process Attributes: process: wrapped psutill.Process object ptype: instance of ProcessType """ _process: typing.Any _ptype: ProcessType def __init__(self, process, ptype: typing.Optional[ProcessType] = None): assert process is not None assert ptype is None or type(ptype) is ProcessType self._process = process if ptype is not None: self._ptype = ptype else: self._ptype = ProcessType.from_process(process) assert type(self._ptype) is ProcessType return def __getattr__(self, name): return getattr(self.process, name) def __repr__(self): return '{}(ptype={}, process={})'.format( self.__class__.__name__, str(self.ptype), repr(self.process)) @property def process(self) -> typing.Any: assert self._process is not None return self._process @property def ptype(self) -> ProcessType: assert type(self._ptype) is ProcessType return self._ptype class PostgresNode(object): # a max number of node start attempts _C_MAX_START_ATEMPTS = 5 _C_PM_PID__IS_NOT_DETECTED = -1 _name: typing.Optional[str] _port: typing.Optional[int] _should_free_port: bool _os_ops: OsOperations _port_manager: typing.Optional[PortManager] _manually_started_pm_pid: typing.Optional[int] def __init__(self, name=None, base_dir=None, port: typing.Optional[int] = None, conn_params: typing.Optional[ConnectionParams] = None, bin_dir=None, prefix=None, os_ops: typing.Optional[OsOperations] = None, port_manager: typing.Optional[PortManager] = None): """ PostgresNode constructor. Args: name: node's application name. port: port to accept connections. base_dir: path to node's data directory. bin_dir: path to node's binary directory. os_ops: None or correct OS operation object. port_manager: None or correct port manager object. """ assert port is None or type(port) is int assert os_ops is None or isinstance(os_ops, OsOperations) assert port_manager is None or isinstance(port_manager, PortManager) if conn_params is not None: assert type(conn_params) is ConnectionParams raise InvalidOperationException("conn_params is deprecated, please use os_ops parameter instead.") # private if os_ops is None: self._os_ops = __class__._get_os_ops() else: assert isinstance(os_ops, OsOperations) self._os_ops = os_ops pass assert self._os_ops is not None assert isinstance(self._os_ops, OsOperations) self._pg_version = PgVer(get_pg_version2(self._os_ops, bin_dir)) self._base_dir = base_dir self._bin_dir = bin_dir self._prefix = prefix self._logger = None self._master = None # basic self._name = name or generate_app_name() if port is not None: assert type(port) is int assert port_manager is None self._port = port self._should_free_port = False self._port_manager = None else: if port_manager is None: self._port_manager = __class__._get_port_manager(self._os_ops) elif os_ops is None: raise InvalidOperationException("When port_manager is not None you have to define os_ops, too.") else: assert isinstance(port_manager, PortManager) assert self._os_ops is os_ops self._port_manager = port_manager assert self._port_manager is not None assert isinstance(self._port_manager, PortManager) self._port = self._port_manager.reserve_port() # raises assert type(self._port) is int self._should_free_port = True assert type(self._port) is int # defaults for __exit__() self.cleanup_on_good_exit = testgres_config.node_cleanup_on_good_exit self.cleanup_on_bad_exit = testgres_config.node_cleanup_on_bad_exit self.shutdown_max_attempts = 3 # NOTE: for compatibility self.utils_log_name = self.utils_log_file self.pg_log_name = self.pg_log_file # Node state self._manually_started_pm_pid = None def __enter__(self): return self def __exit__(self, type, value, traceback): # NOTE: Ctrl+C does not count! got_exception = type is not None and type is not KeyboardInterrupt c1 = self.cleanup_on_good_exit and not got_exception c2 = self.cleanup_on_bad_exit and got_exception attempts = self.shutdown_max_attempts if c1 or c2: self.cleanup(attempts) else: self._try_shutdown(attempts) self._release_resources() def __repr__(self): return "{}(name='{}', port={}, base_dir='{}')".format( self.__class__.__name__, self.name, str(self._port) if self._port is not None else "None", self.base_dir ) @staticmethod def _get_os_ops() -> OsOperations: if testgres_config.os_ops: return testgres_config.os_ops return LocalOperations.get_single_instance() @staticmethod def _get_port_manager(os_ops: OsOperations) -> PortManager: assert os_ops is not None assert isinstance(os_ops, OsOperations) if os_ops is LocalOperations.get_single_instance(): assert utils._old_port_manager is not None assert type(utils._old_port_manager) is PortManager__Generic assert utils._old_port_manager._os_ops is os_ops return PortManager__ThisHost.get_single_instance() # TODO: Throw the exception "Please define a port manager." ? return PortManager__Generic(os_ops) def clone_with_new_name_and_base_dir(self, name: str, base_dir: str): assert name is None or type(name) is str assert base_dir is None or type(base_dir) is str assert __class__ == PostgresNode if self._port_manager is None: raise InvalidOperationException("PostgresNode without PortManager can't be cloned.") assert self._port_manager is not None assert isinstance(self._port_manager, PortManager) assert self._os_ops is not None assert isinstance(self._os_ops, OsOperations) node = PostgresNode( name=name, base_dir=base_dir, bin_dir=self._bin_dir, prefix=self._prefix, os_ops=self._os_ops, port_manager=self._port_manager) return node @property def os_ops(self) -> OsOperations: assert self._os_ops is not None assert isinstance(self._os_ops, OsOperations) return self._os_ops @property def port_manager(self) -> typing.Optional[PortManager]: assert self._port_manager is None or isinstance(self._port_manager, PortManager) return self._port_manager @property def name(self) -> str: if self._name is None: raise InvalidOperationException("PostgresNode name is not defined.") assert type(self._name) is str return self._name @property def host(self) -> str: assert self._os_ops is not None assert isinstance(self._os_ops, OsOperations) return self._os_ops.host @property def port(self) -> int: if self._port is None: raise InvalidOperationException("PostgresNode port is not defined.") assert type(self._port) is int return self._port @property def ssh_key(self) -> typing.Optional[str]: assert self._os_ops is not None assert isinstance(self._os_ops, OsOperations) return self._os_ops.ssh_key @property def pid(self) -> int: """ Return postmaster's PID if node is running, else 0. """ x = self._get_node_state() assert type(x) is utils.PostgresNodeState if x.pid is None: assert x.node_status != NodeStatus.Running return 0 assert x.node_status == NodeStatus.Running assert type(x.pid) is int return x.pid @property def is_started(self) -> bool: if self._manually_started_pm_pid is None: return False assert type(self._manually_started_pm_pid) is int return True @property def auxiliary_pids(self) -> typing.Dict[ProcessType, typing.List[int]]: """ Returns a dict of { ProcessType : PID }. """ result = {} for process in self.auxiliary_processes: assert type(process) is ProcessProxy if process.ptype not in result: result[process.ptype] = [] result[process.ptype].append(process.pid) return result @property def auxiliary_processes(self) -> typing.List[ProcessProxy]: """ Returns a list of auxiliary processes. Each process is represented by :class:`.ProcessProxy` object. """ def is_aux(process: ProcessProxy) -> bool: assert type(process) is ProcessProxy return process.ptype != ProcessType.Unknown return list(filter(is_aux, self.child_processes)) @property def child_processes(self) -> typing.List[ProcessProxy]: """ Returns a list of all child processes. Each process is represented by :class:`.ProcessProxy` object. """ # get a list of postmaster's children x = self._get_node_state() assert type(x) is utils.PostgresNodeState if x.pid is None: assert x.node_status != NodeStatus.Running RaiseError.node_err__cant_enumerate_child_processes( x.node_status ) assert x.node_status == NodeStatus.Running assert type(x.pid) is int return self._get_child_processes(x.pid) def _get_child_processes(self, pid: int) -> typing.List[ProcessProxy]: assert type(pid) is int assert isinstance(self._os_ops, OsOperations) # get a list of postmaster's children children = self._os_ops.get_process_children(pid) return [ProcessProxy(p) for p in children] @property def source_walsender(self): """ Returns master's walsender feeding this replica. """ sql = """ select pid from pg_catalog.pg_stat_replication where application_name = %s """ if self.master is None: raise TestgresException("Node doesn't have a master") assert type(self.master) is PostgresNode # master should be on the same host assert self.master.host == self.host with self.master.connect() as con: for row in con.execute(sql, self.name): for child in self.master.auxiliary_processes: if child.pid == int(row[0]): return child msg = "Master doesn't send WAL to {}".format(self.name) raise TestgresException(msg) @property def master(self): return self._master @property def base_dir(self): if not self._base_dir: self._base_dir = self.os_ops.mkdtemp(prefix=self._prefix or TMP_NODE) # NOTE: it's safe to create a new dir if not self.os_ops.path_exists(self._base_dir): self.os_ops.makedirs(self._base_dir) return self._base_dir @property def bin_dir(self): if not self._bin_dir: self._bin_dir = os.path.dirname(get_bin_path2(self.os_ops, "pg_config")) return self._bin_dir @property def logs_dir(self): assert self._os_ops is not None assert isinstance(self._os_ops, OsOperations) path = self._os_ops.build_path(self.base_dir, LOGS_DIR) assert type(path) is str # NOTE: it's safe to create a new dir if not self.os_ops.path_exists(path): self.os_ops.makedirs(path) return path @property def data_dir(self): assert self._os_ops is not None assert isinstance(self._os_ops, OsOperations) # NOTE: we can't run initdb without user's args path = self._os_ops.build_path(self.base_dir, DATA_DIR) assert type(path) is str return path @property def utils_log_file(self): assert self._os_ops is not None assert isinstance(self._os_ops, OsOperations) path = self._os_ops.build_path(self.logs_dir, UTILS_LOG_FILE) assert type(path) is str return path @property def pg_log_file(self): assert self._os_ops is not None assert isinstance(self._os_ops, OsOperations) path = self._os_ops.build_path(self.logs_dir, PG_LOG_FILE) assert type(path) is str return path @property def version(self): """ Return PostgreSQL version for this node. Returns: Instance of :class:`distutils.version.LooseVersion`. """ return self._pg_version def _try_shutdown(self, max_attempts, with_force=False): assert type(max_attempts) is int assert type(with_force) is bool assert max_attempts > 0 attempts = 0 # try stopping server N times while attempts < max_attempts: attempts += 1 try: self.stop() except ExecUtilException: continue # one more time except Exception: eprint('cannot stop node {}'.format(self.name)) break return # OK # If force stopping is enabled and PID is valid if not with_force: return node_pid = self.pid assert node_pid is not None assert type(node_pid) is int if node_pid == 0: return # TODO: [2025-02-28] It is really the old ugly code. We have to rewrite it! ps_command = ['ps', '-o', 'pid=', '-p', str(node_pid)] ps_output = self.os_ops.exec_command(cmd=ps_command, shell=True, ignore_errors=True).decode('utf-8') assert type(ps_output) is str if ps_output == "": return if ps_output != str(node_pid): __class__._throw_bugcheck__unexpected_result_of_ps( ps_output, ps_command) try: eprint('Force stopping node {0} with PID {1}'.format(self.name, node_pid)) self.os_ops.kill(node_pid, signal.SIGKILL) except Exception: # The node has already stopped pass # Check that node stopped - print only column pid without headers ps_output = self.os_ops.exec_command(cmd=ps_command, shell=True, ignore_errors=True).decode('utf-8') assert type(ps_output) is str if ps_output == "": eprint('Node {0} has been stopped successfully.'.format(self.name)) return if ps_output == str(node_pid): eprint('Failed to stop node {0}.'.format(self.name)) return __class__._throw_bugcheck__unexpected_result_of_ps( ps_output, ps_command) @staticmethod def _throw_bugcheck__unexpected_result_of_ps(result, cmd): assert type(result) is str assert type(cmd) is list errLines = [] errLines.append("[BUG CHECK] Unexpected result of command ps:") errLines.append(result) errLines.append("-----") errLines.append("Command line is {0}".format(cmd)) raise RuntimeError("\n".join(errLines)) def _assign_master(self, master): """NOTE: this is a private method!""" # now this node has a master self._master = master def _create_recovery_conf(self, username, slot=None): """NOTE: this is a private method!""" # fetch master of this node master = self.master assert master is not None conninfo = { "application_name": self.name, "port": master.port, "user": username } # yapf: disable # host is tricky try: import ipaddress ipaddress.ip_address(master.host) conninfo["hostaddr"] = master.host except ValueError: conninfo["host"] = master.host line = ( "primary_conninfo='{}'\n" ).format(options_string(**conninfo)) # yapf: disable # Since 12 recovery.conf had disappeared if self.version >= PgVer('12'): assert self._os_ops is not None assert isinstance(self._os_ops, OsOperations) signal_name = self._os_ops.build_path(self.data_dir, "standby.signal") assert type(signal_name) is str self.os_ops.touch(signal_name) else: line += "standby_mode=on\n" if slot: # Connect to master for some additional actions with master.connect(username=username) as con: # check if slot already exists res = con.execute( """ select exists ( select from pg_catalog.pg_replication_slots where slot_name = %s ) """, slot) if res[0][0]: raise TestgresException( "Slot '{}' already exists".format(slot)) # TODO: we should drop this slot after replica's cleanup() con.execute( """ select pg_catalog.pg_create_physical_replication_slot(%s) """, slot) line += "primary_slot_name={}\n".format(slot) if self.version >= PgVer('12'): self.append_conf(line=line) else: self.append_conf(filename=RECOVERY_CONF_FILE, line=line) def _maybe_start_logger(self): if testgres_config.use_python_logging: # spawn new logger if it doesn't exist or is stopped if not self._logger or not self._logger.is_alive(): self._logger = TestgresLogger(self.name, self.pg_log_file) self._logger.start() def _maybe_stop_logger(self): if self._logger: self._logger.stop() def _collect_special_files(self) -> typing.List[typing.Tuple[str, bytes]]: result = [] # list of important files + last N lines assert self._os_ops is not None assert isinstance(self._os_ops, OsOperations) files = [ (self._os_ops.build_path(self.data_dir, PG_CONF_FILE), 0), (self._os_ops.build_path(self.data_dir, PG_AUTO_CONF_FILE), 0), (self._os_ops.build_path(self.data_dir, RECOVERY_CONF_FILE), 0), (self._os_ops.build_path(self.data_dir, HBA_CONF_FILE), 0), (self.pg_log_file, testgres_config.error_log_lines) ] # yapf: disable for f, num_lines in files: # skip missing files if not self.os_ops.path_exists(f): continue file_lines = self.os_ops.readlines(f, num_lines, binary=True, encoding=None) lines = b''.join(file_lines) # fill list result.append((f, lines)) return result def init(self, initdb_params=None, cached=True, **kwargs): """ Perform initdb for this node. Args: initdb_params: parameters for initdb (list). fsync: should this node use fsync to keep data safe? unix_sockets: should we enable UNIX sockets? allow_streaming: should this node add a hba entry for replication? Returns: This instance of :class:`.PostgresNode` """ # initialize this PostgreSQL node assert self._os_ops is not None assert isinstance(self._os_ops, OsOperations) cached_initdb( data_dir=self.data_dir, logfile=self.utils_log_file, os_ops=self._os_ops, params=initdb_params, bin_path=self.bin_dir, cached=False) # initialize default config files self.default_conf(**kwargs) return self def default_conf(self, fsync=False, unix_sockets=True, allow_streaming=True, allow_logical=False, log_statement='all'): """ Apply default settings to this node. Args: fsync: should this node use fsync to keep data safe? unix_sockets: should we enable UNIX sockets? allow_streaming: should this node add a hba entry for replication? allow_logical: can this node be used as a logical replication publisher? log_statement: one of ('all', 'off', 'mod', 'ddl'). Returns: This instance of :class:`.PostgresNode`. """ assert self._os_ops is not None assert isinstance(self._os_ops, OsOperations) postgres_conf = self._os_ops.build_path(self.data_dir, PG_CONF_FILE) hba_conf = self._os_ops.build_path(self.data_dir, HBA_CONF_FILE) # filter lines in hba file # get rid of comments and blank lines hba_conf_file = self.os_ops.readlines(hba_conf) lines = [ s for s in hba_conf_file if len(s.strip()) > 0 and not s.startswith('#') ] # write filtered lines self.os_ops.write(hba_conf, lines, truncate=True) # replication-related settings if allow_streaming: # get auth method for host or local users def get_auth_method(t): return next((s.split()[-1] for s in lines if s.startswith(t)), 'trust') # get auth methods auth_local = get_auth_method('local') auth_host = get_auth_method('host') subnet_base = ".".join(self.os_ops.host.split('.')[:-1] + ['0']) new_lines = [ u"local\treplication\tall\t\t\t{}\n".format(auth_local), u"host\treplication\tall\t127.0.0.1/32\t{}\n".format(auth_host), u"host\treplication\tall\t::1/128\t\t{}\n".format(auth_host), u"host\treplication\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host), u"host\tall\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host), u"host\tall\tall\tall\t{}\n".format(auth_host), u"host\treplication\tall\tall\t{}\n".format(auth_host) ] # yapf: disable # write missing lines self.os_ops.write(hba_conf, new_lines) # overwrite config file self.os_ops.write(postgres_conf, '', truncate=True) self.append_conf(fsync=fsync, max_worker_processes=MAX_WORKER_PROCESSES, log_statement=log_statement, listen_addresses=self.host, port=self.port) # yapf:disable # common replication settings if allow_streaming or allow_logical: self.append_conf(max_replication_slots=MAX_REPLICATION_SLOTS, max_wal_senders=MAX_WAL_SENDERS) # yapf: disable # binary replication if allow_streaming: # select a proper wal_level for PostgreSQL wal_level = 'replica' if self._pg_version >= PgVer('9.6') else 'hot_standby' if self._pg_version < PgVer('13'): self.append_conf(hot_standby=True, wal_keep_segments=WAL_KEEP_SEGMENTS, wal_level=wal_level) # yapf: disable else: self.append_conf(hot_standby=True, wal_keep_size=WAL_KEEP_SIZE, wal_level=wal_level) # yapf: disable # logical replication if allow_logical: if self._pg_version < PgVer('10'): raise InitNodeException("Logical replication is only " "available on PostgreSQL 10 and newer") self.append_conf( max_logical_replication_workers=MAX_LOGICAL_REPLICATION_WORKERS, wal_level='logical') # disable UNIX sockets if asked to if not unix_sockets: self.append_conf(unix_socket_directories='') return self @method_decorator(positional_args_hack(['filename', 'line'])) def append_conf(self, line='', filename=PG_CONF_FILE, **kwargs): """ Append line to a config file. Args: line: string to be appended to config. filename: config file (postgresql.conf by default). **kwargs: named config options. Returns: This instance of :class:`.PostgresNode`. Examples: >>> append_conf(fsync=False) >>> append_conf('log_connections = yes') >>> append_conf(random_page_cost=1.5, fsync=True, ...) >>> append_conf('postgresql.conf', 'synchronous_commit = off') """ lines = [line] for option, value in iteritems(kwargs): if isinstance(value, bool): value = 'on' if value else 'off' elif not str(value).replace('.', '', 1).isdigit(): value = "'{}'".format(value) if value == '*': lines.append("{} = '*'".format(option)) else: # format a new config line lines.append('{} = {}'.format(option, value)) config_name = self._os_ops.build_path(self.data_dir, filename) conf_text = '' for line in lines: conf_text += text_type(line) + '\n' self.os_ops.write(config_name, conf_text) return self def status(self): """ Check this node's status. Returns: An instance of :class:`.NodeStatus`. """ x = self._get_node_state() assert type(x) is utils.PostgresNodeState return x.node_status def _get_node_state(self) -> utils.PostgresNodeState: return utils.get_pg_node_state( self._os_ops, self.bin_dir, self.data_dir, self.utils_log_file ) def get_control_data(self): """ Return contents of pg_control file. """ # this one is tricky (blame PG 9.4) _params = [self._get_bin_path("pg_controldata")] _params += ["-D"] if self._pg_version >= PgVer('9.5') else [] _params += [self.data_dir] data = execute_utility2(self.os_ops, _params, self.utils_log_file) out_dict = {} for line in data.splitlines(): key, _, value = line.partition(':') out_dict[key.strip()] = value.strip() return out_dict def slow_start(self, replica=False, dbname='template1', username=None, max_attempts=0, exec_env=None): """ Starts the PostgreSQL instance and then polls the instance until it reaches the expected state (primary or replica). The state is checked using the pg_is_in_recovery() function. Args: dbname: username: replica: If True, waits for the instance to be in recovery (i.e., replica mode). If False, waits for the instance to be in primary mode. Default is False. max_attempts: """ assert exec_env is None or type(exec_env) is dict self.start(exec_env=exec_env) try: if replica: query = 'SELECT pg_is_in_recovery()' else: query = 'SELECT not pg_is_in_recovery()' # Call poll_query_until until the expected value is returned suppressed_exceptions = { InternalError, QueryException, ProgrammingError, OperationalError } self.poll_query_until( query=query, dbname=dbname, username=username or self.os_ops.username, suppress=suppressed_exceptions, max_attempts=max_attempts, ) except: # noqa: E722 self.stop() raise return def start( self, params: typing.Optional[typing.List[str]] = None, wait: bool = True, exec_env: typing.Optional[typing.Dict] = None, ) -> PostgresNode: """ Starts the PostgreSQL node using pg_ctl and set flag 'is_started'. By default, it waits for the operation to complete before returning. Optionally, it can return immediately without waiting for the start operation to complete by setting the `wait` parameter to False. Args: params: additional arguments for pg_ctl. wait: wait until operation completes. Returns: This instance of :class:`.PostgresNode`. """ assert params is None or type(params) is list assert type(wait) is bool assert exec_env is None or type(exec_env) is dict self._start(params, wait, exec_env) if not wait: # Postmaster process is starting in background self._manually_started_pm_pid = __class__._C_PM_PID__IS_NOT_DETECTED else: self._manually_started_pm_pid = self._get_node_state().pid if self._manually_started_pm_pid is None: self._raise_cannot_start_node(None, "Cannot detect postmaster pid.") assert type(self._manually_started_pm_pid) is int return self def start2( self, params: typing.Optional[typing.List[str]] = None, wait: bool = True, exec_env: typing.Optional[typing.Dict] = None, ) -> None: """ Starts the PostgreSQL node using pg_ctl. By default, it waits for the operation to complete before returning. Optionally, it can return immediately without waiting for the start operation to complete by setting the `wait` parameter to False. Args: params: additional arguments for pg_ctl. wait: wait until operation completes. Returns: None. """ assert params is None or type(params) is list assert type(wait) is bool assert exec_env is None or type(exec_env) is dict self._start(params, wait, exec_env) return def _start( self, params: typing.Optional[typing.List[str]] = None, wait: bool = True, exec_env: typing.Optional[typing.Dict] = None, ) -> None: assert params is None or type(params) is list assert type(wait) is bool assert exec_env is None or type(exec_env) is dict assert __class__._C_MAX_START_ATEMPTS > 1 if self._port is None: raise InvalidOperationException("Can't start PostgresNode. Port is not defined.") assert type(self._port) is int _params = [ self._get_bin_path("pg_ctl"), "start", "-D", self.data_dir, "-l", self.pg_log_file, "-w" if wait else '-W', # --wait or --no-wait ] if params is not None: assert type(params) is list _params += params def LOCAL__start_node(): # 'error' will be None on Windows _, _, error = execute_utility2(self.os_ops, _params, self.utils_log_file, verbose=True, exec_env=exec_env) assert error is None or type(error) is str if error and 'does not exist' in error: raise Exception(error) def LOCAL__raise_cannot_start_node__std(from_exception): assert isinstance(from_exception, Exception) self._raise_cannot_start_node(from_exception, 'Cannot start node') if not self._should_free_port: try: LOCAL__start_node() except Exception as e: LOCAL__raise_cannot_start_node__std(e) else: assert self._should_free_port assert self._port_manager is not None assert isinstance(self._port_manager, PortManager) assert __class__._C_MAX_START_ATEMPTS > 1 log_reader = PostgresNodeLogReader(self, from_beginnig=False) nAttempt = 0 timeout = 1 while True: assert nAttempt >= 0 assert nAttempt < __class__._C_MAX_START_ATEMPTS nAttempt += 1 try: LOCAL__start_node() except Exception as e: assert nAttempt > 0 assert nAttempt <= __class__._C_MAX_START_ATEMPTS if nAttempt == __class__._C_MAX_START_ATEMPTS: self._raise_cannot_start_node(e, "Cannot start node after multiple attempts.") is_it_port_conflict = PostgresNodeUtils.delect_port_conflict(log_reader) if not is_it_port_conflict: LOCAL__raise_cannot_start_node__std(e) logging.warning( "Detected a conflict with using the port {0}. Trying another port after a {1}-second sleep...".format(self._port, timeout) ) time.sleep(timeout) timeout = min(2 * timeout, 5) cur_port = self._port new_port = self._port_manager.reserve_port() # can raise try: options = {'port': new_port} self.set_auto_conf(options) except: # noqa: E722 self._port_manager.release_port(new_port) raise self._port = new_port self._port_manager.release_port(cur_port) continue break self._maybe_start_logger() return def _raise_cannot_start_node( self, from_exception: typing.Optional[Exception], msg: str ): assert from_exception is None or isinstance(from_exception, Exception) assert type(msg) is str files = self._collect_special_files() raise_from(StartNodeException(msg, files), from_exception) def stop(self, params=[], wait=True): """ Stops the PostgreSQL node using pg_ctl if the node has been started. Args: params: A list of additional arguments for pg_ctl. Defaults to None. wait: If True, waits until the operation is complete. Defaults to True. Returns: This instance of :class:`.PostgresNode`. """ _params = [ self._get_bin_path("pg_ctl"), "-D", self.data_dir, "-w" if wait else '-W', # --wait or --no-wait "stop" ] + params # yapf: disable execute_utility2(self.os_ops, _params, self.utils_log_file) self._manually_started_pm_pid = None self._maybe_stop_logger() return self def kill(self, someone=None): """ Kills the PostgreSQL node or a specified auxiliary process if the node is running. Args: someone: A key to the auxiliary process in the auxiliary_pids dictionary. If None, the main PostgreSQL node process will be killed. Defaults to None. """ x = self._get_node_state() assert type(x) is utils.PostgresNodeState if x.node_status != NodeStatus.Running: RaiseError.node_err__cant_kill(x.node_status) assert False assert x.node_status == NodeStatus.Running assert type(x.pid) is int sig = signal.SIGKILL if os.name != 'nt' else signal.SIGBREAK if someone is None: self._os_ops.kill(x.pid, sig) self._manually_started_pm_pid = None else: childs = self._get_child_processes(x.pid) for c in childs: assert type(c) is ProcessProxy if c.ptype == someone: self._os_ops.kill(c.process.pid, sig) continue return def restart(self, params=[]): """ Restart this node using pg_ctl. Args: params: additional arguments for pg_ctl. Returns: This instance of :class:`.PostgresNode`. """ _params = [ self._get_bin_path("pg_ctl"), "-D", self.data_dir, "-l", self.pg_log_file, "-w", # wait "restart" ] + params # yapf: disable try: error_code, out, error = execute_utility2(self.os_ops, _params, self.utils_log_file, verbose=True) if error and 'could not start server' in error: raise ExecUtilException except ExecUtilException as e: msg = 'Cannot restart node' files = self._collect_special_files() raise_from(StartNodeException(msg, files), e) self._maybe_start_logger() return self def reload(self, params=[]): """ Asynchronously reload config files using pg_ctl. Args: params: additional arguments for pg_ctl. Returns: This instance of :class:`.PostgresNode`. """ _params = [ self._get_bin_path("pg_ctl"), "-D", self.data_dir, "reload" ] + params # yapf: disable execute_utility2(self.os_ops, _params, self.utils_log_file) return self def promote(self, dbname=None, username=None): """ Promote standby instance to master using pg_ctl. For PostgreSQL versions below 10 some additional actions required to ensure that instance became writable and hence `dbname` and `username` parameters may be needed. Returns: This instance of :class:`.PostgresNode`. """ _params = [ self._get_bin_path("pg_ctl"), "-D", self.data_dir, "-w", # wait "promote" ] # yapf: disable execute_utility2(self.os_ops, _params, self.utils_log_file) # for versions below 10 `promote` is asynchronous so we need to wait # until it actually becomes writable if self._pg_version < PgVer('10'): check_query = "SELECT pg_is_in_recovery()" self.poll_query_until(query=check_query, expected=False, dbname=dbname, username=username, max_attempts=0) # infinite # node becomes master itself self._master = None return self def pg_ctl(self, params): """ Invoke pg_ctl with params. Args: params: arguments for pg_ctl. Returns: Stdout + stderr of pg_ctl. """ _params = [ self._get_bin_path("pg_ctl"), "-D", self.data_dir, "-w" # wait ] + params # yapf: disable return execute_utility2(self.os_ops, _params, self.utils_log_file) def release_resources(self): """ Release resorces owned by this node. """ return self._release_resources() def free_port(self): """ Reclaim port owned by this node. NOTE: this method does not release manually defined port but reset it. """ return self._free_port() def cleanup(self, max_attempts=3, full=False, release_resources=False): """ Stop node if needed and remove its data/logs directory. NOTE: take a look at TestgresConfig.node_cleanup_full. Args: max_attempts: how many times should we try to stop()? full: clean full base dir Returns: This instance of :class:`.PostgresNode`. """ self._try_shutdown(max_attempts) # choose directory to be removed if testgres_config.node_cleanup_full or full: rm_dir = self.base_dir # everything else: rm_dir = self.data_dir # just data, save logs self.os_ops.rmdirs(rm_dir, ignore_errors=False) if release_resources: self._release_resources() return self @method_decorator(positional_args_hack(['dbname', 'query'])) def psql(self, query=None, filename=None, dbname=None, username=None, input=None, host: typing.Optional[str] = None, port: typing.Optional[int] = None, **variables): """ Execute a query using psql. Args: query: query to be executed. filename: file with a query. dbname: database name to connect to. username: database user name. input: raw input to be passed. host: an explicit host of server. port: an explicit port of server. **variables: vars to be set before execution. Returns: A tuple of (code, stdout, stderr). Examples: >>> psql('select 1') >>> psql('postgres', 'select 2') >>> psql(query='select 3', ON_ERROR_STOP=1) """ assert host is None or type(host) is str assert port is None or type(port) is int assert type(variables) is dict return self._psql( ignore_errors=True, query=query, filename=filename, dbname=dbname, username=username, input=input, host=host, port=port, **variables ) def _psql( self, ignore_errors, query=None, filename=None, dbname=None, username=None, input=None, host: typing.Optional[str] = None, port: typing.Optional[int] = None, **variables): assert host is None or type(host) is str assert port is None or type(port) is int assert type(variables) is dict # # We do not support encoding. It may be added later. Ok? # if input is None: pass elif type(input) is bytes: pass else: raise Exception("Input data must be None or bytes.") if host is None: host = self.host if port is None: port = self.port assert host is not None assert port is not None assert type(host) is str assert type(port) is int psql_params = [ self._get_bin_path("psql"), "-p", str(port), "-h", host, "-U", username or self.os_ops.username, "-d", dbname or default_dbname(), "-X", # no .psqlrc "-A", # unaligned output "-t", # print rows only "-q" # run quietly ] # yapf: disable # set variables before execution for key, value in iteritems(variables): psql_params.extend(["--set", '{}={}'.format(key, value)]) # select query source if query: psql_params.extend(("-c", query)) elif filename: psql_params.extend(("-f", filename)) else: raise QueryException('Query or filename must be provided') return self.os_ops.exec_command( psql_params, verbose=True, input=input, stderr=subprocess.PIPE, stdout=subprocess.PIPE, ignore_errors=ignore_errors) @method_decorator(positional_args_hack(['dbname', 'query'])) def safe_psql(self, query=None, expect_error=False, **kwargs): """ Execute a query using psql. Args: query: query to be executed. filename: file with a query. dbname: database name to connect to. username: database user name. input: raw input to be passed. expect_error: if True - fail if we didn't get ret if False - fail if we got ret **kwargs are passed to psql(). Returns: psql's output as str. """ assert type(kwargs) is dict assert "ignore_errors" not in kwargs.keys() assert "expect_error" not in kwargs.keys() # force this setting kwargs['ON_ERROR_STOP'] = 1 try: ret, out, err = self._psql(ignore_errors=False, query=query, **kwargs) except ExecUtilException as e: if not expect_error: raise QueryException(e.message, query) if type(e.error) is bytes: return e.error.decode("utf-8") # throw # [2024-12-09] This situation is not expected assert False return e.error if expect_error: raise InvalidOperationException("Exception was expected, but query finished successfully: `{}`.".format(query)) return out def dump(self, filename=None, dbname=None, username=None, format=DumpFormat.Plain, options=None): """ Dump database into a file using pg_dump. NOTE: the file is not removed automatically. Args: filename: database dump taken by pg_dump. dbname: database name to connect to. username: database user name. format: format argument plain/custom/directory/tar. options: additional options for pg_dump (list). Returns: Path to a file containing dump. """ # Check arguments if not isinstance(format, DumpFormat): try: format = DumpFormat(format) except ValueError: msg = 'Invalid format "{}"'.format(format) raise BackupException(msg) # Generate tmpfile or tmpdir def tmpfile(): if format == DumpFormat.Directory: fname = self.os_ops.mkdtemp(prefix=TMP_DUMP) else: fname = self.os_ops.mkstemp(prefix=TMP_DUMP) return fname filename = filename or tmpfile() _params = [ self._get_bin_path("pg_dump"), "-p", str(self.port), "-h", self.host, "-f", filename, "-U", username or self.os_ops.username, "-d", dbname or default_dbname(), "-F", format.value ] # yapf: disable # Add additional options if provided if options: _params.extend(options) execute_utility2(self.os_ops, _params, self.utils_log_file) return filename def restore(self, filename, dbname=None, username=None): """ Restore database from pg_dump's file. Args: filename: database dump taken by pg_dump in custom/directory/tar formats. dbname: database name to connect to. username: database user name. """ # Set default arguments dbname = dbname or default_dbname() username = username or self.os_ops.username _params = [ self._get_bin_path("pg_restore"), "-p", str(self.port), "-h", self.host, "-U", username, "-d", dbname, filename ] # yapf: disable # try pg_restore if dump is binary format, and psql if not try: execute_utility2(self.os_ops, _params, self.utils_log_name) except ExecUtilException: self.psql(filename=filename, dbname=dbname, username=username) @method_decorator(positional_args_hack(['dbname', 'query'])) def poll_query_until(self, query, dbname=None, username=None, max_attempts=0, sleep_time: typing.Union[int, float] = 1, expected=True, commit=True, suppress=None): """ Run a query once per second until it returns 'expected'. Query should return a single value (1 row, 1 column). Args: query: query to be executed. dbname: database name to connect to. username: database user name. max_attempts: how many times should we try? 0 == infinite sleep_time: how much should we sleep after a failure? expected: what should be returned to break the cycle? commit: should (possible) changes be committed? suppress: a collection of exceptions to be suppressed. Examples: >>> poll_query_until('select true') >>> poll_query_until('postgres', "select now() > '01.01.2018'") >>> poll_query_until('select false', expected=True, max_attempts=4) >>> poll_query_until('select 1', suppress={testgres.OperationalError}) """ # sanity checks assert type(max_attempts) is int assert max_attempts >= 0 assert type(sleep_time) in [int, float] assert sleep_time > 0 attempts = 0 while max_attempts == 0 or attempts < max_attempts: try: res = self.execute(dbname=dbname, query=query, username=username, commit=commit) if expected is None and res is None: return # done if res is None: raise QueryException('Query returned None', query) # result set is not empty if len(res): if len(res[0]) == 0: raise QueryException('Query returned 0 columns', query) if res[0][0] == expected: return # done # empty result set is considered as None elif expected is None: return # done except tuple(suppress or []): logging.info(f"Trying execute, attempt {attempts + 1}.\nQuery: {query}") pass # we're suppressing them time.sleep(sleep_time) attempts += 1 raise QueryTimeoutException('Query timeout', query) @method_decorator(positional_args_hack(['dbname', 'query'])) def execute(self, query, dbname=None, username=None, password=None, commit=True): """ Execute a query and return all rows as list. Args: query: query to be executed. dbname: database name to connect to. username: database user name. password: user's password. commit: should we commit this query? Returns: A list of tuples representing rows. """ with self.connect(dbname=dbname, username=username, password=password, autocommit=commit) as node_con: # yapf: disable res = node_con.execute(query) return res def backup(self, **kwargs): """ Perform pg_basebackup. Args: username: database user name. xlog_method: a method for collecting the logs ('fetch' | 'stream'). base_dir: the base directory for data files and logs Returns: A smart object of type NodeBackup. """ return NodeBackup(node=self, **kwargs) def replicate(self, name=None, slot=None, **kwargs): """ Create a binary replica of this node. Args: name: replica's application name. slot: create a replication slot with the specified name. username: database user name. xlog_method: a method for collecting the logs ('fetch' | 'stream'). base_dir: the base directory for data files and logs """ # transform backup into a replica with clean_on_error(self.backup(**kwargs)) as backup: return backup.spawn_replica(name=name, destroy=True, slot=slot) def set_synchronous_standbys(self, standbys): """ Set standby synchronization options. This corresponds to `synchronous_standby_names `_ option. Note that :meth:`~.PostgresNode.reload` or :meth:`~.PostgresNode.restart` is needed for changes to take place. Args: standbys: either :class:`.First` or :class:`.Any` object specifying synchronization parameters or just a plain list of :class:`.PostgresNode`s replicas which would be equivalent to passing ``First(1, )``. For PostgreSQL 9.5 and below it is only possible to specify a plain list of standbys as `FIRST` and `ANY` keywords aren't supported. Example:: from testgres import get_new_node, First master = get_new_node().init().start() with master.replicate().start() as standby: master.append_conf("synchronous_commit = remote_apply") master.set_synchronous_standbys(First(1, [standby])) master.restart() """ if self._pg_version >= PgVer('9.6'): if isinstance(standbys, Iterable): standbys = First(1, standbys) else: if isinstance(standbys, Iterable): standbys = u", ".join(u"\"{}\"".format(r.name) for r in standbys) else: raise TestgresException("Feature isn't supported in " "Postgres 9.5 and below") self.append_conf("synchronous_standby_names = '{}'".format(standbys)) def catchup(self, dbname=None, username=None): """ Wait until async replica catches up with its master. """ if not self.master: raise TestgresException("Node doesn't have a master") if self._pg_version >= PgVer('10'): poll_lsn = "select pg_catalog.pg_current_wal_lsn()::text" wait_lsn = "select pg_catalog.pg_last_wal_replay_lsn() >= '{}'::pg_lsn" else: poll_lsn = "select pg_catalog.pg_current_xlog_location()::text" wait_lsn = "select pg_catalog.pg_last_xlog_replay_location() >= '{}'::pg_lsn" try: # fetch latest LSN lsn = self.master.execute(query=poll_lsn, dbname=dbname, username=username)[0][0] # yapf: disable # wait until this LSN reaches replica self.poll_query_until(query=wait_lsn.format(lsn), dbname=dbname, username=username, max_attempts=0) # infinite except Exception as e: raise_from(CatchUpException("Failed to catch up."), e) def publish(self, name, **kwargs): """ Create publication for logical replication Args: pubname: publication name tables: tables names list dbname: database name where objects or interest are located username: replication username """ return Publication(name=name, node=self, **kwargs) def subscribe(self, publication, name, dbname=None, username=None, **params): """ Create subscription for logical replication Args: name: subscription name publication: publication object obtained from publish() dbname: database name username: replication username params: subscription parameters (see documentation on `CREATE SUBSCRIPTION `_ for details) """ # yapf: disable return Subscription(name=name, node=self, publication=publication, dbname=dbname, username=username, **params) # yapf: enable def pgbench(self, dbname=None, username=None, stdout=None, stderr=None, options=None): """ Spawn a pgbench process. Args: dbname: database name to connect to. username: database user name. stdout: stdout file to be used by Popen. stderr: stderr file to be used by Popen. options: additional options for pgbench (list). Returns: Process created by subprocess.Popen. """ if options is None: options = [] dbname = dbname or default_dbname() _params = [ self._get_bin_path("pgbench"), "-p", str(self.port), "-h", self.host, "-U", username or self.os_ops.username ] + options # yapf: disable # should be the last one _params.append(dbname) proc = self.os_ops.exec_command(_params, stdout=stdout, stderr=stderr, wait_exit=True, get_process=True) return proc def pgbench_with_wait(self, dbname=None, username=None, stdout=None, stderr=None, options=None): """ Do pgbench command and wait. Args: dbname: database name to connect to. username: database user name. stdout: stdout file to be used by Popen. stderr: stderr file to be used by Popen. options: additional options for pgbench (list). """ if options is None: options = [] with self.pgbench(dbname, username, stdout, stderr, options) as pgbench: pgbench.wait() return def pgbench_init(self, **kwargs): """ Small wrapper for pgbench_run(). Sets initialize=True. Returns: This instance of :class:`.PostgresNode`. """ self.pgbench_run(initialize=True, **kwargs) return self def pgbench_run(self, dbname=None, username=None, options=[], **kwargs): """ Run pgbench with some options. This event is logged (see self.utils_log_file). Args: dbname: database name to connect to. username: database user name. options: additional options for pgbench (list). **kwargs: named options for pgbench. Run pgbench --help to learn more. Returns: Stdout produced by pgbench. Examples: >>> pgbench_run(initialize=True, scale=2) >>> pgbench_run(time=10) """ dbname = dbname or default_dbname() _params = [ self._get_bin_path("pgbench"), "-p", str(self.port), "-h", self.host, "-U", username or self.os_ops.username ] + options # yapf: disable for key, value in iteritems(kwargs): # rename keys for pgbench key = key.replace('_', '-') # append option if not isinstance(value, bool): _params.append('--{}={}'.format(key, value)) else: assert value is True # just in case _params.append('--{}'.format(key)) # should be the last one _params.append(dbname) return execute_utility2(self.os_ops, _params, self.utils_log_file) def connect(self, dbname=None, username=None, password=None, autocommit=False): """ Connect to a database. Args: dbname: database name to connect to. username: database user name. password: user's password. autocommit: commit each statement automatically. Also it should be set to `True` for statements requiring to be run outside a transaction? such as `VACUUM` or `CREATE DATABASE`. Returns: An instance of :class:`.NodeConnection`. """ return NodeConnection(node=self, dbname=dbname, username=username, password=password, autocommit=autocommit) # yapf: disable def table_checksum( self, table: str, dbname: str = "postgres" ) -> int: assert type(table) is str assert type(dbname) is str cn = self.connect(dbname=dbname) assert type(cn) is NodeConnection try: sum = __class__._table_checksum__use_cn(cn, table) assert type(sum) is int finally: assert type(cn) is NodeConnection cn.close() assert type(sum) is int return sum sm_pgbench_tables = [ 'pgbench_branches', 'pgbench_tellers', 'pgbench_accounts', 'pgbench_history' ] def pgbench_table_checksums( self, dbname: str = "postgres", pgbench_tables: typing.Iterable[str] = sm_pgbench_tables ) -> typing.Set[typing.Tuple[str, int]]: assert type(dbname) is str r1 = self._tables_checksum(dbname, pgbench_tables) assert type(r1) is list r2 = set(r1) assert type(r2) is set return r2 def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): """ Update or remove configuration options in the specified configuration file, updates the options specified in the options dictionary, removes any options specified in the rm_options set, and writes the updated configuration back to the file. Args: options (dict): A dictionary containing the options to update or add, with the option names as keys and their values as values. config (str, optional): The name of the configuration file to update. Defaults to 'postgresql.auto.conf'. rm_options (set, optional): A set containing the names of the options to remove. Defaults to an empty set. """ assert self._os_ops is not None assert isinstance(self._os_ops, OsOperations) # parse postgresql.auto.conf path = self.os_ops.build_path(self.data_dir, config) lines = self.os_ops.readlines(path) current_options = {} current_directives = [] for line in lines: # ignore comments if line.startswith('#'): continue if line.strip() == '': continue if line.startswith('include'): current_directives.append(line) continue name, var = line.partition('=')[::2] name = name.strip() # Remove options specified in rm_options list if name in rm_options: continue current_options[name] = var for option in options: assert type(option) is str assert option != "" assert option.strip() == option value = options[option] valueType = type(value) if valueType is str: value = __class__._escape_config_value(value) elif valueType is bool: value = "on" if value else "off" current_options[option] = value auto_conf = '' for option in current_options: auto_conf += option + " = " + str(current_options[option]) + "\n" for directive in current_directives: auto_conf += directive + "\n" self.os_ops.write(path, auto_conf, truncate=True) def upgrade_from(self, old_node, options=None, expect_error=False): """ Upgrade this node from an old node using pg_upgrade. Args: old_node: An instance of PostgresNode representing the old node. """ assert isinstance(self._os_ops, OsOperations) if not self._os_ops.path_exists(old_node.data_dir): raise Exception("Old node must be initialized") if not self._os_ops.path_exists(self.data_dir): self.init() if not options: options = [] pg_upgrade_binary = self._get_bin_path("pg_upgrade") if not self._os_ops.path_exists(pg_upgrade_binary): raise Exception("pg_upgrade does not exist in the new node's binary path") upgrade_command = [ pg_upgrade_binary, "--old-bindir", old_node.bin_dir, "--new-bindir", self.bin_dir, "--old-datadir", old_node.data_dir, "--new-datadir", self.data_dir, "--old-port", str(old_node.port), "--new-port", str(self.port) ] upgrade_command += options return self.os_ops.exec_command(upgrade_command, expect_error=expect_error) def _release_resources(self): self._free_port() def _free_port(self): assert type(self._should_free_port) is bool if not self._should_free_port: self._port = None else: assert type(self._port) is int assert self._port_manager is not None assert isinstance(self._port_manager, PortManager) port = self._port self._should_free_port = False self._port = None self._port_manager.release_port(port) def _get_bin_path(self, filename): assert self._os_ops is not None assert isinstance(self._os_ops, OsOperations) if self.bin_dir: bin_path = self._os_ops.build_path(self.bin_dir, filename) else: bin_path = get_bin_path2(self.os_ops, filename) return bin_path @staticmethod def _escape_config_value(value): assert type(value) is str result = "'" for ch in value: if ch == "'": result += "\\'" elif ch == "\n": result += "\\n" elif ch == "\r": result += "\\r" elif ch == "\t": result += "\\t" elif ch == "\b": result += "\\b" elif ch == "\\": result += "\\\\" else: result += ch result += "'" return result def _tables_checksum( self, dbname: str, tables: typing.Iterable[str], ) -> typing.List[typing.Tuple[str, int]]: assert isinstance(tables, typing.Iterable) assert type(dbname) is str result = [] cn = self.connect(dbname=dbname) assert type(cn) is NodeConnection try: cn.begin() for table in tables: assert type(table) is str sum = __class__._table_checksum__use_cn(cn, table) assert type(sum) is int result.append((table, sum)) cn.commit() finally: assert type(cn) is NodeConnection cn.close() assert type(result) is list return result @staticmethod def _table_checksum__use_cn( cn: NodeConnection, table: str, ) -> int: assert type(cn) is NodeConnection assert type(table) is str sum = 0 cursor = cn.connection.cursor() assert cursor is not None try: cursor.execute("SELECT SUM(hashtext(t::text)) FROM {} as t".format( __class__._delim_sql_ident(table) )) row = cursor.fetchone() assert row is not None assert type(row) in [list, tuple] assert len(row) == 1 v = row[0] sum += int(v if v is not None else 0) finally: cursor.close() assert type(sum) is int return sum @staticmethod def _delim_sql_ident(name: str) -> str: assert isinstance(name, str) result = '"' for ch in name: if ch == '"': result = result + '""' else: result = result + ch result = result + '"' return result class PostgresNodeLogReader: class LogInfo: position: int def __init__(self, position: int): self.position = position # -------------------------------------------------------------------- class LogDataBlock: _file_name: str _position: int _data: str def __init__( self, file_name: str, position: int, data: str ): assert type(file_name) is str assert type(position) is int assert type(data) is str assert file_name != "" assert position >= 0 self._file_name = file_name self._position = position self._data = data @property def file_name(self) -> str: assert type(self._file_name) is str assert self._file_name != "" return self._file_name @property def position(self) -> int: assert type(self._position) is int assert self._position >= 0 return self._position @property def data(self) -> str: assert type(self._data) is str return self._data # -------------------------------------------------------------------- _node: PostgresNode _logs: typing.Dict[str, LogInfo] # -------------------------------------------------------------------- def __init__(self, node: PostgresNode, from_beginnig: bool): assert node is not None assert isinstance(node, PostgresNode) assert type(from_beginnig) is bool self._node = node if from_beginnig: self._logs = dict() else: self._logs = self._collect_logs() assert type(self._logs) is dict return def read(self) -> typing.List[LogDataBlock]: assert self._node is not None assert isinstance(self._node, PostgresNode) cur_logs: typing.Dict[str, __class__.LogInfo] = self._collect_logs() assert cur_logs is not None assert type(cur_logs) is dict assert type(self._logs) is dict result = list() for file_name, cur_log_info in cur_logs.items(): assert type(file_name) is str assert type(cur_log_info) is __class__.LogInfo read_pos = 0 if file_name in self._logs.keys(): prev_log_info = self._logs[file_name] assert type(prev_log_info) is __class__.LogInfo read_pos = prev_log_info.position # the previous size file_content_b = self._node.os_ops.read_binary(file_name, read_pos) assert type(file_content_b) is bytes # # A POTENTIAL PROBLEM: file_content_b may contain an incompleted UTF-8 symbol. # file_content_s = file_content_b.decode() assert type(file_content_s) is str next_read_pos = read_pos + len(file_content_b) # It is a research/paranoja check. # When we will process partial UTF-8 symbol, it must be adjusted. assert cur_log_info.position <= next_read_pos cur_log_info.position = next_read_pos block = __class__.LogDataBlock( file_name, read_pos, file_content_s ) result.append(block) # A new check point self._logs = cur_logs return result def _collect_logs(self) -> typing.Dict[str, LogInfo]: assert self._node is not None assert isinstance(self._node, PostgresNode) files = [ self._node.pg_log_file ] # yapf: disable result = dict() for f in files: assert type(f) is str # skip missing files if not self._node.os_ops.path_exists(f): continue file_size = self._node.os_ops.get_file_size(f) assert type(file_size) is int assert file_size >= 0 result[f] = __class__.LogInfo(file_size) return result class PostgresNodeUtils: @staticmethod def delect_port_conflict(log_reader: PostgresNodeLogReader) -> bool: assert type(log_reader) is PostgresNodeLogReader blocks = log_reader.read() assert type(blocks) is list for block in blocks: assert type(block) is PostgresNodeLogReader.LogDataBlock if 'Is another postmaster already running on port' in block.data: return True return False ================================================ FILE: src/node_app.py ================================================ from .node import OsOperations from .node import LocalOperations from .node import PostgresNode from .node import PortManager import os import platform import tempfile import typing T_DICT_STR_STR = typing.Dict[str, str] T_LIST_STR = typing.List[str] class NodeApp: _test_path: str _os_ops: OsOperations _port_manager: PortManager _nodes_to_cleanup: typing.List[PostgresNode] def __init__( self, test_path: typing.Optional[str] = None, nodes_to_cleanup: typing.Optional[list] = None, os_ops: typing.Optional[OsOperations] = None, port_manager: typing.Optional[PortManager] = None, ): assert test_path is None or type(test_path) is str assert os_ops is None or isinstance(os_ops, OsOperations) assert port_manager is None or isinstance(port_manager, PortManager) if os_ops is None: os_ops = LocalOperations.get_single_instance() assert isinstance(os_ops, OsOperations) self._os_ops = os_ops self._port_manager = port_manager if test_path is None: self._test_path = os_ops.cwd() elif os.path.isabs(test_path): self._test_path = test_path else: self._test_path = os_ops.build_path(os_ops.cwd(), test_path) if nodes_to_cleanup is None: self._nodes_to_cleanup = [] else: self._nodes_to_cleanup = nodes_to_cleanup @property def test_path(self) -> str: assert type(self._test_path) is str return self._test_path @property def os_ops(self) -> OsOperations: assert isinstance(self._os_ops, OsOperations) return self._os_ops @property def port_manager(self) -> PortManager: assert self._port_manager is None or isinstance(self._port_manager, PortManager) return self._port_manager @property def nodes_to_cleanup(self) -> typing.List[PostgresNode]: assert type(self._nodes_to_cleanup) is list return self._nodes_to_cleanup def make_empty( self, base_dir: str, port: typing.Optional[int] = None, bin_dir: typing.Optional[str] = None ) -> PostgresNode: assert type(base_dir) is str assert port is None or type(port) is int assert bin_dir is None or type(bin_dir) is str assert isinstance(self._os_ops, OsOperations) assert type(self._test_path) is str if base_dir is None: raise ValueError("Argument 'base_dir' is not defined.") if base_dir == "": raise ValueError("Argument 'base_dir' is empty.") real_base_dir = self._os_ops.build_path(self._test_path, base_dir) self._os_ops.rmdirs(real_base_dir, ignore_errors=True) self._os_ops.makedirs(real_base_dir) port_manager: typing.Optional[PortManager] = None if port is None: port_manager = self._port_manager node = PostgresNode( base_dir=real_base_dir, port=port, bin_dir=bin_dir, os_ops=self._os_ops, port_manager=port_manager ) try: assert type(self._nodes_to_cleanup) is list self._nodes_to_cleanup.append(node) except: # noqa: E722 node.cleanup(release_resources=True) raise return node def make_simple( self, base_dir: str, port: typing.Optional[int] = None, set_replication: bool = False, ptrack_enable: bool = False, initdb_params: typing.Optional[T_LIST_STR] = None, pg_options: typing.Optional[T_DICT_STR_STR] = None, checksum: bool = True, bin_dir: typing.Optional[str] = None ) -> PostgresNode: assert type(base_dir) is str assert port is None or type(port) is int assert type(set_replication) is bool assert type(ptrack_enable) is bool assert initdb_params is None or type(initdb_params) is list assert pg_options is None or type(pg_options) is dict assert type(checksum) is bool assert bin_dir is None or type(bin_dir) is str node = self.make_empty( base_dir, port, bin_dir=bin_dir ) final_initdb_params = initdb_params if checksum: final_initdb_params = __class__._paramlist_append_if_not_exist( initdb_params, final_initdb_params, '--data-checksums' ) assert final_initdb_params is not None assert '--data-checksums' in final_initdb_params node.init( initdb_params=final_initdb_params, allow_streaming=set_replication ) # set major version pg_version_file = self._os_ops.read(self._os_ops.build_path(node.data_dir, 'PG_VERSION')) node.major_version_str = str(pg_version_file.rstrip()) node.major_version = float(node.major_version_str) # Set default parameters options = { 'max_connections': 100, 'shared_buffers': '10MB', 'fsync': 'off', 'wal_level': 'logical', 'hot_standby': 'off', 'log_line_prefix': '%t [%p]: [%l-1] ', 'log_statement': 'none', 'log_duration': 'on', 'log_min_duration_statement': 0, 'log_connections': 'on', 'log_disconnections': 'on', 'restart_after_crash': 'off', 'autovacuum': 'off', # unix_socket_directories will be defined later } # Allow replication in pg_hba.conf if set_replication: options['max_wal_senders'] = 10 if ptrack_enable: options['ptrack.map_size'] = '1' options['shared_preload_libraries'] = 'ptrack' if node.major_version >= 13: options['wal_keep_size'] = '200MB' else: options['wal_keep_segments'] = '12' # Apply given parameters if pg_options is not None: assert type(pg_options) is dict for option_name, option_value in pg_options.items(): options[option_name] = option_value # Define delayed propertyes if "unix_socket_directories" not in options.keys(): options["unix_socket_directories"] = __class__._gettempdir_for_socket() # Set config values node.set_auto_conf(options) # kludge for testgres # https://github.com/postgrespro/testgres/issues/54 # for PG >= 13 remove 'wal_keep_segments' parameter if node.major_version >= 13: node.set_auto_conf({}, 'postgresql.conf', ['wal_keep_segments']) return node @staticmethod def _paramlist_has_param( params: typing.Optional[T_LIST_STR], param: str ) -> bool: assert type(param) is str if params is None: return False assert type(params) is list return param in params @staticmethod def _paramlist_append( user_params: typing.Optional[T_LIST_STR], updated_params: typing.Optional[T_LIST_STR], param: str, ) -> T_LIST_STR: assert user_params is None or type(user_params) is list assert updated_params is None or type(updated_params) is list assert type(param) is str if updated_params is None: if user_params is None: return [param] return [*user_params, param] assert updated_params is not None if updated_params is user_params: return [*user_params, param] updated_params.append(param) return updated_params @staticmethod def _paramlist_append_if_not_exist( user_params: typing.Optional[T_LIST_STR], updated_params: typing.Optional[T_LIST_STR], param: str, ) -> typing.Optional[T_LIST_STR]: if __class__._paramlist_has_param(updated_params, param): return updated_params return __class__._paramlist_append(user_params, updated_params, param) @staticmethod def _gettempdir_for_socket() -> str: platform_system_name = platform.system().lower() if platform_system_name == "windows": return __class__._gettempdir() # # [2025-02-17] Hot fix. # # Let's use hard coded path as Postgres likes. # # pg_config_manual.h: # # #ifndef WIN32 # #define DEFAULT_PGSOCKET_DIR "/tmp" # #else # #define DEFAULT_PGSOCKET_DIR "" # #endif # # On the altlinux-10 tempfile.gettempdir() may return # the path to "private" temp directiry - "/temp/.private//" # # But Postgres want to find a socket file in "/tmp" (see above). # return "/tmp" @staticmethod def _gettempdir() -> str: v = tempfile.gettempdir() # # Paranoid checks # if type(v) is str: __class__._raise_bugcheck("tempfile.gettempdir returned a value with type {0}.".format(type(v).__name__)) if v == "": __class__._raise_bugcheck("tempfile.gettempdir returned an empty string.") if not os.path.exists(v): __class__._raise_bugcheck("tempfile.gettempdir returned a not exist path [{0}].".format(v)) # OK return v @staticmethod def _raise_bugcheck(msg): assert type(msg) is str assert msg != "" raise Exception("[BUG CHECK] " + msg) ================================================ FILE: src/port_manager.py ================================================ class PortManager: def __init__(self): super().__init__() def reserve_port(self) -> int: raise NotImplementedError("PortManager::reserve_port is not implemented.") def release_port(self, number: int) -> None: assert type(number) is int raise NotImplementedError("PortManager::release_port is not implemented.") ================================================ FILE: src/pubsub.py ================================================ # coding: utf-8 """ Unlike physical replication the logical replication allows users replicate only specified databases and tables. It uses publish-subscribe model with possibly multiple publishers and multiple subscribers. When initializing publisher's node ``allow_logical=True`` should be passed to the :meth:`.PostgresNode.init()` method to enable PostgreSQL to write extra information to the WAL needed by logical replication. To replicate table ``X`` from node A to node B the same table structure should be defined on the subscriber's node as logical replication don't replicate DDL. After that :meth:`~.PostgresNode.publish()` and :meth:`~.PostgresNode.subscribe()` methods may be used to setup replication. Example: >>> from testgres import get_new_node >>> with get_new_node() as nodeA, get_new_node() as nodeB: ... nodeA.init(allow_logical=True).start() ... nodeB.init().start() ... ... # create same table both on publisher and subscriber ... create_table = 'create table test (a int, b int)' ... nodeA.safe_psql(create_table) ... nodeB.safe_psql(create_table) ... ... # create publication ... pub = nodeA.publish('mypub') ... # create subscription ... sub = nodeB.subscribe(pub, 'mysub') ... ... # insert some data to the publisher's node ... nodeA.execute('insert into test values (1, 1), (2, 2)') ... ... # wait until changes apply on subscriber and check them ... sub.catchup() ... ... # read the data from subscriber's node ... nodeB.execute('select * from test') PostgresNode(name='...', port=..., base_dir='...') PostgresNode(name='...', port=..., base_dir='...') '' '' [(1, 1), (2, 2)] """ from six import raise_from from .consts import LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS from .defaults import default_dbname, default_username from .exceptions import CatchUpException from .utils import options_string class Publication(object): def __init__(self, name, node, tables=None, dbname=None, username=None): """ Constructor. Use :meth:`.PostgresNode.publish()` instead of direct constructing publication objects. Args: name: publication name. node: publisher's node. tables: tables list or None for all tables. dbname: database name used to connect and perform subscription. username: username used to connect to the database. """ self.name = name self.node = node self.dbname = dbname or default_dbname() self.username = username or default_username() # create publication in database t = "table " + ", ".join(tables) if tables else "all tables" query = "create publication {} for {}" node.execute(query.format(name, t), dbname=dbname, username=username) def drop(self, dbname=None, username=None): """ Drop publication """ self.node.execute("drop publication {}".format(self.name), dbname=dbname, username=username) def add_tables(self, tables, dbname=None, username=None): """ Add tables to the publication. Cannot be used if publication was created with empty tables list. Args: tables: a list of tables to be added to the publication. """ if not tables: raise ValueError("Tables list is empty") query = "alter publication {} add table {}" self.node.execute(query.format(self.name, ", ".join(tables)), dbname=dbname or self.dbname, username=username or self.username) class Subscription(object): def __init__(self, node, publication, name=None, dbname=None, username=None, **params): """ Constructor. Use :meth:`.PostgresNode.subscribe()` instead of direct constructing subscription objects. Args: name: subscription name. node: subscriber's node. publication: :class:`.Publication` object we are subscribing to (see :meth:`.PostgresNode.publish()`). dbname: database name used to connect and perform subscription. username: username used to connect to the database. params: subscription parameters (see documentation on `CREATE SUBSCRIPTION `_ for details). """ self.name = name self.node = node self.pub = publication # connection info conninfo = { "dbname": self.pub.dbname, "user": self.pub.username, "host": self.pub.node.host, "port": self.pub.node.port } query = ( "create subscription {} connection '{}' publication {}").format( name, options_string(**conninfo), self.pub.name) # additional parameters if params: query += " with ({})".format(options_string(**params)) # Note: cannot run 'create subscription' query in transaction mode node.execute(query, dbname=dbname, username=username) def disable(self, dbname=None, username=None): """ Disables the running subscription. """ query = "alter subscription {} disable" self.node.execute(query.format(self.name), dbname=None, username=None) def enable(self, dbname=None, username=None): """ Enables the previously disabled subscription. """ query = "alter subscription {} enable" self.node.execute(query.format(self.name), dbname=None, username=None) def refresh(self, copy_data=True, dbname=None, username=None): """ Disables the running subscription. """ query = "alter subscription {} refresh publication with (copy_data={})" self.node.execute(query.format(self.name, copy_data), dbname=dbname, username=username) def drop(self, dbname=None, username=None): """ Drops subscription """ self.node.execute("drop subscription {}".format(self.name), dbname=dbname, username=username) def catchup(self, username=None): """ Wait until subscription catches up with publication. Args: username: remote node's user name. """ try: pub_lsn = self.pub.node.execute(query="select pg_current_wal_lsn()", dbname=None, username=None)[0][0] # yapf: disable # create dummy xact, as LR replicates only on commit. self.pub.node.execute(query="select txid_current()", dbname=None, username=None) query = """ select '{}'::pg_lsn - replay_lsn <= 0 from pg_catalog.pg_stat_replication where application_name = '{}' """.format(pub_lsn, self.name) # wait until this LSN reaches subscriber self.pub.node.poll_query_until( query=query, dbname=self.pub.dbname, username=username or self.pub.username, max_attempts=LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS) # Now, wait until there are no tablesync workers: probably # replay_lsn above was sent with changes of new tables just skipped; # they will be eaten by tablesync workers. query = """ select count(*) = 0 from pg_subscription_rel where srsubstate != 'r' """ self.node.poll_query_until( query=query, dbname=self.pub.dbname, username=username or self.pub.username, max_attempts=LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS) except Exception as e: raise_from(CatchUpException("Failed to catch up"), e) ================================================ FILE: src/raise_error.py ================================================ from .exceptions import InvalidOperationException from .enums import NodeStatus import typing class RaiseError: @staticmethod def pg_ctl_returns_an_empty_string(_params): errLines = [] errLines.append("Utility pg_ctl returns an empty string.") errLines.append("Command line is {0}".format(_params)) raise RuntimeError("\n".join(errLines)) @staticmethod def pg_ctl_returns_an_unexpected_string(out, _params): errLines = [] errLines.append("Utility pg_ctl returns an unexpected string:") errLines.append(out) errLines.append("------------") errLines.append("Command line is {0}".format(_params)) raise RuntimeError("\n".join(errLines)) @staticmethod def pg_ctl_returns_a_zero_pid(out, _params): errLines = [] errLines.append("Utility pg_ctl returns a zero pid. Output string is:") errLines.append(out) errLines.append("------------") errLines.append("Command line is {0}".format(_params)) raise RuntimeError("\n".join(errLines)) @staticmethod def node_err__cant_enumerate_child_processes( node_status: NodeStatus ): assert type(node_status) is NodeStatus msg = "Can't enumerate node child processes. {}.".format( __class__._map_node_status_to_reason( node_status, None, ) ) raise InvalidOperationException(msg) @staticmethod def node_err__cant_kill( node_status: NodeStatus ): assert type(node_status) is NodeStatus msg = "Can't kill server process. {}.".format( __class__._map_node_status_to_reason( node_status, None, ) ) raise InvalidOperationException(msg) @staticmethod def _map_node_status_to_reason( node_status: NodeStatus, node_pid: typing.Optional[int], ) -> str: assert type(node_status) is NodeStatus assert node_pid is None or type(node_pid) is int if node_status == NodeStatus.Uninitialized: return "Node is not initialized" if node_status == NodeStatus.Stopped: return "Node is not running" if node_status == NodeStatus.Running: return "Node is running (pid: {})".format( node_pid ) # assert False return "Node has unknown status {}".format( node_status ) ================================================ FILE: src/standby.py ================================================ # coding: utf-8 import six @six.python_2_unicode_compatible class First: """ Specifies a priority-based synchronous replication and makes transaction commits wait until their WAL records are replicated to ``num_sync`` synchronous standbys chosen based on their priorities. Args: sync_num (int): the number of standbys that transaction need to wait for replies from standbys (:obj:`list` of :class:`.PostgresNode`): the list of standby nodes """ def __init__(self, sync_num, standbys): self.sync_num = sync_num self.standbys = standbys def __str__(self): return u"{} ({})".format( self.sync_num, u", ".join(u"\"{}\"".format(r.name) for r in self.standbys)) @six.python_2_unicode_compatible class Any: """ Specifies a quorum-based synchronous replication and makes transaction commits wait until their WAL records are replicated to at least ``num_sync`` listed standbys. Only available for Postgres 10 and newer. Args: sync_num (int): the number of standbys that transaction need to wait for replies from standbys (:obj:`list` of :class:`.PostgresNode`): the list of standby nodes """ def __init__(self, sync_num, standbys): self.sync_num = sync_num self.standbys = standbys def __str__(self): return u"ANY {} ({})".format( self.sync_num, u", ".join(u"\"{}\"".format(r.name) for r in self.standbys)) ================================================ FILE: src/utils.py ================================================ # coding: utf-8 from __future__ import division from __future__ import print_function import os import sys import time from contextlib import contextmanager from packaging.version import Version, InvalidVersion import re import typing from six import iteritems from .exceptions import ExecUtilException from .config import testgres_config as tconf from .raise_error import RaiseError from .enums import NodeStatus from .consts import PG_CTL__STATUS__OK from .consts import PG_CTL__STATUS__NODE_IS_STOPPED from .consts import PG_CTL__STATUS__BAD_DATADIR from testgres.operations.os_ops import OsOperations from testgres.operations.remote_ops import RemoteOperations from testgres.operations.local_ops import LocalOperations from testgres.operations.helpers import Helpers as OsHelpers from .impl.port_manager__generic import PortManager__Generic from .impl.platforms import internal_platform_utils_factory from .impl import internal_utils # rows returned by PG_CONFIG _pg_config_data = {} # # The old, global "port manager" always worked with LOCAL system # _old_port_manager = PortManager__Generic(LocalOperations.get_single_instance()) # ports used by nodes bound_ports = _old_port_manager._reserved_ports # re-export version type class PgVer(Version): def __init__(self, version: str) -> None: try: super().__init__(version) except InvalidVersion: version = re.sub(r"[a-zA-Z].*", "", version) super().__init__(version) def internal__reserve_port(): """ Generate a new port and add it to 'bound_ports'. """ return _old_port_manager.reserve_port() def internal__release_port(port): """ Free port provided by reserve_port(). """ assert type(port) is int return _old_port_manager.release_port(port) reserve_port = internal__reserve_port release_port = internal__release_port def execute_utility(args, logfile=None, verbose=False): """ Execute utility (pg_ctl, pg_dump etc). Args: args: utility + arguments (list). logfile: path to file to store stdout and stderr. Returns: stdout of executed utility. """ return execute_utility2(tconf.os_ops, args, logfile, verbose) def execute_utility2( os_ops: OsOperations, args, logfile=None, verbose=False, ignore_errors=False, exec_env=None, ): assert os_ops is not None assert isinstance(os_ops, OsOperations) assert type(verbose) is bool assert type(ignore_errors) is bool assert exec_env is None or type(exec_env) is dict exit_status, out, error = os_ops.exec_command( args, verbose=True, ignore_errors=ignore_errors, encoding=OsHelpers.GetDefaultEncoding(), exec_env=exec_env) out = '' if not out else out # write new log entry if possible if logfile: try: os_ops.write(filename=logfile, data=args, truncate=True) if out: # comment-out lines lines = [u'\n'] + ['# ' + line for line in out.splitlines()] + [u'\n'] os_ops.write(filename=logfile, data=lines) except IOError: raise ExecUtilException( "Problem with writing to logfile `{}` during run command `{}`".format(logfile, args)) if verbose: return exit_status, out, error else: return out def get_bin_path(filename): """ Return absolute path to an executable using PG_BIN or PG_CONFIG. This function does nothing if 'filename' is already absolute. """ return get_bin_path2(tconf.os_ops, filename) def get_bin_path2(os_ops: OsOperations, filename): assert os_ops is not None assert isinstance(os_ops, OsOperations) # check if it's already absolute if os.path.isabs(filename): return filename if isinstance(os_ops, RemoteOperations): pg_config = os.environ.get("PG_CONFIG_REMOTE") or os.environ.get("PG_CONFIG") else: # try PG_CONFIG - get from local machine pg_config = os.environ.get("PG_CONFIG") if pg_config: bindir = get_pg_config(pg_config, os_ops)["BINDIR"] return os_ops.build_path(bindir, filename) # try PG_BIN pg_bin = os_ops.environ("PG_BIN") if pg_bin: return os_ops.build_path(pg_bin, filename) pg_config_path = os_ops.find_executable('pg_config') if pg_config_path: bindir = get_pg_config(pg_config_path)["BINDIR"] return os_ops.build_path(bindir, filename) return filename def get_pg_config(pg_config_path=None, os_ops=None): """ Return output of pg_config (provided that it is installed). NOTE: this function caches the result by default (see GlobalConfig). """ if os_ops is None: os_ops = tconf.os_ops return get_pg_config2(os_ops, pg_config_path) def get_pg_config2(os_ops: OsOperations, pg_config_path): assert os_ops is not None assert isinstance(os_ops, OsOperations) def cache_pg_config_data(cmd): # execute pg_config and get the output out = os_ops.exec_command(cmd, encoding='utf-8') data = {} for line in out.splitlines(): if line and '=' in line: key, _, value = line.partition('=') data[key.strip()] = value.strip() # cache data global _pg_config_data _pg_config_data = data return data # drop cache if asked to if not tconf.cache_pg_config: global _pg_config_data _pg_config_data = {} # return cached data if not pg_config_path and _pg_config_data: return _pg_config_data # try specified pg_config path or PG_CONFIG if pg_config_path: return cache_pg_config_data(pg_config_path) if isinstance(os_ops, RemoteOperations): pg_config = os.environ.get("PG_CONFIG_REMOTE") or os.environ.get("PG_CONFIG") else: # try PG_CONFIG - get from local machine pg_config = os.environ.get("PG_CONFIG") if pg_config: return cache_pg_config_data(pg_config) # try PG_BIN pg_bin = os.environ.get("PG_BIN") if pg_bin: cmd = os_ops.build_path(pg_bin, "pg_config") return cache_pg_config_data(cmd) # try plain name return cache_pg_config_data("pg_config") def get_pg_version2(os_ops: OsOperations, bin_dir=None): """ Return PostgreSQL version provided by postmaster. """ assert os_ops is not None assert isinstance(os_ops, OsOperations) C_POSTGRES_BINARY = "postgres" # Get raw version (e.g., postgres (PostgreSQL) 9.5.7) if bin_dir is None: postgres_path = get_bin_path2(os_ops, C_POSTGRES_BINARY) else: # [2025-06-25] OK ? assert type(bin_dir) is str assert bin_dir != "" postgres_path = os_ops.build_path(bin_dir, 'postgres') cmd = [postgres_path, '--version'] raw_ver = os_ops.exec_command(cmd, encoding='utf-8') return parse_pg_version(raw_ver) def get_pg_version(bin_dir=None): """ Return PostgreSQL version provided by postmaster. """ return get_pg_version2(tconf.os_ops, bin_dir) def parse_pg_version(version_out): # Generalize removal of system-specific suffixes (anything in parentheses) raw_ver = re.sub(r'\([^)]*\)', '', version_out).strip() # Cook version of PostgreSQL version = raw_ver.split(' ')[-1] \ .partition('devel')[0] \ .partition('beta')[0] \ .partition('rc')[0] return version def file_tail(f, num_lines): """ Get last N lines of a file. """ assert num_lines > 0 bufsize = 8192 buffers = 1 f.seek(0, os.SEEK_END) end_pos = f.tell() while True: offset = max(0, end_pos - bufsize * buffers) f.seek(offset, os.SEEK_SET) pos = f.tell() lines = f.readlines() cur_lines = len(lines) if cur_lines > num_lines or pos == 0: return lines[-num_lines:] buffers = int(buffers * max(2, num_lines / max(cur_lines, 1))) def eprint(*args, **kwargs): """ Print stuff to stderr. """ print(*args, file=sys.stderr, **kwargs) def options_string(separator=u" ", **kwargs): return separator.join(u"{}={}".format(k, v) for k, v in iteritems(kwargs)) @contextmanager def clean_on_error(node): """ Context manager to wrap PostgresNode and such. Calls cleanup() method when underlying code raises an exception. """ try: yield node except Exception: # TODO: should we wrap this in try-block? node.cleanup() raise class PostgresNodeState: node_status: NodeStatus pid: typing.Optional[int] def __init__( self, node_status: NodeStatus, pid: typing.Optional[int] ): assert type(node_status) is NodeStatus assert pid is None or type(pid) is int self.node_status = node_status self.pid = pid return def get_pg_node_state( os_ops: OsOperations, bin_dir: str, data_dir: str, utils_log_file: typing.Optional[str], ) -> PostgresNodeState: assert isinstance(os_ops, OsOperations) assert type(bin_dir) is str assert type(data_dir) is str assert utils_log_file is None or type(utils_log_file) is str C_MAX_ATTEMPTS = 3 C_SLEEP_TIME1 = 1 C_SLEEP_TIME_MULT = 2 _params = [ os_ops.build_path(bin_dir, "pg_ctl"), "-D", data_dir, "status", ] attempt = 0 sleep_time = C_SLEEP_TIME1 platform_utils: typing.Optional[internal_platform_utils_factory.InternalPlatformUtils] = None while True: assert type(attempt) is int assert attempt >= 0 assert attempt < C_MAX_ATTEMPTS attempt += 1 if attempt > 1: internal_utils.send_log_debug("Sleep {} second(s) before an attempt #{}".format( sleep_time, attempt )) time.sleep(sleep_time) sleep_time = sleep_time * C_SLEEP_TIME_MULT status_code, out, error = execute_utility2( os_ops, _params, utils_log_file, verbose=True, ignore_errors=True, ) assert type(status_code) is int assert type(out) is str assert type(error) is str # ----------------- if status_code == PG_CTL__STATUS__NODE_IS_STOPPED: return PostgresNodeState(NodeStatus.Stopped, None) # ----------------- if status_code == PG_CTL__STATUS__BAD_DATADIR: return PostgresNodeState(NodeStatus.Uninitialized, None) # ----------------- if status_code == PG_CTL__STATUS__OK: if out == "": RaiseError.pg_ctl_returns_an_empty_string( _params ) C_PID_PREFIX = "(PID: " i = out.find(C_PID_PREFIX) if i == -1: RaiseError.pg_ctl_returns_an_unexpected_string( out, _params ) assert i > 0 assert i < len(out) assert len(C_PID_PREFIX) <= len(out) assert i <= len(out) - len(C_PID_PREFIX) i += len(C_PID_PREFIX) start_pid_s = i while True: if i == len(out): RaiseError.pg_ctl_returns_an_unexpected_string( out, _params ) ch = out[i] if ch == ")": break if ch.isdigit(): i += 1 continue RaiseError.pg_ctl_returns_an_unexpected_string( out, _params ) assert False if i == start_pid_s: RaiseError.pg_ctl_returns_an_unexpected_string( out, _params ) # TODO: Let's verify a length of pid string. pid = int(out[start_pid_s:i]) if pid == 0: RaiseError.pg_ctl_returns_a_zero_pid( out, _params ) assert pid != 0 # ----------------- return PostgresNodeState(NodeStatus.Running, pid) assert status_code != PG_CTL__STATUS__OK errMsg = "Getting of a node status [data_dir is {0}] failed.".format( data_dir ) e1 = ExecUtilException( message=errMsg, command=_params, exit_code=status_code, out=out, error=error, ) pid_file = os_ops.build_path(data_dir, "postmaster.pid") postmaster_pid_is_empty = "pg_ctl: the PID file \"{}\" is empty\n".format( pid_file, ) if error == postmaster_pid_is_empty: internal_utils.send_log_debug( "PID file [{}] is empty. A check is being carried out to ensure that the postmaster is alive [bindir: {}] ...".format( pid_file, bin_dir, )) if platform_utils is None: platform_utils = internal_platform_utils_factory.create_internal_platform_utils(os_ops) assert isinstance(platform_utils, internal_platform_utils_factory.InternalPlatformUtils) assert isinstance(platform_utils, internal_platform_utils_factory.InternalPlatformUtils) try: find_postmaster_r = platform_utils.FindPostmaster( os_ops, bin_dir, data_dir, ) except Exception as e2: e2.__cause__ = e1 raise e2 assert type(find_postmaster_r) is internal_platform_utils_factory.InternalPlatformUtils.FindPostmasterResult if find_postmaster_r.code == internal_platform_utils_factory.InternalPlatformUtils.FindPostmasterResultCode.ok: # Postmaster is alive. Let's wait a few seconds and check its status again. internal_utils.send_log_debug( "Postmaster is found and has PID {}.".format( find_postmaster_r.pid )) if attempt < C_MAX_ATTEMPTS: continue errMsg = "Getting of a node status [data_dir is {0}] failed.".format( data_dir ) raise ExecUtilException( message=errMsg, command=_params, exit_code=status_code, out=out, error=error, ) ================================================ FILE: tests/README.md ================================================ ### How do I run tests? #### Simple ```bash # Setup virtualenv virtualenv venv source venv/bin/activate # Install local version of testgres pip install -U . # Set path to PostgreSQL export PG_BIN=/path/to/pg/bin # Run tests ./tests/test_simple.py ``` #### All configurations + coverage ```bash # Set path to PostgreSQL and python version export PATH=/path/to/pg/bin:$PATH export PYTHON_VERSION=3 # or 2 # Run tests ./run_tests.sh ``` #### Remote host tests 1. Start remote host or docker container 2. Make sure that you run ssh ```commandline sudo apt-get install openssh-server sudo systemctl start sshd ``` 3. You need to connect to the remote host at least once to add it to the known hosts file 4. Generate ssh keys 5. Set up params for tests ```commandline conn_params = ConnectionParams( host='remote_host', username='username', ssh_key=/path/to/your/ssh/key' ) os_ops = RemoteOperations(conn_params) ``` If you have different path to `PG_CONFIG` on your local and remote host you can set up `PG_CONFIG_REMOTE`, this value will be using during work with remote host. `test_remote` - Tests for RemoteOperations class. `test_simple_remote` - Tests that create node and check it. The same as `test_simple`, but for remote node. ================================================ FILE: tests/__init__.py ================================================ ================================================ FILE: tests/conftest.py ================================================ # ///////////////////////////////////////////////////////////////////////////// # PyTest Configuration import pluggy import pytest import os import logging import pathlib import math import datetime import typing import enum import _pytest.outcomes import _pytest.unittest import _pytest.logging from packaging.version import Version # ///////////////////////////////////////////////////////////////////////////// C_ROOT_DIR__RELATIVE = ".." # ///////////////////////////////////////////////////////////////////////////// T_TUPLE__str_int = typing.Tuple[str, int] # ///////////////////////////////////////////////////////////////////////////// # T_PLUGGY_RESULT if Version(pluggy.__version__) <= Version("1.2"): T_PLUGGY_RESULT = pluggy._result._Result else: T_PLUGGY_RESULT = pluggy.Result # ///////////////////////////////////////////////////////////////////////////// g_error_msg_count_key = pytest.StashKey[int]() g_warning_msg_count_key = pytest.StashKey[int]() g_critical_msg_count_key = pytest.StashKey[int]() # ///////////////////////////////////////////////////////////////////////////// # T_TEST_PROCESS_KIND class T_TEST_PROCESS_KIND(enum.Enum): Master = 1 Worker = 2 # ///////////////////////////////////////////////////////////////////////////// # T_TEST_PROCESS_MODE class T_TEST_PROCESS_MODE(enum.Enum): Collect = 1 ExecTests = 2 # ///////////////////////////////////////////////////////////////////////////// g_test_process_kind: typing.Optional[T_TEST_PROCESS_KIND] = None g_test_process_mode: typing.Optional[T_TEST_PROCESS_MODE] = None g_worker_log_is_created: typing.Optional[bool] = None # ///////////////////////////////////////////////////////////////////////////// # TestConfigPropNames class TestConfigPropNames: TEST_CFG__LOG_DIR = "TEST_CFG__LOG_DIR" # ///////////////////////////////////////////////////////////////////////////// # TestStartupData__Helper class TestStartupData__Helper: sm_StartTS = datetime.datetime.now() # -------------------------------------------------------------------- @staticmethod def GetStartTS() -> datetime.datetime: assert type(__class__.sm_StartTS) is datetime.datetime return __class__.sm_StartTS # -------------------------------------------------------------------- @staticmethod def CalcRootDir() -> str: r = os.path.abspath(__file__) r = os.path.dirname(r) r = os.path.join(r, C_ROOT_DIR__RELATIVE) r = os.path.abspath(r) return r # -------------------------------------------------------------------- @staticmethod def CalcRootLogDir() -> str: if TestConfigPropNames.TEST_CFG__LOG_DIR in os.environ: resultPath = os.environ[TestConfigPropNames.TEST_CFG__LOG_DIR] else: rootDir = __class__.CalcRootDir() resultPath = os.path.join(rootDir, "logs") assert type(resultPath) is str return resultPath # -------------------------------------------------------------------- @staticmethod def CalcCurrentTestWorkerSignature() -> str: currentPID = os.getpid() assert type(currentPID) is int startTS = __class__.sm_StartTS assert type(startTS) is datetime.datetime result = "pytest-{0:04d}{1:02d}{2:02d}_{3:02d}{4:02d}{5:02d}".format( startTS.year, startTS.month, startTS.day, startTS.hour, startTS.minute, startTS.second, ) gwid = os.environ.get("PYTEST_XDIST_WORKER") if gwid is not None: result += "--xdist_" + str(gwid) result += "--" + "pid" + str(currentPID) return result # ///////////////////////////////////////////////////////////////////////////// # TestStartupData class TestStartupData: sm_RootDir: str = TestStartupData__Helper.CalcRootDir() sm_CurrentTestWorkerSignature: str = ( TestStartupData__Helper.CalcCurrentTestWorkerSignature() ) sm_RootLogDir: str = TestStartupData__Helper.CalcRootLogDir() # -------------------------------------------------------------------- @staticmethod def GetRootDir() -> str: assert type(__class__.sm_RootDir) is str return __class__.sm_RootDir # -------------------------------------------------------------------- @staticmethod def GetRootLogDir() -> str: assert type(__class__.sm_RootLogDir) is str return __class__.sm_RootLogDir # -------------------------------------------------------------------- @staticmethod def GetCurrentTestWorkerSignature() -> str: assert type(__class__.sm_CurrentTestWorkerSignature) is str return __class__.sm_CurrentTestWorkerSignature # ///////////////////////////////////////////////////////////////////////////// # TEST_PROCESS_STATS class TEST_PROCESS_STATS: cTotalTests: int = 0 cNotExecutedTests: int = 0 cExecutedTests: int = 0 cPassedTests: int = 0 cFailedTests: int = 0 cXFailedTests: int = 0 cSkippedTests: int = 0 cNotXFailedTests: int = 0 cWarningTests: int = 0 cUnexpectedTests: int = 0 cAchtungTests: int = 0 FailedTests: typing.List[T_TUPLE__str_int] = list() XFailedTests: typing.List[T_TUPLE__str_int] = list() NotXFailedTests: typing.List[str] = list() WarningTests: typing.List[T_TUPLE__str_int] = list() AchtungTests: typing.List[str] = list() cTotalDuration: datetime.timedelta = datetime.timedelta() cTotalErrors: int = 0 cTotalWarnings: int = 0 # -------------------------------------------------------------------- @staticmethod def incrementTotalTestCount() -> None: assert type(__class__.cTotalTests) is int assert __class__.cTotalTests >= 0 __class__.cTotalTests += 1 assert __class__.cTotalTests > 0 # -------------------------------------------------------------------- @staticmethod def incrementNotExecutedTestCount() -> None: assert type(__class__.cNotExecutedTests) is int assert __class__.cNotExecutedTests >= 0 __class__.cNotExecutedTests += 1 assert __class__.cNotExecutedTests > 0 # -------------------------------------------------------------------- @staticmethod def incrementExecutedTestCount() -> int: assert type(__class__.cExecutedTests) is int assert __class__.cExecutedTests >= 0 __class__.cExecutedTests += 1 assert __class__.cExecutedTests > 0 return __class__.cExecutedTests # -------------------------------------------------------------------- @staticmethod def incrementPassedTestCount() -> None: assert type(__class__.cPassedTests) is int assert __class__.cPassedTests >= 0 __class__.cPassedTests += 1 assert __class__.cPassedTests > 0 # -------------------------------------------------------------------- @staticmethod def incrementFailedTestCount(testID: str, errCount: int) -> None: assert type(testID) is str assert type(errCount) is int assert errCount > 0 assert type(__class__.FailedTests) is list assert type(__class__.cFailedTests) is int assert __class__.cFailedTests >= 0 __class__.FailedTests.append((testID, errCount)) # raise? __class__.cFailedTests += 1 assert len(__class__.FailedTests) > 0 assert __class__.cFailedTests > 0 assert len(__class__.FailedTests) == __class__.cFailedTests # -------- assert type(__class__.cTotalErrors) is int assert __class__.cTotalErrors >= 0 __class__.cTotalErrors += errCount assert __class__.cTotalErrors > 0 # -------------------------------------------------------------------- @staticmethod def incrementXFailedTestCount(testID: str, errCount: int) -> None: assert type(testID) is str assert type(errCount) is int assert errCount >= 0 assert type(__class__.XFailedTests) is list assert type(__class__.cXFailedTests) is int assert __class__.cXFailedTests >= 0 __class__.XFailedTests.append((testID, errCount)) # raise? __class__.cXFailedTests += 1 assert len(__class__.XFailedTests) > 0 assert __class__.cXFailedTests > 0 assert len(__class__.XFailedTests) == __class__.cXFailedTests # -------------------------------------------------------------------- @staticmethod def incrementSkippedTestCount() -> None: assert type(__class__.cSkippedTests) is int assert __class__.cSkippedTests >= 0 __class__.cSkippedTests += 1 assert __class__.cSkippedTests > 0 # -------------------------------------------------------------------- @staticmethod def incrementNotXFailedTests(testID: str) -> None: assert type(testID) is str assert type(__class__.NotXFailedTests) is list assert type(__class__.cNotXFailedTests) is int assert __class__.cNotXFailedTests >= 0 __class__.NotXFailedTests.append(testID) # raise? __class__.cNotXFailedTests += 1 assert len(__class__.NotXFailedTests) > 0 assert __class__.cNotXFailedTests > 0 assert len(__class__.NotXFailedTests) == __class__.cNotXFailedTests # -------------------------------------------------------------------- @staticmethod def incrementWarningTestCount(testID: str, warningCount: int) -> None: assert type(testID) is str assert type(warningCount) is int assert testID != "" assert warningCount > 0 assert type(__class__.WarningTests) is list assert type(__class__.cWarningTests) is int assert __class__.cWarningTests >= 0 __class__.WarningTests.append((testID, warningCount)) # raise? __class__.cWarningTests += 1 assert len(__class__.WarningTests) > 0 assert __class__.cWarningTests > 0 assert len(__class__.WarningTests) == __class__.cWarningTests # -------- assert type(__class__.cTotalWarnings) is int assert __class__.cTotalWarnings >= 0 __class__.cTotalWarnings += warningCount assert __class__.cTotalWarnings > 0 # -------------------------------------------------------------------- @staticmethod def incrementUnexpectedTests() -> None: assert type(__class__.cUnexpectedTests) is int assert __class__.cUnexpectedTests >= 0 __class__.cUnexpectedTests += 1 assert __class__.cUnexpectedTests > 0 # -------------------------------------------------------------------- @staticmethod def incrementAchtungTestCount(testID: str) -> None: assert type(testID) is str assert type(__class__.AchtungTests) is list assert type(__class__.cAchtungTests) is int assert __class__.cAchtungTests >= 0 __class__.AchtungTests.append(testID) # raise? __class__.cAchtungTests += 1 assert len(__class__.AchtungTests) > 0 assert __class__.cAchtungTests > 0 assert len(__class__.AchtungTests) == __class__.cAchtungTests # ///////////////////////////////////////////////////////////////////////////// def timedelta_to_human_text(delta: datetime.timedelta) -> str: assert isinstance(delta, datetime.timedelta) C_SECONDS_IN_MINUTE = 60 C_SECONDS_IN_HOUR = 60 * C_SECONDS_IN_MINUTE v = delta.seconds cHours = int(v / C_SECONDS_IN_HOUR) v = v - cHours * C_SECONDS_IN_HOUR cMinutes = int(v / C_SECONDS_IN_MINUTE) cSeconds = v - cMinutes * C_SECONDS_IN_MINUTE result = "" if delta.days == 0 else "{0} day(s) ".format(delta.days) result = result + "{:02d}:{:02d}:{:02d}.{:06d}".format( cHours, cMinutes, cSeconds, delta.microseconds ) return result # ///////////////////////////////////////////////////////////////////////////// def helper__build_test_id(item: pytest.Function) -> str: assert item is not None assert isinstance(item, pytest.Function) testID = "" if item.cls is not None: testID = item.cls.__module__ + "." + item.cls.__name__ + "::" testID = testID + item.name return testID # ///////////////////////////////////////////////////////////////////////////// def helper__makereport__setup( item: pytest.Function, call: pytest.CallInfo, outcome: T_PLUGGY_RESULT ): assert item is not None assert call is not None assert outcome is not None # it may be pytest.Function or _pytest.unittest.TestCaseFunction assert isinstance(item, pytest.Function) assert type(call) is pytest.CallInfo assert type(outcome) is T_PLUGGY_RESULT C_LINE1 = "******************************************************" # logging.info("pytest_runtest_makereport - setup") TEST_PROCESS_STATS.incrementTotalTestCount() rep: pytest.TestReport = outcome.get_result() assert rep is not None assert type(rep) is pytest.TestReport if rep.outcome == "skipped": TEST_PROCESS_STATS.incrementNotExecutedTestCount() return testID = helper__build_test_id(item) if rep.outcome == "passed": testNumber = TEST_PROCESS_STATS.incrementExecutedTestCount() logging.info(C_LINE1) logging.info("* START TEST {0}".format(testID)) logging.info("*") logging.info("* Path : {0}".format(item.path)) logging.info("* Number: {0}".format(testNumber)) logging.info("*") return assert rep.outcome != "passed" TEST_PROCESS_STATS.incrementAchtungTestCount(testID) logging.info(C_LINE1) logging.info("* ACHTUNG TEST {0}".format(testID)) logging.info("*") logging.info("* Path : {0}".format(item.path)) logging.info("* Outcome is [{0}]".format(rep.outcome)) if rep.outcome == "failed": assert call.excinfo is not None assert call.excinfo.value is not None logging.info("*") logging.error(call.excinfo.value) logging.info("*") return # ------------------------------------------------------------------------ class ExitStatusNames: FAILED = "FAILED" PASSED = "PASSED" XFAILED = "XFAILED" NOT_XFAILED = "NOT XFAILED" SKIPPED = "SKIPPED" UNEXPECTED = "UNEXPECTED" # ------------------------------------------------------------------------ def helper__makereport__call( item: pytest.Function, call: pytest.CallInfo, outcome: T_PLUGGY_RESULT ): assert item is not None assert call is not None assert outcome is not None # it may be pytest.Function or _pytest.unittest.TestCaseFunction assert isinstance(item, pytest.Function) assert type(call) is pytest.CallInfo assert type(outcome) is T_PLUGGY_RESULT # -------- item_error_msg_count1 = item.stash.get(g_error_msg_count_key, 0) assert type(item_error_msg_count1) is int assert item_error_msg_count1 >= 0 item_error_msg_count2 = item.stash.get(g_critical_msg_count_key, 0) assert type(item_error_msg_count2) is int assert item_error_msg_count2 >= 0 item_error_msg_count = item_error_msg_count1 + item_error_msg_count2 # -------- item_warning_msg_count = item.stash.get(g_warning_msg_count_key, 0) assert type(item_warning_msg_count) is int assert item_warning_msg_count >= 0 # -------- rep = outcome.get_result() assert rep is not None assert type(rep) is pytest.TestReport # -------- testID = helper__build_test_id(item) # -------- assert call.start <= call.stop startDT = datetime.datetime.fromtimestamp(call.start) assert type(startDT) is datetime.datetime stopDT = datetime.datetime.fromtimestamp(call.stop) assert type(stopDT) is datetime.datetime testDurration = stopDT - startDT assert type(testDurration) is datetime.timedelta # -------- exitStatus = None exitStatusInfo = None if rep.outcome == "skipped": assert call.excinfo is not None # research assert call.excinfo.value is not None # research if type(call.excinfo.value) is _pytest.outcomes.Skipped: assert not hasattr(rep, "wasxfail") exitStatus = ExitStatusNames.SKIPPED reasonText = str(call.excinfo.value) reasonMsgTempl = "SKIP REASON: {0}" TEST_PROCESS_STATS.incrementSkippedTestCount() elif type(call.excinfo.value) is _pytest.outcomes.XFailed: exitStatus = ExitStatusNames.XFAILED reasonText = str(call.excinfo.value) reasonMsgTempl = "XFAIL REASON: {0}" TEST_PROCESS_STATS.incrementXFailedTestCount(testID, item_error_msg_count) else: exitStatus = ExitStatusNames.XFAILED assert hasattr(rep, "wasxfail") assert rep.wasxfail is not None assert type(rep.wasxfail) is str reasonText = rep.wasxfail reasonMsgTempl = "XFAIL REASON: {0}" if type(call.excinfo.value) is SIGNAL_EXCEPTION: pass else: logging.error(call.excinfo.value) item_error_msg_count += 1 TEST_PROCESS_STATS.incrementXFailedTestCount(testID, item_error_msg_count) assert type(reasonText) is str if reasonText != "": assert type(reasonMsgTempl) is str logging.info("*") logging.info("* " + reasonMsgTempl.format(reasonText)) elif rep.outcome == "failed": assert call.excinfo is not None assert call.excinfo.value is not None if type(call.excinfo.value) is SIGNAL_EXCEPTION: assert item_error_msg_count > 0 pass else: logging.error(call.excinfo.value) item_error_msg_count += 1 assert item_error_msg_count > 0 TEST_PROCESS_STATS.incrementFailedTestCount(testID, item_error_msg_count) exitStatus = ExitStatusNames.FAILED elif rep.outcome == "passed": assert call.excinfo is None if hasattr(rep, "wasxfail"): assert type(rep.wasxfail) is str TEST_PROCESS_STATS.incrementNotXFailedTests(testID) warnMsg = "NOTE: Test is marked as xfail" if rep.wasxfail != "": warnMsg += " [" + rep.wasxfail + "]" logging.info(warnMsg) exitStatus = ExitStatusNames.NOT_XFAILED else: assert not hasattr(rep, "wasxfail") TEST_PROCESS_STATS.incrementPassedTestCount() exitStatus = ExitStatusNames.PASSED else: TEST_PROCESS_STATS.incrementUnexpectedTests() exitStatus = ExitStatusNames.UNEXPECTED exitStatusInfo = rep.outcome # [2025-03-28] It may create a useless problem in new environment. # assert False # -------- if item_warning_msg_count > 0: TEST_PROCESS_STATS.incrementWarningTestCount(testID, item_warning_msg_count) # -------- assert exitStatus is not None assert type(exitStatus) is str if exitStatus == ExitStatusNames.FAILED: assert item_error_msg_count > 0 pass # -------- assert type(TEST_PROCESS_STATS.cTotalDuration) is datetime.timedelta assert type(testDurration) is datetime.timedelta TEST_PROCESS_STATS.cTotalDuration += testDurration assert testDurration <= TEST_PROCESS_STATS.cTotalDuration # -------- exitStatusLineData = exitStatus if exitStatusInfo is not None: exitStatusLineData += " [{}]".format(exitStatusInfo) # -------- logging.info("*") logging.info("* DURATION : {0}".format(timedelta_to_human_text(testDurration))) logging.info("*") logging.info("* EXIT STATUS : {0}".format(exitStatusLineData)) logging.info("* ERROR COUNT : {0}".format(item_error_msg_count)) logging.info("* WARNING COUNT: {0}".format(item_warning_msg_count)) logging.info("*") logging.info("* STOP TEST {0}".format(testID)) logging.info("*") # ///////////////////////////////////////////////////////////////////////////// @pytest.hookimpl(hookwrapper=True) def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): # # https://docs.pytest.org/en/7.1.x/how-to/writing_hook_functions.html#hookwrapper-executing-around-other-hooks # # Note that hook wrappers don’t return results themselves, # they merely perform tracing or other side effects around the actual hook implementations. # # https://docs.pytest.org/en/7.1.x/reference/reference.html#test-running-runtest-hooks # assert item is not None assert call is not None # it may be pytest.Function or _pytest.unittest.TestCaseFunction assert isinstance(item, pytest.Function) assert type(call) is pytest.CallInfo outcome = yield assert outcome is not None assert type(outcome) is T_PLUGGY_RESULT assert type(call.when) is str if call.when == "collect": return if call.when == "setup": helper__makereport__setup(item, call, outcome) return if call.when == "call": helper__makereport__call(item, call, outcome) return if call.when == "teardown": return errMsg = "[pytest_runtest_makereport] unknown 'call.when' value: [{0}].".format( call.when ) raise RuntimeError(errMsg) # ///////////////////////////////////////////////////////////////////////////// class LogWrapper2: _old_method: typing.Any _err_counter: typing.Optional[int] _warn_counter: typing.Optional[int] _critical_counter: typing.Optional[int] # -------------------------------------------------------------------- def __init__(self): self._old_method = None self._err_counter = None self._warn_counter = None self._critical_counter = None # -------------------------------------------------------------------- def __enter__(self): assert self._old_method is None assert self._err_counter is None assert self._warn_counter is None assert self._critical_counter is None assert logging.root is not None assert isinstance(logging.root, logging.RootLogger) self._old_method = logging.root.handle self._err_counter = 0 self._warn_counter = 0 self._critical_counter = 0 logging.root.handle = self return self # -------------------------------------------------------------------- def __exit__(self, exc_type, exc_val, exc_tb): assert self._old_method is not None assert self._err_counter is not None assert self._warn_counter is not None assert logging.root is not None assert isinstance(logging.root, logging.RootLogger) assert logging.root.handle is self logging.root.handle = self._old_method self._old_method = None self._err_counter = None self._warn_counter = None self._critical_counter = None return False # -------------------------------------------------------------------- def __call__(self, record: logging.LogRecord): assert record is not None assert isinstance(record, logging.LogRecord) assert self._old_method is not None assert self._err_counter is not None assert self._warn_counter is not None assert self._critical_counter is not None assert type(self._err_counter) is int assert self._err_counter >= 0 assert type(self._warn_counter) is int assert self._warn_counter >= 0 assert type(self._critical_counter) is int assert self._critical_counter >= 0 r = self._old_method(record) if record.levelno == logging.ERROR: self._err_counter += 1 assert self._err_counter > 0 elif record.levelno == logging.WARNING: self._warn_counter += 1 assert self._warn_counter > 0 elif record.levelno == logging.CRITICAL: self._critical_counter += 1 assert self._critical_counter > 0 return r # ///////////////////////////////////////////////////////////////////////////// class SIGNAL_EXCEPTION(Exception): def __init__(self): pass # ///////////////////////////////////////////////////////////////////////////// @pytest.hookimpl(hookwrapper=True) def pytest_pyfunc_call(pyfuncitem: pytest.Function): assert pyfuncitem is not None assert isinstance(pyfuncitem, pytest.Function) assert logging.root is not None assert isinstance(logging.root, logging.RootLogger) assert logging.root.handle is not None debug__log_handle_method = logging.root.handle assert debug__log_handle_method is not None debug__log_error_method = logging.error assert debug__log_error_method is not None debug__log_warning_method = logging.warning assert debug__log_warning_method is not None pyfuncitem.stash[g_error_msg_count_key] = 0 pyfuncitem.stash[g_warning_msg_count_key] = 0 pyfuncitem.stash[g_critical_msg_count_key] = 0 try: with LogWrapper2() as logWrapper: assert type(logWrapper) is LogWrapper2 assert logWrapper._old_method is not None assert type(logWrapper._err_counter) is int assert logWrapper._err_counter == 0 assert type(logWrapper._warn_counter) is int assert logWrapper._warn_counter == 0 assert type(logWrapper._critical_counter) is int assert logWrapper._critical_counter == 0 assert logging.root.handle is logWrapper r = yield assert r is not None assert type(r) is T_PLUGGY_RESULT assert logWrapper._old_method is not None assert type(logWrapper._err_counter) is int assert logWrapper._err_counter >= 0 assert type(logWrapper._warn_counter) is int assert logWrapper._warn_counter >= 0 assert type(logWrapper._critical_counter) is int assert logWrapper._critical_counter >= 0 assert logging.root.handle is logWrapper assert g_error_msg_count_key in pyfuncitem.stash assert g_warning_msg_count_key in pyfuncitem.stash assert g_critical_msg_count_key in pyfuncitem.stash assert pyfuncitem.stash[g_error_msg_count_key] == 0 assert pyfuncitem.stash[g_warning_msg_count_key] == 0 assert pyfuncitem.stash[g_critical_msg_count_key] == 0 pyfuncitem.stash[g_error_msg_count_key] = logWrapper._err_counter pyfuncitem.stash[g_warning_msg_count_key] = logWrapper._warn_counter pyfuncitem.stash[g_critical_msg_count_key] = logWrapper._critical_counter if r.exception is not None: pass elif logWrapper._err_counter > 0: r.force_exception(SIGNAL_EXCEPTION()) elif logWrapper._critical_counter > 0: r.force_exception(SIGNAL_EXCEPTION()) finally: assert logging.error is debug__log_error_method assert logging.warning is debug__log_warning_method assert logging.root.handle == debug__log_handle_method pass # ///////////////////////////////////////////////////////////////////////////// def helper__calc_W(n: int) -> int: assert n > 0 x = int(math.log10(n)) assert type(x) is int assert x >= 0 x += 1 return x # ------------------------------------------------------------------------ def helper__print_test_list(tests: typing.List[str]) -> None: assert type(tests) is list assert helper__calc_W(9) == 1 assert helper__calc_W(10) == 2 assert helper__calc_W(11) == 2 assert helper__calc_W(99) == 2 assert helper__calc_W(100) == 3 assert helper__calc_W(101) == 3 assert helper__calc_W(999) == 3 assert helper__calc_W(1000) == 4 assert helper__calc_W(1001) == 4 W = helper__calc_W(len(tests)) templateLine = "{0:0" + str(W) + "d}. {1}" nTest = 0 for t in tests: assert type(t) is str assert t != "" nTest += 1 logging.info(templateLine.format(nTest, t)) # ------------------------------------------------------------------------ def helper__print_test_list2(tests: typing.List[T_TUPLE__str_int]) -> None: assert type(tests) is list assert helper__calc_W(9) == 1 assert helper__calc_W(10) == 2 assert helper__calc_W(11) == 2 assert helper__calc_W(99) == 2 assert helper__calc_W(100) == 3 assert helper__calc_W(101) == 3 assert helper__calc_W(999) == 3 assert helper__calc_W(1000) == 4 assert helper__calc_W(1001) == 4 W = helper__calc_W(len(tests)) templateLine = "{0:0" + str(W) + "d}. {1} ({2})" nTest = 0 for t in tests: assert type(t) is tuple assert len(t) == 2 assert type(t[0]) is str assert type(t[1]) is int assert t[0] != "" assert t[1] >= 0 nTest += 1 logging.info(templateLine.format(nTest, t[0], t[1])) # ///////////////////////////////////////////////////////////////////////////// # SUMMARY BUILDER @pytest.hookimpl(trylast=True) def pytest_sessionfinish(): # # NOTE: It should execute after logging.pytest_sessionfinish # global g_test_process_kind # noqa: F824 global g_test_process_mode # noqa: F824 global g_worker_log_is_created # noqa: F824 assert g_test_process_kind is not None assert type(g_test_process_kind) is T_TEST_PROCESS_KIND if g_test_process_kind == T_TEST_PROCESS_KIND.Master: return assert g_test_process_kind == T_TEST_PROCESS_KIND.Worker assert g_test_process_mode is not None assert type(g_test_process_mode) is T_TEST_PROCESS_MODE if g_test_process_mode == T_TEST_PROCESS_MODE.Collect: return assert g_test_process_mode == T_TEST_PROCESS_MODE.ExecTests assert type(g_worker_log_is_created) is bool assert g_worker_log_is_created C_LINE1 = "---------------------------" def LOCAL__print_line1_with_header(header: str): assert type(C_LINE1) is str assert type(header) is str assert header != "" logging.info(C_LINE1 + " [" + header + "]") def LOCAL__print_test_list( header: str, test_count: int, test_list: typing.List[str] ): assert type(header) is str assert type(test_count) is int assert type(test_list) is list assert header != "" assert test_count >= 0 assert len(test_list) == test_count LOCAL__print_line1_with_header(header) logging.info("") if len(test_list) > 0: helper__print_test_list(test_list) logging.info("") def LOCAL__print_test_list2( header: str, test_count: int, test_list: typing.List[T_TUPLE__str_int] ): assert type(header) is str assert type(test_count) is int assert type(test_list) is list assert header != "" assert test_count >= 0 assert len(test_list) == test_count LOCAL__print_line1_with_header(header) logging.info("") if len(test_list) > 0: helper__print_test_list2(test_list) logging.info("") # fmt: off LOCAL__print_test_list( "ACHTUNG TESTS", TEST_PROCESS_STATS.cAchtungTests, TEST_PROCESS_STATS.AchtungTests, ) LOCAL__print_test_list2( "FAILED TESTS", TEST_PROCESS_STATS.cFailedTests, TEST_PROCESS_STATS.FailedTests ) LOCAL__print_test_list2( "XFAILED TESTS", TEST_PROCESS_STATS.cXFailedTests, TEST_PROCESS_STATS.XFailedTests, ) LOCAL__print_test_list( "NOT XFAILED TESTS", TEST_PROCESS_STATS.cNotXFailedTests, TEST_PROCESS_STATS.NotXFailedTests, ) LOCAL__print_test_list2( "WARNING TESTS", TEST_PROCESS_STATS.cWarningTests, TEST_PROCESS_STATS.WarningTests, ) # fmt: on LOCAL__print_line1_with_header("SUMMARY STATISTICS") logging.info("") logging.info("[TESTS]") logging.info(" TOTAL : {0}".format(TEST_PROCESS_STATS.cTotalTests)) logging.info(" EXECUTED : {0}".format(TEST_PROCESS_STATS.cExecutedTests)) logging.info(" NOT EXECUTED : {0}".format(TEST_PROCESS_STATS.cNotExecutedTests)) logging.info(" ACHTUNG : {0}".format(TEST_PROCESS_STATS.cAchtungTests)) logging.info("") logging.info(" PASSED : {0}".format(TEST_PROCESS_STATS.cPassedTests)) logging.info(" FAILED : {0}".format(TEST_PROCESS_STATS.cFailedTests)) logging.info(" XFAILED : {0}".format(TEST_PROCESS_STATS.cXFailedTests)) logging.info(" NOT XFAILED : {0}".format(TEST_PROCESS_STATS.cNotXFailedTests)) logging.info(" SKIPPED : {0}".format(TEST_PROCESS_STATS.cSkippedTests)) logging.info(" WITH WARNINGS: {0}".format(TEST_PROCESS_STATS.cWarningTests)) logging.info(" UNEXPECTED : {0}".format(TEST_PROCESS_STATS.cUnexpectedTests)) logging.info("") assert type(TEST_PROCESS_STATS.cTotalDuration) is datetime.timedelta LOCAL__print_line1_with_header("TIME") logging.info("") logging.info( " TOTAL DURATION: {0}".format( timedelta_to_human_text(TEST_PROCESS_STATS.cTotalDuration) ) ) logging.info("") LOCAL__print_line1_with_header("TOTAL INFORMATION") logging.info("") logging.info(" TOTAL ERROR COUNT : {0}".format(TEST_PROCESS_STATS.cTotalErrors)) logging.info(" TOTAL WARNING COUNT: {0}".format(TEST_PROCESS_STATS.cTotalWarnings)) logging.info("") # ///////////////////////////////////////////////////////////////////////////// def helper__detect_test_process_kind(config: pytest.Config) -> T_TEST_PROCESS_KIND: assert isinstance(config, pytest.Config) # # xdist' master process registers DSession plugin. # p = config.pluginmanager.get_plugin("dsession") if p is not None: return T_TEST_PROCESS_KIND.Master return T_TEST_PROCESS_KIND.Worker # ------------------------------------------------------------------------ def helper__detect_test_process_mode(config: pytest.Config) -> T_TEST_PROCESS_MODE: assert isinstance(config, pytest.Config) if config.getvalue("collectonly"): return T_TEST_PROCESS_MODE.Collect return T_TEST_PROCESS_MODE.ExecTests # ------------------------------------------------------------------------ @pytest.hookimpl(trylast=True) def helper__pytest_configure__logging(config: pytest.Config) -> None: assert isinstance(config, pytest.Config) log_name = TestStartupData.GetCurrentTestWorkerSignature() log_name += ".log" log_dir = TestStartupData.GetRootLogDir() pathlib.Path(log_dir).mkdir(exist_ok=True) logging_plugin = config.pluginmanager.get_plugin("logging-plugin") assert logging_plugin is not None assert isinstance(logging_plugin, _pytest.logging.LoggingPlugin) log_file_path = os.path.join(log_dir, log_name) assert log_file_path is not None assert type(log_file_path) is str logging_plugin.set_log_path(log_file_path) return # ------------------------------------------------------------------------ @pytest.hookimpl(trylast=True) def pytest_configure(config: pytest.Config) -> None: assert isinstance(config, pytest.Config) global g_test_process_kind global g_test_process_mode global g_worker_log_is_created assert g_test_process_kind is None assert g_test_process_mode is None assert g_worker_log_is_created is None g_test_process_mode = helper__detect_test_process_mode(config) g_test_process_kind = helper__detect_test_process_kind(config) assert type(g_test_process_kind) is T_TEST_PROCESS_KIND assert type(g_test_process_mode) is T_TEST_PROCESS_MODE if g_test_process_kind == T_TEST_PROCESS_KIND.Master: pass else: assert g_test_process_kind == T_TEST_PROCESS_KIND.Worker if g_test_process_mode == T_TEST_PROCESS_MODE.Collect: g_worker_log_is_created = False else: assert g_test_process_mode == T_TEST_PROCESS_MODE.ExecTests helper__pytest_configure__logging(config) g_worker_log_is_created = True return # ///////////////////////////////////////////////////////////////////////////// ================================================ FILE: tests/helpers/__init__.py ================================================ ================================================ FILE: tests/helpers/global_data.py ================================================ from testgres.operations.os_ops import OsOperations from testgres.operations.os_ops import ConnectionParams from testgres.operations.local_ops import LocalOperations from testgres.operations.remote_ops import RemoteOperations from src.node import PortManager from src.node import PortManager__ThisHost from src.node import PortManager__Generic import os class OsOpsDescr: sign: str os_ops: OsOperations def __init__(self, sign: str, os_ops: OsOperations): assert type(sign) is str assert isinstance(os_ops, OsOperations) self.sign = sign self.os_ops = os_ops class OsOpsDescrs: sm_remote_conn_params = ConnectionParams( host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', username=os.getenv('USER'), ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) sm_remote_os_ops = RemoteOperations(sm_remote_conn_params) sm_remote_os_ops_descr = OsOpsDescr("remote_ops", sm_remote_os_ops) sm_local_os_ops = LocalOperations.get_single_instance() sm_local_os_ops_descr = OsOpsDescr("local_ops", sm_local_os_ops) class PortManagers: sm_remote_port_manager = PortManager__Generic(OsOpsDescrs.sm_remote_os_ops) sm_local_port_manager = PortManager__ThisHost.get_single_instance() sm_local2_port_manager = PortManager__Generic(OsOpsDescrs.sm_local_os_ops) class PostgresNodeService: sign: str os_ops: OsOperations port_manager: PortManager def __init__(self, sign: str, os_ops: OsOperations, port_manager: PortManager): assert type(sign) is str assert isinstance(os_ops, OsOperations) assert isinstance(port_manager, PortManager) self.sign = sign self.os_ops = os_ops self.port_manager = port_manager class PostgresNodeServices: sm_remote = PostgresNodeService( "remote", OsOpsDescrs.sm_remote_os_ops, PortManagers.sm_remote_port_manager ) sm_local = PostgresNodeService( "local", OsOpsDescrs.sm_local_os_ops, PortManagers.sm_local_port_manager ) sm_local2 = PostgresNodeService( "local2", OsOpsDescrs.sm_local_os_ops, PortManagers.sm_local2_port_manager ) sm_locals_and_remotes = [ sm_local, sm_local2, sm_remote, ] ================================================ FILE: tests/helpers/pg_node_utils.py ================================================ from src import PostgresNode from src import PortManager from src import OsOperations from src import NodeStatus from src.node import PostgresNodeLogReader from tests.helpers.utils import Utils as HelperUtils from tests.helpers.utils import T_WAIT_TIME from tests.helpers.global_data import PostgresNodeService import typing class PostgresNodeUtils: class PostgresNodeUtilsException(Exception): pass class PortConflictNodeException(PostgresNodeUtilsException): _data_dir: str _port: int def __init__(self, data_dir: str, port: int): assert type(data_dir) is str assert type(port) is int super().__init__() self._data_dir = data_dir self._port = port return @property def data_dir(self) -> str: assert type(self._data_dir) is str return self._data_dir @property def port(self) -> int: assert type(self._port) is int return self._port @property def message(self) -> str: assert type(self._data_dir) is str assert type(self._port) is int r = "PostgresNode [data:{}][port: {}] conflicts with port of another instance.".format( self._data_dir, self._port, ) assert type(r) is str return r def __str__(self) -> str: r = self.message assert type(r) is str return r def __repr__(self) -> str: # It must be overrided! assert type(self) is __class__ r = "{}({}, {})".format( __class__.__name__, repr(self._data_dir), repr(self._port), ) assert type(r) is str return r # -------------------------------------------------------------------- class StartNodeException(PostgresNodeUtilsException): _data_dir: str _files: typing.Optional[typing.Iterable] def __init__( self, data_dir: str, files: typing.Optional[typing.Iterable] = None ): assert type(data_dir) is str assert files is None or isinstance(files, typing.Iterable) super().__init__() self._data_dir = data_dir self._files = files return @property def message(self) -> str: assert self._data_dir is None or type(self._data_dir) is str assert self._files is None or isinstance(self._files, typing.Iterable) msg_parts = [] msg_parts.append("PostgresNode [data_dir: {}] is not started.".format( self._data_dir )) for f, lines in self._files or []: assert type(f) is str assert type(lines) in [str, bytes] msg_parts.append(u'{}\n----\n{}\n'.format(f, lines)) return "\n".join(msg_parts) @property def data_dir(self) -> typing.Optional[str]: assert type(self._data_dir) is str return self._data_dir @property def files(self) -> typing.Optional[typing.Iterable]: assert self._files is None or isinstance(self._files, typing.Iterable) return self._files def __repr__(self) -> str: assert type(self._data_dir) is str assert self._files is None or isinstance(self._files, typing.Iterable) r = "{}({}, {})".format( __class__.__name__, repr(self._data_dir), repr(self._files), ) assert type(r) is str return r # -------------------------------------------------------------------- @staticmethod def get_node( node_svc: PostgresNodeService, name: typing.Optional[str] = None, port: typing.Optional[int] = None, port_manager: typing.Optional[PortManager] = None ) -> PostgresNode: assert isinstance(node_svc, PostgresNodeService) assert isinstance(node_svc.os_ops, OsOperations) assert isinstance(node_svc.port_manager, PortManager) if port_manager is None: port_manager = node_svc.port_manager return PostgresNode( name, port=port, os_ops=node_svc.os_ops, port_manager=port_manager if port is None else None ) # -------------------------------------------------------------------- @staticmethod def wait_for_running_state( node: PostgresNode, node_log_reader: PostgresNodeLogReader, timeout: T_WAIT_TIME, ): assert type(node) is PostgresNode assert type(node_log_reader) is PostgresNodeLogReader assert type(timeout) in [int, float] assert node_log_reader._node is node assert timeout > 0 for _ in HelperUtils.WaitUntil( timeout=timeout ): s = node.status() if s == NodeStatus.Running: return assert s == NodeStatus.Stopped blocks = node_log_reader.read() assert type(blocks) is list for block in blocks: assert type(block) is PostgresNodeLogReader.LogDataBlock if 'Is another postmaster already running on port' in block.data: raise __class__.PortConflictNodeException(node.data_dir, node.port) if 'database system is shut down' in block.data: raise __class__.StartNodeException( node.data_dir, node._collect_special_files(), ) continue ================================================ FILE: tests/helpers/run_conditions.py ================================================ # coding: utf-8 import pytest import platform class RunConditions: # It is not a test kit! __test__ = False @staticmethod def skip_if_windows(): if platform.system().lower() == "windows": pytest.skip("This test does not support Windows.") ================================================ FILE: tests/helpers/utils.py ================================================ import typing import time import logging T_WAIT_TIME = typing.Union[int, float] class Utils: @staticmethod def PrintAndSleep(wait: T_WAIT_TIME): assert type(wait) in [int, float] logging.info("Wait for {} second(s)".format(wait)) time.sleep(wait) return @staticmethod def WaitUntil( error_message: str = "Did not complete", timeout: T_WAIT_TIME = 30, interval: T_WAIT_TIME = 1, notification_interval: T_WAIT_TIME = 5, ): """ Loop until the timeout is reached. If the timeout is reached, raise an exception with the given error message. Source of idea: pgbouncer """ assert type(timeout) in [int, float] assert type(interval) in [int, float] assert type(notification_interval) in [int, float] assert timeout >= 0 assert interval >= 0 assert notification_interval >= 0 start_ts = time.monotonic() end_ts = start_ts + timeout last_printed_progress = start_ts last_iteration_ts = start_ts yield attempt = 1 while end_ts > time.monotonic(): if (timeout > 5 and time.monotonic() - last_printed_progress) > notification_interval: last_printed_progress = time.monotonic() m = "{} in {} seconds and {} attempts - will retry".format( error_message, time.monotonic() - start_ts, attempt, ) logging.info(m) interval_remaining = last_iteration_ts + interval - time.monotonic() if interval_remaining > 0: time.sleep(interval_remaining) last_iteration_ts = time.monotonic() yield attempt += 1 continue raise TimeoutError(error_message + " in time") ================================================ FILE: tests/requirements.txt ================================================ psutil pytest pytest-env pytest-xdist psycopg2 six testgres.os_ops>=2.1.0,<3.0.0 ================================================ FILE: tests/test_config.py ================================================ from src import TestgresConfig from src import configure_testgres from src import scoped_config from src import pop_config import src as testgres import pytest class TestConfig: def test_config_stack(self): # no such option with pytest.raises(expected_exception=TypeError): configure_testgres(dummy=True) # we have only 1 config in stack with pytest.raises(expected_exception=IndexError): pop_config() d0 = TestgresConfig.cached_initdb_dir d1 = 'dummy_abc' d2 = 'dummy_def' with scoped_config(cached_initdb_dir=d1) as c1: assert (c1.cached_initdb_dir == d1) with scoped_config(cached_initdb_dir=d2) as c2: stack_size = len(testgres.config.config_stack) # try to break a stack with pytest.raises(expected_exception=TypeError): with scoped_config(dummy=True): pass assert (c2.cached_initdb_dir == d2) assert (len(testgres.config.config_stack) == stack_size) assert (c1.cached_initdb_dir == d1) assert (TestgresConfig.cached_initdb_dir == d0) ================================================ FILE: tests/test_conftest.py--devel ================================================ import pytest import logging class TestConfest: def test_failed(self): raise Exception("TEST EXCEPTION!") def test_ok(self): pass @pytest.mark.skip() def test_mark_skip__no_reason(self): pass @pytest.mark.xfail() def test_mark_xfail__no_reason(self): raise Exception("XFAIL EXCEPTION") @pytest.mark.xfail() def test_mark_xfail__no_reason___no_error(self): pass @pytest.mark.skip(reason="reason") def test_mark_skip__with_reason(self): pass @pytest.mark.xfail(reason="reason") def test_mark_xfail__with_reason(self): raise Exception("XFAIL EXCEPTION") @pytest.mark.xfail(reason="reason") def test_mark_xfail__with_reason___no_error(self): pass def test_exc_skip__no_reason(self): pytest.skip() def test_exc_xfail__no_reason(self): pytest.xfail() def test_exc_skip__with_reason(self): pytest.skip(reason="SKIP REASON") def test_exc_xfail__with_reason(self): pytest.xfail(reason="XFAIL EXCEPTION") def test_log_error(self): logging.error("IT IS A LOG ERROR!") def test_log_error_and_exc(self): logging.error("IT IS A LOG ERROR!") raise Exception("TEST EXCEPTION!") def test_log_error_and_warning(self): logging.error("IT IS A LOG ERROR!") logging.warning("IT IS A LOG WARNING!") logging.error("IT IS THE SECOND LOG ERROR!") logging.warning("IT IS THE SECOND LOG WARNING!") @pytest.mark.xfail() def test_log_error_and_xfail_mark_without_reason(self): logging.error("IT IS A LOG ERROR!") @pytest.mark.xfail(reason="It is a reason message") def test_log_error_and_xfail_mark_with_reason(self): logging.error("IT IS A LOG ERROR!") @pytest.mark.xfail() def test_two_log_error_and_xfail_mark_without_reason(self): logging.error("IT IS THE FIRST LOG ERROR!") logging.info("----------") logging.error("IT IS THE SECOND LOG ERROR!") @pytest.mark.xfail(reason="It is a reason message") def test_two_log_error_and_xfail_mark_with_reason(self): logging.error("IT IS THE FIRST LOG ERROR!") logging.info("----------") logging.error("IT IS THE SECOND LOG ERROR!") ================================================ FILE: tests/test_os_ops_common.py ================================================ # coding: utf-8 from .helpers.global_data import OsOpsDescr from .helpers.global_data import OsOpsDescrs from .helpers.global_data import OsOperations from .helpers.run_conditions import RunConditions import os import sys import pytest import re import tempfile import logging import socket import threading import typing import uuid import subprocess import psutil import time import signal as os_signal from src import InvalidOperationException from src import ExecUtilException from concurrent.futures import ThreadPoolExecutor from concurrent.futures import Future as ThreadFuture class TestOsOpsCommon: sm_os_ops_descrs: typing.List[OsOpsDescr] = [ OsOpsDescrs.sm_local_os_ops_descr, OsOpsDescrs.sm_remote_os_ops_descr ] @pytest.fixture( params=[descr.os_ops for descr in sm_os_ops_descrs], ids=[descr.sign for descr in sm_os_ops_descrs] ) def os_ops(self, request: pytest.FixtureRequest) -> OsOperations: assert isinstance(request, pytest.FixtureRequest) assert isinstance(request.param, OsOperations) return request.param def test_get_platform(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) p = os_ops.get_platform() assert p is not None assert type(p) is str assert p == sys.platform def test_get_platform__is_known(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) p = os_ops.get_platform() assert p is not None assert type(p) is str assert p in {"win32", "linux"} def test_create_clone(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) clone = os_ops.create_clone() assert clone is not None assert clone is not os_ops assert type(clone) is type(os_ops) def test_exec_command_success(self, os_ops: OsOperations): """ Test exec_command for successful command execution. """ assert isinstance(os_ops, OsOperations) RunConditions.skip_if_windows() cmd = ["sh", "-c", "python3 --version"] response = os_ops.exec_command(cmd) assert b'Python 3.' in response def test_exec_command_failure(self, os_ops: OsOperations): """ Test exec_command for command execution failure. """ assert isinstance(os_ops, OsOperations) RunConditions.skip_if_windows() cmd = ["sh", "-c", "nonexistent_command"] while True: try: os_ops.exec_command(cmd) except ExecUtilException as e: assert type(e.exit_code) is int assert e.exit_code == 127 assert type(e.message) is str assert type(e.error) is bytes assert e.message.startswith("Utility exited with non-zero code (127). Error:") assert "nonexistent_command" in e.message assert "not found" in e.message assert b"nonexistent_command" in e.error assert b"not found" in e.error break raise Exception("We wait an exception!") def test_exec_command_failure__expect_error(self, os_ops: OsOperations): """ Test exec_command for command execution failure. """ assert isinstance(os_ops, OsOperations) RunConditions.skip_if_windows() cmd = ["sh", "-c", "nonexistent_command"] exit_status, result, error = os_ops.exec_command(cmd, verbose=True, expect_error=True) assert exit_status == 127 assert result == b'' assert type(error) is bytes assert b"nonexistent_command" in error assert b"not found" in error def test_exec_command_with_exec_env(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) RunConditions.skip_if_windows() C_ENV_NAME = "TESTGRES_TEST__EXEC_ENV_20250414" cmd = ["sh", "-c", "echo ${}".format(C_ENV_NAME)] exec_env = {C_ENV_NAME: "Hello!"} response = os_ops.exec_command(cmd, exec_env=exec_env) assert response is not None assert type(response) is bytes assert response == b'Hello!\n' response = os_ops.exec_command(cmd) assert response is not None assert type(response) is bytes assert response == b'\n' def test_exec_command_with_exec_env__2(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) RunConditions.skip_if_windows() C_ENV_NAME = "TESTGRES_TEST__EXEC_ENV_20250414" tmp_file_content = "echo ${{{}}}".format(C_ENV_NAME) logging.info("content is [{}]".format(tmp_file_content)) tmp_file = os_ops.mkstemp() assert type(tmp_file) is str assert tmp_file != "" logging.info("file is [{}]".format(tmp_file)) assert os_ops.path_exists(tmp_file) os_ops.write(tmp_file, tmp_file_content) cmd = ["sh", tmp_file] exec_env = {C_ENV_NAME: "Hello!"} response = os_ops.exec_command(cmd, exec_env=exec_env) assert response is not None assert type(response) is bytes assert response == b'Hello!\n' response = os_ops.exec_command(cmd) assert response is not None assert type(response) is bytes assert response == b'\n' os_ops.remove_file(tmp_file) assert not os_ops.path_exists(tmp_file) return def test_exec_command_with_cwd(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) RunConditions.skip_if_windows() cmd = ["pwd"] response = os_ops.exec_command(cmd, cwd="/tmp") assert response is not None assert type(response) is bytes assert response == b'/tmp\n' response = os_ops.exec_command(cmd) assert response is not None assert type(response) is bytes assert response != b'/tmp\n' def test_exec_command__test_unset(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) RunConditions.skip_if_windows() C_ENV_NAME = "LANG" cmd = ["sh", "-c", "echo ${}".format(C_ENV_NAME)] response1 = os_ops.exec_command(cmd) assert response1 is not None assert type(response1) is bytes if response1 == b'\n': logging.warning("Environment variable {} is not defined.".format(C_ENV_NAME)) return exec_env = {C_ENV_NAME: None} response2 = os_ops.exec_command(cmd, exec_env=exec_env) assert response2 is not None assert type(response2) is bytes assert response2 == b'\n' response3 = os_ops.exec_command(cmd) assert response3 is not None assert type(response3) is bytes assert response3 == response1 def test_exec_command__test_unset_dummy_var(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) RunConditions.skip_if_windows() C_ENV_NAME = "TESTGRES_TEST__DUMMY_VAR_20250414" cmd = ["sh", "-c", "echo ${}".format(C_ENV_NAME)] exec_env = {C_ENV_NAME: None} response2 = os_ops.exec_command(cmd, exec_env=exec_env) assert response2 is not None assert type(response2) is bytes assert response2 == b'\n' def test_is_executable_true(self, os_ops: OsOperations): """ Test is_executable for an existing executable. """ assert isinstance(os_ops, OsOperations) RunConditions.skip_if_windows() response = os_ops.is_executable("/bin/sh") assert response is True def test_is_executable_false(self, os_ops: OsOperations): """ Test is_executable for a non-executable. """ assert isinstance(os_ops, OsOperations) response = os_ops.is_executable(__file__) assert response is False def test_makedirs_and_rmdirs_success(self, os_ops: OsOperations): """ Test makedirs and rmdirs for successful directory creation and removal. """ assert isinstance(os_ops, OsOperations) RunConditions.skip_if_windows() cmd = "pwd" pwd = os_ops.exec_command(cmd, wait_exit=True, encoding='utf-8').strip() path = "{}/test_dir".format(pwd) # Test makedirs os_ops.makedirs(path) assert os.path.exists(path) assert os_ops.path_exists(path) # Test rmdirs os_ops.rmdirs(path) assert not os.path.exists(path) assert not os_ops.path_exists(path) def test_makedirs_failure(self, os_ops: OsOperations): """ Test makedirs for failure. """ # Try to create a directory in a read-only location assert isinstance(os_ops, OsOperations) RunConditions.skip_if_windows() path = "/root/test_dir" # Test makedirs with pytest.raises(Exception): os_ops.makedirs(path) def test_listdir(self, os_ops: OsOperations): """ Test listdir for listing directory contents. """ assert isinstance(os_ops, OsOperations) RunConditions.skip_if_windows() path = "/etc" files = os_ops.listdir(path) assert isinstance(files, list) for f in files: assert f is not None assert type(f) is str def test_path_exists_true__directory(self, os_ops: OsOperations): """ Test path_exists for an existing directory. """ assert isinstance(os_ops, OsOperations) RunConditions.skip_if_windows() assert os_ops.path_exists("/etc") is True def test_path_exists_true__file(self, os_ops: OsOperations): """ Test path_exists for an existing file. """ assert isinstance(os_ops, OsOperations) RunConditions.skip_if_windows() assert os_ops.path_exists(__file__) is True def test_path_exists_false__directory(self, os_ops: OsOperations): """ Test path_exists for a non-existing directory. """ assert isinstance(os_ops, OsOperations) RunConditions.skip_if_windows() assert os_ops.path_exists("/nonexistent_path") is False def test_path_exists_false__file(self, os_ops: OsOperations): """ Test path_exists for a non-existing file. """ assert isinstance(os_ops, OsOperations) RunConditions.skip_if_windows() assert os_ops.path_exists("/etc/nonexistent_path.txt") is False def test_mkdtemp__default(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) path = os_ops.mkdtemp() logging.info("Path is [{0}].".format(path)) assert os.path.exists(path) os.rmdir(path) assert not os.path.exists(path) def test_mkdtemp__custom(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) C_TEMPLATE = "abcdef" path = os_ops.mkdtemp(C_TEMPLATE) logging.info("Path is [{0}].".format(path)) assert os.path.exists(path) assert C_TEMPLATE in os.path.basename(path) os.rmdir(path) assert not os.path.exists(path) def test_rmdirs(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) path = os_ops.mkdtemp() assert os.path.exists(path) assert os_ops.rmdirs(path, ignore_errors=False) is True assert not os.path.exists(path) def test_rmdirs__01_with_subfolder(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) # folder with subfolder path = os_ops.mkdtemp() assert os.path.exists(path) dir1 = os.path.join(path, "dir1") assert not os.path.exists(dir1) os_ops.makedirs(dir1) assert os.path.exists(dir1) assert os_ops.rmdirs(path, ignore_errors=False) is True assert not os.path.exists(path) assert not os.path.exists(dir1) def test_rmdirs__02_with_file(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) # folder with file path = os_ops.mkdtemp() assert os.path.exists(path) file1 = os.path.join(path, "file1.txt") assert not os.path.exists(file1) os_ops.touch(file1) assert os.path.exists(file1) assert os_ops.rmdirs(path, ignore_errors=False) is True assert not os.path.exists(path) assert not os.path.exists(file1) def test_rmdirs__03_with_subfolder_and_file(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) # folder with subfolder and file path = os_ops.mkdtemp() assert os.path.exists(path) dir1 = os.path.join(path, "dir1") assert not os.path.exists(dir1) os_ops.makedirs(dir1) assert os.path.exists(dir1) file1 = os.path.join(dir1, "file1.txt") assert not os.path.exists(file1) os_ops.touch(file1) assert os.path.exists(file1) assert os_ops.rmdirs(path, ignore_errors=False) is True assert not os.path.exists(path) assert not os.path.exists(dir1) assert not os.path.exists(file1) def test_write_text_file(self, os_ops: OsOperations): """ Test write for writing data to a text file. """ assert isinstance(os_ops, OsOperations) RunConditions.skip_if_windows() filename = os_ops.mkstemp() data = "Hello, world!" os_ops.write(filename, data, truncate=True) os_ops.write(filename, data) response = os_ops.read(filename) assert response == data + data os_ops.remove_file(filename) def test_write_binary_file(self, os_ops: OsOperations): """ Test write for writing data to a binary file. """ assert isinstance(os_ops, OsOperations) RunConditions.skip_if_windows() filename = "/tmp/test_file.bin" data = b"\x00\x01\x02\x03" os_ops.write(filename, data, binary=True, truncate=True) response = os_ops.read(filename, binary=True) assert response == data def test_read_text_file(self, os_ops: OsOperations): """ Test read for reading data from a text file. """ assert isinstance(os_ops, OsOperations) RunConditions.skip_if_windows() filename = "/etc/hosts" response = os_ops.read(filename) assert isinstance(response, str) def test_read_binary_file(self, os_ops: OsOperations): """ Test read for reading data from a binary file. """ assert isinstance(os_ops, OsOperations) RunConditions.skip_if_windows() filename = "/usr/bin/python3" response = os_ops.read(filename, binary=True) assert isinstance(response, bytes) def test_read__text(self, os_ops: OsOperations): """ Test OsOperations::read for text data. """ assert isinstance(os_ops, OsOperations) filename = __file__ # current file with open(filename, 'r') as file: # open in a text mode response0 = file.read() assert type(response0) is str response1 = os_ops.read(filename) assert type(response1) is str assert response1 == response0 response2 = os_ops.read(filename, encoding=None, binary=False) assert type(response2) is str assert response2 == response0 response3 = os_ops.read(filename, encoding="") assert type(response3) is str assert response3 == response0 response4 = os_ops.read(filename, encoding="UTF-8") assert type(response4) is str assert response4 == response0 def test_read__binary(self, os_ops: OsOperations): """ Test OsOperations::read for binary data. """ filename = __file__ # current file with open(filename, 'rb') as file: # open in a binary mode response0 = file.read() assert type(response0) is bytes response1 = os_ops.read(filename, binary=True) assert type(response1) is bytes assert response1 == response0 def test_read__binary_and_encoding(self, os_ops: OsOperations): """ Test OsOperations::read for binary data and encoding. """ assert isinstance(os_ops, OsOperations) filename = __file__ # current file with pytest.raises( InvalidOperationException, match=re.escape("Enconding is not allowed for read binary operation")): os_ops.read(filename, encoding="", binary=True) def test_read_binary__spec(self, os_ops: OsOperations): """ Test OsOperations::read_binary. """ assert isinstance(os_ops, OsOperations) filename = __file__ # currnt file with open(filename, 'rb') as file: # open in a binary mode response0 = file.read() assert type(response0) is bytes response1 = os_ops.read_binary(filename, 0) assert type(response1) is bytes assert response1 == response0 response2 = os_ops.read_binary(filename, 1) assert type(response2) is bytes assert len(response2) < len(response1) assert len(response2) + 1 == len(response1) assert response2 == response1[1:] response3 = os_ops.read_binary(filename, len(response1)) assert type(response3) is bytes assert len(response3) == 0 response4 = os_ops.read_binary(filename, len(response2)) assert type(response4) is bytes assert len(response4) == 1 assert response4[0] == response1[len(response1) - 1] response5 = os_ops.read_binary(filename, len(response1) + 1) assert type(response5) is bytes assert len(response5) == 0 def test_read_binary__spec__negative_offset(self, os_ops: OsOperations): """ Test OsOperations::read_binary with negative offset. """ assert isinstance(os_ops, OsOperations) with pytest.raises( ValueError, match=re.escape("Negative 'offset' is not supported.")): os_ops.read_binary(__file__, -1) def test_get_file_size(self, os_ops: OsOperations): """ Test OsOperations::get_file_size. """ assert isinstance(os_ops, OsOperations) filename = __file__ # current file sz0 = os.path.getsize(filename) assert type(sz0) is int sz1 = os_ops.get_file_size(filename) assert type(sz1) is int assert sz1 == sz0 def test_isfile_true(self, os_ops: OsOperations): """ Test isfile for an existing file. """ assert isinstance(os_ops, OsOperations) filename = __file__ response = os_ops.isfile(filename) assert response is True def test_isfile_false__not_exist(self, os_ops: OsOperations): """ Test isfile for a non-existing file. """ assert isinstance(os_ops, OsOperations) filename = os.path.join(os.path.dirname(__file__), "nonexistent_file.txt") response = os_ops.isfile(filename) assert response is False def test_isfile_false__directory(self, os_ops: OsOperations): """ Test isfile for a firectory. """ assert isinstance(os_ops, OsOperations) name = os.path.dirname(__file__) assert os_ops.isdir(name) response = os_ops.isfile(name) assert response is False def test_isdir_true(self, os_ops: OsOperations): """ Test isdir for an existing directory. """ assert isinstance(os_ops, OsOperations) name = os.path.dirname(__file__) response = os_ops.isdir(name) assert response is True def test_isdir_false__not_exist(self, os_ops: OsOperations): """ Test isdir for a non-existing directory. """ assert isinstance(os_ops, OsOperations) name = os.path.join(os.path.dirname(__file__), "it_is_nonexistent_directory") response = os_ops.isdir(name) assert response is False def test_isdir_false__file(self, os_ops: OsOperations): """ Test isdir for a file. """ assert isinstance(os_ops, OsOperations) name = __file__ assert os_ops.isfile(name) response = os_ops.isdir(name) assert response is False def test_cwd(self, os_ops: OsOperations): """ Test cwd. """ assert isinstance(os_ops, OsOperations) v = os_ops.cwd() assert v is not None assert type(v) is str assert v != "" class tagWriteData001: def __init__(self, sign, source, cp_rw, cp_truncate, cp_binary, cp_data, result): self.sign = sign self.source = source self.call_param__rw = cp_rw self.call_param__truncate = cp_truncate self.call_param__binary = cp_binary self.call_param__data = cp_data self.result = result sm_write_data001 = [ tagWriteData001("A001", "1234567890", False, False, False, "ABC", "1234567890ABC"), tagWriteData001("A002", b"1234567890", False, False, True, b"ABC", b"1234567890ABC"), tagWriteData001("B001", "1234567890", False, True, False, "ABC", "ABC"), tagWriteData001("B002", "1234567890", False, True, False, "ABC1234567890", "ABC1234567890"), tagWriteData001("B003", b"1234567890", False, True, True, b"ABC", b"ABC"), tagWriteData001("B004", b"1234567890", False, True, True, b"ABC1234567890", b"ABC1234567890"), tagWriteData001("C001", "1234567890", True, False, False, "ABC", "1234567890ABC"), tagWriteData001("C002", b"1234567890", True, False, True, b"ABC", b"1234567890ABC"), tagWriteData001("D001", "1234567890", True, True, False, "ABC", "ABC"), tagWriteData001("D002", "1234567890", True, True, False, "ABC1234567890", "ABC1234567890"), tagWriteData001("D003", b"1234567890", True, True, True, b"ABC", b"ABC"), tagWriteData001("D004", b"1234567890", True, True, True, b"ABC1234567890", b"ABC1234567890"), tagWriteData001("E001", "\0001234567890\000", False, False, False, "\000ABC\000", "\0001234567890\000\000ABC\000"), tagWriteData001("E002", b"\0001234567890\000", False, False, True, b"\000ABC\000", b"\0001234567890\000\000ABC\000"), tagWriteData001("F001", "a\nb\n", False, False, False, ["c", "d"], "a\nb\nc\nd\n"), tagWriteData001("F002", b"a\nb\n", False, False, True, [b"c", b"d"], b"a\nb\nc\nd\n"), tagWriteData001("G001", "a\nb\n", False, False, False, ["c\n\n", "d\n"], "a\nb\nc\nd\n"), tagWriteData001("G002", b"a\nb\n", False, False, True, [b"c\n\n", b"d\n"], b"a\nb\nc\nd\n"), ] @pytest.fixture( params=sm_write_data001, ids=[x.sign for x in sm_write_data001], ) def write_data001(self, request): assert isinstance(request, pytest.FixtureRequest) assert type(request.param) is __class__.tagWriteData001 return request.param def test_write(self, write_data001: tagWriteData001, os_ops: OsOperations): assert type(write_data001) is __class__.tagWriteData001 assert isinstance(os_ops, OsOperations) mode = "w+b" if write_data001.call_param__binary else "w+" with tempfile.NamedTemporaryFile(mode=mode, delete=True) as tmp_file: tmp_file.write(write_data001.source) tmp_file.flush() os_ops.write( tmp_file.name, write_data001.call_param__data, read_and_write=write_data001.call_param__rw, truncate=write_data001.call_param__truncate, binary=write_data001.call_param__binary) tmp_file.seek(0) s = tmp_file.read() assert s == write_data001.result def test_touch(self, os_ops: OsOperations): """ Test touch for creating a new file or updating access and modification times of an existing file. """ assert isinstance(os_ops, OsOperations) filename = os_ops.mkstemp() # TODO: this test does not check the result of 'touch' command! os_ops.touch(filename) assert os_ops.isfile(filename) os_ops.remove_file(filename) def test_is_port_free__true(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) C_LIMIT = 128 ports = set(range(1024, 65535)) assert type(ports) is set ok_count = 0 no_count = 0 for port in ports: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: try: s.bind(("", port)) except OSError: continue r = os_ops.is_port_free(port) if r: ok_count += 1 logging.info("OK. Port {} is free.".format(port)) else: no_count += 1 logging.warning("NO. Port {} is not free.".format(port)) if ok_count == C_LIMIT: return if no_count == C_LIMIT: raise RuntimeError("To many false positive test attempts.") if ok_count == 0: raise RuntimeError("No one free port was found.") def test_is_port_free__false(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) C_LIMIT = 10 ports = set(range(1024, 65535)) assert type(ports) is set def LOCAL_server(s: socket.socket): assert s is not None assert type(s) is socket.socket try: while True: r = s.accept() if r is None: break except Exception as e: assert e is not None pass ok_count = 0 no_count = 0 for port in ports: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: try: s.bind(("", port)) except OSError: continue th = threading.Thread(target=LOCAL_server, args=[s]) s.listen(10) assert type(th) is threading.Thread th.start() try: r = os_ops.is_port_free(port) finally: s.shutdown(2) th.join() if not r: ok_count += 1 logging.info("OK. Port {} is not free.".format(port)) else: no_count += 1 logging.warning("NO. Port {} does not accept connection.".format(port)) if ok_count == C_LIMIT: return if no_count == C_LIMIT: raise RuntimeError("To many false positive test attempts.") if ok_count == 0: raise RuntimeError("No one free port was found.") def test_get_tmpdir(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) dir = os_ops.get_tempdir() assert type(dir) is str assert os_ops.path_exists(dir) assert os.path.exists(dir) file_path = os.path.join(dir, "testgres--" + uuid.uuid4().hex + ".tmp") os_ops.write(file_path, "1234", binary=False) assert os_ops.path_exists(file_path) assert os.path.exists(file_path) d = os_ops.read(file_path, binary=False) assert d == "1234" os_ops.remove_file(file_path) assert not os_ops.path_exists(file_path) assert not os.path.exists(file_path) def test_get_tmpdir__compare_with_py_info(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) actual_dir = os_ops.get_tempdir() assert actual_dir is not None assert type(actual_dir) is str # -------- cmd = [sys.executable, "-c", "import tempfile;print(tempfile.gettempdir());"] expected_dir_b = os_ops.exec_command(cmd) assert type(expected_dir_b) is bytes expected_dir = expected_dir_b.decode() assert type(expected_dir) is str assert actual_dir + "\n" == expected_dir return class tagData_OS_OPS__NUMS: os_ops_descr: OsOpsDescr nums: int def __init__(self, os_ops_descr: OsOpsDescr, nums: int): assert isinstance(os_ops_descr, OsOpsDescr) assert type(nums) is int self.os_ops_descr = os_ops_descr self.nums = nums sm_test_exclusive_creation__mt__data = [ tagData_OS_OPS__NUMS(OsOpsDescrs.sm_local_os_ops_descr, 100000), tagData_OS_OPS__NUMS(OsOpsDescrs.sm_remote_os_ops_descr, 120), ] @pytest.fixture( params=sm_test_exclusive_creation__mt__data, ids=[x.os_ops_descr.sign for x in sm_test_exclusive_creation__mt__data] ) def data001(self, request: pytest.FixtureRequest) -> tagData_OS_OPS__NUMS: assert isinstance(request, pytest.FixtureRequest) return request.param def test_mkdir__mt(self, data001: tagData_OS_OPS__NUMS): assert type(data001) is __class__.tagData_OS_OPS__NUMS N_WORKERS = 4 N_NUMBERS = data001.nums assert type(N_NUMBERS) is int os_ops = data001.os_ops_descr.os_ops assert isinstance(os_ops, OsOperations) lock_dir_prefix = "test_mkdir_mt--" + uuid.uuid4().hex lock_dir = os_ops.mkdtemp(prefix=lock_dir_prefix) logging.info("A lock file [{}] is creating ...".format(lock_dir)) assert os.path.exists(lock_dir) def MAKE_PATH(lock_dir: str, num: int) -> str: assert type(lock_dir) is str assert type(num) is int return os.path.join(lock_dir, str(num) + ".lock") def LOCAL_WORKER(os_ops: OsOperations, workerID: int, lock_dir: str, cNumbers: int, reservedNumbers: typing.Set[int]) -> None: assert isinstance(os_ops, OsOperations) assert type(workerID) is int assert type(lock_dir) is str assert type(cNumbers) is int assert type(reservedNumbers) is set assert cNumbers > 0 assert len(reservedNumbers) == 0 assert os.path.exists(lock_dir) def LOG_INFO(template: str, *args) -> None: assert type(template) is str assert type(args) is tuple msg = template.format(*args) assert type(msg) is str logging.info("[Worker #{}] {}".format(workerID, msg)) return LOG_INFO("HELLO! I am here!") for num in range(cNumbers): assert num not in reservedNumbers file_path = MAKE_PATH(lock_dir, num) try: os_ops.makedir(file_path) except Exception as e: LOG_INFO( "Can't reserve {}. Error ({}): {}", num, type(e).__name__, str(e) ) continue LOG_INFO("Number {} is reserved!", num) assert os_ops.path_exists(file_path) reservedNumbers.add(num) continue n_total = cNumbers n_ok = len(reservedNumbers) assert n_ok <= n_total LOG_INFO("Finish! OK: {}. FAILED: {}.", n_ok, n_total - n_ok) return # ----------------------- logging.info("Worker are creating ...") threadPool = ThreadPoolExecutor( max_workers=N_WORKERS, thread_name_prefix="ex_creator" ) class tadWorkerData: future: ThreadFuture reservedNumbers: typing.Set[int] workerDatas: typing.List[tadWorkerData] = list() nErrors = 0 try: for n in range(N_WORKERS): logging.info("worker #{} is creating ...".format(n)) workerDatas.append(tadWorkerData()) workerDatas[n].reservedNumbers = set() workerDatas[n].future = threadPool.submit( LOCAL_WORKER, os_ops, n, lock_dir, N_NUMBERS, workerDatas[n].reservedNumbers ) assert workerDatas[n].future is not None logging.info("OK. All the workers were created!") except Exception as e: nErrors += 1 logging.error("A problem is detected ({}): {}".format(type(e).__name__, str(e))) logging.info("Will wait for stop of all the workers...") nWorkers = 0 assert type(workerDatas) is list for i in range(len(workerDatas)): worker = workerDatas[i].future if worker is None: continue nWorkers += 1 assert isinstance(worker, ThreadFuture) try: logging.info("Wait for worker #{}".format(i)) worker.result() except Exception as e: nErrors += 1 logging.error("Worker #{} finished with error ({}): {}".format( i, type(e).__name__, str(e), )) continue assert nWorkers == N_WORKERS if nErrors != 0: raise RuntimeError("Some problems were detected. Please examine the log messages.") logging.info("OK. Let's check worker results!") reservedNumbers: typing.Dict[int, int] = dict() for i in range(N_WORKERS): logging.info("Worker #{} is checked ...".format(i)) workerNumbers = workerDatas[i].reservedNumbers assert type(workerNumbers) is set for n in workerNumbers: if n < 0 or n >= N_NUMBERS: nErrors += 1 logging.error("Unexpected number {}".format(n)) continue if n in reservedNumbers.keys(): nErrors += 1 logging.error("Number {} was already reserved by worker #{}".format( n, reservedNumbers[n] )) else: reservedNumbers[n] = i file_path = MAKE_PATH(lock_dir, n) if not os_ops.path_exists(file_path): nErrors += 1 logging.error("File {} is not found!".format(file_path)) continue continue logging.info("OK. Let's check reservedNumbers!") for n in range(N_NUMBERS): if n not in reservedNumbers.keys(): nErrors += 1 logging.error("Number {} is not reserved!".format(n)) continue file_path = MAKE_PATH(lock_dir, n) if not os_ops.path_exists(file_path): nErrors += 1 logging.error("File {} is not found!".format(file_path)) continue # OK! continue logging.info("Verification is finished! Total error count is {}.".format(nErrors)) if nErrors == 0: logging.info("Root lock-directory [{}] will be deleted.".format( lock_dir )) for n in range(N_NUMBERS): file_path = MAKE_PATH(lock_dir, n) try: os_ops.rmdir(file_path) except Exception as e: nErrors += 1 logging.error("Cannot delete directory [{}]. Error ({}): {}".format( file_path, type(e).__name__, str(e) )) continue if os_ops.path_exists(file_path): nErrors += 1 logging.error("Directory {} is not deleted!".format(file_path)) continue if nErrors == 0: try: os_ops.rmdir(lock_dir) except Exception as e: nErrors += 1 logging.error("Cannot delete directory [{}]. Error ({}): {}".format( lock_dir, type(e).__name__, str(e) )) logging.info("Test is finished! Total error count is {}.".format(nErrors)) return T_KILL_SIGNAL_DESCR = typing.Tuple[ str, typing.Union[int, os_signal.Signals], str ] sm_kill_signal_ids: typing.List[T_KILL_SIGNAL_DESCR] = [ ("SIGINT", os_signal.SIGINT, "2"), # ("SIGQUIT", os_signal.SIGQUIT, "3"), # it creates coredump ("SIGKILL", os_signal.SIGKILL, "9"), ("SIGTERM", os_signal.SIGTERM, "15"), ("2", 2, "2"), # ("3", 3, "3"), # it creates coredump ("9", 9, "9"), ("15", 15, "15"), ] @pytest.fixture( params=sm_kill_signal_ids, ids=["signal: {}".format(x[0]) for x in sm_kill_signal_ids], ) def kill_signal_id(self, request: pytest.FixtureRequest) -> T_KILL_SIGNAL_DESCR: assert isinstance(request, pytest.FixtureRequest) assert type(request.param) is tuple return request.param def test_kill_signal( self, kill_signal_id: T_KILL_SIGNAL_DESCR, ): assert type(kill_signal_id) is tuple assert "{}".format(kill_signal_id[1]) == kill_signal_id[2] assert "{}".format(int(kill_signal_id[1])) == kill_signal_id[2] def test_kill( self, os_ops: OsOperations, kill_signal_id: T_KILL_SIGNAL_DESCR, ): """ Test listdir for listing directory contents. """ assert isinstance(os_ops, OsOperations) assert type(kill_signal_id) is tuple cmd = [ sys.executable, "-c", "import time; print('ENTER');time.sleep(300);print('EXIT')" ] logging.info("Local test process is creating ...") proc = subprocess.Popen( cmd, text=True, ) assert proc is not None assert type(proc) is subprocess.Popen proc_pid = proc.pid assert type(proc_pid) is int logging.info("Test process pid is {}".format(proc_pid)) logging.info("Get this test process ...") p1 = psutil.Process(proc_pid) assert p1 is not None del p1 logging.info("Kill this test process ...") os_ops.kill(proc_pid, kill_signal_id[1]) logging.info("Wait for finish ...") proc.wait() logging.info("Try to get this test process ...") attempt = 0 while True: if attempt == 20: raise RuntimeError("Process did not die.") attempt += 1 if attempt > 1: logging.info("Sleep 1 seconds...") time.sleep(1) try: psutil.Process(proc_pid) except psutil.ZombieProcess as e: logging.info("Exception {}: {}".format( type(e).__name__, str(e), )) except psutil.NoSuchProcess: logging.info("OK. Process died.") break logging.info("Process is alive!") continue return def test_kill__unk_pid( self, os_ops: OsOperations, kill_signal_id: T_KILL_SIGNAL_DESCR, ): """ Test listdir for listing directory contents. """ assert isinstance(os_ops, OsOperations) assert type(kill_signal_id) is tuple cmd = [ sys.executable, "-c", "import sys; print(\"a\", file=sys.stdout); print(\"b\", file=sys.stderr)" ] logging.info("Local test process is creating ...") proc = subprocess.Popen( cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) assert proc is not None assert type(proc) is subprocess.Popen proc_pid = proc.pid assert type(proc_pid) is int logging.info("Test process pid is {}".format(proc_pid)) logging.info("Wait for finish ...") pout, perr = proc.communicate() logging.info("STDOUT: {}".format(pout)) logging.info("STDERR: {}".format(pout)) assert type(pout) is str assert type(perr) is str assert pout == "a\n" assert perr == "b\n" assert type(proc.returncode) is int assert proc.returncode == 0 logging.info("Try to get this test process ...") attempt = 0 while True: if attempt == 20: raise RuntimeError("Process did not die.") attempt += 1 if attempt > 1: logging.info("Sleep 1 seconds...") time.sleep(1) try: psutil.Process(proc_pid) except psutil.ZombieProcess as e: logging.info("Exception {}: {}".format( type(e).__name__, str(e), )) except psutil.NoSuchProcess: logging.info("OK. Process died.") break logging.info("Process is alive!") continue # -------------------- with pytest.raises(expected_exception=Exception) as x: os_ops.kill(proc_pid, kill_signal_id[1]) assert x is not None assert isinstance(x.value, Exception) assert not isinstance(x.value, AssertionError) logging.info("Our error is [{}]".format(str(x.value))) logging.info("Our exception has type [{}]".format(type(x.value).__name__)) if type(os_ops).__name__ == "LocalOsOperations": assert type(x.value) is ProcessLookupError assert "No such process" in str(x.value) elif type(os_ops).__name__ == "RemoteOsOperations": assert type(x.value) is ExecUtilException assert "No such process" in str(x.value) else: RuntimeError("Unknown os_ops type: {}".format(type(os_ops).__name__)) return ================================================ FILE: tests/test_os_ops_local.py ================================================ # coding: utf-8 from .helpers.global_data import OsOpsDescrs from .helpers.global_data import OsOperations import os import pytest import re class TestOsOpsLocal: @pytest.fixture def os_ops(self): return OsOpsDescrs.sm_local_os_ops def test_read__unknown_file(self, os_ops: OsOperations): """ Test LocalOperations::read with unknown file. """ with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): os_ops.read("/dummy") def test_read_binary__spec__unk_file(self, os_ops: OsOperations): """ Test LocalOperations::read_binary with unknown file. """ with pytest.raises( FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): os_ops.read_binary("/dummy", 0) def test_get_file_size__unk_file(self, os_ops: OsOperations): """ Test LocalOperations::get_file_size. """ assert isinstance(os_ops, OsOperations) with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): os_ops.get_file_size("/dummy") def test_cwd(self, os_ops: OsOperations): """ Test cwd. """ assert isinstance(os_ops, OsOperations) v = os_ops.cwd() assert v is not None assert type(v) is str expectedValue = os.getcwd() assert expectedValue is not None assert type(expectedValue) is str assert expectedValue != "" # research # Comp result assert v == expectedValue ================================================ FILE: tests/test_os_ops_remote.py ================================================ # coding: utf-8 from .helpers.global_data import OsOpsDescrs from .helpers.global_data import OsOperations from src import ExecUtilException import os import pytest class TestOsOpsRemote: @pytest.fixture def os_ops(self): return OsOpsDescrs.sm_remote_os_ops def test_rmdirs__try_to_delete_nonexist_path(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) path = "/root/test_dir" assert os_ops.rmdirs(path, ignore_errors=False) is True def test_rmdirs__try_to_delete_file(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) path = os_ops.mkstemp() assert type(path) is str assert os.path.exists(path) with pytest.raises(ExecUtilException) as x: os_ops.rmdirs(path, ignore_errors=False) assert os.path.exists(path) assert type(x.value) is ExecUtilException assert type(x.value.description) is str assert x.value.description == "Utility exited with non-zero code (20). Error: `cannot remove '" + path + "': it is not a directory`" assert x.value.message.startswith(x.value.description) assert type(x.value.error) is str assert x.value.error.strip() == "cannot remove '" + path + "': it is not a directory" assert type(x.value.exit_code) is int assert x.value.exit_code == 20 def test_read__unknown_file(self, os_ops: OsOperations): """ Test RemoteOperations::read with unknown file. """ assert isinstance(os_ops, OsOperations) with pytest.raises(ExecUtilException) as x: os_ops.read("/dummy") assert "Utility exited with non-zero code (1)." in str(x.value) assert "No such file or directory" in str(x.value) assert "/dummy" in str(x.value) def test_read_binary__spec__unk_file(self, os_ops: OsOperations): """ Test RemoteOperations::read_binary with unknown file. """ assert isinstance(os_ops, OsOperations) with pytest.raises(ExecUtilException) as x: os_ops.read_binary("/dummy", 0) assert "Utility exited with non-zero code (1)." in str(x.value) assert "No such file or directory" in str(x.value) assert "/dummy" in str(x.value) def test_get_file_size__unk_file(self, os_ops: OsOperations): """ Test RemoteOperations::get_file_size. """ assert isinstance(os_ops, OsOperations) with pytest.raises(ExecUtilException) as x: os_ops.get_file_size("/dummy") assert "Utility exited with non-zero code (1)." in str(x.value) assert "No such file or directory" in str(x.value) assert "/dummy" in str(x.value) ================================================ FILE: tests/test_raise_error.py ================================================ from src import InvalidOperationException from src import NodeStatus from src.raise_error import RaiseError import pytest import typing class TestRaiseError: class tagTestData001: node_status: NodeStatus expected_msg: str def __init__( self, node_status: NodeStatus, expected_msg: str, ): assert type(node_status) is NodeStatus assert type(expected_msg) is str self.node_status = node_status self.expected_msg = expected_msg return @property def sign(self) -> str: assert type(self.node_status) is NodeStatus msg = "status: {}".format(self.node_status) return msg sm_Data001: typing.List[tagTestData001] = [ tagTestData001( NodeStatus.Uninitialized, "Can't enumerate node child processes. Node is not initialized.", ), tagTestData001( NodeStatus.Stopped, "Can't enumerate node child processes. Node is not running.", ), ] @pytest.fixture( params=sm_Data001, ids=[x.sign for x in sm_Data001], ) def data001(self, request: pytest.FixtureRequest) -> tagTestData001: assert isinstance(request, pytest.FixtureRequest) assert type(request.param).__name__ == "tagTestData001" return request.param def test_001__node_err__cant_enumerate_child_processes( self, data001: tagTestData001, ): assert type(data001) is __class__.tagTestData001 with pytest.raises(expected_exception=InvalidOperationException) as x: RaiseError.node_err__cant_enumerate_child_processes( data001.node_status ) assert x is not None assert str(x.value) == data001.expected_msg return sm_Data002: typing.List[tagTestData001] = [ tagTestData001( NodeStatus.Uninitialized, "Can't kill server process. Node is not initialized.", ), tagTestData001( NodeStatus.Stopped, "Can't kill server process. Node is not running.", ), ] @pytest.fixture( params=sm_Data002, ids=[x.sign for x in sm_Data002], ) def data002(self, request: pytest.FixtureRequest) -> tagTestData001: assert isinstance(request, pytest.FixtureRequest) assert type(request.param).__name__ == "tagTestData001" return request.param def test_002__node_err__cant_kill( self, data002: tagTestData001, ): assert type(data002) is __class__.tagTestData001 with pytest.raises(expected_exception=InvalidOperationException) as x: RaiseError.node_err__cant_kill( data002.node_status ) assert x is not None assert str(x.value) == data002.expected_msg return ================================================ FILE: tests/test_testgres_common.py ================================================ from __future__ import annotations from .helpers.global_data import PostgresNodeService from .helpers.global_data import PostgresNodeServices from .helpers.global_data import OsOperations from .helpers.global_data import PortManager from src import __version__ as testgres_version from src.node import PgVer from src.node import PostgresNode from src.node import NodeConnection from src.node import PostgresNodeLogReader from src.node import PostgresNodeUtils from src.node import ProcessProxy from src.utils import get_pg_version2 from src.utils import file_tail from src.utils import get_bin_path2 from src.utils import execute_utility2 from src import ProcessType from src import NodeStatus from src import IsolationLevel from src import NodeApp from src import enums # New name prevents to collect test-functions in TestgresException and fixes # the problem with pytest warning. from src import TestgresException as testgres_TestgresException from src import InitNodeException from src import StartNodeException from src import QueryException from src import ExecUtilException from src import QueryTimeoutException from src import InvalidOperationException from src import BackupException from src import ProgrammingError from src import scoped_config from src import First, Any from contextlib import contextmanager import pytest import six import logging import time import tempfile import uuid import os import re import subprocess import typing import types import psutil from packaging.version import Version @contextmanager def removing(os_ops: OsOperations, f): assert isinstance(os_ops, OsOperations) try: yield f finally: if os_ops.isfile(f): os_ops.remove_file(f) elif os_ops.isdir(f): os_ops.rmdirs(f, ignore_errors=True) class TestTestgresCommon: sm_node_svcs: typing.List[PostgresNodeService] = [ PostgresNodeServices.sm_local, PostgresNodeServices.sm_local2, PostgresNodeServices.sm_remote, ] @pytest.fixture( params=sm_node_svcs, ids=[descr.sign for descr in sm_node_svcs] ) def node_svc(self, request: pytest.FixtureRequest) -> PostgresNodeService: assert isinstance(request, pytest.FixtureRequest) assert isinstance(request.param, PostgresNodeService) assert isinstance(request.param.os_ops, OsOperations) assert isinstance(request.param.port_manager, PortManager) return request.param def test_testgres_version(self): assert type(testgres_version) is str v = Version(testgres_version) # Author: Mark G. assert v.major == 1 assert v.minor == 13 assert v.micro == 7 assert str(v) == testgres_version return def test_version_management(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) a = PgVer('10.0') b = PgVer('10') c = PgVer('9.6.5') d = PgVer('15.0') e = PgVer('15rc1') f = PgVer('15beta4') h = PgVer('15.3biha') i = PgVer('15.3') g = PgVer('15.3.1bihabeta1') k = PgVer('15.3.1') assert (a == b) assert (b > c) assert (a > c) assert (d > e) assert (e > f) assert (d > f) assert (h > f) assert (h == i) assert (g == k) assert (g > h) version = get_pg_version2(node_svc.os_ops) with __class__.helper__get_node(node_svc) as node: assert (isinstance(version, six.string_types)) assert (isinstance(node.version, PgVer)) assert (node.version == PgVer(version)) def test_node_repr(self, node_svc: PostgresNodeService): with __class__.helper__get_node(node_svc).init() as node: pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" assert re.match(pattern, str(node)) is not None def test_custom_init(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: # enable page checksums node.init(initdb_params=['-k']).start() with __class__.helper__get_node(node_svc) as node: node.init( allow_streaming=True, initdb_params=['--auth-local=reject', '--auth-host=reject']) hba_file = os.path.join(node.data_dir, 'pg_hba.conf') lines = node.os_ops.readlines(hba_file) # check number of lines assert (len(lines) >= 6) # there should be no trust entries at all assert not (any('trust' in s for s in lines)) def test_double_init(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc).init() as node: # can't initialize node more than once with pytest.raises(expected_exception=InitNodeException): node.init() def test_init_after_cleanup(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: node.init().start().execute('select 1') node.cleanup() node.init().start().execute('select 1') def test_init_unique_system_id(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) # this function exists in PostgreSQL 9.6+ current_version = get_pg_version2(node_svc.os_ops) __class__.helper__skip_test_if_util_not_exist(node_svc.os_ops, "pg_resetwal") __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, '9.6') query = 'select system_identifier from pg_control_system()' with scoped_config(cache_initdb=False): with __class__.helper__get_node(node_svc).init().start() as node0: id0 = node0.execute(query)[0] with scoped_config(cache_initdb=True, cached_initdb_unique=True) as config: assert (config.cache_initdb) assert (config.cached_initdb_unique) # spawn two nodes; ids must be different with __class__.helper__get_node(node_svc).init().start() as node1, \ __class__.helper__get_node(node_svc).init().start() as node2: id1 = node1.execute(query)[0] id2 = node2.execute(query)[0] # ids must increase assert (id1 > id0) assert (id2 > id1) def test_node_exit(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with pytest.raises(expected_exception=QueryException): with __class__.helper__get_node(node_svc).init() as node: base_dir = node.base_dir node.safe_psql('select 1') # we should save the DB for "debugging" assert (node_svc.os_ops.path_exists(base_dir)) node_svc.os_ops.rmdirs(base_dir, ignore_errors=True) with __class__.helper__get_node(node_svc).init() as node: base_dir = node.base_dir # should have been removed by default assert not (node_svc.os_ops.path_exists(base_dir)) def test_double_start(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: node.init() assert not node.is_started assert node.status() == NodeStatus.Stopped node.start() assert node.is_started assert node.status() == NodeStatus.Running with pytest.raises(expected_exception=StartNodeException) as x: # can't start node more than once node.start() assert x is not None assert type(x.value) is StartNodeException assert type(x.value.description) is str assert type(x.value.message) is str assert x.value.description == "Cannot start node" assert x.value.message.startswith(x.value.description) assert node.is_started assert node.status() == NodeStatus.Running return def test_start__manually_stop__start_again(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: node.init() assert not node.is_started logging.info("Start node") node.start() assert node.is_started assert node.status() == NodeStatus.Running logging.info("Stop node manually via pg_ctl") stop_cmd = [ node.os_ops.build_path(node.bin_dir, "pg_ctl"), "stop", "-D", node.data_dir, ] execute_utility2( node.os_ops, stop_cmd, node.utils_log_file ) assert node.is_started assert node.status() == NodeStatus.Stopped logging.info("Start node again") node.start() assert node.is_started assert node.status() == NodeStatus.Running assert not node.is_started assert node.status() == NodeStatus.Uninitialized return def test_uninitialized_start(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: # node is not initialized yet assert node.status() == NodeStatus.Uninitialized with pytest.raises(expected_exception=StartNodeException): node.start() assert node.status() == NodeStatus.Uninitialized return def test_start2(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: node.init() assert not node.is_started assert node.status() == NodeStatus.Stopped node.start2() assert not node.is_started assert node.status() == NodeStatus.Running with pytest.raises(expected_exception=StartNodeException) as x: # can't start node more than once node.start2() assert x is not None assert type(x.value) is StartNodeException assert type(x.value.description) is str assert type(x.value.message) is str assert x.value.description == "Cannot start node" assert x.value.message.startswith(x.value.description) assert not node.is_started assert node.status() == NodeStatus.Running return def test_restart(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: node.init().start() # restart, ok res = node.execute('select 1') assert (res == [(1,)]) node.restart() res = node.execute('select 2') assert (res == [(2,)]) assert node.status() == NodeStatus.Running # restart, fail with pytest.raises(expected_exception=StartNodeException): node.append_conf('pg_hba.conf', 'DUMMY') node.restart() assert node.status() == NodeStatus.Stopped return def test_double_stop(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: node.init() assert not node.is_started node.start() assert node.is_started node.stop() assert not node.is_started with pytest.raises(expected_exception=Exception) as x: # can't start node more than once node.stop() assert x is not None assert "Is server running?" in str(x.value) assert not node.is_started return def test_reload(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: node.init().start() # change client_min_messages and save old value cmm_old = node.execute('show client_min_messages') node.append_conf(client_min_messages='DEBUG1') # reload config node.reload() # check new value cmm_new = node.execute('show client_min_messages') assert ('debug1' == cmm_new[0][0].lower()) assert (cmm_old != cmm_new) def test_pg_ctl(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: node.init().start() status = node.pg_ctl(['status']) assert ('PID' in status) def test_status(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) assert (NodeStatus.Running) assert not (NodeStatus.Stopped) assert not (NodeStatus.Uninitialized) # check statuses after each operation with __class__.helper__get_node(node_svc) as node: assert (node.pid == 0) assert (node.status() == NodeStatus.Uninitialized) node.init() assert (node.pid == 0) assert (node.status() == NodeStatus.Stopped) node.start() assert (node.pid != 0) assert (node.status() == NodeStatus.Running) node.stop() assert (node.pid == 0) assert (node.status() == NodeStatus.Stopped) node.cleanup() assert (node.pid == 0) assert (node.status() == NodeStatus.Uninitialized) def test_status__empty_postmaster_pid(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) assert (NodeStatus.Running) assert not (NodeStatus.Stopped) assert not (NodeStatus.Uninitialized) # check statuses after each operation with __class__.helper__get_node(node_svc) as node: assert (node.pid == 0) assert (node.status() == NodeStatus.Uninitialized) node.init() postmaster_pid_file = node.os_ops.build_path(node.data_dir, "postmaster.pid") node.os_ops.write( postmaster_pid_file, "" ) with pytest.raises(expected_exception=ExecUtilException) as x: node.status() expected_msg = "pg_ctl: the PID file \"{}\" is empty\n".format( postmaster_pid_file ) assert expected_msg == x.value.error return def test_status__force_clean_postmaster_pid(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) assert (NodeStatus.Running) assert not (NodeStatus.Stopped) assert not (NodeStatus.Uninitialized) # check statuses after each operation with __class__.helper__get_node(node_svc) as node: assert (node.pid == 0) assert (node.status() == NodeStatus.Uninitialized) node.init() node.start() assert node.status() == NodeStatus.Running logging.info("Postmaster PID is {}.".format(node.pid)) postmaster_pid_file = node.os_ops.build_path(node.data_dir, "postmaster.pid") logging.info("Clean postmaster pid file [{}].".format( postmaster_pid_file )) node.os_ops.write( postmaster_pid_file, "", truncate=True, ) x = node.os_ops.read( postmaster_pid_file, encoding="utf-8", binary=False ) assert x == "" with pytest.raises(expected_exception=ExecUtilException) as x: node.status() expected_msg = "pg_ctl: the PID file \"{}\" is empty\n".format( postmaster_pid_file ) assert expected_msg == x.value.error return def test_kill__is_not_initialized( self, node_svc: PostgresNodeService ): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: assert isinstance(node, PostgresNode) assert (node.pid == 0) assert (node.status() == NodeStatus.Uninitialized) with pytest.raises(expected_exception=InvalidOperationException) as x: node.kill() assert x is not None assert str(x.value) == "Can't kill server process. Node is not initialized." return def test_kill__is_not_running( self, node_svc: PostgresNodeService ): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: assert isinstance(node, PostgresNode) assert (node.pid == 0) assert (node.status() == NodeStatus.Uninitialized) node.init() try: with pytest.raises(expected_exception=InvalidOperationException) as x: node.kill() assert x is not None assert str(x.value) == "Can't kill server process. Node is not running." finally: try: node.cleanup(release_resources=True) except Exception as e: logging.error("Exception ({}): {}".format( type(e).__name__, e, )) return def test_kill__ok( self, node_svc: PostgresNodeService ): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: assert isinstance(node, PostgresNode) assert (node.pid == 0) assert (node.status() == NodeStatus.Uninitialized) node.init() assert not node.is_started node.slow_start() assert node.is_started node.kill() assert not node.is_started attempt = 0 while True: if attempt == 60: raise RuntimeError("Node is not stopped.") attempt += 1 if attempt > 1: time.sleep(1) s = node.status() logging.info("Node status is {}".format(s.name)) if s == NodeStatus.Running: continue assert s == NodeStatus.Stopped break return def test_kill_backgroud_writer__ok( self, node_svc: PostgresNodeService ): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: assert isinstance(node, PostgresNode) assert (node.pid == 0) assert (node.status() == NodeStatus.Uninitialized) node.init() assert not node.is_started node.slow_start() assert node.is_started node_pid = node.pid assert type(node_pid) is int aux_pids = node.auxiliary_pids assert type(aux_pids) is dict assert ProcessType.BackgroundWriter in aux_pids bw_pids = aux_pids[ProcessType.BackgroundWriter] assert type(bw_pids) is list assert len(bw_pids) == 1 bw_pid = bw_pids[0] assert type(bw_pid) is int node.kill(ProcessType.BackgroundWriter) assert node.is_started attempt = 0 while True: if attempt == 60: raise RuntimeError("Node is not stopped.") attempt += 1 if attempt > 1: time.sleep(1) try: psutil.Process(bw_pid) except psutil.NoSuchProcess: logging.info("Process is not found") break logging.info("Process is still alive.") continue assert node.is_started assert node.pid == node_pid return def test_child_processes__is_not_initialized( self, node_svc: PostgresNodeService ): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: assert isinstance(node, PostgresNode) assert (node.pid == 0) assert (node.status() == NodeStatus.Uninitialized) with pytest.raises(expected_exception=InvalidOperationException) as x: node.child_processes assert x is not None assert str(x.value) == "Can't enumerate node child processes. Node is not initialized." return def test_child_processes__is_not_running( self, node_svc: PostgresNodeService ): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: assert isinstance(node, PostgresNode) assert (node.pid == 0) assert (node.status() == NodeStatus.Uninitialized) node.init() try: with pytest.raises(expected_exception=InvalidOperationException) as x: node.child_processes assert x is not None assert str(x.value) == "Can't enumerate node child processes. Node is not running." finally: try: node.cleanup(release_resources=True) except Exception as e: logging.error("Exception ({}): {}".format( type(e).__name__, e, )) return def test_child_processes__ok( self, node_svc: PostgresNodeService ): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: assert isinstance(node, PostgresNode) assert (node.pid == 0) assert (node.status() == NodeStatus.Uninitialized) node.init() try: node.slow_start() children = node.child_processes assert children is not None assert type(children) is list logging.info("Children count is {}".format(len(children))) logging.info("") def LOCAL__safe_call_cmdline(p: ProcessProxy) -> str: assert type(p) is ProcessProxy try: return p.cmdline() except Exception as e: return "Exception ({}): {}".format( type(e).__name__, e, ) for i in range(len(children)): logging.info("------ check child [{}]".format(i)) child = children[i] try: assert child is not None assert type(child) is ProcessProxy assert hasattr(child, "process") assert hasattr(child, "ptype") assert hasattr(child, "pid") assert hasattr(child, "cmdline") assert child.process is not None assert child.ptype is not None assert child.pid is not None assert type(child.ptype) is ProcessType assert type(child.pid) is int assert type(child.cmdline) is types.MethodType logging.info("ptype is {}".format(child.ptype)) logging.info("pid is {}".format(child.pid)) logging.info("cmdline is [{}]".format(LOCAL__safe_call_cmdline(child))) except Exception as e: logging.error("Exception ({}): {}".format( type(e).__name__, e, )) continue finally: try: node.cleanup(release_resources=True) except Exception as e: logging.error("Exception ({}): {}".format( type(e).__name__, e, )) return def test_child_pids(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) master_processes = [ ProcessType.AutovacuumLauncher, ProcessType.BackgroundWriter, ProcessType.Checkpointer, ProcessType.StatsCollector, ProcessType.WalSender, ProcessType.WalWriter, ] postgresVersion = get_pg_version2(node_svc.os_ops) if __class__.helper__pg_version_ge(postgresVersion, '10'): master_processes.append(ProcessType.LogicalReplicationLauncher) if __class__.helper__pg_version_ge(postgresVersion, '14'): master_processes.remove(ProcessType.StatsCollector) repl_processes = [ ProcessType.Startup, ProcessType.WalReceiver, ] def LOCAL__test_auxiliary_pids( node: PostgresNode, expectedTypes: typing.List[ProcessType] ) -> typing.List[ProcessType]: # returns list of the absence processes assert node is not None assert type(node) is PostgresNode assert expectedTypes is not None assert type(expectedTypes) is list pids = node.auxiliary_pids assert pids is not None assert type(pids) is dict result: typing.List[ProcessType] = list() for ptype in expectedTypes: if ptype not in pids: result.append(ptype) return result def LOCAL__check_auxiliary_pids__multiple_attempts( node: PostgresNode, expectedTypes: typing.List[ProcessType], ): assert node is not None assert type(node) is PostgresNode assert expectedTypes is not None assert type(expectedTypes) is list nAttempt = 0 while True: nAttempt += 1 logging.info("Test pids of [{0}] node. Attempt #{1}.".format( node.name, nAttempt )) if nAttempt > 1: time.sleep(1) absenceList = LOCAL__test_auxiliary_pids(node, expectedTypes) assert absenceList is not None assert type(absenceList) is list if len(absenceList) == 0: logging.info("Bingo!") break if nAttempt == 5: raise Exception("Node {0} does not have the following processes: {1}.".format( node.name, absenceList, )) logging.info("These processes are not found: {0}.".format(absenceList)) continue return with __class__.helper__get_node(node_svc).init().start() as master: # master node doesn't have a source walsender! with pytest.raises(expected_exception=testgres_TestgresException): master.source_walsender with master.connect() as con: assert (con.pid > 0) with master.replicate().start() as replica: assert type(replica) is PostgresNode # test __str__ method str(master.child_processes[0]) LOCAL__check_auxiliary_pids__multiple_attempts( master, master_processes) LOCAL__check_auxiliary_pids__multiple_attempts( replica, repl_processes) master_pids = master.auxiliary_pids # there should be exactly 1 source walsender for replica assert (len(master_pids[ProcessType.WalSender]) == 1) pid1 = master_pids[ProcessType.WalSender][0] pid2 = replica.source_walsender.pid assert (pid1 == pid2) replica.stop() # there should be no walsender after we've stopped replica with pytest.raises(expected_exception=testgres_TestgresException): replica.source_walsender def test_exceptions(self): str(StartNodeException('msg', [('file', 'lines')])) str(ExecUtilException('msg', 'cmd', 1, 'out')) str(QueryException('msg', 'query')) def test_auto_name(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc).init(allow_streaming=True).start() as m: with m.replicate().start() as r: # check that nodes are running assert (m.status()) assert (r.status()) # check their names assert (m.name != r.name) assert ('testgres' in m.name) assert ('testgres' in r.name) def test_file_tail(self): s1 = "the quick brown fox jumped over that lazy dog\n" s2 = "abc\n" s3 = "def\n" with tempfile.NamedTemporaryFile(mode='r+', delete=True) as f: sz = 0 while sz < 3 * 8192: sz += len(s1) f.write(s1) f.write(s2) f.write(s3) f.seek(0) lines = file_tail(f, 3) assert (lines[0] == s1) assert (lines[1] == s2) assert (lines[2] == s3) f.seek(0) lines = file_tail(f, 1) assert (lines[0] == s3) def test_isolation_levels(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc).init().start() as node: with node.connect() as con: # string levels con.begin('Read Uncommitted').commit() con.begin('Read Committed').commit() con.begin('Repeatable Read').commit() con.begin('Serializable').commit() # enum levels con.begin(IsolationLevel.ReadUncommitted).commit() con.begin(IsolationLevel.ReadCommitted).commit() con.begin(IsolationLevel.RepeatableRead).commit() con.begin(IsolationLevel.Serializable).commit() # check wrong level with pytest.raises(expected_exception=QueryException): con.begin('Garbage').commit() def test_users(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc).init().start() as node: node.psql('create role test_user login') value = node.safe_psql('select 1', username='test_user') value = __class__.helper__rm_carriage_returns(value) assert (value == b'1\n') def test_poll_query_until(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: node.init().start() get_time = 'select extract(epoch from now())' check_time = 'select extract(epoch from now()) - {} >= 5' start_time = node.execute(get_time)[0][0] node.poll_query_until(query=check_time.format(start_time)) end_time = node.execute(get_time)[0][0] assert (end_time - start_time >= 5) # check 0 columns with pytest.raises(expected_exception=QueryException): node.poll_query_until( query='select from pg_catalog.pg_class limit 1') # check None, fail with pytest.raises(expected_exception=QueryException): node.poll_query_until(query='create table abc (val int)') # check None, ok node.poll_query_until(query='create table def()', expected=None) # returns nothing # check 0 rows equivalent to expected=None node.poll_query_until( query='select * from pg_catalog.pg_class where true = false', expected=None) # check arbitrary expected value, fail with pytest.raises(expected_exception=QueryTimeoutException): node.poll_query_until(query='select 3', expected=1, max_attempts=3, sleep_time=0.01) # check arbitrary expected value, ok node.poll_query_until(query='select 2', expected=2) # check timeout with pytest.raises(expected_exception=QueryTimeoutException): node.poll_query_until(query='select 1 > 2', max_attempts=3, sleep_time=0.01) # check ProgrammingError, fail with pytest.raises(expected_exception=ProgrammingError): node.poll_query_until(query='dummy1') # check ProgrammingError, ok with pytest.raises(expected_exception=(QueryTimeoutException)): node.poll_query_until(query='dummy2', max_attempts=3, sleep_time=0.01, suppress={ProgrammingError}) # check 1 arg, ok node.poll_query_until('select true') def test_logging(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) C_MAX_ATTEMPTS = 50 # This name is used for testgres logging, too. C_NODE_NAME = "testgres_tests." + __class__.__name__ + "test_logging-master-" + uuid.uuid4().hex logging.info("Node name is [{0}]".format(C_NODE_NAME)) with tempfile.NamedTemporaryFile('w', delete=True) as logfile: formatter = logging.Formatter(fmt="%(node)-5s: %(message)s") handler = logging.FileHandler(filename=logfile.name) handler.formatter = formatter logger = logging.getLogger(C_NODE_NAME) assert logger is not None assert len(logger.handlers) == 0 try: # It disables to log on the root level logger.propagate = False logger.addHandler(handler) with scoped_config(use_python_logging=True): with __class__.helper__get_node(node_svc, name=C_NODE_NAME) as master: logging.info("Master node is initilizing") master.init() logging.info("Master node is starting") master.start() logging.info("Dummy query is executed a few times") for _ in range(20): master.execute('select 1') time.sleep(0.01) # let logging worker do the job time.sleep(0.1) logging.info("Master node log file is checking") nAttempt = 0 while True: assert nAttempt <= C_MAX_ATTEMPTS if nAttempt == C_MAX_ATTEMPTS: raise Exception("Test failed!") # let logging worker do the job time.sleep(0.1) nAttempt += 1 logging.info("Attempt {0}".format(nAttempt)) # check that master's port is found with open(logfile.name, 'r') as log: lines = log.readlines() assert lines is not None assert type(lines) is list def LOCAL__test_lines(lines: typing.Iterable[str]) -> bool: assert isinstance(lines, typing.Iterable) for s in lines: assert type(s) is str if C_NODE_NAME in s: logging.info("OK. We found the node_name in a line \"{0}\"".format(s)) return True return False if LOCAL__test_lines(lines): break logging.info("Master node log file does not have an expected information.") continue # test logger after stop/start/restart logging.info("Master node is stopping...") master.stop() logging.info("Master node is staring again...") master.start() logging.info("Master node is restaring...") master.restart() assert (master._logger.is_alive()) finally: # It is a hack code to logging cleanup with logging._lock: assert logging.Logger.manager is not None assert C_NODE_NAME in logging.Logger.manager.loggerDict.keys() logging.Logger.manager.loggerDict.pop(C_NODE_NAME, None) assert C_NODE_NAME not in logging.Logger.manager.loggerDict.keys() assert handler not in logging._handlers.values() # GO HOME! return def test_psql(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc).init().start() as node: # check returned values (1 arg) res = node.psql('select 1') assert (__class__.helper__rm_carriage_returns(res) == (0, b'1\n', b'')) # check returned values (2 args) res = node.psql('postgres', 'select 2') assert (__class__.helper__rm_carriage_returns(res) == (0, b'2\n', b'')) # check returned values (named) res = node.psql(query='select 3', dbname='postgres') assert (__class__.helper__rm_carriage_returns(res) == (0, b'3\n', b'')) # check returned values (1 arg) res = node.safe_psql('select 4') assert (__class__.helper__rm_carriage_returns(res) == b'4\n') # check returned values (2 args) res = node.safe_psql('postgres', 'select 5') assert (__class__.helper__rm_carriage_returns(res) == b'5\n') # check returned values (named) res = node.safe_psql(query='select 6', dbname='postgres') assert (__class__.helper__rm_carriage_returns(res) == b'6\n') # check feeding input node.safe_psql('create table horns (w int)') node.safe_psql('copy horns from stdin (format csv)', input=b"1\n2\n3\n\\.\n") _sum = node.safe_psql('select sum(w) from horns') assert (__class__.helper__rm_carriage_returns(_sum) == b'6\n') # check psql's default args, fails with pytest.raises(expected_exception=QueryException): r = node.psql() # raises! logging.error("node.psql returns [{}]".format(r)) node.stop() # check psql on stopped node, fails with pytest.raises(expected_exception=QueryException): # [2025-04-03] This call does not raise exception! I do not know why. r = node.safe_psql('select 1') # raises! logging.error("node.safe_psql returns [{}]".format(r)) def test_psql__another_port(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc).init() as node1: with __class__.helper__get_node(node_svc).init() as node2: node1.start() node2.start() assert node1.port != node2.port assert node1.host == node2.host node1.stop() logging.info("test table in node2 is creating ...") node2.safe_psql( dbname="postgres", query="create table test (id integer);" ) logging.info("try to find test table through node1.psql ...") res = node1.psql( dbname="postgres", query="select count(*) from pg_class where relname='test'", host=node2.host, port=node2.port, ) assert (__class__.helper__rm_carriage_returns(res) == (0, b'1\n', b'')) def test_psql__another_bad_host(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc).init() as node: logging.info("try to execute node1.psql ...") res = node.psql( dbname="postgres", query="select count(*) from pg_class where relname='test'", host="DUMMY_HOST_NAME", port=node.port, ) res2 = __class__.helper__rm_carriage_returns(res) assert res2[0] != 0 assert b"DUMMY_HOST_NAME" in res[2] def test_safe_psql__another_port(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc).init() as node1: with __class__.helper__get_node(node_svc).init() as node2: node1.start() node2.start() assert node1.port != node2.port assert node1.host == node2.host node1.stop() logging.info("test table in node2 is creating ...") node2.safe_psql( dbname="postgres", query="create table test (id integer);" ) logging.info("try to find test table through node1.psql ...") res = node1.safe_psql( dbname="postgres", query="select count(*) from pg_class where relname='test'", host=node2.host, port=node2.port, ) assert (__class__.helper__rm_carriage_returns(res) == b'1\n') def test_safe_psql__another_bad_host(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc).init() as node: logging.info("try to execute node1.psql ...") with pytest.raises(expected_exception=Exception) as x: node.safe_psql( dbname="postgres", query="select count(*) from pg_class where relname='test'", host="DUMMY_HOST_NAME", port=node.port, ) assert "DUMMY_HOST_NAME" in str(x.value) def test_safe_psql__expect_error(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc).init().start() as node: err = node.safe_psql('select_or_not_select 1', expect_error=True) assert (type(err) is str) assert ('select_or_not_select' in err) assert ('ERROR: syntax error at or near "select_or_not_select"' in err) # --------- with pytest.raises( expected_exception=InvalidOperationException, match="^" + re.escape("Exception was expected, but query finished successfully: `select 1;`.") + "$" ): node.safe_psql("select 1;", expect_error=True) # --------- res = node.safe_psql("select 1;", expect_error=False) assert (__class__.helper__rm_carriage_returns(res) == b'1\n') def test_transactions(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc).init().start() as node: with node.connect() as con: con.begin() con.execute('create table test(val int)') con.execute('insert into test values (1)') con.commit() con.begin() con.execute('insert into test values (2)') res = con.execute('select * from test order by val asc') assert (res == [(1, ), (2, )]) con.rollback() con.begin() res = con.execute('select * from test') assert (res == [(1, )]) con.rollback() con.begin() con.execute('drop table test') con.commit() def test_control_data(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: # node is not initialized yet with pytest.raises(expected_exception=ExecUtilException): node.get_control_data() node.init() data = node.get_control_data() # check returned dict assert data is not None assert (any('pg_control' in s for s in data.keys())) def test_backup_simple(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as master: # enable streaming for backups master.init(allow_streaming=True) # node must be running with pytest.raises(expected_exception=BackupException): master.backup() # it's time to start node master.start() # fill node with some data master.psql('create table test as select generate_series(1, 4) i') with master.backup(xlog_method='stream') as backup: with backup.spawn_primary().start() as slave: res = slave.execute('select * from test order by i asc') assert (res == [(1, ), (2, ), (3, ), (4, )]) def test_backup_multiple(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() with node.backup(xlog_method='fetch') as backup1, \ node.backup(xlog_method='fetch') as backup2: assert (backup1.base_dir != backup2.base_dir) with node.backup(xlog_method='fetch') as backup: with backup.spawn_primary('node1', destroy=False) as node1, \ backup.spawn_primary('node2', destroy=False) as node2: assert (node1.base_dir != node2.base_dir) def test_backup_exhaust(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() with node.backup(xlog_method='fetch') as backup: # exhaust backup by creating new node with backup.spawn_primary(): pass # now let's try to create one more node with pytest.raises(expected_exception=BackupException): backup.spawn_primary() def test_backup_wrong_xlog_method(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() with pytest.raises( expected_exception=BackupException, match="^" + re.escape('Invalid xlog_method "wrong"') + "$" ): node.backup(xlog_method='wrong') def test_pg_ctl_wait_option(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) C_MAX_ATTEMPT = 5 nAttempt = 0 while True: if nAttempt == C_MAX_ATTEMPT: raise Exception("PostgresSQL did not start.") nAttempt += 1 logging.info("------------------------ attempt #{}".format( nAttempt )) if nAttempt > 1: logging.info("Sleep 3 seconds") time.sleep(3) port = node_svc.port_manager.reserve_port() assert type(port) is int ok = False try: with __class__.helper__get_node(node_svc, port=port) as node: if self.impl__test_pg_ctl_wait_option(node_svc, node): ok = True finally: node_svc.port_manager.release_port(port) if ok: break continue logging.info("OK. Test is passed. Number of attempts is {}".format( nAttempt )) return def impl__test_pg_ctl_wait_option( self, node_svc: PostgresNodeService, node: PostgresNode ) -> bool: assert isinstance(node_svc, PostgresNodeService) assert isinstance(node, PostgresNode) assert node.status() == NodeStatus.Uninitialized C_MAX_ATTEMPTS = 50 logging.info("init node") node.init() assert node.status() == NodeStatus.Stopped logging.info("node is inited") node_log_reader = PostgresNodeLogReader(node, from_beginnig=True) logging.info("start node") try: node.start(wait=False) except StartNodeException as e: logging.info("Exception ({}): {}".format( type(e).__name__, e, )) return False logging.info("node is started") nAttempt = 0 while True: if PostgresNodeUtils.delect_port_conflict(node_log_reader): logging.info("Node port {} conflicted with another PostgreSQL instance.".format( node.port )) return False if nAttempt == C_MAX_ATTEMPTS: # # [2025-03-11] # We have an unexpected problem with this test in CI # Let's get an additional information about this test failure. # logging.error("Node was not stopped.") if not node.os_ops.path_exists(node.pg_log_file): logging.warning("Node log does not exist.") else: logging.info("Let's read node log file [{0}]".format(node.pg_log_file)) logFileData = node.os_ops.read(node.pg_log_file, binary=False) logging.info("Node log file content:\n{0}".format(logFileData)) raise Exception("Could not stop node.") nAttempt += 1 if nAttempt > 1: logging.info("Wait 1 second.") time.sleep(1) logging.info("") logging.info("Try to stop node. Attempt #{0}.".format(nAttempt)) try: node.stop(wait=False) break except ExecUtilException as e: # it's ok to get this exception here since node # could be not started yet logging.info("Node is not stopped. Exception ({0}): {1}".format(type(e).__name__, e)) continue logging.info("OK. Stop command was executed. Let's wait while our node will stop really.") nAttempt = 0 while True: if nAttempt == C_MAX_ATTEMPTS: raise Exception("Could not stop node.") nAttempt += 1 if nAttempt > 1: logging.info("Wait 1 second.") time.sleep(1) logging.info("") logging.info("Attempt #{0}.".format(nAttempt)) s1 = node.status() if s1 == NodeStatus.Running: continue if s1 == NodeStatus.Stopped: break raise Exception("Unexpected node status: {0}.".format(s1)) logging.info("OK. Node is stopped.") return True def test_replicate(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() with node.replicate().start() as replica: res = replica.execute('select 1') assert (res == [(1, )]) node.execute('create table test (val int)', commit=True) replica.catchup() res = node.execute('select * from test') assert (res == []) def test_synchronous_replication(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) current_version = get_pg_version2(node_svc.os_ops) __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "9.6") with __class__.helper__get_node(node_svc) as master: old_version = not __class__.helper__pg_version_ge(current_version, '9.6') master.init(allow_streaming=True).start() if not old_version: master.append_conf('synchronous_commit = remote_apply') # create standby with master.replicate() as standby1, master.replicate() as standby2: standby1.start() standby2.start() # check formatting assert ( '1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2))) ) # yapf: disable assert ( 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2))) ) # yapf: disable # set synchronous_standby_names master.set_synchronous_standbys(First(2, [standby1, standby2])) master.restart() # the following part of the test is only applicable to newer # versions of PostgresQL if not old_version: master.safe_psql('create table abc(a int)') # Create a large transaction that will take some time to apply # on standby to check that it applies synchronously # (If set synchronous_commit to 'on' or other lower level then # standby most likely won't catchup so fast and test will fail) master.safe_psql( 'insert into abc select generate_series(1, 1000000)') res = standby1.safe_psql('select count(*) from abc') assert (__class__.helper__rm_carriage_returns(res) == b'1000000\n') def test_logical_replication(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) current_version = get_pg_version2(node_svc.os_ops) __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "10") with __class__.helper__get_node(node_svc) as node1, __class__.helper__get_node(node_svc) as node2: node1.init(allow_logical=True) node1.start() node2.init().start() create_table = 'create table test (a int, b int)' node1.safe_psql(create_table) node2.safe_psql(create_table) # create publication / create subscription pub = node1.publish('mypub') sub = node2.subscribe(pub, 'mysub') node1.safe_psql('insert into test values (1, 1), (2, 2)') # wait until changes apply on subscriber and check them sub.catchup() res = node2.execute('select * from test') assert (res == [(1, 1), (2, 2)]) # disable and put some new data sub.disable() node1.safe_psql('insert into test values (3, 3)') # enable and ensure that data successfully transferred sub.enable() sub.catchup() res = node2.execute('select * from test') assert (res == [(1, 1), (2, 2), (3, 3)]) # Add new tables. Since we added "all tables" to publication # (default behaviour of publish() method) we don't need # to explicitly perform pub.add_tables() create_table = 'create table test2 (c char)' node1.safe_psql(create_table) node2.safe_psql(create_table) sub.refresh() # put new data node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') sub.catchup() res = node2.execute('select * from test2') assert (res == [('a', ), ('b', )]) # drop subscription sub.drop() pub.drop() # create new publication and subscription for specific table # (omitting copying data as it's already done) pub = node1.publish('newpub', tables=['test']) sub = node2.subscribe(pub, 'newsub', copy_data=False) node1.safe_psql('insert into test values (4, 4)') sub.catchup() res = node2.execute('select * from test') assert (res == [(1, 1), (2, 2), (3, 3), (4, 4)]) # explicitly add table with pytest.raises(expected_exception=ValueError): pub.add_tables([]) # fail pub.add_tables(['test2']) node1.safe_psql('insert into test2 values (\'c\')') sub.catchup() res = node2.execute('select * from test2') assert (res == [('a', ), ('b', )]) def test_logical_catchup(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) """ Runs catchup for 100 times to be sure that it is consistent """ current_version = get_pg_version2(node_svc.os_ops) __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "10") with __class__.helper__get_node(node_svc) as node1, __class__.helper__get_node(node_svc) as node2: node1.init(allow_logical=True) node1.start() node2.init().start() create_table = 'create table test (key int primary key, val int); ' node1.safe_psql(create_table) node1.safe_psql('alter table test replica identity default') node2.safe_psql(create_table) # create publication / create subscription sub = node2.subscribe(node1.publish('mypub'), 'mysub') for i in range(0, 100): node1.execute('insert into test values ({0}, {0})'.format(i)) sub.catchup() res = node2.execute('select * from test') assert (res == [(i, i, )]) node1.execute('delete from test') def test_logical_replication_fail(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) current_version = get_pg_version2(node_svc.os_ops) __class__.helper__skip_test_if_pg_version_is_ge(current_version, "10") with __class__.helper__get_node(node_svc) as node: with pytest.raises(expected_exception=InitNodeException): node.init(allow_logical=True) def test_replication_slots(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() with node.replicate(slot='slot1').start() as replica: replica.execute('select 1') # cannot create new slot with the same name with pytest.raises(expected_exception=testgres_TestgresException): node.replicate(slot='slot1') def test_incorrect_catchup(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() # node has no master, can't catch up with pytest.raises(expected_exception=testgres_TestgresException): node.catchup() def test_promotion(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as master: master.init().start() master.safe_psql('create table abc(id serial)') with master.replicate().start() as replica: master.stop() replica.promote() # make standby becomes writable master replica.safe_psql('insert into abc values (1)') res = replica.safe_psql('select * from abc') assert (__class__.helper__rm_carriage_returns(res) == b'1\n') @pytest.fixture( params=[ enums.DumpFormat.Plain, enums.DumpFormat.Custom, enums.DumpFormat.Directory, enums.DumpFormat.Tar ] ) def dump_fmt(self, request: pytest.FixtureRequest) -> enums.DumpFormat: assert type(request.param) is enums.DumpFormat return request.param def test_dump(self, node_svc: PostgresNodeService, dump_fmt: enums.DumpFormat): assert isinstance(node_svc, PostgresNodeService) assert type(dump_fmt) is enums.DumpFormat query_create = 'create table test as select generate_series(1, 2) as val' query_select = 'select * from test order by val asc' with __class__.helper__get_node(node_svc).init().start() as node1: node1.execute(query_create) with removing(node_svc.os_ops, node1.dump(format=dump_fmt)) as dump: with __class__.helper__get_node(node_svc).init().start() as node3: if dump_fmt == enums.DumpFormat.Directory: assert (os.path.isdir(dump)) else: assert (os.path.isfile(dump)) # restore dump node3.restore(filename=dump) res = node3.execute(query_select) assert (res == [(1, ), (2, )]) def test_dump_with_options(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) query_create = 'create table test_options as select generate_series(1, 5) as val' with __class__.helper__get_node(node_svc).init().start() as node1: node1.execute(query_create) # Test dump with --schema-only option with removing(node_svc.os_ops, node1.dump(options=['--schema-only'])) as dump: with __class__.helper__get_node(node_svc).init().start() as node2: assert (os.path.isfile(dump)) # restore schema-only dump node2.restore(filename=dump) # Check that table exists but has no data res = node2.execute("SELECT COUNT(*) FROM test_options") assert (res == [(0,)]) # Table exists but empty # Verify table structure exists res = node2.execute(""" SELECT COUNT(*) FROM information_schema.tables WHERE table_name = 'test_options' """) assert (res == [(1,)]) # Table structure exists def test_pgbench(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) __class__.helper__skip_test_if_util_not_exist(node_svc.os_ops, "pgbench") with __class__.helper__get_node(node_svc).init().start() as node: # initialize pgbench DB and run benchmarks node.pgbench_init( scale=2, foreign_keys=True, options=['-q'] ).pgbench_run(time=2) # run TPC-B benchmark proc = node.pgbench(stdout=subprocess.PIPE, stderr=subprocess.STDOUT, options=['-T3']) out = proc.communicate()[0] assert (b'tps = ' in out) def test_unix_sockets(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: node.init(unix_sockets=False, allow_streaming=True) node.start() res_exec = node.execute('select 1') assert (res_exec == [(1,)]) res_psql = node.safe_psql('select 1') assert (res_psql == b'1\n') with node.replicate() as r: assert type(r) is PostgresNode r.start() res_exec = r.execute('select 1') assert (res_exec == [(1,)]) res_psql = r.safe_psql('select 1') assert (res_psql == b'1\n') def test_the_same_port(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: node.init().start() assert (node._should_free_port) assert (type(node.port) is int) node_port_copy = node.port r = node.safe_psql("SELECT 1;") assert (__class__.helper__rm_carriage_returns(r) == b'1\n') with __class__.helper__get_node(node_svc, port=node.port) as node2: assert (type(node2.port) is int) assert (node2.port == node.port) assert (not node2._should_free_port) assert (node2.status() == NodeStatus.Uninitialized) node2.init() with pytest.raises( expected_exception=StartNodeException, match=re.escape("Cannot start node") ): node2.start() assert (node2.status() == NodeStatus.Stopped) # node is still working assert (node.port == node_port_copy) assert (node._should_free_port) r = node.safe_psql("SELECT 3;") assert (__class__.helper__rm_carriage_returns(r) == b'3\n') class tagPortManagerProxy(PortManager): m_PrevPortManager: PortManager m_DummyPortNumber: int m_DummyPortMaxUsage: int m_DummyPortCurrentUsage: int m_DummyPortTotalUsage: int def __init__(self, prevPortManager: PortManager, dummyPortNumber: int, dummyPortMaxUsage: int): assert isinstance(prevPortManager, PortManager) assert type(dummyPortNumber) is int assert type(dummyPortMaxUsage) is int assert dummyPortNumber >= 0 assert dummyPortMaxUsage >= 0 super().__init__() self.m_PrevPortManager = prevPortManager self.m_DummyPortNumber = dummyPortNumber self.m_DummyPortMaxUsage = dummyPortMaxUsage self.m_DummyPortCurrentUsage = 0 self.m_DummyPortTotalUsage = 0 def __enter__(self): return self def __exit__(self, type, value, traceback): assert self.m_DummyPortCurrentUsage == 0 assert self.m_PrevPortManager is not None def reserve_port(self) -> int: assert type(self.m_DummyPortMaxUsage) is int assert type(self.m_DummyPortTotalUsage) is int assert type(self.m_DummyPortCurrentUsage) is int assert self.m_DummyPortTotalUsage >= 0 assert self.m_DummyPortCurrentUsage >= 0 assert self.m_DummyPortTotalUsage <= self.m_DummyPortMaxUsage assert self.m_DummyPortCurrentUsage <= self.m_DummyPortTotalUsage assert self.m_PrevPortManager is not None assert isinstance(self.m_PrevPortManager, PortManager) if self.m_DummyPortTotalUsage == self.m_DummyPortMaxUsage: return self.m_PrevPortManager.reserve_port() self.m_DummyPortTotalUsage += 1 self.m_DummyPortCurrentUsage += 1 return self.m_DummyPortNumber def release_port(self, number: int) -> None: assert type(number) is int assert type(self.m_DummyPortMaxUsage) is int assert type(self.m_DummyPortTotalUsage) is int assert type(self.m_DummyPortCurrentUsage) is int assert self.m_DummyPortTotalUsage >= 0 assert self.m_DummyPortCurrentUsage >= 0 assert self.m_DummyPortTotalUsage <= self.m_DummyPortMaxUsage assert self.m_DummyPortCurrentUsage <= self.m_DummyPortTotalUsage assert self.m_PrevPortManager is not None assert isinstance(self.m_PrevPortManager, PortManager) if self.m_DummyPortCurrentUsage > 0 and number == self.m_DummyPortNumber: assert self.m_DummyPortTotalUsage > 0 self.m_DummyPortCurrentUsage -= 1 return return self.m_PrevPortManager.release_port(number) def test_port_rereserve_during_node_start(self, node_svc: PostgresNodeService): assert type(node_svc) is PostgresNodeService assert PostgresNode._C_MAX_START_ATEMPTS == 5 C_COUNT_OF_BAD_PORT_USAGE = 3 with __class__.helper__get_node(node_svc) as node1: node1.init().start() assert node1._should_free_port assert type(node1.port) is int node1_port_copy = node1.port assert __class__.helper__rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n' with __class__.tagPortManagerProxy(node_svc.port_manager, node1.port, C_COUNT_OF_BAD_PORT_USAGE) as proxy: assert proxy.m_DummyPortNumber == node1.port with __class__.helper__get_node(node_svc, port_manager=proxy) as node2: assert node2._should_free_port assert node2.port == node1.port node2.init().start() assert node2.port != node1.port assert node2._should_free_port assert proxy.m_DummyPortCurrentUsage == 0 assert proxy.m_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE assert node2.is_started r = node2.safe_psql("SELECT 2;") assert __class__.helper__rm_carriage_returns(r) == b'2\n' # node1 is still working assert node1.port == node1_port_copy assert node1._should_free_port r = node1.safe_psql("SELECT 3;") assert __class__.helper__rm_carriage_returns(r) == b'3\n' def test_port_conflict(self, node_svc: PostgresNodeService): assert type(node_svc) is PostgresNodeService assert PostgresNode._C_MAX_START_ATEMPTS > 1 C_COUNT_OF_BAD_PORT_USAGE = PostgresNode._C_MAX_START_ATEMPTS with __class__.helper__get_node(node_svc) as node1: node1.init().start() assert node1._should_free_port assert type(node1.port) is int node1_port_copy = node1.port assert __class__.helper__rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n' with __class__.tagPortManagerProxy(node_svc.port_manager, node1.port, C_COUNT_OF_BAD_PORT_USAGE) as proxy: assert proxy.m_DummyPortNumber == node1.port with __class__.helper__get_node(node_svc, port_manager=proxy) as node2: assert node2._should_free_port assert node2.port == node1.port node2.init() assert node2.status() == NodeStatus.Stopped with pytest.raises( expected_exception=StartNodeException, match=re.escape("Cannot start node after multiple attempts.") ): node2.start() assert node2.port == node1.port assert node2._should_free_port assert proxy.m_DummyPortCurrentUsage == 1 assert proxy.m_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE assert not node2.is_started assert node2.status() == NodeStatus.Stopped # node2 must release our dummyPort (node1.port) assert (proxy.m_DummyPortCurrentUsage == 0) # node1 is still working assert node1.port == node1_port_copy assert node1._should_free_port r = node1.safe_psql("SELECT 3;") assert __class__.helper__rm_carriage_returns(r) == b'3\n' def test_try_to_get_port_after_free_manual_port(self, node_svc: PostgresNodeService): assert type(node_svc) is PostgresNodeService assert node_svc.port_manager is not None assert isinstance(node_svc.port_manager, PortManager) with __class__.helper__get_node(node_svc) as node1: assert node1 is not None assert type(node1) is PostgresNode assert node1.port is not None assert type(node1.port) is int with __class__.helper__get_node(node_svc, port=node1.port, port_manager=None) as node2: assert node2 is not None assert type(node1) is PostgresNode assert node2 is not node1 assert node2.port is not None assert type(node2.port) is int assert node2.port == node1.port logging.info("Release node2 port") node2.free_port() logging.info("try to get node2.port...") with pytest.raises( InvalidOperationException, match="^" + re.escape("PostgresNode port is not defined.") + "$" ): p = node2.port assert p is None def test_try_to_start_node_after_free_manual_port(self, node_svc: PostgresNodeService): assert type(node_svc) is PostgresNodeService assert node_svc.port_manager is not None assert isinstance(node_svc.port_manager, PortManager) with __class__.helper__get_node(node_svc) as node1: assert node1 is not None assert type(node1) is PostgresNode assert node1.port is not None assert type(node1.port) is int with __class__.helper__get_node(node_svc, port=node1.port, port_manager=None) as node2: assert node2 is not None assert type(node1) is PostgresNode assert node2 is not node1 assert node2.port is not None assert type(node2.port) is int assert node2.port == node1.port logging.info("Release node2 port") node2.free_port() logging.info("node2 is trying to start...") with pytest.raises( InvalidOperationException, match="^" + re.escape("Can't start PostgresNode. Port is not defined.") + "$" ): node2.start() def test_node__os_ops(self, node_svc: PostgresNodeService): assert type(node_svc) is PostgresNodeService assert node_svc.os_ops is not None assert isinstance(node_svc.os_ops, OsOperations) with PostgresNode(name="node", os_ops=node_svc.os_ops, port_manager=node_svc.port_manager) as node: # retest assert node_svc.os_ops is not None assert isinstance(node_svc.os_ops, OsOperations) assert node.os_ops is node_svc.os_ops # one more time assert node.os_ops is node_svc.os_ops def test_node__port_manager(self, node_svc: PostgresNodeService): assert type(node_svc) is PostgresNodeService assert node_svc.port_manager is not None assert isinstance(node_svc.port_manager, PortManager) with PostgresNode(name="node", os_ops=node_svc.os_ops, port_manager=node_svc.port_manager) as node: # retest assert node_svc.port_manager is not None assert isinstance(node_svc.port_manager, PortManager) assert node.port_manager is node_svc.port_manager # one more time assert node.port_manager is node_svc.port_manager def test_node__port_manager_and_explicit_port(self, node_svc: PostgresNodeService): assert type(node_svc) is PostgresNodeService assert isinstance(node_svc.os_ops, OsOperations) assert node_svc.port_manager is not None assert isinstance(node_svc.port_manager, PortManager) port = node_svc.port_manager.reserve_port() assert type(port) is int try: with PostgresNode(name="node", port=port, os_ops=node_svc.os_ops) as node: # retest assert isinstance(node_svc.os_ops, OsOperations) assert node_svc.port_manager is not None assert isinstance(node_svc.port_manager, PortManager) assert node.port_manager is None assert node.os_ops is node_svc.os_ops # one more time assert node.port_manager is None assert node.os_ops is node_svc.os_ops finally: node_svc.port_manager.release_port(port) def test_node__no_port_manager(self, node_svc: PostgresNodeService): assert type(node_svc) is PostgresNodeService assert isinstance(node_svc.os_ops, OsOperations) assert node_svc.port_manager is not None assert isinstance(node_svc.port_manager, PortManager) port = node_svc.port_manager.reserve_port() assert type(port) is int try: with PostgresNode(name="node", port=port, os_ops=node_svc.os_ops, port_manager=None) as node: # retest assert isinstance(node_svc.os_ops, OsOperations) assert node_svc.port_manager is not None assert isinstance(node_svc.port_manager, PortManager) assert node.port_manager is None assert node.os_ops is node_svc.os_ops # one more time assert node.port_manager is None assert node.os_ops is node_svc.os_ops finally: node_svc.port_manager.release_port(port) class tagTableChecksumTestData: record_count: int def __init__( self, record_count: int, ): assert type(record_count) is int self.record_count = record_count return sm_TableCheckSumTestDatas = [ tagTableChecksumTestData(0), tagTableChecksumTestData(1), tagTableChecksumTestData(2), tagTableChecksumTestData(3), tagTableChecksumTestData(987), tagTableChecksumTestData(999), tagTableChecksumTestData(1000), tagTableChecksumTestData(1001), tagTableChecksumTestData(1999), tagTableChecksumTestData(19999), tagTableChecksumTestData(199999), tagTableChecksumTestData(1999999), ] @pytest.fixture( params=sm_TableCheckSumTestDatas, ids=[x.record_count for x in sm_TableCheckSumTestDatas], ) def table_checksum_test_data( self, request: pytest.FixtureRequest ) -> tagTableChecksumTestData: assert isinstance(request, pytest.FixtureRequest) assert type(request.param).__name__ == "tagTableChecksumTestData" return request.param def test_node__table_checksum( self, node_svc: PostgresNodeService, table_checksum_test_data: tagTableChecksumTestData, ): assert type(node_svc) is PostgresNodeService assert type(table_checksum_test_data) is __class__.tagTableChecksumTestData assert node_svc.port_manager is not None assert isinstance(node_svc.port_manager, PortManager) with __class__.helper__get_node(node_svc) as node: assert node is not None assert type(node) is PostgresNode assert node.port is not None assert type(node.port) is int assert type(table_checksum_test_data.record_count) is int assert table_checksum_test_data.record_count >= 0 node.init() node.slow_start() C_DB = "postgres" with node.connect(dbname=C_DB) as cn: assert type(cn) is NodeConnection cn.execute("create table t (id integer, data varchar(32));") cn.commit() if table_checksum_test_data.record_count > 0: cn.execute("insert into t (id, data) select x, x from generate_series(1, {}) x".format( table_checksum_test_data.record_count )) cn.commit() with cn.connection.cursor() as cursor: assert cursor is not None cursor.execute("SELECT hashtext(t::text) FROM \"t\" as t;") checksum1 = 0 record_count = 0 while True: row = cursor.fetchone() if row is None: break assert type(row) in [list, tuple] assert len(row) == 1 record_count += 1 checksum1 += int(row[0]) pass assert record_count == table_checksum_test_data.record_count checksum2 = node.table_checksum("t", C_DB) assert type(checksum2) is int assert checksum1 == checksum2 pass return def test_node__pgbench_table_checksums__one_table( self, node_svc: PostgresNodeService, table_checksum_test_data: tagTableChecksumTestData, ): assert type(node_svc) is PostgresNodeService assert type(table_checksum_test_data) is __class__.tagTableChecksumTestData assert node_svc.port_manager is not None assert isinstance(node_svc.port_manager, PortManager) with __class__.helper__get_node(node_svc) as node: assert node is not None assert type(node) is PostgresNode assert node.port is not None assert type(node.port) is int assert type(table_checksum_test_data.record_count) is int assert table_checksum_test_data.record_count >= 0 node.init() node.slow_start() C_DB = "postgres" with node.connect(dbname=C_DB) as cn: assert type(cn) is NodeConnection cn.execute("create table t (id integer, data varchar(32));") cn.commit() if table_checksum_test_data.record_count > 0: cn.execute("insert into t (id, data) select x, x from generate_series(1, {}) x".format( table_checksum_test_data.record_count )) cn.commit() with cn.connection.cursor() as cursor: assert cursor is not None cursor.execute("SELECT hashtext(t::text) FROM \"t\" as t;") checksum1 = 0 record_count = 0 while True: row = cursor.fetchone() if row is None: break assert type(row) in [list, tuple] assert len(row) == 1 record_count += 1 checksum1 += int(row[0]) pass assert record_count == table_checksum_test_data.record_count actual_result = node.pgbench_table_checksums(C_DB, ["t"]) assert type(actual_result) is set actual1 = actual_result.pop() assert type(actual1) is tuple assert len(actual1) == 2 assert type(actual1[0]) is str assert type(actual1[1]) is int assert checksum1 == actual1[1] pass return def test_node__pgbench_table_checksums__pbckp_2278(self, node_svc: PostgresNodeService): assert type(node_svc) is PostgresNodeService assert node_svc.port_manager is not None assert isinstance(node_svc.port_manager, PortManager) with __class__.helper__get_node(node_svc) as node: assert node is not None assert type(node) is PostgresNode assert node.port is not None assert type(node.port) is int node.init() node.slow_start() logging.info("init pgbench database") node.pgbench_init(scale=20) nPass = 0 while nPass < 3: nPass += 1 logging.info("------------------- pass: {}".format(nPass)) if not __class__.helper__call_and_check_pgbench_table_checksums(node): raise RuntimeError("pgbench_table_checksums created a problem. Please, check a test log.") continue return @staticmethod def helper__call_and_check_pgbench_table_checksums( node: PostgresNode ) -> bool: assert node is not None assert type(node) is PostgresNode assert node.status() == NodeStatus.Running # We will check # 1) the structure of result # 2) the release of cursor locks logging.info("run pgbench_table_checksums") full_checksums = node.pgbench_table_checksums() assert full_checksums is not None assert type(full_checksums) is set assert len(full_checksums) == 4 expectedTables: typing.Dict[str, bool] = { 'pgbench_branches': False, 'pgbench_tellers': False, 'pgbench_accounts': False, 'pgbench_history': False, } ok = True for tcs in full_checksums: assert type(tcs) is tuple assert len(tcs) == 2 assert type(tcs[0]) is str assert type(tcs[1]) is int tableName = tcs[0] if tableName not in expectedTables: ok = False logging.error("pgbench_table_checksums returns unknown table [{}].".format( tableName )) continue if expectedTables[tableName]: ok = False logging.error("pgbench_table_checksums returns table [{}] more than one time.".format( tableName )) continue expectedTables[tableName] = True continue C_SQL = """select x.granted, x.mode from pg_locks x join pg_class c on x.relation=c.oid where c.relname=%s;""" cn = node.connect(dbname="postgres") assert type(cn) is NodeConnection try: for tcs in full_checksums: tableName = tcs[0] recs = cn.execute(C_SQL, tableName) assert type(recs) is list if len(recs) == 0: logging.info("Table [{}] does not have a lock. It is ok.".format( tableName, )) else: ok = False assert len(recs) == 1 rec = recs[0] assert type(rec) is tuple assert len(rec) == 2 logging.error("Table [{}] has a lock [granted: {}][mode: {}].".format( tableName, rec[0], rec[1], )) continue finally: try: cn.close() except Exception as e: logging.error("Can't close connection. Exception ({}): {}".format( type(e).__name__, e, )) return ok class tag_rmdirs_protector: _os_ops: OsOperations _cwd: str _old_rmdirs: any _cwd: str def __init__(self, os_ops: OsOperations): self._os_ops = os_ops self._cwd = os.path.abspath(os_ops.cwd()) self._old_rmdirs = os_ops.rmdirs def __enter__(self): assert self._os_ops.rmdirs == self._old_rmdirs self._os_ops.rmdirs = self.proxy__rmdirs return self def __exit__(self, exc_type, exc_val, exc_tb): assert self._os_ops.rmdirs == self.proxy__rmdirs self._os_ops.rmdirs = self._old_rmdirs return False def proxy__rmdirs(self, path, ignore_errors=True): raise Exception("Call of rmdirs is not expected!") def test_node_app__make_empty__base_dir_is_None(self, node_svc: PostgresNodeService): assert type(node_svc) is PostgresNodeService assert isinstance(node_svc.os_ops, OsOperations) assert node_svc.port_manager is not None assert isinstance(node_svc.port_manager, PortManager) tmp_dir = node_svc.os_ops.mkdtemp() assert tmp_dir is not None assert type(tmp_dir) is str logging.info("temp directory is [{}]".format(tmp_dir)) # ----------- os_ops = node_svc.os_ops.create_clone() assert os_ops is not node_svc.os_ops # ----------- with __class__.tag_rmdirs_protector(os_ops): node_app = NodeApp(test_path=tmp_dir, os_ops=os_ops) assert node_app.os_ops is os_ops with pytest.raises(expected_exception=BaseException) as x: node_app.make_empty(base_dir=None) if type(x.value) is AssertionError: pass else: assert type(x.value) is ValueError assert str(x.value) == "Argument 'base_dir' is not defined." # ----------- logging.info("temp directory [{}] is deleting".format(tmp_dir)) node_svc.os_ops.rmdir(tmp_dir) def test_node_app__make_empty__base_dir_is_Empty(self, node_svc: PostgresNodeService): assert type(node_svc) is PostgresNodeService assert isinstance(node_svc.os_ops, OsOperations) assert node_svc.port_manager is not None assert isinstance(node_svc.port_manager, PortManager) tmp_dir = node_svc.os_ops.mkdtemp() assert tmp_dir is not None assert type(tmp_dir) is str logging.info("temp directory is [{}]".format(tmp_dir)) # ----------- os_ops = node_svc.os_ops.create_clone() assert os_ops is not node_svc.os_ops # ----------- with __class__.tag_rmdirs_protector(os_ops): node_app = NodeApp(test_path=tmp_dir, os_ops=os_ops) assert node_app.os_ops is os_ops with pytest.raises(expected_exception=ValueError) as x: node_app.make_empty(base_dir="") assert str(x.value) == "Argument 'base_dir' is empty." # ----------- logging.info("temp directory [{}] is deleting".format(tmp_dir)) node_svc.os_ops.rmdir(tmp_dir) def test_node_app__make_empty(self, node_svc: PostgresNodeService): assert type(node_svc) is PostgresNodeService assert isinstance(node_svc.os_ops, OsOperations) assert node_svc.port_manager is not None assert isinstance(node_svc.port_manager, PortManager) tmp_dir = node_svc.os_ops.mkdtemp() assert tmp_dir is not None assert type(tmp_dir) is str logging.info("temp directory is [{}]".format(tmp_dir)) # ----------- node_app = NodeApp( test_path=tmp_dir, os_ops=node_svc.os_ops, port_manager=node_svc.port_manager ) assert node_app.os_ops is node_svc.os_ops assert node_app.port_manager is node_svc.port_manager assert type(node_app.nodes_to_cleanup) is list assert len(node_app.nodes_to_cleanup) == 0 node: typing.Optional[PostgresNode] = None try: node = node_app.make_simple("node") assert node is not None assert isinstance(node, PostgresNode) assert node.os_ops is node_svc.os_ops assert node.port_manager is node_svc.port_manager assert type(node_app.nodes_to_cleanup) is list assert len(node_app.nodes_to_cleanup) == 1 assert node_app.nodes_to_cleanup[0] is node node.slow_start() finally: if node is not None: node.stop() node.release_resources() if node is not None: node.cleanup(release_resources=True) # ----------- logging.info("temp directory [{}] is deleting".format(tmp_dir)) node_svc.os_ops.rmdir(tmp_dir) def test_node_app__make_simple__checksum(self, node_svc: PostgresNodeService): assert type(node_svc) is PostgresNodeService assert isinstance(node_svc.os_ops, OsOperations) assert node_svc.port_manager is not None assert isinstance(node_svc.port_manager, PortManager) tmp_dir = node_svc.os_ops.mkdtemp() assert tmp_dir is not None assert type(tmp_dir) is str logging.info("temp directory is [{}]".format(tmp_dir)) node_app = NodeApp(test_path=tmp_dir, os_ops=node_svc.os_ops) C_NODE = "node" # ----------- def LOCAL__test(checksum: bool, initdb_params: typing.Optional[list]): initdb_params0 = initdb_params initdb_params0_copy = initdb_params0.copy() if initdb_params0 is not None else None with node_app.make_simple(C_NODE, checksum=checksum, initdb_params=initdb_params): assert initdb_params is initdb_params0 if initdb_params0 is not None: assert initdb_params0 == initdb_params0_copy assert initdb_params is initdb_params0 if initdb_params0 is not None: assert initdb_params0 == initdb_params0_copy # ----------- LOCAL__test(checksum=False, initdb_params=None) LOCAL__test(checksum=True, initdb_params=None) # ----------- params = [] LOCAL__test(checksum=False, initdb_params=params) LOCAL__test(checksum=True, initdb_params=params) # ----------- params = ["--no-sync"] LOCAL__test(checksum=False, initdb_params=params) LOCAL__test(checksum=True, initdb_params=params) # ----------- params = ["--data-checksums"] LOCAL__test(checksum=False, initdb_params=params) LOCAL__test(checksum=True, initdb_params=params) # ----------- logging.info("temp directory [{}] is deleting".format(tmp_dir)) node_svc.os_ops.rmdir(tmp_dir) def test_node_app__make_empty_with_explicit_port(self, node_svc: PostgresNodeService): assert type(node_svc) is PostgresNodeService assert isinstance(node_svc.os_ops, OsOperations) assert node_svc.port_manager is not None assert isinstance(node_svc.port_manager, PortManager) C_MAX_ATTEMPTS = 5 tmp_dir = node_svc.os_ops.mkdtemp() assert tmp_dir is not None assert type(tmp_dir) is str logging.info("temp directory is [{}]".format(tmp_dir)) # ----------- node_app = NodeApp( test_path=tmp_dir, os_ops=node_svc.os_ops, port_manager=node_svc.port_manager ) assert node_app.os_ops is node_svc.os_ops assert node_app.port_manager is node_svc.port_manager assert type(node_app.nodes_to_cleanup) is list assert len(node_app.nodes_to_cleanup) == 0 attempt = 0 ports = [] try: while True: if attempt == C_MAX_ATTEMPTS: raise RuntimeError("Node did not start.") attempt += 1 logging.info("------------- attempt #{}".format( attempt )) port = node_app.port_manager.reserve_port() assert type(port) is int assert port is not ports try: ports.append(port) except: # noqa: E722 node_app.port_manager.release_port(port) raise assert len(ports) == attempt node_name = "node_{}".format(attempt) logging.info("Node [{}] is creating...".format(node_name)) node = node_app.make_simple(node_name, port=port) assert node is not None assert isinstance(node, PostgresNode) assert node.os_ops is node_svc.os_ops assert node.port_manager is None # <--------- assert node.port == port assert node._should_free_port == False # noqa: E712 assert type(node_app.nodes_to_cleanup) is list assert len(node_app.nodes_to_cleanup) == attempt assert node_app.nodes_to_cleanup[-1] is node assert node.status() == NodeStatus.Stopped logging.info("Node is created") logging.info("Try to start a node...") try: node.slow_start() except StartNodeException as e: logging.info("Exception ({}): {}".format( type(e).__name__, e )) assert node.status() == NodeStatus.Stopped continue assert node.status() == NodeStatus.Running logging.info("Node is started") logging.info("Stop node") node.stop() assert node.status() == NodeStatus.Stopped logging.info("Node is stopped") logging.info("OK. Go home.") assert node is not None assert isinstance(node, PostgresNode) assert node._port is not None assert node._port == port assert not node._should_free_port break finally: while len(ports) > 0: node_app.port_manager.release_port(ports.pop()) # ----------- logging.info("temp directory [{}] is deleting".format(tmp_dir)) node_svc.os_ops.rmdirs(tmp_dir) return @staticmethod def helper__get_node( node_svc: PostgresNodeService, name: typing.Optional[str] = None, port: typing.Optional[int] = None, port_manager: typing.Optional[PortManager] = None ) -> PostgresNode: assert isinstance(node_svc, PostgresNodeService) assert isinstance(node_svc.os_ops, OsOperations) assert isinstance(node_svc.port_manager, PortManager) if port_manager is None: port_manager = node_svc.port_manager return PostgresNode( name, port=port, os_ops=node_svc.os_ops, port_manager=port_manager if port is None else None ) @staticmethod def helper__skip_test_if_pg_version_is_not_ge(ver1: str, ver2: str): assert type(ver1) is str assert type(ver2) is str if not __class__.helper__pg_version_ge(ver1, ver2): pytest.skip('requires {0}+'.format(ver2)) @staticmethod def helper__skip_test_if_pg_version_is_ge(ver1: str, ver2: str): assert type(ver1) is str assert type(ver2) is str if __class__.helper__pg_version_ge(ver1, ver2): pytest.skip('requires <{0}'.format(ver2)) @staticmethod def helper__pg_version_ge(ver1: str, ver2: str) -> bool: assert type(ver1) is str assert type(ver2) is str v1 = PgVer(ver1) v2 = PgVer(ver2) return v1 >= v2 @staticmethod def helper__rm_carriage_returns(out): """ In Windows we have additional '\r' symbols in output. Let's get rid of them. """ if isinstance(out, (int, float, complex)): return out if isinstance(out, tuple): return tuple(__class__.helper__rm_carriage_returns(item) for item in out) if isinstance(out, bytes): return out.replace(b'\r', b'') assert type(out) is str return out.replace('\r', '') @staticmethod def helper__skip_test_if_util_not_exist(os_ops: OsOperations, name: str): assert isinstance(os_ops, OsOperations) assert type(name) is str if not __class__.helper__util_exists(os_ops, name): pytest.skip('might be missing') @staticmethod def helper__util_exists(os_ops: OsOperations, util): assert isinstance(os_ops, OsOperations) def good_properties(f): return (os_ops.path_exists(f) and # noqa: W504 os_ops.isfile(f) and # noqa: W504 os_ops.is_executable(f)) # yapf: disable # try to resolve it if good_properties(get_bin_path2(os_ops, util)): return True # check if util is in PATH for path in os_ops.environ("PATH").split(os.pathsep): if good_properties(os.path.join(path, util)): return True ================================================ FILE: tests/test_testgres_local.py ================================================ # coding: utf-8 import os import re import subprocess import pytest import psutil import platform import logging import src as testgres from src import StartNodeException from src import ExecUtilException from src import NodeApp from src import NodeStatus from src import scoped_config from src import get_new_node from src import get_bin_path from src import get_pg_config from src import get_pg_version # NOTE: those are ugly imports from src.utils import bound_ports from src.utils import PgVer from src.node import ProcessProxy def pg_version_ge(version): cur_ver = PgVer(get_pg_version()) min_ver = PgVer(version) return cur_ver >= min_ver def util_exists(util): def good_properties(f): return (os.path.exists(f) and # noqa: W504 os.path.isfile(f) and # noqa: W504 os.access(f, os.X_OK)) # yapf: disable # try to resolve it if good_properties(get_bin_path(util)): return True # check if util is in PATH for path in os.environ["PATH"].split(os.pathsep): if good_properties(os.path.join(path, util)): return True def rm_carriage_returns(out): """ In Windows we have additional '\r' symbols in output. Let's get rid of them. """ if os.name == 'nt': if isinstance(out, (int, float, complex)): return out elif isinstance(out, tuple): return tuple(rm_carriage_returns(item) for item in out) elif isinstance(out, bytes): return out.replace(b'\r', b'') else: return out.replace('\r', '') else: return out class TestTestgresLocal: def test_pg_config(self): # check same instances a = get_pg_config() b = get_pg_config() assert (id(a) == id(b)) # save right before config change c1 = get_pg_config() # modify setting for this scope with scoped_config(cache_pg_config=False) as config: # sanity check for value assert not (config.cache_pg_config) # save right after config change c2 = get_pg_config() # check different instances after config change assert (id(c1) != id(c2)) # check different instances a = get_pg_config() b = get_pg_config() assert (id(a) != id(b)) def test_ports_management(self): assert bound_ports is not None assert type(bound_ports) is set if len(bound_ports) != 0: logging.warning("bound_ports is not empty: {0}".format(bound_ports)) stage0__bound_ports = bound_ports.copy() with get_new_node() as node: assert bound_ports is not None assert type(bound_ports) is set assert node.port is not None assert type(node.port) is int logging.info("node port is {0}".format(node.port)) assert node.port in bound_ports assert node.port not in stage0__bound_ports assert stage0__bound_ports <= bound_ports assert len(stage0__bound_ports) + 1 == len(bound_ports) stage1__bound_ports = stage0__bound_ports.copy() stage1__bound_ports.add(node.port) assert stage1__bound_ports == bound_ports # check that port has been freed successfully assert bound_ports is not None assert type(bound_ports) is set assert bound_ports == stage0__bound_ports def test_child_process_dies(self): # test for FileNotFound exception during child_processes() function cmd = ["timeout", "60"] if os.name == 'nt' else ["sleep", "60"] nAttempt = 0 while True: if nAttempt == 5: raise Exception("Max attempt number is exceed.") nAttempt += 1 logging.info("Attempt #{0}".format(nAttempt)) with subprocess.Popen(cmd, shell=True) as process: # shell=True might be needed on Windows r = process.poll() if r is not None: logging.warning("process.pool() returns an unexpected result: {0}.".format(r)) continue assert r is None # collect list of processes currently running children = psutil.Process(os.getpid()).children() # kill a process, so received children dictionary becomes invalid process.kill() process.wait() # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" [ProcessProxy(p) for p in children] break def test_upgrade_node(self): old_bin_dir = os.path.dirname(get_bin_path("pg_config")) new_bin_dir = os.path.dirname(get_bin_path("pg_config")) with get_new_node(prefix='node_old', bin_dir=old_bin_dir) as node_old: node_old.init() node_old.start() node_old.stop() with get_new_node(prefix='node_new', bin_dir=new_bin_dir) as node_new: node_new.init(cached=False) res = node_new.upgrade_from(old_node=node_old) node_new.start() assert (b'Upgrade Complete' in res) class tagPortManagerProxy: sm_prev_testgres_reserve_port = None sm_prev_testgres_release_port = None sm_DummyPortNumber = None sm_DummyPortMaxUsage = None sm_DummyPortCurrentUsage = None sm_DummyPortTotalUsage = None def __init__(self, dummyPortNumber, dummyPortMaxUsage): assert type(dummyPortNumber) is int assert type(dummyPortMaxUsage) is int assert dummyPortNumber >= 0 assert dummyPortMaxUsage >= 0 assert __class__.sm_prev_testgres_reserve_port is None assert __class__.sm_prev_testgres_release_port is None assert testgres.utils.reserve_port == testgres.utils.internal__reserve_port assert testgres.utils.release_port == testgres.utils.internal__release_port __class__.sm_prev_testgres_reserve_port = testgres.utils.reserve_port __class__.sm_prev_testgres_release_port = testgres.utils.release_port testgres.utils.reserve_port = __class__._proxy__reserve_port testgres.utils.release_port = __class__._proxy__release_port assert testgres.utils.reserve_port == __class__._proxy__reserve_port assert testgres.utils.release_port == __class__._proxy__release_port __class__.sm_DummyPortNumber = dummyPortNumber __class__.sm_DummyPortMaxUsage = dummyPortMaxUsage __class__.sm_DummyPortCurrentUsage = 0 __class__.sm_DummyPortTotalUsage = 0 def __enter__(self): return self def __exit__(self, type, value, traceback): assert __class__.sm_DummyPortCurrentUsage == 0 assert __class__.sm_prev_testgres_reserve_port is not None assert __class__.sm_prev_testgres_release_port is not None assert testgres.utils.reserve_port == __class__._proxy__reserve_port assert testgres.utils.release_port == __class__._proxy__release_port testgres.utils.reserve_port = __class__.sm_prev_testgres_reserve_port testgres.utils.release_port = __class__.sm_prev_testgres_release_port __class__.sm_prev_testgres_reserve_port = None __class__.sm_prev_testgres_release_port = None @staticmethod def _proxy__reserve_port(): assert type(__class__.sm_DummyPortMaxUsage) is int assert type(__class__.sm_DummyPortTotalUsage) is int assert type(__class__.sm_DummyPortCurrentUsage) is int assert __class__.sm_DummyPortTotalUsage >= 0 assert __class__.sm_DummyPortCurrentUsage >= 0 assert __class__.sm_DummyPortTotalUsage <= __class__.sm_DummyPortMaxUsage assert __class__.sm_DummyPortCurrentUsage <= __class__.sm_DummyPortTotalUsage assert __class__.sm_prev_testgres_reserve_port is not None if __class__.sm_DummyPortTotalUsage == __class__.sm_DummyPortMaxUsage: return __class__.sm_prev_testgres_reserve_port() __class__.sm_DummyPortTotalUsage += 1 __class__.sm_DummyPortCurrentUsage += 1 return __class__.sm_DummyPortNumber @staticmethod def _proxy__release_port(dummyPortNumber): assert type(dummyPortNumber) is int assert type(__class__.sm_DummyPortMaxUsage) is int assert type(__class__.sm_DummyPortTotalUsage) is int assert type(__class__.sm_DummyPortCurrentUsage) is int assert __class__.sm_DummyPortTotalUsage >= 0 assert __class__.sm_DummyPortCurrentUsage >= 0 assert __class__.sm_DummyPortTotalUsage <= __class__.sm_DummyPortMaxUsage assert __class__.sm_DummyPortCurrentUsage <= __class__.sm_DummyPortTotalUsage assert __class__.sm_prev_testgres_release_port is not None if __class__.sm_DummyPortCurrentUsage > 0 and dummyPortNumber == __class__.sm_DummyPortNumber: assert __class__.sm_DummyPortTotalUsage > 0 __class__.sm_DummyPortCurrentUsage -= 1 return return __class__.sm_prev_testgres_release_port(dummyPortNumber) def test_port_rereserve_during_node_start(self): assert testgres.PostgresNode._C_MAX_START_ATEMPTS == 5 C_COUNT_OF_BAD_PORT_USAGE = 3 with get_new_node() as node1: node1.init().start() assert (node1._should_free_port) assert (type(node1.port) is int) node1_port_copy = node1.port assert (rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n') with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port with get_new_node() as node2: assert (node2._should_free_port) assert (node2.port == node1.port) node2.init().start() assert (node2.port != node1.port) assert (node2._should_free_port) assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 0) assert (__class__.tagPortManagerProxy.sm_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE) assert (node2.is_started) assert (rm_carriage_returns(node2.safe_psql("SELECT 2;")) == b'2\n') # node1 is still working assert (node1.port == node1_port_copy) assert (node1._should_free_port) assert (rm_carriage_returns(node1.safe_psql("SELECT 3;")) == b'3\n') def test_port_conflict(self): assert testgres.PostgresNode._C_MAX_START_ATEMPTS > 1 C_COUNT_OF_BAD_PORT_USAGE = testgres.PostgresNode._C_MAX_START_ATEMPTS with get_new_node() as node1: node1.init().start() assert (node1._should_free_port) assert (type(node1.port) is int) node1_port_copy = node1.port assert (rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n') with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port with get_new_node() as node2: assert (node2._should_free_port) assert (node2.port == node1.port) node2.init() assert (node2.status() == NodeStatus.Stopped) with pytest.raises( expected_exception=StartNodeException, match=re.escape("Cannot start node after multiple attempts.") ): node2.start() assert (node2.port == node1.port) assert (node2._should_free_port) assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 1) assert (__class__.tagPortManagerProxy.sm_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE) assert (not node2.is_started) assert (node2.status() == NodeStatus.Stopped) # node2 must release our dummyPort (node1.port) assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 0) # node1 is still working assert (node1.port == node1_port_copy) assert (node1._should_free_port) assert (node1.status() == NodeStatus.Running) assert (rm_carriage_returns(node1.safe_psql("SELECT 3;")) == b'3\n') def test_simple_with_bin_dir(self): with get_new_node() as node: node.init().start() bin_dir = node.bin_dir app = NodeApp() with app.make_simple(base_dir=node.base_dir, bin_dir=bin_dir) as correct_bin_dir: correct_bin_dir.slow_start() correct_bin_dir.safe_psql("SELECT 1;") correct_bin_dir.stop() while True: try: app.make_simple(base_dir=node.base_dir, bin_dir="wrong/path") except FileNotFoundError: break # Expected error except ExecUtilException: break # Expected error raise RuntimeError("Error was expected.") # We should not reach this return def test_set_auto_conf(self): # elements contain [property id, value, storage value] testData = [ ["archive_command", "cp '%p' \"/mnt/server/archivedir/%f\"", "'cp \\'%p\\' \"/mnt/server/archivedir/%f\""], ["log_line_prefix", "'\n\r\t\b\\\"", "'\\\'\\n\\r\\t\\b\\\\\""], ["log_connections", True, "on"], ["log_disconnections", False, "off"], ["autovacuum_max_workers", 3, "3"] ] if pg_version_ge('12'): testData.append(["restore_command", 'cp "/mnt/server/archivedir/%f" \'%p\'', "'cp \"/mnt/server/archivedir/%f\" \\'%p\\''"]) with get_new_node() as node: node.init().start() options = {} for x in testData: options[x[0]] = x[1] node.set_auto_conf(options) node.stop() node.slow_start() auto_conf_path = f"{node.data_dir}/postgresql.auto.conf" with open(auto_conf_path, "r") as f: content = f.read() for x in testData: assert x[0] + " = " + x[2] in content @staticmethod def helper__skip_test_if_util_not_exist(name: str): assert type(name) is str if platform.system().lower() == "windows": name2 = name + ".exe" else: name2 = name if not util_exists(name2): pytest.skip('might be missing') ================================================ FILE: tests/test_testgres_remote.py ================================================ # coding: utf-8 import os import pytest import logging import typing from .helpers.global_data import PostgresNodeService from .helpers.global_data import PostgresNodeServices import src as testgres from src.exceptions import InitNodeException from src.exceptions import ExecUtilException from src.config import scoped_config from src.config import testgres_config from src import get_bin_path from src import get_pg_config # NOTE: those are ugly imports def util_exists(util): def good_properties(f): return (testgres_config.os_ops.path_exists(f) and # noqa: W504 testgres_config.os_ops.isfile(f) and # noqa: W504 testgres_config.os_ops.is_executable(f)) # yapf: disable # try to resolve it if good_properties(get_bin_path(util)): return True # check if util is in PATH for path in testgres_config.os_ops.environ("PATH").split(testgres_config.os_ops.pathsep): if good_properties(os.path.join(path, util)): return True class TestTestgresRemote: @pytest.fixture(autouse=True, scope="class") def implicit_fixture(self): cur_os_ops = PostgresNodeServices.sm_remote.os_ops assert cur_os_ops is not None prev_ops = testgres_config.os_ops assert prev_ops is not None testgres_config.set_os_ops(os_ops=cur_os_ops) assert testgres_config.os_ops is cur_os_ops yield assert testgres_config.os_ops is cur_os_ops testgres_config.set_os_ops(os_ops=prev_ops) assert testgres_config.os_ops is prev_ops def test_init__LANG_С(self): # PBCKP-1744 prev_LANG = os.environ.get("LANG") try: os.environ["LANG"] = "C" with __class__.helper__get_node() as node: node.init().start() finally: __class__.helper__restore_envvar("LANG", prev_LANG) def test_init__unk_LANG_and_LC_CTYPE(self): # PBCKP-1744 prev_LANG = os.environ.get("LANG") prev_LANGUAGE = os.environ.get("LANGUAGE") prev_LC_CTYPE = os.environ.get("LC_CTYPE") prev_LC_COLLATE = os.environ.get("LC_COLLATE") try: # TODO: Pass unkData through test parameter. unkDatas = [ ("UNKNOWN_LANG", "UNKNOWN_CTYPE"), ("\"UNKNOWN_LANG\"", "\"UNKNOWN_CTYPE\""), ("\\UNKNOWN_LANG\\", "\\UNKNOWN_CTYPE\\"), ("\"UNKNOWN_LANG", "UNKNOWN_CTYPE\""), ("\\UNKNOWN_LANG", "UNKNOWN_CTYPE\\"), ("\\", "\\"), ("\"", "\""), ] errorIsDetected = False for unkData in unkDatas: logging.info("----------------------") logging.info("Unk LANG is [{0}]".format(unkData[0])) logging.info("Unk LC_CTYPE is [{0}]".format(unkData[1])) os.environ["LANG"] = unkData[0] os.environ.pop("LANGUAGE", None) os.environ["LC_CTYPE"] = unkData[1] os.environ.pop("LC_COLLATE", None) assert os.environ.get("LANG") == unkData[0] assert "LANGUAGE" not in os.environ.keys() assert os.environ.get("LC_CTYPE") == unkData[1] assert "LC_COLLATE" not in os.environ.keys() assert os.getenv('LANG') == unkData[0] assert os.getenv('LANGUAGE') is None assert os.getenv('LC_CTYPE') == unkData[1] assert os.getenv('LC_COLLATE') is None exc: typing.Optional[BaseException] = None with __class__.helper__get_node() as node: try: node.init() # IT RAISES! except InitNodeException as e: exc = e.__cause__ assert exc is not None assert isinstance(exc, ExecUtilException) if exc is None: logging.warning("We expected an error!") continue errorIsDetected = True assert isinstance(exc, ExecUtilException) errMsg = str(exc) logging.info("Error message is {0}: {1}".format(type(exc).__name__, errMsg)) assert "warning: setlocale: LC_CTYPE: cannot change locale (" + unkData[1] + ")" in errMsg assert "initdb: error: invalid locale settings; check LANG and LC_* environment variables" in errMsg continue if not errorIsDetected: pytest.xfail("All the bad data are processed without errors!") finally: __class__.helper__restore_envvar("LANG", prev_LANG) __class__.helper__restore_envvar("LANGUAGE", prev_LANGUAGE) __class__.helper__restore_envvar("LC_CTYPE", prev_LC_CTYPE) __class__.helper__restore_envvar("LC_COLLATE", prev_LC_COLLATE) def test_pg_config(self): # check same instances a = get_pg_config() b = get_pg_config() assert (id(a) == id(b)) # save right before config change c1 = get_pg_config() # modify setting for this scope with scoped_config(cache_pg_config=False) as config: # sanity check for value assert not (config.cache_pg_config) # save right after config change c2 = get_pg_config() # check different instances after config change assert (id(c1) != id(c2)) # check different instances a = get_pg_config() b = get_pg_config() assert (id(a) != id(b)) @staticmethod def helper__get_node(name=None): svc = PostgresNodeServices.sm_remote assert isinstance(svc, PostgresNodeService) assert isinstance(svc.os_ops, testgres.OsOperations) assert isinstance(svc.port_manager, testgres.PortManager) return testgres.PostgresNode( name, os_ops=svc.os_ops, port_manager=svc.port_manager) @staticmethod def helper__restore_envvar(name, prev_value): if prev_value is None: os.environ.pop(name, None) else: os.environ[name] = prev_value @staticmethod def helper__skip_test_if_util_not_exist(name: str): assert type(name) is str if not util_exists(name): pytest.skip('might be missing') ================================================ FILE: tests/test_utils.py ================================================ from .helpers.global_data import OsOpsDescr from .helpers.global_data import OsOpsDescrs from .helpers.global_data import OsOperations from src.utils import parse_pg_version from src.utils import get_pg_config2 from src import scoped_config import pytest import typing class TestUtils: sm_os_ops_descrs: typing.List[OsOpsDescr] = [ OsOpsDescrs.sm_local_os_ops_descr, OsOpsDescrs.sm_remote_os_ops_descr ] @pytest.fixture( params=[descr.os_ops for descr in sm_os_ops_descrs], ids=[descr.sign for descr in sm_os_ops_descrs] ) def os_ops(self, request: pytest.FixtureRequest) -> OsOperations: assert isinstance(request, pytest.FixtureRequest) assert isinstance(request.param, OsOperations) return request.param def test_parse_pg_version(self): # Linux Mint assert parse_pg_version("postgres (PostgreSQL) 15.5 (Ubuntu 15.5-1.pgdg22.04+1)") == "15.5" # Linux Ubuntu assert parse_pg_version("postgres (PostgreSQL) 12.17") == "12.17" # Windows assert parse_pg_version("postgres (PostgreSQL) 11.4") == "11.4" # Macos assert parse_pg_version("postgres (PostgreSQL) 14.9 (Homebrew)") == "14.9" def test_get_pg_config2(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) # check same instances a = get_pg_config2(os_ops, None) b = get_pg_config2(os_ops, None) assert (id(a) == id(b)) # save right before config change c1 = get_pg_config2(os_ops, None) # modify setting for this scope with scoped_config(cache_pg_config=False) as config: # sanity check for value assert not (config.cache_pg_config) # save right after config change c2 = get_pg_config2(os_ops, None) # check different instances after config change assert (id(c1) != id(c2)) # check different instances a = get_pg_config2(os_ops, None) b = get_pg_config2(os_ops, None) assert (id(a) != id(b)) ================================================ FILE: tests/units/__init__.py ================================================ ================================================ FILE: tests/units/exceptions/BackupException/__init__.py ================================================ ================================================ FILE: tests/units/exceptions/BackupException/test_set001__constructor.py ================================================ from src.exceptions import BackupException from src.exceptions import TestgresException as testgres__TestgresException class TestSet001_Constructor: def test_001__default(self): e = BackupException() assert type(e) is BackupException assert isinstance(e, testgres__TestgresException) assert e.source is None assert e.message == "" assert str(e) == "" assert repr(e) == "BackupException()" return def test_002__message(self): e = BackupException(message="abc\n123") assert type(e) is BackupException assert isinstance(e, testgres__TestgresException) assert e.source is None assert e.message == "abc\n123" assert str(e) == "abc\n123" assert repr(e) == "BackupException(message='abc\\n123')" return ================================================ FILE: tests/units/exceptions/CatchUpException/__init__.py ================================================ ================================================ FILE: tests/units/exceptions/CatchUpException/test_set001__constructor.py ================================================ from src.exceptions import CatchUpException from src.exceptions import TestgresException as testgres__TestgresException class TestSet001_Constructor: def test_001__default(self): e = CatchUpException() assert type(e) is CatchUpException assert isinstance(e, testgres__TestgresException) assert e.source is None assert e.message == "" assert str(e) == "" assert repr(e) == "CatchUpException()" return def test_002__message(self): e = CatchUpException(message="abc\n123") assert type(e) is CatchUpException assert isinstance(e, testgres__TestgresException) assert e.source is None assert e.message == "abc\n123" assert str(e) == "abc\n123" assert repr(e) == "CatchUpException(message='abc\\n123')" return ================================================ FILE: tests/units/exceptions/InitNodeException/__init__.py ================================================ ================================================ FILE: tests/units/exceptions/InitNodeException/test_set001__constructor.py ================================================ from src.exceptions import InitNodeException from src.exceptions import TestgresException as testgres__TestgresException class TestSet001_Constructor: def test_001__default(self): e = InitNodeException() assert type(e) is InitNodeException assert isinstance(e, testgres__TestgresException) assert e.source is None assert e.message == "" assert str(e) == "" assert repr(e) == "InitNodeException()" return def test_002__message(self): e = InitNodeException(message="abc\n123") assert type(e) is InitNodeException assert isinstance(e, testgres__TestgresException) assert e.source is None assert e.message == "abc\n123" assert str(e) == "abc\n123" assert repr(e) == "InitNodeException(message='abc\\n123')" return ================================================ FILE: tests/units/exceptions/PortForException/__init__.py ================================================ ================================================ FILE: tests/units/exceptions/PortForException/test_set001__constructor.py ================================================ from src.exceptions import PortForException from src.exceptions import TestgresException as testgres__TestgresException class TestSet001_Constructor: def test_001__default(self): e = PortForException() assert type(e) is PortForException assert isinstance(e, testgres__TestgresException) assert e.source is None assert e.message == "" assert str(e) == "" assert repr(e) == "PortForException()" return def test_002__message(self): e = PortForException(message="abc\n123") assert type(e) is PortForException assert isinstance(e, testgres__TestgresException) assert e.source is None assert e.message == "abc\n123" assert str(e) == "abc\n123" assert repr(e) == "PortForException(message='abc\\n123')" return ================================================ FILE: tests/units/exceptions/QueryException/__init__.py ================================================ ================================================ FILE: tests/units/exceptions/QueryException/test_set001__constructor.py ================================================ from src.exceptions import QueryException from src.exceptions import TestgresException as testgres__TestgresException class TestSet001_Constructor: def test_001__default(self): e = QueryException() assert type(e) is QueryException assert isinstance(e, testgres__TestgresException) assert e.source is None assert e.message == "" assert e.description is None assert e.query is None assert str(e) == "" assert repr(e) == "QueryException()" return def test_002__message(self): e = QueryException(message="abc\n123") assert type(e) is QueryException assert isinstance(e, testgres__TestgresException) assert e.source is None assert e.message == "abc\n123" assert e.description == "abc\n123" assert e.query is None assert str(e) == "abc\n123" assert repr(e) == "QueryException(message='abc\\n123')" return def test_003__query(self): e = QueryException(query="cba\n321") assert type(e) is QueryException assert isinstance(e, testgres__TestgresException) assert e.source is None assert e.message == "Query: cba\n321" assert e.description is None assert e.query == "cba\n321" assert str(e) == "Query: cba\n321" assert repr(e) == "QueryException(query='cba\\n321')" return def test_004__all(self): e = QueryException(message="mmm", query="cba\n321") assert type(e) is QueryException assert isinstance(e, testgres__TestgresException) assert e.source is None assert e.message == "mmm\nQuery: cba\n321" assert e.description == "mmm" assert e.query == "cba\n321" assert str(e) == "mmm\nQuery: cba\n321" assert repr(e) == "QueryException(message='mmm', query='cba\\n321')" return ================================================ FILE: tests/units/exceptions/QueryTimeoutException/__init__.py ================================================ ================================================ FILE: tests/units/exceptions/QueryTimeoutException/test_set001__constructor.py ================================================ from src.exceptions import QueryTimeoutException from src.exceptions import QueryException from src.exceptions import TestgresException as testgres__TestgresException class TestSet001_Constructor: def test_001__default(self): e = QueryTimeoutException() assert type(e) is QueryTimeoutException assert isinstance(e, QueryException) assert isinstance(e, testgres__TestgresException) assert e.source is None assert e.message == "" assert e.description is None assert e.query is None assert str(e) == "" assert repr(e) == "QueryTimeoutException()" return def test_002__message(self): e = QueryTimeoutException(message="abc\n123") assert type(e) is QueryTimeoutException assert isinstance(e, QueryException) assert isinstance(e, testgres__TestgresException) assert e.source is None assert e.message == "abc\n123" assert e.description == "abc\n123" assert e.query is None assert str(e) == "abc\n123" assert repr(e) == "QueryTimeoutException(message='abc\\n123')" return def test_003__query(self): e = QueryTimeoutException(query="cba\n321") assert type(e) is QueryTimeoutException assert isinstance(e, QueryException) assert isinstance(e, testgres__TestgresException) assert e.source is None assert e.message == "Query: cba\n321" assert e.description is None assert e.query == "cba\n321" assert str(e) == "Query: cba\n321" assert repr(e) == "QueryTimeoutException(query='cba\\n321')" return def test_004__all(self): e = QueryTimeoutException(message="mmm", query="cba\n321") assert type(e) is QueryTimeoutException assert isinstance(e, QueryException) assert isinstance(e, testgres__TestgresException) assert e.source is None assert e.message == "mmm\nQuery: cba\n321" assert e.description == "mmm" assert e.query == "cba\n321" assert str(e) == "mmm\nQuery: cba\n321" assert repr(e) == "QueryTimeoutException(message='mmm', query='cba\\n321')" return ================================================ FILE: tests/units/exceptions/StartNodeException/__init__.py ================================================ ================================================ FILE: tests/units/exceptions/StartNodeException/test_set001__constructor.py ================================================ from src.exceptions import StartNodeException from src.exceptions import TestgresException as testgres__TestgresException class TestSet001_Constructor: def test_001__default(self): e = StartNodeException() assert type(e) is StartNodeException assert isinstance(e, testgres__TestgresException) assert e.source is None assert e.message == "" assert e.description is None assert e.files is None assert str(e) == "" assert repr(e) == "StartNodeException()" return def test_002__message(self): e = StartNodeException(message="abc\n123") assert type(e) is StartNodeException assert isinstance(e, testgres__TestgresException) assert e.source is None assert e.message == "abc\n123" assert e.description == "abc\n123" assert e.files is None assert str(e) == "abc\n123" assert repr(e) == "StartNodeException(message='abc\\n123')" return def test_003__files(self): e = StartNodeException(files=[("f\n1", b'line1\nline2')]) assert type(e) is StartNodeException assert isinstance(e, testgres__TestgresException) assert e.source is None assert e.message == "f\n1\n----\nb'line1\\nline2'\n" assert e.description is None assert e.files == [("f\n1", b'line1\nline2')] assert str(e) == "f\n1\n----\nb'line1\\nline2'\n" assert repr(e) == "StartNodeException(files=[('f\\n1', b'line1\\nline2')])" return def test_004__all(self): e = StartNodeException(message="mmm", files=[("f\n1", b'line1\nline2')]) assert type(e) is StartNodeException assert isinstance(e, testgres__TestgresException) assert e.source is None assert e.message == "mmm\nf\n1\n----\nb'line1\\nline2'\n" assert e.description == "mmm" assert e.files == [("f\n1", b'line1\nline2')] assert str(e) == "mmm\nf\n1\n----\nb'line1\\nline2'\n" assert repr(e) == "StartNodeException(message='mmm', files=[('f\\n1', b'line1\\nline2')])" return ================================================ FILE: tests/units/exceptions/TimeoutException/__init__.py ================================================ ================================================ FILE: tests/units/exceptions/TimeoutException/test_set001.py ================================================ from src.exceptions import QueryTimeoutException from src.exceptions import TimeoutException from src.exceptions import QueryException class TestSet001: def test_001__default(self): # It is an alias assert TimeoutException == QueryTimeoutException assert issubclass(TimeoutException, QueryException) return ================================================ FILE: tests/units/exceptions/__init__.py ================================================ ================================================ FILE: tests/units/impl/__init__.py ================================================ ================================================ FILE: tests/units/impl/platforms/__init__.py ================================================ ================================================ FILE: tests/units/impl/platforms/internal_platform_utils/InternalPlatformUtils/__init__.py ================================================ ================================================ FILE: tests/units/impl/platforms/internal_platform_utils/__init__.py ================================================ ================================================ FILE: tests/units/node/PostgresNode/__init__.py ================================================ ================================================ FILE: tests/units/node/PostgresNode/test_setM001__start.py ================================================ from __future__ import annotations from tests.helpers.global_data import PostgresNodeService from tests.helpers.global_data import PostgresNodeServices from tests.helpers.global_data import OsOperations from tests.helpers.global_data import PortManager from tests.helpers.utils import Utils as HelperUtils from tests.helpers.pg_node_utils import PostgresNodeUtils as PostgresNodeTestUtils from src import PostgresNode from src import NodeStatus from src import NodeConnection from src.node import PostgresNodeLogReader import pytest import typing import logging class TestSet001__start: @pytest.fixture( params=PostgresNodeServices.sm_locals_and_remotes, ids=[descr.sign for descr in PostgresNodeServices.sm_locals_and_remotes] ) def node_svc(self, request: pytest.FixtureRequest) -> PostgresNodeService: assert isinstance(request, pytest.FixtureRequest) assert isinstance(request.param, PostgresNodeService) assert isinstance(request.param.os_ops, OsOperations) assert isinstance(request.param.port_manager, PortManager) return request.param class tagData001: wait: typing.Optional[bool] def __init__(self, wait: typing.Optional[bool]): assert wait is None or type(wait) is bool self.wait = wait return sm_Data001: typing.List[tagData001] = [ tagData001(None), tagData001(True) ] @pytest.fixture( params=sm_Data001, ids=["wait={}".format(x.wait) for x in sm_Data001] ) def data001(self, request: pytest.FixtureRequest) -> tagData001: assert isinstance(request, pytest.FixtureRequest) assert type(request.param).__name__ == "tagData001" return request.param def test_001__wait_true( self, node_svc: PostgresNodeService, data001: tagData001 ): assert isinstance(node_svc, PostgresNodeService) assert type(data001) is __class__.tagData001 assert data001.wait is None or type(data001.wait) is bool with PostgresNodeTestUtils.get_node(node_svc) as node: assert type(node) is PostgresNode node.init() assert not node.is_started assert node.status() == NodeStatus.Stopped kwargs = {} if data001.wait is not None: assert data001.wait == True # noqa: E712 kwargs["wait"] = data001.wait node.start(**kwargs) assert node.is_started assert node.status() == NodeStatus.Running # Internals assert type(node._manually_started_pm_pid) is int assert node._manually_started_pm_pid != 0 assert node._manually_started_pm_pid != node._C_PM_PID__IS_NOT_DETECTED assert node._manually_started_pm_pid == node.pid return def test_002__wait_false(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) C_MAX_ATTEMPTS = 3 attempt = 0 while True: assert type(attempt) is int assert attempt >= 0 assert attempt <= C_MAX_ATTEMPTS if attempt == C_MAX_ATTEMPTS: raise RuntimeError("Node is not started") attempt += 1 logging.info("------------- attempt #{}".format(attempt)) if attempt > 1: HelperUtils.PrintAndSleep(5) with PostgresNodeTestUtils.get_node(node_svc) as node: assert type(node) is PostgresNode node.init() assert not node.is_started assert node.status() == NodeStatus.Stopped node_log_reader = PostgresNodeLogReader(node, from_beginnig=False) node.start(wait=False) assert node.is_started assert node.status() in [NodeStatus.Stopped, NodeStatus.Running] # Internals assert type(node._manually_started_pm_pid) is int assert node._manually_started_pm_pid == node._C_PM_PID__IS_NOT_DETECTED logging.info("Wait for running state ...") try: PostgresNodeTestUtils.wait_for_running_state( node=node, node_log_reader=node_log_reader, timeout=60, ) except PostgresNodeTestUtils.PortConflictNodeException as e: logging.warning("Exception {}: {}".format( type(e).__name__, e, )) continue logging.info("Node is running.") assert node.status() == NodeStatus.Running return def test_003__exec_env( self, node_svc: PostgresNodeService, ): assert isinstance(node_svc, PostgresNodeService) with PostgresNodeTestUtils.get_node(node_svc) as node: assert type(node) is PostgresNode node.init() assert not node.is_started assert node.status() == NodeStatus.Stopped C_ENV_NAME = "MYTESTVAR" C_ENV_VALUE = "abcdefg" envs = { C_ENV_NAME: C_ENV_VALUE } node.start(exec_env=envs) assert node.is_started assert node.status() == NodeStatus.Running with node.connect(dbname="postgres") as cn: assert type(cn) is NodeConnection cn.execute("CREATE TEMP TABLE cmd_out(content text);") cn.commit() cn.execute("COPY cmd_out FROM PROGRAM 'bash -c \'\'echo ${}\'\'';".format( C_ENV_NAME, )) cn.commit() recs = cn.execute("select content from cmd_out;") assert type(recs) is list assert len(recs) == 1 assert type(recs[0]) is tuple rec = recs[0] assert len(rec) == 1 assert rec[0] == C_ENV_VALUE logging.info("Env has value [{}]. It is OK!".find(rec[0])) return def test_004__params_is_None( self, node_svc: PostgresNodeService, ): assert isinstance(node_svc, PostgresNodeService) with PostgresNodeTestUtils.get_node(node_svc) as node: assert type(node) is PostgresNode node.init() assert not node.is_started assert node.status() == NodeStatus.Stopped node.start(params=None) assert node.is_started assert node.status() == NodeStatus.Running return def test_005__params_is_empty( self, node_svc: PostgresNodeService, ): assert isinstance(node_svc, PostgresNodeService) with PostgresNodeTestUtils.get_node(node_svc) as node: assert type(node) is PostgresNode node.init() assert not node.is_started assert node.status() == NodeStatus.Stopped node.start(params=[]) assert node.is_started assert node.status() == NodeStatus.Running return ================================================ FILE: tests/units/node/PostgresNode/test_setM002__start2.py ================================================ from __future__ import annotations from tests.helpers.global_data import PostgresNodeService from tests.helpers.global_data import PostgresNodeServices from tests.helpers.global_data import OsOperations from tests.helpers.global_data import PortManager from tests.helpers.utils import Utils as HelperUtils from tests.helpers.pg_node_utils import PostgresNodeUtils as PostgresNodeTestUtils from src import PostgresNode from src import NodeStatus from src import NodeConnection from src.node import PostgresNodeLogReader import pytest import typing import logging class TestSet002__start2: @pytest.fixture( params=PostgresNodeServices.sm_locals_and_remotes, ids=[descr.sign for descr in PostgresNodeServices.sm_locals_and_remotes] ) def node_svc(self, request: pytest.FixtureRequest) -> PostgresNodeService: assert isinstance(request, pytest.FixtureRequest) assert isinstance(request.param, PostgresNodeService) assert isinstance(request.param.os_ops, OsOperations) assert isinstance(request.param.port_manager, PortManager) return request.param class tagData001: wait: typing.Optional[bool] def __init__(self, wait: typing.Optional[bool]): assert wait is None or type(wait) is bool self.wait = wait return sm_Data001: typing.List[tagData001] = [ tagData001(None), tagData001(True) ] @pytest.fixture( params=sm_Data001, ids=["wait={}".format(x.wait) for x in sm_Data001] ) def data001(self, request: pytest.FixtureRequest) -> tagData001: assert isinstance(request, pytest.FixtureRequest) assert type(request.param).__name__ == "tagData001" return request.param def test_001__wait_true( self, node_svc: PostgresNodeService, data001: tagData001 ): assert isinstance(node_svc, PostgresNodeService) assert type(data001) is __class__.tagData001 assert data001.wait is None or type(data001.wait) is bool with PostgresNodeTestUtils.get_node(node_svc) as node: assert type(node) is PostgresNode node.init() assert not node.is_started assert node.status() == NodeStatus.Stopped kwargs = {} if data001.wait is not None: assert data001.wait == True # noqa: E712 kwargs["wait"] = data001.wait node.start2(**kwargs) assert not node.is_started assert node.status() == NodeStatus.Running # Internals assert node._manually_started_pm_pid is None return def test_002__wait_false(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) C_MAX_ATTEMPTS = 3 attempt = 0 while True: assert type(attempt) is int assert attempt >= 0 assert attempt <= C_MAX_ATTEMPTS if attempt == C_MAX_ATTEMPTS: raise RuntimeError("Node is not started") logging.info("------------- attempt #{}".format(attempt)) if attempt > 1: HelperUtils.PrintAndSleep(5) with PostgresNodeTestUtils.get_node(node_svc) as node: assert type(node) is PostgresNode node.init() assert not node.is_started assert node.status() == NodeStatus.Stopped node_log_reader = PostgresNodeLogReader(node, from_beginnig=False) node.start2(wait=False) assert not node.is_started assert node.status() in [NodeStatus.Stopped, NodeStatus.Running] # Internals assert node._manually_started_pm_pid is None logging.info("Wait for running state ...") try: PostgresNodeTestUtils.wait_for_running_state( node=node, node_log_reader=node_log_reader, timeout=60, ) except PostgresNodeTestUtils.PortConflictNodeException as e: logging.warning("Exception {}: {}".format( type(e).__name__, e, )) continue logging.info("Node is running.") assert node.status() == NodeStatus.Running return def test_003__exec_env( self, node_svc: PostgresNodeService, ): assert isinstance(node_svc, PostgresNodeService) with PostgresNodeTestUtils.get_node(node_svc) as node: assert type(node) is PostgresNode node.init() assert not node.is_started assert node.status() == NodeStatus.Stopped C_ENV_NAME = "MYTESTVAR" C_ENV_VALUE = "abcdefg" envs = { C_ENV_NAME: C_ENV_VALUE } node.start2(exec_env=envs) assert not node.is_started assert node.status() == NodeStatus.Running with node.connect(dbname="postgres") as cn: assert type(cn) is NodeConnection cn.execute("CREATE TEMP TABLE cmd_out(content text);") cn.commit() cn.execute("COPY cmd_out FROM PROGRAM 'bash -c \'\'echo ${}\'\'';".format( C_ENV_NAME, )) cn.commit() recs = cn.execute("select content from cmd_out;") assert type(recs) is list assert len(recs) == 1 assert type(recs[0]) is tuple rec = recs[0] assert len(rec) == 1 assert rec[0] == C_ENV_VALUE logging.info("Env has value [{}]. It is OK!".find(rec[0])) return def test_004__params_is_None( self, node_svc: PostgresNodeService, ): assert isinstance(node_svc, PostgresNodeService) with PostgresNodeTestUtils.get_node(node_svc) as node: assert type(node) is PostgresNode node.init() assert not node.is_started assert node.status() == NodeStatus.Stopped node.start(params=None) assert node.is_started assert node.status() == NodeStatus.Running return def test_005__params_is_empty( self, node_svc: PostgresNodeService, ): assert isinstance(node_svc, PostgresNodeService) with PostgresNodeTestUtils.get_node(node_svc) as node: assert type(node) is PostgresNode node.init() assert not node.is_started assert node.status() == NodeStatus.Stopped node.start(params=[]) assert node.is_started assert node.status() == NodeStatus.Running return ================================================ FILE: tests/units/node/__init__.py ================================================