Showing preview only (485K chars total). Download the full file or copy to clipboard to get everything.
Repository: postgrespro/testgres
Branch: master
Commit: 722ef1bb3fa8
Files: 95
Total size: 456.7 KB
Directory structure:
gitextract_6f31v0iv/
├── .coveragerc
├── .dockerignore
├── .github/
│ ├── dependabot.yml
│ └── workflows/
│ ├── package-verification.yml
│ └── python-publish.yml
├── .gitignore
├── .style.yapf
├── Dockerfile--altlinux_10.tmpl
├── Dockerfile--altlinux_11.tmpl
├── Dockerfile--astralinux_1_7.tmpl
├── Dockerfile--std-all.tmpl
├── Dockerfile--std.tmpl
├── Dockerfile--std2-all.tmpl
├── Dockerfile--ubuntu_24_04.tmpl
├── LICENSE
├── README.md
├── docs/
│ ├── Makefile
│ ├── README.md
│ └── source/
│ ├── conf.py
│ ├── index.rst
│ └── testgres.rst
├── pyproject.toml
├── run_tests.sh
├── run_tests2.sh
├── src/
│ ├── __init__.py
│ ├── api.py
│ ├── backup.py
│ ├── cache.py
│ ├── config.py
│ ├── connection.py
│ ├── consts.py
│ ├── decorators.py
│ ├── defaults.py
│ ├── enums.py
│ ├── exceptions.py
│ ├── impl/
│ │ ├── internal_utils.py
│ │ ├── platforms/
│ │ │ ├── internal_platform_utils.py
│ │ │ ├── internal_platform_utils_factory.py
│ │ │ ├── linux/
│ │ │ │ └── internal_platform_utils.py
│ │ │ └── win32/
│ │ │ └── internal_platform_utils.py
│ │ ├── port_manager__generic.py
│ │ └── port_manager__this_host.py
│ ├── logger.py
│ ├── node.py
│ ├── node_app.py
│ ├── port_manager.py
│ ├── pubsub.py
│ ├── raise_error.py
│ ├── standby.py
│ └── utils.py
└── tests/
├── README.md
├── __init__.py
├── conftest.py
├── helpers/
│ ├── __init__.py
│ ├── global_data.py
│ ├── pg_node_utils.py
│ ├── run_conditions.py
│ └── utils.py
├── requirements.txt
├── test_config.py
├── test_conftest.py--devel
├── test_os_ops_common.py
├── test_os_ops_local.py
├── test_os_ops_remote.py
├── test_raise_error.py
├── test_testgres_common.py
├── test_testgres_local.py
├── test_testgres_remote.py
├── test_utils.py
└── units/
├── __init__.py
├── exceptions/
│ ├── BackupException/
│ │ ├── __init__.py
│ │ └── test_set001__constructor.py
│ ├── CatchUpException/
│ │ ├── __init__.py
│ │ └── test_set001__constructor.py
│ ├── InitNodeException/
│ │ ├── __init__.py
│ │ └── test_set001__constructor.py
│ ├── PortForException/
│ │ ├── __init__.py
│ │ └── test_set001__constructor.py
│ ├── QueryException/
│ │ ├── __init__.py
│ │ └── test_set001__constructor.py
│ ├── QueryTimeoutException/
│ │ ├── __init__.py
│ │ └── test_set001__constructor.py
│ ├── StartNodeException/
│ │ ├── __init__.py
│ │ └── test_set001__constructor.py
│ ├── TimeoutException/
│ │ ├── __init__.py
│ │ └── test_set001.py
│ └── __init__.py
├── impl/
│ ├── __init__.py
│ └── platforms/
│ ├── __init__.py
│ └── internal_platform_utils/
│ ├── InternalPlatformUtils/
│ │ └── __init__.py
│ └── __init__.py
└── node/
├── PostgresNode/
│ ├── __init__.py
│ ├── test_setM001__start.py
│ └── test_setM002__start2.py
└── __init__.py
================================================
FILE CONTENTS
================================================
================================================
FILE: .coveragerc
================================================
[run]
source=.
omit=./env/*
================================================
FILE: .dockerignore
================================================
dist
env
venv
*.egg-info
logs
.vscode
.pytest_cache
================================================
FILE: .github/dependabot.yml
================================================
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
open-pull-requests-limit: 5
================================================
FILE: .github/workflows/package-verification.yml
================================================
# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
name: Run Package Verifications
on:
push:
branches: [ "master" ]
paths-ignore:
- "*.md"
pull_request:
branches: [ "master" ]
paths-ignore:
- "*.md"
jobs:
lint:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
python -m pip install flake8 flake8-pyproject ruff
- name: Lint with flake8
run: |
flake8 .
- name: Lint with ruff
run: |
ruff check .
build-check:
runs-on: ubuntu-latest
needs: lint
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Set up Python 3.12
uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Install build tools
run: |
python -m pip install --upgrade pip
python -m pip install build twine
- name: Build package
run: |
python -m build
- name: Check package metadata
run: |
twine check dist/*
test:
runs-on: ubuntu-latest
needs: build-check
strategy:
fail-fast: false
matrix:
include:
- platform: "std2-all"
python: "3.7"
postgres: "17"
- platform: "std2-all"
python: "3.8.0"
postgres: "17"
- platform: "std2-all"
python: "3.8"
postgres: "17"
- platform: "std2-all"
python: "3.9"
postgres: "17"
- platform: "std2-all"
python: "3.10"
postgres: "17"
- platform: "std2-all"
python: "3.11"
postgres: "17"
- platform: "std2-all"
python: "3.12"
postgres: "17"
- platform: "std2-all"
python: "3.13"
postgres: "17"
- platform: "std2-all"
python: "3.14"
postgres: "17"
- platform: "std"
python: "3"
postgres: "10"
- platform: "std"
python: "3"
postgres: "11"
- platform: "std"
python: "3"
postgres: "12"
- platform: "std"
python: "3"
postgres: "13"
- platform: "std"
python: "3"
postgres: "14"
- platform: "std"
python: "3"
postgres: "15"
- platform: "std"
python: "3"
postgres: "16"
- platform: "std-all"
python: "3"
postgres: "17"
- platform: "std-all"
python: "3"
postgres: "18"
- platform: "ubuntu_24_04"
python: "3"
postgres: "17"
- platform: "altlinux_10"
python: "3"
postgres: "17"
- platform: "altlinux_11"
python: "3"
postgres: "17"
- platform: "astralinux_1_7"
python: "3"
postgres: "17"
env:
BASE_SIGN: "${{ matrix.platform }}-py${{ matrix.python }}-pg${{ matrix.postgres }}"
steps:
- name: Prepare variables
run: |
echo "RUN_CFG__NOW=$(date +'%Y%m%d_%H%M%S')" >> $GITHUB_ENV
echo "RUN_CFG__LOGS_DIR=logs-${{ env.BASE_SIGN }}" >> $GITHUB_ENV
echo "RUN_CFG__DOCKER_IMAGE_NAME=tests-${{ env.BASE_SIGN }}" >> $GITHUB_ENV
echo "---------- [$GITHUB_ENV]"
cat $GITHUB_ENV
- name: Checkout
uses: actions/checkout@v6
- name: Prepare logs folder on the host
run: mkdir -p "${{ env.RUN_CFG__LOGS_DIR }}"
- name: Adjust logs folder permission
run: chmod -R 777 "${{ env.RUN_CFG__LOGS_DIR }}"
- name: Build local image ${{ matrix.alpine }}
run: docker build --build-arg PG_VERSION="${{ matrix.postgres }}" --build-arg PYTHON_VERSION="${{ matrix.python }}" -t "${{ env.RUN_CFG__DOCKER_IMAGE_NAME }}" -f Dockerfile--${{ matrix.platform }}.tmpl .
- name: Run
run: docker run $(bash <(curl -s https://codecov.io/env)) -t -v ${{ github.workspace }}/${{ env.RUN_CFG__LOGS_DIR }}:/home/test/testgres/logs "${{ env.RUN_CFG__DOCKER_IMAGE_NAME }}"
- name: Upload Logs
uses: actions/upload-artifact@v7
if: always() # IT IS IMPORTANT!
with:
name: testgres--test_logs--${{ env.RUN_CFG__NOW }}-${{ env.BASE_SIGN }}-id${{ github.run_id }}
path: "${{ env.RUN_CFG__LOGS_DIR }}/"
================================================
FILE: .github/workflows/python-publish.yml
================================================
# This workflow will upload a Python Package to PyPI when a release is created
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
# This workflow uses actions that are not certified by GitHub.
# They are provided by a third-party and are governed by
# separate terms of service, privacy policy, and support
# documentation.
name: Upload Python Package
on:
release:
types: [published]
permissions:
contents: read
jobs:
release-build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v6
with:
python-version: "3.x"
- name: Build release distributions
run: |
# NOTE: put your own distribution build steps here.
python -m pip install build
python -m build
- name: Upload distributions
uses: actions/upload-artifact@v7
with:
name: release-dists
path: dist/
pypi-publish:
runs-on: ubuntu-latest
needs:
- release-build
permissions:
# IMPORTANT: this permission is mandatory for trusted publishing
id-token: write
# Dedicated environments with protections for publishing are strongly recommended.
# For more information, see: https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#deployment-protection-rules
environment:
name: pypi
# OPTIONAL: uncomment and update to include your PyPI project URL in the deployment status:
# url: https://pypi.org/p/YOURPROJECT
#
# ALTERNATIVE: if your GitHub Release name is the PyPI project version string
# ALTERNATIVE: exactly, uncomment the following line instead:
# url: https://pypi.org/project/YOURPROJECT/${{ github.event.release.name }}
steps:
- name: Retrieve release distributions
uses: actions/download-artifact@v8
with:
name: release-dists
path: dist/
- name: Publish release distributions to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
packages-dir: dist/
================================================
FILE: .gitignore
================================================
*.pyc
*.egg
*.egg-info/
.eggs/
dist/
build/
docs/build/
logs/
env/
venv/
.coverage
coverage.xml
Dockerfile
*~
*.swp
tags
================================================
FILE: .style.yapf
================================================
[style]
based_on_style = pep8
spaces_before_comment = 4
split_before_logical_operator = false
column_limit=80
================================================
FILE: Dockerfile--altlinux_10.tmpl
================================================
ARG PG_VERSION
ARG PYTHON_VERSION
# --------------------------------------------- base1
FROM alt:p10 AS base1
RUN apt-get update
RUN apt-get install -y sudo curl ca-certificates
RUN apt-get update
RUN apt-get install -y openssh-server openssh-clients
RUN apt-get install -y time
# RUN apt-get install -y mc
RUN apt-get install -y libsqlite3-devel
EXPOSE 22
RUN ssh-keygen -A
# --------------------------------------------- postgres
FROM base1 AS base1_with_dev_tools
RUN apt-get update
RUN apt-get install -y git
RUN apt-get install -y gcc
RUN apt-get install -y make
RUN apt-get install -y meson
RUN apt-get install -y flex
RUN apt-get install -y bison
RUN apt-get install -y pkg-config
RUN apt-get install -y libssl-devel
RUN apt-get install -y libicu-devel
RUN apt-get install -y libzstd-devel
RUN apt-get install -y zlib-devel
RUN apt-get install -y liblz4-devel
RUN apt-get install -y libzstd-devel
RUN apt-get install -y libxml2-devel
# --------------------------------------------- postgres
FROM base1_with_dev_tools AS base1_with_pg-17
RUN git clone https://github.com/postgres/postgres.git -b REL_17_STABLE /pg/postgres/source
WORKDIR /pg/postgres/source
RUN ./configure --prefix=/pg/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml
RUN make -j 4 install
RUN make -j 4 -C contrib install
# SETUP PG_CONFIG
# When pg_config symlink in /usr/local/bin it returns a real (right) result of --bindir
RUN ln -s /pg/postgres/install/bin/pg_config -t /usr/local/bin
# SETUP PG CLIENT LIBRARY
# libpq.so.5 is enough
RUN ln -s /pg/postgres/install/lib/libpq.so.5.17 /usr/lib64/libpq.so.5
# --------------------------------------------- base2_with_python-3
FROM base1_with_pg-${PG_VERSION} AS base2_with_python-3
RUN apt-get install -y python3
RUN apt-get install -y python3-dev
RUN apt-get install -y python3-modules-sqlite3
ENV PYTHON_BINARY=python3
# --------------------------------------------- final
FROM base2_with_python-${PYTHON_VERSION} AS final
RUN adduser test -G wheel
# It enables execution of "sudo service ssh start" without password
RUN echo "test ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers
ADD --chown=test:test . /home/test/testgres
WORKDIR /home/test/testgres
RUN mkdir /home/test/testgres/logs
RUN chown -R test:test /home/test/testgres/logs
ENV LANG=C.UTF-8
USER test
RUN chmod 700 ~/
RUN mkdir -p ~/.ssh
#
# Altlinux 10 and 11 too slowly create a new SSH connection (x6).
#
ENTRYPOINT sh -c " \
set -eux; \
echo HELLO FROM ENTRYPOINT; \
echo HOME DIR IS [`realpath ~/`]; \
ls -la .; \
sudo /usr/sbin/sshd; \
sudo chmod 777 /home/test/testgres/logs; \
ls -la . | grep logs; \
ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \
ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \
ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \
chmod 600 ~/.ssh/authorized_keys; \
ls -la ~/.ssh/; \
TEST_FILTER=\"\" bash ./run_tests.sh;"
================================================
FILE: Dockerfile--altlinux_11.tmpl
================================================
ARG PG_VERSION
ARG PYTHON_VERSION
# --------------------------------------------- base1
FROM alt:p11 AS base1
RUN apt-get update
RUN apt-get install -y sudo curl ca-certificates
RUN apt-get update
RUN apt-get install -y openssh-server openssh-clients
RUN apt-get install -y time
# pgrep (for testgres.os_ops)
RUN apt-get install -y procps
# RUN apt-get install -y mc
RUN apt-get install -y libsqlite3-devel
EXPOSE 22
RUN ssh-keygen -A
# --------------------------------------------- postgres
FROM base1 AS base1_with_dev_tools
RUN apt-get update
RUN apt-get install -y git
RUN apt-get install -y gcc
RUN apt-get install -y make
RUN apt-get install -y meson
RUN apt-get install -y flex
RUN apt-get install -y bison
RUN apt-get install -y pkg-config
RUN apt-get install -y libssl-devel
RUN apt-get install -y libicu-devel
RUN apt-get install -y libzstd-devel
RUN apt-get install -y zlib-devel
RUN apt-get install -y liblz4-devel
RUN apt-get install -y libzstd-devel
RUN apt-get install -y libxml2-devel
# --------------------------------------------- postgres
FROM base1_with_dev_tools AS base1_with_pg-17
RUN git clone https://github.com/postgres/postgres.git -b REL_17_STABLE /pg/postgres/source
WORKDIR /pg/postgres/source
RUN ./configure --prefix=/pg/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml
RUN make -j 4 install
RUN make -j 4 -C contrib install
# SETUP PG_CONFIG
# When pg_config symlink in /usr/local/bin it returns a real (right) result of --bindir
RUN ln -s /pg/postgres/install/bin/pg_config -t /usr/local/bin
# SETUP PG CLIENT LIBRARY
# libpq.so.5 is enough
RUN ln -s /pg/postgres/install/lib/libpq.so.5.17 /usr/lib64/libpq.so.5
# --------------------------------------------- base2_with_python-3
FROM base1_with_pg-${PG_VERSION} AS base2_with_python-3
RUN apt-get install -y python3
RUN apt-get install -y python3-dev
RUN apt-get install -y python3-modules-sqlite3
ENV PYTHON_BINARY=python3
# --------------------------------------------- final
FROM base2_with_python-${PYTHON_VERSION} AS final
RUN adduser test -G wheel
# It enables execution of "sudo service ssh start" without password
RUN echo "test ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers
ADD --chown=test:test . /home/test/testgres
WORKDIR /home/test/testgres
RUN mkdir /home/test/testgres/logs
RUN chown -R test:test /home/test/testgres/logs
ENV LANG=C.UTF-8
USER test
RUN chmod 700 ~/
RUN mkdir -p ~/.ssh
#
# Altlinux 10 and 11 too slowly create a new SSH connection (x6).
#
ENTRYPOINT sh -c " \
set -eux; \
echo HELLO FROM ENTRYPOINT; \
echo HOME DIR IS [`realpath ~/`]; \
ls -la .; \
sudo /usr/sbin/sshd; \
sudo chmod 777 /home/test/testgres/logs; \
ls -la . | grep logs; \
ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \
ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \
ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \
chmod 600 ~/.ssh/authorized_keys; \
ls -la ~/.ssh/; \
TEST_FILTER=\"\" bash ./run_tests.sh;"
================================================
FILE: Dockerfile--astralinux_1_7.tmpl
================================================
ARG PG_VERSION
ARG PYTHON_VERSION
# --------------------------------------------- base1
FROM packpack/packpack:astra-1.7 AS base1
RUN apt update
RUN apt install -y sudo curl ca-certificates
RUN apt update
RUN apt install -y openssh-server
RUN apt install -y time
# RUN apt install -y netcat-traditional
RUN apt install -y git
# --------------------------------------------- postgres
FROM base1 AS base1_with_dev_tools
RUN apt-get update
RUN apt-get install -y git
RUN apt-get install -y gcc
RUN apt-get install -y make
RUN apt-get install -y meson
RUN apt-get install -y flex
RUN apt-get install -y bison
RUN apt-get install -y pkg-config
RUN apt-get install -y libssl-dev
RUN apt-get install -y libicu-dev
RUN apt-get install -y libzstd-dev
RUN apt-get install -y zlib1g-dev
RUN apt-get install -y liblz4-dev
RUN apt-get install -y libzstd-dev
RUN apt-get install -y libxml2-dev
# --------------------------------------------- postgres
FROM base1_with_dev_tools AS base1_with_pg-17
RUN curl -fsSL https://ftp.postgresql.org/pub/source/v17.7/postgresql-17.7.tar.bz2 -o postgresql.tar.bz2 \
&& mkdir -p /pg/postgres/source \
&& tar -xjf postgresql.tar.bz2 -C /pg/postgres/source --strip-components=1 \
&& rm postgresql.tar.bz2
WORKDIR /pg/postgres/source
RUN ./configure --prefix=/pg/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml
RUN make -j 4 install
RUN make -j 4 -C contrib install
# SETUP PG_CONFIG
# When pg_config symlink in /usr/local/bin it returns a real (right) result of --bindir
RUN ln -s /pg/postgres/install/bin/pg_config -t /usr/local/bin
# SETUP PG CLIENT LIBRARY
# libpq.so.5 is enough
RUN ln -s /pg/postgres/install/lib/libpq.so.5.17 /usr/lib/libpq.so.5
# --------------------------------------------- base2_with_python-3
FROM base1_with_pg-${PG_VERSION} AS base2_with_python-3
RUN apt install -y python3 python3-dev python3-venv
ENV PYTHON_BINARY=python3
# --------------------------------------------- final
FROM base2_with_python-${PYTHON_VERSION} AS final
EXPOSE 22
RUN ssh-keygen -A
RUN useradd -m test
# It enables execution of "sudo service ssh start" without password
# MY OLD:
# RUN sh -c "echo test ALL=NOPASSWD:ALL" >> /etc/sudoers
# AI:
RUN echo "test ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD
RUN sh -c "echo "test:*" | chpasswd -e"
RUN sed -i 's/UsePAM yes/UsePAM no/' /etc/ssh/sshd_config
ADD --chown=test:test . /home/test/testgres
WORKDIR /home/test/testgres
RUN mkdir /home/test/testgres/logs
RUN chown -R test:test /home/test/testgres/logs
ENV LANG=C.UTF-8
USER test
RUN chmod 700 ~/
RUN mkdir -p ~/.ssh
RUN chmod 700 ~/.ssh
ENTRYPOINT sh -c " \
set -eux; \
echo HELLO FROM ENTRYPOINT; \
echo HOME DIR IS [`realpath ~/`]; \
ls -la .; \
sudo service ssh start; \
sudo chmod 777 /home/test/testgres/logs; \
ls -la . | grep logs; \
ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \
ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \
ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \
chmod 600 ~/.ssh/authorized_keys; \
ls -la ~/.ssh/; \
TEST_FILTER=\"\" bash ./run_tests.sh;"
================================================
FILE: Dockerfile--std-all.tmpl
================================================
ARG PG_VERSION
ARG PYTHON_VERSION
# --------------------------------------------- base1
FROM postgres:${PG_VERSION}-alpine AS base1
# --------------------------------------------- base2_with_python-3
FROM base1 AS base2_with_python-3
RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers
ENV PYTHON_BINARY=python3
# --------------------------------------------- final
FROM base2_with_python-${PYTHON_VERSION} AS final
#RUN apk add --no-cache mc
RUN apk add --no-cache git
# Full version of "ps" command
RUN apk add --no-cache procps
RUN apk add --no-cache openssh
RUN apk add --no-cache sudo
ENV LANG=C.UTF-8
RUN addgroup -S sudo
RUN adduser -D test
RUN addgroup test sudo
EXPOSE 22
RUN ssh-keygen -A
ADD --chown=test:test . /home/test/testgres
WORKDIR /home/test/testgres
RUN mkdir /home/test/testgres/logs
RUN chown -R test:test /home/test/testgres/logs
# It allows to use sudo without password
RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers
# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD
RUN echo "test:*" | chpasswd -e
USER test
# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD
RUN chmod 700 ~/
RUN mkdir -p ~/.ssh
#RUN chmod 700 ~/.ssh
#ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh
ENTRYPOINT sh -c " \
set -eux; \
echo HELLO FROM ENTRYPOINT; \
echo HOME DIR IS [`realpath ~/`]; \
ls -la .; \
ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \
chmod 600 ~/.ssh/authorized_keys; \
ls -la ~/.ssh/; \
sudo /usr/sbin/sshd; \
sudo chmod 777 /home/test/testgres/logs; \
ls -la . | grep logs; \
ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \
ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \
TEST_FILTER=\"\" bash run_tests.sh;"
================================================
FILE: Dockerfile--std.tmpl
================================================
ARG PG_VERSION
ARG PYTHON_VERSION
# --------------------------------------------- base1
FROM postgres:${PG_VERSION}-alpine AS base1
# --------------------------------------------- base2_with_python-3
FROM base1 AS base2_with_python-3
RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers
ENV PYTHON_BINARY=python3
# --------------------------------------------- final
FROM base2_with_python-${PYTHON_VERSION} AS final
RUN apk add --no-cache git
RUN apk add --no-cache sudo
ENV LANG=C.UTF-8
RUN addgroup -S sudo
RUN adduser -D test
RUN addgroup test sudo
ADD --chown=test:test . /home/test/testgres
WORKDIR /home/test/testgres
RUN mkdir /home/test/testgres/logs
RUN chown -R test:test /home/test/testgres/logs
# It allows to use sudo without password
RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers
USER test
ENTRYPOINT sh -c " \
set -eux; \
echo HELLO FROM ENTRYPOINT; \
echo HOME DIR IS [`realpath ~/`]; \
ls -la .; \
sudo chmod 777 /home/test/testgres/logs; \
ls -la . | grep logs; \
bash run_tests.sh;"
================================================
FILE: Dockerfile--std2-all.tmpl
================================================
ARG PG_VERSION
ARG PYTHON_VERSION
# --------------------------------------------- base1
FROM postgres:${PG_VERSION}-alpine AS base1
# --------------------------------------------- base2_with_python-3
FROM base1 AS base2_with_python-3
ENV PYTHON_BINARY=python3
RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers
# For pyenv
RUN apk add patch
RUN apk add git
RUN apk add xz-dev
RUN apk add zip
RUN apk add zlib-dev
RUN apk add libffi-dev
RUN apk add readline-dev
RUN apk add openssl openssl-dev
RUN apk add sqlite-dev
RUN apk add bzip2-dev
# --------------------------------------------- base3_with_python-3.7
FROM base2_with_python-3 AS base3_with_python-3.7
ENV PYTHON_VERSION=3.7
# --------------------------------------------- base3_with_python-3.8.0
FROM base2_with_python-3 AS base3_with_python-3.8.0
ENV PYTHON_VERSION=3.8.0
# --------------------------------------------- base3_with_python-3.8
FROM base2_with_python-3 AS base3_with_python-3.8
ENV PYTHON_VERSION=3.8
# --------------------------------------------- base3_with_python-3.9
FROM base2_with_python-3 AS base3_with_python-3.9
ENV PYTHON_VERSION=3.9
# --------------------------------------------- base3_with_python-3.10
FROM base2_with_python-3 AS base3_with_python-3.10
ENV PYTHON_VERSION=3.10
# --------------------------------------------- base3_with_python-3.11
FROM base2_with_python-3 AS base3_with_python-3.11
ENV PYTHON_VERSION=3.11
# --------------------------------------------- base3_with_python-3.12
FROM base2_with_python-3 AS base3_with_python-3.12
ENV PYTHON_VERSION=3.12
# --------------------------------------------- base3_with_python-3.13
FROM base2_with_python-3 AS base3_with_python-3.13
ENV PYTHON_VERSION=3.13
# --------------------------------------------- base3_with_python-3.14
FROM base2_with_python-3 AS base3_with_python-3.14
ENV PYTHON_VERSION=3.14
# --------------------------------------------- final
FROM base3_with_python-${PYTHON_VERSION} AS final
#RUN apk add --no-cache mc
# Full version of "ps" command
RUN apk add --no-cache procps
RUN apk add --no-cache openssh
RUN apk add --no-cache sudo
ENV LANG=C.UTF-8
RUN addgroup -S sudo
RUN adduser -D test
RUN addgroup test sudo
EXPOSE 22
RUN ssh-keygen -A
# It allows to use sudo without password
RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers
# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD
RUN echo "test:*" | chpasswd -e
USER test
RUN curl https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash
RUN ~/.pyenv/bin/pyenv install ${PYTHON_VERSION}
ADD --chown=test:test . /home/test/testgres
WORKDIR /home/test/testgres
RUN mkdir /home/test/testgres/logs
RUN chown -R test:test /home/test/testgres/logs
# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD
RUN chmod 700 ~/
RUN mkdir -p ~/.ssh
ENTRYPOINT sh -c " \
set -eux; \
echo HELLO FROM ENTRYPOINT; \
echo HOME DIR IS [`realpath ~/`]; \
ls -la .; \
ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \
chmod 600 ~/.ssh/authorized_keys; \
ls -la ~/.ssh/; \
sudo /usr/sbin/sshd; \
sudo chmod 777 /home/test/testgres/logs; \
ls -la . | grep logs; \
ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \
ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \
export PATH=\"~/.pyenv/bin:$PATH\"; \
TEST_FILTER=\"\" bash run_tests2.sh;"
================================================
FILE: Dockerfile--ubuntu_24_04.tmpl
================================================
ARG PG_VERSION
ARG PYTHON_VERSION
# --------------------------------------------- base1
FROM ubuntu:24.04 AS base1
ARG PG_VERSION
RUN apt update
RUN apt install -y sudo curl ca-certificates
RUN apt update
RUN apt install -y openssh-server
RUN apt install -y time
RUN apt install -y netcat-traditional
RUN apt install -y git
RUN apt update
RUN apt install -y postgresql-common
RUN bash /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y
RUN install -d /usr/share/postgresql-common/pgdg
RUN curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc
RUN apt update
RUN apt install -y postgresql-${PG_VERSION}
# --------------------------------------------- base2_with_python-3
FROM base1 AS base2_with_python-3
RUN apt install -y python3 python3-dev python3-venv libpq-dev build-essential
ENV PYTHON_BINARY=python3
# --------------------------------------------- final
FROM base2_with_python-${PYTHON_VERSION} AS final
EXPOSE 22
RUN ssh-keygen -A
RUN adduser test
RUN chown postgres:postgres /var/run/postgresql
RUN chmod 775 /var/run/postgresql
RUN usermod -aG postgres test
# It enables execution of "sudo service ssh start" without password
# RUN echo "test ALL=NOPASSWD:/usr/sbin/service ssh start" >> /etc/sudoers
# RUN echo "test ALL=NOPASSWD:ALL" >> /etc/sudoers
RUN echo "test ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
ADD --chown=test:test . /home/test/testgres
WORKDIR /home/test/testgres
RUN mkdir /home/test/testgres/logs
RUN chown -R test:test /home/test/testgres/logs
ENV LANG=C.UTF-8
USER test
RUN chmod 700 ~/
RUN mkdir -p ~/.ssh
ENTRYPOINT sh -c " \
#set -eux; \
echo HELLO FROM ENTRYPOINT; \
echo HOME DIR IS [`realpath ~/`]; \
ls -la .; \
service ssh enable; \
sudo service ssh start; \
sudo chmod 777 /home/test/testgres/logs; \
ls -la . | grep logs; \
ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \
ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \
ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \
chmod 600 ~/.ssh/authorized_keys; \
ls -la ~/.ssh/; \
TEST_FILTER=\"\" bash ./run_tests.sh;"
================================================
FILE: LICENSE
================================================
testgres is released under the PostgreSQL License, a liberal Open Source license, similar to the BSD or MIT licenses.
Copyright (c) 2016-2026, Postgres Professional
Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this paragraph and the following two paragraphs appear in all copies.
IN NO EVENT SHALL POSTGRES PROFESSIONAL BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF POSTGRES PROFESSIONAL HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
POSTGRES PROFESSIONAL SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND POSTGRES PROFESSIONAL HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
================================================
FILE: README.md
================================================
[](https://github.com/postgrespro/testgres/actions/workflows/package-verification.yml)
[](https://codecov.io/gh/postgrespro/testgres)
[](https://badge.fury.io/py/testgres)
[](https://pypi.org/project/testgres)
[](https://pypi.org/project/testgres)
[Documentation](https://postgrespro.github.io/testgres/)
# testgres
Utility for orchestrating temporary PostgreSQL clusters in Python tests. Supports Python 3.7.3 and newer.
## Installation
Install `testgres` from PyPI:
```sh
pip install testgres
```
Use a dedicated virtual environment for isolated test dependencies.
## Usage
### Environment
> Note: by default `testgres` invokes `initdb`, `pg_ctl`, and `psql` binaries found in `PATH`.
Specify a custom PostgreSQL installation in one of the following ways:
- Set the `PG_CONFIG` environment variable to point to the `pg_config` executable.
- Set the `PG_BIN` environment variable to point to the directory with PostgreSQL binaries.
Example:
```sh
export PG_BIN=$HOME/pg_16/bin
python my_tests.py
```
### Examples
Create a temporary node, run queries, and let `testgres` clean up automatically:
```python
# create a node with a random name, port, and data directory
with testgres.get_new_node() as node:
# run initdb
node.init()
# start PostgreSQL
node.start()
# execute a query in the default database
print(node.execute('select 1'))
# the node is stopped and its files are removed automatically
```
### Query helpers
`testgres` provides four helpers for executing queries against the node:
| Command | Description |
|---------|-------------|
| `node.psql(query, ...)` | Runs the query via `psql` and returns a tuple `(returncode, stdout, stderr)`. |
| `node.safe_psql(query, ...)` | Same as `psql()` but returns only `stdout` and raises if the command fails. |
| `node.execute(query, ...)` | Connects via `psycopg2` or `pg8000` (whichever is available) and returns a list of tuples. |
| `node.connect(dbname, ...)` | Returns a `NodeConnection` wrapper for executing multiple statements within a transaction. |
Example of transactional usage:
```python
with node.connect() as con:
con.begin('serializable')
print(con.execute('select %s', 1))
con.rollback()
```
### Logging
By default `cleanup()` removes all temporary files (data directories, logs, and so on) created by the API. Call `configure_testgres(node_cleanup_full=False)` before starting nodes if you want to keep logs for inspection.
> Note: context managers (the `with` statement) call `stop()` and `cleanup()` automatically.
`testgres` integrates with the standard [Python logging](https://docs.python.org/3/library/logging.html) module, so you can aggregate logs from multiple nodes:
```python
import logging
# write everything to /tmp/testgres.log
logging.basicConfig(filename='/tmp/testgres.log')
# enable logging and create two nodes
testgres.configure_testgres(use_python_logging=True)
node1 = testgres.get_new_node().init().start()
node2 = testgres.get_new_node().init().start()
node1.execute('select 1')
node2.execute('select 2')
# disable logging
testgres.configure_testgres(use_python_logging=False)
```
See `tests/test_simple.py` for a complete logging example.
### Backup and replication
Creating backups and spawning replicas is straightforward:
```python
with testgres.get_new_node('master') as master:
master.init().start()
with master.backup() as backup:
replica = backup.spawn_replica('replica').start()
replica.catchup()
print(replica.execute('postgres', 'select 1'))
```
### Benchmarks
Use `pgbench` through `testgres` to run quick benchmarks:
```python
with testgres.get_new_node('master') as master:
master.init().start()
result = master.pgbench_init(scale=2).pgbench_run(time=10)
print(result)
```
### Custom configuration
`testgres` ships with sensible defaults. Adjust them as needed with `default_conf()` and `append_conf()`:
```python
extra_conf = "shared_preload_libraries = 'postgres_fdw'"
with testgres.get_new_node().init() as master:
master.default_conf(fsync=True, allow_streaming=True)
master.append_conf('postgresql.conf', extra_conf)
```
`default_conf()` is called by `init()` and rewrites the configuration file. Apply `append_conf()` afterwards to keep custom lines.
### Remote mode
You can provision nodes on a remote host (Linux only) by wiring `RemoteOperations` into the configuration:
```python
from testgres import ConnectionParams, RemoteOperations, TestgresConfig, get_remote_node
conn_params = ConnectionParams(
host='example.com',
username='postgres',
ssh_key='/path/to/ssh/key'
)
os_ops = RemoteOperations(conn_params)
TestgresConfig.set_os_ops(os_ops=os_ops)
def test_basic_query():
with get_remote_node(conn_params=conn_params) as node:
node.init().start()
assert node.execute('SELECT 1') == [(1,)]
```
### Pytest integration
Use fixtures to create and clean up nodes automatically when testing with `pytest`:
```python
import pytest
import testgres
@pytest.fixture
def pg_node():
node = testgres.get_new_node().init().start()
try:
yield node
finally:
node.stop()
node.cleanup()
def test_simple(pg_node):
assert pg_node.execute('select 1')[0][0] == 1
```
This pattern keeps tests concise and ensures that every node is stopped and removed even if the test fails.
### Scaling tips
- Run tests in parallel with `pytest -n auto` (requires `pytest-xdist`). Ensure each node uses a distinct port by setting `PGPORT` in the fixture or by passing the `port` argument to `get_new_node()`.
- Always call `node.cleanup()` after each test, or rely on context managers/fixtures that do it for you, to avoid leftover data directories.
- Prefer `node.safe_psql()` for lightweight assertions that should fail fast; use `node.execute()` when you need structured Python results.
## Authors
[Ildar Musin](https://github.com/zilder)
[Dmitry Ivanov](https://github.com/funbringer)
[Ildus Kurbangaliev](https://github.com/ildus)
[Yury Zhuravlev](https://github.com/stalkerg)
================================================
FILE: docs/Makefile
================================================
# Minimal makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
SPHINXPROJ = testgres
SOURCEDIR = source
BUILDDIR = build
# Put it first so that "make" without argument is like "make help".
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@pip install --force-reinstall ..
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
================================================
FILE: docs/README.md
================================================
# Building the documentation
Make sure you have `Sphinx` package installed:
```
pip install Sphinx
```
Then just run
```
make html
```
Documentation will be built in `build/html` directory. Other output formats are also available; run `make` without arguments to see the options.
================================================
FILE: docs/source/conf.py
================================================
# -*- coding: utf-8 -*-
#
# Configuration file for the Sphinx documentation builder.
#
# This file does only contain a selection of the most common options. For a
# full list see the documentation:
# http://www.sphinx-doc.org/en/stable/config
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
import testgres
assert testgres.__path__ is not None
assert len(testgres.__path__) == 1
assert type(testgres.__path__[0]) is str
p = os.path.dirname(testgres.__path__[0])
assert type(p) is str
sys.path.insert(0, os.path.abspath(p))
# -- Project information -----------------------------------------------------
project = u'testgres'
package_name = u'testgres'
copyright = u'2016-2026, Postgres Professional'
author = u'Postgres Professional'
# The full version, including alpha/beta/rc tags
release = testgres.__version__
# The short X.Y version
version = '.'.join(release.split('.')[:2])
# -- General configuration ---------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon']
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'
# The master toctree document.
master_doc = 'index'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = "en"
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path .
exclude_patterns = []
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'alabaster'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = []
# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
#
# The default sidebars (for documents that don't match any pattern) are
# defined by theme itself. Builtin themes are using these templates by
# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
# 'searchbox.html']``.
#
# html_sidebars = {}
# -- Options for HTMLHelp output ---------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'testgresdoc'
# -- Options for LaTeX output ------------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#
# 'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#
# 'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#
# 'preamble': '',
# Latex figure (float) alignment
#
# 'figure_align': 'htbp',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'testgres.tex', u'testgres Documentation',
u'Postgres Professional', 'manual'),
]
# -- Options for manual page output ------------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [(master_doc, 'testgres', u'testgres Documentation', [author], 1)]
# -- Options for Texinfo output ----------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(master_doc, 'testgres', u'testgres Documentation', author, 'testgres',
'One line description of project.', 'Miscellaneous'),
]
# -- Extension configuration -------------------------------------------------
================================================
FILE: docs/source/index.rst
================================================
Testgres documentation
======================
Utility for orchestrating temporary PostgreSQL clusters in Python tests. Supports Python 3.7.17 and newer.
Installation
============
To install testgres, run:
.. code-block:: bash
pip install testgres
We encourage you to use ``virtualenv`` for your testing environment.
Usage
=====
Environment
-----------
Note: by default ``testgres`` runs ``initdb``, ``pg_ctl``, and ``psql`` found in ``PATH``.
There are several ways to specify a custom PostgreSQL installation:
- export ``PG_CONFIG`` environment variable pointing to the ``pg_config`` executable;
- export ``PG_BIN`` environment variable pointing to the directory with executable files.
Example:
.. code-block:: bash
export PG_BIN=$HOME/pg_16/bin
python my_tests.py
Examples
--------
Create a temporary node, run queries, and let ``testgres`` clean up automatically:
.. code-block:: python
# create a node with a random name, port, and data directory
with testgres.get_new_node() as node:
# run initdb
node.init()
# start PostgreSQL
node.start()
# execute a query in the default database
print(node.execute('select 1'))
# the node is stopped and its files are removed automatically
Query helpers
-------------
``testgres`` provides four helpers for executing queries against the node:
========================== =======================================================
Command Description
========================== =======================================================
``node.psql(query, ...)`` Runs the query via ``psql`` and returns ``(code, out, err)``.
``node.safe_psql(...)`` Returns only ``stdout`` and raises if the command fails.
``node.execute(...)`` Uses ``psycopg2``/``pg8000`` and returns a list of tuples.
``node.connect(...)`` Returns a ``NodeConnection`` for transactional usage.
========================== =======================================================
Example:
.. code-block:: python
with node.connect() as con:
con.begin('serializable')
print(con.execute('select %s', 1))
con.rollback()
Logging
-------
By default ``cleanup()`` removes all temporary files (data directories, logs, and so on) created by the API. Call ``configure_testgres(node_cleanup_full=False)`` before starting nodes if you want to keep logs for inspection.
Note: context managers (the ``with`` statement) call ``stop()`` and ``cleanup()`` automatically.
``testgres`` integrates with the standard `Python logging <https://docs.python.org/3/library/logging.html>`_ module, so you can aggregate logs from multiple nodes:
.. code-block:: python
import logging
logging.basicConfig(filename='/tmp/testgres.log')
testgres.configure_testgres(use_python_logging=True)
node1 = testgres.get_new_node().init().start()
node2 = testgres.get_new_node().init().start()
node1.execute('select 1')
node2.execute('select 2')
testgres.configure_testgres(use_python_logging=False)
Backup & replication
--------------------
It's quite easy to create a backup and start a new replica:
.. code-block:: python
with testgres.get_new_node('master') as master:
master.init().start()
# create a backup
with master.backup() as backup:
# create and start a new replica
replica = backup.spawn_replica('replica').start()
replica.catchup()
print(replica.execute('postgres', 'select 1'))
Benchmarks
----------
Use ``pgbench`` through ``testgres`` to run quick benchmarks:
.. code-block:: python
with testgres.get_new_node('master') as master:
master.init().start()
result = master.pgbench_init(scale=2).pgbench_run(time=10)
print(result)
Custom configuration
--------------------
``testgres`` ships with sensible defaults. Adjust them as needed with ``default_conf()`` and ``append_conf()``:
.. code-block:: python
extra_conf = "shared_preload_libraries = 'postgres_fdw'"
with testgres.get_new_node().init() as master:
master.default_conf(fsync=True, allow_streaming=True)
master.append_conf('postgresql.conf', extra_conf)
``default_conf()`` is called by ``init()`` and rewrites the configuration file. Apply ``append_conf()`` afterwards to keep custom lines.
Remote mode
-----------
Provision nodes on a remote host (Linux only) by wiring ``RemoteOperations`` into the configuration:
.. code-block:: python
from testgres import ConnectionParams, RemoteOperations, TestgresConfig, get_remote_node
conn_params = ConnectionParams(
host='example.com',
username='postgres',
ssh_key='/path/to/ssh/key'
)
os_ops = RemoteOperations(conn_params)
TestgresConfig.set_os_ops(os_ops=os_ops)
def test_basic_query():
with get_remote_node(conn_params=conn_params) as node:
node.init().start()
assert node.execute('SELECT 1') == [(1,)]
Pytest integration
------------------
Use fixtures to create and clean up nodes automatically when testing with ``pytest``:
.. code-block:: python
import pytest
import testgres
@pytest.fixture
def pg_node():
node = testgres.get_new_node().init().start()
try:
yield node
finally:
node.stop()
node.cleanup()
def test_simple(pg_node):
assert pg_node.execute('select 1')[0][0] == 1
Scaling tips
------------
* Run tests in parallel with ``pytest -n auto`` (requires ``pytest-xdist``). Set unique ports by passing ``port`` to ``get_new_node()`` or exporting ``PGPORT`` in the fixture.
* Always call ``node.cleanup()`` after each test, or rely on context managers/fixtures that do it for you, to avoid leftover data directories.
* Prefer ``safe_psql()`` for quick assertions, and ``execute()`` when you need Python data structures.
Modules
=======
.. toctree::
:maxdepth: 2
testgres
.. Indices and tables
.. ==================
.. * :ref:`genindex`
.. * :ref:`modindex`
.. * :ref:`search`
================================================
FILE: docs/source/testgres.rst
================================================
testgres package
================
testgres.api
------------
.. automodule:: testgres.api
:members:
:undoc-members:
:show-inheritance:
testgres.backup
---------------
.. automodule:: testgres.backup
:members:
:undoc-members:
:show-inheritance:
testgres.config
---------------
.. automodule:: testgres.config
:members:
:undoc-members:
:show-inheritance:
:member-order: groupwise
testgres.connection
-------------------
.. automodule:: testgres.connection
:members:
:undoc-members:
:show-inheritance:
testgres.enums
--------------
.. automodule:: testgres.enums
:members:
:undoc-members:
:show-inheritance:
testgres.exceptions
-------------------
.. automodule:: testgres.exceptions
:members:
:undoc-members:
:show-inheritance:
testgres.node
-------------
.. autoclass:: testgres.node.PostgresNode
:members:
.. automethod:: __init__
.. autoclass:: testgres.node.ProcessProxy
:members:
testgres.standby
----------------
.. automodule:: testgres.standby
:members:
:undoc-members:
:show-inheritance:
testgres.pubsub
---------------
.. automodule:: testgres.pubsub
.. autoclass:: testgres.node.Publication
:members:
.. automethod:: __init__
.. autoclass:: testgres.node.Subscription
:members:
.. automethod:: __init__
================================================
FILE: pyproject.toml
================================================
[build-system]
requires = ["setuptools>=61.0"]
build-backend = "setuptools.build_meta"
[tool.setuptools.package-dir]
"testgres" = "src"
[tool.setuptools.dynamic]
version = {attr = "testgres.__version__"}
[tool.flake8]
extend-ignore = ["E501"]
exclude = [".git", "__pycache__", "env", "venv"]
# Pytest settings
[tool.pytest.ini_options]
testpaths = ["tests"]
log_file_level = "NOTSET"
log_file_format = "%(levelname)8s [%(asctime)s] %(message)s"
log_file_date_format = "%Y-%m-%d %H:%M:%S"
[project]
name = "testgres"
dynamic = ["version"]
description = "Testing utility for PostgreSQL and its extensions"
readme = "README.md"
# [2026-01-05]
# This old format is used to ensure compatibility with Python 3.7.
license = {text = "PostgreSQL"}
authors = [
{name = "Postgres Professional", email = "testgres@postgrespro.ru"},
]
keywords = [
'test',
'testing',
'postgresql',
]
requires-python = ">=3.7.3"
classifiers = [
"Intended Audience :: Developers",
"Operating System :: Unix",
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Programming Language :: Python :: 3.14",
"Topic :: Software Development :: Libraries",
"Topic :: Software Development :: Testing",
]
dependencies = [
"pg8000",
"port-for>=0.4",
"six>=1.9.0",
"psutil",
"packaging",
"testgres.os_ops>=2.1.0,<3.0.0",
]
[project.urls]
"HomePage" = "https://github.com/postgrespro/testgres"
================================================
FILE: run_tests.sh
================================================
#!/usr/bin/env bash
set -eux
if [ -z ${TEST_FILTER+x} ]; \
then export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote))"; \
fi
# fail early
echo check that pg_config is in PATH
command -v pg_config
# prepare python environment
VENV_PATH="/tmp/testgres_venv"
rm -rf $VENV_PATH
${PYTHON_BINARY} -m venv "${VENV_PATH}"
export VIRTUAL_ENV_DISABLE_PROMPT=1
source "${VENV_PATH}/bin/activate"
pip install --upgrade pip setuptools wheel
pip install -r tests/requirements.txt
# remove existing coverage file
export COVERAGE_FILE=.coverage
rm -f $COVERAGE_FILE
pip install coverage
# run tests (PATH)
time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}"
# run tests (PG_BIN)
PG_BIN=$(pg_config --bindir) \
time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}"
# run tests (PG_CONFIG)
PG_CONFIG=$(pg_config --bindir)/pg_config \
time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}"
# test pg8000
pip uninstall -y psycopg2
pip install pg8000
PG_CONFIG=$(pg_config --bindir)/pg_config \
time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}"
# show coverage
coverage report
pip uninstall -y coverage
# build documentation
pip install Sphinx
cd docs
make html
cd ..
pip uninstall -y Sphinx
# attempt to fix codecov
set +eux
# send coverage stats to Codecov
bash <(curl -s https://codecov.io/bash)
================================================
FILE: run_tests2.sh
================================================
#!/usr/bin/env bash
set -eux
eval "$(pyenv init -)"
eval "$(pyenv virtualenv-init -)"
pyenv virtualenv --force ${PYTHON_VERSION} cur
pyenv activate cur
./run_tests.sh
================================================
FILE: src/__init__.py
================================================
from .api import get_new_node, get_remote_node
from .backup import NodeBackup
from .config import \
TestgresConfig, \
configure_testgres, \
scoped_config, \
push_config, \
pop_config
from .connection import \
NodeConnection, \
DatabaseError, \
InternalError, \
ProgrammingError, \
OperationalError
from .exceptions import \
TestgresException, \
ExecUtilException, \
QueryException, \
QueryTimeoutException, \
TimeoutException, \
CatchUpException, \
StartNodeException, \
InitNodeException, \
BackupException, \
InvalidOperationException
from .enums import \
XLogMethod, \
IsolationLevel, \
NodeStatus, \
ProcessType, \
DumpFormat
from .node import PostgresNode
from .node import PortManager
from .node_app import NodeApp
from .utils import \
reserve_port, \
release_port, \
bound_ports, \
get_bin_path, \
get_pg_config, \
get_pg_version
from .standby import \
First, \
Any
from .config import testgres_config
from testgres.operations.os_ops import OsOperations, ConnectionParams
from testgres.operations.local_ops import LocalOperations
from testgres.operations.remote_ops import RemoteOperations
__version__ = "1.13.7"
__all__ = [
"get_new_node",
"get_remote_node",
"NodeBackup", "testgres_config",
"TestgresConfig", "configure_testgres", "scoped_config", "push_config", "pop_config",
"NodeConnection", "DatabaseError", "InternalError", "ProgrammingError", "OperationalError",
"TestgresException", "ExecUtilException", "QueryException",
QueryTimeoutException.__name__,
"TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException",
"XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat",
NodeApp.__name__,
PostgresNode.__name__,
PortManager.__name__,
"reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version",
"First", "Any",
"OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams"
]
================================================
FILE: src/api.py
================================================
# coding: utf-8
"""
Testing framework for PostgreSQL and its extensions
This module was created under influence of Postgres TAP test feature
(PostgresNode.pm module). It can manage Postgres clusters: initialize,
edit configuration files, start/stop cluster, execute queries. The
typical flow may look like:
>>> with get_new_node() as node:
... node.init().start()
... result = node.safe_psql('postgres', 'select 1')
... print(result.decode('utf-8').strip())
... node.stop()
PostgresNode(name='...', port=..., base_dir='...')
1
PostgresNode(name='...', port=..., base_dir='...')
Or:
>>> with get_new_node() as master:
... master.init().start()
... with master.backup() as backup:
... with backup.spawn_replica() as replica:
... replica = replica.start()
... master.execute('postgres', 'create table test (val int4)')
... master.execute('postgres', 'insert into test values (0), (1), (2)')
... replica.catchup() # wait until changes are visible
... print(replica.execute('postgres', 'select count(*) from test'))
PostgresNode(name='...', port=..., base_dir='...')
[(3,)]
"""
from .node import PostgresNode
def get_new_node(name=None, base_dir=None, **kwargs):
"""
Simply a wrapper around :class:`.PostgresNode` constructor.
See :meth:`.PostgresNode.__init__` for details.
"""
# NOTE: leave explicit 'name' and 'base_dir' for compatibility
return PostgresNode(name=name, base_dir=base_dir, **kwargs)
def get_remote_node(name=None, conn_params=None):
"""
Simply a wrapper around :class:`.PostgresNode` constructor for remote node.
See :meth:`.PostgresNode.__init__` for details.
For remote connection you can add the next parameter:
conn_params = ConnectionParams(host='127.0.0.1', ssh_key=None, username=default_username())
"""
return get_new_node(name=name, conn_params=conn_params)
================================================
FILE: src/backup.py
================================================
# coding: utf-8
from six import raise_from
from .enums import XLogMethod
from .consts import \
DATA_DIR, \
TMP_NODE, \
TMP_BACKUP, \
PG_CONF_FILE, \
BACKUP_LOG_FILE
from .exceptions import BackupException
from testgres.operations.os_ops import OsOperations
from .utils import \
get_bin_path2, \
execute_utility2, \
clean_on_error
class NodeBackup(object):
"""
Smart object responsible for backups
"""
@property
def log_file(self):
assert self.os_ops is not None
assert isinstance(self.os_ops, OsOperations)
return self.os_ops.build_path(self.base_dir, BACKUP_LOG_FILE)
def __init__(self,
node,
base_dir=None,
username=None,
xlog_method=XLogMethod.fetch,
options=None):
"""
Create a new backup.
Args:
node: :class:`.PostgresNode` we're going to backup.
base_dir: where should we store it?
username: database user name.
xlog_method: none | fetch | stream (see docs)
"""
assert node.os_ops is not None
assert isinstance(node.os_ops, OsOperations)
if not options:
options = []
self.os_ops = node.os_ops
if not node.status():
raise BackupException('Node must be running')
# Check arguments
if not isinstance(xlog_method, XLogMethod):
try:
xlog_method = XLogMethod(xlog_method)
except ValueError:
msg = 'Invalid xlog_method "{}"'.format(xlog_method)
raise BackupException(msg)
# Set default arguments
username = username or self.os_ops.get_user()
base_dir = base_dir or self.os_ops.mkdtemp(prefix=TMP_BACKUP)
# public
self.original_node = node
self.base_dir = base_dir
self.username = username
# private
self._available = True
data_dir = self.os_ops.build_path(self.base_dir, DATA_DIR)
_params = [
get_bin_path2(self.os_ops, "pg_basebackup"),
"-p", str(node.port),
"-h", node.host,
"-U", username,
"-D", data_dir,
"-X", xlog_method.value
] # yapf: disable
_params += options
execute_utility2(self.os_ops, _params, self.log_file)
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.cleanup()
def _prepare_dir(self, destroy):
"""
Provide a data directory for a copy of node.
Args:
destroy: should we convert this backup into a node?
Returns:
Path to data directory.
"""
if not self._available:
raise BackupException('Backup is exhausted')
# Do we want to use this backup several times?
available = not destroy
if available:
assert self.os_ops is not None
assert isinstance(self.os_ops, OsOperations)
dest_base_dir = self.os_ops.mkdtemp(prefix=TMP_NODE)
data1 = self.os_ops.build_path(self.base_dir, DATA_DIR)
data2 = self.os_ops.build_path(dest_base_dir, DATA_DIR)
try:
# Copy backup to new data dir
self.os_ops.copytree(data1, data2)
except Exception as e:
raise_from(BackupException('Failed to copy files'), e)
else:
dest_base_dir = self.base_dir
# Is this backup exhausted?
self._available = available
# Return path to new node
return dest_base_dir
def spawn_primary(self, name=None, destroy=True):
"""
Create a primary node from a backup.
Args:
name: primary's application name.
destroy: should we convert this backup into a node?
Returns:
New instance of :class:`.PostgresNode`.
"""
# Prepare a data directory for this node
base_dir = self._prepare_dir(destroy)
# Build a new PostgresNode
assert self.original_node is not None
if (hasattr(self.original_node, "clone_with_new_name_and_base_dir")):
node = self.original_node.clone_with_new_name_and_base_dir(name=name, base_dir=base_dir)
else:
# For backward compatibility
NodeClass = self.original_node.__class__
node = NodeClass(name=name, base_dir=base_dir, conn_params=self.original_node.os_ops.conn_params)
assert node is not None
assert type(node) is self.original_node.__class__
with clean_on_error(node) as node:
# Set a new port
node.append_conf(filename=PG_CONF_FILE, line='\n')
node.append_conf(filename=PG_CONF_FILE, port=node.port)
return node
def spawn_replica(self, name=None, destroy=True, slot=None):
"""
Create a replica of the original node from a backup.
Args:
name: replica's application name.
slot: create a replication slot with the specified name.
destroy: should we convert this backup into a node?
Returns:
New instance of :class:`.PostgresNode`.
"""
# Build a new PostgresNode
node = self.spawn_primary(name=name, destroy=destroy)
assert node is not None
try:
# Assign it a master and a recovery file (private magic)
node._assign_master(self.original_node)
node._create_recovery_conf(username=self.username, slot=slot)
except: # noqa: E722
# TODO: Pass 'final=True' ?
node.cleanup(release_resources=True)
raise
return node
def cleanup(self):
"""
Remove all files that belong to this backup.
No-op if it's been converted to a PostgresNode (destroy=True).
"""
if self._available:
self._available = False
self.os_ops.rmdirs(self.base_dir, ignore_errors=True)
================================================
FILE: src/cache.py
================================================
# coding: utf-8
from six import raise_from
from .config import testgres_config
from .consts import XLOG_CONTROL_FILE
from .defaults import generate_system_id
from .exceptions import \
InitNodeException, \
ExecUtilException
from .utils import \
get_bin_path2, \
execute_utility2
from testgres.operations.local_ops import LocalOperations
from testgres.operations.os_ops import OsOperations
def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = None, bin_path=None, cached=True):
"""
Perform initdb or use cached node files.
"""
assert os_ops is None or isinstance(os_ops, OsOperations)
if os_ops is None:
os_ops = LocalOperations.get_single_instance()
assert isinstance(os_ops, OsOperations)
def make_utility_path(name):
assert name is not None
assert type(name) is str
if bin_path:
return os_ops.build_path(bin_path, name)
return get_bin_path2(os_ops, name)
def call_initdb(initdb_dir, log=logfile):
try:
initdb_path = make_utility_path("initdb")
_params = [initdb_path, "-D", initdb_dir, "-N"]
execute_utility2(os_ops, _params + (params or []), log)
except ExecUtilException as e:
raise_from(InitNodeException("Failed to run initdb"), e)
if params or not testgres_config.cache_initdb or not cached:
call_initdb(data_dir, logfile)
else:
# Fetch cached initdb dir
cached_data_dir = testgres_config.cached_initdb_dir
# Initialize cached initdb
if not os_ops.path_exists(cached_data_dir) or \
not os_ops.listdir(cached_data_dir):
call_initdb(cached_data_dir)
try:
# Copy cached initdb to current data dir
os_ops.copytree(cached_data_dir, data_dir)
# Assign this node a unique system id if asked to
if testgres_config.cached_initdb_unique:
# XXX: write new unique system id to control file
# Some users might rely upon unique system ids, but
# our initdb caching mechanism breaks this contract.
pg_control = os_ops.build_path(data_dir, XLOG_CONTROL_FILE)
system_id = generate_system_id()
cur_pg_control = os_ops.read(pg_control, binary=True)
new_pg_control = system_id + cur_pg_control[len(system_id):]
os_ops.write(pg_control, new_pg_control, truncate=True, binary=True, read_and_write=True)
# XXX: build new WAL segment with our system id
_params = [make_utility_path("pg_resetwal"), "-D", data_dir, "-f"]
execute_utility2(os_ops, _params, logfile)
except ExecUtilException as e:
msg = "Failed to reset WAL for system id"
raise_from(InitNodeException(msg), e)
except Exception as e:
raise_from(InitNodeException("Failed to spawn a node"), e)
================================================
FILE: src/config.py
================================================
# coding: utf-8
import atexit
import copy
import logging
import os
import tempfile
from contextlib import contextmanager
from .consts import TMP_CACHE
from testgres.operations.os_ops import OsOperations
from testgres.operations.local_ops import LocalOperations
log_level = os.getenv('LOGGING_LEVEL', 'WARNING').upper()
log_format = os.getenv('LOGGING_FORMAT', '%(asctime)s - %(levelname)s - %(message)s')
logging.basicConfig(level=log_level, format=log_format)
class GlobalConfig(object):
"""
Global configuration object which allows user to override default settings.
"""
# NOTE: attributes must not be callable or begin with __.
cache_initdb = True
""" shall we use cached initdb instance? """
cached_initdb_unique = False
""" shall we give new node a unique system id? """
cache_pg_config = True
""" shall we cache pg_config results? """
use_python_logging = False
""" enable python logging subsystem (see logger.py). """
error_log_lines = 20
""" N of log lines to be shown in exceptions (0=inf). """
node_cleanup_full = True
""" shall we remove EVERYTHING (including logs)? """
node_cleanup_on_good_exit = True
""" remove base_dir on nominal __exit__(). """
node_cleanup_on_bad_exit = False
""" remove base_dir on __exit__() via exception. """
_cached_initdb_dir = None
""" underlying class attribute for cached_initdb_dir property """
os_ops = LocalOperations.get_single_instance()
""" OsOperation object that allows work on remote host """
@property
def cached_initdb_dir(self):
""" path to a temp directory for cached initdb. """
return self._cached_initdb_dir
@cached_initdb_dir.setter
def cached_initdb_dir(self, value):
self._cached_initdb_dir = value
if value:
cached_initdb_dirs.add(value)
return testgres_config.cached_initdb_dir
@property
def temp_dir(self):
""" path to temp dir containing nodes with default 'base_dir'. """
return tempfile.tempdir
@temp_dir.setter
def temp_dir(self, value):
tempfile.tempdir = value
def __init__(self, **options):
self.update(options)
def __setitem__(self, key, value):
setattr(self, key, value)
def __getitem__(self, key):
return getattr(self, key)
def __setattr__(self, name, value):
if name not in self.keys():
raise TypeError('Unknown option {}'.format(name))
super(GlobalConfig, self).__setattr__(name, value)
def keys(self):
"""
Return a list of all available settings.
"""
keys = []
for key in dir(GlobalConfig):
if not key.startswith('__') and not callable(self[key]):
keys.append(key)
return keys
def items(self):
"""
Return setting-value pairs.
"""
return ((key, self[key]) for key in self.keys())
def update(self, config):
"""
Extract setting-value pairs from 'config' and
assign those values to corresponding settings
of this GlobalConfig object.
"""
for key, value in config.items():
self[key] = value
return self
def copy(self):
"""
Return a copy of this object.
"""
return copy.copy(self)
@staticmethod
def set_os_ops(os_ops: OsOperations):
testgres_config.os_ops = os_ops
testgres_config.cached_initdb_dir = os_ops.mkdtemp(prefix=TMP_CACHE)
# cached dirs to be removed
cached_initdb_dirs = set()
# default config object
testgres_config = GlobalConfig()
# NOTE: for compatibility
TestgresConfig = testgres_config
# stack of GlobalConfigs
config_stack = []
@atexit.register
def _rm_cached_initdb_dirs():
for d in cached_initdb_dirs:
testgres_config.os_ops.rmdirs(d, ignore_errors=True)
def push_config(**options):
"""
Permanently set custom GlobalConfig options and
put previous settings on top of the config stack.
"""
# push current config to stack
config_stack.append(testgres_config.copy())
return testgres_config.update(options)
def pop_config():
"""
Set previous GlobalConfig options from stack.
"""
if len(config_stack) == 0:
raise IndexError('Reached initial config')
# restore popped config
return testgres_config.update(config_stack.pop())
@contextmanager
def scoped_config(**options):
"""
Temporarily set custom GlobalConfig options for this context.
Previous options are pushed to the config stack.
Example:
>>> from .api import get_new_node
>>> with scoped_config(cache_initdb=False):
... # create a new node with fresh initdb
... with get_new_node().init().start() as node:
... print(node.execute('select 1'))
[(1,)]
"""
try:
# set a new config with options
config = push_config(**options)
# return it
yield config
finally:
# restore previous config
pop_config()
def configure_testgres(**options):
"""
Adjust current global options.
Look at the GlobalConfig to learn about existing settings.
"""
testgres_config.update(options)
# NOTE: assign initial cached dir for initdb
testgres_config.cached_initdb_dir = testgres_config.os_ops.mkdtemp(prefix=TMP_CACHE)
================================================
FILE: src/connection.py
================================================
# coding: utf-8
import logging
# we support both pg8000 and psycopg2
try:
import psycopg2 as pglib
except ImportError:
try:
import pg8000 as pglib
except ImportError:
raise ImportError("You must have psycopg2 or pg8000 modules installed")
from .enums import IsolationLevel
from .defaults import \
default_dbname, \
default_username
from .exceptions import QueryException
# export some exceptions
DatabaseError = pglib.DatabaseError
InternalError = pglib.InternalError
ProgrammingError = pglib.ProgrammingError
OperationalError = pglib.OperationalError
class NodeConnection(object):
"""
Transaction wrapper returned by Node
"""
def __init__(self,
node,
dbname=None,
username=None,
password=None,
autocommit=False):
# Set default arguments
dbname = dbname or default_dbname()
username = username or default_username()
self._node = node
self._connection = pglib.connect(
database=dbname,
user=username,
password=password,
host=node.host,
port=node.port
)
self._connection.autocommit = autocommit
self._cursor = self.connection.cursor()
@property
def node(self):
return self._node
@property
def connection(self):
return self._connection
@property
def pid(self):
return self.execute("select pg_catalog.pg_backend_pid()")[0][0]
@property
def cursor(self):
return self._cursor
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.close()
def begin(self, isolation_level=IsolationLevel.ReadCommitted):
# Check if level isn't an IsolationLevel
if not isinstance(isolation_level, IsolationLevel):
# Get name of isolation level
level_str = str(isolation_level).lower()
# Validate level string
try:
isolation_level = IsolationLevel(level_str)
except ValueError:
error = 'Invalid isolation level "{}"'
raise QueryException(error.format(level_str))
# Set isolation level
cmd = 'SET TRANSACTION ISOLATION LEVEL {}'
self.cursor.execute(cmd.format(isolation_level.value))
return self
def commit(self):
self.connection.commit()
return self
def rollback(self):
self.connection.rollback()
return self
def execute(self, query, *args):
self.cursor.execute(query, args)
try:
# pg8000 might return tuples
res = [tuple(t) for t in self.cursor.fetchall()]
return res
except ProgrammingError:
return None
except Exception as e:
logging.error("Error executing query: {}\n {}".format(repr(e), query))
return None
def close(self):
self.cursor.close()
self.connection.close()
================================================
FILE: src/consts.py
================================================
# coding: utf-8
# names for dirs in base_dir
DATA_DIR = "data"
LOGS_DIR = "logs"
# prefixes for temp dirs
TMP_NODE = 'tgsn_'
TMP_DUMP = 'tgsd_'
TMP_CACHE = 'tgsc_'
TMP_BACKUP = 'tgsb_'
# path to control file
XLOG_CONTROL_FILE = "global/pg_control"
# names for config files
RECOVERY_CONF_FILE = "recovery.conf"
PG_AUTO_CONF_FILE = "postgresql.auto.conf"
PG_CONF_FILE = "postgresql.conf"
PG_PID_FILE = 'postmaster.pid'
HBA_CONF_FILE = "pg_hba.conf"
# names for log files
PG_LOG_FILE = "postgresql.log"
UTILS_LOG_FILE = "utils.log"
BACKUP_LOG_FILE = "backup.log"
# defaults for node settings
MAX_LOGICAL_REPLICATION_WORKERS = 5
MAX_REPLICATION_SLOTS = 10
MAX_WORKER_PROCESSES = 10
WAL_KEEP_SEGMENTS = 20
WAL_KEEP_SIZE = 320
MAX_WAL_SENDERS = 10
# logical replication settings
LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS = 60
PG_CTL__STATUS__OK = 0
PG_CTL__STATUS__NODE_IS_STOPPED = 3
PG_CTL__STATUS__BAD_DATADIR = 4
================================================
FILE: src/decorators.py
================================================
import six
import functools
def positional_args_hack(*special_cases):
"""
Convert positional args described by
'special_cases' into named args.
Example:
@positional_args_hack(['abc'], ['def', 'abc'])
def some_api_func(...)
This is useful for compatibility.
"""
cases = dict()
for case in special_cases:
k = len(case)
assert k not in six.iterkeys(cases), 'len must be unique'
cases[k] = case
def decorator(function):
@functools.wraps(function)
def wrapper(*args, **kwargs):
k = len(args)
if k in six.iterkeys(cases):
case = cases[k]
for i in range(0, k):
arg_name = case[i]
arg_val = args[i]
# transform into named
kwargs[arg_name] = arg_val
# get rid of them
args = []
return function(*args, **kwargs)
return wrapper
return decorator
def method_decorator(decorator):
"""
Convert a function decorator into a method decorator.
"""
def _dec(func):
def _wrapper(self, *args, **kwargs):
@decorator
def bound_func(*args2, **kwargs2):
return func.__get__(self, type(self))(*args2, **kwargs2)
# 'bound_func' is a closure and can see 'self'
return bound_func(*args, **kwargs)
# preserve docs
functools.update_wrapper(_wrapper, func)
return _wrapper
# preserve docs
functools.update_wrapper(_dec, decorator)
# change name for easier debugging
_dec.__name__ = 'method_decorator({})'.format(decorator.__name__)
return _dec
================================================
FILE: src/defaults.py
================================================
import datetime
import struct
import uuid
from .config import testgres_config as tconf
def default_dbname():
"""
Return default DB name.
"""
return 'postgres'
def default_username():
"""
Return default username (current user).
"""
return tconf.os_ops.get_user()
def generate_app_name():
"""
Generate a new application name for node.
"""
return 'testgres-{}'.format(str(uuid.uuid4()))
def generate_system_id():
"""
Generate a new 64-bit unique system identifier for node.
"""
date1 = datetime.datetime.utcfromtimestamp(0)
date2 = datetime.datetime.utcnow()
secs = int((date2 - date1).total_seconds())
usecs = date2.microsecond
# see pg_resetwal.c : GuessControlValues()
system_id = 0
system_id |= (secs << 32)
system_id |= (usecs << 12)
system_id |= (tconf.os_ops.get_pid() & 0xFFF)
# pack ULL in native byte order
return struct.pack('=Q', system_id)
================================================
FILE: src/enums.py
================================================
from enum import Enum, IntEnum
from six import iteritems
from psutil import NoSuchProcess
class XLogMethod(Enum):
"""
Available WAL methods for :class:`.NodeBackup`
"""
none = 'none'
fetch = 'fetch'
stream = 'stream'
class IsolationLevel(Enum):
"""
Transaction isolation level for :class:`.NodeConnection`
"""
ReadUncommitted = 'read uncommitted'
ReadCommitted = 'read committed'
RepeatableRead = 'repeatable read'
Serializable = 'serializable'
class NodeStatus(IntEnum):
"""
Status of a PostgresNode
"""
Running, Stopped, Uninitialized = range(3)
# for Python 3.x
def __bool__(self):
return self == NodeStatus.Running
# for Python 2.x
__nonzero__ = __bool__
class ProcessType(Enum):
"""
Types of processes
"""
AutovacuumLauncher = 'autovacuum launcher'
BackgroundWriter = 'background writer'
Checkpointer = 'checkpointer'
LogicalReplicationLauncher = 'logical replication launcher'
Startup = 'startup'
StatsCollector = 'stats collector'
WalReceiver = 'wal receiver'
WalSender = 'wal sender'
WalWriter = 'wal writer'
# special value
Unknown = 'unknown'
@staticmethod
def from_process(process):
# legacy names for older releases of PG
alternative_names = {
ProcessType.LogicalReplicationLauncher: [
'logical replication worker'
],
ProcessType.BackgroundWriter: [
'writer'
],
} # yapf: disable
try:
cmdline = ''.join(process.cmdline())
except (FileNotFoundError, ProcessLookupError, NoSuchProcess):
return ProcessType.Unknown
# we deliberately cut special words and spaces
cmdline = cmdline.replace('postgres:', '', 1) \
.replace('bgworker:', '', 1) \
.replace(' ', '')
for ptype in ProcessType:
if cmdline.startswith(ptype.value.replace(' ', '')):
return ptype
for ptype, names in iteritems(alternative_names):
for name in names:
if cmdline.startswith(name.replace(' ', '')):
return ptype
# default
return ProcessType.Unknown
class DumpFormat(Enum):
"""
Available dump formats
"""
Plain = 'plain'
Custom = 'custom'
Directory = 'directory'
Tar = 'tar'
================================================
FILE: src/exceptions.py
================================================
# coding: utf-8
import six
import typing
from testgres.operations.exceptions import TestgresException
from testgres.operations.exceptions import ExecUtilException
from testgres.operations.exceptions import InvalidOperationException
class PortForException(TestgresException):
_message: typing.Optional[str]
def __init__(
self,
message: typing.Optional[str] = None,
):
assert message is None or type(message) is str
super().__init__(message)
self._message = message
return
@property
def message(self) -> str:
assert self._message is None or type(self._message) is str
if self._message is None:
return ""
return self._message
def __repr__(self) -> str:
args = []
if self._message is not None:
args.append(("message", self._message))
result = "{}(".format(type(self).__name__)
sep = ""
for a in args:
result += sep + a[0] + "=" + repr(a[1])
sep = ", "
continue
result += ")"
return result
@six.python_2_unicode_compatible
class QueryException(TestgresException):
_description: typing.Optional[str]
_query: typing.Optional[str]
def __init__(
self,
message: typing.Optional[str] = None,
query: typing.Optional[str] = None
):
assert message is None or type(message) is str
assert query is None or type(query) is str
super().__init__(message)
self._description = message
self._query = query
return
@property
def message(self) -> str:
assert self._description is None or type(self._description) is str
assert self._query is None or type(self._query) is str
msg = []
if self._description:
msg.append(self._description)
if self._query:
msg.append(u'Query: {}'.format(self._query))
r = six.text_type('\n').join(msg)
assert type(r) is str
return r
@property
def description(self) -> typing.Optional[str]:
assert self._description is None or type(self._description) is str
return self._description
@property
def query(self) -> typing.Optional[str]:
assert self._query is None or type(self._query) is str
return self._query
def __repr__(self) -> str:
args = []
if self._description is not None:
args.append(("message", self._description))
if self._query is not None:
args.append(("query", self._query))
result = "{}(".format(type(self).__name__)
sep = ""
for a in args:
result += sep + a[0] + "=" + repr(a[1])
sep = ", "
continue
result += ")"
return result
class QueryTimeoutException(QueryException):
def __init__(
self,
message: typing.Optional[str] = None,
query: typing.Optional[str] = None
):
assert message is None or type(message) is str
assert query is None or type(query) is str
super().__init__(message, query)
return
# [2026-01-10] To backward compatibility.
TimeoutException = QueryTimeoutException
# [2026-01-10] It inherits TestgresException now, not QueryException
class CatchUpException(TestgresException):
_message: typing.Optional[str]
def __init__(
self,
message: typing.Optional[str] = None,
):
assert message is None or type(message) is str
super().__init__(message)
self._message = message
return
@property
def message(self) -> str:
assert self._message is None or type(self._message) is str
if self._message is None:
return ""
return self._message
def __repr__(self) -> str:
args = []
if self._message is not None:
args.append(("message", self._message))
result = "{}(".format(type(self).__name__)
sep = ""
for a in args:
result += sep + a[0] + "=" + repr(a[1])
sep = ", "
continue
result += ")"
return result
@six.python_2_unicode_compatible
class StartNodeException(TestgresException):
_description: typing.Optional[str]
_files: typing.Optional[typing.Iterable]
def __init__(
self,
message: typing.Optional[str] = None,
files: typing.Optional[typing.Iterable] = None
):
assert message is None or type(message) is str
assert files is None or isinstance(files, typing.Iterable)
super().__init__(message)
self._description = message
self._files = files
return
@property
def message(self) -> str:
assert self._description is None or type(self._description) is str
assert self._files is None or isinstance(self._files, typing.Iterable)
msg = []
if self._description:
msg.append(self._description)
for f, lines in self._files or []:
assert type(f) is str
assert type(lines) in [str, bytes]
msg.append(u'{}\n----\n{}\n'.format(f, lines))
return six.text_type('\n').join(msg)
@property
def description(self) -> typing.Optional[str]:
assert self._description is None or type(self._description) is str
return self._description
@property
def files(self) -> typing.Optional[typing.Iterable]:
assert self._files is None or isinstance(self._files, typing.Iterable)
return self._files
def __repr__(self) -> str:
args = []
if self._description is not None:
args.append(("message", self._description))
if self._files is not None:
args.append(("files", self._files))
result = "{}(".format(type(self).__name__)
sep = ""
for a in args:
result += sep + a[0] + "=" + repr(a[1])
sep = ", "
continue
result += ")"
return result
class InitNodeException(TestgresException):
_message: typing.Optional[str]
def __init__(
self,
message: typing.Optional[str] = None,
):
assert message is None or type(message) is str
super().__init__(message)
self._message = message
return
@property
def message(self) -> str:
assert self._message is None or type(self._message) is str
if self._message is None:
return ""
return self._message
def __repr__(self) -> str:
args = []
if self._message is not None:
args.append(("message", self._message))
result = "{}(".format(type(self).__name__)
sep = ""
for a in args:
result += sep + a[0] + "=" + repr(a[1])
sep = ", "
continue
result += ")"
return result
class BackupException(TestgresException):
_message: typing.Optional[str]
def __init__(
self,
message: typing.Optional[str] = None,
):
assert message is None or type(message) is str
super().__init__(message)
self._message = message
return
@property
def message(self) -> str:
assert self._message is None or type(self._message) is str
if self._message is None:
return ""
return self._message
def __repr__(self) -> str:
args = []
if self._message is not None:
args.append(("message", self._message))
result = "{}(".format(type(self).__name__)
sep = ""
for a in args:
result += sep + a[0] + "=" + repr(a[1])
sep = ", "
continue
result += ")"
return result
assert ExecUtilException.__name__ == "ExecUtilException"
assert InvalidOperationException.__name__ == "InvalidOperationException"
================================================
FILE: src/impl/internal_utils.py
================================================
import logging
def send_log(level: int, msg: str) -> None:
assert type(level) is int
assert type(msg) is str
return logging.log(level, "[testgres] " + msg)
def send_log_info(msg: str) -> None:
assert type(msg) is str
return send_log(logging.INFO, msg)
def send_log_debug(msg: str) -> None:
assert type(msg) is str
return send_log(logging.DEBUG, msg)
================================================
FILE: src/impl/platforms/internal_platform_utils.py
================================================
from __future__ import annotations
import enum
import typing
from testgres.operations.os_ops import OsOperations
class InternalPlatformUtils:
class FindPostmasterResultCode(enum.Enum):
ok = 0
not_found = 1,
not_implemented = 2
many_processes = 3
has_problems = 4
class FindPostmasterResult:
code: InternalPlatformUtils.FindPostmasterResultCode
pid: typing.Optional[int]
def __init__(
self,
code: InternalPlatformUtils.FindPostmasterResultCode,
pid: typing.Optional[int]
):
assert type(code) is InternalPlatformUtils.FindPostmasterResultCode
assert pid is None or type(pid) is int
self.code = code
self.pid = pid
return
@staticmethod
def create_ok(pid: int) -> InternalPlatformUtils.FindPostmasterResult:
assert type(pid) is int
return __class__(InternalPlatformUtils.FindPostmasterResultCode.ok, pid)
@staticmethod
def create_not_found() -> InternalPlatformUtils.FindPostmasterResult:
return __class__(InternalPlatformUtils.FindPostmasterResultCode.not_found, None)
@staticmethod
def create_not_implemented() -> InternalPlatformUtils.FindPostmasterResult:
return __class__(InternalPlatformUtils.FindPostmasterResultCode.not_implemented, None)
@staticmethod
def create_many_processes() -> InternalPlatformUtils.FindPostmasterResult:
return __class__(InternalPlatformUtils.FindPostmasterResultCode.many_processes, None)
@staticmethod
def create_has_problems() -> InternalPlatformUtils.FindPostmasterResult:
return __class__(InternalPlatformUtils.FindPostmasterResultCode.has_problems, None)
def FindPostmaster(
self,
os_ops: OsOperations,
bin_dir: str,
data_dir: str
) -> FindPostmasterResult:
assert isinstance(os_ops, OsOperations)
assert type(bin_dir) is str
assert type(data_dir) is str
raise NotImplementedError("InternalPlatformUtils::FindPostmaster is not implemented.")
================================================
FILE: src/impl/platforms/internal_platform_utils_factory.py
================================================
from .internal_platform_utils import InternalPlatformUtils
from testgres.operations.os_ops import OsOperations
def create_internal_platform_utils(
os_ops: OsOperations
) -> InternalPlatformUtils:
assert isinstance(os_ops, OsOperations)
platform_name = os_ops.get_platform()
assert type(platform_name) is str
if platform_name == "linux":
from .linux import internal_platform_utils as x
return x.InternalPlatformUtils()
if platform_name == "win32":
from .win32 import internal_platform_utils as x
return x.InternalPlatformUtils()
# not implemented
return InternalPlatformUtils()
================================================
FILE: src/impl/platforms/linux/internal_platform_utils.py
================================================
from __future__ import annotations
from .. import internal_platform_utils as base
from ... import internal_utils
from testgres.operations.os_ops import OsOperations
from testgres.operations.exceptions import ExecUtilException
import re
import shlex
class InternalPlatformUtils(base.InternalPlatformUtils):
C_BASH_EXE = "/bin/bash"
sm_exec_env = {
"LANG": "en_US.UTF-8",
"LC_ALL": "en_US.UTF-8",
}
# --------------------------------------------------------------------
def FindPostmaster(
self,
os_ops: OsOperations,
bin_dir: str,
data_dir: str
) -> InternalPlatformUtils.FindPostmasterResult:
assert isinstance(os_ops, OsOperations)
assert type(bin_dir) is str
assert type(data_dir) is str
assert type(__class__.C_BASH_EXE) is str
assert type(__class__.sm_exec_env) is dict
assert len(__class__.C_BASH_EXE) > 0
assert len(bin_dir) > 0
assert len(data_dir) > 0
pg_path_e = re.escape(os_ops.build_path(bin_dir, "postgres"))
data_dir_e = re.escape(data_dir)
assert type(pg_path_e) is str
assert type(data_dir_e) is str
regexp = r"^\s*[0-9]+\s+" + pg_path_e + r"(\s+.*)?\s+\-[D]\s+" + data_dir_e + r"(\s+.*)?"
cmd = [
__class__.C_BASH_EXE,
"-c",
"ps -ewwo \"pid=,args=\" | grep -E " + shlex.quote(regexp),
]
exit_status, output_b, error_b = os_ops.exec_command(
cmd=cmd,
ignore_errors=True,
verbose=True,
exec_env=__class__.sm_exec_env,
)
assert type(output_b) is bytes
assert type(error_b) is bytes
output = output_b.decode("utf-8")
error = error_b.decode("utf-8")
assert type(output) is str
assert type(error) is str
if exit_status == 1:
return __class__.FindPostmasterResult.create_not_found()
if exit_status != 0:
errMsg = f"test command returned an unexpected exit code: {exit_status}"
raise ExecUtilException(
message=errMsg,
command=cmd,
exit_code=exit_status,
out=output,
error=error,
)
lines = output.splitlines()
assert type(lines) is list
if len(lines) == 0:
return __class__.FindPostmasterResult.create_not_found()
if len(lines) > 1:
msgs = []
msgs.append("Many processes like a postmaster are found: {}.".format(len(lines)))
for i in range(len(lines)):
assert type(lines[i]) is str
lines.append("[{}] '{}'".format(i, lines[i]))
continue
internal_utils.send_log_debug("\n".join(lines))
return __class__.FindPostmasterResult.create_many_processes()
def is_space_or_tab(ch) -> bool:
assert type(ch) is str
return ch == " " or ch == "\t"
line = lines[0]
start = 0
while start < len(line) and is_space_or_tab(line[start]):
start += 1
pos = start
while pos < len(line) and line[pos].isnumeric():
pos += 1
if pos == start:
return __class__.FindPostmasterResult.create_has_problems()
if pos != len(line) and not line[pos].isspace():
return __class__.FindPostmasterResult.create_has_problems()
pid = int(line[start:pos])
assert type(pid) is int
return __class__.FindPostmasterResult.create_ok(pid)
================================================
FILE: src/impl/platforms/win32/internal_platform_utils.py
================================================
from __future__ import annotations
from .. import internal_platform_utils as base
from testgres.operations.os_ops import OsOperations
class InternalPlatformUtils(base.InternalPlatformUtils):
def FindPostmaster(
self,
os_ops: OsOperations,
bin_dir: str,
data_dir: str
) -> InternalPlatformUtils.FindPostmasterResult:
assert isinstance(os_ops, OsOperations)
assert type(bin_dir) is str
assert type(data_dir) is str
return __class__.FindPostmasterResult.create_not_implemented()
================================================
FILE: src/impl/port_manager__generic.py
================================================
from testgres.operations.os_ops import OsOperations
from ..port_manager import PortManager
from ..exceptions import PortForException
import threading
import random
import typing
import logging
class PortManager__Generic(PortManager):
_C_MIN_PORT_NUMBER = 1024
_C_MAX_PORT_NUMBER = 65535
_os_ops: OsOperations
_guard: object
# TODO: is there better to use bitmap fot _available_ports?
_available_ports: typing.Set[int]
_reserved_ports: typing.Set[int]
def __init__(self, os_ops: OsOperations):
assert __class__._C_MIN_PORT_NUMBER <= __class__._C_MAX_PORT_NUMBER
assert os_ops is not None
assert isinstance(os_ops, OsOperations)
self._os_ops = os_ops
self._guard = threading.Lock()
self._available_ports = set(
range(__class__._C_MIN_PORT_NUMBER, __class__._C_MAX_PORT_NUMBER + 1)
)
assert len(self._available_ports) == (
(__class__._C_MAX_PORT_NUMBER - __class__._C_MIN_PORT_NUMBER) + 1
)
self._reserved_ports = set()
return
def reserve_port(self) -> int:
assert self._guard is not None
assert type(self._available_ports) is set
assert type(self._reserved_ports) is set
with self._guard:
t = tuple(self._available_ports)
assert len(t) == len(self._available_ports)
sampled_ports = random.sample(t, min(len(t), 100))
t = None
for port in sampled_ports:
assert type(port) is int
assert port not in self._reserved_ports
assert port in self._available_ports
assert port >= __class__._C_MIN_PORT_NUMBER
assert port <= __class__._C_MAX_PORT_NUMBER
if not self._os_ops.is_port_free(port):
continue
self._reserved_ports.add(port)
self._available_ports.discard(port)
assert port in self._reserved_ports
assert port not in self._available_ports
__class__.helper__send_debug_msg("Port {} is reserved.", port)
return port
raise PortForException("Can't select a port.")
def release_port(self, number: int) -> None:
assert type(number) is int
assert number >= __class__._C_MIN_PORT_NUMBER
assert number <= __class__._C_MAX_PORT_NUMBER
assert self._guard is not None
assert type(self._reserved_ports) is set
with self._guard:
assert number in self._reserved_ports
assert number not in self._available_ports
self._available_ports.add(number)
self._reserved_ports.discard(number)
assert number not in self._reserved_ports
assert number in self._available_ports
__class__.helper__send_debug_msg("Port {} is released.", number)
return
@staticmethod
def helper__send_debug_msg(msg_template: str, *args) -> None:
assert msg_template is not None
assert args is not None
assert type(msg_template) is str
assert type(args) is tuple
assert msg_template != ""
s = "[port manager] "
s += msg_template.format(*args)
logging.debug(s)
================================================
FILE: src/impl/port_manager__this_host.py
================================================
from ..port_manager import PortManager
from .. import utils
import threading
class PortManager__ThisHost(PortManager):
sm_single_instance: PortManager = None
sm_single_instance_guard = threading.Lock()
@staticmethod
def get_single_instance() -> PortManager:
assert __class__ == PortManager__ThisHost
assert __class__.sm_single_instance_guard is not None
if __class__.sm_single_instance is not None:
assert type(__class__.sm_single_instance) is __class__
return __class__.sm_single_instance
with __class__.sm_single_instance_guard:
if __class__.sm_single_instance is None:
__class__.sm_single_instance = __class__()
assert __class__.sm_single_instance is not None
assert type(__class__.sm_single_instance) is __class__
return __class__.sm_single_instance
def reserve_port(self) -> int:
return utils.reserve_port()
def release_port(self, number: int) -> None:
assert type(number) is int
return utils.release_port(number)
================================================
FILE: src/logger.py
================================================
# coding: utf-8
import logging
import select
import threading
import time
class TestgresLogger(threading.Thread):
"""
Helper class to implement reading from log files.
"""
def __init__(self, node_name, log_file_name):
threading.Thread.__init__(self)
self._node_name = node_name
self._log_file_name = log_file_name
self._stop_event = threading.Event()
self._logger = logging.getLogger(node_name)
self._logger.setLevel(logging.INFO)
def run(self):
# open log file for reading
with open(self._log_file_name, 'r') as fd:
# work until we're asked to stop
while not self._stop_event.is_set():
sleep_time = 0.1
new_lines = False
# do we have new lines?
if fd in select.select([fd], [], [], 0)[0]:
for line in fd.readlines():
line = line.strip()
if line:
new_lines = True
extra = {'node': self._node_name}
self._logger.info(line, extra=extra)
if not new_lines:
time.sleep(sleep_time)
# don't forget to clear event
self._stop_event.clear()
def stop(self, wait=True):
self._stop_event.set()
if wait:
self.join()
================================================
FILE: src/node.py
================================================
# coding: utf-8
from __future__ import annotations
import logging
import os
import signal
import subprocess
import time
import typing
try:
from collections.abc import Iterable
except ImportError:
from collections import Iterable
# we support both pg8000 and psycopg2
try:
import psycopg2 as pglib
except ImportError:
try:
import pg8000 as pglib
except ImportError:
raise ImportError("You must have psycopg2 or pg8000 modules installed")
from six import raise_from, iteritems, text_type
from .enums import \
NodeStatus, \
ProcessType, \
DumpFormat
from .cache import cached_initdb
from .config import testgres_config
from .connection import NodeConnection
from .consts import \
DATA_DIR, \
LOGS_DIR, \
TMP_NODE, \
TMP_DUMP, \
PG_CONF_FILE, \
PG_AUTO_CONF_FILE, \
HBA_CONF_FILE, \
RECOVERY_CONF_FILE, \
PG_LOG_FILE, \
UTILS_LOG_FILE
from .consts import \
MAX_LOGICAL_REPLICATION_WORKERS, \
MAX_REPLICATION_SLOTS, \
MAX_WORKER_PROCESSES, \
MAX_WAL_SENDERS, \
WAL_KEEP_SEGMENTS, \
WAL_KEEP_SIZE
from .decorators import \
method_decorator, \
positional_args_hack
from .defaults import \
default_dbname, \
generate_app_name
from .exceptions import \
CatchUpException, \
ExecUtilException, \
QueryException, \
QueryTimeoutException, \
StartNodeException, \
TimeoutException, \
InitNodeException, \
TestgresException, \
BackupException, \
InvalidOperationException
from .port_manager import PortManager
from .impl.port_manager__this_host import PortManager__ThisHost
from .impl.port_manager__generic import PortManager__Generic
from .logger import TestgresLogger
from .pubsub import Publication, Subscription
from .standby import First
from . import utils
from .utils import \
PgVer, \
eprint, \
get_bin_path2, \
get_pg_version2, \
execute_utility2, \
options_string, \
clean_on_error
from .raise_error import RaiseError
from .backup import NodeBackup
from testgres.operations.os_ops import ConnectionParams
from testgres.operations.os_ops import OsOperations
from testgres.operations.local_ops import LocalOperations
InternalError = pglib.InternalError
ProgrammingError = pglib.ProgrammingError
OperationalError = pglib.OperationalError
assert TimeoutException == QueryTimeoutException
class ProcessProxy(object):
"""
Wrapper for psutil.Process
Attributes:
process: wrapped psutill.Process object
ptype: instance of ProcessType
"""
_process: typing.Any
_ptype: ProcessType
def __init__(self, process, ptype: typing.Optional[ProcessType] = None):
assert process is not None
assert ptype is None or type(ptype) is ProcessType
self._process = process
if ptype is not None:
self._ptype = ptype
else:
self._ptype = ProcessType.from_process(process)
assert type(self._ptype) is ProcessType
return
def __getattr__(self, name):
return getattr(self.process, name)
def __repr__(self):
return '{}(ptype={}, process={})'.format(
self.__class__.__name__,
str(self.ptype),
repr(self.process))
@property
def process(self) -> typing.Any:
assert self._process is not None
return self._process
@property
def ptype(self) -> ProcessType:
assert type(self._ptype) is ProcessType
return self._ptype
class PostgresNode(object):
# a max number of node start attempts
_C_MAX_START_ATEMPTS = 5
_C_PM_PID__IS_NOT_DETECTED = -1
_name: typing.Optional[str]
_port: typing.Optional[int]
_should_free_port: bool
_os_ops: OsOperations
_port_manager: typing.Optional[PortManager]
_manually_started_pm_pid: typing.Optional[int]
def __init__(self,
name=None,
base_dir=None,
port: typing.Optional[int] = None,
conn_params: typing.Optional[ConnectionParams] = None,
bin_dir=None,
prefix=None,
os_ops: typing.Optional[OsOperations] = None,
port_manager: typing.Optional[PortManager] = None):
"""
PostgresNode constructor.
Args:
name: node's application name.
port: port to accept connections.
base_dir: path to node's data directory.
bin_dir: path to node's binary directory.
os_ops: None or correct OS operation object.
port_manager: None or correct port manager object.
"""
assert port is None or type(port) is int
assert os_ops is None or isinstance(os_ops, OsOperations)
assert port_manager is None or isinstance(port_manager, PortManager)
if conn_params is not None:
assert type(conn_params) is ConnectionParams
raise InvalidOperationException("conn_params is deprecated, please use os_ops parameter instead.")
# private
if os_ops is None:
self._os_ops = __class__._get_os_ops()
else:
assert isinstance(os_ops, OsOperations)
self._os_ops = os_ops
pass
assert self._os_ops is not None
assert isinstance(self._os_ops, OsOperations)
self._pg_version = PgVer(get_pg_version2(self._os_ops, bin_dir))
self._base_dir = base_dir
self._bin_dir = bin_dir
self._prefix = prefix
self._logger = None
self._master = None
# basic
self._name = name or generate_app_name()
if port is not None:
assert type(port) is int
assert port_manager is None
self._port = port
self._should_free_port = False
self._port_manager = None
else:
if port_manager is None:
self._port_manager = __class__._get_port_manager(self._os_ops)
elif os_ops is None:
raise InvalidOperationException("When port_manager is not None you have to define os_ops, too.")
else:
assert isinstance(port_manager, PortManager)
assert self._os_ops is os_ops
self._port_manager = port_manager
assert self._port_manager is not None
assert isinstance(self._port_manager, PortManager)
self._port = self._port_manager.reserve_port() # raises
assert type(self._port) is int
self._should_free_port = True
assert type(self._port) is int
# defaults for __exit__()
self.cleanup_on_good_exit = testgres_config.node_cleanup_on_good_exit
self.cleanup_on_bad_exit = testgres_config.node_cleanup_on_bad_exit
self.shutdown_max_attempts = 3
# NOTE: for compatibility
self.utils_log_name = self.utils_log_file
self.pg_log_name = self.pg_log_file
# Node state
self._manually_started_pm_pid = None
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
# NOTE: Ctrl+C does not count!
got_exception = type is not None and type is not KeyboardInterrupt
c1 = self.cleanup_on_good_exit and not got_exception
c2 = self.cleanup_on_bad_exit and got_exception
attempts = self.shutdown_max_attempts
if c1 or c2:
self.cleanup(attempts)
else:
self._try_shutdown(attempts)
self._release_resources()
def __repr__(self):
return "{}(name='{}', port={}, base_dir='{}')".format(
self.__class__.__name__,
self.name,
str(self._port) if self._port is not None else "None",
self.base_dir
)
@staticmethod
def _get_os_ops() -> OsOperations:
if testgres_config.os_ops:
return testgres_config.os_ops
return LocalOperations.get_single_instance()
@staticmethod
def _get_port_manager(os_ops: OsOperations) -> PortManager:
assert os_ops is not None
assert isinstance(os_ops, OsOperations)
if os_ops is LocalOperations.get_single_instance():
assert utils._old_port_manager is not None
assert type(utils._old_port_manager) is PortManager__Generic
assert utils._old_port_manager._os_ops is os_ops
return PortManager__ThisHost.get_single_instance()
# TODO: Throw the exception "Please define a port manager." ?
return PortManager__Generic(os_ops)
def clone_with_new_name_and_base_dir(self, name: str, base_dir: str):
assert name is None or type(name) is str
assert base_dir is None or type(base_dir) is str
assert __class__ == PostgresNode
if self._port_manager is None:
raise InvalidOperationException("PostgresNode without PortManager can't be cloned.")
assert self._port_manager is not None
assert isinstance(self._port_manager, PortManager)
assert self._os_ops is not None
assert isinstance(self._os_ops, OsOperations)
node = PostgresNode(
name=name,
base_dir=base_dir,
bin_dir=self._bin_dir,
prefix=self._prefix,
os_ops=self._os_ops,
port_manager=self._port_manager)
return node
@property
def os_ops(self) -> OsOperations:
assert self._os_ops is not None
assert isinstance(self._os_ops, OsOperations)
return self._os_ops
@property
def port_manager(self) -> typing.Optional[PortManager]:
assert self._port_manager is None or isinstance(self._port_manager, PortManager)
return self._port_manager
@property
def name(self) -> str:
if self._name is None:
raise InvalidOperationException("PostgresNode name is not defined.")
assert type(self._name) is str
return self._name
@property
def host(self) -> str:
assert self._os_ops is not None
assert isinstance(self._os_ops, OsOperations)
return self._os_ops.host
@property
def port(self) -> int:
if self._port is None:
raise InvalidOperationException("PostgresNode port is not defined.")
assert type(self._port) is int
return self._port
@property
def ssh_key(self) -> typing.Optional[str]:
assert self._os_ops is not None
assert isinstance(self._os_ops, OsOperations)
return self._os_ops.ssh_key
@property
def pid(self) -> int:
"""
Return postmaster's PID if node is running, else 0.
"""
x = self._get_node_state()
assert type(x) is utils.PostgresNodeState
if x.pid is None:
assert x.node_status != NodeStatus.Running
return 0
assert x.node_status == NodeStatus.Running
assert type(x.pid) is int
return x.pid
@property
def is_started(self) -> bool:
if self._manually_started_pm_pid is None:
return False
assert type(self._manually_started_pm_pid) is int
return True
@property
def auxiliary_pids(self) -> typing.Dict[ProcessType, typing.List[int]]:
"""
Returns a dict of { ProcessType : PID }.
"""
result = {}
for process in self.auxiliary_processes:
assert type(process) is ProcessProxy
if process.ptype not in result:
result[process.ptype] = []
result[process.ptype].append(process.pid)
return result
@property
def auxiliary_processes(self) -> typing.List[ProcessProxy]:
"""
Returns a list of auxiliary processes.
Each process is represented by :class:`.ProcessProxy` object.
"""
def is_aux(process: ProcessProxy) -> bool:
assert type(process) is ProcessProxy
return process.ptype != ProcessType.Unknown
return list(filter(is_aux, self.child_processes))
@property
def child_processes(self) -> typing.List[ProcessProxy]:
"""
Returns a list of all child processes.
Each process is represented by :class:`.ProcessProxy` object.
"""
# get a list of postmaster's children
x = self._get_node_state()
assert type(x) is utils.PostgresNodeState
if x.pid is None:
assert x.node_status != NodeStatus.Running
RaiseError.node_err__cant_enumerate_child_processes(
x.node_status
)
assert x.node_status == NodeStatus.Running
assert type(x.pid) is int
return self._get_child_processes(x.pid)
def _get_child_processes(self, pid: int) -> typing.List[ProcessProxy]:
assert type(pid) is int
assert isinstance(self._os_ops, OsOperations)
# get a list of postmaster's children
children = self._os_ops.get_process_children(pid)
return [ProcessProxy(p) for p in children]
@property
def source_walsender(self):
"""
Returns master's walsender feeding this replica.
"""
sql = """
select pid
from pg_catalog.pg_stat_replication
where application_name = %s
"""
if self.master is None:
raise TestgresException("Node doesn't have a master")
assert type(self.master) is PostgresNode
# master should be on the same host
assert self.master.host == self.host
with self.master.connect() as con:
for row in con.execute(sql, self.name):
for child in self.master.auxiliary_processes:
if child.pid == int(row[0]):
return child
msg = "Master doesn't send WAL to {}".format(self.name)
raise TestgresException(msg)
@property
def master(self):
return self._master
@property
def base_dir(self):
if not self._base_dir:
self._base_dir = self.os_ops.mkdtemp(prefix=self._prefix or TMP_NODE)
# NOTE: it's safe to create a new dir
if not self.os_ops.path_exists(self._base_dir):
self.os_ops.makedirs(self._base_dir)
return self._base_dir
@property
def bin_dir(self):
if not self._bin_dir:
self._bin_dir = os.path.dirname(get_bin_path2(self.os_ops, "pg_config"))
return self._bin_dir
@property
def logs_dir(self):
assert self._os_ops is not None
assert isinstance(self._os_ops, OsOperations)
path = self._os_ops.build_path(self.base_dir, LOGS_DIR)
assert type(path) is str
# NOTE: it's safe to create a new dir
if not self.os_ops.path_exists(path):
self.os_ops.makedirs(path)
return path
@property
def data_dir(self):
assert self._os_ops is not None
assert isinstance(self._os_ops, OsOperations)
# NOTE: we can't run initdb without user's args
path = self._os_ops.build_path(self.base_dir, DATA_DIR)
assert type(path) is str
return path
@property
def utils_log_file(self):
assert self._os_ops is not None
assert isinstance(self._os_ops, OsOperations)
path = self._os_ops.build_path(self.logs_dir, UTILS_LOG_FILE)
assert type(path) is str
return path
@property
def pg_log_file(self):
assert self._os_ops is not None
assert isinstance(self._os_ops, OsOperations)
path = self._os_ops.build_path(self.logs_dir, PG_LOG_FILE)
assert type(path) is str
return path
@property
def version(self):
"""
Return PostgreSQL version for this node.
Returns:
Instance of :class:`distutils.version.LooseVersion`.
"""
return self._pg_version
def _try_shutdown(self, max_attempts, with_force=False):
assert type(max_attempts) is int
assert type(with_force) is bool
assert max_attempts > 0
attempts = 0
# try stopping server N times
while attempts < max_attempts:
attempts += 1
try:
self.stop()
except ExecUtilException:
continue # one more time
except Exception:
eprint('cannot stop node {}'.format(self.name))
break
return # OK
# If force stopping is enabled and PID is valid
if not with_force:
return
node_pid = self.pid
assert node_pid is not None
assert type(node_pid) is int
if node_pid == 0:
return
# TODO: [2025-02-28] It is really the old ugly code. We have to rewrite it!
ps_command = ['ps', '-o', 'pid=', '-p', str(node_pid)]
ps_output = self.os_ops.exec_command(cmd=ps_command, shell=True, ignore_errors=True).decode('utf-8')
assert type(ps_output) is str
if ps_output == "":
return
if ps_output != str(node_pid):
__class__._throw_bugcheck__unexpected_result_of_ps(
ps_output,
ps_command)
try:
eprint('Force stopping node {0} with PID {1}'.format(self.name, node_pid))
self.os_ops.kill(node_pid, signal.SIGKILL)
except Exception:
# The node has already stopped
pass
# Check that node stopped - print only column pid without headers
ps_output = self.os_ops.exec_command(cmd=ps_command, shell=True, ignore_errors=True).decode('utf-8')
assert type(ps_output) is str
if ps_output == "":
eprint('Node {0} has been stopped successfully.'.format(self.name))
return
if ps_output == str(node_pid):
eprint('Failed to stop node {0}.'.format(self.name))
return
__class__._throw_bugcheck__unexpected_result_of_ps(
ps_output,
ps_command)
@staticmethod
def _throw_bugcheck__unexpected_result_of_ps(result, cmd):
assert type(result) is str
assert type(cmd) is list
errLines = []
errLines.append("[BUG CHECK] Unexpected result of command ps:")
errLines.append(result)
errLines.append("-----")
errLines.append("Command line is {0}".format(cmd))
raise RuntimeError("\n".join(errLines))
def _assign_master(self, master):
"""NOTE: this is a private method!"""
# now this node has a master
self._master = master
def _create_recovery_conf(self, username, slot=None):
"""NOTE: this is a private method!"""
# fetch master of this node
master = self.master
assert master is not None
conninfo = {
"application_name": self.name,
"port": master.port,
"user": username
} # yapf: disable
# host is tricky
try:
import ipaddress
ipaddress.ip_address(master.host)
conninfo["hostaddr"] = master.host
except ValueError:
conninfo["host"] = master.host
line = (
"primary_conninfo='{}'\n"
).format(options_string(**conninfo)) # yapf: disable
# Since 12 recovery.conf had disappeared
if self.version >= PgVer('12'):
assert self._os_ops is not None
assert isinstance(self._os_ops, OsOperations)
signal_name = self._os_ops.build_path(self.data_dir, "standby.signal")
assert type(signal_name) is str
self.os_ops.touch(signal_name)
else:
line += "standby_mode=on\n"
if slot:
# Connect to master for some additional actions
with master.connect(username=username) as con:
# check if slot already exists
res = con.execute(
"""
select exists (
select from pg_catalog.pg_replication_slots
where slot_name = %s
)
""", slot)
if res[0][0]:
raise TestgresException(
"Slot '{}' already exists".format(slot))
# TODO: we should drop this slot after replica's cleanup()
con.execute(
"""
select pg_catalog.pg_create_physical_replication_slot(%s)
""", slot)
line += "primary_slot_name={}\n".format(slot)
if self.version >= PgVer('12'):
self.append_conf(line=line)
else:
self.append_conf(filename=RECOVERY_CONF_FILE, line=line)
def _maybe_start_logger(self):
if testgres_config.use_python_logging:
# spawn new logger if it doesn't exist or is stopped
if not self._logger or not self._logger.is_alive():
self._logger = TestgresLogger(self.name, self.pg_log_file)
self._logger.start()
def _maybe_stop_logger(self):
if self._logger:
self._logger.stop()
def _collect_special_files(self) -> typing.List[typing.Tuple[str, bytes]]:
result = []
# list of important files + last N lines
assert self._os_ops is not None
assert isinstance(self._os_ops, OsOperations)
files = [
(self._os_ops.build_path(self.data_dir, PG_CONF_FILE), 0),
(self._os_ops.build_path(self.data_dir, PG_AUTO_CONF_FILE), 0),
(self._os_ops.build_path(self.data_dir, RECOVERY_CONF_FILE), 0),
(self._os_ops.build_path(self.data_dir, HBA_CONF_FILE), 0),
(self.pg_log_file, testgres_config.error_log_lines)
] # yapf: disable
for f, num_lines in files:
# skip missing files
if not self.os_ops.path_exists(f):
continue
file_lines = self.os_ops.readlines(f, num_lines, binary=True, encoding=None)
lines = b''.join(file_lines)
# fill list
result.append((f, lines))
return result
def init(self, initdb_params=None, cached=True, **kwargs):
"""
Perform initdb for this node.
Args:
initdb_params: parameters for initdb (list).
fsync: should this node use fsync to keep data safe?
unix_sockets: should we enable UNIX sockets?
allow_streaming: should this node add a hba entry for replication?
Returns:
This instance of :class:`.PostgresNode`
"""
# initialize this PostgreSQL node
assert self._os_ops is not None
assert isinstance(self._os_ops, OsOperations)
cached_initdb(
data_dir=self.data_dir,
logfile=self.utils_log_file,
os_ops=self._os_ops,
params=initdb_params,
bin_path=self.bin_dir,
cached=False)
# initialize default config files
self.default_conf(**kwargs)
return self
def default_conf(self,
fsync=False,
unix_sockets=True,
allow_streaming=True,
allow_logical=False,
log_statement='all'):
"""
Apply default settings to this node.
Args:
fsync: should this node use fsync to keep data safe?
unix_sockets: should we enable UNIX sockets?
allow_streaming: should this node add a hba entry for replication?
allow_logical: can this node be used as a logical replication publisher?
log_statement: one of ('all', 'off', 'mod', 'ddl').
Returns:
This instance of :class:`.PostgresNode`.
"""
assert self._os_ops is not None
assert isinstance(self._os_ops, OsOperations)
postgres_conf = self._os_ops.build_path(self.data_dir, PG_CONF_FILE)
hba_conf = self._os_ops.build_path(self.data_dir, HBA_CONF_FILE)
# filter lines in hba file
# get rid of comments and blank lines
hba_conf_file = self.os_ops.readlines(hba_conf)
lines = [
s for s in hba_conf_file
if len(s.strip()) > 0 and not s.startswith('#')
]
# write filtered lines
self.os_ops.write(hba_conf, lines, truncate=True)
# replication-related settings
if allow_streaming:
# get auth method for host or local users
def get_auth_method(t):
return next((s.split()[-1]
for s in lines if s.startswith(t)), 'trust')
# get auth methods
auth_local = get_auth_method('local')
auth_host = get_auth_method('host')
subnet_base = ".".join(self.os_ops.host.split('.')[:-1] + ['0'])
new_lines = [
u"local\treplication\tall\t\t\t{}\n".format(auth_local),
u"host\treplication\tall\t127.0.0.1/32\t{}\n".format(auth_host),
u"host\treplication\tall\t::1/128\t\t{}\n".format(auth_host),
u"host\treplication\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host),
u"host\tall\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host),
u"host\tall\tall\tall\t{}\n".format(auth_host),
u"host\treplication\tall\tall\t{}\n".format(auth_host)
] # yapf: disable
# write missing lines
self.os_ops.write(hba_conf, new_lines)
# overwrite config file
self.os_ops.write(postgres_conf, '', truncate=True)
self.append_conf(fsync=fsync,
max_worker_processes=MAX_WORKER_PROCESSES,
log_statement=log_statement,
listen_addresses=self.host,
port=self.port) # yapf:disable
# common replication settings
if allow_streaming or allow_logical:
self.append_conf(max_replication_slots=MAX_REPLICATION_SLOTS,
max_wal_senders=MAX_WAL_SENDERS) # yapf: disable
# binary replication
if allow_streaming:
# select a proper wal_level for PostgreSQL
wal_level = 'replica' if self._pg_version >= PgVer('9.6') else 'hot_standby'
if self._pg_version < PgVer('13'):
self.append_conf(hot_standby=True,
wal_keep_segments=WAL_KEEP_SEGMENTS,
wal_level=wal_level) # yapf: disable
else:
self.append_conf(hot_standby=True,
wal_keep_size=WAL_KEEP_SIZE,
wal_level=wal_level) # yapf: disable
# logical replication
if allow_logical:
if self._pg_version < PgVer('10'):
raise InitNodeException("Logical replication is only "
"available on PostgreSQL 10 and newer")
self.append_conf(
max_logical_replication_workers=MAX_LOGICAL_REPLICATION_WORKERS,
wal_level='logical')
# disable UNIX sockets if asked to
if not unix_sockets:
self.append_conf(unix_socket_directories='')
return self
@method_decorator(positional_args_hack(['filename', 'line']))
def append_conf(self, line='', filename=PG_CONF_FILE, **kwargs):
"""
Append line to a config file.
Args:
line: string to be appended to config.
filename: config file (postgresql.conf by default).
**kwargs: named config options.
Returns:
This instance of :class:`.PostgresNode`.
Examples:
>>> append_conf(fsync=False)
>>> append_conf('log_connections = yes')
>>> append_conf(random_page_cost=1.5, fsync=True, ...)
>>> append_conf('postgresql.conf', 'synchronous_commit = off')
"""
lines = [line]
for option, value in iteritems(kwargs):
if isinstance(value, bool):
value = 'on' if value else 'off'
elif not str(value).replace('.', '', 1).isdigit():
value = "'{}'".format(value)
if value == '*':
lines.append("{} = '*'".format(option))
else:
# format a new config line
lines.append('{} = {}'.format(option, value))
config_name = self._os_ops.build_path(self.data_dir, filename)
conf_text = ''
for line in lines:
conf_text += text_type(line) + '\n'
self.os_ops.write(config_name, conf_text)
return self
def status(self):
"""
Check this node's status.
Returns:
An instance of :class:`.NodeStatus`.
"""
x = self._get_node_state()
assert type(x) is utils.PostgresNodeState
return x.node_status
def _get_node_state(self) -> utils.PostgresNodeState:
return utils.get_pg_node_state(
self._os_ops,
self.bin_dir,
self.data_dir,
self.utils_log_file
)
def get_control_data(self):
"""
Return contents of pg_control file.
"""
# this one is tricky (blame PG 9.4)
_params = [self._get_bin_path("pg_controldata")]
_params += ["-D"] if self._pg_version >= PgVer('9.5') else []
_params += [self.data_dir]
data = execute_utility2(self.os_ops, _params, self.utils_log_file)
out_dict = {}
for line in data.splitlines():
key, _, value = line.partition(':')
out_dict[key.strip()] = value.strip()
return out_dict
def slow_start(self, replica=False, dbname='template1', username=None, max_attempts=0, exec_env=None):
"""
Starts the PostgreSQL instance and then polls the instance
until it reaches the expected state (primary or replica). The state is checked
using the pg_is_in_recovery() function.
Args:
dbname:
username:
replica: If True, waits for the instance to be in recovery (i.e., replica mode).
If False, waits for the instance to be in primary mode. Default is False.
max_attempts:
"""
assert exec_env is None or type(exec_env) is dict
self.start(exec_env=exec_env)
try:
if replica:
query = 'SELECT pg_is_in_recovery()'
else:
query = 'SELECT not pg_is_in_recovery()'
# Call poll_query_until until the expected value is returned
suppressed_exceptions = {
InternalError,
QueryException,
ProgrammingError,
OperationalError
}
self.poll_query_until(
query=query,
dbname=dbname,
username=username or self.os_ops.username,
suppress=suppressed_exceptions,
max_attempts=max_attempts,
)
except: # noqa: E722
self.stop()
raise
return
def start(
self,
params: typing.Optional[typing.List[str]] = None,
wait: bool = True,
exec_env: typing.Optional[typing.Dict] = None,
) -> PostgresNode:
"""
Starts the PostgreSQL node using pg_ctl and set flag 'is_started'.
By default, it waits for the operation to complete before returning.
Optionally, it can return immediately without waiting for the start operation
to complete by setting the `wait` parameter to False.
Args:
params: additional arguments for pg_ctl.
wait: wait until operation completes.
Returns:
This instance of :class:`.PostgresNode`.
"""
assert params is None or type(params) is list
assert type(wait) is bool
assert exec_env is None or type(exec_env) is dict
self._start(params, wait, exec_env)
if not wait:
# Postmaster process is starting in background
self._manually_started_pm_pid = __class__._C_PM_PID__IS_NOT_DETECTED
else:
self._manually_started_pm_pid = self._get_node_state().pid
if self._manually_started_pm_pid is None:
self._raise_cannot_start_node(None, "Cannot detect postmaster pid.")
assert type(self._manually_started_pm_pid) is int
return self
def start2(
self,
params: typing.Optional[typing.List[str]] = None,
wait: bool = True,
exec_env: typing.Optional[typing.Dict] = None,
) -> None:
"""
Starts the PostgreSQL node using pg_ctl.
By default, it waits for the operation to complete before returning.
Optionally, it can return immediately without waiting for the start operation
to complete by setting the `wait` parameter to False.
Args:
params: additional arguments for pg_ctl.
wait: wait until operation completes.
Returns:
None.
"""
assert params is None or type(params) is list
assert type(wait) is bool
assert exec_env is None or type(exec_env) is dict
self._start(params, wait, exec_env)
return
def _start(
self,
params: typing.Optional[typing.List[str]] = None,
wait: bool = True,
exec_env: typing.Optional[typing.Dict] = None,
) -> None:
assert params is None or type(params) is list
assert type(wait) is bool
assert exec_env is None or type(exec_env) is dict
assert __class__._C_MAX_START_ATEMPTS > 1
if self._port is None:
raise InvalidOperationException("Can't start PostgresNode. Port is not defined.")
assert type(self._port) is int
_params = [
self._get_bin_path("pg_ctl"),
"start",
"-D", self.data_dir,
"-l", self.pg_log_file,
"-w" if wait else '-W', # --wait or --no-wait
]
if params is not None:
assert type(params) is list
_params += params
def LOCAL__start_node():
# 'error' will be None on Windows
_, _, error = execute_utility2(self.os_ops, _params, self.utils_log_file, verbose=True, exec_env=exec_env)
assert error is None or type(error) is str
if error and 'does not exist' in error:
raise Exception(error)
def LOCAL__raise_cannot_start_node__std(from_exception):
assert isinstance(from_exception, Exception)
self._raise_cannot_start_node(from_exception, 'Cannot start node')
if not self._should_free_port:
try:
LOCAL__start_node()
except Exception as e:
LOCAL__raise_cannot_start_node__std(e)
else:
assert self._should_free_port
assert self._port_manager is not None
assert isinstance(self._port_manager, PortManager)
assert __class__._C_MAX_START_ATEMPTS > 1
log_reader = PostgresNodeLogReader(self, from_beginnig=False)
nAttempt = 0
timeout = 1
while True:
assert nAttempt >= 0
assert nAttempt < __class__._C_MAX_START_ATEMPTS
nAttempt += 1
try:
LOCAL__start_node()
except Exception as e:
assert nAttempt > 0
assert nAttempt <= __class__._C_MAX_START_ATEMPTS
if nAttempt == __class__._C_MAX_START_ATEMPTS:
self._raise_cannot_start_node(e, "Cannot start node after multiple attempts.")
is_it_port_conflict = PostgresNodeUtils.delect_port_conflict(log_reader)
if not is_it_port_conflict:
LOCAL__raise_cannot_start_node__std(e)
logging.warning(
"Detected a conflict with using the port {0}. Trying another port after a {1}-second sleep...".format(self._port, timeout)
)
time.sleep(timeout)
timeout = min(2 * timeout, 5)
cur_port = self._port
new_port = self._port_manager.reserve_port() # can raise
try:
options = {'port': new_port}
self.set_auto_conf(options)
except: # noqa: E722
self._port_manager.release_port(new_port)
raise
self._port = new_port
self._port_manager.release_port(cur_port)
continue
break
self._maybe_start_logger()
return
def _raise_cannot_start_node(
self,
from_exception: typing.Optional[Exception],
msg: str
):
assert from_exception is None or isinstance(from_exception, Exception)
assert type(msg) is str
files = self._collect_special_files()
raise_from(StartNodeException(msg, files), from_exception)
def stop(self, params=[], wait=True):
"""
Stops the PostgreSQL node using pg_ctl if the node has been started.
Args:
params: A list of additional arguments for pg_ctl. Defaults to None.
wait: If True, waits until the operation is complete. Defaults to True.
Returns:
This instance of :class:`.PostgresNode`.
"""
_params = [
self._get_bin_path("pg_ctl"),
"-D", self.data_dir,
"-w" if wait else '-W', # --wait or --no-wait
"stop"
] + params # yapf: disable
execute_utility2(self.os_ops, _params, self.utils_log_file)
self._manually_started_pm_pid = None
self._maybe_stop_logger()
return self
def kill(self, someone=None):
"""
Kills the PostgreSQL node or a specified auxiliary process if the node is running.
Args:
someone: A key to the auxiliary process in the auxiliary_pids dictionary.
If None, the main PostgreSQL node process will be killed. Defaults to None.
"""
x = self._get_node_state()
assert type(x) is utils.PostgresNodeState
if x.node_status != NodeStatus.Running:
RaiseError.node_err__cant_kill(x.node_status)
assert False
assert x.node_status == NodeStatus.Running
assert type(x.pid) is int
sig = signal.SIGKILL if os.name != 'nt' else signal.SIGBREAK
if someone is None:
self._os_ops.kill(x.pid, sig)
self._manually_started_pm_pid = None
else:
childs = self._get_child_processes(x.pid)
for c in childs:
assert type(c) is ProcessProxy
if c.ptype == someone:
self._os_ops.kill(c.process.pid, sig)
continue
return
def restart(self, params=[]):
"""
Restart this node using pg_ctl.
Args:
params: additional arguments for pg_ctl.
Returns:
This instance of :class:`.PostgresNode`.
"""
_params = [
self._get_bin_path("pg_ctl"),
"-D", self.data_dir,
"-l", self.pg_log_file,
"-w", # wait
"restart"
] + params # yapf: disable
try:
error_code, out, error = execute_utility2(self.os_ops, _params, self.utils_log_file, verbose=True)
if error and 'could not start server' in error:
raise ExecUtilException
except ExecUtilException as e:
msg = 'Cannot restart node'
files = self._collect_special_files()
raise_from(StartNodeException(msg, files), e)
self._maybe_start_logger()
return self
def reload(self, params=[]):
"""
Asynchronously reload config files using pg_ctl.
Args:
params: additional arguments for pg_ctl.
Returns:
This instance of :class:`.PostgresNode`.
"""
_params = [
self._get_bin_path("pg_ctl"),
"-D", self.data_dir,
"reload"
] + params # yapf: disable
execute_utility2(self.os_ops, _params, self.utils_log_file)
return self
def promote(self, dbname=None, username=None):
"""
Promote standby instance to master using pg_ctl. For PostgreSQL versions
below 10 some additional actions required to ensure that instance
became writable and hence `dbname` and `username` parameters may be
needed.
Returns:
This instance of :class:`.PostgresNode`.
"""
_params = [
self._get_bin_path("pg_ctl"),
"-D", self.data_dir,
"-w", # wait
"promote"
] # yapf: disable
execute_utility2(self.os_ops, _params, self.utils_log_file)
# for versions below 10 `promote` is asynchronous so we need to wait
# until it actually becomes writable
if self._pg_version < PgVer('10'):
check_query = "SELECT pg_is_in_recovery()"
self.poll_query_until(query=check_query,
expected=False,
dbname=dbname,
username=username,
max_attempts=0) # infinite
# node becomes master itself
self._master = None
return self
def pg_ctl(self, params):
"""
Invoke pg_ctl with params.
Args:
params: arguments for pg_ctl.
Returns:
Stdout + stderr of pg_ctl.
"""
_params = [
self._get_bin_path("pg_ctl"),
"-D", self.data_dir,
"-w" # wait
] + params # yapf: disable
return execute_utility2(self.os_ops, _params, self.utils_log_file)
def release_resources(self):
"""
Release resorces owned by this node.
"""
return self._release_resources()
def free_port(self):
"""
Reclaim port owned by this node.
NOTE: this method does not release manually defined port but reset it.
"""
return self._free_port()
def cleanup(self, max_attempts=3, full=False, release_resources=False):
"""
Stop node if needed and remove its data/logs directory.
NOTE: take a look at TestgresConfig.node_cleanup_full.
Args:
max_attempts: how many times should we try to stop()?
full: clean full base dir
Returns:
This instance of :class:`.PostgresNode`.
"""
self._try_shutdown(max_attempts)
# choose directory to be removed
if testgres_config.node_cleanup_full or full:
rm_dir = self.base_dir # everything
else:
rm_dir = self.data_dir # just data, save logs
self.os_ops.rmdirs(rm_dir, ignore_errors=False)
if release_resources:
self._release_resources()
return self
@method_decorator(positional_args_hack(['dbname', 'query']))
def psql(self,
query=None,
filename=None,
dbname=None,
username=None,
input=None,
host: typing.Optional[str] = None,
port: typing.Optional[int] = None,
**variables):
"""
Execute a query using psql.
Args:
query: query to be executed.
filename: file with a query.
dbname: database name to connect to.
username: database user name.
input: raw input to be passed.
host: an explicit host of server.
port: an explicit port of server.
**variables: vars to be set before execution.
Returns:
A tuple of (code, stdout, stderr).
Examples:
>>> psql('select 1')
>>> psql('postgres', 'select 2')
>>> psql(query='select 3', ON_ERROR_STOP=1)
"""
assert host is None or type(host) is str
assert port is None or type(port) is int
assert type(variables) is dict
return self._psql(
ignore_errors=True,
query=query,
filename=filename,
dbname=dbname,
username=username,
input=input,
host=host,
port=port,
**variables
)
def _psql(
self,
ignore_errors,
query=None,
filename=None,
dbname=None,
username=None,
input=None,
host: typing.Optional[str] = None,
port: typing.Optional[int] = None,
**variables):
assert host is None or type(host) is str
assert port is None or type(port) is int
assert type(variables) is dict
#
# We do not support encoding. It may be added later. Ok?
#
if input is None:
pass
elif type(input) is bytes:
pass
else:
raise Exception("Input data must be None or bytes.")
if host is None:
host = self.host
if port is None:
port = self.port
assert host is not None
assert port is not None
assert type(host) is str
assert type(port) is int
psql_params = [
self._get_bin_path("psql"),
"-p", str(port),
"-h", host,
"-U", username or self.os_ops.username,
"-d", dbname or default_dbname(),
"-X", # no .psqlrc
"-A", # unaligned output
"-t", # print rows only
"-q" # run quietly
] # yapf: disable
# set variables before execution
for key, value in iteritems(variables):
psql_params.extend(["--set", '{}={}'.format(key, value)])
# select query source
if query:
psql_params.extend(("-c", query))
elif filename:
psql_params.extend(("-f", filename))
else:
raise QueryException('Query or filename must be provided')
return self.os_ops.exec_command(
psql_params,
verbose=True,
input=input,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
ignore_errors=ignore_errors)
@method_decorator(positional_args_hack(['dbname', 'query']))
def safe_psql(self, query=None, expect_error=False, **kwargs):
"""
Execute a query using psql.
Args:
query: query to be executed.
filename: file with a query.
dbname: database name to connect to.
username: database user name.
input: raw input to be passed.
expect_error: if True - fail if we didn't get ret
if False - fail if we got ret
**kwargs are passed to psql().
Returns:
psql's output as str.
"""
assert type(kwargs) is dict
assert "ignore_errors" not in kwargs.keys()
assert "expect_error" not in kwargs.keys()
# force this setting
kwargs['ON_ERROR_STOP'] = 1
try:
ret, out, err = self._psql(ignore_errors=False, query=query, **kwargs)
except ExecUtilException as e:
if not expect_error:
raise QueryException(e.message, query)
if type(e.error) is bytes:
return e.error.decode("utf-8") # throw
# [2024-12-09] This situation is not expected
assert False
return e.error
if expect_error:
raise InvalidOperationException("Exception was expected, but query finished successfully: `{}`.".format(query))
return out
def dump(self,
filename=None,
dbname=None,
username=None,
format=DumpFormat.Plain,
options=None):
"""
Dump database into a file using pg_dump.
NOTE: the file is not removed automatically.
Args:
filename: database dump taken by pg_dump.
dbname: database name to connect to.
username: database user name.
format: format argument plain/custom/directory/tar.
options: additional options for pg_dump (list).
Returns:
Path to a file containing dump.
"""
# Check arguments
if not isinstance(format, DumpFormat):
try:
format = DumpFormat(format)
except ValueError:
msg = 'Invalid format "{}"'.format(format)
raise BackupException(msg)
# Generate tmpfile or tmpdir
def tmpfile():
if format == DumpFormat.Directory:
fname = self.os_ops.mkdtemp(prefix=TMP_DUMP)
else:
fname = self.os_ops.mkstemp(prefix=TMP_DUMP)
return fname
filename = filename or tmpfile()
_params = [
self._get_bin_path("pg_dump"),
"-p", str(self.port),
"-h", self.host,
"-f", filename,
"-U", username or self.os_ops.username,
"-d", dbname or default_dbname(),
"-F", format.value
] # yapf: disable
# Add additional options if provided
if options:
_params.extend(options)
execute_utility2(self.os_ops, _params, self.utils_log_file)
return filename
def restore(self, filename, dbname=None, username=None):
"""
Restore database from pg_dump's file.
Args:
filename: database dump taken by pg_dump in custom/directory/tar formats.
dbname: database name to connect to.
username: database user name.
"""
# Set default arguments
dbname = dbname or default_dbname()
username = username or self.os_ops.username
_params = [
self._get_bin_path("pg_restore"),
"-p", str(self.port),
"-h", self.host,
"-U", username,
"-d", dbname,
filename
] # yapf: disable
# try pg_restore if dump is binary format, and psql if not
try:
execute_utility2(self.os_ops, _params, self.utils_log_name)
except ExecUtilException:
self.psql(filename=filename, dbname=dbname, username=username)
@method_decorator(positional_args_hack(['dbname', 'query']))
def poll_query_until(self,
query,
dbname=None,
username=None,
max_attempts=0,
sleep_time: typing.Union[int, float] = 1,
expected=True,
commit=True,
suppress=None):
"""
Run a query once per second until it returns 'expected'.
Query should return a single value (1 row, 1 column).
Args:
query: query to be executed.
dbname: database name to connect to.
username: database user name.
max_attempts: how many times should we try? 0 == infinite
sleep_time: how much should we sleep after a failure?
expected: what should be returned to break the cycle?
commit: should (possible) changes be committed?
suppress: a collection of exceptions to be suppressed.
Examples:
>>> poll_query_until('select true')
>>> poll_query_until('postgres', "select now() > '01.01.2018'")
>>> poll_query_until('select false', expected=True, max_attempts=4)
>>> poll_query_until('select 1', suppress={testgres.OperationalError})
"""
# sanity checks
assert type(max_attempts) is int
assert max_attempts >= 0
assert type(sleep_time) in [int, float]
assert sleep_time > 0
attempts = 0
while max_attempts == 0 or attempts < max_attempts:
try:
res = self.execute(dbname=dbname,
query=query,
username=username,
commit=commit)
if expected is None and res is None:
return # done
if res is None:
raise QueryException('Query returned None', query)
# result set is not empty
if len(res):
if len(res[0]) == 0:
raise QueryException('Query returned 0 columns', query)
if res[0][0] == expected:
return # done
# empty result set is considered as None
elif expected is None:
return # done
except tuple(suppress or []):
logging.info(f"Trying execute, attempt {attempts + 1}.\nQuery: {query}")
pass # we're suppressing them
time.sleep(sleep_time)
attempts += 1
raise QueryTimeoutException('Query timeout', query)
@method_decorator(positional_args_hack(['dbname', 'query']))
def execute(self,
query,
dbname=None,
username=None,
password=None,
commit=True):
"""
Execute a query and return all rows as list.
Args:
query: query to be executed.
dbname: database name to connect to.
username: database user name.
password: user's password.
commit: should we commit this query?
Returns:
A list of tuples representing rows.
"""
with self.connect(dbname=dbname,
username=username,
password=password,
autocommit=commit) as node_con: # yapf: disable
res = node_con.execute(query)
return res
def backup(self, **kwargs):
"""
Perform pg_basebackup.
Args:
username: database user name.
xlog_method: a method for collecting the logs ('fetch' | 'stream').
base_dir: the base directory for data files and logs
Returns:
A smart object of type NodeBackup.
"""
return NodeBackup(node=self, **kwargs)
def replicate(self, name=None, slot=None, **kwargs):
"""
Create a binary replica of this node.
Args:
name: replica's application name.
slot: create a replication slot with the specified name.
username: database user name.
xlog_method: a method for collecting the logs ('fetch' | 'stream').
base_dir: the base directory for data files and logs
"""
# transform backup into a replica
with clean_on_error(self.backup(**kwargs)) as backup:
return backup.spawn_replica(name=name, destroy=True, slot=slot)
def set_synchronous_standbys(self, standbys):
"""
Set standby synchronization options. This corresponds to
`synchronous_standby_names <https://www.postgresql.org/docs/current/static/runtime-config-replication.html#GUC-SYNCHRONOUS-STANDBY-NAMES>`_
option. Note that :meth:`~.PostgresNode.reload` or
:meth:`~.PostgresNode.restart` is needed for changes to take place.
Args:
standbys: either :class:`.First` or :class:`.Any` object specifying
synchronization parameters or just a plain list of
:class:`.PostgresNode`s replicas which would be equivalent
to passing ``First(1, <list>)``. For PostgreSQL 9.5 and below
it is only possible to specify a plain list of standbys as
`FIRST` and `ANY` keywords aren't supported.
Example::
from testgres import get_new_node, First
master = get_new_node().init().start()
with master.replicate().start() as standby:
master.append_conf("synchronous_commit = remote_apply")
master.set_synchronous_standbys(First(1, [standby]))
master.restart()
"""
if self._pg_version >= PgVer('9.6'):
if isinstance(standbys, Iterable):
standbys = First(1, standbys)
else:
if isinstance(standbys, Iterable):
standbys = u", ".join(u"\"{}\"".format(r.name)
for r in standbys)
else:
raise TestgresException("Feature isn't supported in "
"Postgres 9.5 and below")
self.append_conf("synchronous_standby_names = '{}'".format(standbys))
def catchup(self, dbname=None, username=None):
"""
Wait until async replica catches up with its master.
"""
if not self.master:
raise TestgresException("Node doesn't have a master")
if self._pg_version >= PgVer('10'):
poll_lsn = "select pg_catalog.pg_current_wal_lsn()::text"
wait_lsn = "select pg_catalog.pg_last_wal_replay_lsn() >= '{}'::pg_lsn"
else:
poll_lsn = "select pg_catalog.pg_current_xlog_location()::text"
wait_lsn = "select pg_catalog.pg_last_xlog_replay_location() >= '{}'::pg_lsn"
try:
# fetch latest LSN
lsn = self.master.execute(query=poll_lsn,
dbname=dbname,
username=username)[0][0] # yapf: disable
# wait until this LSN reaches replica
self.poll_query_until(query=wait_lsn.format(lsn),
dbname=dbname,
username=username,
max_attempts=0) # infinite
except Exception as e:
raise_from(CatchUpException("Failed to catch up."), e)
def publish(self, name, **kwargs):
"""
Create publication for logical replication
Args:
pubname: publication name
tables: tables names list
dbname: database name where objects or interest are located
username: replication username
"""
return Publication(name=name, node=self, **kwargs)
def subscribe(self,
publication,
name,
dbname=None,
username=None,
**params):
"""
Create subscription for logical replication
Args:
name: subscription name
publication: publication object obtained from publish()
dbname: database name
username: replication username
params: subscription parameters (see documentation on `CREATE SUBSCRIPTION
<https://www.postgresql.org/docs/current/static/sql-createsubscription.html>`_
for details)
"""
# yapf: disable
return Subscription(name=name, node=self, publication=publication,
dbname=dbname, username=username, **params)
# yapf: enable
def pgbench(self,
dbname=None,
username=None,
stdout=None,
stderr=None,
options=None):
"""
Spawn a pgbench process.
Args:
dbname: database name to connect to.
username: database user name.
stdout: stdout file to be used by Popen.
stderr: stderr file to be used by Popen.
options: additional options for pgbench (list).
Returns:
Process created by subprocess.Popen.
"""
if options is None:
options = []
dbname = dbname or default_dbname()
_params = [
self._get_bin_path("pgbench"),
"-p", str(self.port),
"-h", self.host,
"-U", username or self.os_ops.username
] + options # yapf: disable
# should be the last one
_params.append(dbname)
proc = self.os_ops.exec_command(_params, stdout=stdout, stderr=stderr, wait_exit=True, get_process=True)
return proc
def pgbench_with_wait(self,
dbname=None,
username=None,
stdout=None,
stderr=None,
options=None):
"""
Do pgbench command and wait.
Args:
dbname: database name to connect to.
username: database user name.
stdout: stdout file to be used by Popen.
stderr: stderr file to be used by Popen.
options: additional options for pgbench (list).
"""
if options is None:
options = []
with self.pgbench(dbname, username, stdout, stderr, options) as pgbench:
pgbench.wait()
return
def pgbench_init(self, **kwargs):
"""
Small wrapper for pgbench_run().
Sets initialize=True.
Returns:
This instance of :class:`.PostgresNode`.
"""
self.pgbench_run(initialize=True, **kwargs)
return self
def pgbench_run(self, dbname=None, username=None, options=[], **kwargs):
"""
Run pgbench with some options.
This event is logged (see self.utils_log_file).
Args:
dbname: database name to connect to.
username: database user name.
options: additional options for pgbench (list).
**kwargs: named options for pgbench.
Run pgbench --help to learn more.
Returns:
Stdout produced by pgbench.
Examples:
>>> pgbench_run(initialize=True, scale=2)
>>> pgbench_run(time=10)
"""
dbname = dbname or default_dbname()
_params = [
self._get_bin_path("pgbench"),
"-p", str(self.port),
"-h", self.host,
"-U", username or self.os_ops.username
] + options # yapf: disable
for key, value in iteritems(kwargs):
# rename keys for pgbench
key = key.replace('_', '-')
# append option
if not isinstance(value, bool):
_params.append('--{}={}'.format(key, value))
else:
assert value is True # just in case
_params.append('--{}'.format(key))
# should be the last one
_params.append(dbname)
return execute_utility2(self.os_ops, _params, self.utils_log_file)
def connect(self,
dbname=None,
username=None,
password=None,
autocommit=False):
"""
Connect to a database.
Args:
dbname: database name to connect to.
username: database user name.
password: user's password.
autocommit: commit each statement automatically. Also it should be
set to `True` for statements requiring to be run outside
a transaction? such as `VACUUM` or `CREATE DATABASE`.
Returns:
An instance of :class:`.NodeConnection`.
"""
return NodeConnection(node=self,
dbname=dbname,
username=username,
password=password,
autocommit=autocommit) # yapf: disable
def table_checksum(
self,
table: str,
dbname: str = "postgres"
) -> int:
assert type(table) is str
assert type(dbname) is str
cn = self.connect(dbname=dbname)
assert type(cn) is NodeConnection
try:
sum = __class__._table_checksum__use_cn(cn, table)
assert type(sum) is int
finally:
assert type(cn) is NodeConnection
cn.close()
assert type(sum) is int
return sum
sm_pgbench_tables = [
'pgbench_branches',
'pgbench_tellers',
'pgbench_accounts',
'pgbench_history'
]
def pgbench_table_checksums(
self,
dbname: str = "postgres",
pgbench_tables: typing.Iterable[str] = sm_pgbench_tables
) -> typing.Set[typing.Tuple[str, int]]:
assert type(dbname) is str
r1 = self._tables_checksum(dbname, pgbench_tables)
assert type(r1) is list
r2 = set(r1)
assert type(r2) is set
return r2
def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}):
"""
Update or remove configuration options in the specified configuration file,
updates the options specified in the options dictionary, removes any options
specified in the rm_options set, and writes the updated configuration back to
the file.
Args:
options (dict): A dictionary containing the options to update or add,
with the option names as keys and their values as values.
config (str, optional): The name of the configuration file to update.
Defaults to 'postgresql.auto.conf'.
rm_options (set, optional): A set containing the names of the options to remove.
Defaults to an empty set.
"""
assert self._os_ops is not None
assert isinstance(self._os_ops, OsOperations)
# parse postgresql.auto.conf
path = self.os_ops.build_path(self.data_dir, config)
lines = self.os_ops.readlines(path)
current_options = {}
current_directives = []
for line in lines:
# ignore comments
if line.startswith('#'):
continue
if line.strip() == '':
continue
if line.startswith('include'):
current_directives.append(line)
continue
name, var = line.partition('=')[::2]
name = name.strip()
# Remove options specified in rm_options list
if name in rm_options:
continue
current_options[name] = var
for option in options:
assert type(option) is str
assert option != ""
assert option.strip() == option
value = options[option]
valueType = type(value)
if valueType is str:
value = __class__._escape_config_value(value)
elif valueType is bool:
value = "on" if value else "off"
current_options[option] = value
auto_conf = ''
for option in current_options:
auto_conf += option + " = " + str(current_options[option]) + "\n"
for directive in current_directives:
auto_conf += directive + "\n"
self.os_ops.write(path, auto_conf, truncate=True)
def upgrade_from(self, old_node, options=None, expect_error=False):
"""
Upgrade this node from an old node using pg_upgrade.
Args:
old_node: An instance of PostgresNode representing the old node.
"""
assert isinstance(self._os_ops, OsOperations)
if not self._os_ops.path_exists(old_node.data_dir):
raise Exception("Old node must be initialized")
if not self._os_ops.path_exists(self.data_dir):
self.init()
if not options:
options = []
pg_upgrade_binary = self._get_bin_path("pg_upgrade")
if not self._os_ops.path_exists(pg_upgrade_binary):
raise Exception("pg_upgrade does not exist in the new node's binary path")
upgrade_command = [
pg_upgrade_binary,
"--old-bindir", old_node.bin_dir,
"--new-bindir", self.bin_dir,
"--old-datadir", old_node.data_dir,
"--new-datadir", self.data_dir,
"--old-port", str(old_node.port),
"--new-port", str(self.port)
]
upgrade_command += options
return self.os_ops.exec_command(upgrade_command, expect_error=expect_error)
def _release_resources(self):
self._free_port()
def _free_port(self):
assert type(self._should_free_port) is bool
if not self._should_free_port:
self._port = None
else:
assert type(self._port) is int
assert self._port_manager is not None
assert isinstance(self._port_manager, PortManager)
port = self._port
self._should_free_port = False
self._port = None
self._port_manager.release_port(port)
def _get_bin_path(self, filename):
assert self._os_ops is not None
assert isinstance(self._os_ops, OsOperations)
if self.bin_dir:
bin_path = self._os_ops.build_path(self.bin_dir, filename)
else:
bin_path = get_bin_path2(self.os_ops, filename)
return bin_path
@staticmethod
def _escape_config_value(value):
assert type(value) is str
result = "'"
for ch in value:
if ch == "'":
result += "\\'"
elif ch == "\n":
result += "\\n"
elif ch == "\r":
result += "\\r"
elif ch == "\t":
result += "\\t"
elif ch == "\b":
result += "\\b"
elif ch == "\\":
result += "\\\\"
else:
result += ch
result += "'"
return result
def _tables_checksum(
self,
dbname: str,
tables: typing.Iterable[str],
) -> typing.List[typing.Tuple[str, int]]:
assert isinstance(tables, typing.Iterable)
assert type(dbname) is str
result = []
cn = self.connect(dbname=dbname)
assert type(cn) is NodeConnection
try:
cn.begin()
for table in tables:
assert type(table) is str
sum = __class__._table_checksum__use_cn(cn, table)
assert type(sum) is int
result.append((table, sum))
cn.commit()
finally:
assert type(cn) is NodeConnection
cn.close()
assert type(result) is list
return result
@staticmethod
def _table_checksum__use_cn(
cn: NodeConnection,
table: str,
) -> int:
assert type(cn) is NodeConnection
assert type(table) is str
sum = 0
cursor = cn.connection.cursor()
assert cursor is not None
try:
cursor.execute("SELECT SUM(hashtext(t::text)) FROM {} as t".format(
__class__._delim_sql_ident(table)
))
row = cursor.fetchone()
assert row is not None
assert type(row) in [list, tuple]
assert len(row) == 1
v = row[0]
sum += int(v if v is not None else 0)
finally:
cursor.close()
assert type(sum) is int
return sum
@staticmethod
def _delim_sql_ident(name: str) -> str:
assert isinstance(name, str)
result = '"'
for ch in name:
if ch == '"':
result = result + '""'
else:
result = result + ch
result = result + '"'
return result
class PostgresNodeLogReader:
class LogInfo:
position: int
def __init__(self, position: int):
self.position = position
# --------------------------------------------------------------------
class LogDataBlock:
_file_name: str
_position: int
_data: str
def __init__(
self,
file_name: str,
position: int,
data: str
):
assert type(file_name) is str
assert type(position) is int
assert type(data) is str
assert file_name != ""
assert position >= 0
self._file_name = file_name
self._position = position
self._data = data
@property
def file_name(self) -> str:
assert type(self._file_name) is str
assert self._file_name != ""
return self._file_name
@property
def position(self) -> int:
assert type(self._position) is int
assert self._position >= 0
return self._position
@property
def data(self) -> str:
assert type(self._data) is str
return self._data
# --------------------------------------------------------------------
_node: PostgresNode
_logs: typing.Dict[str, LogInfo]
# --------------------------------------------------------------------
def __init__(self, node: PostgresNode, from_beginnig: bool):
assert node is not None
assert isinstance(node, PostgresNode)
assert type(from_beginnig) is bool
self._node = node
if from_beginnig:
self._logs = dict()
else:
self._logs = self._collect_logs()
assert type(self._logs) is dict
return
def read(self) -> typing.List[LogDataBlock]:
assert self._node is not None
assert isinstance(self._node, PostgresNode)
cur_logs: typing.Dict[str, __class__.LogInfo] = self._collect_logs()
assert cur_logs is not None
assert type(cur_logs) is dict
assert type(self._logs) is dict
result = list()
for file_name, cur_log_info in cur_logs.items():
assert type(file_name) is str
assert type(cur_log_info) is __class__.LogInfo
read_pos = 0
if file_name in self._logs.keys():
prev_log_info = self._logs[file_name]
assert type(prev_log_info) is __class__.LogInfo
read_pos = prev_log_info.position # the previous size
file_content_b = self._node.os_ops.read_binary(file_name, read_pos)
assert type(file_content_b) is bytes
#
# A POTENTIAL PROBLEM: file_content_b may contain an incompleted UTF-8 symbol.
#
file_content_s = file_content_b.decode()
assert type(file_content_s) is str
next_read_pos = read_pos + len(file_content_b)
# It is a research/paranoja check.
# When we will process partial UTF-8 symbol, it must be adjusted.
assert cur_log_info.position <= next_read_pos
cur_log_info.position = next_read_pos
block = __class__.LogDataBlock(
file_name,
read_pos,
file_content_s
)
result.append(block)
# A new check point
self._logs = cur_logs
return result
def _collect_logs(self) -> typing.Dict[str, LogInfo]:
assert self._node is not None
assert isinstance(self._node, PostgresNode)
files = [
self._node.pg_log_file
] # yapf: disable
result = dict()
for f in files:
assert type(f) is str
# skip missing files
if not self._node.os_ops.path_exists(f):
continue
file_size = self._node.os_ops.get_file_size(f)
assert type(file_size) is int
assert file_size >= 0
result[f] = __class__.LogInfo(file_size)
return result
class PostgresNodeUtils:
@staticmethod
def delect_port_conflict(log_reader: PostgresNodeLogReader) -> bool:
assert type(log_reader) is PostgresNodeLogReader
blocks = log_reader.read()
assert type(blocks) is list
for block in blocks:
assert type(block) is PostgresNodeLogReader.LogDataBlock
if 'Is another postmaster already running on port' in block.data:
return True
return False
================================================
FILE: src/node_app.py
================================================
from .node import OsOperations
from .node import LocalOperations
from .node import PostgresNode
from .node import PortManager
import os
import platform
import tempfile
import typing
T_DICT_STR_STR = typing.Dict[str, str]
T_LIST_STR = typing.List[str]
class NodeApp:
_test_path: str
_os_ops: OsOperations
_port_manager: PortManager
_nodes_to_cleanup: typing.List[PostgresNode]
def __init__(
self,
test_path: typing.Optional[str] = None,
nodes_to_cleanup: typing.Optional[list] = None,
os_ops: typing.Optional[OsOperations] = None,
port_manager: typing.Optional[PortManager] = None,
):
assert test_path is None or type(test_path) is str
assert os_ops is None or isinstance(os_ops, OsOperations)
assert port_manager is None or isinstance(port_manager, PortManager)
if os_ops is None:
os_ops = LocalOperations.get_single_instance()
assert isinstance(os_ops, OsOperations)
self._os_ops = os_ops
self._port_manager = port_manager
if test_path is None:
self._test_path = os_ops.cwd()
elif os.path.isabs(test_path):
self._test_path = test_path
else:
self._test_path = os_ops.build_path(os_ops.cwd(), test_path)
if nodes_to_cleanup is None:
self._nodes_to_cleanup = []
else:
self._nodes_to_cleanup = nodes_to_cleanup
@property
def test_path(self) -> str:
assert type(self._test_path) is str
return self._test_path
@property
def os_ops(self) -> OsOperations:
assert isinstance(self._os_ops, OsOperations)
return self._os_ops
@property
def port_manager(self) -> PortManager:
assert self._port_manager is None or isinstance(self._port_manager, PortManager)
return self._port_manager
@property
def nodes_to_cleanup(self) -> typing.List[PostgresNode]:
assert type(self._nodes_to_cleanup) is list
return self._nodes_to_cleanup
def make_empty(
self,
base_dir: str,
port: typing.Optional[int] = None,
bin_dir: typing.Optional[str] = None
) -> PostgresNode:
assert type(base_dir) is str
assert port is None or type(port) is int
assert bin_dir is None or type(bin_dir) is str
assert isinstance(self._os_ops, OsOperations)
assert type(self._test_path) is str
if base_dir is None:
raise ValueError("Argument 'base_dir' is not defined.")
if base_dir == "":
raise ValueError("Argument 'base_dir' is empty.")
real_base_dir = self._os_ops.build_path(self._test_path, base_dir)
self._os_ops.rmdirs(real_base_dir, ignore_errors=True)
self._os_ops.makedirs(real_base_dir)
port_manager: typing.Optional[PortManager] = None
if port is None:
port_manager = self._port_manager
node = PostgresNode(
base_dir=real_base_dir,
port=port,
bin_dir=bin_dir,
os_ops=self._os_ops,
port_manager=port_manager
)
try:
assert type(self._nodes_to_cleanup) is list
self._nodes_to_cleanup.append(node)
except: # noqa: E722
node.cleanup(release_resources=True)
raise
return node
def make_simple(
self,
base_dir: str,
port: typing.Optional[int] = None,
set_replication: bool = False,
ptrack_enable: bool = False,
initdb_params: typing.Optional[T_LIST_STR] = None,
pg_options: typing.Optional[T_DICT_STR_STR] = None,
checksum: bool = True,
bin_dir: typing.Optional[str] = None
) -> PostgresNode:
assert type(base_dir) is str
assert port is None or type(port) is int
assert type(set_replication) is bool
assert type(ptrack_enable) is bool
assert initdb_params is None or type(initdb_params) is list
assert pg_options is None or type(pg_options) is dict
assert type(checksum) is bool
assert bin_dir is None or type(bin_dir) is str
node = self.make_empty(
base_dir,
port,
bin_dir=bin_dir
)
final_initdb_params = initdb_params
if checksum:
final_initdb_params = __class__._paramlist_append_if_not_exist(
initdb_params,
final_initdb_params,
'--data-checksums'
)
assert final_initdb_params is not None
assert '--data-checksums' in final_initdb_params
node.init(
initdb_params=final_initdb_params,
allow_streaming=set_replication
)
# set major version
pg_version_file = self._os_ops.read(self._os_ops.build_path(node.data_dir, 'PG_VERSION'))
node.major_version_str = str(pg_version_file.rstrip())
node.major_version = float(node.major_version_str)
# Set default parameters
options = {
'max_connections': 100,
'shared_buffers': '10MB',
'fsync': 'off',
'wal_level': 'logical',
'hot_standby': 'off',
'log_line_prefix': '%t [%p]: [%l-1] ',
'log_statement': 'none',
'log_duration': 'on',
'log_min_duration_statement': 0,
'log_connections': 'on',
'log_disconnections': 'on',
'restart_after_crash': 'off',
'autovacuum': 'off',
# unix_socket_directories will be defined later
}
# Allow replication in pg_hba.conf
if set_replication:
options['max_wal_senders'] = 10
if ptrack_enable:
options['ptrack.map_size'] = '1'
options['shared_preload_libraries'] = 'ptrack'
if node.major_version >= 13:
options['wal_keep_size'] = '200MB'
else:
options['wal_keep_segments'] = '12'
# Apply given parameters
if pg_options is not None:
assert type(pg_options) is dict
for option_name, option_value in pg_options.items():
options[option_name] = option_value
# Define delayed propertyes
if "unix_socket_directories" not in options.keys():
options["unix_socket_directories"] = __class__._gettempdir_for_socket()
# Set config values
node.set_auto_conf(options)
# kludge for testgres
# https://github.com/postgrespro/testgres/issues/54
# for PG >= 13 remove 'wal_keep_segments' parameter
if node.major_version >= 13:
node.set_auto_conf({}, 'postgresql.conf', ['wal_keep_segments'])
return node
@staticmethod
def _paramlist_has_param(
params: typing.Optional[T_LIST_STR],
param: str
) -> bool:
assert type(param) is str
if params is None:
return False
assert type(params) is list
return param in params
@staticmethod
def _paramlist_append(
user_params: typing.Optional[T_LIST_STR],
updated_params: typing.Optional[T_LIST_STR],
param: str,
) -> T_LIST_STR:
assert user_params is None or type(user_params) is list
assert updated_params is None or type(updated_params) is list
assert type(param) is str
if updated_params is None:
if user_params is None:
return [param]
return [*user_params, param]
assert updated_params is not None
if updated_params is user_params:
return [*user_params, param]
updated_params.append(param)
return updated_params
@staticmethod
def _paramlist_append_if_not_exist(
user_params: typing.Optional[T_LIST_STR],
updated_params: typing.Optional[T_LIST_STR],
param: str,
) -> typing.Optional[T_LIST_STR]:
if __class__._paramlist_has_param(updated_params, param):
return updated_params
return __class__._paramlist_append(user_params, updated_params, param)
@staticmethod
def _gettempdir_for_socket() -> str:
platform_system_name = platform.system().lower()
if platform_system_name == "windows":
return __class__._gettempdir()
#
# [2025-02-17] Hot fix.
#
# Let's use hard coded path as Postgres likes.
#
# pg_config_manual.h:
#
# #ifndef WIN32
# #define DEFAULT_PGSOCKET_DIR "/tmp"
# #else
# #define DEFAULT_PGSOCKET_DIR ""
# #endif
#
# On the altlinux-10 tempfile.gettempdir() may return
# the path to "private" temp directiry - "/temp/.private/<username>/"
#
# But Postgres want to find a socket file in "/tmp" (see above).
#
return "/tmp"
@staticmethod
def _gettempdir() -> str:
v = tempfile.gettempdir()
#
# Paranoid checks
#
if type(v) is str:
__class__._raise_bugcheck("tempfile.gettempdir returned a value with type {0}.".format(type(v).__name__))
if v == "":
__class__._raise_bugcheck("tempfile.gettempdir returned an empty string.")
if not os.path.exists(v):
__class__._raise_bugcheck("tempfile.gettempdir returned a not exist path [{0}].".format(v))
# OK
return v
@staticmethod
def _raise_bugcheck(msg):
assert type(msg) is str
assert msg != ""
raise Exception("[BUG CHECK] " + msg)
================================================
FILE: src/port_manager.py
================================================
class PortManager:
def __init__(self):
super().__init__()
def reserve_port(self) -> int:
raise NotImplementedError("PortManager::reserve_port is not implemented.")
def release_port(self, number: int) -> None:
assert type(number) is int
raise NotImplementedError("PortManager::release_port is not implemented.")
================================================
FILE: src/pubsub.py
================================================
# coding: utf-8
"""
Unlike physical replication the logical replication allows users replicate only
specified databases and tables. It uses publish-subscribe model with possibly
multiple publishers and multiple subscribers. When initializing publisher's
node ``allow_logical=True`` should be passed to the :meth:`.PostgresNode.init()`
method to enable PostgreSQL to write extra information to the WAL needed by
logical replication.
To replicate table ``X`` from node A to node B the same table structure should
be defined on the subscriber's node as logical replication don't replicate DDL.
After that :meth:`~.PostgresNode.publish()` and :meth:`~.PostgresNode.subscribe()`
methods may be used to setup replication. Example:
>>> from testgres import get_new_node
>>> with get_new_node() as nodeA, get_new_node() as nodeB:
... nodeA.init(allow_logical=True).start()
... nodeB.init().start()
...
... # create same table both on publisher and subscriber
... create_table = 'create table test (a int, b int)'
... nodeA.safe_psql(create_table)
... nodeB.safe_psql(create_table)
...
... # create publication
... pub = nodeA.publish('mypub')
... # create subscription
... sub = nodeB.subscribe(pub, 'mysub')
...
... # insert some data to the publisher's node
... nodeA.execute('insert into test values (1, 1), (2, 2)')
...
... # wait until changes apply on subscriber and check them
... sub.catchup()
...
... # read the data from subscriber's node
... nodeB.execute('select * from test')
PostgresNode(name='...', port=..., base_dir='...')
PostgresNode(name='...', port=..., base_dir='...')
''
''
[(1, 1), (2, 2)]
"""
from six import raise_from
from .consts import LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS
from .defaults import default_dbname, default_username
from .exceptions import CatchUpException
from .utils import options_string
class Publication(object):
def __init__(self, name, node, tables=None, dbname=None, username=None):
"""
Constructor. Use :meth:`.PostgresNode.publish()` instead of direct
constructing publication objects.
Args:
name: publication name.
node: publisher's node.
tables: tables list or None for all tables.
dbname: database name used to connect and perform subscription.
username: username used to connect to the database.
"""
self.name = name
self.node = node
self.dbname = dbname or default_dbname()
self.username = username or default_username()
# create publication in database
t = "table " + ", ".join(tables) if tables else "all tables"
query = "create publication {} for {}"
node.execute(query.format(name, t), dbname=dbname, username=username)
def drop(self, dbname=None, username=None):
"""
Drop publication
"""
self.node.execute("drop publication {}".format(self.name),
dbname=dbname,
username=username)
def add_tables(self, tables, dbname=None, username=None):
"""
Add tables to the publication. Cannot be used if publication was
created with empty tables list.
Args:
tables: a list of tables to be added to the publication.
"""
if not tables:
raise ValueError("Tables list is empty")
query = "alter publication {} add table {}"
self.node.execute(query.format(self.name, ", ".join(tables)),
dbname=dbname or self.dbname,
username=username or self.username)
class Subscription(object):
def __init__(self,
node,
publication,
name=None,
dbname=None,
username=None,
**params):
"""
Constructor. Use :meth:`.PostgresNode.subscribe()` instead of direct
constructing subscription objects.
Args:
name: subscription name.
node: subscriber's node.
publication: :class:`.Publication` object we are subscribing to
(see :meth:`.PostgresNode.publish()`).
dbname: database name used to connect and perform subscription.
username: username used to connect to the database.
params: subscription parameters (see documentation on `CREATE SUBSCRIPTION
<https://www.postgresql.org/docs/current/static/sql-createsubscription.html>`_
for details).
"""
self.name = name
self.node = node
self.pub = publication
# connection info
conninfo = {
"dbname": self.pub.dbname,
"user": self.pub.username,
"host": self.pub.node.host,
"port": self.pub.node.port
}
query = (
"create subscription {} connection '{}' publication {}").format(
name, options_string(**conninfo), self.pub.name)
# additional parameters
if params:
query += " with ({})".format(options_string(**params))
# Note: cannot run 'create subscription' query in transaction mode
node.execute(query, dbname=dbname, username=username)
def disable(self, dbname=None, username=None):
"""
Disables the running subscription.
"""
query = "alter subscription {} disable"
self.node.execute(query.format(self.name), dbname=None, username=None)
def enable(self, dbname=None, username=None):
"""
Enables the previously disabled subscription.
"""
query = "alter subscription {} enable"
self.node.execute(query.format(self.name), dbname=None, username=None)
def refresh(self, copy_data=True, dbname=None, username=None):
"""
Disables the running subscription.
"""
query = "alter subscription {} refresh publication with (copy_data={})"
self.node.execute(query.format(self.name, copy_data),
dbname=dbname,
username=username)
def drop(self, dbname=None, username=None):
"""
Drops subscription
"""
self.node.execute("drop subscription {}".format(self.name),
dbname=dbname,
username=username)
def catchup(self, username=None):
"""
Wait until subscription catches up with publication.
Args:
username: remote node's user name.
"""
try:
pub_lsn = self.pub.node.execute(query="select pg_current_wal_lsn()",
dbname=None,
username=None)[0][0] # yapf: disable
# create dummy xact, as LR replicates only on commit.
self.pub.node.execute(query="select txid_current()",
dbname=None,
username=None)
query = """
select '{}'::pg_lsn - replay_lsn <= 0
from pg_catalog.pg_stat_replication where application_name = '{}'
""".format(pub_lsn, self.name)
# wait until this LSN reaches subscriber
self.pub.node.poll_query_until(
query=query,
dbname=self.pub.dbname,
username=username or self.pub.username,
max_attempts=LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS)
# Now, wait until there are no tablesync workers: probably
# replay_lsn above was sent with changes of new tables just skipped;
# they will be eaten by tablesync workers.
query = """
select count(*) = 0 from pg_subscription_rel where srsubstate != 'r'
"""
self.node.poll_query_until(
query=query,
dbname=self.pub.dbname,
gitextract_6f31v0iv/
├── .coveragerc
├── .dockerignore
├── .github/
│ ├── dependabot.yml
│ └── workflows/
│ ├── package-verification.yml
│ └── python-publish.yml
├── .gitignore
├── .style.yapf
├── Dockerfile--altlinux_10.tmpl
├── Dockerfile--altlinux_11.tmpl
├── Dockerfile--astralinux_1_7.tmpl
├── Dockerfile--std-all.tmpl
├── Dockerfile--std.tmpl
├── Dockerfile--std2-all.tmpl
├── Dockerfile--ubuntu_24_04.tmpl
├── LICENSE
├── README.md
├── docs/
│ ├── Makefile
│ ├── README.md
│ └── source/
│ ├── conf.py
│ ├── index.rst
│ └── testgres.rst
├── pyproject.toml
├── run_tests.sh
├── run_tests2.sh
├── src/
│ ├── __init__.py
│ ├── api.py
│ ├── backup.py
│ ├── cache.py
│ ├── config.py
│ ├── connection.py
│ ├── consts.py
│ ├── decorators.py
│ ├── defaults.py
│ ├── enums.py
│ ├── exceptions.py
│ ├── impl/
│ │ ├── internal_utils.py
│ │ ├── platforms/
│ │ │ ├── internal_platform_utils.py
│ │ │ ├── internal_platform_utils_factory.py
│ │ │ ├── linux/
│ │ │ │ └── internal_platform_utils.py
│ │ │ └── win32/
│ │ │ └── internal_platform_utils.py
│ │ ├── port_manager__generic.py
│ │ └── port_manager__this_host.py
│ ├── logger.py
│ ├── node.py
│ ├── node_app.py
│ ├── port_manager.py
│ ├── pubsub.py
│ ├── raise_error.py
│ ├── standby.py
│ └── utils.py
└── tests/
├── README.md
├── __init__.py
├── conftest.py
├── helpers/
│ ├── __init__.py
│ ├── global_data.py
│ ├── pg_node_utils.py
│ ├── run_conditions.py
│ └── utils.py
├── requirements.txt
├── test_config.py
├── test_conftest.py--devel
├── test_os_ops_common.py
├── test_os_ops_local.py
├── test_os_ops_remote.py
├── test_raise_error.py
├── test_testgres_common.py
├── test_testgres_local.py
├── test_testgres_remote.py
├── test_utils.py
└── units/
├── __init__.py
├── exceptions/
│ ├── BackupException/
│ │ ├── __init__.py
│ │ └── test_set001__constructor.py
│ ├── CatchUpException/
│ │ ├── __init__.py
│ │ └── test_set001__constructor.py
│ ├── InitNodeException/
│ │ ├── __init__.py
│ │ └── test_set001__constructor.py
│ ├── PortForException/
│ │ ├── __init__.py
│ │ └── test_set001__constructor.py
│ ├── QueryException/
│ │ ├── __init__.py
│ │ └── test_set001__constructor.py
│ ├── QueryTimeoutException/
│ │ ├── __init__.py
│ │ └── test_set001__constructor.py
│ ├── StartNodeException/
│ │ ├── __init__.py
│ │ └── test_set001__constructor.py
│ ├── TimeoutException/
│ │ ├── __init__.py
│ │ └── test_set001.py
│ └── __init__.py
├── impl/
│ ├── __init__.py
│ └── platforms/
│ ├── __init__.py
│ └── internal_platform_utils/
│ ├── InternalPlatformUtils/
│ │ └── __init__.py
│ └── __init__.py
└── node/
├── PostgresNode/
│ ├── __init__.py
│ ├── test_setM001__start.py
│ └── test_setM002__start2.py
└── __init__.py
SYMBOL INDEX (628 symbols across 48 files)
FILE: src/api.py
function get_new_node (line 36) | def get_new_node(name=None, base_dir=None, **kwargs):
function get_remote_node (line 45) | def get_remote_node(name=None, conn_params=None):
FILE: src/backup.py
class NodeBackup (line 24) | class NodeBackup(object):
method log_file (line 29) | def log_file(self):
method __init__ (line 34) | def __init__(self,
method __enter__ (line 91) | def __enter__(self):
method __exit__ (line 94) | def __exit__(self, type, value, traceback):
method _prepare_dir (line 97) | def _prepare_dir(self, destroy):
method spawn_primary (line 137) | def spawn_primary(self, name=None, destroy=True):
method spawn_replica (line 172) | def spawn_replica(self, name=None, destroy=True, slot=None):
method cleanup (line 200) | def cleanup(self):
FILE: src/cache.py
function cached_initdb (line 23) | def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperati...
FILE: src/config.py
class GlobalConfig (line 20) | class GlobalConfig(object):
method cached_initdb_dir (line 57) | def cached_initdb_dir(self):
method cached_initdb_dir (line 62) | def cached_initdb_dir(self, value):
method temp_dir (line 70) | def temp_dir(self):
method temp_dir (line 75) | def temp_dir(self, value):
method __init__ (line 78) | def __init__(self, **options):
method __setitem__ (line 81) | def __setitem__(self, key, value):
method __getitem__ (line 84) | def __getitem__(self, key):
method __setattr__ (line 87) | def __setattr__(self, name, value):
method keys (line 93) | def keys(self):
method items (line 106) | def items(self):
method update (line 113) | def update(self, config):
method copy (line 125) | def copy(self):
method set_os_ops (line 133) | def set_os_ops(os_ops: OsOperations):
function _rm_cached_initdb_dirs (line 152) | def _rm_cached_initdb_dirs():
function push_config (line 157) | def push_config(**options):
function pop_config (line 169) | def pop_config():
function scoped_config (line 182) | def scoped_config(**options):
function configure_testgres (line 207) | def configure_testgres(**options):
FILE: src/connection.py
class NodeConnection (line 28) | class NodeConnection(object):
method __init__ (line 32) | def __init__(self,
method node (line 57) | def node(self):
method connection (line 61) | def connection(self):
method pid (line 65) | def pid(self):
method cursor (line 69) | def cursor(self):
method __enter__ (line 72) | def __enter__(self):
method __exit__ (line 75) | def __exit__(self, type, value, traceback):
method begin (line 78) | def begin(self, isolation_level=IsolationLevel.ReadCommitted):
method commit (line 97) | def commit(self):
method rollback (line 102) | def rollback(self):
method execute (line 107) | def execute(self, query, *args):
method close (line 119) | def close(self):
FILE: src/decorators.py
function positional_args_hack (line 5) | def positional_args_hack(*special_cases):
function method_decorator (line 49) | def method_decorator(decorator):
FILE: src/defaults.py
function default_dbname (line 8) | def default_dbname():
function default_username (line 16) | def default_username():
function generate_app_name (line 23) | def generate_app_name():
function generate_system_id (line 31) | def generate_system_id():
FILE: src/enums.py
class XLogMethod (line 6) | class XLogMethod(Enum):
class IsolationLevel (line 16) | class IsolationLevel(Enum):
class NodeStatus (line 27) | class NodeStatus(IntEnum):
method __bool__ (line 35) | def __bool__(self):
class ProcessType (line 42) | class ProcessType(Enum):
method from_process (line 61) | def from_process(process):
class DumpFormat (line 95) | class DumpFormat(Enum):
FILE: src/exceptions.py
class PortForException (line 11) | class PortForException(TestgresException):
method __init__ (line 14) | def __init__(
method message (line 24) | def message(self) -> str:
method __repr__ (line 30) | def __repr__(self) -> str:
class QueryException (line 47) | class QueryException(TestgresException):
method __init__ (line 51) | def __init__(
method message (line 66) | def message(self) -> str:
method description (line 83) | def description(self) -> typing.Optional[str]:
method query (line 88) | def query(self) -> typing.Optional[str]:
method __repr__ (line 92) | def __repr__(self) -> str:
class QueryTimeoutException (line 111) | class QueryTimeoutException(QueryException):
method __init__ (line 112) | def __init__(
class CatchUpException (line 129) | class CatchUpException(TestgresException):
method __init__ (line 132) | def __init__(
method message (line 142) | def message(self) -> str:
method __repr__ (line 148) | def __repr__(self) -> str:
class StartNodeException (line 165) | class StartNodeException(TestgresException):
method __init__ (line 169) | def __init__(
method message (line 184) | def message(self) -> str:
method description (line 201) | def description(self) -> typing.Optional[str]:
method files (line 206) | def files(self) -> typing.Optional[typing.Iterable]:
method __repr__ (line 210) | def __repr__(self) -> str:
class InitNodeException (line 229) | class InitNodeException(TestgresException):
method __init__ (line 232) | def __init__(
method message (line 242) | def message(self) -> str:
method __repr__ (line 248) | def __repr__(self) -> str:
class BackupException (line 264) | class BackupException(TestgresException):
method __init__ (line 267) | def __init__(
method message (line 277) | def message(self) -> str:
method __repr__ (line 283) | def __repr__(self) -> str:
FILE: src/impl/internal_utils.py
function send_log (line 4) | def send_log(level: int, msg: str) -> None:
function send_log_info (line 11) | def send_log_info(msg: str) -> None:
function send_log_debug (line 17) | def send_log_debug(msg: str) -> None:
FILE: src/impl/platforms/internal_platform_utils.py
class InternalPlatformUtils (line 9) | class InternalPlatformUtils:
class FindPostmasterResultCode (line 10) | class FindPostmasterResultCode(enum.Enum):
class FindPostmasterResult (line 17) | class FindPostmasterResult:
method __init__ (line 21) | def __init__(
method create_ok (line 33) | def create_ok(pid: int) -> InternalPlatformUtils.FindPostmasterResult:
method create_not_found (line 38) | def create_not_found() -> InternalPlatformUtils.FindPostmasterResult:
method create_not_implemented (line 42) | def create_not_implemented() -> InternalPlatformUtils.FindPostmaster...
method create_many_processes (line 46) | def create_many_processes() -> InternalPlatformUtils.FindPostmasterR...
method create_has_problems (line 50) | def create_has_problems() -> InternalPlatformUtils.FindPostmasterRes...
method FindPostmaster (line 53) | def FindPostmaster(
FILE: src/impl/platforms/internal_platform_utils_factory.py
function create_internal_platform_utils (line 6) | def create_internal_platform_utils(
FILE: src/impl/platforms/linux/internal_platform_utils.py
class InternalPlatformUtils (line 13) | class InternalPlatformUtils(base.InternalPlatformUtils):
method FindPostmaster (line 22) | def FindPostmaster(
FILE: src/impl/platforms/win32/internal_platform_utils.py
class InternalPlatformUtils (line 7) | class InternalPlatformUtils(base.InternalPlatformUtils):
method FindPostmaster (line 8) | def FindPostmaster(
FILE: src/impl/port_manager__generic.py
class PortManager__Generic (line 12) | class PortManager__Generic(PortManager):
method __init__ (line 22) | def __init__(self, os_ops: OsOperations):
method reserve_port (line 39) | def reserve_port(self) -> int:
method release_port (line 70) | def release_port(self, number: int) -> None:
method helper__send_debug_msg (line 89) | def helper__send_debug_msg(msg_template: str, *args) -> None:
FILE: src/impl/port_manager__this_host.py
class PortManager__ThisHost (line 8) | class PortManager__ThisHost(PortManager):
method get_single_instance (line 13) | def get_single_instance() -> PortManager:
method reserve_port (line 28) | def reserve_port(self) -> int:
method release_port (line 31) | def release_port(self, number: int) -> None:
FILE: src/logger.py
class TestgresLogger (line 9) | class TestgresLogger(threading.Thread):
method __init__ (line 13) | def __init__(self, node_name, log_file_name):
method run (line 22) | def run(self):
method stop (line 45) | def stop(self, wait=True):
FILE: src/node.py
class ProcessProxy (line 116) | class ProcessProxy(object):
method __init__ (line 128) | def __init__(self, process, ptype: typing.Optional[ProcessType] = None):
method __getattr__ (line 140) | def __getattr__(self, name):
method __repr__ (line 143) | def __repr__(self):
method process (line 150) | def process(self) -> typing.Any:
method ptype (line 155) | def ptype(self) -> ProcessType:
class PostgresNode (line 160) | class PostgresNode(object):
method __init__ (line 173) | def __init__(self,
method __enter__ (line 260) | def __enter__(self):
method __exit__ (line 263) | def __exit__(self, type, value, traceback):
method __repr__ (line 279) | def __repr__(self):
method _get_os_ops (line 288) | def _get_os_ops() -> OsOperations:
method _get_port_manager (line 295) | def _get_port_manager(os_ops: OsOperations) -> PortManager:
method clone_with_new_name_and_base_dir (line 308) | def clone_with_new_name_and_base_dir(self, name: str, base_dir: str):
method os_ops (line 333) | def os_ops(self) -> OsOperations:
method port_manager (line 339) | def port_manager(self) -> typing.Optional[PortManager]:
method name (line 344) | def name(self) -> str:
method host (line 351) | def host(self) -> str:
method port (line 357) | def port(self) -> int:
method ssh_key (line 365) | def ssh_key(self) -> typing.Optional[str]:
method pid (line 371) | def pid(self) -> int:
method is_started (line 388) | def is_started(self) -> bool:
method auxiliary_pids (line 396) | def auxiliary_pids(self) -> typing.Dict[ProcessType, typing.List[int]]:
method auxiliary_processes (line 413) | def auxiliary_processes(self) -> typing.List[ProcessProxy]:
method child_processes (line 425) | def child_processes(self) -> typing.List[ProcessProxy]:
method _get_child_processes (line 444) | def _get_child_processes(self, pid: int) -> typing.List[ProcessProxy]:
method source_walsender (line 454) | def source_walsender(self):
method master (line 483) | def master(self):
method base_dir (line 487) | def base_dir(self):
method bin_dir (line 498) | def bin_dir(self):
method logs_dir (line 504) | def logs_dir(self):
method data_dir (line 518) | def data_dir(self):
method utils_log_file (line 528) | def utils_log_file(self):
method pg_log_file (line 537) | def pg_log_file(self):
method version (line 546) | def version(self):
method _try_shutdown (line 555) | def _try_shutdown(self, max_attempts, with_force=False):
method _throw_bugcheck__unexpected_result_of_ps (line 625) | def _throw_bugcheck__unexpected_result_of_ps(result, cmd):
method _assign_master (line 635) | def _assign_master(self, master):
method _create_recovery_conf (line 641) | def _create_recovery_conf(self, username, slot=None):
method _maybe_start_logger (line 705) | def _maybe_start_logger(self):
method _maybe_stop_logger (line 712) | def _maybe_stop_logger(self):
method _collect_special_files (line 716) | def _collect_special_files(self) -> typing.List[typing.Tuple[str, byte...
method init (line 744) | def init(self, initdb_params=None, cached=True, **kwargs):
method default_conf (line 775) | def default_conf(self,
method append_conf (line 882) | def append_conf(self, line='', filename=PG_CONF_FILE, **kwargs):
method status (line 922) | def status(self):
method _get_node_state (line 933) | def _get_node_state(self) -> utils.PostgresNodeState:
method get_control_data (line 941) | def get_control_data(self):
method slow_start (line 961) | def slow_start(self, replica=False, dbname='template1', username=None,...
method start (line 1004) | def start(
method start2 (line 1040) | def start2(
method _start (line 1066) | def _start(
method _raise_cannot_start_node (line 1158) | def _raise_cannot_start_node(
method stop (line 1168) | def stop(self, params=[], wait=True):
method kill (line 1193) | def kill(self, someone=None):
method restart (line 1223) | def restart(self, params=[]):
method reload (line 1255) | def reload(self, params=[]):
method promote (line 1276) | def promote(self, dbname=None, username=None):
method pg_ctl (line 1312) | def pg_ctl(self, params):
method release_resources (line 1331) | def release_resources(self):
method free_port (line 1337) | def free_port(self):
method cleanup (line 1344) | def cleanup(self, max_attempts=3, full=False, release_resources=False):
method psql (line 1373) | def psql(self,
method _psql (line 1420) | def _psql(
method safe_psql (line 1489) | def safe_psql(self, query=None, expect_error=False, **kwargs):
method dump (line 1531) | def dump(self,
method restore (line 1588) | def restore(self, filename, dbname=None, username=None):
method poll_query_until (line 1618) | def poll_query_until(self,
method execute (line 1687) | def execute(self,
method backup (line 1716) | def backup(self, **kwargs):
method replicate (line 1731) | def replicate(self, name=None, slot=None, **kwargs):
method set_synchronous_standbys (line 1747) | def set_synchronous_standbys(self, standbys):
method catchup (line 1786) | def catchup(self, dbname=None, username=None):
method publish (line 1815) | def publish(self, name, **kwargs):
method subscribe (line 1827) | def subscribe(self,
method pgbench (line 1850) | def pgbench(self,
method pgbench_with_wait (line 1888) | def pgbench_with_wait(self,
method pgbench_init (line 1911) | def pgbench_init(self, **kwargs):
method pgbench_run (line 1924) | def pgbench_run(self, dbname=None, username=None, options=[], **kwargs):
method connect (line 1970) | def connect(self,
method table_checksum (line 1996) | def table_checksum(
method pgbench_table_checksums (line 2024) | def pgbench_table_checksums(
method set_auto_conf (line 2038) | def set_auto_conf(self, options, config='postgresql.auto.conf', rm_opt...
method upgrade_from (line 2108) | def upgrade_from(self, old_node, options=None, expect_error=False):
method _release_resources (line 2143) | def _release_resources(self):
method _free_port (line 2146) | def _free_port(self):
method _get_bin_path (line 2162) | def _get_bin_path(self, filename):
method _escape_config_value (line 2173) | def _escape_config_value(value):
method _tables_checksum (line 2197) | def _tables_checksum(
method _table_checksum__use_cn (line 2228) | def _table_checksum__use_cn(
method _delim_sql_ident (line 2258) | def _delim_sql_ident(name: str) -> str:
class PostgresNodeLogReader (line 2274) | class PostgresNodeLogReader:
class LogInfo (line 2275) | class LogInfo:
method __init__ (line 2278) | def __init__(self, position: int):
class LogDataBlock (line 2282) | class LogDataBlock:
method __init__ (line 2287) | def __init__(
method file_name (line 2303) | def file_name(self) -> str:
method position (line 2309) | def position(self) -> int:
method data (line 2315) | def data(self) -> str:
method __init__ (line 2324) | def __init__(self, node: PostgresNode, from_beginnig: bool):
method read (line 2339) | def read(self) -> typing.List[LogDataBlock]:
method _collect_logs (line 2392) | def _collect_logs(self) -> typing.Dict[str, LogInfo]:
class PostgresNodeUtils (line 2418) | class PostgresNodeUtils:
method delect_port_conflict (line 2420) | def delect_port_conflict(log_reader: PostgresNodeLogReader) -> bool:
FILE: src/node_app.py
class NodeApp (line 16) | class NodeApp:
method __init__ (line 22) | def __init__(
method test_path (line 53) | def test_path(self) -> str:
method os_ops (line 58) | def os_ops(self) -> OsOperations:
method port_manager (line 63) | def port_manager(self) -> PortManager:
method nodes_to_cleanup (line 68) | def nodes_to_cleanup(self) -> typing.List[PostgresNode]:
method make_empty (line 72) | def make_empty(
method make_simple (line 117) | def make_simple(
method _paramlist_has_param (line 217) | def _paramlist_has_param(
method _paramlist_append (line 231) | def _paramlist_append(
method _paramlist_append_if_not_exist (line 254) | def _paramlist_append_if_not_exist(
method _gettempdir_for_socket (line 264) | def _gettempdir_for_socket() -> str:
method _gettempdir (line 292) | def _gettempdir() -> str:
method _raise_bugcheck (line 311) | def _raise_bugcheck(msg):
FILE: src/port_manager.py
class PortManager (line 1) | class PortManager:
method __init__ (line 2) | def __init__(self):
method reserve_port (line 5) | def reserve_port(self) -> int:
method release_port (line 8) | def release_port(self, number: int) -> None:
FILE: src/pubsub.py
class Publication (line 53) | class Publication(object):
method __init__ (line 54) | def __init__(self, name, node, tables=None, dbname=None, username=None):
method drop (line 76) | def drop(self, dbname=None, username=None):
method add_tables (line 84) | def add_tables(self, tables, dbname=None, username=None):
class Subscription (line 101) | class Subscription(object):
method __init__ (line 102) | def __init__(self,
method disable (line 147) | def disable(self, dbname=None, username=None):
method enable (line 154) | def enable(self, dbname=None, username=None):
method refresh (line 161) | def refresh(self, copy_data=True, dbname=None, username=None):
method drop (line 170) | def drop(self, dbname=None, username=None):
method catchup (line 178) | def catchup(self, username=None):
FILE: src/raise_error.py
class RaiseError (line 7) | class RaiseError:
method pg_ctl_returns_an_empty_string (line 9) | def pg_ctl_returns_an_empty_string(_params):
method pg_ctl_returns_an_unexpected_string (line 16) | def pg_ctl_returns_an_unexpected_string(out, _params):
method pg_ctl_returns_a_zero_pid (line 25) | def pg_ctl_returns_a_zero_pid(out, _params):
method node_err__cant_enumerate_child_processes (line 34) | def node_err__cant_enumerate_child_processes(
method node_err__cant_kill (line 49) | def node_err__cant_kill(
method _map_node_status_to_reason (line 64) | def _map_node_status_to_reason(
FILE: src/standby.py
class First (line 7) | class First:
method __init__ (line 19) | def __init__(self, sync_num, standbys):
method __str__ (line 23) | def __str__(self):
class Any (line 30) | class Any:
method __init__ (line 42) | def __init__(self, sync_num, standbys):
method __str__ (line 46) | def __str__(self):
FILE: src/utils.py
class PgVer (line 47) | class PgVer(Version):
method __init__ (line 48) | def __init__(self, version: str) -> None:
function internal__reserve_port (line 56) | def internal__reserve_port():
function internal__release_port (line 63) | def internal__release_port(port):
function execute_utility (line 76) | def execute_utility(args, logfile=None, verbose=False):
function execute_utility2 (line 90) | def execute_utility2(
function get_bin_path (line 130) | def get_bin_path(filename):
function get_bin_path2 (line 138) | def get_bin_path2(os_ops: OsOperations, filename):
function get_pg_config (line 168) | def get_pg_config(pg_config_path=None, os_ops=None):
function get_pg_config2 (line 180) | def get_pg_config2(os_ops: OsOperations, pg_config_path):
function get_pg_version2 (line 232) | def get_pg_version2(os_ops: OsOperations, bin_dir=None):
function get_pg_version (line 256) | def get_pg_version(bin_dir=None):
function parse_pg_version (line 264) | def parse_pg_version(version_out):
function file_tail (line 276) | def file_tail(f, num_lines):
function eprint (line 303) | def eprint(*args, **kwargs):
function options_string (line 310) | def options_string(separator=u" ", **kwargs):
function clean_on_error (line 315) | def clean_on_error(node):
class PostgresNodeState (line 329) | class PostgresNodeState:
method __init__ (line 333) | def __init__(
function get_pg_node_state (line 346) | def get_pg_node_state(
FILE: tests/conftest.py
class T_TEST_PROCESS_KIND (line 46) | class T_TEST_PROCESS_KIND(enum.Enum):
class T_TEST_PROCESS_MODE (line 55) | class T_TEST_PROCESS_MODE(enum.Enum):
class TestConfigPropNames (line 71) | class TestConfigPropNames:
class TestStartupData__Helper (line 79) | class TestStartupData__Helper:
method GetStartTS (line 84) | def GetStartTS() -> datetime.datetime:
method CalcRootDir (line 90) | def CalcRootDir() -> str:
method CalcRootLogDir (line 99) | def CalcRootLogDir() -> str:
method CalcCurrentTestWorkerSignature (line 111) | def CalcCurrentTestWorkerSignature() -> str:
class TestStartupData (line 140) | class TestStartupData:
method GetRootDir (line 150) | def GetRootDir() -> str:
method GetRootLogDir (line 156) | def GetRootLogDir() -> str:
method GetCurrentTestWorkerSignature (line 162) | def GetCurrentTestWorkerSignature() -> str:
class TEST_PROCESS_STATS (line 171) | class TEST_PROCESS_STATS:
method incrementTotalTestCount (line 197) | def incrementTotalTestCount() -> None:
method incrementNotExecutedTestCount (line 207) | def incrementNotExecutedTestCount() -> None:
method incrementExecutedTestCount (line 217) | def incrementExecutedTestCount() -> int:
method incrementPassedTestCount (line 228) | def incrementPassedTestCount() -> None:
method incrementFailedTestCount (line 238) | def incrementFailedTestCount(testID: str, errCount: int) -> None:
method incrementXFailedTestCount (line 263) | def incrementXFailedTestCount(testID: str, errCount: int) -> None:
method incrementSkippedTestCount (line 280) | def incrementSkippedTestCount() -> None:
method incrementNotXFailedTests (line 290) | def incrementNotXFailedTests(testID: str) -> None:
method incrementWarningTestCount (line 305) | def incrementWarningTestCount(testID: str, warningCount: int) -> None:
method incrementUnexpectedTests (line 331) | def incrementUnexpectedTests() -> None:
method incrementAchtungTestCount (line 341) | def incrementAchtungTestCount(testID: str) -> None:
function timedelta_to_human_text (line 358) | def timedelta_to_human_text(delta: datetime.timedelta) -> str:
function helper__build_test_id (line 383) | def helper__build_test_id(item: pytest.Function) -> str:
function helper__makereport__setup (line 400) | def helper__makereport__setup(
class ExitStatusNames (line 459) | class ExitStatusNames:
function helper__makereport__call (line 469) | def helper__makereport__call(
function pytest_runtest_makereport (line 646) | def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallIn...
class LogWrapper2 (line 691) | class LogWrapper2:
method __init__ (line 699) | def __init__(self):
method __enter__ (line 707) | def __enter__(self):
method __exit__ (line 727) | def __exit__(self, exc_type, exc_val, exc_tb):
method __call__ (line 746) | def __call__(self, record: logging.LogRecord):
class SIGNAL_EXCEPTION (line 779) | class SIGNAL_EXCEPTION(Exception):
method __init__ (line 780) | def __init__(self):
function pytest_pyfunc_call (line 788) | def pytest_pyfunc_call(pyfuncitem: pytest.Function):
function helper__calc_W (line 863) | def helper__calc_W(n: int) -> int:
function helper__print_test_list (line 874) | def helper__print_test_list(tests: typing.List[str]) -> None:
function helper__print_test_list2 (line 901) | def helper__print_test_list2(tests: typing.List[T_TUPLE__str_int]) -> None:
function pytest_sessionfinish (line 936) | def pytest_sessionfinish():
function helper__detect_test_process_kind (line 1074) | def helper__detect_test_process_kind(config: pytest.Config) -> T_TEST_PR...
function helper__detect_test_process_mode (line 1089) | def helper__detect_test_process_mode(config: pytest.Config) -> T_TEST_PR...
function helper__pytest_configure__logging (line 1100) | def helper__pytest_configure__logging(config: pytest.Config) -> None:
function pytest_configure (line 1125) | def pytest_configure(config: pytest.Config) -> None:
FILE: tests/helpers/global_data.py
class OsOpsDescr (line 13) | class OsOpsDescr:
method __init__ (line 17) | def __init__(self, sign: str, os_ops: OsOperations):
class OsOpsDescrs (line 24) | class OsOpsDescrs:
class PortManagers (line 39) | class PortManagers:
class PostgresNodeService (line 47) | class PostgresNodeService:
method __init__ (line 52) | def __init__(self, sign: str, os_ops: OsOperations, port_manager: Port...
class PostgresNodeServices (line 61) | class PostgresNodeServices:
FILE: tests/helpers/pg_node_utils.py
class PostgresNodeUtils (line 15) | class PostgresNodeUtils:
class PostgresNodeUtilsException (line 16) | class PostgresNodeUtilsException(Exception):
class PortConflictNodeException (line 19) | class PortConflictNodeException(PostgresNodeUtilsException):
method __init__ (line 23) | def __init__(self, data_dir: str, port: int):
method data_dir (line 34) | def data_dir(self) -> str:
method port (line 39) | def port(self) -> int:
method message (line 44) | def message(self) -> str:
method __str__ (line 55) | def __str__(self) -> str:
method __repr__ (line 60) | def __repr__(self) -> str:
class StartNodeException (line 72) | class StartNodeException(PostgresNodeUtilsException):
method __init__ (line 76) | def __init__(
method message (line 91) | def message(self) -> str:
method data_dir (line 109) | def data_dir(self) -> typing.Optional[str]:
method files (line 114) | def files(self) -> typing.Optional[typing.Iterable]:
method __repr__ (line 118) | def __repr__(self) -> str:
method get_node (line 132) | def get_node(
method wait_for_running_state (line 154) | def wait_for_running_state(
FILE: tests/helpers/run_conditions.py
class RunConditions (line 6) | class RunConditions:
method skip_if_windows (line 11) | def skip_if_windows():
FILE: tests/helpers/utils.py
class Utils (line 9) | class Utils:
method PrintAndSleep (line 11) | def PrintAndSleep(wait: T_WAIT_TIME):
method WaitUntil (line 18) | def WaitUntil(
FILE: tests/test_config.py
class TestConfig (line 11) | class TestConfig:
method test_config_stack (line 12) | def test_config_stack(self):
FILE: tests/test_os_ops_common.py
class TestOsOpsCommon (line 30) | class TestOsOpsCommon:
method os_ops (line 40) | def os_ops(self, request: pytest.FixtureRequest) -> OsOperations:
method test_get_platform (line 45) | def test_get_platform(self, os_ops: OsOperations):
method test_get_platform__is_known (line 52) | def test_get_platform__is_known(self, os_ops: OsOperations):
method test_create_clone (line 59) | def test_create_clone(self, os_ops: OsOperations):
method test_exec_command_success (line 66) | def test_exec_command_success(self, os_ops: OsOperations):
method test_exec_command_failure (line 80) | def test_exec_command_failure(self, os_ops: OsOperations):
method test_exec_command_failure__expect_error (line 108) | def test_exec_command_failure__expect_error(self, os_ops: OsOperations):
method test_exec_command_with_exec_env (line 126) | def test_exec_command_with_exec_env(self, os_ops: OsOperations):
method test_exec_command_with_exec_env__2 (line 147) | def test_exec_command_with_exec_env__2(self, os_ops: OsOperations):
method test_exec_command_with_cwd (line 185) | def test_exec_command_with_cwd(self, os_ops: OsOperations):
method test_exec_command__test_unset (line 202) | def test_exec_command__test_unset(self, os_ops: OsOperations):
method test_exec_command__test_unset_dummy_var (line 230) | def test_exec_command__test_unset_dummy_var(self, os_ops: OsOperations):
method test_is_executable_true (line 245) | def test_is_executable_true(self, os_ops: OsOperations):
method test_is_executable_false (line 257) | def test_is_executable_false(self, os_ops: OsOperations):
method test_makedirs_and_rmdirs_success (line 267) | def test_makedirs_and_rmdirs_success(self, os_ops: OsOperations):
method test_makedirs_failure (line 290) | def test_makedirs_failure(self, os_ops: OsOperations):
method test_listdir (line 305) | def test_listdir(self, os_ops: OsOperations):
method test_path_exists_true__directory (line 320) | def test_path_exists_true__directory(self, os_ops: OsOperations):
method test_path_exists_true__file (line 330) | def test_path_exists_true__file(self, os_ops: OsOperations):
method test_path_exists_false__directory (line 340) | def test_path_exists_false__directory(self, os_ops: OsOperations):
method test_path_exists_false__file (line 350) | def test_path_exists_false__file(self, os_ops: OsOperations):
method test_mkdtemp__default (line 360) | def test_mkdtemp__default(self, os_ops: OsOperations):
method test_mkdtemp__custom (line 369) | def test_mkdtemp__custom(self, os_ops: OsOperations):
method test_rmdirs (line 380) | def test_rmdirs(self, os_ops: OsOperations):
method test_rmdirs__01_with_subfolder (line 389) | def test_rmdirs__01_with_subfolder(self, os_ops: OsOperations):
method test_rmdirs__02_with_file (line 406) | def test_rmdirs__02_with_file(self, os_ops: OsOperations):
method test_rmdirs__03_with_subfolder_and_file (line 423) | def test_rmdirs__03_with_subfolder_and_file(self, os_ops: OsOperations):
method test_write_text_file (line 447) | def test_write_text_file(self, os_ops: OsOperations):
method test_write_binary_file (line 467) | def test_write_binary_file(self, os_ops: OsOperations):
method test_read_text_file (line 484) | def test_read_text_file(self, os_ops: OsOperations):
method test_read_binary_file (line 498) | def test_read_binary_file(self, os_ops: OsOperations):
method test_read__text (line 512) | def test_read__text(self, os_ops: OsOperations):
method test_read__binary (line 541) | def test_read__binary(self, os_ops: OsOperations):
method test_read__binary_and_encoding (line 556) | def test_read__binary_and_encoding(self, os_ops: OsOperations):
method test_read_binary__spec (line 569) | def test_read_binary__spec(self, os_ops: OsOperations):
method test_read_binary__spec__negative_offset (line 605) | def test_read_binary__spec__negative_offset(self, os_ops: OsOperations):
method test_get_file_size (line 616) | def test_get_file_size(self, os_ops: OsOperations):
method test_isfile_true (line 631) | def test_isfile_true(self, os_ops: OsOperations):
method test_isfile_false__not_exist (line 643) | def test_isfile_false__not_exist(self, os_ops: OsOperations):
method test_isfile_false__directory (line 655) | def test_isfile_false__directory(self, os_ops: OsOperations):
method test_isdir_true (line 669) | def test_isdir_true(self, os_ops: OsOperations):
method test_isdir_false__not_exist (line 681) | def test_isdir_false__not_exist(self, os_ops: OsOperations):
method test_isdir_false__file (line 693) | def test_isdir_false__file(self, os_ops: OsOperations):
method test_cwd (line 707) | def test_cwd(self, os_ops: OsOperations):
class tagWriteData001 (line 719) | class tagWriteData001:
method __init__ (line 720) | def __init__(self, sign, source, cp_rw, cp_truncate, cp_binary, cp_d...
method write_data001 (line 760) | def write_data001(self, request):
method test_write (line 765) | def test_write(self, write_data001: tagWriteData001, os_ops: OsOperati...
method test_touch (line 788) | def test_touch(self, os_ops: OsOperations):
method test_is_port_free__true (line 804) | def test_is_port_free__true(self, os_ops: OsOperations):
method test_is_port_free__false (line 840) | def test_is_port_free__false(self, os_ops: OsOperations):
method test_get_tmpdir (line 901) | def test_get_tmpdir(self, os_ops: OsOperations):
method test_get_tmpdir__compare_with_py_info (line 925) | def test_get_tmpdir__compare_with_py_info(self, os_ops: OsOperations):
class tagData_OS_OPS__NUMS (line 942) | class tagData_OS_OPS__NUMS:
method __init__ (line 946) | def __init__(self, os_ops_descr: OsOpsDescr, nums: int):
method data001 (line 962) | def data001(self, request: pytest.FixtureRequest) -> tagData_OS_OPS__N...
method test_mkdir__mt (line 966) | def test_mkdir__mt(self, data001: tagData_OS_OPS__NUMS):
method kill_signal_id (line 1227) | def kill_signal_id(self, request: pytest.FixtureRequest) -> T_KILL_SIG...
method test_kill_signal (line 1232) | def test_kill_signal(
method test_kill (line 1240) | def test_kill(
method test_kill__unk_pid (line 1309) | def test_kill__unk_pid(
FILE: tests/test_os_ops_local.py
class TestOsOpsLocal (line 11) | class TestOsOpsLocal:
method os_ops (line 13) | def os_ops(self):
method test_read__unknown_file (line 16) | def test_read__unknown_file(self, os_ops: OsOperations):
method test_read_binary__spec__unk_file (line 24) | def test_read_binary__spec__unk_file(self, os_ops: OsOperations):
method test_get_file_size__unk_file (line 34) | def test_get_file_size__unk_file(self, os_ops: OsOperations):
method test_cwd (line 43) | def test_cwd(self, os_ops: OsOperations):
FILE: tests/test_os_ops_remote.py
class TestOsOpsRemote (line 12) | class TestOsOpsRemote:
method os_ops (line 14) | def os_ops(self):
method test_rmdirs__try_to_delete_nonexist_path (line 17) | def test_rmdirs__try_to_delete_nonexist_path(self, os_ops: OsOperations):
method test_rmdirs__try_to_delete_file (line 24) | def test_rmdirs__try_to_delete_file(self, os_ops: OsOperations):
method test_read__unknown_file (line 44) | def test_read__unknown_file(self, os_ops: OsOperations):
method test_read_binary__spec__unk_file (line 57) | def test_read_binary__spec__unk_file(self, os_ops: OsOperations):
method test_get_file_size__unk_file (line 70) | def test_get_file_size__unk_file(self, os_ops: OsOperations):
FILE: tests/test_raise_error.py
class TestRaiseError (line 9) | class TestRaiseError:
class tagTestData001 (line 10) | class tagTestData001:
method __init__ (line 14) | def __init__(
method sign (line 26) | def sign(self) -> str:
method data001 (line 47) | def data001(self, request: pytest.FixtureRequest) -> tagTestData001:
method test_001__node_err__cant_enumerate_child_processes (line 52) | def test_001__node_err__cant_enumerate_child_processes(
method data002 (line 82) | def data002(self, request: pytest.FixtureRequest) -> tagTestData001:
method test_002__node_err__cant_kill (line 87) | def test_002__node_err__cant_kill(
FILE: tests/test_testgres_common.py
function removing (line 59) | def removing(os_ops: OsOperations, f):
class TestTestgresCommon (line 72) | class TestTestgresCommon:
method node_svc (line 83) | def node_svc(self, request: pytest.FixtureRequest) -> PostgresNodeServ...
method test_testgres_version (line 90) | def test_testgres_version(self):
method test_version_management (line 103) | def test_version_management(self, node_svc: PostgresNodeService):
method test_node_repr (line 135) | def test_node_repr(self, node_svc: PostgresNodeService):
method test_custom_init (line 140) | def test_custom_init(self, node_svc: PostgresNodeService):
method test_double_init (line 161) | def test_double_init(self, node_svc: PostgresNodeService):
method test_init_after_cleanup (line 169) | def test_init_after_cleanup(self, node_svc: PostgresNodeService):
method test_init_unique_system_id (line 177) | def test_init_unique_system_id(self, node_svc: PostgresNodeService):
method test_node_exit (line 207) | def test_node_exit(self, node_svc: PostgresNodeService):
method test_double_start (line 225) | def test_double_start(self, node_svc: PostgresNodeService):
method test_start__manually_stop__start_again (line 253) | def test_start__manually_stop__start_again(self, node_svc: PostgresNod...
method test_uninitialized_start (line 291) | def test_uninitialized_start(self, node_svc: PostgresNodeService):
method test_start2 (line 304) | def test_start2(self, node_svc: PostgresNodeService):
method test_restart (line 332) | def test_restart(self, node_svc: PostgresNodeService):
method test_double_stop (line 355) | def test_double_stop(self, node_svc: PostgresNodeService):
method test_reload (line 377) | def test_reload(self, node_svc: PostgresNodeService):
method test_pg_ctl (line 395) | def test_pg_ctl(self, node_svc: PostgresNodeService):
method test_status (line 404) | def test_status(self, node_svc: PostgresNodeService):
method test_status__empty_postmaster_pid (line 436) | def test_status__empty_postmaster_pid(self, node_svc: PostgresNodeServ...
method test_status__force_clean_postmaster_pid (line 467) | def test_status__force_clean_postmaster_pid(self, node_svc: PostgresNo...
method test_kill__is_not_initialized (line 514) | def test_kill__is_not_initialized(
method test_kill__is_not_running (line 532) | def test_kill__is_not_running(
method test_kill__ok (line 561) | def test_kill__ok(
method test_kill_backgroud_writer__ok (line 601) | def test_kill_backgroud_writer__ok(
method test_child_processes__is_not_initialized (line 653) | def test_child_processes__is_not_initialized(
method test_child_processes__is_not_running (line 671) | def test_child_processes__is_not_running(
method test_child_processes__ok (line 700) | def test_child_processes__ok(
method test_child_pids (line 770) | def test_child_pids(self, node_svc: PostgresNodeService):
method test_exceptions (line 891) | def test_exceptions(self):
method test_auto_name (line 896) | def test_auto_name(self, node_svc: PostgresNodeService):
method test_file_tail (line 910) | def test_file_tail(self):
method test_isolation_levels (line 933) | def test_isolation_levels(self, node_svc: PostgresNodeService):
method test_users (line 953) | def test_users(self, node_svc: PostgresNodeService):
method test_poll_query_until (line 961) | def test_poll_query_until(self, node_svc: PostgresNodeService):
method test_logging (line 1023) | def test_logging(self, node_svc: PostgresNodeService):
method test_psql (line 1116) | def test_psql(self, node_svc: PostgresNodeService):
method test_psql__another_port (line 1164) | def test_psql__another_port(self, node_svc: PostgresNodeService):
method test_psql__another_bad_host (line 1190) | def test_psql__another_bad_host(self, node_svc: PostgresNodeService):
method test_safe_psql__another_port (line 1206) | def test_safe_psql__another_port(self, node_svc: PostgresNodeService):
method test_safe_psql__another_bad_host (line 1232) | def test_safe_psql__another_bad_host(self, node_svc: PostgresNodeServi...
method test_safe_psql__expect_error (line 1247) | def test_safe_psql__expect_error(self, node_svc: PostgresNodeService):
method test_transactions (line 1266) | def test_transactions(self, node_svc: PostgresNodeService):
method test_control_data (line 1291) | def test_control_data(self, node_svc: PostgresNodeService):
method test_backup_simple (line 1306) | def test_backup_simple(self, node_svc: PostgresNodeService):
method test_backup_multiple (line 1328) | def test_backup_multiple(self, node_svc: PostgresNodeService):
method test_backup_exhaust (line 1342) | def test_backup_exhaust(self, node_svc: PostgresNodeService):
method test_backup_wrong_xlog_method (line 1356) | def test_backup_wrong_xlog_method(self, node_svc: PostgresNodeService):
method test_pg_ctl_wait_option (line 1367) | def test_pg_ctl_wait_option(self, node_svc: PostgresNodeService):
method impl__test_pg_ctl_wait_option (line 1407) | def impl__test_pg_ctl_wait_option(
method test_replicate (line 1505) | def test_replicate(self, node_svc: PostgresNodeService):
method test_synchronous_replication (line 1521) | def test_synchronous_replication(self, node_svc: PostgresNodeService):
method test_logical_replication (line 1567) | def test_logical_replication(self, node_svc: PostgresNodeService):
method test_logical_catchup (line 1641) | def test_logical_catchup(self, node_svc: PostgresNodeService):
method test_logical_replication_fail (line 1669) | def test_logical_replication_fail(self, node_svc: PostgresNodeService):
method test_replication_slots (line 1680) | def test_replication_slots(self, node_svc: PostgresNodeService):
method test_incorrect_catchup (line 1692) | def test_incorrect_catchup(self, node_svc: PostgresNodeService):
method test_promotion (line 1701) | def test_promotion(self, node_svc: PostgresNodeService):
method dump_fmt (line 1724) | def dump_fmt(self, request: pytest.FixtureRequest) -> enums.DumpFormat:
method test_dump (line 1728) | def test_dump(self, node_svc: PostgresNodeService, dump_fmt: enums.Dum...
method test_dump_with_options (line 1747) | def test_dump_with_options(self, node_svc: PostgresNodeService):
method test_pgbench (line 1772) | def test_pgbench(self, node_svc: PostgresNodeService):
method test_unix_sockets (line 1792) | def test_unix_sockets(self, node_svc: PostgresNodeService):
method test_the_same_port (line 1812) | def test_the_same_port(self, node_svc: PostgresNodeService):
class tagPortManagerProxy (line 1845) | class tagPortManagerProxy(PortManager):
method __init__ (line 1854) | def __init__(self, prevPortManager: PortManager, dummyPortNumber: in...
method __enter__ (line 1871) | def __enter__(self):
method __exit__ (line 1874) | def __exit__(self, type, value, traceback):
method reserve_port (line 1879) | def reserve_port(self) -> int:
method release_port (line 1899) | def release_port(self, number: int) -> None:
method test_port_rereserve_during_node_start (line 1921) | def test_port_rereserve_during_node_start(self, node_svc: PostgresNode...
method test_port_conflict (line 1956) | def test_port_conflict(self, node_svc: PostgresNodeService):
method test_try_to_get_port_after_free_manual_port (line 2000) | def test_try_to_get_port_after_free_manual_port(self, node_svc: Postgr...
method test_try_to_start_node_after_free_manual_port (line 2030) | def test_try_to_start_node_after_free_manual_port(self, node_svc: Post...
method test_node__os_ops (line 2059) | def test_node__os_ops(self, node_svc: PostgresNodeService):
method test_node__port_manager (line 2074) | def test_node__port_manager(self, node_svc: PostgresNodeService):
method test_node__port_manager_and_explicit_port (line 2089) | def test_node__port_manager_and_explicit_port(self, node_svc: Postgres...
method test_node__no_port_manager (line 2115) | def test_node__no_port_manager(self, node_svc: PostgresNodeService):
class tagTableChecksumTestData (line 2141) | class tagTableChecksumTestData:
method __init__ (line 2144) | def __init__(
method table_checksum_test_data (line 2171) | def table_checksum_test_data(
method test_node__table_checksum (line 2179) | def test_node__table_checksum(
method test_node__pgbench_table_checksums__one_table (line 2239) | def test_node__pgbench_table_checksums__one_table(
method test_node__pgbench_table_checksums__pbckp_2278 (line 2304) | def test_node__pgbench_table_checksums__pbckp_2278(self, node_svc: Pos...
method helper__call_and_check_pgbench_table_checksums (line 2334) | def helper__call_and_check_pgbench_table_checksums(
class tag_rmdirs_protector (line 2422) | class tag_rmdirs_protector:
method __init__ (line 2428) | def __init__(self, os_ops: OsOperations):
method __enter__ (line 2433) | def __enter__(self):
method __exit__ (line 2438) | def __exit__(self, exc_type, exc_val, exc_tb):
method proxy__rmdirs (line 2443) | def proxy__rmdirs(self, path, ignore_errors=True):
method test_node_app__make_empty__base_dir_is_None (line 2446) | def test_node_app__make_empty__base_dir_is_None(self, node_svc: Postgr...
method test_node_app__make_empty__base_dir_is_Empty (line 2480) | def test_node_app__make_empty__base_dir_is_Empty(self, node_svc: Postg...
method test_node_app__make_empty (line 2510) | def test_node_app__make_empty(self, node_svc: PostgresNodeService):
method test_node_app__make_simple__checksum (line 2559) | def test_node_app__make_simple__checksum(self, node_svc: PostgresNodeS...
method test_node_app__make_empty_with_explicit_port (line 2612) | def test_node_app__make_empty_with_explicit_port(self, node_svc: Postg...
method helper__get_node (line 2716) | def helper__get_node(
method helper__skip_test_if_pg_version_is_not_ge (line 2737) | def helper__skip_test_if_pg_version_is_not_ge(ver1: str, ver2: str):
method helper__skip_test_if_pg_version_is_ge (line 2744) | def helper__skip_test_if_pg_version_is_ge(ver1: str, ver2: str):
method helper__pg_version_ge (line 2751) | def helper__pg_version_ge(ver1: str, ver2: str) -> bool:
method helper__rm_carriage_returns (line 2759) | def helper__rm_carriage_returns(out):
method helper__skip_test_if_util_not_exist (line 2777) | def helper__skip_test_if_util_not_exist(os_ops: OsOperations, name: str):
method helper__util_exists (line 2784) | def helper__util_exists(os_ops: OsOperations, util):
FILE: tests/test_testgres_local.py
function pg_version_ge (line 28) | def pg_version_ge(version):
function util_exists (line 34) | def util_exists(util):
function rm_carriage_returns (line 50) | def rm_carriage_returns(out):
class TestTestgresLocal (line 68) | class TestTestgresLocal:
method test_pg_config (line 69) | def test_pg_config(self):
method test_ports_management (line 94) | def test_ports_management(self):
method test_child_process_dies (line 128) | def test_child_process_dies(self):
method test_upgrade_node (line 159) | def test_upgrade_node(self):
class tagPortManagerProxy (line 172) | class tagPortManagerProxy:
method __init__ (line 182) | def __init__(self, dummyPortNumber, dummyPortMaxUsage):
method __enter__ (line 208) | def __enter__(self):
method __exit__ (line 211) | def __exit__(self, type, value, traceback):
method _proxy__reserve_port (line 227) | def _proxy__reserve_port():
method _proxy__release_port (line 247) | def _proxy__release_port(dummyPortNumber):
method test_port_rereserve_during_node_start (line 268) | def test_port_rereserve_during_node_start(self):
method test_port_conflict (line 301) | def test_port_conflict(self):
method test_simple_with_bin_dir (line 343) | def test_simple_with_bin_dir(self):
method test_set_auto_conf (line 366) | def test_set_auto_conf(self):
method helper__skip_test_if_util_not_exist (line 410) | def helper__skip_test_if_util_not_exist(name: str):
FILE: tests/test_testgres_remote.py
function util_exists (line 25) | def util_exists(util):
class TestTestgresRemote (line 41) | class TestTestgresRemote:
method implicit_fixture (line 43) | def implicit_fixture(self):
method test_init__LANG_С (line 56) | def test_init__LANG_С(self):
method test_init__unk_LANG_and_LC_CTYPE (line 68) | def test_init__unk_LANG_and_LC_CTYPE(self):
method test_pg_config (line 142) | def test_pg_config(self):
method helper__get_node (line 168) | def helper__get_node(name=None):
method helper__restore_envvar (line 181) | def helper__restore_envvar(name, prev_value):
method helper__skip_test_if_util_not_exist (line 188) | def helper__skip_test_if_util_not_exist(name: str):
FILE: tests/test_utils.py
class TestUtils (line 13) | class TestUtils:
method os_ops (line 23) | def os_ops(self, request: pytest.FixtureRequest) -> OsOperations:
method test_parse_pg_version (line 28) | def test_parse_pg_version(self):
method test_get_pg_config2 (line 38) | def test_get_pg_config2(self, os_ops: OsOperations):
FILE: tests/units/exceptions/BackupException/test_set001__constructor.py
class TestSet001_Constructor (line 5) | class TestSet001_Constructor:
method test_001__default (line 6) | def test_001__default(self):
method test_002__message (line 16) | def test_002__message(self):
FILE: tests/units/exceptions/CatchUpException/test_set001__constructor.py
class TestSet001_Constructor (line 5) | class TestSet001_Constructor:
method test_001__default (line 6) | def test_001__default(self):
method test_002__message (line 16) | def test_002__message(self):
FILE: tests/units/exceptions/InitNodeException/test_set001__constructor.py
class TestSet001_Constructor (line 5) | class TestSet001_Constructor:
method test_001__default (line 6) | def test_001__default(self):
method test_002__message (line 16) | def test_002__message(self):
FILE: tests/units/exceptions/PortForException/test_set001__constructor.py
class TestSet001_Constructor (line 5) | class TestSet001_Constructor:
method test_001__default (line 6) | def test_001__default(self):
method test_002__message (line 16) | def test_002__message(self):
FILE: tests/units/exceptions/QueryException/test_set001__constructor.py
class TestSet001_Constructor (line 5) | class TestSet001_Constructor:
method test_001__default (line 6) | def test_001__default(self):
method test_002__message (line 18) | def test_002__message(self):
method test_003__query (line 30) | def test_003__query(self):
method test_004__all (line 42) | def test_004__all(self):
FILE: tests/units/exceptions/QueryTimeoutException/test_set001__constructor.py
class TestSet001_Constructor (line 6) | class TestSet001_Constructor:
method test_001__default (line 7) | def test_001__default(self):
method test_002__message (line 20) | def test_002__message(self):
method test_003__query (line 33) | def test_003__query(self):
method test_004__all (line 46) | def test_004__all(self):
FILE: tests/units/exceptions/StartNodeException/test_set001__constructor.py
class TestSet001_Constructor (line 5) | class TestSet001_Constructor:
method test_001__default (line 6) | def test_001__default(self):
method test_002__message (line 18) | def test_002__message(self):
method test_003__files (line 30) | def test_003__files(self):
method test_004__all (line 42) | def test_004__all(self):
FILE: tests/units/exceptions/TimeoutException/test_set001.py
class TestSet001 (line 6) | class TestSet001:
method test_001__default (line 7) | def test_001__default(self):
FILE: tests/units/node/PostgresNode/test_setM001__start.py
class TestSet001__start (line 21) | class TestSet001__start:
method node_svc (line 26) | def node_svc(self, request: pytest.FixtureRequest) -> PostgresNodeServ...
class tagData001 (line 33) | class tagData001:
method __init__ (line 36) | def __init__(self, wait: typing.Optional[bool]):
method data001 (line 50) | def data001(self, request: pytest.FixtureRequest) -> tagData001:
method test_001__wait_true (line 55) | def test_001__wait_true(
method test_002__wait_false (line 87) | def test_002__wait_false(self, node_svc: PostgresNodeService):
method test_003__exec_env (line 143) | def test_003__exec_env(
method test_004__params_is_None (line 185) | def test_004__params_is_None(
method test_005__params_is_empty (line 202) | def test_005__params_is_empty(
FILE: tests/units/node/PostgresNode/test_setM002__start2.py
class TestSet002__start2 (line 21) | class TestSet002__start2:
method node_svc (line 26) | def node_svc(self, request: pytest.FixtureRequest) -> PostgresNodeServ...
class tagData001 (line 33) | class tagData001:
method __init__ (line 36) | def __init__(self, wait: typing.Optional[bool]):
method data001 (line 50) | def data001(self, request: pytest.FixtureRequest) -> tagData001:
method test_001__wait_true (line 55) | def test_001__wait_true(
method test_002__wait_false (line 84) | def test_002__wait_false(self, node_svc: PostgresNodeService):
method test_003__exec_env (line 137) | def test_003__exec_env(
method test_004__params_is_None (line 179) | def test_004__params_is_None(
method test_005__params_is_empty (line 196) | def test_005__params_is_empty(
Condensed preview — 95 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (493K chars).
[
{
"path": ".coveragerc",
"chars": 28,
"preview": "[run]\nsource=.\nomit=./env/*\n"
},
{
"path": ".dockerignore",
"chars": 52,
"preview": "dist\nenv\nvenv\n*.egg-info\nlogs\n.vscode\n.pytest_cache\n"
},
{
"path": ".github/dependabot.yml",
"chars": 150,
"preview": "version: 2\nupdates:\n - package-ecosystem: \"github-actions\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"\n "
},
{
"path": ".github/workflows/package-verification.yml",
"chars": 4981,
"preview": "# This workflow will install Python dependencies, run tests and lint with a variety of Python versions\n# For more inform"
},
{
"path": ".github/workflows/python-publish.yml",
"chars": 2218,
"preview": "# This workflow will upload a Python Package to PyPI when a release is created\n# For more information see: https://docs."
},
{
"path": ".gitignore",
"chars": 125,
"preview": "*.pyc\n*.egg\n*.egg-info/\n.eggs/\ndist/\nbuild/\ndocs/build/\nlogs/\n\nenv/\nvenv/\n\n.coverage\ncoverage.xml\n\nDockerfile\n\n*~\n*.swp\n"
},
{
"path": ".style.yapf",
"chars": 110,
"preview": "[style]\nbased_on_style = pep8\nspaces_before_comment = 4\nsplit_before_logical_operator = false\ncolumn_limit=80\n"
},
{
"path": "Dockerfile--altlinux_10.tmpl",
"chars": 2968,
"preview": "ARG PG_VERSION\nARG PYTHON_VERSION \n\n# --------------------------------------------- base1\nFROM alt:p10 AS base1\n\nRUN apt"
},
{
"path": "Dockerfile--altlinux_11.tmpl",
"chars": 3029,
"preview": "ARG PG_VERSION\nARG PYTHON_VERSION \n\n# --------------------------------------------- base1\nFROM alt:p11 AS base1\n\nRUN apt"
},
{
"path": "Dockerfile--astralinux_1_7.tmpl",
"chars": 3209,
"preview": "ARG PG_VERSION\nARG PYTHON_VERSION \n\n# --------------------------------------------- base1\nFROM packpack/packpack:astra-1"
},
{
"path": "Dockerfile--std-all.tmpl",
"chars": 1778,
"preview": "ARG PG_VERSION\nARG PYTHON_VERSION \n\n# --------------------------------------------- base1\nFROM postgres:${PG_VERSION}-al"
},
{
"path": "Dockerfile--std.tmpl",
"chars": 1056,
"preview": "ARG PG_VERSION\nARG PYTHON_VERSION \n\n# --------------------------------------------- base1\nFROM postgres:${PG_VERSION}-al"
},
{
"path": "Dockerfile--std2-all.tmpl",
"chars": 3398,
"preview": "ARG PG_VERSION\nARG PYTHON_VERSION \n\n# --------------------------------------------- base1\nFROM postgres:${PG_VERSION}-al"
},
{
"path": "Dockerfile--ubuntu_24_04.tmpl",
"chars": 2151,
"preview": "ARG PG_VERSION\nARG PYTHON_VERSION \n\n# --------------------------------------------- base1\nFROM ubuntu:24.04 AS base1\nARG"
},
{
"path": "LICENSE",
"chars": 1106,
"preview": "testgres is released under the PostgreSQL License, a liberal Open Source license, similar to the BSD or MIT licenses.\r\n\r"
},
{
"path": "README.md",
"chars": 6514,
"preview": "[![CI Status](https://img.shields.io/github/actions/workflow/status/postgrespro/testgres/.github/workflows/package-verif"
},
{
"path": "docs/Makefile",
"chars": 644,
"preview": "# Minimal makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHI"
},
{
"path": "docs/README.md",
"chars": 285,
"preview": "# Building the documentation\n\nMake sure you have `Sphinx` package installed:\n\n```\npip install Sphinx\n```\n\nThen just run\n"
},
{
"path": "docs/source/conf.py",
"chars": 5086,
"preview": "# -*- coding: utf-8 -*-\n#\n# Configuration file for the Sphinx documentation builder.\n#\n# This file does only contain a s"
},
{
"path": "docs/source/index.rst",
"chars": 6112,
"preview": "\nTestgres documentation\n======================\n\nUtility for orchestrating temporary PostgreSQL clusters in Python tests."
},
{
"path": "docs/source/testgres.rst",
"chars": 1344,
"preview": "testgres package\n================\n\ntestgres.api\n------------\n\n.. automodule:: testgres.api\n :members:\n :undoc-memb"
},
{
"path": "pyproject.toml",
"chars": 1742,
"preview": "[build-system]\nrequires = [\"setuptools>=61.0\"]\nbuild-backend = \"setuptools.build_meta\"\n\n[tool.setuptools.package-dir]\n\"t"
},
{
"path": "run_tests.sh",
"chars": 1365,
"preview": "#!/usr/bin/env bash\n\nset -eux\n\nif [ -z ${TEST_FILTER+x} ]; \\\nthen export TEST_FILTER=\"TestTestgresLocal or (TestTestgres"
},
{
"path": "run_tests2.sh",
"chars": 171,
"preview": "#!/usr/bin/env bash\n\nset -eux\n\neval \"$(pyenv init -)\"\neval \"$(pyenv virtualenv-init -)\"\n\npyenv virtualenv --force ${PYTH"
},
{
"path": "src/__init__.py",
"chars": 2132,
"preview": "from .api import get_new_node, get_remote_node\nfrom .backup import NodeBackup\n\nfrom .config import \\\n TestgresConfig,"
},
{
"path": "src/api.py",
"chars": 1938,
"preview": "# coding: utf-8\n\"\"\"\nTesting framework for PostgreSQL and its extensions\n\nThis module was created under influence of Post"
},
{
"path": "src/backup.py",
"chars": 6136,
"preview": "# coding: utf-8\n\nfrom six import raise_from\n\nfrom .enums import XLogMethod\n\nfrom .consts import \\\n DATA_DIR, \\\n TM"
},
{
"path": "src/cache.py",
"chars": 3011,
"preview": "# coding: utf-8\n\nfrom six import raise_from\n\nfrom .config import testgres_config\n\nfrom .consts import XLOG_CONTROL_FILE\n"
},
{
"path": "src/config.py",
"chars": 5451,
"preview": "# coding: utf-8\n\nimport atexit\nimport copy\nimport logging\nimport os\nimport tempfile\n\nfrom contextlib import contextmanag"
},
{
"path": "src/connection.py",
"chars": 3078,
"preview": "# coding: utf-8\nimport logging\n\n# we support both pg8000 and psycopg2\ntry:\n import psycopg2 as pglib\nexcept ImportErr"
},
{
"path": "src/consts.py",
"chars": 911,
"preview": "# coding: utf-8\n\n# names for dirs in base_dir\nDATA_DIR = \"data\"\nLOGS_DIR = \"logs\"\n\n# prefixes for temp dirs\nTMP_NODE = '"
},
{
"path": "src/decorators.py",
"chars": 1749,
"preview": "import six\nimport functools\n\n\ndef positional_args_hack(*special_cases):\n \"\"\"\n Convert positional args described by"
},
{
"path": "src/defaults.py",
"chars": 969,
"preview": "import datetime\nimport struct\nimport uuid\n\nfrom .config import testgres_config as tconf\n\n\ndef default_dbname():\n \"\"\"\n"
},
{
"path": "src/enums.py",
"chars": 2451,
"preview": "from enum import Enum, IntEnum\nfrom six import iteritems\nfrom psutil import NoSuchProcess\n\n\nclass XLogMethod(Enum):\n "
},
{
"path": "src/exceptions.py",
"chars": 7960,
"preview": "# coding: utf-8\n\nimport six\nimport typing\n\nfrom testgres.operations.exceptions import TestgresException\nfrom testgres.op"
},
{
"path": "src/impl/internal_utils.py",
"chars": 387,
"preview": "import logging\n\n\ndef send_log(level: int, msg: str) -> None:\n assert type(level) is int\n assert type(msg) is str\n\n"
},
{
"path": "src/impl/platforms/internal_platform_utils.py",
"chars": 2188,
"preview": "from __future__ import annotations\n\nimport enum\nimport typing\n\nfrom testgres.operations.os_ops import OsOperations\n\n\ncla"
},
{
"path": "src/impl/platforms/internal_platform_utils_factory.py",
"chars": 648,
"preview": "from .internal_platform_utils import InternalPlatformUtils\n\nfrom testgres.operations.os_ops import OsOperations\n\n\ndef cr"
},
{
"path": "src/impl/platforms/linux/internal_platform_utils.py",
"chars": 3623,
"preview": "from __future__ import annotations\n\nfrom .. import internal_platform_utils as base\nfrom ... import internal_utils\n\nfrom "
},
{
"path": "src/impl/platforms/win32/internal_platform_utils.py",
"chars": 551,
"preview": "from __future__ import annotations\n\nfrom .. import internal_platform_utils as base\nfrom testgres.operations.os_ops impor"
},
{
"path": "src/impl/port_manager__generic.py",
"chars": 3299,
"preview": "from testgres.operations.os_ops import OsOperations\n\nfrom ..port_manager import PortManager\nfrom ..exceptions import Por"
},
{
"path": "src/impl/port_manager__this_host.py",
"chars": 1084,
"preview": "from ..port_manager import PortManager\n\nfrom .. import utils\n\nimport threading\n\n\nclass PortManager__ThisHost(PortManager"
},
{
"path": "src/logger.py",
"chars": 1428,
"preview": "# coding: utf-8\n\nimport logging\nimport select\nimport threading\nimport time\n\n\nclass TestgresLogger(threading.Thread):\n "
},
{
"path": "src/node.py",
"chars": 77271,
"preview": "# coding: utf-8\nfrom __future__ import annotations\n\nimport logging\nimport os\nimport signal\nimport subprocess\n\nimport tim"
},
{
"path": "src/node_app.py",
"chars": 9732,
"preview": "from .node import OsOperations\nfrom .node import LocalOperations\nfrom .node import PostgresNode\nfrom .node import PortMa"
},
{
"path": "src/port_manager.py",
"chars": 357,
"preview": "class PortManager:\n def __init__(self):\n super().__init__()\n\n def reserve_port(self) -> int:\n raise "
},
{
"path": "src/pubsub.py",
"chars": 8218,
"preview": "# coding: utf-8\n\"\"\"\nUnlike physical replication the logical replication allows users replicate only\nspecified databases "
},
{
"path": "src/raise_error.py",
"chars": 2521,
"preview": "from .exceptions import InvalidOperationException\nfrom .enums import NodeStatus\n\nimport typing\n\n\nclass RaiseError:\n @"
},
{
"path": "src/standby.py",
"chars": 1547,
"preview": "# coding: utf-8\n\nimport six\n\n\n@six.python_2_unicode_compatible\nclass First:\n \"\"\"\n Specifies a priority-based synch"
},
{
"path": "src/utils.py",
"chars": 14855,
"preview": "# coding: utf-8\n\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport os\nimport sys\nimport time"
},
{
"path": "tests/README.md",
"chars": 1259,
"preview": "### How do I run tests?\n\n#### Simple\n\n```bash\n# Setup virtualenv\nvirtualenv venv\nsource venv/bin/activate\n\n# Install loc"
},
{
"path": "tests/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "tests/conftest.py",
"chars": 36658,
"preview": "# /////////////////////////////////////////////////////////////////////////////\n# PyTest Configuration\n\nimport pluggy\nim"
},
{
"path": "tests/helpers/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "tests/helpers/global_data.py",
"chars": 2310,
"preview": "from testgres.operations.os_ops import OsOperations\nfrom testgres.operations.os_ops import ConnectionParams\nfrom testgre"
},
{
"path": "tests/helpers/pg_node_utils.py",
"chars": 5835,
"preview": "from src import PostgresNode\nfrom src import PortManager\nfrom src import OsOperations\nfrom src import NodeStatus\nfrom sr"
},
{
"path": "tests/helpers/run_conditions.py",
"chars": 278,
"preview": "# coding: utf-8\nimport pytest\nimport platform\n\n\nclass RunConditions:\n # It is not a test kit!\n __test__ = False\n\n "
},
{
"path": "tests/helpers/utils.py",
"chars": 1907,
"preview": "import typing\nimport time\nimport logging\n\n\nT_WAIT_TIME = typing.Union[int, float]\n\n\nclass Utils:\n @staticmethod\n d"
},
{
"path": "tests/requirements.txt",
"chars": 81,
"preview": "psutil\npytest\npytest-env\npytest-xdist\npsycopg2\nsix\ntestgres.os_ops>=2.1.0,<3.0.0\n"
},
{
"path": "tests/test_config.py",
"chars": 1212,
"preview": "from src import TestgresConfig\nfrom src import configure_testgres\nfrom src import scoped_config\nfrom src import pop_conf"
},
{
"path": "tests/test_conftest.py--devel",
"chars": 2299,
"preview": "import pytest\nimport logging\n\n\nclass TestConfest:\n def test_failed(self):\n raise Exception(\"TEST EXCEPTION!\")\n"
},
{
"path": "tests/test_os_ops_common.py",
"chars": 43106,
"preview": "# coding: utf-8\nfrom .helpers.global_data import OsOpsDescr\nfrom .helpers.global_data import OsOpsDescrs\nfrom .helpers.g"
},
{
"path": "tests/test_os_ops_local.py",
"chars": 1675,
"preview": "# coding: utf-8\nfrom .helpers.global_data import OsOpsDescrs\nfrom .helpers.global_data import OsOperations\n\nimport os\n\ni"
},
{
"path": "tests/test_os_ops_remote.py",
"chars": 2765,
"preview": "# coding: utf-8\n\nfrom .helpers.global_data import OsOpsDescrs\nfrom .helpers.global_data import OsOperations\n\nfrom src im"
},
{
"path": "tests/test_raise_error.py",
"chars": 2954,
"preview": "from src import InvalidOperationException\nfrom src import NodeStatus\nfrom src.raise_error import RaiseError\n\nimport pyte"
},
{
"path": "tests/test_testgres_common.py",
"chars": 103983,
"preview": "from __future__ import annotations\n\nfrom .helpers.global_data import PostgresNodeService\nfrom .helpers.global_data impor"
},
{
"path": "tests/test_testgres_local.py",
"chars": 15379,
"preview": "# coding: utf-8\nimport os\nimport re\nimport subprocess\nimport pytest\nimport psutil\nimport platform\nimport logging\n\nimport"
},
{
"path": "tests/test_testgres_remote.py",
"chars": 6501,
"preview": "# coding: utf-8\nimport os\n\nimport pytest\nimport logging\nimport typing\n\nfrom .helpers.global_data import PostgresNodeServ"
},
{
"path": "tests/test_utils.py",
"chars": 2115,
"preview": "from .helpers.global_data import OsOpsDescr\nfrom .helpers.global_data import OsOpsDescrs\nfrom .helpers.global_data impor"
},
{
"path": "tests/units/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "tests/units/exceptions/BackupException/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "tests/units/exceptions/BackupException/test_set001__constructor.py",
"chars": 835,
"preview": "from src.exceptions import BackupException\nfrom src.exceptions import TestgresException as testgres__TestgresException\n\n"
},
{
"path": "tests/units/exceptions/CatchUpException/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "tests/units/exceptions/CatchUpException/test_set001__constructor.py",
"chars": 842,
"preview": "from src.exceptions import CatchUpException\nfrom src.exceptions import TestgresException as testgres__TestgresException\n"
},
{
"path": "tests/units/exceptions/InitNodeException/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "tests/units/exceptions/InitNodeException/test_set001__constructor.py",
"chars": 849,
"preview": "from src.exceptions import InitNodeException\nfrom src.exceptions import TestgresException as testgres__TestgresException"
},
{
"path": "tests/units/exceptions/PortForException/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "tests/units/exceptions/PortForException/test_set001__constructor.py",
"chars": 842,
"preview": "from src.exceptions import PortForException\nfrom src.exceptions import TestgresException as testgres__TestgresException\n"
},
{
"path": "tests/units/exceptions/QueryException/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "tests/units/exceptions/QueryException/test_set001__constructor.py",
"chars": 1905,
"preview": "from src.exceptions import QueryException\nfrom src.exceptions import TestgresException as testgres__TestgresException\n\n\n"
},
{
"path": "tests/units/exceptions/QueryTimeoutException/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "tests/units/exceptions/QueryTimeoutException/test_set001__constructor.py",
"chars": 2218,
"preview": "from src.exceptions import QueryTimeoutException\nfrom src.exceptions import QueryException\nfrom src.exceptions import Te"
},
{
"path": "tests/units/exceptions/StartNodeException/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "tests/units/exceptions/StartNodeException/test_set001__constructor.py",
"chars": 2121,
"preview": "from src.exceptions import StartNodeException\nfrom src.exceptions import TestgresException as testgres__TestgresExceptio"
},
{
"path": "tests/units/exceptions/TimeoutException/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "tests/units/exceptions/TimeoutException/test_set001.py",
"chars": 345,
"preview": "from src.exceptions import QueryTimeoutException\nfrom src.exceptions import TimeoutException\nfrom src.exceptions import "
},
{
"path": "tests/units/exceptions/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "tests/units/impl/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "tests/units/impl/platforms/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "tests/units/impl/platforms/internal_platform_utils/InternalPlatformUtils/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "tests/units/impl/platforms/internal_platform_utils/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "tests/units/node/PostgresNode/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "tests/units/node/PostgresNode/test_setM001__start.py",
"chars": 7311,
"preview": "from __future__ import annotations\n\nfrom tests.helpers.global_data import PostgresNodeService\nfrom tests.helpers.global_"
},
{
"path": "tests/units/node/PostgresNode/test_setM002__start2.py",
"chars": 7004,
"preview": "from __future__ import annotations\n\nfrom tests.helpers.global_data import PostgresNodeService\nfrom tests.helpers.global_"
},
{
"path": "tests/units/node/__init__.py",
"chars": 0,
"preview": ""
}
]
About this extraction
This page contains the full source code of the postgrespro/testgres GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 95 files (456.7 KB), approximately 106.4k tokens, and a symbol index with 628 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.