Repository: jnidzwetzki/pg-lock-tracer Branch: main Commit: f4c66c3f0922 Files: 31 Total size: 401.3 KB Directory structure: gitextract_x8d44rn2/ ├── .github/ │ ├── dependabot.yml │ └── workflows/ │ ├── codeql.yml │ ├── integration_tests.yml │ ├── pr_handling.yml │ ├── python_publish.yml │ ├── stale.yml │ └── tests.yml ├── .gitignore ├── LICENSE ├── README.md ├── examples/ │ ├── create_table_trace.html │ └── create_table_trace.json ├── pylintrc ├── pyproject.toml ├── requirements_dev.txt ├── src/ │ └── pg_lock_tracer/ │ ├── __init__.py │ ├── animate_lock_graph.py │ ├── bpf/ │ │ ├── .clang-format │ │ ├── __init__.py │ │ ├── pg_lock_tracer.c │ │ ├── pg_lw_lock_tracer.c │ │ ├── pg_row_lock_tracer.c │ │ └── pg_spinlock_delay_tracer.c │ ├── helper.py │ ├── oid_resolver.py │ ├── pg_lock_tracer.py │ ├── pg_lw_lock_tracer.py │ ├── pg_row_lock_tracer.py │ └── pg_spinlock_delay_tracer.py └── tests/ ├── __init__.py └── test_helper.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/dependabot.yml ================================================ # To get started with Dependabot version updates, you'll need to specify which # package ecosystems to update and where the package manifests are located. # Please see the documentation for all configuration options: # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates version: 2 updates: - package-ecosystem: "pip" # See documentation for possible values directory: "/" # Location of package manifests schedule: interval: "weekly" - package-ecosystem: "github-actions" # See documentation for possible values directory: "/" # Location of package manifests schedule: interval: "weekly" ================================================ FILE: .github/workflows/codeql.yml ================================================ # For most projects, this workflow file will not need changing; you simply need # to commit it to your repository. # # You may wish to alter this file to override the set of languages analyzed, # or to provide custom queries or build logic. # # ******** NOTE ******** # We have attempted to detect the languages in your repository. Please check # the `language` matrix defined below to confirm you have the correct set of # supported CodeQL languages. # name: "CodeQL" on: push: branches: [ "main" ] pull_request: # The branches below must be a subset of the branches above branches: [ "main" ] schedule: - cron: '35 7 * * 6' jobs: analyze: name: Analyze runs-on: ubuntu-latest permissions: actions: read contents: read security-events: write strategy: fail-fast: false matrix: language: [ 'python' ] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] # Use only 'java' to analyze code written in Java, Kotlin or both # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support steps: - name: Checkout repository uses: actions/checkout@v6 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs # queries: security-extended,security-and-quality # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v4 # ℹ️ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun # If the Autobuild fails above, remove it and uncomment the following three lines. # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. # - run: | # echo "Run, Build Application using script" # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4 with: category: "/language:${{matrix.language}}" ================================================ FILE: .github/workflows/integration_tests.yml ================================================ name: Basic Integration Tests on: pull_request: push: branches: - main jobs: config: strategy: fail-fast: false matrix: psql_version: ['14.20', '15.15', '16.11', '17.7', '18.1'] python_version: ['3.10', '3.11', '3.12'] name: PostgreSQL ${{ matrix.psql_version }} and Python ${{ matrix.python_version }} integration test runs-on: ubuntu-24.04 steps: - name: Checkout source code uses: actions/checkout@v6 - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python_version }} - name: Show Python version run: python --version - name: Install needed tools run: | sudo apt install python3-bpfcc systemtap-sdt-dev flex bison lcov build-essential libxml2-dev libssl-dev zlib1g-dev libreadline-dev - name: Cache PostgreSQL build ${{ matrix.psql_version }} id: postgresql-cache uses: actions/cache@v5 with: path: ~/postgresql key: postgresql-${{ matrix.psql_version }}-${{ hashFiles('.github/workflows/*') }} - name: Build PostgreSQL if: steps.postgresql-cache.outputs.cache-hit != 'true' run: | mkdir -p ~/postgresql mkdir -p ~/postgresql/src/${{ matrix.psql_version }} mkdir -p ~/postgresql/bin/${{ matrix.psql_version }} # Data dir is not part of the cache mkdir -p ~/postgresql_data/${{ matrix.psql_version }} cd ~/postgresql/src/${{ matrix.psql_version }} wget -q -O postgresql.tar.bz2 https://ftp.postgresql.org/pub/source/v${{ matrix.psql_version }}/postgresql-${{ matrix.psql_version }}.tar.bz2 tar jxf postgresql.tar.bz2 --strip-components 1 -C . CFLAGS="-ggdb -Og -g3 -fno-omit-frame-pointer" ./configure --prefix=$HOME/postgresql/bin/${{ matrix.psql_version }} --with-openssl --with-readline --with-zlib --with-libxml --enable-dtrace make -j 8 - name: Install PostgreSQL run: | cd ~/postgresql/src/${{ matrix.psql_version }} make install ~/postgresql/bin/${{ matrix.psql_version }}/bin/initdb -D ~/postgresql_data/${{ matrix.psql_version }} ~/postgresql/bin/${{ matrix.psql_version }}/bin/pg_ctl -D ~/postgresql_data/${{ matrix.psql_version }} start #sudo apt install curl ca-certificates gnupg #sudo curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null #sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' #sudo apt update #sudo apt install postgresql-${{ matrix.psql_version }} postgresql-${{ matrix.psql_version }}-dbgsym #sudo /etc/init.d/postgresql start ${{ matrix.psql_version }} #dpkg -L postgresql-${{ matrix.psql_version }}-dbgsym #readelf --string-dump=.gnu_debuglink /usr/lib/postgresql/${{ matrix.psql_version }}/bin/postgres #objdump -TC /usr/lib/postgresql/${{ matrix.psql_version }}/bin/postgres - name: Install package run: sudo python -m pip install . - name: Check that pg_lock_tracer can be executed run: | pg_lock_tracer --version sudo pg_lock_tracer -x ~/postgresql/bin/${{ matrix.psql_version }}/bin/postgres -v -p $(pidof postgres) --statistics --dry-run - name: Check that pg_lw_lock_tracer can be executed run: | pg_lw_lock_tracer --version sudo pg_lw_lock_tracer -p $(pidof postgres) --dry-run - name: Check that pg_row_lock_tracer can be executed run: | pg_row_lock_tracer --version sudo pg_row_lock_tracer -x ~/postgresql/bin/${{ matrix.psql_version }}/bin/postgres -v -p $(pidof postgres) --statistics --dry-run - name: Check that animate_lock_graph can be executed run: | animate_lock_graph --version animate_lock_graph -o animation.html -i examples/create_table_trace.json diff animation.html -i examples/create_table_trace.html - name: Check that pg_spinlock_delay_tracer can be executed run: | pg_spinlock_delay_tracer --version sudo pg_spinlock_delay_tracer -x ~/postgresql/bin/${{ matrix.psql_version }}/bin/postgres -v -p $(pidof postgres) --dry-run ================================================ FILE: .github/workflows/pr_handling.yml ================================================ name: PR Handling ################# # NOTE: We are using pull_request_target here: # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target. # # So, only API calls should be made in the actions defined here. The # committed code should _NOT_ be touched in any case. ################# on: pull_request_target: types: [ opened, reopened ] jobs: assign-pr: name: Assign PR to author runs-on: ubuntu-latest steps: - uses: toshimaru/auto-author-assign@v3.0.2 ================================================ FILE: .github/workflows/python_publish.yml ================================================ # Based on https://github.com/actions/starter-workflows/blob/main/ci/python-publish.yml name: Upload Python Package on: release: types: [published] permissions: contents: read jobs: deploy: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - name: Install dependencies run: | python -m pip install --upgrade pip pip install build pip install twine pip install dunamai # See https://github.com/pypa/twine/issues/1216 pip install -U packaging - name: Show version based on git tag run: dunamai from git --style semver - name: Ensure that program and tag version match if: github.event_name == 'release' run: | PACKAGE_VERSION=$(python -c "from src.pg_lock_tracer import __version__; print(__version__)") GIT_VERSION=$(dunamai from git --style semver) if [[ "$PACKAGE_VERSION" != "$GIT_VERSION" ]]; then echo "Version mismatch ${PACKAGE_VERSION} / ${GIT_VERSION}" exit 1 fi - name: Build package run: python -m build - name: Upload to test.pypi if: github.event_name == 'release' run: 'twine upload --non-interactive --repository testpypi dist/*' env: TWINE_USERNAME: '__token__' TWINE_PASSWORD: '${{ secrets.TWINE_PASSWORD_TEST }}' - name: Upload to pypi if: github.event_name == 'release' run: 'twine upload --non-interactive dist/*' env: TWINE_USERNAME: '__token__' TWINE_PASSWORD: '${{ secrets.TWINE_PASSWORD }}' ================================================ FILE: .github/workflows/stale.yml ================================================ name: 'Close stale issues' "on": schedule: - cron: '30 1 * * *' workflow_dispatch: jobs: stale: runs-on: ubuntu-latest steps: - uses: actions/stale@v10 with: # Don't process PRs days-before-stale: -1 # Process only issues days-before-issue-stale: 60 days-before-issue-close: 30 # Add this label after 'days-before-issue-stale' days to mark it as stale stale-issue-label: 'no-activity' # Process only issues that contain the label 'waiting-for-author' only-labels: 'need-more-info' close-issue-message: 'This issue was closed because it has been stalled for several weeks with no activity.' ================================================ FILE: .github/workflows/tests.yml ================================================ name: Basic Project Tests on: pull_request: push: branches: - main jobs: tests: name: Perform unit tests runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v6 - name: Install needed tools run: | sudo apt update sudo apt install python3-bpfcc - name: Setup python 3.10 uses: actions/setup-python@v6 with: python-version: '3.10' - name: Copy distribution BPF packages run: cp -av /usr/lib/python3/dist-packages/bcc* $(python -c "import sysconfig; print(sysconfig.get_path('platlib'))") - name: Install package run: python -m pip install . - name: Install development requirements run: python -m pip install -r requirements_dev.txt - name: Execute unit tests run: pytest linter: name: Run Linter and check formatting runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v6 - name: Install needed tools run: | sudo apt update sudo apt install clang-format python3-bpfcc - name: Install development requirements run: python -m pip install -r requirements_dev.txt - name: Run pylint run: pylint src tests - name: Check code format with black run: black --check --diff src tests - name: Run clang-format run: clang-format --dry-run --Werror src/pg_lock_tracer/bpf/*.c ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ # VScode .vscode/ ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [2022] [Jan Nidzwetzki] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ # Lock tracing tools for PostgreSQL [![Build Status](https://github.com/jnidzwetzki/pg-lock-tracer/actions/workflows/tests.yml/badge.svg)](https://github.com/jnidzwetzki/pg-lock-tracer/actions/workflows/tests.yml) [![PyPI](https://img.shields.io/pypi/v/pg-lock-tracer?color=green)](https://pypi.org/project/pg-lock-tracer/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/pg-lock-tracer)](https://pypi.org/project/pg-lock-tracer/) [![Release date](https://img.shields.io/github/release-date/jnidzwetzki/pg-lock-tracer)](https://github.com/jnidzwetzki/pg-lock-tracer/) [![GitHub Repo stars](https://img.shields.io/github/stars/jnidzwetzki/pg-lock-tracer?style=social)](https://github.com/jnidzwetzki/pg-lock-tracer/) A set of eBPF-based tools designed to observe and visualize PostgreSQL 🐘 locking activity. These tools help users understand lock timing and contention behavior. Key components: - `pg_lock_tracer`: heavy lock tracer (e.g., table level locks) - `pg_lw_lock_tracer`: lightweight lock (LWLock) tracer - `pg_row_lock_tracer`: row-level lock tracer - `pg_spinlock_delay_tracer`: spinlock delay tracer - `animate_lock_graph`: render animated lock graphs from `pg_lock_tracer` tracer output __Note:__ These tools rely on [eBPF](https://ebpf.io/) (_Extended Berkeley Packet Filter_) technology. At the moment, PostgreSQL 14, 15, 16, 17, and 18 are supported (see additional information below). ## ⚡ Quickstart 1. Install the pg-lock-tracer tools: ``` $ pip install pg-lock-tracer ``` 2. Identify the PostgreSQL server binary used by your instance (e.g., `/usr/lib/postgresql/X/bin/postgres`). 3. Start tracing a running Postgres PID (requires root privileges): ``` $ sudo pg_lock_tracer -x /path/to/postgres ``` This shows the locking activity of all processes running this PostgreSQL binary. # pg_lock_tracer `pg_lock_tracer` observes the locking activity of a running PostgreSQL process (using _eBPF_ and _UProbes_). In contrast to the information that is present in the table `pg_locks` (which provides information about which locks are _currently_ requested), `pg_lock_tracer` gives you a continuous view of the locking activity and collects statistics and timings. The tracer also allows dumping the output as JSON-formatted lines, which allows further processing with additional tools. This repository also contains the script `animate_lock_graph`, which provides an animated version of the taken looks. ## 🧪 Usage Examples ``` # Trace use binary '/home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres' for tracing pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres # Trace use binary '/home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres' for tracing and trace pid 1234 pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 # Trace two PIDs pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 -p 5678 # Be verbose pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 -v # Use the given db connection to access the catalog of PID 1234 to resolve OIDs pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 -r 1234:psql://jan@localhost/test2 # Output in JSON format pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 -j # Print stacktrace on deadlock pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 -s DEADLOCK # Print stacktrace for locks and deadlocks pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 -s LOCK DEADLOCK # Trace only Transaction and Query related events pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 -t TRANSACTION QUERY # Write the output into file 'trace' pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 -o trace # Show statistics about locks pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 --statistics # Create an animated lock graph (with Oids) pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 -j -o locks.json animate_lock_graph -i lock -o locks.html # Create an animated lock graph (with table names) pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 -j -r 1234:psql://jan@localhost/test2 -o locks.json animate_lock_graph -i lock -o locks.html ``` ## 📄 Example Output CLI: `pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_14_2_DEBUG/bin/postgres -p 327578 -r 327578:sql://jan@localhost/test2 --statistics` SQL Query: `create table metrics(ts timestamptz NOT NULL, id int NOT NULL, value float);` Tracer Output: ``` 745064333930117 [Pid 327578] Query begin 'create table metrics(ts timestamptz NOT NULL, id int NOT NULL, value float);' 745064333965769 [Pid 327578] Transaction begin 745064334157640 [Pid 327578] Table open 3079 (pg_catalog.pg_extension) AccessShareLock 745064334176147 [Pid 327578] Lock object 3079 (pg_catalog.pg_extension) AccessShareLock 745064334204453 [Pid 327578] Lock granted (fastpath) 3079 (pg_catalog.pg_extension) AccessShareLock 745064334224361 [Pid 327578] Lock granted (local) 3079 (pg_catalog.pg_extension) AccessShareLock (Already hold local 0) 745064334243659 [Pid 327578] Lock was acquired in 67512 ns [...] ```
Full Output ``` ===> Ready to trace queries 745064333930117 [Pid 327578] Query begin 'create table metrics(ts timestamptz NOT NULL, id int NOT NULL, value float);' 745064333965769 [Pid 327578] Transaction begin 745064334157640 [Pid 327578] Table open 3079 (pg_catalog.pg_extension) AccessShareLock 745064334176147 [Pid 327578] Lock object 3079 (pg_catalog.pg_extension) AccessShareLock 745064334204453 [Pid 327578] Lock granted (fastpath) 3079 (pg_catalog.pg_extension) AccessShareLock 745064334224361 [Pid 327578] Lock granted (local) 3079 (pg_catalog.pg_extension) AccessShareLock (Already hold local 0) 745064334243659 [Pid 327578] Lock was acquired in 67512 ns 745064334285877 [Pid 327578] Lock object 3081 (pg_catalog.pg_extension_name_index) AccessShareLock 745064334309610 [Pid 327578] Lock granted (fastpath) 3081 (pg_catalog.pg_extension_name_index) AccessShareLock 745064334328475 [Pid 327578] Lock granted (local) 3081 (pg_catalog.pg_extension_name_index) AccessShareLock (Already hold local 0) 745064334345266 [Pid 327578] Lock was acquired in 59389 ns 745064334562977 [Pid 327578] Lock ungranted (fastpath) 3081 (pg_catalog.pg_extension_name_index) AccessShareLock 745064334583578 [Pid 327578] Lock ungranted (local) 3081 (pg_catalog.pg_extension_name_index) AccessShareLock (Hold local 0) 745064334608957 [Pid 327578] Table close 3079 (pg_catalog.pg_extension) AccessShareLock 745064334631046 [Pid 327578] Lock ungranted (fastpath) 3079 (pg_catalog.pg_extension) AccessShareLock 745064334649932 [Pid 327578] Lock ungranted (local) 3079 (pg_catalog.pg_extension) AccessShareLock (Hold local 0) 745064334671897 [Pid 327578] Table open 3079 (pg_catalog.pg_extension) AccessShareLock 745064334688382 [Pid 327578] Lock object 3079 (pg_catalog.pg_extension) AccessShareLock 745064334712042 [Pid 327578] Lock granted (fastpath) 3079 (pg_catalog.pg_extension) AccessShareLock 745064334731081 [Pid 327578] Lock granted (local) 3079 (pg_catalog.pg_extension) AccessShareLock (Already hold local 0) 745064334748288 [Pid 327578] Lock was acquired in 59906 ns 745064334772367 [Pid 327578] Lock object 3081 (pg_catalog.pg_extension_name_index) AccessShareLock 745064334795943 [Pid 327578] Lock granted (fastpath) 3081 (pg_catalog.pg_extension_name_index) AccessShareLock 745064334814983 [Pid 327578] Lock granted (local) 3081 (pg_catalog.pg_extension_name_index) AccessShareLock (Already hold local 0) 745064334832570 [Pid 327578] Lock was acquired in 60203 ns 745064334953192 [Pid 327578] Lock ungranted (fastpath) 3081 (pg_catalog.pg_extension_name_index) AccessShareLock 745064334973518 [Pid 327578] Lock ungranted (local) 3081 (pg_catalog.pg_extension_name_index) AccessShareLock (Hold local 0) 745064334997936 [Pid 327578] Table close 3079 (pg_catalog.pg_extension) AccessShareLock 745064335019473 [Pid 327578] Lock ungranted (fastpath) 3079 (pg_catalog.pg_extension) AccessShareLock 745064335037880 [Pid 327578] Lock ungranted (local) 3079 (pg_catalog.pg_extension) AccessShareLock (Hold local 0) 745064335901618 [Pid 327578] Table open 1259 (pg_catalog.pg_class) AccessShareLock 745064335918354 [Pid 327578] Lock object 1259 (pg_catalog.pg_class) AccessShareLock 745064335941911 [Pid 327578] Lock granted (fastpath) 1259 (pg_catalog.pg_class) AccessShareLock 745064335960211 [Pid 327578] Lock granted (local) 1259 (pg_catalog.pg_class) AccessShareLock (Already hold local 0) 745064335976642 [Pid 327578] Lock was acquired in 58288 ns 745064335999654 [Pid 327578] Lock object 2663 (pg_catalog.pg_class_relname_nsp_index) AccessShareLock 745064336022776 [Pid 327578] Lock granted (fastpath) 2663 (pg_catalog.pg_class_relname_nsp_index) AccessShareLock 745064336040926 [Pid 327578] Lock granted (local) 2663 (pg_catalog.pg_class_relname_nsp_index) AccessShareLock (Already hold local 0) 745064336057158 [Pid 327578] Lock was acquired in 57504 ns 745064336187786 [Pid 327578] Lock ungranted (fastpath) 2663 (pg_catalog.pg_class_relname_nsp_index) AccessShareLock 745064336207011 [Pid 327578] Lock ungranted (local) 2663 (pg_catalog.pg_class_relname_nsp_index) AccessShareLock (Hold local 0) 745064336230761 [Pid 327578] Table close 1259 (pg_catalog.pg_class) AccessShareLock 745064336252413 [Pid 327578] Lock ungranted (fastpath) 1259 (pg_catalog.pg_class) AccessShareLock 745064336270811 [Pid 327578] Lock ungranted (local) 1259 (pg_catalog.pg_class) AccessShareLock (Hold local 0) 745064336314237 [Pid 327578] Lock granted 2615 (pg_catalog.pg_namespace) AccessShareLock (Requested locks 1) 745064336331450 [Pid 327578] Lock granted (local) 2615 (pg_catalog.pg_namespace) AccessShareLock (Already hold local 0) 745064336402316 [Pid 327578] Lock granted (local) 2615 (pg_catalog.pg_namespace) AccessShareLock (Already hold local 1) 745064336543618 [Pid 327578] Table open 1259 (pg_catalog.pg_class) RowExclusiveLock 745064336560502 [Pid 327578] Lock object 1259 (pg_catalog.pg_class) RowExclusiveLock 745064336584633 [Pid 327578] Lock granted (fastpath) 1259 (pg_catalog.pg_class) RowExclusiveLock 745064336602915 [Pid 327578] Lock granted (local) 1259 (pg_catalog.pg_class) RowExclusiveLock (Already hold local 0) 745064336619969 [Pid 327578] Lock was acquired in 59467 ns 745064336655328 [Pid 327578] Table open 1247 (pg_catalog.pg_type) AccessShareLock 745064336671769 [Pid 327578] Lock object 1247 (pg_catalog.pg_type) AccessShareLock 745064336696072 [Pid 327578] Lock granted (fastpath) 1247 (pg_catalog.pg_type) AccessShareLock 745064336714540 [Pid 327578] Lock granted (local) 1247 (pg_catalog.pg_type) AccessShareLock (Already hold local 0) 745064336731130 [Pid 327578] Lock was acquired in 59361 ns 745064336755221 [Pid 327578] Lock object 2704 (pg_catalog.pg_type_typname_nsp_index) AccessShareLock 745064336778586 [Pid 327578] Lock granted (fastpath) 2704 (pg_catalog.pg_type_typname_nsp_index) AccessShareLock 745064336797018 [Pid 327578] Lock granted (local) 2704 (pg_catalog.pg_type_typname_nsp_index) AccessShareLock (Already hold local 0) 745064336813397 [Pid 327578] Lock was acquired in 58176 ns 745064336932804 [Pid 327578] Lock ungranted (fastpath) 2704 (pg_catalog.pg_type_typname_nsp_index) AccessShareLock 745064336952174 [Pid 327578] Lock ungranted (local) 2704 (pg_catalog.pg_type_typname_nsp_index) AccessShareLock (Hold local 0) 745064336975237 [Pid 327578] Table close 1247 (pg_catalog.pg_type) AccessShareLock 745064336996581 [Pid 327578] Lock ungranted (fastpath) 1247 (pg_catalog.pg_type) AccessShareLock 745064337014858 [Pid 327578] Lock ungranted (local) 1247 (pg_catalog.pg_type) AccessShareLock (Hold local 0) 745064337047504 [Pid 327578] Lock object 2662 (pg_catalog.pg_class_oid_index) AccessShareLock 745064337070928 [Pid 327578] Lock granted (fastpath) 2662 (pg_catalog.pg_class_oid_index) AccessShareLock 745064337089515 [Pid 327578] Lock granted (local) 2662 (pg_catalog.pg_class_oid_index) AccessShareLock (Already hold local 0) 745064337106032 [Pid 327578] Lock was acquired in 58528 ns 745064337183488 [Pid 327578] Lock ungranted (fastpath) 2662 (pg_catalog.pg_class_oid_index) AccessShareLock 745064337202563 [Pid 327578] Lock ungranted (local) 2662 (pg_catalog.pg_class_oid_index) AccessShareLock (Hold local 0) 745064337621853 [Pid 327578] Table open 1247 (pg_catalog.pg_type) AccessShareLock 745064337638996 [Pid 327578] Lock object 1247 (pg_catalog.pg_type) AccessShareLock 745064337661950 [Pid 327578] Lock granted (fastpath) 1247 (pg_catalog.pg_type) AccessShareLock 745064337681169 [Pid 327578] Lock granted (local) 1247 (pg_catalog.pg_type) AccessShareLock (Already hold local 0) 745064337697945 [Pid 327578] Lock was acquired in 58949 ns 745064337723254 [Pid 327578] Lock object 2703 (pg_catalog.pg_type_oid_index) AccessShareLock 745064337746949 [Pid 327578] Lock granted (fastpath) 2703 (pg_catalog.pg_type_oid_index) AccessShareLock 745064337765491 [Pid 327578] Lock granted (local) 2703 (pg_catalog.pg_type_oid_index) AccessShareLock (Already hold local 0) 745064337781897 [Pid 327578] Lock was acquired in 58643 ns 745064337865717 [Pid 327578] Lock ungranted (fastpath) 2703 (pg_catalog.pg_type_oid_index) AccessShareLock 745064337885245 [Pid 327578] Lock ungranted (local) 2703 (pg_catalog.pg_type_oid_index) AccessShareLock (Hold local 0) 745064337907299 [Pid 327578] Table close 1247 (pg_catalog.pg_type) AccessShareLock 745064337928390 [Pid 327578] Lock ungranted (fastpath) 1247 (pg_catalog.pg_type) AccessShareLock 745064337946792 [Pid 327578] Lock ungranted (local) 1247 (pg_catalog.pg_type) AccessShareLock (Hold local 0) 745064337970694 [Pid 327578] Table open 1247 (pg_catalog.pg_type) RowExclusiveLock 745064337987065 [Pid 327578] Lock object 1247 (pg_catalog.pg_type) RowExclusiveLock 745064338010254 [Pid 327578] Lock granted (fastpath) 1247 (pg_catalog.pg_type) RowExclusiveLock 745064338028898 [Pid 327578] Lock granted (local) 1247 (pg_catalog.pg_type) RowExclusiveLock (Already hold local 0) 745064338045413 [Pid 327578] Lock was acquired in 58348 ns 745064338073508 [Pid 327578] Lock object 2703 (pg_catalog.pg_type_oid_index) AccessShareLock 745064338096688 [Pid 327578] Lock granted (fastpath) 2703 (pg_catalog.pg_type_oid_index) AccessShareLock 745064338114955 [Pid 327578] Lock granted (local) 2703 (pg_catalog.pg_type_oid_index) AccessShareLock (Already hold local 0) 745064338131934 [Pid 327578] Lock was acquired in 58426 ns 745064338198858 [Pid 327578] Lock ungranted (fastpath) 2703 (pg_catalog.pg_type_oid_index) AccessShareLock 745064338218267 [Pid 327578] Lock ungranted (local) 2703 (pg_catalog.pg_type_oid_index) AccessShareLock (Hold local 0) 745064338257754 [Pid 327578] Lock object 2703 (pg_catalog.pg_type_oid_index) RowExclusiveLock 745064338281754 [Pid 327578] Lock granted (fastpath) 2703 (pg_catalog.pg_type_oid_index) RowExclusiveLock 745064338300356 [Pid 327578] Lock granted (local) 2703 (pg_catalog.pg_type_oid_index) RowExclusiveLock (Already hold local 0) 745064338317100 [Pid 327578] Lock was acquired in 59346 ns 745064338343423 [Pid 327578] Lock object 2704 (pg_catalog.pg_type_typname_nsp_index) RowExclusiveLock 745064338366354 [Pid 327578] Lock granted (fastpath) 2704 (pg_catalog.pg_type_typname_nsp_index) RowExclusiveLock 745064338384776 [Pid 327578] Lock granted (local) 2704 (pg_catalog.pg_type_typname_nsp_index) RowExclusiveLock (Already hold local 0) 745064338401188 [Pid 327578] Lock was acquired in 57765 ns 745064338925851 [Pid 327578] Lock ungranted (fastpath) 2703 (pg_catalog.pg_type_oid_index) RowExclusiveLock 745064338945562 [Pid 327578] Lock ungranted (local) 2703 (pg_catalog.pg_type_oid_index) RowExclusiveLock (Hold local 0) 745064338969060 [Pid 327578] Lock ungranted (fastpath) 2704 (pg_catalog.pg_type_typname_nsp_index) RowExclusiveLock 745064338987686 [Pid 327578] Lock ungranted (local) 2704 (pg_catalog.pg_type_typname_nsp_index) RowExclusiveLock (Hold local 0) 745064339016433 [Pid 327578] Table open 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064339032833 [Pid 327578] Lock object 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064339056324 [Pid 327578] Lock granted (fastpath) 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064339075245 [Pid 327578] Lock granted (local) 2608 (pg_catalog.pg_depend) RowExclusiveLock (Already hold local 0) 745064339092056 [Pid 327578] Lock was acquired in 59223 ns 745064339118530 [Pid 327578] Lock object 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064339141606 [Pid 327578] Lock granted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064339159893 [Pid 327578] Lock granted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Already hold local 0) 745064339176278 [Pid 327578] Lock was acquired in 57748 ns 745064339275302 [Pid 327578] Lock ungranted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064339295035 [Pid 327578] Lock ungranted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Hold local 0) 745064339320204 [Pid 327578] Lock object 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064339343605 [Pid 327578] Lock granted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064339362550 [Pid 327578] Lock granted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Already hold local 0) 745064339379189 [Pid 327578] Lock was acquired in 58985 ns 745064339466953 [Pid 327578] Lock ungranted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064339486483 [Pid 327578] Lock ungranted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Hold local 0) 745064339512103 [Pid 327578] Lock object 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064339535492 [Pid 327578] Lock granted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064339554170 [Pid 327578] Lock granted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Already hold local 0) 745064339570785 [Pid 327578] Lock was acquired in 58682 ns 745064339661539 [Pid 327578] Lock ungranted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064339680770 [Pid 327578] Lock ungranted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Hold local 0) 745064339706403 [Pid 327578] Lock object 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064339731859 [Pid 327578] Lock granted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064339750907 [Pid 327578] Lock granted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Already hold local 0) 745064339767580 [Pid 327578] Lock was acquired in 61177 ns 745064339855776 [Pid 327578] Lock ungranted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064339875244 [Pid 327578] Lock ungranted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Hold local 0) 745064339898815 [Pid 327578] Table close 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064339920526 [Pid 327578] Lock ungranted (fastpath) 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064339939861 [Pid 327578] Lock ungranted (local) 2608 (pg_catalog.pg_depend) RowExclusiveLock (Hold local 0) 745064339961989 [Pid 327578] Table open 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064339978907 [Pid 327578] Lock object 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064340001670 [Pid 327578] Lock granted (fastpath) 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064340020405 [Pid 327578] Lock granted (local) 2608 (pg_catalog.pg_depend) RowExclusiveLock (Already hold local 0) 745064340036728 [Pid 327578] Lock was acquired in 57821 ns 745064340060373 [Pid 327578] Lock object 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064340083157 [Pid 327578] Lock granted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064340101890 [Pid 327578] Lock granted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Already hold local 0) 745064340118486 [Pid 327578] Lock was acquired in 58113 ns 745064340198186 [Pid 327578] Lock ungranted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064340219330 [Pid 327578] Lock ungranted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Hold local 0) 745064340250646 [Pid 327578] Lock object 2673 (pg_catalog.pg_depend_depender_index) RowExclusiveLock 745064340274684 [Pid 327578] Lock granted (fastpath) 2673 (pg_catalog.pg_depend_depender_index) RowExclusiveLock 745064340293524 [Pid 327578] Lock granted (local) 2673 (pg_catalog.pg_depend_depender_index) RowExclusiveLock (Already hold local 0) 745064340310202 [Pid 327578] Lock was acquired in 59556 ns 745064340337112 [Pid 327578] Lock object 2674 (pg_catalog.pg_depend_reference_index) RowExclusiveLock 745064340360031 [Pid 327578] Lock granted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) RowExclusiveLock 745064340378739 [Pid 327578] Lock granted (local) 2674 (pg_catalog.pg_depend_reference_index) RowExclusiveLock (Already hold local 0) 745064340395496 [Pid 327578] Lock was acquired in 58384 ns 745064340576666 [Pid 327578] Lock ungranted (fastpath) 2673 (pg_catalog.pg_depend_depender_index) RowExclusiveLock 745064340596299 [Pid 327578] Lock ungranted (local) 2673 (pg_catalog.pg_depend_depender_index) RowExclusiveLock (Hold local 0) 745064340620661 [Pid 327578] Lock ungranted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) RowExclusiveLock 745064340639350 [Pid 327578] Lock ungranted (local) 2674 (pg_catalog.pg_depend_reference_index) RowExclusiveLock (Hold local 0) 745064340660990 [Pid 327578] Table close 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064340682154 [Pid 327578] Lock ungranted (fastpath) 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064340700466 [Pid 327578] Lock ungranted (local) 2608 (pg_catalog.pg_depend) RowExclusiveLock (Hold local 0) 745064340724983 [Pid 327578] Table close 1247 (pg_catalog.pg_type) RowExclusiveLock 745064340745747 [Pid 327578] Lock ungranted (fastpath) 1247 (pg_catalog.pg_type) RowExclusiveLock 745064340763978 [Pid 327578] Lock ungranted (local) 1247 (pg_catalog.pg_type) RowExclusiveLock (Hold local 0) 745064340789474 [Pid 327578] Table open 1247 (pg_catalog.pg_type) AccessShareLock 745064340805855 [Pid 327578] Lock object 1247 (pg_catalog.pg_type) AccessShareLock 745064340828989 [Pid 327578] Lock granted (fastpath) 1247 (pg_catalog.pg_type) AccessShareLock 745064340847526 [Pid 327578] Lock granted (local) 1247 (pg_catalog.pg_type) AccessShareLock (Already hold local 0) 745064340864256 [Pid 327578] Lock was acquired in 58401 ns 745064340885838 [Pid 327578] Lock object 2704 (pg_catalog.pg_type_typname_nsp_index) AccessShareLock 745064340909039 [Pid 327578] Lock granted (fastpath) 2704 (pg_catalog.pg_type_typname_nsp_index) AccessShareLock 745064340928115 [Pid 327578] Lock granted (local) 2704 (pg_catalog.pg_type_typname_nsp_index) AccessShareLock (Already hold local 0) 745064340944601 [Pid 327578] Lock was acquired in 58763 ns 745064341048196 [Pid 327578] Lock ungranted (fastpath) 2704 (pg_catalog.pg_type_typname_nsp_index) AccessShareLock 745064341067296 [Pid 327578] Lock ungranted (local) 2704 (pg_catalog.pg_type_typname_nsp_index) AccessShareLock (Hold local 0) 745064341090013 [Pid 327578] Table close 1247 (pg_catalog.pg_type) AccessShareLock 745064341111287 [Pid 327578] Lock ungranted (fastpath) 1247 (pg_catalog.pg_type) AccessShareLock 745064341129580 [Pid 327578] Lock ungranted (local) 1247 (pg_catalog.pg_type) AccessShareLock (Hold local 0) 745064341157165 [Pid 327578] Table open 1247 (pg_catalog.pg_type) RowExclusiveLock 745064341173685 [Pid 327578] Lock object 1247 (pg_catalog.pg_type) RowExclusiveLock 745064341196776 [Pid 327578] Lock granted (fastpath) 1247 (pg_catalog.pg_type) RowExclusiveLock 745064341215241 [Pid 327578] Lock granted (local) 1247 (pg_catalog.pg_type) RowExclusiveLock (Already hold local 0) 745064341231900 [Pid 327578] Lock was acquired in 58215 ns 745064341268528 [Pid 327578] Lock object 2703 (pg_catalog.pg_type_oid_index) RowExclusiveLock 745064341292333 [Pid 327578] Lock granted (fastpath) 2703 (pg_catalog.pg_type_oid_index) RowExclusiveLock 745064341341162 [Pid 327578] Lock granted (local) 2703 (pg_catalog.pg_type_oid_index) RowExclusiveLock (Already hold local 0) 745064341358832 [Pid 327578] Lock was acquired in 90304 ns 745064341383548 [Pid 327578] Lock object 2704 (pg_catalog.pg_type_typname_nsp_index) RowExclusiveLock 745064341406525 [Pid 327578] Lock granted (fastpath) 2704 (pg_catalog.pg_type_typname_nsp_index) RowExclusiveLock 745064341424954 [Pid 327578] Lock granted (local) 2704 (pg_catalog.pg_type_typname_nsp_index) RowExclusiveLock (Already hold local 0) 745064341441633 [Pid 327578] Lock was acquired in 58085 ns 745064341627523 [Pid 327578] Lock ungranted (fastpath) 2703 (pg_catalog.pg_type_oid_index) RowExclusiveLock 745064341646818 [Pid 327578] Lock ungranted (local) 2703 (pg_catalog.pg_type_oid_index) RowExclusiveLock (Hold local 0) 745064341670307 [Pid 327578] Lock ungranted (fastpath) 2704 (pg_catalog.pg_type_typname_nsp_index) RowExclusiveLock 745064341688874 [Pid 327578] Lock ungranted (local) 2704 (pg_catalog.pg_type_typname_nsp_index) RowExclusiveLock (Hold local 0) 745064341718375 [Pid 327578] Table open 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064341735109 [Pid 327578] Lock object 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064341758861 [Pid 327578] Lock granted (fastpath) 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064341777284 [Pid 327578] Lock granted (local) 2608 (pg_catalog.pg_depend) RowExclusiveLock (Already hold local 0) 745064341793917 [Pid 327578] Lock was acquired in 58808 ns 745064341817673 [Pid 327578] Lock object 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064341840607 [Pid 327578] Lock granted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064341859012 [Pid 327578] Lock granted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Already hold local 0) 745064341875436 [Pid 327578] Lock was acquired in 57763 ns 745064341968082 [Pid 327578] Lock ungranted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064341987340 [Pid 327578] Lock ungranted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Hold local 0) 745064342012766 [Pid 327578] Lock object 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064342036033 [Pid 327578] Lock granted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064342054445 [Pid 327578] Lock granted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Already hold local 0) 745064342070843 [Pid 327578] Lock was acquired in 58077 ns 745064342158890 [Pid 327578] Lock ungranted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064342177887 [Pid 327578] Lock ungranted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Hold local 0) 745064342203087 [Pid 327578] Lock object 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064342226414 [Pid 327578] Lock granted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064342245026 [Pid 327578] Lock granted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Already hold local 0) 745064342261449 [Pid 327578] Lock was acquired in 58362 ns 745064342345876 [Pid 327578] Lock ungranted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064342364857 [Pid 327578] Lock ungranted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Hold local 0) 745064342389842 [Pid 327578] Lock object 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064342412949 [Pid 327578] Lock granted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064342431353 [Pid 327578] Lock granted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Already hold local 0) 745064342488520 [Pid 327578] Lock was acquired in 98678 ns 745064342575520 [Pid 327578] Lock ungranted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064342594610 [Pid 327578] Lock ungranted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Hold local 0) 745064342619914 [Pid 327578] Lock object 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064342643371 [Pid 327578] Lock granted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064342662066 [Pid 327578] Lock granted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Already hold local 0) 745064342678513 [Pid 327578] Lock was acquired in 58599 ns 745064342770413 [Pid 327578] Lock ungranted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064342790169 [Pid 327578] Lock ungranted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Hold local 0) 745064342845082 [Pid 327578] Lock object 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064342868911 [Pid 327578] Lock granted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064342888807 [Pid 327578] Lock granted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Already hold local 0) 745064342905919 [Pid 327578] Lock was acquired in 60837 ns 745064342994114 [Pid 327578] Lock ungranted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064343014416 [Pid 327578] Lock ungranted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Hold local 0) 745064343039456 [Pid 327578] Table close 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064343061781 [Pid 327578] Lock ungranted (fastpath) 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064343081935 [Pid 327578] Lock ungranted (local) 2608 (pg_catalog.pg_depend) RowExclusiveLock (Hold local 0) 745064343104591 [Pid 327578] Table open 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064343121705 [Pid 327578] Lock object 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064343145901 [Pid 327578] Lock granted (fastpath) 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064343165164 [Pid 327578] Lock granted (local) 2608 (pg_catalog.pg_depend) RowExclusiveLock (Already hold local 0) 745064343182282 [Pid 327578] Lock was acquired in 60577 ns 745064343207310 [Pid 327578] Lock object 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064343231307 [Pid 327578] Lock granted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064343250703 [Pid 327578] Lock granted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Already hold local 0) 745064343267770 [Pid 327578] Lock was acquired in 60460 ns 745064343347934 [Pid 327578] Lock ungranted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064343367921 [Pid 327578] Lock ungranted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Hold local 0) 745064343398614 [Pid 327578] Lock object 2673 (pg_catalog.pg_depend_depender_index) RowExclusiveLock 745064343423194 [Pid 327578] Lock granted (fastpath) 2673 (pg_catalog.pg_depend_depender_index) RowExclusiveLock 745064343442803 [Pid 327578] Lock granted (local) 2673 (pg_catalog.pg_depend_depender_index) RowExclusiveLock (Already hold local 0) 745064343459844 [Pid 327578] Lock was acquired in 61230 ns 745064343485455 [Pid 327578] Lock object 2674 (pg_catalog.pg_depend_reference_index) RowExclusiveLock 745064343509461 [Pid 327578] Lock granted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) RowExclusiveLock 745064343528810 [Pid 327578] Lock granted (local) 2674 (pg_catalog.pg_depend_reference_index) RowExclusiveLock (Already hold local 0) 745064343545847 [Pid 327578] Lock was acquired in 60392 ns 745064343695078 [Pid 327578] Lock ungranted (fastpath) 2673 (pg_catalog.pg_depend_depender_index) RowExclusiveLock 745064343715321 [Pid 327578] Lock ungranted (local) 2673 (pg_catalog.pg_depend_depender_index) RowExclusiveLock (Hold local 0) 745064343740180 [Pid 327578] Lock ungranted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) RowExclusiveLock 745064343759691 [Pid 327578] Lock ungranted (local) 2674 (pg_catalog.pg_depend_reference_index) RowExclusiveLock (Hold local 0) 745064343782057 [Pid 327578] Table close 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064343803974 [Pid 327578] Lock ungranted (fastpath) 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064343823230 [Pid 327578] Lock ungranted (local) 2608 (pg_catalog.pg_depend) RowExclusiveLock (Hold local 0) 745064343848289 [Pid 327578] Table close 1247 (pg_catalog.pg_type) RowExclusiveLock 745064343870083 [Pid 327578] Lock ungranted (fastpath) 1247 (pg_catalog.pg_type) RowExclusiveLock 745064343889205 [Pid 327578] Lock ungranted (local) 1247 (pg_catalog.pg_type) RowExclusiveLock (Hold local 0) 745064343926948 [Pid 327578] Lock object 2662 (pg_catalog.pg_class_oid_index) RowExclusiveLock 745064343951401 [Pid 327578] Lock granted (fastpath) 2662 (pg_catalog.pg_class_oid_index) RowExclusiveLock 745064343970873 [Pid 327578] Lock granted (local) 2662 (pg_catalog.pg_class_oid_index) RowExclusiveLock (Already hold local 0) 745064343987792 [Pid 327578] Lock was acquired in 60844 ns 745064344013730 [Pid 327578] Lock object 2663 (pg_catalog.pg_class_relname_nsp_index) RowExclusiveLock 745064344038107 [Pid 327578] Lock granted (fastpath) 2663 (pg_catalog.pg_class_relname_nsp_index) RowExclusiveLock 745064344058417 [Pid 327578] Lock granted (local) 2663 (pg_catalog.pg_class_relname_nsp_index) RowExclusiveLock (Already hold local 0) 745064344075613 [Pid 327578] Lock was acquired in 61883 ns 745064344101275 [Pid 327578] Lock object 3455 (pg_catalog.pg_class_tblspc_relfilenode_index) RowExclusiveLock 745064344125438 [Pid 327578] Lock granted (fastpath) 3455 (pg_catalog.pg_class_tblspc_relfilenode_index) RowExclusiveLock 745064344144823 [Pid 327578] Lock granted (local) 3455 (pg_catalog.pg_class_tblspc_relfilenode_index) RowExclusiveLock (Already hold local 0) 745064344161792 [Pid 327578] Lock was acquired in 60517 ns 745064344396301 [Pid 327578] Lock ungranted (fastpath) 2662 (pg_catalog.pg_class_oid_index) RowExclusiveLock 745064344416716 [Pid 327578] Lock ungranted (local) 2662 (pg_catalog.pg_class_oid_index) RowExclusiveLock (Hold local 0) 745064344442093 [Pid 327578] Lock ungranted (fastpath) 2663 (pg_catalog.pg_class_relname_nsp_index) RowExclusiveLock 745064344461627 [Pid 327578] Lock ungranted (local) 2663 (pg_catalog.pg_class_relname_nsp_index) RowExclusiveLock (Hold local 0) 745064344486604 [Pid 327578] Lock ungranted (fastpath) 3455 (pg_catalog.pg_class_tblspc_relfilenode_index) RowExclusiveLock 745064344506759 [Pid 327578] Lock ungranted (local) 3455 (pg_catalog.pg_class_tblspc_relfilenode_index) RowExclusiveLock (Hold local 0) 745064344530484 [Pid 327578] Table open 1249 (pg_catalog.pg_attribute) RowExclusiveLock 745064344548232 [Pid 327578] Lock object 1249 (pg_catalog.pg_attribute) RowExclusiveLock 745064344572515 [Pid 327578] Lock granted (fastpath) 1249 (pg_catalog.pg_attribute) RowExclusiveLock 745064344592394 [Pid 327578] Lock granted (local) 1249 (pg_catalog.pg_attribute) RowExclusiveLock (Already hold local 0) 745064344609832 [Pid 327578] Lock was acquired in 61600 ns 745064344638223 [Pid 327578] Lock object 2658 (pg_catalog.pg_attribute_relid_attnam_index) RowExclusiveLock 745064344662911 [Pid 327578] Lock granted (fastpath) 2658 (pg_catalog.pg_attribute_relid_attnam_index) RowExclusiveLock 745064344682442 [Pid 327578] Lock granted (local) 2658 (pg_catalog.pg_attribute_relid_attnam_index) RowExclusiveLock (Already hold local 0) 745064344699381 [Pid 327578] Lock was acquired in 61158 ns 745064344726938 [Pid 327578] Lock object 2659 (pg_catalog.pg_attribute_relid_attnum_index) RowExclusiveLock 745064344751669 [Pid 327578] Lock granted (fastpath) 2659 (pg_catalog.pg_attribute_relid_attnum_index) RowExclusiveLock 745064344771221 [Pid 327578] Lock granted (local) 2659 (pg_catalog.pg_attribute_relid_attnum_index) RowExclusiveLock (Already hold local 0) 745064344788278 [Pid 327578] Lock was acquired in 61340 ns 745064345099100 [Pid 327578] Table open 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064345117485 [Pid 327578] Lock object 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064345142284 [Pid 327578] Lock granted (fastpath) 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064345161787 [Pid 327578] Lock granted (local) 2608 (pg_catalog.pg_depend) RowExclusiveLock (Already hold local 0) 745064345178996 [Pid 327578] Lock was acquired in 61511 ns 745064345204505 [Pid 327578] Lock object 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064345228926 [Pid 327578] Lock granted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064345248442 [Pid 327578] Lock granted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Already hold local 0) 745064345265473 [Pid 327578] Lock was acquired in 60968 ns 745064345393173 [Pid 327578] Lock ungranted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064345413932 [Pid 327578] Lock ungranted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Hold local 0) 745064345438503 [Pid 327578] Table close 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064345461015 [Pid 327578] Lock ungranted (fastpath) 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064345480576 [Pid 327578] Lock ungranted (local) 2608 (pg_catalog.pg_depend) RowExclusiveLock (Hold local 0) 745064345502818 [Pid 327578] Table open 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064345519944 [Pid 327578] Lock object 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064345544039 [Pid 327578] Lock granted (fastpath) 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064345563246 [Pid 327578] Lock granted (local) 2608 (pg_catalog.pg_depend) RowExclusiveLock (Already hold local 0) 745064345580361 [Pid 327578] Lock was acquired in 60417 ns 745064345605327 [Pid 327578] Lock object 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064345629516 [Pid 327578] Lock granted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064345648760 [Pid 327578] Lock granted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Already hold local 0) 745064345665722 [Pid 327578] Lock was acquired in 60395 ns 745064345752239 [Pid 327578] Lock ungranted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064345772579 [Pid 327578] Lock ungranted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Hold local 0) 745064345796763 [Pid 327578] Table close 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064345822035 [Pid 327578] Lock ungranted (fastpath) 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064345841159 [Pid 327578] Lock ungranted (local) 2608 (pg_catalog.pg_depend) RowExclusiveLock (Hold local 0) 745064345863107 [Pid 327578] Table open 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064345879945 [Pid 327578] Lock object 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064345904002 [Pid 327578] Lock granted (fastpath) 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064345923102 [Pid 327578] Lock granted (local) 2608 (pg_catalog.pg_depend) RowExclusiveLock (Already hold local 0) 745064345939934 [Pid 327578] Lock was acquired in 59989 ns 745064345964544 [Pid 327578] Lock object 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064345988395 [Pid 327578] Lock granted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064346007719 [Pid 327578] Lock granted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Already hold local 0) 745064346024410 [Pid 327578] Lock was acquired in 59866 ns 745064346111281 [Pid 327578] Lock ungranted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064346131283 [Pid 327578] Lock ungranted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Hold local 0) 745064346155446 [Pid 327578] Table close 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064346177619 [Pid 327578] Lock ungranted (fastpath) 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064346196627 [Pid 327578] Lock ungranted (local) 2608 (pg_catalog.pg_depend) RowExclusiveLock (Hold local 0) 745064346803127 [Pid 327578] Lock ungranted (fastpath) 2658 (pg_catalog.pg_attribute_relid_attnam_index) RowExclusiveLock 745064346823666 [Pid 327578] Lock ungranted (local) 2658 (pg_catalog.pg_attribute_relid_attnam_index) RowExclusiveLock (Hold local 0) 745064346848221 [Pid 327578] Lock ungranted (fastpath) 2659 (pg_catalog.pg_attribute_relid_attnum_index) RowExclusiveLock 745064346867396 [Pid 327578] Lock ungranted (local) 2659 (pg_catalog.pg_attribute_relid_attnum_index) RowExclusiveLock (Hold local 0) 745064346889677 [Pid 327578] Table close 1249 (pg_catalog.pg_attribute) RowExclusiveLock 745064346911402 [Pid 327578] Lock ungranted (fastpath) 1249 (pg_catalog.pg_attribute) RowExclusiveLock 745064346930609 [Pid 327578] Lock ungranted (local) 1249 (pg_catalog.pg_attribute) RowExclusiveLock (Hold local 0) 745064346954599 [Pid 327578] Table open 1214 (pg_catalog.pg_shdepend) RowExclusiveLock 745064346971633 [Pid 327578] Lock object 1214 (pg_catalog.pg_shdepend) RowExclusiveLock 745064347005777 [Pid 327578] Lock granted 1214 (pg_catalog.pg_shdepend) RowExclusiveLock (Requested locks 1) 745064347024111 [Pid 327578] Lock granted (local) 1214 (pg_catalog.pg_shdepend) RowExclusiveLock (Already hold local 0) 745064347042353 [Pid 327578] Lock was acquired in 70720 ns 745064347068452 [Pid 327578] Lock object 1233 (pg_catalog.pg_shdepend_reference_index) AccessShareLock 745064347099104 [Pid 327578] Lock granted 1233 (pg_catalog.pg_shdepend_reference_index) AccessShareLock (Requested locks 1) 745064347116884 [Pid 327578] Lock granted (local) 1233 (pg_catalog.pg_shdepend_reference_index) AccessShareLock (Already hold local 0) 745064347134876 [Pid 327578] Lock was acquired in 66424 ns 745064347218507 [Pid 327578] Lock ungranted 1233 (pg_catalog.pg_shdepend_reference_index) AccessShareLock (Requested locks 1) 745064347242474 [Pid 327578] Lock ungranted (local) 1233 (pg_catalog.pg_shdepend_reference_index) AccessShareLock (Hold local 0) 745064347266550 [Pid 327578] Table close 1214 (pg_catalog.pg_shdepend) RowExclusiveLock 745064347289588 [Pid 327578] Lock ungranted 1214 (pg_catalog.pg_shdepend) RowExclusiveLock (Requested locks 1) 745064347311734 [Pid 327578] Lock ungranted (local) 1214 (pg_catalog.pg_shdepend) RowExclusiveLock (Hold local 0) 745064347336007 [Pid 327578] Table open 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064347353183 [Pid 327578] Lock object 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064347376890 [Pid 327578] Lock granted (fastpath) 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064347396366 [Pid 327578] Lock granted (local) 2608 (pg_catalog.pg_depend) RowExclusiveLock (Already hold local 0) 745064347413477 [Pid 327578] Lock was acquired in 60294 ns 745064347438596 [Pid 327578] Lock object 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064347462497 [Pid 327578] Lock granted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064347481886 [Pid 327578] Lock granted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Already hold local 0) 745064347498752 [Pid 327578] Lock was acquired in 60156 ns 745064347638679 [Pid 327578] Lock ungranted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064347658780 [Pid 327578] Lock ungranted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Hold local 0) 745064347687516 [Pid 327578] Lock object 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064347711680 [Pid 327578] Lock granted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064347730837 [Pid 327578] Lock granted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Already hold local 0) 745064347749468 [Pid 327578] Lock was acquired in 61952 ns 745064347836563 [Pid 327578] Lock ungranted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock 745064347856592 [Pid 327578] Lock ungranted (local) 2674 (pg_catalog.pg_depend_reference_index) AccessShareLock (Hold local 0) 745064347885124 [Pid 327578] Lock object 2673 (pg_catalog.pg_depend_depender_index) RowExclusiveLock 745064347909388 [Pid 327578] Lock granted (fastpath) 2673 (pg_catalog.pg_depend_depender_index) RowExclusiveLock 745064347928524 [Pid 327578] Lock granted (local) 2673 (pg_catalog.pg_depend_depender_index) RowExclusiveLock (Already hold local 0) 745064347945405 [Pid 327578] Lock was acquired in 60281 ns 745064347971113 [Pid 327578] Lock object 2674 (pg_catalog.pg_depend_reference_index) RowExclusiveLock 745064347995140 [Pid 327578] Lock granted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) RowExclusiveLock 745064348014609 [Pid 327578] Lock granted (local) 2674 (pg_catalog.pg_depend_reference_index) RowExclusiveLock (Already hold local 0) 745064348102848 [Pid 327578] Lock was acquired in 131735 ns 745064359396329 [Pid 327578] Lock ungranted (fastpath) 2673 (pg_catalog.pg_depend_depender_index) RowExclusiveLock 745064359418584 [Pid 327578] Lock ungranted (local) 2673 (pg_catalog.pg_depend_depender_index) RowExclusiveLock (Hold local 0) 745064359442539 [Pid 327578] Lock ungranted (fastpath) 2674 (pg_catalog.pg_depend_reference_index) RowExclusiveLock 745064359461367 [Pid 327578] Lock ungranted (local) 2674 (pg_catalog.pg_depend_reference_index) RowExclusiveLock (Hold local 0) 745064359483081 [Pid 327578] Table close 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064359504077 [Pid 327578] Lock ungranted (fastpath) 2608 (pg_catalog.pg_depend) RowExclusiveLock 745064359522493 [Pid 327578] Lock ungranted (local) 2608 (pg_catalog.pg_depend) RowExclusiveLock (Hold local 0) 745064359548580 [Pid 327578] Table close 328332 (public.metrics) NoLock 745064359566541 [Pid 327578] Table close 1259 (pg_catalog.pg_class) RowExclusiveLock 745064359587367 [Pid 327578] Lock ungranted (fastpath) 1259 (pg_catalog.pg_class) RowExclusiveLock 745064359605717 [Pid 327578] Lock ungranted (local) 1259 (pg_catalog.pg_class) RowExclusiveLock (Hold local 0) 745064359651119 [Pid 327578] Table open 1259 (pg_catalog.pg_class) AccessShareLock 745064359667961 [Pid 327578] Lock object 1259 (pg_catalog.pg_class) AccessShareLock 745064359693375 [Pid 327578] Lock granted (fastpath) 1259 (pg_catalog.pg_class) AccessShareLock 745064359714827 [Pid 327578] Lock granted (local) 1259 (pg_catalog.pg_class) AccessShareLock (Already hold local 0) 745064359732703 [Pid 327578] Lock was acquired in 64742 ns 745064359755347 [Pid 327578] Lock object 2662 (pg_catalog.pg_class_oid_index) AccessShareLock 745064359778100 [Pid 327578] Lock granted (fastpath) 2662 (pg_catalog.pg_class_oid_index) AccessShareLock 745064359796247 [Pid 327578] Lock granted (local) 2662 (pg_catalog.pg_class_oid_index) AccessShareLock (Already hold local 0) 745064359812617 [Pid 327578] Lock was acquired in 57270 ns 745064359909910 [Pid 327578] Lock ungranted (fastpath) 2662 (pg_catalog.pg_class_oid_index) AccessShareLock 745064359929042 [Pid 327578] Lock ungranted (local) 2662 (pg_catalog.pg_class_oid_index) AccessShareLock (Hold local 0) 745064359952201 [Pid 327578] Table close 1259 (pg_catalog.pg_class) AccessShareLock 745064359973399 [Pid 327578] Lock ungranted (fastpath) 1259 (pg_catalog.pg_class) AccessShareLock 745064359991320 [Pid 327578] Lock ungranted (local) 1259 (pg_catalog.pg_class) AccessShareLock (Hold local 0) 745064360036095 [Pid 327578] Table open 1249 (pg_catalog.pg_attribute) AccessShareLock 745064360052741 [Pid 327578] Lock object 1249 (pg_catalog.pg_attribute) AccessShareLock 745064360075910 [Pid 327578] Lock granted (fastpath) 1249 (pg_catalog.pg_attribute) AccessShareLock 745064360093944 [Pid 327578] Lock granted (local) 1249 (pg_catalog.pg_attribute) AccessShareLock (Already hold local 0) 745064360110225 [Pid 327578] Lock was acquired in 57484 ns 745064360132111 [Pid 327578] Lock object 2659 (pg_catalog.pg_attribute_relid_attnum_index) AccessShareLock 745064360154727 [Pid 327578] Lock granted (fastpath) 2659 (pg_catalog.pg_attribute_relid_attnum_index) AccessShareLock 745064360172921 [Pid 327578] Lock granted (local) 2659 (pg_catalog.pg_attribute_relid_attnum_index) AccessShareLock (Already hold local 0) 745064360189385 [Pid 327578] Lock was acquired in 57274 ns 745064360354543 [Pid 327578] Lock ungranted (fastpath) 2659 (pg_catalog.pg_attribute_relid_attnum_index) AccessShareLock 745064360373938 [Pid 327578] Lock ungranted (local) 2659 (pg_catalog.pg_attribute_relid_attnum_index) AccessShareLock (Hold local 0) 745064360396537 [Pid 327578] Table close 1249 (pg_catalog.pg_attribute) AccessShareLock 745064360417835 [Pid 327578] Lock ungranted (fastpath) 1249 (pg_catalog.pg_attribute) AccessShareLock 745064360436021 [Pid 327578] Lock ungranted (local) 1249 (pg_catalog.pg_attribute) AccessShareLock (Hold local 0) 745064360483980 [Pid 327578] Lock object 328332 (public.metrics) AccessExclusiveLock 745064360606079 [Pid 327578] Lock granted 328332 (public.metrics) AccessExclusiveLock (Requested locks 1) 745064360623446 [Pid 327578] Lock granted (local) 328332 (public.metrics) AccessExclusiveLock (Already hold local 0) 745064360649586 [Pid 327578] Lock was acquired in 165606 ns 745064360728965 [Pid 327578] Table open 328332 (public.metrics) AccessExclusiveLock 745064360745603 [Pid 327578] Lock object 328332 (public.metrics) AccessExclusiveLock 745064360766604 [Pid 327578] Lock granted (local) 328332 (public.metrics) AccessExclusiveLock (Already hold local 1) 745064360781540 [Pid 327578] Lock was acquired in 35937 ns 745064360803960 [Pid 327578] Table close 328332 (public.metrics) NoLock 745064360915669 [Pid 327578] Table open 1259 (pg_catalog.pg_class) AccessShareLock 745064360932430 [Pid 327578] Lock object 1259 (pg_catalog.pg_class) AccessShareLock 745064360955869 [Pid 327578] Lock granted (fastpath) 1259 (pg_catalog.pg_class) AccessShareLock 745064360974141 [Pid 327578] Lock granted (local) 1259 (pg_catalog.pg_class) AccessShareLock (Already hold local 0) 745064360990725 [Pid 327578] Lock was acquired in 58295 ns 745064361012714 [Pid 327578] Lock object 2662 (pg_catalog.pg_class_oid_index) AccessShareLock 745064361035492 [Pid 327578] Lock granted (fastpath) 2662 (pg_catalog.pg_class_oid_index) AccessShareLock 745064361053701 [Pid 327578] Lock granted (local) 2662 (pg_catalog.pg_class_oid_index) AccessShareLock (Already hold local 0) 745064361070112 [Pid 327578] Lock was acquired in 57398 ns 745064361163634 [Pid 327578] Lock ungranted (fastpath) 2662 (pg_catalog.pg_class_oid_index) AccessShareLock 745064361182887 [Pid 327578] Lock ungranted (local) 2662 (pg_catalog.pg_class_oid_index) AccessShareLock (Hold local 0) 745064361206214 [Pid 327578] Table close 1259 (pg_catalog.pg_class) AccessShareLock 745064361227633 [Pid 327578] Lock ungranted (fastpath) 1259 (pg_catalog.pg_class) AccessShareLock 745064361245881 [Pid 327578] Lock ungranted (local) 1259 (pg_catalog.pg_class) AccessShareLock (Hold local 0) 745064361287525 [Pid 327578] Table open 1259 (pg_catalog.pg_class) AccessShareLock 745064361331732 [Pid 327578] Lock object 1259 (pg_catalog.pg_class) AccessShareLock 745064361356607 [Pid 327578] Lock granted (fastpath) 1259 (pg_catalog.pg_class) AccessShareLock 745064361375138 [Pid 327578] Lock granted (local) 1259 (pg_catalog.pg_class) AccessShareLock (Already hold local 0) 745064361391959 [Pid 327578] Lock was acquired in 60227 ns 745064361424521 [Pid 327578] Table close 1259 (pg_catalog.pg_class) AccessShareLock 745064361445624 [Pid 327578] Lock ungranted (fastpath) 1259 (pg_catalog.pg_class) AccessShareLock 745064361463789 [Pid 327578] Lock ungranted (local) 1259 (pg_catalog.pg_class) AccessShareLock (Hold local 0) 745064361535429 [Pid 327578] Table open 1259 (pg_catalog.pg_class) AccessShareLock 745064361552253 [Pid 327578] Lock object 1259 (pg_catalog.pg_class) AccessShareLock 745064361575061 [Pid 327578] Lock granted (fastpath) 1259 (pg_catalog.pg_class) AccessShareLock 745064361593224 [Pid 327578] Lock granted (local) 1259 (pg_catalog.pg_class) AccessShareLock (Already hold local 0) 745064361609584 [Pid 327578] Lock was acquired in 57331 ns 745064361631520 [Pid 327578] Lock object 2663 (pg_catalog.pg_class_relname_nsp_index) AccessShareLock 745064361654381 [Pid 327578] Lock granted (fastpath) 2663 (pg_catalog.pg_class_relname_nsp_index) AccessShareLock 745064361672562 [Pid 327578] Lock granted (local) 2663 (pg_catalog.pg_class_relname_nsp_index) AccessShareLock (Already hold local 0) 745064361688805 [Pid 327578] Lock was acquired in 57285 ns 745064361788882 [Pid 327578] Lock ungranted (fastpath) 2663 (pg_catalog.pg_class_relname_nsp_index) AccessShareLock 745064361808199 [Pid 327578] Lock ungranted (local) 2663 (pg_catalog.pg_class_relname_nsp_index) AccessShareLock (Hold local 0) 745064361831553 [Pid 327578] Table close 1259 (pg_catalog.pg_class) AccessShareLock 745064361852740 [Pid 327578] Lock ungranted (fastpath) 1259 (pg_catalog.pg_class) AccessShareLock 745064361870841 [Pid 327578] Lock ungranted (local) 1259 (pg_catalog.pg_class) AccessShareLock (Hold local 0) 745064362332644 [Pid 327578] Transaction commit 745064368169138 [Pid 327578] Lock ungranted (local) 2615 (pg_catalog.pg_namespace) AccessShareLock (Hold local 2) 745064368193137 [Pid 327578] Lock ungranted (local) 328332 (public.metrics) AccessExclusiveLock (Hold local 2) 745064368236259 [Pid 327578] Lock ungranted 328332 (public.metrics) AccessExclusiveLock (Requested locks 1) 745064368260426 [Pid 327578] Lock ungranted 2615 (pg_catalog.pg_namespace) AccessShareLock (Requested locks 1) 745064368747863 [Pid 327578] Query done ```
Statistics ``` Lock statistics: ================ Locks per oid +----------------------------------------------+----------+------------------------------+ | Lock Name | Requests | Total Lock Request Time (ns) | +----------------------------------------------+----------+------------------------------+ | pg_catalog.pg_depend_reference_index | 20 | 1174663 | | pg_catalog.pg_depend | 8 | 456525 | | pg_catalog.pg_type | 5 | 282986 | | pg_catalog.pg_type_typname_nsp_index | 4 | 229317 | | pg_catalog.pg_type_oid_index | 4 | 300239 | | pg_catalog.pg_class | 3 | 180540 | | pg_catalog.pg_class_oid_index | 3 | 172549 | | pg_catalog.pg_depend_depender_index | 3 | 171186 | | pg_catalog.pg_class_relname_nsp_index | 2 | 114311 | | pg_catalog.pg_attribute | 2 | 113041 | | pg_catalog.pg_attribute_relid_attnum_index | 2 | 113299 | | public.metrics | 2 | 223162 | | pg_catalog.pg_class_tblspc_relfilenode_index | 1 | 56426 | | pg_catalog.pg_attribute_relid_attnam_index | 1 | 57238 | | pg_catalog.pg_shdepend | 1 | 65878 | | pg_catalog.pg_shdepend_reference_index | 1 | 63127 | +----------------------------------------------+----------+------------------------------+ Lock types +---------------------+---------------------------+ | Lock Type | Number of requested locks | +---------------------+---------------------------+ | AccessShareLock | 32 | | RowExclusiveLock | 28 | | AccessExclusiveLock | 2 | +---------------------+---------------------------+ ``` ## Filter Trace Events `pg_lock_tracer` traces per default all supported events. However, often only certain events are required for the analysis (e.g., _which tables are opened?_). The event tracing can be restricted to certain events using the `-t ` parameter. The following events are currently supported: | Event | Description | |----------------|------------------------------------------------------------------------------------------------------| | `TRANSACTION` | Transactions related events (e.g., `StartTransaction`, `CommitTransaction`, `DeadLockReport`) | | `QUERY` | The executed queries (e.g., `exec_simple_query`) | | `TABLE` | Table open and close events (e.g., `table_open`, `table_openrv`, `table_close`) | | `LOCK` | Lock events (e.g., `LockRelationOid`, `UnlockRelationOid`, `GrantLock`, `FastPathGrantRelationLock`) | | `INVALIDATION` | Processing of cache invalidation messages (e.g., `AcceptInvalidationMessages`) | | `ERROR` | Error related events (e.g., `bpf_errstart`) | ``` pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_14_2_DEBUG/bin/postgres -p 2287921 -r 2287921:psql://jan@localhost/test2 --statistics -t TABLE [...] 4111321097620311 [Pid 2290711] Table open (by range value) .metric_name_local AccessShareLock 4111321097626817 [Pid 2287921] Table close 343035 (public.metric_name_local) NoLock 4111321097722307 [Pid 2287921] Table open 3079 (pg_catalog.pg_extension) AccessShareLock 4111321097877109 [Pid 2287921] Table close 3079 (pg_catalog.pg_extension) AccessShareLock 4111321097904906 [Pid 2287921] Table open 3079 (pg_catalog.pg_extension) AccessShareLock 4111321098012011 [Pid 2287921] Table close 3079 (pg_catalog.pg_extension) AccessShareLock 4111321098049134 [Pid 2287921] Table open 343035 (public.metric_name_local) NoLock 4111321098072567 [Pid 2287921] Table close 343035 (public.metric_name_local) NoLock 4111321098089922 [Pid 2287921] Table open 343035 (public.metric_name_local) NoLock 4111321098116394 [Pid 2287921] Table close 343035 (public.metric_name_local) NoLock 4111321098350309 [Pid 2287921] Table open 343035 (public.metric_name_local) NoLock 4111321098484761 [Pid 2287921] Table close 343035 (public.metric_name_local) NoLock 4111321098795235 [Pid 2287921] Table open 343035 (public.metric_name_local) NoLock 4111321098931302 [Pid 2287921] Table close 343035 (public.metric_name_local) NoLock ``` ## Stack Traces It is sometimes necessary to determine where in the source code a particular lock is requested. For this purpose, the option `-s ` can be used. In addition to the traces, stack traces are now also shown. For example, by specifying `-s LOCK` a stack trace is generated and shown on each lock event. The following example shows where the lock for `pg_catalog.pg_extension` was requested. ``` pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_14_2_DEBUG/bin/postgres -p 1051967 -r 1051967:sql://jan@localhost/test2 -s LOCK [...] 1990162746005798 [Pid 1051967] Lock object 3079 (pg_catalog.pg_extension) AccessShareLock LockRelationOid+0x0 [postgres] table_open+0x1d [postgres] parse_analyze+0xed [postgres] pg_analyze_and_rewrite+0x49 [postgres] exec_simple_query+0x2db [postgres] PostgresMain+0x833 [postgres] ExitPostmaster+0x0 [postgres] BackendStartup+0x1b1 [postgres] ServerLoop+0x2d9 [postgres] PostmasterMain+0x1286 [postgres] startup_hacks+0x0 [postgres] __libc_start_main+0xea [libc-2.31.so] [unknown] [...] ``` ### Animated Lock Graphs See the content of the [examples](examples/) directory for examples. # pg_lw_lock_tracer `pg_lw_lock_trace` allows to trace lightweight locks ([LWLocks](https://github.com/postgres/postgres/blob/c8e1ba736b2b9e8c98d37a5b77c4ed31baf94147/src/backend/storage/lmgr/lwlock.c)) in a PostgreSQL process via _Userland Statically Defined Tracing_ (USDT). ## 🧪 Usage Examples ``` # Trace the LW locks of the PID 1234 pg_lw_lock_tracer -p 1234 # Trace the LW locks of the PIDs 1234 and 5678 pg_lw_lock_tracer -p 1234 -p 5678 # Trace the LW locks of the PID 1234 and be verbose pg_lw_lock_tracer -p 1234 -v # Trace the LW locks of the PID 1234 and collect statistics pg_lw_lock_tracer -p 1234 -v --statistics ``` ## Example output SQL Query: `insert into test values(2);` CLI: `sudo pg_lw_lock_tracer -p 1698108 --statistics` Tracer output: ``` 2904552881615298 [Pid 1704367] Acquired lock LockFastPath (mode LW_EXCLUSIVE) / LWLockAcquire() 2904552881673849 [Pid 1704367] Unlock LockFastPath 2904552881782910 [Pid 1704367] Acquired lock ProcArray (mode LW_SHARED) / LWLockAcquire() 2904552881803614 [Pid 1704367] Unlock ProcArray 2904552881865272 [Pid 1704367] Acquired lock LockFastPath (mode LW_EXCLUSIVE) / LWLockAcquire() 2904552881883641 [Pid 1704367] Unlock LockFastPath [...] ```
Full Output ``` ===> Ready to trace 2904552881615298 [Pid 1704367] Acquired lock LockFastPath (mode LW_EXCLUSIVE) / LWLockAcquire() 2904552881673849 [Pid 1704367] Unlock LockFastPath 2904552881782910 [Pid 1704367] Acquired lock ProcArray (mode LW_SHARED) / LWLockAcquire() 2904552881803614 [Pid 1704367] Unlock ProcArray 2904552881865272 [Pid 1704367] Acquired lock LockFastPath (mode LW_EXCLUSIVE) / LWLockAcquire() 2904552881883641 [Pid 1704367] Unlock LockFastPath 2904552882095131 [Pid 1704367] Acquired lock ProcArray (mode LW_SHARED) / LWLockAcquire() 2904552882114171 [Pid 1704367] Unlock ProcArray 2904552882225372 [Pid 1704367] Acquired lock XidGen (mode LW_EXCLUSIVE) / LWLockAcquire() 2904552882246673 [Pid 1704367] Unlock XidGen 2904552882270279 [Pid 1704367] Acquired lock LockManager (mode LW_EXCLUSIVE) / LWLockAcquire() 2904552882296782 [Pid 1704367] Unlock LockManager 2904552882335466 [Pid 1704367] Acquired lock BufferMapping (mode LW_SHARED) / LWLockAcquire() 2904552882358198 [Pid 1704367] Unlock BufferMapping 2904552882379951 [Pid 1704367] Acquired lock BufferContent (mode LW_EXCLUSIVE) / LWLockAcquire() 2904552882415333 [Pid 1704367] Acquired lock WALInsert (mode LW_EXCLUSIVE) / LWLockAcquire() 2904552882485459 [Pid 1704367] Unlock WALInsert 2904552882506167 [Pid 1704367] Unlock BufferContent 2904552882590752 [Pid 1704367] Acquired lock WALInsert (mode LW_EXCLUSIVE) / LWLockAcquire() 2904552882611656 [Pid 1704367] Unlock WALInsert 2904552882638194 [Pid 1704367] Wait for WALWrite 2904554401202251 [Pid 1704367] Wait for WALWrite lock took 1518564057 ns 2904554401222926 [Pid 1704367] Waited but not acquired WALWrite (mode LW_EXCLUSIVE) / LWLockConditionalAcquire() 2904554401234504 [Pid 1704367] Acquired lock WALWrite (mode LW_EXCLUSIVE) / LWLockConditionalAcquire() 2904554404873664 [Pid 1704367] Unlock WALWrite 2904554404928035 [Pid 1704367] Acquired lock XactSLRU (mode LW_EXCLUSIVE) / LWLockConditionalAcquire() 2904554404950334 [Pid 1704367] Unlock XactSLRU 2904554404972224 [Pid 1704367] Acquired lock ProcArray (mode LW_EXCLUSIVE) / LWLockConditionalAcquire() 2904554404993887 [Pid 1704367] Unlock ProcArray 2904554405022734 [Pid 1704367] Acquired lock LockFastPath (mode LW_EXCLUSIVE) / LWLockAcquire() 2904554405038888 [Pid 1704367] Unlock LockFastPath 2904554405059788 [Pid 1704367] Acquired lock LockFastPath (mode LW_EXCLUSIVE) / LWLockAcquire() 2904554405088143 [Pid 1704367] Unlock LockFastPath 2904554405106194 [Pid 1704367] Acquired lock LockManager (mode LW_EXCLUSIVE) / LWLockAcquire() 2904554405145780 [Pid 1704367] Unlock LockManager 2904554405622791 [Pid 1704367] Acquired lock PgStatsData (mode LW_EXCLUSIVE) / LWLockConditionalAcquire() 2904554405640885 [Pid 1704367] Unlock PgStatsData 2904554405665146 [Pid 1704367] Acquired lock PgStatsData (mode LW_EXCLUSIVE) / LWLockConditionalAcquire() 2904554405682599 [Pid 1704367] Unlock PgStatsData 2904554405704514 [Pid 1704367] Acquired lock PgStatsData (mode LW_EXCLUSIVE) / LWLockConditionalAcquire() 2904554405720734 [Pid 1704367] Unlock PgStatsData 2904554405737937 [Pid 1704367] Acquired lock PgStatsData (mode LW_EXCLUSIVE) / LWLockConditionalAcquire() 2904554405755387 [Pid 1704367] Unlock PgStatsData ```
Statistics ``` Lock statistics: ================ Locks per tranche +---------------+----------+--------------------------+------------------------+-------------------------------+-----------------------------+-------+----------------+ | Tranche | Acquired | AcquireOrWait (Acquired) | AcquireOrWait (Waited) | ConditionalAcquire (Acquired) | ConditionalAcquire (Failed) | Waits | Wait time (ns) | +---------------+----------+--------------------------+------------------------+-------------------------------+-----------------------------+-------+----------------+ | BufferContent | 1 | 0 | 0 | 0 | 0 | 0 | 0 | | BufferMapping | 1 | 0 | 0 | 0 | 0 | 0 | 0 | | LockFastPath | 4 | 0 | 0 | 0 | 0 | 0 | 0 | | LockManager | 2 | 0 | 0 | 0 | 0 | 0 | 0 | | PgStatsData | 0 | 0 | 0 | 4 | 0 | 0 | 0 | | ProcArray | 2 | 0 | 0 | 1 | 0 | 0 | 0 | | WALInsert | 2 | 0 | 0 | 0 | 0 | 0 | 0 | | WALWrite | 0 | 1 | 1 | 0 | 0 | 1 | 1518564057 | | XactSLRU | 0 | 0 | 0 | 1 | 0 | 0 | 0 | | XidGen | 1 | 0 | 0 | 0 | 0 | 0 | 0 | +---------------+----------+--------------------------+------------------------+-------------------------------+-----------------------------+-------+----------------+ Locks per type +--------------+----------+ | Lock type | Requests | +--------------+----------+ | LW_EXCLUSIVE | 18 | | LW_SHARED | 3 | +--------------+----------+ ``` # pg_row_lock_tracer `pg_row_lock_tracer` allows to trace row locks (see the PostgreSQL [documentation](https://www.postgresql.org/docs/current/explicit-locking.html#LOCKING-ROWS)) of a PostgreSQL process using _eBPF_ and _UProbes_ ## 🧪 Usage Examples ``` # Trace the row locks of the given PostgreSQL binary pg_row_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_14_9_DEBUG/bin/postgres # Trace the row locks of the PID 1234 pg_row_lock_tracer -p 1234 -x /home/jan/postgresql-sandbox/bin/REL_14_9_DEBUG/bin/postgres # Trace the row locks of the PID 1234 and 5678 pg_row_lock_tracer -p 1234 -p 5678 -x /home/jan/postgresql-sandbox/bin/REL_14_9_DEBUG/bin/postgres # Trace the row locks of the PID 1234 and be verbose pg_row_lock_tracer -p 1234 -x /home/jan/postgresql-sandbox/bin/REL_14_9_DEBUG/bin/postgres -v # Trace the row locks and show statistics pg_row_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_14_9_DEBUG/bin/postgres --statistics ``` ## Example output SQL Query: `SELECT * FROM temperature FOR UPDATE;` CLI: `sudo pg_row_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_14_9_DEBUG/bin/postgres --statistics` Tracer output: ``` [...] 2783502701862408 [Pid 2604491] LOCK_TUPLE_END TM_OK in 13100 ns 2783502701877081 [Pid 2604491] LOCK_TUPLE (Tablespace 1663 database 305234 relation 313419) - (Block and offset 7 143) - LOCK_TUPLE_EXCLUSIVE LOCK_WAIT_BLOCK 2783502701972367 [Pid 2604491] LOCK_TUPLE_END TM_OK in 95286 ns 2783502701988387 [Pid 2604491] LOCK_TUPLE (Tablespace 1663 database 305234 relation 313419) - (Block and offset 7 144) - LOCK_TUPLE_EXCLUSIVE LOCK_WAIT_BLOCK 2783502702001690 [Pid 2604491] LOCK_TUPLE_END TM_OK in 13303 ns 2783502702016387 [Pid 2604491] LOCK_TUPLE (Tablespace 1663 database 305234 relation 313419) - (Block and offset 7 145) - LOCK_TUPLE_EXCLUSIVE LOCK_WAIT_BLOCK 2783502702029375 [Pid 2604491] LOCK_TUPLE_END TM_OK in 12988 ns ^C Lock statistics: ================ Used wait policies: +---------+-----------------+----------------+-----------------+ | PID | LOCK_WAIT_BLOCK | LOCK_WAIT_SKIP | LOCK_WAIT_ERROR | +---------+-----------------+----------------+-----------------+ | 2604491 | 1440 | 0 | 0 | +---------+-----------------+----------------+-----------------+ Lock modes: +---------+---------------------+------------------+---------------------------+----------------------+ | PID | LOCK_TUPLE_KEYSHARE | LOCK_TUPLE_SHARE | LOCK_TUPLE_NOKEYEXCLUSIVE | LOCK_TUPLE_EXCLUSIVE | +---------+---------------------+------------------+---------------------------+----------------------+ | 2604491 | 0 | 0 | 0 | 1440 | +---------+---------------------+------------------+---------------------------+----------------------+ Lock results: +---------+-------+--------------+-----------------+------------+------------+------------------+---------------+ | PID | TM_OK | TM_INVISIBLE | TM_SELFMODIFIED | TM_UPDATED | TM_DELETED | TM_BEINGMODIFIED | TM_WOULDBLOCK | +---------+-------+--------------+-----------------+------------+------------+------------------+---------------+ | 2604491 | 1440 | 0 | 0 | 0 | 0 | 0 | 0 | +---------+-------+--------------+-----------------+------------+------------+------------------+---------------+ ``` # pg_spinlock_delay_tracer `pg_spinlock_delay_tracer` allows tracing spinlock delays in a PostgreSQL process. Spin locks are used in PostgreSQL to protect short critical sections in the code. If another process already holds a spinlock, the requesting process will repeatedly check (i.e., "spin") until the lock becomes available. If the lock is held for a longer period, PostgreSQL performs a ["spin delay"](https://github.com/postgres/postgres/blob/0c8e082fba8d36434552d3d7800abda54acafd57/src/backend/storage/lmgr/s_lock.c#L106) and yields the CPU for a short time to avoid busy-waiting. Such delays are reported via the `pg_spinlock_delay_tracer`. For each delay, it prints the content of the `SpinDelayStatus` structure, which contains information about the number of spins, delays, and the current delay time. Additionally, the function name and source code location where the spin delay occurred are reported. ## 🧪 Usage Examples ``` # Trace the spinlock delays of the given PostgreSQL binary pg_spinlock_delay_tracer -x /home/jan/postgresql-sandbox/bin/REL_17_1_DEBUG/bin/postgres # Trace the spinlock delays of the PID 1234 pg_spinlock_delay_tracer -p 1234 -x /home/jan/postgresql-sandbox/bin/REL_17_1_DEBUG/bin/postgres ``` ## Example output To reproduce a spinlock delay, follow these steps to simulate a delay at the WAL insert position. First, start two different sessions connected to the same database. Then, create two tables: ```sql CREATE TABLE mydata1 (id INT); CREATE TABLE mydata2 (id INT); ``` Next, open a debugger and set a breakpoint in the function `ReserveXLogInsertLocation` after the spin lock `SpinLockAcquire(&Insert->insertpos_lck);` is acquired. Afterward, start the `pg_spinlock_delay_tracer` and perform two inserts in the two different sessions: ``` # Session 1 INSERT INTO mydata1 VALUES(1); # Session 2 INSERT INTO mydata2 VALUES(2); ``` Note: Two different tables are used to prevent both sessions from trying to lock the same buffer and waiting for each other on a different lock. The debugger should stop in the first session at the breakpoint. Furthermore, the pg_spinlock_delay_tracer should report spinlock delays in the second session, as the first session holds the spinlock for a longer period. ``` pg_spinlock_delay_tracer -x /home/jan/postgresql-sandbox/bin/REL_17_1_DEBUG/bin/postgres [...] 13180680737869452 [Pid 1864403] SpinDelay spins=996 delays=939 cur_delay=566086 at ReserveXLogInsertLocation, xlog.c:1132 13180680737874986 [Pid 1864403] SpinDelay spins=997 delays=939 cur_delay=566086 at ReserveXLogInsertLocation, xlog.c:1132 13180680737880522 [Pid 1864403] SpinDelay spins=998 delays=939 cur_delay=566086 at ReserveXLogInsertLocation, xlog.c:1132 13180680737886009 [Pid 1864403] SpinDelay spins=999 delays=939 cur_delay=566086 at ReserveXLogInsertLocation, xlog.c:1132 13180681304189362 [Pid 1864403] SpinDelay spins=0 delays=940 cur_delay=661655 at ReserveXLogInsertLocation, xlog.c:1132 13180681304227806 [Pid 1864403] SpinDelay spins=1 delays=940 cur_delay=661655 at ReserveXLogInsertLocation, xlog.c:1132 13180681304241759 [Pid 1864403] SpinDelay spins=2 delays=940 cur_delay=661655 at ReserveXLogInsertLocation, xlog.c:1132 13180681304255150 [Pid 1864403] SpinDelay spins=3 delays=940 cur_delay=661655 at ReserveXLogInsertLocation, xlog.c:1132 [...] ``` # Additional Information ## Installation The PostgreSQL lock tracing tools are available as a Python package. These tools depend on the Python package for BPF. Unfortunately, this package is currently not available via `pip` (the Python package manager). Therefore, the package of the Linux distribution needs to be installed to provide this dependency. On Debian and Ubuntu, this can be done by executing the following command: ```shell apt install python3-bpfcc ``` The tracing tools can be installed system-wide or in a dedicated [virtual environment](https://docs.python.org/3/library/venv.html). To create and install the tools in such a virtual environment, the following steps must be performed. To install the tools system-wide, these steps can be skipped. ```shell cd python3 -m venv .venv source .venv/bin/activate # Copy the distribution Python BCC packages into this environment cp -av /usr/lib/python3/dist-packages/bcc* $(python -c "import sysconfig; print(sysconfig.get_path('platlib'))") ``` Now, the tracing tools can be installed directly via `pip` by executing: ```shell pip install pg-lock-tracer ``` The tools are now installed and can be invoked by calling `pg_lock_tracer` or `pg_lw_lock_tracer`. If you want to install the latest development snapshot and development dependencies of the tools, the following commands need to be executed: ```shell pip install -r requirements_dev.txt pip install git+https://github.com/jnidzwetzki/pg-lock-tracer ``` ## PostgreSQL Build The software is tested with PostgreSQL versions 14, 15, 16, 17, and 18. In order to be able to attach the _uprobes_ to the functions, they should not to be optimized away (e.g., inlined) during the compilation of PostgreSQL. Otherwise errors like `Unable to locate function XXX` will occur when `pg_lock_tracer` is started. It is recommended to compile PostgreSQL with the following CFLAGS: `CFLAGS="-ggdb -Og -g3 -fno-omit-frame-pointer"`. `pg_lw_lock_trace` uses [USDT probes](https://www.postgresql.org/docs/current/dynamic-trace.html). Therefore, PostgreSQL has to be compiled with `--enable-dtrace` to use this script. ================================================ FILE: examples/create_table_trace.html ================================================
Frame
(
of
)
================================================ FILE: examples/create_table_trace.json ================================================ {"timestamp": 746704676283500, "pid": 328192, "event": "QUERY_BEGIN", "query": "create table metrics(ts timestamptz NOT NULL, id int NOT NULL, value float);"} {"timestamp": 746704676333954, "pid": 328192, "event": "TRANSACTION_BEGIN"} {"timestamp": 746704676556338, "pid": 328192, "event": "TABLE_OPEN", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class", "oid": 1259} {"timestamp": 746704676573555, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class", "oid": 1259} {"timestamp": 746704676600930, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class", "oid": 1259} {"timestamp": 746704676620078, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class", "oid": 1259, "lock_local_hold": 0} {"timestamp": 746704676637798, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 64243} {"timestamp": 746704676676605, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class_relname_nsp_index", "oid": 2663} {"timestamp": 746704676700278, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class_relname_nsp_index", "oid": 2663} {"timestamp": 746704676718307, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class_relname_nsp_index", "oid": 2663, "lock_local_hold": 0} {"timestamp": 746704676734539, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 57934} {"timestamp": 746704676916162, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class_relname_nsp_index", "oid": 2663} {"timestamp": 746704676935994, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class_relname_nsp_index", "oid": 2663} {"timestamp": 746704676960477, "pid": 328192, "event": "TABLE_CLOSE", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class", "oid": 1259} {"timestamp": 746704676982335, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class", "oid": 1259} {"timestamp": 746704677000364, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class", "oid": 1259} {"timestamp": 746704677042533, "pid": 328192, "event": "LOCK_GRANTED", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_namespace", "oid": 2615} {"timestamp": 746704677059297, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_namespace", "oid": 2615, "lock_local_hold": 0} {"timestamp": 746704677130926, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_namespace", "oid": 2615, "lock_local_hold": 1} {"timestamp": 746704677224521, "pid": 328192, "event": "TABLE_OPEN", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class", "oid": 1259} {"timestamp": 746704677241043, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class", "oid": 1259} {"timestamp": 746704677265037, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class", "oid": 1259} {"timestamp": 746704677282823, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class", "oid": 1259, "lock_local_hold": 0} {"timestamp": 746704677299007, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 57964} {"timestamp": 746704677335168, "pid": 328192, "event": "TABLE_OPEN", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704677350916, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704677373795, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704677391585, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type", "oid": 1247, "lock_local_hold": 0} {"timestamp": 746704677407426, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56510} {"timestamp": 746704677430067, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type_typname_nsp_index", "oid": 2704} {"timestamp": 746704677453292, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type_typname_nsp_index", "oid": 2704} {"timestamp": 746704677470917, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type_typname_nsp_index", "oid": 2704, "lock_local_hold": 0} {"timestamp": 746704677486958, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56891} {"timestamp": 746704677654979, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type_typname_nsp_index", "oid": 2704} {"timestamp": 746704677673853, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type_typname_nsp_index", "oid": 2704} {"timestamp": 746704677697409, "pid": 328192, "event": "TABLE_CLOSE", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704677718980, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704677783328, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704677818632, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class_oid_index", "oid": 2662} {"timestamp": 746704677842706, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class_oid_index", "oid": 2662} {"timestamp": 746704677860846, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class_oid_index", "oid": 2662, "lock_local_hold": 0} {"timestamp": 746704677876838, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 58206} {"timestamp": 746704677954619, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class_oid_index", "oid": 2662} {"timestamp": 746704677973108, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class_oid_index", "oid": 2662} {"timestamp": 746704678328344, "pid": 328192, "event": "TABLE_OPEN", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704678345124, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704678367909, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704678385577, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type", "oid": 1247, "lock_local_hold": 0} {"timestamp": 746704678401548, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56424} {"timestamp": 746704678425073, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type_oid_index", "oid": 2703} {"timestamp": 746704678518530, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type_oid_index", "oid": 2703} {"timestamp": 746704678536627, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type_oid_index", "oid": 2703, "lock_local_hold": 0} {"timestamp": 746704678552723, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 127650} {"timestamp": 746704678630504, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type_oid_index", "oid": 2703} {"timestamp": 746704678649158, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type_oid_index", "oid": 2703} {"timestamp": 746704678671189, "pid": 328192, "event": "TABLE_CLOSE", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704678692375, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704678710119, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704678733846, "pid": 328192, "event": "TABLE_OPEN", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704678749716, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704678772331, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704678790185, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type", "oid": 1247, "lock_local_hold": 0} {"timestamp": 746704678806681, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56965} {"timestamp": 746704678834178, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type_oid_index", "oid": 2703} {"timestamp": 746704678857069, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type_oid_index", "oid": 2703} {"timestamp": 746704678874840, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type_oid_index", "oid": 2703, "lock_local_hold": 0} {"timestamp": 746704678890971, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56793} {"timestamp": 746704678960527, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type_oid_index", "oid": 2703} {"timestamp": 746704678979302, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type_oid_index", "oid": 2703} {"timestamp": 746704679016766, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type_oid_index", "oid": 2703} {"timestamp": 746704679040320, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type_oid_index", "oid": 2703} {"timestamp": 746704679058271, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type_oid_index", "oid": 2703, "lock_local_hold": 0} {"timestamp": 746704679074562, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 57796} {"timestamp": 746704679100293, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type_typname_nsp_index", "oid": 2704} {"timestamp": 746704679123512, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type_typname_nsp_index", "oid": 2704} {"timestamp": 746704679141251, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type_typname_nsp_index", "oid": 2704, "lock_local_hold": 0} {"timestamp": 746704679157310, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 57017} {"timestamp": 746704679402113, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type_oid_index", "oid": 2703} {"timestamp": 746704679421747, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type_oid_index", "oid": 2703} {"timestamp": 746704679446017, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type_typname_nsp_index", "oid": 2704} {"timestamp": 746704679464279, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type_typname_nsp_index", "oid": 2704} {"timestamp": 746704679493790, "pid": 328192, "event": "TABLE_OPEN", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704679510271, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704679533670, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704679552280, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608, "lock_local_hold": 0} {"timestamp": 746704679568587, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 58316} {"timestamp": 746704679594742, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704679618295, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704679636182, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674, "lock_local_hold": 0} {"timestamp": 746704679652110, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 57368} {"timestamp": 746704679752050, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704679770865, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704679795403, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704679818254, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704679836029, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674, "lock_local_hold": 0} {"timestamp": 746704679852212, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56809} {"timestamp": 746704679940788, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704679959228, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704679983803, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704680006618, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704680024575, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674, "lock_local_hold": 0} {"timestamp": 746704680040606, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56803} {"timestamp": 746704680131786, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704680150220, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704680174890, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704680197309, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704680214981, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674, "lock_local_hold": 0} {"timestamp": 746704680231071, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56181} {"timestamp": 746704680317312, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704680336323, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704680359602, "pid": 328192, "event": "TABLE_CLOSE", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704680381513, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704680399519, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704680421854, "pid": 328192, "event": "TABLE_OPEN", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704680437476, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704680459915, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704680477484, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608, "lock_local_hold": 0} {"timestamp": 746704680493479, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56003} {"timestamp": 746704680516046, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704680539000, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704680556426, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674, "lock_local_hold": 0} {"timestamp": 746704680572353, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56307} {"timestamp": 746704680659789, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704680678134, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704680708322, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_depender_index", "oid": 2673} {"timestamp": 746704680731675, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_depender_index", "oid": 2673} {"timestamp": 746704680749555, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_depender_index", "oid": 2673, "lock_local_hold": 0} {"timestamp": 746704680765842, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 57520} {"timestamp": 746704680790981, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704680813999, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704680831675, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674, "lock_local_hold": 0} {"timestamp": 746704680847806, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56825} {"timestamp": 746704680989448, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_depender_index", "oid": 2673} {"timestamp": 746704681010372, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_depender_index", "oid": 2673} {"timestamp": 746704681034018, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704681052346, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704681073720, "pid": 328192, "event": "TABLE_CLOSE", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704681095005, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704681112656, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704681137265, "pid": 328192, "event": "TABLE_CLOSE", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704681158209, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704681175907, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704681201239, "pid": 328192, "event": "TABLE_OPEN", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704681217273, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704681240163, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704681257962, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type", "oid": 1247, "lock_local_hold": 0} {"timestamp": 746704681273940, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56667} {"timestamp": 746704681294500, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type_typname_nsp_index", "oid": 2704} {"timestamp": 746704681317288, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type_typname_nsp_index", "oid": 2704} {"timestamp": 746704681334828, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type_typname_nsp_index", "oid": 2704, "lock_local_hold": 0} {"timestamp": 746704681350835, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56335} {"timestamp": 746704681457495, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type_typname_nsp_index", "oid": 2704} {"timestamp": 746704681475984, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type_typname_nsp_index", "oid": 2704} {"timestamp": 746704681499135, "pid": 328192, "event": "TABLE_CLOSE", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704681520310, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704681537949, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704681565426, "pid": 328192, "event": "TABLE_OPEN", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704681581218, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704681603947, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704681621646, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type", "oid": 1247, "lock_local_hold": 0} {"timestamp": 746704681637638, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56420} {"timestamp": 746704681674028, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type_oid_index", "oid": 2703} {"timestamp": 746704681697804, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type_oid_index", "oid": 2703} {"timestamp": 746704681715696, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type_oid_index", "oid": 2703, "lock_local_hold": 0} {"timestamp": 746704681732028, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 58000} {"timestamp": 746704681789261, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type_typname_nsp_index", "oid": 2704} {"timestamp": 746704681813741, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type_typname_nsp_index", "oid": 2704} {"timestamp": 746704681831950, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type_typname_nsp_index", "oid": 2704, "lock_local_hold": 0} {"timestamp": 746704681848335, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 59074} {"timestamp": 746704681986220, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type_oid_index", "oid": 2703} {"timestamp": 746704682005166, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type_oid_index", "oid": 2703} {"timestamp": 746704682029300, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type_typname_nsp_index", "oid": 2704} {"timestamp": 746704682047617, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type_typname_nsp_index", "oid": 2704} {"timestamp": 746704682078008, "pid": 328192, "event": "TABLE_OPEN", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704682094093, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704682117111, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704682134832, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608, "lock_local_hold": 0} {"timestamp": 746704682151056, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56963} {"timestamp": 746704682174084, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704682196843, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704682214674, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674, "lock_local_hold": 0} {"timestamp": 746704682230736, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56652} {"timestamp": 746704682323576, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704682342453, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704682367129, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704682390600, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704682409007, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674, "lock_local_hold": 0} {"timestamp": 746704682424956, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 57827} {"timestamp": 746704682826824, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704682845721, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704682870818, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704682894489, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704682912369, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674, "lock_local_hold": 0} {"timestamp": 746704682929015, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 58197} {"timestamp": 746704683015112, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704683033858, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704683057951, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704683080989, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704683098619, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674, "lock_local_hold": 0} {"timestamp": 746704683114669, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56718} {"timestamp": 746704683203101, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704683221927, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704683245965, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704683269041, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704683286719, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674, "lock_local_hold": 0} {"timestamp": 746704683302827, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56862} {"timestamp": 746704683395707, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704683414442, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704683438595, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704683496427, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704683514717, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674, "lock_local_hold": 0} {"timestamp": 746704683531225, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 92630} {"timestamp": 746704683616675, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704683635066, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704683658102, "pid": 328192, "event": "TABLE_CLOSE", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704683679540, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704683697497, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704683719389, "pid": 328192, "event": "TABLE_OPEN", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704683735158, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704683758064, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704683775755, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608, "lock_local_hold": 0} {"timestamp": 746704683791860, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56702} {"timestamp": 746704683814656, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704683837597, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704683855313, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674, "lock_local_hold": 0} {"timestamp": 746704683871469, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56813} {"timestamp": 746704683953051, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704683971689, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704684000333, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_depender_index", "oid": 2673} {"timestamp": 746704684023429, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_depender_index", "oid": 2673} {"timestamp": 746704684040973, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_depender_index", "oid": 2673, "lock_local_hold": 0} {"timestamp": 746704684057015, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56682} {"timestamp": 746704684080689, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704684103803, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704684121274, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674, "lock_local_hold": 0} {"timestamp": 746704684137207, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56518} {"timestamp": 746704684268675, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_depender_index", "oid": 2673} {"timestamp": 746704684287973, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_depender_index", "oid": 2673} {"timestamp": 746704684311853, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704684329708, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704684353666, "pid": 328192, "event": "TABLE_CLOSE", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704684375030, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704684392997, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704684417335, "pid": 328192, "event": "TABLE_CLOSE", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704684438750, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704684456447, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_type", "oid": 1247} {"timestamp": 746704684493136, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class_oid_index", "oid": 2662} {"timestamp": 746704684516615, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class_oid_index", "oid": 2662} {"timestamp": 746704684534434, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class_oid_index", "oid": 2662, "lock_local_hold": 0} {"timestamp": 746704684550823, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 57687} {"timestamp": 746704684574481, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class_relname_nsp_index", "oid": 2663} {"timestamp": 746704684596862, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class_relname_nsp_index", "oid": 2663} {"timestamp": 746704684614532, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class_relname_nsp_index", "oid": 2663, "lock_local_hold": 0} {"timestamp": 746704684630858, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56377} {"timestamp": 746704684655806, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class_tblspc_relfilenode_index", "oid": 3455} {"timestamp": 746704684678566, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class_tblspc_relfilenode_index", "oid": 3455} {"timestamp": 746704684696029, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class_tblspc_relfilenode_index", "oid": 3455, "lock_local_hold": 0} {"timestamp": 746704684712232, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56426} {"timestamp": 746704684891052, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class_oid_index", "oid": 2662} {"timestamp": 746704684910356, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class_oid_index", "oid": 2662} {"timestamp": 746704684934496, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class_relname_nsp_index", "oid": 2663} {"timestamp": 746704684952910, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class_relname_nsp_index", "oid": 2663} {"timestamp": 746704684976724, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class_tblspc_relfilenode_index", "oid": 3455} {"timestamp": 746704684995182, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class_tblspc_relfilenode_index", "oid": 3455} {"timestamp": 746704685017751, "pid": 328192, "event": "TABLE_OPEN", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_attribute", "oid": 1249} {"timestamp": 746704685033785, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_attribute", "oid": 1249} {"timestamp": 746704685056178, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_attribute", "oid": 1249} {"timestamp": 746704685073827, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_attribute", "oid": 1249, "lock_local_hold": 0} {"timestamp": 746704685089841, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56056} {"timestamp": 746704685115419, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_attribute_relid_attnam_index", "oid": 2658} {"timestamp": 746704685138626, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_attribute_relid_attnam_index", "oid": 2658} {"timestamp": 746704685156409, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_attribute_relid_attnam_index", "oid": 2658, "lock_local_hold": 0} {"timestamp": 746704685172657, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 57238} {"timestamp": 746704685198120, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_attribute_relid_attnum_index", "oid": 2659} {"timestamp": 746704685220872, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_attribute_relid_attnum_index", "oid": 2659} {"timestamp": 746704685238463, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_attribute_relid_attnum_index", "oid": 2659, "lock_local_hold": 0} {"timestamp": 746704685254553, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56433} {"timestamp": 746704685806308, "pid": 328192, "event": "TABLE_OPEN", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704685822934, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704685846226, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704685864348, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608, "lock_local_hold": 0} {"timestamp": 746704685880703, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 57769} {"timestamp": 746704685904218, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704685927560, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704685945300, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674, "lock_local_hold": 0} {"timestamp": 746704685962114, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 57896} {"timestamp": 746704686056103, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704686074625, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704686098054, "pid": 328192, "event": "TABLE_CLOSE", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704686119932, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704686139591, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704686161273, "pid": 328192, "event": "TABLE_OPEN", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704686176771, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704686199544, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704686217503, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608, "lock_local_hold": 0} {"timestamp": 746704686233510, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56739} {"timestamp": 746704686256109, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704686278939, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704686296774, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674, "lock_local_hold": 0} {"timestamp": 746704686312823, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56714} {"timestamp": 746704686401528, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704686420177, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704686487864, "pid": 328192, "event": "TABLE_CLOSE", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704686510017, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704686527884, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704686549434, "pid": 328192, "event": "TABLE_OPEN", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704686565046, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704686587786, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704686605722, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608, "lock_local_hold": 0} {"timestamp": 746704686621830, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56784} {"timestamp": 746704686644498, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704686667282, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704686685041, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674, "lock_local_hold": 0} {"timestamp": 746704686701211, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56713} {"timestamp": 746704686788616, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704686806947, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704686830285, "pid": 328192, "event": "TABLE_CLOSE", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704686851804, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704686869782, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704687352765, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_attribute_relid_attnam_index", "oid": 2658} {"timestamp": 746704687371651, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_attribute_relid_attnam_index", "oid": 2658} {"timestamp": 746704687395545, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_attribute_relid_attnum_index", "oid": 2659} {"timestamp": 746704687413756, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_attribute_relid_attnum_index", "oid": 2659} {"timestamp": 746704687435363, "pid": 328192, "event": "TABLE_CLOSE", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_attribute", "oid": 1249} {"timestamp": 746704687456973, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_attribute", "oid": 1249} {"timestamp": 746704687474838, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_attribute", "oid": 1249} {"timestamp": 746704687496452, "pid": 328192, "event": "TABLE_OPEN", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_shdepend", "oid": 1214} {"timestamp": 746704687512662, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_shdepend", "oid": 1214} {"timestamp": 746704687544183, "pid": 328192, "event": "LOCK_GRANTED", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_shdepend", "oid": 1214} {"timestamp": 746704687560878, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_shdepend", "oid": 1214, "lock_local_hold": 0} {"timestamp": 746704687578540, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 65878} {"timestamp": 746704687602868, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_shdepend_reference_index", "oid": 1233} {"timestamp": 746704687632025, "pid": 328192, "event": "LOCK_GRANTED", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_shdepend_reference_index", "oid": 1233} {"timestamp": 746704687648452, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_shdepend_reference_index", "oid": 1233, "lock_local_hold": 0} {"timestamp": 746704687665995, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 63127} {"timestamp": 746704687750816, "pid": 328192, "event": "LOCK_UNGRANTED", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_shdepend_reference_index", "oid": 1233} {"timestamp": 746704687773201, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_shdepend_reference_index", "oid": 1233} {"timestamp": 746704687796972, "pid": 328192, "event": "TABLE_CLOSE", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_shdepend", "oid": 1214} {"timestamp": 746704687819103, "pid": 328192, "event": "LOCK_UNGRANTED", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_shdepend", "oid": 1214} {"timestamp": 746704687839622, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_shdepend", "oid": 1214} {"timestamp": 746704687864038, "pid": 328192, "event": "TABLE_OPEN", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704687880413, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704687903277, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704687921449, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608, "lock_local_hold": 0} {"timestamp": 746704687937662, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 57249} {"timestamp": 746704687960911, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704687983767, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704688001903, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674, "lock_local_hold": 0} {"timestamp": 746704688018077, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 57166} {"timestamp": 746704688155154, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704688173690, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704688200496, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704688223459, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704688241210, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674, "lock_local_hold": 0} {"timestamp": 746704688257544, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 57048} {"timestamp": 746704688345993, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704688364398, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704688391430, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_depender_index", "oid": 2673} {"timestamp": 746704688414429, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_depender_index", "oid": 2673} {"timestamp": 746704688432343, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_depender_index", "oid": 2673, "lock_local_hold": 0} {"timestamp": 746704688448414, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56984} {"timestamp": 746704688472121, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704688494705, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704688512407, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674, "lock_local_hold": 0} {"timestamp": 746704688528737, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56616} {"timestamp": 746704688665251, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_depender_index", "oid": 2673} {"timestamp": 746704688684168, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_depender_index", "oid": 2673} {"timestamp": 746704688708032, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704688726163, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend_reference_index", "oid": 2674} {"timestamp": 746704688747719, "pid": 328192, "event": "TABLE_CLOSE", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704688768970, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704688786812, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_depend", "oid": 2608} {"timestamp": 746704688812704, "pid": 328192, "event": "TABLE_CLOSE", "lock_type": "NoLock", "table": "public.metrics", "oid": 328345} {"timestamp": 746704688830423, "pid": 328192, "event": "TABLE_CLOSE", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class", "oid": 1259} {"timestamp": 746704688851604, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class", "oid": 1259} {"timestamp": 746704688869376, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "RowExclusiveLock", "table": "pg_catalog.pg_class", "oid": 1259} {"timestamp": 746704688912604, "pid": 328192, "event": "TABLE_OPEN", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class", "oid": 1259} {"timestamp": 746704688928682, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class", "oid": 1259} {"timestamp": 746704688952151, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class", "oid": 1259} {"timestamp": 746704688970901, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class", "oid": 1259, "lock_local_hold": 0} {"timestamp": 746704688987015, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 58333} {"timestamp": 746704689009486, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class_oid_index", "oid": 2662} {"timestamp": 746704689032310, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class_oid_index", "oid": 2662} {"timestamp": 746704689050044, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class_oid_index", "oid": 2662, "lock_local_hold": 0} {"timestamp": 746704689066142, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56656} {"timestamp": 746704689153458, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class_oid_index", "oid": 2662} {"timestamp": 746704689171818, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class_oid_index", "oid": 2662} {"timestamp": 746704689194876, "pid": 328192, "event": "TABLE_CLOSE", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class", "oid": 1259} {"timestamp": 746704689216684, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class", "oid": 1259} {"timestamp": 746704689234574, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_class", "oid": 1259} {"timestamp": 746704689277572, "pid": 328192, "event": "TABLE_OPEN", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_attribute", "oid": 1249} {"timestamp": 746704689294145, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_attribute", "oid": 1249} {"timestamp": 746704689316974, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_attribute", "oid": 1249} {"timestamp": 746704689334868, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_attribute", "oid": 1249, "lock_local_hold": 0} {"timestamp": 746704689351130, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56985} {"timestamp": 746704689371726, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_attribute_relid_attnum_index", "oid": 2659} {"timestamp": 746704689394701, "pid": 328192, "event": "LOCK_GRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_attribute_relid_attnum_index", "oid": 2659} {"timestamp": 746704689412360, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_attribute_relid_attnum_index", "oid": 2659, "lock_local_hold": 0} {"timestamp": 746704689428592, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 56866} {"timestamp": 746704689658305, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_attribute_relid_attnum_index", "oid": 2659} {"timestamp": 746704689677092, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_attribute_relid_attnum_index", "oid": 2659} {"timestamp": 746704689699954, "pid": 328192, "event": "TABLE_CLOSE", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_attribute", "oid": 1249} {"timestamp": 746704689721716, "pid": 328192, "event": "LOCK_UNGRANTED_FASTPATH", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_attribute", "oid": 1249} {"timestamp": 746704689768652, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_attribute", "oid": 1249} {"timestamp": 746704689815332, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessExclusiveLock", "table": "public.metrics", "oid": 328345} {"timestamp": 746704689959625, "pid": 328192, "event": "LOCK_GRANTED", "lock_type": "AccessExclusiveLock", "table": "public.metrics", "oid": 328345} {"timestamp": 746704689976218, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessExclusiveLock", "table": "public.metrics", "oid": 328345, "lock_local_hold": 0} {"timestamp": 746704690002386, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 187054} {"timestamp": 746704690056281, "pid": 328192, "event": "TABLE_OPEN", "lock_type": "AccessExclusiveLock", "table": "public.metrics", "oid": 328345} {"timestamp": 746704690072336, "pid": 328192, "event": "LOCK_RELATION_OID", "lock_type": "AccessExclusiveLock", "table": "public.metrics", "oid": 328345} {"timestamp": 746704690093376, "pid": 328192, "event": "LOCK_GRANTED_LOCAL", "lock_type": "AccessExclusiveLock", "table": "public.metrics", "oid": 328345, "lock_local_hold": 1} {"timestamp": 746704690108444, "pid": 328192, "event": "LOCK_RELATION_OID_END", "lock_time": 36108} {"timestamp": 746704690131202, "pid": 328192, "event": "TABLE_CLOSE", "lock_type": "NoLock", "table": "public.metrics", "oid": 328345} {"timestamp": 746704690193553, "pid": 328192, "event": "TRANSACTION_COMMIT"} {"timestamp": 746704694233743, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_namespace", "oid": 2615} {"timestamp": 746704694257757, "pid": 328192, "event": "LOCK_UNGRANTED_LOCAL", "lock_type": "AccessExclusiveLock", "table": "public.metrics", "oid": 328345} {"timestamp": 746704694313884, "pid": 328192, "event": "LOCK_UNGRANTED", "lock_type": "AccessExclusiveLock", "table": "public.metrics", "oid": 328345} {"timestamp": 746704694337362, "pid": 328192, "event": "LOCK_UNGRANTED", "lock_type": "AccessShareLock", "table": "pg_catalog.pg_namespace", "oid": 2615} {"timestamp": 746704694803804, "pid": 328192, "event": "QUERY_END"} ================================================ FILE: pylintrc ================================================ [MAIN] # Analyse import fallback blocks. This can be used to support both Python 2 and # 3 compatible code, which means that the block might have code that exists # only in one or another interpreter, leading to false positives when analysed. analyse-fallback-blocks=no # Clear in-memory caches upon conclusion of linting. Useful if running pylint # in a server-like mode. clear-cache-post-run=no # Load and enable all available extensions. Use --list-extensions to see a list # all available extensions. #enable-all-extensions= # In error mode, messages with a category besides ERROR or FATAL are # suppressed, and no reports are done by default. Error mode is compatible with # disabling specific errors. #errors-only= # Always return a 0 (non-error) status code, even if lint errors are found. # This is primarily useful in continuous integration scripts. #exit-zero= # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code. extension-pkg-allow-list= # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code. (This is an alternative name to extension-pkg-allow-list # for backward compatibility.) extension-pkg-whitelist= # Return non-zero exit code if any of these messages/categories are detected, # even if score is above --fail-under value. Syntax same as enable. Messages # specified are enabled, while categories only check already-enabled messages. fail-on= # Specify a score threshold under which the program will exit with error. fail-under=10.0 # Interpret the stdin as a python script, whose filename needs to be passed as # the module_or_package argument. #from-stdin= # Files or directories to be skipped. They should be base names, not paths. ignore=CVS # Add files or directories matching the regular expressions patterns to the # ignore-list. The regex matches against paths and can be in Posix or Windows # format. Because '\\' represents the directory delimiter on Windows systems, # it can't be used as an escape character. ignore-paths= # Files or directories matching the regular expression patterns are skipped. # The regex matches against base names, not paths. The default value ignores # Emacs file locks ignore-patterns=^\.# # List of module names for which member attributes should not be checked # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis). It # supports qualified module names, as well as Unix pattern matching. ignored-modules= # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). #init-hook= # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the # number of processors available to use, and will cap the count on Windows to # avoid hangs. jobs=1 # Control the amount of potential inferred values when inferring a single # object. This can help the performance when dealing with large functions or # complex, nested conditions. limit-inference-results=100 # List of plugins (as comma separated values of python module names) to load, # usually to register additional checkers. load-plugins= # Pickle collected data for later comparisons. persistent=yes # Minimum Python version to use for version dependent checks. Will default to # the version used to run pylint. py-version=3.9 # Discover python modules and packages in the file system subtree. recursive=no # When enabled, pylint would attempt to guess common misconfiguration and emit # user-friendly hints instead of false-positive error messages. suggestion-mode=yes # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no # In verbose mode, extra non-checker-related info will be displayed. #verbose= [BASIC] # Naming style matching correct argument names. argument-naming-style=snake_case # Regular expression matching correct argument names. Overrides argument- # naming-style. If left empty, argument names will be checked with the set # naming style. #argument-rgx= # Naming style matching correct attribute names. attr-naming-style=snake_case # Regular expression matching correct attribute names. Overrides attr-naming- # style. If left empty, attribute names will be checked with the set naming # style. #attr-rgx= # Bad variable names which should always be refused, separated by a comma. bad-names=foo, bar, baz, toto, tutu, tata # Bad variable names regexes, separated by a comma. If names match any regex, # they will always be refused bad-names-rgxs= # Naming style matching correct class attribute names. class-attribute-naming-style=any # Regular expression matching correct class attribute names. Overrides class- # attribute-naming-style. If left empty, class attribute names will be checked # with the set naming style. #class-attribute-rgx= # Naming style matching correct class constant names. class-const-naming-style=UPPER_CASE # Regular expression matching correct class constant names. Overrides class- # const-naming-style. If left empty, class constant names will be checked with # the set naming style. #class-const-rgx= # Naming style matching correct class names. class-naming-style=PascalCase # Regular expression matching correct class names. Overrides class-naming- # style. If left empty, class names will be checked with the set naming style. #class-rgx= # Naming style matching correct constant names. const-naming-style=UPPER_CASE # Regular expression matching correct constant names. Overrides const-naming- # style. If left empty, constant names will be checked with the set naming # style. #const-rgx= # Minimum line length for functions/classes that require docstrings, shorter # ones are exempt. docstring-min-length=-1 # Naming style matching correct function names. function-naming-style=snake_case # Regular expression matching correct function names. Overrides function- # naming-style. If left empty, function names will be checked with the set # naming style. #function-rgx= # Good variable names which should always be accepted, separated by a comma. good-names=i, j, k, ex, Run, _ # Good variable names regexes, separated by a comma. If names match any regex, # they will always be accepted good-names-rgxs= # Include a hint for the correct naming format with invalid-name. include-naming-hint=no # Naming style matching correct inline iteration names. inlinevar-naming-style=any # Regular expression matching correct inline iteration names. Overrides # inlinevar-naming-style. If left empty, inline iteration names will be checked # with the set naming style. #inlinevar-rgx= # Naming style matching correct method names. method-naming-style=snake_case # Regular expression matching correct method names. Overrides method-naming- # style. If left empty, method names will be checked with the set naming style. #method-rgx= # Naming style matching correct module names. module-naming-style=snake_case # Regular expression matching correct module names. Overrides module-naming- # style. If left empty, module names will be checked with the set naming style. #module-rgx= # Colon-delimited sets of names that determine each other's naming style when # the name regexes allow several styles. name-group= # Regular expression which should only match function or class names that do # not require a docstring. no-docstring-rgx=^_ # List of decorators that produce properties, such as abc.abstractproperty. Add # to this list to register other decorators that produce valid properties. # These decorators are taken in consideration only for invalid-name. property-classes=abc.abstractproperty # Regular expression matching correct type variable names. If left empty, type # variable names will be checked with the set naming style. #typevar-rgx= # Naming style matching correct variable names. variable-naming-style=snake_case # Regular expression matching correct variable names. Overrides variable- # naming-style. If left empty, variable names will be checked with the set # naming style. #variable-rgx= [CLASSES] # Warn about protected attribute access inside special methods check-protected-access-in-special-methods=no # List of method names used to declare (i.e. assign) instance attributes. defining-attr-methods=__init__, __new__, setUp, __post_init__ # List of member names, which should be excluded from the protected access # warning. exclude-protected=_asdict, _fields, _replace, _source, _make # List of valid names for the first argument in a class method. valid-classmethod-first-arg=cls # List of valid names for the first argument in a metaclass class method. valid-metaclass-classmethod-first-arg=cls [DESIGN] # List of regular expressions of class ancestor names to ignore when counting # public methods (see R0903) exclude-too-few-public-methods= # List of qualified class names to ignore when counting class parents (see # R0901) ignored-parents= # Maximum number of arguments for function / method. max-args=10 # Maximum number of attributes for a class (see R0902). max-attributes=10 # Maximum number of boolean expressions in an if statement (see R0916). max-bool-expr=5 # Maximum number of branch for function / method body. max-branches=12 # Maximum number of locals for function / method body. max-locals=30 # Maximum number of parents for a class (see R0901). max-parents=7 # Maximum number of public methods for a class (see R0904). max-public-methods=25 # Maximum number of return / yield for function / method body. max-returns=20 # Maximum number of statements in function / method body. max-statements=50 # Minimum number of public methods for a class (see R0903). min-public-methods=2 max-positional-arguments=8 [EXCEPTIONS] # Exceptions that will emit a warning when caught. overgeneral-exceptions=builtins.BaseException,builtins.Exception [FORMAT] # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. expected-line-ending-format= # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ # Number of spaces of indent required inside a hanging or continued line. indent-after-paren=4 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). indent-string=' ' # Maximum number of characters on a single line. max-line-length=100 # Maximum number of lines in a module. max-module-lines=1000 # Allow the body of a class to be on the same line as the declaration if body # contains single statement. single-line-class-stmt=no # Allow the body of an if to be on the same line as the test if there is no # else. single-line-if-stmt=no [IMPORTS] # List of modules that can be imported at any level, not just the top level # one. allow-any-import-level= # Allow explicit reexports by alias from a package __init__. allow-reexport-from-package=no # Allow wildcard imports from modules that define __all__. allow-wildcard-with-all=no # Deprecated modules which should not be used, separated by a comma. deprecated-modules= # Output a graph (.gv or any supported image format) of external dependencies # to the given file (report RP0402 must not be disabled). ext-import-graph= # Output a graph (.gv or any supported image format) of all (i.e. internal and # external) dependencies to the given file (report RP0402 must not be # disabled). import-graph= # Output a graph (.gv or any supported image format) of internal dependencies # to the given file (report RP0402 must not be disabled). int-import-graph= # Force import order to recognize a module as part of the standard # compatibility libraries. known-standard-library= # Force import order to recognize a module as part of a third party library. known-third-party=enchant # Couples of modules and preferred modules, separated by a comma. preferred-modules= [LOGGING] # The type of string formatting that logging methods do. `old` means using % # formatting, `new` is for `{}` formatting. logging-format-style=old # Logging modules to check that the string format arguments are in logging # function parameter format. logging-modules=logging [MESSAGES CONTROL] # Only show warnings with the listed confidence levels. Leave empty to show # all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, # UNDEFINED. confidence=HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, UNDEFINED # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifiers separated by comma (,) or put this # option multiple times (only on the command line, not in the configuration # file where it should appear only once). You can also use "--disable=all" to # disable everything first and then re-enable specific checks. For example, if # you want to run only the similarities checker, you can use "--disable=all # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use "--disable=all --enable=classes # --disable=W". disable=raw-checker-failed, bad-inline-option, locally-disabled, file-ignored, suppressed-message, useless-suppression, deprecated-pragma, use-symbolic-message-instead, missing-module-docstring, missing-class-docstring, missing-function-docstring # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where # it should appear only once). See also the "--disable" option for examples. enable=c-extension-no-member [METHOD_ARGS] # List of qualified names (i.e., library.method) which require a timeout # parameter e.g. 'requests.api.get,requests.api.post' timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. notes=FIXME, XXX, TODO # Regular expression of note tags to take in consideration. notes-rgx= [REFACTORING] # Maximum number of nested blocks for function / method body max-nested-blocks=5 # Complete name of functions that never returns. When checking for # inconsistent-return-statements if a never returning function is called then # it will be considered as an explicit return statement and no message will be # printed. never-returning-functions=sys.exit,argparse.parse_error [REPORTS] # Python expression which should return a score less than or equal to 10. You # have access to the variables 'fatal', 'error', 'warning', 'refactor', # 'convention', and 'info' which contain the number of messages in each # category, as well as 'statement' which is the total number of statements # analyzed. This score is used by the global evaluation report (RP0004). evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) # Template used to display messages. This is a python new-style format string # used to format the message information. See doc for all details. msg-template= # Set the output format. Available formats are text, parseable, colorized, json # and msvs (visual studio). You can also give a reporter class, e.g. # mypackage.mymodule.MyReporterClass. #output-format= # Tells whether to display a full report or only the messages. reports=no # Activate the evaluation score. score=yes [SIMILARITIES] # Comments are removed from the similarity computation ignore-comments=yes # Docstrings are removed from the similarity computation ignore-docstrings=yes # Imports are removed from the similarity computation ignore-imports=no # Signatures are removed from the similarity computation ignore-signatures=no # Minimum lines number of a similarity. min-similarity-lines=4 [SPELLING] # Limits count of emitted suggestions for spelling mistakes. max-spelling-suggestions=4 # Spelling dictionary name. Available dictionaries: none. To make it work, # install the 'python-enchant' package. spelling-dict= # List of comma separated words that should be considered directives if they # appear at the beginning of a comment and should not be checked. spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: # List of comma separated words that should not be checked. spelling-ignore-words= # A path to a file that contains the private dictionary; one word per line. spelling-private-dict-file= # Tells whether to store unknown words to the private dictionary (see the # --spelling-private-dict-file option) instead of raising a message. spelling-store-unknown-words=no [STRING] # This flag controls whether inconsistent-quotes generates a warning when the # character used as a quote delimiter is used inconsistently within a module. check-quote-consistency=no # This flag controls whether the implicit-str-concat should generate a warning # on implicit string concatenation in sequences defined over several lines. check-str-concat-over-line-jumps=no [TYPECHECK] # List of decorators that produce context managers, such as # contextlib.contextmanager. Add to this list to register other decorators that # produce valid context managers. contextmanager-decorators=contextlib.contextmanager # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular # expressions are accepted. generated-members= # Tells whether to warn about missing members when the owner of the attribute # is inferred to be None. ignore-none=yes # This flag controls whether pylint should warn about no-member and similar # checks whenever an opaque object is returned when inferring. The inference # can return multiple potential results while evaluating a Python object, but # some branches might not be evaluated, which results in partial inference. In # that case, it might be useful to still emit no-member and other checks for # the rest of the inferred objects. ignore-on-opaque-inference=yes # List of symbolic message names to ignore for Mixin members. ignored-checks-for-mixins=no-member, not-async-context-manager, not-context-manager, attribute-defined-outside-init # List of class names for which member attributes should not be checked (useful # for classes with dynamically set attributes). This supports the use of # qualified names. ignored-classes=optparse.Values,thread._local,_thread._local # Show a hint with possible names when a member name was not found. The aspect # of finding the hint is based on edit distance. missing-member-hint=yes # The minimum edit distance a name should have in order to be considered a # similar match for a missing member name. missing-member-hint-distance=1 # The total number of similar names that should be taken in consideration when # showing a hint for a missing member. missing-member-max-choices=1 # Regex pattern to define which classes are considered mixins. mixin-class-rgx=.*[Mm]ixin # List of decorators that change the signature of a decorated function. signature-mutators= [VARIABLES] # List of additional names supposed to be defined in builtins. Remember that # you should avoid defining new builtins when possible. additional-builtins= # Tells whether unused global variables should be treated as a violation. allow-global-unused-variables=yes # List of names allowed to shadow builtins allowed-redefined-builtins= # List of strings which can identify a callback function by name. A callback # name must start or end with one of those strings. callbacks=cb_, _cb # A regular expression matching the name of dummy variables (i.e. expected to # not be used). dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ # Argument names that match this expression will be ignored. ignored-argument-names=_.*|^ignored_|^unused_ # Tells whether we should check for unused import in __init__ files. init-import=no # List of qualified module names which can have objects that can redefine # builtins. redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io ================================================ FILE: pyproject.toml ================================================ [build-system] requires = ["setuptools>=59"] build-backend = "setuptools.build_meta" [project] name = "pg_lock_tracer" description = "An eBPF based lock tracer for PostgreSQL" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.10" license = "Apache-2.0" license-files = [ "LICENSE", ] authors = [ { name = "Jan Nidzwetzki", email = "jnidzwetzki@gmx.de" }, ] keywords = ["postgresql", "postgres", "ebpf", "locktracer"] classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Operating System :: POSIX :: Linux", "Programming Language :: Python", "Topic :: Software Development :: Debuggers" ] dependencies = [ "graphviz", "igraph", "prettytable", "psycopg2", ] dynamic = ["version"] [project.urls] Homepage = "https://github.com/jnidzwetzki/pg-lock-tracer" "Bug Tracker" = "https://github.com/jnidzwetzki/pg-lock-tracer/issues" [project.scripts] pg_lock_tracer = "pg_lock_tracer.pg_lock_tracer:main" pg_lw_lock_tracer = "pg_lock_tracer.pg_lw_lock_tracer:main" pg_row_lock_tracer = "pg_lock_tracer.pg_row_lock_tracer:main" pg_spinlock_delay_tracer = "pg_lock_tracer.pg_spinlock_delay_tracer:main" animate_lock_graph = "pg_lock_tracer.animate_lock_graph:main" [tool.setuptools] package-dir = { "" = "src" } [tool.setuptools.packages.find] where = ["src"] [tool.setuptools.package-data] "pg_lock_tracer.bpf" = ["*.c"] [tool.setuptools.dynamic] version = { attr = "pg_lock_tracer.__version__" } [tool.pytest.ini_options] testpaths = ["tests/"] addopts = [ "--verbose", "--ignore-glob=**/_*.py", ] ================================================ FILE: requirements_dev.txt ================================================ astroid==4.0.3 attrs==26.1.0 black==26.3.1 click==8.3.3 dill==0.4.1 exceptiongroup==1.3.1 graphviz==0.21 igraph==1.0.0 iniconfig==2.3.0 isort==8.0.1 lazy-object-proxy==1.12.0 mccabe==0.7.0 mypy-extensions==1.1.0 packaging==26.2 pathspec==1.1.1 platformdirs==4.9.6 pluggy==1.6.0 prettytable==3.17.0 psycopg2==2.9.12 pylint==4.0.5 pytest==9.0.3 texttable==1.7.0 tomli==2.4.1 typing-extensions==4.15.0 wcwidth==0.7.0 wrapt==2.1.2 ================================================ FILE: src/pg_lock_tracer/__init__.py ================================================ __version__ = "0.7.1" ================================================ FILE: src/pg_lock_tracer/animate_lock_graph.py ================================================ #!/usr/bin/env python3 # # Generate an animated version of the locks # of a query. Input data is the JSON output of # pg_lock_tracker. # # Current limitations: # * Only lock traces of one query are supported. ########################## import os import json import argparse import igraph import graphviz from pg_lock_tracer import __version__ from pg_lock_tracer.helper import PostgreSQLLockHelper # See https://github.com/magjac/d3-graphviz/blob/master/examples/basic-unpkg-worker.html HTML_TEMPLATE = """
Frame
(
of
)
""" EXAMPLES = """ usage examples: # Read events from 'test_trace.json' and generate 'test.html' animate_lock_graph.py -i test_trace.json -o test.html """ parser = argparse.ArgumentParser( description="", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=EXAMPLES, ) parser.add_argument( "-V", "--version", action="version", version=f"{parser.prog} ({__version__})", ) parser.add_argument("-v", "--verbose", action="store_true", help="be verbose") parser.add_argument( "-o", "--output", type=str, dest="output_file", default=None, help="write the trace into output file", required=True, ) parser.add_argument( "-i", "--input", type=str, dest="input_file", default=None, help="the input file with the events", required=True, ) parser.add_argument( "-f", "--force", action="store_true", help="overwrite the output file if it already exists", ) class DOTModel: def __init__(self, input_file, verbose) -> None: self.input_file = input_file self.verbose = verbose self.dot_graphs = [] self.graph = igraph.Graph(directed=True) self.calculate_graphs() def calculate_graphs(self): """ Calculate the dot graphs for each line of the input """ with open(self.input_file, "r", encoding="utf-8") as input_file_handle: for line in input_file_handle: json_data = json.loads(line) self.handle_json(json_data) def handle_json(self, event): """ Handle the JSON encoded graph event """ vertex_name_query = f"query_{event['pid']}" if event["event"] == "QUERY_BEGIN": self.graph.add_vertex(vertex_name_query, type="query", label=event["query"]) self.generate_graph() return if event["event"] == "LOCK_GRANTED_LOCAL": if self.verbose: print(f"Processing {event}") tablename = StringHelper.get_tablename(event) lock_type = event["lock_type"] if not self.graph.vs.select(label_eq=tablename): self.graph.add_vertex(tablename, label=tablename) # Get numeric value of lock lock_numeric_value = PostgreSQLLockHelper.lock_type_to_int(lock_type) # Get existing edge to this table edge_id = self.graph.get_eid(vertex_name_query, tablename, error=False) # If not exist, create if edge_id == -1: lock_value = PostgreSQLLockHelper.encode_locks_into_value( [lock_numeric_value] ) self.graph.add_edge(vertex_name_query, tablename, lock_value=lock_value) self.generate_graph() return # Else, modify edge and add lock edge = self.graph.es[edge_id] decoded_lock_value = PostgreSQLLockHelper.decode_locks_from_value( edge["lock_value"] ) decoded_lock_value.append(lock_numeric_value) edge["lock_value"] = PostgreSQLLockHelper.encode_locks_into_value( decoded_lock_value ) return if event["event"] == "LOCK_UNGRANTED_LOCAL": if self.verbose: print(f"Processing {event}") tablename = StringHelper.get_tablename(event) lock_type = event["lock_type"] lock_numeric_value = PostgreSQLLockHelper.lock_type_to_int(lock_type) edge_id = self.graph.get_eid(vertex_name_query, tablename) edge = self.graph.es[edge_id] decoded_lock_value = PostgreSQLLockHelper.decode_locks_from_value( edge["lock_value"] ) if lock_numeric_value in decoded_lock_value: decoded_lock_value.remove(lock_numeric_value) else: print( f"Lock {lock_numeric_value} for table {tablename} removed but not requested" ) edge["lock_value"] = PostgreSQLLockHelper.encode_locks_into_value( decoded_lock_value ) # Delete the edge if not locks hold if len(decoded_lock_value) == 0: self.graph.delete_edges(edge) # Is the vertex unconnected (degree == 0) after we deleted the edge? vertex = self.graph.vs.select(label_eq=tablename) if self.graph.degree(vertex)[0] == 0: # self.graph.delete_vertices(tablename) self.graph.delete_vertices(vertex) self.generate_graph() return def generate_graph(self): """ Generate a DOT graph based on the igraph """ vertices = self.graph.vs edges = self.graph.es # Reduce space if more nodes are present if len(vertices) > 15: mindist = "0.2" elif len(vertices) > 10: mindist = "0.4" elif len(vertices) > 6: mindist = "0.75" elif len(vertices) > 4: mindist = "1.0" else: mindist = "1.7" dot = graphviz.Digraph("lock-graph", graph_attr={"mindist": mindist}) for vertex in vertices: if vertex["type"] == "query": dot.node( vertex["name"], vertex["label"], fillcolor="gray", style="filled" ) else: display_name = vertex["label"] display_name = StringHelper.split_string(display_name, 20) dot.node( vertex["name"], display_name, shape="box", fillcolor="lightgray", style="filled", ) for edge in edges: # Decode graph lock value into list of lock values decoded_lock_value = PostgreSQLLockHelper.decode_locks_from_value( edge["lock_value"] ) # Convert numeric lock values into a string values and join them into a single string label = ",".join( map(PostgreSQLLockHelper.lock_type_to_str, decoded_lock_value) ) # Add edge to dot graph dot.edge( vertices[edge.source]["name"], vertices[edge.target]["name"], label=label, ) # Convert Dot graph into string. # Every line needs to be quoted and line breaks need a ","" # # For example: # # ['digraph "lock-graph" {', # ' query_171379 [label="select count(*) from sensor_data ;"]', # ' "public.sensor_data" [label="public.sensor_data"]', # ' query_171379 -> "public.sensor_data"', # '}'], lines = dot.source.split("\n") lines = [f"'{line}'" for line in lines if line] lines = ",\n".join(lines) # Add dot string to list of graphs self.dot_graphs.append(f"[{lines}]") def get_html(self): """ Convert all dot graphs into a single string """ result = "var dots = [\n" graphs = ",\n".join(self.dot_graphs) result += graphs result += "];\n" return result # pylint: disable=too-few-public-methods class StringHelper: @staticmethod def split_string(mystring, max_length): """ Add linebreaks to the string after max_length chars at the next dot or underscore. """ result = "" chars_without_break = 0 for element in mystring: result += element chars_without_break += 1 if chars_without_break > max_length and element in (".", "_"): chars_without_break = 0 result += "\\n" return result @staticmethod def get_tablename(event): """ Get the tablename or the Oid of the event. """ if "table" in event: return event["table"] return f"Oid {event['oid']}" def main(): """ Entry point for the animate_lock_graph script """ args = parser.parse_args() if not os.path.exists(args.input_file): raise ValueError(f"Input file does not exist {args.input_file}") if os.path.exists(args.output_file) and not args.force: raise ValueError(f"Output file already exists {args.output_file}") # Create a new dot model, process the events in the input file # and generate the HTML output dot_model_instance = DOTModel(args.input_file, args.verbose) with open(args.output_file, "w", encoding="utf-8") as output: dots = dot_model_instance.get_html() output_html = HTML_TEMPLATE.replace("%%DOT_TEMPLATE%%", dots) output.write(output_html) if __name__ == "__main__": main() ================================================ FILE: src/pg_lock_tracer/bpf/.clang-format ================================================ --- Language: Cpp # BasedOnStyle: Google AccessModifierOffset: -1 AlignAfterOpenBracket: Align AlignArrayOfStructures: None AlignConsecutiveMacros: None AlignConsecutiveAssignments: None AlignConsecutiveBitFields: None AlignConsecutiveDeclarations: None AlignEscapedNewlines: Left AlignOperands: Align AlignTrailingComments: true AllowAllArgumentsOnNextLine: true AllowAllParametersOfDeclarationOnNextLine: true AllowShortEnumsOnASingleLine: true AllowShortBlocksOnASingleLine: Never AllowShortCaseLabelsOnASingleLine: false AllowShortFunctionsOnASingleLine: All AllowShortLambdasOnASingleLine: All AllowShortIfStatementsOnASingleLine: WithoutElse AllowShortLoopsOnASingleLine: true AlwaysBreakAfterDefinitionReturnType: None AlwaysBreakAfterReturnType: None AlwaysBreakBeforeMultilineStrings: true AlwaysBreakTemplateDeclarations: Yes AttributeMacros: - __capability BinPackArguments: true BinPackParameters: true BraceWrapping: AfterCaseLabel: false AfterClass: false AfterControlStatement: Never AfterEnum: false AfterFunction: false AfterNamespace: false AfterObjCDeclaration: false AfterStruct: false AfterUnion: false AfterExternBlock: false BeforeCatch: false BeforeElse: false BeforeLambdaBody: false BeforeWhile: false IndentBraces: false SplitEmptyFunction: true SplitEmptyRecord: true SplitEmptyNamespace: true BreakBeforeBinaryOperators: None BreakBeforeConceptDeclarations: true BreakBeforeBraces: Attach BreakBeforeInheritanceComma: false BreakInheritanceList: BeforeColon BreakBeforeTernaryOperators: true BreakConstructorInitializersBeforeComma: false BreakConstructorInitializers: BeforeColon BreakAfterJavaFieldAnnotations: false BreakStringLiterals: true ColumnLimit: 80 CommentPragmas: '^ IWYU pragma:' QualifierAlignment: Leave CompactNamespaces: false ConstructorInitializerIndentWidth: 4 ContinuationIndentWidth: 4 Cpp11BracedListStyle: true DeriveLineEnding: true DerivePointerAlignment: true DisableFormat: false EmptyLineAfterAccessModifier: Never EmptyLineBeforeAccessModifier: LogicalBlock ExperimentalAutoDetectBinPacking: false PackConstructorInitializers: NextLine BasedOnStyle: '' ConstructorInitializerAllOnOneLineOrOnePerLine: false AllowAllConstructorInitializersOnNextLine: true FixNamespaceComments: true ForEachMacros: - foreach - Q_FOREACH - BOOST_FOREACH IfMacros: - KJ_IF_MAYBE IncludeBlocks: Regroup IncludeCategories: - Regex: '^' Priority: 2 SortPriority: 0 CaseSensitive: false - Regex: '^<.*\.h>' Priority: 1 SortPriority: 0 CaseSensitive: false - Regex: '^<.*' Priority: 2 SortPriority: 0 CaseSensitive: false - Regex: '.*' Priority: 3 SortPriority: 0 CaseSensitive: false IncludeIsMainRegex: '([-_](test|unittest))?$' IncludeIsMainSourceRegex: '' IndentAccessModifiers: false IndentCaseLabels: true IndentCaseBlocks: false IndentGotoLabels: true IndentPPDirectives: None IndentExternBlock: AfterExternBlock IndentRequires: false IndentWidth: 2 IndentWrappedFunctionNames: false InsertTrailingCommas: None JavaScriptQuotes: Leave JavaScriptWrapImports: true KeepEmptyLinesAtTheStartOfBlocks: false LambdaBodyIndentation: Signature MacroBlockBegin: '' MacroBlockEnd: '' MaxEmptyLinesToKeep: 1 NamespaceIndentation: None ObjCBinPackProtocolList: Never ObjCBlockIndentWidth: 2 ObjCBreakBeforeNestedBlockParam: true ObjCSpaceAfterProperty: false ObjCSpaceBeforeProtocolList: true PenaltyBreakAssignment: 2 PenaltyBreakBeforeFirstCallParameter: 1 PenaltyBreakComment: 300 PenaltyBreakFirstLessLess: 120 PenaltyBreakOpenParenthesis: 0 PenaltyBreakString: 1000 PenaltyBreakTemplateDeclaration: 10 PenaltyExcessCharacter: 1000000 PenaltyReturnTypeOnItsOwnLine: 200 PenaltyIndentedWhitespace: 0 PointerAlignment: Left PPIndentWidth: -1 RawStringFormats: - Language: Cpp Delimiters: - cc - CC - cpp - Cpp - CPP - 'c++' - 'C++' CanonicalDelimiter: '' BasedOnStyle: google - Language: TextProto Delimiters: - pb - PB - proto - PROTO EnclosingFunctions: - EqualsProto - EquivToProto - PARSE_PARTIAL_TEXT_PROTO - PARSE_TEST_PROTO - PARSE_TEXT_PROTO - ParseTextOrDie - ParseTextProtoOrDie - ParseTestProto - ParsePartialTestProto CanonicalDelimiter: pb BasedOnStyle: google ReferenceAlignment: Pointer ReflowComments: true RemoveBracesLLVM: false SeparateDefinitionBlocks: Leave ShortNamespaceLines: 1 SortIncludes: CaseSensitive SortJavaStaticImport: Before SortUsingDeclarations: true SpaceAfterCStyleCast: false SpaceAfterLogicalNot: false SpaceAfterTemplateKeyword: true SpaceBeforeAssignmentOperators: true SpaceBeforeCaseColon: false SpaceBeforeCpp11BracedList: false SpaceBeforeCtorInitializerColon: true SpaceBeforeInheritanceColon: true SpaceBeforeParens: ControlStatements SpaceBeforeParensOptions: AfterControlStatements: true AfterForeachMacros: true AfterFunctionDefinitionName: false AfterFunctionDeclarationName: false AfterIfMacros: true AfterOverloadedOperator: false BeforeNonEmptyParentheses: false SpaceAroundPointerQualifiers: Default SpaceBeforeRangeBasedForLoopColon: true SpaceInEmptyBlock: false SpaceInEmptyParentheses: false SpacesBeforeTrailingComments: 2 SpacesInAngles: Never SpacesInConditionalStatement: false SpacesInContainerLiterals: true SpacesInCStyleCastParentheses: false SpacesInLineCommentPrefix: Minimum: 1 Maximum: -1 SpacesInParentheses: false SpacesInSquareBrackets: false SpaceBeforeSquareBrackets: false BitFieldColonSpacing: Both Standard: Auto StatementAttributeLikeMacros: - Q_EMIT StatementMacros: - Q_UNUSED - QT_REQUIRE_VERSION TabWidth: 8 UseCRLF: false UseTab: Never WhitespaceSensitiveMacros: - STRINGIZE - PP_STRINGIZE - BOOST_PP_STRINGIZE - NS_SWIFT_NAME - CF_SWIFT_NAME ... ================================================ FILE: src/pg_lock_tracer/bpf/__init__.py ================================================ ================================================ FILE: src/pg_lock_tracer/bpf/pg_lock_tracer.c ================================================ #include /* * Placeholder for EVENT_* and ERROR_* defines * * #define EVENT_.... n * * Will by automatically generated from python Events ENUM */ __DEFINES__ typedef struct PostgreSQLEvent { u32 pid; u64 timestamp; u32 event_type; u32 object; // typedef unsigned int Oid; int mode; // typedef int LOCKMODE; u32 requested; // Requested locks s64 lock_local_hold; // Requested local locks char payload_str1[127]; // Generic payload string data 1 (e.g., a query / a // schema) char payload_str2[127]; // Generic payload string data 2 (e.g., a table) int stackid; // The id of the stack } PostgreSQLEvent; BPF_PERF_OUTPUT(lockevents); #if defined(STACKTRACE_DEADLOCK) || defined(STACKTRACE_LOCK) || \ defined(STACKTRACE_UNLOCK) BPF_STACK_TRACE(stacks, 4096); #endif static void fill_basic_data(PostgreSQLEvent *event) { event->pid = bpf_get_current_pid_tgid(); event->timestamp = bpf_ktime_get_ns(); } /* * ==================================== * Table handling * ==================================== */ static void handle_table_event(PostgreSQLEvent *event, struct pt_regs *ctx) { fill_basic_data(event); bpf_probe_read_kernel(&(event->mode), sizeof(event->mode), &(PT_REGS_PARM2(ctx))); // bpf_trace_printk("Event: %d %d\\n", event.object, event.mode); lockevents.perf_submit(ctx, event, sizeof(PostgreSQLEvent)); } /* * PostgreSQL: table_open * Parameter 1: Oid relationId * Parameter 2: LOCKMODE lockmode */ int bpf_table_open(struct pt_regs *ctx) { PostgreSQLEvent event = {.event_type = EVENT_TABLE_OPEN}; bpf_probe_read_kernel(&(event.object), sizeof(event.object), &(PT_REGS_PARM1(ctx))); handle_table_event(&event, ctx); return 0; } /* * ptype /o RangeVar * type = struct RangeVar { * 0 | 4 NodeTag type; * XXX 4-byte hole * 8 | 8 char *catalogname; * 16 | 8 char *schemaname; * 24 | 8 char *relname; * [...] */ typedef struct RangeVar { u8 enumvalue; char *catalogname; char *schemaname; char *relname; } RangeVar; /* * PostgreSQL: table_openrv * Parameter 1: const RangeVar *relation * Parameter 2: LOCKMODE lockmode */ int bpf_table_openrv(struct pt_regs *ctx, RangeVar *relation) { PostgreSQLEvent event = {.event_type = EVENT_TABLE_OPEN_RV}; bpf_probe_read_user_str(event.payload_str1, sizeof(event.payload_str1), (void *)relation->schemaname); bpf_probe_read_user_str(event.payload_str2, sizeof(event.payload_str2), (void *)relation->relname); handle_table_event(&event, ctx); return 0; } /* * PostgreSQL: table_openrv_extended * Parameter 1: const RangeVar *relation * Parameter 2: LOCKMODE lockmode * Parameter 3: bool missing_ok */ int bpf_table_openrv_extended(struct pt_regs *ctx, RangeVar *relation) { PostgreSQLEvent event = {.event_type = EVENT_TABLE_OPEN_RV_EXTENDED}; bpf_probe_read_user_str(event.payload_str1, sizeof(event.payload_str1), (void *)relation->schemaname); bpf_probe_read_user_str(event.payload_str2, sizeof(event.payload_str2), (void *)relation->relname); handle_table_event(&event, ctx); return 0; } int bpf_table_close(struct pt_regs *ctx) { PostgreSQLEvent event = {.event_type = EVENT_TABLE_CLOSE}; // Param 1 is a Relation struct // The Oid rd_id is stored at byte 72 (PG 15.1) // gdb: ptype /o Relation // // type = struct RelationData { // /* 0 | 12 */ RelFileNode rd_node; // /* XXX 4-byte hole */ // /* 16 | 8 */ SMgrRelation rd_smgr; // /* 24 | 4 */ int rd_refcnt; // /* 28 | 4 */ BackendId rd_backend; // /* 32 | 1 */ _Bool rd_islocaltemp; // /* 33 | 1 */ _Bool rd_isnailed; // /* 34 | 1 */ _Bool rd_isvalid; // /* 35 | 1 */ _Bool rd_indexvalid; // /* 36 | 1 */ _Bool rd_statvalid; // /* XXX 3-byte hole */ // /* 40 | 4 */ SubTransactionId rd_createSubid; // /* 44 | 4 */ SubTransactionId rd_newRelfilenodeSubid; // /* 48 | 4 */ SubTransactionId rd_firstRelfilenodeSubid; // /* 52 | 4 */ SubTransactionId rd_droppedSubid; // /* 56 | 8 */ Form_pg_class rd_rel; // /* 64 | 8 */ TupleDesc rd_att; // /* 72 | 4 */ Oid rd_id; // [....] char buffer[76]; bpf_probe_read_user(buffer, sizeof(buffer), (void *)PT_REGS_PARM1(ctx)); bpf_probe_read_kernel(&(event.object), sizeof(event.object), &(buffer[72])); // Oid oid; // bpf_probe_read_kernel(&oid, sizeof(oid), &(buffer[72])); // bpf_probe_read_kernel(&(event.object), sizeof(event.object), // &(PT_REGS_PARM1(ctx))); bpf_trace_printk("Oid: %d \\n", oid); handle_table_event(&event, ctx); return 0; } static void fill_basic_data_and_submit(PostgreSQLEvent *event, struct pt_regs *ctx) { fill_basic_data(event); lockevents.perf_submit(ctx, event, sizeof(PostgreSQLEvent)); } /* * ==================================== * Query handling * ==================================== */ int bpf_query_begin(struct pt_regs *ctx) { PostgreSQLEvent event = {.event_type = EVENT_QUERY_BEGIN}; bpf_probe_read_user_str(event.payload_str1, sizeof(event.payload_str1), (void *)PT_REGS_PARM1(ctx)); fill_basic_data_and_submit(&event, ctx); return 0; } /* * Query return probe */ int bpf_query_end(struct pt_regs *ctx) { PostgreSQLEvent event = {.event_type = EVENT_QUERY_END}; fill_basic_data_and_submit(&event, ctx); return 0; } /* * ==================================== * Error handling * ==================================== */ int bpf_errstart(struct pt_regs *ctx) { PostgreSQLEvent event = {.event_type = EVENT_ERROR}; fill_basic_data(&event); bpf_probe_read_kernel(&event.mode, sizeof(event.mode), (void *)&(PT_REGS_PARM1(ctx))); if (event.mode >= PGERROR_ERROR) { lockevents.perf_submit(ctx, &event, sizeof(PostgreSQLEvent)); } return 0; } /* * ==================================== * Lock handling * ==================================== */ /* * PSQL: LockRelationOid * Parameter 1: Oid * Parameter 2: LOCKMODE */ int bpf_lock_relation_oid(struct pt_regs *ctx) { PostgreSQLEvent event = {.event_type = EVENT_LOCK_RELATION_OID}; bpf_probe_read_kernel(&(event.object), sizeof(event.object), &(PT_REGS_PARM1(ctx))); #ifdef STACKTRACE_LOCK event.stackid = stacks.get_stackid(ctx, BPF_F_USER_STACK); #endif handle_table_event(&event, ctx); return 0; } /* * PSQL: LockRelationOid - Return probe */ int bpf_lock_relation_oid_end(struct pt_regs *ctx) { PostgreSQLEvent event = {.event_type = EVENT_LOCK_RELATION_OID_END}; fill_basic_data_and_submit(&event, ctx); return 0; } /* * PSQL: UnlockRelationOid * Parameter 1: Oid * Parameter 2: LOCKMODE */ int bpf_unlock_relation_oid(struct pt_regs *ctx) { PostgreSQLEvent event = {.event_type = EVENT_UNLOCK_RELATION_OID}; bpf_probe_read_kernel(&(event.object), sizeof(event.object), &(PT_REGS_PARM1(ctx))); handle_table_event(&event, ctx); return 0; } /* * Parse the LOCK Structure * PG 14.2 * (gdb) ptype /o lock * type = struct LOCK { * 0 | 16 * LOCKTAG tag; * 16 | 4 * LOCKMASK grantMask; * 20 | 4 * LOCKMASK waitMask; * 24 | 16 * SHM_QUEUE procLocks; * 40 | 24 * PROC_QUEUE waitProcs; * 64 | 40 * int requested[10]; * 104 | 4 * int nRequested; * 108 | 40 * int granted[10]; * 148 | 4 * int nGranted; * * (gdb) ptype /o LOCKTAG * type = struct LOCKTAG { * 0 | 4 * uint32 locktag_field1; // Database OID (see SET_LOCKTAG_RELATION) * 4 | 4 * uint32 locktag_field2; // Relation OID (see SET_LOCKTAG_RELATION) * 8 | 4 * uint32 locktag_field3; * 12 | 2 * uint16 locktag_field4; * 14 | 1 * uint8 locktag_type; * 15 | 1 * uint8 locktag_lockmethodid; */ static void fill_lock_object(PostgreSQLEvent *event, void *param) { char buffer[108]; bpf_probe_read_user(buffer, sizeof(buffer), param); bpf_probe_read_kernel(&(event->object), sizeof(event->object), &(buffer[4])); bpf_probe_read_kernel(&(event->requested), sizeof(event->requested), &(buffer[104])); } /* * PSQL: GrantLock * Parameter 1 LOCK * Parameter 2 PROCLOCK * Parameter 3 LOCKMODE */ int bpf_lock_grant(struct pt_regs *ctx) { PostgreSQLEvent event = {.event_type = EVENT_LOCK_GRANTED}; bpf_probe_read_kernel(&(event.mode), sizeof(event.mode), &(PT_REGS_PARM3(ctx))); fill_lock_object(&event, (void *)PT_REGS_PARM1(ctx)); if (event.object != 0) fill_basic_data_and_submit(&event, ctx); return 0; } /* * PSQL: FastPathGrantRelationLock * Parameter 1 OID * Parameter 2 LOCKMODE */ int bpf_lock_fastpath_grant(struct pt_regs *ctx) { PostgreSQLEvent event = {.event_type = EVENT_LOCK_GRANTED_FASTPATH}; bpf_probe_read_kernel(&(event.object), sizeof(event.object), &(PT_REGS_PARM1(ctx))); bpf_probe_read_kernel(&(event.mode), sizeof(event.mode), &(PT_REGS_PARM2(ctx))); fill_basic_data_and_submit(&event, ctx); return 0; } /* * Local lock grant * Parameter 1: LOCALLOCK *locallock * Parameter 2: ResourceOwner owner * * PG 14 * * ptype /o locallock * type = struct LOCALLOCK { * 0 | 20 * LOCALLOCKTAG tag; * 20 | 4 * uint32 hashcode; * 24 | 8 * LOCK *lock; * 32 | 8 * PROCLOCK *proclock; * 40 | 8 * int64 nLocks; * 48 | 4 * int numLockOwners; * 52 | 4 * int maxLockOwners; * 56 | 8 * LOCALLOCKOWNER *lockOwners; * 64 | 1 * _Bool holdsStrongLockCount; * 65 | 1 * _Bool lockCleared; * XXX 6-byte padding * * type = struct LOCALLOCKTAG { * 0 | 16 * LOCKTAG lock; -- See LOCKTAG above * 16 | 4 * LOCKMODE mode; * */ static void fill_locallock_object(PostgreSQLEvent *event, void *param) { char buffer[48]; bpf_probe_read_user(buffer, sizeof(buffer), param); bpf_probe_read_kernel(&(event->object), sizeof(event->object), &(buffer[4])); bpf_probe_read_kernel(&(event->lock_local_hold), sizeof(event->lock_local_hold), &(buffer[40])); bpf_probe_read_kernel(&(event->mode), sizeof(event->mode), &(buffer[16])); } /* * PSQL: GrantLockLocal * Parameter 1 LOCALLOCK * Parameter 2 ResourceOwner */ int bpf_lock_local_grant(struct pt_regs *ctx) { PostgreSQLEvent event = {.event_type = EVENT_LOCK_GRANTED_LOCAL}; fill_locallock_object(&event, (void *)PT_REGS_PARM1(ctx)); if (event.object != 0) fill_basic_data_and_submit(&event, ctx); return 0; } /* * PSQL: UnGrantLock * Parameter 1 LOCK * Parameter 2 LOCKMODE */ int bpf_lock_ungrant(struct pt_regs *ctx) { PostgreSQLEvent event = {.event_type = EVENT_LOCK_UNGRANTED}; bpf_probe_read_kernel(&(event.mode), sizeof(event.mode), &(PT_REGS_PARM2(ctx))); fill_lock_object(&event, (void *)PT_REGS_PARM1(ctx)); #ifdef STACKTRACE_UNLOCK event.stackid = stacks.get_stackid(ctx, BPF_F_USER_STACK); #endif if (event.object != 0) fill_basic_data_and_submit(&event, ctx); return 0; } /* * PSQL: FastPathUnGrantRelationLock * Parameter 1 Oid * Parameter 2 LOCKMODE */ int bpf_lock_fastpath_ungrant(struct pt_regs *ctx) { PostgreSQLEvent event = {.event_type = EVENT_LOCK_UNGRANTED_FASTPATH}; bpf_probe_read_kernel(&(event.object), sizeof(event.object), &(PT_REGS_PARM1(ctx))); bpf_probe_read_kernel(&(event.mode), sizeof(event.mode), &(PT_REGS_PARM2(ctx))); #ifdef STACKTRACE_UNLOCK event.stackid = stacks.get_stackid(ctx, BPF_F_USER_STACK); #endif fill_basic_data_and_submit(&event, ctx); return 0; } /* * PSQL: RemoveLocalLock * Parameter 1: LOCALLOCK */ int bfp_local_lock_ungrant(struct pt_regs *ctx) { PostgreSQLEvent event = {.event_type = EVENT_LOCK_UNGRANTED_LOCAL}; fill_locallock_object(&event, (void *)PT_REGS_PARM1(ctx)); if (event.object != 0) fill_basic_data_and_submit(&event, ctx); return 0; } /* * Deadlock detected * PSQL: DeadLockReport */ int bpf_deadlock(struct pt_regs *ctx) { PostgreSQLEvent event = {.event_type = EVENT_DEADLOCK}; #ifdef STACKTRACE_DEADLOCK event.stackid = stacks.get_stackid(ctx, BPF_F_USER_STACK); #endif fill_basic_data_and_submit(&event, ctx); return 0; } /* * ==================================== * Transaction handling * ==================================== */ /* * PSQL: StartTransaction */ int bpf_transaction_begin(struct pt_regs *ctx) { PostgreSQLEvent event = {.event_type = EVENT_TRANSACTION_BEGIN}; fill_basic_data_and_submit(&event, ctx); return 0; } /* * PSQL: CommitTransaction */ int bpf_transaction_commit(struct pt_regs *ctx) { PostgreSQLEvent event = {.event_type = EVENT_TRANSACTION_COMMIT}; fill_basic_data_and_submit(&event, ctx); return 0; } /* * PSQL: AbortTransaction */ int bpf_transaction_abort(struct pt_regs *ctx) { PostgreSQLEvent event = {.event_type = EVENT_TRANSACTION_ABORT}; fill_basic_data_and_submit(&event, ctx); return 0; } /* * ==================================== * Process Invalidation Messages * ==================================== */ /* * PSQL: AcceptInvalidationMessages */ int bpf_accept_invalidation_messages(struct pt_regs *ctx) { PostgreSQLEvent event = {.event_type = EVENT_INVALIDATION_MESSAGES_ACCEPT}; fill_basic_data_and_submit(&event, ctx); return 0; } ================================================ FILE: src/pg_lock_tracer/bpf/pg_lw_lock_tracer.c ================================================ // https://github.com/postgres/postgres/blob/a4adc31f6902f6fc29d74868e8969412fc590da9/src/include/storage/lwlock.h#L110 typedef enum LWLockMode { LW_EXCLUSIVE, LW_SHARED, LW_WAIT_UNTIL_FREE /* A special mode used in PGPROC->lwWaitMode, * when waiting for lock to become free. Not * to be used as LWLockAcquire argument */ } LWLockMode; /* Placeholder for auto generated defines */ __DEFINES__ typedef struct LockEvent_t { u32 pid; u64 timestamp; u32 event_type; /* LWLock tranche name (see T_NAME / GetLWTrancheName() * in lwlock.c) */ char tranche[255]; /* LWLockMode */ u32 mode; } LockEvent; BPF_PERF_OUTPUT(lockevents); static void fill_and_submit(struct pt_regs *ctx, LockEvent *event, uint64_t tranche_addr) { bpf_probe_read_user_str(event->tranche, sizeof(event->tranche), (void *)tranche_addr); event->pid = bpf_get_current_pid_tgid(); event->timestamp = bpf_ktime_get_ns(); // sudo cat /sys/kernel/debug/tracing/trace_pipe // bpf_trace_printk("LW lock event for trance: %s\\n", tranche); lockevents.perf_submit(ctx, event, sizeof(LockEvent)); } /* * Acquire a LW Lock * Arguments: TRACE_POSTGRESQL_LWLOCK_ACQUIRE(T_NAME(lock), mode) */ int lwlock_acquire(struct pt_regs *ctx) { uint64_t tranche_addr = 0; LWLockMode mode; LockEvent event = {.event_type = EVENT_LOCK}; // The usdt does not support using bpf_usdt_readarg outside the main probe // function See: https://github.com/iovisor/bcc/issues/2265 bpf_usdt_readarg(1, ctx, &tranche_addr); bpf_usdt_readarg(2, ctx, &mode); event.mode = mode; fill_and_submit(ctx, &event, tranche_addr); return 0; } /* * Release a LW Lock * Arguments: TRACE_POSTGRESQL_LWLOCK_RELEASE(T_NAME(lock)) */ int lwlock_release(struct pt_regs *ctx) { uint64_t tranche_addr = 0; LockEvent event = {.event_type = EVENT_UNLOCK}; bpf_usdt_readarg(1, ctx, &tranche_addr); fill_and_submit(ctx, &event, tranche_addr); return 0; } /* * Wait for a LW Lock * Arguments: TRACE_POSTGRESQL_LWLOCK_WAIT_START(T_NAME(lock), mode) */ int lwlock_wait_start(struct pt_regs *ctx) { uint64_t tranche_addr = 0; LWLockMode mode; LockEvent event = {.event_type = EVENT_WAIT_START}; // The usdt does not support using bpf_usdt_readarg outside the main probe // function. See: https://github.com/iovisor/bcc/issues/2265 bpf_usdt_readarg(1, ctx, &tranche_addr); bpf_usdt_readarg(2, ctx, &mode); event.mode = mode; fill_and_submit(ctx, &event, tranche_addr); return 0; } /* * Wait for a LW Lock is done * Arguments: TRACE_POSTGRESQL_LWLOCK_WAIT_DONE(T_NAME(lock), mode) */ int lwlock_wait_done(struct pt_regs *ctx) { uint64_t tranche_addr = 0; LWLockMode mode; LockEvent event = {.event_type = EVENT_WAIT_DONE}; bpf_usdt_readarg(1, ctx, &tranche_addr); bpf_usdt_readarg(2, ctx, &mode); event.mode = mode; fill_and_submit(ctx, &event, tranche_addr); return 0; } /* * LWLock was successfully acquired when the caller specified no waiting * Arguments: TRACE_POSTGRESQL_LWLOCK_CONDACQUIRE(T_NAME(lock), mode) */ int lwlock_condacquire(struct pt_regs *ctx) { uint64_t tranche_addr = 0; LWLockMode mode; LockEvent event = {.event_type = EVENT_COND_ACQUIRE}; bpf_usdt_readarg(1, ctx, &tranche_addr); bpf_usdt_readarg(2, ctx, &mode); event.mode = mode; fill_and_submit(ctx, &event, tranche_addr); return 0; } /* * LWLock was successfully acquired when the caller specified no waiting * Arguments: TRACE_POSTGRESQL_LWLOCK_CONDACQUIRE_FAIL(T_NAME(lock), mode) */ int lwlock_condacquire_fail(struct pt_regs *ctx) { uint64_t tranche_addr = 0; LWLockMode mode; LockEvent event = {.event_type = EVENT_COND_ACQUIRE_FAIL}; bpf_usdt_readarg(1, ctx, &tranche_addr); bpf_usdt_readarg(2, ctx, &mode); event.mode = mode; fill_and_submit(ctx, &event, tranche_addr); return 0; } /* * Acquire a LWLock or wait if already locked see LWLockAcquireOrWait * Arguments: TRACE_POSTGRESQL_LWLOCK_ACQUIRE_OR_WAIT(T_NAME(lock), mode); */ int lwlock_acquire_or_wait(struct pt_regs *ctx) { uint64_t tranche_addr = 0; LWLockMode mode; LockEvent event = {.event_type = EVENT_LOCK_OR_WAIT}; bpf_usdt_readarg(1, ctx, &tranche_addr); bpf_usdt_readarg(2, ctx, &mode); event.mode = mode; fill_and_submit(ctx, &event, tranche_addr); return 0; } /* * Acquire a LWLock or wait if already locked (failed) see LWLockAcquireOrWait * Arguments: TRACE_POSTGRESQL_LWLOCK_ACQUIRE_OR_WAIT_FAIL(T_NAME(lock), mode); */ int lwlock_acquire_or_wait_fail(struct pt_regs *ctx) { uint64_t tranche_addr = 0; LWLockMode mode; LockEvent event = {.event_type = EVENT_LOCK_OR_WAIT_FAIL}; bpf_usdt_readarg(1, ctx, &tranche_addr); bpf_usdt_readarg(2, ctx, &mode); event.mode = mode; fill_and_submit(ctx, &event, tranche_addr); return 0; } ================================================ FILE: src/pg_lock_tracer/bpf/pg_row_lock_tracer.c ================================================ #include /* Placeholder for auto generated defines */ __DEFINES__ typedef struct RowLockEvent_t { u32 pid; u64 timestamp; u32 event_type; /* See RelFileNode - Oid is u32 */ u32 tablespace; u32 database; u32 relation; /* LockTupleMode */ u8 locktuplemode; /* LockWaitPolicy */ u8 lockwaitpolicy; /* Locked tuple */ u32 blockid; u16 offset; /* TM_Result */ int lockresult; } RowLockEvent; BPF_PERF_OUTPUT(lockevents); static void fill_and_submit(struct pt_regs *ctx, RowLockEvent *event) { event->pid = bpf_get_current_pid_tgid(); event->timestamp = bpf_ktime_get_ns(); // sudo cat /sys/kernel/debug/tracing/trace_pipe // bpf_trace_printk("LW lock event for trance: %s\\n", tranche); lockevents.perf_submit(ctx, event, sizeof(RowLockEvent)); } /* * Acquire a tuple lock * * Arguments: * 1. Relation relation (1st member RelFileNode) * 2. ItemPointer tid * 3. Snapshot snapshot, * 4. TupleTableSlot *slot, * 5. CommandId cid, * 6. LockTupleMode mode, * 7. LockWaitPolicy wait_policy, * 8. uint8 flags, * 9. TM_FailureData *tmfd * */ int heapam_tuple_lock(struct pt_regs *ctx) { RowLockEvent event = {.event_type = EVENT_LOCK_TUPLE}; /* * (gdb) ptype /o RelFileNode * 0 | 4 Oid spcNode; * 4 | 4 Oid dbNode; * 8 | 4 Oid relNode; */ char buffer_relation[12]; bpf_probe_read_user(buffer_relation, sizeof(buffer_relation), (void *)PT_REGS_PARM1(ctx)); bpf_probe_read_kernel(&(event.tablespace), sizeof(event.tablespace), &(buffer_relation[0])); bpf_probe_read_kernel(&(event.database), sizeof(event.database), &(buffer_relation[4])); bpf_probe_read_kernel(&(event.relation), sizeof(event.relation), &(buffer_relation[8])); /* Locked tuple */ char buffer_item_pointer[6]; u16 bi_hi; u16 bi_lo; bpf_probe_read_user(buffer_item_pointer, sizeof(buffer_item_pointer), (void *)PT_REGS_PARM2(ctx)); bpf_probe_read_kernel(&(bi_hi), sizeof(bi_hi), &(buffer_item_pointer[0])); bpf_probe_read_kernel(&(bi_lo), sizeof(bi_lo), &(buffer_item_pointer[2])); bpf_probe_read_kernel(&(event.offset), sizeof(event.offset), &(buffer_item_pointer[4])); /* See #define BlockIdGetBlockNumber(blockId) */ event.blockid = (bi_hi) << 16 | bi_lo; /* Locking options */ bpf_probe_read_kernel(&(event.locktuplemode), sizeof(event.locktuplemode), &(PT_REGS_PARM6(ctx))); /* Only the first six function parameters are passed via register. All * remaining parameters are stored on the stack. * * See: System V Application Binary Interface—AMD64 Architecture Processor * Supplement. */ void *ptr = 0; bpf_probe_read(&ptr, sizeof(ptr), (void *)(PT_REGS_SP(ctx) + (1 * 8))); bpf_probe_read_kernel(&(event.lockwaitpolicy), sizeof(event.lockwaitpolicy), &ptr); fill_and_submit(ctx, &event); return 0; } /* * Acquire a tuple lock - Function done */ int heapam_tuple_lock_end(struct pt_regs *ctx) { RowLockEvent event = {.event_type = EVENT_LOCK_TUPLE_END}; event.lockresult = PT_REGS_RC(ctx); fill_and_submit(ctx, &event); return 0; } ================================================ FILE: src/pg_lock_tracer/bpf/pg_spinlock_delay_tracer.c ================================================ #include #define MAX_STR_LEN 128 typedef struct SpinDelayStatus_t { int spins; int delays; int cur_delay; const char *file; int line; const char *func; } SpinDelayStatus; typedef struct SpinDelayEvent_t { u32 pid; u64 timestamp; int spins; int delays; int cur_delay; int line; char file[MAX_STR_LEN]; char func[MAX_STR_LEN]; } SpinDelayEvent; BPF_PERF_OUTPUT(lockevents); int spin_delay(struct pt_regs *ctx) { SpinDelayEvent event = {}; SpinDelayStatus status = {}; SpinDelayStatus *status_ptr = (SpinDelayStatus *)PT_REGS_PARM1(ctx); event.pid = bpf_get_current_pid_tgid(); event.timestamp = bpf_ktime_get_ns(); if (!status_ptr) { lockevents.perf_submit(ctx, &event, sizeof(SpinDelayEvent)); return 0; } bpf_probe_read_user(&status, sizeof(status), status_ptr); event.spins = status.spins; event.delays = status.delays; event.cur_delay = status.cur_delay; event.line = status.line; if (status.file) { bpf_probe_read_user_str(&event.file, sizeof(event.file), status.file); } if (status.func) { bpf_probe_read_user_str(&event.func, sizeof(event.func), status.func); } lockevents.perf_submit(ctx, &event, sizeof(SpinDelayEvent)); return 0; } ================================================ FILE: src/pg_lock_tracer/helper.py ================================================ """ Helper classes """ import os from pathlib import Path from bcc import BPF class PostgreSQLLockHelper: """ # Defines taken from: src/include/storage/lockdefs.h # # NoLock is not a lock mode, but a flag value meaning "don't get a lock" # define NoLock 0 # # define AccessShareLock 1 /* SELECT */ # define RowShareLock 2 /* SELECT FOR UPDATE/FOR SHARE */ # define RowExclusiveLock 3 /* INSERT, UPDATE, DELETE */ # define ShareUpdateExclusiveLock 4 /* VACUUM (non-FULL), ANALYZE, CREATE # * INDEX CONCURRENTLY */ # define ShareLock 5 /* CREATE INDEX (WITHOUT CONCURRENTLY) */ # define ShareRowExclusiveLock 6 /* like EXCLUSIVE MODE, but allows ROW # * SHARE */ # define ExclusiveLock 7 /* blocks ROW SHARE/SELECT...FOR UPDATE */ # define AccessExclusiveLock 8 /* ALTER TABLE, DROP TABLE, VACUUM FULL, # * and unqualified LOCK TABLE */ """ locks = {} locks["NoLock"] = 0 locks["AccessShareLock"] = 1 locks["RowShareLock"] = 2 locks["RowExclusiveLock"] = 3 locks["ShareUpdateExclusiveLock"] = 4 locks["ShareLock"] = 5 locks["ShareRowExclusiveLock"] = 6 locks["ExclusiveLock"] = 7 locks["AccessExclusiveLock"] = 8 @staticmethod def encode_locks_into_value(locks): """ Encode a list of lock types into a single value """ result = 0 for lock in locks: lock_value = 1 << int(lock) result |= lock_value return result @staticmethod def decode_locks_from_value(encoded_value): """ Decode a value of locks into a list of locks """ result = [] for lock in range(0, 9): lock_value = 1 << int(lock) if encoded_value & lock_value == lock_value: result.append(lock) return result @staticmethod def lock_type_to_str(lock_type): """ Return the name of a lock based on the numeric value """ for lock_name, lock_numeric_value in PostgreSQLLockHelper.locks.items(): if lock_numeric_value == lock_type: return lock_name raise ValueError(f"Unsupported lock type {lock_type}") @staticmethod def lock_type_to_int(lock_name): """ Return the numeric value of a lock based on the name """ if lock_name not in PostgreSQLLockHelper.locks: raise ValueError(f"Unknown lock type {lock_name}") return PostgreSQLLockHelper.locks[lock_name] class BPFHelper: # The size of the kernel ring buffer page_cnt = 2048 @staticmethod def enum_to_defines(enum_instance, prefix): """ Convert a IntEnum into C '#define' statements """ result = "" for instance in enum_instance: result += f"#define {prefix}_{instance.name} {instance.value}\n" return result @staticmethod def read_bpf_program(program_name): """ Load the BPF program from a file and return it as a string. """ program_file = Path(__file__).parent / "bpf" / program_name if not os.path.exists(program_file): raise ValueError(f"BPF program file not found {program_file}") with program_file.open("r") as bpf_program: return bpf_program.read() @staticmethod def check_pid_exe(pids, executable): """ Do the given PIDs belong to the executable """ if not pids: return for pid in pids: if not os.path.isdir(f"/proc/{pid}"): raise ValueError( f"/proc entry for pid {pid} not found, does the process exist?" ) binary = os.readlink(f"/proc/{pid}/exe") if binary != executable: raise ValueError( f"Pid {pid} does not belong to binary {executable}. Executable is {binary}" ) @staticmethod def register_ebpf_probe( path, bpf_instance, function_regex, bpf_fn_name, verbose, probe_on_enter=True ): """ Register a BPF probe """ addresses = set() func_and_addr = BPF.get_user_functions_and_addresses(path, function_regex) if not func_and_addr: raise ValueError(f"Unable to locate function {function_regex}") # Handle address duplicates for function, address in func_and_addr: if address in addresses: continue addresses.add(address) if probe_on_enter: bpf_instance.attach_uprobe(name=path, sym=function, fn_name=bpf_fn_name) if verbose: print(f"Attaching to {function} at address {address} on enter") else: bpf_instance.attach_uretprobe( name=path, sym=function, fn_name=bpf_fn_name ) if verbose: print(f"Attaching to {function} at address {address} on return") ================================================ FILE: src/pg_lock_tracer/oid_resolver.py ================================================ """Resolve PostgreSQL OIDs to names and cache the result""" import sys from urllib.parse import urlparse import psycopg2 class OIDResolver: def __init__(self, connection_url): self.connection_url = connection_url self.cache = {} self.connection = None self.cur = None self.connect() def connect(self): """ Open the database connection """ connection_url_parsed = urlparse(self.connection_url) username = connection_url_parsed.username password = connection_url_parsed.password database = connection_url_parsed.path[1:] hostname = connection_url_parsed.hostname port = connection_url_parsed.port try: self.connection = psycopg2.connect( database=database, user=username, password=password, host=hostname, port=port, ) self.connection.set_session(autocommit=True) self.cur = self.connection.cursor() # Warmup cache self.fetch_all_oids() except psycopg2.OperationalError as error: print(f"Unable to connect to the database {self.connection_url}") print(f"{error}") sys.exit(1) def disconnect(self): """ Close the database connection. """ if self.cur: self.cur.close() self.cur = None if self.connection: self.connection.close() self.connection = None def fetch_all_oids(self): """ Fetch all Oid mappings from the catalog and cache them. This is done because: (1) Cache Oid cache lookups have to be fast and we want a warm cache. (2) Operations such as DROP delete objects from the database. Fetching the oid mapping afterwards is not possible. """ select_stmt = """ SELECT n.nspname, c.relname, c.oid FROM pg_namespace n JOIN pg_class c ON n.oid = c.relnamespace """ self.cur.execute(select_stmt) oids = self.cur.fetchall() for result_row in oids: oid = result_row[2] name = f"{result_row[0]}.{result_row[1]}" self.cache[oid] = name def fetch_oid_from_db(self, oid): """ Resolve the given OID into a name """ select_stmt = """ SELECT n.nspname, c.relname FROM pg_namespace n JOIN pg_class c ON n.oid = c.relnamespace WHERE c.oid = %s; """ oid = str(oid) try: self.cur.execute( select_stmt, [ oid, ], ) result_row = self.cur.fetchone() # Unable to get name, return Oid instead if result_row is None: return f"Oid {oid}" name = f"{result_row[0]}.{result_row[1]}" # Cache result self.cache[oid] = name return name except psycopg2.Error as error: print(f"Error while executing SQL statement: {error}") print(f"pgerror: {error.pgerror}") print(f"pgcode: {error.pgcode}") return "" def resolve_oid(self, oid): """ Resolve the given OID into a name. """ # OID cache hit if oid in self.cache: return self.cache[oid] # OID cache miss return self.fetch_oid_from_db(oid) ================================================ FILE: src/pg_lock_tracer/pg_lock_tracer.py ================================================ #!/usr/bin/env python3 # # PostgreSQL lock tracer # # This tool traces the lock operations that PostgreSQL performs. # # Unfortunately, PostgreSQL does not provide USDT probes for # these locks. Therefore BPF, UProbes, and parameter parsing # is used to trace these events. ############################################### import os import sys import json import argparse from abc import ABC from enum import IntEnum, auto, unique from bcc import BPF from prettytable import PrettyTable from pg_lock_tracer import __version__ from pg_lock_tracer.oid_resolver import OIDResolver from pg_lock_tracer.helper import PostgreSQLLockHelper, BPFHelper EXAMPLES = """ usage examples: # Trace use binary '/home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres' for tracing and trace pid 1234 pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 # Trace two PIDs pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 -p 5678 # Be verbose pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 -v # Use the given db connection to access the catalog of PID 1234 to resolve OIDs pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 -r 1234:psql://jan@localhost/test2 # Output in JSON format pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 -j # Print stacktrace on deadlock pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 -s DEADLOCK # Print stacktrace for locks and deadlocks pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 -s LOCK DEADLOCK # Trace only Transaction and Query related events pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 -t TRANSACTION QUERY # Write the output into file 'trace' pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 -o trace # Show statistics about locks pg_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres -p 1234 --statistics """ class TraceEvents(IntEnum): """ Events to trace """ TRANSACTION = auto() QUERY = auto() TABLE = auto() LOCK = auto() INVALIDATION = auto() ERROR = auto() parser = argparse.ArgumentParser( description="", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=EXAMPLES, ) parser.add_argument( "-V", "--version", action="version", version=f"{parser.prog} ({__version__})", ) parser.add_argument("-v", "--verbose", action="store_true", help="be verbose") parser.add_argument( "-j", "--json", action="store_true", help="generate output as JSON data" ) parser.add_argument( "-p", "--pid", type=int, nargs="+", action="extend", dest="pids", metavar="PID", help="the pid(s) to trace", ) parser.add_argument( "-x", "--exe", type=str, required=True, dest="path", metavar="PATH", help="path to binary", ) parser.add_argument( "-r", "--oid-resolver", type=str, action="extend", default=[], nargs="*", dest="oid_resolver_urls", metavar="OIDResolver", help="OID resolver for a PID. The resolver has to be specified in format ", ) parser.add_argument( "-s", "--stacktrace", type=str, dest="stacktrace", action="extend", default=None, nargs="*", choices=["DEADLOCK", "LOCK", "UNLOCK"], help="print stacktrace on every of these events", ) parser.add_argument( "-t", "--trace", type=str, dest="trace", action="extend", default=None, nargs="*", choices=[event.name for event in TraceEvents], help="events to trace (default: All events are traced)", ) parser.add_argument( "-o", "--output", type=str, dest="output_file", default=None, help="write the trace into output file", ) parser.add_argument("--statistics", action="store_true", help="print lock statistics") parser.add_argument( "-d", "--dry-run", action="store_true", help="compile and load the BPF program but exit afterward", ) @unique class Events(IntEnum): TABLE_OPEN = 1 TABLE_OPEN_RV = 2 TABLE_OPEN_RV_EXTENDED = 3 TABLE_CLOSE = 4 ERROR = 5 QUERY_BEGIN = 20 QUERY_END = 21 LOCK_RELATION_OID = 30 LOCK_RELATION_OID_END = 31 UNLOCK_RELATION_OID = 32 LOCK_GRANTED = 33 LOCK_GRANTED_FASTPATH = 34 LOCK_GRANTED_LOCAL = 35 LOCK_UNGRANTED = 36 LOCK_UNGRANTED_FASTPATH = 37 LOCK_UNGRANTED_LOCAL = 38 TRANSACTION_BEGIN = 40 TRANSACTION_COMMIT = 41 TRANSACTION_ABORT = 42 INVALIDATION_MESSAGES_ACCEPT = 50 # Events over 1000 are handled regardless of any pid filter GLOBAL = 1000 DEADLOCK = 1001 # From elog.h @unique class PGError(IntEnum): ERROR = 21 FATAL = 22 PANIC = 23 class LockStatisticsEntry: def __init__(self) -> None: # The number of requested locks self._lock_count = 0 # The total time spend for lock requests self._lock_time_ns = 0 # A list with the requested locks self._requested_locks = [] @property def lock_count(self): return self._lock_count @lock_count.setter def lock_count(self, value): self._lock_count = value @property def lock_time_ns(self): return self._lock_time_ns @lock_time_ns.setter def lock_time_ns(self, value): self._lock_time_ns = value @property def requested_locks(self): return self._requested_locks @requested_locks.setter def requested_locks(self, lock_type): self._requested_locks.append(lock_type) class PGLockTraceOutput(ABC): def __init__(self) -> None: super().__init__() self.statistics = {} self.bpf_instance = None self.bpf_stacks = None self.output_file = None self.oid_resolvers = None self.pids = None # Variables for lock timing self.last_lock_request_time = {} self.last_lock_relation = {} def set_context( self, bpf_instance, bpf_stacks, output_file, oid_resolvers, pids ) -> None: """ Set the needed context variables """ self.bpf_instance = bpf_instance self.bpf_stacks = bpf_stacks self.output_file = output_file self.oid_resolvers = oid_resolvers self.pids = pids def print_event(self, _cpu, data, _size): """ Handle the output of the given event. Subclasses will implement the concrete logic. """ def update_statistics(self, event, oid_value): """ Add the lock call to the statistics and measure lock request time """ if event.event_type == Events.LOCK_RELATION_OID: if oid_value not in self.statistics: self.statistics[oid_value] = LockStatisticsEntry() statistics_entry = self.statistics.get(oid_value) statistics_entry.lock_count += 1 statistics_entry.requested_locks = event.mode self.last_lock_request_time[event.pid] = event.timestamp self.last_lock_relation[event.pid] = oid_value if event.event_type == Events.LOCK_RELATION_OID_END: lock_relation = self.last_lock_relation[event.pid] statistics_entry = self.statistics.get(lock_relation) statistics_entry.lock_time_ns += self.get_lock_wait_time(event) self.last_lock_relation[event.pid] = None def get_lock_wait_time(self, event): """ Get the last lock wait time (LOCK_RELATION_OID updates last_lock_request_time). This method should be called on LOCK_RELATION_OID_END. """ if event.event_type != Events.LOCK_RELATION_OID_END: return None return event.timestamp - self.last_lock_request_time[event.pid] def print_statistics(self): """ Print lock statistics """ print("\nLock statistics:\n================") # Oid lock statistics print("\nLocks per OID") table = PrettyTable(["Lock Name", "Requests", "Total Lock Request Time (ns)"]) sorted_keys = sorted( self.statistics.keys(), key=lambda key: self.statistics.get(key).lock_count, reverse=True, ) for key in sorted_keys: statistics = self.statistics[key] table.add_row([key, statistics.lock_count, statistics.lock_time_ns]) print(table) # Lock type statistics print("\nLock types") table = PrettyTable(["Lock Type", "Number of requested locks"]) # Map: Key = Lock type, Value = Number of requested locks requested_locks = {} # Gather per lock type statistics for statistics in self.statistics.values(): for lock_type in statistics.requested_locks: locks = requested_locks.get(lock_type, 0) + 1 requested_locks[lock_type] = locks # Print statistics for lock_type in sorted(requested_locks): locks = requested_locks[lock_type] lock_name = PostgreSQLLockHelper.lock_type_to_str(lock_type) table.add_row([lock_name, locks]) print(table) def handle_output_line(self, line): """ Handle a output line """ if self.output_file: self.output_file.write(line + "\n") else: print(line) class PGLockTraceOutputHuman(PGLockTraceOutput): # pylint: disable=too-many-branches, too-many-statements def print_event(self, _cpu, data, _size): """ Print event in a human readable format """ event = self.bpf_instance["lockevents"].event(data) if ( self.pids and event.pid not in self.pids and event.event_type < Events.GLOBAL ): return print_prefix = f"{event.timestamp} [Pid {event.pid}]" # Resolve the OID to a name if event.object > 0: tablename = event.object # Resolve the OID to a table name if event.pid in self.oid_resolvers and event.object: resolver = self.oid_resolvers[event.pid] oid_value = resolver.resolve_oid(event.object) tablename = f"{tablename} ({oid_value})" self.update_statistics(event, oid_value) else: self.update_statistics(event, event.object) output = None if event.event_type == Events.TABLE_OPEN: lock_type = PostgreSQLLockHelper.lock_type_to_str(event.mode) output = f"{print_prefix} Table open {tablename} {lock_type}" elif event.event_type in (Events.TABLE_OPEN_RV, Events.TABLE_OPEN_RV_EXTENDED): # Table is opened using a (string) range value lock_type = PostgreSQLLockHelper.lock_type_to_str(event.mode) schema = event.payload_str1.decode("utf-8") table = event.payload_str2.decode("utf-8") tablename = f"{schema}.{table}" output = ( f"{print_prefix} Table open (by range value) {tablename} {lock_type}" ) elif event.event_type == Events.TABLE_CLOSE: lock_type = PostgreSQLLockHelper.lock_type_to_str(event.mode) output = f"{print_prefix} Table close {tablename} {lock_type}" elif event.event_type == Events.LOCK_RELATION_OID: lock_type = PostgreSQLLockHelper.lock_type_to_str(event.mode) output = f"{print_prefix} Lock object {tablename} {lock_type}" elif event.event_type == Events.LOCK_RELATION_OID_END: lock_time = self.get_lock_wait_time(event) output = f"{print_prefix} Lock was acquired in {lock_time} ns" elif event.event_type == Events.UNLOCK_RELATION_OID: lock_type = PostgreSQLLockHelper.lock_type_to_str(event.mode) output = f"{print_prefix} Unlock relation {tablename} {lock_type}" elif event.event_type == Events.LOCK_GRANTED: lock_type = PostgreSQLLockHelper.lock_type_to_str(event.mode) output = ( f"{print_prefix} Lock granted {tablename} {lock_type} " f"(Requested locks {event.requested})" ) elif event.event_type == Events.LOCK_GRANTED_FASTPATH: lock_type = PostgreSQLLockHelper.lock_type_to_str(event.mode) output = f"{print_prefix} Lock granted (fastpath) {tablename} {lock_type}" elif event.event_type == Events.LOCK_GRANTED_LOCAL: lock_type = PostgreSQLLockHelper.lock_type_to_str(event.mode) output = ( f"{print_prefix} Lock granted (local) {tablename} {lock_type} " f"(Already hold local {event.lock_local_hold})" ) elif event.event_type == Events.LOCK_UNGRANTED: lock_type = PostgreSQLLockHelper.lock_type_to_str(event.mode) output = ( f"{print_prefix} Lock ungranted {tablename} {lock_type} " f"(Requested locks {event.requested})" ) elif event.event_type == Events.LOCK_UNGRANTED_FASTPATH: lock_type = PostgreSQLLockHelper.lock_type_to_str(event.mode) output = f"{print_prefix} Lock ungranted (fastpath) {tablename} {lock_type}" elif event.event_type == Events.LOCK_UNGRANTED_LOCAL: lock_type = PostgreSQLLockHelper.lock_type_to_str(event.mode) output = ( f"{print_prefix} Lock ungranted (local) {tablename} {lock_type} " f"(Hold local {event.lock_local_hold})" ) elif event.event_type == Events.INVALIDATION_MESSAGES_ACCEPT: output = f"{print_prefix} Accept invalidation messages" elif event.event_type == Events.ERROR: pgerror_value = PGError(event.mode).name output = f"{print_prefix} Error occurred servity: {pgerror_value}" elif event.event_type == Events.QUERY_BEGIN: query = event.payload_str1.decode("utf-8") output = f"{print_prefix} Query begin '{query}'" elif event.event_type == Events.QUERY_END: output = f"{print_prefix} Query done\n" elif event.event_type == Events.TRANSACTION_BEGIN: output = f"{print_prefix} Transaction begin" elif event.event_type == Events.TRANSACTION_COMMIT: output = f"{print_prefix} Transaction commit" elif event.event_type == Events.TRANSACTION_ABORT: output = f"{print_prefix} Transaction abort" elif event.event_type == Events.DEADLOCK: output = f"{print_prefix} DEADLOCK DETECTED" else: raise ValueError(f"Unsupported event type {event.event_type}") self.handle_output_line(output) self.print_stacktace_if_available(event) def print_stacktace_if_available(self, event): """ Print the stacktrace of an event if available """ if event.stackid == 0 or self.bpf_stacks is None: return if event.stackid < 0: print( "Error stack is missing. Try to increase BPF_STACK_TRACE buffer size." ) else: for frame in self.bpf_stacks.walk(event.stackid): line = self.bpf_instance.sym( frame, event.pid, show_offset=True, show_module=True ) line = line.decode("utf-8") # Get line with: 'gdb info line *(symbol+0x1111)' line = f"\t{line}" self.handle_output_line(line) class PGLockTraceOutputJSON(PGLockTraceOutput): def print_event(self, _cpu, data, _size): """ Print event in JSON format """ event = self.bpf_instance["lockevents"].event(data) if ( self.pids and event.pid not in self.pids and event.event_type < Events.GLOBAL ): return output = {} output["timestamp"] = event.timestamp output["pid"] = event.pid output["event"] = Events(event.event_type).name if event.event_type in ( Events.TABLE_OPEN, Events.TABLE_OPEN_RV, Events.TABLE_OPEN_RV_EXTENDED, Events.TABLE_CLOSE, Events.LOCK_RELATION_OID, Events.UNLOCK_RELATION_OID, Events.LOCK_GRANTED, Events.LOCK_GRANTED_FASTPATH, Events.LOCK_GRANTED_LOCAL, Events.LOCK_UNGRANTED, Events.LOCK_UNGRANTED_FASTPATH, Events.LOCK_UNGRANTED_LOCAL, ): output["lock_type"] = PostgreSQLLockHelper.lock_type_to_str(event.mode) # Resolve OID to tablename if event.pid in self.oid_resolvers and event.object: resolver = self.oid_resolvers[event.pid] oid_value = resolver.resolve_oid(event.object) output["table"] = oid_value self.update_statistics(event, oid_value) else: self.update_statistics(event, event.object) # Resolve the OID to a name if event.object: output["oid"] = event.object if event.event_type == Events.ERROR: pgerror_value = PGError(event.mode).name output["servity"] = pgerror_value elif event.event_type == Events.QUERY_BEGIN: output["query"] = event.payload_str1.decode("utf-8") elif event.event_type in (Events.TABLE_OPEN_RV, Events.TABLE_OPEN_RV_EXTENDED): # Table is opened using a (string) range value schema = event.payload_str1.decode("utf-8") table = event.payload_str2.decode("utf-8") output["table"] = f"{schema}.{table}" elif event.event_type == Events.LOCK_GRANTED_LOCAL: output["lock_local_hold"] = event.lock_local_hold elif event.event_type == Events.LOCK_RELATION_OID_END: lock_time = self.get_lock_wait_time(event) output["lock_time"] = lock_time self.add_stacktrace_if_available(output, event) self.handle_output_line(json.dumps(output)) def add_stacktrace_if_available(self, output, event): """ Add a stacktrace to the JSON structure if available """ if event.stackid == 0 or self.bpf_stacks is None: return if event.stackid < 0: output["stacktrace"] = "MISSING" else: lines = [] # Get stacktrace symbol with module for frame in self.bpf_stacks.walk(event.stackid): line = self.bpf_instance.sym( frame, event.pid, show_offset=True, show_module=True ) lines.append(line.decode("utf-8")) # Merge lines into a single string stacktrace = ", ".join(lines) output["stacktrace"] = stacktrace class PGLockTracer: def __init__(self, prog_args): self.bpf_instance = None self.bpf_stacks = None self.output_file = None self.output_class = None self.args = prog_args # A map of OID resolvers. One resolver per PID is needed # because the Oid depend on the catalog of the database. self.oid_resolvers = {} for oid_resolver_url in self.args.oid_resolver_urls: if ":" not in oid_resolver_url: raise ValueError( f"Resolver URL has to be in format: 'PID:URL' ({oid_resolver_url} was provided)" ) split_url = oid_resolver_url.split(":", 1) resolver_pid = int(split_url[0]) database_url = split_url[1] if resolver_pid not in self.args.pids: print( f"Specified resolver for PID {resolver_pid}, but PID is not monitored" ) sys.exit(1) if self.args.verbose: print(f"Add resolver for PID {resolver_pid} with URL {database_url}") oid_resolver = OIDResolver(database_url) self.oid_resolvers[resolver_pid] = oid_resolver # Belong the processes to the binary? BPFHelper.check_pid_exe(self.args.pids, self.args.path) # Does the output file already exists? if self.args.output_file and os.path.exists(self.args.output_file): raise ValueError(f"Output file {self.args.output_file} already exists") @staticmethod def generate_c_defines(stacktrace_events, verbose): """ Create C defines from python enums """ enum_defines = BPFHelper.enum_to_defines(Events, "EVENT") error_defines = BPFHelper.enum_to_defines(PGError, "PGERROR") defines = enum_defines + error_defines # Print stacktrace for each lock if stacktrace_events and "LOCK" in stacktrace_events: defines += "#define STACKTRACE_LOCK\n" if verbose: print("Print stacktrace on each lock event") # Print stacktrace for each unlock if stacktrace_events and "UNLOCK" in stacktrace_events: defines += "#define STACKTRACE_UNLOCK\n" if verbose: print("Print stacktrace on each unlock event") # Print stacktrace on deadlock if stacktrace_events and "DEADLOCK" in stacktrace_events: defines += "#define STACKTRACE_DEADLOCK\n" if verbose: print("Print stacktrace on each deadlock") return defines def init(self): """ Init the PostgreSQL lock tracer """ defines = PGLockTracer.generate_c_defines( self.args.stacktrace, self.args.verbose ) bpf_program = BPFHelper.read_bpf_program("pg_lock_tracer.c") bpf_program_final = bpf_program.replace("__DEFINES__", defines) if self.args.verbose: print(bpf_program_final) # Disable warnings like # 'warning: '__HAVE_BUILTIN_BSWAP32__' macro redefined [-Wmacro-redefined]' bpf_cflags = ["-Wno-macro-redefined"] if not self.args.verbose else [] print("===> Compiling BPF program") self.bpf_instance = BPF(text=bpf_program_final, cflags=bpf_cflags) print("===> Attaching BPF probes") self.attach_probes() # Stack traces requested? if self.args.stacktrace: self.bpf_stacks = self.bpf_instance.get_table("stacks") # Open file for output if provided if self.args.output_file: # pylint: disable=consider-using-with self.output_file = open(self.args.output_file, "w", encoding="utf-8") if not self.output_file.writable(): raise ValueError( f"Output file {self.args.output_file} is not writeable" ) # Output as human readable text or as json? self.output_class = ( PGLockTraceOutputJSON() if self.args.json else PGLockTraceOutputHuman() ) # Init the output class self.output_class.set_context( self.bpf_instance, self.bpf_stacks, self.output_file, self.oid_resolvers, self.args.pids, ) # Open the event queue self.bpf_instance["lockevents"].open_perf_buffer( self.output_class.print_event, page_cnt=BPFHelper.page_cnt ) def attach_probes(self): """ Attach BPF probes """ # Transaction probes if self.args.trace is None or TraceEvents.TRANSACTION.name in self.args.trace: BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^StartTransaction$", "bpf_transaction_begin", self.args.verbose, ) BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^CommitTransaction$", "bpf_transaction_commit", self.args.verbose, ) BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^AbortTransaction$", "bpf_transaction_abort", self.args.verbose, ) BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^DeadLockReport$", "bpf_deadlock", self.args.verbose, ) # Query probes if self.args.trace is None or TraceEvents.QUERY.name in self.args.trace: BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^exec_simple_query$", "bpf_query_begin", self.args.verbose, ) BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^exec_simple_query$", "bpf_query_end", self.args.verbose, False, ) # Table probes if self.args.trace is None or TraceEvents.TABLE.name in self.args.trace: BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^table_open$", "bpf_table_open", self.args.verbose, ) BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^table_openrv$", "bpf_table_openrv", self.args.verbose, ) BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^table_openrv_extended$", "bpf_table_openrv_extended", self.args.verbose, ) BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^table_close$", "bpf_table_close", self.args.verbose, ) # Lock probes if self.args.trace is None or TraceEvents.LOCK.name in self.args.trace: BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^LockRelationOid$", "bpf_lock_relation_oid", self.args.verbose, ) BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^LockRelationOid$", "bpf_lock_relation_oid_end", self.args.verbose, False, ) BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^UnlockRelationOid$", "bpf_unlock_relation_oid", self.args.verbose, ) BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^GrantLock$", "bpf_lock_grant", self.args.verbose, ) BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^FastPathGrantRelationLock$", "bpf_lock_fastpath_grant", self.args.verbose, ) BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^GrantLockLocal$", "bpf_lock_local_grant", self.args.verbose, ) BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^UnGrantLock$", "bpf_lock_ungrant", self.args.verbose, ) BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^FastPathUnGrantRelationLock$", "bpf_lock_fastpath_ungrant", self.args.verbose, ) BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^RemoveLocalLock$", "bfp_local_lock_ungrant", self.args.verbose, ) # Invalidation messages probes if self.args.trace is None or TraceEvents.INVALIDATION.name in self.args.trace: BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^AcceptInvalidationMessages$", "bpf_accept_invalidation_messages", self.args.verbose, ) # Error probes if self.args.trace is None or TraceEvents.ERROR.name in self.args.trace: BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^errstart$", "bpf_errstart", self.args.verbose, ) def run(self): """ Run the BPF program and read results """ print("===> Ready to trace queries") while True: try: self.bpf_instance.perf_buffer_poll() except KeyboardInterrupt: if self.output_file: self.output_file.close() if self.args.statistics: self.output_class.print_statistics() sys.exit(0) def main(): """ Entry point for the BPF based PostgreSQL lock tracer. """ args = parser.parse_args() pg_lock_tracer = PGLockTracer(args) pg_lock_tracer.init() if not args.dry_run: pg_lock_tracer.run() if __name__ == "__main__": main() ================================================ FILE: src/pg_lock_tracer/pg_lw_lock_tracer.py ================================================ #!/usr/bin/env python3 # # PostgreSQL LW lock tracer. To use this script, PostgreSQL has to be # compiled with '--enable-dtrace'. # # See https://www.postgresql.org/docs/current/dynamic-trace.html # # List all available USDT probes # sudo bpftrace -l "usdt:/home/jan/postgresql-sandbox/bin/REL_15_1_DEBUG/bin/postgres:*" ############################################### import sys import argparse from enum import IntEnum, unique from bcc import BPF, USDT from prettytable import PrettyTable from pg_lock_tracer import __version__ from pg_lock_tracer.helper import BPFHelper EXAMPLES = """examples: # Trace the LW locks of the PID 1234 pg_lw_lock_tracer -p 1234 # Trace the LW locks of the PIDs 1234 and 5678 pg_lw_lock_tracer -p 1234 -p 5678 # Trace the LW locks of the PID 1234 and be verbose pg_lw_lock_tracer -p 1234 -v # Trace the LW locks of the PID 1234 and collect statistics pg_lw_lock_tracer -p 1234 -v --statistics """ parser = argparse.ArgumentParser( description="", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=EXAMPLES, ) parser.add_argument( "-V", "--version", action="version", version=f"{parser.prog} ({__version__})", ) parser.add_argument("-v", "--verbose", action="store_true", help="Be verbose") parser.add_argument( "-p", "--pid", type=int, nargs="+", dest="pids", metavar="PID", help="the pid(s) to trace", required=True, ) parser.add_argument( "-d", "--dry-run", action="store_true", help="compile and load the BPF program but exit afterward", ) parser.add_argument("--statistics", action="store_true", help="print lock statistics") @unique class Events(IntEnum): LOCK = 0 LOCK_OR_WAIT = 1 LOCK_OR_WAIT_FAIL = 2 UNLOCK = 3 WAIT_START = 4 WAIT_DONE = 5 COND_ACQUIRE = 6 COND_ACQUIRE_FAIL = 7 @unique class LWLockMode(IntEnum): LW_EXCLUSIVE = 0 LW_SHARED = 1 LW_WAIT_UNTIL_FREE = 2 class LockStatisticsEntry: def __init__(self) -> None: # The number of non-waited requested locks self._direct_lock_count = 0 # The number of acquire lock or wait calls self._acquire_or_wait_count = 0 # The number of failed acquire lock or wait calls self._acquire_or_wait_failed_count = 0 # The number of locks with condition self._lock_cond_count = 0 # The number of failed lock with condition self._lock_cond_failed_count = 0 # The number of lock waits self._wait_lock_count = 0 # The total time spend for lock wait requests self._lock_wait_time_ns = 0 # A list with the requested locks self._requested_locks = [] @property def direct_lock_count(self): return self._direct_lock_count @direct_lock_count.setter def direct_lock_count(self, value): self._direct_lock_count = value @property def acquire_or_wait_count(self): return self._acquire_or_wait_count @acquire_or_wait_count.setter def acquire_or_wait_count(self, value): self._acquire_or_wait_count = value @property def acquire_or_wait_failed_count(self): return self._acquire_or_wait_failed_count @acquire_or_wait_failed_count.setter def acquire_or_wait_failed_count(self, value): self._acquire_or_wait_failed_count = value @property def wait_lock_count(self): return self._wait_lock_count @wait_lock_count.setter def wait_lock_count(self, value): self._wait_lock_count = value @property def lock_cond_count(self): return self._lock_cond_count @lock_cond_count.setter def lock_cond_count(self, value): self._lock_cond_count = value @property def lock_cond_failed_count(self): return self._lock_cond_failed_count @lock_cond_failed_count.setter def lock_cond_failed_count(self, value): self._lock_cond_failed_count = value @property def lock_wait_time_ns(self): return self._lock_wait_time_ns @lock_wait_time_ns.setter def lock_wait_time_ns(self, value): self._lock_wait_time_ns = value @property def requested_locks(self): return self._requested_locks @requested_locks.setter def requested_locks(self, lock_type): self._requested_locks.append(lock_type) class PGLWLockTracer: def __init__(self, prog_args): self.bpf_instance = None self.usdts = None self.prog_args = prog_args self.statistics = {} # Variables for lock timing self.last_lock_request_time = {} def update_statistics(self, event, tranche, lock_mode): """ Update the statistics """ if tranche not in self.statistics: self.statistics[tranche] = LockStatisticsEntry() statistics_entry = self.statistics.get(tranche) # Lock directly requested if event.event_type == Events.LOCK: statistics_entry.direct_lock_count += 1 statistics_entry.requested_locks = lock_mode return # LWLockAcquireOrWait - Acquired if event.event_type == Events.LOCK_OR_WAIT: statistics_entry.acquire_or_wait_count += 1 statistics_entry.requested_locks = lock_mode return # LWLockAcquireOrWait - Waited if event.event_type == Events.LOCK_OR_WAIT_FAIL: statistics_entry.acquire_or_wait_failed_count += 1 statistics_entry.requested_locks = lock_mode return # Wait for lock if event.event_type == Events.WAIT_START: statistics_entry.wait_lock_count += 1 self.last_lock_request_time[event.pid] = event.timestamp return # Wait for lock done if event.event_type == Events.WAIT_DONE: wait_time = self.get_lock_wait_time(event) statistics_entry.lock_wait_time_ns += wait_time return # LWLockConditionalAcquire - Acquire with condition if event.event_type == Events.COND_ACQUIRE: statistics_entry.lock_cond_count += 1 statistics_entry.requested_locks = lock_mode return # LWLockConditionalAcquire - Condition not possible if event.event_type == Events.COND_ACQUIRE_FAIL: statistics_entry.lock_cond_failed_count += 1 statistics_entry.requested_locks = lock_mode return def get_lock_wait_time(self, event): """ Get the last lock wait time (WAIT_START updates last_lock_request_time). """ if event.event_type != Events.WAIT_DONE: return None return event.timestamp - self.last_lock_request_time[event.pid] def print_lock_event(self, _cpu, data, _size): """ Print a new lock event. Developer note: Wait events can be tested with second PostgreSQL process and gdb b LWLockAcquireOrWait """ event = self.bpf_instance["lockevents"].event(data) tranche = event.tranche.decode("utf-8") print_prefix = f"{event.timestamp} [Pid {event.pid}]" lock_mode = LWLockMode(event.mode).name self.update_statistics(event, tranche, lock_mode) if event.event_type == Events.LOCK: print( f"{print_prefix} Acquired lock {tranche} (mode {lock_mode}) / LWLockAcquire()" ) elif event.event_type == Events.LOCK_OR_WAIT: print( f"{print_prefix} Acquired lock {tranche} (mode {lock_mode}) " "/ LWLockConditionalAcquire()" ) elif event.event_type == Events.LOCK_OR_WAIT_FAIL: print( f"{print_prefix} Waited but not acquired {tranche} (mode {lock_mode}) " "/ LWLockConditionalAcquire()" ) elif event.event_type == Events.UNLOCK: print(f"{print_prefix} Unlock {tranche}") elif event.event_type == Events.WAIT_START: print(f"{print_prefix} Wait for {tranche}") elif event.event_type == Events.WAIT_DONE: lock_time = self.get_lock_wait_time(event) print(f"{print_prefix} Wait for {tranche} lock took {lock_time} ns") elif event.event_type == Events.COND_ACQUIRE: print( f"{print_prefix} Acquired lock {tranche} (mode {lock_mode}) " "/ LWLockConditionalAcquire()" ) elif event.event_type == Events.COND_ACQUIRE_FAIL: print( f"{print_prefix} Failed to acquire lock {tranche} (mode {lock_mode}) " "/ LWLockConditionalAcquire()" ) else: raise ValueError(f"Unknown event type {event.event_type}") def init(self): """ Compile and load the BPF program """ print(f"==> Attaching to PIDs {self.prog_args.pids}") self.usdts = list(map(lambda pid: USDT(pid=pid), self.prog_args.pids)) # See https://www.postgresql.org/docs/15/dynamic-trace.html for usdt in self.usdts: usdt.enable_probe("lwlock__acquire", "lwlock_acquire") usdt.enable_probe("lwlock__acquire__or__wait", "lwlock_acquire_or_wait") usdt.enable_probe( "lwlock__acquire__or__wait__fail", "lwlock_acquire_or_wait_fail" ) usdt.enable_probe("lwlock__release", "lwlock_release") usdt.enable_probe("lwlock__wait__start", "lwlock_wait_start") usdt.enable_probe("lwlock__wait__done", "lwlock_wait_done") usdt.enable_probe("lwlock__condacquire", "lwlock_condacquire") usdt.enable_probe("lwlock__condacquire__fail", "lwlock_condacquire_fail") if self.prog_args.verbose: print("=======") print("\n".join(map(lambda u: u.get_text(), self.usdts))) print("=======") enum_defines = BPFHelper.enum_to_defines(Events, "EVENT") bpf_program = BPFHelper.read_bpf_program("pg_lw_lock_tracer.c") bpf_program_final = bpf_program.replace("__DEFINES__", enum_defines) if self.prog_args.verbose: print(bpf_program_final) # Disable warnings like # 'warning: '__HAVE_BUILTIN_BSWAP32__' macro redefined [-Wmacro-redefined]' bpf_cflags = ["-Wno-macro-redefined"] if not self.prog_args.verbose else [] print("===> Compiling BPF program") self.bpf_instance = BPF( text=bpf_program_final, cflags=bpf_cflags, usdt_contexts=self.usdts ) self.bpf_instance["lockevents"].open_perf_buffer( self.print_lock_event, page_cnt=BPFHelper.page_cnt ) def print_statistics(self): """ Print lock statistics """ print("\nLock statistics:\n================") # Tranche lock statistics print("\nLocks per tranche") table = PrettyTable( [ "Tranche", "Acquired", "AcquireOrWait (Acquired)", "AcquireOrWait (Waited)", "ConditionalAcquire (Acquired)", "ConditionalAcquire (Failed)", "Waits", "Wait time (ns)", ] ) for key in sorted(self.statistics): statistics = self.statistics[key] table.add_row( [ key, statistics.direct_lock_count, statistics.acquire_or_wait_count, statistics.acquire_or_wait_failed_count, statistics.lock_cond_count, statistics.lock_cond_failed_count, statistics.wait_lock_count, statistics.lock_wait_time_ns, ] ) print(table) # Type lock statistics print("\nLocks per type") table = PrettyTable(["Lock type", "Requests"]) # Map: Key = Lock type, Value = Number of requested locks requested_locks = {} for statistics in self.statistics.values(): for lock_type in statistics.requested_locks: locks = requested_locks.get(lock_type, 0) + 1 requested_locks[lock_type] = locks for lock_type in sorted(requested_locks): locks = requested_locks[lock_type] table.add_row([lock_type, locks]) print(table) def run(self): """ Run the BPF program and read results """ print("===> Ready to trace") while True: try: self.bpf_instance.perf_buffer_poll() except KeyboardInterrupt: if self.prog_args.statistics: self.print_statistics() sys.exit(0) def main(): """ Entry point for the BPF based PostgreSQL LW lock tracer. """ args = parser.parse_args() pg_lock_tracer = PGLWLockTracer(args) pg_lock_tracer.init() if not args.dry_run: pg_lock_tracer.run() if __name__ == "__main__": main() ================================================ FILE: src/pg_lock_tracer/pg_row_lock_tracer.py ================================================ #!/usr/bin/env python3 # # PostgreSQL row lock tracer. # # See https://www.postgresql.org/docs/current/explicit-locking.html#LOCKING-ROWS # ############################################### import sys import argparse from enum import IntEnum, unique from bcc import BPF from prettytable import PrettyTable from pg_lock_tracer import __version__ from pg_lock_tracer.helper import BPFHelper EXAMPLES = """examples: # Trace the row locks of the given PostgreSQL binary pg_row_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_14_9_DEBUG/bin/postgres # Trace the row locks of the PID 1234 pg_row_lock_tracer -p 1234 -x /home/jan/postgresql-sandbox/bin/REL_14_9_DEBUG/bin/postgres # Trace the row locks of the PID 1234 and 5678 pg_row_lock_tracer -p 1234 -p 5678 -x /home/jan/postgresql-sandbox/bin/REL_14_9_DEBUG/bin/postgres # Trace the row locks of the PID 1234 and be verbose pg_row_lock_tracer -p 1234 -x /home/jan/postgresql-sandbox/bin/REL_14_9_DEBUG/bin/postgres -v # Trace the row locks and show statistics pg_row_lock_tracer -x /home/jan/postgresql-sandbox/bin/REL_14_9_DEBUG/bin/postgres --statistics """ parser = argparse.ArgumentParser( description="", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=EXAMPLES, ) parser.add_argument( "-V", "--version", action="version", version=f"{parser.prog} ({__version__})", ) parser.add_argument("-v", "--verbose", action="store_true", help="Be verbose") parser.add_argument( "-p", "--pid", type=int, nargs="+", dest="pids", metavar="PID", help="the pid(s) to trace", ) parser.add_argument( "-x", "--exe", type=str, required=True, dest="path", metavar="PATH", help="path to binary", ) parser.add_argument( "-d", "--dry-run", action="store_true", help="compile and load the BPF program but exit afterward", ) parser.add_argument("--statistics", action="store_true", help="print lock statistics") @unique class Events(IntEnum): LOCK_TUPLE = 0 LOCK_TUPLE_END = 1 # See lockoptions.h in PostgreSQL @unique class TMResult(IntEnum): TM_OK = 0 TM_INVISIBLE = 1 TM_SELFMODIFIED = 2 TM_UPDATED = 3 TM_DELETED = 4 TM_BEINGMODIFIED = 5 TM_WOULDBLOCK = 6 # See lockoptions.h in PostgreSQL @unique class LockWaitPolicy(IntEnum): LOCK_WAIT_BLOCK = 0 LOCK_WAIT_SKIP = 1 LOCK_WAIT_ERROR = 2 # See lockoptions.h in PostgreSQL @unique class LockTupleMode(IntEnum): LOCK_TUPLE_KEYSHARE = 0 LOCK_TUPLE_SHARE = 1 LOCK_TUPLE_NOKEYEXCLUSIVE = 2 LOCK_TUPLE_EXCLUSIVE = 3 class LockStatisticsEntry: def __init__(self) -> None: # The requested locks self._lock_modes = {} # The used lock policies self._lock_policies = {} # The lock results self._lock_results = {} @property def lock_modes(self): return self._lock_modes @lock_modes.setter def lock_modes(self, value): self._lock_modes = value @property def lock_policies(self): return self._lock_policies @lock_policies.setter def lock_policies(self, value): self._lock_policies = value @property def lock_results(self): return self._lock_results @lock_results.setter def lock_results(self, value): self._results = value class PGRowLockTracer: def __init__(self, prog_args): self.bpf_instance = None self.args = prog_args self.statistics = {} # Variables for lock timing self.last_lock_request_time = {} # Belong the processes to the binary? BPFHelper.check_pid_exe(self.args.pids, self.args.path) def get_lock_wait_time(self, event): """ Get the last lock wait time (WAIT_START updates last_lock_request_time). """ if event.event_type != Events.LOCK_TUPLE_END: return None return event.timestamp - self.last_lock_request_time[event.pid] def update_statistics(self, event): """ Update the statistics """ if event.pid not in self.statistics: self.statistics[event.pid] = LockStatisticsEntry() statistics_entry = self.statistics.get(event.pid) # Lock requested if event.event_type == Events.LOCK_TUPLE: lock_wait_policy = LockWaitPolicy(event.lockwaitpolicy) if lock_wait_policy in statistics_entry.lock_policies: statistics_entry.lock_policies[lock_wait_policy] += 1 else: statistics_entry.lock_policies[lock_wait_policy] = 1 lock_tuple_mode = LockTupleMode(event.locktuplemode) if lock_tuple_mode in statistics_entry.lock_modes: statistics_entry.lock_modes[lock_tuple_mode] += 1 else: statistics_entry.lock_modes[lock_tuple_mode] = 1 return # Lock request done if event.event_type == Events.LOCK_TUPLE_END: lock_result = TMResult(event.lockresult) if lock_result in statistics_entry.lock_results: statistics_entry.lock_results[lock_result] += 1 else: statistics_entry.lock_results[lock_result] = 1 return return def print_lock_event(self, _cpu, data, _size): """ Print a new lock event. """ event = self.bpf_instance["lockevents"].event(data) if self.args.pids and event.pid not in self.args.pids: return print_prefix = f"{event.timestamp} [Pid {event.pid}]" self.update_statistics(event) if event.event_type == Events.LOCK_TUPLE: self.last_lock_request_time[event.pid] = event.timestamp locktuplemode = LockTupleMode(event.locktuplemode).name lockwaitpolicy = LockWaitPolicy(event.lockwaitpolicy).name print( f"{print_prefix} LOCK_TUPLE (Tablespace {event.tablespace} " f"database {event.database} relation {event.relation}) " f"- (Block and offset {event.blockid} {event.offset}) " f"- {locktuplemode} {lockwaitpolicy}" ) elif event.event_type == Events.LOCK_TUPLE_END: lockresult = TMResult(event.lockresult).name needed_time = self.get_lock_wait_time(event) print(f"{print_prefix} LOCK_TUPLE_END {lockresult} in {needed_time} ns") else: raise ValueError(f"Unknown event type {event.event_type}") def init(self): """ Init the PostgreSQL lock tracer """ enum_defines = BPFHelper.enum_to_defines(Events, "EVENT") bpf_program = BPFHelper.read_bpf_program("pg_row_lock_tracer.c") bpf_program_final = bpf_program.replace("__DEFINES__", enum_defines) if self.args.verbose: print(bpf_program_final) # Disable warnings like # 'warning: '__HAVE_BUILTIN_BSWAP32__' macro redefined [-Wmacro-redefined]' bpf_cflags = ["-Wno-macro-redefined"] if not self.args.verbose else [] print("===> Compiling BPF program") self.bpf_instance = BPF(text=bpf_program_final, cflags=bpf_cflags) print("===> Attaching BPF probes") self.attach_probes() # Open the event queue self.bpf_instance["lockevents"].open_perf_buffer( self.print_lock_event, page_cnt=BPFHelper.page_cnt ) def attach_probes(self): """ Attach BPF probes """ BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^heapam_tuple_lock$", "heapam_tuple_lock", self.args.verbose, ) BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^heapam_tuple_lock$", "heapam_tuple_lock_end", self.args.verbose, False, ) def print_statistics(self): """ Print lock statistics """ print("\nLock statistics:\n================") # Wait policies print("\nUsed wait policies:") wait_polices = ["PID"] for wait_policy in LockWaitPolicy: wait_polices.append(wait_policy.name) table = PrettyTable(wait_polices) for pid in sorted(self.statistics): statistics = self.statistics[pid] pid_statistics = [pid] for wait_policy in LockWaitPolicy: pid_statistics.append(statistics.lock_policies.get(wait_policy, 0)) table.add_row(pid_statistics) print(table) # Lock modes print("\nLock modes:") lock_modes = ["PID"] for lock_mode in LockTupleMode: lock_modes.append(lock_mode.name) table = PrettyTable(lock_modes) for pid in sorted(self.statistics): statistics = self.statistics[pid] pid_statistics = [pid] for lock_mode in LockTupleMode: pid_statistics.append(statistics.lock_modes.get(lock_mode, 0)) table.add_row(pid_statistics) print(table) # Lock results print("\nLock results:") lock_results = ["PID"] for lock_result in TMResult: lock_results.append(lock_result.name) table = PrettyTable(lock_results) for pid in sorted(self.statistics): statistics = self.statistics[pid] pid_statistics = [pid] for lock_result in TMResult: pid_statistics.append(statistics.lock_results.get(lock_result, 0)) table.add_row(pid_statistics) print(table) def run(self): """ Run the BPF program and read results """ print("===> Ready to trace") while True: try: self.bpf_instance.perf_buffer_poll() except KeyboardInterrupt: if self.args.statistics: self.print_statistics() sys.exit(0) def main(): """ Entry point for the BPF based PostgreSQL row lock tracer. """ args = parser.parse_args() pg_lock_tracer = PGRowLockTracer(args) pg_lock_tracer.init() if not args.dry_run: pg_lock_tracer.run() if __name__ == "__main__": main() ================================================ FILE: src/pg_lock_tracer/pg_spinlock_delay_tracer.py ================================================ #!/usr/bin/env python3 # # PostgreSQL spinlock delay tracer. # ############################################### import sys import argparse from bcc import BPF from pg_lock_tracer import __version__ from pg_lock_tracer.helper import BPFHelper EXAMPLES = """examples: # Trace spin delays of the given PostgreSQL binary pg_spinlock_delay_tracer -x /home/jan/postgresql-sandbox/bin/REL_14_9_DEBUG/bin/postgres # Trace spin delays of the PID 1234 pg_spinlock_delay_tracer -p 1234 -x /home/jan/postgresql-sandbox/bin/REL_14_9_DEBUG/bin/postgres # Trace spin delays of the PID 1234 and 5678 pg_spinlock_delay_tracer -p 1234 -p 5678 -x /home/jan/postgresql-sandbox/bin/REL_14_9_DEBUG/bin/postgres # Trace spin delays of the PID 1234 and be verbose pg_spinlock_delay_tracer -p 1234 -x /home/jan/postgresql-sandbox/bin/REL_14_9_DEBUG/bin/postgres -v """ parser = argparse.ArgumentParser( description="", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=EXAMPLES, ) parser.add_argument( "-V", "--version", action="version", version=f"{parser.prog} ({__version__})", ) parser.add_argument("-v", "--verbose", action="store_true", help="Be verbose") parser.add_argument( "-p", "--pid", type=int, nargs="+", dest="pids", metavar="PID", help="the pid(s) to trace", ) parser.add_argument( "-x", "--exe", type=str, required=True, dest="path", metavar="PATH", help="path to binary", ) parser.add_argument( "-d", "--dry-run", action="store_true", help="compile and load the BPF program but exit afterward", ) class PGSpinDelayTracer: def __init__(self, prog_args): self.bpf_instance = None self.args = prog_args # Belong the processes to the binary? BPFHelper.check_pid_exe(self.args.pids, self.args.path) @staticmethod def _decode_field(value): return value.decode("utf-8", "replace").rstrip("\x00") def print_lock_event(self, _cpu, data, _size): """ Print a new spin delay event. """ event = self.bpf_instance["lockevents"].event(data) if self.args.pids and event.pid not in self.args.pids: return file_name = self._decode_field(event.file) or "(unknown)" func_name = self._decode_field(event.func) or "(unknown)" print( f"{event.timestamp} [Pid {event.pid}] SpinDelay " f"spins={event.spins} delays={event.delays} " f"cur_delay={event.cur_delay} at {func_name}, {file_name}:{event.line}" ) def init(self): """ Init the PostgreSQL spin delay tracer """ bpf_program = BPFHelper.read_bpf_program("pg_spinlock_delay_tracer.c") if self.args.verbose: print(bpf_program) # Disable warnings like # 'warning: '__HAVE_BUILTIN_BSWAP32__' macro redefined [-Wmacro-redefined]' bpf_cflags = ["-Wno-macro-redefined"] if not self.args.verbose else [] print("===> Compiling BPF program") self.bpf_instance = BPF(text=bpf_program, cflags=bpf_cflags) print("===> Attaching BPF probes") self.attach_probes() # Open the event queue self.bpf_instance["lockevents"].open_perf_buffer( self.print_lock_event, page_cnt=BPFHelper.page_cnt ) def attach_probes(self): """ Attach BPF probes """ BPFHelper.register_ebpf_probe( self.args.path, self.bpf_instance, "^perform_spin_delay$", "spin_delay", self.args.verbose, ) def run(self): """ Run the BPF program and read results """ print("===> Ready to trace") while True: try: self.bpf_instance.perf_buffer_poll() except KeyboardInterrupt: sys.exit(0) def main(): """ Entry point for the BPF based PostgreSQL spin delay tracer. """ args = parser.parse_args() pg_spin_delay_tracer = PGSpinDelayTracer(args) pg_spin_delay_tracer.init() if not args.dry_run: pg_spin_delay_tracer.run() if __name__ == "__main__": main() ================================================ FILE: tests/__init__.py ================================================ ================================================ FILE: tests/test_helper.py ================================================ #!/usr/bin/env python3 import unittest from src.pg_lock_tracer.helper import PostgreSQLLockHelper class UNITTests(unittest.TestCase): def test_encode_locks(self): """ Test the encoding of locks """ my_locks = [ "NoLock", "AccessShareLock", "ShareRowExclusiveLock", "AccessExclusiveLock", ] my_locks_numeric = list(map(PostgreSQLLockHelper.lock_type_to_int, my_locks)) # Test conversion was ok self.assertListEqual([0, 1, 6, 8], my_locks_numeric) def encode_and_decode_locks(self, locks): """ Decode and encode the list of locks and test for errors """ # Convert into numeric values my_locks_numeric = list(map(PostgreSQLLockHelper.lock_type_to_int, locks)) # Encode and decode into single value single_value = PostgreSQLLockHelper.encode_locks_into_value(my_locks_numeric) decoded_locks = PostgreSQLLockHelper.decode_locks_from_value(single_value) decoded_my_locks = list( map(PostgreSQLLockHelper.lock_type_to_str, decoded_locks) ) self.assertListEqual(locks, decoded_my_locks) def test_parse_lock_decoding_and_encoding0(self): """ Test encoding and decoding of locks into a single value """ my_locks = [ "AccessExclusiveLock", ] self.encode_and_decode_locks(my_locks) def test_parse_lock_decoding_and_encoding1(self): """ Test encoding and decoding of locks into a single value """ my_locks = [ "NoLock", "AccessShareLock", "ShareRowExclusiveLock", "AccessExclusiveLock", ] self.encode_and_decode_locks(my_locks) def test_parse_lock_decoding_and_encoding2(self): """ Test encoding and decoding of locks into a single value """ my_locks = ["NoLock"] self.encode_and_decode_locks(my_locks) def test_parse_lock_decoding_and_encoding3(self): """ Test encoding and decoding of locks into a single value """ my_locks = [] self.encode_and_decode_locks(my_locks) def test_parse_lock_decoding_and_encoding_dup(self): """ Test encoding and decoding with duplicates """ my_locks = [ "AccessShareLock", "AccessShareLock", "AccessShareLock", "ShareRowExclusiveLock", "ShareRowExclusiveLock", ] my_locks_numeric = list(map(PostgreSQLLockHelper.lock_type_to_int, my_locks)) # Encode and decode into single value single_value = PostgreSQLLockHelper.encode_locks_into_value(my_locks_numeric) decoded_locks = PostgreSQLLockHelper.decode_locks_from_value(single_value) decoded_my_locks = list( map(PostgreSQLLockHelper.lock_type_to_str, decoded_locks) ) # Only unique values are present self.assertListEqual([my_locks[0], my_locks[4]], decoded_my_locks)