Repository: ShahriyarR/MySQL-AutoXtraBackup Branch: master Commit: dfdf86ba4d1f Files: 75 Total size: 171.6 KB Directory structure: gitextract_74cfdu6s/ ├── .deepsource.toml ├── .flake8 ├── .github/ │ └── workflows/ │ └── publish.yml ├── .gitignore ├── Dockerfile ├── HISTORY.md ├── LICENSE ├── README.md ├── changes/ │ ├── README.md │ └── make_history.py ├── docker-compose-test.yaml ├── docker-compose.yaml ├── docs/ │ ├── Makefile │ ├── advance_features.rst │ ├── api.rst │ ├── backup_tags.rst │ ├── basic_features.rst │ ├── basic_overview.rst │ ├── conf.py │ ├── config_file.rst │ ├── index.rst │ ├── installation.rst │ ├── intro.rst │ ├── option_reference.rst │ └── what_is_new_in_2_0.rst ├── mypy.ini ├── mysql_autoxtrabackup/ │ ├── __init__.py │ ├── api/ │ │ ├── __init__.py │ │ ├── controller/ │ │ │ ├── __init__.py │ │ │ └── controller.py │ │ └── main.py │ ├── autoxtrabackup.py │ ├── backup_backup/ │ │ ├── __init__.py │ │ ├── backup_archive.py │ │ ├── backup_builder.py │ │ └── backuper.py │ ├── backup_prepare/ │ │ ├── __init__.py │ │ ├── copy_back.py │ │ ├── prepare.py │ │ └── prepare_builder.py │ ├── general_conf/ │ │ ├── __init__.py │ │ ├── check_env.py │ │ ├── generalops.py │ │ ├── generate_default_conf.py │ │ └── path_config.py │ ├── process_runner/ │ │ ├── __init__.py │ │ ├── errors.py │ │ └── process_runner.py │ └── utils/ │ ├── __init__.py │ ├── helpers.py │ ├── mysql_cli.py │ └── version.py ├── netlify.toml ├── pyproject.toml ├── scripts/ │ ├── api_calls/ │ │ ├── delete_all_backups.sh │ │ ├── get_all_backups.sh │ │ ├── prepare_backup.sh │ │ └── take_backup.sh │ ├── build-docs.sh │ ├── format-imports.sh │ ├── format.sh │ ├── lint.sh │ ├── netlify-docs.sh │ └── publish.sh └── tests/ ├── Dockerfile ├── README.md ├── __init__.py ├── conftest.py ├── entrypoint.sh ├── requirements.txt ├── test_api.py ├── test_backup.bats ├── test_backup.py ├── test_helpers.py └── test_mysql_cli.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .deepsource.toml ================================================ version = 1 [[analyzers]] name = "python" enabled = true [analyzers.meta] runtime_version = "3.x.x" ================================================ FILE: .flake8 ================================================ [flake8] max-line-length = 88 select = C,E,F,W,B,B9 ignore = E203, E501, W503 exclude = __init__.py ================================================ FILE: .github/workflows/publish.yml ================================================ name: Publish on: release: types: - created jobs: publish: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v1 with: python-version: "3.6" - name: Install Flit run: pip install flit - name: Install Dependencies run: flit install --symlink - name: Publish env: FLIT_USERNAME: ${{ secrets.FLIT_USERNAME }} FLIT_PASSWORD: ${{ secrets.FLIT_PASSWORD }} FLIT_INDEX_URL: ${{ secrets.FLIT_INDEX_URL }} run: bash scripts/publish.sh ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python env/ build/ _build develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ *.egg-info/ .installed.cfg *.egg .vscode # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit tests / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *,cover .hypothesis/ # Translations *.mo *.pot # PyCharm stuff .idea .vagrant # Django stuff: *.log local_settings.py # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # pyenv .python-version # celery beat schedule file celerybeat-schedule # dotenv .env # virtualenv .venv venv/ ENV/ # Spyder project settings .spyderproject # Rope project settings .ropeproject # Local Netlify folder .netlify ================================================ FILE: Dockerfile ================================================ FROM tiangolo/uvicorn-gunicorn-fastapi:python3.8 WORKDIR /app RUN git clone https://github.com/sstephenson/bats.git && \ cd bats && \ ./install.sh /usr/local RUN apt-get update && apt-get install -y lsb-release RUN apt install -y libncurses5 RUN apt-get install -y vim RUN wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb RUN dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb RUN percona-release enable-only tools release RUN apt-get update RUN apt-get install -y percona-xtrabackup-80 RUN apt-get install -y qpress COPY ./ /app RUN pip install --upgrade pip && \ pip install -r requirements.txt RUN python3 setup.py install ENV MODULE_NAME="api.main" ================================================ FILE: HISTORY.md ================================================ ## v2.0.2 (2021-05-06) * Increased code coverage and did code base refactoring, #444 by @shahriyarr ## v2.0.1 (2021-05-05) * Added change handler for each release(so-called HISTORY.md), #442 by @shahriyarr * Added new version.py file for getting the exact version from one place, #440 by @shahriyarr * New option --run-server was added to start API server, #437 by @shahriyarr ================================================ FILE: LICENSE ================================================ The MIT License (MIT) Copyright (c) 2016 Shahriyar Rzayev Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ MySQL-AutoXtrabackup ==================== MySQL AutoXtrabackup commandline tool written in Python 3. The source code fully typed with hints - structured the project mostly similar to [FastAPI](https://fastapi.tiangolo.com/) and [Pydantic](https://github.com/samuelcolvin/pydantic) For community from Azerbaijan MySQL User Community: [Python Azerbaijan Community](https://www.facebook.com/groups/python.az). For any question please open an issue here. What this project is about? --------------------------- The idea for this tool, came from my hard times after accidentally deleting the table data. There was a full backup and 12 incremental backups. It took me 20 minutes to prepare necessary commands for preparing backups. If you have compressed + encrypted backups you need also, decrypt and decompress, which is going to add extra time for preparing backups. Then I decided to automate this process. In other words, preparing necessary commands for backup and prepare stage were automated. We have nice CLI with necessary options: ``` autoxtrabackup --help Usage: autoxtrabackup [OPTIONS] Options: --dry-run Enable the dry run. --prepare Prepare/recover backups. --run-server Start the FastAPI app for serving API --backup Take full and incremental backups. --version Version information. --defaults-file TEXT Read options from the given file [default: / home/shako/.autoxtrabackup/autoxtrabackup.cn f] --tag TEXT Pass the tag string for each backup --show-tags Show backup tags and exit -v, --verbose Be verbose (print to console) -lf, --log-file TEXT Set log file [default: /home/shako/.autoxtr abackup/autoxtrabackup.log] -l, --log, --log-level [DEBUG|INFO|WARNING|ERROR|CRITICAL] Set log level [default: INFO] --log-file-max-bytes INTEGER Set log file max size in bytes [default: 1073741824] --log-file-backup-count INTEGER Set log file backup count [default: 7] --help Print help message and exit. ``` If you think, CLI is not for you. We have experimental feature where you can start API server and take backups using API call(ATTENTION: FastAPI involved) ``` sudo `which autoxtrabackup` --run-server INFO: Started server process [30238] INFO: Waiting for application startup. app started INFO: Application startup complete. INFO: Uvicorn running on http://127.0.0.1:5555 (Press CTRL+C to quit) ``` ``` $ curl -X POST http://127.0.0.1:5555/backup {"result":"Successfully finished the backup process"} ``` For the rest please read the full documentation. Development: ------------------- Current major version is >= 2.0 - so if you want to help, please do changes on this branch and then kindly send PR :) I also encourage you to upgrade from older version as the code base fully updated. Do you have an idea, question please open an issue. Read full documentation here: ---------------------------------------------- [**MySQL-AutoXtrabackup documentation!**](https://autoxtrabackup.azepug.az/) ================================================ FILE: changes/README.md ================================================ # Pending Changes This directory contains files describing changes to `mysql-autoxtrabackup` since the last release. If you're creating a pull request, please add a new file to this directory called `-.md`. It should be formatted as a single paragraph of markdown The contents of this file will be used to update `HISTORY.md` before the next release. ================================================ FILE: changes/make_history.py ================================================ #!/usr/bin/env python3 import re import sys from datetime import date from importlib.machinery import SourceFileLoader from pathlib import Path THIS_DIR = Path(__file__).parent name_regex = re.compile(r'(\d+)-(.*?)\.md') bullet_list = [] for p in THIS_DIR.glob('*.md'): if p.name == 'README.md': continue m = name_regex.fullmatch(p.name) if not m: raise RuntimeError(f'{p.name!r}: invalid change file name') gh_id, creator = m.groups() content = p.read_text().replace('\r\n', '\n').strip('\n. ') if '\n\n' in content: raise RuntimeError(f'{p.name!r}: content includes multiple paragraphs') content = content.replace('\n', '\n ') priority = 0 if '**breaking change' in content.lower(): priority = 2 elif content.startswith('**'): priority = 1 bullet_list.append((priority, int(gh_id), f'* {content}, #{gh_id} by @{creator}')) if not bullet_list: print('no changes found') sys.exit(0) version = SourceFileLoader('version', 'mysql_autoxtrabackup/utils/version.py').load_module() chunk_title = f'v{version.VERSION} ({date.today():%Y-%m-%d})' new_chunk = '## {}\n\n{}\n\n'.format(chunk_title, '\n'.join(c for *_, c in sorted(bullet_list, reverse=True))) print(f'{chunk_title}...{len(bullet_list)} items') history_path = THIS_DIR / '..' / 'HISTORY.md' history = new_chunk + history_path.read_text() history_path.write_text(history) for p in THIS_DIR.glob('*.md'): if p.name != 'README.md': p.unlink() print( 'changes deleted and HISTORY.md successfully updated, to reset use:\n\n' ' git checkout -- changes/*-*.md HISTORY.md\n' ) ================================================ FILE: docker-compose-test.yaml ================================================ version: "3.7" services: api_v1: build: context: tests dockerfile: Dockerfile args: GIT_BRANCH_NAME: ${GIT_BRANCH_NAME} container_name: autoxtrabackup_apiv1_test volumes: - ./tests/entrypoint.sh:/entrypoint.sh - /dev/log:/dev/log ports: - "8080:8080" network_mode: host ================================================ FILE: docker-compose.yaml ================================================ version: "3.7" services: api_v1: build: context: . dockerfile: Dockerfile container_name: autoxtrabackup_apiv1 volumes: - /dev/log:/dev/log - /var/lib/mysql:/var/lib/mysql - /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock - /usr/bin/mysqladmin:/usr/bin/mysqladmin - /usr/bin/mysql:/usr/bin/mysql ports: - "80:80" network_mode: host ================================================ FILE: docs/Makefile ================================================ # Minimal makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build SPHINXPROJ = MySQLAutoXtrabackup SOURCEDIR = . BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) ================================================ FILE: docs/advance_features.rst ================================================ Advance features ================= Compressed backups ------------------ To enable compression support just uncomment the options under [Compress] category inside main configuration file: :: [Compress] #Optional #Enable only if you want to use compression. compress=quicklz compress_chunk_size=65536 compress_threads=4 decompress=TRUE #Enable if you want to remove .qp files after decompression.(Available from PXB 2.3.7 and 2.4.6) remove_original=FALSE Encrypted backups ----------------- To enable encryption support uncomment the options under [Encryption] category: :: [Encrypt] #optional #enable only if you want to create encrypted backups xbcrypt = /usr/bin/xbcrypt encrypt = AES256 #please note that --encrypt-key and --encrypt-key-file are mutually exclusive encrypt_key = VVTBwgM4UhwkTTV98fhuj+D1zyWoA89K #encrypt_key_file = /path/to/file/with_encrypt_key encrypt_threads = 4 encrypt_chunk_size = 65536 decrypt = AES256 #enable if you want to remove .qp files after decompression.(Available from PXB 2.3.7 and 2.4.6) remove_original = FALSE Partial backups --------------- It is possible to take partial full and incremental backups. The idea is, to take specified table(or database) as full backup, then to take incremental backups based on this one table. You can achieve this by enabling ``partial_list`` option from config file: :: [Backup] #optional: set pid directory pid_dir = /tmp/MySQL-AutoXtraBackup tmp_dir = /home/shako/XB_TEST/mysql_datadirs #optional: set warning if pid of backup us running for longer than x pid_runtime_warning = 2 Hours backup_dir = /home/shako/XB_TEST/backup_dir backup_tool = /usr/bin/xtrabackup #optional: specify different path/version of xtrabackup here for prepare #prepare_tool = xtra_prepare = --apply-log-only #optional: pass additional options for backup stage #xtra_backup = --compact #optional: pass additional options for prepare stage #xtra_prepare_options = --rebuild-indexes #optional: pass general additional options; it will go to both for backup and prepare #xtra_options = --binlog-info=ON --galera-info #optional: set archive and rotation #archive_dir = /home/shako/XB_TEST/backup_archives #prepare_archive = 1 #move_archive = 0 #full_backup_interval = 1 day #archive_max_size = 100GiB #archive_max_duration = 4 Days #optional: warning(enable this if you want to take partial backups). specify database names or table names. #partial_list = test.t1 test.t2 dbtest Run it and notice that backup command has changed (see ``--databases`` option for xtrabackup command): In the same way you can take incremental backup of specified tables. The prepare process is the same as ordinary prepare, just run autoxtrabackup with ``--prepare`` option. Decompressing and Decrypting backups ------------------------------------ We took Compressed and Encrypted backups. It is time to prepare them. autoxtrabackup will prepare all backups automatically, by first decrypting then decompressing step-by-step. All backups first will be decrypted then decompressed and then prepared. You can also optionally enable ``--remove-original`` option to remove ``.xbcrypt`` and ``.qp`` files from backup directory during prepare process. Read about this option here -> `--remove-original `_ autoxtrabackup with --dry_run option ------------------------------------ For testing purposes or just to show what is going on, with autoxtrabackup backup and prepare steps. You can append ``--dry_run`` option, to show commands but not to run them. Taking backup: :: $ sudo autoxtrabackup -v -lf /home/shahriyar.rzaev/autoxtrabackup_2_3_5_6.log -l DEBUG \ --defaults-file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.cnf --backup --dry_run Preparing backups: :: $ sudo autoxtrabackup -v -lf /home/shahriyar.rzaev/autoxtrabackup_2_3_5_6.log -l DEBUG \ --defaults-file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.cnf --prepare --dry_run The end. ================================================ FILE: docs/api.rst ================================================ Here is the basic ideas how to use API calls to operate on backups ================================================================== Currently, we have only 4 endpoints. .. image:: doc_api.png :width: 400 :alt: DOC API Sample cURLs: ------------- After starting API server: :: $ sudo `which autoxtrabackup` --run-server Of course you can use the Web UI for taking backups, but in case you want to send requests, below I have provided sample cURLs. Taking backup: :: $ curl -X POST http://127.0.0.1:5555/backup {"result":"Successfully finished the backup process"} Preparing backup: :: $ curl -X POST http://127.0.0.1:5555/prepare {"result":"Successfully prepared all the backups"} Listing backups: :: $ curl http://127.0.0.1:5555/backups {"backups":{"full":[{"2021-05-05_18-37-36":"Full-Prepared"}],"inc":["2021-05-05_18-38-19"]}} Removing backups: :: $ curl -X DELETE http://127.0.0.1:5555/delete {"result":"There is no backups or backups removed successfully"} ================================================ FILE: docs/backup_tags.rst ================================================ Backup Tags =========== The backup tags actually is a result of feature requests by community member `Yusif Yusifli `_. Read discussions about feature requests below: `#163 `_. `#164 `_. `#210 `_. So basically how to take backups and create a tag for it? Taking full backup: ------------------- :: $ sudo autoxtrabackup --tag="My Full backup" -v \ -lf /home/shahriyar.rzaev/autoxtrabackup_2_4_5_7.log \ -l DEBUG --defaults-file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf --backup Taking incremental one: ----------------------- :: $ autoxtrabackup --tag="First incremental backup" -v \ -lf /home/shahriyar.rzaev/autoxtrabackup_2_4_5_7.log \ -l DEBUG --defaults-file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf --backup Taking second incremental: -------------------------- :: $ autoxtrabackup --tag="Second incremental backup" -v \ -lf /home/shahriyar.rzaev/autoxtrabackup_2_4_5_7.log \ -l DEBUG --defaults-file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf --backup To list available tags(backups): -------------------------------- For eg, if full backup failed, the result will be something like this: :: $ sudo autoxtrabackup --show_tags \ --defaults-file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf Backup Type Status Completion_time Size TAG ---------------------------------------------------------------------------------- 2017-12-14_12-01-11 Full FAILED 2017-12-14_12-01-11 4,0K 'My Full backup' backup_tags.txt file -------------------- All tags are stored inside backup_tags.txt file, which will be created in backup directory: :: [vagrant@localhost ps_5_7_x_2_4]$ ls backup_tags.txt full inc [vagrant@localhost ps_5_7_x_2_4]$ cat backup_tags.txt $ cat backup_tags.txt 2017-12-14_12-01-11 Full FAILED 2017-12-14_12-01-11 4,0K 'My Full backup' Preparing with tag ------------------ That's very nice. Now you can use those tags to prepare your backups. Say you want to prepare only first incremental and ignore second one(or others). :: $ autoxtrabackup --tag="First incremental backup" -v \ -lf /home/shahriyar.rzaev/autoxtrabackup_2_4_5_7.log \ -l DEBUG --defaults-file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf --prepare As you see it will mark given incremental backup as last one, because you have specified it in --tag option. **If you pass wrong/non-existing tag name the tool will raise RuntimeError.** ================================================ FILE: docs/basic_features.rst ================================================ Basic features ============== Backup ------ Yes you are right, this tool is for taking backups. It should take care for automating this process for you. You can specify the backup directory in config file (default ~/.autoxtrabackup/autoxtrabackup.cnf) under [Backup] category. So you have prepared your config and now you are ready for start. The command for taking full backup with DEBUG enabled, i.e first run of the tool. :: $ sudo autoxtrabackup -v -lf /home/shako/.autoxtrabackup/autoxtrabackup.log \ -l DEBUG --defaults-file=/home/shako/.autoxtrabackup/autoxtrabackup.cnf --backup The result of second run; it will take an incremental backup. :: $ sudo autoxtrabackup -v -lf /home/shako/.autoxtrabackup/autoxtrabackup.log \ -l DEBUG --defaults-file=/home/shako/.autoxtrabackup/autoxtrabackup.cnf --backup You will have 2 separate folders inside backup directory: :: (.venv) shako@shako-localhost:~/XB_TEST$ cd backup_dir/ (.venv) shako@shako-localhost:~/XB_TEST/backup_dir$ ls full inc We took full backup and it should be under the ``full`` directory: :: (.venv) shako@shako-localhost:~/XB_TEST/backup_dir$ ls full/ 2019-01-20_13-52-07 Incremental backups are inside ``inc`` directory: :: (.venv) shako@shako-localhost:~/XB_TEST/backup_dir$ ls inc/ 2019-01-20_13-53-59 If you want more incremental backups just run the same command again and again. Prepare ------- For preparing backups just use --prepare option. For our case we have a full and 2 incremental backups: :: (.venv) shako@shako-localhost:~/XB_TEST/backup_dir$ ls full/ 2019-01-20_13-52-07 (.venv) shako@shako-localhost:~/XB_TEST/backup_dir$ ls inc/ 2019-01-20_13-53-59 2019-01-20_13-56-42 All backups will be prepared automatically. You are going to have 3 options to choose: 1. Only prepare backups. 2. Prepare backups and restore immediately. 3. Restore from already prepared backup. For now let's choose 1: :: $ sudo `which autoxtrabackup` --prepare -v -l DEBUG That's it. Your backup is ready to restore/recovery. ================================================ FILE: docs/basic_overview.rst ================================================ Basic Overview ============== Project Structure ----------------- XtraBackup is a powerful open-source hot online backup tool for MySQL from Percona. This script is using XtraBackup for full and incremental backups, also for preparing backups, as well as to restore. Here is project path tree: :: * mysql_autoxtrabackup -- source directory * mysql_autoxtrabackup/backup_backup -- Full and Incremental backup taker script. * mysql_autoxtrabackup/backup_prepare -- Backup prepare and restore script. * mysql_autoxtrabackup/general_conf -- All-in-one config file's and config reader class folder. * mysql_autoxtrabackup/process_runner -- The directory for process runner script. * mysql_autoxtrabackup/autoxtrabackup.py -- Commandline Tool provider script. * mysql_autoxtrabackup/api -- The API server written in FastAPI. * tests -- The directory for test things. * ~/.autoxtrabackup/autoxtrabackup.cnf -- Config file will be created during setup. Available Options ----------------- .. code-block:: shell $ autoxtrabackup --help Usage: autoxtrabackup [OPTIONS] Options: --dry-run Enable the dry run. --prepare Prepare/recover backups. --run-server Start the FastAPI app for serving API --backup Take full and incremental backups. --version Version information. --defaults-file TEXT Read options from the given file [default: / home/shako/.autoxtrabackup/autoxtrabackup.cn f] --tag TEXT Pass the tag string for each backup --show-tags Show backup tags and exit -v, --verbose Be verbose (print to console) -lf, --log-file TEXT Set log file [default: /home/shako/.autoxtr abackup/autoxtrabackup.log] -l, --log, --log-level [DEBUG|INFO|WARNING|ERROR|CRITICAL] Set log level [default: INFO] --log-file-max-bytes INTEGER Set log file max size in bytes [default: 1073741824] --log-file-backup-count INTEGER Set log file backup count [default: 7] --help Print help message and exit. Usage ----- :: 1. Install it. 2. Edit ~/.autoxtrabackup/autoxtrabackup.cnf(default config) file to reflect your environment or create your own config. 3. Pass config file to autoxtrabackup with --defaults_file option(if you are not using default config) and begin to backup/prepare/restore. Logging -------- The logging mechanism is using Python3 logging. It lets to log directly to console and also to file. ================================================ FILE: docs/conf.py ================================================ # -*- coding: utf-8 -*- # # MySQL AutoXtrabackup documentation build configuration file, created by # sphinx-quickstart on Fri Feb 24 23:39:55 2017. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import os import sys # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # import sphinx_rtd_theme from mysql_autoxtrabackup.utils.version import VERSION sys.path.insert(0, os.path.abspath("../mysql_autoxtrabackup")) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. # # needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ "sphinx.ext.autodoc", "sphinx.ext.napoleon", "sphinx.ext.viewcode", ] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] source_suffix = ".rst" # The master toctree document. master_doc = "index" # General information about the project. project = u"MySQL-AutoXtrabackup" copyright = u"2020, Shahriyar Rzayev" author = u"Shahriyar Rzayev" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = u"{}".format(VERSION) # The full version, including alpha/beta/rc tags. release = u"{}".format(VERSION) # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = None # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = "sphinx_rtd_theme" html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # # html_theme_options = {} # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". # html_static_path = ['_static'] html_static_path = [] # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. htmlhelp_basename = "MySQLAutoXtrabackupdoc" # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # # 'preamble': '', # Latex figure (float) alignment # # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ ( master_doc, "MySQLAutoXtrabackup.tex", u"MySQL AutoXtrabackup Documentation", u"Shahriyar Rzayev", "manual", ), ] # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ( master_doc, "mysqlautoxtrabackup", u"MySQL AutoXtrabackup Documentation", [author], 1, ) ] # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ( master_doc, "MySQLAutoXtrabackup", u"MySQL AutoXtrabackup Documentation", author, "MySQLAutoXtrabackup", "One line description of project.", "Miscellaneous", ), ] ================================================ FILE: docs/config_file.rst ================================================ The structure of configuration file =================================== Defaults file explained ----------------------- There are some changes related to default config file. First and foremost it is renamed and now located in home of user in .autoxtrabackup folder. Keep in mind that config file is going to be dynamically generated during setup process. The default config will be located ~/.autoxtrabackup/autoxtrabackup.cnf. The available options are divided into optional and primary options. Options are quite self-explanatory. I have tried to make them similar to existing options in XtraBackup and MySQL. You can use another configuration file using ``--defaults_file`` option. Let's clarify the config file structure a bit. [MySQL] -------- The [MySQL] category is for specifying information about MySQL instance. :: [MySQL] mysql=/usr/bin/mysql mycnf=/etc/my.cnf mysqladmin=/usr/bin/mysqladmin mysql_user=root mysql_password=12345 ## Set either mysql_socket only, OR host + port. If both are set mysql_socket is used #mysql_socket=/var/lib/mysql/mysql.sock mysql_host=127.0.0.1 mysql_port=3306 datadir=/var/lib/mysql [Logging] --------- Options for logging mechanism of tool.(added in 1.5.4 version) :: [Logging] #[debug,info,warning,error,critical] log = DEBUG log_file_max_bytes = 1073741824 log_file_backup_count = 7 [Backup] -------- The [Backup] category is for specifying information about backup/prepare process itself. :: [Backup] #optional: set pid directory pid_dir = /tmp/MySQL-AutoXtraBackup tmp_dir = /home/shako/XB_TEST/mysql_datadirs #optional: set warning if pid of backup us running for longer than x pid_runtime_warning = 2 Hours backup_dir = /home/shako/XB_TEST/backup_dir backup_tool = /usr/bin/xtrabackup #optional: specify different path/version of xtrabackup here for prepare #prepare_tool = xtra_prepare = --apply-log-only #optional: pass additional options for backup stage #xtra_backup = --compact #optional: pass additional options for prepare stage #xtra_prepare_options = --rebuild-indexes #optional: pass general additional options; it will go to both for backup and prepare #xtra_options = --binlog-info=ON --galera-info #optional: set archive and rotation #archive_dir = /home/shako/XB_TEST/backup_archives #prepare_archive = 1 #move_archive = 0 #full_backup_interval = 1 day #archive_max_size = 100GiB #archive_max_duration = 4 Days #optional: warning(enable this if you want to take partial backups). specify database names or table names. #partial_list = test.t1 test.t2 dbtest +----------------------+----------+-----------------------------------------------------------------------------+ | **Key** | Required | **Description** | +----------------------+----------+-----------------------------------------------------------------------------+ | pid_dir | no | Directory where the PID file will be created in | +----------------------+----------+-----------------------------------------------------------------------------+ | tmp_dir | yes | Used for moving current running mysql-datadir to when copying-back | | | | (restoring) an archive | +----------------------+----------+-----------------------------------------------------------------------------+ | backup_dir | yes | Directory will be used for storing the backups. Subdirs ./full and ./inc | | | | will be created | +----------------------+----------+-----------------------------------------------------------------------------+ | backup_tool | yes | Full path to Percona xtrabackup executable used when making backup | +----------------------+----------+-----------------------------------------------------------------------------+ | prepare_tool | no | Full path to Percona xtrabackup executable used when preparing (restoring) | +----------------------+----------+-----------------------------------------------------------------------------+ | xtra_prepare | yes | Options passed to xtrabackup when preparing. | | | | '--apply-log-only' is essential to allow further incremental | | | | backups to be made. See[1] | +----------------------+----------+-----------------------------------------------------------------------------+ | xtra_backup | no | pass additional options for backup stage | +----------------------+----------+-----------------------------------------------------------------------------+ | xtra_prepare_options | no | pass additional options for prepare stage | +----------------------+----------+-----------------------------------------------------------------------------+ | xtra_options | no | pass general additional options; it will go to both for backup and prepare | +----------------------+----------+-----------------------------------------------------------------------------+ | archive_dir | no | Directory for storing archives (tar.gz or otherwise). Cannot be inside the | | | | 'backupdir' above | +----------------------+----------+-----------------------------------------------------------------------------+ | prepare_archive | no | Prepare backups before archiving them. | +----------------------+----------+-----------------------------------------------------------------------------+ | move_archive | no | When rotating backups to archive move instead of compressing with tar.gz | +----------------------+----------+-----------------------------------------------------------------------------+ | full_backup_interval | no | Maximum interval after which a new full backup will be made | +----------------------+----------+-----------------------------------------------------------------------------+ | archive_max_size | no | Delete archived backups after X GiB | +----------------------+----------+-----------------------------------------------------------------------------+ | archive_max_duration | no | Delete archived backups after X Days | +----------------------+----------+-----------------------------------------------------------------------------+ | partial_list | no | Specify database names or table names. | | | | **WARNING**: Enable this if you want to take partial backups | +----------------------+----------+-----------------------------------------------------------------------------+ [Compress] ---------- The [Compress] category is for enabling backup compression. The options will be passed to XtraBackup. :: [Compress] #optional #enable only if you want to use compression. compress = quicklz compress_chunk_size = 65536 compress_threads = 4 decompress = TRUE #enable if you want to remove .qp files after decompression.(Available from PXB 2.3.7 and 2.4.6) remove_original = FALSE [Encrypt] --------- The [Encrypt] category is for enabling backup encryption. The options will be passed to XtraBackup. :: [Encrypt] #optional #enable only if you want to create encrypted backups xbcrypt = /usr/bin/xbcrypt encrypt = AES256 #please note that --encrypt-key and --encrypt-key-file are mutually exclusive encrypt_key = VVTBwgM4UhwkTTV98fhuj+D1zyWoA89K #encrypt_key_file = /path/to/file/with_encrypt_key encrypt_threads = 4 encrypt_chunk_size = 65536 decrypt = AES256 #enable if you want to remove .qp files after decompression.(Available from PXB 2.3.7 and 2.4.6) remove_original = FALSE [Xbstream] ---------- The [Xbstream] category is for enabling backup streaming. The options will be passed to XtraBackup. :: [Xbstream] #experimental #enable this, if you want to stream your backups xbstream = /usr/bin/xbstream stream = xbstream xbstream_options = -x --parallel=100 xbs_decrypt = 1 # warn, enable this, if you want to stream your backups to remote host #remote_stream = ssh xxx.xxx.xxx.xxx Deprecated feature, will be removed in next releases[Do not use] :: #Optional remote syncing #[Remote] #remote_conn=root@xxx.xxx.xxx.xxx #remote_dir=/home/sh/Documents [Commands] ---------- The [Commands] category is for specifying some options for copy-back/restore actions. :: [Commands] start_mysql_command=service mysql start stop_mysql_command=service mysql stop #Change user:group respectively chown_command=chown -R mysql:mysql [1]: https://www.percona.com/doc/percona-xtrabackup/LATEST/xtrabackup_bin/incremental_backups.html#preparing-the-incremental-backups ================================================ FILE: docs/index.rst ================================================ .. MySQL AutoXtrabackup documentation master file, created by sphinx-quickstart on Fri Feb 24 23:39:55 2017. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. Welcome to MySQL-AutoXtrabackup's documentation! ================================================ .. toctree:: :maxdepth: 2 what_is_new_in_2_0 intro basic_overview installation config_file basic_features backup_tags advance_features option_reference api .. automodule:: backup_backup :members: .. automodule:: backup_prepare :members: .. automodule:: general_conf :members: .. automodule:: process_runner :members: .. automodule:: utils :members: .. automodule:: api :members: Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` ================================================ FILE: docs/installation.rst ================================================ Installation ============ System requirements ------------------- Following packages should be already there: - Percona Xtrabackup (>= 2.3.5) - Python 3 (>= 3.6) Installing MySQL-AutoXtraBackup ------------------------------- Using pip3: :: pip3 install mysql-autoxtrabackup Installing from source for development purposes: :: git clone https://github.com/ShahriyarR/MySQL-AutoXtraBackup.git cd to MySQL-AutoXtraBackup flit install --symlink As Percona XtraBackup requires root privileges in order to backup the MySQL server, it is convenient to install, autoxtrabackup globally. But, if you think no, then install it to virtualenv and then call as root/sudo :) ================================================ FILE: docs/intro.rst ================================================ Intro ===== What is this? ------------- MySQL-AutoXtraBackup is a commandline tool written in Python3 based on Percona XtraBackup. It is aimed to simplify the usage of XtraBackup in daily basis and to help backup admin in certain tasks. Why you need this? ------------------ The idea for this tool, came from my hard times after accidentally deleting the table data. There was a full backup and 12 incremental backups. It took me 20 minutes to prepare necessary commands for preparing backups. If you have compressed + encrypted backups you need also, decrypt and decompress, which is going to add extra time for preparing backups. Then I decided to automate this process. In other words, preparing necessary commands for backup and prepare stage were automated. If you think that those reasons are not enough - Then just believe me you need this :) Is it production ready? ----------------------- Well, we have famous answer for this - "It depends!". Basically this tool is based on Percona XtraBackup and it is using XtraBackup's functionality. For me, I have used it in production environment after testing for a while in test servers. But to be clear, just test it enough to be confident. ================================================ FILE: docs/option_reference.rst ================================================ Option Reference ================= The command line options to use: .. code-block:: shell $ autoxtrabackup --help Usage: autoxtrabackup [OPTIONS] Options: --dry-run Enable the dry run. --prepare Prepare/recover backups. --run-server Start the FastAPI app for serving API --backup Take full and incremental backups. --version Version information. --defaults-file TEXT Read options from the given file [default: / home/shako/.autoxtrabackup/autoxtrabackup.cn f] --tag TEXT Pass the tag string for each backup --show-tags Show backup tags and exit -v, --verbose Be verbose (print to console) -lf, --log-file TEXT Set log file [default: /home/shako/.autoxtr abackup/autoxtrabackup.log] -l, --log, --log-level [DEBUG|INFO|WARNING|ERROR|CRITICAL] Set log level [default: INFO] --log-file-max-bytes INTEGER Set log file max size in bytes [default: 1073741824] --log-file-backup-count INTEGER Set log file backup count [default: 7] --help Print help message and exit. dry-run ------- --dry-run Use this option to enable dry running. If you enable this, actual commands will not be executed but, will only be shown. It is useful, when you want to see what is going to happen, if you run the actual process. prepare ------- --prepare This option is for prepare and copy-back(recover) the backup. run-server ---------- --run-server This option is for starting API server[Defaults are localhost and port 5555] backup ------ --backup This option for taking backups. If it is first run, it will take full backup. If you want incremental backups, just run same command as much as you want take incremental backups. version ------- --version Prints version information. defaults-file ------------- --defaults-file The main config file to path to ``autoxtrabackup``. The default one is ``~/.autoxtrabackup/autoxtrabackup.cnf``. In default config, the compression, encryption and streaming backups are disabled by defualt. tag ---- --tag This option enables creation of tags for backups. The backup_tags.txt file will be created and stored inside backup directory. show-tags --------- It will show the backup tags and exit. verbose, v ---------- --verbose, -v This option enables to print to console the logging messages. log-file, lf ------------ -lf, --log-file Pass, the path for log file, for autoxtrabackup. Default is ``~/.autoxtrabackup/autoxtrabackup.log`` log-file-backup_count --------------------- --log_file_backup_count Set log file backup count. Default is 7 log-file-max-bytes ------------------ --log_file_max_bytes Set log file max size in bytes. Default: 1073741824 bytes. log, log-level -------------- -l, --log, --log-level Set the log level for tool. Can be DEBUG, INFO, WARNING, ERROR or CRITICAL. Default is DEBUG. help ---- --help As name indicates. ================================================ FILE: docs/what_is_new_in_2_0.rst ================================================ What is new in >= 2.0 major version? ==================================== Let me put it concise --------------------- The whole project was completely rewritten. It is incompatible with previous versions. Just imagine nearly ~6754 lines of code was thrown and only ~3200 was added. Added mypy, black, flake8 and now our project is fully typed and has support with type checkers. Removed Pipenv, setup.py and added pyproject.yaml with flit support. There is a scripts folder with helper scripts if you decide to contribute. We have experimental API built with FastAPI - to operate on backups using API calls.(experimental) ================================================ FILE: mypy.ini ================================================ [mypy] # --strict disallow_any_generics = True disallow_subclassing_any = True disallow_untyped_calls = True disallow_untyped_defs = True disallow_incomplete_defs = True check_untyped_defs = True disallow_untyped_decorators = True no_implicit_optional = True warn_redundant_casts = True warn_unused_ignores = True warn_return_any = True implicit_reexport = False strict_equality = True # --strict end ================================================ FILE: mysql_autoxtrabackup/__init__.py ================================================ """MySQL-AutoXtrabackup command-line tool, for automating tedious MySQL physical backups management using Percona Xtrabackup""" from .utils import version __version__ = version.VERSION ================================================ FILE: mysql_autoxtrabackup/api/__init__.py ================================================ ================================================ FILE: mysql_autoxtrabackup/api/controller/__init__.py ================================================ ================================================ FILE: mysql_autoxtrabackup/api/controller/controller.py ================================================ from fastapi import APIRouter, status from fastapi.responses import JSONResponse from starlette.responses import RedirectResponse from mysql_autoxtrabackup.backup_backup.backuper import Backup from mysql_autoxtrabackup.backup_prepare.prepare import Prepare from mysql_autoxtrabackup.utils.helpers import list_available_backups router = APIRouter() @router.get( "/", tags=["MySQL-AutoXtrabackup"], response_description="Home Page", description="Redirecting to DOC", include_in_schema=False, ) async def home() -> RedirectResponse: return RedirectResponse(url="/docs") @router.post( "/backup", tags=["MySQL-AutoXtrabackup"], response_description="Json response ", description="Run all backup process", ) async def backup() -> JSONResponse: backup_ = Backup() result = backup_.all_backup() if result: return JSONResponse( content={"result": "Successfully finished the backup process"}, status_code=status.HTTP_201_CREATED, ) return JSONResponse( content={"result": "[FAILED] to take backup"}, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @router.post( "/prepare", tags=["MySQL-AutoXtrabackup"], response_description="Json response", description="Prepare all backups", ) async def prepare() -> JSONResponse: prepare_ = Prepare() result = prepare_.prepare_inc_full_backups() if result: return JSONResponse( content={"result": "Successfully prepared all the backups"}, status_code=status.HTTP_200_OK, ) return JSONResponse( content={"result": "[FAILED] to prepare backup"}, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @router.get( "/backups", tags=["MySQL-AutoXtrabackup"], response_description="Json response", description="List all available backups", ) async def backups() -> JSONResponse: backup_ = Backup() result = list_available_backups( str(backup_.builder_obj.backup_options.get("backup_dir")) ) if result: return JSONResponse(content={"backups": result}, status_code=status.HTTP_200_OK) return JSONResponse(content={"backups": {}}, status_code=status.HTTP_200_OK) @router.delete( "/delete", tags=["MySQL-AutoXtrabackup"], response_description="Json response", description="Remove all available backups", ) async def delete() -> JSONResponse: backup_ = Backup() delete_full = backup_.clean_full_backup_dir(remove_all=True) delete_inc = backup_.clean_inc_backup_dir() if delete_full and delete_inc: return JSONResponse( content={"result": "There is no backups or backups removed successfully"}, status_code=status.HTTP_200_OK, ) return JSONResponse( content={"result": "[FAILED] to remove/delete available backups"}, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) ================================================ FILE: mysql_autoxtrabackup/api/main.py ================================================ from typing import Any, Dict, Optional import uvicorn # type: ignore from fastapi import FastAPI from fastapi.openapi.utils import get_openapi from mysql_autoxtrabackup.api.controller.controller import router from mysql_autoxtrabackup.utils.version import VERSION app = FastAPI() @app.on_event("startup") async def startup() -> None: """startup.""" print("app started") @app.on_event("shutdown") async def shutdown() -> None: """shutdown.""" print("SHUTDOWN") def modify_openapi() -> Dict[str, Any]: """modify_openapi.""" if app.openapi_schema: return app.openapi_schema openapi_schema = get_openapi( title="MySQL-AutoXtrabackup", version=f"{VERSION}", description="Rest API doc for MySQL-AutoXtrabackup", routes=app.routes, ) app.openapi_schema = openapi_schema return app.openapi_schema app.openapi = modify_openapi # type: ignore app.include_router(router) def run_server(host: Optional[str] = None, port: Optional[int] = None) -> None: host = host or "127.0.0.1" port = port or 5555 uvicorn.run(app, host=host, port=port) if __name__ == "__main__": run_server() ================================================ FILE: mysql_autoxtrabackup/autoxtrabackup.py ================================================ import logging import logging.handlers import os import re import time from logging.handlers import RotatingFileHandler from sys import exit from sys import platform as _platform from typing import Optional import click import humanfriendly # type: ignore import pid # type: ignore from mysql_autoxtrabackup.api import main from mysql_autoxtrabackup.backup_backup.backuper import Backup from mysql_autoxtrabackup.backup_prepare.prepare import Prepare from mysql_autoxtrabackup.general_conf import path_config from mysql_autoxtrabackup.general_conf.generalops import GeneralClass from mysql_autoxtrabackup.process_runner.process_runner import ProcessRunner from mysql_autoxtrabackup.utils import version logger = logging.getLogger("") destinations_hash = { "linux": "/dev/log", "linux2": "/dev/log", "darwin": "/var/run/syslog", } def address_matcher(plt: str) -> str: return destinations_hash.get(plt, ("localhost", 514)) # type: ignore handler = logging.handlers.SysLogHandler(address=address_matcher(_platform)) # Set syslog for the root logger logger.addHandler(handler) def print_help(ctx: click.Context, param: None, value: bool) -> None: if not value: return click.echo(ctx.get_help()) ctx.exit() def print_version(ctx: click.Context, param: None, value: bool) -> None: if not value or ctx.resilient_parsing: return click.echo("Developed by Shahriyar Rzayev from Azerbaijan PUG(http://azepug.az)") click.echo("Link : https://github.com/ShahriyarR/MySQL-AutoXtraBackup") click.echo("Email: rzayev.sehriyar@gmail.com") click.echo( "Based on Percona XtraBackup: https://github.com/percona/percona-xtrabackup/" ) click.echo(f"MySQL-AutoXtraBackup Version: {version.VERSION}") ctx.exit() def check_file_content(file: str) -> Optional[bool]: """Check if all mandatory headers and keys exist in file""" with open(file, "r") as config_file: file_content = config_file.read() config_headers = ["MySQL", "Backup", "Encrypt", "Compress", "Commands"] config_keys = [ "mysql", "mycnf", "mysqladmin", "mysql_user", "mysql_password", "mysql_host", "datadir", "tmp_dir", "backup_dir", "backup_tool", "xtra_prepare", "start_mysql_command", "stop_mysql_command", "chown_command", ] for header in config_headers: if header not in file_content: raise KeyError("Mandatory header [%s] doesn't exist in %s" % (header, file)) for key in config_keys: if key not in file_content: raise KeyError("Mandatory key '%s' doesn't exists in %s." % (key, file)) return True def validate_file(file: str) -> Optional[bool]: """ Check for validity of the file given in file path. If file doesn't exist or invalid configuration file, throw error. """ if not os.path.isfile(file): raise FileNotFoundError("Specified file does not exist.") # filename extension should be .cnf pattern = re.compile(r".*\.cnf") if pattern.match(file): # Lastly the file should have all 5 required headers if check_file_content(file): return None else: raise ValueError("Invalid file extension. Expecting .cnf") return None @click.command() @click.option("--dry-run", is_flag=True, help="Enable the dry run.") @click.option("--prepare", is_flag=True, help="Prepare/recover backups.") @click.option( "--run-server", is_flag=True, help="Start the FastAPI app for serving API" ) @click.option("--backup", is_flag=True, help="Take full and incremental backups.") @click.option( "--version", is_flag=True, callback=print_version, # type: ignore expose_value=False, is_eager=True, help="Version information.", ) @click.option( "--defaults-file", default=path_config.config_path_file, # type: ignore show_default=True, help="Read options from the given file", ) @click.option("--tag", help="Pass the tag string for each backup") @click.option("--show-tags", is_flag=True, help="Show backup tags and exit") @click.option("-v", "--verbose", is_flag=True, help="Be verbose (print to console)") @click.option( "-lf", "--log-file", default=path_config.log_file_path, show_default=True, help="Set log file", ) @click.option( "-l", "--log", "--log-level", default="INFO", show_default=True, type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]), help="Set log level", ) @click.option( "--log-file-max-bytes", default=1073741824, show_default=True, nargs=1, type=int, help="Set log file max size in bytes", ) @click.option( "--log-file-backup-count", default=7, show_default=True, nargs=1, type=int, help="Set log file backup count", ) @click.option( "--help", is_flag=True, callback=print_help, # type: ignore expose_value=False, is_eager=False, help="Print help message and exit.", ) @click.pass_context def all_procedure( ctx, prepare, backup, run_server, tag, show_tags, verbose, log_file, log, defaults_file, dry_run, log_file_max_bytes, log_file_backup_count, ): options = GeneralClass(defaults_file) logging_options = options.logging_options backup_options = options.backup_options formatter = logging.Formatter( fmt="%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s", datefmt="%Y-%m-%d %H:%M:%S", ) if verbose: ch = logging.StreamHandler() # control console output log level ch.setLevel(logging.INFO) ch.setFormatter(formatter) logger.addHandler(ch) if log_file: try: if logging_options.get("log_file_max_bytes") and logging_options.get( "log_file_backup_count" ): file_handler = RotatingFileHandler( log_file, mode="a", maxBytes=int(str(logging_options.get("log_file_max_bytes"))), backupCount=int(str(logging_options.get("log_file_backup_count"))), ) else: file_handler = RotatingFileHandler( log_file, mode="a", maxBytes=log_file_max_bytes, backupCount=log_file_backup_count, ) file_handler.setFormatter(formatter) logger.addHandler(file_handler) except PermissionError as err: exit("{} Please consider to run as root or sudo".format(err)) # set log level in order: 1. user argument 2. config file 3. @click default if log is not None: logger.setLevel(log) elif logging_options.get("log_level"): logger.setLevel(str(logging_options.get("log_level"))) else: # this is the fallback default log-level. logger.setLevel("INFO") validate_file(defaults_file) pid_file = pid.PidFile(piddir=backup_options.get("pid_dir")) try: with pid_file: # User PidFile for locking to single instance dry_run_ = dry_run if dry_run_: dry_run_ = 1 logger.warning("Dry run enabled!") if ( prepare is False and backup is False and verbose is False and dry_run is False and show_tags is False and run_server is False ): print_help(ctx, None, value=True) elif run_server: main.run_server() elif show_tags and defaults_file: backup_ = Backup(config=defaults_file) backup_.show_tags(backup_dir=str(backup_options.get("backup_dir"))) elif prepare: prepare_ = Prepare(config=defaults_file, dry_run=dry_run_, tag=tag) prepare_.prepare_backup_and_copy_back() elif backup: backup_ = Backup(config=defaults_file, dry_run=dry_run_, tag=tag) backup_.all_backup() except (pid.PidFileAlreadyLockedError, pid.PidFileAlreadyRunningError) as error: if float( str(backup_options.get("pid_runtime_warning")) ) and time.time() - os.stat(pid_file.filename).st_ctime > float( str(backup_options.get("pid_runtime_warning")) ): pid.fh.seek(0) pid_str = pid.fh.read(16).split("\n", 1)[0].strip() logger.warning( "Pid file already exists or Pid already running! : ", str(error) ) logger.critical( "Backup (pid: " + pid_str + ") has been running for logger than: " + str( humanfriendly.format_timespan( backup_options.get("pid_runtime_warning") ) ) ) except pid.PidFileUnreadableError as error: logger.warning("Pid file can not be read: " + str(error)) except pid.PidFileError as error: logger.warning("Generic error with pid file: " + str(error)) logger.info("Xtrabackup command history:") for i in ProcessRunner.xtrabackup_history_log: logger.info(str(i)) logger.info("Autoxtrabackup completed successfully!") return True if __name__ == "__main__": all_procedure() ================================================ FILE: mysql_autoxtrabackup/backup_backup/__init__.py ================================================ from .backup_archive import BackupArchive as BackupArchive from .backup_builder import BackupBuilderChecker as BackupBuilderChecker from .backuper import Backup as Backup ================================================ FILE: mysql_autoxtrabackup/backup_backup/backup_archive.py ================================================ import logging import os import shutil from datetime import datetime from typing import Union from mysql_autoxtrabackup.backup_backup.backup_builder import BackupBuilderChecker from mysql_autoxtrabackup.general_conf import path_config from mysql_autoxtrabackup.general_conf.generalops import GeneralClass from mysql_autoxtrabackup.process_runner.process_runner import ProcessRunner from mysql_autoxtrabackup.utils import helpers logger = logging.getLogger(__name__) class BackupArchive: def __init__( self, config: str = path_config.config_path_file, dry_run: Union[bool, None] = None, tag: Union[str, None] = None, ) -> None: self.conf = config self.dry = dry_run self.tag = tag options_obj = GeneralClass(config=self.conf) self.backup_options = BackupBuilderChecker( config=self.conf, dry_run=self.dry ).backup_options self.backup_archive_options = options_obj.backup_archive_options def create_backup_archives(self) -> bool: from mysql_autoxtrabackup.backup_prepare.prepare import Prepare # Creating .tar.gz archive files of taken backups file_list = os.listdir(str(self.backup_options.get("full_dir"))) for i in file_list: if len(file_list) == 1 or i != max(file_list): logger.info("Preparing backups prior archiving them...") if self.backup_archive_options.get("prepare_archive"): logger.info("Started to prepare backups, prior archiving!") prepare_obj = Prepare( config=self.conf, dry_run=self.dry, tag=self.tag ) status = prepare_obj.prepare_inc_full_backups() if status: logger.info( "Backups Prepared successfully... {}".format(status) ) if self.backup_archive_options.get("move_archive") and ( int(str(self.backup_archive_options.get("move_archive"))) == 1 ): dir_name = ( str(self.backup_archive_options.get("archive_dir")) + "/" + i + "_archive" ) logger.info( "move_archive enabled. Moving {} to {}".format( self.backup_options.get("backup_dir"), dir_name ) ) try: shutil.copytree( str(self.backup_options.get("backup_dir")), dir_name ) except Exception as err: logger.error("FAILED: Move Archive") logger.error(err) raise else: return True else: logger.info( "move_archive is disabled. archiving / compressing current_backup." ) # Multi-core tar utilizing pigz. # Pigz default to number of cores available, or 8 if cannot be read. # Test if pigz is available. logger.info("testing for pigz...") status = ProcessRunner.run_command("pigz --version") archive_file = ( str(self.backup_archive_options.get("archive_dir")) + "/" + i + ".tar.gz" ) if status: logger.info("Found pigz...") # run_tar = "tar cvvf - {} {} | pigz -v > {}" \ run_tar = ( "tar --use-compress-program=pigz -cvf {} {} {}".format( archive_file, self.backup_options.get("full_dir"), self.backup_options.get("inc_dir"), ) ) else: # handle file not found error. logger.warning( "pigz executeable not available. Defaulting to singlecore tar" ) run_tar = "tar -zcf {} {} {}".format( archive_file, self.backup_options.get("full_dir"), self.backup_options.get("inc_dir"), ) status = ProcessRunner.run_command(run_tar) if status: logger.info( "OK: Old full backup and incremental backups archived!" ) return True logger.error("FAILED: Archiving ") raise RuntimeError("FAILED: Archiving -> {}".format(run_tar)) return True def clean_old_archives(self) -> None: logger.info("Starting cleaning of old archives") archive_dir = str(self.backup_archive_options.get("archive_dir")) # Finding if last full backup older than the interval or more from now! cleanup_msg = "Removing archive {}/{} due to {}" for archive in helpers.sorted_ls(archive_dir): if "_archive" in archive: archive_date = datetime.strptime(archive, "%Y-%m-%d_%H-%M-%S_archive") else: archive_date = datetime.strptime(archive, "%Y-%m-%d_%H-%M-%S.tar.gz") now = datetime.now() if ( self.backup_archive_options.get("archive_max_duration") or self.backup_archive_options.get("archive_max_size") ) and ( float((now - archive_date).total_seconds()) >= float(str(self.backup_archive_options.get("archive_max_duration"))) or float(helpers.get_directory_size(archive_dir)) > float(str(self.backup_archive_options.get("archive_max_size"))) ): logger.info( cleanup_msg.format( archive_dir, archive, "archive_max_duration exceeded." ) ) logger.info("OR") logger.info( cleanup_msg.format( archive_dir, archive, "archive_max_size exceeded." ) ) full_archive_path = os.path.join(archive_dir, archive) if os.path.isdir(full_archive_path): shutil.rmtree(full_archive_path) else: os.remove(full_archive_path) ================================================ FILE: mysql_autoxtrabackup/backup_backup/backup_builder.py ================================================ # Will store necessary checks and command building actions here import logging from os.path import isfile from typing import Optional, Union from mysql_autoxtrabackup.general_conf import path_config from mysql_autoxtrabackup.general_conf.generalops import GeneralClass from mysql_autoxtrabackup.process_runner.process_runner import ProcessRunner logger = logging.getLogger(__name__) class BackupBuilderChecker: # General pre-backup checking/extracting/untar/streaming etc. should happen here def __init__( self, config: str = path_config.config_path_file, dry_run: Union[bool, None] = None, ) -> None: self.conf = config self.dry = dry_run options_obj = GeneralClass(config=self.conf) self.mysql_options = options_obj.mysql_options self.compression_options = options_obj.compression_options self.encryption_options = options_obj.encryption_options self.backup_options = options_obj.backup_options self.xbstream_options = options_obj.xbstream_options def general_command_builder(self) -> str: """ Method for building general options for backup command. :return: String of constructed options. """ args = "" if self.mysql_options.get("mysql_socket"): args += " --socket={}".format(self.mysql_options.get("mysql_socket")) else: args += " --host={} --port={}".format( self.mysql_options.get("mysql_host"), self.mysql_options.get("mysql_port"), ) # Adding compression support for backup if ( self.compression_options.get("compress") and self.compression_options.get("compress_chunk_size") and self.compression_options.get("compress_threads") ): args += ( " --compress={}" " --compress-chunk-size={}" " --compress-threads={}".format( self.compression_options.get("compress"), self.compression_options.get("compress_chunk_size"), self.compression_options.get("compress_threads"), ) ) # Adding encryption support for full backup if self.encryption_options.get("encrypt"): args += ( " --encrypt={}" " --encrypt-threads={}" " --encrypt-chunk-size={}".format( self.encryption_options.get("encrypt"), self.encryption_options.get("encrypt_threads"), self.encryption_options.get("encrypt_chunk_size"), ) ) if self.encryption_options.get("encrypt_key"): if self.encryption_options.get("encrypt_key_file"): raise AttributeError( "--encrypt-key and --encrypt-key-file are mutually exclusive" ) args += " --encrypt-key={}".format( self.encryption_options.get("encrypt_key") ) elif self.encryption_options.get("encrypt_key_file"): args += " --encrypt-key-file={}".format( self.encryption_options.get("encrypt_key_file") ) # Checking if extra options were passed: if self.backup_options.get("xtra_options"): args += " {}".format(self.backup_options.get("xtra_options")) # Checking if extra backup options were passed: if self.backup_options.get("xtra_backup"): args += " {}".format(self.backup_options.get("xtra_backup")) # Checking if partial recovery list is available if self.backup_options.get("partial_list"): logger.warning("Partial Backup is enabled!") args += ' --databases="{}"'.format(self.backup_options.get("partial_list")) return args def extract_decrypt_from_stream_backup( self, recent_full_bck: Optional[str] = None, recent_inc_bck: Optional[str] = None, flag: Optional[bool] = None, ) -> None: """ Method for extracting and if necessary decrypting from streamed backup. If the recent_full_bck passed then it means you want to extract the full backup. If the recent_int_bck passed then it means you want to extract the inc backup. """ # Extract and decrypt streamed full backup prior to executing incremental backup file_name = "{}/{}/inc_backup.stream".format( self.backup_options.get("inc_dir"), recent_inc_bck ) file_place_holder = "< {} -C {}/{}".format( file_name, self.backup_options.get("inc_dir"), recent_inc_bck ) if not recent_inc_bck: file_name = "{}/{}/full_backup.stream".format( self.backup_options.get("full_dir"), recent_full_bck ) file_place_holder = "< {} -C {}/{}".format( file_name, self.backup_options.get("full_dir"), recent_full_bck ) xbstream_command = None if self.xbstream_options.get("stream") == "xbstream": xbstream_command = "{} {}".format( self.xbstream_options.get("xbstream"), self.xbstream_options.get("xbstream_options"), ) if ( self.encryption_options.get("encrypt") and self.xbstream_options.get("xbs_decrypt") and not flag ): logger.info( "Using xbstream to extract and decrypt from {}".format(file_name) ) xbstream_command += ( " --decrypt={} --encrypt-key={} --encrypt-threads={} ".format( self.encryption_options.get("decrypt"), self.encryption_options.get("encrypt_key"), self.encryption_options.get("encrypt_threads"), ) ) if xbstream_command: xbstream_command += file_place_holder logger.info( "The following xbstream command will be executed {}".format( xbstream_command ) ) if self.dry == 0 and isfile(file_name): ProcessRunner.run_command(xbstream_command) def stream_encrypt_compress_tar_checker(self) -> None: if self.xbstream_options.get("stream") == "tar" and ( self.encryption_options.get("encrypt") or self.compression_options.get("compress") ): logger.error( "xtrabackup: error: compressed and encrypted backups are " "incompatible with the 'tar' streaming format. Use --stream=xbstream instead." ) raise RuntimeError( "xtrabackup: error: compressed and encrypted backups are " "incompatible with the 'tar' streaming format. Use --stream=xbstream instead." ) def stream_tar_incremental_checker(self) -> None: if self.xbstream_options.get("stream") == "tar": logger.error( "xtrabackup: error: streaming incremental backups are incompatible with the " "'tar' streaming format. Use --stream=xbstream instead." ) raise RuntimeError( "xtrabackup: error: streaming incremental backups are incompatible with the " "'tar' streaming format. Use --stream=xbstream instead." ) def full_backup_command_builder(self, full_backup_dir: str) -> str: """ Method for creating Full Backup command. :param full_backup_dir the path of backup directory """ xtrabackup_cmd = ( "{} --defaults-file={} --user={} --password={} " " --target-dir={} --backup".format( self.backup_options.get("backup_tool"), self.mysql_options.get("mycnf"), self.mysql_options.get("mysql_user"), self.mysql_options.get("mysql_password"), full_backup_dir, ) ) # Calling general options/command builder to add extra options xtrabackup_cmd += self.general_command_builder() stream = self.backup_options.get("stream") if stream: logger.warning("Streaming is enabled!") xtrabackup_cmd += ' --stream="{}"'.format(stream) if stream == "xbstream": xtrabackup_cmd += " > {}/full_backup.stream".format(full_backup_dir) elif stream == "tar": xtrabackup_cmd += " > {}/full_backup.tar".format(full_backup_dir) return xtrabackup_cmd def inc_backup_command_builder( self, recent_full_bck: Optional[str], inc_backup_dir: Optional[str], recent_inc_bck: Optional[str] = None, ) -> str: xtrabackup_inc_cmd_base = ( "{} --defaults-file={} --user={} --password={}".format( self.backup_options.get("backup_tool"), self.mysql_options.get("mycnf"), self.mysql_options.get("mysql_user"), self.mysql_options.get("mysql_password"), ) ) if not recent_inc_bck: xtrabackup_inc_cmd_base += ( " --target-dir={} --incremental-basedir={}/{} --backup".format( inc_backup_dir, self.backup_options.get("full_dir"), recent_full_bck ) ) else: xtrabackup_inc_cmd_base += ( " --target-dir={} --incremental-basedir={}/{} --backup".format( inc_backup_dir, self.backup_options.get("inc_dir"), recent_inc_bck ) ) # Calling general options/command builder to add extra options xtrabackup_inc_cmd_base += self.general_command_builder() # Checking if streaming enabled for backups # There is no need to check for 'tar' streaming type -> see the method: stream_tar_incremental_checker() if ( hasattr(self, "stream") and self.xbstream_options.get("stream") == "xbstream" ): xtrabackup_inc_cmd_base += ' --stream="{}"'.format( self.xbstream_options.get("stream") ) xtrabackup_inc_cmd_base += " > {}/inc_backup.stream".format(inc_backup_dir) logger.warning("Streaming xbstream is enabled!") return xtrabackup_inc_cmd_base def decrypter( self, recent_full_bck: Optional[str] = None, xtrabackup_inc_cmd: Optional[str] = None, recent_inc_bck: Optional[str] = None, ) -> None: logger.info("Applying workaround for LP #1444255") logger.info("See more -> https://jira.percona.com/browse/PXB-934") # With recent PXB 8 it seems to be there is no need for this workaround. # Due to this moving this feature to this method and keeping just in case. # Deprecated as hell. if "encrypt" not in xtrabackup_inc_cmd: # type: ignore return if not isfile( "{}/{}/xtrabackup_checkpoints.xbcrypt".format( self.backup_options.get("full_dir"), recent_full_bck ) ): logger.info("Skipping...") return xbcrypt_command = "{} -d -k {} -a {}".format( self.encryption_options.get("xbcrypt"), self.encryption_options.get("encrypt_key"), self.encryption_options.get("encrypt"), ) xbcrypt_command_extra = ( " -i {}/{}/xtrabackup_checkpoints.xbcrypt -o {}/{}/xtrabackup_checkpoints" ) xbcrypt_command += xbcrypt_command_extra.format( self.backup_options.get("full_dir"), recent_full_bck, self.backup_options.get("full_dir"), recent_full_bck, ) if recent_inc_bck: if not isfile( "{}/{}/xtrabackup_checkpoints.xbcrypt".format( self.backup_options.get("inc_dir"), recent_inc_bck ) ): logger.info("Skipping...") return xbcrypt_command += xbcrypt_command_extra.format( self.backup_options.get("inc_dir"), recent_inc_bck, self.backup_options.get("inc_dir"), recent_inc_bck, ) logger.info( "The following xbcrypt command will be executed {}".format(xbcrypt_command) ) if self.dry == 0: ProcessRunner.run_command(xbcrypt_command) ================================================ FILE: mysql_autoxtrabackup/backup_backup/backuper.py ================================================ # MySQL Backuper Script using Percona Xtrabackup # Originally Developed by # Shahriyar Rzayev (Shako)-> https://mysql.az/ https://azepug.az/ # / rzayev.sehriyar@gmail.com / rzayev.shahriyar@yandex.com # This comment is from 2014 - keeping it here import logging import os import shutil import time from datetime import datetime from typing import Optional, Union from mysql_autoxtrabackup.backup_backup.backup_archive import BackupArchive from mysql_autoxtrabackup.backup_backup.backup_builder import BackupBuilderChecker from mysql_autoxtrabackup.general_conf import path_config from mysql_autoxtrabackup.general_conf.check_env import CheckEnv from mysql_autoxtrabackup.process_runner.process_runner import ProcessRunner from mysql_autoxtrabackup.utils import helpers, mysql_cli logger = logging.getLogger(__name__) class Backup: def __init__( self, config: str = path_config.config_path_file, dry_run: Union[bool, None] = None, tag: Union[str, None] = None, ) -> None: self.conf = config self.dry = dry_run self.tag = tag self.mysql_cli = mysql_cli.MySQLClientHelper(config=self.conf) self.builder_obj = BackupBuilderChecker(config=self.conf, dry_run=self.dry) self.archive_obj = BackupArchive( config=self.conf, dry_run=self.dry, tag=self.tag ) def add_tag( self, backup_type: str, backup_size: Optional[str], backup_status: Optional[str] ) -> bool: """ Method for adding backup tags :param backup_type: The backup type - Full/Inc :param backup_size: The size of the backup in human readable format :param backup_status: Status: OK or Status: Failed :return: True if no exception """ # skip tagging unless self.tag if not self.tag: logger.info("TAGGING SKIPPED") return True # Currently only support Inc and Full types, calculate name based on this assert backup_type in ( "Full", "Inc", ), "add_tag(): backup_type {}: must be 'Full' or 'Inc'".format(backup_type) backup_name = ( helpers.get_latest_dir_name( str(self.builder_obj.backup_options.get("full_dir")) ) if backup_type == "Full" else helpers.get_latest_dir_name( str(self.builder_obj.backup_options.get("inc_dir")) ) ) # Calculate more tag fields, create string backup_timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") backup_tag_str = ( "{bk_name}\t{bk_type}\t{bk_status}\t{bk_timestamp}\t{bk_size}\t'{bk_tag}'\n" ) # Apply tag with open( "{}/backup_tags.txt".format( self.builder_obj.backup_options.get("backup_dir") ), "a", ) as backup_tags_file: backup_tag_final = backup_tag_str.format( bk_name=backup_name, bk_type=backup_type, bk_status=backup_status, bk_timestamp=backup_timestamp, bk_size=backup_size, bk_tag=self.tag, ) backup_tags_file.write(backup_tag_final) return True @staticmethod def show_tags(backup_dir: str, tag_file: Optional[str] = None) -> Optional[bool]: tag_file = tag_file or "{}/backup_tags.txt".format(backup_dir) if os.path.isfile(tag_file): with open("{}/backup_tags.txt".format(backup_dir), "r") as backup_tags: from_file = backup_tags.read() column_names = "{0}\t{1}\t{2}\t{3}\t{4}\tTAG\n".format( "Backup".ljust(19), "Type".ljust(4), "Status".ljust(2), "Completion_time".ljust(19), "Size", ) extra_str = "{}\n".format("-" * (len(column_names) + 21)) print(column_names + extra_str + from_file) logger.info(column_names + extra_str + from_file) return True else: logger.warning( "Could not find backup_tags.txt inside given backup directory. Can't print tags." ) print( "WARNING: Could not find backup_tags.txt inside given backup directory. Can't print tags." ) return None def last_full_backup_date( self, path: Optional[str] = None, full_backup_interval: Optional[float] = None ) -> bool: """ Check if last full backup date retired or not. :return: True if last full backup date older than given interval, False if it is newer. """ # Finding last full backup date from dir/folder name full_dir = path or str(self.builder_obj.backup_options.get("full_dir")) backup_interval = full_backup_interval or str( self.builder_obj.backup_options.get("full_backup_interval") ) max_dir = helpers.get_latest_dir_name(full_dir) dir_date = datetime.strptime(str(max_dir), "%Y-%m-%d_%H-%M-%S") now = datetime.now() return float((now - dir_date).total_seconds()) >= float(backup_interval) def clean_full_backup_dir( self, full_dir: Optional[str] = None, remove_all: Optional[bool] = None, ) -> Optional[bool]: # Deleting old full backup after taking new full backup. # Keeping the latest in order not to lose everything. logger.info("starting clean_full_backup_dir") full_dir = full_dir or str(self.builder_obj.backup_options.get("full_dir")) if not os.path.isdir(full_dir): return True if remove_all: for i in os.listdir(full_dir): rm_dir = full_dir + "/" + i shutil.rmtree(rm_dir) return True for i in os.listdir(full_dir): rm_dir = full_dir + "/" + i if i != max(os.listdir(full_dir)): shutil.rmtree(rm_dir) logger.info("DELETING {}".format(rm_dir)) else: logger.info("KEEPING {}".format(rm_dir)) return True def clean_inc_backup_dir(self, inc_dir: Optional[str] = None) -> Optional[bool]: # Deleting incremental backups after taking new fresh full backup. inc_dir = inc_dir or str(self.builder_obj.backup_options.get("inc_dir")) if not os.path.isdir(inc_dir): return True for i in os.listdir(inc_dir): rm_dir = inc_dir + "/" + i shutil.rmtree(str(rm_dir)) return True def full_backup(self) -> bool: """ Method for taking full backups. It will construct the backup command based on config file. :return: True on success. :raise: RuntimeError on error. """ logger.info( "starting full backup to {}".format( self.builder_obj.backup_options.get("full_dir") ) ) full_backup_dir = helpers.create_backup_directory( str(self.builder_obj.backup_options.get("full_dir")) ) # Creating Full Backup command. xtrabackup_cmd = self.builder_obj.full_backup_command_builder( full_backup_dir=full_backup_dir ) # Extra checks. self.builder_obj.stream_encrypt_compress_tar_checker() if self.dry: # If it's a dry run, skip running & tagging return True logger.debug( "Starting {}".format(self.builder_obj.backup_options.get("backup_tool")) ) status = ProcessRunner.run_command(xtrabackup_cmd) status_str = "OK" if status is True else "FAILED" self.add_tag( backup_type="Full", backup_size=helpers.get_folder_size(full_backup_dir), backup_status=status_str, ) return status def inc_backup(self) -> bool: """ Method for taking incremental backups. :return: True on success. :raise: RuntimeError on error. """ # Get the recent full backup path recent_full_bck = helpers.get_latest_dir_name( str(self.builder_obj.backup_options.get("full_dir")) ) if not recent_full_bck: raise RuntimeError( "Failed to get Full backup path. Are you sure you have one?" ) # Get the recent incremental backup path recent_inc_bck = helpers.get_latest_dir_name( str(self.builder_obj.backup_options.get("inc_dir")) ) # Creating time-stamped incremental backup directory inc_backup_dir = helpers.create_backup_directory( str(self.builder_obj.backup_options.get("inc_dir")) ) # Check here if stream=tar enabled. # Because it is impossible to take incremental backup with streaming tar. # raise RuntimeError. self.builder_obj.stream_tar_incremental_checker() xtrabackup_inc_cmd = self.builder_obj.inc_backup_command_builder( recent_full_bck=recent_full_bck, inc_backup_dir=inc_backup_dir, recent_inc_bck=recent_inc_bck, ) self.builder_obj.extract_decrypt_from_stream_backup( recent_full_bck=recent_full_bck, recent_inc_bck=recent_inc_bck ) # Deprecated workaround for LP #1444255 self.builder_obj.decrypter( recent_full_bck=recent_full_bck, xtrabackup_inc_cmd=xtrabackup_inc_cmd, recent_inc_bck=recent_inc_bck, ) if self.dry: # If it's a dry run, skip running & tagging return True logger.debug( "Starting {}".format(self.builder_obj.backup_options.get("backup_tool")) ) status = ProcessRunner.run_command(xtrabackup_inc_cmd) status_str = "OK" if status is True else "FAILED" self.add_tag( backup_type="Inc", backup_size=helpers.get_folder_size(inc_backup_dir), backup_status=status_str, ) return status def all_backup(self) -> bool: """ This method at first checks full backup directory, if it is empty takes full backup. If it is not empty then checks for full backup time. If the recent full backup is taken 1 day ago, it takes full backup. In any other conditions it takes incremental backup. """ # Workaround for circular import dependency error in Python # Creating object from CheckEnv class check_env_obj = CheckEnv( self.conf, full_dir=str(self.builder_obj.backup_options.get("full_dir")), inc_dir=str(self.builder_obj.backup_options.get("inc_dir")), ) assert check_env_obj.check_all_env() is True, "environment checks failed!" if not helpers.get_latest_dir_name( str(self.builder_obj.backup_options.get("full_dir")) ): logger.info( "- - - - You have no backups : Taking very first Full Backup! - - - -" ) if self.mysql_cli.mysql_run_command("flush logs") and self.full_backup(): # Removing old inc backups self.clean_inc_backup_dir() elif self.last_full_backup_date(): logger.info( "- - - - Your full backup is timeout : Taking new Full Backup! - - - -" ) # Archiving backups if self.archive_obj.backup_archive_options.get("archive_dir"): logger.info( "Archiving enabled; cleaning archive_dir & archiving previous Full Backup" ) if self.archive_obj.backup_archive_options.get( "archive_max_duration" ) or self.archive_obj.backup_archive_options.get("archive_max_size"): self.archive_obj.clean_old_archives() self.archive_obj.create_backup_archives() else: logger.info("Archiving disabled. Skipping!") if self.mysql_cli.mysql_run_command("flush logs") and self.full_backup(): # Removing full backups self.clean_full_backup_dir() # Removing inc backups self.clean_inc_backup_dir() else: logger.info( "- - - - You have a full backup that is less than {} seconds old. - - - -".format( self.builder_obj.backup_options.get("full_backup_interval") ) ) logger.info( "- - - - We will take an incremental one based on recent Full Backup - - - -" ) time.sleep(3) # Taking incremental backup self.inc_backup() return True ================================================ FILE: mysql_autoxtrabackup/backup_prepare/__init__.py ================================================ from .copy_back import CopyBack as CopyBack from .prepare import Prepare as Prepare from .prepare_builder import BackupPrepareBuilderChecker as BackupPrepareBuilderChecker ================================================ FILE: mysql_autoxtrabackup/backup_prepare/copy_back.py ================================================ import logging import os import shutil from typing import Optional, Union from mysql_autoxtrabackup.general_conf import path_config from mysql_autoxtrabackup.general_conf.generalops import GeneralClass from mysql_autoxtrabackup.process_runner.process_runner import ProcessRunner from mysql_autoxtrabackup.utils import helpers logger = logging.getLogger(__name__) class CopyBack: def __init__(self, config: str = path_config.config_path_file) -> None: self.conf = config options_obj = GeneralClass(config=self.conf) self.command_options = options_obj.command_options self.mysql_options = options_obj.backup_options self.backup_options = options_obj.backup_options def shutdown_mysql(self) -> Union[None, bool, Exception]: # Shut Down MySQL logger.info("Shutting Down MySQL server:") args = self.command_options.get("stop_mysql_command") return ProcessRunner.run_command(args) def move_to_tmp_dir(self) -> None: try: shutil.move( str(self.mysql_options.get("data_dir")), str(self.backup_options.get("tmp_dir")), ) logger.info( "Moved data_dir to {} ...".format(self.backup_options.get("tmp_dir")) ) except shutil.Error as err: logger.error("Error occurred while moving data_dir") logger.error(err) raise RuntimeError(err) def create_empty_data_dir(self) -> Union[None, bool, Exception]: logger.info("Creating an empty data directory ...") makedir = "mkdir {}".format(self.mysql_options.get("data_dir")) return ProcessRunner.run_command(makedir) def move_data_dir(self) -> bool: # Move data_dir to new directory tmp_dir = self.backup_options.get("tmp_dir") logger.info("Moving MySQL data_dir to {}".format(tmp_dir)) if os.path.isdir(str(self.backup_options.get("tmp_dir"))): rmdir_ = "rm -rf {}".format(tmp_dir) ProcessRunner.run_command(rmdir_) self.move_to_tmp_dir() self.create_empty_data_dir() return True def run_xtra_copyback(self, data_dir: Optional[str] = None) -> Optional[bool]: # Running Xtrabackup with --copy-back option copy_back = "{} --copy-back {} --target-dir={}/{} --data_dir={}".format( self.backup_options.get("backup_tool"), self.backup_options.get("xtra_options"), self.backup_options.get("full_dir"), helpers.get_latest_dir_name(str(self.backup_options.get("full_dir"))), self.mysql_options.get("data_dir") if data_dir is None else data_dir, ) return ProcessRunner.run_command(copy_back) def giving_chown(self, data_dir: Optional[str] = None) -> Optional[bool]: # Changing owner of data_dir to given user:group give_chown = "{} {}".format( self.command_options.get("chown_command"), self.mysql_options.get("data_dir") if data_dir is None else data_dir, ) return ProcessRunner.run_command(give_chown) def start_mysql_func( self, start_tool: Optional[str] = None, options: Optional[str] = None ) -> Union[None, bool, Exception]: # Starting MySQL logger.info("Starting MySQL server: ") args = ( self.command_options.get("start_mysql_command") if start_tool is None else start_tool ) start_command = "{} {}".format(args, options) if options is not None else args return ProcessRunner.run_command(start_command) @staticmethod def check_if_backup_prepared( full_dir: Optional[str], full_backup_file: Optional[str] ) -> Optional[bool]: """ This method is for checking if the backup can be copied-back. It is going to check xtrabackup_checkpoints file inside backup directory for backup_type column. backup_type column must be equal to 'full-prepared' :return: True if backup is already prepared; RuntimeError if it is not. """ with open( "{}/{}/xtrabackup_checkpoints".format(full_dir, full_backup_file), "r" ) as xchk_file: # This thing seems to be complicated bu it is not: # Trying to get 'full-prepared' from ['backup_type ', ' full-prepared\n'] if ( xchk_file.readline().split("=")[1].strip("\n").lstrip() == "full-prepared" ): return True raise RuntimeError( "This full backup is not fully prepared, not doing copy-back!" ) def copy( self, options: Optional[str] = None, data_dir: Optional[str] = None ) -> bool: """ Function for running: xtrabackup --copy-back giving chown to data_dir starting mysql :return: True if succeeded. Error if failed """ logger.info("Copying Back Already Prepared Final Backup:") if ( len( os.listdir( str(self.mysql_options.get("data_dir")) if data_dir is None else data_dir ) ) > 0 ): logger.info("MySQL data_dir is not empty!") return False else: self.run_xtra_copyback(data_dir=data_dir) self.giving_chown(data_dir=data_dir) self.start_mysql_func(options=options) return True def copy_back_action(self, options: Optional[str] = None) -> Optional[bool]: """ Function for complete recover/copy-back actions :return: True if succeeded. Error if failed. """ try: self.check_if_backup_prepared( str(self.backup_options.get("full_dir")), helpers.get_latest_dir_name(str(self.backup_options.get("full_dir"))), ) self.shutdown_mysql() if self.move_data_dir() and self.copy(options=options): logger.info("All data copied back successfully. ") logger.info("Your MySQL server is UP again") return True except Exception as err: logger.error("{}: {}".format(type(err).__name__, err)) return None ================================================ FILE: mysql_autoxtrabackup/backup_prepare/prepare.py ================================================ import logging import os import time from typing import Optional, Union from mysql_autoxtrabackup.backup_backup.backup_builder import BackupBuilderChecker from mysql_autoxtrabackup.backup_prepare.copy_back import CopyBack from mysql_autoxtrabackup.backup_prepare.prepare_builder import ( BackupPrepareBuilderChecker, ) from mysql_autoxtrabackup.general_conf import path_config from mysql_autoxtrabackup.process_runner.process_runner import ProcessRunner from mysql_autoxtrabackup.utils import helpers logger = logging.getLogger(__name__) class Prepare: def __init__( self, config: str = path_config.config_path_file, dry_run: Optional[bool] = None, tag: Optional[str] = None, ) -> None: self.conf = config self.dry = dry_run self.tag = tag self.prepare_options = BackupPrepareBuilderChecker( config=self.conf, dry_run=self.dry ) # If prepare_tool option enabled in config, make backup_tool to use this. # The reason is maybe you have backup taken with 2.4 version but your are going to prepare # with newer version. It is somehow unlike to do this but still. if self.prepare_options.backup_options.get("prepare_tool"): self.prepare_options.backup_options["backup_tool"] = str( self.prepare_options.backup_options.get("prepare_tool") ) if self.tag and not os.path.isfile( "{}/backup_tags.txt".format( self.prepare_options.backup_options.get("backup_dir") ) ): raise RuntimeError( "Could not find backup_tags.txt inside backup directory. " "Please run without --tag option" ) def run_prepare_command( self, base_dir: Optional[str], actual_dir: Optional[str], cmd: Optional[str] ) -> Optional[bool]: # Decrypt backup self.prepare_options.decrypt_backup(base_dir, actual_dir) # Decompress backup self.prepare_options.decompress_backup(base_dir, actual_dir) logger.info("Running prepare command -> {}".format(cmd)) if self.dry: return True return ProcessRunner.run_command(cmd) def prepare_with_tags(self) -> Optional[bool]: # Method for preparing backups based on passed backup tags found_backups = BackupPrepareBuilderChecker.parse_backup_tags( backup_dir=str(self.prepare_options.backup_options.get("backup_dir")), tag_name=self.tag, ) recent_bck = helpers.get_latest_dir_name( str(self.prepare_options.backup_options.get("full_dir")) ) # I am not going to initialize this object in Prepare class constructor as I thin there is no need. backup_builder = BackupBuilderChecker(self.conf, dry_run=self.dry) if found_backups[1] == "Full": # type: ignore if recent_bck: logger.info("- - - - Preparing Full Backup - - - -") # Extracting/decrypting from streamed backup and extra checks goes here. backup_builder.extract_decrypt_from_stream_backup( recent_full_bck=recent_bck ) # Prepare command backup_prepare_cmd = self.prepare_options.prepare_command_builder( full_backup=recent_bck ) self.run_prepare_command( str(self.prepare_options.backup_options.get("full_dir")), recent_bck, backup_prepare_cmd, ) elif found_backups[1] == "Inc": # type: ignore if not os.listdir(str(self.prepare_options.backup_options.get("inc_dir"))): logger.info( "- - - - You have no Incremental backups. So will prepare only latest Full backup - - - -" ) self.prepare_only_full_backup() else: logger.info("- - - - You have Incremental backups. - - - -") if self.prepare_only_full_backup(): logger.info("Preparing Incs: ") list_of_dir = helpers.sorted_ls( str(self.prepare_options.backup_options.get("inc_dir")) ) # Find the index number inside all list for backup(which was found via tag) index_num = list_of_dir.index(found_backups[0]) # type: ignore # Limit the iteration until this found backup for dir_ in list_of_dir[: index_num + 1]: apply_log_only = None if dir_ != found_backups[0]: # type: ignore logger.info( "Preparing inc backups in sequence. inc backup dir/name is {}".format( dir_ ) ) apply_log_only = True else: logger.info( "Preparing last incremental backup, inc backup dir/name is {}".format( dir_ ) ) # Extracting/decrypting from streamed backup and extra checks goes here backup_builder.extract_decrypt_from_stream_backup( recent_inc_bck=dir_, flag=True ) # Prepare command backup_prepare_cmd = ( self.prepare_options.prepare_command_builder( full_backup=recent_bck, incremental=dir_, apply_log_only=apply_log_only, ) ) self.run_prepare_command( str(self.prepare_options.backup_options.get("inc_dir")), dir_, backup_prepare_cmd, ) logger.info("- - - - The end of the Prepare Stage. - - - -") return True def prepare_only_full_backup(self) -> Union[None, bool, Exception]: recent_bck = helpers.get_latest_dir_name( str(self.prepare_options.backup_options.get("full_dir")) ) backup_builder = BackupBuilderChecker(self.conf, dry_run=self.dry) if recent_bck: apply_log_only = None if not os.listdir(str(self.prepare_options.backup_options.get("inc_dir"))): logger.info("- - - - Preparing Full Backup - - - -") self.prepare_options.untar_backup(recent_bck=recent_bck) # Extracting/decrypting from streamed backup and extra checks goes here backup_builder.extract_decrypt_from_stream_backup( recent_full_bck=recent_bck ) else: logger.info("- - - - Preparing Full backup for incrementals - - - -") logger.info( "- - - - Final prepare,will occur after preparing all inc backups - - - -" ) time.sleep(3) apply_log_only = True # Prepare command backup_prepare_cmd = self.prepare_options.prepare_command_builder( full_backup=recent_bck, apply_log_only=apply_log_only ) self.run_prepare_command( str(self.prepare_options.backup_options.get("full_dir")), recent_bck, backup_prepare_cmd, ) return True def prepare_inc_full_backups(self) -> Union[None, bool, Exception]: backup_builder = BackupBuilderChecker(self.conf, dry_run=self.dry) if not os.listdir(str(self.prepare_options.backup_options.get("inc_dir"))): logger.info( "- - - - You have no Incremental backups. So will prepare only latest Full backup - - - -" ) return self.prepare_only_full_backup() else: logger.info("- - - - You have Incremental backups. - - - -") recent_bck = helpers.get_latest_dir_name( str(self.prepare_options.backup_options.get("full_dir")) ) if self.prepare_only_full_backup(): logger.info("Preparing Incs: ") list_of_dir = sorted( os.listdir(str(self.prepare_options.backup_options.get("inc_dir"))) ) for inc_backup_dir in list_of_dir: apply_log_only = None if inc_backup_dir != max( os.listdir( str(self.prepare_options.backup_options.get("inc_dir")) ) ): logger.info( "Preparing Incremental backups in sequence. Incremental backup dir/name is {}".format( inc_backup_dir ) ) apply_log_only = True else: logger.info( "Preparing last Incremental backup, inc backup dir/name is {}".format( inc_backup_dir ) ) # Extracting/decrypting from streamed backup and extra checks goes here backup_builder.extract_decrypt_from_stream_backup( recent_inc_bck=inc_backup_dir, flag=True ) # Prepare command backup_prepare_cmd = self.prepare_options.prepare_command_builder( full_backup=recent_bck, incremental=inc_backup_dir, apply_log_only=apply_log_only, ) self.run_prepare_command( str(self.prepare_options.backup_options.get("inc_dir")), inc_backup_dir, backup_prepare_cmd, ) logger.info("- - - - The end of the Prepare Stage. - - - -") return True def prepare_backup_and_copy_back(self) -> None: copy_back_obj = CopyBack(config=self.conf) # Recovering/Copying Back Prepared Backup x = "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -" print(x) print("") print("Preparing full/inc backups!") print("What do you want to do?") print( "1. Prepare Backups and keep for future usage. NOTE('Once Prepared Backups Can not be prepared Again')" ) print("2. Prepare Backups and restore/recover/copy-back immediately") print("3. Just copy-back previously prepared backups") prepare = int(input("Please Choose one of options and type 1 or 2 or 3: ")) print("") print(x) time.sleep(3) if prepare == 1: if not self.tag: self.prepare_inc_full_backups() else: logger.info("Backup tag will be used to prepare backups") self.prepare_with_tags() elif prepare == 2: if not self.tag: self.prepare_inc_full_backups() else: self.prepare_with_tags() if not self.dry: copy_back_obj.copy_back_action() else: logger.critical( "Dry run is not implemented for copy-back/recovery actions!" ) elif prepare == 3: if not self.dry: copy_back_obj.copy_back_action() else: logger.critical( "Dry run is not implemented for copy-back/recovery actions!" ) else: print("Please type 1 or 2 or 3 and nothing more!") ================================================ FILE: mysql_autoxtrabackup/backup_prepare/prepare_builder.py ================================================ import logging import os from typing import Optional, Tuple from mysql_autoxtrabackup.general_conf import path_config from mysql_autoxtrabackup.general_conf.generalops import GeneralClass from mysql_autoxtrabackup.process_runner.process_runner import ProcessRunner logger = logging.getLogger(__name__) class BackupPrepareBuilderChecker: def __init__( self, config: str = path_config.config_path_file, dry_run: Optional[bool] = None ) -> None: self.conf = config self.dry = dry_run options_obj = GeneralClass(config=self.conf) self.backup_options = options_obj.backup_options self.compression_options = options_obj.compression_options self.encryption_options = options_obj.encryption_options self.xbstream_options = options_obj.xbstream_options @staticmethod def parse_backup_tags( backup_dir: Optional[str], tag_name: Optional[str] ) -> Optional[Tuple[str, str]]: """ Static Method for returning the backup directory name and backup type :param backup_dir: The backup directory path :param tag_name: The tag name to search :return: Tuple of (backup directory, backup type) (2017-11-09_19-37-16, Full). :raises: RuntimeError if there is no such tag inside backup_tags.txt """ if os.path.isfile("{}/backup_tags.txt".format(backup_dir)): with open("{}/backup_tags.txt".format(backup_dir), "r") as backup_tags: f = backup_tags.readlines() for i in f: split_ = i.split("\t") if tag_name == split_[-1].rstrip("'\n\r").lstrip("'"): return split_[0], split_[1] else: raise RuntimeError("There is no such tag for backups") return None def decompress_backup( self, path: Optional[str], dir_name: Optional[str] ) -> Optional[bool]: """ Method for backup decompression. Check if decompression enabled, if it is, decompress backup prior prepare. :param path: the basedir path i.e full backup dir or incremental dir. :param dir_name: the exact name backup folder(likely timestamped folder name). :return: None or RuntimeError """ if self.compression_options.get("decompress"): # The base decompression command dec_cmd = "{} --decompress={} --target-dir={}/{}".format( self.backup_options.get("backup_tool"), self.compression_options.get("decompress"), path, dir_name, ) if self.compression_options.get("remove_original_comp"): dec_cmd += " --remove-original" logger.info("Trying to decompress backup") logger.info("Running decompress command -> {}".format(dec_cmd)) if self.dry: return None return ProcessRunner.run_command(dec_cmd) return None def decrypt_backup( self, path: Optional[str], dir_name: Optional[str] ) -> Optional[bool]: """ Method for decrypting backups. If you use crypted backups it should be decrypted prior preparing. :param path: the basedir path i.e full backup dir or incremental dir. :param dir_name: the exact name backup folder(likely timestamped folder name). :return: None or RuntimeError """ if self.encryption_options.get("decrypt"): # The base decryption command decr_cmd = "{} --decrypt={} --encrypt-key={} --target-dir={}/{}".format( self.backup_options.get("backup_tool"), self.encryption_options.get("decrypt"), self.encryption_options.get("encrypt_key"), path, dir_name, ) if self.encryption_options.get("remove_original_comp"): decr_cmd += " --remove-original" logger.info("Trying to decrypt backup") logger.info("Running decrypt command -> {}".format(decr_cmd)) if self.dry: return None return ProcessRunner.run_command(decr_cmd) return None def prepare_command_builder( self, full_backup: Optional[str], incremental: Optional[str] = None, apply_log_only: Optional[bool] = None, ) -> str: """ Method for building prepare command as it is repeated several times. :param full_backup: The full backup directory name :param incremental: The incremental backup directory name :param apply_log_only: The flag to add --apply-log-only :return: The prepare command string """ # Base prepare command xtrabackup_prepare_cmd = "{} --prepare --target-dir={}/{}".format( self.backup_options.get("backup_tool"), self.backup_options.get("full_dir"), full_backup, ) if incremental: xtrabackup_prepare_cmd += " --incremental-dir={}/{}".format( self.backup_options.get("inc_dir"), incremental ) if apply_log_only: xtrabackup_prepare_cmd += " --apply-log-only" # Checking if extra options were passed: if self.backup_options.get("xtra_options"): xtrabackup_prepare_cmd += " {}".format( self.backup_options.get("xtra_options") ) # Checking of extra prepare options were passed: if self.backup_options.get("xtra_prepare_options"): xtrabackup_prepare_cmd += " {}".format( self.backup_options.get("xtra_prepare_options") ) return xtrabackup_prepare_cmd def untar_backup(self, recent_bck: str) -> Optional[bool]: if self.xbstream_options.get("stream") == "tar": full_dir = self.backup_options.get("full_dir") untar_cmd = "tar -xf {}/{}/full_backup.tar -C {}/{}".format( full_dir, recent_bck, full_dir, recent_bck ) logger.info( "The following tar command will be executed -> {}".format(untar_cmd) ) if self.dry == 0 and os.path.isfile( "{}/{}/full_backup.tar".format(full_dir, recent_bck) ): return ProcessRunner.run_command(untar_cmd) return None ================================================ FILE: mysql_autoxtrabackup/general_conf/__init__.py ================================================ from . import path_config as path_config from .check_env import CheckEnv as CheckEnv from .generalops import GeneralClass as GeneralClass from .generate_default_conf import GenerateDefaultConfig as GenerateDefaultConfig ================================================ FILE: mysql_autoxtrabackup/general_conf/check_env.py ================================================ import logging import os import re from typing import Optional, Union from mysql_autoxtrabackup.process_runner.process_runner import ProcessRunner from mysql_autoxtrabackup.utils.helpers import create_directory from . import path_config from .generalops import GeneralClass logger = logging.getLogger(__name__) class CheckEnv: def __init__( self, config: str = path_config.config_path_file, full_dir: Union[str, None] = None, inc_dir: Union[str, None] = None, ) -> None: self.conf = config options = GeneralClass(config=self.conf) self.backup_options = options.backup_options self.mysql_options = options.mysql_options self.archive_options = options.backup_archive_options if full_dir: self.backup_options["full_dir"] = full_dir if inc_dir: self.backup_options["ind_dir"] = inc_dir def check_mysql_uptime(self, options: Optional[str] = None) -> Optional[bool]: """ Method for checking if MySQL server is up or not. :param options: Passed options to connect to MySQL server if None, then going to get it from conf file :return: True on success, raise RuntimeError on error. """ if not options: status_args = ( "{} --defaults-file={} " "--user={} --password='{}' status".format( self.mysql_options.get("mysqladmin"), self.mysql_options.get("mycnf"), self.mysql_options.get("mysql_user"), self.mysql_options.get("mysql_password"), ) ) if self.mysql_options.get("mysql_socket"): status_args += " --socket={}".format( self.mysql_options.get("mysql_socket") ) elif self.mysql_options.get("mysql_host") and self.mysql_options.get( "mysql_port" ): status_args += " --host={}".format(self.mysql_options.get("mysql_host")) status_args += " --port={}".format(self.mysql_options.get("mysql_port")) else: logger.critical( "Neither mysql_socket nor mysql_host and mysql_port are defined in config!" ) raise RuntimeError( "Neither mysql_socket nor mysql_host and mysql_port are defined in config!" ) else: status_args = "{} {} status".format( self.mysql_options.get("mysqladmin"), options ) # filter out password from argument list filtered_args = re.sub("--password='?\w+'?", "--password='*'", status_args) logger.info("Running mysqladmin command -> {}".format(filtered_args)) return ProcessRunner.run_command(status_args) def check_mysql_conf(self) -> Union[bool, Exception]: """ Method for checking passed MySQL my.cnf defaults file. If it is not passed then skip this check :return: True on success, raise RuntimeError on error. """ my_cnf = self.mysql_options.get("mycnf") if not my_cnf or my_cnf == "": logger.info("Skipping my.cnf check, because it is not specified") return True elif not os.path.exists(my_cnf): logger.error("FAILED: MySQL configuration file path does NOT exist") raise RuntimeError("FAILED: MySQL configuration file path does NOT exist") logger.info("OK: MySQL configuration file exists") return True def check_mysql_mysql(self) -> Union[bool, Exception]: """ Method for checking mysql client path :return: True on success, raise RuntimeError on error. """ mysql = self.mysql_options.get("mysql") if os.path.exists(str(mysql)): logger.info("OK: {} exists".format(mysql)) return True logger.error("FAILED: {} doest NOT exist".format(mysql)) raise RuntimeError("FAILED: {} doest NOT exist".format(mysql)) def check_mysql_mysqladmin(self) -> Union[bool, Exception]: """ Method for checking mysqladmin path :return: True on success, raise RuntimeError on error. """ mysqladmin = self.mysql_options.get("mysqladmin") if os.path.exists(str(mysqladmin)): logger.info("OK: {} exists".format(mysqladmin)) return True logger.error("FAILED: {} does NOT exist".format(mysqladmin)) raise RuntimeError("FAILED: {} does NOT exist".format(mysqladmin)) def check_mysql_backuptool(self) -> Union[bool, Exception]: """ Method for checking if given backup tool path is there or not. :return: RuntimeError on failure, True on success """ if os.path.exists(str(self.backup_options.get("backup_tool"))): logger.info("OK: XtraBackup exists") return True logger.error("FAILED: XtraBackup does NOT exist") raise RuntimeError("FAILED: XtraBackup does NOT exist") def check_mysql_backup_dir(self) -> Optional[bool]: """ Check for MySQL backup directory. If directory exists already then, return True. If not, try to create it. :return: True on success. RuntimeError on failure. """ if os.path.exists(str(self.backup_options.get("backup_dir"))): logger.info("OK: Main backup directory exists") return True return create_directory(str(self.backup_options.get("backup_dir"))) def check_mysql_archive_dir(self) -> Optional[bool]: """ Check for archive directory. If archive_dir is given in config file and if it does not exist, try to create. :return: True on success. RuntimeError on failure. """ if not self.archive_options.get("archive_dir"): logger.info("Skipping check as this option not specified in config file...") return True if os.path.exists(str(self.archive_options.get("archive_dir"))): logger.info("OK: Archive folder directory exists") return True return create_directory(str(self.archive_options.get("archive_dir"))) def check_mysql_full_backup_dir(self) -> Optional[bool]: """ Check full backup directory path. If this path exists return True if not try to create. :return: True on success. """ if os.path.exists(str(self.backup_options.get("full_dir"))): logger.info("OK: Full Backup directory exists") return True return create_directory(str(self.backup_options.get("full_dir"))) def check_mysql_inc_backup_dir(self) -> Optional[bool]: """ Check incremental backup directory path. If this path exists return True if not try to create. :return: True on success. """ if os.path.exists(str(self.backup_options.get("inc_dir"))): logger.info("OK: Increment directory exists") return True return create_directory(str(self.backup_options.get("inc_dir"))) def check_all_env(self) -> Union[bool, Exception]: """ Method for running all checks :return: True on success, raise RuntimeError on error. """ try: self.check_mysql_uptime() self.check_mysql_mysql() self.check_mysql_mysqladmin() self.check_mysql_conf() self.check_mysql_backuptool() self.check_mysql_backup_dir() self.check_mysql_full_backup_dir() self.check_mysql_inc_backup_dir() self.check_mysql_archive_dir() except Exception as err: logger.critical("FAILED: Check status") logger.error(err) raise RuntimeError("FAILED: Check status") else: logger.info("OK: Check status") return True ================================================ FILE: mysql_autoxtrabackup/general_conf/generalops.py ================================================ import configparser import logging from os.path import isfile from typing import Dict, Union import humanfriendly # type: ignore from . import path_config logger = logging.getLogger(__name__) class GeneralClass: def __init__(self, config: str = path_config.config_path_file) -> None: if isfile(config): self.con = configparser.ConfigParser() self.con.read(config) else: logger.critical( "Missing config file : {}".format(path_config.config_path_file) ) @property def mysql_options(self) -> Dict[str, str]: section = "MySQL" return { "mysql": self.con.get(section, "mysql"), "mycnf": self.con.get(section, "mycnf"), "mysqladmin": self.con.get(section, "mysqladmin"), "mysql_user": self.con.get(section, "mysql_user"), "mysql_password": self.con.get(section, "mysql_password"), "mysql_socket": self.con.get(section, "mysql_socket", fallback=None), # type: ignore "mysql_host": self.con.get(section, "mysql_host", fallback=None), # type: ignore "mysql_port": self.con.get(section, "mysql_port", fallback=None), # type: ignore "data_dir": self.con.get(section, "datadir"), } @property def logging_options(self) -> Dict[str, str]: section = "Logging" return { "log_level": self.con.get(section, "log"), "log_file_max_bytes": self.con.get(section, "log_file_max_bytes"), "log_file_backup_count": self.con.get(section, "log_file_backup_count"), } @property def compression_options(self) -> Dict[str, str]: section = "Compress" return { "compress": self.con.get(section, "compress", fallback=None), # type: ignore "compress_chunk_size": self.con.get(section, "compress_chunk_size", fallback=None), # type: ignore "compress_threads": self.con.get(section, "compress_threads", fallback=None), # type: ignore "decompress": self.con.get(section, "decompress", fallback=None), # type: ignore "remove_original": self.con.get(section, "remove_original", fallback=None), # type: ignore } @property def xbstream_options(self) -> Dict[str, str]: section = "Xbstream" return { "xbstream": self.con.get(section, "xbstream", fallback=None), # type: ignore "stream": self.con.get(section, "stream", fallback=None), # type: ignore "xbstream_options": self.con.get(section, "xbstream_options", fallback=None), # type: ignore "xbs_decrypt": self.con.get(section, "xbs_decrypt", fallback=None), # type: ignore } @property def command_options(self) -> Dict[str, str]: section = "Commands" return { "start_mysql_command": self.con.get(section, "start_mysql_command"), "stop_mysql_command": self.con.get(section, "stop_mysql_command"), "chown_command": self.con.get(section, "chown_command"), } @property def encryption_options(self) -> Dict[str, str]: section = "Encrypt" return { "xbcrypt": self.con.get(section, "xbcrypt", fallback=None), # type: ignore "encrypt": self.con.get(section, "encrypt", fallback=None), # type: ignore "encrypt_key": self.con.get(section, "encrypt_key", fallback=None), # type: ignore "encrypt_key_file": self.con.get(section, "encrypt_key_file", fallback=None), # type: ignore "encrypt_threads": self.con.get(section, "encrypt_threads", fallback=None), # type: ignore "encrypt_chunk_size": self.con.get(section, "encrypt_chunk_size", fallback=None), # type: ignore "decrypt": self.con.get(section, "decrypt", fallback=None), # type: ignore "remove_original": self.con.get(section, "remove_original", fallback=None), # type: ignore } @property def backup_archive_options(self) -> Dict[str, Union[str, float]]: section = "Backup" # backward compatible with old config 'max_archive_size' and newer 'archive_max_size' archive_max_size = self.con.get(section, "max_archive_size", fallback=None) if archive_max_size: archive_max_size = humanfriendly.parse_size(archive_max_size) else: if self.con.get(section, "archive_max_size", fallback=None): archive_max_size = humanfriendly.parse_size( self.con.get(section, "archive_max_size", fallback=None) ) # backward compatible with old config 'max_archive_duration' and newer 'archive_max_duration' archive_max_duration = self.con.get( section, "max_archive_duration", fallback=None ) if archive_max_duration: archive_max_duration = humanfriendly.parse_timespan(archive_max_duration) else: if self.con.get(section, "archive_max_size", fallback=None): archive_max_duration = humanfriendly.parse_timespan( self.con.get(section, "archive_max_size", fallback=None) ) return { "archive_dir": self.con.get(section, "archive_dir", fallback=None), # type: ignore "prepare_archive": self.con.get(section, "prepare_archive", fallback=None), # type: ignore "move_archive": self.con.get(section, "move_archive", fallback=None), # type: ignore "archive_max_size": str(archive_max_size), "archive_max_duration": str(archive_max_duration), } @property def backup_options(self) -> Dict[str, Union[str, float]]: section = "Backup" return { "pid_dir": self.con.get(section, "pid_dir", fallback="/tmp/"), "tmp_dir": self.con.get(section, "tmp_dir"), "pid_runtime_warning": humanfriendly.parse_timespan( self.con.get(section, "pid_runtime_warning") ), "backup_dir": self.con.get(section, "backup_dir"), "full_dir": self.con.get(section, "backup_dir") + "/full", "inc_dir": self.con.get(section, "backup_dir") + "/inc", "backup_tool": self.con.get(section, "backup_tool"), "prepare_tool": self.con.get(section, "prepare_tool", fallback=None), # type: ignore "xtra_backup": self.con.get(section, "xtra_backup", fallback=None), # type: ignore "xtra_prepare_options": self.con.get(section, "xtra_prepare_options", fallback=None), # type: ignore "xtra_options": self.con.get(section, "xtra_options", fallback=None), # type: ignore "full_backup_interval": humanfriendly.parse_timespan( self.con.get(section, "full_backup_interval", fallback="86400.0") ), "partial_list": self.con.get(section, "partial_list", fallback=None), # type: ignore } ================================================ FILE: mysql_autoxtrabackup/general_conf/generate_default_conf.py ================================================ # Generate the default config file dynamically. # As part of - https://github.com/ShahriyarR/MySQL-AutoXtraBackup/issues/331 import configparser from os import makedirs from os.path import exists, join from . import path_config class GenerateDefaultConfig: def __init__(self, config: str = path_config.config_path_file) -> None: self.conf = config self.home = path_config.home try: if not exists(path_config.config_path): makedirs(path_config.config_path) except: pass def generate_config_file(self) -> None: with open(self.conf, "w+") as cfgfile: config = configparser.ConfigParser(allow_no_value=True) section1 = "MySQL" config.add_section(section1) config.set(section1, "mysql", "/usr/bin/mysql") config.set(section1, "mycnf", "") config.set(section1, "mysqladmin", "/usr/bin/mysqladmin") config.set(section1, "mysql_user", "root") config.set(section1, "mysql_password", "12345") config.set( section1, "## Set either mysql_socket only, OR host + port. If both are set mysql_socket is used", ) config.set(section1, "mysql_socket", "/var/lib/mysql/mysql.sock") config.set(section1, "#mysql_host", "127.0.0.1") config.set(section1, "#mysql_port", "3306") config.set(section1, "datadir", "/var/lib/mysql") section2 = "Logging" config.add_section(section2) config.set(section2, "#[DEBUG,INFO,WARNING,ERROR,CRITICAL]") config.set(section2, "log", "DEBUG") config.set(section2, "log_file_max_bytes", "1073741824") config.set(section2, "log_file_backup_count", "7") section3 = "Backup" config.add_section(section3) config.set(section3, "#Optional: set pid directory") config.set(section3, "pid_dir", "/tmp/MySQL-AutoXtraBackup") config.set(section3, "tmp_dir", join(self.home, "XB_TEST/mysql_datadirs")) config.set( section3, "#Optional: set warning if pid of backup us running for longer than X", ) config.set(section3, "pid_runtime_warning", "2 Hours") config.set(section3, "backup_dir", join(self.home, "XB_TEST/backup_dir")) config.set(section3, "backup_tool", "/usr/bin/xtrabackup") config.set( section3, "#Optional: specify different path/version of xtrabackup here for prepare", ) config.set(section3, "#prepare_tool", "") config.set(section3, "#Optional: pass additional options for backup stage") config.set(section3, "#xtra_backup", "--compact") config.set(section3, "#Optional: pass additional options for prepare stage") config.set(section3, "#xtra_prepare_options", "--rebuild-indexes") config.set( section3, "#Optional: pass general additional options; it will go to both for backup and prepare", ) config.set(section3, "#xtra_options", "--binlog-info=ON --galera-info") config.set(section3, "#Optional: set archive and rotation") config.set( section3, "#archive_dir", join(self.home, "XB_TEST/backup_archives") ) config.set(section3, "#prepare_archive", "1") config.set(section3, "#move_archive", "0") config.set(section3, "#full_backup_interval", "1 day") config.set(section3, "#archive_max_size", "100GiB") config.set(section3, "#archive_max_duration", "4 Days") config.set( section3, "#Optional: WARNING(Enable this if you want to take partial backups). " "Specify database names or table names.", ) config.set(section3, "#partial_list", "tests.t1 tests.t2 dbtest") section4 = "Compress" config.add_section(section4) config.set(section4, "#optional") config.set(section4, "#Enable only if you want to use compression.") config.set(section4, "#compress", "quicklz") config.set(section4, "#compress_chunk_size", "65536") config.set(section4, "#compress_threads", "4") config.set(section4, "#decompress", "TRUE") config.set( section4, "#Enable if you want to remove .qp files after decompression." "(Available from PXB 2.3.7 and 2.4.6)", ) config.set(section4, "#remove_original", "FALSE") section5 = "Encrypt" config.add_section(section5) config.set(section5, "#Optional") config.set(section5, "#Enable only if you want to create encrypted backups") config.set(section5, "#xbcrypt", "/usr/bin/xbcrypt") config.set(section5, "#encrypt", "AES256") config.set( section5, "#Please note that --encrypt-key and --encrypt-key-file are mutually exclusive", ) config.set(section5, "#encrypt_key", "VVTBwgM4UhwkTTV98fhuj+D1zyWoA89K") config.set(section5, "#encrypt_key_file", "/path/to/file/with_encrypt_key") config.set(section5, "#encrypt_threads", "4") config.set(section5, "#encrypt_chunk_size", "65536") config.set(section5, "#decrypt", "AES256") config.set( section5, "#Enable if you want to remove .qp files after decompression." "(Available from PXB 2.3.7 and 2.4.6)", ) config.set(section5, "#remove_original", "FALSE") section6 = "Xbstream" config.add_section(section6) config.set(section6, "#EXPERIMENTAL/OPTIONAL") config.set(section6, "#Enable this, if you want to stream your backups") config.set(section6, "#xbstream", "/usr/bin/xbstream") config.set(section6, "#stream", "xbstream") config.set(section6, "#xbstream_options", "-x --parallel=100") config.set(section6, "#xbs_decrypt", "1") config.set( section6, "# WARN, enable this, if you want to stream your backups to remote host", ) config.set(section6, "#remote_stream", "ssh xxx.xxx.xxx.xxx") section8 = "Commands" config.add_section(section8) config.set(section8, "start_mysql_command", "service mysql start") config.set(section8, "stop_mysql_command", "service mysql stop") config.set(section8, "chown_command", "chown -R mysql:mysql") config.write(cfgfile) ================================================ FILE: mysql_autoxtrabackup/general_conf/path_config.py ================================================ # This file is simply place holder for default config file path, which is used in many places. # If you decide to change the default config path then please edit this file and reinstall tool. from os.path import expanduser, join home: str = expanduser("~") config_path: str = join(home, ".autoxtrabackup") config_path_file: str = join(config_path, "autoxtrabackup.cnf") log_file_path: str = join(config_path, "autoxtrabackup.log") ================================================ FILE: mysql_autoxtrabackup/process_runner/__init__.py ================================================ from .process_runner import ProcessHandler as ProcessHandler ================================================ FILE: mysql_autoxtrabackup/process_runner/errors.py ================================================ import logging logger = logging.getLogger(__name__) # TODO: use these errors in the future - keeping it for future def log_error(expression: str, message: str) -> None: logger.error("FAILED: " + expression + " " + message) class Error(Exception): """Base class for all sort of exceptions""" class ExternalCommandFailed(Error): """ Exception raised for external tool/command fails. """ def __init__(self, expression: str, message: str) -> None: self.expression = expression self.message = message log_error(self.expression, self.message) class FullBackupFailed(Error): """ Exception raised for full backup error. """ def __init__(self, expression: str, message: str) -> None: self.expression = expression self.message = message log_error(self.expression, self.message) class IncrementalBackupFailed(Error): """ Exception raised for incremental backup error. """ def __init__(self, expression: str, message: str) -> None: self.expression = expression self.message = message log_error(self.expression, self.message) class SomethingWentWrong(Error): """ Exception raised for all general failed commands. """ def __init__(self, expression: str, message: str) -> None: self.expression = expression self.message = message log_error(self.expression, self.message) ================================================ FILE: mysql_autoxtrabackup/process_runner/process_runner.py ================================================ import datetime import logging import re import shlex import subprocess import typing from subprocess import PIPE, STDOUT from mysql_autoxtrabackup.general_conf import path_config from mysql_autoxtrabackup.general_conf.generalops import GeneralClass logger = logging.getLogger(__name__) class ProcessHandler(GeneralClass): """ Class to run a command with real-time logging for process centralizes logic for subprocess calls, and is available to all other classes (Prepare, Backup, etc) """ def __init__(self, config: str = path_config.config_path_file) -> None: self.conf = config GeneralClass.__init__(self, self.conf) self._xtrabackup_history_log = [ [ "command", "xtrabackup_function", "start time", "end time", "duration", "exit code", ] ] @property def xtrabackup_history_log(self) -> typing.List[typing.List[str]]: return self._xtrabackup_history_log def run_command(self, command: typing.Optional[str]) -> bool: """ executes a prepared command, enables real-time console & log output. This function should eventually be used for all subprocess calls. :param command: bash command to be executed :type command: str :return: True if success, False if failure :rtype: bool """ # filter out password from argument list, print command to execute filtered_command = re.sub("--password='?\w+'?", "--password='*'", command) # type: ignore logger.info("SUBPROCESS STARTING: {}".format(str(filtered_command))) subprocess_args = self.command_to_args(command_str=command) # start the command subprocess cmd_start = datetime.datetime.now() with subprocess.Popen(subprocess_args, stdout=PIPE, stderr=STDOUT) as process: for line in process.stdout: # type: ignore logger.debug( "[{}:{}] {}".format( subprocess_args[0], process.pid, line.decode("utf-8").strip("\n"), ) ) logger.info( "SUBPROCESS {} COMPLETED with exit code: {}".format( subprocess_args[0], process.returncode ) ) cmd_end = datetime.datetime.now() self.summarize_process(subprocess_args, cmd_start, cmd_end, process.returncode) # return True or False. if process.returncode == 0: return True else: raise ChildProcessError("SUBPROCESS FAILED! >> {}".format(filtered_command)) @staticmethod def command_to_args(command_str: typing.Optional[str]) -> typing.List[str]: """ convert a string bash command to an arguments list, to use with subprocess Most autoxtrabackup code creates a string command, e.g. "xtrabackup --prepare --target-dir..." If we run a string command with subprocess.Popen, we require shell=True. shell=True has security considerations (below), and we run autoxtrabackup with privileges (!). https://docs.python.org/3/library/subprocess.html#security-considerations So, convert to an args list and call Popen without shell=True. :param command_str: string command to execute as a subprocess :type command_str: str :return: list of args to pass to subprocess.Popen. :rtype: list """ if isinstance(command_str, list): # already a list args = command_str elif isinstance(command_str, str): args = shlex.split(command_str) else: raise TypeError logger.debug("subprocess args are: {}".format(args)) return args @staticmethod def represent_duration( start_time: datetime.datetime, end_time: datetime.datetime ) -> str: # https://gist.github.com/thatalextaylor/7408395 duration_delta = end_time - start_time seconds = int(duration_delta.seconds) days, seconds = divmod(seconds, 86400) hours, seconds = divmod(seconds, 3600) minutes, seconds = divmod(seconds, 60) if days > 0: return "%dd%dh%dm%ds" % (days, hours, minutes, seconds) elif hours > 0: return "%dh%dm%ds" % (hours, minutes, seconds) elif minutes > 0: return "%dm%ds" % (minutes, seconds) else: return "%ds" % (seconds,) def summarize_process( self, args: typing.List[str], cmd_start: datetime.datetime, cmd_end: datetime.datetime, return_code: int, ) -> bool: cmd_root: str = args[0].split("/")[-1:][0] xtrabackup_function = None if cmd_root == "xtrabackup": if "--backup" in args: xtrabackup_function = "backup" elif "--prepare" in args and "--apply-log-only" not in args: xtrabackup_function = "prepare" elif "--prepare" in args: xtrabackup_function = "prepare/apply-log-only" if not xtrabackup_function: for arg in args: if re.search(r"(--decrypt)=?[\w]*", arg): xtrabackup_function = "decrypt" elif re.search(r"(--decompress)=?[\w]*", arg): xtrabackup_function = "decompress" if cmd_root != "pigz": # this will be just the pigz --version call self._xtrabackup_history_log.append( [ cmd_root, str(xtrabackup_function), cmd_start.strftime("%Y-%m-%d %H:%M:%S"), cmd_end.strftime("%Y-%m-%d %H:%M:%S"), self.represent_duration(cmd_start, cmd_end), str(return_code), ] ) return True ProcessRunner = ProcessHandler() ================================================ FILE: mysql_autoxtrabackup/utils/__init__.py ================================================ ================================================ FILE: mysql_autoxtrabackup/utils/helpers.py ================================================ # General helpers file for adding all sort of simple helper functions. # Trying to use here type hints as well. import logging import os import subprocess from datetime import datetime from typing import Dict, List, Optional, Union logger = logging.getLogger(__name__) def get_folder_size(path: str) -> Union[str, None]: """ Function to calculate given folder size. Using 'du' command here. :param path: The full path to be calculated :return: String with human readable size info, for eg, 5.3M """ du_cmd = "du -hs {}".format(path) status, output = subprocess.getstatusoutput(du_cmd) if status == 0: return output.split()[0] else: logger.error("Failed to get the folder size") return None def sorted_ls(path: Optional[str]) -> List[str]: """ Function for sorting given path :param path: Directory path :return: The list of sorted directories """ mtime = lambda f: os.stat(os.path.join(path, f)).st_mtime # type: ignore return list(sorted(os.listdir(path), key=mtime)) def get_directory_size(path: str) -> int: """ Calculate total size of given directory path :param path: Directory path :return: Total size of directory """ # I am not sure why we have 2 separate functions for same thing but, # I assume it is there on purpose total_size = 0 for dir_path, dir_names, file_names in os.walk(path): for f in file_names: fp = os.path.join(dir_path, f) total_size += os.path.getsize(fp) return total_size def create_backup_directory(directory: str, forced_dir: Optional[str] = None) -> str: """ Function for creating timestamped directory on given path :param directory: Directory path :param forced_dir: Full Directory path forced to be created :return: Created new directory path """ new_dir = os.path.join(directory, datetime.now().strftime("%Y-%m-%d_%H-%M-%S")) if forced_dir: new_dir = os.path.join(directory, forced_dir) try: # Creating directory os.makedirs(new_dir) return new_dir except Exception as err: logger.error( "Something went wrong in create_backup_directory(): {}".format(err) ) raise RuntimeError( "Something went wrong in create_backup_directory(): {}".format(err) ) def get_latest_dir_name(path: Optional[str]) -> Optional[str]: # Return last backup dir name either incremental or full backup dir if len(os.listdir(path)) > 0: return max(os.listdir(path)) return None def create_directory(path: str) -> Optional[bool]: logger.info("Creating given directory...") try: os.makedirs(path) logger.info("OK: Created") return True except Exception as err: logger.error("FAILED: Could not create directory, ", err) raise RuntimeError("FAILED: Could not create directory") def check_if_backup_prepared(type_: str, path: str) -> str: """ Helper function for checking if given backup already prepared or not. :param type_: Type of backup full or inc :param path: path string of the backup folder :return: True if given backup is prepared, False otherwise """ if type_ == "full" and os.path.isfile(path + "/xtrabackup_checkpoints"): with open(path + "/xtrabackup_checkpoints", "r") as f: if f.readline().split()[-1] == "full-prepared": return "Full-Prepared" # TODO: add the possible way of checking for incremental backups as well. return "Not-Prepared" def list_available_backups(path: str) -> Dict[str, List[Dict[str, str]]]: """ Helper function for returning Dict of backups; and the statuses - if they are already prepared or not :param path: General backup directory path :return: dictionary of backups full and incremental """ backups = {} full_backup_dir = path + "/full" inc_backup_dir = path + "/inc" if os.path.isdir(full_backup_dir): backups = { "full": [ {dir_: check_if_backup_prepared("full", full_backup_dir + f"/{dir_}")} ] for dir_ in os.listdir(full_backup_dir) } if os.path.isdir(inc_backup_dir): backups["inc"] = sorted_ls(inc_backup_dir) # type: ignore logger.info( "Listing all available backups from full and incremental backup directories..." ) return backups ================================================ FILE: mysql_autoxtrabackup/utils/mysql_cli.py ================================================ # This file will consist of some wrapper for using MySQL # It is mainly used for preparing and calling mysql cli import logging from mysql_autoxtrabackup.general_conf import path_config from mysql_autoxtrabackup.general_conf.generalops import GeneralClass from mysql_autoxtrabackup.process_runner.process_runner import ProcessRunner logger = logging.getLogger(__name__) class MySQLClientHelper: def __init__(self, config: str = path_config.config_path_file): self.conf = config # Using Composition instead of Inheritance here options_obj = GeneralClass(config=self.conf) self.mysql_options = options_obj.mysql_options def create_mysql_client_command(self, statement: str) -> str: command_connection = "{} --defaults-file={} -u{} --password={}".format( self.mysql_options.get("mysql"), self.mysql_options.get("mycnf"), self.mysql_options.get("mysql_user"), self.mysql_options.get("mysql_password"), ) command_execute = ' -e "{}"' if self.mysql_options.get("mysql_socket"): command_connection += " --socket={}" new_command = command_connection.format( self.mysql_options.get("mysql_socket") ) else: command_connection += " --host={} --port={}" new_command = command_connection.format( self.mysql_options.get("mysql_host"), self.mysql_options.get("mysql_port"), ) new_command += command_execute return new_command.format(statement) def mysql_run_command(self, statement: str) -> bool: command = self.create_mysql_client_command(statement=statement) return ProcessRunner.run_command(command) ================================================ FILE: mysql_autoxtrabackup/utils/version.py ================================================ __all__ = "VERSION" VERSION = "2.0.2" ================================================ FILE: netlify.toml ================================================ # example netlify.toml [build] command = "bash scripts/netlify-docs.sh" functions = "netlify/functions" publish = "docs/_build/html" ## Uncomment to use this redirect for Single Page Applications like create-react-app. ## Not needed for static site generators. #[[redirects]] # from = "/*" # to = "/index.html" # status = 200 ## (optional) Settings for Netlify Dev ## https://github.com/netlify/cli/blob/main/docs/netlify-dev.md#project-detection #[dev] # command = "yarn start" # Command to start your dev server # port = 3000 # Port that the dev server will be listening on # publish = "dist" # Folder with the static content for _redirect file ## more info on configuring this file: https://www.netlify.com/docs/netlify-toml-reference/ ================================================ FILE: pyproject.toml ================================================ [build-system] requires = ["flit"] build-backend = "flit.buildapi" [tool.flit.metadata] module = "mysql_autoxtrabackup" author = "Shahriyar(Shako) Rzayev" author-email = "rzayev.sehriyar@gmail.com" home-page = "https://github.com/ShahriyarR/MySQL-AutoXtraBackup" classifiers = [ "Intended Audience :: Information Technology", "Intended Audience :: System Administrators", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 3", "Programming Language :: Python", "Topic :: Internet", "Topic :: Software Development :: Libraries :: Application Frameworks", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Software Development :: Libraries", "Topic :: Software Development", "Typing :: Typed", "Development Status :: 4 - Beta", "Environment :: Console", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", ] requires = [ "click >=3.3", "pid >=2.0", "humanfriendly >=2.0", "fastapi >= 0.63.0", "uvicorn >= 0.13.4", ] description-file = "README.md" requires-python = ">=3.6" [tool.flit.metadata.urls] Documentation = "https://autoxtrabackup.azepug.az/" [tool.flit.metadata.requires-extra] test = [ "pytest ==5.4.3", "pytest-cov ==2.10.0", "mypy ==0.812", "isort >=5.0.6,<6.0.0", "flake8 >=3.8.3,<4.0.0", "black ==20.8b1", ] doc = [ "sphinx", "sphinx_rtd_theme", "sphinx-autobuild", "jinja2 >=2.11.3" ] dev = [ "click >=3.3", "pid >=2.0", "humanfriendly >=2.0", "pytest ==5.4.3", "pytest-cov ==2.10.0", "mypy ==0.812", "isort >=5.0.6,<6.0.0", "flake8 >=3.8.3,<4.0.0", "black ==20.8b1", "fastapi >= 0.63.0", "uvicorn >= 0.13.4", ] all = [ "click >=3.3", "pid >=2.0", "humanfriendly >=2.0", "sphinx", "sphinx_rtd_theme", "sphinx-autobuild", "jinja2 >=2.11.3", "fastapi >= 0.63.0", "uvicorn >= 0.13.4", ] [tool.isort] profile = "black" known_third_party = ["click", "pid", "humanfriendly"] [tool.flit.scripts] autoxtrabackup = "mysql_autoxtrabackup.autoxtrabackup:all_procedure" ================================================ FILE: scripts/api_calls/delete_all_backups.sh ================================================ #!/usr/bin/env bash curl -X DELETE http://127.0.0.1:5555/delete ================================================ FILE: scripts/api_calls/get_all_backups.sh ================================================ #!/usr/bin/env bash curl http://127.0.0.1:5555/backups ================================================ FILE: scripts/api_calls/prepare_backup.sh ================================================ #!/usr/bin/env bash curl -X POST http://127.0.0.1:5555/prepare ================================================ FILE: scripts/api_calls/take_backup.sh ================================================ #!/usr/bin/env bash curl -X POST http://127.0.0.1:5555/backup ================================================ FILE: scripts/build-docs.sh ================================================ #!/usr/bin/env bash set -e set -x make html ================================================ FILE: scripts/format-imports.sh ================================================ #!/bin/sh -e set -x # Sort imports one per line, so autoflake can remove unused imports isort --recursive mysql_autoxtrabackup tests docs scripts --force-single-line-imports sh ./scripts/format.sh ================================================ FILE: scripts/format.sh ================================================ #!/bin/sh -e set -x autoflake --remove-all-unused-imports --recursive --remove-unused-variables --in-place mysql_autoxtrabackup docs scripts tests --exclude=__init__.py black mysql_autoxtrabackup docs scripts tests isort --recursive mysql_autoxtrabackup docs scripts tests ================================================ FILE: scripts/lint.sh ================================================ #!/usr/bin/env bash set -e set -x mypy mysql_autoxtrabackup flake8 mysql_autoxtrabackup tests black mysql_autoxtrabackup tests --check isort --recursive mysql_autoxtrabackup tests docs scripts --check-only ================================================ FILE: scripts/netlify-docs.sh ================================================ #!/usr/bin/env bash set -x set -e # Install pip cd /tmp curl https://bootstrap.pypa.io/pip/3.5/get-pip.py -o get-pip.py python3 get-pip.py --user cd - # Install Flit to be able to install all python3 -m pip install --user flit # Install with Flit python3 -m flit install --user --deps=production # Finally, run sphinx make python3 -m make html ================================================ FILE: scripts/publish.sh ================================================ #!/usr/bin/env bash set -e flit publish ================================================ FILE: tests/Dockerfile ================================================ FROM mysql/mysql-server:8.0 USER root WORKDIR /opt RUN yum install -y git RUN yum install -y python3 RUN yum install -y vim RUN yum install -y perl RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm RUN yum install -y https://repo.percona.com/yum/percona-release-latest.noarch.rpm RUN yum install -y libev RUN percona-release enable-only tools RUN yum install -y --exclude=Percona-Server\* percona-xtrabackup-80 RUN yum install -y qpress RUN yum install -y python3-pip RUN cd /opt && \ git clone https://github.com/sstephenson/bats.git && \ cd bats && \ ./install.sh /usr/local ARG GIT_BRANCH_NAME RUN cd /opt && \ git clone -b $GIT_BRANCH_NAME https://github.com/ShahriyarR/MySQL-AutoXtraBackup.git && \ cd /opt/MySQL-AutoXtraBackup && \ python3 setup.py install RUN yum groupinstall -y "Development Tools" RUN yum -y install python3-devel.x86_64 --enablerepo=rhel-7-server-optional-rpms RUN cd /opt/MySQL-AutoXtraBackup/test && \ pip3 install -r requirements.txt EXPOSE 8080 RUN cd /opt/MySQL-AutoXtraBackup && \ git pull && \ pipenv --python `which python3` install WORKDIR /opt/MySQL-AutoXtraBackup RUN cd /opt/MySQL-AutoXtraBackup && git pull RUN pip3 install uvicorn RUN pip3 install fastapi COPY entrypoint.sh / RUN chmod +x /entrypoint.sh ENTRYPOINT ["./entrypoint.sh"] #CMD ["uvicorn", "api.main:app", "--port", "8080"] ================================================ FILE: tests/README.md ================================================ # The place for testing this project. All you need is to checkout to your test branch and run docker. ================================================ FILE: tests/__init__.py ================================================ ================================================ FILE: tests/conftest.py ================================================ import pytest from fastapi.testclient import TestClient from mysql_autoxtrabackup.api.main import app from mysql_autoxtrabackup.backup_backup.backuper import Backup bck_obj = Backup() client = TestClient(app) @pytest.fixture() def return_bck_obj(): return bck_obj @pytest.fixture() def fastapi_client(): return client ================================================ FILE: tests/entrypoint.sh ================================================ #!/bin/bash # Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA set -e echo "[Entrypoint] MySQL Docker Image 8.0.20-1.1.16" echo "[Entrypoint] Modified by Shako(mysql.az)" # Fetch value from server config # We use mysqld --verbose --help instead of my_print_defaults because the # latter only show values present in config files, and not server defaults _get_config() { local conf="$1"; shift "$@" --verbose --help 2>/dev/null | grep "^$conf" | awk '$1 == "'"$conf"'" { print $2; exit }' } # If command starts with an option, prepend mysqld # This allows users to add command-line options without # needing to specify the "mysqld" command if [ "${1:0:1}" = '-' ]; then set -- mysqld "$@" fi if [ "$1" = 'mysqld' ]; then # Test that the server can start. We redirect stdout to /dev/null so # only the error messages are left. result=0 output=$("$@" --validate-config) || result=$? if [ ! "$result" = "0" ]; then echo >&2 '[Entrypoint] ERROR: Unable to start MySQL. Please check your configuration.' echo >&2 "[Entrypoint] $output" exit 1 fi # Get config DATADIR="$(_get_config 'datadir' "$@")" SOCKET="$(_get_config 'socket' "$@")" if [ -n "$MYSQL_LOG_CONSOLE" ] || [ -n "console" ]; then # Don't touch bind-mounted config files if ! cat /proc/1/mounts | grep "etc/my.cnf"; then sed -i 's/^log-error=/#&/' /etc/my.cnf fi fi if [ ! -d "$DATADIR/mysql" ]; then # If the password variable is a filename we use the contents of the file. We # read this first to make sure that a proper error is generated for empty files. if [ -f "$MYSQL_ROOT_PASSWORD" ]; then MYSQL_ROOT_PASSWORD="$(cat $MYSQL_ROOT_PASSWORD)" if [ -z "$MYSQL_ROOT_PASSWORD" ]; then echo >&2 '[Entrypoint] Empty MYSQL_ROOT_PASSWORD file specified.' exit 1 fi fi if [ -z "$MYSQL_ROOT_PASSWORD" -a -z "$MYSQL_ALLOW_EMPTY_PASSWORD" -a -z "$MYSQL_RANDOM_ROOT_PASSWORD" ]; then echo >&2 '[Entrypoint] No password option specified for new database.' echo >&2 '[Entrypoint] A random onetime password will be generated.' MYSQL_RANDOM_ROOT_PASSWORD=true MYSQL_ONETIME_PASSWORD=true fi mkdir -p "$DATADIR" chown -R mysql:mysql "$DATADIR" echo '[Entrypoint] Initializing database' "$@" --initialize-insecure echo '[Entrypoint] Database initialized' "$@" --daemonize --skip-networking --socket="$SOCKET" # To avoid using password on commandline, put it in a temporary file. # The file is only populated when and if the root password is set. PASSFILE=$(mktemp -u /var/lib/mysql-files/XXXXXXXXXX) install /dev/null -m0600 -omysql -gmysql "$PASSFILE" # Define the client command used throughout the script # "SET @@SESSION.SQL_LOG_BIN=0;" is required for products like group replication to work properly mysql=( mysql --defaults-extra-file="$PASSFILE" --protocol=socket -uroot -hlocalhost --socket="$SOCKET" --init-command="SET @@SESSION.SQL_LOG_BIN=0;") if [ ! -z "" ]; then for i in {30..0}; do if mysqladmin --socket="$SOCKET" ping &>/dev/null; then break fi echo '[Entrypoint] Waiting for server...' sleep 1 done if [ "$i" = 0 ]; then echo >&2 '[Entrypoint] Timeout during MySQL init.' exit 1 fi fi mysql_tzinfo_to_sql /usr/share/zoneinfo | "${mysql[@]}" mysql if [ ! -z "$MYSQL_RANDOM_ROOT_PASSWORD" ]; then MYSQL_ROOT_PASSWORD="$(pwmake 128)" echo "[Entrypoint] GENERATED ROOT PASSWORD: $MYSQL_ROOT_PASSWORD" fi if [ -z "$MYSQL_ROOT_HOST" ]; then ROOTCREATE="ALTER USER 'root'@'localhost' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}';" else ROOTCREATE="ALTER USER 'root'@'localhost' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}'; \ CREATE USER 'root'@'${MYSQL_ROOT_HOST}' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}'; \ GRANT ALL ON *.* TO 'root'@'${MYSQL_ROOT_HOST}' WITH GRANT OPTION ; \ GRANT PROXY ON ''@'' TO 'root'@'${MYSQL_ROOT_HOST}' WITH GRANT OPTION ;" fi "${mysql[@]}" <<-EOSQL DELETE FROM mysql.user WHERE user NOT IN ('mysql.infoschema', 'mysql.session', 'mysql.sys', 'root') OR host NOT IN ('localhost'); CREATE USER 'healthchecker'@'localhost' IDENTIFIED BY 'healthcheckpass'; ${ROOTCREATE} FLUSH PRIVILEGES ; EOSQL if [ ! -z "$MYSQL_ROOT_PASSWORD" ]; then # Put the password into the temporary config file cat >"$PASSFILE" < "$SQL" ALTER USER 'root'@'localhost' IDENTIFIED BY '12345'; #ALTER USER 'root'@'${MYSQL_ROOT_HOST}' PASSWORD EXPIRE; #ALTER USER 'root'@'localhost' PASSWORD EXPIRE; EOF else cat << EOF > "$SQL" #ALTER USER 'root'@'localhost' PASSWORD EXPIRE; ALTER USER 'root'@'localhost' IDENTIFIED BY '12345'; EOF fi set -- "$@" --init-file="$SQL" unset SQL fi fi echo echo '[Entrypoint] MySQL init process done. Ready for start up.' echo fi # Used by healthcheck to make sure it doesn't mistakenly report container # healthy during startup # Put the password into the temporary config file touch /healthcheck.cnf cat >"/healthcheck.cnf" <