[ { "path": ".deepsource.toml", "content": "version = 1\n\n[[analyzers]]\nname = \"python\"\nenabled = true\n\n [analyzers.meta]\n runtime_version = \"3.x.x\"" }, { "path": ".flake8", "content": "[flake8]\nmax-line-length = 88\nselect = C,E,F,W,B,B9\nignore = E203, E501, W503\nexclude = __init__.py\n" }, { "path": ".github/workflows/publish.yml", "content": "name: Publish\n\non:\n release:\n types:\n - created\n\njobs:\n publish:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v2\n - name: Set up Python\n uses: actions/setup-python@v1\n with:\n python-version: \"3.6\"\n - name: Install Flit\n run: pip install flit\n - name: Install Dependencies\n run: flit install --symlink\n - name: Publish\n env:\n FLIT_USERNAME: ${{ secrets.FLIT_USERNAME }}\n FLIT_PASSWORD: ${{ secrets.FLIT_PASSWORD }}\n FLIT_INDEX_URL: ${{ secrets.FLIT_INDEX_URL }}\n run: bash scripts/publish.sh\n" }, { "path": ".gitignore", "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nenv/\nbuild/\n_build\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\n*.egg-info/\n.installed.cfg\n*.egg\n.vscode\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit tests / coverage reports\nhtmlcov/\n.tox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*,cover\n.hypothesis/\n\n# Translations\n*.mo\n*.pot\n\n# PyCharm stuff\n.idea\n.vagrant\n\n# Django stuff:\n*.log\nlocal_settings.py\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# pyenv\n.python-version\n\n# celery beat schedule file\ncelerybeat-schedule\n\n# dotenv\n.env\n\n# virtualenv\n.venv\nvenv/\nENV/\n\n# Spyder project settings\n.spyderproject\n\n# Rope project settings\n.ropeproject\n\n# Local Netlify folder\n.netlify" }, { "path": "Dockerfile", "content": "FROM tiangolo/uvicorn-gunicorn-fastapi:python3.8\n\nWORKDIR /app\n\nRUN git clone https://github.com/sstephenson/bats.git && \\\n cd bats && \\\n ./install.sh /usr/local\n\nRUN apt-get update && apt-get install -y lsb-release\nRUN apt install -y libncurses5\nRUN apt-get install -y vim\nRUN wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb\nRUN dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb\nRUN percona-release enable-only tools release\nRUN apt-get update\nRUN apt-get install -y percona-xtrabackup-80\nRUN apt-get install -y qpress\n\nCOPY ./ /app\nRUN pip install --upgrade pip && \\\n pip install -r requirements.txt\n\nRUN python3 setup.py install\n\nENV MODULE_NAME=\"api.main\"\n" }, { "path": "HISTORY.md", "content": "## v2.0.2 (2021-05-06)\n\n* Increased code coverage and did code base refactoring, #444 by @shahriyarr\n\n## v2.0.1 (2021-05-05)\n\n* Added change handler for each release(so-called HISTORY.md), #442 by @shahriyarr\n* Added new version.py file for getting the exact version from one place, #440 by @shahriyarr\n* New option --run-server was added to start API server, #437 by @shahriyarr" }, { "path": "LICENSE", "content": "The MIT License (MIT)\n\nCopyright (c) 2016 Shahriyar Rzayev\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "README.md", "content": "MySQL-AutoXtrabackup\n====================\n\nMySQL AutoXtrabackup commandline tool written in Python 3.\nThe source code fully typed with hints - structured the project mostly similar to\n[FastAPI](https://fastapi.tiangolo.com/) and [Pydantic](https://github.com/samuelcolvin/pydantic)\n\nFor community from Azerbaijan MySQL User Community: [Python Azerbaijan Community](https://www.facebook.com/groups/python.az).\n\nFor any question please open an issue here.\n\nWhat this project is about?\n---------------------------\n\nThe idea for this tool, came from my hard times after accidentally\ndeleting the table data.\nThere was a full backup and 12 incremental backups.\nIt took me 20 minutes to prepare necessary commands for preparing\nbackups. If you have compressed + encrypted backups you need also,\ndecrypt and decompress, which is going to add extra time for preparing\nbackups. Then I decided to automate this process. In other words,\npreparing necessary commands for backup and prepare stage were\nautomated.\n\nWe have nice CLI with necessary options:\n\n```\nautoxtrabackup --help\nUsage: autoxtrabackup [OPTIONS]\n\nOptions:\n --dry-run Enable the dry run.\n --prepare Prepare/recover backups.\n --run-server Start the FastAPI app for serving API\n --backup Take full and incremental backups.\n --version Version information.\n --defaults-file TEXT Read options from the given file [default: /\n home/shako/.autoxtrabackup/autoxtrabackup.cn\n f]\n\n --tag TEXT Pass the tag string for each backup\n --show-tags Show backup tags and exit\n -v, --verbose Be verbose (print to console)\n -lf, --log-file TEXT Set log file [default: /home/shako/.autoxtr\n abackup/autoxtrabackup.log]\n\n -l, --log, --log-level [DEBUG|INFO|WARNING|ERROR|CRITICAL]\n Set log level [default: INFO]\n --log-file-max-bytes INTEGER Set log file max size in bytes [default:\n 1073741824]\n\n --log-file-backup-count INTEGER\n Set log file backup count [default: 7]\n --help Print help message and exit.\n```\n\n\nIf you think, CLI is not for you. We have experimental feature where you can start API server \nand take backups using API call(ATTENTION: FastAPI involved)\n\n```\nsudo `which autoxtrabackup` --run-server\nINFO: Started server process [30238]\nINFO: Waiting for application startup.\napp started\nINFO: Application startup complete.\nINFO: Uvicorn running on http://127.0.0.1:5555 (Press CTRL+C to quit)\n```\n\n```\n$ curl -X POST http://127.0.0.1:5555/backup\n{\"result\":\"Successfully finished the backup process\"}\n```\n\nFor the rest please read the full documentation.\n\nDevelopment:\n-------------------\n\nCurrent major version is >= 2.0 - so if you want to help, please do changes on this branch and then kindly send PR :)\nI also encourage you to upgrade from older version as the code base fully updated.\nDo you have an idea, question please open an issue.\n\nRead full documentation here:\n----------------------------------------------\n\n[**MySQL-AutoXtrabackup documentation!**](https://autoxtrabackup.azepug.az/)\n" }, { "path": "changes/README.md", "content": "# Pending Changes\n\nThis directory contains files describing changes to `mysql-autoxtrabackup` since the last release.\n\nIf you're creating a pull request, please add a new file to this directory called\n`-.md`. It should be formatted as a single paragraph of markdown\n\nThe contents of this file will be used to update `HISTORY.md` before the next release.\n" }, { "path": "changes/make_history.py", "content": "#!/usr/bin/env python3\nimport re\nimport sys\nfrom datetime import date\nfrom importlib.machinery import SourceFileLoader\nfrom pathlib import Path\n\nTHIS_DIR = Path(__file__).parent\nname_regex = re.compile(r'(\\d+)-(.*?)\\.md')\nbullet_list = []\n\nfor p in THIS_DIR.glob('*.md'):\n if p.name == 'README.md':\n continue\n m = name_regex.fullmatch(p.name)\n if not m:\n raise RuntimeError(f'{p.name!r}: invalid change file name')\n gh_id, creator = m.groups()\n content = p.read_text().replace('\\r\\n', '\\n').strip('\\n. ')\n if '\\n\\n' in content:\n raise RuntimeError(f'{p.name!r}: content includes multiple paragraphs')\n content = content.replace('\\n', '\\n ')\n priority = 0\n if '**breaking change' in content.lower():\n priority = 2\n elif content.startswith('**'):\n priority = 1\n bullet_list.append((priority, int(gh_id), f'* {content}, #{gh_id} by @{creator}'))\n\nif not bullet_list:\n print('no changes found')\n sys.exit(0)\n\nversion = SourceFileLoader('version', 'mysql_autoxtrabackup/utils/version.py').load_module()\nchunk_title = f'v{version.VERSION} ({date.today():%Y-%m-%d})'\nnew_chunk = '## {}\\n\\n{}\\n\\n'.format(chunk_title, '\\n'.join(c for *_, c in sorted(bullet_list, reverse=True)))\n\nprint(f'{chunk_title}...{len(bullet_list)} items')\nhistory_path = THIS_DIR / '..' / 'HISTORY.md'\nhistory = new_chunk + history_path.read_text()\n\nhistory_path.write_text(history)\nfor p in THIS_DIR.glob('*.md'):\n if p.name != 'README.md':\n p.unlink()\n\nprint(\n 'changes deleted and HISTORY.md successfully updated, to reset use:\\n\\n'\n ' git checkout -- changes/*-*.md HISTORY.md\\n'\n)\n" }, { "path": "docker-compose-test.yaml", "content": "version: \"3.7\"\n\nservices:\n api_v1:\n build:\n context: tests\n dockerfile: Dockerfile\n args:\n GIT_BRANCH_NAME: ${GIT_BRANCH_NAME}\n container_name: autoxtrabackup_apiv1_test\n volumes:\n - ./tests/entrypoint.sh:/entrypoint.sh\n - /dev/log:/dev/log\n ports:\n - \"8080:8080\"\n network_mode: host\n\n\n" }, { "path": "docker-compose.yaml", "content": "version: \"3.7\"\n\nservices:\n api_v1:\n build:\n context: .\n dockerfile: Dockerfile\n container_name: autoxtrabackup_apiv1\n volumes:\n - /dev/log:/dev/log\n - /var/lib/mysql:/var/lib/mysql\n - /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock\n - /usr/bin/mysqladmin:/usr/bin/mysqladmin\n - /usr/bin/mysql:/usr/bin/mysql\n ports:\n - \"80:80\"\n network_mode: host\n\n\n" }, { "path": "docs/Makefile", "content": "# Minimal makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHINXBUILD = sphinx-build\nSPHINXPROJ = MySQLAutoXtrabackup\nSOURCEDIR = .\nBUILDDIR = _build\n\n# Put it first so that \"make\" without argument is like \"make help\".\nhelp:\n\t@$(SPHINXBUILD) -M help \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)\n\n.PHONY: help Makefile\n\n# Catch-all target: route all unknown targets to Sphinx using the new\n# \"make mode\" option. $(O) is meant as a shortcut for $(SPHINXOPTS).\n%: Makefile\n\t@$(SPHINXBUILD) -M $@ \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)" }, { "path": "docs/advance_features.rst", "content": "Advance features\n=================\n\nCompressed backups\n------------------\n\nTo enable compression support just uncomment the options under\n[Compress] category inside main configuration file:\n\n::\n\n [Compress]\n #Optional\n #Enable only if you want to use compression.\n compress=quicklz\n compress_chunk_size=65536\n compress_threads=4\n decompress=TRUE\n #Enable if you want to remove .qp files after decompression.(Available from PXB 2.3.7 and 2.4.6)\n remove_original=FALSE\n\n\nEncrypted backups\n-----------------\n\nTo enable encryption support uncomment the options under [Encryption]\ncategory:\n\n::\n\n [Encrypt]\n #optional\n #enable only if you want to create encrypted backups\n xbcrypt = /usr/bin/xbcrypt\n encrypt = AES256\n #please note that --encrypt-key and --encrypt-key-file are mutually exclusive\n encrypt_key = VVTBwgM4UhwkTTV98fhuj+D1zyWoA89K\n #encrypt_key_file = /path/to/file/with_encrypt_key\n encrypt_threads = 4\n encrypt_chunk_size = 65536\n decrypt = AES256\n #enable if you want to remove .qp files after decompression.(Available from PXB 2.3.7 and 2.4.6)\n remove_original = FALSE\n\n\nPartial backups\n---------------\n\nIt is possible to take partial full and incremental backups. The idea is, to take specified table(or database) as full backup,\nthen to take incremental backups based on this one table.\nYou can achieve this by enabling ``partial_list`` option from config file:\n\n\n::\n\n [Backup]\n #optional: set pid directory\n pid_dir = /tmp/MySQL-AutoXtraBackup\n tmp_dir = /home/shako/XB_TEST/mysql_datadirs\n #optional: set warning if pid of backup us running for longer than x\n pid_runtime_warning = 2 Hours\n backup_dir = /home/shako/XB_TEST/backup_dir\n backup_tool = /usr/bin/xtrabackup\n #optional: specify different path/version of xtrabackup here for prepare\n #prepare_tool =\n xtra_prepare = --apply-log-only\n #optional: pass additional options for backup stage\n #xtra_backup = --compact\n #optional: pass additional options for prepare stage\n #xtra_prepare_options = --rebuild-indexes\n #optional: pass general additional options; it will go to both for backup and prepare\n #xtra_options = --binlog-info=ON --galera-info\n #optional: set archive and rotation\n #archive_dir = /home/shako/XB_TEST/backup_archives\n #prepare_archive = 1\n #move_archive = 0\n #full_backup_interval = 1 day\n #archive_max_size = 100GiB\n #archive_max_duration = 4 Days\n #optional: warning(enable this if you want to take partial backups). specify database names or table names.\n #partial_list = test.t1 test.t2 dbtest\n\n\nRun it and notice that backup command has changed (see ``--databases`` option for xtrabackup command):\n\nIn the same way you can take incremental backup of specified tables.\n\nThe prepare process is the same as ordinary prepare, just run autoxtrabackup with ``--prepare`` option.\n\nDecompressing and Decrypting backups\n------------------------------------\n\nWe took Compressed and Encrypted backups.\nIt is time to prepare them.\nautoxtrabackup will prepare all backups automatically, by first decrypting then\ndecompressing step-by-step.\nAll backups first will be decrypted then decompressed and then\nprepared.\nYou can also optionally enable ``--remove-original`` option to\nremove ``.xbcrypt`` and ``.qp`` files from backup directory during prepare\nprocess.\nRead about this option here -> `--remove-original `_\n\nautoxtrabackup with --dry_run option\n------------------------------------\n\nFor testing purposes or just to show what is going on, with autoxtrabackup backup and prepare steps.\nYou can append ``--dry_run`` option, to show commands but not to run them.\nTaking backup:\n\n::\n\n\n $ sudo autoxtrabackup -v -lf /home/shahriyar.rzaev/autoxtrabackup_2_3_5_6.log -l DEBUG \\\n --defaults-file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.cnf --backup --dry_run\n\n\nPreparing backups:\n\n::\n\n\n $ sudo autoxtrabackup -v -lf /home/shahriyar.rzaev/autoxtrabackup_2_3_5_6.log -l DEBUG \\\n --defaults-file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.cnf --prepare --dry_run\n\n\nThe end.\n" }, { "path": "docs/api.rst", "content": "Here is the basic ideas how to use API calls to operate on backups\n==================================================================\n\nCurrently, we have only 4 endpoints.\n\n.. image:: doc_api.png\n :width: 400\n :alt: DOC API\n\n\nSample cURLs:\n-------------\n\nAfter starting API server:\n\n::\n\n $ sudo `which autoxtrabackup` --run-server\n\nOf course you can use the Web UI for taking backups, but in case you want to send requests,\nbelow I have provided sample cURLs.\n\nTaking backup:\n\n::\n\n $ curl -X POST http://127.0.0.1:5555/backup\n {\"result\":\"Successfully finished the backup process\"}\n\nPreparing backup:\n\n::\n\n $ curl -X POST http://127.0.0.1:5555/prepare\n {\"result\":\"Successfully prepared all the backups\"}\n\nListing backups:\n\n::\n\n $ curl http://127.0.0.1:5555/backups\n {\"backups\":{\"full\":[{\"2021-05-05_18-37-36\":\"Full-Prepared\"}],\"inc\":[\"2021-05-05_18-38-19\"]}}\n\n\nRemoving backups:\n\n::\n\n $ curl -X DELETE http://127.0.0.1:5555/delete\n {\"result\":\"There is no backups or backups removed successfully\"}" }, { "path": "docs/backup_tags.rst", "content": "Backup Tags\n===========\n\nThe backup tags actually is a result of feature requests by community member `Yusif Yusifli `_.\nRead discussions about feature requests below:\n\n`#163 `_.\n`#164 `_.\n`#210 `_.\n\nSo basically how to take backups and create a tag for it?\n\nTaking full backup:\n-------------------\n\n::\n\n $ sudo autoxtrabackup --tag=\"My Full backup\" -v \\\n -lf /home/shahriyar.rzaev/autoxtrabackup_2_4_5_7.log \\\n -l DEBUG --defaults-file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf --backup\n\nTaking incremental one:\n-----------------------\n\n::\n\n $ autoxtrabackup --tag=\"First incremental backup\" -v \\\n -lf /home/shahriyar.rzaev/autoxtrabackup_2_4_5_7.log \\\n -l DEBUG --defaults-file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf --backup\n\nTaking second incremental:\n--------------------------\n\n::\n\n $ autoxtrabackup --tag=\"Second incremental backup\" -v \\\n -lf /home/shahriyar.rzaev/autoxtrabackup_2_4_5_7.log \\\n -l DEBUG --defaults-file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf --backup\n\nTo list available tags(backups):\n--------------------------------\nFor eg, if full backup failed, the result will be something like this:\n\n::\n\n $ sudo autoxtrabackup --show_tags \\\n --defaults-file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf\n Backup \tType\tStatus\tCompletion_time \tSize\tTAG\n ----------------------------------------------------------------------------------\n 2017-12-14_12-01-11\tFull\tFAILED\t2017-12-14_12-01-11\t4,0K\t'My Full backup'\n\n\nbackup_tags.txt file\n--------------------\nAll tags are stored inside backup_tags.txt file, which will be created in backup directory:\n\n::\n\n [vagrant@localhost ps_5_7_x_2_4]$ ls\n backup_tags.txt full inc\n [vagrant@localhost ps_5_7_x_2_4]$ cat backup_tags.txt\n $ cat backup_tags.txt\n 2017-12-14_12-01-11\tFull\tFAILED\t2017-12-14_12-01-11\t4,0K\t'My Full backup'\n\nPreparing with tag\n------------------\n\nThat's very nice. Now you can use those tags to prepare your backups.\nSay you want to prepare only first incremental and ignore second one(or others).\n\n::\n\n $ autoxtrabackup --tag=\"First incremental backup\" -v \\\n -lf /home/shahriyar.rzaev/autoxtrabackup_2_4_5_7.log \\\n -l DEBUG --defaults-file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf --prepare\n\nAs you see it will mark given incremental backup as last one, because you have specified it in --tag option.\n\n**If you pass wrong/non-existing tag name the tool will raise RuntimeError.**\n\n" }, { "path": "docs/basic_features.rst", "content": "Basic features\n==============\n\nBackup\n------\n\nYes you are right, this tool is for taking backups.\nIt should take care for automating this process for you.\nYou can specify the backup directory in config file (default ~/.autoxtrabackup/autoxtrabackup.cnf) under [Backup] category.\nSo you have prepared your config and now you are ready for start.\n\nThe command for taking full backup with DEBUG enabled, i.e first run of the tool.\n\n::\n\n $ sudo autoxtrabackup -v -lf /home/shako/.autoxtrabackup/autoxtrabackup.log \\\n -l DEBUG --defaults-file=/home/shako/.autoxtrabackup/autoxtrabackup.cnf --backup\n\n\nThe result of second run; it will take an incremental backup.\n\n::\n\n $ sudo autoxtrabackup -v -lf /home/shako/.autoxtrabackup/autoxtrabackup.log \\\n -l DEBUG --defaults-file=/home/shako/.autoxtrabackup/autoxtrabackup.cnf --backup\n\n\n\n\nYou will have 2 separate folders inside backup directory:\n\n::\n\n (.venv) shako@shako-localhost:~/XB_TEST$ cd backup_dir/\n (.venv) shako@shako-localhost:~/XB_TEST/backup_dir$ ls\n full inc\n\n\n\nWe took full backup and it should be under the ``full`` directory:\n\n::\n\n (.venv) shako@shako-localhost:~/XB_TEST/backup_dir$ ls full/\n 2019-01-20_13-52-07\n\n\nIncremental backups are inside ``inc`` directory:\n\n::\n\n (.venv) shako@shako-localhost:~/XB_TEST/backup_dir$ ls inc/\n 2019-01-20_13-53-59\n\nIf you want more incremental backups just run the same command again and again.\n\n\nPrepare\n-------\nFor preparing backups just use --prepare option. For our case we have a\nfull and 2 incremental backups:\n\n::\n\n (.venv) shako@shako-localhost:~/XB_TEST/backup_dir$ ls full/\n 2019-01-20_13-52-07\n (.venv) shako@shako-localhost:~/XB_TEST/backup_dir$ ls inc/\n 2019-01-20_13-53-59 2019-01-20_13-56-42\n\n\nAll backups will be prepared\nautomatically.\n\nYou are going to have 3 options to choose:\n\n1. Only prepare backups.\n2. Prepare backups and restore immediately.\n3. Restore from already prepared backup.\n\nFor now let's choose 1:\n\n::\n\n $ sudo `which autoxtrabackup` --prepare -v -l DEBUG\n\n\nThat's it. Your backup is ready to restore/recovery.\n" }, { "path": "docs/basic_overview.rst", "content": "Basic Overview\n==============\n\nProject Structure\n-----------------\n\nXtraBackup is a powerful open-source hot online backup tool for MySQL\nfrom Percona. This script is using XtraBackup for full and incremental\nbackups, also for preparing backups, as well as to restore. Here is project path tree:\n\n::\n\n * mysql_autoxtrabackup -- source directory\n * mysql_autoxtrabackup/backup_backup -- Full and Incremental backup taker script.\n * mysql_autoxtrabackup/backup_prepare -- Backup prepare and restore script.\n * mysql_autoxtrabackup/general_conf -- All-in-one config file's and config reader class folder.\n * mysql_autoxtrabackup/process_runner -- The directory for process runner script.\n * mysql_autoxtrabackup/autoxtrabackup.py -- Commandline Tool provider script.\n * mysql_autoxtrabackup/api -- The API server written in FastAPI.\n * tests -- The directory for test things.\n * ~/.autoxtrabackup/autoxtrabackup.cnf -- Config file will be created during setup.\n\n\nAvailable Options\n-----------------\n\n.. code-block:: shell\n\n $ autoxtrabackup --help\n Usage: autoxtrabackup [OPTIONS]\n\n Options:\n --dry-run Enable the dry run.\n --prepare Prepare/recover backups.\n --run-server Start the FastAPI app for serving API\n --backup Take full and incremental backups.\n --version Version information.\n --defaults-file TEXT Read options from the given file [default: /\n home/shako/.autoxtrabackup/autoxtrabackup.cn\n f]\n\n --tag TEXT Pass the tag string for each backup\n --show-tags Show backup tags and exit\n -v, --verbose Be verbose (print to console)\n -lf, --log-file TEXT Set log file [default: /home/shako/.autoxtr\n abackup/autoxtrabackup.log]\n\n -l, --log, --log-level [DEBUG|INFO|WARNING|ERROR|CRITICAL]\n Set log level [default: INFO]\n --log-file-max-bytes INTEGER Set log file max size in bytes [default:\n 1073741824]\n\n --log-file-backup-count INTEGER\n Set log file backup count [default: 7]\n --help Print help message and exit.\n\n\n\n\n\n\nUsage\n-----\n\n::\n\n 1. Install it.\n 2. Edit ~/.autoxtrabackup/autoxtrabackup.cnf(default config) file to reflect your environment or create your own config.\n 3. Pass config file to autoxtrabackup with --defaults_file option(if you are not using default config) and begin to backup/prepare/restore.\n\n\n\n\nLogging\n--------\n\nThe logging mechanism is using Python3 logging.\nIt lets to log directly to console and also to file.\n" }, { "path": "docs/conf.py", "content": "# -*- coding: utf-8 -*-\n#\n# MySQL AutoXtrabackup documentation build configuration file, created by\n# sphinx-quickstart on Fri Feb 24 23:39:55 2017.\n#\n# This file is execfile()d with the current directory set to its\n# containing dir.\n#\n# Note that not all possible configuration values are present in this\n# autogenerated file.\n#\n# All configuration values have a default; values that are commented out\n# serve to show the default.\n\nimport os\nimport sys\n\n# If extensions (or modules to document with autodoc) are in another directory,\n# add these directories to sys.path here. If the directory is relative to the\n# documentation root, use os.path.abspath to make it absolute, like shown here.\n#\nimport sphinx_rtd_theme\n\nfrom mysql_autoxtrabackup.utils.version import VERSION\n\nsys.path.insert(0, os.path.abspath(\"../mysql_autoxtrabackup\"))\n\n# -- General configuration ------------------------------------------------\n\n# If your documentation needs a minimal Sphinx version, state it here.\n#\n# needs_sphinx = '1.0'\n\n# Add any Sphinx extension module names here, as strings. They can be\n# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom\n# ones.\nextensions = [\n \"sphinx.ext.autodoc\",\n \"sphinx.ext.napoleon\",\n \"sphinx.ext.viewcode\",\n]\n\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = [\"_templates\"]\n\n# The suffix(es) of source filenames.\n# You can specify multiple suffix as a list of string:\n#\n# source_suffix = ['.rst', '.md']\nsource_suffix = \".rst\"\n\n# The master toctree document.\nmaster_doc = \"index\"\n\n# General information about the project.\nproject = u\"MySQL-AutoXtrabackup\"\ncopyright = u\"2020, Shahriyar Rzayev\"\nauthor = u\"Shahriyar Rzayev\"\n\n# The version info for the project you're documenting, acts as replacement for\n# |version| and |release|, also used in various other places throughout the\n# built documents.\n#\n# The short X.Y version.\nversion = u\"{}\".format(VERSION)\n# The full version, including alpha/beta/rc tags.\nrelease = u\"{}\".format(VERSION)\n\n# The language for content autogenerated by Sphinx. Refer to documentation\n# for a list of supported languages.\n#\n# This is also used if you do content translation via gettext catalogs.\n# Usually you set \"language\" from the command line for these cases.\nlanguage = None\n\n# List of patterns, relative to source directory, that match files and\n# directories to ignore when looking for source files.\n# This patterns also effect to html_static_path and html_extra_path\nexclude_patterns = [\"_build\", \"Thumbs.db\", \".DS_Store\"]\n\n# The name of the Pygments (syntax highlighting) style to use.\npygments_style = \"sphinx\"\n\n# If true, `todo` and `todoList` produce output, else they produce nothing.\ntodo_include_todos = False\n\n\n# -- Options for HTML output ----------------------------------------------\n\n# The theme to use for HTML and HTML Help pages. See the documentation for\n# a list of builtin themes.\n#\nhtml_theme = \"sphinx_rtd_theme\"\n\nhtml_theme_path = [sphinx_rtd_theme.get_html_theme_path()]\n\n# Theme options are theme-specific and customize the look and feel of a theme\n# further. For a list of options available for each theme, see the\n# documentation.\n#\n# html_theme_options = {}\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\n\n# html_static_path = ['_static']\n\nhtml_static_path = []\n\n\n# -- Options for HTMLHelp output ------------------------------------------\n\n# Output file base name for HTML help builder.\nhtmlhelp_basename = \"MySQLAutoXtrabackupdoc\"\n\n\n# -- Options for LaTeX output ---------------------------------------------\n\nlatex_elements = {\n # The paper size ('letterpaper' or 'a4paper').\n #\n # 'papersize': 'letterpaper',\n # The font size ('10pt', '11pt' or '12pt').\n #\n # 'pointsize': '10pt',\n # Additional stuff for the LaTeX preamble.\n #\n # 'preamble': '',\n # Latex figure (float) alignment\n #\n # 'figure_align': 'htbp',\n}\n\n# Grouping the document tree into LaTeX files. List of tuples\n# (source start file, target name, title,\n# author, documentclass [howto, manual, or own class]).\nlatex_documents = [\n (\n master_doc,\n \"MySQLAutoXtrabackup.tex\",\n u\"MySQL AutoXtrabackup Documentation\",\n u\"Shahriyar Rzayev\",\n \"manual\",\n ),\n]\n\n\n# -- Options for manual page output ---------------------------------------\n\n# One entry per manual page. List of tuples\n# (source start file, name, description, authors, manual section).\nman_pages = [\n (\n master_doc,\n \"mysqlautoxtrabackup\",\n u\"MySQL AutoXtrabackup Documentation\",\n [author],\n 1,\n )\n]\n\n\n# -- Options for Texinfo output -------------------------------------------\n\n# Grouping the document tree into Texinfo files. List of tuples\n# (source start file, target name, title, author,\n# dir menu entry, description, category)\ntexinfo_documents = [\n (\n master_doc,\n \"MySQLAutoXtrabackup\",\n u\"MySQL AutoXtrabackup Documentation\",\n author,\n \"MySQLAutoXtrabackup\",\n \"One line description of project.\",\n \"Miscellaneous\",\n ),\n]\n" }, { "path": "docs/config_file.rst", "content": "The structure of configuration file\n===================================\n\nDefaults file explained\n-----------------------\n\nThere are some changes related to default config file.\nFirst and foremost it is renamed and now located in home of user in .autoxtrabackup folder.\nKeep in mind that config file is going to be dynamically generated during setup process.\nThe default config will be located ~/.autoxtrabackup/autoxtrabackup.cnf.\nThe available options are divided into optional and primary options.\nOptions are quite self-explanatory.\nI have tried to make them similar to existing options in XtraBackup and MySQL.\nYou can use another configuration file using ``--defaults_file`` option.\n\nLet's clarify the config file structure a bit.\n\n[MySQL]\n--------\n\nThe [MySQL] category is for specifying information about MySQL instance.\n\n::\n\n [MySQL]\n mysql=/usr/bin/mysql\n mycnf=/etc/my.cnf\n mysqladmin=/usr/bin/mysqladmin\n mysql_user=root\n mysql_password=12345\n ## Set either mysql_socket only, OR host + port. If both are set mysql_socket is used\n #mysql_socket=/var/lib/mysql/mysql.sock\n mysql_host=127.0.0.1\n mysql_port=3306\n datadir=/var/lib/mysql\n\n\n[Logging]\n---------\n\nOptions for logging mechanism of tool.(added in 1.5.4 version)\n\n::\n\n [Logging]\n #[debug,info,warning,error,critical]\n log = DEBUG\n log_file_max_bytes = 1073741824\n log_file_backup_count = 7\n\n[Backup]\n--------\n\nThe [Backup] category is for specifying information about backup/prepare process itself.\n\n::\n\n [Backup]\n #optional: set pid directory\n pid_dir = /tmp/MySQL-AutoXtraBackup\n tmp_dir = /home/shako/XB_TEST/mysql_datadirs\n #optional: set warning if pid of backup us running for longer than x\n pid_runtime_warning = 2 Hours\n backup_dir = /home/shako/XB_TEST/backup_dir\n backup_tool = /usr/bin/xtrabackup\n #optional: specify different path/version of xtrabackup here for prepare\n #prepare_tool =\n xtra_prepare = --apply-log-only\n #optional: pass additional options for backup stage\n #xtra_backup = --compact\n #optional: pass additional options for prepare stage\n #xtra_prepare_options = --rebuild-indexes\n #optional: pass general additional options; it will go to both for backup and prepare\n #xtra_options = --binlog-info=ON --galera-info\n #optional: set archive and rotation\n #archive_dir = /home/shako/XB_TEST/backup_archives\n #prepare_archive = 1\n #move_archive = 0\n #full_backup_interval = 1 day\n #archive_max_size = 100GiB\n #archive_max_duration = 4 Days\n #optional: warning(enable this if you want to take partial backups). specify database names or table names.\n #partial_list = test.t1 test.t2 dbtest\n\n+----------------------+----------+-----------------------------------------------------------------------------+\n| **Key** | Required | **Description** |\n+----------------------+----------+-----------------------------------------------------------------------------+\n| pid_dir | no | Directory where the PID file will be created in |\n+----------------------+----------+-----------------------------------------------------------------------------+\n| tmp_dir | yes | Used for moving current running mysql-datadir to when copying-back |\n| | | (restoring) an archive |\n+----------------------+----------+-----------------------------------------------------------------------------+\n| backup_dir | yes | Directory will be used for storing the backups. Subdirs ./full and ./inc |\n| | | will be created |\n+----------------------+----------+-----------------------------------------------------------------------------+\n| backup_tool | yes | Full path to Percona xtrabackup executable used when making backup |\n+----------------------+----------+-----------------------------------------------------------------------------+\n| prepare_tool | no | Full path to Percona xtrabackup executable used when preparing (restoring) |\n+----------------------+----------+-----------------------------------------------------------------------------+\n| xtra_prepare | yes | Options passed to xtrabackup when preparing. |\n| | | '--apply-log-only' is essential to allow further incremental |\n| | | backups to be made. See[1] |\n+----------------------+----------+-----------------------------------------------------------------------------+\n| xtra_backup | no | pass additional options for backup stage |\n+----------------------+----------+-----------------------------------------------------------------------------+\n| xtra_prepare_options | no | pass additional options for prepare stage |\n+----------------------+----------+-----------------------------------------------------------------------------+\n| xtra_options | no | pass general additional options; it will go to both for backup and prepare |\n+----------------------+----------+-----------------------------------------------------------------------------+\n| archive_dir | no | Directory for storing archives (tar.gz or otherwise). Cannot be inside the |\n| | | 'backupdir' above |\n+----------------------+----------+-----------------------------------------------------------------------------+\n| prepare_archive | no | Prepare backups before archiving them. |\n+----------------------+----------+-----------------------------------------------------------------------------+\n| move_archive | no | When rotating backups to archive move instead of compressing with tar.gz |\n+----------------------+----------+-----------------------------------------------------------------------------+\n| full_backup_interval | no | Maximum interval after which a new full backup will be made |\n+----------------------+----------+-----------------------------------------------------------------------------+\n| archive_max_size | no | Delete archived backups after X GiB |\n+----------------------+----------+-----------------------------------------------------------------------------+\n| archive_max_duration | no | Delete archived backups after X Days |\n+----------------------+----------+-----------------------------------------------------------------------------+\n| partial_list | no | Specify database names or table names. |\n| | | **WARNING**: Enable this if you want to take partial backups |\n+----------------------+----------+-----------------------------------------------------------------------------+\n\n[Compress]\n----------\n\nThe [Compress] category is for enabling backup compression.\n\nThe options will be passed to XtraBackup.\n\n::\n\n [Compress]\n #optional\n #enable only if you want to use compression.\n compress = quicklz\n compress_chunk_size = 65536\n compress_threads = 4\n decompress = TRUE\n #enable if you want to remove .qp files after decompression.(Available from PXB 2.3.7 and 2.4.6)\n remove_original = FALSE\n\n[Encrypt]\n---------\n\nThe [Encrypt] category is for enabling backup encryption.\n\nThe options will be passed to XtraBackup.\n\n::\n\n [Encrypt]\n #optional\n #enable only if you want to create encrypted backups\n xbcrypt = /usr/bin/xbcrypt\n encrypt = AES256\n #please note that --encrypt-key and --encrypt-key-file are mutually exclusive\n encrypt_key = VVTBwgM4UhwkTTV98fhuj+D1zyWoA89K\n #encrypt_key_file = /path/to/file/with_encrypt_key\n encrypt_threads = 4\n encrypt_chunk_size = 65536\n decrypt = AES256\n #enable if you want to remove .qp files after decompression.(Available from PXB 2.3.7 and 2.4.6)\n remove_original = FALSE\n\n[Xbstream]\n----------\n\nThe [Xbstream] category is for enabling backup streaming.\n\nThe options will be passed to XtraBackup.\n\n::\n\n [Xbstream]\n #experimental\n #enable this, if you want to stream your backups\n xbstream = /usr/bin/xbstream\n stream = xbstream\n xbstream_options = -x --parallel=100\n xbs_decrypt = 1\n # warn, enable this, if you want to stream your backups to remote host\n #remote_stream = ssh xxx.xxx.xxx.xxx\n\n\nDeprecated feature, will be removed in next releases[Do not use]\n\n::\n\n #Optional remote syncing\n #[Remote]\n #remote_conn=root@xxx.xxx.xxx.xxx\n #remote_dir=/home/sh/Documents\n\n[Commands]\n----------\n\nThe [Commands] category is for specifying some options for copy-back/restore actions.\n\n::\n\n [Commands]\n start_mysql_command=service mysql start\n stop_mysql_command=service mysql stop\n #Change user:group respectively\n chown_command=chown -R mysql:mysql\n\n\n[1]: https://www.percona.com/doc/percona-xtrabackup/LATEST/xtrabackup_bin/incremental_backups.html#preparing-the-incremental-backups\n\n" }, { "path": "docs/index.rst", "content": ".. MySQL AutoXtrabackup documentation master file, created by\n sphinx-quickstart on Fri Feb 24 23:39:55 2017.\n You can adapt this file completely to your liking, but it should at least\n contain the root `toctree` directive.\n\nWelcome to MySQL-AutoXtrabackup's documentation!\n================================================\n\n.. toctree::\n :maxdepth: 2\n\n what_is_new_in_2_0\n intro\n basic_overview\n installation\n config_file\n basic_features\n backup_tags\n advance_features\n option_reference\n api\n\n.. automodule:: backup_backup\n :members:\n\n.. automodule:: backup_prepare\n :members:\n\n.. automodule:: general_conf\n :members:\n\n.. automodule:: process_runner\n :members:\n\n.. automodule:: utils\n :members:\n\n.. automodule:: api\n :members:\n\n\nIndices and tables\n==================\n\n* :ref:`genindex`\n* :ref:`modindex`\n* :ref:`search`\n" }, { "path": "docs/installation.rst", "content": "Installation\n============\n\nSystem requirements\n-------------------\n\nFollowing packages should be already there:\n\n- Percona Xtrabackup (>= 2.3.5)\n- Python 3 (>= 3.6)\n\nInstalling MySQL-AutoXtraBackup\n-------------------------------\n\nUsing pip3:\n\n::\n\n pip3 install mysql-autoxtrabackup\n\n\nInstalling from source for development purposes:\n\n::\n\n git clone https://github.com/ShahriyarR/MySQL-AutoXtraBackup.git\n cd to MySQL-AutoXtraBackup\n flit install --symlink\n\n\nAs Percona XtraBackup requires root privileges in order to backup the MySQL server, it is convenient to install,\nautoxtrabackup globally. But, if you think no, then install it to virtualenv and then call as root/sudo :)" }, { "path": "docs/intro.rst", "content": "Intro\n=====\n\nWhat is this?\n-------------\n\nMySQL-AutoXtraBackup is a commandline tool written in Python3 based on\nPercona XtraBackup.\nIt is aimed to simplify the usage of XtraBackup in\ndaily basis and to help backup admin in certain tasks.\n\nWhy you need this?\n------------------\n\nThe idea for this tool, came from my hard times after accidentally\ndeleting the table data.\nThere was a full backup and 12 incremental backups.\nIt took me 20 minutes to prepare necessary commands for preparing\nbackups. If you have compressed + encrypted backups you need also,\ndecrypt and decompress, which is going to add extra time for preparing\nbackups. Then I decided to automate this process. In other words,\npreparing necessary commands for backup and prepare stage were\nautomated.\n\nIf you think that those reasons are not enough - Then just believe me\nyou need this :)\n\nIs it production ready?\n-----------------------\n\nWell, we have famous answer for this - \"It depends!\".\nBasically this tool is based on Percona XtraBackup and it is using XtraBackup's\nfunctionality.\nFor me, I have used it in production environment after testing for a while in test servers.\nBut to be clear, just test it enough to be confident.\n" }, { "path": "docs/option_reference.rst", "content": "Option Reference\n=================\n\nThe command line options to use:\n\n.. code-block:: shell\n\n $ autoxtrabackup --help\n Usage: autoxtrabackup [OPTIONS]\n\n Options:\n --dry-run Enable the dry run.\n --prepare Prepare/recover backups.\n --run-server Start the FastAPI app for serving API\n --backup Take full and incremental backups.\n --version Version information.\n --defaults-file TEXT Read options from the given file [default: /\n home/shako/.autoxtrabackup/autoxtrabackup.cn\n f]\n\n --tag TEXT Pass the tag string for each backup\n --show-tags Show backup tags and exit\n -v, --verbose Be verbose (print to console)\n -lf, --log-file TEXT Set log file [default: /home/shako/.autoxtr\n abackup/autoxtrabackup.log]\n\n -l, --log, --log-level [DEBUG|INFO|WARNING|ERROR|CRITICAL]\n Set log level [default: INFO]\n --log-file-max-bytes INTEGER Set log file max size in bytes [default:\n 1073741824]\n\n --log-file-backup-count INTEGER\n Set log file backup count [default: 7]\n --help Print help message and exit.\n\n\n\ndry-run\n-------\n\n--dry-run\nUse this option to enable dry running. If you enable this, actual commands will not be executed but, will only be shown.\nIt is useful, when you want to see what is going to happen, if you run the actual process.\n\nprepare\n-------\n\n--prepare\nThis option is for prepare and copy-back(recover) the backup.\n\nrun-server\n----------\n\n--run-server\nThis option is for starting API server[Defaults are localhost and port 5555]\n\nbackup\n------\n\n--backup\nThis option for taking backups. If it is first run, it will take full backup.\nIf you want incremental backups, just run same command as much as you want take incremental backups.\n\nversion\n-------\n\n--version\nPrints version information.\n\ndefaults-file\n-------------\n\n--defaults-file\nThe main config file to path to ``autoxtrabackup``. The default one is ``~/.autoxtrabackup/autoxtrabackup.cnf``.\nIn default config, the compression, encryption and streaming backups are disabled by defualt.\n\ntag\n----\n--tag\nThis option enables creation of tags for backups.\nThe backup_tags.txt file will be created and stored inside backup directory.\n\nshow-tags\n---------\nIt will show the backup tags and exit.\n\nverbose, v\n----------\n\n--verbose, -v\nThis option enables to print to console the logging messages.\n\nlog-file, lf\n------------\n\n-lf, --log-file\nPass, the path for log file, for autoxtrabackup. Default is ``~/.autoxtrabackup/autoxtrabackup.log``\n\nlog-file-backup_count\n---------------------\n\n--log_file_backup_count\nSet log file backup count. Default is 7\n\nlog-file-max-bytes\n------------------\n\n--log_file_max_bytes\nSet log file max size in bytes. Default: 1073741824 bytes.\n\nlog, log-level\n--------------\n\n-l, --log, --log-level\n\nSet the log level for tool. Can be DEBUG, INFO, WARNING, ERROR or CRITICAL. Default is DEBUG.\n\n\nhelp\n----\n\n--help\nAs name indicates.\n\n" }, { "path": "docs/what_is_new_in_2_0.rst", "content": "What is new in >= 2.0 major version?\n====================================\n\nLet me put it concise\n---------------------\n\nThe whole project was completely rewritten. It is incompatible with previous versions.\nJust imagine nearly ~6754 lines of code was thrown and only ~3200 was added.\n\nAdded mypy, black, flake8 and now our project is fully typed and has support with type checkers.\nRemoved Pipenv, setup.py and added pyproject.yaml with flit support.\nThere is a scripts folder with helper scripts if you decide to contribute.\n\nWe have experimental API built with FastAPI - to operate on backups using API calls.(experimental)" }, { "path": "mypy.ini", "content": "[mypy]\n\n# --strict\ndisallow_any_generics = True\ndisallow_subclassing_any = True\ndisallow_untyped_calls = True\ndisallow_untyped_defs = True\ndisallow_incomplete_defs = True\ncheck_untyped_defs = True\ndisallow_untyped_decorators = True\nno_implicit_optional = True\nwarn_redundant_casts = True\nwarn_unused_ignores = True\nwarn_return_any = True\nimplicit_reexport = False\nstrict_equality = True\n# --strict end\n" }, { "path": "mysql_autoxtrabackup/__init__.py", "content": "\"\"\"MySQL-AutoXtrabackup command-line tool, for automating tedious MySQL physical backups management\nusing Percona Xtrabackup\"\"\"\nfrom .utils import version\n\n__version__ = version.VERSION\n" }, { "path": "mysql_autoxtrabackup/api/__init__.py", "content": "" }, { "path": "mysql_autoxtrabackup/api/controller/__init__.py", "content": "" }, { "path": "mysql_autoxtrabackup/api/controller/controller.py", "content": "from fastapi import APIRouter, status\nfrom fastapi.responses import JSONResponse\nfrom starlette.responses import RedirectResponse\n\nfrom mysql_autoxtrabackup.backup_backup.backuper import Backup\nfrom mysql_autoxtrabackup.backup_prepare.prepare import Prepare\nfrom mysql_autoxtrabackup.utils.helpers import list_available_backups\n\nrouter = APIRouter()\n\n\n@router.get(\n \"/\",\n tags=[\"MySQL-AutoXtrabackup\"],\n response_description=\"Home Page\",\n description=\"Redirecting to DOC\",\n include_in_schema=False,\n)\nasync def home() -> RedirectResponse:\n return RedirectResponse(url=\"/docs\")\n\n\n@router.post(\n \"/backup\",\n tags=[\"MySQL-AutoXtrabackup\"],\n response_description=\"Json response \",\n description=\"Run all backup process\",\n)\nasync def backup() -> JSONResponse:\n backup_ = Backup()\n result = backup_.all_backup()\n if result:\n return JSONResponse(\n content={\"result\": \"Successfully finished the backup process\"},\n status_code=status.HTTP_201_CREATED,\n )\n return JSONResponse(\n content={\"result\": \"[FAILED] to take backup\"},\n status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,\n )\n\n\n@router.post(\n \"/prepare\",\n tags=[\"MySQL-AutoXtrabackup\"],\n response_description=\"Json response\",\n description=\"Prepare all backups\",\n)\nasync def prepare() -> JSONResponse:\n prepare_ = Prepare()\n result = prepare_.prepare_inc_full_backups()\n if result:\n return JSONResponse(\n content={\"result\": \"Successfully prepared all the backups\"},\n status_code=status.HTTP_200_OK,\n )\n return JSONResponse(\n content={\"result\": \"[FAILED] to prepare backup\"},\n status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,\n )\n\n\n@router.get(\n \"/backups\",\n tags=[\"MySQL-AutoXtrabackup\"],\n response_description=\"Json response\",\n description=\"List all available backups\",\n)\nasync def backups() -> JSONResponse:\n backup_ = Backup()\n result = list_available_backups(\n str(backup_.builder_obj.backup_options.get(\"backup_dir\"))\n )\n if result:\n return JSONResponse(content={\"backups\": result}, status_code=status.HTTP_200_OK)\n return JSONResponse(content={\"backups\": {}}, status_code=status.HTTP_200_OK)\n\n\n@router.delete(\n \"/delete\",\n tags=[\"MySQL-AutoXtrabackup\"],\n response_description=\"Json response\",\n description=\"Remove all available backups\",\n)\nasync def delete() -> JSONResponse:\n backup_ = Backup()\n delete_full = backup_.clean_full_backup_dir(remove_all=True)\n delete_inc = backup_.clean_inc_backup_dir()\n if delete_full and delete_inc:\n return JSONResponse(\n content={\"result\": \"There is no backups or backups removed successfully\"},\n status_code=status.HTTP_200_OK,\n )\n return JSONResponse(\n content={\"result\": \"[FAILED] to remove/delete available backups\"},\n status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,\n )\n" }, { "path": "mysql_autoxtrabackup/api/main.py", "content": "from typing import Any, Dict, Optional\n\nimport uvicorn # type: ignore\nfrom fastapi import FastAPI\nfrom fastapi.openapi.utils import get_openapi\n\nfrom mysql_autoxtrabackup.api.controller.controller import router\nfrom mysql_autoxtrabackup.utils.version import VERSION\n\napp = FastAPI()\n\n\n@app.on_event(\"startup\")\nasync def startup() -> None:\n \"\"\"startup.\"\"\"\n print(\"app started\")\n\n\n@app.on_event(\"shutdown\")\nasync def shutdown() -> None:\n \"\"\"shutdown.\"\"\"\n print(\"SHUTDOWN\")\n\n\ndef modify_openapi() -> Dict[str, Any]:\n \"\"\"modify_openapi.\"\"\"\n if app.openapi_schema:\n return app.openapi_schema\n openapi_schema = get_openapi(\n title=\"MySQL-AutoXtrabackup\",\n version=f\"{VERSION}\",\n description=\"Rest API doc for MySQL-AutoXtrabackup\",\n routes=app.routes,\n )\n app.openapi_schema = openapi_schema\n return app.openapi_schema\n\n\napp.openapi = modify_openapi # type: ignore\n\napp.include_router(router)\n\n\ndef run_server(host: Optional[str] = None, port: Optional[int] = None) -> None:\n host = host or \"127.0.0.1\"\n port = port or 5555\n uvicorn.run(app, host=host, port=port)\n\n\nif __name__ == \"__main__\":\n run_server()\n" }, { "path": "mysql_autoxtrabackup/autoxtrabackup.py", "content": "import logging\nimport logging.handlers\nimport os\nimport re\nimport time\nfrom logging.handlers import RotatingFileHandler\nfrom sys import exit\nfrom sys import platform as _platform\nfrom typing import Optional\n\nimport click\nimport humanfriendly # type: ignore\nimport pid # type: ignore\n\nfrom mysql_autoxtrabackup.api import main\nfrom mysql_autoxtrabackup.backup_backup.backuper import Backup\nfrom mysql_autoxtrabackup.backup_prepare.prepare import Prepare\nfrom mysql_autoxtrabackup.general_conf import path_config\nfrom mysql_autoxtrabackup.general_conf.generalops import GeneralClass\nfrom mysql_autoxtrabackup.process_runner.process_runner import ProcessRunner\nfrom mysql_autoxtrabackup.utils import version\n\nlogger = logging.getLogger(\"\")\ndestinations_hash = {\n \"linux\": \"/dev/log\",\n \"linux2\": \"/dev/log\",\n \"darwin\": \"/var/run/syslog\",\n}\n\n\ndef address_matcher(plt: str) -> str:\n return destinations_hash.get(plt, (\"localhost\", 514)) # type: ignore\n\n\nhandler = logging.handlers.SysLogHandler(address=address_matcher(_platform))\n\n# Set syslog for the root logger\nlogger.addHandler(handler)\n\n\ndef print_help(ctx: click.Context, param: None, value: bool) -> None:\n if not value:\n return\n click.echo(ctx.get_help())\n ctx.exit()\n\n\ndef print_version(ctx: click.Context, param: None, value: bool) -> None:\n if not value or ctx.resilient_parsing:\n return\n click.echo(\"Developed by Shahriyar Rzayev from Azerbaijan PUG(http://azepug.az)\")\n click.echo(\"Link : https://github.com/ShahriyarR/MySQL-AutoXtraBackup\")\n click.echo(\"Email: rzayev.sehriyar@gmail.com\")\n click.echo(\n \"Based on Percona XtraBackup: https://github.com/percona/percona-xtrabackup/\"\n )\n click.echo(f\"MySQL-AutoXtraBackup Version: {version.VERSION}\")\n ctx.exit()\n\n\ndef check_file_content(file: str) -> Optional[bool]:\n \"\"\"Check if all mandatory headers and keys exist in file\"\"\"\n with open(file, \"r\") as config_file:\n file_content = config_file.read()\n\n config_headers = [\"MySQL\", \"Backup\", \"Encrypt\", \"Compress\", \"Commands\"]\n config_keys = [\n \"mysql\",\n \"mycnf\",\n \"mysqladmin\",\n \"mysql_user\",\n \"mysql_password\",\n \"mysql_host\",\n \"datadir\",\n \"tmp_dir\",\n \"backup_dir\",\n \"backup_tool\",\n \"xtra_prepare\",\n \"start_mysql_command\",\n \"stop_mysql_command\",\n \"chown_command\",\n ]\n\n for header in config_headers:\n if header not in file_content:\n raise KeyError(\"Mandatory header [%s] doesn't exist in %s\" % (header, file))\n\n for key in config_keys:\n if key not in file_content:\n raise KeyError(\"Mandatory key '%s' doesn't exists in %s.\" % (key, file))\n\n return True\n\n\ndef validate_file(file: str) -> Optional[bool]:\n \"\"\"\n Check for validity of the file given in file path. If file doesn't exist or invalid\n configuration file, throw error.\n \"\"\"\n if not os.path.isfile(file):\n raise FileNotFoundError(\"Specified file does not exist.\")\n\n # filename extension should be .cnf\n pattern = re.compile(r\".*\\.cnf\")\n\n if pattern.match(file):\n # Lastly the file should have all 5 required headers\n if check_file_content(file):\n return None\n else:\n raise ValueError(\"Invalid file extension. Expecting .cnf\")\n return None\n\n\n@click.command()\n@click.option(\"--dry-run\", is_flag=True, help=\"Enable the dry run.\")\n@click.option(\"--prepare\", is_flag=True, help=\"Prepare/recover backups.\")\n@click.option(\n \"--run-server\", is_flag=True, help=\"Start the FastAPI app for serving API\"\n)\n@click.option(\"--backup\", is_flag=True, help=\"Take full and incremental backups.\")\n@click.option(\n \"--version\",\n is_flag=True,\n callback=print_version, # type: ignore\n expose_value=False,\n is_eager=True,\n help=\"Version information.\",\n)\n@click.option(\n \"--defaults-file\",\n default=path_config.config_path_file, # type: ignore\n show_default=True,\n help=\"Read options from the given file\",\n)\n@click.option(\"--tag\", help=\"Pass the tag string for each backup\")\n@click.option(\"--show-tags\", is_flag=True, help=\"Show backup tags and exit\")\n@click.option(\"-v\", \"--verbose\", is_flag=True, help=\"Be verbose (print to console)\")\n@click.option(\n \"-lf\",\n \"--log-file\",\n default=path_config.log_file_path,\n show_default=True,\n help=\"Set log file\",\n)\n@click.option(\n \"-l\",\n \"--log\",\n \"--log-level\",\n default=\"INFO\",\n show_default=True,\n type=click.Choice([\"DEBUG\", \"INFO\", \"WARNING\", \"ERROR\", \"CRITICAL\"]),\n help=\"Set log level\",\n)\n@click.option(\n \"--log-file-max-bytes\",\n default=1073741824,\n show_default=True,\n nargs=1,\n type=int,\n help=\"Set log file max size in bytes\",\n)\n@click.option(\n \"--log-file-backup-count\",\n default=7,\n show_default=True,\n nargs=1,\n type=int,\n help=\"Set log file backup count\",\n)\n@click.option(\n \"--help\",\n is_flag=True,\n callback=print_help, # type: ignore\n expose_value=False,\n is_eager=False,\n help=\"Print help message and exit.\",\n)\n@click.pass_context\ndef all_procedure(\n ctx,\n prepare,\n backup,\n run_server,\n tag,\n show_tags,\n verbose,\n log_file,\n log,\n defaults_file,\n dry_run,\n log_file_max_bytes,\n log_file_backup_count,\n):\n options = GeneralClass(defaults_file)\n logging_options = options.logging_options\n backup_options = options.backup_options\n\n formatter = logging.Formatter(\n fmt=\"%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s\",\n datefmt=\"%Y-%m-%d %H:%M:%S\",\n )\n\n if verbose:\n ch = logging.StreamHandler()\n # control console output log level\n ch.setLevel(logging.INFO)\n ch.setFormatter(formatter)\n logger.addHandler(ch)\n\n if log_file:\n try:\n if logging_options.get(\"log_file_max_bytes\") and logging_options.get(\n \"log_file_backup_count\"\n ):\n file_handler = RotatingFileHandler(\n log_file,\n mode=\"a\",\n maxBytes=int(str(logging_options.get(\"log_file_max_bytes\"))),\n backupCount=int(str(logging_options.get(\"log_file_backup_count\"))),\n )\n else:\n file_handler = RotatingFileHandler(\n log_file,\n mode=\"a\",\n maxBytes=log_file_max_bytes,\n backupCount=log_file_backup_count,\n )\n file_handler.setFormatter(formatter)\n logger.addHandler(file_handler)\n except PermissionError as err:\n exit(\"{} Please consider to run as root or sudo\".format(err))\n\n # set log level in order: 1. user argument 2. config file 3. @click default\n if log is not None:\n logger.setLevel(log)\n elif logging_options.get(\"log_level\"):\n logger.setLevel(str(logging_options.get(\"log_level\")))\n else:\n # this is the fallback default log-level.\n logger.setLevel(\"INFO\")\n\n validate_file(defaults_file)\n pid_file = pid.PidFile(piddir=backup_options.get(\"pid_dir\"))\n\n try:\n with pid_file: # User PidFile for locking to single instance\n dry_run_ = dry_run\n if dry_run_:\n dry_run_ = 1\n logger.warning(\"Dry run enabled!\")\n if (\n prepare is False\n and backup is False\n and verbose is False\n and dry_run is False\n and show_tags is False\n and run_server is False\n ):\n print_help(ctx, None, value=True)\n\n elif run_server:\n main.run_server()\n elif show_tags and defaults_file:\n backup_ = Backup(config=defaults_file)\n backup_.show_tags(backup_dir=str(backup_options.get(\"backup_dir\")))\n elif prepare:\n prepare_ = Prepare(config=defaults_file, dry_run=dry_run_, tag=tag)\n prepare_.prepare_backup_and_copy_back()\n elif backup:\n backup_ = Backup(config=defaults_file, dry_run=dry_run_, tag=tag)\n backup_.all_backup()\n\n except (pid.PidFileAlreadyLockedError, pid.PidFileAlreadyRunningError) as error:\n if float(\n str(backup_options.get(\"pid_runtime_warning\"))\n ) and time.time() - os.stat(pid_file.filename).st_ctime > float(\n str(backup_options.get(\"pid_runtime_warning\"))\n ):\n pid.fh.seek(0)\n pid_str = pid.fh.read(16).split(\"\\n\", 1)[0].strip()\n logger.warning(\n \"Pid file already exists or Pid already running! : \", str(error)\n )\n logger.critical(\n \"Backup (pid: \"\n + pid_str\n + \") has been running for logger than: \"\n + str(\n humanfriendly.format_timespan(\n backup_options.get(\"pid_runtime_warning\")\n )\n )\n )\n\n except pid.PidFileUnreadableError as error:\n logger.warning(\"Pid file can not be read: \" + str(error))\n except pid.PidFileError as error:\n logger.warning(\"Generic error with pid file: \" + str(error))\n\n logger.info(\"Xtrabackup command history:\")\n for i in ProcessRunner.xtrabackup_history_log:\n logger.info(str(i))\n logger.info(\"Autoxtrabackup completed successfully!\")\n return True\n\n\nif __name__ == \"__main__\":\n all_procedure()\n" }, { "path": "mysql_autoxtrabackup/backup_backup/__init__.py", "content": "from .backup_archive import BackupArchive as BackupArchive\nfrom .backup_builder import BackupBuilderChecker as BackupBuilderChecker\nfrom .backuper import Backup as Backup\n" }, { "path": "mysql_autoxtrabackup/backup_backup/backup_archive.py", "content": "import logging\nimport os\nimport shutil\nfrom datetime import datetime\nfrom typing import Union\n\nfrom mysql_autoxtrabackup.backup_backup.backup_builder import BackupBuilderChecker\nfrom mysql_autoxtrabackup.general_conf import path_config\nfrom mysql_autoxtrabackup.general_conf.generalops import GeneralClass\nfrom mysql_autoxtrabackup.process_runner.process_runner import ProcessRunner\nfrom mysql_autoxtrabackup.utils import helpers\n\nlogger = logging.getLogger(__name__)\n\n\nclass BackupArchive:\n def __init__(\n self,\n config: str = path_config.config_path_file,\n dry_run: Union[bool, None] = None,\n tag: Union[str, None] = None,\n ) -> None:\n self.conf = config\n self.dry = dry_run\n self.tag = tag\n options_obj = GeneralClass(config=self.conf)\n self.backup_options = BackupBuilderChecker(\n config=self.conf, dry_run=self.dry\n ).backup_options\n self.backup_archive_options = options_obj.backup_archive_options\n\n def create_backup_archives(self) -> bool:\n from mysql_autoxtrabackup.backup_prepare.prepare import Prepare\n\n # Creating .tar.gz archive files of taken backups\n file_list = os.listdir(str(self.backup_options.get(\"full_dir\")))\n for i in file_list:\n if len(file_list) == 1 or i != max(file_list):\n logger.info(\"Preparing backups prior archiving them...\")\n\n if self.backup_archive_options.get(\"prepare_archive\"):\n logger.info(\"Started to prepare backups, prior archiving!\")\n prepare_obj = Prepare(\n config=self.conf, dry_run=self.dry, tag=self.tag\n )\n status = prepare_obj.prepare_inc_full_backups()\n if status:\n logger.info(\n \"Backups Prepared successfully... {}\".format(status)\n )\n\n if self.backup_archive_options.get(\"move_archive\") and (\n int(str(self.backup_archive_options.get(\"move_archive\"))) == 1\n ):\n dir_name = (\n str(self.backup_archive_options.get(\"archive_dir\"))\n + \"/\"\n + i\n + \"_archive\"\n )\n logger.info(\n \"move_archive enabled. Moving {} to {}\".format(\n self.backup_options.get(\"backup_dir\"), dir_name\n )\n )\n try:\n shutil.copytree(\n str(self.backup_options.get(\"backup_dir\")), dir_name\n )\n except Exception as err:\n logger.error(\"FAILED: Move Archive\")\n logger.error(err)\n raise\n else:\n return True\n else:\n logger.info(\n \"move_archive is disabled. archiving / compressing current_backup.\"\n )\n # Multi-core tar utilizing pigz.\n\n # Pigz default to number of cores available, or 8 if cannot be read.\n\n # Test if pigz is available.\n logger.info(\"testing for pigz...\")\n status = ProcessRunner.run_command(\"pigz --version\")\n archive_file = (\n str(self.backup_archive_options.get(\"archive_dir\"))\n + \"/\"\n + i\n + \".tar.gz\"\n )\n if status:\n logger.info(\"Found pigz...\")\n # run_tar = \"tar cvvf - {} {} | pigz -v > {}\" \\\n run_tar = (\n \"tar --use-compress-program=pigz -cvf {} {} {}\".format(\n archive_file,\n self.backup_options.get(\"full_dir\"),\n self.backup_options.get(\"inc_dir\"),\n )\n )\n else:\n # handle file not found error.\n logger.warning(\n \"pigz executeable not available. Defaulting to singlecore tar\"\n )\n run_tar = \"tar -zcf {} {} {}\".format(\n archive_file,\n self.backup_options.get(\"full_dir\"),\n self.backup_options.get(\"inc_dir\"),\n )\n status = ProcessRunner.run_command(run_tar)\n if status:\n logger.info(\n \"OK: Old full backup and incremental backups archived!\"\n )\n return True\n\n logger.error(\"FAILED: Archiving \")\n raise RuntimeError(\"FAILED: Archiving -> {}\".format(run_tar))\n return True\n\n def clean_old_archives(self) -> None:\n logger.info(\"Starting cleaning of old archives\")\n archive_dir = str(self.backup_archive_options.get(\"archive_dir\"))\n # Finding if last full backup older than the interval or more from now!\n cleanup_msg = \"Removing archive {}/{} due to {}\"\n for archive in helpers.sorted_ls(archive_dir):\n if \"_archive\" in archive:\n archive_date = datetime.strptime(archive, \"%Y-%m-%d_%H-%M-%S_archive\")\n else:\n archive_date = datetime.strptime(archive, \"%Y-%m-%d_%H-%M-%S.tar.gz\")\n\n now = datetime.now()\n\n if (\n self.backup_archive_options.get(\"archive_max_duration\")\n or self.backup_archive_options.get(\"archive_max_size\")\n ) and (\n float((now - archive_date).total_seconds())\n >= float(str(self.backup_archive_options.get(\"archive_max_duration\")))\n or float(helpers.get_directory_size(archive_dir))\n > float(str(self.backup_archive_options.get(\"archive_max_size\")))\n ):\n logger.info(\n cleanup_msg.format(\n archive_dir, archive, \"archive_max_duration exceeded.\"\n )\n )\n logger.info(\"OR\")\n logger.info(\n cleanup_msg.format(\n archive_dir, archive, \"archive_max_size exceeded.\"\n )\n )\n full_archive_path = os.path.join(archive_dir, archive)\n if os.path.isdir(full_archive_path):\n shutil.rmtree(full_archive_path)\n else:\n os.remove(full_archive_path)\n" }, { "path": "mysql_autoxtrabackup/backup_backup/backup_builder.py", "content": "# Will store necessary checks and command building actions here\nimport logging\nfrom os.path import isfile\nfrom typing import Optional, Union\n\nfrom mysql_autoxtrabackup.general_conf import path_config\nfrom mysql_autoxtrabackup.general_conf.generalops import GeneralClass\nfrom mysql_autoxtrabackup.process_runner.process_runner import ProcessRunner\n\nlogger = logging.getLogger(__name__)\n\n\nclass BackupBuilderChecker:\n # General pre-backup checking/extracting/untar/streaming etc. should happen here\n\n def __init__(\n self,\n config: str = path_config.config_path_file,\n dry_run: Union[bool, None] = None,\n ) -> None:\n self.conf = config\n self.dry = dry_run\n options_obj = GeneralClass(config=self.conf)\n self.mysql_options = options_obj.mysql_options\n self.compression_options = options_obj.compression_options\n self.encryption_options = options_obj.encryption_options\n self.backup_options = options_obj.backup_options\n self.xbstream_options = options_obj.xbstream_options\n\n def general_command_builder(self) -> str:\n \"\"\"\n Method for building general options for backup command.\n :return: String of constructed options.\n \"\"\"\n args = \"\"\n\n if self.mysql_options.get(\"mysql_socket\"):\n args += \" --socket={}\".format(self.mysql_options.get(\"mysql_socket\"))\n else:\n args += \" --host={} --port={}\".format(\n self.mysql_options.get(\"mysql_host\"),\n self.mysql_options.get(\"mysql_port\"),\n )\n # Adding compression support for backup\n if (\n self.compression_options.get(\"compress\")\n and self.compression_options.get(\"compress_chunk_size\")\n and self.compression_options.get(\"compress_threads\")\n ):\n args += (\n \" --compress={}\"\n \" --compress-chunk-size={}\"\n \" --compress-threads={}\".format(\n self.compression_options.get(\"compress\"),\n self.compression_options.get(\"compress_chunk_size\"),\n self.compression_options.get(\"compress_threads\"),\n )\n )\n\n # Adding encryption support for full backup\n if self.encryption_options.get(\"encrypt\"):\n args += (\n \" --encrypt={}\"\n \" --encrypt-threads={}\"\n \" --encrypt-chunk-size={}\".format(\n self.encryption_options.get(\"encrypt\"),\n self.encryption_options.get(\"encrypt_threads\"),\n self.encryption_options.get(\"encrypt_chunk_size\"),\n )\n )\n\n if self.encryption_options.get(\"encrypt_key\"):\n if self.encryption_options.get(\"encrypt_key_file\"):\n raise AttributeError(\n \"--encrypt-key and --encrypt-key-file are mutually exclusive\"\n )\n args += \" --encrypt-key={}\".format(\n self.encryption_options.get(\"encrypt_key\")\n )\n elif self.encryption_options.get(\"encrypt_key_file\"):\n args += \" --encrypt-key-file={}\".format(\n self.encryption_options.get(\"encrypt_key_file\")\n )\n\n # Checking if extra options were passed:\n if self.backup_options.get(\"xtra_options\"):\n args += \" {}\".format(self.backup_options.get(\"xtra_options\"))\n # Checking if extra backup options were passed:\n if self.backup_options.get(\"xtra_backup\"):\n args += \" {}\".format(self.backup_options.get(\"xtra_backup\"))\n\n # Checking if partial recovery list is available\n if self.backup_options.get(\"partial_list\"):\n logger.warning(\"Partial Backup is enabled!\")\n args += ' --databases=\"{}\"'.format(self.backup_options.get(\"partial_list\"))\n\n return args\n\n def extract_decrypt_from_stream_backup(\n self,\n recent_full_bck: Optional[str] = None,\n recent_inc_bck: Optional[str] = None,\n flag: Optional[bool] = None,\n ) -> None:\n \"\"\"\n Method for extracting and if necessary decrypting from streamed backup.\n If the recent_full_bck passed then it means you want to extract the full backup.\n If the recent_int_bck passed then it means you want to extract the inc backup.\n \"\"\"\n # Extract and decrypt streamed full backup prior to executing incremental backup\n file_name = \"{}/{}/inc_backup.stream\".format(\n self.backup_options.get(\"inc_dir\"), recent_inc_bck\n )\n file_place_holder = \"< {} -C {}/{}\".format(\n file_name, self.backup_options.get(\"inc_dir\"), recent_inc_bck\n )\n\n if not recent_inc_bck:\n file_name = \"{}/{}/full_backup.stream\".format(\n self.backup_options.get(\"full_dir\"), recent_full_bck\n )\n file_place_holder = \"< {} -C {}/{}\".format(\n file_name, self.backup_options.get(\"full_dir\"), recent_full_bck\n )\n\n xbstream_command = None\n\n if self.xbstream_options.get(\"stream\") == \"xbstream\":\n xbstream_command = \"{} {}\".format(\n self.xbstream_options.get(\"xbstream\"),\n self.xbstream_options.get(\"xbstream_options\"),\n )\n if (\n self.encryption_options.get(\"encrypt\")\n and self.xbstream_options.get(\"xbs_decrypt\")\n and not flag\n ):\n logger.info(\n \"Using xbstream to extract and decrypt from {}\".format(file_name)\n )\n xbstream_command += (\n \" --decrypt={} --encrypt-key={} --encrypt-threads={} \".format(\n self.encryption_options.get(\"decrypt\"),\n self.encryption_options.get(\"encrypt_key\"),\n self.encryption_options.get(\"encrypt_threads\"),\n )\n )\n\n if xbstream_command:\n xbstream_command += file_place_holder\n logger.info(\n \"The following xbstream command will be executed {}\".format(\n xbstream_command\n )\n )\n if self.dry == 0 and isfile(file_name):\n ProcessRunner.run_command(xbstream_command)\n\n def stream_encrypt_compress_tar_checker(self) -> None:\n if self.xbstream_options.get(\"stream\") == \"tar\" and (\n self.encryption_options.get(\"encrypt\")\n or self.compression_options.get(\"compress\")\n ):\n logger.error(\n \"xtrabackup: error: compressed and encrypted backups are \"\n \"incompatible with the 'tar' streaming format. Use --stream=xbstream instead.\"\n )\n raise RuntimeError(\n \"xtrabackup: error: compressed and encrypted backups are \"\n \"incompatible with the 'tar' streaming format. Use --stream=xbstream instead.\"\n )\n\n def stream_tar_incremental_checker(self) -> None:\n if self.xbstream_options.get(\"stream\") == \"tar\":\n logger.error(\n \"xtrabackup: error: streaming incremental backups are incompatible with the \"\n \"'tar' streaming format. Use --stream=xbstream instead.\"\n )\n raise RuntimeError(\n \"xtrabackup: error: streaming incremental backups are incompatible with the \"\n \"'tar' streaming format. Use --stream=xbstream instead.\"\n )\n\n def full_backup_command_builder(self, full_backup_dir: str) -> str:\n \"\"\"\n Method for creating Full Backup command.\n :param full_backup_dir the path of backup directory\n \"\"\"\n xtrabackup_cmd = (\n \"{} --defaults-file={} --user={} --password={} \"\n \" --target-dir={} --backup\".format(\n self.backup_options.get(\"backup_tool\"),\n self.mysql_options.get(\"mycnf\"),\n self.mysql_options.get(\"mysql_user\"),\n self.mysql_options.get(\"mysql_password\"),\n full_backup_dir,\n )\n )\n # Calling general options/command builder to add extra options\n xtrabackup_cmd += self.general_command_builder()\n\n stream = self.backup_options.get(\"stream\")\n if stream:\n logger.warning(\"Streaming is enabled!\")\n xtrabackup_cmd += ' --stream=\"{}\"'.format(stream)\n if stream == \"xbstream\":\n xtrabackup_cmd += \" > {}/full_backup.stream\".format(full_backup_dir)\n elif stream == \"tar\":\n xtrabackup_cmd += \" > {}/full_backup.tar\".format(full_backup_dir)\n\n return xtrabackup_cmd\n\n def inc_backup_command_builder(\n self,\n recent_full_bck: Optional[str],\n inc_backup_dir: Optional[str],\n recent_inc_bck: Optional[str] = None,\n ) -> str:\n xtrabackup_inc_cmd_base = (\n \"{} --defaults-file={} --user={} --password={}\".format(\n self.backup_options.get(\"backup_tool\"),\n self.mysql_options.get(\"mycnf\"),\n self.mysql_options.get(\"mysql_user\"),\n self.mysql_options.get(\"mysql_password\"),\n )\n )\n if not recent_inc_bck:\n xtrabackup_inc_cmd_base += (\n \" --target-dir={} --incremental-basedir={}/{} --backup\".format(\n inc_backup_dir, self.backup_options.get(\"full_dir\"), recent_full_bck\n )\n )\n else:\n xtrabackup_inc_cmd_base += (\n \" --target-dir={} --incremental-basedir={}/{} --backup\".format(\n inc_backup_dir, self.backup_options.get(\"inc_dir\"), recent_inc_bck\n )\n )\n\n # Calling general options/command builder to add extra options\n xtrabackup_inc_cmd_base += self.general_command_builder()\n\n # Checking if streaming enabled for backups\n # There is no need to check for 'tar' streaming type -> see the method: stream_tar_incremental_checker()\n if (\n hasattr(self, \"stream\")\n and self.xbstream_options.get(\"stream\") == \"xbstream\"\n ):\n xtrabackup_inc_cmd_base += ' --stream=\"{}\"'.format(\n self.xbstream_options.get(\"stream\")\n )\n xtrabackup_inc_cmd_base += \" > {}/inc_backup.stream\".format(inc_backup_dir)\n logger.warning(\"Streaming xbstream is enabled!\")\n\n return xtrabackup_inc_cmd_base\n\n def decrypter(\n self,\n recent_full_bck: Optional[str] = None,\n xtrabackup_inc_cmd: Optional[str] = None,\n recent_inc_bck: Optional[str] = None,\n ) -> None:\n logger.info(\"Applying workaround for LP #1444255\")\n logger.info(\"See more -> https://jira.percona.com/browse/PXB-934\")\n # With recent PXB 8 it seems to be there is no need for this workaround.\n # Due to this moving this feature to this method and keeping just in case.\n # Deprecated as hell.\n if \"encrypt\" not in xtrabackup_inc_cmd: # type: ignore\n return\n if not isfile(\n \"{}/{}/xtrabackup_checkpoints.xbcrypt\".format(\n self.backup_options.get(\"full_dir\"), recent_full_bck\n )\n ):\n logger.info(\"Skipping...\")\n return\n\n xbcrypt_command = \"{} -d -k {} -a {}\".format(\n self.encryption_options.get(\"xbcrypt\"),\n self.encryption_options.get(\"encrypt_key\"),\n self.encryption_options.get(\"encrypt\"),\n )\n xbcrypt_command_extra = (\n \" -i {}/{}/xtrabackup_checkpoints.xbcrypt -o {}/{}/xtrabackup_checkpoints\"\n )\n xbcrypt_command += xbcrypt_command_extra.format(\n self.backup_options.get(\"full_dir\"),\n recent_full_bck,\n self.backup_options.get(\"full_dir\"),\n recent_full_bck,\n )\n\n if recent_inc_bck:\n if not isfile(\n \"{}/{}/xtrabackup_checkpoints.xbcrypt\".format(\n self.backup_options.get(\"inc_dir\"), recent_inc_bck\n )\n ):\n logger.info(\"Skipping...\")\n return\n xbcrypt_command += xbcrypt_command_extra.format(\n self.backup_options.get(\"inc_dir\"),\n recent_inc_bck,\n self.backup_options.get(\"inc_dir\"),\n recent_inc_bck,\n )\n logger.info(\n \"The following xbcrypt command will be executed {}\".format(xbcrypt_command)\n )\n if self.dry == 0:\n ProcessRunner.run_command(xbcrypt_command)\n" }, { "path": "mysql_autoxtrabackup/backup_backup/backuper.py", "content": "# MySQL Backuper Script using Percona Xtrabackup\n# Originally Developed by\n# Shahriyar Rzayev (Shako)-> https://mysql.az/ https://azepug.az/\n# / rzayev.sehriyar@gmail.com / rzayev.shahriyar@yandex.com\n# This comment is from 2014 - keeping it here\n\n\nimport logging\nimport os\nimport shutil\nimport time\nfrom datetime import datetime\nfrom typing import Optional, Union\n\nfrom mysql_autoxtrabackup.backup_backup.backup_archive import BackupArchive\nfrom mysql_autoxtrabackup.backup_backup.backup_builder import BackupBuilderChecker\nfrom mysql_autoxtrabackup.general_conf import path_config\nfrom mysql_autoxtrabackup.general_conf.check_env import CheckEnv\nfrom mysql_autoxtrabackup.process_runner.process_runner import ProcessRunner\nfrom mysql_autoxtrabackup.utils import helpers, mysql_cli\n\nlogger = logging.getLogger(__name__)\n\n\nclass Backup:\n def __init__(\n self,\n config: str = path_config.config_path_file,\n dry_run: Union[bool, None] = None,\n tag: Union[str, None] = None,\n ) -> None:\n self.conf = config\n self.dry = dry_run\n self.tag = tag\n self.mysql_cli = mysql_cli.MySQLClientHelper(config=self.conf)\n self.builder_obj = BackupBuilderChecker(config=self.conf, dry_run=self.dry)\n self.archive_obj = BackupArchive(\n config=self.conf, dry_run=self.dry, tag=self.tag\n )\n\n def add_tag(\n self, backup_type: str, backup_size: Optional[str], backup_status: Optional[str]\n ) -> bool:\n \"\"\"\n Method for adding backup tags\n :param backup_type: The backup type - Full/Inc\n :param backup_size: The size of the backup in human readable format\n :param backup_status: Status: OK or Status: Failed\n :return: True if no exception\n \"\"\"\n # skip tagging unless self.tag\n if not self.tag:\n logger.info(\"TAGGING SKIPPED\")\n return True\n\n # Currently only support Inc and Full types, calculate name based on this\n assert backup_type in (\n \"Full\",\n \"Inc\",\n ), \"add_tag(): backup_type {}: must be 'Full' or 'Inc'\".format(backup_type)\n backup_name = (\n helpers.get_latest_dir_name(\n str(self.builder_obj.backup_options.get(\"full_dir\"))\n )\n if backup_type == \"Full\"\n else helpers.get_latest_dir_name(\n str(self.builder_obj.backup_options.get(\"inc_dir\"))\n )\n )\n\n # Calculate more tag fields, create string\n backup_timestamp = datetime.now().strftime(\"%Y-%m-%d_%H-%M-%S\")\n backup_tag_str = (\n \"{bk_name}\\t{bk_type}\\t{bk_status}\\t{bk_timestamp}\\t{bk_size}\\t'{bk_tag}'\\n\"\n )\n\n # Apply tag\n with open(\n \"{}/backup_tags.txt\".format(\n self.builder_obj.backup_options.get(\"backup_dir\")\n ),\n \"a\",\n ) as backup_tags_file:\n backup_tag_final = backup_tag_str.format(\n bk_name=backup_name,\n bk_type=backup_type,\n bk_status=backup_status,\n bk_timestamp=backup_timestamp,\n bk_size=backup_size,\n bk_tag=self.tag,\n )\n\n backup_tags_file.write(backup_tag_final)\n return True\n\n @staticmethod\n def show_tags(backup_dir: str, tag_file: Optional[str] = None) -> Optional[bool]:\n tag_file = tag_file or \"{}/backup_tags.txt\".format(backup_dir)\n if os.path.isfile(tag_file):\n with open(\"{}/backup_tags.txt\".format(backup_dir), \"r\") as backup_tags:\n from_file = backup_tags.read()\n column_names = \"{0}\\t{1}\\t{2}\\t{3}\\t{4}\\tTAG\\n\".format(\n \"Backup\".ljust(19),\n \"Type\".ljust(4),\n \"Status\".ljust(2),\n \"Completion_time\".ljust(19),\n \"Size\",\n )\n extra_str = \"{}\\n\".format(\"-\" * (len(column_names) + 21))\n print(column_names + extra_str + from_file)\n logger.info(column_names + extra_str + from_file)\n return True\n else:\n logger.warning(\n \"Could not find backup_tags.txt inside given backup directory. Can't print tags.\"\n )\n print(\n \"WARNING: Could not find backup_tags.txt inside given backup directory. Can't print tags.\"\n )\n return None\n\n def last_full_backup_date(\n self, path: Optional[str] = None, full_backup_interval: Optional[float] = None\n ) -> bool:\n \"\"\"\n Check if last full backup date retired or not.\n :return: True if last full backup date older than given interval, False if it is newer.\n \"\"\"\n # Finding last full backup date from dir/folder name\n full_dir = path or str(self.builder_obj.backup_options.get(\"full_dir\"))\n backup_interval = full_backup_interval or str(\n self.builder_obj.backup_options.get(\"full_backup_interval\")\n )\n max_dir = helpers.get_latest_dir_name(full_dir)\n\n dir_date = datetime.strptime(str(max_dir), \"%Y-%m-%d_%H-%M-%S\")\n now = datetime.now()\n return float((now - dir_date).total_seconds()) >= float(backup_interval)\n\n def clean_full_backup_dir(\n self,\n full_dir: Optional[str] = None,\n remove_all: Optional[bool] = None,\n ) -> Optional[bool]:\n # Deleting old full backup after taking new full backup.\n # Keeping the latest in order not to lose everything.\n logger.info(\"starting clean_full_backup_dir\")\n full_dir = full_dir or str(self.builder_obj.backup_options.get(\"full_dir\"))\n if not os.path.isdir(full_dir):\n return True\n if remove_all:\n for i in os.listdir(full_dir):\n rm_dir = full_dir + \"/\" + i\n shutil.rmtree(rm_dir)\n return True\n\n for i in os.listdir(full_dir):\n rm_dir = full_dir + \"/\" + i\n if i != max(os.listdir(full_dir)):\n shutil.rmtree(rm_dir)\n logger.info(\"DELETING {}\".format(rm_dir))\n else:\n logger.info(\"KEEPING {}\".format(rm_dir))\n return True\n\n def clean_inc_backup_dir(self, inc_dir: Optional[str] = None) -> Optional[bool]:\n # Deleting incremental backups after taking new fresh full backup.\n inc_dir = inc_dir or str(self.builder_obj.backup_options.get(\"inc_dir\"))\n if not os.path.isdir(inc_dir):\n return True\n for i in os.listdir(inc_dir):\n rm_dir = inc_dir + \"/\" + i\n shutil.rmtree(str(rm_dir))\n return True\n\n def full_backup(self) -> bool:\n \"\"\"\n Method for taking full backups. It will construct the backup command based on config file.\n :return: True on success.\n :raise: RuntimeError on error.\n \"\"\"\n logger.info(\n \"starting full backup to {}\".format(\n self.builder_obj.backup_options.get(\"full_dir\")\n )\n )\n full_backup_dir = helpers.create_backup_directory(\n str(self.builder_obj.backup_options.get(\"full_dir\"))\n )\n\n # Creating Full Backup command.\n xtrabackup_cmd = self.builder_obj.full_backup_command_builder(\n full_backup_dir=full_backup_dir\n )\n\n # Extra checks.\n self.builder_obj.stream_encrypt_compress_tar_checker()\n\n if self.dry:\n # If it's a dry run, skip running & tagging\n return True\n\n logger.debug(\n \"Starting {}\".format(self.builder_obj.backup_options.get(\"backup_tool\"))\n )\n status = ProcessRunner.run_command(xtrabackup_cmd)\n status_str = \"OK\" if status is True else \"FAILED\"\n self.add_tag(\n backup_type=\"Full\",\n backup_size=helpers.get_folder_size(full_backup_dir),\n backup_status=status_str,\n )\n return status\n\n def inc_backup(self) -> bool:\n \"\"\"\n Method for taking incremental backups.\n :return: True on success.\n :raise: RuntimeError on error.\n \"\"\"\n # Get the recent full backup path\n recent_full_bck = helpers.get_latest_dir_name(\n str(self.builder_obj.backup_options.get(\"full_dir\"))\n )\n if not recent_full_bck:\n raise RuntimeError(\n \"Failed to get Full backup path. Are you sure you have one?\"\n )\n\n # Get the recent incremental backup path\n recent_inc_bck = helpers.get_latest_dir_name(\n str(self.builder_obj.backup_options.get(\"inc_dir\"))\n )\n\n # Creating time-stamped incremental backup directory\n inc_backup_dir = helpers.create_backup_directory(\n str(self.builder_obj.backup_options.get(\"inc_dir\"))\n )\n\n # Check here if stream=tar enabled.\n # Because it is impossible to take incremental backup with streaming tar.\n # raise RuntimeError.\n self.builder_obj.stream_tar_incremental_checker()\n\n xtrabackup_inc_cmd = self.builder_obj.inc_backup_command_builder(\n recent_full_bck=recent_full_bck,\n inc_backup_dir=inc_backup_dir,\n recent_inc_bck=recent_inc_bck,\n )\n\n self.builder_obj.extract_decrypt_from_stream_backup(\n recent_full_bck=recent_full_bck, recent_inc_bck=recent_inc_bck\n )\n\n # Deprecated workaround for LP #1444255\n self.builder_obj.decrypter(\n recent_full_bck=recent_full_bck,\n xtrabackup_inc_cmd=xtrabackup_inc_cmd,\n recent_inc_bck=recent_inc_bck,\n )\n\n if self.dry:\n # If it's a dry run, skip running & tagging\n return True\n\n logger.debug(\n \"Starting {}\".format(self.builder_obj.backup_options.get(\"backup_tool\"))\n )\n status = ProcessRunner.run_command(xtrabackup_inc_cmd)\n status_str = \"OK\" if status is True else \"FAILED\"\n self.add_tag(\n backup_type=\"Inc\",\n backup_size=helpers.get_folder_size(inc_backup_dir),\n backup_status=status_str,\n )\n return status\n\n def all_backup(self) -> bool:\n \"\"\"\n This method at first checks full backup directory, if it is empty takes full backup.\n If it is not empty then checks for full backup time.\n If the recent full backup is taken 1 day ago, it takes full backup.\n In any other conditions it takes incremental backup.\n \"\"\"\n # Workaround for circular import dependency error in Python\n\n # Creating object from CheckEnv class\n check_env_obj = CheckEnv(\n self.conf,\n full_dir=str(self.builder_obj.backup_options.get(\"full_dir\")),\n inc_dir=str(self.builder_obj.backup_options.get(\"inc_dir\")),\n )\n\n assert check_env_obj.check_all_env() is True, \"environment checks failed!\"\n if not helpers.get_latest_dir_name(\n str(self.builder_obj.backup_options.get(\"full_dir\"))\n ):\n logger.info(\n \"- - - - You have no backups : Taking very first Full Backup! - - - -\"\n )\n\n if self.mysql_cli.mysql_run_command(\"flush logs\") and self.full_backup():\n # Removing old inc backups\n self.clean_inc_backup_dir()\n\n elif self.last_full_backup_date():\n logger.info(\n \"- - - - Your full backup is timeout : Taking new Full Backup! - - - -\"\n )\n\n # Archiving backups\n if self.archive_obj.backup_archive_options.get(\"archive_dir\"):\n logger.info(\n \"Archiving enabled; cleaning archive_dir & archiving previous Full Backup\"\n )\n if self.archive_obj.backup_archive_options.get(\n \"archive_max_duration\"\n ) or self.archive_obj.backup_archive_options.get(\"archive_max_size\"):\n self.archive_obj.clean_old_archives()\n self.archive_obj.create_backup_archives()\n else:\n logger.info(\"Archiving disabled. Skipping!\")\n\n if self.mysql_cli.mysql_run_command(\"flush logs\") and self.full_backup():\n # Removing full backups\n self.clean_full_backup_dir()\n\n # Removing inc backups\n self.clean_inc_backup_dir()\n\n else:\n\n logger.info(\n \"- - - - You have a full backup that is less than {} seconds old. - - - -\".format(\n self.builder_obj.backup_options.get(\"full_backup_interval\")\n )\n )\n logger.info(\n \"- - - - We will take an incremental one based on recent Full Backup - - - -\"\n )\n\n time.sleep(3)\n\n # Taking incremental backup\n self.inc_backup()\n\n return True\n" }, { "path": "mysql_autoxtrabackup/backup_prepare/__init__.py", "content": "from .copy_back import CopyBack as CopyBack\nfrom .prepare import Prepare as Prepare\nfrom .prepare_builder import BackupPrepareBuilderChecker as BackupPrepareBuilderChecker\n" }, { "path": "mysql_autoxtrabackup/backup_prepare/copy_back.py", "content": "import logging\nimport os\nimport shutil\nfrom typing import Optional, Union\n\nfrom mysql_autoxtrabackup.general_conf import path_config\nfrom mysql_autoxtrabackup.general_conf.generalops import GeneralClass\nfrom mysql_autoxtrabackup.process_runner.process_runner import ProcessRunner\nfrom mysql_autoxtrabackup.utils import helpers\n\nlogger = logging.getLogger(__name__)\n\n\nclass CopyBack:\n def __init__(self, config: str = path_config.config_path_file) -> None:\n self.conf = config\n options_obj = GeneralClass(config=self.conf)\n self.command_options = options_obj.command_options\n self.mysql_options = options_obj.backup_options\n self.backup_options = options_obj.backup_options\n\n def shutdown_mysql(self) -> Union[None, bool, Exception]:\n # Shut Down MySQL\n logger.info(\"Shutting Down MySQL server:\")\n args = self.command_options.get(\"stop_mysql_command\")\n return ProcessRunner.run_command(args)\n\n def move_to_tmp_dir(self) -> None:\n try:\n shutil.move(\n str(self.mysql_options.get(\"data_dir\")),\n str(self.backup_options.get(\"tmp_dir\")),\n )\n logger.info(\n \"Moved data_dir to {} ...\".format(self.backup_options.get(\"tmp_dir\"))\n )\n except shutil.Error as err:\n logger.error(\"Error occurred while moving data_dir\")\n logger.error(err)\n raise RuntimeError(err)\n\n def create_empty_data_dir(self) -> Union[None, bool, Exception]:\n logger.info(\"Creating an empty data directory ...\")\n makedir = \"mkdir {}\".format(self.mysql_options.get(\"data_dir\"))\n return ProcessRunner.run_command(makedir)\n\n def move_data_dir(self) -> bool:\n # Move data_dir to new directory\n tmp_dir = self.backup_options.get(\"tmp_dir\")\n logger.info(\"Moving MySQL data_dir to {}\".format(tmp_dir))\n if os.path.isdir(str(self.backup_options.get(\"tmp_dir\"))):\n rmdir_ = \"rm -rf {}\".format(tmp_dir)\n ProcessRunner.run_command(rmdir_)\n self.move_to_tmp_dir()\n self.create_empty_data_dir()\n return True\n\n def run_xtra_copyback(self, data_dir: Optional[str] = None) -> Optional[bool]:\n # Running Xtrabackup with --copy-back option\n copy_back = \"{} --copy-back {} --target-dir={}/{} --data_dir={}\".format(\n self.backup_options.get(\"backup_tool\"),\n self.backup_options.get(\"xtra_options\"),\n self.backup_options.get(\"full_dir\"),\n helpers.get_latest_dir_name(str(self.backup_options.get(\"full_dir\"))),\n self.mysql_options.get(\"data_dir\") if data_dir is None else data_dir,\n )\n return ProcessRunner.run_command(copy_back)\n\n def giving_chown(self, data_dir: Optional[str] = None) -> Optional[bool]:\n # Changing owner of data_dir to given user:group\n give_chown = \"{} {}\".format(\n self.command_options.get(\"chown_command\"),\n self.mysql_options.get(\"data_dir\") if data_dir is None else data_dir,\n )\n return ProcessRunner.run_command(give_chown)\n\n def start_mysql_func(\n self, start_tool: Optional[str] = None, options: Optional[str] = None\n ) -> Union[None, bool, Exception]:\n # Starting MySQL\n logger.info(\"Starting MySQL server: \")\n args = (\n self.command_options.get(\"start_mysql_command\")\n if start_tool is None\n else start_tool\n )\n start_command = \"{} {}\".format(args, options) if options is not None else args\n return ProcessRunner.run_command(start_command)\n\n @staticmethod\n def check_if_backup_prepared(\n full_dir: Optional[str], full_backup_file: Optional[str]\n ) -> Optional[bool]:\n \"\"\"\n This method is for checking if the backup can be copied-back.\n It is going to check xtrabackup_checkpoints file inside backup directory for backup_type column.\n backup_type column must be equal to 'full-prepared'\n :return: True if backup is already prepared; RuntimeError if it is not.\n \"\"\"\n with open(\n \"{}/{}/xtrabackup_checkpoints\".format(full_dir, full_backup_file), \"r\"\n ) as xchk_file:\n # This thing seems to be complicated bu it is not:\n # Trying to get 'full-prepared' from ['backup_type ', ' full-prepared\\n']\n if (\n xchk_file.readline().split(\"=\")[1].strip(\"\\n\").lstrip()\n == \"full-prepared\"\n ):\n return True\n raise RuntimeError(\n \"This full backup is not fully prepared, not doing copy-back!\"\n )\n\n def copy(\n self, options: Optional[str] = None, data_dir: Optional[str] = None\n ) -> bool:\n \"\"\"\n Function for running:\n xtrabackup --copy-back\n giving chown to data_dir\n starting mysql\n :return: True if succeeded. Error if failed\n \"\"\"\n logger.info(\"Copying Back Already Prepared Final Backup:\")\n if (\n len(\n os.listdir(\n str(self.mysql_options.get(\"data_dir\"))\n if data_dir is None\n else data_dir\n )\n )\n > 0\n ):\n logger.info(\"MySQL data_dir is not empty!\")\n return False\n else:\n self.run_xtra_copyback(data_dir=data_dir)\n self.giving_chown(data_dir=data_dir)\n self.start_mysql_func(options=options)\n return True\n\n def copy_back_action(self, options: Optional[str] = None) -> Optional[bool]:\n \"\"\"\n Function for complete recover/copy-back actions\n :return: True if succeeded. Error if failed.\n \"\"\"\n try:\n self.check_if_backup_prepared(\n str(self.backup_options.get(\"full_dir\")),\n helpers.get_latest_dir_name(str(self.backup_options.get(\"full_dir\"))),\n )\n self.shutdown_mysql()\n if self.move_data_dir() and self.copy(options=options):\n logger.info(\"All data copied back successfully. \")\n logger.info(\"Your MySQL server is UP again\")\n return True\n except Exception as err:\n logger.error(\"{}: {}\".format(type(err).__name__, err))\n return None\n" }, { "path": "mysql_autoxtrabackup/backup_prepare/prepare.py", "content": "import logging\nimport os\nimport time\nfrom typing import Optional, Union\n\nfrom mysql_autoxtrabackup.backup_backup.backup_builder import BackupBuilderChecker\nfrom mysql_autoxtrabackup.backup_prepare.copy_back import CopyBack\nfrom mysql_autoxtrabackup.backup_prepare.prepare_builder import (\n BackupPrepareBuilderChecker,\n)\nfrom mysql_autoxtrabackup.general_conf import path_config\nfrom mysql_autoxtrabackup.process_runner.process_runner import ProcessRunner\nfrom mysql_autoxtrabackup.utils import helpers\n\nlogger = logging.getLogger(__name__)\n\n\nclass Prepare:\n def __init__(\n self,\n config: str = path_config.config_path_file,\n dry_run: Optional[bool] = None,\n tag: Optional[str] = None,\n ) -> None:\n self.conf = config\n self.dry = dry_run\n self.tag = tag\n self.prepare_options = BackupPrepareBuilderChecker(\n config=self.conf, dry_run=self.dry\n )\n # If prepare_tool option enabled in config, make backup_tool to use this.\n # The reason is maybe you have backup taken with 2.4 version but your are going to prepare\n # with newer version. It is somehow unlike to do this but still.\n if self.prepare_options.backup_options.get(\"prepare_tool\"):\n self.prepare_options.backup_options[\"backup_tool\"] = str(\n self.prepare_options.backup_options.get(\"prepare_tool\")\n )\n\n if self.tag and not os.path.isfile(\n \"{}/backup_tags.txt\".format(\n self.prepare_options.backup_options.get(\"backup_dir\")\n )\n ):\n raise RuntimeError(\n \"Could not find backup_tags.txt inside backup directory. \"\n \"Please run without --tag option\"\n )\n\n def run_prepare_command(\n self, base_dir: Optional[str], actual_dir: Optional[str], cmd: Optional[str]\n ) -> Optional[bool]:\n # Decrypt backup\n self.prepare_options.decrypt_backup(base_dir, actual_dir)\n\n # Decompress backup\n self.prepare_options.decompress_backup(base_dir, actual_dir)\n\n logger.info(\"Running prepare command -> {}\".format(cmd))\n if self.dry:\n return True\n return ProcessRunner.run_command(cmd)\n\n def prepare_with_tags(self) -> Optional[bool]:\n # Method for preparing backups based on passed backup tags\n found_backups = BackupPrepareBuilderChecker.parse_backup_tags(\n backup_dir=str(self.prepare_options.backup_options.get(\"backup_dir\")),\n tag_name=self.tag,\n )\n recent_bck = helpers.get_latest_dir_name(\n str(self.prepare_options.backup_options.get(\"full_dir\"))\n )\n # I am not going to initialize this object in Prepare class constructor as I thin there is no need.\n backup_builder = BackupBuilderChecker(self.conf, dry_run=self.dry)\n\n if found_backups[1] == \"Full\": # type: ignore\n if recent_bck:\n logger.info(\"- - - - Preparing Full Backup - - - -\")\n\n # Extracting/decrypting from streamed backup and extra checks goes here.\n backup_builder.extract_decrypt_from_stream_backup(\n recent_full_bck=recent_bck\n )\n\n # Prepare command\n backup_prepare_cmd = self.prepare_options.prepare_command_builder(\n full_backup=recent_bck\n )\n\n self.run_prepare_command(\n str(self.prepare_options.backup_options.get(\"full_dir\")),\n recent_bck,\n backup_prepare_cmd,\n )\n\n elif found_backups[1] == \"Inc\": # type: ignore\n if not os.listdir(str(self.prepare_options.backup_options.get(\"inc_dir\"))):\n logger.info(\n \"- - - - You have no Incremental backups. So will prepare only latest Full backup - - - -\"\n )\n self.prepare_only_full_backup()\n else:\n logger.info(\"- - - - You have Incremental backups. - - - -\")\n if self.prepare_only_full_backup():\n logger.info(\"Preparing Incs: \")\n list_of_dir = helpers.sorted_ls(\n str(self.prepare_options.backup_options.get(\"inc_dir\"))\n )\n # Find the index number inside all list for backup(which was found via tag)\n index_num = list_of_dir.index(found_backups[0]) # type: ignore\n # Limit the iteration until this found backup\n for dir_ in list_of_dir[: index_num + 1]:\n apply_log_only = None\n if dir_ != found_backups[0]: # type: ignore\n logger.info(\n \"Preparing inc backups in sequence. inc backup dir/name is {}\".format(\n dir_\n )\n )\n apply_log_only = True\n\n else:\n logger.info(\n \"Preparing last incremental backup, inc backup dir/name is {}\".format(\n dir_\n )\n )\n\n # Extracting/decrypting from streamed backup and extra checks goes here\n backup_builder.extract_decrypt_from_stream_backup(\n recent_inc_bck=dir_, flag=True\n )\n\n # Prepare command\n backup_prepare_cmd = (\n self.prepare_options.prepare_command_builder(\n full_backup=recent_bck,\n incremental=dir_,\n apply_log_only=apply_log_only,\n )\n )\n\n self.run_prepare_command(\n str(self.prepare_options.backup_options.get(\"inc_dir\")),\n dir_,\n backup_prepare_cmd,\n )\n\n logger.info(\"- - - - The end of the Prepare Stage. - - - -\")\n return True\n\n def prepare_only_full_backup(self) -> Union[None, bool, Exception]:\n recent_bck = helpers.get_latest_dir_name(\n str(self.prepare_options.backup_options.get(\"full_dir\"))\n )\n backup_builder = BackupBuilderChecker(self.conf, dry_run=self.dry)\n if recent_bck:\n apply_log_only = None\n if not os.listdir(str(self.prepare_options.backup_options.get(\"inc_dir\"))):\n logger.info(\"- - - - Preparing Full Backup - - - -\")\n self.prepare_options.untar_backup(recent_bck=recent_bck)\n # Extracting/decrypting from streamed backup and extra checks goes here\n backup_builder.extract_decrypt_from_stream_backup(\n recent_full_bck=recent_bck\n )\n\n else:\n logger.info(\"- - - - Preparing Full backup for incrementals - - - -\")\n logger.info(\n \"- - - - Final prepare,will occur after preparing all inc backups - - - -\"\n )\n time.sleep(3)\n\n apply_log_only = True\n # Prepare command\n\n backup_prepare_cmd = self.prepare_options.prepare_command_builder(\n full_backup=recent_bck, apply_log_only=apply_log_only\n )\n\n self.run_prepare_command(\n str(self.prepare_options.backup_options.get(\"full_dir\")),\n recent_bck,\n backup_prepare_cmd,\n )\n return True\n\n def prepare_inc_full_backups(self) -> Union[None, bool, Exception]:\n backup_builder = BackupBuilderChecker(self.conf, dry_run=self.dry)\n if not os.listdir(str(self.prepare_options.backup_options.get(\"inc_dir\"))):\n logger.info(\n \"- - - - You have no Incremental backups. So will prepare only latest Full backup - - - -\"\n )\n return self.prepare_only_full_backup()\n else:\n logger.info(\"- - - - You have Incremental backups. - - - -\")\n recent_bck = helpers.get_latest_dir_name(\n str(self.prepare_options.backup_options.get(\"full_dir\"))\n )\n\n if self.prepare_only_full_backup():\n logger.info(\"Preparing Incs: \")\n list_of_dir = sorted(\n os.listdir(str(self.prepare_options.backup_options.get(\"inc_dir\")))\n )\n for inc_backup_dir in list_of_dir:\n apply_log_only = None\n if inc_backup_dir != max(\n os.listdir(\n str(self.prepare_options.backup_options.get(\"inc_dir\"))\n )\n ):\n logger.info(\n \"Preparing Incremental backups in sequence. Incremental backup dir/name is {}\".format(\n inc_backup_dir\n )\n )\n apply_log_only = True\n else:\n logger.info(\n \"Preparing last Incremental backup, inc backup dir/name is {}\".format(\n inc_backup_dir\n )\n )\n\n # Extracting/decrypting from streamed backup and extra checks goes here\n backup_builder.extract_decrypt_from_stream_backup(\n recent_inc_bck=inc_backup_dir, flag=True\n )\n # Prepare command\n backup_prepare_cmd = self.prepare_options.prepare_command_builder(\n full_backup=recent_bck,\n incremental=inc_backup_dir,\n apply_log_only=apply_log_only,\n )\n\n self.run_prepare_command(\n str(self.prepare_options.backup_options.get(\"inc_dir\")),\n inc_backup_dir,\n backup_prepare_cmd,\n )\n\n logger.info(\"- - - - The end of the Prepare Stage. - - - -\")\n return True\n\n def prepare_backup_and_copy_back(self) -> None:\n copy_back_obj = CopyBack(config=self.conf)\n # Recovering/Copying Back Prepared Backup\n x = \"- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\"\n\n print(x)\n print(\"\")\n print(\"Preparing full/inc backups!\")\n print(\"What do you want to do?\")\n print(\n \"1. Prepare Backups and keep for future usage. NOTE('Once Prepared Backups Can not be prepared Again')\"\n )\n print(\"2. Prepare Backups and restore/recover/copy-back immediately\")\n print(\"3. Just copy-back previously prepared backups\")\n\n prepare = int(input(\"Please Choose one of options and type 1 or 2 or 3: \"))\n print(\"\")\n print(x)\n\n time.sleep(3)\n\n if prepare == 1:\n if not self.tag:\n self.prepare_inc_full_backups()\n else:\n logger.info(\"Backup tag will be used to prepare backups\")\n self.prepare_with_tags()\n elif prepare == 2:\n if not self.tag:\n self.prepare_inc_full_backups()\n else:\n self.prepare_with_tags()\n if not self.dry:\n copy_back_obj.copy_back_action()\n else:\n logger.critical(\n \"Dry run is not implemented for copy-back/recovery actions!\"\n )\n elif prepare == 3:\n if not self.dry:\n copy_back_obj.copy_back_action()\n else:\n logger.critical(\n \"Dry run is not implemented for copy-back/recovery actions!\"\n )\n else:\n print(\"Please type 1 or 2 or 3 and nothing more!\")\n" }, { "path": "mysql_autoxtrabackup/backup_prepare/prepare_builder.py", "content": "import logging\nimport os\nfrom typing import Optional, Tuple\n\nfrom mysql_autoxtrabackup.general_conf import path_config\nfrom mysql_autoxtrabackup.general_conf.generalops import GeneralClass\nfrom mysql_autoxtrabackup.process_runner.process_runner import ProcessRunner\n\nlogger = logging.getLogger(__name__)\n\n\nclass BackupPrepareBuilderChecker:\n def __init__(\n self, config: str = path_config.config_path_file, dry_run: Optional[bool] = None\n ) -> None:\n self.conf = config\n self.dry = dry_run\n\n options_obj = GeneralClass(config=self.conf)\n self.backup_options = options_obj.backup_options\n self.compression_options = options_obj.compression_options\n self.encryption_options = options_obj.encryption_options\n self.xbstream_options = options_obj.xbstream_options\n\n @staticmethod\n def parse_backup_tags(\n backup_dir: Optional[str], tag_name: Optional[str]\n ) -> Optional[Tuple[str, str]]:\n \"\"\"\n Static Method for returning the backup directory name and backup type\n :param backup_dir: The backup directory path\n :param tag_name: The tag name to search\n :return: Tuple of (backup directory, backup type) (2017-11-09_19-37-16, Full).\n :raises: RuntimeError if there is no such tag inside backup_tags.txt\n \"\"\"\n if os.path.isfile(\"{}/backup_tags.txt\".format(backup_dir)):\n with open(\"{}/backup_tags.txt\".format(backup_dir), \"r\") as backup_tags:\n f = backup_tags.readlines()\n\n for i in f:\n split_ = i.split(\"\\t\")\n if tag_name == split_[-1].rstrip(\"'\\n\\r\").lstrip(\"'\"):\n return split_[0], split_[1]\n else:\n raise RuntimeError(\"There is no such tag for backups\")\n return None\n\n def decompress_backup(\n self, path: Optional[str], dir_name: Optional[str]\n ) -> Optional[bool]:\n \"\"\"\n Method for backup decompression.\n Check if decompression enabled, if it is, decompress\n backup prior prepare.\n :param path: the basedir path i.e full backup dir or incremental dir.\n :param dir_name: the exact name backup folder(likely timestamped folder name).\n :return: None or RuntimeError\n \"\"\"\n if self.compression_options.get(\"decompress\"):\n # The base decompression command\n dec_cmd = \"{} --decompress={} --target-dir={}/{}\".format(\n self.backup_options.get(\"backup_tool\"),\n self.compression_options.get(\"decompress\"),\n path,\n dir_name,\n )\n if self.compression_options.get(\"remove_original_comp\"):\n dec_cmd += \" --remove-original\"\n\n logger.info(\"Trying to decompress backup\")\n logger.info(\"Running decompress command -> {}\".format(dec_cmd))\n if self.dry:\n return None\n return ProcessRunner.run_command(dec_cmd)\n return None\n\n def decrypt_backup(\n self, path: Optional[str], dir_name: Optional[str]\n ) -> Optional[bool]:\n \"\"\"\n Method for decrypting backups.\n If you use crypted backups it should be decrypted prior preparing.\n :param path: the basedir path i.e full backup dir or incremental dir.\n :param dir_name: the exact name backup folder(likely timestamped folder name).\n :return: None or RuntimeError\n \"\"\"\n if self.encryption_options.get(\"decrypt\"):\n # The base decryption command\n decr_cmd = \"{} --decrypt={} --encrypt-key={} --target-dir={}/{}\".format(\n self.backup_options.get(\"backup_tool\"),\n self.encryption_options.get(\"decrypt\"),\n self.encryption_options.get(\"encrypt_key\"),\n path,\n dir_name,\n )\n if self.encryption_options.get(\"remove_original_comp\"):\n decr_cmd += \" --remove-original\"\n logger.info(\"Trying to decrypt backup\")\n logger.info(\"Running decrypt command -> {}\".format(decr_cmd))\n if self.dry:\n return None\n return ProcessRunner.run_command(decr_cmd)\n return None\n\n def prepare_command_builder(\n self,\n full_backup: Optional[str],\n incremental: Optional[str] = None,\n apply_log_only: Optional[bool] = None,\n ) -> str:\n \"\"\"\n Method for building prepare command as it is repeated several times.\n :param full_backup: The full backup directory name\n :param incremental: The incremental backup directory name\n :param apply_log_only: The flag to add --apply-log-only\n :return: The prepare command string\n \"\"\"\n # Base prepare command\n xtrabackup_prepare_cmd = \"{} --prepare --target-dir={}/{}\".format(\n self.backup_options.get(\"backup_tool\"),\n self.backup_options.get(\"full_dir\"),\n full_backup,\n )\n\n if incremental:\n xtrabackup_prepare_cmd += \" --incremental-dir={}/{}\".format(\n self.backup_options.get(\"inc_dir\"), incremental\n )\n if apply_log_only:\n xtrabackup_prepare_cmd += \" --apply-log-only\"\n\n # Checking if extra options were passed:\n if self.backup_options.get(\"xtra_options\"):\n xtrabackup_prepare_cmd += \" {}\".format(\n self.backup_options.get(\"xtra_options\")\n )\n\n # Checking of extra prepare options were passed:\n if self.backup_options.get(\"xtra_prepare_options\"):\n xtrabackup_prepare_cmd += \" {}\".format(\n self.backup_options.get(\"xtra_prepare_options\")\n )\n\n return xtrabackup_prepare_cmd\n\n def untar_backup(self, recent_bck: str) -> Optional[bool]:\n if self.xbstream_options.get(\"stream\") == \"tar\":\n full_dir = self.backup_options.get(\"full_dir\")\n untar_cmd = \"tar -xf {}/{}/full_backup.tar -C {}/{}\".format(\n full_dir, recent_bck, full_dir, recent_bck\n )\n logger.info(\n \"The following tar command will be executed -> {}\".format(untar_cmd)\n )\n if self.dry == 0 and os.path.isfile(\n \"{}/{}/full_backup.tar\".format(full_dir, recent_bck)\n ):\n return ProcessRunner.run_command(untar_cmd)\n return None\n" }, { "path": "mysql_autoxtrabackup/general_conf/__init__.py", "content": "from . import path_config as path_config\nfrom .check_env import CheckEnv as CheckEnv\nfrom .generalops import GeneralClass as GeneralClass\nfrom .generate_default_conf import GenerateDefaultConfig as GenerateDefaultConfig\n" }, { "path": "mysql_autoxtrabackup/general_conf/check_env.py", "content": "import logging\nimport os\nimport re\nfrom typing import Optional, Union\n\nfrom mysql_autoxtrabackup.process_runner.process_runner import ProcessRunner\nfrom mysql_autoxtrabackup.utils.helpers import create_directory\n\nfrom . import path_config\nfrom .generalops import GeneralClass\n\nlogger = logging.getLogger(__name__)\n\n\nclass CheckEnv:\n def __init__(\n self,\n config: str = path_config.config_path_file,\n full_dir: Union[str, None] = None,\n inc_dir: Union[str, None] = None,\n ) -> None:\n self.conf = config\n options = GeneralClass(config=self.conf)\n self.backup_options = options.backup_options\n self.mysql_options = options.mysql_options\n self.archive_options = options.backup_archive_options\n if full_dir:\n self.backup_options[\"full_dir\"] = full_dir\n if inc_dir:\n self.backup_options[\"ind_dir\"] = inc_dir\n\n def check_mysql_uptime(self, options: Optional[str] = None) -> Optional[bool]:\n \"\"\"\n Method for checking if MySQL server is up or not.\n :param options: Passed options to connect to MySQL server if None, then going to get it from conf file\n :return: True on success, raise RuntimeError on error.\n \"\"\"\n if not options:\n\n status_args = (\n \"{} --defaults-file={} \"\n \"--user={} --password='{}' status\".format(\n self.mysql_options.get(\"mysqladmin\"),\n self.mysql_options.get(\"mycnf\"),\n self.mysql_options.get(\"mysql_user\"),\n self.mysql_options.get(\"mysql_password\"),\n )\n )\n\n if self.mysql_options.get(\"mysql_socket\"):\n status_args += \" --socket={}\".format(\n self.mysql_options.get(\"mysql_socket\")\n )\n elif self.mysql_options.get(\"mysql_host\") and self.mysql_options.get(\n \"mysql_port\"\n ):\n status_args += \" --host={}\".format(self.mysql_options.get(\"mysql_host\"))\n status_args += \" --port={}\".format(self.mysql_options.get(\"mysql_port\"))\n else:\n logger.critical(\n \"Neither mysql_socket nor mysql_host and mysql_port are defined in config!\"\n )\n raise RuntimeError(\n \"Neither mysql_socket nor mysql_host and mysql_port are defined in config!\"\n )\n else:\n status_args = \"{} {} status\".format(\n self.mysql_options.get(\"mysqladmin\"), options\n )\n\n # filter out password from argument list\n filtered_args = re.sub(\"--password='?\\w+'?\", \"--password='*'\", status_args)\n\n logger.info(\"Running mysqladmin command -> {}\".format(filtered_args))\n\n return ProcessRunner.run_command(status_args)\n\n def check_mysql_conf(self) -> Union[bool, Exception]:\n \"\"\"\n Method for checking passed MySQL my.cnf defaults file. If it is not passed then skip this check\n :return: True on success, raise RuntimeError on error.\n \"\"\"\n my_cnf = self.mysql_options.get(\"mycnf\")\n if not my_cnf or my_cnf == \"\":\n logger.info(\"Skipping my.cnf check, because it is not specified\")\n return True\n elif not os.path.exists(my_cnf):\n logger.error(\"FAILED: MySQL configuration file path does NOT exist\")\n raise RuntimeError(\"FAILED: MySQL configuration file path does NOT exist\")\n\n logger.info(\"OK: MySQL configuration file exists\")\n return True\n\n def check_mysql_mysql(self) -> Union[bool, Exception]:\n \"\"\"\n Method for checking mysql client path\n :return: True on success, raise RuntimeError on error.\n \"\"\"\n mysql = self.mysql_options.get(\"mysql\")\n if os.path.exists(str(mysql)):\n logger.info(\"OK: {} exists\".format(mysql))\n return True\n\n logger.error(\"FAILED: {} doest NOT exist\".format(mysql))\n raise RuntimeError(\"FAILED: {} doest NOT exist\".format(mysql))\n\n def check_mysql_mysqladmin(self) -> Union[bool, Exception]:\n \"\"\"\n Method for checking mysqladmin path\n :return: True on success, raise RuntimeError on error.\n \"\"\"\n mysqladmin = self.mysql_options.get(\"mysqladmin\")\n if os.path.exists(str(mysqladmin)):\n logger.info(\"OK: {} exists\".format(mysqladmin))\n return True\n\n logger.error(\"FAILED: {} does NOT exist\".format(mysqladmin))\n raise RuntimeError(\"FAILED: {} does NOT exist\".format(mysqladmin))\n\n def check_mysql_backuptool(self) -> Union[bool, Exception]:\n \"\"\"\n Method for checking if given backup tool path is there or not.\n :return: RuntimeError on failure, True on success\n \"\"\"\n if os.path.exists(str(self.backup_options.get(\"backup_tool\"))):\n logger.info(\"OK: XtraBackup exists\")\n return True\n\n logger.error(\"FAILED: XtraBackup does NOT exist\")\n raise RuntimeError(\"FAILED: XtraBackup does NOT exist\")\n\n def check_mysql_backup_dir(self) -> Optional[bool]:\n \"\"\"\n Check for MySQL backup directory.\n If directory exists already then, return True. If not, try to create it.\n :return: True on success. RuntimeError on failure.\n \"\"\"\n if os.path.exists(str(self.backup_options.get(\"backup_dir\"))):\n logger.info(\"OK: Main backup directory exists\")\n return True\n\n return create_directory(str(self.backup_options.get(\"backup_dir\")))\n\n def check_mysql_archive_dir(self) -> Optional[bool]:\n \"\"\"\n Check for archive directory.\n If archive_dir is given in config file and if it does not exist, try to create.\n :return: True on success. RuntimeError on failure.\n \"\"\"\n if not self.archive_options.get(\"archive_dir\"):\n logger.info(\"Skipping check as this option not specified in config file...\")\n return True\n\n if os.path.exists(str(self.archive_options.get(\"archive_dir\"))):\n logger.info(\"OK: Archive folder directory exists\")\n return True\n\n return create_directory(str(self.archive_options.get(\"archive_dir\")))\n\n def check_mysql_full_backup_dir(self) -> Optional[bool]:\n \"\"\"\n Check full backup directory path.\n If this path exists return True if not try to create.\n :return: True on success.\n \"\"\"\n if os.path.exists(str(self.backup_options.get(\"full_dir\"))):\n logger.info(\"OK: Full Backup directory exists\")\n return True\n\n return create_directory(str(self.backup_options.get(\"full_dir\")))\n\n def check_mysql_inc_backup_dir(self) -> Optional[bool]:\n \"\"\"\n Check incremental backup directory path.\n If this path exists return True if not try to create.\n :return: True on success.\n \"\"\"\n if os.path.exists(str(self.backup_options.get(\"inc_dir\"))):\n logger.info(\"OK: Increment directory exists\")\n return True\n\n return create_directory(str(self.backup_options.get(\"inc_dir\")))\n\n def check_all_env(self) -> Union[bool, Exception]:\n \"\"\"\n Method for running all checks\n :return: True on success, raise RuntimeError on error.\n \"\"\"\n try:\n self.check_mysql_uptime()\n self.check_mysql_mysql()\n self.check_mysql_mysqladmin()\n self.check_mysql_conf()\n self.check_mysql_backuptool()\n self.check_mysql_backup_dir()\n self.check_mysql_full_backup_dir()\n self.check_mysql_inc_backup_dir()\n self.check_mysql_archive_dir()\n except Exception as err:\n logger.critical(\"FAILED: Check status\")\n logger.error(err)\n raise RuntimeError(\"FAILED: Check status\")\n else:\n logger.info(\"OK: Check status\")\n return True\n" }, { "path": "mysql_autoxtrabackup/general_conf/generalops.py", "content": "import configparser\nimport logging\nfrom os.path import isfile\nfrom typing import Dict, Union\n\nimport humanfriendly # type: ignore\n\nfrom . import path_config\n\nlogger = logging.getLogger(__name__)\n\n\nclass GeneralClass:\n def __init__(self, config: str = path_config.config_path_file) -> None:\n if isfile(config):\n self.con = configparser.ConfigParser()\n self.con.read(config)\n else:\n logger.critical(\n \"Missing config file : {}\".format(path_config.config_path_file)\n )\n\n @property\n def mysql_options(self) -> Dict[str, str]:\n section = \"MySQL\"\n return {\n \"mysql\": self.con.get(section, \"mysql\"),\n \"mycnf\": self.con.get(section, \"mycnf\"),\n \"mysqladmin\": self.con.get(section, \"mysqladmin\"),\n \"mysql_user\": self.con.get(section, \"mysql_user\"),\n \"mysql_password\": self.con.get(section, \"mysql_password\"),\n \"mysql_socket\": self.con.get(section, \"mysql_socket\", fallback=None), # type: ignore\n \"mysql_host\": self.con.get(section, \"mysql_host\", fallback=None), # type: ignore\n \"mysql_port\": self.con.get(section, \"mysql_port\", fallback=None), # type: ignore\n \"data_dir\": self.con.get(section, \"datadir\"),\n }\n\n @property\n def logging_options(self) -> Dict[str, str]:\n section = \"Logging\"\n return {\n \"log_level\": self.con.get(section, \"log\"),\n \"log_file_max_bytes\": self.con.get(section, \"log_file_max_bytes\"),\n \"log_file_backup_count\": self.con.get(section, \"log_file_backup_count\"),\n }\n\n @property\n def compression_options(self) -> Dict[str, str]:\n section = \"Compress\"\n return {\n \"compress\": self.con.get(section, \"compress\", fallback=None), # type: ignore\n \"compress_chunk_size\": self.con.get(section, \"compress_chunk_size\", fallback=None), # type: ignore\n \"compress_threads\": self.con.get(section, \"compress_threads\", fallback=None), # type: ignore\n \"decompress\": self.con.get(section, \"decompress\", fallback=None), # type: ignore\n \"remove_original\": self.con.get(section, \"remove_original\", fallback=None), # type: ignore\n }\n\n @property\n def xbstream_options(self) -> Dict[str, str]:\n section = \"Xbstream\"\n return {\n \"xbstream\": self.con.get(section, \"xbstream\", fallback=None), # type: ignore\n \"stream\": self.con.get(section, \"stream\", fallback=None), # type: ignore\n \"xbstream_options\": self.con.get(section, \"xbstream_options\", fallback=None), # type: ignore\n \"xbs_decrypt\": self.con.get(section, \"xbs_decrypt\", fallback=None), # type: ignore\n }\n\n @property\n def command_options(self) -> Dict[str, str]:\n section = \"Commands\"\n return {\n \"start_mysql_command\": self.con.get(section, \"start_mysql_command\"),\n \"stop_mysql_command\": self.con.get(section, \"stop_mysql_command\"),\n \"chown_command\": self.con.get(section, \"chown_command\"),\n }\n\n @property\n def encryption_options(self) -> Dict[str, str]:\n section = \"Encrypt\"\n return {\n \"xbcrypt\": self.con.get(section, \"xbcrypt\", fallback=None), # type: ignore\n \"encrypt\": self.con.get(section, \"encrypt\", fallback=None), # type: ignore\n \"encrypt_key\": self.con.get(section, \"encrypt_key\", fallback=None), # type: ignore\n \"encrypt_key_file\": self.con.get(section, \"encrypt_key_file\", fallback=None), # type: ignore\n \"encrypt_threads\": self.con.get(section, \"encrypt_threads\", fallback=None), # type: ignore\n \"encrypt_chunk_size\": self.con.get(section, \"encrypt_chunk_size\", fallback=None), # type: ignore\n \"decrypt\": self.con.get(section, \"decrypt\", fallback=None), # type: ignore\n \"remove_original\": self.con.get(section, \"remove_original\", fallback=None), # type: ignore\n }\n\n @property\n def backup_archive_options(self) -> Dict[str, Union[str, float]]:\n section = \"Backup\"\n # backward compatible with old config 'max_archive_size' and newer 'archive_max_size'\n archive_max_size = self.con.get(section, \"max_archive_size\", fallback=None)\n if archive_max_size:\n archive_max_size = humanfriendly.parse_size(archive_max_size)\n else:\n if self.con.get(section, \"archive_max_size\", fallback=None):\n archive_max_size = humanfriendly.parse_size(\n self.con.get(section, \"archive_max_size\", fallback=None)\n )\n\n # backward compatible with old config 'max_archive_duration' and newer 'archive_max_duration'\n archive_max_duration = self.con.get(\n section, \"max_archive_duration\", fallback=None\n )\n if archive_max_duration:\n archive_max_duration = humanfriendly.parse_timespan(archive_max_duration)\n else:\n if self.con.get(section, \"archive_max_size\", fallback=None):\n archive_max_duration = humanfriendly.parse_timespan(\n self.con.get(section, \"archive_max_size\", fallback=None)\n )\n\n return {\n \"archive_dir\": self.con.get(section, \"archive_dir\", fallback=None), # type: ignore\n \"prepare_archive\": self.con.get(section, \"prepare_archive\", fallback=None), # type: ignore\n \"move_archive\": self.con.get(section, \"move_archive\", fallback=None), # type: ignore\n \"archive_max_size\": str(archive_max_size),\n \"archive_max_duration\": str(archive_max_duration),\n }\n\n @property\n def backup_options(self) -> Dict[str, Union[str, float]]:\n section = \"Backup\"\n return {\n \"pid_dir\": self.con.get(section, \"pid_dir\", fallback=\"/tmp/\"),\n \"tmp_dir\": self.con.get(section, \"tmp_dir\"),\n \"pid_runtime_warning\": humanfriendly.parse_timespan(\n self.con.get(section, \"pid_runtime_warning\")\n ),\n \"backup_dir\": self.con.get(section, \"backup_dir\"),\n \"full_dir\": self.con.get(section, \"backup_dir\") + \"/full\",\n \"inc_dir\": self.con.get(section, \"backup_dir\") + \"/inc\",\n \"backup_tool\": self.con.get(section, \"backup_tool\"),\n \"prepare_tool\": self.con.get(section, \"prepare_tool\", fallback=None), # type: ignore\n \"xtra_backup\": self.con.get(section, \"xtra_backup\", fallback=None), # type: ignore\n \"xtra_prepare_options\": self.con.get(section, \"xtra_prepare_options\", fallback=None), # type: ignore\n \"xtra_options\": self.con.get(section, \"xtra_options\", fallback=None), # type: ignore\n \"full_backup_interval\": humanfriendly.parse_timespan(\n self.con.get(section, \"full_backup_interval\", fallback=\"86400.0\")\n ),\n \"partial_list\": self.con.get(section, \"partial_list\", fallback=None), # type: ignore\n }\n" }, { "path": "mysql_autoxtrabackup/general_conf/generate_default_conf.py", "content": "# Generate the default config file dynamically.\n# As part of - https://github.com/ShahriyarR/MySQL-AutoXtraBackup/issues/331\n\nimport configparser\nfrom os import makedirs\nfrom os.path import exists, join\n\nfrom . import path_config\n\n\nclass GenerateDefaultConfig:\n def __init__(self, config: str = path_config.config_path_file) -> None:\n self.conf = config\n self.home = path_config.home\n try:\n if not exists(path_config.config_path):\n makedirs(path_config.config_path)\n except:\n pass\n\n def generate_config_file(self) -> None:\n with open(self.conf, \"w+\") as cfgfile:\n config = configparser.ConfigParser(allow_no_value=True)\n section1 = \"MySQL\"\n config.add_section(section1)\n config.set(section1, \"mysql\", \"/usr/bin/mysql\")\n config.set(section1, \"mycnf\", \"\")\n config.set(section1, \"mysqladmin\", \"/usr/bin/mysqladmin\")\n config.set(section1, \"mysql_user\", \"root\")\n config.set(section1, \"mysql_password\", \"12345\")\n config.set(\n section1,\n \"## Set either mysql_socket only, OR host + port. If both are set mysql_socket is used\",\n )\n config.set(section1, \"mysql_socket\", \"/var/lib/mysql/mysql.sock\")\n config.set(section1, \"#mysql_host\", \"127.0.0.1\")\n config.set(section1, \"#mysql_port\", \"3306\")\n config.set(section1, \"datadir\", \"/var/lib/mysql\")\n\n section2 = \"Logging\"\n config.add_section(section2)\n config.set(section2, \"#[DEBUG,INFO,WARNING,ERROR,CRITICAL]\")\n config.set(section2, \"log\", \"DEBUG\")\n config.set(section2, \"log_file_max_bytes\", \"1073741824\")\n config.set(section2, \"log_file_backup_count\", \"7\")\n\n section3 = \"Backup\"\n config.add_section(section3)\n config.set(section3, \"#Optional: set pid directory\")\n config.set(section3, \"pid_dir\", \"/tmp/MySQL-AutoXtraBackup\")\n config.set(section3, \"tmp_dir\", join(self.home, \"XB_TEST/mysql_datadirs\"))\n config.set(\n section3,\n \"#Optional: set warning if pid of backup us running for longer than X\",\n )\n config.set(section3, \"pid_runtime_warning\", \"2 Hours\")\n config.set(section3, \"backup_dir\", join(self.home, \"XB_TEST/backup_dir\"))\n config.set(section3, \"backup_tool\", \"/usr/bin/xtrabackup\")\n config.set(\n section3,\n \"#Optional: specify different path/version of xtrabackup here for prepare\",\n )\n config.set(section3, \"#prepare_tool\", \"\")\n config.set(section3, \"#Optional: pass additional options for backup stage\")\n config.set(section3, \"#xtra_backup\", \"--compact\")\n config.set(section3, \"#Optional: pass additional options for prepare stage\")\n config.set(section3, \"#xtra_prepare_options\", \"--rebuild-indexes\")\n config.set(\n section3,\n \"#Optional: pass general additional options; it will go to both for backup and prepare\",\n )\n config.set(section3, \"#xtra_options\", \"--binlog-info=ON --galera-info\")\n config.set(section3, \"#Optional: set archive and rotation\")\n config.set(\n section3, \"#archive_dir\", join(self.home, \"XB_TEST/backup_archives\")\n )\n config.set(section3, \"#prepare_archive\", \"1\")\n config.set(section3, \"#move_archive\", \"0\")\n config.set(section3, \"#full_backup_interval\", \"1 day\")\n config.set(section3, \"#archive_max_size\", \"100GiB\")\n config.set(section3, \"#archive_max_duration\", \"4 Days\")\n config.set(\n section3,\n \"#Optional: WARNING(Enable this if you want to take partial backups). \"\n \"Specify database names or table names.\",\n )\n config.set(section3, \"#partial_list\", \"tests.t1 tests.t2 dbtest\")\n\n section4 = \"Compress\"\n config.add_section(section4)\n config.set(section4, \"#optional\")\n config.set(section4, \"#Enable only if you want to use compression.\")\n config.set(section4, \"#compress\", \"quicklz\")\n config.set(section4, \"#compress_chunk_size\", \"65536\")\n config.set(section4, \"#compress_threads\", \"4\")\n config.set(section4, \"#decompress\", \"TRUE\")\n config.set(\n section4,\n \"#Enable if you want to remove .qp files after decompression.\"\n \"(Available from PXB 2.3.7 and 2.4.6)\",\n )\n config.set(section4, \"#remove_original\", \"FALSE\")\n\n section5 = \"Encrypt\"\n config.add_section(section5)\n config.set(section5, \"#Optional\")\n config.set(section5, \"#Enable only if you want to create encrypted backups\")\n config.set(section5, \"#xbcrypt\", \"/usr/bin/xbcrypt\")\n config.set(section5, \"#encrypt\", \"AES256\")\n config.set(\n section5,\n \"#Please note that --encrypt-key and --encrypt-key-file are mutually exclusive\",\n )\n config.set(section5, \"#encrypt_key\", \"VVTBwgM4UhwkTTV98fhuj+D1zyWoA89K\")\n config.set(section5, \"#encrypt_key_file\", \"/path/to/file/with_encrypt_key\")\n config.set(section5, \"#encrypt_threads\", \"4\")\n config.set(section5, \"#encrypt_chunk_size\", \"65536\")\n config.set(section5, \"#decrypt\", \"AES256\")\n config.set(\n section5,\n \"#Enable if you want to remove .qp files after decompression.\"\n \"(Available from PXB 2.3.7 and 2.4.6)\",\n )\n config.set(section5, \"#remove_original\", \"FALSE\")\n\n section6 = \"Xbstream\"\n config.add_section(section6)\n config.set(section6, \"#EXPERIMENTAL/OPTIONAL\")\n config.set(section6, \"#Enable this, if you want to stream your backups\")\n config.set(section6, \"#xbstream\", \"/usr/bin/xbstream\")\n config.set(section6, \"#stream\", \"xbstream\")\n config.set(section6, \"#xbstream_options\", \"-x --parallel=100\")\n config.set(section6, \"#xbs_decrypt\", \"1\")\n config.set(\n section6,\n \"# WARN, enable this, if you want to stream your backups to remote host\",\n )\n config.set(section6, \"#remote_stream\", \"ssh xxx.xxx.xxx.xxx\")\n\n section8 = \"Commands\"\n config.add_section(section8)\n config.set(section8, \"start_mysql_command\", \"service mysql start\")\n config.set(section8, \"stop_mysql_command\", \"service mysql stop\")\n config.set(section8, \"chown_command\", \"chown -R mysql:mysql\")\n\n config.write(cfgfile)\n" }, { "path": "mysql_autoxtrabackup/general_conf/path_config.py", "content": "# This file is simply place holder for default config file path, which is used in many places.\n# If you decide to change the default config path then please edit this file and reinstall tool.\nfrom os.path import expanduser, join\n\nhome: str = expanduser(\"~\")\nconfig_path: str = join(home, \".autoxtrabackup\")\nconfig_path_file: str = join(config_path, \"autoxtrabackup.cnf\")\nlog_file_path: str = join(config_path, \"autoxtrabackup.log\")\n" }, { "path": "mysql_autoxtrabackup/process_runner/__init__.py", "content": "from .process_runner import ProcessHandler as ProcessHandler\n" }, { "path": "mysql_autoxtrabackup/process_runner/errors.py", "content": "import logging\n\nlogger = logging.getLogger(__name__)\n\n# TODO: use these errors in the future - keeping it for future\n\n\ndef log_error(expression: str, message: str) -> None:\n logger.error(\"FAILED: \" + expression + \" \" + message)\n\n\nclass Error(Exception):\n \"\"\"Base class for all sort of exceptions\"\"\"\n\n\nclass ExternalCommandFailed(Error):\n \"\"\"\n Exception raised for external tool/command fails.\n \"\"\"\n\n def __init__(self, expression: str, message: str) -> None:\n self.expression = expression\n self.message = message\n log_error(self.expression, self.message)\n\n\nclass FullBackupFailed(Error):\n \"\"\"\n Exception raised for full backup error.\n \"\"\"\n\n def __init__(self, expression: str, message: str) -> None:\n self.expression = expression\n self.message = message\n log_error(self.expression, self.message)\n\n\nclass IncrementalBackupFailed(Error):\n \"\"\"\n Exception raised for incremental backup error.\n \"\"\"\n\n def __init__(self, expression: str, message: str) -> None:\n self.expression = expression\n self.message = message\n log_error(self.expression, self.message)\n\n\nclass SomethingWentWrong(Error):\n \"\"\"\n Exception raised for all general failed commands.\n \"\"\"\n\n def __init__(self, expression: str, message: str) -> None:\n self.expression = expression\n self.message = message\n log_error(self.expression, self.message)\n" }, { "path": "mysql_autoxtrabackup/process_runner/process_runner.py", "content": "import datetime\nimport logging\nimport re\nimport shlex\nimport subprocess\nimport typing\nfrom subprocess import PIPE, STDOUT\n\nfrom mysql_autoxtrabackup.general_conf import path_config\nfrom mysql_autoxtrabackup.general_conf.generalops import GeneralClass\n\nlogger = logging.getLogger(__name__)\n\n\nclass ProcessHandler(GeneralClass):\n \"\"\"\n Class to run a command with real-time logging for process\n\n centralizes logic for subprocess calls, and is available to all other classes (Prepare, Backup, etc)\n \"\"\"\n\n def __init__(self, config: str = path_config.config_path_file) -> None:\n self.conf = config\n GeneralClass.__init__(self, self.conf)\n self._xtrabackup_history_log = [\n [\n \"command\",\n \"xtrabackup_function\",\n \"start time\",\n \"end time\",\n \"duration\",\n \"exit code\",\n ]\n ]\n\n @property\n def xtrabackup_history_log(self) -> typing.List[typing.List[str]]:\n return self._xtrabackup_history_log\n\n def run_command(self, command: typing.Optional[str]) -> bool:\n \"\"\"\n executes a prepared command, enables real-time console & log output.\n\n This function should eventually be used for all subprocess calls.\n\n :param command: bash command to be executed\n :type command: str\n :return: True if success, False if failure\n :rtype: bool\n \"\"\"\n # filter out password from argument list, print command to execute\n\n filtered_command = re.sub(\"--password='?\\w+'?\", \"--password='*'\", command) # type: ignore\n logger.info(\"SUBPROCESS STARTING: {}\".format(str(filtered_command)))\n subprocess_args = self.command_to_args(command_str=command)\n # start the command subprocess\n cmd_start = datetime.datetime.now()\n with subprocess.Popen(subprocess_args, stdout=PIPE, stderr=STDOUT) as process:\n for line in process.stdout: # type: ignore\n logger.debug(\n \"[{}:{}] {}\".format(\n subprocess_args[0],\n process.pid,\n line.decode(\"utf-8\").strip(\"\\n\"),\n )\n )\n logger.info(\n \"SUBPROCESS {} COMPLETED with exit code: {}\".format(\n subprocess_args[0], process.returncode\n )\n )\n cmd_end = datetime.datetime.now()\n self.summarize_process(subprocess_args, cmd_start, cmd_end, process.returncode)\n # return True or False.\n if process.returncode == 0:\n return True\n else:\n raise ChildProcessError(\"SUBPROCESS FAILED! >> {}\".format(filtered_command))\n\n @staticmethod\n def command_to_args(command_str: typing.Optional[str]) -> typing.List[str]:\n \"\"\"\n convert a string bash command to an arguments list, to use with subprocess\n\n Most autoxtrabackup code creates a string command, e.g. \"xtrabackup --prepare --target-dir...\"\n If we run a string command with subprocess.Popen, we require shell=True.\n shell=True has security considerations (below), and we run autoxtrabackup with privileges (!).\n https://docs.python.org/3/library/subprocess.html#security-considerations\n So, convert to an args list and call Popen without shell=True.\n\n :param command_str: string command to execute as a subprocess\n :type command_str: str\n :return: list of args to pass to subprocess.Popen.\n :rtype: list\n \"\"\"\n if isinstance(command_str, list):\n # already a list\n args = command_str\n elif isinstance(command_str, str):\n args = shlex.split(command_str)\n else:\n raise TypeError\n logger.debug(\"subprocess args are: {}\".format(args))\n return args\n\n @staticmethod\n def represent_duration(\n start_time: datetime.datetime, end_time: datetime.datetime\n ) -> str:\n # https://gist.github.com/thatalextaylor/7408395\n duration_delta = end_time - start_time\n seconds = int(duration_delta.seconds)\n days, seconds = divmod(seconds, 86400)\n hours, seconds = divmod(seconds, 3600)\n minutes, seconds = divmod(seconds, 60)\n if days > 0:\n return \"%dd%dh%dm%ds\" % (days, hours, minutes, seconds)\n elif hours > 0:\n return \"%dh%dm%ds\" % (hours, minutes, seconds)\n elif minutes > 0:\n return \"%dm%ds\" % (minutes, seconds)\n else:\n return \"%ds\" % (seconds,)\n\n def summarize_process(\n self,\n args: typing.List[str],\n cmd_start: datetime.datetime,\n cmd_end: datetime.datetime,\n return_code: int,\n ) -> bool:\n cmd_root: str = args[0].split(\"/\")[-1:][0]\n xtrabackup_function = None\n if cmd_root == \"xtrabackup\":\n if \"--backup\" in args:\n xtrabackup_function = \"backup\"\n elif \"--prepare\" in args and \"--apply-log-only\" not in args:\n xtrabackup_function = \"prepare\"\n elif \"--prepare\" in args:\n xtrabackup_function = \"prepare/apply-log-only\"\n if not xtrabackup_function:\n for arg in args:\n if re.search(r\"(--decrypt)=?[\\w]*\", arg):\n xtrabackup_function = \"decrypt\"\n elif re.search(r\"(--decompress)=?[\\w]*\", arg):\n xtrabackup_function = \"decompress\"\n\n if cmd_root != \"pigz\":\n # this will be just the pigz --version call\n self._xtrabackup_history_log.append(\n [\n cmd_root,\n str(xtrabackup_function),\n cmd_start.strftime(\"%Y-%m-%d %H:%M:%S\"),\n cmd_end.strftime(\"%Y-%m-%d %H:%M:%S\"),\n self.represent_duration(cmd_start, cmd_end),\n str(return_code),\n ]\n )\n return True\n\n\nProcessRunner = ProcessHandler()\n" }, { "path": "mysql_autoxtrabackup/utils/__init__.py", "content": "" }, { "path": "mysql_autoxtrabackup/utils/helpers.py", "content": "# General helpers file for adding all sort of simple helper functions.\n# Trying to use here type hints as well.\n\nimport logging\nimport os\nimport subprocess\nfrom datetime import datetime\nfrom typing import Dict, List, Optional, Union\n\nlogger = logging.getLogger(__name__)\n\n\ndef get_folder_size(path: str) -> Union[str, None]:\n \"\"\"\n Function to calculate given folder size. Using 'du' command here.\n :param path: The full path to be calculated\n :return: String with human readable size info, for eg, 5.3M\n \"\"\"\n du_cmd = \"du -hs {}\".format(path)\n status, output = subprocess.getstatusoutput(du_cmd)\n if status == 0:\n return output.split()[0]\n else:\n logger.error(\"Failed to get the folder size\")\n return None\n\n\ndef sorted_ls(path: Optional[str]) -> List[str]:\n \"\"\"\n Function for sorting given path\n :param path: Directory path\n :return: The list of sorted directories\n \"\"\"\n mtime = lambda f: os.stat(os.path.join(path, f)).st_mtime # type: ignore\n return list(sorted(os.listdir(path), key=mtime))\n\n\ndef get_directory_size(path: str) -> int:\n \"\"\"\n Calculate total size of given directory path\n :param path: Directory path\n :return: Total size of directory\n \"\"\"\n # I am not sure why we have 2 separate functions for same thing but,\n # I assume it is there on purpose\n total_size = 0\n for dir_path, dir_names, file_names in os.walk(path):\n for f in file_names:\n fp = os.path.join(dir_path, f)\n total_size += os.path.getsize(fp)\n return total_size\n\n\ndef create_backup_directory(directory: str, forced_dir: Optional[str] = None) -> str:\n \"\"\"\n Function for creating timestamped directory on given path\n :param directory: Directory path\n :param forced_dir: Full Directory path forced to be created\n :return: Created new directory path\n \"\"\"\n new_dir = os.path.join(directory, datetime.now().strftime(\"%Y-%m-%d_%H-%M-%S\"))\n if forced_dir:\n new_dir = os.path.join(directory, forced_dir)\n try:\n # Creating directory\n os.makedirs(new_dir)\n return new_dir\n except Exception as err:\n logger.error(\n \"Something went wrong in create_backup_directory(): {}\".format(err)\n )\n raise RuntimeError(\n \"Something went wrong in create_backup_directory(): {}\".format(err)\n )\n\n\ndef get_latest_dir_name(path: Optional[str]) -> Optional[str]:\n # Return last backup dir name either incremental or full backup dir\n if len(os.listdir(path)) > 0:\n return max(os.listdir(path))\n return None\n\n\ndef create_directory(path: str) -> Optional[bool]:\n logger.info(\"Creating given directory...\")\n try:\n os.makedirs(path)\n logger.info(\"OK: Created\")\n return True\n except Exception as err:\n logger.error(\"FAILED: Could not create directory, \", err)\n raise RuntimeError(\"FAILED: Could not create directory\")\n\n\ndef check_if_backup_prepared(type_: str, path: str) -> str:\n \"\"\"\n Helper function for checking if given backup already prepared or not.\n :param type_: Type of backup full or inc\n :param path: path string of the backup folder\n :return: True if given backup is prepared, False otherwise\n \"\"\"\n if type_ == \"full\" and os.path.isfile(path + \"/xtrabackup_checkpoints\"):\n with open(path + \"/xtrabackup_checkpoints\", \"r\") as f:\n if f.readline().split()[-1] == \"full-prepared\":\n return \"Full-Prepared\"\n # TODO: add the possible way of checking for incremental backups as well.\n return \"Not-Prepared\"\n\n\ndef list_available_backups(path: str) -> Dict[str, List[Dict[str, str]]]:\n \"\"\"\n Helper function for returning\n Dict of backups;\n and the statuses - if they are already prepared or not\n :param path: General backup directory path\n :return: dictionary of backups full and incremental\n \"\"\"\n backups = {}\n full_backup_dir = path + \"/full\"\n inc_backup_dir = path + \"/inc\"\n if os.path.isdir(full_backup_dir):\n backups = {\n \"full\": [\n {dir_: check_if_backup_prepared(\"full\", full_backup_dir + f\"/{dir_}\")}\n ]\n for dir_ in os.listdir(full_backup_dir)\n }\n if os.path.isdir(inc_backup_dir):\n backups[\"inc\"] = sorted_ls(inc_backup_dir) # type: ignore\n logger.info(\n \"Listing all available backups from full and incremental backup directories...\"\n )\n return backups\n" }, { "path": "mysql_autoxtrabackup/utils/mysql_cli.py", "content": "# This file will consist of some wrapper for using MySQL\n# It is mainly used for preparing and calling mysql cli\nimport logging\n\nfrom mysql_autoxtrabackup.general_conf import path_config\nfrom mysql_autoxtrabackup.general_conf.generalops import GeneralClass\nfrom mysql_autoxtrabackup.process_runner.process_runner import ProcessRunner\n\nlogger = logging.getLogger(__name__)\n\n\nclass MySQLClientHelper:\n def __init__(self, config: str = path_config.config_path_file):\n self.conf = config\n # Using Composition instead of Inheritance here\n options_obj = GeneralClass(config=self.conf)\n self.mysql_options = options_obj.mysql_options\n\n def create_mysql_client_command(self, statement: str) -> str:\n command_connection = \"{} --defaults-file={} -u{} --password={}\".format(\n self.mysql_options.get(\"mysql\"),\n self.mysql_options.get(\"mycnf\"),\n self.mysql_options.get(\"mysql_user\"),\n self.mysql_options.get(\"mysql_password\"),\n )\n command_execute = ' -e \"{}\"'\n if self.mysql_options.get(\"mysql_socket\"):\n command_connection += \" --socket={}\"\n new_command = command_connection.format(\n self.mysql_options.get(\"mysql_socket\")\n )\n else:\n command_connection += \" --host={} --port={}\"\n new_command = command_connection.format(\n self.mysql_options.get(\"mysql_host\"),\n self.mysql_options.get(\"mysql_port\"),\n )\n new_command += command_execute\n return new_command.format(statement)\n\n def mysql_run_command(self, statement: str) -> bool:\n command = self.create_mysql_client_command(statement=statement)\n return ProcessRunner.run_command(command)\n" }, { "path": "mysql_autoxtrabackup/utils/version.py", "content": "__all__ = \"VERSION\"\n\nVERSION = \"2.0.2\"\n" }, { "path": "netlify.toml", "content": "# example netlify.toml\n[build]\n command = \"bash scripts/netlify-docs.sh\"\n functions = \"netlify/functions\"\n publish = \"docs/_build/html\"\n\n ## Uncomment to use this redirect for Single Page Applications like create-react-app.\n ## Not needed for static site generators.\n #[[redirects]]\n # from = \"/*\"\n # to = \"/index.html\"\n # status = 200\n\n ## (optional) Settings for Netlify Dev\n ## https://github.com/netlify/cli/blob/main/docs/netlify-dev.md#project-detection\n #[dev]\n # command = \"yarn start\" # Command to start your dev server\n # port = 3000 # Port that the dev server will be listening on\n # publish = \"dist\" # Folder with the static content for _redirect file\n\n ## more info on configuring this file: https://www.netlify.com/docs/netlify-toml-reference/\n" }, { "path": "pyproject.toml", "content": "[build-system]\nrequires = [\"flit\"]\nbuild-backend = \"flit.buildapi\"\n\n[tool.flit.metadata]\nmodule = \"mysql_autoxtrabackup\"\nauthor = \"Shahriyar(Shako) Rzayev\"\nauthor-email = \"rzayev.sehriyar@gmail.com\"\nhome-page = \"https://github.com/ShahriyarR/MySQL-AutoXtraBackup\"\nclassifiers = [\n \"Intended Audience :: Information Technology\",\n \"Intended Audience :: System Administrators\",\n \"Operating System :: POSIX :: Linux\",\n \"Programming Language :: Python :: 3\",\n \"Programming Language :: Python\",\n \"Topic :: Internet\",\n \"Topic :: Software Development :: Libraries :: Application Frameworks\",\n \"Topic :: Software Development :: Libraries :: Python Modules\",\n \"Topic :: Software Development :: Libraries\",\n \"Topic :: Software Development\",\n \"Typing :: Typed\",\n \"Development Status :: 4 - Beta\",\n \"Environment :: Console\",\n \"Intended Audience :: Developers\",\n \"License :: OSI Approved :: MIT License\",\n \"Programming Language :: Python :: 3 :: Only\",\n \"Programming Language :: Python :: 3.6\",\n \"Programming Language :: Python :: 3.7\",\n \"Programming Language :: Python :: 3.8\",\n]\n\nrequires = [\n \"click >=3.3\",\n \"pid >=2.0\",\n \"humanfriendly >=2.0\",\n \"fastapi >= 0.63.0\",\n \"uvicorn >= 0.13.4\",\n]\n\ndescription-file = \"README.md\"\nrequires-python = \">=3.6\"\n\n[tool.flit.metadata.urls]\nDocumentation = \"https://autoxtrabackup.azepug.az/\"\n\n[tool.flit.metadata.requires-extra]\ntest = [\n \"pytest ==5.4.3\",\n \"pytest-cov ==2.10.0\",\n \"mypy ==0.812\",\n \"isort >=5.0.6,<6.0.0\",\n \"flake8 >=3.8.3,<4.0.0\",\n \"black ==20.8b1\",\n]\n\ndoc = [\n \"sphinx\",\n \"sphinx_rtd_theme\",\n \"sphinx-autobuild\",\n \"jinja2 >=2.11.3\"\n]\n\ndev = [\n \"click >=3.3\",\n \"pid >=2.0\",\n \"humanfriendly >=2.0\",\n \"pytest ==5.4.3\",\n \"pytest-cov ==2.10.0\",\n \"mypy ==0.812\",\n \"isort >=5.0.6,<6.0.0\",\n \"flake8 >=3.8.3,<4.0.0\",\n \"black ==20.8b1\",\n \"fastapi >= 0.63.0\",\n \"uvicorn >= 0.13.4\",\n]\n\nall = [\n \"click >=3.3\",\n \"pid >=2.0\",\n \"humanfriendly >=2.0\",\n \"sphinx\",\n \"sphinx_rtd_theme\",\n \"sphinx-autobuild\",\n \"jinja2 >=2.11.3\",\n \"fastapi >= 0.63.0\",\n \"uvicorn >= 0.13.4\",\n]\n\n[tool.isort]\nprofile = \"black\"\nknown_third_party = [\"click\", \"pid\", \"humanfriendly\"]\n\n[tool.flit.scripts]\nautoxtrabackup = \"mysql_autoxtrabackup.autoxtrabackup:all_procedure\"" }, { "path": "scripts/api_calls/delete_all_backups.sh", "content": "#!/usr/bin/env bash\n\ncurl -X DELETE http://127.0.0.1:5555/delete" }, { "path": "scripts/api_calls/get_all_backups.sh", "content": "#!/usr/bin/env bash\n\ncurl http://127.0.0.1:5555/backups" }, { "path": "scripts/api_calls/prepare_backup.sh", "content": "#!/usr/bin/env bash\n\ncurl -X POST http://127.0.0.1:5555/prepare" }, { "path": "scripts/api_calls/take_backup.sh", "content": "#!/usr/bin/env bash\n\ncurl -X POST http://127.0.0.1:5555/backup" }, { "path": "scripts/build-docs.sh", "content": "#!/usr/bin/env bash\n\nset -e\nset -x\n\nmake html" }, { "path": "scripts/format-imports.sh", "content": "#!/bin/sh -e\nset -x\n\n# Sort imports one per line, so autoflake can remove unused imports\nisort --recursive mysql_autoxtrabackup tests docs scripts --force-single-line-imports\nsh ./scripts/format.sh\n" }, { "path": "scripts/format.sh", "content": "#!/bin/sh -e\nset -x\n\nautoflake --remove-all-unused-imports --recursive --remove-unused-variables --in-place mysql_autoxtrabackup docs scripts tests --exclude=__init__.py\nblack mysql_autoxtrabackup docs scripts tests\nisort --recursive mysql_autoxtrabackup docs scripts tests\n" }, { "path": "scripts/lint.sh", "content": "#!/usr/bin/env bash\n\nset -e\nset -x\n\nmypy mysql_autoxtrabackup\nflake8 mysql_autoxtrabackup tests\nblack mysql_autoxtrabackup tests --check\nisort --recursive mysql_autoxtrabackup tests docs scripts --check-only" }, { "path": "scripts/netlify-docs.sh", "content": "#!/usr/bin/env bash\nset -x\nset -e\n# Install pip\ncd /tmp\ncurl https://bootstrap.pypa.io/pip/3.5/get-pip.py -o get-pip.py\npython3 get-pip.py --user\ncd -\n# Install Flit to be able to install all\npython3 -m pip install --user flit\n# Install with Flit\npython3 -m flit install --user --deps=production\n# Finally, run sphinx make\npython3 -m make html" }, { "path": "scripts/publish.sh", "content": "#!/usr/bin/env bash\n\nset -e\n\nflit publish" }, { "path": "tests/Dockerfile", "content": "FROM mysql/mysql-server:8.0\nUSER root\nWORKDIR /opt\nRUN yum install -y git\nRUN yum install -y python3\nRUN yum install -y vim\nRUN yum install -y perl\nRUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm\nRUN yum install -y https://repo.percona.com/yum/percona-release-latest.noarch.rpm\nRUN yum install -y libev\nRUN percona-release enable-only tools\nRUN yum install -y --exclude=Percona-Server\\* percona-xtrabackup-80\nRUN yum install -y qpress\nRUN yum install -y python3-pip\nRUN cd /opt && \\\n git clone https://github.com/sstephenson/bats.git && \\\n cd bats && \\\n ./install.sh /usr/local\nARG GIT_BRANCH_NAME\nRUN cd /opt && \\\n git clone -b $GIT_BRANCH_NAME https://github.com/ShahriyarR/MySQL-AutoXtraBackup.git && \\\n cd /opt/MySQL-AutoXtraBackup && \\\n python3 setup.py install\n\nRUN yum groupinstall -y \"Development Tools\"\nRUN yum -y install python3-devel.x86_64 --enablerepo=rhel-7-server-optional-rpms\nRUN cd /opt/MySQL-AutoXtraBackup/test && \\\n pip3 install -r requirements.txt\n\nEXPOSE 8080\n\nRUN cd /opt/MySQL-AutoXtraBackup && \\\n git pull && \\\n pipenv --python `which python3` install\n\nWORKDIR /opt/MySQL-AutoXtraBackup\nRUN cd /opt/MySQL-AutoXtraBackup && git pull\nRUN pip3 install uvicorn\nRUN pip3 install fastapi\n\nCOPY entrypoint.sh /\nRUN chmod +x /entrypoint.sh\nENTRYPOINT [\"./entrypoint.sh\"]\n#CMD [\"uvicorn\", \"api.main:app\", \"--port\", \"8080\"]\n" }, { "path": "tests/README.md", "content": "# The place for testing this project. All you need is to checkout to your test branch and run docker." }, { "path": "tests/__init__.py", "content": "" }, { "path": "tests/conftest.py", "content": "import pytest\nfrom fastapi.testclient import TestClient\n\nfrom mysql_autoxtrabackup.api.main import app\nfrom mysql_autoxtrabackup.backup_backup.backuper import Backup\n\nbck_obj = Backup()\nclient = TestClient(app)\n\n\n@pytest.fixture()\ndef return_bck_obj():\n return bck_obj\n\n\n@pytest.fixture()\ndef fastapi_client():\n return client\n" }, { "path": "tests/entrypoint.sh", "content": "#!/bin/bash\n# Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.\n#\n# This program is free software; you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation; version 2 of the License.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program; if not, write to the Free Software\n# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA\nset -e\n\necho \"[Entrypoint] MySQL Docker Image 8.0.20-1.1.16\"\necho \"[Entrypoint] Modified by Shako(mysql.az)\"\n# Fetch value from server config\n# We use mysqld --verbose --help instead of my_print_defaults because the\n# latter only show values present in config files, and not server defaults\n_get_config() {\n\tlocal conf=\"$1\"; shift\n\t\"$@\" --verbose --help 2>/dev/null | grep \"^$conf\" | awk '$1 == \"'\"$conf\"'\" { print $2; exit }'\n}\n\n# If command starts with an option, prepend mysqld\n# This allows users to add command-line options without\n# needing to specify the \"mysqld\" command\nif [ \"${1:0:1}\" = '-' ]; then\n\tset -- mysqld \"$@\"\nfi\n\nif [ \"$1\" = 'mysqld' ]; then\n\t# Test that the server can start. We redirect stdout to /dev/null so\n\t# only the error messages are left.\n\tresult=0\n\toutput=$(\"$@\" --validate-config) || result=$?\n\tif [ ! \"$result\" = \"0\" ]; then\n\t\techo >&2 '[Entrypoint] ERROR: Unable to start MySQL. Please check your configuration.'\n\t\techo >&2 \"[Entrypoint] $output\"\n\t\texit 1\n\tfi\n\n\t# Get config\n\tDATADIR=\"$(_get_config 'datadir' \"$@\")\"\n\tSOCKET=\"$(_get_config 'socket' \"$@\")\"\n\n\tif [ -n \"$MYSQL_LOG_CONSOLE\" ] || [ -n \"console\" ]; then\n\t\t# Don't touch bind-mounted config files\n\t\tif ! cat /proc/1/mounts | grep \"etc/my.cnf\"; then\n\t\t\tsed -i 's/^log-error=/#&/' /etc/my.cnf\n\t\tfi\n\tfi\n\n\tif [ ! -d \"$DATADIR/mysql\" ]; then\n\t\t# If the password variable is a filename we use the contents of the file. We\n\t\t# read this first to make sure that a proper error is generated for empty files.\n\t\tif [ -f \"$MYSQL_ROOT_PASSWORD\" ]; then\n\t\t\tMYSQL_ROOT_PASSWORD=\"$(cat $MYSQL_ROOT_PASSWORD)\"\n\t\t\tif [ -z \"$MYSQL_ROOT_PASSWORD\" ]; then\n\t\t\t\techo >&2 '[Entrypoint] Empty MYSQL_ROOT_PASSWORD file specified.'\n\t\t\t\texit 1\n\t\t\tfi\n\t\tfi\n\t\tif [ -z \"$MYSQL_ROOT_PASSWORD\" -a -z \"$MYSQL_ALLOW_EMPTY_PASSWORD\" -a -z \"$MYSQL_RANDOM_ROOT_PASSWORD\" ]; then\n\t\t\techo >&2 '[Entrypoint] No password option specified for new database.'\n\t\t\techo >&2 '[Entrypoint] A random onetime password will be generated.'\n\t\t\tMYSQL_RANDOM_ROOT_PASSWORD=true\n\t\t\tMYSQL_ONETIME_PASSWORD=true\n\t\tfi\n\t\tmkdir -p \"$DATADIR\"\n\t\tchown -R mysql:mysql \"$DATADIR\"\n\n\t\techo '[Entrypoint] Initializing database'\n\t\t\"$@\" --initialize-insecure\n\t\techo '[Entrypoint] Database initialized'\n\n\t\t\"$@\" --daemonize --skip-networking --socket=\"$SOCKET\"\n\n\t\t# To avoid using password on commandline, put it in a temporary file.\n\t\t# The file is only populated when and if the root password is set.\n\t\tPASSFILE=$(mktemp -u /var/lib/mysql-files/XXXXXXXXXX)\n\t\tinstall /dev/null -m0600 -omysql -gmysql \"$PASSFILE\"\n\t\t# Define the client command used throughout the script\n\t\t# \"SET @@SESSION.SQL_LOG_BIN=0;\" is required for products like group replication to work properly\n\t\tmysql=( mysql --defaults-extra-file=\"$PASSFILE\" --protocol=socket -uroot -hlocalhost --socket=\"$SOCKET\" --init-command=\"SET @@SESSION.SQL_LOG_BIN=0;\")\n\n\t\tif [ ! -z \"\" ];\n\t\tthen\n\t\t\tfor i in {30..0}; do\n\t\t\t\tif mysqladmin --socket=\"$SOCKET\" ping &>/dev/null; then\n\t\t\t\t\tbreak\n\t\t\t\tfi\n\t\t\t\techo '[Entrypoint] Waiting for server...'\n\t\t\t\tsleep 1\n\t\t\tdone\n\t\t\tif [ \"$i\" = 0 ]; then\n\t\t\t\techo >&2 '[Entrypoint] Timeout during MySQL init.'\n\t\t\t\texit 1\n\t\t\tfi\n\t\tfi\n\n\t\tmysql_tzinfo_to_sql /usr/share/zoneinfo | \"${mysql[@]}\" mysql\n\t\t\n\t\tif [ ! -z \"$MYSQL_RANDOM_ROOT_PASSWORD\" ]; then\n\t\t\tMYSQL_ROOT_PASSWORD=\"$(pwmake 128)\"\n\t\t\techo \"[Entrypoint] GENERATED ROOT PASSWORD: $MYSQL_ROOT_PASSWORD\"\n\t\tfi\n\t\tif [ -z \"$MYSQL_ROOT_HOST\" ]; then\n\t\t\tROOTCREATE=\"ALTER USER 'root'@'localhost' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}';\"\n\t\telse\n\t\t\tROOTCREATE=\"ALTER USER 'root'@'localhost' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}'; \\\n\t\t\tCREATE USER 'root'@'${MYSQL_ROOT_HOST}' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}'; \\\n\t\t\tGRANT ALL ON *.* TO 'root'@'${MYSQL_ROOT_HOST}' WITH GRANT OPTION ; \\\n\t\t\tGRANT PROXY ON ''@'' TO 'root'@'${MYSQL_ROOT_HOST}' WITH GRANT OPTION ;\"\n\t\tfi\n\t\t\"${mysql[@]}\" <<-EOSQL\n\t\t\tDELETE FROM mysql.user WHERE user NOT IN ('mysql.infoschema', 'mysql.session', 'mysql.sys', 'root') OR host NOT IN ('localhost');\n\t\t\tCREATE USER 'healthchecker'@'localhost' IDENTIFIED BY 'healthcheckpass';\n\t\t\t${ROOTCREATE}\n\t\t\tFLUSH PRIVILEGES ;\n\t\tEOSQL\n\t\tif [ ! -z \"$MYSQL_ROOT_PASSWORD\" ]; then\n\t\t\t# Put the password into the temporary config file\n\t\t\tcat >\"$PASSFILE\" < \"$SQL\"\nALTER USER 'root'@'localhost' IDENTIFIED BY '12345';\n#ALTER USER 'root'@'${MYSQL_ROOT_HOST}' PASSWORD EXPIRE;\n#ALTER USER 'root'@'localhost' PASSWORD EXPIRE;\nEOF\n\t\t\t\telse\n\t\t\t\t\tcat << EOF > \"$SQL\"\n#ALTER USER 'root'@'localhost' PASSWORD EXPIRE;\nALTER USER 'root'@'localhost' IDENTIFIED BY '12345';\nEOF\n\t\t\t\tfi\n\t\t\t\tset -- \"$@\" --init-file=\"$SQL\"\n\t\t\t\tunset SQL\n\t\t\tfi\n\t\tfi\n\n\t\techo\n\t\techo '[Entrypoint] MySQL init process done. Ready for start up.'\n\t\techo\n\tfi\n\n\t# Used by healthcheck to make sure it doesn't mistakenly report container\n\t# healthy during startup\n\t# Put the password into the temporary config file\n\ttouch /healthcheck.cnf\n\tcat >\"/healthcheck.cnf\" <